From 7d07c0623d9f95ac704c1cd7b9d91638abb90d41 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 20 Feb 2024 15:15:49 +0100 Subject: [PATCH 1/9] refactor: optimize memory usage of blaze --- bbmri/docker-compose.yml | 4 ++-- bridgehead | 1 + ccp/docker-compose.yml | 3 ++- lib/functions.sh | 10 ++++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 9bc05cc..dcd446c 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -8,8 +8,8 @@ services: container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx4g" - LOG_LEVEL: "debug" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "blaze-data:/app/data" diff --git a/bridgehead b/bridgehead index 31a838e..68e29e1 100755 --- a/bridgehead +++ b/bridgehead @@ -50,6 +50,7 @@ loadVars() { source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import" fi fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" + setBlazeMemoryCap [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index d92ccfb..823bcdf 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -6,7 +6,8 @@ services: container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx4g" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "blaze-data:/app/data" diff --git a/lib/functions.sh b/lib/functions.sh index 6a45d35..66fb701 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -155,6 +155,16 @@ setHostname() { fi } +# blaze memory cap should be approximately a quarter of the system memory +# the memory cap will be applied to both the java heap size and db clock cache +setBlazeMemoryCap() { + if [ -z "$BLAZE_MEMORY_CAP" ]; then + system_memory=$(grep MemTotal /proc/meminfo | grep -Po '\d+'); + system_memory_in_mb=$(("$system_memory"/1024)); + export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); + fi +} + # Takes 1) The Backup Directory Path 2) The name of the Service to be backuped # Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week createEncryptedPostgresBackup(){ From b5ef856f1283c3074a25add9eec116ec7ecd4578 Mon Sep 17 00:00:00 2001 From: Torben Brenner <76154651+torbrenner@users.noreply.github.com> Date: Fri, 23 Feb 2024 08:27:06 +0100 Subject: [PATCH 2/9] refactor: calculate memory using free Co-authored-by: Tobias Kussel --- lib/functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 66fb701..ab904d3 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -159,8 +159,7 @@ setHostname() { # the memory cap will be applied to both the java heap size and db clock cache setBlazeMemoryCap() { if [ -z "$BLAZE_MEMORY_CAP" ]; then - system_memory=$(grep MemTotal /proc/meminfo | grep -Po '\d+'); - system_memory_in_mb=$(("$system_memory"/1024)); + system_memory_in_mb=$(free -m | grep 'Mem:' | awk '{print $2}'); export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); fi } From ad4430e48034c855e25c0618d47226cf6fa8e627 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Wed, 28 Feb 2024 10:11:03 +0000 Subject: [PATCH 3/9] Create env to control dnpm synthetic data generation --- ccp/modules/dnpm-node-compose.yml | 1 + ccp/modules/dnpm-node-setup.sh | 21 +++++++++++---------- minimal/modules/dnpm-node-compose.yml | 1 + minimal/modules/dnpm-node-setup.sh | 21 +++++++++++---------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/ccp/modules/dnpm-node-compose.yml b/ccp/modules/dnpm-node-compose.yml index 8a39223..ee84d89 100644 --- a/ccp/modules/dnpm-node-compose.yml +++ b/ccp/modules/dnpm-node-compose.yml @@ -6,6 +6,7 @@ services: container_name: bridgehead-dnpm-backend environment: - ZPM_SITE=${ZPM_SITE} + - N_RANDOM_FILES=${DNPM_SYNTH_NUM} volumes: - /etc/bridgehead/dnpm:/bwhc_config:ro - ${DNPM_DATA_DIR}:/bwhc_data diff --git a/ccp/modules/dnpm-node-setup.sh b/ccp/modules/dnpm-node-setup.sh index 88447ef..bf8fd26 100644 --- a/ccp/modules/dnpm-node-setup.sh +++ b/ccp/modules/dnpm-node-setup.sh @@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" exit 1 fi - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} + if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then + echo "Override of landing page url already in place" + else + echo "Adding override of landing page url" + if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then + echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + else + echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + fi + fi fi diff --git a/minimal/modules/dnpm-node-compose.yml b/minimal/modules/dnpm-node-compose.yml index 8a39223..ee84d89 100644 --- a/minimal/modules/dnpm-node-compose.yml +++ b/minimal/modules/dnpm-node-compose.yml @@ -6,6 +6,7 @@ services: container_name: bridgehead-dnpm-backend environment: - ZPM_SITE=${ZPM_SITE} + - N_RANDOM_FILES=${DNPM_SYNTH_NUM} volumes: - /etc/bridgehead/dnpm:/bwhc_config:ro - ${DNPM_DATA_DIR}:/bwhc_data diff --git a/minimal/modules/dnpm-node-setup.sh b/minimal/modules/dnpm-node-setup.sh index 88447ef..bf8fd26 100644 --- a/minimal/modules/dnpm-node-setup.sh +++ b/minimal/modules/dnpm-node-setup.sh @@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" exit 1 fi - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} + if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then + echo "Override of landing page url already in place" + else + echo "Adding override of landing page url" + if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then + echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + else + echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + fi + fi fi From 3777d4bf054622e9e7ada7041dc590cafb9d2d25 Mon Sep 17 00:00:00 2001 From: Torben Brenner <76154651+torbrenner@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:34:00 +0100 Subject: [PATCH 4/9] Add default value for BLAZE_MEMORY_CAP Co-authored-by: Tobias Kussel --- bbmri/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index dcd446c..378b64a 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -8,7 +8,7 @@ services: container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 823bcdf..a87754a 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -6,7 +6,7 @@ services: container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: From b5c35211f625f1a8714685808fcb7fc37d89ef65 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 5 Mar 2024 14:58:06 +0000 Subject: [PATCH 5/9] Dont test clock skew and priv key for minimal bridgeheads --- lib/prerequisites.sh | 45 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 10166e0..235826a 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -67,29 +67,30 @@ log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..." source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars -set +e -SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" -RET=$? -set -e -if [ $RET -ne 0 ]; then - log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." - log WARN "Unable to check clock skew due to previous error." -else - log INFO "Checking clock skew ..." +if [ "${PROJECT}" != "minimal" ]; then + set +e + SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" + RET=$? + set -e + if [ $RET -ne 0 ]; then + log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." + log WARN "Unable to check clock skew due to previous error." + else + log INFO "Checking clock skew ..." - SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) - MYTIME=$(date +%s) - SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) - SKEW=$(echo $SKEW | awk -F- '{print $NF}') - SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." - if [ $SKEW -ge 300 ]; then - report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" - exit 1 - elif [ $SKEW -ge 60 ]; then - log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" - fi + SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) + MYTIME=$(date +%s) + SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) + SKEW=$(echo $SKEW | awk -F- '{print $NF}') + SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." + if [ $SKEW -ge 300 ]; then + report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" + exit 1 + elif [ $SKEW -ge 60 ]; then + log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" + fi + fi fi - checkPrivKey() { if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then log INFO "Success - private key found." @@ -100,7 +101,7 @@ checkPrivKey() { return 0 } -if [[ "$@" =~ "noprivkey" ]]; then +if [[ "$@" =~ "noprivkey" || "${PROJECT}" != "minimal" ]]; then log INFO "Skipping check for private key for now." else checkPrivKey || exit 1 From ea6441fbcbe8fc0510ef0bb70f7d57a4e511f9aa Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Fri, 8 Mar 2024 08:33:15 +0100 Subject: [PATCH 6/9] Updated ccp focus to 0.4.1 --- ccp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 5e26878..ac80334 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -19,7 +19,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" focus: - image: docker.verbis.dkfz.de/cache/samply/focus:0.4.0 + image: docker.verbis.dkfz.de/cache/samply/focus:0.4.1 container_name: bridgehead-focus environment: API_KEY: ${FOCUS_BEAM_SECRET_SHORT} From eeb17e7bfef300367787d8262583e8074d6c4acb Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 8 Mar 2024 13:33:30 +0100 Subject: [PATCH 7/9] feat: added optional resource cache cap --- bbmri/docker-compose.yml | 1 + ccp/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 378b64a..e48bd77 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -9,6 +9,7 @@ services: environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index a87754a..331ce0d 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -7,6 +7,7 @@ services: environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: From e1e523f1acf09cdd11810bc6f551becf1f46665b Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Mon, 11 Mar 2024 09:58:30 +0100 Subject: [PATCH 8/9] refactor: tune configuration of blaze according to system memory --- bridgehead | 2 +- lib/functions.sh | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/bridgehead b/bridgehead index 68e29e1..62cb38e 100755 --- a/bridgehead +++ b/bridgehead @@ -50,7 +50,7 @@ loadVars() { source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import" fi fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" - setBlazeMemoryCap + optimizeBlazeMemoryUsage [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a diff --git a/lib/functions.sh b/lib/functions.sh index ab904d3..ecd330f 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -155,13 +155,26 @@ setHostname() { fi } -# blaze memory cap should be approximately a quarter of the system memory -# the memory cap will be applied to both the java heap size and db clock cache -setBlazeMemoryCap() { +# This function optimizes the usage of memory through blaze, according to the official performance tuning guide: +# https://github.com/samply/blaze/blob/master/docs/tuning-guide.md +# Short summary of the adjustments made: +# - set blaze memory cap to a quarter of the system memory +# - set db block cache size to a quarter of the system memory +# - limit resource count allowed in blaze to 1,25M per 4GB available system memory +optimizeBlazeMemoryUsage() { if [ -z "$BLAZE_MEMORY_CAP" ]; then system_memory_in_mb=$(free -m | grep 'Mem:' | awk '{print $2}'); export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); fi + if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then + available_system_memory_chuncks=$((BLAZE_MEMORY_CAP / 1000)) + if [ $available_system_memory_chuncks -eq 0 ]; then + log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower." + export BLAZE_RESOURCE_CACHE_CAP=128000; + else + export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chuncks * 312500)) + fi + fi } # Takes 1) The Backup Directory Path 2) The name of the Service to be backuped From 6217e285902e64589289c293eaeda89d3fff2c2a Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 15 Mar 2024 11:48:25 +0100 Subject: [PATCH 9/9] fix: use always english output of free command --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index ecd330f..b4dc876 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -163,7 +163,7 @@ setHostname() { # - limit resource count allowed in blaze to 1,25M per 4GB available system memory optimizeBlazeMemoryUsage() { if [ -z "$BLAZE_MEMORY_CAP" ]; then - system_memory_in_mb=$(free -m | grep 'Mem:' | awk '{print $2}'); + system_memory_in_mb=$(LC_ALL=C free -m | grep 'Mem:' | awk '{print $2}'); export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); fi if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then