diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 9bc05cc..e48bd77 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -8,8 +8,9 @@ services: container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx4g" - LOG_LEVEL: "debug" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "blaze-data:/app/data" diff --git a/bridgehead b/bridgehead index 2706df8..dc82d80 100755 --- a/bridgehead +++ b/bridgehead @@ -51,6 +51,7 @@ loadVars() { fi fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" setHostname + optimizeBlazeMemoryUsage [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 5e26878..0407e07 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -6,7 +6,9 @@ services: container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx4g" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "blaze-data:/app/data" @@ -19,7 +21,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" focus: - image: docker.verbis.dkfz.de/cache/samply/focus:0.4.0 + image: docker.verbis.dkfz.de/cache/samply/focus:0.4.1 container_name: bridgehead-focus environment: API_KEY: ${FOCUS_BEAM_SECRET_SHORT} diff --git a/ccp/modules/dnpm-node-compose.yml b/ccp/modules/dnpm-node-compose.yml index 8a39223..ee84d89 100644 --- a/ccp/modules/dnpm-node-compose.yml +++ b/ccp/modules/dnpm-node-compose.yml @@ -6,6 +6,7 @@ services: container_name: bridgehead-dnpm-backend environment: - ZPM_SITE=${ZPM_SITE} + - N_RANDOM_FILES=${DNPM_SYNTH_NUM} volumes: - /etc/bridgehead/dnpm:/bwhc_config:ro - ${DNPM_DATA_DIR}:/bwhc_data diff --git a/ccp/modules/dnpm-node-setup.sh b/ccp/modules/dnpm-node-setup.sh index 88447ef..bf8fd26 100644 --- a/ccp/modules/dnpm-node-setup.sh +++ b/ccp/modules/dnpm-node-setup.sh @@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" exit 1 fi - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} + if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then + echo "Override of landing page url already in place" + else + echo "Adding override of landing page url" + if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then + echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + else + echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + fi + fi fi diff --git a/lib/functions.sh b/lib/functions.sh index 1d6bfe9..f277368 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -155,6 +155,28 @@ setHostname() { fi } +# This function optimizes the usage of memory through blaze, according to the official performance tuning guide: +# https://github.com/samply/blaze/blob/master/docs/tuning-guide.md +# Short summary of the adjustments made: +# - set blaze memory cap to a quarter of the system memory +# - set db block cache size to a quarter of the system memory +# - limit resource count allowed in blaze to 1,25M per 4GB available system memory +optimizeBlazeMemoryUsage() { + if [ -z "$BLAZE_MEMORY_CAP" ]; then + system_memory_in_mb=$(LC_ALL=C free -m | grep 'Mem:' | awk '{print $2}'); + export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); + fi + if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then + available_system_memory_chuncks=$((BLAZE_MEMORY_CAP / 1000)) + if [ $available_system_memory_chuncks -eq 0 ]; then + log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower." + export BLAZE_RESOURCE_CACHE_CAP=128000; + else + export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chuncks * 312500)) + fi + fi +} + # Takes 1) The Backup Directory Path 2) The name of the Service to be backuped # Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week createEncryptedPostgresBackup(){ diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 10166e0..235826a 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -67,29 +67,30 @@ log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..." source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars -set +e -SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" -RET=$? -set -e -if [ $RET -ne 0 ]; then - log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." - log WARN "Unable to check clock skew due to previous error." -else - log INFO "Checking clock skew ..." +if [ "${PROJECT}" != "minimal" ]; then + set +e + SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" + RET=$? + set -e + if [ $RET -ne 0 ]; then + log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." + log WARN "Unable to check clock skew due to previous error." + else + log INFO "Checking clock skew ..." - SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) - MYTIME=$(date +%s) - SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) - SKEW=$(echo $SKEW | awk -F- '{print $NF}') - SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." - if [ $SKEW -ge 300 ]; then - report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" - exit 1 - elif [ $SKEW -ge 60 ]; then - log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" - fi + SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) + MYTIME=$(date +%s) + SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) + SKEW=$(echo $SKEW | awk -F- '{print $NF}') + SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." + if [ $SKEW -ge 300 ]; then + report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" + exit 1 + elif [ $SKEW -ge 60 ]; then + log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" + fi + fi fi - checkPrivKey() { if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then log INFO "Success - private key found." @@ -100,7 +101,7 @@ checkPrivKey() { return 0 } -if [[ "$@" =~ "noprivkey" ]]; then +if [[ "$@" =~ "noprivkey" || "${PROJECT}" != "minimal" ]]; then log INFO "Skipping check for private key for now." else checkPrivKey || exit 1 diff --git a/minimal/modules/dnpm-node-compose.yml b/minimal/modules/dnpm-node-compose.yml index 8a39223..ee84d89 100644 --- a/minimal/modules/dnpm-node-compose.yml +++ b/minimal/modules/dnpm-node-compose.yml @@ -6,6 +6,7 @@ services: container_name: bridgehead-dnpm-backend environment: - ZPM_SITE=${ZPM_SITE} + - N_RANDOM_FILES=${DNPM_SYNTH_NUM} volumes: - /etc/bridgehead/dnpm:/bwhc_config:ro - ${DNPM_DATA_DIR}:/bwhc_data diff --git a/minimal/modules/dnpm-node-setup.sh b/minimal/modules/dnpm-node-setup.sh index 88447ef..bf8fd26 100644 --- a/minimal/modules/dnpm-node-setup.sh +++ b/minimal/modules/dnpm-node-setup.sh @@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" exit 1 fi - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} + if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then + echo "Override of landing page url already in place" + else + echo "Adding override of landing page url" + if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then + echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + else + echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml + fi + fi fi