samply
/
bridgehead
mirror of https://github.com/samply/bridgehead.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/datashield

This commit is contained in:
patrickskowronekdkfz 2024-03-19 09:52:40 +01:00 committed by GitHub
parent ec6f9302a1 8ea7da64b7
commit 4fac079aec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 77 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bbmri/docker-compose.yml
View File

@ -8,8 +8,9 @@ services:
container_name: bridgehead-bbmri-blaze container_name: bridgehead-bbmri-blaze
environment: environment:
BASE_URL: "http://bridgehead-bbmri-blaze:8080" BASE_URL: "http://bridgehead-bbmri-blaze:8080"
JAVA_TOOL_OPTIONS: "-Xmx4g" JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
LOG_LEVEL: "debug" DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
ENFORCE_REFERENTIAL_INTEGRITY: "false" ENFORCE_REFERENTIAL_INTEGRITY: "false"
volumes: volumes:
- "blaze-data:/app/data" - "blaze-data:/app/data"

1
bridgehead
View File

@ -51,6 +51,7 @@ loadVars() {
fi fi
fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile"
setHostname setHostname
optimizeBlazeMemoryUsage
[ -e ./$PROJECT/vars ] && source ./$PROJECT/vars [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars
set +a set +a

6
ccp/docker-compose.yml
View File

@ -6,7 +6,9 @@ services:
container_name: bridgehead-ccp-blaze container_name: bridgehead-ccp-blaze
environment: environment:
BASE_URL: "http://bridgehead-ccp-blaze:8080" BASE_URL: "http://bridgehead-ccp-blaze:8080"
JAVA_TOOL_OPTIONS: "-Xmx4g" JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
ENFORCE_REFERENTIAL_INTEGRITY: "false" ENFORCE_REFERENTIAL_INTEGRITY: "false"
volumes: volumes:
- "blaze-data:/app/data" - "blaze-data:/app/data"
@ -19,7 +21,7 @@ services:
- "traefik.http.routers.blaze_ccp.tls=true" - "traefik.http.routers.blaze_ccp.tls=true"
focus: focus:
image: docker.verbis.dkfz.de/cache/samply/focus:0.4.0 image: docker.verbis.dkfz.de/cache/samply/focus:0.4.1
container_name: bridgehead-focus container_name: bridgehead-focus
environment: environment:
API_KEY: ${FOCUS_BEAM_SECRET_SHORT} API_KEY: ${FOCUS_BEAM_SECRET_SHORT}

1
ccp/modules/dnpm-node-compose.yml
View File

@ -6,6 +6,7 @@ services:
container_name: bridgehead-dnpm-backend container_name: bridgehead-dnpm-backend
environment: environment:
- ZPM_SITE=${ZPM_SITE} - ZPM_SITE=${ZPM_SITE}
- N_RANDOM_FILES=${DNPM_SYNTH_NUM}
volumes: volumes:
- /etc/bridgehead/dnpm:/bwhc_config:ro - /etc/bridgehead/dnpm:/bwhc_config:ro
- ${DNPM_DATA_DIR}:/bwhc_data - ${DNPM_DATA_DIR}:/bwhc_data

21
ccp/modules/dnpm-node-setup.sh
View File

@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then
log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" log ERROR "Mandatory variable DNPM_DATA_DIR not defined!"
exit 1 exit 1
fi fi
if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0}
echo "Override of landing page url already in place" if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then
else echo "Override of landing page url already in place"
echo "Adding override of landing page url" else
if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then echo "Adding override of landing page url"
echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then
else echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml else
fi echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
fi fi
fi
fi fi

22
lib/functions.sh
View File

@ -155,6 +155,28 @@ setHostname() {
fi fi
} }
# This function optimizes the usage of memory through blaze, according to the official performance tuning guide:
# https://github.com/samply/blaze/blob/master/docs/tuning-guide.md
# Short summary of the adjustments made:
# - set blaze memory cap to a quarter of the system memory
# - set db block cache size to a quarter of the system memory
# - limit resource count allowed in blaze to 1,25M per 4GB available system memory
optimizeBlazeMemoryUsage() {
if [ -z "$BLAZE_MEMORY_CAP" ]; then
system_memory_in_mb=$(LC_ALL=C free -m | grep 'Mem:' | awk '{print $2}');
export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4));
fi
if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then
available_system_memory_chuncks=$((BLAZE_MEMORY_CAP / 1000))
if [ $available_system_memory_chuncks -eq 0 ]; then
log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower."
export BLAZE_RESOURCE_CACHE_CAP=128000;
else
export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chuncks * 312500))
fi
fi
}
# Takes 1) The Backup Directory Path 2) The name of the Service to be backuped # Takes 1) The Backup Directory Path 2) The name of the Service to be backuped
# Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week # Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week
createEncryptedPostgresBackup(){ createEncryptedPostgresBackup(){

45
lib/prerequisites.sh
View File

@ -67,29 +67,30 @@ log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..."
source /etc/bridgehead/${PROJECT}.conf source /etc/bridgehead/${PROJECT}.conf
source ${PROJECT}/vars source ${PROJECT}/vars
set +e if [ "${PROJECT}" != "minimal" ]; then
SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" set +e
RET=$? SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
set -e RET=$?
if [ $RET -ne 0 ]; then set -e
log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." if [ $RET -ne 0 ]; then
log WARN "Unable to check clock skew due to previous error." log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication."
else log WARN "Unable to check clock skew due to previous error."
log INFO "Checking clock skew ..." else
log INFO "Checking clock skew ..."
SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s)
MYTIME=$(date +%s) MYTIME=$(date +%s)
SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME))
SKEW=$(echo $SKEW | awk -F- '{print $NF}') SKEW=$(echo $SKEW | awk -F- '{print $NF}')
SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd."
if [ $SKEW -ge 300 ]; then if [ $SKEW -ge 300 ]; then
report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT"
exit 1 exit 1
elif [ $SKEW -ge 60 ]; then elif [ $SKEW -ge 60 ]; then
log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT"
fi fi
fi
fi fi
checkPrivKey() { checkPrivKey() {
if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then
log INFO "Success - private key found." log INFO "Success - private key found."
@ -100,7 +101,7 @@ checkPrivKey() {
return 0 return 0
} }
if [[ "$@" =~ "noprivkey" ]]; then if [[ "$@" =~ "noprivkey" || "${PROJECT}" != "minimal" ]]; then
log INFO "Skipping check for private key for now." log INFO "Skipping check for private key for now."
else else
checkPrivKey || exit 1 checkPrivKey || exit 1

1
minimal/modules/dnpm-node-compose.yml
View File

@ -6,6 +6,7 @@ services:
container_name: bridgehead-dnpm-backend container_name: bridgehead-dnpm-backend
environment: environment:
- ZPM_SITE=${ZPM_SITE} - ZPM_SITE=${ZPM_SITE}
- N_RANDOM_FILES=${DNPM_SYNTH_NUM}
volumes: volumes:
- /etc/bridgehead/dnpm:/bwhc_config:ro - /etc/bridgehead/dnpm:/bwhc_config:ro
- ${DNPM_DATA_DIR}:/bwhc_data - ${DNPM_DATA_DIR}:/bwhc_data

21
minimal/modules/dnpm-node-setup.sh
View File

@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then
log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" log ERROR "Mandatory variable DNPM_DATA_DIR not defined!"
exit 1 exit 1
fi fi
if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0}
echo "Override of landing page url already in place" if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then
else echo "Override of landing page url already in place"
echo "Adding override of landing page url" else
if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then echo "Adding override of landing page url"
echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then
else echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml else
fi echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
fi fi
fi
fi fi