diff --git a/README.md b/README.md index 012d28c..99f1157 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Additionally, your site might use its own proxy. You should discuss this with yo Note that git and Docker may also need to be configured to use this proxy. This is a job for your systems administrators. -If there is a site firewall, this needs to be configured so that git and Docker can reach the outside world. Another job for the systems administrators. +If there is a site firewall, this needs to be configured so that outgoing calls to the following URLs are allowed: *.dkfz.de, github.com, docker.io, *.docker.io, *.samply.de. Note for Ubuntu: Please note that the uncomplicated firewall (ufw) is known to conflict with Docker [here](https://github.com/chaifeng/ufw-docker). diff --git a/bbmri/directory-sync.sh b/bbmri/directory-sync.sh deleted file mode 100755 index 2eeef4a..0000000 --- a/bbmri/directory-sync.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -function dirSetup() { - if [ -n "$DS_DIRECTORY_USER_NAME" ]; then - log INFO "Directory sync setup detected -- will start directory sync service." - OVERRIDE+=" -f ./$PROJECT/directory-sync-compose.yml" - fi -} diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 07f1a2f..9bc05cc 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -1,5 +1,7 @@ version: "3.7" +# This includes only the shared persistence for BBMRI-ERIC and GBN. Federation components are included as modules, see vars. + services: blaze: image: docker.verbis.dkfz.de/cache/samply/blaze:latest @@ -19,42 +21,10 @@ services: - "traefik.http.routers.blaze_ccp.middlewares=ccp_b_strip,auth" - "traefik.http.routers.blaze_ccp.tls=true" - focus: - image: docker.verbis.dkfz.de/cache/samply/focus:main - container_name: bridgehead-focus - environment: - API_KEY: ${FOCUS_BEAM_SECRET_SHORT} - BEAM_APP_ID_LONG: focus.${PROXY_ID} - PROXY_ID: ${PROXY_ID} - BLAZE_URL: "http://bridgehead-bbmri-blaze:8080/fhir/" - BEAM_PROXY_URL: http://beam-proxy:8081 - RETRY_COUNT: ${FOCUS_RETRY_COUNT} - depends_on: - - "beam-proxy" - - "blaze" - - beam-proxy: - image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop - container_name: bridgehead-beam-proxy - environment: - BROKER_URL: ${BROKER_URL} - PROXY_ID: ${PROXY_ID} - APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT} - PRIVKEY_FILE: /run/secrets/proxy.pem - ALL_PROXY: http://forward_proxy:3128 - TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs - ROOTCERT_FILE: /conf/root.crt.pem - secrets: - - proxy.pem - depends_on: - - "forward_proxy" - volumes: - - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - - /srv/docker/bridgehead/bbmri/root.crt.pem:/conf/root.crt.pem:ro - volumes: blaze-data: +# used in modules *-locator.yml secrets: proxy.pem: file: /etc/bridgehead/pki/${SITE_ID}.priv.pem diff --git a/bbmri/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml similarity index 100% rename from bbmri/directory-sync-compose.yml rename to bbmri/modules/directory-sync-compose.yml diff --git a/bbmri/modules/directory-sync.sh b/bbmri/modules/directory-sync.sh new file mode 100755 index 0000000..2765dfb --- /dev/null +++ b/bbmri/modules/directory-sync.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ -n "${DS_DIRECTORY_USER_NAME}" ]; then + log INFO "Directory sync setup detected -- will start directory sync service." + OVERRIDE+=" -f ./$PROJECT/modules/directory-sync-compose.yml" +fi diff --git a/bbmri/modules/dnpm-setup.sh b/bbmri/modules/dnpm-setup.sh index 7b3be9a..ce37632 100644 --- a/bbmri/modules/dnpm-setup.sh +++ b/bbmri/modules/dnpm-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -n "${ENABLE_DNPM}" ]; then - log DEBUG "DNPM setup detected (Beam.Connect) -- will start Beam and Beam.Connect for DNPM." + log INFO "DNPM setup detected (Beam.Connect) -- will start Beam and Beam.Connect for DNPM." OVERRIDE+=" -f ./$PROJECT/modules/dnpm-compose.yml" # Set variables required for Beam-Connect diff --git a/bbmri/modules/eric-compose.yml b/bbmri/modules/eric-compose.yml new file mode 100644 index 0000000..2bf7dd6 --- /dev/null +++ b/bbmri/modules/eric-compose.yml @@ -0,0 +1,36 @@ +version: "3.7" + +services: + focus-eric: + image: docker.verbis.dkfz.de/cache/samply/focus:main + container_name: bridgehead-focus-eric + environment: + API_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT} + BEAM_APP_ID_LONG: focus.${ERIC_PROXY_ID} + PROXY_ID: ${ERIC_PROXY_ID} + BLAZE_URL: "http://blaze:8080/fhir/" + BEAM_PROXY_URL: http://beam-proxy-eric:8081 + RETRY_COUNT: ${FOCUS_RETRY_COUNT} + depends_on: + - "beam-proxy-eric" + - "blaze" + + beam-proxy-eric: + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop + container_name: bridgehead-beam-proxy-eric + environment: + BROKER_URL: ${ERIC_BROKER_URL} + PROXY_ID: ${ERIC_PROXY_ID} + APP_focus_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT} + PRIVKEY_FILE: /run/secrets/proxy.pem + ALL_PROXY: http://forward_proxy:3128 + TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs + ROOTCERT_FILE: /conf/root.crt.pem + secrets: + - proxy.pem + depends_on: + - "forward_proxy" + volumes: + - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro + - /srv/docker/bridgehead/bbmri/modules/eric.root.crt.pem:/conf/root.crt.pem:ro + diff --git a/bbmri/modules/eric-setup.sh b/bbmri/modules/eric-setup.sh new file mode 100644 index 0000000..364281b --- /dev/null +++ b/bbmri/modules/eric-setup.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ "${ENABLE_ERIC}" == "true" ]; then + log INFO "BBMRI-ERIC setup detected -- will start services for BBMRI-ERIC." + OVERRIDE+=" -f ./$PROJECT/modules/eric-compose.yml" + + # Set required variables + ERIC_BROKER_ID=broker.bbmri.samply.de + ERIC_BROKER_URL=https://${ERIC_BROKER_ID} + ERIC_PROXY_ID=${SITE_ID}.${ERIC_BROKER_ID} + ERIC_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + ERIC_SUPPORT_EMAIL=bridgehead@helpdesk.bbmri-eric.eu +fi diff --git a/bbmri/root.crt.pem b/bbmri/modules/eric.root.crt.pem similarity index 100% rename from bbmri/root.crt.pem rename to bbmri/modules/eric.root.crt.pem diff --git a/bbmri/modules/gbn-compose.yml b/bbmri/modules/gbn-compose.yml new file mode 100644 index 0000000..47b1d36 --- /dev/null +++ b/bbmri/modules/gbn-compose.yml @@ -0,0 +1,36 @@ +version: "3.7" + +services: + focus-gbn: + image: docker.verbis.dkfz.de/cache/samply/focus:main + container_name: bridgehead-focus-gbn + environment: + API_KEY: ${GBN_FOCUS_BEAM_SECRET_SHORT} + BEAM_APP_ID_LONG: focus.${GBN_PROXY_ID} + PROXY_ID: ${GBN_PROXY_ID} + BLAZE_URL: "http://blaze:8080/fhir/" + BEAM_PROXY_URL: http://beam-proxy-gbn:8081 + RETRY_COUNT: ${FOCUS_RETRY_COUNT} + depends_on: + - "beam-proxy-gbn" + - "blaze" + + beam-proxy-gbn: + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop + container_name: bridgehead-beam-proxy-gbn + environment: + BROKER_URL: ${GBN_BROKER_URL} + PROXY_ID: ${GBN_PROXY_ID} + APP_focus_KEY: ${GBN_FOCUS_BEAM_SECRET_SHORT} + PRIVKEY_FILE: /run/secrets/proxy.pem + ALL_PROXY: http://forward_proxy:3128 + TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs + ROOTCERT_FILE: /conf/root.crt.pem + secrets: + - proxy.pem + depends_on: + - "forward_proxy" + volumes: + - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro + - /srv/docker/bridgehead/bbmri/modules/gbn.root.crt.pem:/conf/root.crt.pem:ro + diff --git a/bbmri/modules/gbn-setup.sh b/bbmri/modules/gbn-setup.sh new file mode 100644 index 0000000..b032c8e --- /dev/null +++ b/bbmri/modules/gbn-setup.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ "${ENABLE_GBN}" == "true" ]; then + log INFO "GBN setup detected -- will start services for German Biobank Node." + OVERRIDE+=" -f ./$PROJECT/modules/gbn-compose.yml" + + # Set required variables + GBN_BROKER_ID='#TODO#' + GBN_BROKER_URL=https://${GBN_BROKER_ID} + GBN_PROXY_ID=${SITE_ID}.${GBN_BROKER_ID} + GBN_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + GBN_SUPPORT_EMAIL=todo@verbis.dkfz.de +fi diff --git a/bbmri/modules/gbn.root.crt.pem b/bbmri/modules/gbn.root.crt.pem new file mode 100644 index 0000000..0c2ad6f --- /dev/null +++ b/bbmri/modules/gbn.root.crt.pem @@ -0,0 +1 @@ +#TODO# diff --git a/bbmri/vars b/bbmri/vars index 21aeaec..1cb6128 100644 --- a/bbmri/vars +++ b/bbmri/vars @@ -1,9 +1,10 @@ -BROKER_ID=broker.bbmri.samply.de -BROKER_URL=https://${BROKER_ID} -PROXY_ID=${SITE_ID}.${BROKER_ID} -FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" +# Makes sense for all European Biobanks +: ${ENABLE_ERIC:=true} + +# Makes only sense for German Biobanks +: ${ENABLE_GBN:=false} + FOCUS_RETRY_COUNT=32 -SUPPORT_EMAIL=bridgehead@helpdesk.bbmri-eric.eu PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem for module in $PROJECT/modules/*.sh @@ -12,6 +13,25 @@ do source $module done -# This will load directory-sync setup. -source $PROJECT/directory-sync.sh -dirSetup +SUPPORT_EMAIL=$ERIC_SUPPORT_EMAIL +BROKER_URL_FOR_PREREQ=$ERIC_BROKER_URL + +if [ -n "$GBN_SUPPORT_EMAIL" ]; then + SUPPORT_EMAIL=$GBN_SUPPORT_EMAIL +fi + +function do_enroll { + COUNT=0 + if [ "$ENABLE_ERIC" == "true" ]; then + do_enroll_inner $ERIC_PROXY_ID $ERIC_SUPPORT_EMAIL + COUNT=$((COUNT+1)) + fi + if [ "$ENABLE_GBN" == "true" ]; then + do_enroll_inner $GBN_PROXY_ID $GBN_SUPPORT_EMAIL + COUNT=$((COUNT+1)) + fi + if [ $COUNT -ge 2 ]; then + echo + echo "You just received $COUNT certificate signing requests (CSR). Please send $COUNT e-mails, with 1 CSR each, to the respective e-mail address." + fi +} \ No newline at end of file diff --git a/bridgehead b/bridgehead index 4f023fa..192cc0e 100755 --- a/bridgehead +++ b/bridgehead @@ -113,22 +113,7 @@ case "$ACTION" in enroll) loadVars - MANUAL_PROXY_ID="${3:-$PROXY_ID}" - if [ -z "$MANUAL_PROXY_ID" ]; then - log ERROR "No Proxy ID set" - exit 1 - else - log INFO "Enrolling Beam Proxy Id $MANUAL_PROXY_ID" - fi - - if [ -z "$SUPPORT_EMAIL" ]; then - EMAIL_PARAM="" - else - EMAIL_PARAM="--admin-email $SUPPORT_EMAIL" - fi - - docker run --rm -ti -v /etc/bridgehead/pki:/etc/bridgehead/pki samply/beam-enroll:latest --output-file $PRIVATEKEYFILENAME --proxy-id $MANUAL_PROXY_ID $EMAIL_PARAM - chmod 600 $PRIVATEKEYFILENAME + do_enroll $PROXY_ID ;; preRun | preUpdate) fixPermissions diff --git a/ccp/modules/dnpm-setup.sh b/ccp/modules/dnpm-setup.sh index 04659eb..b3dd636 100644 --- a/ccp/modules/dnpm-setup.sh +++ b/ccp/modules/dnpm-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -n "${ENABLE_DNPM}" ]; then - log DEBUG "DNPM setup detected (Beam.Connect) -- will start Beam.Connect for DNPM." + log INFO "DNPM setup detected (Beam.Connect) -- will start Beam.Connect for DNPM." OVERRIDE+=" -f ./$PROJECT/modules/dnpm-compose.yml" # Set variables required for Beam-Connect diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index be1375e..f7a48fd 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -5,6 +5,7 @@ services: container_name: bridgehead-id-manager environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST} + TOMCAT_REVERSEPROXY_SSL: "true" MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID} MAGICPL_ALLOWED_ORIGINS: https://${HOST} MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} diff --git a/ccp/vars b/ccp/vars index 420b0d0..61d5d2e 100644 --- a/ccp/vars +++ b/ccp/vars @@ -6,6 +6,8 @@ FOCUS_RETRY_COUNT=32 SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +BROKER_URL_FOR_PREREQ=$BROKER_URL + # This will load id-management setup. Effective only if id-management configuration is defined. source $PROJECT/modules/id-management-setup.sh idManagementSetup diff --git a/lib/functions.sh b/lib/functions.sh index d6c1b85..9173e86 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -172,24 +172,43 @@ function bk_is_running { fi } -##Setting Network properties -# currently not needed -#export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); +function do_enroll_inner { + PARAMS="" + + MANUAL_PROXY_ID="${1:-$PROXY_ID}" + if [ -z "$MANUAL_PROXY_ID" ]; then + log ERROR "No Proxy ID set" + exit 1 + else + log INFO "Enrolling Beam Proxy Id $MANUAL_PROXY_ID" + fi + SUPPORT_EMAIL="${2:-$SUPPORT_EMAIL}" + if [ -n "$SUPPORT_EMAIL" ]; then + PARAMS+="--admin-email $SUPPORT_EMAIL" + fi + + docker run --rm -ti -v /etc/bridgehead/pki:/etc/bridgehead/pki samply/beam-enroll:latest --output-file $PRIVATEKEYFILENAME --proxy-id $MANUAL_PROXY_ID $PARAMS + chmod 600 $PRIVATEKEYFILENAME +} + +function do_enroll { + do_enroll_inner $@ +} add_basic_auth_user() { - USER="${1}" - PASSWORD="${2}" - NAME="${3}" - PROJECT="${4}" - FILE="/etc/bridgehead/${PROJECT}.local.conf" - ENCRY_CREDENTIALS="$(docker run --rm docker.verbis.dkfz.de/cache/httpd:alpine htpasswd -nb $USER $PASSWORD | tr -d '\n' | tr -d '\r')" - if [ -f $FILE ] && grep -R -q "$NAME=" $FILE # if a specific basic auth user already exists: - then - sed -i "/$NAME/ s|='|='$ENCRY_CREDENTIALS,|" $FILE - else - echo -e "\n## Basic Authentication Credentials for:\n$NAME='$ENCRY_CREDENTIALS'" >> $FILE; - fi - log DEBUG "Saving clear text credentials in $FILE. If wanted, delete them manually." - sed -i "/^$NAME/ s|$|\n# User: $USER\n# Password: $PASSWORD|" $FILE + USER="${1}" + PASSWORD="${2}" + NAME="${3}" + PROJECT="${4}" + FILE="/etc/bridgehead/${PROJECT}.local.conf" + ENCRY_CREDENTIALS="$(docker run --rm docker.verbis.dkfz.de/cache/httpd:alpine htpasswd -nb $USER $PASSWORD | tr -d '\n' | tr -d '\r')" + if [ -f $FILE ] && grep -R -q "$NAME=" $FILE # if a specific basic auth user already exists: + then + sed -i "/$NAME/ s|='|='$ENCRY_CREDENTIALS,|" $FILE + else + echo -e "\n## Basic Authentication Credentials for:\n$NAME='$ENCRY_CREDENTIALS'" >> $FILE; + fi + log DEBUG "Saving clear text credentials in $FILE. If wanted, delete them manually." + sed -i "/^$NAME/ s|$|\n# User: $USER\n# Password: $PASSWORD|" $FILE } \ No newline at end of file diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index 1bddf3e..cd470b2 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -71,8 +71,12 @@ if [ -d /etc/bridgehead ]; then fi elif [[ "$DEV_MODE" == "NODEV" ]]; then log "INFO" "Now cloning your site configuration repository for you." - read -p "Please enter your site: " site - read -s -p "Please enter the bridgehead's access token for your site configuration repository (will not be echoed): " access_token + if [ -z "$site" ]; then + read -p "Please enter your site: " site + fi + if [ -z "$access_token" ]; then + read -s -p "Please enter the bridgehead's access token for your site configuration repository (will not be echoed): " access_token + fi site_configuration_repository_url="https://bytoken:${access_token}@${site_configuration_repository_middle}$(echo $site | tr '[:upper:]' '[:lower:]').git" git clone $site_configuration_repository_url /etc/bridgehead if [ $? -gt 0 ]; then diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 8ce7051..062f065 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -62,6 +62,34 @@ if [ -e /etc/bridgehead/vault.conf ]; then fi fi +log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..." + +source /etc/bridgehead/${PROJECT}.conf +source ${PROJECT}/vars + +set +e +SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" +RET=$? +set -e +if [ $RET -ne 0 ]; then + log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." + log WARN "Unable to check clock skew due to previous error." +else + log INFO "Checking clock skew ..." + + SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) + MYTIME=$(date +%s) + SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) + SKEW=$(echo $SKEW | awk -F- '{print $NF}') + SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." + if [ $SKEW -ge 300 ]; then + report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" + exit 1 + elif [ $SKEW -ge 60 ]; then + log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" + fi +fi + checkPrivKey() { if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then log INFO "Success - private key found." @@ -69,8 +97,6 @@ checkPrivKey() { log ERROR "Unable to find private key at /etc/bridgehead/pki/${SITE_ID}.priv.pem. To fix, please run\n bridgehead enroll ${PROJECT}\nand follow the instructions." return 1 fi - log INFO "Success - all prerequisites are met!" - hc_send log "Success - all prerequisites are met!" return 0 } @@ -80,4 +106,7 @@ else checkPrivKey || exit 1 fi +log INFO "Success - all prerequisites are met!" +hc_send log "Success - all prerequisites are met!" + exit 0 diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index 055f683..c0eb353 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -37,6 +37,9 @@ services: HTTPS_PROXY: ${HTTPS_PROXY_URL} USERNAME: ${HTTPS_PROXY_USERNAME} PASSWORD: ${HTTPS_PROXY_PASSWORD} + tmpfs: + - /var/log/squid + - /var/spool/squid volumes: - /etc/bridgehead/trusted-ca-certs:/docker/custom-certs/:ro