Merge branch 'main' into feature/custom-basic-auth

README.md

@@ -64,7 +64,7 @@ Additionally, your site might use its own proxy. You should discuss this with yo
 
 Note that git and Docker may also need to be configured to use this proxy. This is a job for your systems administrators.
 
-If there is a site firewall, this needs to be configured so that git and Docker can reach the outside world. Another job for the systems administrators.
+If there is a site firewall, this needs to be configured so that outgoing calls to the following URLs are allowed: *.dkfz.de, github.com, docker.io, *.docker.io, *.samply.de.
 
 Note for Ubuntu: Please note that the uncomplicated firewall (ufw) is known to conflict with Docker [here](https://github.com/chaifeng/ufw-docker).
 

bbmri/directory-sync.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-function dirSetup() {
-	if [ -n "$DS_DIRECTORY_USER_NAME" ]; then
-		log INFO "Directory sync setup detected -- will start directory sync service."
-		OVERRIDE+=" -f ./$PROJECT/directory-sync-compose.yml"
-	fi
-}

bbmri/docker-compose.yml

@@ -1,5 +1,7 @@
 version: "3.7"
 
+# This includes only the shared persistence for BBMRI-ERIC and GBN. Federation components are included as modules, see vars.
+
 services:
   blaze:
     image: docker.verbis.dkfz.de/cache/samply/blaze:latest
@@ -19,42 +21,10 @@ services:
       - "traefik.http.routers.blaze_ccp.middlewares=ccp_b_strip,auth"
       - "traefik.http.routers.blaze_ccp.tls=true"
 
-  focus:
-    image: docker.verbis.dkfz.de/cache/samply/focus:main
-    container_name: bridgehead-focus
-    environment:
-      API_KEY: ${FOCUS_BEAM_SECRET_SHORT}
-      BEAM_APP_ID_LONG: focus.${PROXY_ID}
-      PROXY_ID: ${PROXY_ID}
-      BLAZE_URL: "http://bridgehead-bbmri-blaze:8080/fhir/"
-      BEAM_PROXY_URL: http://beam-proxy:8081
-      RETRY_COUNT: ${FOCUS_RETRY_COUNT}
-    depends_on:
-      - "beam-proxy"
-      - "blaze"
-
-  beam-proxy:
-    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop
-    container_name: bridgehead-beam-proxy
-    environment:
-      BROKER_URL: ${BROKER_URL}
-      PROXY_ID: ${PROXY_ID}
-      APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT}
-      PRIVKEY_FILE: /run/secrets/proxy.pem
-      ALL_PROXY: http://forward_proxy:3128
-      TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
-      ROOTCERT_FILE: /conf/root.crt.pem
-    secrets:
-      - proxy.pem
-    depends_on:
-      - "forward_proxy"
-    volumes:
-      - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
-      - /srv/docker/bridgehead/bbmri/root.crt.pem:/conf/root.crt.pem:ro
-
 volumes:
   blaze-data:
 
+# used in modules *-locator.yml
 secrets:
   proxy.pem:
     file: /etc/bridgehead/pki/${SITE_ID}.priv.pem

bbmri/modules/directory-sync.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+if [ -n "${DS_DIRECTORY_USER_NAME}" ]; then
+	log INFO "Directory sync setup detected -- will start directory sync service."
+	OVERRIDE+=" -f ./$PROJECT/modules/directory-sync-compose.yml"
+fi

bbmri/modules/dnpm-setup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [ -n "${ENABLE_DNPM}" ]; then
-	log DEBUG "DNPM setup detected (Beam.Connect) -- will start Beam and Beam.Connect for DNPM."
+	log INFO "DNPM setup detected (Beam.Connect) -- will start Beam and Beam.Connect for DNPM."
 	OVERRIDE+=" -f ./$PROJECT/modules/dnpm-compose.yml"
 
 	# Set variables required for Beam-Connect

bbmri/modules/eric-compose.yml

@@ -0,0 +1,36 @@
+version: "3.7"
+
+services:
+  focus-eric:
+    image: docker.verbis.dkfz.de/cache/samply/focus:main
+    container_name: bridgehead-focus-eric
+    environment:
+      API_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT}
+      BEAM_APP_ID_LONG: focus.${ERIC_PROXY_ID}
+      PROXY_ID: ${ERIC_PROXY_ID}
+      BLAZE_URL: "http://blaze:8080/fhir/"
+      BEAM_PROXY_URL: http://beam-proxy-eric:8081
+      RETRY_COUNT: ${FOCUS_RETRY_COUNT}
+    depends_on:
+      - "beam-proxy-eric"
+      - "blaze"
+
+  beam-proxy-eric:
+    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop
+    container_name: bridgehead-beam-proxy-eric
+    environment:
+      BROKER_URL: ${ERIC_BROKER_URL}
+      PROXY_ID: ${ERIC_PROXY_ID}
+      APP_focus_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT}
+      PRIVKEY_FILE: /run/secrets/proxy.pem
+      ALL_PROXY: http://forward_proxy:3128
+      TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
+      ROOTCERT_FILE: /conf/root.crt.pem
+    secrets:
+      - proxy.pem
+    depends_on:
+      - "forward_proxy"
+    volumes:
+      - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
+      - /srv/docker/bridgehead/bbmri/modules/eric.root.crt.pem:/conf/root.crt.pem:ro
+

bbmri/modules/eric-setup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ "${ENABLE_ERIC}" == "true" ]; then
+	log INFO "BBMRI-ERIC setup detected -- will start services for BBMRI-ERIC."
+	OVERRIDE+=" -f ./$PROJECT/modules/eric-compose.yml"
+
+	# Set required variables
+	ERIC_BROKER_ID=broker.bbmri.samply.de
+	ERIC_BROKER_URL=https://${ERIC_BROKER_ID}
+	ERIC_PROXY_ID=${SITE_ID}.${ERIC_BROKER_ID}
+	ERIC_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
+	ERIC_SUPPORT_EMAIL=bridgehead@helpdesk.bbmri-eric.eu
+fi

bbmri/modules/gbn-compose.yml

@@ -0,0 +1,36 @@
+version: "3.7"
+
+services:
+  focus-gbn:
+    image: docker.verbis.dkfz.de/cache/samply/focus:main
+    container_name: bridgehead-focus-gbn
+    environment:
+      API_KEY: ${GBN_FOCUS_BEAM_SECRET_SHORT}
+      BEAM_APP_ID_LONG: focus.${GBN_PROXY_ID}
+      PROXY_ID: ${GBN_PROXY_ID}
+      BLAZE_URL: "http://blaze:8080/fhir/"
+      BEAM_PROXY_URL: http://beam-proxy-gbn:8081
+      RETRY_COUNT: ${FOCUS_RETRY_COUNT}
+    depends_on:
+      - "beam-proxy-gbn"
+      - "blaze"
+
+  beam-proxy-gbn:
+    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop
+    container_name: bridgehead-beam-proxy-gbn
+    environment:
+      BROKER_URL: ${GBN_BROKER_URL}
+      PROXY_ID: ${GBN_PROXY_ID}
+      APP_focus_KEY: ${GBN_FOCUS_BEAM_SECRET_SHORT}
+      PRIVKEY_FILE: /run/secrets/proxy.pem
+      ALL_PROXY: http://forward_proxy:3128
+      TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
+      ROOTCERT_FILE: /conf/root.crt.pem
+    secrets:
+      - proxy.pem
+    depends_on:
+      - "forward_proxy"
+    volumes:
+      - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
+      - /srv/docker/bridgehead/bbmri/modules/gbn.root.crt.pem:/conf/root.crt.pem:ro
+

bbmri/modules/gbn-setup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ "${ENABLE_GBN}" == "true" ]; then
+	log INFO "GBN setup detected -- will start services for German Biobank Node."
+	OVERRIDE+=" -f ./$PROJECT/modules/gbn-compose.yml"
+
+	# Set required variables
+	GBN_BROKER_ID='#TODO#'
+	GBN_BROKER_URL=https://${GBN_BROKER_ID}
+	GBN_PROXY_ID=${SITE_ID}.${GBN_BROKER_ID}
+	GBN_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
+	GBN_SUPPORT_EMAIL=todo@verbis.dkfz.de
+fi

bbmri/modules/gbn.root.crt.pem

@@ -0,0 +1 @@
+#TODO#

bbmri/vars

@@ -1,9 +1,10 @@
-BROKER_ID=broker.bbmri.samply.de
-BROKER_URL=https://${BROKER_ID}
-PROXY_ID=${SITE_ID}.${BROKER_ID}
-FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
+# Makes sense for all European Biobanks
+: ${ENABLE_ERIC:=true}
+
+# Makes only sense for German Biobanks
+: ${ENABLE_GBN:=false}
+
 FOCUS_RETRY_COUNT=32
-SUPPORT_EMAIL=bridgehead@helpdesk.bbmri-eric.eu
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 
 for module in $PROJECT/modules/*.sh
@@ -12,6 +13,25 @@ do
     source $module
 done
 
-# This will load directory-sync setup.
-source $PROJECT/directory-sync.sh
-dirSetup
+SUPPORT_EMAIL=$ERIC_SUPPORT_EMAIL
+BROKER_URL_FOR_PREREQ=$ERIC_BROKER_URL
+
+if [ -n "$GBN_SUPPORT_EMAIL" ]; then
+    SUPPORT_EMAIL=$GBN_SUPPORT_EMAIL
+fi
+
+function do_enroll {
+    COUNT=0
+    if [ "$ENABLE_ERIC" == "true" ]; then
+        do_enroll_inner $ERIC_PROXY_ID $ERIC_SUPPORT_EMAIL
+        COUNT=$((COUNT+1))
+    fi
+    if [ "$ENABLE_GBN" == "true" ]; then
+        do_enroll_inner $GBN_PROXY_ID $GBN_SUPPORT_EMAIL
+        COUNT=$((COUNT+1))
+    fi
+    if [ $COUNT -ge 2 ]; then
+        echo
+        echo "You just received $COUNT certificate signing requests (CSR). Please send $COUNT e-mails, with 1 CSR each, to the respective e-mail address."
+    fi
+}

bridgehead

@@ -113,22 +113,7 @@ case "$ACTION" in
 	enroll)
 		loadVars
 
-		MANUAL_PROXY_ID="${3:-$PROXY_ID}"
-		if [ -z "$MANUAL_PROXY_ID" ]; then
-			log ERROR "No Proxy ID set"
-			exit 1
-		else
-			log INFO "Enrolling Beam Proxy Id $MANUAL_PROXY_ID"
-		fi
-
-		if [ -z "$SUPPORT_EMAIL" ]; then
-			EMAIL_PARAM=""
-		else
-			EMAIL_PARAM="--admin-email $SUPPORT_EMAIL"
-		fi
-			
-		docker run --rm -ti -v /etc/bridgehead/pki:/etc/bridgehead/pki samply/beam-enroll:latest --output-file $PRIVATEKEYFILENAME --proxy-id $MANUAL_PROXY_ID $EMAIL_PARAM
-		chmod 600 $PRIVATEKEYFILENAME
+		do_enroll $PROXY_ID
 		;;
 	preRun | preUpdate)
 		fixPermissions

ccp/modules/dnpm-setup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [ -n "${ENABLE_DNPM}" ]; then
-	log DEBUG "DNPM setup detected (Beam.Connect) -- will start Beam.Connect for DNPM."
+	log INFO "DNPM setup detected (Beam.Connect) -- will start Beam.Connect for DNPM."
 	OVERRIDE+=" -f ./$PROJECT/modules/dnpm-compose.yml"
 
 	# Set variables required for Beam-Connect

ccp/modules/id-management-compose.yml

@@ -5,6 +5,7 @@ services:
     container_name: bridgehead-id-manager
     environment:
       TOMCAT_REVERSEPROXY_FQDN: ${HOST}
+      TOMCAT_REVERSEPROXY_SSL: "true"
       MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID}
       MAGICPL_ALLOWED_ORIGINS: https://${HOST}
       MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}

ccp/vars

@@ -6,6 +6,8 @@ FOCUS_RETRY_COUNT=32
 SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 
+BROKER_URL_FOR_PREREQ=$BROKER_URL
+
 # This will load id-management setup. Effective only if id-management configuration is defined.
 source $PROJECT/modules/id-management-setup.sh
 idManagementSetup

lib/functions.sh

@@ -172,10 +172,29 @@ function bk_is_running {
 	fi
 }
 
-##Setting Network properties
-# currently not needed
-#export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}');
+function do_enroll_inner {
+	PARAMS=""
 	
+	MANUAL_PROXY_ID="${1:-$PROXY_ID}"
+	if [ -z "$MANUAL_PROXY_ID" ]; then
+		log ERROR "No Proxy ID set"
+		exit 1
+	else
+		log INFO "Enrolling Beam Proxy Id $MANUAL_PROXY_ID"
+	fi
+
+	SUPPORT_EMAIL="${2:-$SUPPORT_EMAIL}"
+	if [ -n "$SUPPORT_EMAIL" ]; then
+		PARAMS+="--admin-email $SUPPORT_EMAIL"
+	fi
+
+	docker run --rm -ti -v /etc/bridgehead/pki:/etc/bridgehead/pki samply/beam-enroll:latest --output-file $PRIVATEKEYFILENAME --proxy-id $MANUAL_PROXY_ID $PARAMS
+	chmod 600 $PRIVATEKEYFILENAME
+}
+
+function do_enroll {
+	do_enroll_inner $@
+}
 
 add_basic_auth_user() {
    USER="${1}"

lib/prepare-system.sh

@@ -71,8 +71,12 @@ if [ -d /etc/bridgehead ]; then
     fi
 elif [[ "$DEV_MODE" == "NODEV" ]]; then
     log "INFO" "Now cloning your site configuration repository for you."
+    if [ -z "$site" ]; then
         read -p "Please enter your site: " site
+    fi
+    if [ -z "$access_token" ]; then
         read -s -p "Please enter the bridgehead's access token for your site configuration repository (will not be echoed): " access_token
+    fi
     site_configuration_repository_url="https://bytoken:${access_token}@${site_configuration_repository_middle}$(echo $site | tr '[:upper:]' '[:lower:]').git"
     git clone $site_configuration_repository_url /etc/bridgehead
     if [ $? -gt 0 ]; then

lib/prerequisites.sh

@@ -62,6 +62,34 @@ if [ -e /etc/bridgehead/vault.conf ]; then
   fi
 fi
 
+log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..."
+
+source /etc/bridgehead/${PROJECT}.conf
+source ${PROJECT}/vars
+
+set +e
+SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
+RET=$?
+set -e
+if [ $RET -ne 0 ]; then
+	log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication."
+	log WARN "Unable to check clock skew due to previous error."
+else
+	log INFO "Checking clock skew ..."
+
+	SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s)
+	MYTIME=$(date +%s)
+	SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME))
+	SKEW=$(echo $SKEW | awk -F- '{print $NF}')
+	SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd."
+	if [ $SKEW -ge 300 ]; then
+		report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT"
+		exit 1
+	elif [ $SKEW -ge 60 ]; then
+		log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT"
+	fi
+fi
+
 checkPrivKey() {
   if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then
     log INFO "Success - private key found."
@@ -69,8 +97,6 @@ checkPrivKey() {
     log ERROR "Unable to find private key at /etc/bridgehead/pki/${SITE_ID}.priv.pem. To fix, please run\n  bridgehead enroll ${PROJECT}\nand follow the instructions."
     return 1
   fi
-  log INFO "Success - all prerequisites are met!"
-  hc_send log "Success - all prerequisites are met!"
   return 0
 }
 
@@ -80,4 +106,7 @@ else
   checkPrivKey || exit 1
 fi
 
+log INFO "Success - all prerequisites are met!"
+hc_send log "Success - all prerequisites are met!"
+
 exit 0

minimal/docker-compose.yml

@@ -37,6 +37,9 @@ services:
       HTTPS_PROXY: ${HTTPS_PROXY_URL}
       USERNAME: ${HTTPS_PROXY_USERNAME}
       PASSWORD: ${HTTPS_PROXY_PASSWORD}
+    tmpfs:
+      - /var/log/squid
+      - /var/spool/squid
     volumes:
       - /etc/bridgehead/trusted-ca-certs:/docker/custom-certs/:ro