From 054d71538da4d3344b03442a103ea039a2e66fdd Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Wed, 28 Sep 2022 17:37:11 +0200 Subject: [PATCH 01/24] Check time sync in prereqs --- lib/prerequisites.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 4e03530..a89aa54 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -65,6 +65,16 @@ if [ -e /etc/bridgehead/vault.conf ]; then fi fi +if ! which timedatectl > /dev/null; then + log ERROR "systemd time sync is missing -- please install package containing timedatectl" + exit 1 +fi + +if ! timedatectl show | grep NTPSynchronized=yes >/dev/null; then + log ERROR "This server's clock is not synchronized with a time server. This will cause Samply.Beam's certificate will fail. Please enter a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." + exit 1 +fi + log INFO "Success - all prerequisites are met!" exit 0 From 7d13eace3233a74fcc4ec5fb7ae02d230bfd9f11 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Fri, 30 Sep 2022 17:02:05 +0200 Subject: [PATCH 02/24] Check clock skew even smarter --- lib/prerequisites.sh | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index a89aa54..b338d7c 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -65,13 +65,28 @@ if [ -e /etc/bridgehead/vault.conf ]; then fi fi -if ! which timedatectl > /dev/null; then - log ERROR "systemd time sync is missing -- please install package containing timedatectl" +log INFO "Checking network access ($BROKER_URL) ..." + +source /etc/bridgehead/${PROJECT}.conf +source ${PROJECT}/vars + +set +e +SERVERTIME=$(curl -I $BROKER_URL) +if [ $? -ne 0 ]; then + log ERROR "Unable to connect to Samply.Beam broker at $BROKER_URL. Please check your proxy settings." exit 1 fi +set -e -if ! timedatectl show | grep NTPSynchronized=yes >/dev/null; then - log ERROR "This server's clock is not synchronized with a time server. This will cause Samply.Beam's certificate will fail. Please enter a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." +log INFO "Checking clock skew ..." + +SERVERTIME=$(echo $SERVERTIME | grep -i ^Date: | cut -d: -f2- | sed 's/^ *\(.*\).*/\1/') +SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) +MYTIME=$(date +%s) +SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) +SKEW=$(echo $SKEW | awk -F- '{print $NF}') +if [ $SKEW -ge 300 ]; then + log ERROR "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." exit 1 fi From 1d05137bb9cb99e56b246512e1936edfb99f9ba7 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:13:28 +0200 Subject: [PATCH 03/24] Use proxy in time check --- lib/prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index b338d7c..9a1e39f 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -71,7 +71,7 @@ source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars set +e -SERVERTIME=$(curl -I $BROKER_URL) +SERVERTIME=$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -I $BROKER_URL) if [ $? -ne 0 ]; then log ERROR "Unable to connect to Samply.Beam broker at $BROKER_URL. Please check your proxy settings." exit 1 From f52012008d22d4fab53cb1418234b344e1b6aacf Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:36:07 +0200 Subject: [PATCH 04/24] Update prerequisites.sh --- lib/prerequisites.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 9a1e39f..28958b3 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -71,16 +71,16 @@ source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars set +e -SERVERTIME=$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -I $BROKER_URL) +SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -v $BROKER_URL 2>&1)" if [ $? -ne 0 ]; then - log ERROR "Unable to connect to Samply.Beam broker at $BROKER_URL. Please check your proxy settings." + log ERROR "Unable to connect to Samply.Beam broker at $BROKER_URL. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\"." exit 1 fi set -e log INFO "Checking clock skew ..." -SERVERTIME=$(echo $SERVERTIME | grep -i ^Date: | cut -d: -f2- | sed 's/^ *\(.*\).*/\1/') +SERVERTIME=$(echo -e "$SERVERTIME" | grep Date | sed -e 's/< Date: //') SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) MYTIME=$(date +%s) SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) @@ -88,6 +88,8 @@ SKEW=$(echo $SKEW | awk -F- '{print $NF}') if [ $SKEW -ge 300 ]; then log ERROR "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." exit 1 +elif [ $SKEW -ge 60 ]; then + log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server." fi log INFO "Success - all prerequisites are met!" From 091402eea03ebab5159cce0eb33eaafdd932ee83 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:42:53 +0200 Subject: [PATCH 05/24] Update prerequisites.sh --- lib/prerequisites.sh | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 28958b3..0bb7882 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -72,24 +72,26 @@ source ${PROJECT}/vars set +e SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -v $BROKER_URL 2>&1)" -if [ $? -ne 0 ]; then - log ERROR "Unable to connect to Samply.Beam broker at $BROKER_URL. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\"." - exit 1 -fi +RET=$? set -e +if [ $RET -ne 0 ]; then + log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." + log WARN "Unable to check clock skew due to previous error." +else + log INFO "Checking clock skew ..." -log INFO "Checking clock skew ..." - -SERVERTIME=$(echo -e "$SERVERTIME" | grep Date | sed -e 's/< Date: //') -SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) -MYTIME=$(date +%s) -SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) -SKEW=$(echo $SKEW | awk -F- '{print $NF}') -if [ $SKEW -ge 300 ]; then - log ERROR "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." - exit 1 -elif [ $SKEW -ge 60 ]; then - log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server." + SERVERTIME=$(echo -e "$SERVERTIME" | grep Date | sed -e 's/< Date: //') + SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) + MYTIME=$(date +%s) + SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) + SKEW=$(echo $SKEW | awk -F- '{print $NF}') + SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." + if [ $SKEW -ge 300 ]; then + log ERROR "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" + exit 1 + elif [ $SKEW -ge 60 ]; then + log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" + fi fi log INFO "Success - all prerequisites are met!" From a23f1ae07593838e2bd1e2e14c7924556dc93972 Mon Sep 17 00:00:00 2001 From: lablans Date: Wed, 22 Feb 2023 12:06:28 +0000 Subject: [PATCH 06/24] Automate installation --- lib/prepare-system.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index 2cba2e2..a0a0215 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -52,8 +52,12 @@ if [ -d /etc/bridgehead ]; then fi else log "INFO" "Now cloning your site configuration repository for you." - read -p "Please enter your site: " site - read -s -p "Please enter the bridgehead's access token for your site configuration repository (will not be echoed): " access_token + if [ -z "$site" ]; then + read -p "Please enter your site: " site + fi + if [ -z "$access_token" ]; then + read -s -p "Please enter the bridgehead's access token for your site configuration repository (will not be echoed): " access_token + fi site_configuration_repository_url="https://bytoken:${access_token}@${site_configuration_repository_middle}$(echo $site | tr '[:upper:]' '[:lower:]').git" git clone $site_configuration_repository_url /etc/bridgehead if [ $? -gt 0 ]; then From b311ff7831d476d86675a55f0785b3d5bb900749 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Wed, 12 Jul 2023 08:53:16 +0200 Subject: [PATCH 07/24] Docs: Move info about BBMRI-ERIC Directory --- README.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 0c7e1fb..012d28c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ This repository is the starting point for any information and tools you will nee - [Hardware](#hardware) - [Software](#software) - [Network](#network) - - [Register with the Directory](#register-with-the-directory) 2. [Deployment](#deployment) - [Site name](#site-name) - [Projects](#projects) @@ -22,6 +21,7 @@ This repository is the starting point for any information and tools you will nee - [HTTPS Access](#https-access) - [TLS terminating proxies](#tls-terminating-proxies) - [File structure](#file-structure) + - [BBMRI-ERIC Directory](#bbmri-eric-directory) 4. [Things you should know](#things-you-should-know) - [Auto-Updates](#auto-updates) - [Auto-Backups](#auto-backups) @@ -68,14 +68,6 @@ If there is a site firewall, this needs to be configured so that git and Docker Note for Ubuntu: Please note that the uncomplicated firewall (ufw) is known to conflict with Docker [here](https://github.com/chaifeng/ufw-docker). -### Register with the Directory - -If you run a biobank, you should register with the [Directory](https://directory.bbmri-eric.eu), a BBMRI project that catalogs biobanks. - -To do this, contact the BBMRI national node for the country where your biobank is based, see [the list of nodes](http://www.bbmri-eric.eu/national-nodes/). - -Once you have registered, **you should choose one of your sample collections as a default collection for your biobank**. This is the collection that will be automatically used to label any samples that have not been assigned a collection ID in your ETL process. Make a note of this ID, you will need it later on in the installation process. - ## Deployment ### Site name @@ -255,9 +247,15 @@ All of the Bridgehead's outgoing connections are secured by transport encryption Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. -### Directory sync +### BBMRI-ERIC Directory -This is an optional feature for bbmri projects. It keeps the [BBMRI Directory](https://directory.bbmri-eric.eu/) up to date with your local data eg. number of samples. It also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly set your country specific directory url, username and password to enable this feature. +If you run a biobank, you should register with the [Directory](https://directory.bbmri-eric.eu), a BBMRI-ERIC project that catalogs biobanks. + +To do this, contact the BBMRI-ERIC national node for the country where your biobank is based, see [the list of nodes](http://www.bbmri-eric.eu/national-nodes/). + +Once you have registered, **you should choose one of your sample collections as a default collection for your biobank**. This is the collection that will be automatically used to label any samples that have not been assigned a collection ID in your ETL process. Make a note of this ID, you will need it later on in the installation process. + +The Bridgehead's **Directory Sync** is an optional feature that keeps the Directory up to date with your local data, e.g. number of samples. Conversely, it also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly set your country specific directory url, username and password to enable this feature. Full details can be found in [directory_sync_service](https://github.com/samply/directory_sync_service). @@ -274,7 +272,7 @@ You must contact the Directory for your national node to find the URL, and to re Additionally, you should choose when you want Directory sync to run. In the example above, this is set to happen at 10 pm every evening. You can modify this to suit your requirements. The timer specification should follow the [cron](https://crontab.guru) convention. -Once you edited the gitlab config. The bridgehead will autoupdate the config with the values and will sync the data. +Once you edited the gitlab config, the bridgehead will autoupdate the config with the values and will sync the data. There will be a delay before the effects of Directory sync become visible. First, you will need to wait until the time you have specified in ```TIMER_CRON```. Second, the information will then be synchronized from your national node with the central European Directory. This can take up to 24 hours. From 0a12720e4cdfee34133eee9e0c993ccf878e5fd8 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 25 Jul 2023 13:27:21 +0200 Subject: [PATCH 08/24] fix: ensure id-management redirects with ssl --- ccp/modules/id-management-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index be1375e..f7a48fd 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -5,6 +5,7 @@ services: container_name: bridgehead-id-manager environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST} + TOMCAT_REVERSEPROXY_SSL: "true" MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID} MAGICPL_ALLOWED_ORIGINS: https://${HOST} MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} From 8c45e1da80d2e66bb54989afca25d9fe7468b5ff Mon Sep 17 00:00:00 2001 From: Croft Date: Thu, 27 Jul 2023 09:44:55 +0200 Subject: [PATCH 09/24] Added URLs that need to be accessible for the installation to work. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 012d28c..98f18b1 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Additionally, your site might use its own proxy. You should discuss this with yo Note that git and Docker may also need to be configured to use this proxy. This is a job for your systems administrators. -If there is a site firewall, this needs to be configured so that git and Docker can reach the outside world. Another job for the systems administrators. +If there is a site firewall, this needs to be configured so that outgoing calls to the following URLs are allowed: *.dkfz.de, github.com, docker.com, *.docker.io, *.samply.de. Note for Ubuntu: Please note that the uncomplicated firewall (ufw) is known to conflict with Docker [here](https://github.com/chaifeng/ufw-docker). From 788e4ea9f76c936c1a79cf9afd06d015da144f37 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 25 Jul 2023 14:08:54 +0200 Subject: [PATCH 10/24] add generic bash function addBasicAuthUser --- bridgehead | 4 ++++ ccp/nngm-compose.yml | 7 ++++++- lib/functions.sh | 24 ++++++++++++++++++++++++ lib/install-bridgehead.sh | 2 +- lib/sitespecific-functions.sh | 7 +++++++ 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 lib/sitespecific-functions.sh diff --git a/bridgehead b/bridgehead index 2709f8c..1a88973 100755 --- a/bridgehead +++ b/bridgehead @@ -103,6 +103,10 @@ case "$ACTION" in uninstall) exec ./lib/uninstall-bridgehead.sh $PROJECT ;; + addUser) + loadVars + exec ./lib/sitespecific-functions.sh $PROJECT + ;; enroll) loadVars diff --git a/ccp/nngm-compose.yml b/ccp/nngm-compose.yml index 47bfa70..18be977 100644 --- a/ccp/nngm-compose.yml +++ b/ccp/nngm-compose.yml @@ -18,7 +18,12 @@ services: - "traefik.http.middlewares.connector_strip.stripprefix.prefixes=/nngm-connector" - "traefik.http.services.connector.loadbalancer.server.port=8080" - "traefik.http.routers.connector.tls=true" - - "traefik.http.routers.connector.middlewares=connector_strip,auth" + - "traefik.http.routers.connector.middlewares=connector_strip,auth-nngm" volumes: - nngm-rest:/var/log + traefik: + labels: + - "traefik.http.middlewares.auth-nngm.basicauth.users=$NNGM_AUTH" + + diff --git a/lib/functions.sh b/lib/functions.sh index a243842..cd73186 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -183,3 +183,27 @@ function bk_is_running { ##Setting Network properties # currently not needed #export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); +addBasicAuthUser() { + USER="${1}" + PASSWORD="${2}" + NAME="${3}" + PROJECT="${4}" + FILE="/etc/bridgehead/${PROJECT}.local.conf" + ENCRY_CREDENTIALS="$(docker run --rm docker.verbis.dkfz.de/cache/httpd:alpine htpasswd -nb $USER $PASSWORD | tr -d '\n' | tr -d '\r')" + if [ -f $FILE ] && grep -R -q "$NAME=" $FILE # if a basic auth user already exists: + then + sed -i "/$NAME/ s/$/,$ENCRY_CREDENTIALS/" $FILE + else + echo -e "\n## Basic Authentication Credentials for:\n$NAME=$ENCRY_CREDENTIALS" >> $FILE; + fi + read -p "Do you want to have your cleartext credentials saved in your $FILE: [y/n]" yn + case $yn in + [yYjJ] ) + echo "ok, variables are saved in $FILE" + echo -e "# User: $USER\n# Password: $PASSWORD" >> $FILE; + ;; + [nN] ) + echo "skip saving cleartext LDM credentials; make sure to save them somewhere else" + ;; + esac +} \ No newline at end of file diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index c42119f..906e4a1 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -34,7 +34,7 @@ if [ -z "$LDM_PASSWORD" ]; then generated_passwd="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 32)" log "INFO" "Your generated credentials are:\n user: $PROJECT\n password: $generated_passwd" - echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; + addBasicAuthUser $PROJECT $generated_passwd "LDM_LOGIN" $PROJECT fi log "INFO" "Registering system units for bridgehead and bridgehead-update" diff --git a/lib/sitespecific-functions.sh b/lib/sitespecific-functions.sh new file mode 100644 index 0000000..55c3142 --- /dev/null +++ b/lib/sitespecific-functions.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e +source lib/functions.sh +PROJECT="ccp" +log "INFO" "Adding custom encrypted credentials in /etc/bridgehead/$PROJECT.local.conf" +read -p "Please enter custom user: " user +read -s -p "Please enter password (will not be echoed): "$'\n' password +addBasicAuthUser $user $password "NNGM_AUTH" $PROJECT \ No newline at end of file From 9facafd0c4939e2d6b7fab6a83a9b6593223f139 Mon Sep 17 00:00:00 2001 From: lablans Date: Fri, 28 Jul 2023 11:23:54 +0000 Subject: [PATCH 11/24] Only read headers from Broker to check clock skew --- lib/prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 0bb7882..9b2272f 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -71,7 +71,7 @@ source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars set +e -SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -v $BROKER_URL 2>&1)" +SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL 2>&1)" RET=$? set -e if [ $RET -ne 0 ]; then From b7ed90c5c8c2df37be390d631fe9ac935d3b9cc6 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 28 Jul 2023 11:32:48 +0000 Subject: [PATCH 12/24] Change logging/reporting order --- lib/prerequisites.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index d719aa4..0cec06b 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -84,7 +84,7 @@ else SKEW=$(echo $SKEW | awk -F- '{print $NF}') SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." if [ $SKEW -ge 300 ]; then - log ERROR "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" + report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" exit 1 elif [ $SKEW -ge 60 ]; then log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" @@ -98,8 +98,6 @@ checkPrivKey() { log ERROR "Unable to find private key at /etc/bridgehead/pki/${SITE_ID}.priv.pem. To fix, please run\n bridgehead enroll ${PROJECT}\nand follow the instructions." return 1 fi - log INFO "Success - all prerequisites are met!" - hc_send log "Success - all prerequisites are met!" return 0 } @@ -110,5 +108,6 @@ else fi log INFO "Success - all prerequisites are met!" +hc_send log "Success - all prerequisites are met!" exit 0 From 20359fde713aa74a684fde5b65a675f9e97d7f8e Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 1 Aug 2023 10:23:14 +0200 Subject: [PATCH 13/24] fix: correctly parse curl output --- lib/prerequisites.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 0cec06b..396b6e5 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -68,7 +68,7 @@ source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars set +e -SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL 2>&1)" +SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL 2>&1 | grep -e '^Date: ' | sed -e 's/^Date: //')" RET=$? set -e if [ $RET -ne 0 ]; then @@ -77,7 +77,6 @@ if [ $RET -ne 0 ]; then else log INFO "Checking clock skew ..." - SERVERTIME=$(echo -e "$SERVERTIME" | grep Date | sed -e 's/< Date: //') SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) MYTIME=$(date +%s) SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) From 7093166a53e1ea308e1af9d014de4be5c9bb7377 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 1 Aug 2023 10:25:23 +0200 Subject: [PATCH 14/24] fix: make the check case insensitive --- lib/prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 396b6e5..810be31 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -68,7 +68,7 @@ source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars set +e -SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL 2>&1 | grep -e '^Date: ' | sed -e 's/^Date: //')" +SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" RET=$? set -e if [ $RET -ne 0 ]; then From e775ec58347c8e7ca676bf02f3ee3e561736cd98 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Tue, 1 Aug 2023 10:34:06 +0200 Subject: [PATCH 15/24] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98f18b1..99f1157 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Additionally, your site might use its own proxy. You should discuss this with yo Note that git and Docker may also need to be configured to use this proxy. This is a job for your systems administrators. -If there is a site firewall, this needs to be configured so that outgoing calls to the following URLs are allowed: *.dkfz.de, github.com, docker.com, *.docker.io, *.samply.de. +If there is a site firewall, this needs to be configured so that outgoing calls to the following URLs are allowed: *.dkfz.de, github.com, docker.io, *.docker.io, *.samply.de. Note for Ubuntu: Please note that the uncomplicated firewall (ufw) is known to conflict with Docker [here](https://github.com/chaifeng/ufw-docker). From acc1e2361a3a6f490914dbe5f197e55bbc8184d2 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 4 Aug 2023 17:20:46 +0200 Subject: [PATCH 16/24] Prevent creation of many anonymous volumes In combination with https://github.com/samply/bridgehead-forward-proxy/pull/10, this will prevent the creation of two anonymous volumes per startup for the bridgehead-forward-proxy. --- minimal/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index cd4c2e5..e941350 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -37,6 +37,9 @@ services: HTTPS_PROXY: ${HTTPS_PROXY_URL} USERNAME: ${HTTPS_PROXY_USERNAME} PASSWORD: ${HTTPS_PROXY_PASSWORD} + tmpfs: + - /var/log/squid + - /var/spool/squid volumes: - /etc/bridgehead/trusted-ca-certs:/docker/custom-certs/:ro From b32a19a7b52ba52d284c54aed81efda08076e84a Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 7 Aug 2023 13:00:24 +0200 Subject: [PATCH 17/24] Make Directory Sync a module --- bbmri/directory-sync.sh | 8 -------- bbmri/{ => modules}/directory-sync-compose.yml | 0 bbmri/modules/directory-sync.sh | 6 ++++++ bbmri/vars | 4 ---- 4 files changed, 6 insertions(+), 12 deletions(-) delete mode 100755 bbmri/directory-sync.sh rename bbmri/{ => modules}/directory-sync-compose.yml (100%) create mode 100755 bbmri/modules/directory-sync.sh diff --git a/bbmri/directory-sync.sh b/bbmri/directory-sync.sh deleted file mode 100755 index 2eeef4a..0000000 --- a/bbmri/directory-sync.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -function dirSetup() { - if [ -n "$DS_DIRECTORY_USER_NAME" ]; then - log INFO "Directory sync setup detected -- will start directory sync service." - OVERRIDE+=" -f ./$PROJECT/directory-sync-compose.yml" - fi -} diff --git a/bbmri/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml similarity index 100% rename from bbmri/directory-sync-compose.yml rename to bbmri/modules/directory-sync-compose.yml diff --git a/bbmri/modules/directory-sync.sh b/bbmri/modules/directory-sync.sh new file mode 100755 index 0000000..2765dfb --- /dev/null +++ b/bbmri/modules/directory-sync.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ -n "${DS_DIRECTORY_USER_NAME}" ]; then + log INFO "Directory sync setup detected -- will start directory sync service." + OVERRIDE+=" -f ./$PROJECT/modules/directory-sync-compose.yml" +fi diff --git a/bbmri/vars b/bbmri/vars index 21aeaec..d86930c 100644 --- a/bbmri/vars +++ b/bbmri/vars @@ -11,7 +11,3 @@ do log DEBUG "sourcing $module" source $module done - -# This will load directory-sync setup. -source $PROJECT/directory-sync.sh -dirSetup From 54d83736c32c27e39a6e338673b3c0efadc19c83 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 7 Aug 2023 13:00:37 +0200 Subject: [PATCH 18/24] Move BBMRI-ERIC, GBN to modules --- bbmri/docker-compose.yml | 36 ++----------------- bbmri/modules/dnpm-setup.sh | 2 +- bbmri/modules/eric-compose.yml | 36 +++++++++++++++++++ bbmri/modules/eric-setup.sh | 13 +++++++ .../eric.root.crt.pem} | 0 bbmri/modules/gbn-compose.yml | 36 +++++++++++++++++++ bbmri/modules/gbn-setup.sh | 13 +++++++ bbmri/modules/gbn.root.crt.pem | 1 + bbmri/vars | 34 +++++++++++++++--- bridgehead | 17 +-------- ccp/modules/dnpm-setup.sh | 2 +- ccp/vars | 2 ++ lib/functions.sh | 26 ++++++++++++-- lib/prerequisites.sh | 6 ++-- 14 files changed, 162 insertions(+), 62 deletions(-) create mode 100644 bbmri/modules/eric-compose.yml create mode 100644 bbmri/modules/eric-setup.sh rename bbmri/{root.crt.pem => modules/eric.root.crt.pem} (100%) create mode 100644 bbmri/modules/gbn-compose.yml create mode 100644 bbmri/modules/gbn-setup.sh create mode 100644 bbmri/modules/gbn.root.crt.pem diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 07f1a2f..9bc05cc 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -1,5 +1,7 @@ version: "3.7" +# This includes only the shared persistence for BBMRI-ERIC and GBN. Federation components are included as modules, see vars. + services: blaze: image: docker.verbis.dkfz.de/cache/samply/blaze:latest @@ -19,42 +21,10 @@ services: - "traefik.http.routers.blaze_ccp.middlewares=ccp_b_strip,auth" - "traefik.http.routers.blaze_ccp.tls=true" - focus: - image: docker.verbis.dkfz.de/cache/samply/focus:main - container_name: bridgehead-focus - environment: - API_KEY: ${FOCUS_BEAM_SECRET_SHORT} - BEAM_APP_ID_LONG: focus.${PROXY_ID} - PROXY_ID: ${PROXY_ID} - BLAZE_URL: "http://bridgehead-bbmri-blaze:8080/fhir/" - BEAM_PROXY_URL: http://beam-proxy:8081 - RETRY_COUNT: ${FOCUS_RETRY_COUNT} - depends_on: - - "beam-proxy" - - "blaze" - - beam-proxy: - image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop - container_name: bridgehead-beam-proxy - environment: - BROKER_URL: ${BROKER_URL} - PROXY_ID: ${PROXY_ID} - APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT} - PRIVKEY_FILE: /run/secrets/proxy.pem - ALL_PROXY: http://forward_proxy:3128 - TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs - ROOTCERT_FILE: /conf/root.crt.pem - secrets: - - proxy.pem - depends_on: - - "forward_proxy" - volumes: - - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - - /srv/docker/bridgehead/bbmri/root.crt.pem:/conf/root.crt.pem:ro - volumes: blaze-data: +# used in modules *-locator.yml secrets: proxy.pem: file: /etc/bridgehead/pki/${SITE_ID}.priv.pem diff --git a/bbmri/modules/dnpm-setup.sh b/bbmri/modules/dnpm-setup.sh index 7b3be9a..ce37632 100644 --- a/bbmri/modules/dnpm-setup.sh +++ b/bbmri/modules/dnpm-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -n "${ENABLE_DNPM}" ]; then - log DEBUG "DNPM setup detected (Beam.Connect) -- will start Beam and Beam.Connect for DNPM." + log INFO "DNPM setup detected (Beam.Connect) -- will start Beam and Beam.Connect for DNPM." OVERRIDE+=" -f ./$PROJECT/modules/dnpm-compose.yml" # Set variables required for Beam-Connect diff --git a/bbmri/modules/eric-compose.yml b/bbmri/modules/eric-compose.yml new file mode 100644 index 0000000..2bf7dd6 --- /dev/null +++ b/bbmri/modules/eric-compose.yml @@ -0,0 +1,36 @@ +version: "3.7" + +services: + focus-eric: + image: docker.verbis.dkfz.de/cache/samply/focus:main + container_name: bridgehead-focus-eric + environment: + API_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT} + BEAM_APP_ID_LONG: focus.${ERIC_PROXY_ID} + PROXY_ID: ${ERIC_PROXY_ID} + BLAZE_URL: "http://blaze:8080/fhir/" + BEAM_PROXY_URL: http://beam-proxy-eric:8081 + RETRY_COUNT: ${FOCUS_RETRY_COUNT} + depends_on: + - "beam-proxy-eric" + - "blaze" + + beam-proxy-eric: + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop + container_name: bridgehead-beam-proxy-eric + environment: + BROKER_URL: ${ERIC_BROKER_URL} + PROXY_ID: ${ERIC_PROXY_ID} + APP_focus_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT} + PRIVKEY_FILE: /run/secrets/proxy.pem + ALL_PROXY: http://forward_proxy:3128 + TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs + ROOTCERT_FILE: /conf/root.crt.pem + secrets: + - proxy.pem + depends_on: + - "forward_proxy" + volumes: + - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro + - /srv/docker/bridgehead/bbmri/modules/eric.root.crt.pem:/conf/root.crt.pem:ro + diff --git a/bbmri/modules/eric-setup.sh b/bbmri/modules/eric-setup.sh new file mode 100644 index 0000000..364281b --- /dev/null +++ b/bbmri/modules/eric-setup.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ "${ENABLE_ERIC}" == "true" ]; then + log INFO "BBMRI-ERIC setup detected -- will start services for BBMRI-ERIC." + OVERRIDE+=" -f ./$PROJECT/modules/eric-compose.yml" + + # Set required variables + ERIC_BROKER_ID=broker.bbmri.samply.de + ERIC_BROKER_URL=https://${ERIC_BROKER_ID} + ERIC_PROXY_ID=${SITE_ID}.${ERIC_BROKER_ID} + ERIC_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + ERIC_SUPPORT_EMAIL=bridgehead@helpdesk.bbmri-eric.eu +fi diff --git a/bbmri/root.crt.pem b/bbmri/modules/eric.root.crt.pem similarity index 100% rename from bbmri/root.crt.pem rename to bbmri/modules/eric.root.crt.pem diff --git a/bbmri/modules/gbn-compose.yml b/bbmri/modules/gbn-compose.yml new file mode 100644 index 0000000..47b1d36 --- /dev/null +++ b/bbmri/modules/gbn-compose.yml @@ -0,0 +1,36 @@ +version: "3.7" + +services: + focus-gbn: + image: docker.verbis.dkfz.de/cache/samply/focus:main + container_name: bridgehead-focus-gbn + environment: + API_KEY: ${GBN_FOCUS_BEAM_SECRET_SHORT} + BEAM_APP_ID_LONG: focus.${GBN_PROXY_ID} + PROXY_ID: ${GBN_PROXY_ID} + BLAZE_URL: "http://blaze:8080/fhir/" + BEAM_PROXY_URL: http://beam-proxy-gbn:8081 + RETRY_COUNT: ${FOCUS_RETRY_COUNT} + depends_on: + - "beam-proxy-gbn" + - "blaze" + + beam-proxy-gbn: + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop + container_name: bridgehead-beam-proxy-gbn + environment: + BROKER_URL: ${GBN_BROKER_URL} + PROXY_ID: ${GBN_PROXY_ID} + APP_focus_KEY: ${GBN_FOCUS_BEAM_SECRET_SHORT} + PRIVKEY_FILE: /run/secrets/proxy.pem + ALL_PROXY: http://forward_proxy:3128 + TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs + ROOTCERT_FILE: /conf/root.crt.pem + secrets: + - proxy.pem + depends_on: + - "forward_proxy" + volumes: + - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro + - /srv/docker/bridgehead/bbmri/modules/gbn.root.crt.pem:/conf/root.crt.pem:ro + diff --git a/bbmri/modules/gbn-setup.sh b/bbmri/modules/gbn-setup.sh new file mode 100644 index 0000000..b032c8e --- /dev/null +++ b/bbmri/modules/gbn-setup.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ "${ENABLE_GBN}" == "true" ]; then + log INFO "GBN setup detected -- will start services for German Biobank Node." + OVERRIDE+=" -f ./$PROJECT/modules/gbn-compose.yml" + + # Set required variables + GBN_BROKER_ID='#TODO#' + GBN_BROKER_URL=https://${GBN_BROKER_ID} + GBN_PROXY_ID=${SITE_ID}.${GBN_BROKER_ID} + GBN_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + GBN_SUPPORT_EMAIL=todo@verbis.dkfz.de +fi diff --git a/bbmri/modules/gbn.root.crt.pem b/bbmri/modules/gbn.root.crt.pem new file mode 100644 index 0000000..0c2ad6f --- /dev/null +++ b/bbmri/modules/gbn.root.crt.pem @@ -0,0 +1 @@ +#TODO# diff --git a/bbmri/vars b/bbmri/vars index d86930c..1cb6128 100644 --- a/bbmri/vars +++ b/bbmri/vars @@ -1,9 +1,10 @@ -BROKER_ID=broker.bbmri.samply.de -BROKER_URL=https://${BROKER_ID} -PROXY_ID=${SITE_ID}.${BROKER_ID} -FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" +# Makes sense for all European Biobanks +: ${ENABLE_ERIC:=true} + +# Makes only sense for German Biobanks +: ${ENABLE_GBN:=false} + FOCUS_RETRY_COUNT=32 -SUPPORT_EMAIL=bridgehead@helpdesk.bbmri-eric.eu PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem for module in $PROJECT/modules/*.sh @@ -11,3 +12,26 @@ do log DEBUG "sourcing $module" source $module done + +SUPPORT_EMAIL=$ERIC_SUPPORT_EMAIL +BROKER_URL_FOR_PREREQ=$ERIC_BROKER_URL + +if [ -n "$GBN_SUPPORT_EMAIL" ]; then + SUPPORT_EMAIL=$GBN_SUPPORT_EMAIL +fi + +function do_enroll { + COUNT=0 + if [ "$ENABLE_ERIC" == "true" ]; then + do_enroll_inner $ERIC_PROXY_ID $ERIC_SUPPORT_EMAIL + COUNT=$((COUNT+1)) + fi + if [ "$ENABLE_GBN" == "true" ]; then + do_enroll_inner $GBN_PROXY_ID $GBN_SUPPORT_EMAIL + COUNT=$((COUNT+1)) + fi + if [ $COUNT -ge 2 ]; then + echo + echo "You just received $COUNT certificate signing requests (CSR). Please send $COUNT e-mails, with 1 CSR each, to the respective e-mail address." + fi +} \ No newline at end of file diff --git a/bridgehead b/bridgehead index 2709f8c..5793370 100755 --- a/bridgehead +++ b/bridgehead @@ -106,22 +106,7 @@ case "$ACTION" in enroll) loadVars - MANUAL_PROXY_ID="${3:-$PROXY_ID}" - if [ -z "$MANUAL_PROXY_ID" ]; then - log ERROR "No Proxy ID set" - exit 1 - else - log INFO "Enrolling Beam Proxy Id $MANUAL_PROXY_ID" - fi - - if [ -z "$SUPPORT_EMAIL" ]; then - EMAIL_PARAM="" - else - EMAIL_PARAM="--admin-email $SUPPORT_EMAIL" - fi - - docker run --rm -ti -v /etc/bridgehead/pki:/etc/bridgehead/pki samply/beam-enroll:latest --output-file $PRIVATEKEYFILENAME --proxy-id $MANUAL_PROXY_ID $EMAIL_PARAM - chmod 600 $PRIVATEKEYFILENAME + do_enroll $PROXY_ID ;; preRun | preUpdate) fixPermissions diff --git a/ccp/modules/dnpm-setup.sh b/ccp/modules/dnpm-setup.sh index 04659eb..b3dd636 100644 --- a/ccp/modules/dnpm-setup.sh +++ b/ccp/modules/dnpm-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -n "${ENABLE_DNPM}" ]; then - log DEBUG "DNPM setup detected (Beam.Connect) -- will start Beam.Connect for DNPM." + log INFO "DNPM setup detected (Beam.Connect) -- will start Beam.Connect for DNPM." OVERRIDE+=" -f ./$PROJECT/modules/dnpm-compose.yml" # Set variables required for Beam-Connect diff --git a/ccp/vars b/ccp/vars index 420b0d0..61d5d2e 100644 --- a/ccp/vars +++ b/ccp/vars @@ -6,6 +6,8 @@ FOCUS_RETRY_COUNT=32 SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +BROKER_URL_FOR_PREREQ=$BROKER_URL + # This will load id-management setup. Effective only if id-management configuration is defined. source $PROJECT/modules/id-management-setup.sh idManagementSetup diff --git a/lib/functions.sh b/lib/functions.sh index a243842..48bc650 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -180,6 +180,26 @@ function bk_is_running { fi } -##Setting Network properties -# currently not needed -#export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); +function do_enroll_inner { + PARAMS="" + + MANUAL_PROXY_ID="${1:-$PROXY_ID}" + if [ -z "$MANUAL_PROXY_ID" ]; then + log ERROR "No Proxy ID set" + exit 1 + else + log INFO "Enrolling Beam Proxy Id $MANUAL_PROXY_ID" + fi + + SUPPORT_EMAIL="${2:-$SUPPORT_EMAIL}" + if [ -n "$SUPPORT_EMAIL" ]; then + PARAMS+="--admin-email $SUPPORT_EMAIL" + fi + + docker run --rm -ti -v /etc/bridgehead/pki:/etc/bridgehead/pki samply/beam-enroll:latest --output-file $PRIVATEKEYFILENAME --proxy-id $MANUAL_PROXY_ID $PARAMS + chmod 600 $PRIVATEKEYFILENAME +} + +function do_enroll { + do_enroll_inner $@ +} \ No newline at end of file diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 810be31..062f065 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -62,17 +62,17 @@ if [ -e /etc/bridgehead/vault.conf ]; then fi fi -log INFO "Checking network access ($BROKER_URL) ..." +log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..." source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars set +e -SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" +SERVERTIME="$(https_proxy=$HTTPS_PROXY_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" RET=$? set -e if [ $RET -ne 0 ]; then - log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." + log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." log WARN "Unable to check clock skew due to previous error." else log INFO "Checking clock skew ..." From dc0fc286b1fb356293934f30b90e4983e6b6974c Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Thu, 27 Jul 2023 15:38:29 +0200 Subject: [PATCH 19/24] add generic bash function addBasicAuthUser --- bridgehead | 1 - ccp/nngm-compose.yml | 2 +- ccp/nngm-setup.sh | 4 ---- lib/functions.sh | 32 +++++++++----------------------- lib/install-bridgehead.sh | 12 ++++++++---- lib/sitespecific-functions.sh | 14 +++++++++----- minimal/docker-compose.yml | 2 +- 7 files changed, 28 insertions(+), 39 deletions(-) mode change 100644 => 100755 lib/sitespecific-functions.sh diff --git a/bridgehead b/bridgehead index 1a88973..5319620 100755 --- a/bridgehead +++ b/bridgehead @@ -73,7 +73,6 @@ case "$ACTION" in hc_send log "Bridgehead $PROJECT startup: Checking requirements ..." checkRequirements hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..." - export LDM_LOGIN=$(getLdmPassword) exec $COMPOSE -p $PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit ;; stop) diff --git a/ccp/nngm-compose.yml b/ccp/nngm-compose.yml index 18be977..e61532d 100644 --- a/ccp/nngm-compose.yml +++ b/ccp/nngm-compose.yml @@ -24,6 +24,6 @@ services: traefik: labels: - - "traefik.http.middlewares.auth-nngm.basicauth.users=$NNGM_AUTH" + - "traefik.http.middlewares.auth-nngm.basicauth.users=${NNGM_AUTH}" diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index bcc4cd1..cb9590a 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -1,8 +1,4 @@ #!/bin/bash -##nNGM vars: -#NNGM_MAGICPL_APIKEY -#NNGM_CTS_APIKEY -#NNGM_CRYPTKEY function nngmSetup() { if [ -n "$NNGM_CTS_APIKEY" ]; then diff --git a/lib/functions.sh b/lib/functions.sh index cd73186..d6c1b85 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -9,14 +9,6 @@ detectCompose() { fi } -getLdmPassword() { - if [ -n "$LDM_PASSWORD" ]; then - docker run --rm docker.verbis.dkfz.de/cache/httpd:alpine htpasswd -nb $PROJECT $LDM_PASSWORD | tr -d '\n' | tr -d '\r' - else - echo -n "" - fi -} - exitIfNotRoot() { if [ "$EUID" -ne 0 ]; then log "ERROR" "Please run as root" @@ -34,7 +26,7 @@ checkOwner(){ } printUsage() { - echo "Usage: bridgehead start|stop|is-running|update|install|uninstall|enroll PROJECTNAME" + echo "Usage: bridgehead start|stop|is-running|update|install|uninstall|addUser|enroll PROJECTNAME" echo "PROJECTNAME should be one of ccp|bbmri" } @@ -183,27 +175,21 @@ function bk_is_running { ##Setting Network properties # currently not needed #export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); -addBasicAuthUser() { + + +add_basic_auth_user() { USER="${1}" PASSWORD="${2}" NAME="${3}" PROJECT="${4}" FILE="/etc/bridgehead/${PROJECT}.local.conf" ENCRY_CREDENTIALS="$(docker run --rm docker.verbis.dkfz.de/cache/httpd:alpine htpasswd -nb $USER $PASSWORD | tr -d '\n' | tr -d '\r')" - if [ -f $FILE ] && grep -R -q "$NAME=" $FILE # if a basic auth user already exists: + if [ -f $FILE ] && grep -R -q "$NAME=" $FILE # if a specific basic auth user already exists: then - sed -i "/$NAME/ s/$/,$ENCRY_CREDENTIALS/" $FILE + sed -i "/$NAME/ s|='|='$ENCRY_CREDENTIALS,|" $FILE else - echo -e "\n## Basic Authentication Credentials for:\n$NAME=$ENCRY_CREDENTIALS" >> $FILE; + echo -e "\n## Basic Authentication Credentials for:\n$NAME='$ENCRY_CREDENTIALS'" >> $FILE; fi - read -p "Do you want to have your cleartext credentials saved in your $FILE: [y/n]" yn - case $yn in - [yYjJ] ) - echo "ok, variables are saved in $FILE" - echo -e "# User: $USER\n# Password: $PASSWORD" >> $FILE; - ;; - [nN] ) - echo "skip saving cleartext LDM credentials; make sure to save them somewhere else" - ;; - esac + log DEBUG "Saving clear text credentials in $FILE. If wanted, delete them manually." + sed -i "/^$NAME/ s|$|\n# User: $USER\n# Password: $PASSWORD|" $FILE } \ No newline at end of file diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 906e4a1..d0e9964 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -29,12 +29,16 @@ bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF # TODO: Determine whether this should be located in setup-bridgehead (triggered through bridgehead install) or in update bridgehead (triggered every hour) -if [ -z "$LDM_PASSWORD" ]; then - log "INFO" "Now generating a password for the local data management. Please save the password for your ETL process!" +if [ -z "$LDM_AUTH" ]; then + log "INFO" "Now generating basic auth for the local data management (see addUser in bridgehead for more information). " generated_passwd="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 32)" + add_basic_auth_user $PROJECT $generated_passwd "LDM_AUTH" $PROJECT +fi - log "INFO" "Your generated credentials are:\n user: $PROJECT\n password: $generated_passwd" - addBasicAuthUser $PROJECT $generated_passwd "LDM_LOGIN" $PROJECT +if [ -z "$NNGM_CTS_APIKEY" ] && [ -z "$NNGM_AUTH" ]; then + log "INFO" "Now generating basic auth for nNGM upload API (see addUser in bridgehead for more information). " + generated_passwd="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 32)" + add_basic_auth_user "nngm" $generated_passwd "NNGM_AUTH" $PROJECT fi log "INFO" "Registering system units for bridgehead and bridgehead-update" diff --git a/lib/sitespecific-functions.sh b/lib/sitespecific-functions.sh old mode 100644 new mode 100755 index 55c3142..04db02e --- a/lib/sitespecific-functions.sh +++ b/lib/sitespecific-functions.sh @@ -1,7 +1,11 @@ #!/bin/bash -e + source lib/functions.sh -PROJECT="ccp" -log "INFO" "Adding custom encrypted credentials in /etc/bridgehead/$PROJECT.local.conf" -read -p "Please enter custom user: " user -read -s -p "Please enter password (will not be echoed): "$'\n' password -addBasicAuthUser $user $password "NNGM_AUTH" $PROJECT \ No newline at end of file + +PROJECT=$1 + +log "INFO" "Adding encrypted credentials in /etc/bridgehead/$PROJECT.local.conf" +read -p "Please choose the component (LDM_AUTH|NNGM_AUTH) you want to add a user to : " COMPONENT +read -p "Please enter a username: " USER +read -s -p "Please enter a password (will not be echoed): "$'\n' PASSWORD +add_basic_auth_user $USER $PASSWORD $COMPONENT $PROJECT \ No newline at end of file diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index cd4c2e5..055f683 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -21,7 +21,7 @@ services: - "traefik.http.routers.dashboard.service=api@internal" - "traefik.http.routers.dashboard.tls=true" - "traefik.http.routers.dashboard.middlewares=auth" - - "traefik.http.middlewares.auth.basicauth.users=${LDM_LOGIN}" + - "traefik.http.middlewares.auth.basicauth.users=${LDM_AUTH}" ports: - 80:80 - 443:443 From 3f43c32bd258f51be769ef645671694eb0aa5dad Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 9 Aug 2023 09:10:20 +0200 Subject: [PATCH 20/24] refactor addUser code --- bridgehead | 6 +++++- lib/sitespecific-functions.sh | 11 ----------- 2 files changed, 5 insertions(+), 12 deletions(-) delete mode 100755 lib/sitespecific-functions.sh diff --git a/bridgehead b/bridgehead index 5319620..37e7e7b 100755 --- a/bridgehead +++ b/bridgehead @@ -104,7 +104,11 @@ case "$ACTION" in ;; addUser) loadVars - exec ./lib/sitespecific-functions.sh $PROJECT + log "INFO" "Adding encrypted credentials in /etc/bridgehead/$PROJECT.local.conf" + read -p "Please choose the component (LDM_AUTH|NNGM_AUTH) you want to add a user to : " COMPONENT + read -p "Please enter a username: " USER + read -s -p "Please enter a password (will not be echoed): "$'\n' PASSWORD + add_basic_auth_user $USER $PASSWORD $COMPONENT $PROJECT ;; enroll) loadVars diff --git a/lib/sitespecific-functions.sh b/lib/sitespecific-functions.sh deleted file mode 100755 index 04db02e..0000000 --- a/lib/sitespecific-functions.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -e - -source lib/functions.sh - -PROJECT=$1 - -log "INFO" "Adding encrypted credentials in /etc/bridgehead/$PROJECT.local.conf" -read -p "Please choose the component (LDM_AUTH|NNGM_AUTH) you want to add a user to : " COMPONENT -read -p "Please enter a username: " USER -read -s -p "Please enter a password (will not be echoed): "$'\n' PASSWORD -add_basic_auth_user $USER $PASSWORD $COMPONENT $PROJECT \ No newline at end of file From 2c7de6c8b40ad812be93f6daa947e4c7778cdf13 Mon Sep 17 00:00:00 2001 From: Pierre Delpy <75260699+PierreDelpy@users.noreply.github.com> Date: Wed, 9 Aug 2023 09:24:23 +0200 Subject: [PATCH 21/24] refactor strange formatting --- bridgehead | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridgehead b/bridgehead index 37e7e7b..4f023fa 100755 --- a/bridgehead +++ b/bridgehead @@ -105,10 +105,10 @@ case "$ACTION" in addUser) loadVars log "INFO" "Adding encrypted credentials in /etc/bridgehead/$PROJECT.local.conf" - read -p "Please choose the component (LDM_AUTH|NNGM_AUTH) you want to add a user to : " COMPONENT - read -p "Please enter a username: " USER - read -s -p "Please enter a password (will not be echoed): "$'\n' PASSWORD - add_basic_auth_user $USER $PASSWORD $COMPONENT $PROJECT + read -p "Please choose the component (LDM_AUTH|NNGM_AUTH) you want to add a user to : " COMPONENT + read -p "Please enter a username: " USER + read -s -p "Please enter a password (will not be echoed): "$'\n' PASSWORD + add_basic_auth_user $USER $PASSWORD $COMPONENT $PROJECT ;; enroll) loadVars From 4754eb282b68d0d60442eec03ab2cb6206251339 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 15 Aug 2023 13:55:07 +0200 Subject: [PATCH 22/24] add migration for old credentials --- lib/install-bridgehead.sh | 2 +- lib/update-bridgehead.sh | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index d0e9964..784020e 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -35,7 +35,7 @@ if [ -z "$LDM_AUTH" ]; then add_basic_auth_user $PROJECT $generated_passwd "LDM_AUTH" $PROJECT fi -if [ -z "$NNGM_CTS_APIKEY" ] && [ -z "$NNGM_AUTH" ]; then +if [ ! -z "$NNGM_CTS_APIKEY" ] && [ -z "$NNGM_AUTH" ]; then log "INFO" "Now generating basic auth for nNGM upload API (see addUser in bridgehead for more information). " generated_passwd="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 32)" add_basic_auth_user "nngm" $generated_passwd "NNGM_AUTH" $PROJECT diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 89db369..0d985f7 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -139,6 +139,13 @@ else log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" fi +if [ ! -z "$LDM_PASSWORD" ]; then + FILE="/etc/bridgehead/$PROJECT.local.conf" + log "INFO" "Migrating LDM_PASSWORD to encrypted credentials in $FILE" + add_basic_auth_user $PROJECT $LDM_PASSWORD "LDM_AUTH" $PROJECT + sed -i "/LDM_PASSWORD/{d;}" $FILE +fi + exit 0 # TODO: Print last commit explicit From b58348328c91c06a102ac323f38df60239d4351b Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 15 Aug 2023 15:34:49 +0200 Subject: [PATCH 23/24] fix nngm migration --- lib/update-bridgehead.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 0d985f7..aea043a 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -139,10 +139,12 @@ else log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" fi +#TODO: the following block can be deleted after successful update at all sites if [ ! -z "$LDM_PASSWORD" ]; then FILE="/etc/bridgehead/$PROJECT.local.conf" log "INFO" "Migrating LDM_PASSWORD to encrypted credentials in $FILE" add_basic_auth_user $PROJECT $LDM_PASSWORD "LDM_AUTH" $PROJECT + add_basic_auth_user $PROJECT $LDM_PASSWORD "NNGM_AUTH" $PROJECT sed -i "/LDM_PASSWORD/{d;}" $FILE fi From af25df79e3f97cf6c181e28aea550ebf221934c8 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 15 Aug 2023 15:42:42 +0200 Subject: [PATCH 24/24] refactor addUser to adduser - lowercase --- bridgehead | 2 +- lib/functions.sh | 2 +- lib/install-bridgehead.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bridgehead b/bridgehead index 192cc0e..4665f5a 100755 --- a/bridgehead +++ b/bridgehead @@ -102,7 +102,7 @@ case "$ACTION" in uninstall) exec ./lib/uninstall-bridgehead.sh $PROJECT ;; - addUser) + adduser) loadVars log "INFO" "Adding encrypted credentials in /etc/bridgehead/$PROJECT.local.conf" read -p "Please choose the component (LDM_AUTH|NNGM_AUTH) you want to add a user to : " COMPONENT diff --git a/lib/functions.sh b/lib/functions.sh index 9173e86..ed0478e 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -26,7 +26,7 @@ checkOwner(){ } printUsage() { - echo "Usage: bridgehead start|stop|is-running|update|install|uninstall|addUser|enroll PROJECTNAME" + echo "Usage: bridgehead start|stop|is-running|update|install|uninstall|adduser|enroll PROJECTNAME" echo "PROJECTNAME should be one of ccp|bbmri" } diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 784020e..38c3874 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -30,13 +30,13 @@ EOF # TODO: Determine whether this should be located in setup-bridgehead (triggered through bridgehead install) or in update bridgehead (triggered every hour) if [ -z "$LDM_AUTH" ]; then - log "INFO" "Now generating basic auth for the local data management (see addUser in bridgehead for more information). " + log "INFO" "Now generating basic auth for the local data management (see adduser in bridgehead for more information). " generated_passwd="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 32)" add_basic_auth_user $PROJECT $generated_passwd "LDM_AUTH" $PROJECT fi if [ ! -z "$NNGM_CTS_APIKEY" ] && [ -z "$NNGM_AUTH" ]; then - log "INFO" "Now generating basic auth for nNGM upload API (see addUser in bridgehead for more information). " + log "INFO" "Now generating basic auth for nNGM upload API (see adduser in bridgehead for more information). " generated_passwd="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 32)" add_basic_auth_user "nngm" $generated_passwd "NNGM_AUTH" $PROJECT fi