fix(dnpm): rewrite old etl endpoint to new one

When a OIDC provider is configured, you'll get redirected to authup by Keycloak which redirects you to the DNPM:DIP. 

Currently the url looks like this: https://myserver/authup//someurl
and produces an error. Manually removing the additional / fixes the issue.

.github/CODEOWNERS

@@ -1 +0,0 @@
-@samply/bridgehead-developers

LICENSE

@@ -1,4 +1,4 @@
-                                 Apache License test
+                                 Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 

README.md

@@ -76,7 +76,7 @@ The following URLs need to be accessible (prefix with `https://`):
   * git.verbis.dkfz.de
 * To fetch docker images
   * docker.verbis.dkfz.de
-  * Official Docker, Inc. URLs (subject to change, see [official list](https://docs.docker.com/desktop/setup/allow-list/))
+  * Official Docker, Inc. URLs (subject to change, see [official list](https://docs.docker.com/desktop/all))
     * hub.docker.com
     * registry-1.docker.io
     * production.cloudflare.docker.com
@@ -154,7 +154,7 @@ Pay special attention to:
 Clone the bridgehead repository:
 ```shell
 sudo mkdir -p /srv/docker/
-sudo git clone -b main https://github.com/samply/bridgehead.git /srv/docker/bridgehead
+sudo git clone https://github.com/samply/bridgehead.git /srv/docker/bridgehead
 ```
 
 Then, run the installation script:

bridgehead

@@ -53,44 +53,17 @@ case "$PROJECT" in
 		;;
 esac
 
-# Loads config variables and runs the projects setup script
 loadVars() {
+	# Load variables from /etc/bridgehead and /srv/docker/bridgehead
 	set -a
-	# Source the project specific config file
 	source /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "/etc/bridgehead/$PROJECT.conf not found"
-	# Source the project specific local config file if present
-	# This file is ignored by git as oposed to the regular config file as it contains private site information like etl auth data
 	if [ -e /etc/bridgehead/$PROJECT.local.conf ]; then
 		log INFO "Applying /etc/bridgehead/$PROJECT.local.conf"
 		source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import"
 	fi
-	# Set execution environment on main default to prod else test
-	if [[ -z "${ENVIRONMENT+x}" ]]; then
-		if [ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]; then
-			ENVIRONMENT="production"
-		else
-			ENVIRONMENT="test"
-		fi
-	fi
-	# Source the versions of the images components 
-	case "$ENVIRONMENT" in
-		"production")
-			source ./versions/prod
-			;;
-		"test")
-			source ./versions/test
-			;;
-		*)
-			report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!"
-			source ./versions/prod
-			;;
-	esac
 	fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile"
 	setHostname
 	optimizeBlazeMemoryUsage
-	# Run project specific setup if it exists
-	# This will ususally modiy the `OVERRIDE` to include all the compose files that the project depends on
-	# This is also where projects specify which modules to load
 	[ -e ./$PROJECT/vars ] && source ./$PROJECT/vars
 	set +a
 
@@ -106,6 +79,26 @@ loadVars() {
 	fi
 	detectCompose
 	setupProxy
+
+	# Set some project-independent default values
+	: ${ENVIRONMENT:=production}
+	export ENVIRONMENT
+
+	case "$ENVIRONMENT" in
+		"production")
+			export FOCUS_TAG=main
+			export BEAM_TAG=main
+			;;
+		"test")
+			export FOCUS_TAG=develop
+			export BEAM_TAG=develop
+			;;
+		*)
+			report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!"
+			export FOCUS_TAG=main
+			export BEAM_TAG=main
+			;;
+	esac
 }
 
 case "$ACTION" in

ccp/modules/dnpm-node-compose.yml

@@ -83,6 +83,10 @@ services:
       - "traefik.http.routers.dnpm-backend-etl.rule=PathRegexp(`^/api(/.*)?etl(/.*)?$`)"
       - "traefik.http.routers.dnpm-backend-etl.tls=true"
       - "traefik.http.routers.dnpm-backend-etl.service=dnpm-backend"
+      # TODO: add to minimal and document
+      - "traefik.http.middlewares.rewrite-mtbfile.replacepathregex.regex=^(.*)/MTBFile$"
+      - "traefik.http.middlewares.rewrite-mtbfile.replacepathregex.replacement=$1"
+      - "traefik.http.routers.dnpm-backend-etl.middlewares=rewrite-mtbfile"
       # this needs an ETL processor with support for basic auth
       - "traefik.http.routers.dnpm-backend-etl.middlewares=auth"
       # except peer-to-peer

ccp/modules/dnpm-node-setup.sh

@@ -10,7 +10,7 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then
 		exit 1
 	fi
     mkdir -p /var/cache/bridgehead/dnpm/ || fail_and_report 1 "Failed to create '/var/cache/bridgehead/dnpm/'. Please run sudo './bridgehead install $PROJECT' again to fix the permissions."
-    DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1}
+	DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1}
     DNPM_MYSQL_ROOT_PASSWORD="$(generate_simple_password 'dnpm mysql')"
     DNPM_AUTHUP_SECRET="$(generate_simple_password 'dnpm authup')"
 fi

ccp/modules/obds2fhir-rest-compose.yml

@@ -3,7 +3,7 @@ version: "3.7"
 services:
   obds2fhir-rest:
     container_name: bridgehead-obds2fhir-rest
-    image: docker.verbis.dkfz.de/samply/obds2fhir-rest:main
+    image: docker.verbis.dkfz.de/ccp/obds2fhir-rest:main
     environment:
       IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID
       MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}

dhki/vars

@@ -17,12 +17,4 @@ do
 done
 
 idManagementSetup
-obds2fhirRestSetup
+obds2fhirRestSetup
-
-for module in modules/*.sh
-do
-    log DEBUG "sourcing $module"
-    source $module
-done
-
-transfairSetup

lib/functions.sh

@@ -116,7 +116,7 @@ assertVarsNotEmpty() {
 	MISSING_VARS=""
 
 	for VAR in $@; do
-		if [ -z "${!VAR}" ]; then
+	if [ -z "${!VAR}" ]; then
 			MISSING_VARS+="$VAR "
 		fi
 	done
@@ -318,7 +318,7 @@ function sync_secrets() {
         docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest
 
     set -a # Export variables as environment variables
-    source /var/cache/bridgehead/secrets/oidc
+    source /var/cache/bridgehead/secrets/*
     set +a # Export variables in the regular way
 }
 

lib/gitlab-token-helper.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-[ "$1" = "get" ] || exit
-
-source /var/cache/bridgehead/secrets/gitlab_token
-
-# Any non-empty username works, only the token matters
-cat << EOF
-username=bk
-password=$BRIDGEHEAD_CONFIG_REPO_TOKEN
-EOF

lib/gitpassword.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+if [ "$1" != "get" ]; then
+	echo "Usage: $0 get"
+	exit 1
+fi
+
+baseDir() {
+	# see https://stackoverflow.com/questions/59895
+	SOURCE=${BASH_SOURCE[0]}
+	while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+		DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+		SOURCE=$(readlink "$SOURCE")
+		[[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+        done
+        DIR=$( cd -P "$( dirname "$SOURCE" )/.." >/dev/null 2>&1 && pwd )
+        echo $DIR
+}
+
+BASE=$(baseDir)
+cd $BASE
+
+source lib/functions.sh
+
+assertVarsNotEmpty SITE_ID || fail_and_report 1 "gitpassword.sh failed: SITE_ID is empty."
+
+PARAMS="$(cat)"
+GITHOST=$(echo "$PARAMS" | grep "^host=" | sed 's/host=\(.*\)/\1/g')
+
+fetchVarsFromVault GIT_PASSWORD
+
+if [ -z "${GIT_PASSWORD}" ]; then
+	fail_and_report 1 "gitpassword.sh failed: Git password not found."
+fi
+
+cat <<EOF
+protocol=https
+host=$GITHOST
+username=bk-${SITE_ID}
+password=${GIT_PASSWORD}
+EOF

lib/update-bridgehead.sh

@@ -19,7 +19,7 @@ fi
 
 hc_send log "Checking for bridgehead updates ..."
 
-CONFFILE=/etc/bridgehead/$PROJECT.conf
+CONFFILE=/etc/bridgehead/$1.conf
 
 if [ ! -e $CONFFILE ]; then
   fail_and_report 1 "Configuration file $CONFFILE not found."
@@ -33,43 +33,7 @@ export SITE_ID
 checkOwner /srv/docker/bridgehead bridgehead || fail_and_report 1 "Update failed: Wrong permissions in /srv/docker/bridgehead"
 checkOwner /etc/bridgehead bridgehead || fail_and_report 1 "Update failed: Wrong permissions in /etc/bridgehead"
 
-# Use Secret Sync to validate the GitLab token in /var/cache/bridgehead/secrets/gitlab_token.
+CREDHELPER="/srv/docker/bridgehead/lib/gitpassword.sh"
-# If it is missing or expired, Secret Sync will create a new token and write it to the file.
-# The git credential helper reads the token from the file during git pull.
-mkdir -p /var/cache/bridgehead/secrets
-touch /var/cache/bridgehead/secrets/gitlab_token # the file has to exist to be mounted correctly in the Docker container
-log "INFO" "Running Secret Sync for the GitLab token"
-docker pull docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest # make sure we have the latest image
-docker run --rm \
-  -v /var/cache/bridgehead/secrets/gitlab_token:/usr/local/cache \
-  -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \
-  -v /srv/docker/bridgehead/$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \
-  -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \
-  -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \
-  -e NO_PROXY=localhost,127.0.0.1 \
-  -e ALL_PROXY=$HTTPS_PROXY_FULL_URL \
-  -e PROXY_ID=$PROXY_ID \
-  -e BROKER_URL=$BROKER_URL \
-  -e GITLAB_PROJECT_ACCESS_TOKEN_PROVIDER=secret-sync-central.oidc-client-enrollment.$BROKER_ID \
-  -e SECRET_DEFINITIONS=GitLabProjectAccessToken:BRIDGEHEAD_CONFIG_REPO_TOKEN: \
-  docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest
-if [ $? -eq 0 ]; then
-  log "INFO" "Secret Sync was successful"
-  # In the past we used to hardcode tokens into the repository URL. We have to remove those now for the git credential helper to become effective.
-  CLEAN_REPO="$(git -C /etc/bridgehead remote get-url origin | sed -E 's|https://[^@]+@|https://|')"
-  git -C /etc/bridgehead remote set-url origin "$CLEAN_REPO"
-  # Set the git credential helper
-  git -C /etc/bridgehead config credential.helper /srv/docker/bridgehead/lib/gitlab-token-helper.sh
-else
-  log "WARN" "Secret Sync failed"
-  # Remove the git credential helper
-  git -C /etc/bridgehead config --unset credential.helper
-fi
-
-# In the past the git credential helper was also set for /srv/docker/bridgehead but never used.
-# Let's remove it to avoid confusion. This line can be removed at some point the future when we
-# believe that it was removed on all/most production servers.
-git -C /srv/docker/bridgehead config --unset credential.helper
 
 CHANGES=""
 
@@ -81,6 +45,10 @@ for DIR in /etc/bridgehead $(pwd); do
   if [ -n "$OUT" ]; then
     report_error log "The working directory $DIR is modified. Changed files: $OUT"
   fi
+  if [ "$(git -C $DIR config --get credential.helper)" != "$CREDHELPER" ]; then
+    log "INFO" "Configuring repo to use bridgehead git credential helper."
+    git -C $DIR config credential.helper "$CREDHELPER"
+  fi
   old_git_hash="$(git -C $DIR rev-parse --verify HEAD)"
   if [ -z "$HTTPS_PROXY_FULL_URL" ]; then
     log "INFO" "Git is using no proxy!"

minimal/modules/dnpm-node-setup.sh

@@ -10,7 +10,7 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then
 		exit 1
 	fi
     mkdir -p /var/cache/bridgehead/dnpm/ || fail_and_report 1 "Failed to create '/var/cache/bridgehead/dnpm/'. Please run sudo './bridgehead install $PROJECT' again to fix the permissions."
-    DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1}
+	DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1}
     DNPM_MYSQL_ROOT_PASSWORD="$(generate_simple_password 'dnpm mysql')"
     DNPM_AUTHUP_SECRET="$(generate_simple_password 'dnpm authup')"
 fi

modules/transfair-compose.yml

@@ -1,51 +0,0 @@
-
-services:
-  transfair:
-    image: docker.verbis.dkfz.de/cache/samply/transfair:latest
-    container_name: bridgehead-transfair
-    environment:
-      # NOTE: Those 3 variables need only to be passed if their set, otherwise transfair will complain about empty url values
-      - INSTITUTE_TTP_URL
-      - INSTITUTE_TTP_API_KEY
-      - PROJECT_ID_SYSTEM
-      - FHIR_REQUEST_URL=${FHIR_REQUEST_URL}
-      - FHIR_INPUT_URL=${FHIR_INPUT_URL}
-      - FHIR_OUTPUT_URL=${FHIR_OUTPUT_URL:-http://blaze:8080}
-      - FHIR_REQUEST_CREDENTIALS=${FHIR_REQUEST_CREDENTIALS}
-      - FHIR_INPUT_CREDENTIALS=${FHIR_INPUT_CREDENTIALS}
-      - FHIR_OUTPUT_CREDENTIALS=${FHIR_OUTPUT_CREDENTIALS}
-      - EXCHANGE_ID_SYSTEM=${EXCHANGE_ID_SYSTEM:-SESSION_ID}
-      - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc
-      - RUST_LOG=${RUST_LOG:-info}
-    volumes:
-      - /var/cache/bridgehead/${PROJECT}/transfair:/transfair
-
-  transfair-input-blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
-    container_name: bridgehead-transfair-input-blaze
-    environment:
-      BASE_URL: "http://bridgehead-transfair-input-blaze:8080"
-      JAVA_TOOL_OPTIONS: "-Xmx1024m"
-      DB_BLOCK_CACHE_SIZE: 1024
-      CQL_EXPR_CACHE_SIZE: 8
-      ENFORCE_REFERENTIAL_INTEGRITY: "false"
-    volumes:
-      - "transfair-input-blaze-data:/app/data"
-    profiles: ["transfair-input-blaze"]
-
-  transfair-request-blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
-    container_name: bridgehead-transfair-requests-blaze
-    environment:
-      BASE_URL: "http://bridgehead-transfair-requests-blaze:8080"
-      JAVA_TOOL_OPTIONS: "-Xmx1024m"
-      DB_BLOCK_CACHE_SIZE: 1024
-      CQL_EXPR_CACHE_SIZE: 8
-      ENFORCE_REFERENTIAL_INTEGRITY: "false"
-    volumes:
-      - "transfair-request-blaze-data:/app/data"
-    profiles: ["transfair-request-blaze"]
-
-volumes:
-  transfair-input-blaze-data:
-  transfair-request-blaze-data:

modules/transfair-setup.sh

@@ -1,22 +0,0 @@
-#!/bin/bash -e
-
-function transfairSetup() {
-    if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then
-        echo "Starting transfair."
-	    OVERRIDE+=" -f ./modules/transfair-compose.yml"
-	    if [ -n "$FHIR_INPUT_URL" ]; then
-		    log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL"
-	    else
-		    log INFO "TransFAIR input fhir store not set writing to internal blaze"
-		    FHIR_INPUT_URL="http://transfair-input-blaze:8080"
-		    OVERRIDE+=" --profile transfair-input-blaze"
-	    fi
-	    if [ -n "$FHIR_REQUEST_URL" ]; then
-		    log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL"
-	    else
-		    log INFO "TransFAIR request fhir store not set writing to internal blaze"
-		    FHIR_REQUEST_URL="http://transfair-requests-blaze:8080"
-		    OVERRIDE+=" --profile transfair-request-blaze"
-	    fi
-    fi
-}

versions/prod

@@ -1,2 +0,0 @@
-FOCUS_TAG=main
-BEAM_TAG=main

versions/test

@@ -1,2 +0,0 @@
-FOCUS_TAG=develop
-BEAM_TAG=develop