Test

Merge develop into main

bbmri/modules/eric-setup.sh

@@ -10,6 +10,10 @@ if [ "${ENABLE_ERIC}" == "true" ]; then
 			export ERIC_BROKER_ID=broker.bbmri.samply.de
 			export ERIC_ROOT_CERT=eric
 			;;
+		"acceptance")
+			export ERIC_BROKER_ID=broker-acc.bbmri-acc.samply.de
+			export ERIC_ROOT_CERT=eric.acc
+			;;
 		"test")
 			export ERIC_BROKER_ID=broker-test.bbmri-test.samply.de
 			export ERIC_ROOT_CERT=eric.test

bbmri/modules/eric.acc.root.crt.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNTCCAh2gAwIBAgIUE/wu6FmI+KSMOalI65b+lI3HI4cwDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjQwOTE2MTUyMzU0WhcNMzQw
+OTE0MTUyNDI0WjAWMRQwEgYDVQQDEwtCcm9rZXItUm9vdDCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAOt1I1FQt2bI4Nnjtg8JBYid29cBIkDT4MMb45Jr
+ays24y4R3WO7VJK9UjNduSq/A1jlA0W0A/szDf8Ojq6bBtg+uL92PTDjYH1QXwX0
+c7eMo2tvvyyrs/cb2/ovDBQ1lpibcxVmVAv042ASmil3SdqKKXpv3ATnF9I7V4cv
+fwB56FChaGIov5EK+9JOMjTx6oMlBEgUFR6qq/lSqM9my0HYwUFbX2W+nT9EKEIP
+9UP1eyfRZR3E/+oticnm/cS20BGCbjoYrNgLthXKyaASuhGoElKs8EZ3h9MiI+u0
+DpR0KpePhAkMLugBrgYWqkMwwD1684LfC4YVQrsLwzo5OW8CAwEAAaN7MHkwDgYD
+VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFPbXs3g3lMjH
+1JMe0a5aVbN7lB92MB8GA1UdIwQYMBaAFPbXs3g3lMjH1JMe0a5aVbN7lB92MBYG
+A1UdEQQPMA2CC0Jyb2tlci1Sb290MA0GCSqGSIb3DQEBCwUAA4IBAQBM5RsXb2HN
+FpC1mYfocXAn20Zu4d603qmc/IqkiOWbp36pWo+jk1AxejyRS9hEpQalgSnvcRPQ
+1hPEhGU+wvI0WWVi/01iNjVbXmJNPQEouXQWAT17dyp9vqQkPw8LNzpSV/qdPgbT
+Z9o3sZrjUsSLsK7A7Q5ky4ePkiJBaMsHeAD+wqGwpiJ4D2Xhp8e1v36TWM0qt2EA
+gySx9isx/jeGGPBmDqYB9BCal5lrihPN56jd+5pCkyXeZqKWiiXFJKXwcwxctYZc
+ADHIiTLLPXE8LHTUJAO51it1NAZ1S24aMzax4eWDXcWO7/ybbx5pkYkMd6EqlKHd
+8riQJIhY4huX
+-----END CERTIFICATE-----

bridgehead

@@ -69,7 +69,7 @@ loadVars() {
 		if [ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]; then
 			ENVIRONMENT="production"
 		else
-			ENVIRONMENT="test"
+			ENVIRONMENT="test" # we have acceptance environment in BBMRI ERIC and it would be more appropriate to default to that one in case the data they have in BH is real, but I'm gonna leave it as is for backward compatibility
 		fi
 	fi
 	# Source the versions of the images components 
@@ -80,6 +80,9 @@ loadVars() {
 		"test")
 			source ./versions/test
 			;;
+		"acceptance")
+			source ./versions/acceptance
+			;;
 		*)
 			report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!"
 			source ./versions/prod

ccp/modules/exporter-compose.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   exporter:
-    image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest
+    image: docker.verbis.dkfz.de/ccp/dktk-exporter:test
     container_name: bridgehead-ccp-exporter
     environment:
       JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC"

ccp/modules/teiler-compose.yml

@@ -19,7 +19,8 @@ services:
       HTTP_RELATIVE_PATH: "/ccp-teiler"
 
   teiler-dashboard:
-    image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:develop
+    #image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:develop
+    image: samply/teiler-dashboard:develop
     container_name: bridgehead-teiler-dashboard
     labels:
       - "traefik.enable=true"
@@ -31,6 +32,7 @@ services:
     environment:
       DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE}"
       TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend"
+      TEILER_DASHBOARD_URL: "https://${HOST}/ccp-teiler-dashboard"
       OIDC_URL: "${OIDC_URL}"
       OIDC_REALM: "${OIDC_REALM}"
       OIDC_CLIENT_ID: "${OIDC_PUBLIC_CLIENT_ID}"
@@ -41,7 +43,6 @@ services:
       TEILER_PROJECT: "${PROJECT}"
       EXPORTER_API_KEY: "${EXPORTER_API_KEY}"
       TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler"
-      TEILER_DASHBOARD_HTTP_RELATIVE_PATH: "/ccp-teiler-dashboard"
       TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler"
       TEILER_USER: "${OIDC_USER_GROUP}"
       TEILER_ADMIN: "${OIDC_ADMIN_GROUP}"
@@ -69,10 +70,10 @@ services:
       TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler"
       TEILER_DASHBOARD_DE_URL: "https://${HOST}/ccp-teiler-dashboard/de"
       TEILER_DASHBOARD_EN_URL: "https://${HOST}/ccp-teiler-dashboard/en"
-      CENTRAX_URL: "${CENTRAXX_URL}"
       HTTP_PROXY: "http://forward_proxy:3128"
       ENABLE_MTBA: "${ENABLE_MTBA}"
       ENABLE_DATASHIELD: "${ENABLE_DATASHIELD}"
+      IDMANAGER_UPLOAD_APIKEY: "${IDMANAGER_UPLOAD_APIKEY}" # Only used to check if the ID Manager is active
     secrets:
       - ccp.conf
 

ccp/vars

@@ -30,3 +30,11 @@ idManagementSetup
 mtbaSetup
 obds2fhirRestSetup
 blazeSecondarySetup
+
+for module in modules/*.sh
+do
+    log DEBUG "sourcing $module"
+    source $module
+done
+
+transfairSetup

kr/modules/teiler-compose.yml

@@ -31,6 +31,7 @@ services:
     environment:
       DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE}"
       TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend"
+      TEILER_DASHBOARD_URL: "https://${HOST}/ccp-teiler-dashboard"
       OIDC_URL: "${OIDC_URL}"
       OIDC_REALM: "${OIDC_REALM}"
       OIDC_CLIENT_ID: "${OIDC_PUBLIC_CLIENT_ID}"
@@ -41,7 +42,6 @@ services:
       TEILER_PROJECT: "${PROJECT}"
       EXPORTER_API_KEY: "${EXPORTER_API_KEY}"
       TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler"
-      TEILER_DASHBOARD_HTTP_RELATIVE_PATH: "/ccp-teiler-dashboard"
       TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler"
       TEILER_USER: "${OIDC_USER_GROUP}"
       TEILER_ADMIN: "${OIDC_ADMIN_GROUP}"
@@ -69,7 +69,6 @@ services:
       TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler"
       TEILER_DASHBOARD_DE_URL: "https://${HOST}/ccp-teiler-dashboard/de"
       TEILER_DASHBOARD_EN_URL: "https://${HOST}/ccp-teiler-dashboard/en"
-      CENTRAX_URL: "${CENTRAXX_URL}"
       HTTP_PROXY: "http://forward_proxy:3128"
       ENABLE_MTBA: "${ENABLE_MTBA}"
       ENABLE_DATASHIELD: "${ENABLE_DATASHIELD}"

lib/functions.sh

@@ -334,24 +334,40 @@ function secret_sync_gitlab_token() {
             ;;
     esac
 
-    # Use Secret Sync to validate the GitLab token in /var/cache/bridgehead/secrets/gitlab_token.
+    if [ "$PROJECT" == "bbmri" ]; then
+        # If the project is BBMRI, use the BBMRI-ERIC broker and not the GBN broker
+        proxy_id=$ERIC_PROXY_ID
+        broker_url=$ERIC_BROKER_URL
+        broker_id=$ERIC_BROKER_ID
+        root_crt_file="/srv/docker/bridgehead/bbmri/modules/${ERIC_ROOT_CERT}.root.crt.pem"
+    else
+        proxy_id=$PROXY_ID
+        broker_url=$BROKER_URL
+        broker_id=$BROKER_ID
+        root_crt_file="/srv/docker/bridgehead/$PROJECT/root.crt.pem"
+    fi
+
+    # Create a temporary directory for Secret Sync that is valid per boot
+    secret_sync_tempdir="/tmp/bridgehead/secret-sync.boot-$(cat /proc/sys/kernel/random/boot_id)"
+    mkdir -p $secret_sync_tempdir
+
+    # Use Secret Sync to validate the GitLab token in $secret_sync_tempdir/cache.
     # If it is missing or expired, Secret Sync will create a new token and write it to the file.
     # The git credential helper reads the token from the file during git pull.
-    mkdir -p /var/cache/bridgehead/secrets
-    touch /var/cache/bridgehead/secrets/gitlab_token # the file has to exist to be mounted correctly in the Docker container
     log "INFO" "Running Secret Sync for the GitLab token (gitlab=$gitlab)"
     docker pull docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest # make sure we have the latest image
     docker run --rm \
-        -v /var/cache/bridgehead/secrets/gitlab_token:/usr/local/cache \
         -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \
-        -v /srv/docker/bridgehead/$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \
+        -v $root_crt_file:/run/secrets/root.crt.pem:ro \
         -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \
+        -v $secret_sync_tempdir:/secret-sync/ \
+        -e CACHE_PATH=/secret-sync/gitlab-token \
         -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \
         -e NO_PROXY=localhost,127.0.0.1 \
         -e ALL_PROXY=$HTTPS_PROXY_FULL_URL \
-        -e PROXY_ID=$PROXY_ID \
+        -e PROXY_ID=$proxy_id \
-        -e BROKER_URL=$BROKER_URL \
+        -e BROKER_URL=$broker_url \
-        -e GITLAB_PROJECT_ACCESS_TOKEN_PROVIDER=secret-sync-central.central-secret-sync.$BROKER_ID \
+        -e GITLAB_PROJECT_ACCESS_TOKEN_PROVIDER=secret-sync-central.central-secret-sync.$broker_id \
         -e SECRET_DEFINITIONS=GitLabProjectAccessToken:BRIDGEHEAD_CONFIG_REPO_TOKEN:$gitlab \
         docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest
     if [ $? -eq 0 ]; then

lib/gitlab-token-helper.sh

@@ -2,7 +2,7 @@
 
 [ "$1" = "get" ] || exit
 
-source /var/cache/bridgehead/secrets/gitlab_token
+source "/tmp/bridgehead/secret-sync.boot-$(cat /proc/sys/kernel/random/boot_id)/gitlab-token"
 
 # Any non-empty username works, only the token matters
 cat << EOF

lib/install-bridgehead.sh

@@ -41,6 +41,14 @@ if [ ! -z "$NNGM_CTS_APIKEY" ] && [ -z "$NNGM_AUTH" ]; then
   add_basic_auth_user "nngm" $generated_passwd "NNGM_AUTH" $PROJECT
 fi
 
+if [ -z "$TRANSFAIR_AUTH" ]; then
+  if [[ -n "$TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then
+    log "INFO" "Now generating basic auth user for transfair API (see adduser in bridgehead for more information). "
+    generated_passwd="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 32)"
+    add_basic_auth_user "transfair" $generated_passwd "TRANSFAIR_AUTH" $PROJECT
+  fi
+fi
+
 log "INFO" "Registering system units for bridgehead and bridgehead-update"
 cp -v \
     lib/systemd/bridgehead\@.service \

modules/ssh-tunnel-compose.yml

@@ -0,0 +1,17 @@
+version: "3.7"
+
+services:
+  ssh-tunnel:
+    image: docker.verbis.dkfz.de/cache/samply/ssh-tunnel
+    container_name: bridgehead-ccp-ssh-tunnel
+    environment:
+      SSH_TUNNEL_USERNAME: "${SSH_TUNNEL_USERNAME}"
+      SSH_TUNNEL_HOST: "${SSH_TUNNEL_HOST}"
+      SSH_TUNNEL_PORT: "${SSH_TUNNEL_PORT:-22}"
+    volumes:
+      - "/etc/bridgehead/ssh-tunnel.conf:/ssh-tunnel.conf:ro"
+    secrets:
+      - privkey
+secrets:
+  privkey:
+    file: /etc/bridgehead/pki/ssh-tunnel.priv.pem

modules/ssh-tunnel-setup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+if [ -n "$ENABLE_SSH_TUNNEL" ]; then
+	log INFO "SSH Tunnel setup detected -- will start SSH Tunnel."
+	OVERRIDE+=" -f ./modules/ssh-tunnel-compose.yml"
+fi

modules/ssh-tunnel.md

@@ -0,0 +1,19 @@
+# SSH Tunnel Module
+
+This module enables SSH tunneling capabilities for the Bridgehead installation.
+The primary use case for this is to connect bridgehead components that are hosted externally due to security concerns.
+To connect the new components to the locally running bridgehead infra one is supposed to write a docker-compose.override.yml changing the urls to point to the corresponding forwarded port of the ssh-tunnel container.
+
+## Configuration Variables
+
+- `ENABLE_SSH_TUNNEL`: Required to enable the module
+- `SSH_TUNNEL_USERNAME`: Username for SSH connection
+- `SSH_TUNNEL_HOST`: Target host for SSH tunnel
+- `SSH_TUNNEL_PORT`: SSH port (defaults to 22)
+
+## Configuration Files
+
+The module requires the following files to be present:
+
+- `/etc/bridgehead/ssh-tunnel.conf`: SSH tunnel configuration file. Detailed information can be found [here](https://github.com/samply/ssh-tunnel?tab=readme-ov-file#configuration).
+- `/etc/bridgehead/pki/ssh-tunnel.priv.pem`: The SSH private key used to connect to the `SSH_TUNNEL_HOST`. **Passphrases for the key are not supported!**

modules/transfair-compose.yml

@@ -5,8 +5,13 @@ services:
     container_name: bridgehead-transfair
     environment:
       # NOTE: Those 3 variables need only to be passed if their set, otherwise transfair will complain about empty url values
-      - INSTITUTE_TTP_URL
+      - TTP_URL
-      - INSTITUTE_TTP_API_KEY
+      - TTP_ML_API_KEY
+      - TTP_GW_SOURCE
+      - TTP_GW_EPIX_DOMAIN
+      - TTP_GW_GPAS_DOMAIN
+      - TTP_TYPE
+      - TTP_AUTH
       - PROJECT_ID_SYSTEM
       - FHIR_REQUEST_URL=${FHIR_REQUEST_URL}
       - FHIR_INPUT_URL=${FHIR_INPUT_URL}
@@ -18,9 +23,23 @@ services:
       - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc
       - RUST_LOG=${RUST_LOG:-info}
       - TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs
+      - TLS_DISABLE=${TRANSFAIR_TLS_DISABLE:-false}
+      - NO_PROXY=${TRANSFAIR_NO_PROXIES}
+      - ALL_PROXY=http://forward_proxy:3128
     volumes:
       - /var/cache/bridgehead/${PROJECT}/transfair:/transfair
       - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.middlewares.transfair-strip.stripprefix.prefixes=/transfair"
+      - "traefik.http.routers.transfair.middlewares=transfair-strip,transfair-auth"
+      - "traefik.http.routers.transfair.rule=PathPrefix(`/transfair`)"
+      - "traefik.http.services.transfair.loadbalancer.server.port=8080"
+      - "traefik.http.routers.transfair.tls=true"
+  
+  traefik:
+    labels:
+      - "traefik.http.middlewares.transfair-auth.basicauth.users=${TRANSFAIR_AUTH}"
 
   transfair-input-blaze:
     image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG}
@@ -34,12 +53,19 @@ services:
     volumes:
       - "transfair-input-blaze-data:/app/data"
     profiles: ["transfair-input-blaze"]
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.transfair-input-blaze.rule=PathPrefix(`/data-delivery`)"
+      - "traefik.http.middlewares.transfair-input-strip.stripprefix.prefixes=/data-delivery"
+      - "traefik.http.services.transfair-input-blaze.loadbalancer.server.port=8080"
+      - "traefik.http.routers.transfair-input-blaze.middlewares=transfair-input-strip,transfair-auth"
+      - "traefik.http.routers.transfair-input-blaze.tls=true"
 
   transfair-request-blaze:
     image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG}
-    container_name: bridgehead-transfair-requests-blaze
+    container_name: bridgehead-transfair-request-blaze
     environment:
-      BASE_URL: "http://bridgehead-transfair-requests-blaze:8080"
+      BASE_URL: "http://bridgehead-transfair-request-blaze:8080"
       JAVA_TOOL_OPTIONS: "-Xmx1024m"
       DB_BLOCK_CACHE_SIZE: 1024
       CQL_EXPR_CACHE_SIZE: 8
@@ -47,6 +73,13 @@ services:
     volumes:
       - "transfair-request-blaze-data:/app/data"
     profiles: ["transfair-request-blaze"]
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.transfair-request-blaze.rule=PathPrefix(`/data-requests`)"
+      - "traefik.http.middlewares.transfair-request-strip.stripprefix.prefixes=/data-requests"
+      - "traefik.http.services.transfair-request-blaze.loadbalancer.server.port=8080"
+      - "traefik.http.routers.transfair-request-blaze.middlewares=transfair-request-strip,transfair-auth"
+      - "traefik.http.routers.transfair-request-blaze.tls=true"
 
 volumes:
   transfair-input-blaze-data:

modules/transfair-setup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 function transfairSetup() {
-    if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then
+    if [[ -n "$TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then
         echo "Starting transfair."
 	    OVERRIDE+=" -f ./modules/transfair-compose.yml"
 	    if [ -n "$FHIR_INPUT_URL" ]; then
@@ -15,8 +15,21 @@ function transfairSetup() {
 		    log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL"
 	    else
 		    log INFO "TransFAIR request fhir store not set writing to internal blaze"
-		    FHIR_REQUEST_URL="http://transfair-requests-blaze:8080"
+		    FHIR_REQUEST_URL="http://transfair-request-blaze:8080"
 		    OVERRIDE+=" --profile transfair-request-blaze"
 	    fi
+	    if [ -n "$TTP_GW_SOURCE" ]; then
+		    log INFO "TransFAIR configured with greifswald as ttp"
+		    TTP_TYPE="greifswald"
+	    elif [ -n "$TTP_ML_API_KEY" ]; then
+		    log INFO "TransFAIR configured with mainzelliste as ttp"
+		    TTP_TYPE="mainzelliste"
+        else
+		    log INFO "TransFAIR configured without ttp"
+	    fi
+        TRANSFAIR_NO_PROXIES="transfair-input-blaze,blaze,transfair-requests-blaze"
+        if [ -n "${TRANSFAIR_NO_PROXY}" ]; then
+            TRANSFAIR_NO_PROXIES+=",${TRANSFAIR_NO_PROXY}"
+        fi
     fi
 }

versions/acceptance

@@ -0,0 +1,3 @@
+FOCUS_TAG=develop
+BEAM_TAG=develop
+BLAZE_TAG=main