Merge branch 'main' into dnpmconnector

ccp/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.7"
 services:
   traefik:
     container_name: bridgehead-traefik
-    image: traefik:latest
+    image: docker.verbis.dkfz.de/cache/traefik:latest
     command:
       - --entrypoints.web.address=:80
       - --entrypoints.websecure.address=:443
@@ -32,7 +32,7 @@ services:
 
   forward_proxy:
     container_name: bridgehead-forward-proxy
-    image: samply/bridgehead-forward-proxy:latest
+    image: docker.verbis.dkfz.de/cache/samply/bridgehead-forward-proxy:latest
     environment:
       HTTPS_PROXY: ${HTTPS_PROXY_URL}
       USERNAME: ${HTTPS_PROXY_USERNAME}
@@ -42,7 +42,7 @@ services:
 
   landing:
     container_name: bridgehead-landingpage
-    image: samply/bridgehead-landingpage:master
+    image: docker.verbis.dkfz.de/cache/samply/bridgehead-landingpage:master
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.landing.rule=PathPrefix(`/`)"
@@ -54,7 +54,7 @@ services:
       SITE_NAME: ${SITE_NAME}
 
   blaze:
-    image: "samply/blaze:0.18"
+    image: docker.verbis.dkfz.de/cache/samply/blaze:0.19
     container_name: bridgehead-ccp-blaze
     environment:
       BASE_URL: "http://bridgehead-ccp-blaze:8080"
@@ -72,7 +72,7 @@ services:
       - "traefik.http.routers.blaze_ccp.tls=true"
 
   spot:
-    image: samply/spot:latest
+    image: docker.verbis.dkfz.de/cache/samply/spot:latest
     container_name: bridgehead-spot
     environment:
       SECRET: ${SPOT_BEAM_SECRET_LONG}
@@ -85,7 +85,7 @@ services:
       - "blaze"
 
   beam-proxy:
-    image: "samply/beam-proxy:develop"
+    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop
     container_name: bridgehead-beam-proxy
     environment:
       BROKER_URL: ${BROKER_URL}
@@ -95,15 +95,16 @@ services:
       APP_1_ID: report-hub
       APP_1_KEY: ${REPORTHUB_BEAM_SECRET_SHORT}
       PRIVKEY_FILE: /run/secrets/proxy.pem
-      RUST_LOG: debug
       ALL_PROXY: http://forward_proxy:3128
       TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
+      ROOTCERT_FILE: /conf/root.crt.pem
     secrets:
       - proxy.pem
     depends_on:
       - "forward_proxy"
     volumes:
       - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
+      - ./root.crt.pem:/conf/root.crt.pem:ro
 
 
 volumes:

ccp/exliquid-compose.yml

@@ -0,0 +1,34 @@
+version: "3.7"
+
+services:
+  exliquid-task-store:
+    image: docker.verbis.dkfz.de/cache/samply/blaze:0.19
+    container_name: bridgehead-exliquid-task-store
+    environment:
+      BASE_URL: "http://bridgehead-exliquid-task-store:8080"
+      JAVA_TOOL_OPTIONS: "-Xmx1g"
+    volumes:
+    - "exliquid-task-store-data:/app/data"
+    labels:
+      - "traefik.enable=false"
+
+  exliquid-report-hub:
+    image: docker.verbis.dkfz.de/cache/samply/report-hub:latest
+    container_name: bridgehead-exliquid-report-hub
+    environment:
+      SPRING_WEBFLUX_BASE_PATH: "/exliquid"
+      JAVA_TOOL_OPTIONS: "-Xmx1g"
+      APP_BEAM_APPID: "report-hub.${PROXY_ID}"
+      APP_BEAM_SECRET: ${REPORTHUB_BEAM_SECRET_SHORT}
+      APP_BEAM_PROXY_BASEURL: http://beam-proxy:8081
+      APP_TASKSTORE_BASEURL: "http://bridgehead-exliquid-task-store:8080/fhir"
+      APP_DATASTORE_BASEURL:  http://bridgehead-ccp-blaze:8080/fhir
+    restart: always
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.report-ccp.rule=PathPrefix(`/exliquid`)"
+      - "traefik.http.services.report-ccp.loadbalancer.server.port=8080"
+      - "traefik.http.routers.report-ccp.tls=true"
+
+volumes:
+  exliquid-task-store-data:

ccp/exliquid-setup.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+function exliquidSetup() {
+	case ${SITE_ID} in
+		berlin|dresden|essen|frankfurt|freiburg|luebeck|mainz|muenchen-lmu|muenchen-tum|mannheim|tuebingen)
+			EXLIQUID=1
+			;;
+		dktk-test)
+			EXLIQUID=1
+			;;
+		*)
+			EXLIQUID=0
+			;;
+	esac
+	if [[ $EXLIQUID -eq 1 ]]; then
+		log INFO "EXLIQUID setup detected -- will start Report-Hub."
+		OVERRIDE+=" -f ./$PROJECT/exliquid-compose.yml"
+	fi
+}

ccp/modules/id-management-compose.yml

@@ -0,0 +1,57 @@
+version: "3.7"
+services:
+  id-manager:
+    image: docker.verbis.dkfz.de/bridgehead/magicpl
+    container_name: bridgehead-id-manager
+    environment:
+      TOMCAT_REVERSEPROXY_FQDN: ${HOST}
+      MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID}
+      MAGICPL_ALLOWED_ORIGINS: https://${HOST}
+      MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
+      MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_UPLOAD_APIKEY}
+      MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY}
+      MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY}
+      MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY}
+      MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID}
+      MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET}
+    depends_on:
+      - patientlist
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)"
+      - "traefik.http.services.id-manager.loadbalancer.server.port=8080"
+      - "traefik.http.routers.id-manager.tls=true"
+
+  patientlist:
+    image: docker.verbis.dkfz.de/bridgehead/mainzelliste
+    container_name: bridgehead-patientlist
+    environment:
+      - TOMCAT_REVERSEPROXY_FQDN=${HOST}
+      - ML_SITE=${IDMANAGEMENT_FRIENDLY_ID}
+      - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD}
+      - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
+      - ML_UPLOAD_API_KEY=${IDMANAGER_UPLOAD_APIKEY}
+      # Add Variables from /etc/patientlist-id-generators.env
+      - PATIENTLIST_SEEDS_TRANSFORMED
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)"
+      - "traefik.http.services.patientlist.loadbalancer.server.port=8080"
+      - "traefik.http.routers.patientlist.tls=true"
+    depends_on:
+      - patientlist-db
+
+  patientlist-db:
+    image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine
+    container_name: bridgehead-patientlist-db
+    environment:
+      POSTGRES_USER: "mainzelliste"
+      POSTGRES_DB: "mainzelliste"
+      POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
+    volumes:
+      - "patientlist-db-data:/var/lib/postgresql/data"
+      # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!!
+      - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/"
+
+volumes:
+  patientlist-db-data:

ccp/modules/id-management-setup.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+function idManagementSetup() {
+	if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then
+		log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)."
+		OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml"
+
+		# Auto Generate local Passwords
+		PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+		IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
+
+		# Transform Seeds Configuration to pass it to the Mainzelliste Container
+		PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')"
+
+		# Ensure old ids are working !!!
+		export IDMANAGEMENT_FRIENDLY_ID=$(legacyIdMapping "$SITE_ID")
+	fi
+}
+
+# Transform into single string array, e.g. 'dktk-test' to 'dktk test'
+# Usage: transformToSingleStringArray 'dktk-test' -> 'dktk test'
+function transformToSingleStringArray() {
+	echo "${1//-/ }";
+}
+
+# Ensure all Words are Uppercase
+# Usage: transformToUppercase 'dktk test' -> 'Dktk Test'
+function transformToUppercase() {
+	result="";
+	for word in $1; do
+		result+=" ${word^}";
+	done
+	echo "$result";
+}
+
+# Handle all execeptions from the norm (e.g LMU, TUM)
+# Usage: applySpecialCases 'Muenchen Lmu Test' -> 'Muenchen LMU Test'
+function applySpecialCases() {
+	result="$1";
+	result="${result/Lmu/LMU}";
+	result="${result/Tum/TUM}";
+	echo "$result";
+}
+
+# Transform current siteids to legacy version
+# Usage: legacyIdMapping "dktk-test" -> "DktkTest"
+function legacyIdMapping() {
+	single_string_array=$(transformToSingleStringArray "$1");
+	uppercase_string=$(transformToUppercase "$single_string_array");
+	normalized_string=$(applySpecialCases "$uppercase_string");
+	echo "$normalized_string" | tr -d ' '
+}

ccp/modules/id-management.md

@@ -0,0 +1,66 @@
+# Module: Id-Management
+This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP Data Protection Concept](https://dktk.dkfz.de/klinische-plattformen/documents-download).
+
+## Getting Started
+The following configuration variables are added to your sites-configuration repository:
+
+```
+IDMANAGER_UPLOAD_APIKEY="<random-string>"
+IDMANAGER_READ_APIKEY="<random-string>"
+IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="<given-to-you-by-ccp-it>"
+IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="<given-to-you-by-ccp-it>"
+IDMANAGER_AUTH_CLIENT_ID="<given-to-you-by-ccp-it>"
+IDMANAGER_AUTH_CLIENT_SECRET="<given-to-you-by-ccp-it>"
+
+IDMANAGER_SEEDS_BK="<three-numbers>"
+IDMANAGER_SEEDS_MDS="<three-numbers>"
+IDMANAGER_SEEDS_DKTK000001985="<three-numbers>"
+```
+> NOTE: Additionally, the CCP-IT adds lines declaring the `PATIENTLIST_SEEDS` array in your site configuration. This will contain the seeds for the different id-generators used in all projects.
+
+Once your Bridgehead is updated and restarted, you're all set!
+
+## Additional information you may want to know
+
+### Services
+
+Upon configuration, the Bridgehead will spawn the following services:
+
+- The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead.
+- The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects.
+- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data`.
+
+### How to import an existing database (e.g from Legacy Windows or from Backups)
+First you must shutdown your local bridgehead instance:
+```
+systemctl stop bridgehead@ccp
+```
+
+Next you need to remove the current patientlist database:
+```
+docker volume rm patientlist-db-data;
+```
+
+Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then the volume `patientlist-db-data` was removed previously. 
+> NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL).
+
+After this, you can restart your bridgehead and the dump will be imported:
+```
+systemctl start bridgehead@ccp
+```
+
+### How to connect your local data-management
+Typically, the sites connect their local data-management for the pseudonym creation with the id-management in the bridgehead. In the following two sections, you can read where you can change the configuration:
+#### Sites using CentraXX
+On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file.
+```
+dktk.idmanagement.url=https://<your-linux-bk-host>/id-manager/translator/getId
+dktk.idmanagement.apiKey=<your-setting-for-IDMANAGER_UPLOAD_APIKEY>
+```
+They typically already exist, but need to be changed to the new values!
+#### Sites using ADT2FHIR
+@Pierre
+
+
+### How to connect the legacy windows bridgehead
+You need to change the configuration file "..." of your Windows Bridgehead. TODO... 

ccp/mtba-compose.yml

@@ -0,0 +1,36 @@
+version: "3.7"
+
+services:
+  mtba:
+    image: docker.verbis.dkfz.de/cache/samply/mtba:develop
+    container_name: bridgehead-mtba
+    environment:
+      BLAZE_STORE_URL: http://blaze:8080
+      # NOTE: Aktuell Berechtigungen wie MagicPL!!!
+      # TODO: Add separate ApiKey to MagicPL only for MTBA!
+      ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY}
+      ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID
+      ID_MANAGER_URL: http://id-manager:8080/id-manager
+      PATIENT_CSV_FIRST_NAME_HEADER: ${MTBA_PATIENT_CSV_FIRST_NAME_HEADER}
+      PATIENT_CSV_LAST_NAME_HEADER: ${MTBA_PATIENT_CSV_LAST_NAME_HEADER}
+      PATIENT_CSV_GENDER_HEADER: ${MTBA_PATIENT_CSV_GENDER_HEADER}
+      PATIENT_CSV_BIRTHDAY_HEADER: ${MTBA_PATIENT_CSV_BIRTHDAY_HEADER}
+      CBIOPORTAL_URL:  http://cbioportal:8080
+      FILE_CHARSET: ${MTBA_FILE_CHARSET}
+      FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE}
+      CSV_DELIMITER: ${MTBA_CSV_DELIMITER}
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.mtba.rule=PathPrefix(`/`)"
+      - "traefik.http.services.mtba.loadbalancer.server.port=80"
+      - "traefik.http.routers.mtba.tls=true"
+    volumes:
+      - /tmp/bridgehead/mtba/input:/app/input
+      - /tmp/bridgehead/mtba/persist:/app/persist
+
+  # TODO: Include CBioPortal in Deployment ...
+  # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted!
+  # TODO: Find a trigger to let mtba signal a restart for CBioPortal
+
+volumes:
+  mtba-data:

ccp/nngm-compose.yml

@@ -18,7 +18,7 @@ services:
       - "traefik.http.routers.connector.tls=true"
 
   connector_db:
-    image: postgres:9.5-alpine
+    image: docker.verbis.dkfz.de/cache/postgres:9.5-alpine
     container_name: bridgehead-ccp-connector-db
     volumes:
       - "connector_db_data:/var/lib/postgresql/data"

ccp/nngm-setup.sh

@@ -3,7 +3,19 @@
 function nngmSetup() {
 	if [ -n "$NNGM_CTS_APIKEY" ]; then
 		log INFO "nNGM setup detected -- will start nNGM Connector."
-		OVERRIDE+="-f ./$PROJECT/nngm-compose.yml"
+		OVERRIDE+=" -f ./$PROJECT/nngm-compose.yml"
 	fi
 	CONNECTOR_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
 }
+
+function mtbaSetup() {
+	# TODO: Check if ID-Management Module is activated!
+	if [ -n "$ENABLE_MTBA" ];then
+		log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal."
+		if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then
+			log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!"
+			exit 1;
+		fi
+		OVERRIDE+=" -f ./$PROJECT/mtba-compose.yml"
+	fi
+}

ccp/root.crt.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNTCCAh2gAwIBAgIUMeGRSrNPhRdQ1tU7uK5+lUa4f38wDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjIwOTI5MTQxMjU1WhcNMzIw
+OTI2MTQxMzI1WjAWMRQwEgYDVQQDEwtCcm9rZXItUm9vdDCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAMYyroOUeb27mYzClOrjCmgIceLalsFA0aVCh5mZ
+KtP8+1U3oq/7exP30gXiJojxW7xoerfyQY9s0Sz5YYbxYbuskFOYEtyAILB/pxgd
++k+J3tlZKolpfmo7WT5tZiHxH/zjrtAYGnuB2xPHRMCWh/tHYrELgXQuilNol24y
+GBa1plTlARy0aKEDUHp87WLhD2qH7B8sFlLgo0+gunE1UtR2HMSPF45w3VXszyG6
+fJNrAj0yPnKy3Dm1BMO3jDO2e0A9lCQ71a4j4TeKePfCk1xCArSu6PpiwiacKplF
+c6CRR6KrWVm2g+8Y2hFcOBG/Py2xusm3PWbpylGq6vtFRkkCAwEAAaN7MHkwDgYD
+VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEFxD6BQwQO5
+xsJ+3cvZypsnh6dDMB8GA1UdIwQYMBaAFEFxD6BQwQO5xsJ+3cvZypsnh6dDMBYG
+A1UdEQQPMA2CC0Jyb2tlci1Sb290MA0GCSqGSIb3DQEBCwUAA4IBAQB5zTeIhV/3
+3Am6O144EFtnIeaZ2w0D6aEHqHAZp50vJv3+uQfOliCOzgw7VDxI4Zz2JALjlR/i
+uOYHsu3YIRMIOmPOjqrdDJa6auB0ufL4oUPfCRln7Fh0f3JVlz3BUoHsSDt949p4
+g0nnsciL2JHuzlqjn7Jyt3L7dAHrlFKulCcuidG5D3cqXrRCbF83f+k3TC/HRiNd
+25oMi7I4MP/SOCdfQGUGIsHIf/0hSm3pNjDOrC/XuI/8gh2f5io+Y8V+hMwMBcm4
+JbH8bdyBB+EIhsNbTwf2MWntD5bmg47sf7hh23aNvKXI67Li1pTI2t1CqiGnFR0U
+fCEpeaEAHs0k
+-----END CERTIFICATE-----

ccp/vars

@@ -8,10 +8,17 @@ REPORTHUB_BEAM_SECRET_LONG="ApiKey report-hub.${PROXY_ID} ${REPORTHUB_BEAM_SECRE
 SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 
+# This will load id-management setup. Effective only if id-management configuration is defined.
+source $PROJECT/modules/id-management-setup.sh
+idManagementSetup
 # This will load nngm setup. Effective only if nngm configuration is defined.
 source $PROJECT/nngm-setup.sh
 nngmSetup
 
+source $PROJECT/exliquid-setup.sh
+exliquidSetup
+mtbaSetup
+
 # This will load DNPM setup. Effective only if DNPM configuration is defined in /etc/bridgehead/dnpm.
 source dnpm/dnpm-setup.sh
 dnpmSetup