From 99c0e7f28354bcb6ae49f30556df78d1a9e1da07 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 7 Dec 2022 15:46:19 +0100 Subject: [PATCH 01/34] Added Configuration for Local ID-Management --- ccp/modules/id-management-compose.yml | 75 +++++++++++++++++++++++++++ ccp/modules/id-management-setup.sh | 17 ++++++ ccp/vars | 3 ++ lib/functions.sh | 2 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 ccp/modules/id-management-compose.yml create mode 100644 ccp/modules/id-management-setup.sh diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml new file mode 100644 index 0000000..0968048 --- /dev/null +++ b/ccp/modules/id-management-compose.yml @@ -0,0 +1,75 @@ +version: "3.7" +services: + id-manager: + image: docker.verbis.dkfz.de/bridgehead/magicpl + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST} + MAGICPL_SITE: ${SITE_ID} + MAGICPL_ALLOWED_ORIGINS: https://${HOST} + MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} + MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} + MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET} + depends_on: + - patientlist + labels: + - "traefik.enable=true" + - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)" + - "traefik.http.services.id-manager.loadbalancer.server.port=8080" + - "traefik.http.routers.id-manager.tls=true" + + patientlist: + image: docker.verbis.dkfz.de/bridgehead/mainzelliste + environment: + - TOMCAT_REVERSEPROXY_FQDN=${HOST} + - ML_SITE=${SITE_ID} + - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} + - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # Add Variables from /etc/patientlist-id-generators.env + - ML_BK_IDGENERATOR_RANDOM_1 + - ML_BK_IDGENERATOR_RANDOM_2 + - ML_BK_IDGENERATOR_RANDOM_3 + - ML_MDS_IDGENERATOR_RANDOM_1 + - ML_MDS_IDGENERATOR_RANDOM_2 + - ML_MDS_IDGENERATOR_RANDOM_3 + - ML_DKTK000001985_IDGENERATOR_RANDOM_1 + - ML_DKTK000001985_IDGENERATOR_RANDOM_2 + - ML_DKTK000001985_IDGENERATOR_RANDOM_3 + - ML_DKTK000001986_IDGENERATOR_RANDOM_1 + - ML_DKTK000001986_IDGENERATOR_RANDOM_2 + - ML_DKTK000001986_IDGENERATOR_RANDOM_3 + - ML_DKTK000001950_IDGENERATOR_RANDOM_1 + - ML_DKTK000001950_IDGENERATOR_RANDOM_2 + - ML_DKTK000001950_IDGENERATOR_RANDOM_3 + - ML_DKTK000001951_IDGENERATOR_RANDOM_1 + - ML_DKTK000001951_IDGENERATOR_RANDOM_2 + - ML_DKTK000001951_IDGENERATOR_RANDOM_3 + - ML_DKTK999999999_IDGENERATOR_RANDOM_1 + - ML_DKTK999999999_IDGENERATOR_RANDOM_2 + - ML_DKTK999999999_IDGENERATOR_RANDOM_3 + - ML_DKTK000002089_IDGENERATOR_RANDOM_1 + - ML_DKTK000002089_IDGENERATOR_RANDOM_2 + - ML_DKTK000002089_IDGENERATOR_RANDOM_3 + labels: + - "traefik.enable=true" + - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" + - "traefik.http.services.patientlist.loadbalancer.server.port=8080" + - "traefik.http.routers.patientlist.tls=true" + depends_on: + - patientlist-db + + patientlist-db: + image: postgres:14-alpine + environment: + POSTGRES_USER: "mainzelliste" + POSTGRES_DB: "mainzelliste" + POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} + volumes: + - "patientlist-db-data:/var/lib/postgresql/data" + +volumes: + patientlist-db-data: + diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh new file mode 100644 index 0000000..ca939fb --- /dev/null +++ b/ccp/modules/id-management-setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +function idManagementSetup() { + if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." + OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" + + # Auto Generate local Passwords + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + + # Source the ID Generators Configuration + source /etc/bridgehead/patientlist-id-generators.env + log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + fi + +} diff --git a/ccp/vars b/ccp/vars index 63def80..89deae0 100644 --- a/ccp/vars +++ b/ccp/vars @@ -8,6 +8,9 @@ REPORTHUB_BEAM_SECRET_LONG="ApiKey report-hub.${PROXY_ID} ${REPORTHUB_BEAM_SECRE SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +# This will load id-management setup. Effective only if id-management configuration is defined. +source $PROJECT/modules/id-management-setup.sh +idManagementSetup # This will load nngm setup. Effective only if nngm configuration is defined. source $PROJECT/nngm-setup.sh nngmSetup diff --git a/lib/functions.sh b/lib/functions.sh index 4f40fd0..355ebaa 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -131,7 +131,7 @@ fail_and_report() { setHostname() { if [ -z "$HOST" ]; then - export HOST=$(hostname -f) + export HOST=$(hostname -f | tr "[:upper:]" "[:lower:]") log DEBUG "Using auto-detected hostname $HOST." fi } From 5e063003ace490b3835e30fcebbafd6465ed4586 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 13 Dec 2022 16:51:32 +0100 Subject: [PATCH 02/34] feature: Added automated Backups for PostgreSQL --- README.md | 9 +++++++ ccp/modules/id-management-compose.yml | 11 ++++---- lib/functions.sh | 11 ++++++++ lib/install-bridgehead.sh | 10 ++++--- lib/update-bridgehead.sh | 39 +++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b57c10e..24ebdf2 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. +- `/var/data/bridgehead` contains persistent data of the bridgehead + - `/var/data/bridgehead/backups` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. @@ -139,6 +141,13 @@ Your Bridgehead will automatically and regularly check for updates. Whenever som If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@.service` and `systemctl cat bridgehead-update@ \ + "$1/$2/$(date +%Y-KW%V).sql" +} + + # from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 # ex. use: retry 5 /bin/false function retry { diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 5e3add3..7cbd8ef 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,6 +37,10 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi +log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data." +mkdir -p /var/data/bridgehead +chown -R bridgehead /var/data/bridgehead + log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ lib/systemd/bridgehead\@.service \ @@ -63,4 +67,4 @@ else STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR" fi -log "INFO" "$STR" \ No newline at end of file +log "INFO" "$STR" diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 897c8a2..cc1d55f 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,6 +1,45 @@ #!/bin/bash source lib/functions.sh +AUTO_BACKUP=${AUTO_BACKUP:-true} + +if [ "$AUTO_BACKUP" == "true" ]; then + BACKUP_DIRECTORY="/var/data/bridgehead/backups" + if [ ! -d /var/data ]; then + log DEBUG "Created /var/data" + mkdir /var/data + fi + if [ ! -d /var/data/bridgehead ]; then + log DEBUG "Created /var/data/bridgehead" + mkdir /var/data/bridgehead + fi + if [ ! -d $BACKUP_DIRECTORY ]; then + message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY + fi + BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d $BACKUP_DIRECTORY/$service ]; then + message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY/$service + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then From 125bb5e26232be6f5279d1fb771a99710d8fe012 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:03 +0100 Subject: [PATCH 03/34] refactor: Changed Trigger of the IDM Module --- ccp/modules/id-management-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index ca939fb..790f846 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From bc72093be9c42db534d12a4d418106cf7e21409d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:23 +0100 Subject: [PATCH 04/34] docs: Added Documentation for the IDM Module --- ccp/modules/id-management.md | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ccp/modules/id-management.md diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md new file mode 100644 index 0000000..8de2f5a --- /dev/null +++ b/ccp/modules/id-management.md @@ -0,0 +1,58 @@ +# Module: Id-Management +This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP-DSK](https://dktk.dkfz.de/application/files/5016/2030/2474/20_11_23_Datenschutzkonzept_CCP-IT_inkl_Anlagen.pdf). + +## Getting Started +You must add following configuration variables to your sites-configuration repository: + +``` +IDMANAGER_CENTRAXX_APIKEY="" +IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" +IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" +IDMANAGER_AUTH_CLIENT_ID="" +IDMANAGER_AUTH_CLIENT_SECRET="" +``` + +Additionally, the ccp-it needs to add a new file "patientlist-id-generators.env" to your site configuration. This file will hold the seeds for the different id-generators used in all projects. + +After adding the configuration, you simply need to update your bridgehead and 3 new services will run on your server: + +- `bridgehead-id-manager`, accessible by "https:///id-manager". This component adds a common interface for creating pseudonymes in the bridgehead. +- `bridgehead-patientlist`, accessible by "https:// NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). + +After this, you can restart your bridgehead and the dump will be imported: +``` +systemctl start bridgehead@ccp +``` + +### How to connect your local data-management +Typically, the sites connect their local data-management for the pseudonym creation with the id-management in the bridgehead. In the following two sections, you can read where you can change the configuration: +#### Sites using CentraXX +On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. +``` +dktk.idmanagement.url=https:///id-manager/translator/getId +dktk.idmanagement.apiKey= +``` +They typically already exist, but need to be changed to the new values! +#### Sites using ADT2FHIR +@Pierre + + +### How to connect the legacy windows bridgehead +You need to change the configuration file "..." of your Windows Bridgehead. TODO... From 276be28db1792c4f276191e24bbb3b7cbd8f852c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 16 Dec 2022 12:02:49 +0100 Subject: [PATCH 05/34] fix: Included Legacy ID-Mapping (Please Complete) --- ccp/modules/id-management-compose.yml | 6 +++--- ccp/modules/id-management-setup.sh | 24 ++++++++++++++++++++++-- ccp/modules/id-management.md | 6 +++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index d1639af..4baaba7 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -5,11 +5,11 @@ services: container_name: bridgehead-id-manager environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST} - MAGICPL_SITE: ${SITE_ID} + MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID} MAGICPL_ALLOWED_ORIGINS: https://${HOST} MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} - MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_UPLOAD_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY} MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 790f846..d2449c7 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" @@ -11,7 +11,27 @@ function idManagementSetup() { # Source the ID Generators Configuration source /etc/bridgehead/patientlist-id-generators.env - log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + + # Ensure old ids are working !!! + legacyIdMapping fi } + +# TODO: Map all old site ids to the new ones +function legacyIdMapping() { + case ${SITE_ID} in + "berlin") + export IDMANAGEMENT_FRIENDLY_ID=Berlin + ;; + "dresden") + export IDMANAGEMENT_FRIENDLY_ID=Dresden + ;; + "frankfurt") + export IDMANAGEMENT_FRIENDLY_ID=Frankfurt + ;; + *) + export IDMANAGEMENT_FRIENDLY_ID=$SITE_ID + ;; + esac +} diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 8de2f5a..89ff65c 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -5,8 +5,8 @@ This module provides integration with the CCP-Pseudonymiziation Service. To lear You must add following configuration variables to your sites-configuration repository: ``` -IDMANAGER_CENTRAXX_APIKEY="" -IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_UPLOAD_APIKEY="" +IDMANAGER_READ_APIKEY="" IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" IDMANAGER_AUTH_CLIENT_ID="" @@ -47,7 +47,7 @@ Typically, the sites connect their local data-management for the pseudonym creat On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. ``` dktk.idmanagement.url=https:///id-manager/translator/getId -dktk.idmanagement.apiKey= +dktk.idmanagement.apiKey= ``` They typically already exist, but need to be changed to the new values! #### Sites using ADT2FHIR From 828312e04510315dd9ee87c3d5b9f53a4099852d Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 2 Dec 2022 15:38:33 +0100 Subject: [PATCH 06/34] Added MTBA Module --- ccp/mtba-compose.yml | 33 +++++++++++++++++++++++++++++++++ ccp/nngm-setup.sh | 12 ++++++++++++ ccp/vars | 1 + 3 files changed, 46 insertions(+) create mode 100644 ccp/mtba-compose.yml diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml new file mode 100644 index 0000000..53fd327 --- /dev/null +++ b/ccp/mtba-compose.yml @@ -0,0 +1,33 @@ +version: "3.7" + +services: + mtba: + image: samply/mtba:develop + container_name: bridgehead-mtba + environment: + BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir + # NOTE: Aktuell Berechtigungen wie MagicPL!!! + # TODO: Add separate ApiKey to Patientlist only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID + ID_MANAGER_URL: http://bridgehead-id-manager:8080 + PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} + PATIENT_CSV_LAST_NAME_HEADER: ${PATIENT_CSV_LAST_NAME_HEADER:-"LAST_NAME"} + PATIENT_CSV_GENDER_HEADER: ${PATIENT_CSV_GENDER_HEADER:-"GENDER"} + PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} + CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 + MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" + labels: + - "traefik.enable=true" + - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" + - "traefik.http.services.mtba.loadbalancer.server.port=80" + - "traefik.http.routers.mtba.tls=true" + volumes: + # This directory persists the FHIR Resources that are needed to import data into blaze. + - /var/data/bridgehead/mtba:/app/mtba-files/persist + # Place new import files in this directory + - /tmp/bridgehead/mtba/:/app/mtba-files/input + + # TODO: Include CBioPortal in Deployment ... + # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! + # TODO: Find a trigger to let mtba signal a restart for CBioPortal diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index d5b80eb..b55d48e 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -7,3 +7,15 @@ function nngmSetup() { fi CONNECTOR_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" } + +function mtbaSetup() { + # TODO: Check if ID-Management Module is activated! + if [ -n "$ENABLE_MTBA" ];then + log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" + exit 1; + fi + OVERRIDE+=" -f ./$PROJECT/mtba-compose.yml" + fi +} diff --git a/ccp/vars b/ccp/vars index 89deae0..5172d24 100644 --- a/ccp/vars +++ b/ccp/vars @@ -16,3 +16,4 @@ source $PROJECT/nngm-setup.sh nngmSetup source $PROJECT/exliquid-setup.sh exliquidSetup +mtbaSetup \ No newline at end of file From e1e7ffece8899a57ce6c22b1e5f55ebdf083225c Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 16 Dec 2022 15:01:04 +0100 Subject: [PATCH 07/34] set magicpl instead of patientlist --- ccp/mtba-compose.yml | 4 ++-- ccp/nngm-setup.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 53fd327..d492ae0 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -7,8 +7,8 @@ services: environment: BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir # NOTE: Aktuell Berechtigungen wie MagicPL!!! - # TODO: Add separate ApiKey to Patientlist only for MTBA! - ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # TODO: Add separate ApiKey to MagicPL only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID ID_MANAGER_URL: http://bridgehead-id-manager:8080 PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index b55d48e..ba2e77f 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -12,7 +12,7 @@ function mtbaSetup() { # TODO: Check if ID-Management Module is activated! if [ -n "$ENABLE_MTBA" ];then log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." - if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" exit 1; fi From 6d24dbce7f9710dc4050ec62fc6016273bd38cd4 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 7 Dec 2022 15:46:19 +0100 Subject: [PATCH 08/34] Added Configuration for Local ID-Management --- ccp/modules/id-management-compose.yml | 75 +++++++++++++++++++++++++++ ccp/modules/id-management-setup.sh | 17 ++++++ ccp/vars | 3 ++ lib/functions.sh | 2 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 ccp/modules/id-management-compose.yml create mode 100644 ccp/modules/id-management-setup.sh diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml new file mode 100644 index 0000000..0968048 --- /dev/null +++ b/ccp/modules/id-management-compose.yml @@ -0,0 +1,75 @@ +version: "3.7" +services: + id-manager: + image: docker.verbis.dkfz.de/bridgehead/magicpl + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST} + MAGICPL_SITE: ${SITE_ID} + MAGICPL_ALLOWED_ORIGINS: https://${HOST} + MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} + MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} + MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET} + depends_on: + - patientlist + labels: + - "traefik.enable=true" + - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)" + - "traefik.http.services.id-manager.loadbalancer.server.port=8080" + - "traefik.http.routers.id-manager.tls=true" + + patientlist: + image: docker.verbis.dkfz.de/bridgehead/mainzelliste + environment: + - TOMCAT_REVERSEPROXY_FQDN=${HOST} + - ML_SITE=${SITE_ID} + - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} + - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # Add Variables from /etc/patientlist-id-generators.env + - ML_BK_IDGENERATOR_RANDOM_1 + - ML_BK_IDGENERATOR_RANDOM_2 + - ML_BK_IDGENERATOR_RANDOM_3 + - ML_MDS_IDGENERATOR_RANDOM_1 + - ML_MDS_IDGENERATOR_RANDOM_2 + - ML_MDS_IDGENERATOR_RANDOM_3 + - ML_DKTK000001985_IDGENERATOR_RANDOM_1 + - ML_DKTK000001985_IDGENERATOR_RANDOM_2 + - ML_DKTK000001985_IDGENERATOR_RANDOM_3 + - ML_DKTK000001986_IDGENERATOR_RANDOM_1 + - ML_DKTK000001986_IDGENERATOR_RANDOM_2 + - ML_DKTK000001986_IDGENERATOR_RANDOM_3 + - ML_DKTK000001950_IDGENERATOR_RANDOM_1 + - ML_DKTK000001950_IDGENERATOR_RANDOM_2 + - ML_DKTK000001950_IDGENERATOR_RANDOM_3 + - ML_DKTK000001951_IDGENERATOR_RANDOM_1 + - ML_DKTK000001951_IDGENERATOR_RANDOM_2 + - ML_DKTK000001951_IDGENERATOR_RANDOM_3 + - ML_DKTK999999999_IDGENERATOR_RANDOM_1 + - ML_DKTK999999999_IDGENERATOR_RANDOM_2 + - ML_DKTK999999999_IDGENERATOR_RANDOM_3 + - ML_DKTK000002089_IDGENERATOR_RANDOM_1 + - ML_DKTK000002089_IDGENERATOR_RANDOM_2 + - ML_DKTK000002089_IDGENERATOR_RANDOM_3 + labels: + - "traefik.enable=true" + - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" + - "traefik.http.services.patientlist.loadbalancer.server.port=8080" + - "traefik.http.routers.patientlist.tls=true" + depends_on: + - patientlist-db + + patientlist-db: + image: postgres:14-alpine + environment: + POSTGRES_USER: "mainzelliste" + POSTGRES_DB: "mainzelliste" + POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} + volumes: + - "patientlist-db-data:/var/lib/postgresql/data" + +volumes: + patientlist-db-data: + diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh new file mode 100644 index 0000000..ca939fb --- /dev/null +++ b/ccp/modules/id-management-setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +function idManagementSetup() { + if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." + OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" + + # Auto Generate local Passwords + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + + # Source the ID Generators Configuration + source /etc/bridgehead/patientlist-id-generators.env + log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + fi + +} diff --git a/ccp/vars b/ccp/vars index 63def80..89deae0 100644 --- a/ccp/vars +++ b/ccp/vars @@ -8,6 +8,9 @@ REPORTHUB_BEAM_SECRET_LONG="ApiKey report-hub.${PROXY_ID} ${REPORTHUB_BEAM_SECRE SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +# This will load id-management setup. Effective only if id-management configuration is defined. +source $PROJECT/modules/id-management-setup.sh +idManagementSetup # This will load nngm setup. Effective only if nngm configuration is defined. source $PROJECT/nngm-setup.sh nngmSetup diff --git a/lib/functions.sh b/lib/functions.sh index 4f40fd0..355ebaa 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -131,7 +131,7 @@ fail_and_report() { setHostname() { if [ -z "$HOST" ]; then - export HOST=$(hostname -f) + export HOST=$(hostname -f | tr "[:upper:]" "[:lower:]") log DEBUG "Using auto-detected hostname $HOST." fi } From 1ffc9b9cd581e592c42a8ac2c881045aa39c1c3c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 13 Dec 2022 16:51:32 +0100 Subject: [PATCH 09/34] feature: Added automated Backups for PostgreSQL --- README.md | 9 +++++++ ccp/modules/id-management-compose.yml | 11 ++++---- lib/functions.sh | 11 ++++++++ lib/install-bridgehead.sh | 10 ++++--- lib/update-bridgehead.sh | 39 +++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b57c10e..24ebdf2 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. +- `/var/data/bridgehead` contains persistent data of the bridgehead + - `/var/data/bridgehead/backups` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. @@ -139,6 +141,13 @@ Your Bridgehead will automatically and regularly check for updates. Whenever som If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@.service` and `systemctl cat bridgehead-update@ \ + "$1/$2/$(date +%Y-KW%V).sql" +} + + # from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 # ex. use: retry 5 /bin/false function retry { diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 5e3add3..7cbd8ef 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,6 +37,10 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi +log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data." +mkdir -p /var/data/bridgehead +chown -R bridgehead /var/data/bridgehead + log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ lib/systemd/bridgehead\@.service \ @@ -63,4 +67,4 @@ else STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR" fi -log "INFO" "$STR" \ No newline at end of file +log "INFO" "$STR" diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 897c8a2..cc1d55f 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,6 +1,45 @@ #!/bin/bash source lib/functions.sh +AUTO_BACKUP=${AUTO_BACKUP:-true} + +if [ "$AUTO_BACKUP" == "true" ]; then + BACKUP_DIRECTORY="/var/data/bridgehead/backups" + if [ ! -d /var/data ]; then + log DEBUG "Created /var/data" + mkdir /var/data + fi + if [ ! -d /var/data/bridgehead ]; then + log DEBUG "Created /var/data/bridgehead" + mkdir /var/data/bridgehead + fi + if [ ! -d $BACKUP_DIRECTORY ]; then + message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY + fi + BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d $BACKUP_DIRECTORY/$service ]; then + message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY/$service + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then From 2a46bd00fd31a5278ac23f1f434b21ca355d516d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:03 +0100 Subject: [PATCH 10/34] refactor: Changed Trigger of the IDM Module --- ccp/modules/id-management-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index ca939fb..790f846 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From 4f0526ddf4de73836c74f0607676f3f2fc583f40 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:23 +0100 Subject: [PATCH 11/34] docs: Added Documentation for the IDM Module --- ccp/modules/id-management.md | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ccp/modules/id-management.md diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md new file mode 100644 index 0000000..8de2f5a --- /dev/null +++ b/ccp/modules/id-management.md @@ -0,0 +1,58 @@ +# Module: Id-Management +This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP-DSK](https://dktk.dkfz.de/application/files/5016/2030/2474/20_11_23_Datenschutzkonzept_CCP-IT_inkl_Anlagen.pdf). + +## Getting Started +You must add following configuration variables to your sites-configuration repository: + +``` +IDMANAGER_CENTRAXX_APIKEY="" +IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" +IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" +IDMANAGER_AUTH_CLIENT_ID="" +IDMANAGER_AUTH_CLIENT_SECRET="" +``` + +Additionally, the ccp-it needs to add a new file "patientlist-id-generators.env" to your site configuration. This file will hold the seeds for the different id-generators used in all projects. + +After adding the configuration, you simply need to update your bridgehead and 3 new services will run on your server: + +- `bridgehead-id-manager`, accessible by "https:///id-manager". This component adds a common interface for creating pseudonymes in the bridgehead. +- `bridgehead-patientlist`, accessible by "https:// NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). + +After this, you can restart your bridgehead and the dump will be imported: +``` +systemctl start bridgehead@ccp +``` + +### How to connect your local data-management +Typically, the sites connect their local data-management for the pseudonym creation with the id-management in the bridgehead. In the following two sections, you can read where you can change the configuration: +#### Sites using CentraXX +On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. +``` +dktk.idmanagement.url=https:///id-manager/translator/getId +dktk.idmanagement.apiKey= +``` +They typically already exist, but need to be changed to the new values! +#### Sites using ADT2FHIR +@Pierre + + +### How to connect the legacy windows bridgehead +You need to change the configuration file "..." of your Windows Bridgehead. TODO... From 4d49351ad21f734ec1fcd7aaf9f4ee9bf475a10d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 16 Dec 2022 12:02:49 +0100 Subject: [PATCH 12/34] fix: Included Legacy ID-Mapping (Please Complete) --- ccp/modules/id-management-compose.yml | 6 +++--- ccp/modules/id-management-setup.sh | 24 ++++++++++++++++++++++-- ccp/modules/id-management.md | 6 +++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index d1639af..4baaba7 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -5,11 +5,11 @@ services: container_name: bridgehead-id-manager environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST} - MAGICPL_SITE: ${SITE_ID} + MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID} MAGICPL_ALLOWED_ORIGINS: https://${HOST} MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} - MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_UPLOAD_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY} MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 790f846..d2449c7 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" @@ -11,7 +11,27 @@ function idManagementSetup() { # Source the ID Generators Configuration source /etc/bridgehead/patientlist-id-generators.env - log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + + # Ensure old ids are working !!! + legacyIdMapping fi } + +# TODO: Map all old site ids to the new ones +function legacyIdMapping() { + case ${SITE_ID} in + "berlin") + export IDMANAGEMENT_FRIENDLY_ID=Berlin + ;; + "dresden") + export IDMANAGEMENT_FRIENDLY_ID=Dresden + ;; + "frankfurt") + export IDMANAGEMENT_FRIENDLY_ID=Frankfurt + ;; + *) + export IDMANAGEMENT_FRIENDLY_ID=$SITE_ID + ;; + esac +} diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 8de2f5a..89ff65c 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -5,8 +5,8 @@ This module provides integration with the CCP-Pseudonymiziation Service. To lear You must add following configuration variables to your sites-configuration repository: ``` -IDMANAGER_CENTRAXX_APIKEY="" -IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_UPLOAD_APIKEY="" +IDMANAGER_READ_APIKEY="" IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" IDMANAGER_AUTH_CLIENT_ID="" @@ -47,7 +47,7 @@ Typically, the sites connect their local data-management for the pseudonym creat On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. ``` dktk.idmanagement.url=https:///id-manager/translator/getId -dktk.idmanagement.apiKey= +dktk.idmanagement.apiKey= ``` They typically already exist, but need to be changed to the new values! #### Sites using ADT2FHIR From e0c9a5ced38a61da718b0bdf2b01208c2371f094 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 2 Dec 2022 15:38:33 +0100 Subject: [PATCH 13/34] Added MTBA Module --- ccp/mtba-compose.yml | 33 +++++++++++++++++++++++++++++++++ ccp/nngm-setup.sh | 12 ++++++++++++ ccp/vars | 1 + 3 files changed, 46 insertions(+) create mode 100644 ccp/mtba-compose.yml diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml new file mode 100644 index 0000000..53fd327 --- /dev/null +++ b/ccp/mtba-compose.yml @@ -0,0 +1,33 @@ +version: "3.7" + +services: + mtba: + image: samply/mtba:develop + container_name: bridgehead-mtba + environment: + BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir + # NOTE: Aktuell Berechtigungen wie MagicPL!!! + # TODO: Add separate ApiKey to Patientlist only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID + ID_MANAGER_URL: http://bridgehead-id-manager:8080 + PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} + PATIENT_CSV_LAST_NAME_HEADER: ${PATIENT_CSV_LAST_NAME_HEADER:-"LAST_NAME"} + PATIENT_CSV_GENDER_HEADER: ${PATIENT_CSV_GENDER_HEADER:-"GENDER"} + PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} + CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 + MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" + labels: + - "traefik.enable=true" + - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" + - "traefik.http.services.mtba.loadbalancer.server.port=80" + - "traefik.http.routers.mtba.tls=true" + volumes: + # This directory persists the FHIR Resources that are needed to import data into blaze. + - /var/data/bridgehead/mtba:/app/mtba-files/persist + # Place new import files in this directory + - /tmp/bridgehead/mtba/:/app/mtba-files/input + + # TODO: Include CBioPortal in Deployment ... + # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! + # TODO: Find a trigger to let mtba signal a restart for CBioPortal diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index d5b80eb..b55d48e 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -7,3 +7,15 @@ function nngmSetup() { fi CONNECTOR_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" } + +function mtbaSetup() { + # TODO: Check if ID-Management Module is activated! + if [ -n "$ENABLE_MTBA" ];then + log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" + exit 1; + fi + OVERRIDE+=" -f ./$PROJECT/mtba-compose.yml" + fi +} diff --git a/ccp/vars b/ccp/vars index 89deae0..5172d24 100644 --- a/ccp/vars +++ b/ccp/vars @@ -16,3 +16,4 @@ source $PROJECT/nngm-setup.sh nngmSetup source $PROJECT/exliquid-setup.sh exliquidSetup +mtbaSetup \ No newline at end of file From e7b238de9d023abf69b88699eb132f67acdbe8f9 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 16 Dec 2022 15:01:04 +0100 Subject: [PATCH 14/34] set magicpl instead of patientlist --- ccp/mtba-compose.yml | 4 ++-- ccp/nngm-setup.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 53fd327..d492ae0 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -7,8 +7,8 @@ services: environment: BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir # NOTE: Aktuell Berechtigungen wie MagicPL!!! - # TODO: Add separate ApiKey to Patientlist only for MTBA! - ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # TODO: Add separate ApiKey to MagicPL only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID ID_MANAGER_URL: http://bridgehead-id-manager:8080 PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index b55d48e..ba2e77f 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -12,7 +12,7 @@ function mtbaSetup() { # TODO: Check if ID-Management Module is activated! if [ -n "$ENABLE_MTBA" ];then log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." - if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" exit 1; fi From af7960e036deb3ddff22d42354f554ee71942649 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Thu, 19 Jan 2023 10:05:03 +0000 Subject: [PATCH 15/34] Improve documentation --- ccp/modules/id-management.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 89ff65c..4c929b9 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -1,8 +1,8 @@ # Module: Id-Management -This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP-DSK](https://dktk.dkfz.de/application/files/5016/2030/2474/20_11_23_Datenschutzkonzept_CCP-IT_inkl_Anlagen.pdf). +This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP Data Protection Concept](https://dktk.dkfz.de/klinische-plattformen/documents-download). ## Getting Started -You must add following configuration variables to your sites-configuration repository: +The following configuration variables are added to your sites-configuration repository: ``` IDMANAGER_UPLOAD_APIKEY="" @@ -13,15 +13,19 @@ IDMANAGER_AUTH_CLIENT_ID="" IDMANAGER_AUTH_CLIENT_SECRET="" ``` -Additionally, the ccp-it needs to add a new file "patientlist-id-generators.env" to your site configuration. This file will hold the seeds for the different id-generators used in all projects. +Most of the configuration is kept identical across all sites. Some site-specific variables, however, go into a file called `/etc/bridgehead/ccp/id-management.local`. -After adding the configuration, you simply need to update your bridgehead and 3 new services will run on your server: +After adding the configuration, you simply need to update your bridgehead. You're all set! -- `bridgehead-id-manager`, accessible by "https:///id-manager". This component adds a common interface for creating pseudonymes in the bridgehead. -- `bridgehead-patientlist`, accessible by "https:// Date: Thu, 19 Jan 2023 10:15:12 +0000 Subject: [PATCH 16/34] Made it even easier for sites --- ccp/modules/id-management.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 4c929b9..60cb5b1 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -11,11 +11,13 @@ IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" IDMANAGER_AUTH_CLIENT_ID="" IDMANAGER_AUTH_CLIENT_SECRET="" + +IDMANAGER_SEEDS_BK="" +IDMANAGER_SEEDS_MDS="" +IDMANAGER_SEEDS_DKTK000001985="" ``` -Most of the configuration is kept identical across all sites. Some site-specific variables, however, go into a file called `/etc/bridgehead/ccp/id-management.local`. - -After adding the configuration, you simply need to update your bridgehead. You're all set! +Once your Bridgehead is updated and restarted, you're all set! ## Additional information you may want to know ### Services From 1c3fa4fa3fb6c21f886f2c9fdb4fe19b73a6545c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 17 Jan 2023 08:56:02 +0100 Subject: [PATCH 17/34] refactor: Changed Paths for persistent Data --- README.md | 4 ++-- ccp/modules/id-management-compose.yml | 4 ++-- ccp/modules/id-management.md | 6 +++--- ccp/mtba-compose.yml | 2 +- lib/install-bridgehead.sh | 15 ++++++++++----- lib/update-bridgehead.sh | 10 +--------- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 24ebdf2..134e1ae 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. -- `/var/data/bridgehead` contains persistent data of the bridgehead - - `/var/data/bridgehead/backups` contains automatically created backups of the databases. + - `/var/lib/bridgehead/data` contains persistent data of the bridgehead + - `/var/cache/bridgehead/backup` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 4baaba7..896663b 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -71,6 +71,6 @@ services: POSTGRES_DB: "mainzelliste" POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} volumes: - - "/var/data/bridgehead/patientlist:/var/lib/postgresql/data" - # NOTE: Add backups here. This is only imported if /var/data/bridgehead/patientlist/ is empty!!! + - "/var/lib/bridgehead/data/patientlist:/var/lib/postgresql/data" + # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!! - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/" diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 60cb5b1..c45b95a 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -26,7 +26,7 @@ Upon configuration, the Bridgehead will spawn the following services: - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. -- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/data/bridgehead/patientlist` and backups are automatically created in `/var/data/bridgehead/backups/bridgehead-patientlist-db`. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/lib/bridgehead/data/patientlist` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. ### How to import an existing database (e.g from Legacy Windows or from Backups) First you must shutdown your local bridgehead instance: @@ -36,10 +36,10 @@ systemctl stop bridgehead@ccp Next you need to remove the current patientlist database: ``` -rm -rf /var/data/bridgehead/patientlist +rm -rf /var/lib/bridgehead/data/patientlist ``` -Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/data/bridgehead/patientlist is empty. +Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/lib/bridgehead/data/patientlist is empty. > NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). After this, you can restart your bridgehead and the dump will be imported: diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index d492ae0..1c62989 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -24,7 +24,7 @@ services: - "traefik.http.routers.mtba.tls=true" volumes: # This directory persists the FHIR Resources that are needed to import data into blaze. - - /var/data/bridgehead/mtba:/app/mtba-files/persist + - /var/lib/bridgehead/data/mtba:/app/mtba-files/persist # Place new import files in this directory - /tmp/bridgehead/mtba/:/app/mtba-files/input diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 7cbd8ef..04503e3 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,9 +37,14 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi -log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data." -mkdir -p /var/data/bridgehead -chown -R bridgehead /var/data/bridgehead +log "INFO" "Creating directory /var/lib/bridgehead for storage of persistent data." +mkdir -p /var/lib/bridgehead +chown -R bridgehead /var/lib/bridgehead + + +log "INFO" "Creating directory /var/cache/bridgehead for storage of backups." +mkdir -p /var/cache/bridgehead +chown -R bridgehead /var/cache/bridgehead log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index cc1d55f..65560ea 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -4,15 +4,7 @@ source lib/functions.sh AUTO_BACKUP=${AUTO_BACKUP:-true} if [ "$AUTO_BACKUP" == "true" ]; then - BACKUP_DIRECTORY="/var/data/bridgehead/backups" - if [ ! -d /var/data ]; then - log DEBUG "Created /var/data" - mkdir /var/data - fi - if [ ! -d /var/data/bridgehead ]; then - log DEBUG "Created /var/data/bridgehead" - mkdir /var/data/bridgehead - fi + BACKUP_DIRECTORY="/var/cache/bridgehead/backup" if [ ! -d $BACKUP_DIRECTORY ]; then message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." hc_send log "$message" From 0d998ab8716a2e2c5c92fa84e65ccc608524ccef Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 10:55:44 +0100 Subject: [PATCH 18/34] refactor: Move persistent data to named volumes --- README.md | 1 - ccp/modules/id-management-compose.yml | 5 ++++- ccp/modules/id-management.md | 6 +++--- ccp/mtba-compose.yml | 5 ++++- lib/install-bridgehead.sh | 9 ++------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 134e1ae..e1d3286 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,6 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. - - `/var/lib/bridgehead/data` contains persistent data of the bridgehead - `/var/cache/bridgehead/backup` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 896663b..72aebcc 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -71,6 +71,9 @@ services: POSTGRES_DB: "mainzelliste" POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} volumes: - - "/var/lib/bridgehead/data/patientlist:/var/lib/postgresql/data" + - "patientlist-db-data:/var/lib/postgresql/data" # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!! - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/" + +volumes: + patientlist-db-data: diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index c45b95a..e18d3f8 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -26,7 +26,7 @@ Upon configuration, the Bridgehead will spawn the following services: - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. -- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/lib/bridgehead/data/patientlist` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. ### How to import an existing database (e.g from Legacy Windows or from Backups) First you must shutdown your local bridgehead instance: @@ -36,10 +36,10 @@ systemctl stop bridgehead@ccp Next you need to remove the current patientlist database: ``` -rm -rf /var/lib/bridgehead/data/patientlist +docker volume rm patientlist-db-data; ``` -Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/lib/bridgehead/data/patientlist is empty. +Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then the volume `patientlist-db-data` was removed previously. > NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). After this, you can restart your bridgehead and the dump will be imported: diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 1c62989..857ff34 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -24,10 +24,13 @@ services: - "traefik.http.routers.mtba.tls=true" volumes: # This directory persists the FHIR Resources that are needed to import data into blaze. - - /var/lib/bridgehead/data/mtba:/app/mtba-files/persist + - mtba-data:/app/mtba-files/persist # Place new import files in this directory - /tmp/bridgehead/mtba/:/app/mtba-files/input # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! # TODO: Find a trigger to let mtba signal a restart for CBioPortal + +volumes: + mtba-data: diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 04503e3..f1aff73 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,11 +37,6 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi -log "INFO" "Creating directory /var/lib/bridgehead for storage of persistent data." -mkdir -p /var/lib/bridgehead -chown -R bridgehead /var/lib/bridgehead - - log "INFO" "Creating directory /var/cache/bridgehead for storage of backups." mkdir -p /var/cache/bridgehead chown -R bridgehead /var/cache/bridgehead From 1befa65f35033f9383aae88b7fdcf5fd0244e156 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 10:59:29 +0100 Subject: [PATCH 19/34] refactor: Changed Salt for patientlist db password --- ccp/modules/id-management-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index d2449c7..98c4217 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -6,7 +6,7 @@ function idManagementSetup() { OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" # Auto Generate local Passwords - PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" # Source the ID Generators Configuration From 713dc5f4e9564596fb2a47afaceeb2a28cd2ea1d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:15:09 +0100 Subject: [PATCH 20/34] refactor: Move Backups after the Update --- lib/update-bridgehead.sh | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 65560ea..7fb3688 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,37 +1,6 @@ #!/bin/bash source lib/functions.sh -AUTO_BACKUP=${AUTO_BACKUP:-true} - -if [ "$AUTO_BACKUP" == "true" ]; then - BACKUP_DIRECTORY="/var/cache/bridgehead/backup" - if [ ! -d $BACKUP_DIRECTORY ]; then - message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - mkdir -p $BACKUP_DIRECTORY - fi - BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" - log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; - for service in $BACKUP_SERVICES; do - if [ ! -d $BACKUP_DIRECTORY/$service ]; then - message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - mkdir -p $BACKUP_DIRECTORY/$service - fi - if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then - message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - else - fail_and_report 5 "Failed to create encrypted update for $service" - fi - done -else - log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" -fi - AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then @@ -134,6 +103,37 @@ else hc_send log "$RES" fi +AUTO_BACKUP=${AUTO_BACKUP:-true} + +if [ "$AUTO_BACKUP" == "true" ]; then + BACKUP_DIRECTORY="/var/cache/bridgehead/backup" + if [ ! -d $BACKUP_DIRECTORY ]; then + message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY + fi + BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d $BACKUP_DIRECTORY/$service ]; then + message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY/$service + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + exit 0 # TODO: Print last commit explicit From 17d48a3636c812d7d2ac1a07b6adab20c930d75c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:26:31 +0100 Subject: [PATCH 21/34] refactor: Expect User to select a Backup Directory --- README.md | 3 ++- ccp/modules/id-management.md | 2 +- lib/install-bridgehead.sh | 8 ++------ lib/update-bridgehead.sh | 19 ++++++++----------- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index e1d3286..8fba0c9 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,6 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. - - `/var/cache/bridgehead/backup` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. @@ -147,6 +146,8 @@ Some of the components in the bridgehead will store persistent data. For those c 2) Year-KW-XX, were XX represents the calendar week to allow re-import of at least one version per calendar week 3) Year-Month, to allow re-import of at least one version per month +To enable the Auto-Backup feature, please set the Variable `BACKUP_DIRECTORY` in your sites configuration. + ### Monitoring To keep all Bridgeheads up and working and detect any errors before a user does, a central monitoring diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index e18d3f8..98da3d1 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -26,7 +26,7 @@ Upon configuration, the Bridgehead will spawn the following services: - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. -- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data`. ### How to import an existing database (e.g from Legacy Windows or from Backups) First you must shutdown your local bridgehead instance: diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index f1aff73..c42119f 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,10 +37,6 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi -log "INFO" "Creating directory /var/cache/bridgehead for storage of backups." -mkdir -p /var/cache/bridgehead -chown -R bridgehead /var/cache/bridgehead - log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ lib/systemd/bridgehead\@.service \ diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 7fb3688..276f60c 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -103,27 +103,24 @@ else hc_send log "$RES" fi -AUTO_BACKUP=${AUTO_BACKUP:-true} - -if [ "$AUTO_BACKUP" == "true" ]; then - BACKUP_DIRECTORY="/var/cache/bridgehead/backup" - if [ ! -d $BACKUP_DIRECTORY ]; then - message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." +if [ -z "${BACKUP_DIRECTORY}" ]; then + if [ ! -d "$BACKUP_DIRECTORY" ]; then + message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY." hc_send log "$message" log INFO "$message" - mkdir -p $BACKUP_DIRECTORY + mkdir -p "$BACKUP_DIRECTORY" fi BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; for service in $BACKUP_SERVICES; do - if [ ! -d $BACKUP_DIRECTORY/$service ]; then - message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then + message="Performing automatic maintenance: Attempting to create backup directory for $service in $BACKUP_DIRECTORY." hc_send log "$message" log INFO "$message" - mkdir -p $BACKUP_DIRECTORY/$service + mkdir -p "$BACKUP_DIRECTORY/$service" fi if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then - message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + message="Performing automatic maintenance: Stored encrypted backup for $service in $BACKUP_DIRECTORY." hc_send log "$message" log INFO "$message" else From a37bf79c030908e80284c981975161bb31b989cc Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:31:36 +0100 Subject: [PATCH 22/34] refactor: Added Monitoring for backup permissions --- lib/update-bridgehead.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 276f60c..0f1c413 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -109,7 +109,9 @@ if [ -z "${BACKUP_DIRECTORY}" ]; then hc_send log "$message" log INFO "$message" mkdir -p "$BACKUP_DIRECTORY" + chown -R "$BACKUP_DIRECTORY" bridgehead; fi + checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)" BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; for service in $BACKUP_SERVICES; do From 98afeac701f0065d0052f07a8c9b27ce9e2ff779 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:49:18 +0100 Subject: [PATCH 23/34] refactor: Use -db as Indicator for Backups --- lib/update-bridgehead.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 0f1c413..ab39cca 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -112,7 +112,8 @@ if [ -z "${BACKUP_DIRECTORY}" ]; then chown -R "$BACKUP_DIRECTORY" bridgehead; fi checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)" - BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + # Collect all container names that contain '-db' + BACKUP_SERVICES="$(docker ps --filter name=-db --format "{{.Names}}" | tr "\n" "\ ")" log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; for service in $BACKUP_SERVICES; do if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then From 2445a5978506bd7d3b84d85fafe7e8a16d6f239d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:52:09 +0100 Subject: [PATCH 24/34] refacotr: Restructured passing Patientlist Seeds --- ccp/modules/id-management-compose.yml | 25 +------------------------ ccp/modules/id-management-setup.sh | 4 ++-- ccp/modules/id-management.md | 2 ++ 3 files changed, 5 insertions(+), 26 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 72aebcc..3f13519 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -31,30 +31,7 @@ services: - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} # Add Variables from /etc/patientlist-id-generators.env - - ML_BK_IDGENERATOR_RANDOM_1 - - ML_BK_IDGENERATOR_RANDOM_2 - - ML_BK_IDGENERATOR_RANDOM_3 - - ML_MDS_IDGENERATOR_RANDOM_1 - - ML_MDS_IDGENERATOR_RANDOM_2 - - ML_MDS_IDGENERATOR_RANDOM_3 - - ML_DKTK000001985_IDGENERATOR_RANDOM_1 - - ML_DKTK000001985_IDGENERATOR_RANDOM_2 - - ML_DKTK000001985_IDGENERATOR_RANDOM_3 - - ML_DKTK000001986_IDGENERATOR_RANDOM_1 - - ML_DKTK000001986_IDGENERATOR_RANDOM_2 - - ML_DKTK000001986_IDGENERATOR_RANDOM_3 - - ML_DKTK000001950_IDGENERATOR_RANDOM_1 - - ML_DKTK000001950_IDGENERATOR_RANDOM_2 - - ML_DKTK000001950_IDGENERATOR_RANDOM_3 - - ML_DKTK000001951_IDGENERATOR_RANDOM_1 - - ML_DKTK000001951_IDGENERATOR_RANDOM_2 - - ML_DKTK000001951_IDGENERATOR_RANDOM_3 - - ML_DKTK999999999_IDGENERATOR_RANDOM_1 - - ML_DKTK999999999_IDGENERATOR_RANDOM_2 - - ML_DKTK999999999_IDGENERATOR_RANDOM_3 - - ML_DKTK000002089_IDGENERATOR_RANDOM_1 - - ML_DKTK000002089_IDGENERATOR_RANDOM_2 - - ML_DKTK000002089_IDGENERATOR_RANDOM_3 + - PATIENTLIST_SEEDS_TRANSFORMED labels: - "traefik.enable=true" - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 98c4217..0f361dc 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -9,8 +9,8 @@ function idManagementSetup() { PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" - # Source the ID Generators Configuration - source /etc/bridgehead/patientlist-id-generators.env + # Transform Seeds Configuration to pass it to the Mainzelliste Container + PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')" # Ensure old ids are working !!! legacyIdMapping diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 98da3d1..66f9f71 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -16,10 +16,12 @@ IDMANAGER_SEEDS_BK="" IDMANAGER_SEEDS_MDS="" IDMANAGER_SEEDS_DKTK000001985="" ``` +> NOTE: Additionally, the CCP-IT adds lines declaring the `PATIENTLIST_SEEDS` array in your site configuration. This will contain the seeds for the different id-generators used in all projects. Once your Bridgehead is updated and restarted, you're all set! ## Additional information you may want to know + ### Services Upon configuration, the Bridgehead will spawn the following services: From a3ba98a2fdb1d950ab4fbcb2d1a9302692086816 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 7 Dec 2022 15:46:19 +0100 Subject: [PATCH 25/34] Added Configuration for Local ID-Management --- ccp/modules/id-management-setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 0f361dc..b02ef66 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,8 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then +# if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + if [ -n "$ENABLE_ID_MANAGEMENT" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From e2d90447f7e3ad6ee981c16de6d02b048cfdebf2 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:03 +0100 Subject: [PATCH 26/34] refactor: Changed Trigger of the IDM Module --- ccp/modules/id-management-setup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index b02ef66..c8ea707 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,6 @@ #!/bin/bash function idManagementSetup() { -# if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then if [ -n "$ENABLE_ID_MANAGEMENT" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From bce28342f90482205a5eba1d284fb5dffb6e6cce Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 18 Jan 2023 13:42:36 +0100 Subject: [PATCH 27/34] update docker and host paths --- ccp/mtba-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 857ff34..ad25dd0 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -17,16 +17,16 @@ services: PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" + NEW_FILES_DIRECTORY: "/app/input" + PERSIST_DIRECTORY: "/app/persist" labels: - "traefik.enable=true" - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" - "traefik.http.services.mtba.loadbalancer.server.port=80" - "traefik.http.routers.mtba.tls=true" volumes: - # This directory persists the FHIR Resources that are needed to import data into blaze. - - mtba-data:/app/mtba-files/persist - # Place new import files in this directory - - /tmp/bridgehead/mtba/:/app/mtba-files/input + - /tmp/bridgehead/mtba/input:/app/input + - /tmp/bridgehead/mtba/persist:/app/persist # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! From e9455a5558dd33de59bb46e3c048368f4c30df34 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 27 Jan 2023 11:13:49 +0100 Subject: [PATCH 28/34] update mtba config --- ccp/modules/id-management-setup.sh | 2 +- ccp/mtba-compose.yml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index c8ea707..0f361dc 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index ad25dd0..bb90059 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -5,20 +5,20 @@ services: image: samply/mtba:develop container_name: bridgehead-mtba environment: - BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir + BLAZE_STORE_URL: http://blaze:8080 # NOTE: Aktuell Berechtigungen wie MagicPL!!! # TODO: Add separate ApiKey to MagicPL only for MTBA! ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID - ID_MANAGER_URL: http://bridgehead-id-manager:8080 - PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} - PATIENT_CSV_LAST_NAME_HEADER: ${PATIENT_CSV_LAST_NAME_HEADER:-"LAST_NAME"} - PATIENT_CSV_GENDER_HEADER: ${PATIENT_CSV_GENDER_HEADER:-"GENDER"} - PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} - CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 - MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" - NEW_FILES_DIRECTORY: "/app/input" - PERSIST_DIRECTORY: "/app/persist" + ID_MANAGER_URL: http://id-manager:8080/id-manager + PATIENT_CSV_FIRST_NAME_HEADER: ${MTBA_PATIENT_CSV_FIRST_NAME_HEADER} + PATIENT_CSV_LAST_NAME_HEADER: ${MTBA_PATIENT_CSV_LAST_NAME_HEADER} + PATIENT_CSV_GENDER_HEADER: ${MTBA_PATIENT_CSV_GENDER_HEADER} + PATIENT_CSV_BIRTHDAY_HEADER: ${MTBA_PATIENT_CSV_BIRTHDAY_HEADER} + CBIOPORTAL_URL: http://cbioportal:8080 + FILE_CHARSET: ${MTBA_FILE_CHARSET} + FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} + CSV_DELIMITER: ${MTBA_CSV_DELIMITER} labels: - "traefik.enable=true" - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" From c88919c926e1b07b0d5bfe8a21d32dfaee3c267e Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 8 Feb 2023 14:42:52 +0100 Subject: [PATCH 29/34] feature: Ensured the mapping to legacy ids works --- ccp/modules/id-management-setup.sh | 49 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 0f361dc..ba8ad45 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -13,25 +13,40 @@ function idManagementSetup() { PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')" # Ensure old ids are working !!! - legacyIdMapping + export IDMANAGEMENT_FRIENDLY_ID=$(legacyIdMapping "$SITE_ID") fi - } -# TODO: Map all old site ids to the new ones +# Transform into single string array, e.g. 'dktk-test' to 'dktk test' +# Usage: transformToSingleStringArray 'dktk-test' -> 'dktk test' +function transformToSingleStringArray() { + echo "${1//-/ }"; +} + +# Ensure all Words are Uppercase +# Usage: transformToUppercase 'dktk test' -> 'Dktk Test' +function transformToUppercase() { + result=""; + for word in $1; do + result+=" ${word^}"; + done + echo "$result"; +} + +# Handle all execeptions from the norm (e.g LMU, TUM) +# Usage: applySpecialCases 'Muenchen Lmu Test' -> 'Muenchen LMU Test' +function applySpecialCases() { + result="$1"; + result="${result/Lmu/LMU}"; + result="${result/Tum/TUM}"; + echo "$result"; +} + +# Transform current siteids to legacy version +# Usage: legacyIdMapping "dktk-test" -> "DktkTest" function legacyIdMapping() { - case ${SITE_ID} in - "berlin") - export IDMANAGEMENT_FRIENDLY_ID=Berlin - ;; - "dresden") - export IDMANAGEMENT_FRIENDLY_ID=Dresden - ;; - "frankfurt") - export IDMANAGEMENT_FRIENDLY_ID=Frankfurt - ;; - *) - export IDMANAGEMENT_FRIENDLY_ID=$SITE_ID - ;; - esac + single_string_array=$(transformToSingleStringArray "$1"); + uppercase_string=$(transformToUppercase "$single_string_array"); + normalized_string=$(applySpecialCases "$uppercase_string"); + echo "$normalized_string" | tr -d ' ' } From d728ccd88657358374f107ee0a03b06442a5c4e1 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 8 Feb 2023 15:02:35 +0100 Subject: [PATCH 30/34] refactor: Fixated Postgres Version to 15.1 All Minor Updates will cause a crashing postgres on start --- ccp/modules/id-management-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 3f13519..2f26ce4 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -41,7 +41,7 @@ services: - patientlist-db patientlist-db: - image: postgres:14-alpine + image: postgres:15.1-alpine container_name: bridgehead-patientlist-db environment: POSTGRES_USER: "mainzelliste" From 64095ac8dd3b0b0816ff6aefc77fe5d56a8efb16 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 09:15:29 +0100 Subject: [PATCH 31/34] feat: Added Upload ApiKey to Patientlist --- ccp/modules/id-management-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 2f26ce4..8e5bab8 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -30,6 +30,7 @@ services: - ML_SITE=${SITE_ID} - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + - ML_UPLOAD_API_KEY=${IDMANAGER_UPLOAD_APIKEY} # Add Variables from /etc/patientlist-id-generators.env - PATIENTLIST_SEEDS_TRANSFORMED labels: From a1cd5a206e402922a22f037c4bedaa1488642150 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 09:50:30 +0100 Subject: [PATCH 32/34] fix: Repaired check for Auto Updates --- lib/update-bridgehead.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index ab39cca..c661595 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -103,7 +103,7 @@ else hc_send log "$RES" fi -if [ -z "${BACKUP_DIRECTORY}" ]; then +if [ -n "${BACKUP_DIRECTORY}" ]; then if [ ! -d "$BACKUP_DIRECTORY" ]; then message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY." hc_send log "$message" From a6975e37ac8433d615b8a8ee3d36c219e047c194 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 09:58:51 +0100 Subject: [PATCH 33/34] fix: Added Image Updates for activated Modules --- lib/update-bridgehead.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index c661595..c53b731 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -81,7 +81,7 @@ done # Check docker updates log "INFO" "Checking for updates to running docker images ..." docker_updated="false" -for IMAGE in $(cat $PROJECT/docker-compose.yml | grep -v "^#" | grep "image:" | sed -e 's_^.*image: \(.*\).*$_\1_g; s_\"__g'); do +for IMAGE in $(cat $PROJECT/docker-compose.yml ${OVERRIDE//-f/} | grep -v "^#" | grep "image:" | sed -e 's_^.*image: \(.*\).*$_\1_g; s_\"__g'); do log "INFO" "Checking for Updates of Image: $IMAGE" if docker pull $IMAGE | grep "Downloaded newer image"; then CHANGE="Image $IMAGE updated." From 5a7dd1d0aedeba61c6d8c2b954f0bc0cc8ef5114 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 10:07:05 +0100 Subject: [PATCH 34/34] fix: Use Mapped Site Id in Patientlist --- ccp/modules/id-management-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 8e5bab8..cb0c89a 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -27,7 +27,7 @@ services: container_name: bridgehead-patientlist environment: - TOMCAT_REVERSEPROXY_FQDN=${HOST} - - ML_SITE=${SITE_ID} + - ML_SITE=${IDMANAGEMENT_FRIENDLY_ID} - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - ML_UPLOAD_API_KEY=${IDMANAGER_UPLOAD_APIKEY}