From 99c0e7f28354bcb6ae49f30556df78d1a9e1da07 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 7 Dec 2022 15:46:19 +0100 Subject: [PATCH 01/52] Added Configuration for Local ID-Management --- ccp/modules/id-management-compose.yml | 75 +++++++++++++++++++++++++++ ccp/modules/id-management-setup.sh | 17 ++++++ ccp/vars | 3 ++ lib/functions.sh | 2 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 ccp/modules/id-management-compose.yml create mode 100644 ccp/modules/id-management-setup.sh diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml new file mode 100644 index 0000000..0968048 --- /dev/null +++ b/ccp/modules/id-management-compose.yml @@ -0,0 +1,75 @@ +version: "3.7" +services: + id-manager: + image: docker.verbis.dkfz.de/bridgehead/magicpl + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST} + MAGICPL_SITE: ${SITE_ID} + MAGICPL_ALLOWED_ORIGINS: https://${HOST} + MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} + MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} + MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET} + depends_on: + - patientlist + labels: + - "traefik.enable=true" + - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)" + - "traefik.http.services.id-manager.loadbalancer.server.port=8080" + - "traefik.http.routers.id-manager.tls=true" + + patientlist: + image: docker.verbis.dkfz.de/bridgehead/mainzelliste + environment: + - TOMCAT_REVERSEPROXY_FQDN=${HOST} + - ML_SITE=${SITE_ID} + - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} + - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # Add Variables from /etc/patientlist-id-generators.env + - ML_BK_IDGENERATOR_RANDOM_1 + - ML_BK_IDGENERATOR_RANDOM_2 + - ML_BK_IDGENERATOR_RANDOM_3 + - ML_MDS_IDGENERATOR_RANDOM_1 + - ML_MDS_IDGENERATOR_RANDOM_2 + - ML_MDS_IDGENERATOR_RANDOM_3 + - ML_DKTK000001985_IDGENERATOR_RANDOM_1 + - ML_DKTK000001985_IDGENERATOR_RANDOM_2 + - ML_DKTK000001985_IDGENERATOR_RANDOM_3 + - ML_DKTK000001986_IDGENERATOR_RANDOM_1 + - ML_DKTK000001986_IDGENERATOR_RANDOM_2 + - ML_DKTK000001986_IDGENERATOR_RANDOM_3 + - ML_DKTK000001950_IDGENERATOR_RANDOM_1 + - ML_DKTK000001950_IDGENERATOR_RANDOM_2 + - ML_DKTK000001950_IDGENERATOR_RANDOM_3 + - ML_DKTK000001951_IDGENERATOR_RANDOM_1 + - ML_DKTK000001951_IDGENERATOR_RANDOM_2 + - ML_DKTK000001951_IDGENERATOR_RANDOM_3 + - ML_DKTK999999999_IDGENERATOR_RANDOM_1 + - ML_DKTK999999999_IDGENERATOR_RANDOM_2 + - ML_DKTK999999999_IDGENERATOR_RANDOM_3 + - ML_DKTK000002089_IDGENERATOR_RANDOM_1 + - ML_DKTK000002089_IDGENERATOR_RANDOM_2 + - ML_DKTK000002089_IDGENERATOR_RANDOM_3 + labels: + - "traefik.enable=true" + - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" + - "traefik.http.services.patientlist.loadbalancer.server.port=8080" + - "traefik.http.routers.patientlist.tls=true" + depends_on: + - patientlist-db + + patientlist-db: + image: postgres:14-alpine + environment: + POSTGRES_USER: "mainzelliste" + POSTGRES_DB: "mainzelliste" + POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} + volumes: + - "patientlist-db-data:/var/lib/postgresql/data" + +volumes: + patientlist-db-data: + diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh new file mode 100644 index 0000000..ca939fb --- /dev/null +++ b/ccp/modules/id-management-setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +function idManagementSetup() { + if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." + OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" + + # Auto Generate local Passwords + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + + # Source the ID Generators Configuration + source /etc/bridgehead/patientlist-id-generators.env + log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + fi + +} diff --git a/ccp/vars b/ccp/vars index 63def80..89deae0 100644 --- a/ccp/vars +++ b/ccp/vars @@ -8,6 +8,9 @@ REPORTHUB_BEAM_SECRET_LONG="ApiKey report-hub.${PROXY_ID} ${REPORTHUB_BEAM_SECRE SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +# This will load id-management setup. Effective only if id-management configuration is defined. +source $PROJECT/modules/id-management-setup.sh +idManagementSetup # This will load nngm setup. Effective only if nngm configuration is defined. source $PROJECT/nngm-setup.sh nngmSetup diff --git a/lib/functions.sh b/lib/functions.sh index 4f40fd0..355ebaa 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -131,7 +131,7 @@ fail_and_report() { setHostname() { if [ -z "$HOST" ]; then - export HOST=$(hostname -f) + export HOST=$(hostname -f | tr "[:upper:]" "[:lower:]") log DEBUG "Using auto-detected hostname $HOST." fi } From 5e063003ace490b3835e30fcebbafd6465ed4586 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 13 Dec 2022 16:51:32 +0100 Subject: [PATCH 02/52] feature: Added automated Backups for PostgreSQL --- README.md | 9 +++++++ ccp/modules/id-management-compose.yml | 11 ++++---- lib/functions.sh | 11 ++++++++ lib/install-bridgehead.sh | 10 ++++--- lib/update-bridgehead.sh | 39 +++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b57c10e..24ebdf2 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. +- `/var/data/bridgehead` contains persistent data of the bridgehead + - `/var/data/bridgehead/backups` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. @@ -139,6 +141,13 @@ Your Bridgehead will automatically and regularly check for updates. Whenever som If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@.service` and `systemctl cat bridgehead-update@ \ + "$1/$2/$(date +%Y-KW%V).sql" +} + + # from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 # ex. use: retry 5 /bin/false function retry { diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 5e3add3..7cbd8ef 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,6 +37,10 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi +log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data." +mkdir -p /var/data/bridgehead +chown -R bridgehead /var/data/bridgehead + log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ lib/systemd/bridgehead\@.service \ @@ -63,4 +67,4 @@ else STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR" fi -log "INFO" "$STR" \ No newline at end of file +log "INFO" "$STR" diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 897c8a2..cc1d55f 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,6 +1,45 @@ #!/bin/bash source lib/functions.sh +AUTO_BACKUP=${AUTO_BACKUP:-true} + +if [ "$AUTO_BACKUP" == "true" ]; then + BACKUP_DIRECTORY="/var/data/bridgehead/backups" + if [ ! -d /var/data ]; then + log DEBUG "Created /var/data" + mkdir /var/data + fi + if [ ! -d /var/data/bridgehead ]; then + log DEBUG "Created /var/data/bridgehead" + mkdir /var/data/bridgehead + fi + if [ ! -d $BACKUP_DIRECTORY ]; then + message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY + fi + BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d $BACKUP_DIRECTORY/$service ]; then + message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY/$service + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then From 125bb5e26232be6f5279d1fb771a99710d8fe012 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:03 +0100 Subject: [PATCH 03/52] refactor: Changed Trigger of the IDM Module --- ccp/modules/id-management-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index ca939fb..790f846 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From bc72093be9c42db534d12a4d418106cf7e21409d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:23 +0100 Subject: [PATCH 04/52] docs: Added Documentation for the IDM Module --- ccp/modules/id-management.md | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ccp/modules/id-management.md diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md new file mode 100644 index 0000000..8de2f5a --- /dev/null +++ b/ccp/modules/id-management.md @@ -0,0 +1,58 @@ +# Module: Id-Management +This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP-DSK](https://dktk.dkfz.de/application/files/5016/2030/2474/20_11_23_Datenschutzkonzept_CCP-IT_inkl_Anlagen.pdf). + +## Getting Started +You must add following configuration variables to your sites-configuration repository: + +``` +IDMANAGER_CENTRAXX_APIKEY="" +IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" +IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" +IDMANAGER_AUTH_CLIENT_ID="" +IDMANAGER_AUTH_CLIENT_SECRET="" +``` + +Additionally, the ccp-it needs to add a new file "patientlist-id-generators.env" to your site configuration. This file will hold the seeds for the different id-generators used in all projects. + +After adding the configuration, you simply need to update your bridgehead and 3 new services will run on your server: + +- `bridgehead-id-manager`, accessible by "https:///id-manager". This component adds a common interface for creating pseudonymes in the bridgehead. +- `bridgehead-patientlist`, accessible by "https:// NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). + +After this, you can restart your bridgehead and the dump will be imported: +``` +systemctl start bridgehead@ccp +``` + +### How to connect your local data-management +Typically, the sites connect their local data-management for the pseudonym creation with the id-management in the bridgehead. In the following two sections, you can read where you can change the configuration: +#### Sites using CentraXX +On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. +``` +dktk.idmanagement.url=https:///id-manager/translator/getId +dktk.idmanagement.apiKey= +``` +They typically already exist, but need to be changed to the new values! +#### Sites using ADT2FHIR +@Pierre + + +### How to connect the legacy windows bridgehead +You need to change the configuration file "..." of your Windows Bridgehead. TODO... From 276be28db1792c4f276191e24bbb3b7cbd8f852c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 16 Dec 2022 12:02:49 +0100 Subject: [PATCH 05/52] fix: Included Legacy ID-Mapping (Please Complete) --- ccp/modules/id-management-compose.yml | 6 +++--- ccp/modules/id-management-setup.sh | 24 ++++++++++++++++++++++-- ccp/modules/id-management.md | 6 +++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index d1639af..4baaba7 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -5,11 +5,11 @@ services: container_name: bridgehead-id-manager environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST} - MAGICPL_SITE: ${SITE_ID} + MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID} MAGICPL_ALLOWED_ORIGINS: https://${HOST} MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} - MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_UPLOAD_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY} MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 790f846..d2449c7 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" @@ -11,7 +11,27 @@ function idManagementSetup() { # Source the ID Generators Configuration source /etc/bridgehead/patientlist-id-generators.env - log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + + # Ensure old ids are working !!! + legacyIdMapping fi } + +# TODO: Map all old site ids to the new ones +function legacyIdMapping() { + case ${SITE_ID} in + "berlin") + export IDMANAGEMENT_FRIENDLY_ID=Berlin + ;; + "dresden") + export IDMANAGEMENT_FRIENDLY_ID=Dresden + ;; + "frankfurt") + export IDMANAGEMENT_FRIENDLY_ID=Frankfurt + ;; + *) + export IDMANAGEMENT_FRIENDLY_ID=$SITE_ID + ;; + esac +} diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 8de2f5a..89ff65c 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -5,8 +5,8 @@ This module provides integration with the CCP-Pseudonymiziation Service. To lear You must add following configuration variables to your sites-configuration repository: ``` -IDMANAGER_CENTRAXX_APIKEY="" -IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_UPLOAD_APIKEY="" +IDMANAGER_READ_APIKEY="" IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" IDMANAGER_AUTH_CLIENT_ID="" @@ -47,7 +47,7 @@ Typically, the sites connect their local data-management for the pseudonym creat On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. ``` dktk.idmanagement.url=https:///id-manager/translator/getId -dktk.idmanagement.apiKey= +dktk.idmanagement.apiKey= ``` They typically already exist, but need to be changed to the new values! #### Sites using ADT2FHIR From 828312e04510315dd9ee87c3d5b9f53a4099852d Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 2 Dec 2022 15:38:33 +0100 Subject: [PATCH 06/52] Added MTBA Module --- ccp/mtba-compose.yml | 33 +++++++++++++++++++++++++++++++++ ccp/nngm-setup.sh | 12 ++++++++++++ ccp/vars | 1 + 3 files changed, 46 insertions(+) create mode 100644 ccp/mtba-compose.yml diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml new file mode 100644 index 0000000..53fd327 --- /dev/null +++ b/ccp/mtba-compose.yml @@ -0,0 +1,33 @@ +version: "3.7" + +services: + mtba: + image: samply/mtba:develop + container_name: bridgehead-mtba + environment: + BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir + # NOTE: Aktuell Berechtigungen wie MagicPL!!! + # TODO: Add separate ApiKey to Patientlist only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID + ID_MANAGER_URL: http://bridgehead-id-manager:8080 + PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} + PATIENT_CSV_LAST_NAME_HEADER: ${PATIENT_CSV_LAST_NAME_HEADER:-"LAST_NAME"} + PATIENT_CSV_GENDER_HEADER: ${PATIENT_CSV_GENDER_HEADER:-"GENDER"} + PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} + CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 + MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" + labels: + - "traefik.enable=true" + - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" + - "traefik.http.services.mtba.loadbalancer.server.port=80" + - "traefik.http.routers.mtba.tls=true" + volumes: + # This directory persists the FHIR Resources that are needed to import data into blaze. + - /var/data/bridgehead/mtba:/app/mtba-files/persist + # Place new import files in this directory + - /tmp/bridgehead/mtba/:/app/mtba-files/input + + # TODO: Include CBioPortal in Deployment ... + # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! + # TODO: Find a trigger to let mtba signal a restart for CBioPortal diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index d5b80eb..b55d48e 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -7,3 +7,15 @@ function nngmSetup() { fi CONNECTOR_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" } + +function mtbaSetup() { + # TODO: Check if ID-Management Module is activated! + if [ -n "$ENABLE_MTBA" ];then + log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" + exit 1; + fi + OVERRIDE+=" -f ./$PROJECT/mtba-compose.yml" + fi +} diff --git a/ccp/vars b/ccp/vars index 89deae0..5172d24 100644 --- a/ccp/vars +++ b/ccp/vars @@ -16,3 +16,4 @@ source $PROJECT/nngm-setup.sh nngmSetup source $PROJECT/exliquid-setup.sh exliquidSetup +mtbaSetup \ No newline at end of file From e1e7ffece8899a57ce6c22b1e5f55ebdf083225c Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 16 Dec 2022 15:01:04 +0100 Subject: [PATCH 07/52] set magicpl instead of patientlist --- ccp/mtba-compose.yml | 4 ++-- ccp/nngm-setup.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 53fd327..d492ae0 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -7,8 +7,8 @@ services: environment: BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir # NOTE: Aktuell Berechtigungen wie MagicPL!!! - # TODO: Add separate ApiKey to Patientlist only for MTBA! - ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # TODO: Add separate ApiKey to MagicPL only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID ID_MANAGER_URL: http://bridgehead-id-manager:8080 PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index b55d48e..ba2e77f 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -12,7 +12,7 @@ function mtbaSetup() { # TODO: Check if ID-Management Module is activated! if [ -n "$ENABLE_MTBA" ];then log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." - if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" exit 1; fi From 6d24dbce7f9710dc4050ec62fc6016273bd38cd4 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 7 Dec 2022 15:46:19 +0100 Subject: [PATCH 08/52] Added Configuration for Local ID-Management --- ccp/modules/id-management-compose.yml | 75 +++++++++++++++++++++++++++ ccp/modules/id-management-setup.sh | 17 ++++++ ccp/vars | 3 ++ lib/functions.sh | 2 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 ccp/modules/id-management-compose.yml create mode 100644 ccp/modules/id-management-setup.sh diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml new file mode 100644 index 0000000..0968048 --- /dev/null +++ b/ccp/modules/id-management-compose.yml @@ -0,0 +1,75 @@ +version: "3.7" +services: + id-manager: + image: docker.verbis.dkfz.de/bridgehead/magicpl + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST} + MAGICPL_SITE: ${SITE_ID} + MAGICPL_ALLOWED_ORIGINS: https://${HOST} + MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} + MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} + MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET} + depends_on: + - patientlist + labels: + - "traefik.enable=true" + - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)" + - "traefik.http.services.id-manager.loadbalancer.server.port=8080" + - "traefik.http.routers.id-manager.tls=true" + + patientlist: + image: docker.verbis.dkfz.de/bridgehead/mainzelliste + environment: + - TOMCAT_REVERSEPROXY_FQDN=${HOST} + - ML_SITE=${SITE_ID} + - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} + - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # Add Variables from /etc/patientlist-id-generators.env + - ML_BK_IDGENERATOR_RANDOM_1 + - ML_BK_IDGENERATOR_RANDOM_2 + - ML_BK_IDGENERATOR_RANDOM_3 + - ML_MDS_IDGENERATOR_RANDOM_1 + - ML_MDS_IDGENERATOR_RANDOM_2 + - ML_MDS_IDGENERATOR_RANDOM_3 + - ML_DKTK000001985_IDGENERATOR_RANDOM_1 + - ML_DKTK000001985_IDGENERATOR_RANDOM_2 + - ML_DKTK000001985_IDGENERATOR_RANDOM_3 + - ML_DKTK000001986_IDGENERATOR_RANDOM_1 + - ML_DKTK000001986_IDGENERATOR_RANDOM_2 + - ML_DKTK000001986_IDGENERATOR_RANDOM_3 + - ML_DKTK000001950_IDGENERATOR_RANDOM_1 + - ML_DKTK000001950_IDGENERATOR_RANDOM_2 + - ML_DKTK000001950_IDGENERATOR_RANDOM_3 + - ML_DKTK000001951_IDGENERATOR_RANDOM_1 + - ML_DKTK000001951_IDGENERATOR_RANDOM_2 + - ML_DKTK000001951_IDGENERATOR_RANDOM_3 + - ML_DKTK999999999_IDGENERATOR_RANDOM_1 + - ML_DKTK999999999_IDGENERATOR_RANDOM_2 + - ML_DKTK999999999_IDGENERATOR_RANDOM_3 + - ML_DKTK000002089_IDGENERATOR_RANDOM_1 + - ML_DKTK000002089_IDGENERATOR_RANDOM_2 + - ML_DKTK000002089_IDGENERATOR_RANDOM_3 + labels: + - "traefik.enable=true" + - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" + - "traefik.http.services.patientlist.loadbalancer.server.port=8080" + - "traefik.http.routers.patientlist.tls=true" + depends_on: + - patientlist-db + + patientlist-db: + image: postgres:14-alpine + environment: + POSTGRES_USER: "mainzelliste" + POSTGRES_DB: "mainzelliste" + POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} + volumes: + - "patientlist-db-data:/var/lib/postgresql/data" + +volumes: + patientlist-db-data: + diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh new file mode 100644 index 0000000..ca939fb --- /dev/null +++ b/ccp/modules/id-management-setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +function idManagementSetup() { + if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." + OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" + + # Auto Generate local Passwords + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + + # Source the ID Generators Configuration + source /etc/bridgehead/patientlist-id-generators.env + log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + fi + +} diff --git a/ccp/vars b/ccp/vars index 63def80..89deae0 100644 --- a/ccp/vars +++ b/ccp/vars @@ -8,6 +8,9 @@ REPORTHUB_BEAM_SECRET_LONG="ApiKey report-hub.${PROXY_ID} ${REPORTHUB_BEAM_SECRE SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +# This will load id-management setup. Effective only if id-management configuration is defined. +source $PROJECT/modules/id-management-setup.sh +idManagementSetup # This will load nngm setup. Effective only if nngm configuration is defined. source $PROJECT/nngm-setup.sh nngmSetup diff --git a/lib/functions.sh b/lib/functions.sh index 4f40fd0..355ebaa 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -131,7 +131,7 @@ fail_and_report() { setHostname() { if [ -z "$HOST" ]; then - export HOST=$(hostname -f) + export HOST=$(hostname -f | tr "[:upper:]" "[:lower:]") log DEBUG "Using auto-detected hostname $HOST." fi } From 1ffc9b9cd581e592c42a8ac2c881045aa39c1c3c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 13 Dec 2022 16:51:32 +0100 Subject: [PATCH 09/52] feature: Added automated Backups for PostgreSQL --- README.md | 9 +++++++ ccp/modules/id-management-compose.yml | 11 ++++---- lib/functions.sh | 11 ++++++++ lib/install-bridgehead.sh | 10 ++++--- lib/update-bridgehead.sh | 39 +++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b57c10e..24ebdf2 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. +- `/var/data/bridgehead` contains persistent data of the bridgehead + - `/var/data/bridgehead/backups` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. @@ -139,6 +141,13 @@ Your Bridgehead will automatically and regularly check for updates. Whenever som If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@.service` and `systemctl cat bridgehead-update@ \ + "$1/$2/$(date +%Y-KW%V).sql" +} + + # from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 # ex. use: retry 5 /bin/false function retry { diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 5e3add3..7cbd8ef 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,6 +37,10 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi +log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data." +mkdir -p /var/data/bridgehead +chown -R bridgehead /var/data/bridgehead + log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ lib/systemd/bridgehead\@.service \ @@ -63,4 +67,4 @@ else STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR" fi -log "INFO" "$STR" \ No newline at end of file +log "INFO" "$STR" diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 897c8a2..cc1d55f 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,6 +1,45 @@ #!/bin/bash source lib/functions.sh +AUTO_BACKUP=${AUTO_BACKUP:-true} + +if [ "$AUTO_BACKUP" == "true" ]; then + BACKUP_DIRECTORY="/var/data/bridgehead/backups" + if [ ! -d /var/data ]; then + log DEBUG "Created /var/data" + mkdir /var/data + fi + if [ ! -d /var/data/bridgehead ]; then + log DEBUG "Created /var/data/bridgehead" + mkdir /var/data/bridgehead + fi + if [ ! -d $BACKUP_DIRECTORY ]; then + message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY + fi + BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d $BACKUP_DIRECTORY/$service ]; then + message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY/$service + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then From 2a46bd00fd31a5278ac23f1f434b21ca355d516d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:03 +0100 Subject: [PATCH 10/52] refactor: Changed Trigger of the IDM Module --- ccp/modules/id-management-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index ca939fb..790f846 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From 4f0526ddf4de73836c74f0607676f3f2fc583f40 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:23 +0100 Subject: [PATCH 11/52] docs: Added Documentation for the IDM Module --- ccp/modules/id-management.md | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ccp/modules/id-management.md diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md new file mode 100644 index 0000000..8de2f5a --- /dev/null +++ b/ccp/modules/id-management.md @@ -0,0 +1,58 @@ +# Module: Id-Management +This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP-DSK](https://dktk.dkfz.de/application/files/5016/2030/2474/20_11_23_Datenschutzkonzept_CCP-IT_inkl_Anlagen.pdf). + +## Getting Started +You must add following configuration variables to your sites-configuration repository: + +``` +IDMANAGER_CENTRAXX_APIKEY="" +IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" +IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" +IDMANAGER_AUTH_CLIENT_ID="" +IDMANAGER_AUTH_CLIENT_SECRET="" +``` + +Additionally, the ccp-it needs to add a new file "patientlist-id-generators.env" to your site configuration. This file will hold the seeds for the different id-generators used in all projects. + +After adding the configuration, you simply need to update your bridgehead and 3 new services will run on your server: + +- `bridgehead-id-manager`, accessible by "https:///id-manager". This component adds a common interface for creating pseudonymes in the bridgehead. +- `bridgehead-patientlist`, accessible by "https:// NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). + +After this, you can restart your bridgehead and the dump will be imported: +``` +systemctl start bridgehead@ccp +``` + +### How to connect your local data-management +Typically, the sites connect their local data-management for the pseudonym creation with the id-management in the bridgehead. In the following two sections, you can read where you can change the configuration: +#### Sites using CentraXX +On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. +``` +dktk.idmanagement.url=https:///id-manager/translator/getId +dktk.idmanagement.apiKey= +``` +They typically already exist, but need to be changed to the new values! +#### Sites using ADT2FHIR +@Pierre + + +### How to connect the legacy windows bridgehead +You need to change the configuration file "..." of your Windows Bridgehead. TODO... From 4d49351ad21f734ec1fcd7aaf9f4ee9bf475a10d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 16 Dec 2022 12:02:49 +0100 Subject: [PATCH 12/52] fix: Included Legacy ID-Mapping (Please Complete) --- ccp/modules/id-management-compose.yml | 6 +++--- ccp/modules/id-management-setup.sh | 24 ++++++++++++++++++++++-- ccp/modules/id-management.md | 6 +++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index d1639af..4baaba7 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -5,11 +5,11 @@ services: container_name: bridgehead-id-manager environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST} - MAGICPL_SITE: ${SITE_ID} + MAGICPL_SITE: ${IDMANAGEMENT_FRIENDLY_ID} MAGICPL_ALLOWED_ORIGINS: https://${HOST} MAGICPL_LOCAL_PATIENTLIST_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_CENTRAXX_APIKEY} - MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_CENTRAXX_APIKEY: ${IDMANAGER_UPLOAD_APIKEY} + MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY} MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 790f846..d2449c7 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$IDMANAGER_CENTRAXX_APIKEY" ]; then + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" @@ -11,7 +11,27 @@ function idManagementSetup() { # Source the ID Generators Configuration source /etc/bridgehead/patientlist-id-generators.env - log INFO "ID-Management Generator 1: ${ML_BK_IDGENERATOR_RANDOM_1}" + + # Ensure old ids are working !!! + legacyIdMapping fi } + +# TODO: Map all old site ids to the new ones +function legacyIdMapping() { + case ${SITE_ID} in + "berlin") + export IDMANAGEMENT_FRIENDLY_ID=Berlin + ;; + "dresden") + export IDMANAGEMENT_FRIENDLY_ID=Dresden + ;; + "frankfurt") + export IDMANAGEMENT_FRIENDLY_ID=Frankfurt + ;; + *) + export IDMANAGEMENT_FRIENDLY_ID=$SITE_ID + ;; + esac +} diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 8de2f5a..89ff65c 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -5,8 +5,8 @@ This module provides integration with the CCP-Pseudonymiziation Service. To lear You must add following configuration variables to your sites-configuration repository: ``` -IDMANAGER_CENTRAXX_APIKEY="" -IDMANAGER_CONNECTOR_APIKEY="" +IDMANAGER_UPLOAD_APIKEY="" +IDMANAGER_READ_APIKEY="" IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" IDMANAGER_AUTH_CLIENT_ID="" @@ -47,7 +47,7 @@ Typically, the sites connect their local data-management for the pseudonym creat On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. ``` dktk.idmanagement.url=https:///id-manager/translator/getId -dktk.idmanagement.apiKey= +dktk.idmanagement.apiKey= ``` They typically already exist, but need to be changed to the new values! #### Sites using ADT2FHIR From e0c9a5ced38a61da718b0bdf2b01208c2371f094 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 2 Dec 2022 15:38:33 +0100 Subject: [PATCH 13/52] Added MTBA Module --- ccp/mtba-compose.yml | 33 +++++++++++++++++++++++++++++++++ ccp/nngm-setup.sh | 12 ++++++++++++ ccp/vars | 1 + 3 files changed, 46 insertions(+) create mode 100644 ccp/mtba-compose.yml diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml new file mode 100644 index 0000000..53fd327 --- /dev/null +++ b/ccp/mtba-compose.yml @@ -0,0 +1,33 @@ +version: "3.7" + +services: + mtba: + image: samply/mtba:develop + container_name: bridgehead-mtba + environment: + BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir + # NOTE: Aktuell Berechtigungen wie MagicPL!!! + # TODO: Add separate ApiKey to Patientlist only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID + ID_MANAGER_URL: http://bridgehead-id-manager:8080 + PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} + PATIENT_CSV_LAST_NAME_HEADER: ${PATIENT_CSV_LAST_NAME_HEADER:-"LAST_NAME"} + PATIENT_CSV_GENDER_HEADER: ${PATIENT_CSV_GENDER_HEADER:-"GENDER"} + PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} + CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 + MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" + labels: + - "traefik.enable=true" + - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" + - "traefik.http.services.mtba.loadbalancer.server.port=80" + - "traefik.http.routers.mtba.tls=true" + volumes: + # This directory persists the FHIR Resources that are needed to import data into blaze. + - /var/data/bridgehead/mtba:/app/mtba-files/persist + # Place new import files in this directory + - /tmp/bridgehead/mtba/:/app/mtba-files/input + + # TODO: Include CBioPortal in Deployment ... + # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! + # TODO: Find a trigger to let mtba signal a restart for CBioPortal diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index d5b80eb..b55d48e 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -7,3 +7,15 @@ function nngmSetup() { fi CONNECTOR_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" } + +function mtbaSetup() { + # TODO: Check if ID-Management Module is activated! + if [ -n "$ENABLE_MTBA" ];then + log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" + exit 1; + fi + OVERRIDE+=" -f ./$PROJECT/mtba-compose.yml" + fi +} diff --git a/ccp/vars b/ccp/vars index 89deae0..5172d24 100644 --- a/ccp/vars +++ b/ccp/vars @@ -16,3 +16,4 @@ source $PROJECT/nngm-setup.sh nngmSetup source $PROJECT/exliquid-setup.sh exliquidSetup +mtbaSetup \ No newline at end of file From e7b238de9d023abf69b88699eb132f67acdbe8f9 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 16 Dec 2022 15:01:04 +0100 Subject: [PATCH 14/52] set magicpl instead of patientlist --- ccp/mtba-compose.yml | 4 ++-- ccp/nngm-setup.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 53fd327..d492ae0 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -7,8 +7,8 @@ services: environment: BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir # NOTE: Aktuell Berechtigungen wie MagicPL!!! - # TODO: Add separate ApiKey to Patientlist only for MTBA! - ID_MANAGER_API_KEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + # TODO: Add separate ApiKey to MagicPL only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID ID_MANAGER_URL: http://bridgehead-id-manager:8080 PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index b55d48e..ba2e77f 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -12,7 +12,7 @@ function mtbaSetup() { # TODO: Check if ID-Management Module is activated! if [ -n "$ENABLE_MTBA" ];then log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." - if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" exit 1; fi From af7960e036deb3ddff22d42354f554ee71942649 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Thu, 19 Jan 2023 10:05:03 +0000 Subject: [PATCH 15/52] Improve documentation --- ccp/modules/id-management.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 89ff65c..4c929b9 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -1,8 +1,8 @@ # Module: Id-Management -This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP-DSK](https://dktk.dkfz.de/application/files/5016/2030/2474/20_11_23_Datenschutzkonzept_CCP-IT_inkl_Anlagen.pdf). +This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP Data Protection Concept](https://dktk.dkfz.de/klinische-plattformen/documents-download). ## Getting Started -You must add following configuration variables to your sites-configuration repository: +The following configuration variables are added to your sites-configuration repository: ``` IDMANAGER_UPLOAD_APIKEY="" @@ -13,15 +13,19 @@ IDMANAGER_AUTH_CLIENT_ID="" IDMANAGER_AUTH_CLIENT_SECRET="" ``` -Additionally, the ccp-it needs to add a new file "patientlist-id-generators.env" to your site configuration. This file will hold the seeds for the different id-generators used in all projects. +Most of the configuration is kept identical across all sites. Some site-specific variables, however, go into a file called `/etc/bridgehead/ccp/id-management.local`. -After adding the configuration, you simply need to update your bridgehead and 3 new services will run on your server: +After adding the configuration, you simply need to update your bridgehead. You're all set! -- `bridgehead-id-manager`, accessible by "https:///id-manager". This component adds a common interface for creating pseudonymes in the bridgehead. -- `bridgehead-patientlist`, accessible by "https:// Date: Thu, 19 Jan 2023 10:15:12 +0000 Subject: [PATCH 16/52] Made it even easier for sites --- ccp/modules/id-management.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 4c929b9..60cb5b1 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -11,11 +11,13 @@ IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" IDMANAGER_AUTH_CLIENT_ID="" IDMANAGER_AUTH_CLIENT_SECRET="" + +IDMANAGER_SEEDS_BK="" +IDMANAGER_SEEDS_MDS="" +IDMANAGER_SEEDS_DKTK000001985="" ``` -Most of the configuration is kept identical across all sites. Some site-specific variables, however, go into a file called `/etc/bridgehead/ccp/id-management.local`. - -After adding the configuration, you simply need to update your bridgehead. You're all set! +Once your Bridgehead is updated and restarted, you're all set! ## Additional information you may want to know ### Services From eb37d16b847800cfb4ba31bb7423a7815ceb0a4c Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Tue, 31 Jan 2023 15:44:30 +0100 Subject: [PATCH 17/52] Updated Blaze to 0.19 --- bbmri/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- ccp/exliquid-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 2433234..cbe2263 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -54,7 +54,7 @@ services: SITE_NAME: ${SITE_NAME} blaze: - image: "samply/blaze:0.18" + image: "samply/blaze:0.19" container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 54b2daa..6715a69 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -54,7 +54,7 @@ services: SITE_NAME: ${SITE_NAME} blaze: - image: "samply/blaze:0.18" + image: "samply/blaze:0.19" container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" diff --git a/ccp/exliquid-compose.yml b/ccp/exliquid-compose.yml index e21617a..eb8d194 100644 --- a/ccp/exliquid-compose.yml +++ b/ccp/exliquid-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: exliquid-task-store: - image: "samply/blaze:0.18" + image: "samply/blaze:0.19" container_name: bridgehead-exliquid-task-store environment: BASE_URL: "http://bridgehead-exliquid-task-store:8080" From 6d8e877899870ba8a9a0191d80995f2b947db110 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Fri, 3 Feb 2023 13:28:44 +0100 Subject: [PATCH 18/52] Use beam-proxy "main" tag --- bbmri/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index cbe2263..3291326 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: "samply/beam-proxy:develop" + image: samply/beam-proxy:main container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 6715a69..4c26d5e 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: "samply/beam-proxy:develop" + image: samply/beam-proxy:main container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} From 1c3fa4fa3fb6c21f886f2c9fdb4fe19b73a6545c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 17 Jan 2023 08:56:02 +0100 Subject: [PATCH 19/52] refactor: Changed Paths for persistent Data --- README.md | 4 ++-- ccp/modules/id-management-compose.yml | 4 ++-- ccp/modules/id-management.md | 6 +++--- ccp/mtba-compose.yml | 2 +- lib/install-bridgehead.sh | 15 ++++++++++----- lib/update-bridgehead.sh | 10 +--------- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 24ebdf2..134e1ae 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. -- `/var/data/bridgehead` contains persistent data of the bridgehead - - `/var/data/bridgehead/backups` contains automatically created backups of the databases. + - `/var/lib/bridgehead/data` contains persistent data of the bridgehead + - `/var/cache/bridgehead/backup` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 4baaba7..896663b 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -71,6 +71,6 @@ services: POSTGRES_DB: "mainzelliste" POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} volumes: - - "/var/data/bridgehead/patientlist:/var/lib/postgresql/data" - # NOTE: Add backups here. This is only imported if /var/data/bridgehead/patientlist/ is empty!!! + - "/var/lib/bridgehead/data/patientlist:/var/lib/postgresql/data" + # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!! - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/" diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 60cb5b1..c45b95a 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -26,7 +26,7 @@ Upon configuration, the Bridgehead will spawn the following services: - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. -- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/data/bridgehead/patientlist` and backups are automatically created in `/var/data/bridgehead/backups/bridgehead-patientlist-db`. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/lib/bridgehead/data/patientlist` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. ### How to import an existing database (e.g from Legacy Windows or from Backups) First you must shutdown your local bridgehead instance: @@ -36,10 +36,10 @@ systemctl stop bridgehead@ccp Next you need to remove the current patientlist database: ``` -rm -rf /var/data/bridgehead/patientlist +rm -rf /var/lib/bridgehead/data/patientlist ``` -Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/data/bridgehead/patientlist is empty. +Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/lib/bridgehead/data/patientlist is empty. > NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). After this, you can restart your bridgehead and the dump will be imported: diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index d492ae0..1c62989 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -24,7 +24,7 @@ services: - "traefik.http.routers.mtba.tls=true" volumes: # This directory persists the FHIR Resources that are needed to import data into blaze. - - /var/data/bridgehead/mtba:/app/mtba-files/persist + - /var/lib/bridgehead/data/mtba:/app/mtba-files/persist # Place new import files in this directory - /tmp/bridgehead/mtba/:/app/mtba-files/input diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 7cbd8ef..04503e3 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,9 +37,14 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi -log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data." -mkdir -p /var/data/bridgehead -chown -R bridgehead /var/data/bridgehead +log "INFO" "Creating directory /var/lib/bridgehead for storage of persistent data." +mkdir -p /var/lib/bridgehead +chown -R bridgehead /var/lib/bridgehead + + +log "INFO" "Creating directory /var/cache/bridgehead for storage of backups." +mkdir -p /var/cache/bridgehead +chown -R bridgehead /var/cache/bridgehead log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index cc1d55f..65560ea 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -4,15 +4,7 @@ source lib/functions.sh AUTO_BACKUP=${AUTO_BACKUP:-true} if [ "$AUTO_BACKUP" == "true" ]; then - BACKUP_DIRECTORY="/var/data/bridgehead/backups" - if [ ! -d /var/data ]; then - log DEBUG "Created /var/data" - mkdir /var/data - fi - if [ ! -d /var/data/bridgehead ]; then - log DEBUG "Created /var/data/bridgehead" - mkdir /var/data/bridgehead - fi + BACKUP_DIRECTORY="/var/cache/bridgehead/backup" if [ ! -d $BACKUP_DIRECTORY ]; then message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." hc_send log "$message" From 0d998ab8716a2e2c5c92fa84e65ccc608524ccef Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 10:55:44 +0100 Subject: [PATCH 20/52] refactor: Move persistent data to named volumes --- README.md | 1 - ccp/modules/id-management-compose.yml | 5 ++++- ccp/modules/id-management.md | 6 +++--- ccp/mtba-compose.yml | 5 ++++- lib/install-bridgehead.sh | 9 ++------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 134e1ae..e1d3286 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,6 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. - - `/var/lib/bridgehead/data` contains persistent data of the bridgehead - `/var/cache/bridgehead/backup` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 896663b..72aebcc 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -71,6 +71,9 @@ services: POSTGRES_DB: "mainzelliste" POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} volumes: - - "/var/lib/bridgehead/data/patientlist:/var/lib/postgresql/data" + - "patientlist-db-data:/var/lib/postgresql/data" # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!! - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/" + +volumes: + patientlist-db-data: diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index c45b95a..e18d3f8 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -26,7 +26,7 @@ Upon configuration, the Bridgehead will spawn the following services: - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. -- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/lib/bridgehead/data/patientlist` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. ### How to import an existing database (e.g from Legacy Windows or from Backups) First you must shutdown your local bridgehead instance: @@ -36,10 +36,10 @@ systemctl stop bridgehead@ccp Next you need to remove the current patientlist database: ``` -rm -rf /var/lib/bridgehead/data/patientlist +docker volume rm patientlist-db-data; ``` -Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/lib/bridgehead/data/patientlist is empty. +Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then the volume `patientlist-db-data` was removed previously. > NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). After this, you can restart your bridgehead and the dump will be imported: diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 1c62989..857ff34 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -24,10 +24,13 @@ services: - "traefik.http.routers.mtba.tls=true" volumes: # This directory persists the FHIR Resources that are needed to import data into blaze. - - /var/lib/bridgehead/data/mtba:/app/mtba-files/persist + - mtba-data:/app/mtba-files/persist # Place new import files in this directory - /tmp/bridgehead/mtba/:/app/mtba-files/input # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! # TODO: Find a trigger to let mtba signal a restart for CBioPortal + +volumes: + mtba-data: diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 04503e3..f1aff73 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/lib/bridgehead /var/cache/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,11 +37,6 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi -log "INFO" "Creating directory /var/lib/bridgehead for storage of persistent data." -mkdir -p /var/lib/bridgehead -chown -R bridgehead /var/lib/bridgehead - - log "INFO" "Creating directory /var/cache/bridgehead for storage of backups." mkdir -p /var/cache/bridgehead chown -R bridgehead /var/cache/bridgehead From 1befa65f35033f9383aae88b7fdcf5fd0244e156 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 10:59:29 +0100 Subject: [PATCH 21/52] refactor: Changed Salt for patientlist db password --- ccp/modules/id-management-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index d2449c7..98c4217 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -6,7 +6,7 @@ function idManagementSetup() { OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" # Auto Generate local Passwords - PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" # Source the ID Generators Configuration From 713dc5f4e9564596fb2a47afaceeb2a28cd2ea1d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:15:09 +0100 Subject: [PATCH 22/52] refactor: Move Backups after the Update --- lib/update-bridgehead.sh | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 65560ea..7fb3688 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,37 +1,6 @@ #!/bin/bash source lib/functions.sh -AUTO_BACKUP=${AUTO_BACKUP:-true} - -if [ "$AUTO_BACKUP" == "true" ]; then - BACKUP_DIRECTORY="/var/cache/bridgehead/backup" - if [ ! -d $BACKUP_DIRECTORY ]; then - message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - mkdir -p $BACKUP_DIRECTORY - fi - BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" - log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; - for service in $BACKUP_SERVICES; do - if [ ! -d $BACKUP_DIRECTORY/$service ]; then - message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - mkdir -p $BACKUP_DIRECTORY/$service - fi - if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then - message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - else - fail_and_report 5 "Failed to create encrypted update for $service" - fi - done -else - log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" -fi - AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then @@ -134,6 +103,37 @@ else hc_send log "$RES" fi +AUTO_BACKUP=${AUTO_BACKUP:-true} + +if [ "$AUTO_BACKUP" == "true" ]; then + BACKUP_DIRECTORY="/var/cache/bridgehead/backup" + if [ ! -d $BACKUP_DIRECTORY ]; then + message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY + fi + BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d $BACKUP_DIRECTORY/$service ]; then + message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p $BACKUP_DIRECTORY/$service + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + exit 0 # TODO: Print last commit explicit From 17d48a3636c812d7d2ac1a07b6adab20c930d75c Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:26:31 +0100 Subject: [PATCH 23/52] refactor: Expect User to select a Backup Directory --- README.md | 3 ++- ccp/modules/id-management.md | 2 +- lib/install-bridgehead.sh | 8 ++------ lib/update-bridgehead.sh | 19 ++++++++----------- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index e1d3286..8fba0c9 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,6 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. - - `/var/cache/bridgehead/backup` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. @@ -147,6 +146,8 @@ Some of the components in the bridgehead will store persistent data. For those c 2) Year-KW-XX, were XX represents the calendar week to allow re-import of at least one version per calendar week 3) Year-Month, to allow re-import of at least one version per month +To enable the Auto-Backup feature, please set the Variable `BACKUP_DIRECTORY` in your sites configuration. + ### Monitoring To keep all Bridgeheads up and working and detect any errors before a user does, a central monitoring diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index e18d3f8..98da3d1 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -26,7 +26,7 @@ Upon configuration, the Bridgehead will spawn the following services: - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. -- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data` and backups are automatically created in `/var/cache/bridgehead/backup/bridgehead-patientlist-db`. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data`. ### How to import an existing database (e.g from Legacy Windows or from Backups) First you must shutdown your local bridgehead instance: diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index f1aff73..c42119f 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/cache/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,10 +37,6 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi -log "INFO" "Creating directory /var/cache/bridgehead for storage of backups." -mkdir -p /var/cache/bridgehead -chown -R bridgehead /var/cache/bridgehead - log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ lib/systemd/bridgehead\@.service \ diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 7fb3688..276f60c 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -103,27 +103,24 @@ else hc_send log "$RES" fi -AUTO_BACKUP=${AUTO_BACKUP:-true} - -if [ "$AUTO_BACKUP" == "true" ]; then - BACKUP_DIRECTORY="/var/cache/bridgehead/backup" - if [ ! -d $BACKUP_DIRECTORY ]; then - message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." +if [ -z "${BACKUP_DIRECTORY}" ]; then + if [ ! -d "$BACKUP_DIRECTORY" ]; then + message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY." hc_send log "$message" log INFO "$message" - mkdir -p $BACKUP_DIRECTORY + mkdir -p "$BACKUP_DIRECTORY" fi BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; for service in $BACKUP_SERVICES; do - if [ ! -d $BACKUP_DIRECTORY/$service ]; then - message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." + if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then + message="Performing automatic maintenance: Attempting to create backup directory for $service in $BACKUP_DIRECTORY." hc_send log "$message" log INFO "$message" - mkdir -p $BACKUP_DIRECTORY/$service + mkdir -p "$BACKUP_DIRECTORY/$service" fi if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then - message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." + message="Performing automatic maintenance: Stored encrypted backup for $service in $BACKUP_DIRECTORY." hc_send log "$message" log INFO "$message" else From a37bf79c030908e80284c981975161bb31b989cc Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:31:36 +0100 Subject: [PATCH 24/52] refactor: Added Monitoring for backup permissions --- lib/update-bridgehead.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 276f60c..0f1c413 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -109,7 +109,9 @@ if [ -z "${BACKUP_DIRECTORY}" ]; then hc_send log "$message" log INFO "$message" mkdir -p "$BACKUP_DIRECTORY" + chown -R "$BACKUP_DIRECTORY" bridgehead; fi + checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)" BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; for service in $BACKUP_SERVICES; do From 98afeac701f0065d0052f07a8c9b27ce9e2ff779 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:49:18 +0100 Subject: [PATCH 25/52] refactor: Use -db as Indicator for Backups --- lib/update-bridgehead.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 0f1c413..ab39cca 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -112,7 +112,8 @@ if [ -z "${BACKUP_DIRECTORY}" ]; then chown -R "$BACKUP_DIRECTORY" bridgehead; fi checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)" - BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" + # Collect all container names that contain '-db' + BACKUP_SERVICES="$(docker ps --filter name=-db --format "{{.Names}}" | tr "\n" "\ ")" log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; for service in $BACKUP_SERVICES; do if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then From 2445a5978506bd7d3b84d85fafe7e8a16d6f239d Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 27 Jan 2023 11:52:09 +0100 Subject: [PATCH 26/52] refacotr: Restructured passing Patientlist Seeds --- ccp/modules/id-management-compose.yml | 25 +------------------------ ccp/modules/id-management-setup.sh | 4 ++-- ccp/modules/id-management.md | 2 ++ 3 files changed, 5 insertions(+), 26 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 72aebcc..3f13519 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -31,30 +31,7 @@ services: - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} # Add Variables from /etc/patientlist-id-generators.env - - ML_BK_IDGENERATOR_RANDOM_1 - - ML_BK_IDGENERATOR_RANDOM_2 - - ML_BK_IDGENERATOR_RANDOM_3 - - ML_MDS_IDGENERATOR_RANDOM_1 - - ML_MDS_IDGENERATOR_RANDOM_2 - - ML_MDS_IDGENERATOR_RANDOM_3 - - ML_DKTK000001985_IDGENERATOR_RANDOM_1 - - ML_DKTK000001985_IDGENERATOR_RANDOM_2 - - ML_DKTK000001985_IDGENERATOR_RANDOM_3 - - ML_DKTK000001986_IDGENERATOR_RANDOM_1 - - ML_DKTK000001986_IDGENERATOR_RANDOM_2 - - ML_DKTK000001986_IDGENERATOR_RANDOM_3 - - ML_DKTK000001950_IDGENERATOR_RANDOM_1 - - ML_DKTK000001950_IDGENERATOR_RANDOM_2 - - ML_DKTK000001950_IDGENERATOR_RANDOM_3 - - ML_DKTK000001951_IDGENERATOR_RANDOM_1 - - ML_DKTK000001951_IDGENERATOR_RANDOM_2 - - ML_DKTK000001951_IDGENERATOR_RANDOM_3 - - ML_DKTK999999999_IDGENERATOR_RANDOM_1 - - ML_DKTK999999999_IDGENERATOR_RANDOM_2 - - ML_DKTK999999999_IDGENERATOR_RANDOM_3 - - ML_DKTK000002089_IDGENERATOR_RANDOM_1 - - ML_DKTK000002089_IDGENERATOR_RANDOM_2 - - ML_DKTK000002089_IDGENERATOR_RANDOM_3 + - PATIENTLIST_SEEDS_TRANSFORMED labels: - "traefik.enable=true" - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 98c4217..0f361dc 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -9,8 +9,8 @@ function idManagementSetup() { PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" - # Source the ID Generators Configuration - source /etc/bridgehead/patientlist-id-generators.env + # Transform Seeds Configuration to pass it to the Mainzelliste Container + PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')" # Ensure old ids are working !!! legacyIdMapping diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 98da3d1..66f9f71 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -16,10 +16,12 @@ IDMANAGER_SEEDS_BK="" IDMANAGER_SEEDS_MDS="" IDMANAGER_SEEDS_DKTK000001985="" ``` +> NOTE: Additionally, the CCP-IT adds lines declaring the `PATIENTLIST_SEEDS` array in your site configuration. This will contain the seeds for the different id-generators used in all projects. Once your Bridgehead is updated and restarted, you're all set! ## Additional information you may want to know + ### Services Upon configuration, the Bridgehead will spawn the following services: From a3ba98a2fdb1d950ab4fbcb2d1a9302692086816 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 7 Dec 2022 15:46:19 +0100 Subject: [PATCH 27/52] Added Configuration for Local ID-Management --- ccp/modules/id-management-setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 0f361dc..b02ef66 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,8 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then +# if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + if [ -n "$ENABLE_ID_MANAGEMENT" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From e2d90447f7e3ad6ee981c16de6d02b048cfdebf2 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 15 Dec 2022 16:39:03 +0100 Subject: [PATCH 28/52] refactor: Changed Trigger of the IDM Module --- ccp/modules/id-management-setup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index b02ef66..c8ea707 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,6 @@ #!/bin/bash function idManagementSetup() { -# if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then if [ -n "$ENABLE_ID_MANAGEMENT" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" From bce28342f90482205a5eba1d284fb5dffb6e6cce Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 18 Jan 2023 13:42:36 +0100 Subject: [PATCH 29/52] update docker and host paths --- ccp/mtba-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index 857ff34..ad25dd0 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -17,16 +17,16 @@ services: PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" + NEW_FILES_DIRECTORY: "/app/input" + PERSIST_DIRECTORY: "/app/persist" labels: - "traefik.enable=true" - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" - "traefik.http.services.mtba.loadbalancer.server.port=80" - "traefik.http.routers.mtba.tls=true" volumes: - # This directory persists the FHIR Resources that are needed to import data into blaze. - - mtba-data:/app/mtba-files/persist - # Place new import files in this directory - - /tmp/bridgehead/mtba/:/app/mtba-files/input + - /tmp/bridgehead/mtba/input:/app/input + - /tmp/bridgehead/mtba/persist:/app/persist # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! From e9455a5558dd33de59bb46e3c048368f4c30df34 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 27 Jan 2023 11:13:49 +0100 Subject: [PATCH 30/52] update mtba config --- ccp/modules/id-management-setup.sh | 2 +- ccp/mtba-compose.yml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index c8ea707..0f361dc 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash function idManagementSetup() { - if [ -n "$ENABLE_ID_MANAGEMENT" ]; then + if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index ad25dd0..bb90059 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -5,20 +5,20 @@ services: image: samply/mtba:develop container_name: bridgehead-mtba environment: - BLAZE_STORE_URL: http://bridgehead-ccp-blaze:8080/fhir + BLAZE_STORE_URL: http://blaze:8080 # NOTE: Aktuell Berechtigungen wie MagicPL!!! # TODO: Add separate ApiKey to MagicPL only for MTBA! ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID - ID_MANAGER_URL: http://bridgehead-id-manager:8080 - PATIENT_CSV_FIRST_NAME_HEADER: ${PATIENT_CSV_FIRST_NAME_HEADER:-"FIRST_NAME"} - PATIENT_CSV_LAST_NAME_HEADER: ${PATIENT_CSV_LAST_NAME_HEADER:-"LAST_NAME"} - PATIENT_CSV_GENDER_HEADER: ${PATIENT_CSV_GENDER_HEADER:-"GENDER"} - PATIENT_CSV_BIRTHDAY_HEADER: ${PATIENT_CSV_BIRTHDAY_HEADER:-"BIRTHDAY"} - CBIOPORTAL_URL: http://bridgehead-ccp-cbioportal:8080 - MUTATIONS_CSV_SCRIPT_INTERPRETER: "python3" - NEW_FILES_DIRECTORY: "/app/input" - PERSIST_DIRECTORY: "/app/persist" + ID_MANAGER_URL: http://id-manager:8080/id-manager + PATIENT_CSV_FIRST_NAME_HEADER: ${MTBA_PATIENT_CSV_FIRST_NAME_HEADER} + PATIENT_CSV_LAST_NAME_HEADER: ${MTBA_PATIENT_CSV_LAST_NAME_HEADER} + PATIENT_CSV_GENDER_HEADER: ${MTBA_PATIENT_CSV_GENDER_HEADER} + PATIENT_CSV_BIRTHDAY_HEADER: ${MTBA_PATIENT_CSV_BIRTHDAY_HEADER} + CBIOPORTAL_URL: http://cbioportal:8080 + FILE_CHARSET: ${MTBA_FILE_CHARSET} + FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} + CSV_DELIMITER: ${MTBA_CSV_DELIMITER} labels: - "traefik.enable=true" - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" From c88919c926e1b07b0d5bfe8a21d32dfaee3c267e Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 8 Feb 2023 14:42:52 +0100 Subject: [PATCH 31/52] feature: Ensured the mapping to legacy ids works --- ccp/modules/id-management-setup.sh | 49 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 0f361dc..ba8ad45 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -13,25 +13,40 @@ function idManagementSetup() { PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')" # Ensure old ids are working !!! - legacyIdMapping + export IDMANAGEMENT_FRIENDLY_ID=$(legacyIdMapping "$SITE_ID") fi - } -# TODO: Map all old site ids to the new ones +# Transform into single string array, e.g. 'dktk-test' to 'dktk test' +# Usage: transformToSingleStringArray 'dktk-test' -> 'dktk test' +function transformToSingleStringArray() { + echo "${1//-/ }"; +} + +# Ensure all Words are Uppercase +# Usage: transformToUppercase 'dktk test' -> 'Dktk Test' +function transformToUppercase() { + result=""; + for word in $1; do + result+=" ${word^}"; + done + echo "$result"; +} + +# Handle all execeptions from the norm (e.g LMU, TUM) +# Usage: applySpecialCases 'Muenchen Lmu Test' -> 'Muenchen LMU Test' +function applySpecialCases() { + result="$1"; + result="${result/Lmu/LMU}"; + result="${result/Tum/TUM}"; + echo "$result"; +} + +# Transform current siteids to legacy version +# Usage: legacyIdMapping "dktk-test" -> "DktkTest" function legacyIdMapping() { - case ${SITE_ID} in - "berlin") - export IDMANAGEMENT_FRIENDLY_ID=Berlin - ;; - "dresden") - export IDMANAGEMENT_FRIENDLY_ID=Dresden - ;; - "frankfurt") - export IDMANAGEMENT_FRIENDLY_ID=Frankfurt - ;; - *) - export IDMANAGEMENT_FRIENDLY_ID=$SITE_ID - ;; - esac + single_string_array=$(transformToSingleStringArray "$1"); + uppercase_string=$(transformToUppercase "$single_string_array"); + normalized_string=$(applySpecialCases "$uppercase_string"); + echo "$normalized_string" | tr -d ' ' } From d728ccd88657358374f107ee0a03b06442a5c4e1 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Wed, 8 Feb 2023 15:02:35 +0100 Subject: [PATCH 32/52] refactor: Fixated Postgres Version to 15.1 All Minor Updates will cause a crashing postgres on start --- ccp/modules/id-management-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 3f13519..2f26ce4 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -41,7 +41,7 @@ services: - patientlist-db patientlist-db: - image: postgres:14-alpine + image: postgres:15.1-alpine container_name: bridgehead-patientlist-db environment: POSTGRES_USER: "mainzelliste" From 64095ac8dd3b0b0816ff6aefc77fe5d56a8efb16 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 09:15:29 +0100 Subject: [PATCH 33/52] feat: Added Upload ApiKey to Patientlist --- ccp/modules/id-management-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 2f26ce4..8e5bab8 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -30,6 +30,7 @@ services: - ML_SITE=${SITE_ID} - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + - ML_UPLOAD_API_KEY=${IDMANAGER_UPLOAD_APIKEY} # Add Variables from /etc/patientlist-id-generators.env - PATIENTLIST_SEEDS_TRANSFORMED labels: From a1cd5a206e402922a22f037c4bedaa1488642150 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 09:50:30 +0100 Subject: [PATCH 34/52] fix: Repaired check for Auto Updates --- lib/update-bridgehead.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index ab39cca..c661595 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -103,7 +103,7 @@ else hc_send log "$RES" fi -if [ -z "${BACKUP_DIRECTORY}" ]; then +if [ -n "${BACKUP_DIRECTORY}" ]; then if [ ! -d "$BACKUP_DIRECTORY" ]; then message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY." hc_send log "$message" From a6975e37ac8433d615b8a8ee3d36c219e047c194 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 09:58:51 +0100 Subject: [PATCH 35/52] fix: Added Image Updates for activated Modules --- lib/update-bridgehead.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index c661595..c53b731 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -81,7 +81,7 @@ done # Check docker updates log "INFO" "Checking for updates to running docker images ..." docker_updated="false" -for IMAGE in $(cat $PROJECT/docker-compose.yml | grep -v "^#" | grep "image:" | sed -e 's_^.*image: \(.*\).*$_\1_g; s_\"__g'); do +for IMAGE in $(cat $PROJECT/docker-compose.yml ${OVERRIDE//-f/} | grep -v "^#" | grep "image:" | sed -e 's_^.*image: \(.*\).*$_\1_g; s_\"__g'); do log "INFO" "Checking for Updates of Image: $IMAGE" if docker pull $IMAGE | grep "Downloaded newer image"; then CHANGE="Image $IMAGE updated." From 5a7dd1d0aedeba61c6d8c2b954f0bc0cc8ef5114 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Feb 2023 10:07:05 +0100 Subject: [PATCH 36/52] fix: Use Mapped Site Id in Patientlist --- ccp/modules/id-management-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 8e5bab8..cb0c89a 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -27,7 +27,7 @@ services: container_name: bridgehead-patientlist environment: - TOMCAT_REVERSEPROXY_FQDN=${HOST} - - ML_SITE=${SITE_ID} + - ML_SITE=${IDMANAGEMENT_FRIENDLY_ID} - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - ML_UPLOAD_API_KEY=${IDMANAGER_UPLOAD_APIKEY} From 90773ea92afb409bd15c6205531398e6f464d3f4 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 21 Feb 2023 09:26:53 +0100 Subject: [PATCH 37/52] Switch beam images to develop tag --- bbmri/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 3291326..89ed1a9 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: samply/beam-proxy:main + image: samply/beam-proxy:develop container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 4c26d5e..65e57d2 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: samply/beam-proxy:main + image: samply/beam-proxy:develop container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} From 191e9863642119a5088124f714bb045de29e3348 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Wed, 22 Feb 2023 15:32:21 +0100 Subject: [PATCH 38/52] Add check for installation in WSL and for systemd --- lib/prepare-system.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index 2cba2e2..2a4fdae 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -5,6 +5,15 @@ source lib/functions.sh log "INFO" "Preparing your system for bridgehead installation ..." +# Check, if running in WSL +if [[ $(grep -i Microsoft /proc/version) ]]; then + # Check, if systemd is available + if [ ! $(systemctl) ]; then + log "ERROR" "It seems, that you have no active systemd environment in your WSL. Please follow the guide in https://devblogs.microsoft.com/commandline/systemd-support-is-now-available-in-wsl/" + exit 1 + fi +fi + # Create the bridgehead user if id bridgehead &>/dev/null; then log "INFO" "Existing user with id $(id -u bridgehead) will be used by the bridgehead system units." From 4578c77d4bf18d2275d45a94532cdfed04673bb8 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Wed, 22 Feb 2023 15:42:52 +0100 Subject: [PATCH 39/52] Fix systemd check --- lib/prepare-system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index 2a4fdae..cfede1e 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -8,7 +8,7 @@ log "INFO" "Preparing your system for bridgehead installation ..." # Check, if running in WSL if [[ $(grep -i Microsoft /proc/version) ]]; then # Check, if systemd is available - if [ ! $(systemctl) ]; then + if [ $(systemctl is-system-running) -eq "offline" ]; then log "ERROR" "It seems, that you have no active systemd environment in your WSL. Please follow the guide in https://devblogs.microsoft.com/commandline/systemd-support-is-now-available-in-wsl/" exit 1 fi From fdda14c1bea574c9a8eef11e8459056ae2ed3f16 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Thu, 23 Feb 2023 14:26:59 +0100 Subject: [PATCH 40/52] Fixed naming of site in exliquid script --- ccp/exliquid-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/exliquid-setup.sh b/ccp/exliquid-setup.sh index 83daa45..91909eb 100644 --- a/ccp/exliquid-setup.sh +++ b/ccp/exliquid-setup.sh @@ -2,7 +2,7 @@ function exliquidSetup() { case ${SITE_ID} in - berlin|dresden|essen|frankfurt|freiburg|luebeck|mainz|muenchen-lmu|muenchen-tu|mannheim|tuebingen) + berlin|dresden|essen|frankfurt|freiburg|luebeck|mainz|muenchen-lmu|muenchen-tum|mannheim|tuebingen) EXLIQUID=1 ;; dktk-test) From 3023b82bb164a99d331f86a113418e6c78b90895 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 21 Feb 2023 09:26:53 +0100 Subject: [PATCH 41/52] Switch beam images to develop tag --- bbmri/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 3291326..89ed1a9 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: samply/beam-proxy:main + image: samply/beam-proxy:develop container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 4c26d5e..65e57d2 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: samply/beam-proxy:main + image: samply/beam-proxy:develop container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} From 2dc36433bf38527970160942e793cfc21a91e6d3 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Thu, 23 Feb 2023 14:26:59 +0100 Subject: [PATCH 42/52] Fixed naming of site in exliquid script --- ccp/exliquid-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/exliquid-setup.sh b/ccp/exliquid-setup.sh index 83daa45..91909eb 100644 --- a/ccp/exliquid-setup.sh +++ b/ccp/exliquid-setup.sh @@ -2,7 +2,7 @@ function exliquidSetup() { case ${SITE_ID} in - berlin|dresden|essen|frankfurt|freiburg|luebeck|mainz|muenchen-lmu|muenchen-tu|mannheim|tuebingen) + berlin|dresden|essen|frankfurt|freiburg|luebeck|mainz|muenchen-lmu|muenchen-tum|mannheim|tuebingen) EXLIQUID=1 ;; dktk-test) From 8b2e99200ed0586faf6ed7e7df229908c9809072 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Thu, 23 Feb 2023 18:05:34 +0100 Subject: [PATCH 43/52] Fix typo --- lib/prepare-system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index 2cba2e2..f54ee07 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -26,7 +26,7 @@ if [ -d "/srv/docker/bridgehead" ]; then else log "INFO" "Cloning $bridgehead_repository_url to /srv/docker/bridgehead" mkdir -p /srv/docker/ - git clone bridgehead_repository_url /srv/docker/bridgehead + git clone $bridgehead_repository_url /srv/docker/bridgehead fi case "$PROJECT" in From 857e351b884b11d123d4516e4fadea04f893addb Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Thu, 23 Feb 2023 18:05:53 +0100 Subject: [PATCH 44/52] Support gitmirror for github.com repo --- lib/prepare-system.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index f54ee07..8cacdf0 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -14,7 +14,12 @@ else fi # Clone the OpenSource repository of bridgehead -bridgehead_repository_url="https://github.com/samply/bridgehead.git" +set +e +bridgehead_repository_url=$(git remote get-url origin) +if [ $? -ne 0 ]; then + bridgehead_repository_url="https://github.com/samply/bridgehead.git" +fi +set -e if [ -d "/srv/docker/bridgehead" ]; then current_owner=$(stat -c '%U' /srv/docker/bridgehead) if [ "$(su -c 'git -C /srv/docker/bridgehead remote get-url origin' $current_owner)" == "$bridgehead_repository_url" ]; then From 7a350a8c9b2e3c1d5a8c3a56d0e7958e468ea39a Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Fri, 24 Feb 2023 11:29:06 +0100 Subject: [PATCH 45/52] Fix string comparison in WSL check --- lib/prepare-system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index cfede1e..169ad2c 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -8,7 +8,7 @@ log "INFO" "Preparing your system for bridgehead installation ..." # Check, if running in WSL if [[ $(grep -i Microsoft /proc/version) ]]; then # Check, if systemd is available - if [ $(systemctl is-system-running) -eq "offline" ]; then + if [ $(systemctl is-system-running) = "offline" ]; then log "ERROR" "It seems, that you have no active systemd environment in your WSL. Please follow the guide in https://devblogs.microsoft.com/commandline/systemd-support-is-now-available-in-wsl/" exit 1 fi From bfc00b99676ce51c821bd9fd38739ed5e3495843 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Fri, 24 Feb 2023 11:41:05 +0100 Subject: [PATCH 46/52] Prevent variable splitting in wsl check and improve error message --- lib/prepare-system.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index 169ad2c..7e9f24f 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -8,8 +8,8 @@ log "INFO" "Preparing your system for bridgehead installation ..." # Check, if running in WSL if [[ $(grep -i Microsoft /proc/version) ]]; then # Check, if systemd is available - if [ $(systemctl is-system-running) = "offline" ]; then - log "ERROR" "It seems, that you have no active systemd environment in your WSL. Please follow the guide in https://devblogs.microsoft.com/commandline/systemd-support-is-now-available-in-wsl/" + if [ "$(systemctl is-system-running)" = "offline" ]; then + log "ERROR" "It seems you have no active systemd environment in your WSL environment. Please follow the guide in https://devblogs.microsoft.com/commandline/systemd-support-is-now-available-in-wsl/" exit 1 fi fi From 5d38f48f682065d3ec1ce70275452a57ea9da932 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Fri, 24 Feb 2023 16:32:17 +0100 Subject: [PATCH 47/52] Add developer install --- bridgehead | 7 ++++++- lib/prepare-system.sh | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/bridgehead b/bridgehead index ecf4ec0..227a0d7 100755 --- a/bridgehead +++ b/bridgehead @@ -77,7 +77,12 @@ case "$ACTION" in exec ./lib/update-bridgehead.sh $PROJECT ;; install) - source ./lib/prepare-system.sh + source ./lib/prepare-system.sh NODEV + loadVars + exec ./lib/install-bridgehead.sh $PROJECT + ;; + dev-install) + exec ./lib/prepare-system.sh DEV loadVars exec ./lib/install-bridgehead.sh $PROJECT ;; diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index 8cacdf0..765c6d3 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +DEV_MODE="${1:-NODEV}" + source lib/log.sh source lib/functions.sh @@ -55,7 +57,7 @@ if [ -d /etc/bridgehead ]; then else log "WARN" "Your site configuration repository in /etc/bridgehead seems to have another origin than git.verbis.dkfz.de. Please check if the repository is correctly cloned!" fi -else +elif [[ "$DEV_MODE" == "NODEV" ]]; then log "INFO" "Now cloning your site configuration repository for you." read -p "Please enter your site: " site read -s -p "Please enter the bridgehead's access token for your site configuration repository (will not be echoed): " access_token @@ -64,9 +66,13 @@ else if [ $? -gt 0 ]; then log "ERROR" "Unable to clone your configuration repository. Please obtain correct access data and try again." fi +elif [[ "$DEV_MODE" == "DEV" ]]; then + log "INFO" "Now cloning your developer configuration repository for you." + read -p "Please enter your config repository URL: " url + git clone "$url" /etc/bridgehead fi chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead -log INFO "System preparation is completed and private key is present." +log INFO "System preparation is completed and configuration is present." From cedc97477f3bc094cdb35daf1d6b2a8ba81513c6 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Mon, 27 Feb 2023 13:02:59 +0100 Subject: [PATCH 48/52] Add developer install option to the documentation --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8fba0c9..2951102 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,10 @@ To keep all Bridgeheads up and working and detect any errors before a user does, In all monitoring cases, obviously no sensitive information is transmitted, in particular not any patient-related data. Aggregated data, e.g. total amount of datasets, may be transmitted for diagnostic purposes. +### Development Installation + +By using `./bridgehead dev-install ` instead of `install`, you can install a developer bridgehead. The difference is, that you can provide an arbitrary configuration repository during the installation, meaning that it does not have to adhere to the usual naming scheme. This allows for better decoupling between development and production configurations. + ## Troubleshooting ### Docker Daemon Proxy Configuration From 0ff153ef22ab71ad6ca6e31d37f59950062a9545 Mon Sep 17 00:00:00 2001 From: lablans Date: Wed, 8 Mar 2023 09:00:38 +0000 Subject: [PATCH 49/52] Use project name. Add is-running function. --- bridgehead | 7 +++++-- lib/functions.sh | 12 +++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/bridgehead b/bridgehead index 227a0d7..60f4aa9 100755 --- a/bridgehead +++ b/bridgehead @@ -66,11 +66,14 @@ case "$ACTION" in checkRequirements hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..." export LDM_LOGIN=$(getLdmPassword) - exec $COMPOSE -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit + exec $COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit ;; stop) loadVars - exec $COMPOSE -f ./$PROJECT/docker-compose.yml $OVERRIDE down + exec $COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE down + ;; + is-running) + exit bk_is_running ;; update) loadVars diff --git a/lib/functions.sh b/lib/functions.sh index 23fb939..6954cd2 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -34,7 +34,7 @@ checkOwner(){ } printUsage() { - echo "Usage: bridgehead start|stop|update|install|uninstall|enroll PROJECTNAME" + echo "Usage: bridgehead start|stop|is-running|update|install|uninstall|enroll PROJECTNAME" echo "PROJECTNAME should be one of ccp|bbmri" } @@ -169,6 +169,16 @@ function retry { return 0 } +function bk_is_running { + RUNNING="$($COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE ps -q)" + NUMBEROFRUNNING=$(echo "$RUNNING" | wc -l) + if [ $NUMBEROFRUNNING -gt 0 ]; then + return 0 + else + return 1 + fi +} + ##Setting Network properties # currently not needed #export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); From 380511d3bb5ff95422f269621eb49484b231199a Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Wed, 8 Mar 2023 10:37:37 +0100 Subject: [PATCH 50/52] Don't delete docker images if BK is not running --- bridgehead | 3 ++- lib/functions.sh | 3 ++- lib/update-bridgehead.sh | 9 +++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bridgehead b/bridgehead index 60f4aa9..f2eaa6b 100755 --- a/bridgehead +++ b/bridgehead @@ -73,7 +73,8 @@ case "$ACTION" in exec $COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE down ;; is-running) - exit bk_is_running + bk_is_running + exit $? ;; update) loadVars diff --git a/lib/functions.sh b/lib/functions.sh index 6954cd2..836ffcc 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -170,9 +170,10 @@ function retry { } function bk_is_running { + detectCompose RUNNING="$($COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE ps -q)" NUMBEROFRUNNING=$(echo "$RUNNING" | wc -l) - if [ $NUMBEROFRUNNING -gt 0 ]; then + if [ $NUMBEROFRUNNING -ge 2 ]; then return 0 else return 1 diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index c53b731..bce720d 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -4,10 +4,15 @@ source lib/functions.sh AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then - A="Performing automatic maintenance: Cleaning docker images." + A="Performing automatic maintenance: " + if bk_is_running; then + A="$A Cleaning docker images." + docker system prune -a -f + else + A="$A Not cleaning docker images since BK is not running." + fi hc_send log "$A" log INFO "$A" - docker system prune -a -f else log WARN "Automatic housekeeping disabled (variable AUTO_HOUSEKEEPING != \"true\")" fi From 33b50372c6d264ce17811656ab2b2d74bf846404 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Thu, 9 Mar 2023 11:16:34 +0100 Subject: [PATCH 51/52] Pull docker images from DKFZ mirror --- bbmri/docker-compose.yml | 12 ++++++------ ccp/docker-compose.yml | 12 ++++++------ ccp/exliquid-compose.yml | 4 ++-- ccp/modules/id-management-compose.yml | 2 +- ccp/mtba-compose.yml | 2 +- ccp/nngm-compose.yml | 2 +- lib/functions.sh | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 89ed1a9..6a6d0a8 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: traefik: container_name: bridgehead-traefik - image: traefik:latest + image: docker.verbis.dkfz.de/cache/traefik:latest command: - --entrypoints.web.address=:80 - --entrypoints.websecure.address=:443 @@ -32,7 +32,7 @@ services: forward_proxy: container_name: bridgehead-forward-proxy - image: samply/bridgehead-forward-proxy:latest + image: docker.verbis.dkfz.de/cache/samply/bridgehead-forward-proxy:latest environment: HTTPS_PROXY: ${HTTPS_PROXY_URL} USERNAME: ${HTTPS_PROXY_USERNAME} @@ -42,7 +42,7 @@ services: landing: container_name: bridgehead-landingpage - image: samply/bridgehead-landingpage:master + image: docker.verbis.dkfz.de/cache/samply/bridgehead-landingpage:master labels: - "traefik.enable=true" - "traefik.http.routers.landing.rule=PathPrefix(`/`)" @@ -54,7 +54,7 @@ services: SITE_NAME: ${SITE_NAME} blaze: - image: "samply/blaze:0.19" + image: docker.verbis.dkfz.de/cache/samply/blaze:0.19 container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" @@ -72,7 +72,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" spot: - image: samply/spot:latest + image: docker.verbis.dkfz.de/cache/samply/spot:latest container_name: bridgehead-spot environment: SECRET: ${SPOT_BEAM_SECRET_LONG} @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: samply/beam-proxy:develop + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 65e57d2..209ee70 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: traefik: container_name: bridgehead-traefik - image: traefik:latest + image: docker.verbis.dkfz.de/cache/traefik:latest command: - --entrypoints.web.address=:80 - --entrypoints.websecure.address=:443 @@ -32,7 +32,7 @@ services: forward_proxy: container_name: bridgehead-forward-proxy - image: samply/bridgehead-forward-proxy:latest + image: docker.verbis.dkfz.de/cache/samply/bridgehead-forward-proxy:latest environment: HTTPS_PROXY: ${HTTPS_PROXY_URL} USERNAME: ${HTTPS_PROXY_USERNAME} @@ -42,7 +42,7 @@ services: landing: container_name: bridgehead-landingpage - image: samply/bridgehead-landingpage:master + image: docker.verbis.dkfz.de/cache/samply/bridgehead-landingpage:master labels: - "traefik.enable=true" - "traefik.http.routers.landing.rule=PathPrefix(`/`)" @@ -54,7 +54,7 @@ services: SITE_NAME: ${SITE_NAME} blaze: - image: "samply/blaze:0.19" + image: docker.verbis.dkfz.de/cache/samply/blaze:0.19 container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" @@ -72,7 +72,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" spot: - image: samply/spot:latest + image: docker.verbis.dkfz.de/cache/samply/spot:latest container_name: bridgehead-spot environment: SECRET: ${SPOT_BEAM_SECRET_LONG} @@ -85,7 +85,7 @@ services: - "blaze" beam-proxy: - image: samply/beam-proxy:develop + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} diff --git a/ccp/exliquid-compose.yml b/ccp/exliquid-compose.yml index eb8d194..d5bb351 100644 --- a/ccp/exliquid-compose.yml +++ b/ccp/exliquid-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: exliquid-task-store: - image: "samply/blaze:0.19" + image: docker.verbis.dkfz.de/cache/samply/blaze:0.19 container_name: bridgehead-exliquid-task-store environment: BASE_URL: "http://bridgehead-exliquid-task-store:8080" @@ -13,7 +13,7 @@ services: - "traefik.enable=false" exliquid-report-hub: - image: "samply/report-hub:latest" + image: docker.verbis.dkfz.de/cache/samply/report-hub:latest container_name: bridgehead-exliquid-report-hub environment: SPRING_WEBFLUX_BASE_PATH: "/exliquid" diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index cb0c89a..be1375e 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -42,7 +42,7 @@ services: - patientlist-db patientlist-db: - image: postgres:15.1-alpine + image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine container_name: bridgehead-patientlist-db environment: POSTGRES_USER: "mainzelliste" diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index bb90059..01cff12 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: mtba: - image: samply/mtba:develop + image: docker.verbis.dkfz.de/cache/samply/mtba:develop container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 diff --git a/ccp/nngm-compose.yml b/ccp/nngm-compose.yml index c212fed..bd189fb 100644 --- a/ccp/nngm-compose.yml +++ b/ccp/nngm-compose.yml @@ -18,7 +18,7 @@ services: - "traefik.http.routers.connector.tls=true" connector_db: - image: postgres:9.5-alpine + image: docker.verbis.dkfz.de/cache/postgres:9.5-alpine container_name: bridgehead-ccp-connector-db volumes: - "connector_db_data:/var/lib/postgresql/data" diff --git a/lib/functions.sh b/lib/functions.sh index 836ffcc..fb93e87 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -11,7 +11,7 @@ detectCompose() { getLdmPassword() { if [ -n "$LDM_PASSWORD" ]; then - docker run --rm httpd:alpine htpasswd -nb $PROJECT $LDM_PASSWORD | tr -d '\n' | tr -d '\r' + docker run --rm docker.verbis.dkfz.de/cache/httpd:alpine htpasswd -nb $PROJECT $LDM_PASSWORD | tr -d '\n' | tr -d '\r' else echo -n "" fi From 25081c1bf48b53d097579468642a079afffd4146 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 9 Mar 2023 14:56:45 +0100 Subject: [PATCH 52/52] hotfix: Switch to old Project Name --- bridgehead | 6 ++++-- lib/functions.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bridgehead b/bridgehead index f2eaa6b..13cb682 100755 --- a/bridgehead +++ b/bridgehead @@ -66,11 +66,13 @@ case "$ACTION" in checkRequirements hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..." export LDM_LOGIN=$(getLdmPassword) - exec $COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit + exec $COMPOSE -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit ;; stop) loadVars - exec $COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE down + # HACK: This is tempoarily to properly shut down false bridgehead instances (bridgehead-ccp instead ccp) + $COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE down + exec $COMPOSE -f ./$PROJECT/docker-compose.yml $OVERRIDE down ;; is-running) bk_is_running diff --git a/lib/functions.sh b/lib/functions.sh index fb93e87..ac5ae6b 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -171,7 +171,7 @@ function retry { function bk_is_running { detectCompose - RUNNING="$($COMPOSE -p bridgehead-$PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE ps -q)" + RUNNING="$($COMPOSE -p $PROJECT -f ./$PROJECT/docker-compose.yml $OVERRIDE ps -q)" NUMBEROFRUNNING=$(echo "$RUNNING" | wc -l) if [ $NUMBEROFRUNNING -ge 2 ]; then return 0