diff --git a/README.md b/README.md index b57c10e..8fba0c9 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,15 @@ Your Bridgehead will automatically and regularly check for updates. Whenever som If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@.service` and `systemctl cat bridgehead-update@ 'dktk test' +function transformToSingleStringArray() { + echo "${1//-/ }"; +} + +# Ensure all Words are Uppercase +# Usage: transformToUppercase 'dktk test' -> 'Dktk Test' +function transformToUppercase() { + result=""; + for word in $1; do + result+=" ${word^}"; + done + echo "$result"; +} + +# Handle all execeptions from the norm (e.g LMU, TUM) +# Usage: applySpecialCases 'Muenchen Lmu Test' -> 'Muenchen LMU Test' +function applySpecialCases() { + result="$1"; + result="${result/Lmu/LMU}"; + result="${result/Tum/TUM}"; + echo "$result"; +} + +# Transform current siteids to legacy version +# Usage: legacyIdMapping "dktk-test" -> "DktkTest" +function legacyIdMapping() { + single_string_array=$(transformToSingleStringArray "$1"); + uppercase_string=$(transformToUppercase "$single_string_array"); + normalized_string=$(applySpecialCases "$uppercase_string"); + echo "$normalized_string" | tr -d ' ' +} diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md new file mode 100644 index 0000000..66f9f71 --- /dev/null +++ b/ccp/modules/id-management.md @@ -0,0 +1,66 @@ +# Module: Id-Management +This module provides integration with the CCP-Pseudonymiziation Service. To learn more on the backgrounds of this service, you can refer to the [CCP Data Protection Concept](https://dktk.dkfz.de/klinische-plattformen/documents-download). + +## Getting Started +The following configuration variables are added to your sites-configuration repository: + +``` +IDMANAGER_UPLOAD_APIKEY="" +IDMANAGER_READ_APIKEY="" +IDMANAGER_CENTRAL_PATIENTLIST_APIKEY="" +IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY="" +IDMANAGER_AUTH_CLIENT_ID="" +IDMANAGER_AUTH_CLIENT_SECRET="" + +IDMANAGER_SEEDS_BK="" +IDMANAGER_SEEDS_MDS="" +IDMANAGER_SEEDS_DKTK000001985="" +``` +> NOTE: Additionally, the CCP-IT adds lines declaring the `PATIENTLIST_SEEDS` array in your site configuration. This will contain the seeds for the different id-generators used in all projects. + +Once your Bridgehead is updated and restarted, you're all set! + +## Additional information you may want to know + +### Services + +Upon configuration, the Bridgehead will spawn the following services: + +- The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. +- The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data`. + +### How to import an existing database (e.g from Legacy Windows or from Backups) +First you must shutdown your local bridgehead instance: +``` +systemctl stop bridgehead@ccp +``` + +Next you need to remove the current patientlist database: +``` +docker volume rm patientlist-db-data; +``` + +Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then the volume `patientlist-db-data` was removed previously. +> NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). + +After this, you can restart your bridgehead and the dump will be imported: +``` +systemctl start bridgehead@ccp +``` + +### How to connect your local data-management +Typically, the sites connect their local data-management for the pseudonym creation with the id-management in the bridgehead. In the following two sections, you can read where you can change the configuration: +#### Sites using CentraXX +On your CentraXX Server, you need to change following settings in the "centraxx-dev.properties" file. +``` +dktk.idmanagement.url=https:///id-manager/translator/getId +dktk.idmanagement.apiKey= +``` +They typically already exist, but need to be changed to the new values! +#### Sites using ADT2FHIR +@Pierre + + +### How to connect the legacy windows bridgehead +You need to change the configuration file "..." of your Windows Bridgehead. TODO... diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml new file mode 100644 index 0000000..bb90059 --- /dev/null +++ b/ccp/mtba-compose.yml @@ -0,0 +1,36 @@ +version: "3.7" + +services: + mtba: + image: samply/mtba:develop + container_name: bridgehead-mtba + environment: + BLAZE_STORE_URL: http://blaze:8080 + # NOTE: Aktuell Berechtigungen wie MagicPL!!! + # TODO: Add separate ApiKey to MagicPL only for MTBA! + ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} + ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID + ID_MANAGER_URL: http://id-manager:8080/id-manager + PATIENT_CSV_FIRST_NAME_HEADER: ${MTBA_PATIENT_CSV_FIRST_NAME_HEADER} + PATIENT_CSV_LAST_NAME_HEADER: ${MTBA_PATIENT_CSV_LAST_NAME_HEADER} + PATIENT_CSV_GENDER_HEADER: ${MTBA_PATIENT_CSV_GENDER_HEADER} + PATIENT_CSV_BIRTHDAY_HEADER: ${MTBA_PATIENT_CSV_BIRTHDAY_HEADER} + CBIOPORTAL_URL: http://cbioportal:8080 + FILE_CHARSET: ${MTBA_FILE_CHARSET} + FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} + CSV_DELIMITER: ${MTBA_CSV_DELIMITER} + labels: + - "traefik.enable=true" + - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" + - "traefik.http.services.mtba.loadbalancer.server.port=80" + - "traefik.http.routers.mtba.tls=true" + volumes: + - /tmp/bridgehead/mtba/input:/app/input + - /tmp/bridgehead/mtba/persist:/app/persist + + # TODO: Include CBioPortal in Deployment ... + # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! + # TODO: Find a trigger to let mtba signal a restart for CBioPortal + +volumes: + mtba-data: diff --git a/ccp/nngm-setup.sh b/ccp/nngm-setup.sh index d5b80eb..ba2e77f 100644 --- a/ccp/nngm-setup.sh +++ b/ccp/nngm-setup.sh @@ -7,3 +7,15 @@ function nngmSetup() { fi CONNECTOR_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" } + +function mtbaSetup() { + # TODO: Check if ID-Management Module is activated! + if [ -n "$ENABLE_MTBA" ];then + log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." + if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + log ERROR "Detected MTBA Module configuration but ID-Management Module seems not to be configured!" + exit 1; + fi + OVERRIDE+=" -f ./$PROJECT/mtba-compose.yml" + fi +} diff --git a/ccp/vars b/ccp/vars index 63def80..5172d24 100644 --- a/ccp/vars +++ b/ccp/vars @@ -8,8 +8,12 @@ REPORTHUB_BEAM_SECRET_LONG="ApiKey report-hub.${PROXY_ID} ${REPORTHUB_BEAM_SECRE SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +# This will load id-management setup. Effective only if id-management configuration is defined. +source $PROJECT/modules/id-management-setup.sh +idManagementSetup # This will load nngm setup. Effective only if nngm configuration is defined. source $PROJECT/nngm-setup.sh nngmSetup source $PROJECT/exliquid-setup.sh exliquidSetup +mtbaSetup \ No newline at end of file diff --git a/lib/functions.sh b/lib/functions.sh index 4f40fd0..23fb939 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -131,11 +131,22 @@ fail_and_report() { setHostname() { if [ -z "$HOST" ]; then - export HOST=$(hostname -f) + export HOST=$(hostname -f | tr "[:upper:]" "[:lower:]") log DEBUG "Using auto-detected hostname $HOST." fi } +# Takes 1) The Backup Directory Path 2) The name of the Service to be backuped +# Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week +createEncryptedPostgresBackup(){ + docker exec "$2" bash -c 'pg_dump -U $POSTGRES_USER $POSTGRES_DB --format=p --no-owner --no-privileges' | \ + # TODO: Encrypt using /etc/bridgehead/pki/${SITE_ID}.priv.pem | \ + tee "$1/$2/$(date +Last-%A).sql" | \ + tee "$1/$2/$(date +%Y-%m).sql" > \ + "$1/$2/$(date +%Y-KW%V).sql" +} + + # from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 # ex. use: retry 5 /bin/false function retry { diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 5e3add3..c42119f 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -63,4 +63,4 @@ else STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR" fi -log "INFO" "$STR" \ No newline at end of file +log "INFO" "$STR" diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 897c8a2..c53b731 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -81,7 +81,7 @@ done # Check docker updates log "INFO" "Checking for updates to running docker images ..." docker_updated="false" -for IMAGE in $(cat $PROJECT/docker-compose.yml | grep -v "^#" | grep "image:" | sed -e 's_^.*image: \(.*\).*$_\1_g; s_\"__g'); do +for IMAGE in $(cat $PROJECT/docker-compose.yml ${OVERRIDE//-f/} | grep -v "^#" | grep "image:" | sed -e 's_^.*image: \(.*\).*$_\1_g; s_\"__g'); do log "INFO" "Checking for Updates of Image: $IMAGE" if docker pull $IMAGE | grep "Downloaded newer image"; then CHANGE="Image $IMAGE updated." @@ -103,6 +103,37 @@ else hc_send log "$RES" fi +if [ -n "${BACKUP_DIRECTORY}" ]; then + if [ ! -d "$BACKUP_DIRECTORY" ]; then + message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p "$BACKUP_DIRECTORY" + chown -R "$BACKUP_DIRECTORY" bridgehead; + fi + checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)" + # Collect all container names that contain '-db' + BACKUP_SERVICES="$(docker ps --filter name=-db --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then + message="Performing automatic maintenance: Attempting to create backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p "$BACKUP_DIRECTORY/$service" + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + exit 0 # TODO: Print last commit explicit