diff --git a/README.md b/README.md index 24ebdf2..8fba0c9 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,6 @@ All of the Bridgehead's outgoing connections are secured by transport encryption - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. -- `/var/data/bridgehead` contains persistent data of the bridgehead - - `/var/data/bridgehead/backups` contains automatically created backups of the databases. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. @@ -148,6 +146,8 @@ Some of the components in the bridgehead will store persistent data. For those c 2) Year-KW-XX, were XX represents the calendar week to allow re-import of at least one version per calendar week 3) Year-Month, to allow re-import of at least one version per month +To enable the Auto-Backup feature, please set the Variable `BACKUP_DIRECTORY` in your sites configuration. + ### Monitoring To keep all Bridgeheads up and working and detect any errors before a user does, a central monitoring diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 4baaba7..3f13519 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -31,30 +31,7 @@ services: - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} # Add Variables from /etc/patientlist-id-generators.env - - ML_BK_IDGENERATOR_RANDOM_1 - - ML_BK_IDGENERATOR_RANDOM_2 - - ML_BK_IDGENERATOR_RANDOM_3 - - ML_MDS_IDGENERATOR_RANDOM_1 - - ML_MDS_IDGENERATOR_RANDOM_2 - - ML_MDS_IDGENERATOR_RANDOM_3 - - ML_DKTK000001985_IDGENERATOR_RANDOM_1 - - ML_DKTK000001985_IDGENERATOR_RANDOM_2 - - ML_DKTK000001985_IDGENERATOR_RANDOM_3 - - ML_DKTK000001986_IDGENERATOR_RANDOM_1 - - ML_DKTK000001986_IDGENERATOR_RANDOM_2 - - ML_DKTK000001986_IDGENERATOR_RANDOM_3 - - ML_DKTK000001950_IDGENERATOR_RANDOM_1 - - ML_DKTK000001950_IDGENERATOR_RANDOM_2 - - ML_DKTK000001950_IDGENERATOR_RANDOM_3 - - ML_DKTK000001951_IDGENERATOR_RANDOM_1 - - ML_DKTK000001951_IDGENERATOR_RANDOM_2 - - ML_DKTK000001951_IDGENERATOR_RANDOM_3 - - ML_DKTK999999999_IDGENERATOR_RANDOM_1 - - ML_DKTK999999999_IDGENERATOR_RANDOM_2 - - ML_DKTK999999999_IDGENERATOR_RANDOM_3 - - ML_DKTK000002089_IDGENERATOR_RANDOM_1 - - ML_DKTK000002089_IDGENERATOR_RANDOM_2 - - ML_DKTK000002089_IDGENERATOR_RANDOM_3 + - PATIENTLIST_SEEDS_TRANSFORMED labels: - "traefik.enable=true" - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" @@ -71,6 +48,9 @@ services: POSTGRES_DB: "mainzelliste" POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} volumes: - - "/var/data/bridgehead/patientlist:/var/lib/postgresql/data" - # NOTE: Add backups here. This is only imported if /var/data/bridgehead/patientlist/ is empty!!! + - "patientlist-db-data:/var/lib/postgresql/data" + # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!! - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/" + +volumes: + patientlist-db-data: diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index d2449c7..0f361dc 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -6,11 +6,11 @@ function idManagementSetup() { OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" # Auto Generate local Passwords - PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" - # Source the ID Generators Configuration - source /etc/bridgehead/patientlist-id-generators.env + # Transform Seeds Configuration to pass it to the Mainzelliste Container + PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')" # Ensure old ids are working !!! legacyIdMapping diff --git a/ccp/modules/id-management.md b/ccp/modules/id-management.md index 60cb5b1..66f9f71 100644 --- a/ccp/modules/id-management.md +++ b/ccp/modules/id-management.md @@ -16,17 +16,19 @@ IDMANAGER_SEEDS_BK="" IDMANAGER_SEEDS_MDS="" IDMANAGER_SEEDS_DKTK000001985="" ``` +> NOTE: Additionally, the CCP-IT adds lines declaring the `PATIENTLIST_SEEDS` array in your site configuration. This will contain the seeds for the different id-generators used in all projects. Once your Bridgehead is updated and restarted, you're all set! ## Additional information you may want to know + ### Services Upon configuration, the Bridgehead will spawn the following services: - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead. - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects. -- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/data/bridgehead/patientlist` and backups are automatically created in `/var/data/bridgehead/backups/bridgehead-patientlist-db`. +- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data`. ### How to import an existing database (e.g from Legacy Windows or from Backups) First you must shutdown your local bridgehead instance: @@ -36,10 +38,10 @@ systemctl stop bridgehead@ccp Next you need to remove the current patientlist database: ``` -rm -rf /var/data/bridgehead/patientlist +docker volume rm patientlist-db-data; ``` -Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/data/bridgehead/patientlist is empty. +Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then the volume `patientlist-db-data` was removed previously. > NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL). After this, you can restart your bridgehead and the dump will be imported: diff --git a/ccp/mtba-compose.yml b/ccp/mtba-compose.yml index d492ae0..857ff34 100644 --- a/ccp/mtba-compose.yml +++ b/ccp/mtba-compose.yml @@ -24,10 +24,13 @@ services: - "traefik.http.routers.mtba.tls=true" volumes: # This directory persists the FHIR Resources that are needed to import data into blaze. - - /var/data/bridgehead/mtba:/app/mtba-files/persist + - mtba-data:/app/mtba-files/persist # Place new import files in this directory - /tmp/bridgehead/mtba/:/app/mtba-files/input # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! # TODO: Find a trigger to let mtba signal a restart for CBioPortal + +volumes: + mtba-data: diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index 7cbd8ef..c42119f 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@*.service, \\ - /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\ - /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF @@ -37,10 +37,6 @@ if [ -z "$LDM_PASSWORD" ]; then echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; fi -log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data." -mkdir -p /var/data/bridgehead -chown -R bridgehead /var/data/bridgehead - log "INFO" "Registering system units for bridgehead and bridgehead-update" cp -v \ lib/systemd/bridgehead\@.service \ diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index cc1d55f..ab39cca 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,45 +1,6 @@ #!/bin/bash source lib/functions.sh -AUTO_BACKUP=${AUTO_BACKUP:-true} - -if [ "$AUTO_BACKUP" == "true" ]; then - BACKUP_DIRECTORY="/var/data/bridgehead/backups" - if [ ! -d /var/data ]; then - log DEBUG "Created /var/data" - mkdir /var/data - fi - if [ ! -d /var/data/bridgehead ]; then - log DEBUG "Created /var/data/bridgehead" - mkdir /var/data/bridgehead - fi - if [ ! -d $BACKUP_DIRECTORY ]; then - message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - mkdir -p $BACKUP_DIRECTORY - fi - BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")" - log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; - for service in $BACKUP_SERVICES; do - if [ ! -d $BACKUP_DIRECTORY/$service ]; then - message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - mkdir -p $BACKUP_DIRECTORY/$service - fi - if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then - message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY." - hc_send log "$message" - log INFO "$message" - else - fail_and_report 5 "Failed to create encrypted update for $service" - fi - done -else - log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" -fi - AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} if [ "$AUTO_HOUSEKEEPING" == "true" ]; then @@ -142,6 +103,37 @@ else hc_send log "$RES" fi +if [ -z "${BACKUP_DIRECTORY}" ]; then + if [ ! -d "$BACKUP_DIRECTORY" ]; then + message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p "$BACKUP_DIRECTORY" + chown -R "$BACKUP_DIRECTORY" bridgehead; + fi + checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)" + # Collect all container names that contain '-db' + BACKUP_SERVICES="$(docker ps --filter name=-db --format "{{.Names}}" | tr "\n" "\ ")" + log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES"; + for service in $BACKUP_SERVICES; do + if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then + message="Performing automatic maintenance: Attempting to create backup directory for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + mkdir -p "$BACKUP_DIRECTORY/$service" + fi + if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then + message="Performing automatic maintenance: Stored encrypted backup for $service in $BACKUP_DIRECTORY." + hc_send log "$message" + log INFO "$message" + else + fail_and_report 5 "Failed to create encrypted update for $service" + fi + done +else + log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")" +fi + exit 0 # TODO: Print last commit explicit