Merge pull request #51 from samply/refactor/directoryNames

refactor: Changed Paths for persistent Data

README.md

@@ -128,8 +128,6 @@ All of the Bridgehead's outgoing connections are secured by transport encryption
   - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access).
   - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam).
   - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here.
-- `/var/data/bridgehead` contains persistent data of the bridgehead
-  - `/var/data/bridgehead/backups` contains automatically created backups of the databases.
 
 Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect <volume_name>`.
 
@@ -148,6 +146,8 @@ Some of the components in the bridgehead will store persistent data. For those c
 2) Year-KW-XX, were XX represents the calendar week to allow re-import of at least one version per calendar week
 3) Year-Month, to allow re-import of at least one version per month
 
+To enable the Auto-Backup feature, please set the Variable `BACKUP_DIRECTORY` in your sites configuration.
+
 ### Monitoring
 
 To keep all Bridgeheads up and working and detect any errors before a user does, a central monitoring 

ccp/modules/id-management-compose.yml

@@ -31,30 +31,7 @@ services:
       - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD}
       - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
       # Add Variables from /etc/patientlist-id-generators.env
-      - ML_BK_IDGENERATOR_RANDOM_1
+      - PATIENTLIST_SEEDS_TRANSFORMED
-      - ML_BK_IDGENERATOR_RANDOM_2
-      - ML_BK_IDGENERATOR_RANDOM_3
-      - ML_MDS_IDGENERATOR_RANDOM_1
-      - ML_MDS_IDGENERATOR_RANDOM_2
-      - ML_MDS_IDGENERATOR_RANDOM_3
-      - ML_DKTK000001985_IDGENERATOR_RANDOM_1
-      - ML_DKTK000001985_IDGENERATOR_RANDOM_2
-      - ML_DKTK000001985_IDGENERATOR_RANDOM_3
-      - ML_DKTK000001986_IDGENERATOR_RANDOM_1
-      - ML_DKTK000001986_IDGENERATOR_RANDOM_2
-      - ML_DKTK000001986_IDGENERATOR_RANDOM_3
-      - ML_DKTK000001950_IDGENERATOR_RANDOM_1
-      - ML_DKTK000001950_IDGENERATOR_RANDOM_2
-      - ML_DKTK000001950_IDGENERATOR_RANDOM_3
-      - ML_DKTK000001951_IDGENERATOR_RANDOM_1
-      - ML_DKTK000001951_IDGENERATOR_RANDOM_2
-      - ML_DKTK000001951_IDGENERATOR_RANDOM_3
-      - ML_DKTK999999999_IDGENERATOR_RANDOM_1
-      - ML_DKTK999999999_IDGENERATOR_RANDOM_2
-      - ML_DKTK999999999_IDGENERATOR_RANDOM_3
-      - ML_DKTK000002089_IDGENERATOR_RANDOM_1
-      - ML_DKTK000002089_IDGENERATOR_RANDOM_2
-      - ML_DKTK000002089_IDGENERATOR_RANDOM_3
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)"
@@ -71,6 +48,9 @@ services:
       POSTGRES_DB: "mainzelliste"
       POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
     volumes:
-      - "/var/data/bridgehead/patientlist:/var/lib/postgresql/data"
+      - "patientlist-db-data:/var/lib/postgresql/data"
-      # NOTE: Add backups here. This is only imported if /var/data/bridgehead/patientlist/ is empty!!!
+      # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!!
       - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/"
+
+volumes:
+  patientlist-db-data:

ccp/modules/id-management-setup.sh

@@ -6,11 +6,11 @@ function idManagementSetup() {
 		OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml"
 
 		# Auto Generate local Passwords
-		PATIENTLIST_POSTGRES_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+		PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
 		IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
 
-		# Source the ID Generators Configuration
+		# Transform Seeds Configuration to pass it to the Mainzelliste Container
-		source /etc/bridgehead/patientlist-id-generators.env
+		PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')"
 
 		# Ensure old ids are working !!!
 		legacyIdMapping

ccp/modules/id-management.md

@@ -16,17 +16,19 @@ IDMANAGER_SEEDS_BK="<three-numbers>"
 IDMANAGER_SEEDS_MDS="<three-numbers>"
 IDMANAGER_SEEDS_DKTK000001985="<three-numbers>"
 ```
+> NOTE: Additionally, the CCP-IT adds lines declaring the `PATIENTLIST_SEEDS` array in your site configuration. This will contain the seeds for the different id-generators used in all projects.
 
 Once your Bridgehead is updated and restarted, you're all set!
 
 ## Additional information you may want to know
+
 ### Services
 
 Upon configuration, the Bridgehead will spawn the following services:
 
 - The `bridgehead-id-manager` at https://bridgehead.local/id-manager, provides a common interface for creating pseudonyms in the bridgehead.
 - The `bridgehead-patientlist` at https://bridgehead.local/patientlist is a local instance of the open-source software [Mainzelliste](https://mainzelliste.de). This service's primary task is to map patients IDAT to pseudonyms identifying them along the different CCP projects.
-- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted in `/var/data/bridgehead/patientlist` and backups are automatically created in `/var/data/bridgehead/backups/bridgehead-patientlist-db`.
+- The `bridgehead-patientlist-db` is only accessible within the Bridgehead itself. This is a local postgresql instance storing the database for `bridgehead-patientlist`. The data is persisted as a named volume `patientlist-db-data`.
 
 ### How to import an existing database (e.g from Legacy Windows or from Backups)
 First you must shutdown your local bridgehead instance:
@@ -36,10 +38,10 @@ systemctl stop bridgehead@ccp
 
 Next you need to remove the current patientlist database:
 ```
-rm -rf /var/data/bridgehead/patientlist
+docker volume rm patientlist-db-data;
 ```
 
-Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then /var/data/bridgehead/patientlist is empty. 
+Third, you need to place your postgres dump in the import directory `/tmp/bridgehead/patientlist/some-dump.sql`. This will only be imported, then the volume `patientlist-db-data` was removed previously. 
 > NOTE: Please create the postgres dump with the options "--no-owner" and "--no-privileges". Additionally ensure the dump is created in the plain format (SQL).
 
 After this, you can restart your bridgehead and the dump will be imported:

ccp/mtba-compose.yml

@@ -24,10 +24,13 @@ services:
       - "traefik.http.routers.mtba.tls=true"
     volumes:
       # This directory persists the FHIR Resources that are needed to import data into blaze.
-      - /var/data/bridgehead/mtba:/app/mtba-files/persist
+      - mtba-data:/app/mtba-files/persist
       # Place new import files in this directory
       - /tmp/bridgehead/mtba/:/app/mtba-files/input
 
   # TODO: Include CBioPortal in Deployment ...
   # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted!
   # TODO: Find a trigger to let mtba signal a restart for CBioPortal
+
+volumes:
+  mtba-data:

lib/install-bridgehead.sh

@@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\
     /bin/systemctl stop bridgehead@${PROJECT}.service, \\
     /bin/systemctl restart bridgehead@${PROJECT}.service, \\
     /bin/systemctl restart bridgehead@*.service, \\
-    /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\
+    /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\
-    /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead
+    /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead
 
 bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^}
 EOF
@@ -37,10 +37,6 @@ if [ -z "$LDM_PASSWORD" ]; then
   echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf;
 fi
 
-log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data."
-mkdir -p /var/data/bridgehead
-chown -R bridgehead /var/data/bridgehead
-
 log "INFO" "Registering system units for bridgehead and bridgehead-update"
 cp -v \
     lib/systemd/bridgehead\@.service \

lib/update-bridgehead.sh

@@ -1,45 +1,6 @@
 #!/bin/bash
 source lib/functions.sh
 
-AUTO_BACKUP=${AUTO_BACKUP:-true}
-
-if [ "$AUTO_BACKUP" == "true" ]; then
-  BACKUP_DIRECTORY="/var/data/bridgehead/backups"
-  if [ ! -d /var/data ]; then
-    log DEBUG "Created /var/data"
-    mkdir /var/data
-  fi
-  if [ ! -d /var/data/bridgehead ]; then
-    log DEBUG "Created /var/data/bridgehead"
-    mkdir /var/data/bridgehead
-  fi
-  if [ ! -d $BACKUP_DIRECTORY ]; then
-    message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY."
-    hc_send log "$message"
-    log INFO "$message"
-    mkdir -p $BACKUP_DIRECTORY
-  fi
-  BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")"
-  log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES";
-  for service in $BACKUP_SERVICES; do
-    if [ ! -d $BACKUP_DIRECTORY/$service ]; then
-      message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY."
-      hc_send log "$message"
-      log INFO "$message"
-      mkdir -p $BACKUP_DIRECTORY/$service
-    fi
-    if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then
-      message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY."
-      hc_send log "$message"
-      log INFO "$message"
-    else
-      fail_and_report 5 "Failed to create encrypted update for $service"
-    fi
-  done
-else
-  log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")"
-fi
-
 AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true}
 
 if [ "$AUTO_HOUSEKEEPING" == "true" ]; then
@@ -142,6 +103,37 @@ else
   hc_send log "$RES"
 fi
 
+if [ -z "${BACKUP_DIRECTORY}" ]; then
+  if [ ! -d "$BACKUP_DIRECTORY" ]; then
+    message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY."
+    hc_send log "$message"
+    log INFO "$message"
+    mkdir -p "$BACKUP_DIRECTORY"
+    chown -R "$BACKUP_DIRECTORY" bridgehead;
+  fi
+  checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)"
+  # Collect all container names that contain '-db'
+  BACKUP_SERVICES="$(docker ps --filter name=-db --format "{{.Names}}" | tr "\n" "\ ")"
+  log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES";
+  for service in $BACKUP_SERVICES; do
+    if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then
+      message="Performing automatic maintenance: Attempting to create backup directory for $service in $BACKUP_DIRECTORY."
+      hc_send log "$message"
+      log INFO "$message"
+      mkdir -p "$BACKUP_DIRECTORY/$service"
+    fi
+    if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then
+      message="Performing automatic maintenance: Stored encrypted backup for $service in $BACKUP_DIRECTORY."
+      hc_send log "$message"
+      log INFO "$message"
+    else
+      fail_and_report 5 "Failed to create encrypted update for $service"
+    fi
+  done
+else
+  log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")"
+fi
+
 exit 0
 
 # TODO: Print last commit explicit