feature: Added automated Backups for PostgreSQL

This commit is contained in:
Torben Brenner 2022-12-13 16:51:32 +01:00
parent 99c0e7f283
commit 5e063003ac
5 changed files with 72 additions and 8 deletions

View File

@ -128,6 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption
- `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access). - `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access).
- `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam). - `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam).
- `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here. - `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here.
- `/var/data/bridgehead` contains persistent data of the bridgehead
- `/var/data/bridgehead/backups` contains automatically created backups of the databases.
Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect <volume_name>`. Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect <volume_name>`.
@ -139,6 +141,13 @@ Your Bridgehead will automatically and regularly check for updates. Whenever som
If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@<PROJECT>.service` and `systemctl cat bridgehead-update@<PROJECT.timer`. If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@<PROJECT>.service` and `systemctl cat bridgehead-update@<PROJECT.timer`.
### Auto-Backups
Some of the components in the bridgehead will store persistent data. For those components, we integrated an automated backup solution in the bridgehead updates. It will automatically save the backup in multiple files
1) Last-XX, were XX represents a weekday to allow re-import of at least one version of the database for each of the past seven days.
2) Year-KW-XX, were XX represents the calendar week to allow re-import of at least one version per calendar week
3) Year-Month, to allow re-import of at least one version per month
### Monitoring ### Monitoring
To keep all Bridgeheads up and working and detect any errors before a user does, a central monitoring To keep all Bridgeheads up and working and detect any errors before a user does, a central monitoring

View File

@ -2,6 +2,7 @@ version: "3.7"
services: services:
id-manager: id-manager:
image: docker.verbis.dkfz.de/bridgehead/magicpl image: docker.verbis.dkfz.de/bridgehead/magicpl
container_name: bridgehead-id-manager
environment: environment:
TOMCAT_REVERSEPROXY_FQDN: ${HOST} TOMCAT_REVERSEPROXY_FQDN: ${HOST}
MAGICPL_SITE: ${SITE_ID} MAGICPL_SITE: ${SITE_ID}
@ -23,6 +24,7 @@ services:
patientlist: patientlist:
image: docker.verbis.dkfz.de/bridgehead/mainzelliste image: docker.verbis.dkfz.de/bridgehead/mainzelliste
container_name: bridgehead-patientlist
environment: environment:
- TOMCAT_REVERSEPROXY_FQDN=${HOST} - TOMCAT_REVERSEPROXY_FQDN=${HOST}
- ML_SITE=${SITE_ID} - ML_SITE=${SITE_ID}
@ -63,13 +65,12 @@ services:
patientlist-db: patientlist-db:
image: postgres:14-alpine image: postgres:14-alpine
container_name: bridgehead-patientlist-db
environment: environment:
POSTGRES_USER: "mainzelliste" POSTGRES_USER: "mainzelliste"
POSTGRES_DB: "mainzelliste" POSTGRES_DB: "mainzelliste"
POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
volumes: volumes:
- "patientlist-db-data:/var/lib/postgresql/data" - "/var/data/bridgehead/patientlist:/var/lib/postgresql/data"
# NOTE: Add backups here. This is only imported if /var/data/bridgehead/patientlist/ is empty!!!
volumes: - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/"
patientlist-db-data:

View File

@ -136,6 +136,17 @@ setHostname() {
fi fi
} }
# Takes 1) The Backup Directory Path 2) The name of the Service to be backuped
# Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week
createEncryptedPostgresBackup(){
docker exec "$2" bash -c 'pg_dump -U $POSTGRES_USER $POSTGRES_DB --format=p --no-owner --no-privileges' | \
# TODO: Encrypt using /etc/bridgehead/pki/${SITE_ID}.priv.pem | \
tee "$1/$2/$(date +Last-%A).sql" | \
tee "$1/$2/$(date +%Y-%m).sql" > \
"$1/$2/$(date +%Y-KW%V).sql"
}
# from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 # from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
# ex. use: retry 5 /bin/false # ex. use: retry 5 /bin/false
function retry { function retry {

View File

@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\
/bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\
/bin/systemctl restart bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\
/bin/systemctl restart bridgehead@*.service, \\ /bin/systemctl restart bridgehead@*.service, \\
/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\
/usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead
bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^}
EOF EOF
@ -37,6 +37,10 @@ if [ -z "$LDM_PASSWORD" ]; then
echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf; echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf;
fi fi
log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data."
mkdir -p /var/data/bridgehead
chown -R bridgehead /var/data/bridgehead
log "INFO" "Registering system units for bridgehead and bridgehead-update" log "INFO" "Registering system units for bridgehead and bridgehead-update"
cp -v \ cp -v \
lib/systemd/bridgehead\@.service \ lib/systemd/bridgehead\@.service \
@ -63,4 +67,4 @@ else
STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR" STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR"
fi fi
log "INFO" "$STR" log "INFO" "$STR"

View File

@ -1,6 +1,45 @@
#!/bin/bash #!/bin/bash
source lib/functions.sh source lib/functions.sh
AUTO_BACKUP=${AUTO_BACKUP:-true}
if [ "$AUTO_BACKUP" == "true" ]; then
BACKUP_DIRECTORY="/var/data/bridgehead/backups"
if [ ! -d /var/data ]; then
log DEBUG "Created /var/data"
mkdir /var/data
fi
if [ ! -d /var/data/bridgehead ]; then
log DEBUG "Created /var/data/bridgehead"
mkdir /var/data/bridgehead
fi
if [ ! -d $BACKUP_DIRECTORY ]; then
message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY."
hc_send log "$message"
log INFO "$message"
mkdir -p $BACKUP_DIRECTORY
fi
BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")"
log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES";
for service in $BACKUP_SERVICES; do
if [ ! -d $BACKUP_DIRECTORY/$service ]; then
message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY."
hc_send log "$message"
log INFO "$message"
mkdir -p $BACKUP_DIRECTORY/$service
fi
if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then
message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY."
hc_send log "$message"
log INFO "$message"
else
fail_and_report 5 "Failed to create encrypted update for $service"
fi
done
else
log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")"
fi
AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true} AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true}
if [ "$AUTO_HOUSEKEEPING" == "true" ]; then if [ "$AUTO_HOUSEKEEPING" == "true" ]; then