feature: Added automated Backups for PostgreSQL
This commit is contained in:
Torben Brenner
p.delpy@dkfz-heidelberg.de
parent
6d24dbce7f
commit
1ffc9b9cd5
|
|
@ -128,6 +128,8 @@ All of the Bridgehead's outgoing connections are secured by transport encryption
|
|||
- `/etc/bridgehead/traefik-tls` contains your Bridgehead's reverse proxies TLS certificates for [HTTPS access](#https-access).
|
||||
- `/etc/bridgehead/pki` contains your Bridgehead's private key (e.g., but not limited to Samply.Beam), generated as part of the [Samply.Beam enrollment](#register-with-samplybeam).
|
||||
- `/etc/bridgehead/trusted-ca-certs` contains third-party certificates to be trusted by the Bridgehead. For example, you want to place the certificates of your [TLS-terminating proxy](#network) here.
|
||||
- `/var/data/bridgehead` contains persistent data of the bridgehead
|
||||
- `/var/data/bridgehead/backups` contains automatically created backups of the databases.
|
||||
|
||||
Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect <volume_name>`.
|
||||
|
||||
|
|
@ -139,6 +141,13 @@ Your Bridgehead will automatically and regularly check for updates. Whenever som
|
|||
|
||||
If you would like to understand what happens exactly and when, please check the systemd units deployed during the [installation](#base-installation) via `systemctl cat bridgehead-update@<PROJECT>.service` and `systemctl cat bridgehead-update@<PROJECT.timer`.
|
||||
|
||||
### Auto-Backups
|
||||
Some of the components in the bridgehead will store persistent data. For those components, we integrated an automated backup solution in the bridgehead updates. It will automatically save the backup in multiple files
|
||||
|
||||
1) Last-XX, were XX represents a weekday to allow re-import of at least one version of the database for each of the past seven days.
|
||||
2) Year-KW-XX, were XX represents the calendar week to allow re-import of at least one version per calendar week
|
||||
3) Year-Month, to allow re-import of at least one version per month
|
||||
|
||||
### Monitoring
|
||||
|
||||
To keep all Bridgeheads up and working and detect any errors before a user does, a central monitoring
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ version: "3.7"
|
|||
services:
|
||||
id-manager:
|
||||
image: docker.verbis.dkfz.de/bridgehead/magicpl
|
||||
container_name: bridgehead-id-manager
|
||||
environment:
|
||||
TOMCAT_REVERSEPROXY_FQDN: ${HOST}
|
||||
MAGICPL_SITE: ${SITE_ID}
|
||||
|
|
@ -23,6 +24,7 @@ services:
|
|||
|
||||
patientlist:
|
||||
image: docker.verbis.dkfz.de/bridgehead/mainzelliste
|
||||
container_name: bridgehead-patientlist
|
||||
environment:
|
||||
- TOMCAT_REVERSEPROXY_FQDN=${HOST}
|
||||
- ML_SITE=${SITE_ID}
|
||||
|
|
@ -63,13 +65,12 @@ services:
|
|||
|
||||
patientlist-db:
|
||||
image: postgres:14-alpine
|
||||
container_name: bridgehead-patientlist-db
|
||||
environment:
|
||||
POSTGRES_USER: "mainzelliste"
|
||||
POSTGRES_DB: "mainzelliste"
|
||||
POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- "patientlist-db-data:/var/lib/postgresql/data"
|
||||
|
||||
volumes:
|
||||
patientlist-db-data:
|
||||
|
||||
- "/var/data/bridgehead/patientlist:/var/lib/postgresql/data"
|
||||
# NOTE: Add backups here. This is only imported if /var/data/bridgehead/patientlist/ is empty!!!
|
||||
- "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/"
|
||||
|
|
|
|||
|
|
@ -136,6 +136,17 @@ setHostname() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Takes 1) The Backup Directory Path 2) The name of the Service to be backuped
|
||||
# Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week
|
||||
createEncryptedPostgresBackup(){
|
||||
docker exec "$2" bash -c 'pg_dump -U $POSTGRES_USER $POSTGRES_DB --format=p --no-owner --no-privileges' | \
|
||||
# TODO: Encrypt using /etc/bridgehead/pki/${SITE_ID}.priv.pem | \
|
||||
tee "$1/$2/$(date +Last-%A).sql" | \
|
||||
tee "$1/$2/$(date +%Y-%m).sql" > \
|
||||
"$1/$2/$(date +%Y-KW%V).sql"
|
||||
}
|
||||
|
||||
|
||||
# from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
|
||||
# ex. use: retry 5 /bin/false
|
||||
function retry {
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\
|
|||
/bin/systemctl stop bridgehead@${PROJECT}.service, \\
|
||||
/bin/systemctl restart bridgehead@${PROJECT}.service, \\
|
||||
/bin/systemctl restart bridgehead@*.service, \\
|
||||
/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\
|
||||
/usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead
|
||||
/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead, \\
|
||||
/usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead /var/data/bridgehead
|
||||
|
||||
bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^}
|
||||
EOF
|
||||
|
|
@ -37,6 +37,10 @@ if [ -z "$LDM_PASSWORD" ]; then
|
|||
echo -e "## Local Data Management Basic Authentication\n# User: $PROJECT\nLDM_PASSWORD=$generated_passwd" >> /etc/bridgehead/${PROJECT}.local.conf;
|
||||
fi
|
||||
|
||||
log "INFO" "Creating directory /var/data/bridgehead for storage of persistent data."
|
||||
mkdir -p /var/data/bridgehead
|
||||
chown -R bridgehead /var/data/bridgehead
|
||||
|
||||
log "INFO" "Registering system units for bridgehead and bridgehead-update"
|
||||
cp -v \
|
||||
lib/systemd/bridgehead\@.service \
|
||||
|
|
@ -63,4 +67,4 @@ else
|
|||
STR="Success. Next, enroll into the $PROJECT broker by creating a cryptographic certificate. To do so, run\n\n /srv/docker/bridgehead/bridgehead enroll $PROJECT\n\nThen, you may start the bridgehead by running$STR"
|
||||
fi
|
||||
|
||||
log "INFO" "$STR"
|
||||
log "INFO" "$STR"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,45 @@
|
|||
#!/bin/bash
|
||||
source lib/functions.sh
|
||||
|
||||
AUTO_BACKUP=${AUTO_BACKUP:-true}
|
||||
|
||||
if [ "$AUTO_BACKUP" == "true" ]; then
|
||||
BACKUP_DIRECTORY="/var/data/bridgehead/backups"
|
||||
if [ ! -d /var/data ]; then
|
||||
log DEBUG "Created /var/data"
|
||||
mkdir /var/data
|
||||
fi
|
||||
if [ ! -d /var/data/bridgehead ]; then
|
||||
log DEBUG "Created /var/data/bridgehead"
|
||||
mkdir /var/data/bridgehead
|
||||
fi
|
||||
if [ ! -d $BACKUP_DIRECTORY ]; then
|
||||
message="Performing automatic maintenance: Creating Backup directory $BACKUP_DIRECTORY."
|
||||
hc_send log "$message"
|
||||
log INFO "$message"
|
||||
mkdir -p $BACKUP_DIRECTORY
|
||||
fi
|
||||
BACKUP_SERVICES="$(docker ps --filter ancestor=postgres:14-alpine --format "{{.Names}}" | tr "\n" "\ ")"
|
||||
log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES";
|
||||
for service in $BACKUP_SERVICES; do
|
||||
if [ ! -d $BACKUP_DIRECTORY/$service ]; then
|
||||
message="Performing automatic maintenance: Creating Backup directory for $service in $BACKUP_DIRECTORY."
|
||||
hc_send log "$message"
|
||||
log INFO "$message"
|
||||
mkdir -p $BACKUP_DIRECTORY/$service
|
||||
fi
|
||||
if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then
|
||||
message="Performing automatic maintenance: Stored encrypted Backup for $service in $BACKUP_DIRECTORY."
|
||||
hc_send log "$message"
|
||||
log INFO "$message"
|
||||
else
|
||||
fail_and_report 5 "Failed to create encrypted update for $service"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")"
|
||||
fi
|
||||
|
||||
AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true}
|
||||
|
||||
if [ "$AUTO_HOUSEKEEPING" == "true" ]; then
|
||||
|
|
|
|||
Loading…
Reference in New Issue