2021-12-22 13:18:41 +01:00
|
|
|
#!/bin/bash
|
2022-01-10 16:31:48 +01:00
|
|
|
source lib/functions.sh
|
2021-12-22 13:18:41 +01:00
|
|
|
|
2022-10-20 16:55:05 +02:00
|
|
|
AUTO_HOUSEKEEPING=${AUTO_HOUSEKEEPING:-true}
|
|
|
|
|
|
|
|
if [ "$AUTO_HOUSEKEEPING" == "true" ]; then
|
2023-03-08 10:37:37 +01:00
|
|
|
A="Performing automatic maintenance: "
|
|
|
|
if bk_is_running; then
|
|
|
|
A="$A Cleaning docker images."
|
|
|
|
docker system prune -a -f
|
|
|
|
else
|
|
|
|
A="$A Not cleaning docker images since BK is not running."
|
|
|
|
fi
|
2022-10-20 16:55:05 +02:00
|
|
|
hc_send log "$A"
|
|
|
|
log INFO "$A"
|
|
|
|
else
|
|
|
|
log WARN "Automatic housekeeping disabled (variable AUTO_HOUSEKEEPING != \"true\")"
|
|
|
|
fi
|
|
|
|
|
2022-10-06 12:05:36 +02:00
|
|
|
hc_send log "Checking for bridgehead updates ..."
|
2022-10-06 10:45:50 +02:00
|
|
|
|
2022-05-31 13:56:44 +02:00
|
|
|
CONFFILE=/etc/bridgehead/$1.conf
|
|
|
|
|
|
|
|
if [ ! -e $CONFFILE ]; then
|
2022-10-06 10:45:50 +02:00
|
|
|
fail_and_report 1 "Configuration file $CONFFILE not found."
|
2022-05-31 13:56:44 +02:00
|
|
|
fi
|
|
|
|
|
|
|
|
source $CONFFILE
|
|
|
|
|
2022-10-06 10:45:50 +02:00
|
|
|
assertVarsNotEmpty SITE_ID || fail_and_report 1 "Update failed: SITE_ID empty"
|
2022-05-31 13:56:44 +02:00
|
|
|
export SITE_ID
|
|
|
|
|
2022-10-06 10:45:50 +02:00
|
|
|
checkOwner . bridgehead || fail_and_report 1 "Update failed: Wrong permissions in $(pwd)"
|
|
|
|
checkOwner /etc/bridgehead bridgehead || fail_and_report 1 "Update failed: Wrong permissions in /etc/bridgehead"
|
2022-05-31 13:56:44 +02:00
|
|
|
|
|
|
|
CREDHELPER="/srv/docker/bridgehead/lib/gitpassword.sh"
|
2022-01-11 14:40:16 +01:00
|
|
|
|
2022-10-17 16:07:25 +02:00
|
|
|
CHANGES=""
|
|
|
|
|
2022-05-11 10:30:18 +02:00
|
|
|
# Check git updates
|
2022-10-17 16:11:34 +02:00
|
|
|
git_updated="false"
|
2022-05-11 10:30:18 +02:00
|
|
|
for DIR in /etc/bridgehead $(pwd); do
|
2022-05-31 13:56:44 +02:00
|
|
|
log "INFO" "Checking for updates to git repo $DIR ..."
|
2022-11-04 16:12:08 +01:00
|
|
|
OUT="$(git -C $DIR status --porcelain)"
|
|
|
|
if [ -n "$OUT" ]; then
|
2022-11-04 17:24:23 +01:00
|
|
|
report_error log "The working directory $DIR is modified. Changed files: $OUT"
|
2022-11-04 16:12:08 +01:00
|
|
|
fi
|
2022-05-31 13:56:44 +02:00
|
|
|
if [ "$(git -C $DIR config --get credential.helper)" != "$CREDHELPER" ]; then
|
|
|
|
log "INFO" "Configuring repo to use bridgehead git credential helper."
|
|
|
|
git -C $DIR config credential.helper "$CREDHELPER"
|
|
|
|
fi
|
2022-05-11 10:30:18 +02:00
|
|
|
old_git_hash="$(git -C $DIR rev-parse --verify HEAD)"
|
2022-09-30 11:49:18 +02:00
|
|
|
if [ -z "$HTTP_PROXY_URL" ]; then
|
|
|
|
log "INFO" "Git is using no proxy!"
|
2022-11-29 08:36:05 +01:00
|
|
|
OUT=$(retry 5 git -C $DIR fetch 2>&1 && retry 5 git -C $DIR pull 2>&1)
|
2022-09-30 11:49:18 +02:00
|
|
|
else
|
|
|
|
log "INFO" "Git is using proxy ${HTTP_PROXY_URL} from ${CONFFILE}"
|
2022-11-29 08:36:05 +01:00
|
|
|
OUT=$(retry 5 git -c http.proxy=$HTTP_PROXY_URL -c https.proxy=$HTTPS_PROXY_URL -C $DIR fetch 2>&1 && retry 5 git -c http.proxy=$HTTP_PROXY_URL -c https.proxy=$HTTPS_PROXY_URL -C $DIR pull 2>&1)
|
2022-11-04 15:26:27 +01:00
|
|
|
fi
|
|
|
|
if [ $? -ne 0 ]; then
|
2022-11-04 17:24:23 +01:00
|
|
|
report_error log "Unable to update git $DIR: $OUT"
|
2022-09-30 13:42:41 +02:00
|
|
|
fi
|
2022-11-04 15:54:08 +01:00
|
|
|
|
2022-09-30 13:42:41 +02:00
|
|
|
new_git_hash="$(git -C $DIR rev-parse --verify HEAD)"
|
2022-05-11 10:30:18 +02:00
|
|
|
if [ "$old_git_hash" != "$new_git_hash" ]; then
|
2022-10-17 16:07:25 +02:00
|
|
|
CHANGE="Updated git repository in ${DIR} from commit $old_git_hash to $new_git_hash"
|
2022-10-17 16:11:34 +02:00
|
|
|
CHANGES+="- $CHANGE\n"
|
2022-10-17 16:07:25 +02:00
|
|
|
log "INFO" "$CHANGE"
|
2022-05-16 11:02:51 +02:00
|
|
|
# NOTE: Link generation doesn't work on repositories placed at an self-hosted instance of bitbucket.
|
|
|
|
# See: https://community.atlassian.com/t5/Bitbucket-questions/BitBucket-4-14-diff-between-any-two-commits/qaq-p/632974
|
2022-05-19 12:11:37 +02:00
|
|
|
git_repository_url="$(git -C $DIR remote get-url origin)"
|
|
|
|
git_repository_url=${git_repository_url/.git/}
|
|
|
|
if [ "$( echo $git_repository_url | grep "github.com")" ]; then
|
|
|
|
# Ensure web link even if ssh is used
|
|
|
|
git_repository_url="${git_repository_url/git@github.com/https:\/\/github.com\/}"
|
|
|
|
log "INFO" "You can review all changes on the repository with $git_repository_url/compare/$old_git_hash...$new_git_hash"
|
|
|
|
elif [ "$( echo $git_repository_url | grep "git.verbis.dkfz.de")" ]; then
|
|
|
|
git_repository_url="${git_repository_url/ssh:\/\/git@git.verbis.dkfz.de/https:\/\/git.verbis.dkfz.de\/}"
|
|
|
|
git_repository_url="https://$(echo $git_repository_url | awk -F '@' '{print $2}')"
|
|
|
|
log "INFO" "You can review all changes on the repository with $git_repository_url/compare?from=$old_git_hash&to=$new_git_hash"
|
2022-05-16 11:02:51 +02:00
|
|
|
fi
|
2022-05-11 10:30:18 +02:00
|
|
|
git_updated="true"
|
2023-06-30 10:51:33 +02:00
|
|
|
git -C "$DIR" checkout main
|
|
|
|
REPORT_BRANCH_NAME=$(git -C "$DIR" branch --show-current)
|
|
|
|
REPORT_STATUS_CHECK=$(git -C "$DIR" status --porcelain)
|
|
|
|
report_error 7 "MAINTENANCE: Switched Branch to main, you are now on branch $REPORT_BRANCH_NAME \n see status \n $REPORT_STATUS_CHECK"
|
2022-05-11 10:30:18 +02:00
|
|
|
fi
|
|
|
|
done
|
2022-01-11 14:40:16 +01:00
|
|
|
|
2022-05-11 10:30:18 +02:00
|
|
|
# Check docker updates
|
2022-05-31 13:56:44 +02:00
|
|
|
log "INFO" "Checking for updates to running docker images ..."
|
2021-12-22 13:18:41 +01:00
|
|
|
docker_updated="false"
|
2023-02-09 09:58:51 +01:00
|
|
|
for IMAGE in $(cat $PROJECT/docker-compose.yml ${OVERRIDE//-f/} | grep -v "^#" | grep "image:" | sed -e 's_^.*image: \(.*\).*$_\1_g; s_\"__g'); do
|
2022-05-11 10:30:18 +02:00
|
|
|
log "INFO" "Checking for Updates of Image: $IMAGE"
|
|
|
|
if docker pull $IMAGE | grep "Downloaded newer image"; then
|
2022-10-17 16:07:25 +02:00
|
|
|
CHANGE="Image $IMAGE updated."
|
2022-10-17 16:11:34 +02:00
|
|
|
CHANGES+="- $CHANGE\n"
|
2022-10-17 16:07:25 +02:00
|
|
|
log "INFO" "$CHANGE"
|
2021-12-22 13:18:41 +01:00
|
|
|
docker_updated="true"
|
|
|
|
fi
|
|
|
|
done
|
2022-05-11 10:30:18 +02:00
|
|
|
|
|
|
|
# If anything is updated, restart service
|
2021-12-22 13:18:41 +01:00
|
|
|
if [ $git_updated = "true" ] || [ $docker_updated = "true" ]; then
|
2022-10-17 16:07:25 +02:00
|
|
|
RES="Updates detected, now restarting bridgehead:\n$CHANGES"
|
2022-10-06 10:45:50 +02:00
|
|
|
log "INFO" "$RES"
|
|
|
|
hc_send log "$RES"
|
2022-09-30 15:35:37 +02:00
|
|
|
sudo /bin/systemctl restart bridgehead@*.service
|
2022-05-31 13:56:44 +02:00
|
|
|
else
|
2022-10-06 10:45:50 +02:00
|
|
|
RES="Nothing updated, nothing to restart."
|
|
|
|
log "INFO" "$RES"
|
|
|
|
hc_send log "$RES"
|
2021-12-22 13:18:41 +01:00
|
|
|
fi
|
2022-05-31 13:56:44 +02:00
|
|
|
|
2023-02-09 09:50:30 +01:00
|
|
|
if [ -n "${BACKUP_DIRECTORY}" ]; then
|
2023-01-27 11:26:31 +01:00
|
|
|
if [ ! -d "$BACKUP_DIRECTORY" ]; then
|
|
|
|
message="Performing automatic maintenance: Attempting to create backup directory $BACKUP_DIRECTORY."
|
2023-01-27 11:15:09 +01:00
|
|
|
hc_send log "$message"
|
|
|
|
log INFO "$message"
|
2023-01-27 11:26:31 +01:00
|
|
|
mkdir -p "$BACKUP_DIRECTORY"
|
2023-01-27 11:31:36 +01:00
|
|
|
chown -R "$BACKUP_DIRECTORY" bridgehead;
|
2023-01-27 11:15:09 +01:00
|
|
|
fi
|
2023-01-27 11:31:36 +01:00
|
|
|
checkOwner "$BACKUP_DIRECTORY" bridgehead || fail_and_report 1 "Automatic maintenance failed: Wrong permissions for backup directory $(pwd)"
|
2023-01-27 11:49:18 +01:00
|
|
|
# Collect all container names that contain '-db'
|
|
|
|
BACKUP_SERVICES="$(docker ps --filter name=-db --format "{{.Names}}" | tr "\n" "\ ")"
|
2023-01-27 11:15:09 +01:00
|
|
|
log INFO "Performing automatic maintenance: Creating Backups for $BACKUP_SERVICES";
|
|
|
|
for service in $BACKUP_SERVICES; do
|
2023-01-27 11:26:31 +01:00
|
|
|
if [ ! -d "$BACKUP_DIRECTORY/$service" ]; then
|
|
|
|
message="Performing automatic maintenance: Attempting to create backup directory for $service in $BACKUP_DIRECTORY."
|
2023-01-27 11:15:09 +01:00
|
|
|
hc_send log "$message"
|
|
|
|
log INFO "$message"
|
2023-01-27 11:26:31 +01:00
|
|
|
mkdir -p "$BACKUP_DIRECTORY/$service"
|
2023-01-27 11:15:09 +01:00
|
|
|
fi
|
|
|
|
if createEncryptedPostgresBackup "$BACKUP_DIRECTORY" "$service"; then
|
2023-01-27 11:26:31 +01:00
|
|
|
message="Performing automatic maintenance: Stored encrypted backup for $service in $BACKUP_DIRECTORY."
|
2023-01-27 11:15:09 +01:00
|
|
|
hc_send log "$message"
|
|
|
|
log INFO "$message"
|
|
|
|
else
|
|
|
|
fail_and_report 5 "Failed to create encrypted update for $service"
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
else
|
|
|
|
log WARN "Automated backups are disabled (variable AUTO_BACKUPS != \"true\")"
|
|
|
|
fi
|
|
|
|
|
2022-05-16 11:02:51 +02:00
|
|
|
exit 0
|
|
|
|
|
|
|
|
# TODO: Print last commit explicit
|