From 8a6274389459c0c15a8dfcac852c2238c1081b07 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Thu, 6 Oct 2022 10:45:50 +0200 Subject: [PATCH] Monitoring for bridgehead startup and update (#22) --- bridgehead | 12 +++++--- lib/functions.sh | 23 +++++++++++---- lib/gitpassword.sh | 5 ++-- lib/log.sh | 5 ++++ lib/monitoring.sh | 41 ++++++++++++++++++++++++++ lib/prerequisites.sh | 16 ++++------ lib/setup-bridgehead-units.sh | 4 ++- lib/systemd/bridgehead-update@.service | 3 +- lib/systemd/bridgehead@.service | 3 +- lib/update-bridgehead.sh | 19 +++++++----- 10 files changed, 98 insertions(+), 33 deletions(-) create mode 100644 lib/log.sh create mode 100755 lib/monitoring.sh diff --git a/bridgehead b/bridgehead index eba542d..87d6a1f 100755 --- a/bridgehead +++ b/bridgehead @@ -43,14 +43,16 @@ esac # Load variables from /etc/bridgehead and /srv/docker/bridgehead set -a -source /etc/bridgehead/$PROJECT.conf -fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || exit 1 +source /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "/etc/bridgehead/$PROJECT.conf not found" +fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a case "$ACTION" in start) + hc_send log "Bridgehead $PROJECT startup: Checking requirements ..." checkRequirements + hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..." exec docker-compose -f ./$PROJECT/docker-compose.yml up --abort-on-container-exit ;; stop) @@ -65,8 +67,10 @@ case "$ACTION" in uninstall) exec ./lib/remove-bridgehead-units.sh $PROJECT ;; - fixPermissions) - chown -R bridgehead /etc/bridgehead . + preRun | preUpdate) + fixPermissions + ;; + postRun | postUpdate) ;; *) printUsage diff --git a/lib/functions.sh b/lib/functions.sh index 3d5a88f..ded0cd9 100755 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -1,9 +1,11 @@ #!/bin/bash -e +source lib/log.sh + exitIfNotRoot() { if [ "$EUID" -ne 0 ]; then log "ERROR" "Please run as root" - exit 1 + fail_and_report 1 "Please run as root" fi } @@ -16,10 +18,6 @@ checkOwner(){ return 0 } -log() { - echo -e "$(date +'%Y-%m-%d %T')" "$1:" "$2" -} - printUsage() { echo "Usage: bridgehead start|stop|update|install|uninstall PROJECTNAME" echo "PROJECTNAME should be one of ccp|nngm|gbn" @@ -28,7 +26,7 @@ printUsage() { checkRequirements() { if ! lib/prerequisites.sh; then log "ERROR" "Validating Prerequisites failed, please fix the error(s) above this line." - exit 1 + fail_and_report 1 "Validating prerequisites failed." else return 0 fi @@ -97,6 +95,19 @@ assertVarsNotEmpty() { return 0 } +fixPermissions() { + CHOWN=$(which chown) + sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead +} + +source lib/monitoring.sh + +fail_and_report() { + log ERROR "$2" + hc_send $1 "$2" + exit $1 +} + ##Setting Network properties export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); export HOST=$(hostname) diff --git a/lib/gitpassword.sh b/lib/gitpassword.sh index 25eb9ce..17756d6 100755 --- a/lib/gitpassword.sh +++ b/lib/gitpassword.sh @@ -22,7 +22,7 @@ cd $BASE source lib/functions.sh -assertVarsNotEmpty SITE_ID || exit 1 +assertVarsNotEmpty SITE_ID || fail_and_report 1 "gitpassword.sh failed: SITE_ID is empty." PARAMS="$(cat)" GITHOST=$(echo "$PARAMS" | grep "^host=" | sed 's/host=\(.*\)/\1/g') @@ -30,8 +30,7 @@ GITHOST=$(echo "$PARAMS" | grep "^host=" | sed 's/host=\(.*\)/\1/g') fetchVarsFromVault GIT_PASSWORD if [ -z "${GIT_PASSWORD}" ]; then - log ERROR "Git password not found." - exit 1 + fail_and_report 1 "gitpassword.sh failed: Git password not found." fi cat <&1 is_available=$? if [ $is_available -gt 0 ]; then - log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" - exit 79 + fail_and_report 79 "Prerequisite not fulfilled - $prerequisite is not available!" fi # TODO: Check for specific version done log INFO "Checking if sudo is installed ..." if [ ! -d /etc/sudoers.d ]; then - log ERROR "/etc/sudoers.d does not exist. Please install sudo package." - exit 1 + fail_and_report 1 "/etc/sudoers.d does not exist. Please install sudo package." fi log INFO "Checking configuration ..." ## Download submodule if [ ! -d "/etc/bridgehead/" ]; then - log ERROR "Please set up the config folder at /etc/bridgehead. Instruction are in the readme." - exit 1 + fail_and_report 1 "Please set up the config folder at /etc/bridgehead. Instruction are in the readme." fi # TODO: Check all required variables here in a generic loop #check if project env is present if [ -d "/etc/bridgehead/${PROJECT}.conf" ]; then - log ERROR "Project config not found. Please copy the template from ${PROJECT} and put it under /etc/bridgehead-config/${PROJECT}.conf." - exit 1 + fail_and_report 1 "Project config not found. Please copy the template from ${PROJECT} and put it under /etc/bridgehead-config/${PROJECT}.conf." fi # TODO: Make sure you're in the right directory, or, even better, be independent from the working directory. @@ -60,11 +56,11 @@ fi if [ -e /etc/bridgehead/vault.conf ]; then if [ "$(stat -c "%a %U" /etc/bridgehead/vault.conf)" != "600 bridgehead" ]; then - log ERROR "/etc/bridgehead/vault.conf has wrong owner/permissions. To correct this issue, run chmod 600 /etc/bridgehead/vault.conf && chown bridgehead /etc/bridgehead/vault.conf." - exit 1 + fail_and_report 1 "/etc/bridgehead/vault.conf has wrong owner/permissions. To correct this issue, run chmod 600 /etc/bridgehead/vault.conf && chown bridgehead /etc/bridgehead/vault.conf." fi fi log INFO "Success - all prerequisites are met!" +hc_send log "Success - all prerequisites are met!" exit 0 diff --git a/lib/setup-bridgehead-units.sh b/lib/setup-bridgehead-units.sh index a1393c2..a96e583 100755 --- a/lib/setup-bridgehead-units.sh +++ b/lib/setup-bridgehead-units.sh @@ -26,7 +26,9 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl start bridgehead@${PROJECT}.service, \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ - /bin/systemctl restart bridgehead@*.service + /bin/systemctl restart bridgehead@*.service, \\ + /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ + /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^} EOF diff --git a/lib/systemd/bridgehead-update@.service b/lib/systemd/bridgehead-update@.service index e8b42ea..3a7f347 100644 --- a/lib/systemd/bridgehead-update@.service +++ b/lib/systemd/bridgehead-update@.service @@ -4,8 +4,9 @@ Description=Bridgehead (%i) Update Service [Service] Type=oneshot User=bridgehead -ExecStartPre=-/srv/docker/bridgehead/bridgehead fixPermissions %i +ExecStartPre=-/srv/docker/bridgehead/bridgehead preUpdate %i ExecStart=/srv/docker/bridgehead/bridgehead update %i +ExecStopPost=-/srv/docker/bridgehead/bridgehead postUpdate %i [Install] WantedBy=multi-user.target diff --git a/lib/systemd/bridgehead@.service b/lib/systemd/bridgehead@.service index f109e5a..7645793 100644 --- a/lib/systemd/bridgehead@.service +++ b/lib/systemd/bridgehead@.service @@ -6,9 +6,10 @@ Requires=docker.service User=bridgehead Restart=always RestartSec=30 -ExecStartPre=-/srv/docker/bridgehead/bridgehead fixPermissions %i +ExecStartPre=-/srv/docker/bridgehead/bridgehead preRun %i ExecStart=/srv/docker/bridgehead/bridgehead start %i ExecStop=/srv/docker/bridgehead/bridgehead stop %i +ExecStopPost=-/srv/docker/bridgehead/bridgehead postRun %i [Install] WantedBy=multi-user.target diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 9bb73bd..162d592 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -1,20 +1,21 @@ #!/bin/bash source lib/functions.sh +hc_send log "Updating bridgehead ..." + CONFFILE=/etc/bridgehead/$1.conf if [ ! -e $CONFFILE ]; then - log ERROR "Configuration file $CONFFILE not found." - exit 1 + fail_and_report 1 "Configuration file $CONFFILE not found." fi source $CONFFILE -assertVarsNotEmpty SITE_ID || exit 1 +assertVarsNotEmpty SITE_ID || fail_and_report 1 "Update failed: SITE_ID empty" export SITE_ID -checkOwner . bridgehead || exit 1 -checkOwner /etc/bridgehead bridgehead || exit 1 +checkOwner . bridgehead || fail_and_report 1 "Update failed: Wrong permissions in $(pwd)" +checkOwner /etc/bridgehead bridgehead || fail_and_report 1 "Update failed: Wrong permissions in /etc/bridgehead" CREDHELPER="/srv/docker/bridgehead/lib/gitpassword.sh" @@ -69,10 +70,14 @@ done # If anything is updated, restart service if [ $git_updated = "true" ] || [ $docker_updated = "true" ]; then - log "INFO" "Update detected, now restarting bridgehead" + RES="Update detected, now restarting bridgehead" + log "INFO" "$RES" + hc_send log "$RES" sudo /bin/systemctl restart bridgehead@*.service else - log "INFO" "Nothing updated, nothing to restart." + RES="Nothing updated, nothing to restart." + log "INFO" "$RES" + hc_send log "$RES" fi exit 0