Monitoring for bridgehead startup and update (#22)

bridgehead

@@ -43,14 +43,16 @@ esac
 
 # Load variables from /etc/bridgehead and /srv/docker/bridgehead
 set -a
-source /etc/bridgehead/$PROJECT.conf
+source /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "/etc/bridgehead/$PROJECT.conf not found"
-fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || exit 1
+fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile"
 [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars
 set +a
 
 case "$ACTION" in
 	start)
+		hc_send log "Bridgehead $PROJECT startup: Checking requirements ..."
 		checkRequirements
+		hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..."
 		exec docker-compose -f ./$PROJECT/docker-compose.yml up --abort-on-container-exit
 		;;
 	stop)
@@ -65,8 +67,10 @@ case "$ACTION" in
 	uninstall)
 		exec ./lib/remove-bridgehead-units.sh $PROJECT
 		;;
-	fixPermissions)
+	preRun | preUpdate)
-		chown -R bridgehead /etc/bridgehead .
+		fixPermissions
+		;;
+	postRun | postUpdate)
 		;;
 	*)
 		printUsage

lib/functions.sh

@@ -1,9 +1,11 @@
 #!/bin/bash -e
 
+source lib/log.sh
+
 exitIfNotRoot() {
   if [ "$EUID" -ne 0 ]; then
     log "ERROR" "Please run as root"
-    exit 1
+    fail_and_report 1 "Please run as root"
   fi
 }
 
@@ -16,10 +18,6 @@ checkOwner(){
   return 0
 }
 
-log() {
-  echo -e "$(date +'%Y-%m-%d %T')" "$1:" "$2"
-}
-
 printUsage() {
 	echo "Usage: bridgehead start|stop|update|install|uninstall PROJECTNAME"
 	echo "PROJECTNAME should be one of ccp|nngm|gbn"
@@ -28,7 +26,7 @@ printUsage() {
 checkRequirements() {
 	if ! lib/prerequisites.sh; then
 		log "ERROR" "Validating Prerequisites failed, please fix the error(s) above this line."
-		exit 1
+		fail_and_report 1 "Validating prerequisites failed."
 	else
 		return 0
 	fi
@@ -97,6 +95,19 @@ assertVarsNotEmpty() {
 	return 0
 }
 
+fixPermissions() {
+	CHOWN=$(which chown)
+	sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead
+}
+
+source lib/monitoring.sh
+
+fail_and_report() {
+	log ERROR "$2"
+	hc_send $1 "$2"
+	exit $1
+}
+
 ##Setting Network properties
 export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}');
 export HOST=$(hostname)

lib/gitpassword.sh

@@ -22,7 +22,7 @@ cd $BASE
 
 source lib/functions.sh
 
-assertVarsNotEmpty SITE_ID || exit 1
+assertVarsNotEmpty SITE_ID || fail_and_report 1 "gitpassword.sh failed: SITE_ID is empty."
 
 PARAMS="$(cat)"
 GITHOST=$(echo "$PARAMS" | grep "^host=" | sed 's/host=\(.*\)/\1/g')
@@ -30,8 +30,7 @@ GITHOST=$(echo "$PARAMS" | grep "^host=" | sed 's/host=\(.*\)/\1/g')
 fetchVarsFromVault GIT_PASSWORD
 
 if [ -z "${GIT_PASSWORD}" ]; then
-	log ERROR "Git password not found."
+	fail_and_report 1 "gitpassword.sh failed: Git password not found."
-	exit 1
 fi
 
 cat <<EOF

lib/log.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+log() {
+  echo -e "$(date +'%Y-%m-%d %T')" "$1:" "$2"
+}

lib/monitoring.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+source lib/log.sh
+
+function hc_set_uuid(){
+    HCUUID="$1"
+}
+
+function hc_set_service(){
+    HCSERVICE="$1"
+}
+
+UPTIME=
+
+function hc_send(){
+    if [ -n "$MONITOR_APIKEY" ]; then
+        hc_set_uuid $MONITOR_APIKEY
+    fi
+
+    if [ -n "$HCSERVICE" ]; then
+        HCURL="https://hc-ping.com/$PING_KEY/$HCSERVICE"
+    fi
+    if [ -n "$HCUUID" ]; then
+        HCURL="https://hc-ping.com/$HCUUID"
+    fi
+    if [ ! -n "$HCURL" ]; then
+        log WARN "Healthcheck reporting failed: Neither Healthcheck UUID nor service set - please check config in /etc/bridgehead"
+        return 1
+    fi
+
+    if [ -z "$UPTIME" ]; then
+        UPTIME=$(docker ps --format '{{.Names}} {{.RunningFor}}' --filter name=bridgehead || echo "Unable to get docker statistics")
+    fi
+
+    if [ -n "$2" ]; then
+        MSG="$2\n\nDocker stats:\n$UPTIME"
+        echo -e "$MSG" | https_proxy=$HTTPS_PROXY_URL curl -s -o /dev/null -X POST --data-binary @- "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
+    else
+        https_proxy=$HTTPS_PROXY_URL curl -s -o /dev/null "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
+    fi
+}

lib/prerequisites.sh

@@ -17,32 +17,28 @@ for prerequisite in $prerequisites; do
   $prerequisite --version 2>&1
   is_available=$?
   if [ $is_available -gt 0 ]; then
-    log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!"
+    fail_and_report 79 "Prerequisite not fulfilled - $prerequisite is not available!"
-    exit 79
   fi
   # TODO: Check for specific version
 done
 
 log INFO "Checking if sudo is installed ..."
 if [ ! -d /etc/sudoers.d ]; then
-  log ERROR "/etc/sudoers.d does not exist. Please install sudo package."
+  fail_and_report 1 "/etc/sudoers.d does not exist. Please install sudo package."
-  exit 1
 fi
 
 log INFO "Checking configuration ..."
 
 ## Download submodule
 if [ ! -d "/etc/bridgehead/" ]; then
-  log ERROR "Please set up the config folder at /etc/bridgehead. Instruction are in the readme."
+  fail_and_report 1 "Please set up the config folder at /etc/bridgehead. Instruction are in the readme."
-  exit 1
 fi
 
 # TODO: Check all required variables here in a generic loop
 
 #check if project env is present
 if [ -d "/etc/bridgehead/${PROJECT}.conf" ]; then
-   log ERROR "Project config not found. Please copy the template from ${PROJECT} and put it under /etc/bridgehead-config/${PROJECT}.conf."
+   fail_and_report 1 "Project config not found. Please copy the template from ${PROJECT} and put it under /etc/bridgehead-config/${PROJECT}.conf."
-   exit 1
 fi
 
 # TODO: Make sure you're in the right directory, or, even better, be independent from the working directory.
@@ -60,11 +56,11 @@ fi
 
 if [ -e /etc/bridgehead/vault.conf ]; then
 	if [ "$(stat -c "%a %U" /etc/bridgehead/vault.conf)" != "600 bridgehead" ]; then
-		log ERROR "/etc/bridgehead/vault.conf has wrong owner/permissions. To correct this issue, run chmod 600 /etc/bridgehead/vault.conf && chown bridgehead /etc/bridgehead/vault.conf."
+    fail_and_report 1 "/etc/bridgehead/vault.conf has wrong owner/permissions. To correct this issue, run chmod 600 /etc/bridgehead/vault.conf && chown bridgehead /etc/bridgehead/vault.conf."
-		exit 1
 	fi
 fi
 
 log INFO "Success - all prerequisites are met!"
+hc_send log "Success - all prerequisites are met!"
 
 exit 0

lib/setup-bridgehead-units.sh

@@ -26,7 +26,9 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\
     /bin/systemctl start bridgehead@${PROJECT}.service, \\
     /bin/systemctl stop bridgehead@${PROJECT}.service, \\
     /bin/systemctl restart bridgehead@${PROJECT}.service, \\
-    /bin/systemctl restart bridgehead@*.service
+    /bin/systemctl restart bridgehead@*.service, \\
+    /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\
+    /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead
 
 bridgehead ALL= NOPASSWD: BRIDGEHEAD${PROJECT^^}
 EOF

lib/systemd/bridgehead-update@.service

@@ -4,8 +4,9 @@ Description=Bridgehead (%i) Update Service
 [Service]
 Type=oneshot
 User=bridgehead
-ExecStartPre=-/srv/docker/bridgehead/bridgehead fixPermissions %i
+ExecStartPre=-/srv/docker/bridgehead/bridgehead preUpdate %i
 ExecStart=/srv/docker/bridgehead/bridgehead update %i
+ExecStopPost=-/srv/docker/bridgehead/bridgehead postUpdate %i
 
 [Install]
 WantedBy=multi-user.target

lib/systemd/bridgehead@.service

@@ -6,9 +6,10 @@ Requires=docker.service
 User=bridgehead
 Restart=always
 RestartSec=30
-ExecStartPre=-/srv/docker/bridgehead/bridgehead fixPermissions %i
+ExecStartPre=-/srv/docker/bridgehead/bridgehead preRun %i
 ExecStart=/srv/docker/bridgehead/bridgehead start %i
 ExecStop=/srv/docker/bridgehead/bridgehead stop %i
+ExecStopPost=-/srv/docker/bridgehead/bridgehead postRun %i
 
 [Install]
 WantedBy=multi-user.target

lib/update-bridgehead.sh

@@ -1,20 +1,21 @@
 #!/bin/bash
 source lib/functions.sh
 
+hc_send log "Updating bridgehead ..."
+
 CONFFILE=/etc/bridgehead/$1.conf
 
 if [ ! -e $CONFFILE ]; then
-  log ERROR "Configuration file $CONFFILE not found."
+  fail_and_report 1 "Configuration file $CONFFILE not found."
-  exit 1
 fi
 
 source $CONFFILE
 
-assertVarsNotEmpty SITE_ID || exit 1
+assertVarsNotEmpty SITE_ID || fail_and_report 1 "Update failed: SITE_ID empty"
 export SITE_ID
 
-checkOwner . bridgehead || exit 1
+checkOwner . bridgehead || fail_and_report 1 "Update failed: Wrong permissions in $(pwd)"
-checkOwner /etc/bridgehead bridgehead || exit 1
+checkOwner /etc/bridgehead bridgehead || fail_and_report 1 "Update failed: Wrong permissions in /etc/bridgehead"
 
 CREDHELPER="/srv/docker/bridgehead/lib/gitpassword.sh"
 
@@ -69,10 +70,14 @@ done
 
 # If anything is updated, restart service
 if [ $git_updated = "true" ] || [ $docker_updated = "true" ]; then
-  log "INFO" "Update detected, now restarting bridgehead"
+  RES="Update detected, now restarting bridgehead"
+  log "INFO" "$RES"
+  hc_send log "$RES"
   sudo /bin/systemctl restart bridgehead@*.service
 else
-  log "INFO" "Nothing updated, nothing to restart."
+  RES="Nothing updated, nothing to restart."
+  log "INFO" "$RES"
+  hc_send log "$RES"
 fi
 
 exit 0