From 5593ae38bb969288372c4975f2c70aecbef256e4 Mon Sep 17 00:00:00 2001
From: Martin Lablans <m.lablans@dkfz-heidelberg.de>
Date: Tue, 29 Nov 2022 07:36:05 +0000
Subject: [PATCH] Retry git fetch/pull to make it more resilient to e.g.
 network outages

---
 lib/functions.sh         | 22 ++++++++++++++++++++++
 lib/update-bridgehead.sh |  4 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/lib/functions.sh b/lib/functions.sh
index a539e0d..4f40fd0 100644
--- a/lib/functions.sh
+++ b/lib/functions.sh
@@ -136,6 +136,28 @@ setHostname() {
 	fi
 }
 
+# from: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
+# ex. use: retry 5 /bin/false
+function retry {
+  local retries=$1
+  shift
+
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** $count))
+    count=$(($count + 1))
+    if [ $count -lt $retries ]; then
+      echo "Retry $count/$retries exited with code $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      echo "Retry $count/$retries exited with code $exit, giving up."
+      return $exit
+    fi
+  done
+  return 0
+}
+
 ##Setting Network properties
 # currently not needed
 #export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}');
diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh
index 3201fc5..fd4806b 100755
--- a/lib/update-bridgehead.sh
+++ b/lib/update-bridgehead.sh
@@ -48,10 +48,10 @@ for DIR in /etc/bridgehead $(pwd); do
   old_git_hash="$(git -C $DIR rev-parse --verify HEAD)"
   if [ -z "$HTTP_PROXY_URL" ]; then
     log "INFO" "Git is using no proxy!"
-    OUT=$(git -C $DIR fetch 2>&1 && git -C $DIR pull 2>&1)
+    OUT=$(retry 5 git -C $DIR fetch 2>&1 && retry 5 git -C $DIR pull 2>&1)
   else
     log "INFO" "Git is using proxy ${HTTP_PROXY_URL} from ${CONFFILE}"
-    OUT=$(git -c http.proxy=$HTTP_PROXY_URL -c https.proxy=$HTTPS_PROXY_URL -C $DIR fetch 2>&1 && git -c http.proxy=$HTTP_PROXY_URL -c https.proxy=$HTTPS_PROXY_URL -C $DIR pull 2>&1)
+    OUT=$(retry 5 git -c http.proxy=$HTTP_PROXY_URL -c https.proxy=$HTTPS_PROXY_URL -C $DIR fetch 2>&1 && retry 5 git -c http.proxy=$HTTP_PROXY_URL -c https.proxy=$HTTPS_PROXY_URL -C $DIR pull 2>&1)
   fi
   if [ $? -ne 0 ]; then
     report_error log "Unable to update git $DIR: $OUT"