Secret Sync for the config repo token, one-time enrollment code, check output (#401)

* cce: Fetch the config repository token via Secret Sync

* Keep the Secret Sync GitLab token across reboots and outages

* Install site configuration with a one-time enrollment code

* Align the itcc and dhki cases in prepare-system.sh

* Report an unreachable broker instead of a clock skew during install

* Show bridgehead check results as OK, WARN or CRIT and run them during install

* Refresh the config repo token before bridgehead check fetches

* Skip Secret Sync until the private key and root certificate exist

* Indent the case block in prepare-system.sh with tabs only

* Hide the config repo URL, token and enrollment code from other local users

* remove the unreachable second dhki case

---------

Co-authored-by: Martin Lablans <m.lablans@dkfz-heidelberg.de>
This commit is contained in:
Pierre Delpy
2026-10-08 14:16:00 +02:00
committed by GitHub
co-authored by Martin Lablans
parent 59a9fd688e
commit 00192eaefc
8 changed files with 287 additions and 144 deletions
+90 -69
View File
@@ -1,74 +1,95 @@
#!/bin/bash
source lib/functions.sh
log INFO "Running Bridgehead checks..."
# Directory ownership
log INFO "Checking directory ownership..."
OWNERSHIP_OK=true
if ! checkOwner /srv/docker/bridgehead bridgehead &> /dev/null; then
log ERROR "Wrong ownership for /srv/docker/bridgehead."
log INFO "Hint: Run 'sudo chown -R bridgehead /srv/docker/bridgehead'."
OWNERSHIP_OK=false
fi
if ! checkOwner /etc/bridgehead bridgehead &> /dev/null; then
log ERROR "Wrong ownership for /etc/bridgehead."
log INFO "Hint: Run 'sudo chown -R bridgehead /etc/bridgehead'."
OWNERSHIP_OK=false
fi
if [ "$OWNERSHIP_OK" = true ]; then
log INFO "Directory ownership is correct."
fi
# Git repository status
log INFO "Checking Git repository status..."
GIT_OK=true
if [ -d "/etc/bridgehead/.git" ]; then
if [ -n "$(git -C "/etc/bridgehead" status --porcelain)" ]; then
log ERROR "The config repo at /etc/bridgehead is modified.\n$(git -C /etc/bridgehead status -s)"
log INFO "Hint: Review your changes with git diff if they are already upstreamed use git stash and git pull to update the repo"
GIT_OK=false
fi
fi
if [ -n "$(git -C "$(pwd)" status --porcelain)" ]; then
log ERROR "$(pwd) is modified. \n$(git -C "$(pwd)" status -s)"
log INFO "Hint: If these are site specific changes to docker compose files consider moving them to $PROJECT/docker-compose.override.yml which is ignored by git."
log INFO " If they are already upstreamed use git stash and git pull to update the repo"
GIT_OK=false
fi
if [ "$GIT_OK" = true ]; then
log INFO "Git repositories are clean."
fi
# Git remote connection
log INFO "Checking Git remote connection..."
GIT_REMOTE_OK=true
if [ -d "/etc/bridgehead/.git" ]; then
if ! git -C "/etc/bridgehead" fetch --dry-run >/dev/null 2>&1; then
log ERROR "Cannot connect to the Git remote for /etc/bridgehead."
log INFO "Hint: Check your network connection and Git remote configuration for /etc/bridgehead."
GIT_REMOTE_OK=false
fi
fi
if [ -d "$(pwd)/.git" ]; then
if ! git -C "$(pwd)" fetch --dry-run >/dev/null 2>&1; then
log ERROR "Cannot connect to the Git remote for $(pwd)."
log INFO "Hint: Check your network connection and Git remote configuration for $(pwd)."
GIT_REMOTE_OK=false
fi
fi
if [ "$GIT_REMOTE_OK" = true ]; then
log INFO "Git remote connection successful."
fi
if [ "$OWNERSHIP_OK" = true ] && [ "$GIT_OK" = true ] && [ "$GIT_REMOTE_OK" = true ]; then
log INFO "All checks passed."
exit 0
if [ -t 1 ]; then
GREEN=$(printf '\033[32m')
YELLOW=$(printf '\033[33m')
RED=$(printf '\033[31m')
RESET=$(printf '\033[0m')
else
log ERROR "Some checks failed. Please review the hints and fix the issues."
log ERROR "Without fixing these issues bridgehead updates may not work correctly."
exit 1
GREEN=""
YELLOW=""
RED=""
RESET=""
fi
WORST=0
report() {
local status=$1 message=$2 color level label
shift 2
case "$status" in
OK) color=$GREEN; level=0; label=" OK " ;;
WARN) color=$YELLOW; level=1; label="WARN" ;;
*) color=$RED; level=2; label="CRIT" ;;
esac
printf '[%s%s%s] %s\n' "$color" "$label" "$RESET" "$message"
local hint
for hint in "$@"; do
printf '%s\n' "$hint" | sed 's/^/ /'
done
if [ $level -gt $WORST ]; then
WORST=$level
fi
}
GIT_PROXY_ARGS=()
if [ -n "$HTTPS_PROXY_FULL_URL" ]; then
GIT_PROXY_ARGS=(-c http.proxy=$HTTPS_PROXY_FULL_URL -c https.proxy=$HTTPS_PROXY_FULL_URL)
fi
echo "Running Bridgehead checks for project $PROJECT ..."
if [ -n "$BROKER_URL_FOR_PREREQ" ]; then
if https_proxy=$HTTPS_PROXY_FULL_URL curl -m 10 -sS -o /dev/null "$BROKER_URL_FOR_PREREQ" 2>/dev/null; then
report OK "Network connection to $BROKER_URL_FOR_PREREQ"
else
report CRIT "Cannot connect to $BROKER_URL_FOR_PREREQ with the configured proxy \"$HTTPS_PROXY_URL\"" \
"Hint: If your server needs a proxy, HTTPS_PROXY_URL (and, if required, HTTPS_PROXY_USERNAME and HTTPS_PROXY_PASSWORD) must be set in /etc/bridgehead/$PROJECT.conf. Please ask ${SUPPORT_EMAIL:-the support team of your project} to add it to your site configuration."
fi
fi
for DIR in /srv/docker/bridgehead /etc/bridgehead; do
if checkOwner $DIR bridgehead &> /dev/null; then
report OK "Ownership of $DIR"
else
report CRIT "Wrong ownership for $DIR" "Hint: Run 'sudo chown -R bridgehead $DIR'."
fi
done
if [ -d "/etc/bridgehead/.git" ]; then
if [ -z "$(git -C "/etc/bridgehead" status --porcelain)" ]; then
report OK "The config repo at /etc/bridgehead is clean"
else
report WARN "The config repo at /etc/bridgehead is modified:" \
"$(git -C /etc/bridgehead status -s)" \
"Hint: Review your changes with git diff if they are already upstreamed use git stash and git pull to update the repo"
fi
fi
if [ -z "$(git -C "$(pwd)" status --porcelain)" ]; then
report OK "$(pwd) is clean"
else
report WARN "$(pwd) is modified:" \
"$(git -C "$(pwd)" status -s)" \
"Hint: If these are site specific changes to docker compose files consider moving them to $PROJECT/docker-compose.override.yml which is ignored by git." \
" If they are already upstreamed use git stash and git pull to update the repo"
fi
secret_sync_gitlab_token &> /dev/null
for DIR in /etc/bridgehead "$(pwd)"; do
if [ -d "$DIR/.git" ]; then
if git "${GIT_PROXY_ARGS[@]}" -C "$DIR" fetch --dry-run >/dev/null 2>&1; then
report OK "Git remote connection for $DIR"
else
report CRIT "Cannot connect to the Git remote for $DIR" "Hint: Check your network connection and Git remote configuration for $DIR."
fi
fi
done
case $WORST in
0) echo "All checks passed." ;;
1) echo "Some checks reported warnings. Please review the hints." ;;
*) echo "Some checks failed. Please review the hints and fix the issues. Without fixing these issues bridgehead updates may not work correctly." ;;
esac
exit $WORST
+126 -37
View File
@@ -336,21 +336,57 @@ function sync_secrets() {
set +a # Export variables in the regular way
}
# Map a GitLab URL to the prefix recognized by Secret Sync
function secret_sync_gitlab_instance() {
case "$1" in
*git.verbis.dkfz.de*) echo verbis;;
*gitlab.bbmri-eric.eu*) echo bbmri;;
*) return 1;;
esac
}
# Use Secret Sync to validate the GitLab token in /var/cache/bridgehead/secrets/gitlab-token.
# If it is missing or expired, Secret Sync will create a new token and write it to the file.
# The git credential helper reads the token from the file during git pull.
function secret_sync_fetch_gitlab_token() {
local gitlab=$1 proxy_id=$2 broker_url=$3 broker_id=$4 privkey_file=$5 root_crt_file=$6
if [ ! -f "$privkey_file" ] || [ ! -f "$root_crt_file" ]; then
log "WARN" "Not running Secret Sync because $privkey_file or $root_crt_file is missing"
return 1
fi
local trusted_ca_args=()
if [ -d /etc/bridgehead/trusted-ca-certs ]; then
trusted_ca_args=(-v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs)
fi
mkdir -p /var/cache/bridgehead/secrets
chmod 700 /var/cache/bridgehead/secrets
log "INFO" "Running Secret Sync for the GitLab token (gitlab=$gitlab)"
docker pull docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest # make sure we have the latest image
docker run --rm \
-v $privkey_file:/run/secrets/privkey.pem:ro \
-v $root_crt_file:/run/secrets/root.crt.pem:ro \
"${trusted_ca_args[@]}" \
-v /var/cache/bridgehead/secrets:/secret-sync/ \
-e CACHE_PATH=/secret-sync/gitlab-token \
-e NO_PROXY=localhost,127.0.0.1 \
-e ALL_PROXY=$HTTPS_PROXY_FULL_URL \
-e PROXY_ID=$proxy_id \
-e BROKER_URL=$broker_url \
-e GITLAB_PROJECT_ACCESS_TOKEN_PROVIDER=secret-sync-central.central-secret-sync.$broker_id \
-e SECRET_DEFINITIONS=GitLabProjectAccessToken:BRIDGEHEAD_CONFIG_REPO_TOKEN:$gitlab \
docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest
}
function secret_sync_gitlab_token() {
if [[ "$PROJECT" != "ccp" && "$PROJECT" != "bbmri" ]]; then
log "INFO" "Not running Secret Sync for project minimal"
if [[ "$PROJECT" != "ccp" && "$PROJECT" != "bbmri" && "$PROJECT" != "cce" ]] && [ -z "$(git -C /etc/bridgehead config credential.helper)" ]; then
log "INFO" "Not running Secret Sync for project $PROJECT"
return
fi
# Map the origin of the git repository /etc/bridgehead to the prefix recognized by Secret Sync
local gitlab
case "$(git -C /etc/bridgehead remote get-url origin)" in
*git.verbis.dkfz.de*) gitlab=verbis;;
*gitlab.bbmri-eric.eu*) gitlab=bbmri;;
*)
log "WARN" "Not running Secret Sync because the git repository /etc/bridgehead has unknown origin"
return
;;
esac
if ! gitlab=$(secret_sync_gitlab_instance "$(git -C /etc/bridgehead remote get-url origin)"); then
log "WARN" "Not running Secret Sync because the git repository /etc/bridgehead has unknown origin"
return
fi
if [ "$PROJECT" == "bbmri" ]; then
# If the project is BBMRI, use the BBMRI-ERIC broker and not the GBN broker
@@ -365,30 +401,7 @@ function secret_sync_gitlab_token() {
root_crt_file="/srv/docker/bridgehead/$PROJECT/root.crt.pem"
fi
# Create a temporary directory for Secret Sync that is valid per boot
secret_sync_tempdir="/tmp/bridgehead/secret-sync.boot-$(cat /proc/sys/kernel/random/boot_id)"
mkdir -p $secret_sync_tempdir
# Use Secret Sync to validate the GitLab token in $secret_sync_tempdir/cache.
# If it is missing or expired, Secret Sync will create a new token and write it to the file.
# The git credential helper reads the token from the file during git pull.
log "INFO" "Running Secret Sync for the GitLab token (gitlab=$gitlab)"
docker pull docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest # make sure we have the latest image
docker run --rm \
-v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \
-v $root_crt_file:/run/secrets/root.crt.pem:ro \
-v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \
-v $secret_sync_tempdir:/secret-sync/ \
-e CACHE_PATH=/secret-sync/gitlab-token \
-e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \
-e NO_PROXY=localhost,127.0.0.1 \
-e ALL_PROXY=$HTTPS_PROXY_FULL_URL \
-e PROXY_ID=$proxy_id \
-e BROKER_URL=$broker_url \
-e GITLAB_PROJECT_ACCESS_TOKEN_PROVIDER=secret-sync-central.central-secret-sync.$broker_id \
-e SECRET_DEFINITIONS=GitLabProjectAccessToken:BRIDGEHEAD_CONFIG_REPO_TOKEN:$gitlab \
docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest
if [ $? -eq 0 ]; then
if secret_sync_fetch_gitlab_token "$gitlab" "$proxy_id" "$broker_url" "$broker_id" "$PRIVATEKEYFILENAME" "$root_crt_file"; then
log "INFO" "Secret Sync was successful"
# In the past we used to hardcode tokens into the repository URL. We have to remove those now for the git credential helper to become effective.
CLEAN_REPO="$(git -C /etc/bridgehead remote get-url origin | sed -E 's|https://[^@]+@|https://|')"
@@ -397,8 +410,6 @@ function secret_sync_gitlab_token() {
git -C /etc/bridgehead config credential.helper /srv/docker/bridgehead/lib/gitlab-token-helper.sh
else
log "WARN" "Secret Sync failed"
# Remove the git credential helper
git -C /etc/bridgehead config --unset credential.helper
fi
# In the past the git credential helper was also set for /srv/docker/bridgehead but never used.
@@ -407,6 +418,84 @@ function secret_sync_gitlab_token() {
git -C /srv/docker/bridgehead config --unset credential.helper
}
function bootstrap_site_configuration() {
local site=$1 enrollment_code=$2 repository_url=$3
local SITE_ID=$site BROKER_ID="" BROKER_URL="" PROXY_ID="" PRIVATEKEYFILENAME=""
local HTTPS_PROXY_FULL_URL=${HTTPS_PROXY_FULL_URL:-${https_proxy:-${HTTPS_PROXY:-}}}
eval "$(grep -E '^(BROKER_ID|BROKER_URL|PROXY_ID|PRIVATEKEYFILENAME)=' "$PROJECT/vars")"
if [ -z "$BROKER_ID" ]; then
log "ERROR" "Project $PROJECT has no Samply.Beam broker to enroll with."
return 1
fi
local gitlab
if ! gitlab=$(secret_sync_gitlab_instance "$repository_url"); then
log "ERROR" "Secret Sync does not support the GitLab server of $repository_url."
return 1
fi
local enroll_dir
enroll_dir=$(mktemp -d /run/bridgehead-enrollment.XXXXXX)
local key_file="$enroll_dir/$SITE_ID.priv.pem"
log "INFO" "Enrolling Beam Proxy Id $PROXY_ID"
if ! docker run --rm -v "$enroll_dir:/pki" docker.verbis.dkfz.de/cache/samply/beam-enroll:latest --output-file "/pki/$SITE_ID.priv.pem" --proxy-id "$PROXY_ID" > "$enroll_dir/enroll.out"; then
log "ERROR" "Unable to generate the private key for $PROXY_ID."
rm -rf "$enroll_dir"
return 1
fi
local csr
csr=$(sed -n '/BEGIN CERTIFICATE REQUEST/,/END CERTIFICATE REQUEST/p' "$enroll_dir/enroll.out")
printf '%s' "$enrollment_code" > "$enroll_dir/enrollment-code"
local response
response=$(curl -sS --data-urlencode "csr=$csr" --data-urlencode "token@$enroll_dir/enrollment-code" "$BROKER_URL/csr" 2>&1)
if [[ "$response" != *"Successfully registered CSR"* ]]; then
log "ERROR" "$BROKER_URL did not accept the certificate request with your one-time enrollment code: $(echo "$response" | sed -e 's/<[^>]*>//g' | tr -s '[:space:]' ' ' | head -c 300)"
rm -rf "$enroll_dir"
return 1
fi
local answer
until retry 3 secret_sync_fetch_gitlab_token "$gitlab" "$PROXY_ID" "$BROKER_URL" "$BROKER_ID" "$key_file" "/srv/docker/bridgehead/$PROJECT/root.crt.pem" \
&& git -c credential.helper=/srv/docker/bridgehead/lib/gitlab-token-helper.sh clone "$repository_url" /etc/bridgehead; do
log "ERROR" "Unable to download $repository_url."
read -r -p "Fix the cause and retry? If you don't, you will need a new one-time enrollment code. [Y/n] " answer || answer=n
if [[ "$answer" == [Nn]* ]]; then
rm -rf "$enroll_dir"
return 1
fi
done
local configured_site_id
configured_site_id=$(grep -m1 -E '^SITE_ID=' "/etc/bridgehead/$PROJECT.conf" | cut -d= -f2- | tr -d "\"'")
if [ "$configured_site_id" != "$SITE_ID" ]; then
log "ERROR" "Your configuration repository is for site '$configured_site_id', but you entered '$SITE_ID'. Ask for a one-time enrollment code for '$configured_site_id' and run the installation again."
rm -rf /etc/bridgehead "$enroll_dir"
return 1
fi
git -C /etc/bridgehead config credential.helper /srv/docker/bridgehead/lib/gitlab-token-helper.sh
mkdir -p "$(dirname "$PRIVATEKEYFILENAME")"
mv "$key_file" "$PRIVATEKEYFILENAME"
chmod 600 "$PRIVATEKEYFILENAME"
rm -rf "$enroll_dir"
}
function read_masked() {
local prompt=$1 char value=""
local -n read_masked_result=$2
printf '%s' "$prompt"
while IFS= read -r -s -n1 char && [ -n "$char" ]; do
if [[ "$char" == $'\x7f' || "$char" == $'\b' ]]; then
if [ -n "$value" ]; then
value=${value%?}
printf '\b \b'
fi
else
value+=$char
printf '*'
fi
done
echo
read_masked_result=$value
}
capitalize_first_letter() {
input="$1"
capitalized="$(tr '[:lower:]' '[:upper:]' <<< ${input:0:1})${input:1}"
+2 -2
View File
@@ -2,10 +2,10 @@
[ "$1" = "get" ] || exit
source "/tmp/bridgehead/secret-sync.boot-$(cat /proc/sys/kernel/random/boot_id)/gitlab-token"
source /var/cache/bridgehead/secrets/gitlab-token
# Any non-empty username works, only the token matters
cat << EOF
username=bk
password=$BRIDGEHEAD_CONFIG_REPO_TOKEN
EOF
EOF
+10
View File
@@ -63,6 +63,16 @@ cp -v \
systemctl daemon-reload
fixPermissions
log "INFO" "Checking your bridgehead ..."
CHECK_RESULT=0
/srv/docker/bridgehead/bridgehead check "${PROJECT}" || CHECK_RESULT=$?
if [ $CHECK_RESULT -ge 2 ]; then
log "ERROR" "Some checks failed. Fix them and run the installation again."
exit 1
fi
log INFO "Trying to update your bridgehead ..."
systemctl start bridgehead-update@"${PROJECT}".service
+39 -16
View File
@@ -60,16 +60,13 @@ case "$PROJECT" in
;;
itcc)
site_configuration_repository_middle="git.verbis.dkfz.de/itcc-sites/"
;;
dhki)
site_configuration_repository_middle="git.verbis.dkfz.de/dhki/"
;;
kr)
site_configuration_repository_middle="git.verbis.dkfz.de/krebsregister-sites/"
;;
dhki)
site_configuration_repository_middle="git.verbis.dkfz.de/dhki/"
;;
kr)
site_configuration_repository_middle="git.verbis.dkfz.de/krebsregister-sites/"
;;
nngm)
site_configuration_repository_middle="git.verbis.dkfz.de/nngm/"
;;
@@ -78,7 +75,7 @@ case "$PROJECT" in
;;
*)
log ERROR "Internal error, this should not happen."
exit 1
exit 1
;;
esac
@@ -92,16 +89,42 @@ if [ -d /etc/bridgehead ]; then
fi
elif [[ "$DEV_MODE" == "NODEV" ]]; then
log "INFO" "Now cloning your site configuration repository for you."
if [ -z "$site" ]; then
read -p "Please enter your site: " site
if [ -z "$access_token" ] && [ -z "$enrollment_code" ]; then
echo "Way A: You received a one-time enrollment code."
echo "Way B: You received a repository URL including access credentials."
read -p "Please choose your way of installation [A/B]: " way
case "$way" in
[Aa])
read -p "Please enter your one-time enrollment code: " enrollment_code
;;
[Bb])
read_masked "Please enter your repository URL: " site_configuration_repository_url
;;
*)
log "ERROR" "Please choose A or B."
exit 1
;;
esac
fi
if [ -z "$access_token" ]; then
read -s -p "Please enter the bridgehead's access token for your site configuration repository (will not be echoed): " access_token
if [ -z "$site_configuration_repository_url" ]; then
if [ -z "$site" ]; then
read -p "Please enter your site: " site
fi
site=$(echo $site | tr '[:upper:]' '[:lower:]')
fi
site_configuration_repository_url="https://bytoken:${access_token}@${site_configuration_repository_middle}$(echo $site | tr '[:upper:]' '[:lower:]').git"
git clone $site_configuration_repository_url /etc/bridgehead
if [ $? -gt 0 ]; then
log "ERROR" "Unable to clone your configuration repository. Please obtain correct access data and try again."
if [ -z "$access_token" ] && [ -n "$enrollment_code" ]; then
if ! bootstrap_site_configuration "$site" "$enrollment_code" "https://${site_configuration_repository_middle}${site}.git"; then
log "ERROR" "Unable to set up your configuration repository with your one-time enrollment code (Way A)."
exit 1
fi
else
if [ -n "$access_token" ]; then
site_configuration_repository_url="https://bytoken:${access_token}@${site_configuration_repository_middle}${site}.git"
fi
git clone $site_configuration_repository_url /etc/bridgehead
if [ $? -gt 0 ]; then
log "ERROR" "Unable to clone your configuration repository. Please obtain correct access data and try again."
fi
fi
elif [[ "$DEV_MODE" == "DEV" ]]; then
log "INFO" "Now cloning your developer configuration repository for you."
@@ -109,7 +132,7 @@ elif [[ "$DEV_MODE" == "DEV" ]]; then
git clone "$url" /etc/bridgehead
fi
chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead
fixPermissions
mkdir -p /tmp/bridgehead /var/cache/bridgehead
chown -R bridgehead:docker /tmp/bridgehead /var/cache/bridgehead
chmod -R g+wr /var/cache/bridgehead /tmp/bridgehead
+1
View File
@@ -72,6 +72,7 @@ source ${PROJECT}/vars
if [ "${PROJECT}" != "minimal" ]; then
set +e
SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
[ -n "$SERVERTIME" ]
RET=$?
set -e
if [ $RET -ne 0 ]; then