Merge pull request #379 from samply/develop

* feat: add Data Quality Agent configuration and setup

* fix: correct environment variable name in Data Quality Agent configuration

* feat: update Data Quality Agent configuration and setup instructions

* feat: update Data Quality Agent setup and documentation

* feat: add volume configuration for agent data in Data Quality Agent compose file

* feat: update volume configuration for Data Quality Agent in compose file

* Update README.md

Co-authored-by: Tobias Kussel <TKussel@users.noreply.github.com>

---------

Co-authored-by: Tobias Kussel <TKussel@users.noreply.github.com>

README.md

@@ -27,6 +27,7 @@ This repository is the starting point for any information and tools you will nee
     - [Teiler (Frontend)](#teiler-frontend)
     - [Data Exporter Service](#data-exporter-service)
       - [Data Quality Report](#data-quality-report)
+    - [Data Quality Agent](#data-quality-agent)
 4. [Things you should know](#things-you-should-know)
     - [Auto-Updates](#auto-updates)
     - [Auto-Backups](#auto-backups)
@@ -424,6 +425,32 @@ ENABLE_EXPORTER=true
 ```
 [For further information](docs/exporter.md)  
 
+### Data Quality Agent
+
+The Data Quality Agent is an optional module that periodically evaluates the quality of FHIR data stored in Blaze. It generates local data quality reports accessible via the Bridgehead web interface.
+
+To enable the service, set the following variable in your `<PROJECT>.conf` file:
+
+```bash
+ENABLE_DATA_QUALITY_AGENT=true
+```
+
+#### Sharing Data Quality Reports (recommended)
+
+We encourage sharing your data quality reports with the central BBMRI-ERIC quality dashboard. The reports contain only aggregated, non-patient-identifiable statistics and help the network to monitor and improve overall data quality. However, quality reporting is completely optional and opt-in.
+
+To opt in, additionally set the following variables in your `<PROJECT>.conf` file:
+
+```bash
+DATA_QUALITY_SERVER_URL=https://quality-dashboard.bbmri-eric.eu
+DATA_QUALITY_SERVER_NAME=Central Data Quality Server of BBMRI
+```
+
+If these variables are not set, the Data Quality Agent will still run and generate local reports, but no data will be shared externally.
+
+Reports are accessible at `https://<your-host>/bbmri-data-quality-agent` (default credentials are admin:admin, please change it after first login!!).
+
+[Official documentation](https://fdqf.bbmri-eric.eu/user/deployment.html)
 ## Things you should know
 
 ### Auto-Updates

bbmri/modules/data-quality-agent-compose.yml

@@ -0,0 +1,23 @@
+version: "3.7"
+
+services:
+  data-quality-agent:
+    image: ghcr.io/bbmri-cz/data-quality-server:${DATA_QUALITY_AGENT_TAG}
+    container_name: bridgehead-bbmri-data-quality-agent
+    environment:
+      APP_SETTING_FHIR_URL: http://bridgehead-bbmri-blaze:8080/fhir
+      REPORTING_SERVER_URL: ${DATA_QUALITY_SERVER_URL}
+      REPORTING_SERVER_NAME: ${DATA_QUALITY_SERVER_NAME}
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.data_quality_agent_bbmri.rule=PathPrefix(`/bbmri-data-quality-agent`)"
+      - "traefik.http.services.data_quality_agent_bbmri.loadbalancer.server.port=8082"
+      - "traefik.http.routers.data_quality_agent_bbmri.tls=true"
+      - "traefik.http.middlewares.data_quality_agent_bbmri_strip.stripprefix.prefixes=/bbmri-data-quality-agent"
+      - "traefik.http.routers.data_quality_agent_bbmri.middlewares=data_quality_agent_bbmri_strip,auth"
+    depends_on:
+      - "blaze"
+    volumes:
+      - /var/cache/bridgehead/bbmri/agent-db:/app/data
+      - /etc/localtime:/etc/localtime:ro
+      - /etc/timezone:/etc/timezone:ro

bbmri/modules/data-quality-agent-setup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [ "$ENABLE_DATA_QUALITY_AGENT" == "true" ]; then
+    log INFO "Data Quality Agent setup detected -- will start data-quality-agent service."
+    OVERRIDE+=" -f ./$PROJECT/modules/data-quality-agent-compose.yml"
+fi
+

cce/modules/osiris2fhir-setup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ -n "$ENABLE_OSIRIS2FHIR" ]; then
+  log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module."
+  OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml"
+  LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+fi

lib/functions.sh

@@ -9,15 +9,6 @@ detectCompose() {
 	fi
 }
 
-# Encodes all characters not in unrestricted character set of RFC3986 Section 2.3
-urlencode() {
-  for ((i=0;i<${#1};i++)); do
-    local c=${1:i:1}
-    [[ "$c" =~ [a-zA-Z0-9._~-] ]] && printf '%s' "$c" || printf '%%%02X' "'$c"
-  done
-  echo
-}
-
 setupProxy() {
 	### Note: As the current data protection concepts do not allow communication via HTTP,
 	### we are not setting a proxy for HTTP requests.
@@ -31,12 +22,9 @@ setupProxy() {
 		HTTPS_PROXY_HOST="$(echo $hostport | sed -e 's,:.*,,g')"
 		HTTPS_PROXY_PORT="$(echo $hostport | sed -e 's,^.*:,:,g' -e 's,.*:\([0-9]*\).*,\1,g' -e 's,[^0-9],,g')"
 		if [[ ! -z "$HTTPS_PROXY_USERNAME" && ! -z "$HTTPS_PROXY_PASSWORD" ]]; then
-			local ESCAPED_PASSWORD="$(echo $HTTPS_PROXY_PASSWORD | od -An -v -t x1 | sed -e 's/[[:space:]]//g' -e 's/\([0-9a-f][0-9a-f]\)/%\1/g' | tr -d '\n')"
-			local CURL_ESCAPED_PW="$(urlencode $HTTPS_PROXY_PASSWORD)"
 			local proto="$(echo $HTTPS_PROXY_URL | grep :// | sed -e 's,^\(.*://\).*,\1,g')"
 			local fqdn="$(echo ${HTTPS_PROXY_URL/$proto/})"
-			HTTPS_PROXY_FULL_URL="$(echo $proto$HTTPS_PROXY_USERNAME:$ESCAPED_PASSWORD@$fqdn)"
+			HTTPS_PROXY_FULL_URL="$(echo $proto$HTTPS_PROXY_USERNAME:$HTTPS_PROXY_PASSWORD@$fqdn)"
-			CURL_HTTPS_PROXY_FULL_URL="$(echo $proto$HTTPS_PROXY_USERNAME:$CURL_ESCAPED_PW@$fqdn)"
 			https="authenticated"
 		else
 			HTTPS_PROXY_FULL_URL=$HTTPS_PROXY_URL
@@ -45,7 +33,7 @@ setupProxy() {
 	fi
 
 	log INFO "Configuring proxy servers: $http http proxy (we're not supporting unencrypted comms), $https https proxy"
-	export HTTPS_PROXY_HOST HTTPS_PROXY_PORT HTTPS_PROXY_FULL_URL CURL_HTTPS_PROXY_FULL_URL
+	export HTTPS_PROXY_HOST HTTPS_PROXY_PORT HTTPS_PROXY_FULL_URL
 }
 
 exitIfNotRoot() {
@@ -349,7 +337,7 @@ function sync_secrets() {
 }
 
 function secret_sync_gitlab_token() {
-    if [[ "$PROJECT" != "dktk" && "$PROJECT" != "bbmri" ]]; then
+    if [[ "$PROJECT" != "ccp" && "$PROJECT" != "bbmri" ]]; then
         log "INFO" "Not running Secret Sync for project minimal"
         return
     fi

lib/monitoring.sh

@@ -47,8 +47,8 @@ function hc_send(){
 
     if [ -n "$2" ]; then
         MSG="$2\n\nDocker stats:\n$UPTIME"
-        echo -e "$MSG" | https_proxy=$CURL_HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null -X POST --data-binary @- "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
+        echo -e "$MSG" | https_proxy=$HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null -X POST --data-binary @- "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
     else
-        https_proxy=$CURL_HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
+        https_proxy=$HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
     fi
 }

lib/prerequisites.sh

@@ -71,7 +71,7 @@ source ${PROJECT}/vars
 
 if [ "${PROJECT}" != "minimal" ]; then
   set +e
-  SERVERTIME="$(https_proxy=$CURL_HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
+  SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
   RET=$?
   set -e
   if [ $RET -ne 0 ]; then

lib/tests/test_proxyparsing.sh

@@ -1,123 +0,0 @@
-source ../functions.sh
-
-test_setupProxy() {
-  # simple logger for tests
-  log() { :; }
-
-  local failures=0
-  local total=0
-
-  assert_eq() {
-    local label="$1" got="$2" expected="$3"
-    total=$((total + 1))
-    if [[ "$got" != "$expected" ]]; then
-      failures=$((failures + 1))
-      printf 'FAIL: %s\n  got:      %q\n  expected: %q\n\n' "$label" "$got" "$expected"
-    else
-      printf 'ok: %s\n' "$label"
-    fi
-  }
-
-  run_case() {
-    local name="$1"
-    local url="$2"
-    local u="$3"
-    local p="$4"
-    local exp_host="$5"
-    local exp_port="$6"
-    local exp_full="$7"
-
-    HTTPS_PROXY_URL="$url"
-    HTTPS_PROXY_USERNAME="$u"
-    HTTPS_PROXY_PASSWORD="$p"
-
-    setupProxy >/dev/null 2>&1
-
-    assert_eq "$name host" "$HTTPS_PROXY_HOST" "$exp_host"
-    assert_eq "$name port" "$HTTPS_PROXY_PORT" "$exp_port"
-    assert_eq "$name full" "$HTTPS_PROXY_FULL_URL" "$exp_full"
-  }
-
-  echo "Running setupProxy tests..."
-  echo
-
-  # 1) Basic https host:port
-  run_case "basic https" \
-    "https://proxy.example.org:8443" "" "" \
-    "proxy.example.org" "8443" \
-    "https://proxy.example.org:8443"
-
-  # 2) https without port -> default 443
-  run_case "https no port" \
-    "https://proxy.example.org" "" "" \
-    "proxy.example.org" "443" \
-    "https://proxy.example.org"
-
-  # 3) no scheme, host:port -> defaults scheme=https
-  run_case "no scheme hostport" \
-    "proxy.example.org:3128" "" "" \
-    "proxy.example.org" "3128" \
-    "https://proxy.example.org:3128"
-
-  # 4) URL with path/query/fragment
-  run_case "ignores path" \
-    "https://proxy.example.org:8443/some/path?x=1#y" "" "" \
-    "proxy.example.org" "8443" \
-    "https://proxy.example.org:8443"
-
-  # 5) explicit env creds inserted
-  run_case "env creds override" \
-    "https://proxy.example.org:8443" "alice" "secret" \
-    "proxy.example.org" "8443" \
-    "https://alice:secret@proxy.example.org:8443"
-
-  # 6) embedded creds used if env creds absent
-  run_case "embedded creds" \
-    "https://bob:pw@proxy.example.org:8443" "" "" \
-    "proxy.example.org" "8443" \
-    "https://bob:pw@proxy.example.org:8443"
-
-  # 7) env creds override embedded creds
-  run_case "env overrides embedded" \
-    "https://bob:pw@proxy.example.org:8443" "alice" "secret" \
-    "proxy.example.org" "8443" \
-    "https://alice:secret@proxy.example.org:8443"
-
-  # 8) IPv6 literal with port
-  run_case "ipv6 with port" \
-    "https://[2001:db8::1]:8080" "" "" \
-    "2001:db8::1" "8080" \
-    "https://[2001:db8::1]:8080"
-
-  # 9) IPv6 literal without port -> default 443
-  run_case "ipv6 no port" \
-    "https://[2001:db8::1]" "" "" \
-    "2001:db8::1" "443" \
-    "https://[2001:db8::1]"
-
-  # 10) http scheme rejected -> outputs empty
-  HTTPS_PROXY_URL="http://proxy.example.org:8080"
-  HTTPS_PROXY_USERNAME=""
-  HTTPS_PROXY_PASSWORD=""
-  setupProxy >/dev/null 2>&1
-  assert_eq "http rejected host" "${HTTPS_PROXY_HOST:-}" ""
-  assert_eq "http rejected port" "${HTTPS_PROXY_PORT:-}" ""
-  assert_eq "http rejected full" "${HTTPS_PROXY_FULL_URL:-}" ""
-
-  # 11) empty URL -> outputs empty but no failure
-  HTTPS_PROXY_URL=""
-  setupProxy >/dev/null 2>&1
-  assert_eq "empty url host" "${HTTPS_PROXY_HOST:-}" ""
-  assert_eq "empty url port" "${HTTPS_PROXY_PORT:-}" ""
-  assert_eq "empty url full" "${HTTPS_PROXY_FULL_URL:-}" ""
-
-  echo
-  echo "Tests complete: $((total - failures))/$total passed."
-  if (( failures > 0 )); then
-    echo "Some tests failed."
-    return 1
-  fi
-  return 0
-}
-
-test_setupProxy

minimal/docker-compose.yml

@@ -32,7 +32,7 @@ services:
 
   forward_proxy:
     container_name: bridgehead-forward-proxy
-    image: samply/bridgehead-forward-proxy:pr-16
+    image: docker.verbis.dkfz.de/cache/samply/bridgehead-forward-proxy:latest
     environment:
       HTTPS_PROXY: ${HTTPS_PROXY_URL}
       HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USERNAME}

pscc/modules/osiris2fhir-compose.yml

@@ -1,8 +1,10 @@
 services:
   osiris2fhir:
     container_name: bridgehead-osiris2fhir
-    image: docker.verbis.dkfz.de/ccp/osiris2fhir:${SITE_ID}
+    image: docker.verbis.dkfz.de/ccp/osiris2fhir
     environment:
+      FHIR_PROFILE: ${PROJECT:-pscc}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
       SALT: ${LOCAL_SALT}
     labels:
       - "traefik.enable=true"

pscc/modules/osiris2fhir-setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 if [ -n "$ENABLE_OSIRIS2FHIR" ]; then
-  log INFO "oBDS2FHIR-REST setup detected -- will start osiris2fhir module."
+  log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module."
   OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml"
   LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
 fi

versions/acceptance

@@ -3,4 +3,5 @@ BEAM_TAG=develop
 BLAZE_TAG=0.32
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=develop
-MTBA_TAG=develop
+MTBA_TAG=develop
+DATA_QUALITY_AGENT_TAG=latest

versions/prod

@@ -3,4 +3,5 @@ BEAM_TAG=main
 BLAZE_TAG=0.32
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=main
-MTBA_TAG=main
+MTBA_TAG=main
+DATA_QUALITY_AGENT_TAG=0.1

versions/test

@@ -4,3 +4,4 @@ BLAZE_TAG=0.32
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=develop
 MTBA_TAG=develop
+DATA_QUALITY_AGENT_TAG=latest