secure db and ml

bbmri/modules/directory-sync.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [ -n "${DS_DIRECTORY_USER_NAME}" ] || [ -n "${DS_DIRECTORY_USER_TOKEN}" ]; then
+if [ -n "${DS_DIRECTORY_USER_NAME}" ]; then
 	log INFO "Directory sync setup detected -- will start directory sync service."
 	OVERRIDE+=" -f ./$PROJECT/modules/directory-sync-compose.yml"
 fi

itcc/docker-compose.yml

@@ -32,7 +32,7 @@ services:
       BEAM_PROXY_URL: http://beam-proxy:8081
       RETRY_COUNT: ${FOCUS_RETRY_COUNT}
       EPSILON: 0.28
-      QUERIES_TO_CACHE: '/queries_to_cache.conf'
+      QUERIES_TO_CACHE: "/queries_to_cache.conf"
       ENDPOINT_TYPE: ${FOCUS_ENDPOINT_TYPE:-blaze}
     volumes:
       - /srv/docker/bridgehead/itcc/queries_to_cache.conf:/queries_to_cache.conf:ro
@@ -41,12 +41,13 @@ services:
       - "blaze"
 
   beam-proxy:
-    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop-sockets
     container_name: bridgehead-beam-proxy
     environment:
       BROKER_URL: ${BROKER_URL}
       PROXY_ID: ${PROXY_ID}
       APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT}
+      APP_omics-endpoint_KEY: ${FOCUS_BEAM_SECRET_SHORT}
       PRIVKEY_FILE: /run/secrets/proxy.pem
       ALL_PROXY: http://forward_proxy:3128
       TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
@@ -59,7 +60,6 @@ services:
       - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
       - /srv/docker/bridgehead/itcc/root.crt.pem:/conf/root.crt.pem:ro
 
-
 volumes:
   blaze-data:
 

itcc/modules/itcc-omics-ingest.sh

@@ -3,4 +3,5 @@
 if [ -n "$ENABLE_OMICS" ];then
   OVERRIDE+=" -f ./$PROJECT/modules/itcc-omics-ingest.yaml"
   GENERATE_API_KEY="$(generate_simple_password 'omics')"
+  PATIENTLIST_POSTGRES_PASSWORD=="$(generate_simple_password 'mainzelliste')"
 fi

itcc/modules/itcc-omics-ingest.yaml

@@ -1,8 +1,20 @@
 services:
   omics-endpoint:
-    image: ghcr.io/samply/itcc-omics-ingest:main
+    image: ghcr.io/samply/itcc-omics-ingest:fix-task
     environment:
-      - API_KEY=${GENERATE_API_KEY}
+      API_KEY: ${GENERATE_API_KEY}
+      RUST_LOG: debug
+      BEAM_ID: "omics-endpoint.${PROXY_ID}"
+      BEAM_SECRET: "${FOCUS_BEAM_SECRET_SHORT}"
+      BEAM_URL: http://beam-proxy:8081
+      ML_URL: http://mainzelliste:8080
+      BLAZE_URL: http://bridgehead-itcc-blaze:8080/fhir/
+      DWH_SOCKET_ID: "${DWH_SOCKET_ID}"
+      DWH_TASK_ID: "${DWH_TASK_ID}"
+      ENABLE_SOCKETS: true
+      PARTNER_ID: itcc-inform
+      ML_API_KEY: ${GENERATE_API_KEY}
+
     volumes:
       - /var/cache/bridgehead/omics/data:/data/uploads
     labels:
@@ -12,3 +24,58 @@ services:
       - "traefik.http.routers.omics.tls=true"
       - "traefik.http.middlewares.omics-stripprefix.stripprefix.prefixes=/api"
       - "traefik.http.routers.omics.middlewares=omics-stripprefix"
+
+  db:
+    image: postgres:15
+    container_name: db
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: mainzelliste
+      POSTGRES_USER: ${ML_DB_USER}
+      POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
+    ports:
+      - "5432:5432"
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${ML_DB_USER} -d mainzelliste"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  mainzelliste:
+    image: medicalinformatics/mainzelliste:latest
+    container_name: mainzelliste
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    ports:
+      - 7887:8080
+
+    environment:
+      ML_API_KEY: ${GENERATE_API_KEY}
+      ML_DB_HOST: db
+      ML_DB_PORT: "5432"
+      ML_DB_NAME: mainzelliste
+      ML_DB_USER: ${ML_DB_USER}
+      ML_DB_PASS: ${PATIENTLIST_POSTGRES_PASSWORD}
+      ML_DB_DRIVER: org.postgresql.Driver
+      ML_DB_TYPE: postgresql
+      ML_LOG_LEVEL: INFO
+      ML_ALLOWEDREMOTEADDRESSES: "127.0.0.1,::1,172.16.0.0/12"
+
+    secrets:
+      - mainzelliste.docker.conf
+      - source: symmetric_key
+        target: /etc/resources/keys/symmetric_key.der
+
+volumes:
+  db_data:
+secrets:
+  mainzelliste.docker.conf:
+    file: /etc/bridgehead/mainzelliste/mainzelliste.docker.conf
+
+  symmetric_key:
+    file: /etc/bridgehead/mainzelliste/keyset_siv.json

itcc/vars

@@ -7,6 +7,9 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 BROKER_URL_FOR_PREREQ=$BROKER_URL
 PUBLIC_ENVIRONMENT=prod
+DWH_SOCKET_ID=socket.itcc-datalake.${BROKER_ID}
+DWH_TASK_ID=task.itcc-datalake.${BROKER_ID}
+ML_DB_USER=mainzelliste
 
 for module in $PROJECT/modules/*.sh
 do

kr/docker-compose.yml

@@ -12,8 +12,7 @@ services:
       BASE_URL: "http://bridgehead-kr-blaze:8080"
       JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
       DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
-      DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP}
+      DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
-      CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32}
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
     volumes:
       - "blaze-data:/app/data"

kr/modules/export-and-qb.curl-templates

@@ -0,0 +1,6 @@
+# Full Excel Export
+curl --location --request POST 'https://${HOST}/ccp-exporter/request?query=Patient&query-format=FHIR_PATH&template-id=ccp&output-format=EXCEL' \
+--header 'x-api-key: ${EXPORT_API_KEY}'
+
+# QB
+curl --location --request POST 'https://${HOST}/ccp-reporter/generate?template-id=ccp'

kr/modules/lens-compose.yml

@@ -4,41 +4,32 @@ services:
     deploy:
       replicas: 1 #reactivate if lens is in use
     container_name: lens_federated-search
-    image: docker.verbis.dkfz.de/ccp/kr-explorer:main
+    image: docker.verbis.dkfz.de/ccp/lens:${SITE_ID}
-    environment:
-      PUBLIC_SPOT_URL: https://${HOST}/prod
     labels:
-      - "traefik.http.services.lens.loadbalancer.server.port=3000"
       - "traefik.enable=true"
-      - "traefik.http.routers.lens.rule=Host(`${HOST}`)"
+      - "traefik.http.routers.landing.rule=PathPrefix(`/`)"
-      - "traefik.http.routers.lens.tls=true"
+      - "traefik.http.services.landing.loadbalancer.server.port=80"
+      - "traefik.http.routers.landing.tls=true"
 
   spot:
-    image: samply/rustyspot:latest
+    image: docker.verbis.dkfz.de/ccp-private/central-spot
     environment:
       BEAM_SECRET: "${FOCUS_BEAM_SECRET_SHORT}"
-      BEAM_PROXY_URL: http://beam-proxy:8081
+      BEAM_URL: http://beam-proxy:8081
-      BEAM_APP_ID: "spot.${SITE_ID}.${BROKER_ID}"
+      BEAM_PROXY_ID: ${SITE_ID}
-      CORS_ORIGIN: "https://${HOST}"
+      BEAM_BROKER_ID: ${BROKER_ID}
-      SITES: ${SITES}
+      BEAM_APP_ID: "focus"
-      TRANSFORM: LENS
+      PROJECT_METADATA: "kr_supervisors"
-      PROJECT: kr
-      BIND_ADDR: 0.0.0.0:8055
     depends_on:
       - "beam-proxy"
     labels:
       - "traefik.enable=true"
-      - "traefik.http.services.spot.loadbalancer.server.port=8055"
+      - "traefik.http.services.spot.loadbalancer.server.port=8080"
       - "traefik.http.middlewares.corsheaders2.headers.accesscontrolallowmethods=GET,OPTIONS,POST"
-      - "traefik.http.middlewares.corsheaders2.headers.accesscontrolallowheaders=content-type"
       - "traefik.http.middlewares.corsheaders2.headers.accesscontrolalloworiginlist=https://${HOST}"
       - "traefik.http.middlewares.corsheaders2.headers.accesscontrolallowcredentials=true"
       - "traefik.http.middlewares.corsheaders2.headers.accesscontrolmaxage=-1"
-      - "traefik.http.routers.spot.rule=Host(`${HOST}`) && PathPrefix(`/prod`)"
+      - "traefik.http.routers.spot.rule=Host(`${HOST}`) && PathPrefix(`/backend`)"
-      - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/prod"
+      - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/backend"
       - "traefik.http.routers.spot.tls=true"
-      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth"
+      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot"
-
-  beam-proxy:
-    environment:
-      APP_spot_KEY: ${FOCUS_BEAM_SECRET_SHORT}

kr/modules/obds2fhir-rest-compose.yml

@@ -3,7 +3,7 @@ version: "3.7"
 services:
   obds2fhir-rest:
     container_name: bridgehead-obds2fhir-rest
-    image: docker.verbis.dkfz.de/samply/obds2fhir-rest:main
+    image: docker.verbis.dkfz.de/ccp/obds2fhir-rest:main
     environment:
       IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID
       MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}

kr/vars

@@ -3,7 +3,7 @@ BROKER_URL=https://${BROKER_ID}
 PROXY_ID=${SITE_ID}.${BROKER_ID}
 FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
 FOCUS_RETRY_COUNT=${FOCUS_RETRY_COUNT:-64}
-SUPPORT_EMAIL=p.delpy@dkfz-heidelberg.de
+SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 BROKER_URL_FOR_PREREQ=$BROKER_URL
 

lib/functions.sh

@@ -9,15 +9,6 @@ detectCompose() {
 	fi
 }
 
-# Encodes all characters not in unrestricted character set of RFC3986 Section 2.3
-urlencode() {
-  for ((i=0;i<${#1};i++)); do
-    local c=${1:i:1}
-    [[ "$c" =~ [a-zA-Z0-9._~-] ]] && printf '%s' "$c" || printf '%%%02X' "'$c"
-  done
-  echo
-}
-
 setupProxy() {
 	### Note: As the current data protection concepts do not allow communication via HTTP,
 	### we are not setting a proxy for HTTP requests.
@@ -31,12 +22,9 @@ setupProxy() {
 		HTTPS_PROXY_HOST="$(echo $hostport | sed -e 's,:.*,,g')"
 		HTTPS_PROXY_PORT="$(echo $hostport | sed -e 's,^.*:,:,g' -e 's,.*:\([0-9]*\).*,\1,g' -e 's,[^0-9],,g')"
 		if [[ ! -z "$HTTPS_PROXY_USERNAME" && ! -z "$HTTPS_PROXY_PASSWORD" ]]; then
-			local ESCAPED_PASSWORD="$(echo $HTTPS_PROXY_PASSWORD | od -An -v -t x1 | sed -e 's/[[:space:]]//g' -e 's/\([0-9a-f][0-9a-f]\)/%\1/g' | tr -d '\n')"
-			local CURL_ESCAPED_PW="$(urlencode $HTTPS_PROXY_PASSWORD)"
 			local proto="$(echo $HTTPS_PROXY_URL | grep :// | sed -e 's,^\(.*://\).*,\1,g')"
 			local fqdn="$(echo ${HTTPS_PROXY_URL/$proto/})"
-			HTTPS_PROXY_FULL_URL="$(echo $proto$HTTPS_PROXY_USERNAME:$ESCAPED_PASSWORD@$fqdn)"
+			HTTPS_PROXY_FULL_URL="$(echo $proto$HTTPS_PROXY_USERNAME:$HTTPS_PROXY_PASSWORD@$fqdn)"
-			CURL_HTTPS_PROXY_FULL_URL="$(echo $proto$HTTPS_PROXY_USERNAME:$CURL_ESCAPED_PW@$fqdn)"
 			https="authenticated"
 		else
 			HTTPS_PROXY_FULL_URL=$HTTPS_PROXY_URL
@@ -45,7 +33,7 @@ setupProxy() {
 	fi
 
 	log INFO "Configuring proxy servers: $http http proxy (we're not supporting unencrypted comms), $https https proxy"
-	export HTTPS_PROXY_HOST HTTPS_PROXY_PORT HTTPS_PROXY_FULL_URL CURL_HTTPS_PROXY_FULL_URL
+	export HTTPS_PROXY_HOST HTTPS_PROXY_PORT HTTPS_PROXY_FULL_URL
 }
 
 exitIfNotRoot() {

lib/monitoring.sh

@@ -47,8 +47,8 @@ function hc_send(){
 
     if [ -n "$2" ]; then
         MSG="$2\n\nDocker stats:\n$UPTIME"
-        echo -e "$MSG" | https_proxy=$CURL_HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null -X POST --data-binary @- "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
+        echo -e "$MSG" | https_proxy=$HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null -X POST --data-binary @- "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
     else
-        https_proxy=$CURL_HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
+        https_proxy=$HTTPS_PROXY_FULL_URL curl --max-time 5 -A "$USER_AGENT" -s -o /dev/null "$HCURL"/"$1" || log WARN "Monitoring failed: Unable to send data to $HCURL/$1"
     fi
 }

lib/prerequisites.sh

@@ -71,7 +71,7 @@ source ${PROJECT}/vars
 
 if [ "${PROJECT}" != "minimal" ]; then
   set +e
-  SERVERTIME="$(https_proxy=$CURL_HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
+  SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
   RET=$?
   set -e
   if [ $RET -ne 0 ]; then

lib/tests/test_proxyparsing.sh

@@ -1,123 +0,0 @@
-source ../functions.sh
-
-test_setupProxy() {
-  # simple logger for tests
-  log() { :; }
-
-  local failures=0
-  local total=0
-
-  assert_eq() {
-    local label="$1" got="$2" expected="$3"
-    total=$((total + 1))
-    if [[ "$got" != "$expected" ]]; then
-      failures=$((failures + 1))
-      printf 'FAIL: %s\n  got:      %q\n  expected: %q\n\n' "$label" "$got" "$expected"
-    else
-      printf 'ok: %s\n' "$label"
-    fi
-  }
-
-  run_case() {
-    local name="$1"
-    local url="$2"
-    local u="$3"
-    local p="$4"
-    local exp_host="$5"
-    local exp_port="$6"
-    local exp_full="$7"
-
-    HTTPS_PROXY_URL="$url"
-    HTTPS_PROXY_USERNAME="$u"
-    HTTPS_PROXY_PASSWORD="$p"
-
-    setupProxy >/dev/null 2>&1
-
-    assert_eq "$name host" "$HTTPS_PROXY_HOST" "$exp_host"
-    assert_eq "$name port" "$HTTPS_PROXY_PORT" "$exp_port"
-    assert_eq "$name full" "$HTTPS_PROXY_FULL_URL" "$exp_full"
-  }
-
-  echo "Running setupProxy tests..."
-  echo
-
-  # 1) Basic https host:port
-  run_case "basic https" \
-    "https://proxy.example.org:8443" "" "" \
-    "proxy.example.org" "8443" \
-    "https://proxy.example.org:8443"
-
-  # 2) https without port -> default 443
-  run_case "https no port" \
-    "https://proxy.example.org" "" "" \
-    "proxy.example.org" "443" \
-    "https://proxy.example.org"
-
-  # 3) no scheme, host:port -> defaults scheme=https
-  run_case "no scheme hostport" \
-    "proxy.example.org:3128" "" "" \
-    "proxy.example.org" "3128" \
-    "https://proxy.example.org:3128"
-
-  # 4) URL with path/query/fragment
-  run_case "ignores path" \
-    "https://proxy.example.org:8443/some/path?x=1#y" "" "" \
-    "proxy.example.org" "8443" \
-    "https://proxy.example.org:8443"
-
-  # 5) explicit env creds inserted
-  run_case "env creds override" \
-    "https://proxy.example.org:8443" "alice" "secret" \
-    "proxy.example.org" "8443" \
-    "https://alice:secret@proxy.example.org:8443"
-
-  # 6) embedded creds used if env creds absent
-  run_case "embedded creds" \
-    "https://bob:pw@proxy.example.org:8443" "" "" \
-    "proxy.example.org" "8443" \
-    "https://bob:pw@proxy.example.org:8443"
-
-  # 7) env creds override embedded creds
-  run_case "env overrides embedded" \
-    "https://bob:pw@proxy.example.org:8443" "alice" "secret" \
-    "proxy.example.org" "8443" \
-    "https://alice:secret@proxy.example.org:8443"
-
-  # 8) IPv6 literal with port
-  run_case "ipv6 with port" \
-    "https://[2001:db8::1]:8080" "" "" \
-    "2001:db8::1" "8080" \
-    "https://[2001:db8::1]:8080"
-
-  # 9) IPv6 literal without port -> default 443
-  run_case "ipv6 no port" \
-    "https://[2001:db8::1]" "" "" \
-    "2001:db8::1" "443" \
-    "https://[2001:db8::1]"
-
-  # 10) http scheme rejected -> outputs empty
-  HTTPS_PROXY_URL="http://proxy.example.org:8080"
-  HTTPS_PROXY_USERNAME=""
-  HTTPS_PROXY_PASSWORD=""
-  setupProxy >/dev/null 2>&1
-  assert_eq "http rejected host" "${HTTPS_PROXY_HOST:-}" ""
-  assert_eq "http rejected port" "${HTTPS_PROXY_PORT:-}" ""
-  assert_eq "http rejected full" "${HTTPS_PROXY_FULL_URL:-}" ""
-
-  # 11) empty URL -> outputs empty but no failure
-  HTTPS_PROXY_URL=""
-  setupProxy >/dev/null 2>&1
-  assert_eq "empty url host" "${HTTPS_PROXY_HOST:-}" ""
-  assert_eq "empty url port" "${HTTPS_PROXY_PORT:-}" ""
-  assert_eq "empty url full" "${HTTPS_PROXY_FULL_URL:-}" ""
-
-  echo
-  echo "Tests complete: $((total - failures))/$total passed."
-  if (( failures > 0 )); then
-    echo "Some tests failed."
-    return 1
-  fi
-  return 0
-}
-
-test_setupProxy

minimal/docker-compose.yml

@@ -32,7 +32,7 @@ services:
 
   forward_proxy:
     container_name: bridgehead-forward-proxy
-    image: samply/bridgehead-forward-proxy:pr-16
+    image: docker.verbis.dkfz.de/cache/samply/bridgehead-forward-proxy:latest
     environment:
       HTTPS_PROXY: ${HTTPS_PROXY_URL}
       HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USERNAME}

pscc/modules/osiris2fhir-compose.yml

@@ -1,13 +0,0 @@
-services:
-  osiris2fhir:
-    container_name: bridgehead-osiris2fhir
-    image: docker.verbis.dkfz.de/ccp/osiris2fhir:${SITE_ID}
-    environment:
-      SALT: ${LOCAL_SALT}
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.osiris2fhir.rule=PathPrefix(`/osiris2fhir`)"
-      - "traefik.http.middlewares.osiris2fhir_strip.stripprefix.prefixes=/osiris2fhir"
-      - "traefik.http.services.osiris2fhir.loadbalancer.server.port=8080"
-      - "traefik.http.routers.osiris2fhir.tls=true"
-      - "traefik.http.routers.osiris2fhir.middlewares=osiris2fhir_strip,auth"

pscc/modules/osiris2fhir-setup.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-if [ -n "$ENABLE_OSIRIS2FHIR" ]; then
-  log INFO "oBDS2FHIR-REST setup detected -- will start osiris2fhir module."
-  OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml"
-  LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
-fi