secure db and ml

* Changes to make deployed CCE explorer work properly.

In the lens environment section in services:
- add PUBLIC_SPOT_URL value

bbmri/modules/eric-compose.yml

@@ -11,6 +11,7 @@ services:
       BLAZE_URL: "http://blaze:8080/fhir/"
       BEAM_PROXY_URL: http://beam-proxy-eric:8081
       RETRY_COUNT: ${FOCUS_RETRY_COUNT}
+      OBFUSCATE_BBMRI_ERIC_WAY: "true"
     depends_on:
       - "beam-proxy-eric"
       - "blaze"

cce/modules/lens-compose.yml

@@ -2,7 +2,9 @@ version: "3.7"
 services:
   lens:
     container_name: lens_federated-search
-    image: ghcr.io/samply/cce-explorer:pr-1
+    image: samply/cce-explorer:main
+    environment:
+      PUBLIC_SPOT_URL: https://${HOST}/prod
     labels:
       - "traefik.http.services.lens.loadbalancer.server.port=3000"
       - "traefik.enable=true"
@@ -36,4 +38,9 @@ services:
       - "traefik.http.routers.spot.rule=Host(`${HOST}`) && PathPrefix(`/prod`)"
       - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/prod"
       - "traefik.http.routers.spot.tls=true"
-      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth"
+      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth"
+
+  beam-proxy:
+    environment:
+      APP_spot_KEY: ${FOCUS_BEAM_SECRET_SHORT}
+      

itcc/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "blaze-data:/app/data"
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.blaze_itcc.rule=PathPrefix(`/itcc-localdatamanagement`)"
+      - "traefik.http.routers.blaze_itcc.rule=Host(`${HOST}`) && PathPrefix(`/itcc-localdatamanagement`)"
       - "traefik.http.middlewares.itcc_b_strip.stripprefix.prefixes=/itcc-localdatamanagement"
       - "traefik.http.services.blaze_itcc.loadbalancer.server.port=8080"
       - "traefik.http.routers.blaze_itcc.middlewares=itcc_b_strip,auth"
@@ -32,7 +32,7 @@ services:
       BEAM_PROXY_URL: http://beam-proxy:8081
       RETRY_COUNT: ${FOCUS_RETRY_COUNT}
       EPSILON: 0.28
-      QUERIES_TO_CACHE: '/queries_to_cache.conf'
+      QUERIES_TO_CACHE: "/queries_to_cache.conf"
       ENDPOINT_TYPE: ${FOCUS_ENDPOINT_TYPE:-blaze}
     volumes:
       - /srv/docker/bridgehead/itcc/queries_to_cache.conf:/queries_to_cache.conf:ro
@@ -41,12 +41,13 @@ services:
       - "blaze"
 
   beam-proxy:
-    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop-sockets
     container_name: bridgehead-beam-proxy
     environment:
       BROKER_URL: ${BROKER_URL}
       PROXY_ID: ${PROXY_ID}
       APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT}
+      APP_omics-endpoint_KEY: ${FOCUS_BEAM_SECRET_SHORT}
       PRIVKEY_FILE: /run/secrets/proxy.pem
       ALL_PROXY: http://forward_proxy:3128
       TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
@@ -59,7 +60,6 @@ services:
       - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
       - /srv/docker/bridgehead/itcc/root.crt.pem:/conf/root.crt.pem:ro
 
-
 volumes:
   blaze-data:
 

itcc/modules/id-management-compose.yml

@@ -1,37 +0,0 @@
-version: "3.7"
-
-services:
-  patientlist:
-    image: docker.verbis.dkfz.de/bridgehead/mainzelliste
-    container_name: bridgehead-patientlist
-    environment:
-      - TOMCAT_REVERSEPROXY_FQDN=${HOST}
-      - TOMCAT_REVERSEPROXY_SSL=true
-      - ML_SITE=${IDMANAGEMENT_FRIENDLY_ID}
-      - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD}
-      - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
-      - ML_UPLOAD_API_KEY=${IDMANAGER_UPLOAD_APIKEY}
-      # Add Variables from /etc/patientlist-id-generators.env
-      - PATIENTLIST_SEEDS_TRANSFORMED
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)"
-      - "traefik.http.services.patientlist.loadbalancer.server.port=8080"
-      - "traefik.http.routers.patientlist.tls=true"
-    depends_on:
-      - patientlist-db
-
-  patientlist-db:
-    image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG}
-    container_name: bridgehead-patientlist-db
-    environment:
-      POSTGRES_USER: "mainzelliste"
-      POSTGRES_DB: "mainzelliste"
-      POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
-    volumes:
-      - "patientlist-db-data:/var/lib/postgresql/data"
-      # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!!
-      - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/"
-
-volumes:
-  patientlist-db-data:

itcc/modules/id-management-setup.sh

@@ -1,54 +0,0 @@
-#!/bin/bash -e
-
-function idManagementSetup() {
-	if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then
-		log INFO "id-management setup detected -- will start id-management (mainzelliste)."
-		OVERRIDE+=" -f ./itcc/modules/id-management-compose.yml"
-
-		# Auto Generate local Passwords
-		PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
-		IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
-
-		# Transform Seeds Configuration to pass it to the Mainzelliste Container
-		PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')"
-
-		# Ensure old ids are working !!!
-		export IDMANAGEMENT_FRIENDLY_ID=$(legacyIdMapping "$SITE_ID")
-
-	fi
-}
-
-# Transform into single string array, e.g. 'dktk-test' to 'dktk test'
-# Usage: transformToSingleStringArray 'dktk-test' -> 'dktk test'
-function transformToSingleStringArray() {
-	echo "${1//-/ }";
-}
-
-# Ensure all Words are Uppercase
-# Usage: transformToUppercase 'dktk test' -> 'Dktk Test'
-function transformToUppercase() {
-	result="";
-	for word in $1; do
-		result+=" ${word^}";
-	done
-	echo "$result";
-}
-
-# Handle all execeptions from the norm (e.g LMU, TUM)
-# Usage: applySpecialCases 'Muenchen Lmu Test' -> 'Muenchen LMU Test'
-function applySpecialCases() {
-	result="$1";
-	result="${result/Lmu/LMU}";
-	result="${result/Tum/TUM}";
-	result="${result/Dktk Test/Teststandort}";
-	echo "$result";
-}
-
-# Transform current siteids to legacy version
-# Usage: legacyIdMapping "dktk-test" -> "DktkTest"
-function legacyIdMapping() {
-	single_string_array=$(transformToSingleStringArray "$1");
-	uppercase_string=$(transformToUppercase "$single_string_array");
-	normalized_string=$(applySpecialCases "$uppercase_string");
-	echo "$normalized_string" | tr -d ' '
-}

itcc/modules/itcc-omics-ingest.sh

@@ -3,4 +3,5 @@
 if [ -n "$ENABLE_OMICS" ];then
   OVERRIDE+=" -f ./$PROJECT/modules/itcc-omics-ingest.yaml"
   GENERATE_API_KEY="$(generate_simple_password 'omics')"
+  PATIENTLIST_POSTGRES_PASSWORD=="$(generate_simple_password 'mainzelliste')"
 fi

itcc/modules/itcc-omics-ingest.yaml

@@ -1,8 +1,20 @@
 services:
   omics-endpoint:
-    image: ghcr.io/samply/itcc-omics-ingest:main
+    image: ghcr.io/samply/itcc-omics-ingest:fix-task
     environment:
-      - API_KEY=${GENERATE_API_KEY}
+      API_KEY: ${GENERATE_API_KEY}
+      RUST_LOG: debug
+      BEAM_ID: "omics-endpoint.${PROXY_ID}"
+      BEAM_SECRET: "${FOCUS_BEAM_SECRET_SHORT}"
+      BEAM_URL: http://beam-proxy:8081
+      ML_URL: http://mainzelliste:8080
+      BLAZE_URL: http://bridgehead-itcc-blaze:8080/fhir/
+      DWH_SOCKET_ID: "${DWH_SOCKET_ID}"
+      DWH_TASK_ID: "${DWH_TASK_ID}"
+      ENABLE_SOCKETS: true
+      PARTNER_ID: itcc-inform
+      ML_API_KEY: ${GENERATE_API_KEY}
+
     volumes:
       - /var/cache/bridgehead/omics/data:/data/uploads
     labels:
@@ -12,3 +24,58 @@ services:
       - "traefik.http.routers.omics.tls=true"
       - "traefik.http.middlewares.omics-stripprefix.stripprefix.prefixes=/api"
       - "traefik.http.routers.omics.middlewares=omics-stripprefix"
+
+  db:
+    image: postgres:15
+    container_name: db
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: mainzelliste
+      POSTGRES_USER: ${ML_DB_USER}
+      POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
+    ports:
+      - "5432:5432"
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${ML_DB_USER} -d mainzelliste"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  mainzelliste:
+    image: medicalinformatics/mainzelliste:latest
+    container_name: mainzelliste
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    ports:
+      - 7887:8080
+
+    environment:
+      ML_API_KEY: ${GENERATE_API_KEY}
+      ML_DB_HOST: db
+      ML_DB_PORT: "5432"
+      ML_DB_NAME: mainzelliste
+      ML_DB_USER: ${ML_DB_USER}
+      ML_DB_PASS: ${PATIENTLIST_POSTGRES_PASSWORD}
+      ML_DB_DRIVER: org.postgresql.Driver
+      ML_DB_TYPE: postgresql
+      ML_LOG_LEVEL: INFO
+      ML_ALLOWEDREMOTEADDRESSES: "127.0.0.1,::1,172.16.0.0/12"
+
+    secrets:
+      - mainzelliste.docker.conf
+      - source: symmetric_key
+        target: /etc/resources/keys/symmetric_key.der
+
+volumes:
+  db_data:
+secrets:
+  mainzelliste.docker.conf:
+    file: /etc/bridgehead/mainzelliste/mainzelliste.docker.conf
+
+  symmetric_key:
+    file: /etc/bridgehead/mainzelliste/keyset_siv.json

itcc/modules/lens-compose.yml

@@ -1,17 +1,18 @@
 version: "3.7"
 services:
-  landing:
+  itcc-explorer:
     container_name: lens_itcc_explorer
-    image: ghcr.io/samply/itcc-explorer:fix-setup
+    image: samply/itcc-explorer:main
     environment:
       HOST: "0.0.0.0"
       BIND_ADDR: "0.0.0.0:3000"
       PUBLIC_ENVIRONMENT: ${PUBLIC_ENVIRONMENT}
     labels:
-      - "traefik.http.routers.lens.rule=Host(`${HOST}`)"
       - "traefik.enable=true"
-      - "traefik.http.services.landing.loadbalancer.server.port=3000"
-      - "traefik.http.routers.landing.tls=true"
+      - "traefik.http.routers.itcc.rule=Host(`${HOST}`) && PathPrefix(`/`)"
+      - "traefik.http.routers.itcc.entrypoints=websecure"
+      - "traefik.http.services.itcc.loadbalancer.server.port=3000"
+      - "traefik.http.routers.itcc.tls=true"
 
   spot:
     image: samply/rustyspot:latest

itcc/vars

@@ -7,7 +7,9 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 BROKER_URL_FOR_PREREQ=$BROKER_URL
 PUBLIC_ENVIRONMENT=prod
-SITES=itcc-inform,itcc-ither,itcc-mappyacts,itcc-profyle,itcc-smpaeds,itcc-zero
+DWH_SOCKET_ID=socket.itcc-datalake.${BROKER_ID}
+DWH_TASK_ID=task.itcc-datalake.${BROKER_ID}
+ML_DB_USER=mainzelliste
 
 for module in $PROJECT/modules/*.sh
 do

lib/functions.sh

@@ -327,7 +327,7 @@ function sync_secrets() {
         -e ALL_PROXY=$HTTPS_PROXY_FULL_URL \
         -e PROXY_ID=$proxy_id \
         -e BROKER_URL=$broker_url \
-        -e OIDC_PROVIDER=secret-sync-central.test-secret-sync.$broker_id \
+        -e OIDC_PROVIDER=secret-sync-central.central-secret-sync.$broker_id \
         -e SECRET_DEFINITIONS=$secret_sync_args \
         docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest
 

pscc/root.crt.pem

@@ -1,20 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDNTCCAh2gAwIBAgIUW34NEb7bl0+Ywx+I1VKtY5vpAOowDQYJKoZIhvcNAQEL
-BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjQwMTIyMTMzNzEzWhcNMzQw
-MTE5MTMzNzQzWjAWMRQwEgYDVQQDEwtCcm9rZXItUm9vdDCCASIwDQYJKoZIhvcN
-AQEBBQADggEPADCCAQoCggEBAL5UegLXTlq3XRRj8LyFs3aF0tpRPVoW9RXp5kFI
-TnBvyO6qjNbMDT/xK+4iDtEX4QQUvsxAKxfXbe9i1jpdwjgH7JHaSGm2IjAiKLqO
-OXQQtguWwfNmmp96Ql13ArLj458YH08xMO/w2NFWGwB/hfARa4z/T0afFuc/tKJf
-XbGCG9xzJ9tmcG45QN8NChGhVvaTweNdVxGWlpHxmi0Mn8OM9CEuB7nPtTTiBuiu
-pRC2zVVmNjVp4ktkAqL7IHOz+/F5nhiz6tOika9oD3376Xj055lPznLcTQn2+4d7
-K7ZrBopCFxIQPjkgmYRLfPejbpdUjK1UVJw7hbWkqWqH7JMCAwEAAaN7MHkwDgYD
-VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFGjvRcaIP4HM
-poIguUAK9YL2n7fbMB8GA1UdIwQYMBaAFGjvRcaIP4HMpoIguUAK9YL2n7fbMBYG
-A1UdEQQPMA2CC0Jyb2tlci1Sb290MA0GCSqGSIb3DQEBCwUAA4IBAQCbzycJSaDm
-AXXNJqQ88djrKs5MDXS8RIjS/cu2ayuLaYDe+BzVmUXNA0Vt9nZGdaz63SLLcjpU
-fNSxBfKbwmf7s30AK8Cnfj9q4W/BlBeVizUHQsg1+RQpDIdMrRQrwkXv8mfLw+w5
-3oaXNW6W/8KpBp/H8TBZ6myl6jCbeR3T8EMXBwipMGop/1zkbF01i98Xpqmhx2+l
-n+80ofPsSspOo5XmgCZym8CD/m/oFHmjcvOfpOCvDh4PZ+i37pmbSlCYoMpla3u/
-7MJMP5lugfLBYNDN2p+V4KbHP/cApCDT5UWLOeAWjgiZQtHH5ilDeYqEc1oPjyJt
-Rtup0MTxSJtN
+MIIDNTCCAh2gAwIBAgIUVC1Y1tx0q5PNR33gArAyyBm8PMQwDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjUxMTAzMTQxODQ5WhcNMzUx
+MTAxMTQxOTE5WjAWMRQwEgYDVQQDEwtCcm9rZXItUm9vdDCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAMB1yd7zkh7Io/ReQYindBcAdA1b4ogdVnrdSLRN
+N3zLSh6jN5KIXgs34BdRXx0so0m96q+9xlgacTXGRBn1Tu5SKMRyXdxnCLMzHAYU
+rNKhqF5HeZCYkVyh/tsAyFfDwZDVzsdX64V+0r5+raev2X0gJnlgmF83DIKjkVUS
+2+c+3BnXa9LOdXks0qygJjvaFyi+5MA3DinLnmMLCQ3yAvaZYWyP3xCnGIoVrZFq
+a+YioMCmHrbByuXPoZsXcFY7Z85LQkCtSVt1dH4kkN2/JehXG099nqwMqO8FpLZZ
+xG7/U3P/slX1MMLs97nqRCRoW7Cha2ci1NBYLll+34ekhxMCAwEAAaN7MHkwDgYD
+VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJHTpnuyIGHw
+yvC/mmh+S/JKYVrAMB8GA1UdIwQYMBaAFJHTpnuyIGHwyvC/mmh+S/JKYVrAMBYG
+A1UdEQQPMA2CC0Jyb2tlci1Sb290MA0GCSqGSIb3DQEBCwUAA4IBAQAeDc/k28yb
+I5MLC/LdaA+MKsW2FWF9HT+tsbtltTaQIRnnkwfU/40Ius3gzUU5z+kPqq5+kxhy
+3T646Rbau85Zw24gdNmiVKAAG5ntKoQ7XnyR/06PYyXNGLqnb6aKvbcIPoWtU/+2
+8f5hHdQ/4271aHws7dKcBNWu9V5WmxMZ3YTfnBR5lEda+DhVwHqtmun8EpSbwthD
+aLLIOHJpetr+KWUVFHQdGbO23Qg1Else0Akcn5Gzf/sKkVCVxjHE6jeo4ZwHtstG
+KMoff+ETC+DL5kMZ4CV5VaQ4HxVK7N0qiUxmijWe+EyRZseum1c0s2OEi2L52Q9K
+P4N3yD4ed4p/
 -----END CERTIFICATE-----

pscc/vars

@@ -1,4 +1,4 @@
-BROKER_ID=test-no-real-data.broker.samply.de
+BROKER_ID=broker.pscc.org
 BROKER_URL=https://${BROKER_ID}
 PROXY_ID=${SITE_ID}.${BROKER_ID}
 FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"

versions/acceptance

@@ -1,6 +1,6 @@
 FOCUS_TAG=develop
 BEAM_TAG=develop
-BLAZE_TAG=main
+BLAZE_TAG=0.32
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=develop
 MTBA_TAG=develop

versions/test

@@ -1,6 +1,6 @@
 FOCUS_TAG=develop
 BEAM_TAG=develop
-BLAZE_TAG=main
+BLAZE_TAG=0.32
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=develop
-MTBA_TAG=develop
+MTBA_TAG=develop