Merge remote-tracking branch 'origin/develop' into feat/non-ccp-ovis-bridgehead

README.md

@@ -27,6 +27,7 @@ This repository is the starting point for any information and tools you will nee
     - [Teiler (Frontend)](#teiler-frontend)
     - [Data Exporter Service](#data-exporter-service)
       - [Data Quality Report](#data-quality-report)
+    - [Data Quality Agent](#data-quality-agent)
 4. [Things you should know](#things-you-should-know)
     - [Auto-Updates](#auto-updates)
     - [Auto-Backups](#auto-backups)
@@ -41,7 +42,7 @@ This repository is the starting point for any information and tools you will nee
 
 The data protection officer at your site will probably want to know exactly what our software does with patient data, and you may need to get their approval before you are allowed to install a Bridgehead. To help you with this, we have provided some data protection concepts:
 
-- [Germany](https://www.bbmri.de/biobanking/it/infrastruktur/datenschutzkonzept/)
+- [Germany](https://www.netzwerk-universitaetsmedizin.de/plattformen/gbn/biobanking/it/infrastruktur/datenschutzkonzept)
 
 ### Hardware
 
@@ -424,6 +425,32 @@ ENABLE_EXPORTER=true
 ```
 [For further information](docs/exporter.md)  
 
+### Data Quality Agent
+
+The Data Quality Agent is an optional module that periodically evaluates the quality of FHIR data stored in Blaze. It generates local data quality reports accessible via the Bridgehead web interface.
+
+To enable the service, set the following variable in your `<PROJECT>.conf` file:
+
+```bash
+ENABLE_DATA_QUALITY_AGENT=true
+```
+
+#### Sharing Data Quality Reports (recommended)
+
+We encourage sharing your data quality reports with the central BBMRI-ERIC quality dashboard. The reports contain only aggregated, non-patient-identifiable statistics and help the network to monitor and improve overall data quality. However, quality reporting is completely optional and opt-in.
+
+To opt in, additionally set the following variables in your `<PROJECT>.conf` file:
+
+```bash
+DATA_QUALITY_SERVER_URL=https://quality-dashboard.bbmri-eric.eu
+DATA_QUALITY_SERVER_NAME=Central Data Quality Server of BBMRI
+```
+
+If these variables are not set, the Data Quality Agent will still run and generate local reports, but no data will be shared externally.
+
+Reports are accessible at `https://<your-host>/bbmri-data-quality-agent` (default credentials are admin:admin, please change it after first login!!).
+
+[Official documentation](https://fdqf.bbmri-eric.eu/user/deployment.html)
 ## Things you should know
 
 ### Auto-Updates

bbmri/modules/data-quality-agent-compose.yml

@@ -0,0 +1,23 @@
+version: "3.7"
+
+services:
+  data-quality-agent:
+    image: ghcr.io/bbmri-cz/data-quality-agent:${DATA_QUALITY_AGENT_TAG}
+    container_name: bridgehead-bbmri-data-quality-agent
+    environment:
+      APP_SETTING_FHIR_URL: http://bridgehead-bbmri-blaze:8080/fhir
+      REPORTING_SERVER_URL: ${DATA_QUALITY_SERVER_URL}
+      REPORTING_SERVER_NAME: ${DATA_QUALITY_SERVER_NAME}
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.data_quality_agent_bbmri.rule=PathPrefix(`/bbmri-data-quality-agent`)"
+      - "traefik.http.services.data_quality_agent_bbmri.loadbalancer.server.port=8082"
+      - "traefik.http.routers.data_quality_agent_bbmri.tls=true"
+      - "traefik.http.middlewares.data_quality_agent_bbmri_strip.stripprefix.prefixes=/bbmri-data-quality-agent"
+      - "traefik.http.routers.data_quality_agent_bbmri.middlewares=data_quality_agent_bbmri_strip,auth"
+    depends_on:
+      - "blaze"
+    volumes:
+      - /var/cache/bridgehead/bbmri/agent-db:/app/data
+      - /etc/localtime:/etc/localtime:ro
+      - /etc/timezone:/etc/timezone:ro

bbmri/modules/data-quality-agent-setup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [ "$ENABLE_DATA_QUALITY_AGENT" == "true" ]; then
+    log INFO "Data Quality Agent setup detected -- will start data-quality-agent service."
+    OVERRIDE+=" -f ./$PROJECT/modules/data-quality-agent-compose.yml"
+fi
+

cce/modules/osiris2fhir-setup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ -n "$ENABLE_OSIRIS2FHIR" ]; then
+  log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module."
+  OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml"
+  LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+fi

itcc/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/blaze:latest
     container_name: bridgehead-itcc-blaze
     environment:
       BASE_URL: "http://bridgehead-itcc-blaze:8080"
@@ -32,7 +32,7 @@ services:
       BEAM_PROXY_URL: http://beam-proxy:8081
       RETRY_COUNT: ${FOCUS_RETRY_COUNT}
       EPSILON: 0.28
-      QUERIES_TO_CACHE: '/queries_to_cache.conf'
+      QUERIES_TO_CACHE: "/queries_to_cache.conf"
       ENDPOINT_TYPE: ${FOCUS_ENDPOINT_TYPE:-blaze}
     volumes:
       - /srv/docker/bridgehead/itcc/queries_to_cache.conf:/queries_to_cache.conf:ro
@@ -41,12 +41,13 @@ services:
       - "blaze"
 
   beam-proxy:
-    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop-sockets
     container_name: bridgehead-beam-proxy
     environment:
       BROKER_URL: ${BROKER_URL}
       PROXY_ID: ${PROXY_ID}
       APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT}
+      APP_omics-endpoint_KEY: ${FOCUS_BEAM_SECRET_SHORT}
       PRIVKEY_FILE: /run/secrets/proxy.pem
       ALL_PROXY: http://forward_proxy:3128
       TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
@@ -59,7 +60,6 @@ services:
       - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
       - /srv/docker/bridgehead/itcc/root.crt.pem:/conf/root.crt.pem:ro
 
-
 volumes:
   blaze-data:
 

itcc/modules/itcc-omics-ingest.sh

@@ -3,4 +3,25 @@
 if [ -n "$ENABLE_OMICS" ];then
   OVERRIDE+=" -f ./$PROJECT/modules/itcc-omics-ingest.yaml"
   GENERATE_API_KEY="$(generate_simple_password 'omics')"
+  PATIENTLIST_POSTGRES_PASSWORD=="$(generate_simple_password 'mainzelliste')"
+  KEYSET=/var/bridgehead/mainzelliste/keyset_siv.json
+  if [ ! -f "$KEYSET" ]; then
+    mkdir -p "$(dirname "$KEYSET")"
+    KEY_ID=$(($(openssl rand -hex 4 | sed 's/^/0x/') & 0x7FFFFFFF))
+    VALUE=$({ printf '\x12\x40'; openssl rand 64; } | base64 | tr -d '\n')
+    jq -n --argjson id "$KEY_ID" --arg value "$VALUE" '{
+      primaryKeyId: $id,
+      key: [{
+        keyData: {
+          typeUrl: "type.googleapis.com/google.crypto.tink.AesSivKey",
+          value: $value,
+          keyMaterialType: "SYMMETRIC"
+        },
+        status: "ENABLED",
+        keyId: $id,
+        outputPrefixType: "TINK"
+      }]
+    }' > "$KEYSET"
+    chmod 600 "$KEYSET"
+  fi
 fi

itcc/modules/itcc-omics-ingest.yaml

@@ -1,14 +1,69 @@
 services:
   omics-endpoint:
-    image: ghcr.io/samply/itcc-omics-ingest:main
+    image: samply/itcc-omics-ingest:main
     environment:
-      - API_KEY=${GENERATE_API_KEY}
-    volumes:
-      - /var/cache/bridgehead/omics/data:/data/uploads
+      API_KEY: ${GENERATE_API_KEY}
+      BEAM_APP_ID_LONG: omics-endpoint.${PROXY_ID}
+      BEAM_SECRET: ${FOCUS_BEAM_SECRET_SHORT}
+      DWH_SOCKET_ID: ${DWH_SOCKET_ID}
+      DWH_TASK_ID: ${DWH_TASK_ID}
+      PARTNER_ID: ${SITE_ID}
+      ML_API_KEY: ${GENERATE_API_KEY}
     labels:
-      - "traefik.http.routers.omics.rule=Host(`${HOST}`) && PathPrefix(`/api/omics`)"
+      - "traefik.http.routers.omics.rule=Host(`${HOST}`) &&
+        PathPrefix(`/api/upload`)"
       - "traefik.enable=true"
       - "traefik.http.services.omics.loadbalancer.server.port=6080"
       - "traefik.http.routers.omics.tls=true"
       - "traefik.http.middlewares.omics-stripprefix.stripprefix.prefixes=/api"
       - "traefik.http.routers.omics.middlewares=omics-stripprefix"
+
+  patientlist-db:
+    image: postgres:${POSTGRES_TAG}
+    container_name: bridgehead-patientlist-db
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: mainzelliste
+      POSTGRES_USER: ${ML_DB_USER}
+      POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
+    volumes:
+      - "patientlist-db-data:/var/lib/postgresql/data"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${ML_DB_USER} -d mainzelliste"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  patientlist:
+    image: medicalinformatics/mainzelliste:latest
+    container_name: bridgehead-patientlist
+    restart: unless-stopped
+    depends_on:
+      patientlist-db:
+        condition: service_healthy
+    environment:
+      ML_API_KEY: ${GENERATE_API_KEY}
+      ML_DB_HOST: patientlist-db
+      ML_DB_PORT: "5432"
+      ML_DB_NAME: mainzelliste
+      ML_DB_USER: ${ML_DB_USER}
+      ML_DB_PASS: ${PATIENTLIST_POSTGRES_PASSWORD}
+      ML_DB_DRIVER: org.postgresql.Driver
+      ML_DB_TYPE: postgresql
+      ML_LOG_LEVEL: INFO
+      ML_ALLOWEDREMOTEADDRESSES: "127.0.0.1,::1,172.16.0.0/12"
+
+    secrets:
+      - mainzelliste.docker.conf
+      - source: symmetric_key
+        target: /etc/resources/keys/symmetric_key.json
+
+volumes:
+  patientlist-db-data:
+secrets:
+  mainzelliste.docker.conf:
+    file: /etc/bridgehead/mainzelliste/mainzelliste.docker.conf
+
+  symmetric_key:
+    file: /var/bridgehead/mainzelliste/keyset_siv.json

itcc/modules/lens-compose.yml

@@ -7,6 +7,7 @@ services:
       HOST: "0.0.0.0"
       BIND_ADDR: "0.0.0.0:3000"
       PUBLIC_ENVIRONMENT: ${PUBLIC_ENVIRONMENT}
+      PUBLIC_SPOT_URL: https://${HOST}/prod
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.itcc.rule=Host(`${HOST}`) && PathPrefix(`/`)"

itcc/vars

@@ -7,6 +7,9 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 BROKER_URL_FOR_PREREQ=$BROKER_URL
 PUBLIC_ENVIRONMENT=prod
+DWH_SOCKET_ID=socket.itcc-datalake.${BROKER_ID}
+DWH_TASK_ID=task.itcc-datalake.${BROKER_ID}
+ML_DB_USER=mainzelliste
 
 for module in $PROJECT/modules/*.sh
 do

lib/functions.sh

@@ -337,7 +337,7 @@ function sync_secrets() {
 }
 
 function secret_sync_gitlab_token() {
-    if [[ "$PROJECT" != "dktk" && "$PROJECT" != "bbmri" ]]; then
+    if [[ "$PROJECT" != "ccp" && "$PROJECT" != "bbmri" ]]; then
         log "INFO" "Not running Secret Sync for project minimal"
         return
     fi

minimal/modules/dnpm-central-targets.json

@@ -49,7 +49,7 @@
             "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de"
         },
         {
-            "id": "Charite",
+            "id": "Charité",
             "name": "Berlin",
             "virtualhost": "charite.dnpm.de",
             "beamconnect": "dnpm-connect.berlin-test.broker.ccp-it.dktk.dkfz.de"

pscc/modules/lens-compose.yml

@@ -2,7 +2,9 @@ version: "3.7"
 services:
   lens:
     container_name: lens-federated-search
-    image: docker.verbis.dkfz.de/ccp/lens:${SITE_ID}
+    image: docker.verbis.dkfz.de/ccp/lens:pscc
+    environment:
+      PUBLIC_SPOT_URL: https://${HOST}/prod
     labels:
       - "traefik.http.services.lens.loadbalancer.server.port=3000"
       - "traefik.enable=true"
@@ -11,7 +13,6 @@ services:
 
   spot:
     image: samply/rustyspot:latest
-    platform: linux/amd64
     environment:
       HTTP_PROXY: ${HTTP_PROXY_URL}
       HTTPS_PROXY: ${HTTPS_PROXY_URL}
@@ -38,3 +39,7 @@ services:
       - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/prod"
       - "traefik.http.routers.spot.tls=true"
       - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth"
+
+  beam-proxy:
+    environment:
+      APP_spot_KEY: ${FOCUS_BEAM_SECRET_SHORT}

pscc/modules/osiris2fhir-compose.yml

@@ -1,8 +1,10 @@
 services:
   osiris2fhir:
     container_name: bridgehead-osiris2fhir
-    image: docker.verbis.dkfz.de/ccp/osiris2fhir:${SITE_ID}
+    image: docker.verbis.dkfz.de/ccp/osiris2fhir
     environment:
+      FHIR_PROFILE: ${PROJECT:-pscc}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
       SALT: ${LOCAL_SALT}
     labels:
       - "traefik.enable=true"

pscc/modules/osiris2fhir-setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 if [ -n "$ENABLE_OSIRIS2FHIR" ]; then
-  log INFO "oBDS2FHIR-REST setup detected -- will start osiris2fhir module."
+  log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module."
   OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml"
   LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
 fi

versions/acceptance

@@ -1,6 +1,7 @@
 FOCUS_TAG=develop
 BEAM_TAG=develop
-BLAZE_TAG=0.32
+BLAZE_TAG=latest
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=develop
 MTBA_TAG=develop
+DATA_QUALITY_AGENT_TAG=latest

versions/prod

@@ -1,6 +1,7 @@
 FOCUS_TAG=main
 BEAM_TAG=main
-BLAZE_TAG=0.32
+BLAZE_TAG=1.8
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=main
 MTBA_TAG=main
+DATA_QUALITY_AGENT_TAG=0.1

versions/test

@@ -1,6 +1,7 @@
 FOCUS_TAG=develop
 BEAM_TAG=develop
-BLAZE_TAG=0.32
+BLAZE_TAG=latest
 POSTGRES_TAG=15.13-alpine
 TEILER_DASHBOARD_TAG=develop
 MTBA_TAG=develop
+DATA_QUALITY_AGENT_TAG=latest