diff --git a/README.md b/README.md index d0e462a7..03a1b637 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ This repository is the starting point for any information and tools you will nee - [Teiler (Frontend)](#teiler-frontend) - [Data Exporter Service](#data-exporter-service) - [Data Quality Report](#data-quality-report) + - [Data Quality Agent](#data-quality-agent) 4. [Things you should know](#things-you-should-know) - [Auto-Updates](#auto-updates) - [Auto-Backups](#auto-backups) @@ -41,7 +42,7 @@ This repository is the starting point for any information and tools you will nee The data protection officer at your site will probably want to know exactly what our software does with patient data, and you may need to get their approval before you are allowed to install a Bridgehead. To help you with this, we have provided some data protection concepts: -- [Germany](https://www.bbmri.de/biobanking/it/infrastruktur/datenschutzkonzept/) +- [Germany](https://www.netzwerk-universitaetsmedizin.de/plattformen/gbn/biobanking/it/infrastruktur/datenschutzkonzept) ### Hardware @@ -424,6 +425,32 @@ ENABLE_EXPORTER=true ``` [For further information](docs/exporter.md) +### Data Quality Agent + +The Data Quality Agent is an optional module that periodically evaluates the quality of FHIR data stored in Blaze. It generates local data quality reports accessible via the Bridgehead web interface. + +To enable the service, set the following variable in your `.conf` file: + +```bash +ENABLE_DATA_QUALITY_AGENT=true +``` + +#### Sharing Data Quality Reports (recommended) + +We encourage sharing your data quality reports with the central BBMRI-ERIC quality dashboard. The reports contain only aggregated, non-patient-identifiable statistics and help the network to monitor and improve overall data quality. However, quality reporting is completely optional and opt-in. + +To opt in, additionally set the following variables in your `.conf` file: + +```bash +DATA_QUALITY_SERVER_URL=https://quality-dashboard.bbmri-eric.eu +DATA_QUALITY_SERVER_NAME=Central Data Quality Server of BBMRI +``` + +If these variables are not set, the Data Quality Agent will still run and generate local reports, but no data will be shared externally. + +Reports are accessible at `https:///bbmri-data-quality-agent` (default credentials are admin:admin, please change it after first login!!). + +[Official documentation](https://fdqf.bbmri-eric.eu/user/deployment.html) ## Things you should know ### Auto-Updates diff --git a/bbmri/modules/data-quality-agent-compose.yml b/bbmri/modules/data-quality-agent-compose.yml new file mode 100644 index 00000000..272f982b --- /dev/null +++ b/bbmri/modules/data-quality-agent-compose.yml @@ -0,0 +1,23 @@ +version: "3.7" + +services: + data-quality-agent: + image: ghcr.io/bbmri-cz/data-quality-agent:${DATA_QUALITY_AGENT_TAG} + container_name: bridgehead-bbmri-data-quality-agent + environment: + APP_SETTING_FHIR_URL: http://bridgehead-bbmri-blaze:8080/fhir + REPORTING_SERVER_URL: ${DATA_QUALITY_SERVER_URL} + REPORTING_SERVER_NAME: ${DATA_QUALITY_SERVER_NAME} + labels: + - "traefik.enable=true" + - "traefik.http.routers.data_quality_agent_bbmri.rule=PathPrefix(`/bbmri-data-quality-agent`)" + - "traefik.http.services.data_quality_agent_bbmri.loadbalancer.server.port=8082" + - "traefik.http.routers.data_quality_agent_bbmri.tls=true" + - "traefik.http.middlewares.data_quality_agent_bbmri_strip.stripprefix.prefixes=/bbmri-data-quality-agent" + - "traefik.http.routers.data_quality_agent_bbmri.middlewares=data_quality_agent_bbmri_strip,auth" + depends_on: + - "blaze" + volumes: + - /var/cache/bridgehead/bbmri/agent-db:/app/data + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro diff --git a/bbmri/modules/data-quality-agent-setup.sh b/bbmri/modules/data-quality-agent-setup.sh new file mode 100644 index 00000000..f0a0e840 --- /dev/null +++ b/bbmri/modules/data-quality-agent-setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ "$ENABLE_DATA_QUALITY_AGENT" == "true" ]; then + log INFO "Data Quality Agent setup detected -- will start data-quality-agent service." + OVERRIDE+=" -f ./$PROJECT/modules/data-quality-agent-compose.yml" +fi + diff --git a/cce/modules/osiris2fhir-setup.sh b/cce/modules/osiris2fhir-setup.sh new file mode 100644 index 00000000..ce1d6d73 --- /dev/null +++ b/cce/modules/osiris2fhir-setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +if [ -n "$ENABLE_OSIRIS2FHIR" ]; then + log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module." + OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml" + LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +fi \ No newline at end of file diff --git a/itcc/docker-compose.yml b/itcc/docker-compose.yml index 49edff0a..ae9e09fb 100644 --- a/itcc/docker-compose.yml +++ b/itcc/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} + image: docker.verbis.dkfz.de/cache/samply/blaze:latest container_name: bridgehead-itcc-blaze environment: BASE_URL: "http://bridgehead-itcc-blaze:8080" @@ -32,7 +32,7 @@ services: BEAM_PROXY_URL: http://beam-proxy:8081 RETRY_COUNT: ${FOCUS_RETRY_COUNT} EPSILON: 0.28 - QUERIES_TO_CACHE: '/queries_to_cache.conf' + QUERIES_TO_CACHE: "/queries_to_cache.conf" ENDPOINT_TYPE: ${FOCUS_ENDPOINT_TYPE:-blaze} volumes: - /srv/docker/bridgehead/itcc/queries_to_cache.conf:/queries_to_cache.conf:ro @@ -41,12 +41,13 @@ services: - "blaze" beam-proxy: - image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG} + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop-sockets container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} PROXY_ID: ${PROXY_ID} APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT} + APP_omics-endpoint_KEY: ${FOCUS_BEAM_SECRET_SHORT} PRIVKEY_FILE: /run/secrets/proxy.pem ALL_PROXY: http://forward_proxy:3128 TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs @@ -59,7 +60,6 @@ services: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - /srv/docker/bridgehead/itcc/root.crt.pem:/conf/root.crt.pem:ro - volumes: blaze-data: diff --git a/itcc/modules/itcc-omics-ingest.sh b/itcc/modules/itcc-omics-ingest.sh index a078140a..9d0dcb41 100644 --- a/itcc/modules/itcc-omics-ingest.sh +++ b/itcc/modules/itcc-omics-ingest.sh @@ -3,4 +3,25 @@ if [ -n "$ENABLE_OMICS" ];then OVERRIDE+=" -f ./$PROJECT/modules/itcc-omics-ingest.yaml" GENERATE_API_KEY="$(generate_simple_password 'omics')" + PATIENTLIST_POSTGRES_PASSWORD=="$(generate_simple_password 'mainzelliste')" + KEYSET=/var/bridgehead/mainzelliste/keyset_siv.json + if [ ! -f "$KEYSET" ]; then + mkdir -p "$(dirname "$KEYSET")" + KEY_ID=$(($(openssl rand -hex 4 | sed 's/^/0x/') & 0x7FFFFFFF)) + VALUE=$({ printf '\x12\x40'; openssl rand 64; } | base64 | tr -d '\n') + jq -n --argjson id "$KEY_ID" --arg value "$VALUE" '{ + primaryKeyId: $id, + key: [{ + keyData: { + typeUrl: "type.googleapis.com/google.crypto.tink.AesSivKey", + value: $value, + keyMaterialType: "SYMMETRIC" + }, + status: "ENABLED", + keyId: $id, + outputPrefixType: "TINK" + }] + }' > "$KEYSET" + chmod 600 "$KEYSET" + fi fi \ No newline at end of file diff --git a/itcc/modules/itcc-omics-ingest.yaml b/itcc/modules/itcc-omics-ingest.yaml index 81084331..74ae5274 100644 --- a/itcc/modules/itcc-omics-ingest.yaml +++ b/itcc/modules/itcc-omics-ingest.yaml @@ -1,14 +1,69 @@ services: omics-endpoint: - image: ghcr.io/samply/itcc-omics-ingest:main + image: samply/itcc-omics-ingest:main environment: - - API_KEY=${GENERATE_API_KEY} - volumes: - - /var/cache/bridgehead/omics/data:/data/uploads + API_KEY: ${GENERATE_API_KEY} + BEAM_APP_ID_LONG: omics-endpoint.${PROXY_ID} + BEAM_SECRET: ${FOCUS_BEAM_SECRET_SHORT} + DWH_SOCKET_ID: ${DWH_SOCKET_ID} + DWH_TASK_ID: ${DWH_TASK_ID} + PARTNER_ID: ${SITE_ID} + ML_API_KEY: ${GENERATE_API_KEY} labels: - - "traefik.http.routers.omics.rule=Host(`${HOST}`) && PathPrefix(`/api/omics`)" + - "traefik.http.routers.omics.rule=Host(`${HOST}`) && + PathPrefix(`/api/upload`)" - "traefik.enable=true" - "traefik.http.services.omics.loadbalancer.server.port=6080" - "traefik.http.routers.omics.tls=true" - "traefik.http.middlewares.omics-stripprefix.stripprefix.prefixes=/api" - "traefik.http.routers.omics.middlewares=omics-stripprefix" + + patientlist-db: + image: postgres:${POSTGRES_TAG} + container_name: bridgehead-patientlist-db + restart: unless-stopped + environment: + POSTGRES_DB: mainzelliste + POSTGRES_USER: ${ML_DB_USER} + POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} + volumes: + - "patientlist-db-data:/var/lib/postgresql/data" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${ML_DB_USER} -d mainzelliste"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + + patientlist: + image: medicalinformatics/mainzelliste:latest + container_name: bridgehead-patientlist + restart: unless-stopped + depends_on: + patientlist-db: + condition: service_healthy + environment: + ML_API_KEY: ${GENERATE_API_KEY} + ML_DB_HOST: patientlist-db + ML_DB_PORT: "5432" + ML_DB_NAME: mainzelliste + ML_DB_USER: ${ML_DB_USER} + ML_DB_PASS: ${PATIENTLIST_POSTGRES_PASSWORD} + ML_DB_DRIVER: org.postgresql.Driver + ML_DB_TYPE: postgresql + ML_LOG_LEVEL: INFO + ML_ALLOWEDREMOTEADDRESSES: "127.0.0.1,::1,172.16.0.0/12" + + secrets: + - mainzelliste.docker.conf + - source: symmetric_key + target: /etc/resources/keys/symmetric_key.json + +volumes: + patientlist-db-data: +secrets: + mainzelliste.docker.conf: + file: /etc/bridgehead/mainzelliste/mainzelliste.docker.conf + + symmetric_key: + file: /var/bridgehead/mainzelliste/keyset_siv.json diff --git a/itcc/modules/lens-compose.yml b/itcc/modules/lens-compose.yml index 5a5b78cc..a548cde9 100644 --- a/itcc/modules/lens-compose.yml +++ b/itcc/modules/lens-compose.yml @@ -7,6 +7,7 @@ services: HOST: "0.0.0.0" BIND_ADDR: "0.0.0.0:3000" PUBLIC_ENVIRONMENT: ${PUBLIC_ENVIRONMENT} + PUBLIC_SPOT_URL: https://${HOST}/prod labels: - "traefik.enable=true" - "traefik.http.routers.itcc.rule=Host(`${HOST}`) && PathPrefix(`/`)" diff --git a/itcc/vars b/itcc/vars index 3eee6525..26fbc254 100644 --- a/itcc/vars +++ b/itcc/vars @@ -7,6 +7,9 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL PUBLIC_ENVIRONMENT=prod +DWH_SOCKET_ID=socket.itcc-datalake.${BROKER_ID} +DWH_TASK_ID=task.itcc-datalake.${BROKER_ID} +ML_DB_USER=mainzelliste for module in $PROJECT/modules/*.sh do diff --git a/lib/functions.sh b/lib/functions.sh index 268434e2..686ff8a8 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -337,7 +337,7 @@ function sync_secrets() { } function secret_sync_gitlab_token() { - if [[ "$PROJECT" != "dktk" && "$PROJECT" != "bbmri" ]]; then + if [[ "$PROJECT" != "ccp" && "$PROJECT" != "bbmri" ]]; then log "INFO" "Not running Secret Sync for project minimal" return fi diff --git a/minimal/modules/dnpm-central-targets.json b/minimal/modules/dnpm-central-targets.json index 5469da03..9b28c373 100644 --- a/minimal/modules/dnpm-central-targets.json +++ b/minimal/modules/dnpm-central-targets.json @@ -49,7 +49,7 @@ "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" }, { - "id": "Charite", + "id": "Charité", "name": "Berlin", "virtualhost": "charite.dnpm.de", "beamconnect": "dnpm-connect.berlin-test.broker.ccp-it.dktk.dkfz.de" diff --git a/pscc/modules/lens-compose.yml b/pscc/modules/lens-compose.yml index 4571c7b2..f5ef163e 100644 --- a/pscc/modules/lens-compose.yml +++ b/pscc/modules/lens-compose.yml @@ -2,7 +2,9 @@ version: "3.7" services: lens: container_name: lens-federated-search - image: docker.verbis.dkfz.de/ccp/lens:${SITE_ID} + image: docker.verbis.dkfz.de/ccp/lens:pscc + environment: + PUBLIC_SPOT_URL: https://${HOST}/prod labels: - "traefik.http.services.lens.loadbalancer.server.port=3000" - "traefik.enable=true" @@ -11,7 +13,6 @@ services: spot: image: samply/rustyspot:latest - platform: linux/amd64 environment: HTTP_PROXY: ${HTTP_PROXY_URL} HTTPS_PROXY: ${HTTPS_PROXY_URL} @@ -37,4 +38,8 @@ services: - "traefik.http.routers.spot.rule=Host(`${HOST}`) && PathPrefix(`/prod`)" - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/prod" - "traefik.http.routers.spot.tls=true" - - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth" \ No newline at end of file + - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth" + + beam-proxy: + environment: + APP_spot_KEY: ${FOCUS_BEAM_SECRET_SHORT} \ No newline at end of file diff --git a/pscc/modules/osiris2fhir-compose.yml b/pscc/modules/osiris2fhir-compose.yml index a5fbffbe..6b346d93 100644 --- a/pscc/modules/osiris2fhir-compose.yml +++ b/pscc/modules/osiris2fhir-compose.yml @@ -1,8 +1,10 @@ services: osiris2fhir: container_name: bridgehead-osiris2fhir - image: docker.verbis.dkfz.de/ccp/osiris2fhir:${SITE_ID} + image: docker.verbis.dkfz.de/ccp/osiris2fhir environment: + FHIR_PROFILE: ${PROJECT:-pscc} + LOG_LEVEL: ${LOG_LEVEL:-INFO} SALT: ${LOCAL_SALT} labels: - "traefik.enable=true" diff --git a/pscc/modules/osiris2fhir-setup.sh b/pscc/modules/osiris2fhir-setup.sh index 852a3a85..ce1d6d73 100644 --- a/pscc/modules/osiris2fhir-setup.sh +++ b/pscc/modules/osiris2fhir-setup.sh @@ -1,6 +1,6 @@ #!/bin/bash if [ -n "$ENABLE_OSIRIS2FHIR" ]; then - log INFO "oBDS2FHIR-REST setup detected -- will start osiris2fhir module." + log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module." OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml" LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" fi \ No newline at end of file diff --git a/versions/acceptance b/versions/acceptance index 89d509ec..648e371e 100644 --- a/versions/acceptance +++ b/versions/acceptance @@ -1,6 +1,7 @@ FOCUS_TAG=develop BEAM_TAG=develop -BLAZE_TAG=0.32 +BLAZE_TAG=latest POSTGRES_TAG=15.13-alpine TEILER_DASHBOARD_TAG=develop -MTBA_TAG=develop \ No newline at end of file +MTBA_TAG=develop +DATA_QUALITY_AGENT_TAG=latest diff --git a/versions/prod b/versions/prod index 7f6642df..a336e123 100644 --- a/versions/prod +++ b/versions/prod @@ -1,6 +1,7 @@ FOCUS_TAG=main BEAM_TAG=main -BLAZE_TAG=0.32 +BLAZE_TAG=1.8 POSTGRES_TAG=15.13-alpine TEILER_DASHBOARD_TAG=main -MTBA_TAG=main \ No newline at end of file +MTBA_TAG=main +DATA_QUALITY_AGENT_TAG=0.1 diff --git a/versions/test b/versions/test index b1a3a402..648e371e 100644 --- a/versions/test +++ b/versions/test @@ -1,6 +1,7 @@ FOCUS_TAG=develop BEAM_TAG=develop -BLAZE_TAG=0.32 +BLAZE_TAG=latest POSTGRES_TAG=15.13-alpine TEILER_DASHBOARD_TAG=develop MTBA_TAG=develop +DATA_QUALITY_AGENT_TAG=latest