diff --git a/README.md b/README.md index d0e462a..03a1b63 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ This repository is the starting point for any information and tools you will nee - [Teiler (Frontend)](#teiler-frontend) - [Data Exporter Service](#data-exporter-service) - [Data Quality Report](#data-quality-report) + - [Data Quality Agent](#data-quality-agent) 4. [Things you should know](#things-you-should-know) - [Auto-Updates](#auto-updates) - [Auto-Backups](#auto-backups) @@ -41,7 +42,7 @@ This repository is the starting point for any information and tools you will nee The data protection officer at your site will probably want to know exactly what our software does with patient data, and you may need to get their approval before you are allowed to install a Bridgehead. To help you with this, we have provided some data protection concepts: -- [Germany](https://www.bbmri.de/biobanking/it/infrastruktur/datenschutzkonzept/) +- [Germany](https://www.netzwerk-universitaetsmedizin.de/plattformen/gbn/biobanking/it/infrastruktur/datenschutzkonzept) ### Hardware @@ -424,6 +425,32 @@ ENABLE_EXPORTER=true ``` [For further information](docs/exporter.md) +### Data Quality Agent + +The Data Quality Agent is an optional module that periodically evaluates the quality of FHIR data stored in Blaze. It generates local data quality reports accessible via the Bridgehead web interface. + +To enable the service, set the following variable in your `.conf` file: + +```bash +ENABLE_DATA_QUALITY_AGENT=true +``` + +#### Sharing Data Quality Reports (recommended) + +We encourage sharing your data quality reports with the central BBMRI-ERIC quality dashboard. The reports contain only aggregated, non-patient-identifiable statistics and help the network to monitor and improve overall data quality. However, quality reporting is completely optional and opt-in. + +To opt in, additionally set the following variables in your `.conf` file: + +```bash +DATA_QUALITY_SERVER_URL=https://quality-dashboard.bbmri-eric.eu +DATA_QUALITY_SERVER_NAME=Central Data Quality Server of BBMRI +``` + +If these variables are not set, the Data Quality Agent will still run and generate local reports, but no data will be shared externally. + +Reports are accessible at `https:///bbmri-data-quality-agent` (default credentials are admin:admin, please change it after first login!!). + +[Official documentation](https://fdqf.bbmri-eric.eu/user/deployment.html) ## Things you should know ### Auto-Updates diff --git a/bbmri/modules/data-quality-agent-compose.yml b/bbmri/modules/data-quality-agent-compose.yml new file mode 100644 index 0000000..272f982 --- /dev/null +++ b/bbmri/modules/data-quality-agent-compose.yml @@ -0,0 +1,23 @@ +version: "3.7" + +services: + data-quality-agent: + image: ghcr.io/bbmri-cz/data-quality-agent:${DATA_QUALITY_AGENT_TAG} + container_name: bridgehead-bbmri-data-quality-agent + environment: + APP_SETTING_FHIR_URL: http://bridgehead-bbmri-blaze:8080/fhir + REPORTING_SERVER_URL: ${DATA_QUALITY_SERVER_URL} + REPORTING_SERVER_NAME: ${DATA_QUALITY_SERVER_NAME} + labels: + - "traefik.enable=true" + - "traefik.http.routers.data_quality_agent_bbmri.rule=PathPrefix(`/bbmri-data-quality-agent`)" + - "traefik.http.services.data_quality_agent_bbmri.loadbalancer.server.port=8082" + - "traefik.http.routers.data_quality_agent_bbmri.tls=true" + - "traefik.http.middlewares.data_quality_agent_bbmri_strip.stripprefix.prefixes=/bbmri-data-quality-agent" + - "traefik.http.routers.data_quality_agent_bbmri.middlewares=data_quality_agent_bbmri_strip,auth" + depends_on: + - "blaze" + volumes: + - /var/cache/bridgehead/bbmri/agent-db:/app/data + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro diff --git a/bbmri/modules/data-quality-agent-setup.sh b/bbmri/modules/data-quality-agent-setup.sh new file mode 100644 index 0000000..f0a0e84 --- /dev/null +++ b/bbmri/modules/data-quality-agent-setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ "$ENABLE_DATA_QUALITY_AGENT" == "true" ]; then + log INFO "Data Quality Agent setup detected -- will start data-quality-agent service." + OVERRIDE+=" -f ./$PROJECT/modules/data-quality-agent-compose.yml" +fi + diff --git a/bbmri/modules/eric-compose.yml b/bbmri/modules/eric-compose.yml index e6ed044..65eb6b6 100644 --- a/bbmri/modules/eric-compose.yml +++ b/bbmri/modules/eric-compose.yml @@ -12,6 +12,7 @@ services: BEAM_PROXY_URL: http://beam-proxy-eric:8081 RETRY_COUNT: ${FOCUS_RETRY_COUNT} OBFUSCATE_BBMRI_ERIC_WAY: "true" + CQL_FLAVOUR: "${FOCUS_CQL_FLAVOUR}" depends_on: - "beam-proxy-eric" - "blaze" diff --git a/cce/modules/osiris2fhir-setup.sh b/cce/modules/osiris2fhir-setup.sh new file mode 100644 index 0000000..ce1d6d7 --- /dev/null +++ b/cce/modules/osiris2fhir-setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +if [ -n "$ENABLE_OSIRIS2FHIR" ]; then + log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module." + OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml" + LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +fi \ No newline at end of file diff --git a/itcc/docker-compose.yml b/itcc/docker-compose.yml index 49edff0..ae9e09f 100644 --- a/itcc/docker-compose.yml +++ b/itcc/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} + image: docker.verbis.dkfz.de/cache/samply/blaze:latest container_name: bridgehead-itcc-blaze environment: BASE_URL: "http://bridgehead-itcc-blaze:8080" @@ -32,7 +32,7 @@ services: BEAM_PROXY_URL: http://beam-proxy:8081 RETRY_COUNT: ${FOCUS_RETRY_COUNT} EPSILON: 0.28 - QUERIES_TO_CACHE: '/queries_to_cache.conf' + QUERIES_TO_CACHE: "/queries_to_cache.conf" ENDPOINT_TYPE: ${FOCUS_ENDPOINT_TYPE:-blaze} volumes: - /srv/docker/bridgehead/itcc/queries_to_cache.conf:/queries_to_cache.conf:ro @@ -41,12 +41,13 @@ services: - "blaze" beam-proxy: - image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG} + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop-sockets container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} PROXY_ID: ${PROXY_ID} APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT} + APP_omics-endpoint_KEY: ${FOCUS_BEAM_SECRET_SHORT} PRIVKEY_FILE: /run/secrets/proxy.pem ALL_PROXY: http://forward_proxy:3128 TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs @@ -59,7 +60,6 @@ services: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - /srv/docker/bridgehead/itcc/root.crt.pem:/conf/root.crt.pem:ro - volumes: blaze-data: diff --git a/itcc/modules/itcc-omics-ingest.sh b/itcc/modules/itcc-omics-ingest.sh index a078140..9d0dcb4 100644 --- a/itcc/modules/itcc-omics-ingest.sh +++ b/itcc/modules/itcc-omics-ingest.sh @@ -3,4 +3,25 @@ if [ -n "$ENABLE_OMICS" ];then OVERRIDE+=" -f ./$PROJECT/modules/itcc-omics-ingest.yaml" GENERATE_API_KEY="$(generate_simple_password 'omics')" + PATIENTLIST_POSTGRES_PASSWORD=="$(generate_simple_password 'mainzelliste')" + KEYSET=/var/bridgehead/mainzelliste/keyset_siv.json + if [ ! -f "$KEYSET" ]; then + mkdir -p "$(dirname "$KEYSET")" + KEY_ID=$(($(openssl rand -hex 4 | sed 's/^/0x/') & 0x7FFFFFFF)) + VALUE=$({ printf '\x12\x40'; openssl rand 64; } | base64 | tr -d '\n') + jq -n --argjson id "$KEY_ID" --arg value "$VALUE" '{ + primaryKeyId: $id, + key: [{ + keyData: { + typeUrl: "type.googleapis.com/google.crypto.tink.AesSivKey", + value: $value, + keyMaterialType: "SYMMETRIC" + }, + status: "ENABLED", + keyId: $id, + outputPrefixType: "TINK" + }] + }' > "$KEYSET" + chmod 600 "$KEYSET" + fi fi \ No newline at end of file diff --git a/itcc/modules/itcc-omics-ingest.yaml b/itcc/modules/itcc-omics-ingest.yaml index 8108433..74ae527 100644 --- a/itcc/modules/itcc-omics-ingest.yaml +++ b/itcc/modules/itcc-omics-ingest.yaml @@ -1,14 +1,69 @@ services: omics-endpoint: - image: ghcr.io/samply/itcc-omics-ingest:main + image: samply/itcc-omics-ingest:main environment: - - API_KEY=${GENERATE_API_KEY} - volumes: - - /var/cache/bridgehead/omics/data:/data/uploads + API_KEY: ${GENERATE_API_KEY} + BEAM_APP_ID_LONG: omics-endpoint.${PROXY_ID} + BEAM_SECRET: ${FOCUS_BEAM_SECRET_SHORT} + DWH_SOCKET_ID: ${DWH_SOCKET_ID} + DWH_TASK_ID: ${DWH_TASK_ID} + PARTNER_ID: ${SITE_ID} + ML_API_KEY: ${GENERATE_API_KEY} labels: - - "traefik.http.routers.omics.rule=Host(`${HOST}`) && PathPrefix(`/api/omics`)" + - "traefik.http.routers.omics.rule=Host(`${HOST}`) && + PathPrefix(`/api/upload`)" - "traefik.enable=true" - "traefik.http.services.omics.loadbalancer.server.port=6080" - "traefik.http.routers.omics.tls=true" - "traefik.http.middlewares.omics-stripprefix.stripprefix.prefixes=/api" - "traefik.http.routers.omics.middlewares=omics-stripprefix" + + patientlist-db: + image: postgres:${POSTGRES_TAG} + container_name: bridgehead-patientlist-db + restart: unless-stopped + environment: + POSTGRES_DB: mainzelliste + POSTGRES_USER: ${ML_DB_USER} + POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD} + volumes: + - "patientlist-db-data:/var/lib/postgresql/data" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${ML_DB_USER} -d mainzelliste"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + + patientlist: + image: medicalinformatics/mainzelliste:latest + container_name: bridgehead-patientlist + restart: unless-stopped + depends_on: + patientlist-db: + condition: service_healthy + environment: + ML_API_KEY: ${GENERATE_API_KEY} + ML_DB_HOST: patientlist-db + ML_DB_PORT: "5432" + ML_DB_NAME: mainzelliste + ML_DB_USER: ${ML_DB_USER} + ML_DB_PASS: ${PATIENTLIST_POSTGRES_PASSWORD} + ML_DB_DRIVER: org.postgresql.Driver + ML_DB_TYPE: postgresql + ML_LOG_LEVEL: INFO + ML_ALLOWEDREMOTEADDRESSES: "127.0.0.1,::1,172.16.0.0/12" + + secrets: + - mainzelliste.docker.conf + - source: symmetric_key + target: /etc/resources/keys/symmetric_key.json + +volumes: + patientlist-db-data: +secrets: + mainzelliste.docker.conf: + file: /etc/bridgehead/mainzelliste/mainzelliste.docker.conf + + symmetric_key: + file: /var/bridgehead/mainzelliste/keyset_siv.json diff --git a/itcc/modules/lens-compose.yml b/itcc/modules/lens-compose.yml index 5a5b78c..a548cde 100644 --- a/itcc/modules/lens-compose.yml +++ b/itcc/modules/lens-compose.yml @@ -7,6 +7,7 @@ services: HOST: "0.0.0.0" BIND_ADDR: "0.0.0.0:3000" PUBLIC_ENVIRONMENT: ${PUBLIC_ENVIRONMENT} + PUBLIC_SPOT_URL: https://${HOST}/prod labels: - "traefik.enable=true" - "traefik.http.routers.itcc.rule=Host(`${HOST}`) && PathPrefix(`/`)" diff --git a/itcc/vars b/itcc/vars index 3eee652..26fbc25 100644 --- a/itcc/vars +++ b/itcc/vars @@ -7,6 +7,9 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL PUBLIC_ENVIRONMENT=prod +DWH_SOCKET_ID=socket.itcc-datalake.${BROKER_ID} +DWH_TASK_ID=task.itcc-datalake.${BROKER_ID} +ML_DB_USER=mainzelliste for module in $PROJECT/modules/*.sh do diff --git a/lib/functions.sh b/lib/functions.sh index 520d86a..3a5cfe4 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -337,7 +337,7 @@ function sync_secrets() { } function secret_sync_gitlab_token() { - if [[ "$PROJECT" != "dktk" && "$PROJECT" != "bbmri" ]]; then + if [[ "$PROJECT" != "ccp" && "$PROJECT" != "bbmri" ]]; then log "INFO" "Not running Secret Sync for project minimal" return fi diff --git a/lib/install-bridgehead.sh b/lib/install-bridgehead.sh index f712153..2e2ec69 100755 --- a/lib/install-bridgehead.sh +++ b/lib/install-bridgehead.sh @@ -21,7 +21,6 @@ Cmnd_Alias BRIDGEHEAD${PROJECT^^} = \\ /bin/systemctl start bridgehead@${PROJECT}.service, \\ /bin/systemctl stop bridgehead@${PROJECT}.service, \\ /bin/systemctl restart bridgehead@${PROJECT}.service, \\ - /bin/systemctl restart bridgehead@*.service, \\ /bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead, \\ /usr/bin/chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead diff --git a/lib/update-bridgehead.sh b/lib/update-bridgehead.sh index 8ae3bde..cf846db 100755 --- a/lib/update-bridgehead.sh +++ b/lib/update-bridgehead.sh @@ -98,7 +98,7 @@ if [ $git_updated = "true" ] || [ $docker_updated = "true" ]; then RES="Updates detected, now restarting bridgehead:\n$CHANGES" log "INFO" "$RES" hc_send log "$RES" - sudo /bin/systemctl restart bridgehead@*.service + sudo /bin/systemctl restart bridgehead@${PROJECT}.service else RES="Nothing updated, nothing to restart." log "INFO" "$RES" diff --git a/minimal/modules/dnpm-central-targets.json b/minimal/modules/dnpm-central-targets.json index 5469da0..9b28c37 100644 --- a/minimal/modules/dnpm-central-targets.json +++ b/minimal/modules/dnpm-central-targets.json @@ -49,7 +49,7 @@ "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" }, { - "id": "Charite", + "id": "Charité", "name": "Berlin", "virtualhost": "charite.dnpm.de", "beamconnect": "dnpm-connect.berlin-test.broker.ccp-it.dktk.dkfz.de" diff --git a/pscc/modules/lens-compose.yml b/pscc/modules/lens-compose.yml index 4571c7b..f5ef163 100644 --- a/pscc/modules/lens-compose.yml +++ b/pscc/modules/lens-compose.yml @@ -2,7 +2,9 @@ version: "3.7" services: lens: container_name: lens-federated-search - image: docker.verbis.dkfz.de/ccp/lens:${SITE_ID} + image: docker.verbis.dkfz.de/ccp/lens:pscc + environment: + PUBLIC_SPOT_URL: https://${HOST}/prod labels: - "traefik.http.services.lens.loadbalancer.server.port=3000" - "traefik.enable=true" @@ -11,7 +13,6 @@ services: spot: image: samply/rustyspot:latest - platform: linux/amd64 environment: HTTP_PROXY: ${HTTP_PROXY_URL} HTTPS_PROXY: ${HTTPS_PROXY_URL} @@ -37,4 +38,8 @@ services: - "traefik.http.routers.spot.rule=Host(`${HOST}`) && PathPrefix(`/prod`)" - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/prod" - "traefik.http.routers.spot.tls=true" - - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth" \ No newline at end of file + - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth" + + beam-proxy: + environment: + APP_spot_KEY: ${FOCUS_BEAM_SECRET_SHORT} \ No newline at end of file diff --git a/pscc/modules/osiris2fhir-compose.yml b/pscc/modules/osiris2fhir-compose.yml index a5fbffb..6b346d9 100644 --- a/pscc/modules/osiris2fhir-compose.yml +++ b/pscc/modules/osiris2fhir-compose.yml @@ -1,8 +1,10 @@ services: osiris2fhir: container_name: bridgehead-osiris2fhir - image: docker.verbis.dkfz.de/ccp/osiris2fhir:${SITE_ID} + image: docker.verbis.dkfz.de/ccp/osiris2fhir environment: + FHIR_PROFILE: ${PROJECT:-pscc} + LOG_LEVEL: ${LOG_LEVEL:-INFO} SALT: ${LOCAL_SALT} labels: - "traefik.enable=true" diff --git a/pscc/modules/osiris2fhir-setup.sh b/pscc/modules/osiris2fhir-setup.sh index 852a3a8..ce1d6d7 100644 --- a/pscc/modules/osiris2fhir-setup.sh +++ b/pscc/modules/osiris2fhir-setup.sh @@ -1,6 +1,6 @@ #!/bin/bash if [ -n "$ENABLE_OSIRIS2FHIR" ]; then - log INFO "oBDS2FHIR-REST setup detected -- will start osiris2fhir module." + log INFO "OSIRIS2FHIR-REST setup detected -- will start osiris2fhir module." OVERRIDE+=" -f ./pscc/modules/osiris2fhir-compose.yml" LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" fi \ No newline at end of file diff --git a/versions/acceptance b/versions/acceptance index 89d509e..648e371 100644 --- a/versions/acceptance +++ b/versions/acceptance @@ -1,6 +1,7 @@ FOCUS_TAG=develop BEAM_TAG=develop -BLAZE_TAG=0.32 +BLAZE_TAG=latest POSTGRES_TAG=15.13-alpine TEILER_DASHBOARD_TAG=develop -MTBA_TAG=develop \ No newline at end of file +MTBA_TAG=develop +DATA_QUALITY_AGENT_TAG=latest diff --git a/versions/prod b/versions/prod index 7f6642d..a336e12 100644 --- a/versions/prod +++ b/versions/prod @@ -1,6 +1,7 @@ FOCUS_TAG=main BEAM_TAG=main -BLAZE_TAG=0.32 +BLAZE_TAG=1.8 POSTGRES_TAG=15.13-alpine TEILER_DASHBOARD_TAG=main -MTBA_TAG=main \ No newline at end of file +MTBA_TAG=main +DATA_QUALITY_AGENT_TAG=0.1 diff --git a/versions/test b/versions/test index b1a3a40..648e371 100644 --- a/versions/test +++ b/versions/test @@ -1,6 +1,7 @@ FOCUS_TAG=develop BEAM_TAG=develop -BLAZE_TAG=0.32 +BLAZE_TAG=latest POSTGRES_TAG=15.13-alpine TEILER_DASHBOARD_TAG=develop MTBA_TAG=develop +DATA_QUALITY_AGENT_TAG=latest