diff --git a/README.md b/README.md index 2534ee7..3c36053 100644 --- a/README.md +++ b/README.md @@ -301,19 +301,19 @@ Once you have added your biobank to the Directory you got persistent identifier The Bridgehead's **Directory Sync** is an optional feature that keeps the Directory up to date with your local data, e.g. number of samples. Conversely, it also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly set your country specific directory URL, username and password to enable this feature. +You should talk with your local data protection group regarding the information that is published by Directory sync. + Full details can be found in [directory_sync_service](https://github.com/samply/directory_sync_service). To enable it, you will need to set these variables to the ```bbmri.conf``` file of your GitLab repository. Here is an example config: ``` -DS_DIRECTORY_URL=https://directory.bbmri-eric.eu DS_DIRECTORY_USER_NAME=your_directory_username -DS_DIRECTORY_USER_PASS=qwdnqwswdvqHBVGFR9887 -DS_TIMER_CRON="0 22 * * *" +DS_DIRECTORY_USER_PASS=your_directory_password ``` -You must contact the Directory team for your national node to find the URL, and to register as a user. +Please contact your National Node to obtain this information. -Additionally, you should choose when you want Directory sync to run. In the example above, this is set to happen at 10 pm every evening. You can modify this to suit your requirements. The timer specification should follow the [cron](https://crontab.guru) convention. +Optionally, you **may** change when you want Directory sync to run by specifying a [cron](https://crontab.guru) expression, e.g. `DS_TIMER_CRON="0 22 * * *"` for 10 pm every evening. Once you edited the gitlab config, the bridgehead will autoupdate the config with the values and will sync the data. diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 178f46c..33a7d31 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -4,7 +4,13 @@ services: directory_sync_service: image: "docker.verbis.dkfz.de/cache/samply/directory_sync_service" environment: - DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} + DS_DIRECTORY_URL: ${DS_DIRECTORY_URL:-https://directory.bbmri-eric.eu} DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} - DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} - DS_TIMER_CRON: ${DS_TIMER_CRON} + DS_DIRECTORY_USER_PASS: ${DS_DIRECTORY_USER_PASS} + DS_TIMER_CRON: ${DS_TIMER_CRON:-0 22 * * *} + DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL:-true} + DS_DIRECTORY_MOCK: ${DS_DIRECTORY_MOCK} + DS_DIRECTORY_DEFAULT_COLLECTION_ID: ${DS_DIRECTORY_DEFAULT_COLLECTION_ID} + DS_DIRECTORY_COUNTRY: ${DS_DIRECTORY_COUNTRY} + depends_on: + - "blaze" diff --git a/bridgehead b/bridgehead index 81b19b3..37b3047 100755 --- a/bridgehead +++ b/bridgehead @@ -76,6 +76,7 @@ loadVars() { # Set some project-independent default values : ${ENVIRONMENT:=production} + export ENVIRONMENT case "$ENVIRONMENT" in "production") diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 95ff9c3..fcabc9b 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -31,6 +31,9 @@ services: BEAM_PROXY_URL: http://beam-proxy:8081 RETRY_COUNT: ${FOCUS_RETRY_COUNT} EPSILON: 0.28 + QUERIES_TO_CACHE: '/queries_to_cache.conf' + volumes: + - /srv/docker/bridgehead/ccp/queries_to_cache.conf:/queries_to_cache.conf depends_on: - "beam-proxy" - "blaze" diff --git a/ccp/modules/fhir2sql-compose.yml b/ccp/modules/fhir2sql-compose.yml new file mode 100644 index 0000000..7733787 --- /dev/null +++ b/ccp/modules/fhir2sql-compose.yml @@ -0,0 +1,25 @@ +version: "3.7" + +services: + fhir2sql: + depends_on: + - "dashboard-db" + - "blaze" + image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest + container_name: bridgehead-ccp-dashboard-fhir2sql + environment: + BLAZE_BASE_URL: "http://bridgehead-ccp-blaze:8080" + PG_HOST: "dashboard-db" + PG_USERNAME: "dashboard" + PG_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in dashboard-setup.sh + PG_DBNAME: "dashboard" + + dashboard-db: + image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} + container_name: bridgehead-ccp-dashboard-db + environment: + POSTGRES_USER: "dashboard" + POSTGRES_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in dashboard-setup.sh + POSTGRES_DB: "dashboard" + volumes: + - "/var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data" diff --git a/ccp/modules/fhir2sql-setup.sh b/ccp/modules/fhir2sql-setup.sh new file mode 100644 index 0000000..64fa7c0 --- /dev/null +++ b/ccp/modules/fhir2sql-setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +if [ "$ENABLE_FHIR2SQL" == true ]; then + log INFO "Dashboard setup detected -- will start Dashboard backend and FHIR2SQL service." + OVERRIDE+=" -f ./$PROJECT/modules/fhir2sql-compose.yml" + DASHBOARD_DB_PASSWORD="$(generate_simple_password 'fhir2sql')" +fi diff --git a/ccp/modules/fhir2sql.md b/ccp/modules/fhir2sql.md new file mode 100644 index 0000000..deea710 --- /dev/null +++ b/ccp/modules/fhir2sql.md @@ -0,0 +1,36 @@ +# fhir2sql +fhir2sql connects to Blaze, retrieves data, and syncs it with a PostgreSQL database. The application is designed to run continuously, syncing data at regular intervals. +The Dashboard module is a optional component of the Bridgehead CCP setup. When enabled, it starts two Docker services: **fhir2sql** and **dashboard-db**. Data held in PostgreSQL is only stored temporarily and Blaze is considered to be the 'leading system' or 'source of truth'. + +## Services +### fhir2sql +* Image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest +* Container name: bridgehead-ccp-dashboard-fhir2sql +* Depends on: dashboard-db +* Environment variables: + - BLAZE_BASE_URL: The base URL of the Blaze FHIR server (set to http://blaze:8080/fhir/) + - PG_HOST: The hostname of the PostgreSQL database (set to dashboard-db) + - PG_USERNAME: The username for the PostgreSQL database (set to dashboard) + - PG_PASSWORD: The password for the PostgreSQL database (set to the value of DASHBOARD_DB_PASSWORD) + - PG_DBNAME: The name of the PostgreSQL database (set to dashboard) + +### dashboard-db + +* Image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} +* Container name: bridgehead-ccp-dashboard-db +* Environment variables: + - POSTGRES_USER: The username for the PostgreSQL database (set to dashboard) + - POSTGRES_PASSWORD: The password for the PostgreSQL database (set to the value of DASHBOARD_DB_PASSWORD) + - POSTGRES_DB: The name of the PostgreSQL database (set to dashboard) +* Volumes: + - /var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data + +The volume used by dashboard-db can be removed safely and should be restored to a working order by re-importing data from Blaze. + +### Environment Variables +* DASHBOARD_DB_PASSWORD: A generated password for the PostgreSQL database, created using a salt string and the SHA1 hash function. +* POSTGRES_TAG: The tag of the PostgreSQL image to use (not set in this module, but required by the dashboard-db service). + + +### Setup +To enable the Dashboard module, set the ENABLE_FHIR2SQL environment variable to true. The dashboard-setup.sh script will then start the fhir2sql and dashboard-db services, using the environment variables and volumes defined above. \ No newline at end of file diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 61a4733..f9156cf 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -14,15 +14,15 @@ services: MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY} MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} - MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} - MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET} depends_on: - patientlist + - traefik-forward-auth labels: - "traefik.enable=true" - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)" - "traefik.http.services.id-manager.loadbalancer.server.port=8080" - "traefik.http.routers.id-manager.tls=true" + - "traefik.http.routers.id-manager.middlewares=traefik-forward-auth-idm" patientlist: image: docker.verbis.dkfz.de/bridgehead/mainzelliste @@ -56,5 +56,41 @@ services: # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!! - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/" + traefik-forward-auth: + image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:v7.6.0 + environment: + - http_proxy=http://forward_proxy:3128 + - https_proxy=http://forward_proxy:3128 + - OAUTH2_PROXY_PROVIDER=oidc + - OAUTH2_PROXY_SKIP_PROVIDER_BUTTON=true + - OAUTH2_PROXY_OIDC_ISSUER_URL=https://login.verbis.dkfz.de/realms/master + - OAUTH2_PROXY_CLIENT_ID=bridgehead-${SITE_ID} + - OAUTH2_PROXY_CLIENT_SECRET=${IDMANAGER_AUTH_CLIENT_SECRET} + - OAUTH2_PROXY_COOKIE_SECRET=${IDMANAGER_AUTH_COOKIE_SECRET} + - OAUTH2_PROXY_COOKIE_DOMAINS=.${HOST} + - OAUTH2_PROXY_HTTP_ADDRESS=:4180 + - OAUTH2_PROXY_REVERSE_PROXY=true + - OAUTH2_PROXY_WHITELIST_DOMAINS=.${HOST} + - OAUTH2_PROXY_UPSTREAMS=static://202 + - OAUTH2_PROXY_EMAIL_DOMAINS=* + - OAUTH2_PROXY_SCOPE=openid profile email + # Pass Authorization Header and some user information to backend services + - OAUTH2_PROXY_SET_AUTHORIZATION_HEADER=true + - OAUTH2_PROXY_SET_XAUTHREQUEST=true + # Keycloak has an expiration time of 60s therefore oauth2-proxy needs to refresh after that + - OAUTH2_PROXY_COOKIE_REFRESH=60s + - OAUTH2_PROXY_ALLOWED_GROUPS=DKTK-CCP-PPSN + - OAUTH2_PROXY_PROXY_PREFIX=/oauth2-idm + labels: + - "traefik.enable=true" + - "traefik.http.services.traefik-forward-auth.loadbalancer.server.port=4180" + - "traefik.http.routers.traefik-forward-auth.rule=Host(`${HOST}`) && PathPrefix(`/oauth2-idm`)" + - "traefik.http.routers.traefik-forward-auth.tls=true" + - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.address=http://traefik-forward-auth:4180" + - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.authResponseHeaders=Authorization" + depends_on: + forward_proxy: + condition: service_healthy + volumes: patientlist-db-data: diff --git a/ccp/queries_to_cache.conf b/ccp/queries_to_cache.conf new file mode 100644 index 0000000..b950312 --- /dev/null +++ b/ccp/queries_to_cache.conf @@ -0,0 +1,2 @@ +bGlicmFyeSBSZXRyaWV2ZQp1c2luZyBGSElSIHZlcnNpb24gJzQuMC4wJwppbmNsdWRlIEZISVJIZWxwZXJzIHZlcnNpb24gJzQuMC4wJwoKY29kZXN5c3RlbSBsb2luYzogJ2h0dHA6Ly9sb2luYy5vcmcnCgpjb250ZXh0IFBhdGllbnQKCgpES1RLX1NUUkFUX0dFTkRFUl9TVFJBVElGSUVSCgpES1RLX1NUUkFUX1BSSU1BUllfRElBR05PU0lTX05PX1NPUlRfU1RSQVRJRklFUgpES1RLX1NUUkFUX0FHRV9DTEFTU19TVFJBVElGSUVSCgpES1RLX1NUUkFUX0RFQ0VBU0VEX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfRElBR05PU0lTX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfU1BFQ0lNRU5fU1RSQVRJRklFUgoKREtUS19TVFJBVF9QUk9DRURVUkVfU1RSQVRJRklFUgoKREtUS19TVFJBVF9NRURJQ0FUSU9OX1NUUkFUSUZJRVIKCiAgREtUS19TVFJBVF9ISVNUT0xPR1lfU1RSQVRJRklFUgpES1RLX1NUUkFUX0RFRl9JTl9JTklUSUFMX1BPUFVMQVRJT04KdHJ1ZQ== +bGlicmFyeSBSZXRyaWV2ZQp1c2luZyBGSElSIHZlcnNpb24gJzQuMC4wJwppbmNsdWRlIEZISVJIZWxwZXJzIHZlcnNpb24gJzQuMC4wJwoKY29kZXN5c3RlbSBsb2luYzogJ2h0dHA6Ly9sb2luYy5vcmcnCmNvZGVzeXN0ZW0gaWNkMTA6ICdodHRwOi8vZmhpci5kZS9Db2RlU3lzdGVtL2JmYXJtL2ljZC0xMC1nbScKY29kZXN5c3RlbSBtb3JwaDogJ3VybjpvaWQ6Mi4xNi44NDAuMS4xMTM4ODMuNi40My4xJwoKY29udGV4dCBQYXRpZW50CgoKREtUS19TVFJBVF9HRU5ERVJfU1RSQVRJRklFUgoKREtUS19TVFJBVF9QUklNQVJZX0RJQUdOT1NJU19OT19TT1JUX1NUUkFUSUZJRVIKREtUS19TVFJBVF9BR0VfQ0xBU1NfU1RSQVRJRklFUgoKREtUS19TVFJBVF9ERUNFQVNFRF9TVFJBVElGSUVSCgpES1RLX1NUUkFUX0RJQUdOT1NJU19TVFJBVElGSUVSCgpES1RLX1NUUkFUX1NQRUNJTUVOX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfUFJPQ0VEVVJFX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfTUVESUNBVElPTl9TVFJBVElGSUVSCgogIERLVEtfU1RSQVRfSElTVE9MT0dZX1NUUkFUSUZJRVIKREtUS19TVFJBVF9ERUZfSU5fSU5JVElBTF9QT1BVTEFUSU9OKGV4aXN0cyBbQ29uZGl0aW9uOiBDb2RlICdDNjEnIGZyb20gaWNkMTBdKSBhbmQgCigoZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzgxNDAvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzgxNDcvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzg0ODAvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzg1MDAvMycpKQ== \ No newline at end of file diff --git a/lib/systemd/bridgehead-update@.timer b/lib/systemd/bridgehead-update@.timer index 4c8fada..d9abdf4 100644 --- a/lib/systemd/bridgehead-update@.timer +++ b/lib/systemd/bridgehead-update@.timer @@ -1,8 +1,9 @@ [Unit] -Description=Hourly Updates of Bridgehead (%i) +Description=Daily Updates at 6am of Bridgehead (%i) [Timer] -OnCalendar=*-*-* *:00:00 +OnCalendar=*-*-* 06:00:00 +Persistent=true [Install] WantedBy=basic.target diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index e9f53d6..6e8818f 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -58,3 +58,4 @@ services: HOST: ${HOST} PROJECT: ${PROJECT} SITE_NAME: ${SITE_NAME} + ENVIRONMENT: ${ENVIRONMENT}