feature: add github action for automated pull request for the pilot branch, after pull requests on main

fix: specify host for id-management login

.github/workflows/auto-pr.yml

@@ -0,0 +1,21 @@
+name: Automatically generate Pull Requests for feature/pilot-projects
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  create_pr:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Create Pull Request to feature/pilot-projects branch
+        run: gh pr create -B feature/pilot-projects -H main --title 'Create Pull Request to feature/pilot-projects branch' --body 'Created by Github action'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -200,7 +200,7 @@ sudo systemctl [enable|disable] bridgehead@<PROJECT>.service
 After starting the Bridgehead, you can watch the initialization process with the following command:
 
 ```shell
-journalctl -u bridgehead@bbmri -f
+/srv/docker/bridgehead/bridgehead logs <project> -f
 ```
 
 if this exits with something similar to the following:
@@ -220,8 +220,9 @@ docker ps
 There should be 6 - 10 Docker proceses. If there are fewer, then you know that something has gone wrong. To see what is going on, run:
 
 ```shell
-journalctl -u bridgehead@bbmri -f
+/srv/docker/bridgehead/bridgehead logs <Project> -f
 ```
+This translates to a journalctl command so all the regular journalctl flags can be used.
 
 Once the Bridgehead has passed these checks, take a look at the landing page:
 
@@ -235,7 +236,7 @@ You can either do this in a browser or with curl. If you visit the URL in the br
 curl -k https://localhost
 ```
 
-If you get errors when you do this, you need to use ```docker logs``` to examine your landing page container in order to determine what is going wrong.
+Should the landing page not show anything, you can inspect the logs of the containers to determine what is going wrong. To do this you can use `./bridgehead docker-logs <Project> -f` to follow the logs of the container. This transaltes to a docker compose logs command meaning all the ususal docker logs flags work.
 
 If you have chosen to take part in our monitoring program (by setting the ```MONITOR_APIKEY``` variable in the configuration), you will be informed by email when problems are detected in your Bridgehead.
 
@@ -298,19 +299,19 @@ Once you have added your biobank to the Directory you got persistent identifier
 
 The Bridgehead's **Directory Sync** is an optional feature that keeps the Directory up to date with your local data, e.g. number of samples. Conversely, it also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly set your country specific directory URL, username and password to enable this feature.
 
+You should talk with your local data protection group regarding the information that is published by Directory sync.
+
 Full details can be found in [directory_sync_service](https://github.com/samply/directory_sync_service).
 
 To enable it, you will need to set these variables to the ```bbmri.conf``` file of your GitLab repository. Here is an example config:
 
 ```
-DS_DIRECTORY_URL=https://directory.bbmri-eric.eu
 DS_DIRECTORY_USER_NAME=your_directory_username
-DS_DIRECTORY_USER_PASS=qwdnqwswdvqHBVGFR9887
+DS_DIRECTORY_USER_PASS=your_directory_password
-DS_TIMER_CRON="0 22 * * *"
 ```
-You must contact the Directory team for your national node to find the URL, and to register as a user.
+Please contact your National Node to obtain this information.
 
-Additionally, you should choose when you want Directory sync to run. In the example above, this is set to happen at 10 pm every evening. You can modify this to suit your requirements. The timer specification should follow the [cron](https://crontab.guru) convention.
+Optionally, you **may** change when you want Directory sync to run by specifying a [cron](https://crontab.guru) expression, e.g. `DS_TIMER_CRON="0 22 * * *"` for 10 pm every evening.
 
 Once you edited the gitlab config, the bridgehead will autoupdate the config with the values and will sync the data.
 

bbmri/docker-compose.yml

@@ -4,7 +4,7 @@ version: "3.7"
 
 services:
   blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:latest
+    image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
     container_name: bridgehead-bbmri-blaze
     environment:
       BASE_URL: "http://bridgehead-bbmri-blaze:8080"

bbmri/modules/directory-sync-compose.yml

@@ -4,7 +4,13 @@ services:
   directory_sync_service:
     image: "docker.verbis.dkfz.de/cache/samply/directory_sync_service"
     environment:
-      DS_DIRECTORY_URL: ${DS_DIRECTORY_URL}
+      DS_DIRECTORY_URL: ${DS_DIRECTORY_URL:-https://directory.bbmri-eric.eu}
       DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME}
-      DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE}
+      DS_DIRECTORY_USER_PASS: ${DS_DIRECTORY_USER_PASS}
-      DS_TIMER_CRON: ${DS_TIMER_CRON}
+      DS_TIMER_CRON: ${DS_TIMER_CRON:-0 22 * * *}
+      DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL:-true}
+      DS_DIRECTORY_MOCK: ${DS_DIRECTORY_MOCK}
+      DS_DIRECTORY_DEFAULT_COLLECTION_ID: ${DS_DIRECTORY_DEFAULT_COLLECTION_ID}
+      DS_DIRECTORY_COUNTRY: ${DS_DIRECTORY_COUNTRY}
+    depends_on:
+      - "blaze"

bbmri/vars

@@ -4,7 +4,7 @@
 # Makes only sense for German Biobanks
 : ${ENABLE_GBN:=false}
 
-FOCUS_RETRY_COUNT=32
+FOCUS_RETRY_COUNT=${FOCUS_RETRY_COUNT:-64}
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 
 for module in $PROJECT/modules/*.sh

bridgehead

@@ -70,6 +70,7 @@ loadVars() {
 
 	# Set some project-independent default values
 	: ${ENVIRONMENT:=production}
+	export ENVIRONMENT
 
 	case "$ENVIRONMENT" in
 		"production")
@@ -107,6 +108,11 @@ case "$ACTION" in
 		exit $?
 		;;
 	logs)
+		loadVars
+		shift 2
+		exec journalctl -u bridgehead@$PROJECT -u bridgehead-update@$PROJECT -a $@
+		;;
+	docker-logs)
 		loadVars
 		shift 2
 		exec $COMPOSE -p $PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE logs -f $@

ccp/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:latest
+    image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
     container_name: bridgehead-ccp-blaze
     environment:
       BASE_URL: "http://bridgehead-ccp-blaze:8080"
@@ -21,7 +21,7 @@ services:
       - "traefik.http.routers.blaze_ccp.tls=true"
 
   focus:
-    image: docker.verbis.dkfz.de/cache/samply/focus:0.4.4
+    image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG}
     container_name: bridgehead-focus
     environment:
       API_KEY: ${FOCUS_BEAM_SECRET_SHORT}
@@ -31,6 +31,9 @@ services:
       BEAM_PROXY_URL: http://beam-proxy:8081
       RETRY_COUNT: ${FOCUS_RETRY_COUNT}
       EPSILON: 0.28
+      QUERIES_TO_CACHE: '/queries_to_cache.conf'
+    volumes:
+      - /srv/docker/bridgehead/ccp/queries_to_cache.conf:/queries_to_cache.conf
     depends_on:
       - "beam-proxy"
       - "blaze"

ccp/modules/adt2fhir-rest-compose.yml

@@ -1,18 +0,0 @@
-version: "3.7"
-
-services:
-  adt2fhir-rest:
-    container_name: bridgehead-adt2fhir-rest
-    image: docker.verbis.dkfz.de/ccp/adt2fhir-rest:main
-    environment:
-      IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID
-      MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
-      SALT: ${LOCAL_SALT}
-    restart: always
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.adt2fhir-rest.rule=PathPrefix(`/adt2fhir-rest`)"
-      - "traefik.http.middlewares.adt2fhir-rest_strip.stripprefix.prefixes=/adt2fhir-rest"
-      - "traefik.http.services.adt2fhir-rest.loadbalancer.server.port=8080"
-      - "traefik.http.routers.adt2fhir-rest.tls=true"
-      - "traefik.http.routers.adt2fhir-rest.middlewares=adt2fhir-rest_strip,auth"

ccp/modules/adt2fhir-rest-setup.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-function adt2fhirRestSetup() {
-  if [ -n "$ENABLE_ADT2FHIR_REST" ]; then
-    log INFO "ADT2FHIR-REST setup detected -- will start adt2fhir-rest API."
-    if [ ! -n "$IDMANAGER_LOCAL_PATIENTLIST_APIKEY" ]; then
-      log ERROR "Missing ID-Management Module! Fix this by setting up ID Management:"
-      exit 1;
-    fi
-    OVERRIDE+=" -f ./$PROJECT/modules/adt2fhir-rest-compose.yml"
-    LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
-  fi
-}

ccp/modules/blaze-secondary-compose.yml

@@ -0,0 +1,32 @@
+version: "3.7"
+
+services:
+  blaze-secondary:
+    image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
+    container_name: bridgehead-ccp-blaze-secondary
+    environment:
+      BASE_URL: "http://bridgehead-ccp-blaze-secondary:8080"
+      JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
+      DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
+      DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
+      ENFORCE_REFERENTIAL_INTEGRITY: "false"
+    volumes:
+      - "blaze-secondary-data:/app/data"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.blaze-secondary_ccp.rule=PathPrefix(`/ccp-localdatamanagement-secondary`)"
+      - "traefik.http.middlewares.ccp_b-secondary_strip.stripprefix.prefixes=/ccp-localdatamanagement-secondary"
+      - "traefik.http.services.blaze-secondary_ccp.loadbalancer.server.port=8080"
+      - "traefik.http.routers.blaze-secondary_ccp.middlewares=ccp_b-secondary_strip,auth"
+      - "traefik.http.routers.blaze-secondary_ccp.tls=true"
+
+  obds2fhir-rest:
+    environment:
+      STORE_PATH: ${STORE_PATH:-http://blaze:8080/fhir}
+
+  exporter:
+    environment:
+      BLAZE_HOST: "blaze-secondary"
+
+volumes:
+  blaze-secondary-data:

ccp/modules/blaze-secondary-setup.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+function blazeSecondarySetup() {
+  if [ -n "$ENABLE_SECONDARY_BLAZE" ]; then
+    log INFO "Secondary Blaze setup detected -- will start second blaze."
+    OVERRIDE+=" -f ./$PROJECT/modules/blaze-secondary-compose.yml"
+    #make oBDS2FHIR ignore ID-Management and replace target Blaze
+    PATIENTLIST_URL=" "
+    STORE_PATH="http://blaze-secondary:8080/fhir"
+  fi
+}

ccp/modules/datashield-compose.yml

@@ -1,6 +1,25 @@
 version: "3.7"
 
 services:
+  rstudio:
+    container_name: bridgehead-rstudio
+    image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest
+    environment:
+      #DEFAULT_USER: "rstudio" # This line is kept for informational purposes
+      PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" # It is required, even if the authentication is disabled
+      DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use
+      HTTP_RELATIVE_PATH: "/rstudio"
+      ALL_PROXY: "http://forward_proxy:3128" # https://rocker-project.org/use/networking.html
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)"
+      - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787"
+      - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio"
+      - "traefik.http.routers.rstudio_ccp.tls=true"
+      - "traefik.http.routers.rstudio_ccp.middlewares=oidcAuth,rstudio_ccp_strip"
+    networks:
+      - rstudio
+
   opal:
     container_name: bridgehead-opal
     image: docker.verbis.dkfz.de/ccp/dktk-opal:latest
@@ -74,14 +93,79 @@ services:
       - beam-proxy
     volumes:
       - /tmp/bridgehead/opal-map/:/map/:ro
+    networks:
+      - default
+      - rstudio
+
+  traefik:
+    labels:
+      - "traefik.http.middlewares.oidcAuth.forwardAuth.address=http://oauth2-proxy:4180/"
+      - "traefik.http.middlewares.oidcAuth.forwardAuth.trustForwardHeader=true"
+      - "traefik.http.middlewares.oidcAuth.forwardAuth.authResponseHeaders=X-Auth-Request-Access-Token,Authorization"
+    networks:
+      - default
+      - rstudio
+  forward_proxy:
+    networks:
+      - default
+      - rstudio
 
   beam-proxy:
     environment:
       APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET}
       APP_token-manager_KEY: ${TOKEN_MANAGER_SECRET}
 
+  # TODO: Allow users of group /DataSHIELD and OIDC_USER_GROUP at the same time:
+  # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/configuration/oauth_provider):
+  # --allowed-groups=/DataSHIELD,OIDC_USER_GROUP
+  oauth2-proxy:
+    image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:latest
+    container_name: bridgehead-oauth2proxy
+    command: >-
+      --allowed-group=DataSHIELD
+      --oidc-groups-claim=${OIDC_GROUP_CLAIM}
+      --auth-logging=true
+      --whitelist-domain=${HOST}
+      --http-address="0.0.0.0:4180"
+      --reverse-proxy=true
+      --upstream="static://202"
+      --email-domain="*"
+      --cookie-name="_BRIDGEHEAD_oauth2"
+      --cookie-secret="${OAUTH2_PROXY_SECRET}"
+      --cookie-expire="12h"
+      --cookie-secure="true"
+      --cookie-httponly="true"
+      #OIDC settings
+      --provider="keycloak-oidc"
+      --provider-display-name="VerbIS Login"
+      --client-id="${OIDC_PRIVATE_CLIENT_ID}"
+      --client-secret="${OIDC_CLIENT_SECRET}"
+      --redirect-url="https://${HOST}${OAUTH2_CALLBACK}"
+      --oidc-issuer-url="${OIDC_ISSUER_URL}"
+      --scope="openid email profile"
+      --code-challenge-method="S256"
+      --skip-provider-button=true
+      #X-Forwarded-Header settings - true/false depending on your needs
+      --pass-basic-auth=true
+      --pass-user-headers=false
+      --pass-access-token=false
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`)"
+      - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180"
+      - "traefik.http.routers.oauth2_proxy.tls=true"
+    environment:
+      http_proxy: "http://forward_proxy:3128"
+      https_proxy: "http://forward_proxy:3128"
+    depends_on:
+      forward_proxy:
+        condition: service_healthy
+
 secrets:
   opal-cert.pem:
     file: /tmp/bridgehead/opal-cert.pem
   opal-key.pem:
     file: /tmp/bridgehead/opal-key.pem
+
+networks:
+  rstudio:

ccp/modules/datashield-setup.sh

@@ -5,12 +5,17 @@ if [ "$ENABLE_DATASHIELD" == true ]; then
   if [ -z "${ENABLE_EXPORTER}" ] || [ "${ENABLE_EXPORTER}" != "true" ]; then
     log WARN "The ENABLE_EXPORTER variable is either not set or not set to 'true'."
   fi
+  OAUTH2_CALLBACK=/oauth2/callback
+  OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)"
+  add_private_oidc_redirect_url "${OAUTH2_CALLBACK}"
+
   log INFO "DataSHIELD setup detected -- will start DataSHIELD services."
   OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml"
   EXPORTER_OPAL_PASSWORD="$(generate_password \"exporter in Opal\")"
   TOKEN_MANAGER_OPAL_PASSWORD="$(generate_password \"Token Manager in Opal\")"
   OPAL_DB_PASSWORD="$(echo \"Opal DB\" | generate_simple_password)"
   OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")"
+  RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")"
   DATASHIELD_CONNECT_SECRET="$(echo \"DataShield Connect\" | generate_simple_password)"
   TOKEN_MANAGER_SECRET="$(echo \"Token Manager\" | generate_simple_password)"
   if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then
@@ -18,12 +23,18 @@ if [ "$ENABLE_DATASHIELD" == true ]; then
     openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=opal/C=DE"
   fi
   mkdir -p /tmp/bridgehead/opal-map
-  echo '{"sites": []}' >/tmp/bridgehead/opal-map/central.json
+  sites="$(cat ./$PROJECT/modules/datashield-sites.json)"
-  echo '[{
+  echo "$sites" | docker_jq -n --args '{"sites": input | map({
-    "external": "'$SITE_ID':443",
+    "name": .,
+    "id": .,
+    "virtualhost": "\(.):443",
+    "beamconnect": "datashield-connect.\(.).'"$BROKER_ID"'"
+  })}' $sites >/tmp/bridgehead/opal-map/central.json
+  echo "$sites" | docker_jq -n --args '[{
+    "external": "'"$SITE_ID"':443",
     "internal": "opal:8443",
-    "allowed": ["central-ds-orchestrator.'$BROKER_ID'"]
+    "allowed": input | map("datashield-connect.\(.).'"$BROKER_ID"'")
-  }]' > /tmp/bridgehead/opal-map/local.json
+  }]' >/tmp/bridgehead/opal-map/local.json
   if [ "$USER" == "root" ]; then
     chown -R bridgehead:docker /tmp/bridgehead
     chmod g+wr /tmp/bridgehead/opal-map/*

ccp/modules/datashield-sites.json

@@ -0,0 +1,14 @@
+[
+    "berlin",
+    "muenchen-lmu",
+    "dresden",
+    "freiburg",
+    "muenchen-tum",
+    "tuebingen",
+    "mainz",
+    "frankfurt",
+    "essen",
+    "dktk-datashield-test",
+    "dktk-test",
+    "mannheim"
+]

ccp/modules/datashield.md

@@ -1,5 +1,5 @@
 # DataSHIELD
-This module constitutes the infrastructure to run DataSHIELD within the bridghead. 
+This module constitutes the infrastructure to run DataSHIELD within the bridgehead. 
 For more information about DataSHIELD, please visit https://www.datashield.org/
 
 ## R-Studio

ccp/modules/exporter-compose.yml

@@ -65,16 +65,3 @@ services:
       - "traefik.http.routers.reporter_ccp.tls=true"
       - "traefik.http.middlewares.reporter_ccp_strip.stripprefix.prefixes=/ccp-reporter"
       - "traefik.http.routers.reporter_ccp.middlewares=reporter_ccp_strip"
-
-  file-dispatcher:
-    image: docker.verbis.dkfz.de/cache/samply/file-dispatcher:latest
-    environment:
-      - BEAM_ID=file-dispatcher.${PROXY_ID}
-      - PROJECT_MANAGER_ID=project-manager.request-manager.${BROKER_ID}
-      - BEAM_SECRET=${FILE_DISPATCHER_BEAM_SECRET} # Generated in exporter-setup.sh
-      - BEAM_URL=http://beam-proxy:8081
-      - EXPORTER_URL=http://exporter:8092
-  
-  beam-proxy:
-    environment:
-      - APP_file-dispatcher_KEY=${FILE_DISPATCHER_BEAM_SECRET}

ccp/modules/exporter-setup.sh

@@ -5,5 +5,4 @@ if [ "$ENABLE_EXPORTER" == true ]; then
   OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml"
   EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
   EXPORTER_API_KEY="$(echo \"This is a salt string to generate one consistent API KEY for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 64)"
-  FILE_DISPATCHER_BEAM_SECRET="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
 fi

ccp/modules/fhir2sql-compose.yml

@@ -0,0 +1,25 @@
+version: "3.7"
+
+services:
+  fhir2sql:
+    depends_on:
+      - "dashboard-db"
+      - "blaze"
+    image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest
+    container_name: bridgehead-ccp-dashboard-fhir2sql
+    environment:
+      BLAZE_BASE_URL: "http://bridgehead-ccp-blaze:8080"
+      PG_HOST: "dashboard-db"
+      PG_USERNAME: "dashboard"
+      PG_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in dashboard-setup.sh
+      PG_DBNAME: "dashboard"
+
+  dashboard-db:
+    image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG}
+    container_name: bridgehead-ccp-dashboard-db
+    environment:
+      POSTGRES_USER: "dashboard"
+      POSTGRES_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in dashboard-setup.sh
+      POSTGRES_DB: "dashboard"
+    volumes:
+      - "/var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data"

ccp/modules/fhir2sql-setup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+if [ "$ENABLE_FHIR2SQL" == true ]; then
+  log INFO "Dashboard setup detected -- will start Dashboard backend and FHIR2SQL service."
+  OVERRIDE+=" -f ./$PROJECT/modules/fhir2sql-compose.yml"
+  DASHBOARD_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the Dashboard database. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+fi

ccp/modules/fhir2sql.md

@@ -0,0 +1,36 @@
+# fhir2sql
+fhir2sql connects to Blaze, retrieves data, and syncs it with a PostgreSQL database. The application is designed to run continuously, syncing data at regular intervals.
+The Dashboard module is a optional component of the Bridgehead CCP setup. When enabled, it starts two Docker services: **fhir2sql** and **dashboard-db**. Data held in PostgreSQL is only stored temporarily and Blaze is considered to be the 'leading system' or 'source of truth'.
+
+## Services
+### fhir2sql
+* Image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest
+* Container name: bridgehead-ccp-dashboard-fhir2sql
+* Depends on: dashboard-db
+* Environment variables:
+  - BLAZE_BASE_URL: The base URL of the Blaze FHIR server (set to http://blaze:8080/fhir/)
+  - PG_HOST: The hostname of the PostgreSQL database (set to dashboard-db)
+  - PG_USERNAME: The username for the PostgreSQL database (set to dashboard)
+  - PG_PASSWORD: The password for the PostgreSQL database (set to the value of DASHBOARD_DB_PASSWORD)
+  - PG_DBNAME: The name of the PostgreSQL database (set to dashboard)
+
+### dashboard-db
+
+* Image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG}
+* Container name: bridgehead-ccp-dashboard-db
+* Environment variables:
+  - POSTGRES_USER: The username for the PostgreSQL database (set to dashboard)
+  - POSTGRES_PASSWORD: The password for the PostgreSQL database (set to the value of DASHBOARD_DB_PASSWORD)
+  - POSTGRES_DB: The name of the PostgreSQL database (set to dashboard)
+* Volumes:
+  - /var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data
+
+The volume used by dashboard-db can be removed safely and should be restored to a working order by re-importing data from Blaze.
+
+### Environment Variables
+* DASHBOARD_DB_PASSWORD: A generated password for the PostgreSQL database, created using a salt string and the SHA1 hash function.
+* POSTGRES_TAG: The tag of the PostgreSQL image to use (not set in this module, but required by the dashboard-db service).
+
+
+### Setup
+To enable the Dashboard module, set the ENABLE_FHIR2SQL environment variable to true. The dashboard-setup.sh script will then start the fhir2sql and dashboard-db services, using the environment variables and volumes defined above.

ccp/modules/id-management-compose.yml

@@ -14,21 +14,22 @@ services:
       MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY}
       MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY}
       MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY}
-      MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID}
-      MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET}
     depends_on:
       - patientlist
+      - traefik-forward-auth
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)"
       - "traefik.http.services.id-manager.loadbalancer.server.port=8080"
       - "traefik.http.routers.id-manager.tls=true"
+      - "traefik.http.routers.id-manager.middlewares=traefik-forward-auth-idm"
 
   patientlist:
     image: docker.verbis.dkfz.de/bridgehead/mainzelliste
     container_name: bridgehead-patientlist
     environment:
       - TOMCAT_REVERSEPROXY_FQDN=${HOST}
+      - TOMCAT_REVERSEPROXY_SSL=true
       - ML_SITE=${IDMANAGEMENT_FRIENDLY_ID}
       - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD}
       - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
@@ -55,5 +56,41 @@ services:
       # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!!
       - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/"
 
+  traefik-forward-auth:
+    image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:v7.6.0
+    environment:
+      - http_proxy=http://forward_proxy:3128
+      - https_proxy=http://forward_proxy:3128
+      - OAUTH2_PROXY_PROVIDER=oidc
+      - OAUTH2_PROXY_SKIP_PROVIDER_BUTTON=true
+      - OAUTH2_PROXY_OIDC_ISSUER_URL=https://login.verbis.dkfz.de/realms/master
+      - OAUTH2_PROXY_CLIENT_ID=bridgehead-${SITE_ID}
+      - OAUTH2_PROXY_CLIENT_SECRET=${IDMANAGER_AUTH_CLIENT_SECRET}
+      - OAUTH2_PROXY_COOKIE_SECRET=${IDMANAGER_AUTH_COOKIE_SECRET}
+      - OAUTH2_PROXY_COOKIE_DOMAINS=.${HOST}
+      - OAUTH2_PROXY_HTTP_ADDRESS=:4180
+      - OAUTH2_PROXY_REVERSE_PROXY=true
+      - OAUTH2_PROXY_WHITELIST_DOMAINS=.${HOST}
+      - OAUTH2_PROXY_UPSTREAMS=static://202
+      - OAUTH2_PROXY_EMAIL_DOMAINS=*
+      - OAUTH2_PROXY_SCOPE=openid profile email
+      # Pass Authorization Header and some user information to backend services
+      - OAUTH2_PROXY_SET_AUTHORIZATION_HEADER=true
+      - OAUTH2_PROXY_SET_XAUTHREQUEST=true
+      # Keycloak has an expiration time of 60s therefore oauth2-proxy needs to refresh after that
+      - OAUTH2_PROXY_COOKIE_REFRESH=60s
+      - OAUTH2_PROXY_ALLOWED_GROUPS=DKTK-CCP-PPSN
+      - OAUTH2_PROXY_PROXY_PREFIX=/oauth2-idm
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.traefik-forward-auth.loadbalancer.server.port=4180"
+      - "traefik.http.routers.traefik-forward-auth.rule=Host(`${HOST}`) && PathPrefix(`/oauth2-idm`)"
+      - "traefik.http.routers.traefik-forward-auth.tls=true"
+      - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.address=http://traefik-forward-auth:4180"
+      - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.authResponseHeaders=Authorization"
+    depends_on:
+      forward_proxy:
+        condition: service_healthy
+
 volumes:
   patientlist-db-data:

ccp/modules/mtba-setup.sh

@@ -5,7 +5,6 @@ function mtbaSetup() {
     log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal."
     if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then
       log ERROR "Missing ID-Management Module! Fix this by setting up ID Management:"
-      exit 1;
     fi
     OVERRIDE+=" -f ./$PROJECT/modules/mtba-compose.yml"
     add_private_oidc_redirect_url "/mtba/*"

ccp/modules/obds2fhir-rest-compose.yml

@@ -0,0 +1,20 @@
+version: "3.7"
+
+services:
+  obds2fhir-rest:
+    container_name: bridgehead-obds2fhir-rest
+    image: docker.verbis.dkfz.de/ccp/obds2fhir-rest:main
+    environment:
+      IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID
+      MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
+      SALT: ${LOCAL_SALT}
+      KEEP_INTERNAL_ID: ${KEEP_INTERNAL_ID:-false}
+      MAINZELLISTE_URL: ${PATIENTLIST_URL:-http://patientlist:8080/patientlist}
+    restart: always
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.obds2fhir-rest.rule=PathPrefix(`/obds2fhir-rest`) || PathPrefix(`/adt2fhir-rest`)"
+      - "traefik.http.middlewares.obds2fhir-rest_strip.stripprefix.prefixes=/obds2fhir-rest,/adt2fhir-rest"
+      - "traefik.http.services.obds2fhir-rest.loadbalancer.server.port=8080"
+      - "traefik.http.routers.obds2fhir-rest.tls=true"
+      - "traefik.http.routers.obds2fhir-rest.middlewares=obds2fhir-rest_strip,auth"

ccp/modules/obds2fhir-rest-setup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+function obds2fhirRestSetup() {
+  if [ -n "$ENABLE_OBDS2FHIR_REST" ]; then
+    log INFO "oBDS2FHIR-REST setup detected -- will start obds2fhir-rest module."
+    if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then
+      log ERROR "Missing ID-Management Module! Fix this by setting up ID Management:"
+      PATIENTLIST_URL=" "
+    fi
+    OVERRIDE+=" -f ./$PROJECT/modules/obds2fhir-rest-compose.yml"
+    LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+  fi
+}

ccp/queries_to_cache.conf

@@ -0,0 +1,2 @@
+bGlicmFyeSBSZXRyaWV2ZQp1c2luZyBGSElSIHZlcnNpb24gJzQuMC4wJwppbmNsdWRlIEZISVJIZWxwZXJzIHZlcnNpb24gJzQuMC4wJwoKY29kZXN5c3RlbSBsb2luYzogJ2h0dHA6Ly9sb2luYy5vcmcnCgpjb250ZXh0IFBhdGllbnQKCgpES1RLX1NUUkFUX0dFTkRFUl9TVFJBVElGSUVSCgpES1RLX1NUUkFUX1BSSU1BUllfRElBR05PU0lTX05PX1NPUlRfU1RSQVRJRklFUgpES1RLX1NUUkFUX0FHRV9DTEFTU19TVFJBVElGSUVSCgpES1RLX1NUUkFUX0RFQ0VBU0VEX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfRElBR05PU0lTX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfU1BFQ0lNRU5fU1RSQVRJRklFUgoKREtUS19TVFJBVF9QUk9DRURVUkVfU1RSQVRJRklFUgoKREtUS19TVFJBVF9NRURJQ0FUSU9OX1NUUkFUSUZJRVIKCiAgREtUS19TVFJBVF9ISVNUT0xPR1lfU1RSQVRJRklFUgpES1RLX1NUUkFUX0RFRl9JTl9JTklUSUFMX1BPUFVMQVRJT04KdHJ1ZQ==
+bGlicmFyeSBSZXRyaWV2ZQp1c2luZyBGSElSIHZlcnNpb24gJzQuMC4wJwppbmNsdWRlIEZISVJIZWxwZXJzIHZlcnNpb24gJzQuMC4wJwoKY29kZXN5c3RlbSBsb2luYzogJ2h0dHA6Ly9sb2luYy5vcmcnCmNvZGVzeXN0ZW0gaWNkMTA6ICdodHRwOi8vZmhpci5kZS9Db2RlU3lzdGVtL2JmYXJtL2ljZC0xMC1nbScKY29kZXN5c3RlbSBtb3JwaDogJ3VybjpvaWQ6Mi4xNi44NDAuMS4xMTM4ODMuNi40My4xJwoKY29udGV4dCBQYXRpZW50CgoKREtUS19TVFJBVF9HRU5ERVJfU1RSQVRJRklFUgoKREtUS19TVFJBVF9QUklNQVJZX0RJQUdOT1NJU19OT19TT1JUX1NUUkFUSUZJRVIKREtUS19TVFJBVF9BR0VfQ0xBU1NfU1RSQVRJRklFUgoKREtUS19TVFJBVF9ERUNFQVNFRF9TVFJBVElGSUVSCgpES1RLX1NUUkFUX0RJQUdOT1NJU19TVFJBVElGSUVSCgpES1RLX1NUUkFUX1NQRUNJTUVOX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfUFJPQ0VEVVJFX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfTUVESUNBVElPTl9TVFJBVElGSUVSCgogIERLVEtfU1RSQVRfSElTVE9MT0dZX1NUUkFUSUZJRVIKREtUS19TVFJBVF9ERUZfSU5fSU5JVElBTF9QT1BVTEFUSU9OKGV4aXN0cyBbQ29uZGl0aW9uOiBDb2RlICdDNjEnIGZyb20gaWNkMTBdKSBhbmQgCigoZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzgxNDAvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzgxNDcvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzg0ODAvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzg1MDAvMycpKQ==

ccp/vars

@@ -2,7 +2,7 @@ BROKER_ID=broker.ccp-it.dktk.dkfz.de
 BROKER_URL=https://${BROKER_ID}
 PROXY_ID=${SITE_ID}.${BROKER_ID}
 FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
-FOCUS_RETRY_COUNT=32
+FOCUS_RETRY_COUNT=${FOCUS_RETRY_COUNT:-64}
 SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 
@@ -28,4 +28,5 @@ done
 
 idManagementSetup
 mtbaSetup
-adt2fhirRestSetup
+obds2fhirRestSetup
+blazeSecondarySetup

lib/functions.sh

@@ -53,7 +53,7 @@ checkOwner(){
 }
 
 printUsage() {
-	echo "Usage: bridgehead start|stop|logs|is-running|update|install|uninstall|adduser|enroll PROJECTNAME"
+	echo "Usage: bridgehead start|stop|logs|docker-logs|is-running|update|install|uninstall|adduser|enroll PROJECTNAME"
 	echo "PROJECTNAME should be one of ccp|bbmri"
 }
 
@@ -367,3 +367,7 @@ generate_simple_password(){
   local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret."
   echo "${combined_text}" | sha1sum | openssl pkeyutl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/[+\/]/A/g'
 }
+
+docker_jq() {
+    docker run --rm -i docker.verbis.dkfz.de/cache/jqlang/jq:latest "$@"
+}

lib/systemd/bridgehead-update@.timer

@@ -1,8 +1,9 @@
 [Unit]
-Description=Hourly Updates of Bridgehead (%i)
+Description=Daily Updates at 6am of Bridgehead (%i)
 
 [Timer]
-OnCalendar=*-*-* *:00:00
+OnCalendar=*-*-* 06:00:00
+Persistent=true
 
 [Install]
 WantedBy=basic.target

minimal/docker-compose.yml

@@ -42,6 +42,9 @@ services:
       - /var/spool/squid
     volumes:
       - /etc/bridgehead/trusted-ca-certs:/docker/custom-certs/:ro
+    healthcheck:
+      # Wait 1s before marking this service healthy. Required for the oauth2-proxy to talk to the OIDC provider on startup which will fail if the forward proxy is not started yet.
+      test: ["CMD", "sleep", "1"]
 
   landing:
     container_name: bridgehead-landingpage
@@ -55,3 +58,4 @@ services:
       HOST: ${HOST}
       PROJECT: ${PROJECT}
       SITE_NAME: ${SITE_NAME}
+      ENVIRONMENT: ${ENVIRONMENT}