From 8104711075101b84d35dad9bb800a4a0d520097e Mon Sep 17 00:00:00 2001 From: Croft Date: Wed, 6 Mar 2024 11:26:07 +0100 Subject: [PATCH 01/24] Allow user to push star model facts to Directory This takes advantage of new functionality added to Directory sync. Defaults to false. --- bbmri/modules/directory-sync-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 9776ecb..99cb467 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -6,3 +6,4 @@ services: DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} DS_TIMER_CRON: ${DS_TIMER_CRON} + DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL} From 1a928e670187ae3c29a75d576df4406cbfa02c70 Mon Sep 17 00:00:00 2001 From: Croft Date: Wed, 6 Mar 2024 11:35:17 +0100 Subject: [PATCH 02/24] Included the new functionality into the README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 05038ae..4f03fd5 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,8 @@ Once you have added your biobank to the Directory you got persistent identifier The Bridgehead's **Directory Sync** is an optional feature that keeps the Directory up to date with your local data, e.g. number of samples. Conversely, it also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly set your country specific directory URL, username and password to enable this feature. +You should talk with your local data protection group regarding the information that is published by Directory sync. + Full details can be found in [directory_sync_service](https://github.com/samply/directory_sync_service). To enable it, you will need to set these variables to the ```bbmri.conf``` file of your GitLab repository. Here is an example config: @@ -306,6 +308,7 @@ To enable it, you will need to set these variables to the ```bbmri.conf``` file DS_DIRECTORY_URL=https://directory.bbmri-eric.eu DS_DIRECTORY_USER_NAME=your_directory_username DS_DIRECTORY_USER_PASS=qwdnqwswdvqHBVGFR9887 +DS_DIRECTORY_ALLOW_STAR_MODEL=true DS_TIMER_CRON="0 22 * * *" ``` You must contact the Directory team for your national node to find the URL, and to register as a user. From 91ff51304b25a8d0d8bc272ea3faca87ed9a8c3b Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 1 Jul 2024 14:54:04 +0200 Subject: [PATCH 03/24] Add new dashboard backend --- ccp/modules/dashboard-compose.yml | 28 ++++++++++++++++++++++++++++ ccp/modules/dashboard-setup.sh | 7 +++++++ ccp/modules/dashboard.md | 1 + 3 files changed, 36 insertions(+) create mode 100644 ccp/modules/dashboard-compose.yml create mode 100644 ccp/modules/dashboard-setup.sh create mode 100644 ccp/modules/dashboard.md diff --git a/ccp/modules/dashboard-compose.yml b/ccp/modules/dashboard-compose.yml new file mode 100644 index 0000000..43b109d --- /dev/null +++ b/ccp/modules/dashboard-compose.yml @@ -0,0 +1,28 @@ +version: "3.7" + +services: + fhir2sql: + depends_on: + - "dashboard-db" + image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest + container_name: bridgehead-ccp-dashboard-fhir2sql + environment: + BLAZE_BASE_URL: "http://blaze:8080/fhir/" + PG_HOST: "dashboard-db" + PG_PORT: 5432 + PG_USERNAME: "dashboard" + PG_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in exporter-setup.sh + PG_DBNAME: "dashboard" + # TODO: Remove the following, replace with defaults in app + BLAZE_PAGE_RESOURCE_COUNT: 10000 + PG_BATCH_SIZE: 10000 + + dashboard-db: + image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} + container_name: bridgehead-ccp-dashboard-db + environment: + POSTGRES_USER: "dashboard" + POSTGRES_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in exporter-setup.sh + POSTGRES_DB: "dashboard" + volumes: + - "/var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data" diff --git a/ccp/modules/dashboard-setup.sh b/ccp/modules/dashboard-setup.sh new file mode 100644 index 0000000..aee79fa --- /dev/null +++ b/ccp/modules/dashboard-setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +if [ "$ENABLE_DASHBOARD" == true ]; then + log INFO "Dashboard setup detected -- will start Dashboard backend and FHIR2SQL service." + OVERRIDE+=" -f ./$PROJECT/modules/dashboard-compose.yml" + DASHBOARD_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the Dashboard database. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +fi diff --git a/ccp/modules/dashboard.md b/ccp/modules/dashboard.md new file mode 100644 index 0000000..ed00305 --- /dev/null +++ b/ccp/modules/dashboard.md @@ -0,0 +1 @@ +# TODO David Scholz From f28e3c2cd2bd311271c6eff7e47b48eb4418e23c Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 1 Jul 2024 15:19:44 +0200 Subject: [PATCH 04/24] Remove unnecessary default values --- ccp/modules/dashboard-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ccp/modules/dashboard-compose.yml b/ccp/modules/dashboard-compose.yml index 43b109d..037f88f 100644 --- a/ccp/modules/dashboard-compose.yml +++ b/ccp/modules/dashboard-compose.yml @@ -9,13 +9,9 @@ services: environment: BLAZE_BASE_URL: "http://blaze:8080/fhir/" PG_HOST: "dashboard-db" - PG_PORT: 5432 PG_USERNAME: "dashboard" PG_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in exporter-setup.sh PG_DBNAME: "dashboard" - # TODO: Remove the following, replace with defaults in app - BLAZE_PAGE_RESOURCE_COUNT: 10000 - PG_BATCH_SIZE: 10000 dashboard-db: image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} From d316f1c798f7b92629e887ffe5a3784457bdc66b Mon Sep 17 00:00:00 2001 From: Pierre Delpy Date: Tue, 23 Jul 2024 13:01:07 +0200 Subject: [PATCH 05/24] add caching in focus --- ccp/docker-compose.yml | 3 +++ ccp/queries_to_cache.conf | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 ccp/queries_to_cache.conf diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 52e7eb5..2395d8c 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -31,6 +31,9 @@ services: BEAM_PROXY_URL: http://beam-proxy:8081 RETRY_COUNT: ${FOCUS_RETRY_COUNT} EPSILON: 0.28 + QUERIES_TO_CACHE: '/queries_to_cache.conf' + volumes: + - /srv/docker/bridgehead/ccp/queries_to_cache.conf:/queries_to_cache.conf depends_on: - "beam-proxy" - "blaze" diff --git a/ccp/queries_to_cache.conf b/ccp/queries_to_cache.conf new file mode 100644 index 0000000..b950312 --- /dev/null +++ b/ccp/queries_to_cache.conf @@ -0,0 +1,2 @@ +bGlicmFyeSBSZXRyaWV2ZQp1c2luZyBGSElSIHZlcnNpb24gJzQuMC4wJwppbmNsdWRlIEZISVJIZWxwZXJzIHZlcnNpb24gJzQuMC4wJwoKY29kZXN5c3RlbSBsb2luYzogJ2h0dHA6Ly9sb2luYy5vcmcnCgpjb250ZXh0IFBhdGllbnQKCgpES1RLX1NUUkFUX0dFTkRFUl9TVFJBVElGSUVSCgpES1RLX1NUUkFUX1BSSU1BUllfRElBR05PU0lTX05PX1NPUlRfU1RSQVRJRklFUgpES1RLX1NUUkFUX0FHRV9DTEFTU19TVFJBVElGSUVSCgpES1RLX1NUUkFUX0RFQ0VBU0VEX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfRElBR05PU0lTX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfU1BFQ0lNRU5fU1RSQVRJRklFUgoKREtUS19TVFJBVF9QUk9DRURVUkVfU1RSQVRJRklFUgoKREtUS19TVFJBVF9NRURJQ0FUSU9OX1NUUkFUSUZJRVIKCiAgREtUS19TVFJBVF9ISVNUT0xPR1lfU1RSQVRJRklFUgpES1RLX1NUUkFUX0RFRl9JTl9JTklUSUFMX1BPUFVMQVRJT04KdHJ1ZQ== +bGlicmFyeSBSZXRyaWV2ZQp1c2luZyBGSElSIHZlcnNpb24gJzQuMC4wJwppbmNsdWRlIEZISVJIZWxwZXJzIHZlcnNpb24gJzQuMC4wJwoKY29kZXN5c3RlbSBsb2luYzogJ2h0dHA6Ly9sb2luYy5vcmcnCmNvZGVzeXN0ZW0gaWNkMTA6ICdodHRwOi8vZmhpci5kZS9Db2RlU3lzdGVtL2JmYXJtL2ljZC0xMC1nbScKY29kZXN5c3RlbSBtb3JwaDogJ3VybjpvaWQ6Mi4xNi44NDAuMS4xMTM4ODMuNi40My4xJwoKY29udGV4dCBQYXRpZW50CgoKREtUS19TVFJBVF9HRU5ERVJfU1RSQVRJRklFUgoKREtUS19TVFJBVF9QUklNQVJZX0RJQUdOT1NJU19OT19TT1JUX1NUUkFUSUZJRVIKREtUS19TVFJBVF9BR0VfQ0xBU1NfU1RSQVRJRklFUgoKREtUS19TVFJBVF9ERUNFQVNFRF9TVFJBVElGSUVSCgpES1RLX1NUUkFUX0RJQUdOT1NJU19TVFJBVElGSUVSCgpES1RLX1NUUkFUX1NQRUNJTUVOX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfUFJPQ0VEVVJFX1NUUkFUSUZJRVIKCkRLVEtfU1RSQVRfTUVESUNBVElPTl9TVFJBVElGSUVSCgogIERLVEtfU1RSQVRfSElTVE9MT0dZX1NUUkFUSUZJRVIKREtUS19TVFJBVF9ERUZfSU5fSU5JVElBTF9QT1BVTEFUSU9OKGV4aXN0cyBbQ29uZGl0aW9uOiBDb2RlICdDNjEnIGZyb20gaWNkMTBdKSBhbmQgCigoZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzgxNDAvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzgxNDcvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzg0ODAvMycpIG9yIAooZXhpc3RzIGZyb20gW09ic2VydmF0aW9uOiBDb2RlICc1OTg0Ny00JyBmcm9tIGxvaW5jXSBPCndoZXJlIE8udmFsdWUuY29kaW5nLmNvZGUgY29udGFpbnMgJzg1MDAvMycpKQ== \ No newline at end of file From 8942b923b38efd16ae49bb03f6712e944dc1cd28 Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Fri, 26 Jul 2024 09:57:40 +0200 Subject: [PATCH 06/24] Added comment for consistency with Directory Sync README --- bbmri/modules/directory-sync-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index da329f8..60998f3 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -8,4 +8,5 @@ services: DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} DS_TIMER_CRON: ${DS_TIMER_CRON} + # It is recommended to check the enabling of this flag with your local data protection group DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL} From df08d678398a1b719af426a16d4655c2a42364e4 Mon Sep 17 00:00:00 2001 From: davidmscholz Date: Mon, 29 Jul 2024 10:45:00 +0200 Subject: [PATCH 07/24] add optional dashboard module --- ccp/modules/dashboard-compose.yml | 4 ++-- ccp/modules/dashboard.md | 37 ++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/ccp/modules/dashboard-compose.yml b/ccp/modules/dashboard-compose.yml index 037f88f..e2756d4 100644 --- a/ccp/modules/dashboard-compose.yml +++ b/ccp/modules/dashboard-compose.yml @@ -10,7 +10,7 @@ services: BLAZE_BASE_URL: "http://blaze:8080/fhir/" PG_HOST: "dashboard-db" PG_USERNAME: "dashboard" - PG_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in exporter-setup.sh + PG_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in dashboard-setup.sh PG_DBNAME: "dashboard" dashboard-db: @@ -18,7 +18,7 @@ services: container_name: bridgehead-ccp-dashboard-db environment: POSTGRES_USER: "dashboard" - POSTGRES_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in exporter-setup.sh + POSTGRES_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in dashboard-setup.sh POSTGRES_DB: "dashboard" volumes: - "/var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data" diff --git a/ccp/modules/dashboard.md b/ccp/modules/dashboard.md index ed00305..defdf39 100644 --- a/ccp/modules/dashboard.md +++ b/ccp/modules/dashboard.md @@ -1 +1,36 @@ -# TODO David Scholz +# fhir2sql +fhir2sql connects to Blaze, retrieves data, and syncs it with a PostgreSQL database. The application is designed to run continuously, syncing data at regular intervals. +The Dashboard module is a optional component of the Bridgehead CCP setup. When enabled, it starts two Docker services: **fhir2sql** and **dashboard-db**. Data held in PostgreSQL is only stored temporarily and Blaze is considered to be the 'leading system' or 'source of truth'. + +## Services +### fhir2sql +* Image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest +* Container name: bridgehead-ccp-dashboard-fhir2sql +* Depends on: dashboard-db +* Environment variables: + - BLAZE_BASE_URL: The base URL of the Blaze FHIR server (set to http://blaze:8080/fhir/) + - PG_HOST: The hostname of the PostgreSQL database (set to dashboard-db) + - PG_USERNAME: The username for the PostgreSQL database (set to dashboard) + - PG_PASSWORD: The password for the PostgreSQL database (set to the value of DASHBOARD_DB_PASSWORD) + - PG_DBNAME: The name of the PostgreSQL database (set to dashboard) + +### dashboard-db + +* Image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} +* Container name: bridgehead-ccp-dashboard-db +* Environment variables: + - POSTGRES_USER: The username for the PostgreSQL database (set to dashboard) + - POSTGRES_PASSWORD: The password for the PostgreSQL database (set to the value of DASHBOARD_DB_PASSWORD) + - POSTGRES_DB: The name of the PostgreSQL database (set to dashboard) +* Volumes: + - /var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data + +The volume used by dashboard-db can be removed safely and should be restored to a working order by re-importing data from Blaze. + +### Environment Variables +* DASHBOARD_DB_PASSWORD: A generated password for the PostgreSQL database, created using a salt string and the SHA1 hash function. +* POSTGRES_TAG: The tag of the PostgreSQL image to use (not set in this module, but required by the dashboard-db service). + + +### Setup +To enable the Dashboard module, set the ENABLE_DASHBOARD environment variable to true. The dashboard-setup.sh script will then start the fhir2sql and dashboard-db services, using the environment variables and volumes defined above. \ No newline at end of file From 5ed07423f3a04132c08f59294909534d4f1e8038 Mon Sep 17 00:00:00 2001 From: davidmscholz Date: Tue, 30 Jul 2024 09:24:07 +0200 Subject: [PATCH 08/24] fix dashboard-compose --- ccp/modules/dashboard-compose.yml | 3 ++- ccp/modules/dashboard-setup.sh | 2 +- ccp/modules/dashboard.md | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ccp/modules/dashboard-compose.yml b/ccp/modules/dashboard-compose.yml index e2756d4..a84ff24 100644 --- a/ccp/modules/dashboard-compose.yml +++ b/ccp/modules/dashboard-compose.yml @@ -4,10 +4,11 @@ services: fhir2sql: depends_on: - "dashboard-db" + - [ blaze ] image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest container_name: bridgehead-ccp-dashboard-fhir2sql environment: - BLAZE_BASE_URL: "http://blaze:8080/fhir/" + BLAZE_BASE_URL: "http://bridgehead-ccp-blaze:8080" PG_HOST: "dashboard-db" PG_USERNAME: "dashboard" PG_PASSWORD: "${DASHBOARD_DB_PASSWORD}" # Set in dashboard-setup.sh diff --git a/ccp/modules/dashboard-setup.sh b/ccp/modules/dashboard-setup.sh index aee79fa..e1a33af 100644 --- a/ccp/modules/dashboard-setup.sh +++ b/ccp/modules/dashboard-setup.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -if [ "$ENABLE_DASHBOARD" == true ]; then +if [ "$ENABLE_FHIR2SQL" == true ]; then log INFO "Dashboard setup detected -- will start Dashboard backend and FHIR2SQL service." OVERRIDE+=" -f ./$PROJECT/modules/dashboard-compose.yml" DASHBOARD_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the Dashboard database. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" diff --git a/ccp/modules/dashboard.md b/ccp/modules/dashboard.md index defdf39..deea710 100644 --- a/ccp/modules/dashboard.md +++ b/ccp/modules/dashboard.md @@ -33,4 +33,4 @@ The volume used by dashboard-db can be removed safely and should be restored to ### Setup -To enable the Dashboard module, set the ENABLE_DASHBOARD environment variable to true. The dashboard-setup.sh script will then start the fhir2sql and dashboard-db services, using the environment variables and volumes defined above. \ No newline at end of file +To enable the Dashboard module, set the ENABLE_FHIR2SQL environment variable to true. The dashboard-setup.sh script will then start the fhir2sql and dashboard-db services, using the environment variables and volumes defined above. \ No newline at end of file From af44b6b4462e9ba874db9917a8d070a9df05aca4 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 30 Jul 2024 07:40:49 +0000 Subject: [PATCH 09/24] Fix depends_on syntax --- ccp/modules/dashboard-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/dashboard-compose.yml b/ccp/modules/dashboard-compose.yml index a84ff24..7733787 100644 --- a/ccp/modules/dashboard-compose.yml +++ b/ccp/modules/dashboard-compose.yml @@ -4,7 +4,7 @@ services: fhir2sql: depends_on: - "dashboard-db" - - [ blaze ] + - "blaze" image: docker.verbis.dkfz.de/cache/samply/fhir2sql:latest container_name: bridgehead-ccp-dashboard-fhir2sql environment: From 2e5aeabca8cc252c431dfb05ce964740fd0755e3 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 30 Jul 2024 07:44:47 +0000 Subject: [PATCH 10/24] Rename fhir2sql module files --- ccp/modules/{dashboard-compose.yml => fhir2sql-compose.yml} | 0 ccp/modules/{dashboard-setup.sh => fhir2sql-setup.sh} | 2 +- ccp/modules/{dashboard.md => fhir2sql.md} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename ccp/modules/{dashboard-compose.yml => fhir2sql-compose.yml} (100%) rename ccp/modules/{dashboard-setup.sh => fhir2sql-setup.sh} (87%) rename ccp/modules/{dashboard.md => fhir2sql.md} (100%) diff --git a/ccp/modules/dashboard-compose.yml b/ccp/modules/fhir2sql-compose.yml similarity index 100% rename from ccp/modules/dashboard-compose.yml rename to ccp/modules/fhir2sql-compose.yml diff --git a/ccp/modules/dashboard-setup.sh b/ccp/modules/fhir2sql-setup.sh similarity index 87% rename from ccp/modules/dashboard-setup.sh rename to ccp/modules/fhir2sql-setup.sh index e1a33af..6b27571 100644 --- a/ccp/modules/dashboard-setup.sh +++ b/ccp/modules/fhir2sql-setup.sh @@ -2,6 +2,6 @@ if [ "$ENABLE_FHIR2SQL" == true ]; then log INFO "Dashboard setup detected -- will start Dashboard backend and FHIR2SQL service." - OVERRIDE+=" -f ./$PROJECT/modules/dashboard-compose.yml" + OVERRIDE+=" -f ./$PROJECT/modules/fhir2sql-compose.yml" DASHBOARD_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the Dashboard database. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" fi diff --git a/ccp/modules/dashboard.md b/ccp/modules/fhir2sql.md similarity index 100% rename from ccp/modules/dashboard.md rename to ccp/modules/fhir2sql.md From 62edaf99e035fbd8d507a904878f261346ee8cbd Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Thu, 1 Aug 2024 11:23:56 +0200 Subject: [PATCH 11/24] Reduce bridgehead update interval to once a day at 6am --- lib/systemd/bridgehead-update@.timer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/systemd/bridgehead-update@.timer b/lib/systemd/bridgehead-update@.timer index 4c8fada..21415c8 100644 --- a/lib/systemd/bridgehead-update@.timer +++ b/lib/systemd/bridgehead-update@.timer @@ -2,7 +2,7 @@ Description=Hourly Updates of Bridgehead (%i) [Timer] -OnCalendar=*-*-* *:00:00 +OnCalendar=*-*-* 6:00:00 [Install] WantedBy=basic.target From 5227dc57a78087b9d67143a2a2c03db6143d6515 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Thu, 1 Aug 2024 11:32:15 +0200 Subject: [PATCH 12/24] Fix systemd timer description Co-authored-by: Jan <59206115+Threated@users.noreply.github.com> --- lib/systemd/bridgehead-update@.timer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/systemd/bridgehead-update@.timer b/lib/systemd/bridgehead-update@.timer index 21415c8..0790baf 100644 --- a/lib/systemd/bridgehead-update@.timer +++ b/lib/systemd/bridgehead-update@.timer @@ -1,5 +1,5 @@ [Unit] -Description=Hourly Updates of Bridgehead (%i) +Description=Daily Updates at 6am of Bridgehead (%i) [Timer] OnCalendar=*-*-* 6:00:00 From ecd92690220540726004b8c65847dcb87997edef Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Thu, 1 Aug 2024 11:38:25 +0200 Subject: [PATCH 13/24] Add bridgehead update timer persistance Co-authored-by: Martin Lablans <6804500+lablans@users.noreply.github.com> --- lib/systemd/bridgehead-update@.timer | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/systemd/bridgehead-update@.timer b/lib/systemd/bridgehead-update@.timer index 0790baf..c2ce82e 100644 --- a/lib/systemd/bridgehead-update@.timer +++ b/lib/systemd/bridgehead-update@.timer @@ -3,6 +3,7 @@ Description=Daily Updates at 6am of Bridgehead (%i) [Timer] OnCalendar=*-*-* 6:00:00 +Persistent=true [Install] WantedBy=basic.target From 35d6a1777871b9f0c606c2b74d492d33a37a5585 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Thu, 1 Aug 2024 11:39:03 +0200 Subject: [PATCH 14/24] Fix bridgehead update timer time convention Co-authored-by: Martin Lablans <6804500+lablans@users.noreply.github.com> --- lib/systemd/bridgehead-update@.timer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/systemd/bridgehead-update@.timer b/lib/systemd/bridgehead-update@.timer index c2ce82e..d9abdf4 100644 --- a/lib/systemd/bridgehead-update@.timer +++ b/lib/systemd/bridgehead-update@.timer @@ -2,7 +2,7 @@ Description=Daily Updates at 6am of Bridgehead (%i) [Timer] -OnCalendar=*-*-* 6:00:00 +OnCalendar=*-*-* 06:00:00 Persistent=true [Install] From 7c560a2e9313f6a6c87bd7e3d26f0e44a2280fe5 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Thu, 15 Aug 2024 09:10:37 +0200 Subject: [PATCH 15/24] Added env to landing-page --- minimal/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index e9f53d6..6e8818f 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -58,3 +58,4 @@ services: HOST: ${HOST} PROJECT: ${PROJECT} SITE_NAME: ${SITE_NAME} + ENVIRONMENT: ${ENVIRONMENT} From b8b81b1242f97ad8b7206224953f478eefd3514e Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:17:34 +0200 Subject: [PATCH 16/24] Fixed environment variable passing for Directory sync There were problems with the passing of environment variables from bbmri.conf to the Directory synce container: * The Directory password variable was misspellt. * Some useful variables were missing. Additionally, a delay was added before launching Directory sync, to give Blaze time to start up. --- bbmri/modules/directory-sync-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 60998f3..0a58cd1 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -6,7 +6,11 @@ services: environment: DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} - DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} + DS_DIRECTORY_USER_PASS: ${DS_DIRECTORY_USER_PASS} DS_TIMER_CRON: ${DS_TIMER_CRON} # It is recommended to check the enabling of this flag with your local data protection group DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL} + DS_DIRECTORY_MOCK: ${DS_DIRECTORY_MOCK} + DS_DIRECTORY_DEFAULT_COLLECTION_ID: ${DS_DIRECTORY_DEFAULT_COLLECTION_ID} + DS_DIRECTORY_COUNTRY: ${DS_DIRECTORY_COUNTRY} + command: sh -c "sleep 90 && java -jar directory_sync_service.jar" # Wait for Blaze before start From 95574f38befffe81c796178098d4c98704d6dad0 Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:33:28 +0200 Subject: [PATCH 17/24] Included Blaze dependency --- bbmri/modules/directory-sync-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 0a58cd1..17929cc 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -14,3 +14,5 @@ services: DS_DIRECTORY_DEFAULT_COLLECTION_ID: ${DS_DIRECTORY_DEFAULT_COLLECTION_ID} DS_DIRECTORY_COUNTRY: ${DS_DIRECTORY_COUNTRY} command: sh -c "sleep 90 && java -jar directory_sync_service.jar" # Wait for Blaze before start + depends_on: + - "blaze" From 3496fa7a0f0669f891aee1af0914a8e248398440 Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:36:57 +0200 Subject: [PATCH 18/24] Let Directory sync handle connection with Blaze Remove the delayed start, because Directory sync will automatically keep trying to connect to Blaze if not initially present. --- bbmri/modules/directory-sync-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 17929cc..1afc46a 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -13,6 +13,5 @@ services: DS_DIRECTORY_MOCK: ${DS_DIRECTORY_MOCK} DS_DIRECTORY_DEFAULT_COLLECTION_ID: ${DS_DIRECTORY_DEFAULT_COLLECTION_ID} DS_DIRECTORY_COUNTRY: ${DS_DIRECTORY_COUNTRY} - command: sh -c "sleep 90 && java -jar directory_sync_service.jar" # Wait for Blaze before start depends_on: - "blaze" From de847f309c52ba5604d6f8768bb237440b47db90 Mon Sep 17 00:00:00 2001 From: lablans Date: Thu, 15 Aug 2024 11:40:02 +0000 Subject: [PATCH 19/24] Provide defaults --- README.md | 9 +++------ bbmri/modules/directory-sync-compose.yml | 6 +++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6c0e480..12984d1 100644 --- a/README.md +++ b/README.md @@ -306,15 +306,12 @@ Full details can be found in [directory_sync_service](https://github.com/samply/ To enable it, you will need to set these variables to the ```bbmri.conf``` file of your GitLab repository. Here is an example config: ``` -DS_DIRECTORY_URL=https://directory.bbmri-eric.eu DS_DIRECTORY_USER_NAME=your_directory_username -DS_DIRECTORY_USER_PASS=qwdnqwswdvqHBVGFR9887 -DS_DIRECTORY_ALLOW_STAR_MODEL=true -DS_TIMER_CRON="0 22 * * *" +DS_DIRECTORY_USER_PASS=your_directory_password ``` -You must contact the Directory team for your national node to find the URL, and to register as a user. +Please contact your National Node to obtain this information. -Additionally, you should choose when you want Directory sync to run. In the example above, this is set to happen at 10 pm every evening. You can modify this to suit your requirements. The timer specification should follow the [cron](https://crontab.guru) convention. +Optionally, you **may** change when you want Directory sync to run by specifying a [cron](https://crontab.guru) expression, e.g. `DS_TIMER_CRON="0 22 * * *"` for 10 pm every evening. Once you edited the gitlab config, the bridgehead will autoupdate the config with the values and will sync the data. diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 1afc46a..215acd4 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -4,12 +4,12 @@ services: directory_sync_service: image: "docker.verbis.dkfz.de/cache/samply/directory_sync_service" environment: - DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} + DS_DIRECTORY_URL: ${DS_DIRECTORY_URL:-https://directory.bbmri-eric.eu} DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} DS_DIRECTORY_USER_PASS: ${DS_DIRECTORY_USER_PASS} - DS_TIMER_CRON: ${DS_TIMER_CRON} + DS_TIMER_CRON: ${DS_TIMER_CRON:-0 22 * * *} # It is recommended to check the enabling of this flag with your local data protection group - DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL} + DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL:-true} DS_DIRECTORY_MOCK: ${DS_DIRECTORY_MOCK} DS_DIRECTORY_DEFAULT_COLLECTION_ID: ${DS_DIRECTORY_DEFAULT_COLLECTION_ID} DS_DIRECTORY_COUNTRY: ${DS_DIRECTORY_COUNTRY} From 18c9e1bb308f62c1a74cb11eadf3fba0953494aa Mon Sep 17 00:00:00 2001 From: lablans Date: Thu, 15 Aug 2024 11:43:14 +0000 Subject: [PATCH 20/24] Remove DP statement already present in readme. --- bbmri/modules/directory-sync-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 215acd4..33a7d31 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -8,7 +8,6 @@ services: DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} DS_DIRECTORY_USER_PASS: ${DS_DIRECTORY_USER_PASS} DS_TIMER_CRON: ${DS_TIMER_CRON:-0 22 * * *} - # It is recommended to check the enabling of this flag with your local data protection group DS_DIRECTORY_ALLOW_STAR_MODEL: ${DS_DIRECTORY_ALLOW_STAR_MODEL:-true} DS_DIRECTORY_MOCK: ${DS_DIRECTORY_MOCK} DS_DIRECTORY_DEFAULT_COLLECTION_ID: ${DS_DIRECTORY_DEFAULT_COLLECTION_ID} From ae95f1403013dc43c96457b830200e285217988b Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Mon, 19 Aug 2024 08:27:20 +0200 Subject: [PATCH 21/24] export ENVIRONMENT --- bridgehead | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bridgehead b/bridgehead index 85593b0..7a66262 100755 --- a/bridgehead +++ b/bridgehead @@ -74,13 +74,18 @@ loadVars() { case "$ENVIRONMENT" in "production") export FOCUS_TAG=main + export ENVIRONMENT="production" ;; "test") export FOCUS_TAG=develop + export ENVIRONMENT="test" + ;; *) report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!" export FOCUS_TAG=main + export ENVIRONMENT="production" + ;; esac } From 60acac619de3876db753a5da54fb2ae571825022 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 19 Aug 2024 08:38:34 +0200 Subject: [PATCH 22/24] Don't repeat definition of ENVIRONMENT var --- bridgehead | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bridgehead b/bridgehead index 7a66262..db1a469 100755 --- a/bridgehead +++ b/bridgehead @@ -70,22 +70,18 @@ loadVars() { # Set some project-independent default values : ${ENVIRONMENT:=production} + export ENVIRONMENT case "$ENVIRONMENT" in "production") export FOCUS_TAG=main - export ENVIRONMENT="production" ;; "test") export FOCUS_TAG=develop - export ENVIRONMENT="test" - ;; *) report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!" export FOCUS_TAG=main - export ENVIRONMENT="production" - ;; esac } From 33843fe961458730bc70b99f314311487c67aa6a Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 25 Jul 2024 15:56:47 +0200 Subject: [PATCH 23/24] fix: switch id-management to keycloak --- ccp/modules/id-management-compose.yml | 40 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 61a4733..7eef387 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -14,15 +14,15 @@ services: MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY} MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY} MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY} - MAGICPL_OIDC_CLIENT_ID: ${IDMANAGER_AUTH_CLIENT_ID} - MAGICPL_OIDC_CLIENT_SECRET: ${IDMANAGER_AUTH_CLIENT_SECRET} depends_on: - patientlist + - traefik-forward-auth labels: - "traefik.enable=true" - "traefik.http.routers.id-manager.rule=PathPrefix(`/id-manager`)" - "traefik.http.services.id-manager.loadbalancer.server.port=8080" - "traefik.http.routers.id-manager.tls=true" + - "traefik.http.routers.id-manager.middlewares=traefik-forward-auth-idm" patientlist: image: docker.verbis.dkfz.de/bridgehead/mainzelliste @@ -56,5 +56,41 @@ services: # NOTE: Add backups here. This is only imported if /var/lib/bridgehead/data/patientlist/ is empty!!! - "/tmp/bridgehead/patientlist/:/docker-entrypoint-initdb.d/" + traefik-forward-auth: + image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:v7.6.0 + environment: + - http_proxy=http://forward_proxy:3128 + - https_proxy=http://forward_proxy:3128 + - OAUTH2_PROXY_PROVIDER=oidc + - OAUTH2_PROXY_SKIP_PROVIDER_BUTTON=true + - OAUTH2_PROXY_OIDC_ISSUER_URL=https://login.verbis.dkfz.de/realms/master + - OAUTH2_PROXY_CLIENT_ID=bridgehead-${SITE_ID} + - OAUTH2_PROXY_CLIENT_SECRET=${IDMANAGER_AUTH_CLIENT_SECRET} + - OAUTH2_PROXY_COOKIE_SECRET=${IDMANAGER_AUTH_COOKIE_SECRET} + - OAUTH2_PROXY_COOKIE_DOMAINS=.${HOST} + - OAUTH2_PROXY_HTTP_ADDRESS=:4180 + - OAUTH2_PROXY_REVERSE_PROXY=true + - OAUTH2_PROXY_WHITELIST_DOMAINS=.${HOST} + - OAUTH2_PROXY_UPSTREAMS=static://202 + - OAUTH2_PROXY_EMAIL_DOMAINS=* + - OAUTH2_PROXY_SCOPE=openid profile email + # Pass Authorization Header and some user information to backend services + - OAUTH2_PROXY_SET_AUTHORIZATION_HEADER=true + - OAUTH2_PROXY_SET_XAUTHREQUEST=true + # Keycloak has an expiration time of 60s therefore oauth2-proxy needs to refresh after that + - OAUTH2_PROXY_COOKIE_REFRESH=60s + - OAUTH2_PROXY_ALLOWED_GROUPS=DKTK-CCP-PPSN + - OAUTH2_PROXY_PROXY_PREFIX=/oauth2-idm + labels: + - "traefik.enable=true" + - "traefik.http.services.traefik-forward-auth.loadbalancer.server.port=4180" + - "traefik.http.routers.oauth2.rule=PathPrefix(`/oauth2-idm/`)" + - "traefik.http.routers.oauth2.tls=true" + - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.address=http://traefik-forward-auth:4180" + - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.authResponseHeaders=Authorization" + depends_on: + forward_proxy: + condition: service_healthy + volumes: patientlist-db-data: From 6228cb376202d6b35e623b2d44e91e4ef6044acf Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Mon, 19 Aug 2024 17:09:10 +0200 Subject: [PATCH 24/24] fix: specify host for id-management login Otherwise traefik will match the route with the one specified in datashield-compose.yml --- ccp/modules/id-management-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 7eef387..f9156cf 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -84,8 +84,8 @@ services: labels: - "traefik.enable=true" - "traefik.http.services.traefik-forward-auth.loadbalancer.server.port=4180" - - "traefik.http.routers.oauth2.rule=PathPrefix(`/oauth2-idm/`)" - - "traefik.http.routers.oauth2.tls=true" + - "traefik.http.routers.traefik-forward-auth.rule=Host(`${HOST}`) && PathPrefix(`/oauth2-idm`)" + - "traefik.http.routers.traefik-forward-auth.tls=true" - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.address=http://traefik-forward-auth:4180" - "traefik.http.middlewares.traefik-forward-auth-idm.forwardauth.authResponseHeaders=Authorization" depends_on: