From 3e55030b1bfa1871d66b31503bb8ed44d535a312 Mon Sep 17 00:00:00 2001 From: Croft Date: Fri, 27 Jan 2023 13:49:52 +0100 Subject: [PATCH 1/8] Added a Directory sync component * Added new container to bbmri/docker-compose.yml. * Added set up documentation to README. --- README.md | 35 +++++++++++++++++++++++++++++++++++ bbmri/docker-compose.yml | 11 +++++++++++ 2 files changed, 46 insertions(+) diff --git a/README.md b/README.md index b57c10e..79655a8 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,41 @@ All of the Bridgehead's outgoing connections are secured by transport encryption Your Bridgehead's actual data is not stored in the above directories, but in named docker volumes, see `docker volume ls` and `docker volume inspect `. +### Directory sync + +This is an optional feature for bbmri projects. It keeps the [BBMRI Directory](https://directory.bbmri-eric.eu/) up to date with the number of samples, etc. kept in your biobank. It also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly enable this feature if you want to make use of it. + +Full details can be found in [directory_sync_service](https://github.com/samply/directory_sync_service). + +To enable it, you will need to add some extra variables to the ```bbmri.conf``` file in your GitLab repository, like so: + +``` +### Directory sync service +DIRECTORY_URL=https://directory.bbmri-eric.eu +DIRECTORY_USER_NAME=your_directory_username +DIRECTORY_PASS_CODE=qwdnqwswdvqHBVGFR9887 +TIMER_CRON="0 22 * * *" +``` +You must contact the Directory for your national node to find the URL, and to register as a user. + +Additionally, you should choose when you want Directory sync to run. In the example above, this is set to happen at 10 pm every evening. You can modify this to suit your requirements. The timer specification should follow the cron convention. + +Once you have made the changes, update your local configuration: + +```shell +cd /etc/bridgehead +sudo git pull +sudo chown -R bridgehead * .git* +``` + +Then restart the Bridgehead: + +```shell +sudo systemctl restart bridgehead@bbmri.service +``` + +There will be a delay before the effects of Directory sync become visible. First, you will need to wait until the time you have specified in ```TIMER_CRON```. Second, the information will then be synchronized from your national node with the central European Directory. This can take up to 24 hours. + ## Things you should know ### Auto-Updates diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 2433234..b4bc9a9 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -104,6 +104,17 @@ services: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - ./root.crt.pem:/conf/root.crt.pem:ro + directory_sync_service: + image: "samply/directory_sync_service" + environment: + DIRECTORY_URL: ${DIRECTORY_URL} + DIRECTORY_USER_NAME: ${DIRECTORY_USER_NAME} + DIRECTORY_PASS_CODE: ${DIRECTORY_PASS_CODE} + FHIR_STORE_URL: "http://bridgehead-bbmri-blaze:8080" + TIMER_CRON: ${TIMER_CRON} + RETRY_MAX: ${RETRY_MAX} + RETRY_INTERVAL: ${RETRY_INTERVAL} + volumes: blaze-data: From 92dd4b84c1e78eca5942535f100ae1f16bc961a2 Mon Sep 17 00:00:00 2001 From: Croft Date: Tue, 31 Jan 2023 09:43:26 +0100 Subject: [PATCH 2/8] Incorporated new environemnt variable nameing for Directory sync --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 79655a8..3cb8822 100644 --- a/README.md +++ b/README.md @@ -137,14 +137,14 @@ This is an optional feature for bbmri projects. It keeps the [BBMRI Directory]( Full details can be found in [directory_sync_service](https://github.com/samply/directory_sync_service). -To enable it, you will need to add some extra variables to the ```bbmri.conf``` file in your GitLab repository, like so: +To enable it, you will need to add some extra variables to the ```bbmri.conf``` file in your GitLab repository, for example: ``` ### Directory sync service -DIRECTORY_URL=https://directory.bbmri-eric.eu -DIRECTORY_USER_NAME=your_directory_username -DIRECTORY_PASS_CODE=qwdnqwswdvqHBVGFR9887 -TIMER_CRON="0 22 * * *" +DS_DIRECTORY_URL=https://directory.bbmri-eric.eu +DS_DIRECTORY_USER_NAME=your_directory_username +DS_DIRECTORY_USER_PASS=qwdnqwswdvqHBVGFR9887 +DS_TIMER_CRON="0 22 * * *" ``` You must contact the Directory for your national node to find the URL, and to register as a user. From 068125c0623137aeb00e40a1ae217279487cb868 Mon Sep 17 00:00:00 2001 From: Croft Date: Wed, 8 Feb 2023 11:03:35 +0100 Subject: [PATCH 3/8] Updated environemnt variable names so that they start with "DS_" --- bbmri/docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index b4bc9a9..1824ef6 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -107,13 +107,13 @@ services: directory_sync_service: image: "samply/directory_sync_service" environment: - DIRECTORY_URL: ${DIRECTORY_URL} - DIRECTORY_USER_NAME: ${DIRECTORY_USER_NAME} - DIRECTORY_PASS_CODE: ${DIRECTORY_PASS_CODE} - FHIR_STORE_URL: "http://bridgehead-bbmri-blaze:8080" - TIMER_CRON: ${TIMER_CRON} - RETRY_MAX: ${RETRY_MAX} - RETRY_INTERVAL: ${RETRY_INTERVAL} + DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} + DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} + DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} + DS_FHIR_STORE_URL: "http://bridgehead-bbmri-blaze:8080" + DS_TIMER_CRON: ${DS_TIMER_CRON} + DS_RETRY_MAX: ${DS_RETRY_MAX} + DS_RETRY_INTERVAL: ${DS_RETRY_INTERVAL} volumes: blaze-data: From bedad57f416ceade24d337418cca4c4dec4cd49e Mon Sep 17 00:00:00 2001 From: Croft Date: Tue, 14 Mar 2023 09:59:37 +0100 Subject: [PATCH 4/8] Changes for Directory sync PR 53 * Change docker-compose.yml to reduce the number of environment variables being passed to Directory sync. * Improve documentation. --- README.md | 20 ++++---------------- bbmri/docker-compose.yml | 3 --- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 3cb8822..283b81b 100644 --- a/README.md +++ b/README.md @@ -133,11 +133,11 @@ Your Bridgehead's actual data is not stored in the above directories, but in nam ### Directory sync -This is an optional feature for bbmri projects. It keeps the [BBMRI Directory](https://directory.bbmri-eric.eu/) up to date with the number of samples, etc. kept in your biobank. It also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly enable this feature if you want to make use of it. +This is an optional feature for bbmri projects. It keeps the [BBMRI Directory](https://directory.bbmri-eric.eu/) up to date with your local data eg. number of samples. It also updates the local FHIR store with the latest contact details etc. from the Directory. You must explicitly set your country specific directory url, username and password to enable this feature. Full details can be found in [directory_sync_service](https://github.com/samply/directory_sync_service). -To enable it, you will need to add some extra variables to the ```bbmri.conf``` file in your GitLab repository, for example: +To enable it, you will need to set these variables to the ```bbmri.conf``` file of your GitLab repository. Here is an example config: ``` ### Directory sync service @@ -148,21 +148,9 @@ DS_TIMER_CRON="0 22 * * *" ``` You must contact the Directory for your national node to find the URL, and to register as a user. -Additionally, you should choose when you want Directory sync to run. In the example above, this is set to happen at 10 pm every evening. You can modify this to suit your requirements. The timer specification should follow the cron convention. +Additionally, you should choose when you want Directory sync to run. In the example above, this is set to happen at 10 pm every evening. You can modify this to suit your requirements. The timer specification should follow the [cron](https://crontab.guru) convention. -Once you have made the changes, update your local configuration: - -```shell -cd /etc/bridgehead -sudo git pull -sudo chown -R bridgehead * .git* -``` - -Then restart the Bridgehead: - -```shell -sudo systemctl restart bridgehead@bbmri.service -``` +Once you edited the gitlab config. The bridgehead will autoupdate the config with the values and will sync the data. There will be a delay before the effects of Directory sync become visible. First, you will need to wait until the time you have specified in ```TIMER_CRON```. Second, the information will then be synchronized from your national node with the central European Directory. This can take up to 24 hours. diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 1824ef6..26f2378 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -110,10 +110,7 @@ services: DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} - DS_FHIR_STORE_URL: "http://bridgehead-bbmri-blaze:8080" DS_TIMER_CRON: ${DS_TIMER_CRON} - DS_RETRY_MAX: ${DS_RETRY_MAX} - DS_RETRY_INTERVAL: ${DS_RETRY_INTERVAL} volumes: blaze-data: From df74d6d768dd4d9b69b1c044e17d9cd2aab75429 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Fri, 31 Mar 2023 08:04:28 +0200 Subject: [PATCH 5/8] Make directory sync opt service --- bbmri/directory-sync-compose.yml | 8 ++++++++ bbmri/directory-sync.sh | 8 ++++++++ bbmri/docker-compose.yml | 8 -------- bbmri/vars | 4 ++++ 4 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 bbmri/directory-sync-compose.yml create mode 100755 bbmri/directory-sync.sh diff --git a/bbmri/directory-sync-compose.yml b/bbmri/directory-sync-compose.yml new file mode 100644 index 0000000..486b924 --- /dev/null +++ b/bbmri/directory-sync-compose.yml @@ -0,0 +1,8 @@ +services: + directory_sync_service: + image: "samply/directory_sync_service" + environment: + DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} + DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} + DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} + DS_TIMER_CRON: ${DS_TIMER_CRON} \ No newline at end of file diff --git a/bbmri/directory-sync.sh b/bbmri/directory-sync.sh new file mode 100755 index 0000000..2eeef4a --- /dev/null +++ b/bbmri/directory-sync.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +function dirSetup() { + if [ -n "$DS_DIRECTORY_USER_NAME" ]; then + log INFO "Directory sync setup detected -- will start directory sync service." + OVERRIDE+=" -f ./$PROJECT/directory-sync-compose.yml" + fi +} diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 26f2378..2433234 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -104,14 +104,6 @@ services: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - ./root.crt.pem:/conf/root.crt.pem:ro - directory_sync_service: - image: "samply/directory_sync_service" - environment: - DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} - DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} - DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} - DS_TIMER_CRON: ${DS_TIMER_CRON} - volumes: blaze-data: diff --git a/bbmri/vars b/bbmri/vars index 6fb693d..a9c65cb 100644 --- a/bbmri/vars +++ b/bbmri/vars @@ -5,3 +5,7 @@ SPOT_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | he SPOT_BEAM_SECRET_LONG="ApiKey spot.${PROXY_ID} ${SPOT_BEAM_SECRET_SHORT}" SUPPORT_EMAIL=bridgehead@helpdesk.bbmri-eric.eu PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem + +# This will load directory-sync setup. +source $PROJECT/directory-sync.sh +dirSetup From fa41f8d77f3313a836ce9003d981a1951cf913d1 Mon Sep 17 00:00:00 2001 From: Croft Date: Fri, 31 Mar 2023 10:01:51 +0200 Subject: [PATCH 6/8] Changed image to docker.verbis.dkfz.de/cache/ Requested by Torben Brenner in PR 53 --- bbmri/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 26f2378..0ec9ca3 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -105,7 +105,7 @@ services: - ./root.crt.pem:/conf/root.crt.pem:ro directory_sync_service: - image: "samply/directory_sync_service" + image: "docker.verbis.dkfz.de/cache/samply/directory_sync_service" environment: DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} From f4e65cc3d00cdc06bab7860474beb02a221a5d09 Mon Sep 17 00:00:00 2001 From: Croft Date: Fri, 31 Mar 2023 11:55:19 +0200 Subject: [PATCH 7/8] Implemented Torbens request for PR 53 --- bbmri/directory-sync-compose.yml | 4 ++-- bbmri/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bbmri/directory-sync-compose.yml b/bbmri/directory-sync-compose.yml index 486b924..9776ecb 100644 --- a/bbmri/directory-sync-compose.yml +++ b/bbmri/directory-sync-compose.yml @@ -1,8 +1,8 @@ services: directory_sync_service: - image: "samply/directory_sync_service" + image: "docker.verbis.dkfz.de/cache/samply/directory_sync_service" environment: DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} - DS_TIMER_CRON: ${DS_TIMER_CRON} \ No newline at end of file + DS_TIMER_CRON: ${DS_TIMER_CRON} diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 0ec9ca3..26f2378 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -105,7 +105,7 @@ services: - ./root.crt.pem:/conf/root.crt.pem:ro directory_sync_service: - image: "docker.verbis.dkfz.de/cache/samply/directory_sync_service" + image: "samply/directory_sync_service" environment: DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} From 5b926ba20c58b106b7d8fc1b09e7f09bbfb36025 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Fri, 31 Mar 2023 12:15:43 +0200 Subject: [PATCH 8/8] Remove opt directory_sync from compose --- bbmri/docker-compose.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 26f2378..2433234 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -104,14 +104,6 @@ services: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - ./root.crt.pem:/conf/root.crt.pem:ro - directory_sync_service: - image: "samply/directory_sync_service" - environment: - DS_DIRECTORY_URL: ${DS_DIRECTORY_URL} - DS_DIRECTORY_USER_NAME: ${DS_DIRECTORY_USER_NAME} - DS_DIRECTORY_PASS_CODE: ${DS_DIRECTORY_PASS_CODE} - DS_TIMER_CRON: ${DS_TIMER_CRON} - volumes: blaze-data: