From 5c28e704d2b04d2404d2d71cd9abbb6aae35c32a Mon Sep 17 00:00:00 2001 From: Jan <59206115+Threated@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:27:27 +0100 Subject: [PATCH 01/19] fix: remove `restart: always` in compose files (#261) --- ccp/modules/nngm-compose.yml | 1 - ccp/modules/obds2fhir-rest-compose.yml | 1 - kr/modules/obds2fhir-rest-compose.yml | 1 - minimal/modules/nngm-compose.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/ccp/modules/nngm-compose.yml b/ccp/modules/nngm-compose.yml index 7ffa190..36e9f27 100644 --- a/ccp/modules/nngm-compose.yml +++ b/ccp/modules/nngm-compose.yml @@ -12,7 +12,6 @@ services: CTS_API_KEY: ${NNGM_CTS_APIKEY} CRYPT_KEY: ${NNGM_CRYPTKEY} #CTS_MAGICPL_SITE: ${SITE_ID}TODO - restart: always labels: - "traefik.enable=true" - "traefik.http.routers.connector.rule=PathPrefix(`/nngm-connector`)" diff --git a/ccp/modules/obds2fhir-rest-compose.yml b/ccp/modules/obds2fhir-rest-compose.yml index f201e23..833580d 100644 --- a/ccp/modules/obds2fhir-rest-compose.yml +++ b/ccp/modules/obds2fhir-rest-compose.yml @@ -10,7 +10,6 @@ services: SALT: ${LOCAL_SALT} KEEP_INTERNAL_ID: ${KEEP_INTERNAL_ID:-false} MAINZELLISTE_URL: ${PATIENTLIST_URL:-http://patientlist:8080/patientlist} - restart: always labels: - "traefik.enable=true" - "traefik.http.routers.obds2fhir-rest.rule=PathPrefix(`/obds2fhir-rest`) || PathPrefix(`/adt2fhir-rest`)" diff --git a/kr/modules/obds2fhir-rest-compose.yml b/kr/modules/obds2fhir-rest-compose.yml index f201e23..833580d 100644 --- a/kr/modules/obds2fhir-rest-compose.yml +++ b/kr/modules/obds2fhir-rest-compose.yml @@ -10,7 +10,6 @@ services: SALT: ${LOCAL_SALT} KEEP_INTERNAL_ID: ${KEEP_INTERNAL_ID:-false} MAINZELLISTE_URL: ${PATIENTLIST_URL:-http://patientlist:8080/patientlist} - restart: always labels: - "traefik.enable=true" - "traefik.http.routers.obds2fhir-rest.rule=PathPrefix(`/obds2fhir-rest`) || PathPrefix(`/adt2fhir-rest`)" diff --git a/minimal/modules/nngm-compose.yml b/minimal/modules/nngm-compose.yml index e61532d..8e42e71 100644 --- a/minimal/modules/nngm-compose.yml +++ b/minimal/modules/nngm-compose.yml @@ -11,7 +11,6 @@ services: CTS_API_KEY: ${NNGM_CTS_APIKEY} CRYPT_KEY: ${NNGM_CRYPTKEY} #CTS_MAGICPL_SITE: ${SITE_ID}TODO - restart: always labels: - "traefik.enable=true" - "traefik.http.routers.connector.rule=PathPrefix(`/nngm-connector`)" From 3d1105b97ca64aa42b03f2f928d530303a25bb45 Mon Sep 17 00:00:00 2001 From: patrickskowronekdkfz <86347677+patrickskowronekdkfz@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:28:47 +0100 Subject: [PATCH 02/19] Update: Blaze to version 0.31 (#260) --- bbmri/docker-compose.yml | 2 +- cce/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- ccp/modules/blaze-secondary-compose.yml | 2 +- dhki/docker-compose.yml | 2 +- itcc/docker-compose.yml | 2 +- kr/docker-compose.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 000df01..1903c62 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.31 container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" diff --git a/cce/docker-compose.yml b/cce/docker-compose.yml index 87b6b1c..0641af7 100644 --- a/cce/docker-compose.yml +++ b/cce/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.31 container_name: bridgehead-cce-blaze environment: BASE_URL: "http://bridgehead-cce-blaze:8080" diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index b7a71b2..871eec2 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.31 container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" diff --git a/ccp/modules/blaze-secondary-compose.yml b/ccp/modules/blaze-secondary-compose.yml index b57bfbe..ad748a6 100644 --- a/ccp/modules/blaze-secondary-compose.yml +++ b/ccp/modules/blaze-secondary-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze-secondary: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.31 container_name: bridgehead-ccp-blaze-secondary environment: BASE_URL: "http://bridgehead-ccp-blaze-secondary:8080" diff --git a/dhki/docker-compose.yml b/dhki/docker-compose.yml index ee8cd17..d37f1a2 100644 --- a/dhki/docker-compose.yml +++ b/dhki/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.31 container_name: bridgehead-dhki-blaze environment: BASE_URL: "http://bridgehead-dhki-blaze:8080" diff --git a/itcc/docker-compose.yml b/itcc/docker-compose.yml index 7aab26d..c9bce0c 100644 --- a/itcc/docker-compose.yml +++ b/itcc/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.31 container_name: bridgehead-itcc-blaze environment: BASE_URL: "http://bridgehead-itcc-blaze:8080" diff --git a/kr/docker-compose.yml b/kr/docker-compose.yml index 47a9db6..17b36b7 100644 --- a/kr/docker-compose.yml +++ b/kr/docker-compose.yml @@ -6,7 +6,7 @@ services: replicas: 0 #deactivate landing page blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.31 container_name: bridgehead-kr-blaze environment: BASE_URL: "http://bridgehead-kr-blaze:8080" From 1003cd73cf6c8a089366f0a72fdeccf0d720b90f Mon Sep 17 00:00:00 2001 From: patrickskowronekdkfz <86347677+patrickskowronekdkfz@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:30:20 +0100 Subject: [PATCH 03/19] fix: changed ccp_ppi to use IDMANAGEMENT_FRIENDLY_ID instead of SITE_NAME (#259) --- ccp/modules/id-management-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index b7c3f61..4e3e90a 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -106,7 +106,7 @@ services: container_name: bridgehead-ccp-patient-project-identificator environment: MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - SITE_NAME: ${SITE_NAME} + SITE_NAME: ${IDMANAGEMENT_FRIENDLY_ID} volumes: patientlist-db-data: From 910289079b7defbef187666700ea7d9c6f0e9df6 Mon Sep 17 00:00:00 2001 From: patrickskowronekdkfz <86347677+patrickskowronekdkfz@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:35:25 +0100 Subject: [PATCH 04/19] docs: documentation for changing your configuration repository access token (#256) --- README.md | 2 ++ docs/update-access-token.md | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 docs/update-access-token.md diff --git a/README.md b/README.md index 3c36053..b7e60ad 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,8 @@ sh bridgehead uninstall ## Site-specific configuration +[How to Change Config Access Token](docs/update-access-token.md) + ### HTTPS Access Even within your internal network, the Bridgehead enforces HTTPS for all services. During the installation, a self-signed, long-lived certificate was created for you. To increase security, you can simply replace the files under `/etc/bridgehead/traefik-tls` with ones from established certification authorities such as [Let's Encrypt](https://letsencrypt.org) or [DFN-AAI](https://www.aai.dfn.de). diff --git a/docs/update-access-token.md b/docs/update-access-token.md new file mode 100644 index 0000000..d608d45 --- /dev/null +++ b/docs/update-access-token.md @@ -0,0 +1,42 @@ +## How to Change Config Access Token + +### 1. Generate a New Access Token + +1. Go to your Git configuration repository provider, it might be either [git.verbis.dkfz.de](https://git.verbis.dkfz.de) or [gitlab.bbmri-eric.eu](https://gitlab.bbmri-eric.eu). +2. Navigate to the configuration repository for your site. +3. Go to **Settings → Access Tokens** to check if your Access Token is valid or expired. + - **If expired**, create a new Access Token. +4. Configure the new Access Token with the following settings: + - **Expiration date**: One year from today, minus one day. + - **Role**: Developer. + - **Scope**: Only `read_repository`. +5. Save the newly generated Access Token in a secure location. + +--- + +### 2. Replace the Old Access Token + +1. Navigate to `/etc/bridgehead` in your system. +2. Run the following command to retrieve the current Git remote URL: + ```bash + git remote get-url origin + ``` + Example output: + ``` + https://name40dkfz-heidelberg.de:@git.verbis.dkfz.de/bbmri-bridgehead-configs/test.git + ``` +3. Replace `` with your new Access Token in the URL. +4. Set the updated URL using the following command: + ```bash + git remote set-url origin https://name40dkfz-heidelberg.de:@git.verbis.dkfz.de/bbmri-bridgehead-configs/test.git + + ``` + +5. Start the Bridgehead update service by running: + ```bash + systemctl start bridgehead-update@ + ``` +6. View the output to ensure the update process is successful: + ```bash + journalctl -u bridgehead-update@ -f + ``` \ No newline at end of file From 47364f999eee89275fb2fef7570e7f1da66d4b33 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 8 Jan 2025 15:27:00 +0000 Subject: [PATCH 05/19] wip: routine connector --- dhki/vars | 10 +++++++++- modules/transfair-compose.yml | 19 +++++++++++++++++++ modules/transfair-output-blaze-compose.yml | 17 +++++++++++++++++ modules/transfair-setup.sh | 13 +++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 modules/transfair-compose.yml create mode 100644 modules/transfair-output-blaze-compose.yml create mode 100755 modules/transfair-setup.sh diff --git a/dhki/vars b/dhki/vars index b728925..f7f7ecd 100644 --- a/dhki/vars +++ b/dhki/vars @@ -17,4 +17,12 @@ do done idManagementSetup -obds2fhirRestSetup \ No newline at end of file +obds2fhirRestSetup + +for module in modules/*.sh +do + log DEBUG "sourcing $module" + source $module +done + +routineConnectorSetup \ No newline at end of file diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml new file mode 100644 index 0000000..863ae4a --- /dev/null +++ b/modules/transfair-compose.yml @@ -0,0 +1,19 @@ + +services: + transfair: + image: samply/routine-connector:latest # TODO: Harbor image and new name ofc + container_name: bridgehead-transfair + environment: + INSTITUTE_TTP_URL: "${INSTITUTE_TTP_URL}" + INSTITUTE_TTP_API_KEY: "${INSTITUTE_TTP_API_KEY}" + PROJECT_ID_SYSTEM: "${PROJECT_ID_SYSTEM}" + FHIR_REQUEST_URL: "${FHIR_REQUEST_URL}" + FHIR_INPUT_URL: "${FHIR_INPUT_URL}" + FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL}" + FHIR_REQUEST_CREDENTIALS: "${FHIR_REQUEST_CREDENTIALS}" + FHIR_INPUT_CREDENTIALS: "${FHIR_INPUT_CREDENTIALS}" + FHIR_OUTPUT_CREDENTIALS: "${FHIR_OUTPUT_CREDENTIALS}" + EXCHANGE_ID_SYSTEM: "SESSION_ID" + DATABASE_URL: "sqlite://transfair/data_requests.sql?mode=rwc" + volumes: + - /var/cache/bridgehead/${PROJECT}/transfair:/transfair diff --git a/modules/transfair-output-blaze-compose.yml b/modules/transfair-output-blaze-compose.yml new file mode 100644 index 0000000..9f8583b --- /dev/null +++ b/modules/transfair-output-blaze-compose.yml @@ -0,0 +1,17 @@ + +services: + transfair-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-blaze + environment: + BASE_URL: "http://bridgehead-ccp-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} + CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-blaze-data:/app/data" + +volumes: + transfair-balze-data: \ No newline at end of file diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh new file mode 100755 index 0000000..6dff9c2 --- /dev/null +++ b/modules/transfair-setup.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e + +function transfairSetup() { + assertVarsNotEmpty INSTITUTE_TTP_URL INSTITUTE_TTP_API_KEY PROJECT_ID_SYSTEM FHIR_REQUEST_URL FHIR_INPUT_URL + OVERRIDE+=" -f ./modules/transfair-compose.yml" + if [ -n "$FHIR_OUTPUT_URL" ]; then + log INFO "TransFAIR output fhir store set to external $FHIR_OUTPUT_URL" + else + log INFO "TransFAIR output fhir store not set writing to internal blaze" + FHIR_OUTPUT_URL="http://transfair-blaze:8080" + OVERRIDE+=" -f ./modules/transfair-compose.yml" + fi +} From 2058a7a5c922515f3b6a38a1d23c3b2be3062c80 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 23 Jan 2025 12:06:43 +0000 Subject: [PATCH 06/19] update image url --- modules/transfair-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 863ae4a..2fa980f 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -1,7 +1,7 @@ services: transfair: - image: samply/routine-connector:latest # TODO: Harbor image and new name ofc + image: docker.verbis.dkfz.de/cache/samply/transfair:latest container_name: bridgehead-transfair environment: INSTITUTE_TTP_URL: "${INSTITUTE_TTP_URL}" From 139fcecabe2825ad3c253b4565baae92f4d01459 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 23 Jan 2025 12:42:59 +0000 Subject: [PATCH 07/19] redo transfair setup --- dhki/vars | 2 +- modules/transfair-compose.yml | 36 ++++++++++++++++++++-- modules/transfair-output-blaze-compose.yml | 17 ---------- modules/transfair-setup.sh | 23 ++++++++++---- 4 files changed, 52 insertions(+), 26 deletions(-) delete mode 100644 modules/transfair-output-blaze-compose.yml diff --git a/dhki/vars b/dhki/vars index f7f7ecd..d043dd2 100644 --- a/dhki/vars +++ b/dhki/vars @@ -25,4 +25,4 @@ do source $module done -routineConnectorSetup \ No newline at end of file +transfairSetup \ No newline at end of file diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 2fa980f..3927fd9 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -9,11 +9,43 @@ services: PROJECT_ID_SYSTEM: "${PROJECT_ID_SYSTEM}" FHIR_REQUEST_URL: "${FHIR_REQUEST_URL}" FHIR_INPUT_URL: "${FHIR_INPUT_URL}" - FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL}" + FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL:-http://blaze:8080}" FHIR_REQUEST_CREDENTIALS: "${FHIR_REQUEST_CREDENTIALS}" FHIR_INPUT_CREDENTIALS: "${FHIR_INPUT_CREDENTIALS}" FHIR_OUTPUT_CREDENTIALS: "${FHIR_OUTPUT_CREDENTIALS}" - EXCHANGE_ID_SYSTEM: "SESSION_ID" + EXCHANGE_ID_SYSTEM: "${EXCHANGE_ID_SYSTEM:-SESSION_ID}" DATABASE_URL: "sqlite://transfair/data_requests.sql?mode=rwc" volumes: - /var/cache/bridgehead/${PROJECT}/transfair:/transfair + + transfair-input-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-input-blaze + environment: + BASE_URL: "http://bridgehead-transfair-input-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} + CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-input-blaze-data:/app/data" + profiles: ["transfair-input-blaze"] + + transfair-request-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-requests-blaze + environment: + BASE_URL: "http://bridgehead-transfair-requests-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} + CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-request-blaze-data:/app/data" + profiles: ["transfair-request-blaze"] + +volumes: + transfair-input-blaze-data: + transfair-request-blaze-data: \ No newline at end of file diff --git a/modules/transfair-output-blaze-compose.yml b/modules/transfair-output-blaze-compose.yml deleted file mode 100644 index 9f8583b..0000000 --- a/modules/transfair-output-blaze-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ - -services: - transfair-blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 - container_name: bridgehead-transfair-blaze - environment: - BASE_URL: "http://bridgehead-ccp-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" - DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} - DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} - CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} - ENFORCE_REFERENTIAL_INTEGRITY: "false" - volumes: - - "transfair-blaze-data:/app/data" - -volumes: - transfair-balze-data: \ No newline at end of file diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh index 6dff9c2..720bb25 100755 --- a/modules/transfair-setup.sh +++ b/modules/transfair-setup.sh @@ -1,13 +1,24 @@ #!/bin/bash -e function transfairSetup() { - assertVarsNotEmpty INSTITUTE_TTP_URL INSTITUTE_TTP_API_KEY PROJECT_ID_SYSTEM FHIR_REQUEST_URL FHIR_INPUT_URL + if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then + echo "Starting transfair." + else + return + fi OVERRIDE+=" -f ./modules/transfair-compose.yml" - if [ -n "$FHIR_OUTPUT_URL" ]; then - log INFO "TransFAIR output fhir store set to external $FHIR_OUTPUT_URL" + if [ -n "$FHIR_INPUT_URL" ]; then + log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" else - log INFO "TransFAIR output fhir store not set writing to internal blaze" - FHIR_OUTPUT_URL="http://transfair-blaze:8080" - OVERRIDE+=" -f ./modules/transfair-compose.yml" + log INFO "TransFAIR input fhir store not set writing to internal blaze" + FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" + OVERRIDE+=" --profile transfair-input-blaze" + fi + if [ -n "$FHIR_REQUEST_URL" ]; then + log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" + else + log INFO "TransFAIR request fhir store not set writing to internal blaze" + FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" + OVERRIDE+=" --profile transfair-request-blaze" fi } From fa0d9fb8b4a47e11f36b37a6999685f20a44ba89 Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 24 Jan 2025 09:23:08 +0000 Subject: [PATCH 08/19] restrict additional blaze memory usage --- modules/transfair-compose.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 3927fd9..08d1a09 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -23,10 +23,9 @@ services: container_name: bridgehead-transfair-input-blaze environment: BASE_URL: "http://bridgehead-transfair-input-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" - DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} - DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} - CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + JAVA_TOOL_OPTIONS: "-Xmx1024m" + DB_BLOCK_CACHE_SIZE: 1024 + CQL_EXPR_CACHE_SIZE: 8 ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "transfair-input-blaze-data:/app/data" @@ -37,10 +36,9 @@ services: container_name: bridgehead-transfair-requests-blaze environment: BASE_URL: "http://bridgehead-transfair-requests-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" - DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} - DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} - CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + JAVA_TOOL_OPTIONS: "-Xmx1024m" + DB_BLOCK_CACHE_SIZE: 1024 + CQL_EXPR_CACHE_SIZE: 8 ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "transfair-request-blaze-data:/app/data" From db950d6d870246141c2b11c6d34755143dec42b8 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 28 Jan 2025 08:59:57 +0000 Subject: [PATCH 09/19] Fixed Docker Hub URL list link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7e60ad..86bd1d9 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ The following URLs need to be accessible (prefix with `https://`): * git.verbis.dkfz.de * To fetch docker images * docker.verbis.dkfz.de - * Official Docker, Inc. URLs (subject to change, see [official list](https://docs.docker.com/desktop/all)) + * Official Docker, Inc. URLs (subject to change, see [official list](https://docs.docker.com/desktop/setup/allow-list/)) * hub.docker.com * registry-1.docker.io * production.cloudflare.docker.com From 615990b92a5c7066f51c9e4563fcda2deff351d0 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:53:49 +0100 Subject: [PATCH 10/19] Use secret-sync for gitpassword (#257) --------- Co-authored-by: Tim Schumacher Co-authored-by: Jan <59206115+Threated@users.noreply.github.com> Co-authored-by: Tim Schumacher --- lib/functions.sh | 2 +- lib/gitlab-token-helper.sh | 11 ++++++++++ lib/gitpassword.sh | 41 ----------------------------------- lib/update-bridgehead.sh | 44 ++++++++++++++++++++++++++++++++------ 4 files changed, 50 insertions(+), 48 deletions(-) create mode 100755 lib/gitlab-token-helper.sh delete mode 100755 lib/gitpassword.sh diff --git a/lib/functions.sh b/lib/functions.sh index 3fcae38..ed57293 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -116,7 +116,7 @@ assertVarsNotEmpty() { MISSING_VARS="" for VAR in $@; do - if [ -z "${!VAR}" ]; then + if [ -z "${!VAR}" ]; then MISSING_VARS+="$VAR " fi done diff --git a/lib/gitlab-token-helper.sh b/lib/gitlab-token-helper.sh new file mode 100755 index 0000000..e618029 --- /dev/null +++ b/lib/gitlab-token-helper.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +[ "$1" = "get" ] || exit + +source /var/cache/bridgehead/secrets/gitlab_token + +# Any non-empty username works, only the token matters +cat << EOF +username=bk +password=$BRIDGEHEAD_CONFIG_REPO_TOKEN +EOF \ No newline at end of file diff --git a/lib/gitpassword.sh b/lib/gitpassword.sh deleted file mode 100755 index 17756d6..0000000 --- a/lib/gitpassword.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -if [ "$1" != "get" ]; then - echo "Usage: $0 get" - exit 1 -fi - -baseDir() { - # see https://stackoverflow.com/questions/59895 - SOURCE=${BASH_SOURCE[0]} - while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - DIR=$( cd -P "$( dirname "$SOURCE" )/.." >/dev/null 2>&1 && pwd ) - echo $DIR -} - -BASE=$(baseDir) -cd $BASE - -source lib/functions.sh - -assertVarsNotEmpty SITE_ID || fail_and_report 1 "gitpassword.sh failed: SITE_ID is empty." - -PARAMS="$(cat)" -GITHOST=$(echo "$PARAMS" | grep "^host=" | sed 's/host=\(.*\)/\1/g') - -fetchVarsFromVault GIT_PASSWORD - -if [ -z "${GIT_PASSWORD}" ]; then - fail_and_report 1 "gitpassword.sh failed: Git password not found." -fi - -cat < Date: Wed, 29 Jan 2025 10:59:27 +0100 Subject: [PATCH 11/19] fix: properly load oidc secrets (#267) --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index ed57293..ffdc234 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -318,7 +318,7 @@ function sync_secrets() { docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest set -a # Export variables as environment variables - source /var/cache/bridgehead/secrets/* + source /var/cache/bridgehead/secrets/oidc set +a # Export variables in the regular way } From e3553370b6e4518122a9d14000fb1b87e6f73d8e Mon Sep 17 00:00:00 2001 From: Jan <59206115+Threated@users.noreply.github.com> Date: Wed, 29 Jan 2025 13:17:59 +0100 Subject: [PATCH 12/19] feat: unify version handeling (#265) --- bridgehead | 49 ++++++++++++++++++++++++++++--------------------- versions/prod | 2 ++ versions/test | 2 ++ 3 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 versions/prod create mode 100644 versions/test diff --git a/bridgehead b/bridgehead index d5d3a20..cbe7527 100755 --- a/bridgehead +++ b/bridgehead @@ -53,17 +53,44 @@ case "$PROJECT" in ;; esac +# Loads config variables and runs the projects setup script loadVars() { - # Load variables from /etc/bridgehead and /srv/docker/bridgehead set -a + # Source the project specific config file source /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "/etc/bridgehead/$PROJECT.conf not found" + # Source the project specific local config file if present + # This file is ignored by git as oposed to the regular config file as it contains private site information like etl auth data if [ -e /etc/bridgehead/$PROJECT.local.conf ]; then log INFO "Applying /etc/bridgehead/$PROJECT.local.conf" source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import" fi + # Set execution environment on main default to prod else test + if [[ -z "${ENVIRONMENT+x}" ]]; then + if [ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]; then + ENVIRONMENT="production" + else + ENVIRONMENT="test" + fi + fi + # Source the versions of the images components + case "$ENVIRONMENT" in + "production") + source ./versions/prod + ;; + "test") + source ./versions/test + ;; + *) + report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!" + source ./versions/prod + ;; + esac fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" setHostname optimizeBlazeMemoryUsage + # Run project specific setup if it exists + # This will ususally modiy the `OVERRIDE` to include all the compose files that the project depends on + # This is also where projects specify which modules to load [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a @@ -79,26 +106,6 @@ loadVars() { fi detectCompose setupProxy - - # Set some project-independent default values - : ${ENVIRONMENT:=production} - export ENVIRONMENT - - case "$ENVIRONMENT" in - "production") - export FOCUS_TAG=main - export BEAM_TAG=main - ;; - "test") - export FOCUS_TAG=develop - export BEAM_TAG=develop - ;; - *) - report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!" - export FOCUS_TAG=main - export BEAM_TAG=main - ;; - esac } case "$ACTION" in diff --git a/versions/prod b/versions/prod new file mode 100644 index 0000000..1dd754f --- /dev/null +++ b/versions/prod @@ -0,0 +1,2 @@ +FOCUS_TAG=main +BEAM_TAG=main \ No newline at end of file diff --git a/versions/test b/versions/test new file mode 100644 index 0000000..10ae062 --- /dev/null +++ b/versions/test @@ -0,0 +1,2 @@ +FOCUS_TAG=develop +BEAM_TAG=develop \ No newline at end of file From 721627a78fed32dfb8472dcec9321f4e4999c108 Mon Sep 17 00:00:00 2001 From: Jan <59206115+Threated@users.noreply.github.com> Date: Wed, 5 Feb 2025 13:24:28 +0100 Subject: [PATCH 13/19] feat: migrate to new dnpm:dip node (#251) * feat: migrate to new dnpm:dip node * hardcode dnpm connector type to broker * use `SITE_NAME` for dnpm `LOCAL_SITE` * host central targets in git * dnpm: add goettingen to central targets * dnpm: add uksh to central targets * dnpm: replace named volumes with fs volumes * chore: change dnpm images * chore: pin mysql * dnpm: Secure endpoints for ETL and p2p communications (#254) * fix authup redirect (#262) When a OIDC provider is configured, you'll get redirected to authup by Keycloak which redirects you to the DNPM:DIP. Currently the url looks like this: https://myserver/authup//someurl and produces an error. Manually removing the additional / fixes the issue. * Whitespace formatting --------- Co-authored-by: Niklas Co-authored-by: Niklas Reimer Co-authored-by: Martin Lablans <6804500+lablans@users.noreply.github.com> --- ccp/modules/dnpm-compose.yml | 4 +- ccp/modules/dnpm-node-compose.yml | 115 ++++++++++++++---- ccp/modules/dnpm-node-setup.sh | 22 +--- minimal/docker-compose.yml | 2 +- minimal/modules/dnpm-central-targets.json | 142 ++++++++++++++++++++++ minimal/modules/dnpm-compose.yml | 4 +- minimal/modules/dnpm-node-compose.yml | 115 ++++++++++++++---- minimal/modules/dnpm-node-setup.sh | 22 +--- 8 files changed, 337 insertions(+), 89 deletions(-) create mode 100644 minimal/modules/dnpm-central-targets.json diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index c32426f..0ce7f74 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -13,7 +13,7 @@ services: PROXY_APIKEY: ${DNPM_BEAM_SECRET_SHORT} APP_ID: dnpm-connect.${PROXY_ID} DISCOVERY_URL: "./conf/central_targets.json" - LOCAL_TARGETS_FILE: "./conf/connect_targets.json" + LOCAL_TARGETS_FILE: "/conf/connect_targets.json" HTTP_PROXY: "http://forward_proxy:3128" HTTPS_PROXY: "http://forward_proxy:3128" NO_PROXY: beam-proxy,dnpm-backend,host.docker.internal${DNPM_ADDITIONAL_NO_PROXY} @@ -25,7 +25,7 @@ services: volumes: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - /etc/bridgehead/dnpm/local_targets.json:/conf/connect_targets.json:ro - - /etc/bridgehead/dnpm/central_targets.json:/conf/central_targets.json:ro + - /srv/docker/bridgehead/minimal/modules/dnpm-central-targets.json:/conf/central_targets.json:ro labels: - "traefik.enable=true" - "traefik.http.routers.dnpm-connect.rule=PathPrefix(`/dnpm-connect`)" diff --git a/ccp/modules/dnpm-node-compose.yml b/ccp/modules/dnpm-node-compose.yml index ee84d89..c1f7dde 100644 --- a/ccp/modules/dnpm-node-compose.yml +++ b/ccp/modules/dnpm-node-compose.yml @@ -1,34 +1,99 @@ version: "3.7" services: - dnpm-backend: - image: ghcr.io/kohlbacherlab/bwhc-backend:1.0-snapshot-broker-connector - container_name: bridgehead-dnpm-backend + dnpm-mysql: + image: mysql:9 + healthcheck: + test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ] + interval: 3s + timeout: 5s + retries: 5 environment: - - ZPM_SITE=${ZPM_SITE} - - N_RANDOM_FILES=${DNPM_SYNTH_NUM} + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD: ${DNPM_MYSQL_ROOT_PASSWORD} volumes: - - /etc/bridgehead/dnpm:/bwhc_config:ro - - ${DNPM_DATA_DIR}:/bwhc_data - labels: - - "traefik.enable=true" - - "traefik.http.routers.bwhc-backend.rule=PathPrefix(`/bwhc`)" - - "traefik.http.services.bwhc-backend.loadbalancer.server.port=9000" - - "traefik.http.routers.bwhc-backend.tls=true" + - /var/cache/bridgehead/dnpm/mysql:/var/lib/mysql - dnpm-frontend: - image: ghcr.io/kohlbacherlab/bwhc-frontend:2209 - container_name: bridgehead-dnpm-frontend - links: - - dnpm-backend + dnpm-authup: + image: authup/authup:latest + container_name: bridgehead-dnpm-authup + volumes: + - /var/cache/bridgehead/dnpm/authup:/usr/src/app/writable + depends_on: + dnpm-mysql: + condition: service_healthy + command: server/core start environment: - - NUXT_HOST=0.0.0.0 - - NUXT_PORT=8080 - - BACKEND_PROTOCOL=https - - BACKEND_HOSTNAME=$HOST - - BACKEND_PORT=443 + - PUBLIC_URL=https://${HOST}/auth/ + - AUTHORIZE_REDIRECT_URL=https://${HOST} + - ROBOT_ADMIN_ENABLED=true + - ROBOT_ADMIN_SECRET=${DNPM_AUTHUP_SECRET} + - ROBOT_ADMIN_SECRET_RESET=true + - DB_TYPE=mysql + - DB_HOST=dnpm-mysql + - DB_USERNAME=root + - DB_PASSWORD=${DNPM_MYSQL_ROOT_PASSWORD} + - DB_DATABASE=auth labels: - "traefik.enable=true" - - "traefik.http.routers.bwhc-frontend.rule=PathPrefix(`/`)" - - "traefik.http.services.bwhc-frontend.loadbalancer.server.port=8080" - - "traefik.http.routers.bwhc-frontend.tls=true" + - "traefik.http.middlewares.authup-strip.stripprefix.prefixes=/auth" + - "traefik.http.routers.dnpm-auth.middlewares=authup-strip" + - "traefik.http.routers.dnpm-auth.rule=PathPrefix(`/auth`)" + - "traefik.http.services.dnpm-auth.loadbalancer.server.port=3000" + - "traefik.http.routers.dnpm-auth.tls=true" + + dnpm-portal: + image: ghcr.io/dnpm-dip/portal:latest + container_name: bridgehead-dnpm-portal + environment: + - NUXT_API_URL=http://dnpm-backend:9000/ + - NUXT_PUBLIC_API_URL=https://${HOST}/api/ + - NUXT_AUTHUP_URL=http://dnpm-authup:3000/ + - NUXT_PUBLIC_AUTHUP_URL=https://${HOST}/auth/ + labels: + - "traefik.enable=true" + - "traefik.http.routers.dnpm-frontend.rule=PathPrefix(`/`)" + - "traefik.http.services.dnpm-frontend.loadbalancer.server.port=3000" + - "traefik.http.routers.dnpm-frontend.tls=true" + + dnpm-backend: + container_name: bridgehead-dnpm-backend + image: ghcr.io/dnpm-dip/backend:latest + environment: + - LOCAL_SITE=${ZPM_SITE}:${SITE_NAME} # Format: {Site-ID}:{Site-name}, e.g. UKT:Tübingen + - RD_RANDOM_DATA=${DNPM_SYNTH_NUM:--1} + - MTB_RANDOM_DATA=${DNPM_SYNTH_NUM:--1} + - HATEOAS_HOST=https://${HOST} + - CONNECTOR_TYPE=broker + - AUTHUP_URL=robot://system:${DNPM_AUTHUP_SECRET}@http://dnpm-authup:3000 + volumes: + - /etc/bridgehead/dnpm/config:/dnpm_config + - /var/cache/bridgehead/dnpm/backend-data:/dnpm_data + depends_on: + dnpm-authup: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.services.dnpm-backend.loadbalancer.server.port=9000" + # expose everything + - "traefik.http.routers.dnpm-backend.rule=PathPrefix(`/api`)" + - "traefik.http.routers.dnpm-backend.tls=true" + - "traefik.http.routers.dnpm-backend.service=dnpm-backend" + # except ETL + - "traefik.http.routers.dnpm-backend-etl.rule=PathRegexp(`^/api(/.*)?etl(/.*)?$`)" + - "traefik.http.routers.dnpm-backend-etl.tls=true" + - "traefik.http.routers.dnpm-backend-etl.service=dnpm-backend" + # this needs an ETL processor with support for basic auth + - "traefik.http.routers.dnpm-backend-etl.middlewares=auth" + # except peer-to-peer + - "traefik.http.routers.dnpm-backend-peer.rule=PathRegexp(`^/api(/.*)?/peer2peer(/.*)?$`)" + - "traefik.http.routers.dnpm-backend-peer.tls=true" + - "traefik.http.routers.dnpm-backend-peer.service=dnpm-backend" + - "traefik.http.routers.dnpm-backend-peer.middlewares=dnpm-backend-peer" + # this effectively denies all requests + # this is okay, because requests from peers don't go through Traefik + - "traefik.http.middlewares.dnpm-backend-peer.ipWhiteList.sourceRange=0.0.0.0/32" + + landing: + labels: + - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)" diff --git a/ccp/modules/dnpm-node-setup.sh b/ccp/modules/dnpm-node-setup.sh index bf8fd26..f3681b5 100644 --- a/ccp/modules/dnpm-node-setup.sh +++ b/ccp/modules/dnpm-node-setup.sh @@ -1,28 +1,16 @@ #!/bin/bash if [ -n "${ENABLE_DNPM_NODE}" ]; then - log INFO "DNPM setup detected (BwHC Node) -- will start BwHC node." + log INFO "DNPM setup detected -- will start DNPM:DIP node." OVERRIDE+=" -f ./$PROJECT/modules/dnpm-node-compose.yml" # Set variables required for BwHC Node. ZPM_SITE is assumed to be set in /etc/bridgehead/.conf - DNPM_APPLICATION_SECRET="$(echo \"This is a salt string to generate one consistent password for DNPM. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ -z "${ZPM_SITE+x}" ]; then log ERROR "Mandatory variable ZPM_SITE not defined!" exit 1 fi - if [ -z "${DNPM_DATA_DIR+x}" ]; then - log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" - exit 1 - fi - DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + mkdir -p /var/cache/bridgehead/dnpm/ || fail_and_report 1 "Failed to create '/var/cache/bridgehead/dnpm/'. Please run sudo './bridgehead install $PROJECT' again to fix the permissions." + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1} + DNPM_MYSQL_ROOT_PASSWORD="$(generate_simple_password 'dnpm mysql')" + DNPM_AUTHUP_SECRET="$(generate_simple_password 'dnpm authup')" fi diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index dc76331..159276a 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -16,7 +16,7 @@ services: - --entrypoints.web.http.redirections.entrypoint.scheme=https labels: - "traefik.enable=true" - - "traefik.http.routers.dashboard.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard/`)" + - "traefik.http.routers.dashboard.rule=PathPrefix(`/dashboard/`)" - "traefik.http.routers.dashboard.entrypoints=websecure" - "traefik.http.routers.dashboard.service=api@internal" - "traefik.http.routers.dashboard.tls=true" diff --git a/minimal/modules/dnpm-central-targets.json b/minimal/modules/dnpm-central-targets.json new file mode 100644 index 0000000..5469da0 --- /dev/null +++ b/minimal/modules/dnpm-central-targets.json @@ -0,0 +1,142 @@ +{ + "sites": [ + { + "id": "UKFR", + "name": "Freiburg", + "virtualhost": "ukfr.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKHD", + "name": "Heidelberg", + "virtualhost": "ukhd.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKT", + "name": "Tübingen", + "virtualhost": "ukt.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKU", + "name": "Ulm", + "virtualhost": "uku.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UM", + "name": "Mainz", + "virtualhost": "um.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKMR", + "name": "Marburg", + "virtualhost": "ukmr.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKE", + "name": "Hamburg", + "virtualhost": "uke.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKA", + "name": "Aachen", + "virtualhost": "uka.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "Charite", + "name": "Berlin", + "virtualhost": "charite.dnpm.de", + "beamconnect": "dnpm-connect.berlin-test.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "MRI", + "name": "Muenchen-tum", + "virtualhost": "mri.dnpm.de", + "beamconnect": "dnpm-connect.muenchen-tum.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "KUM", + "name": "Muenchen-lmu", + "virtualhost": "kum.dnpm.de", + "beamconnect": "dnpm-connect.muenchen-lmu.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "MHH", + "name": "Hannover", + "virtualhost": "mhh.dnpm.de", + "beamconnect": "dnpm-connect.hannover.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKDD", + "name": "dresden-dnpm", + "virtualhost": "ukdd.dnpm.de", + "beamconnect": "dnpm-connect.dresden-dnpm.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKB", + "name": "Bonn", + "virtualhost": "ukb.dnpm.de", + "beamconnect": "dnpm-connect.bonn-dnpm.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKD", + "name": "Duesseldorf", + "virtualhost": "ukd.dnpm.de", + "beamconnect": "dnpm-connect.duesseldorf-dnpm.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKK", + "name": "Koeln", + "virtualhost": "ukk.dnpm.de", + "beamconnect": "dnpm-connect.dnpm-bridge.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UME", + "name": "Essen", + "virtualhost": "ume.dnpm.de", + "beamconnect": "dnpm-connect.essen.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKM", + "name": "Muenster", + "virtualhost": "ukm.dnpm.de", + "beamconnect": "dnpm-connect.muenster-dnpm.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKF", + "name": "Frankfurt", + "virtualhost": "ukf.dnpm.de", + "beamconnect": "dnpm-connect.frankfurt.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UMG", + "name": "Goettingen", + "virtualhost": "umg.dnpm.de", + "beamconnect": "dnpm-connect.goettingen.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKW", + "name": "Würzburg", + "virtualhost": "ukw.dnpm.de", + "beamconnect": "dnpm-connect.wuerzburg-dnpm.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "UKSH", + "name": "Schleswig-Holstein", + "virtualhost": "uksh.dnpm.de", + "beamconnect": "dnpm-connect.uksh-dnpm.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "TKT", + "name": "Test", + "virtualhost": "tkt.dnpm.de", + "beamconnect": "dnpm-connect.tobias-develop.broker.ccp-it.dktk.dkfz.de" + } + ] +} diff --git a/minimal/modules/dnpm-compose.yml b/minimal/modules/dnpm-compose.yml index 646a457..1c9a36a 100644 --- a/minimal/modules/dnpm-compose.yml +++ b/minimal/modules/dnpm-compose.yml @@ -29,7 +29,7 @@ services: PROXY_APIKEY: ${DNPM_BEAM_SECRET_SHORT} APP_ID: dnpm-connect.${DNPM_PROXY_ID} DISCOVERY_URL: "./conf/central_targets.json" - LOCAL_TARGETS_FILE: "./conf/connect_targets.json" + LOCAL_TARGETS_FILE: "/conf/connect_targets.json" HTTP_PROXY: http://forward_proxy:3128 HTTPS_PROXY: http://forward_proxy:3128 NO_PROXY: dnpm-beam-proxy,dnpm-backend, host.docker.internal${DNPM_ADDITIONAL_NO_PROXY} @@ -41,7 +41,7 @@ services: volumes: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - /etc/bridgehead/dnpm/local_targets.json:/conf/connect_targets.json:ro - - /etc/bridgehead/dnpm/central_targets.json:/conf/central_targets.json:ro + - /srv/docker/bridgehead/minimal/modules/dnpm-central-targets.json:/conf/central_targets.json:ro labels: - "traefik.enable=true" - "traefik.http.routers.dnpm-connect.rule=PathPrefix(`/dnpm-connect`)" diff --git a/minimal/modules/dnpm-node-compose.yml b/minimal/modules/dnpm-node-compose.yml index ee84d89..8c2b146 100644 --- a/minimal/modules/dnpm-node-compose.yml +++ b/minimal/modules/dnpm-node-compose.yml @@ -1,34 +1,99 @@ version: "3.7" services: - dnpm-backend: - image: ghcr.io/kohlbacherlab/bwhc-backend:1.0-snapshot-broker-connector - container_name: bridgehead-dnpm-backend + dnpm-mysql: + image: mysql:9 + healthcheck: + test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ] + interval: 3s + timeout: 5s + retries: 5 environment: - - ZPM_SITE=${ZPM_SITE} - - N_RANDOM_FILES=${DNPM_SYNTH_NUM} + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD: ${DNPM_MYSQL_ROOT_PASSWORD} volumes: - - /etc/bridgehead/dnpm:/bwhc_config:ro - - ${DNPM_DATA_DIR}:/bwhc_data - labels: - - "traefik.enable=true" - - "traefik.http.routers.bwhc-backend.rule=PathPrefix(`/bwhc`)" - - "traefik.http.services.bwhc-backend.loadbalancer.server.port=9000" - - "traefik.http.routers.bwhc-backend.tls=true" + - /var/cache/bridgehead/dnpm/mysql:/var/lib/mysql - dnpm-frontend: - image: ghcr.io/kohlbacherlab/bwhc-frontend:2209 - container_name: bridgehead-dnpm-frontend - links: - - dnpm-backend + dnpm-authup: + image: authup/authup:latest + container_name: bridgehead-dnpm-authup + volumes: + - /var/cache/bridgehead/dnpm/authup:/usr/src/app/writable + depends_on: + dnpm-mysql: + condition: service_healthy + command: server/core start environment: - - NUXT_HOST=0.0.0.0 - - NUXT_PORT=8080 - - BACKEND_PROTOCOL=https - - BACKEND_HOSTNAME=$HOST - - BACKEND_PORT=443 + - PUBLIC_URL=https://${HOST}/auth/ + - AUTHORIZE_REDIRECT_URL=https://${HOST} + - ROBOT_ADMIN_ENABLED=true + - ROBOT_ADMIN_SECRET=${DNPM_AUTHUP_SECRET} + - ROBOT_ADMIN_SECRET_RESET=true + - DB_TYPE=mysql + - DB_HOST=dnpm-mysql + - DB_USERNAME=root + - DB_PASSWORD=${DNPM_MYSQL_ROOT_PASSWORD} + - DB_DATABASE=auth labels: - "traefik.enable=true" - - "traefik.http.routers.bwhc-frontend.rule=PathPrefix(`/`)" - - "traefik.http.services.bwhc-frontend.loadbalancer.server.port=8080" - - "traefik.http.routers.bwhc-frontend.tls=true" + - "traefik.http.middlewares.authup-strip.stripprefix.prefixes=/auth/" + - "traefik.http.routers.dnpm-auth.middlewares=authup-strip" + - "traefik.http.routers.dnpm-auth.rule=PathPrefix(`/auth`)" + - "traefik.http.services.dnpm-auth.loadbalancer.server.port=3000" + - "traefik.http.routers.dnpm-auth.tls=true" + + dnpm-portal: + image: ghcr.io/dnpm-dip/portal:latest + container_name: bridgehead-dnpm-portal + environment: + - NUXT_API_URL=http://dnpm-backend:9000/ + - NUXT_PUBLIC_API_URL=https://${HOST}/api/ + - NUXT_AUTHUP_URL=http://dnpm-authup:3000/ + - NUXT_PUBLIC_AUTHUP_URL=https://${HOST}/auth/ + labels: + - "traefik.enable=true" + - "traefik.http.routers.dnpm-frontend.rule=PathPrefix(`/`)" + - "traefik.http.services.dnpm-frontend.loadbalancer.server.port=3000" + - "traefik.http.routers.dnpm-frontend.tls=true" + + dnpm-backend: + container_name: bridgehead-dnpm-backend + image: ghcr.io/dnpm-dip/backend:latest + environment: + - LOCAL_SITE=${ZPM_SITE}:${SITE_NAME} # Format: {Site-ID}:{Site-name}, e.g. UKT:Tübingen + - RD_RANDOM_DATA=${DNPM_SYNTH_NUM:--1} + - MTB_RANDOM_DATA=${DNPM_SYNTH_NUM:--1} + - HATEOAS_HOST=https://${HOST} + - CONNECTOR_TYPE=broker + - AUTHUP_URL=robot://system:${DNPM_AUTHUP_SECRET}@http://dnpm-authup:3000 + volumes: + - /etc/bridgehead/dnpm/config:/dnpm_config + - /var/cache/bridgehead/dnpm/backend-data:/dnpm_data + depends_on: + dnpm-authup: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.services.dnpm-backend.loadbalancer.server.port=9000" + # expose everything + - "traefik.http.routers.dnpm-backend.rule=PathPrefix(`/api`)" + - "traefik.http.routers.dnpm-backend.tls=true" + - "traefik.http.routers.dnpm-backend.service=dnpm-backend" + # except ETL + - "traefik.http.routers.dnpm-backend-etl.rule=PathRegexp(`^/api(/.*)?etl(/.*)?$`)" + - "traefik.http.routers.dnpm-backend-etl.tls=true" + - "traefik.http.routers.dnpm-backend-etl.service=dnpm-backend" + # this needs an ETL processor with support for basic auth + - "traefik.http.routers.dnpm-backend-etl.middlewares=auth" + # except peer-to-peer + - "traefik.http.routers.dnpm-backend-peer.rule=PathRegexp(`^/api(/.*)?/peer2peer(/.*)?$`)" + - "traefik.http.routers.dnpm-backend-peer.tls=true" + - "traefik.http.routers.dnpm-backend-peer.service=dnpm-backend" + - "traefik.http.routers.dnpm-backend-peer.middlewares=dnpm-backend-peer" + # this effectively denies all requests + # this is okay, because requests from peers don't go through Traefik + - "traefik.http.middlewares.dnpm-backend-peer.ipWhiteList.sourceRange=0.0.0.0/32" + + landing: + labels: + - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)" diff --git a/minimal/modules/dnpm-node-setup.sh b/minimal/modules/dnpm-node-setup.sh index bf8fd26..f3681b5 100644 --- a/minimal/modules/dnpm-node-setup.sh +++ b/minimal/modules/dnpm-node-setup.sh @@ -1,28 +1,16 @@ #!/bin/bash if [ -n "${ENABLE_DNPM_NODE}" ]; then - log INFO "DNPM setup detected (BwHC Node) -- will start BwHC node." + log INFO "DNPM setup detected -- will start DNPM:DIP node." OVERRIDE+=" -f ./$PROJECT/modules/dnpm-node-compose.yml" # Set variables required for BwHC Node. ZPM_SITE is assumed to be set in /etc/bridgehead/.conf - DNPM_APPLICATION_SECRET="$(echo \"This is a salt string to generate one consistent password for DNPM. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ -z "${ZPM_SITE+x}" ]; then log ERROR "Mandatory variable ZPM_SITE not defined!" exit 1 fi - if [ -z "${DNPM_DATA_DIR+x}" ]; then - log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" - exit 1 - fi - DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + mkdir -p /var/cache/bridgehead/dnpm/ || fail_and_report 1 "Failed to create '/var/cache/bridgehead/dnpm/'. Please run sudo './bridgehead install $PROJECT' again to fix the permissions." + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1} + DNPM_MYSQL_ROOT_PASSWORD="$(generate_simple_password 'dnpm mysql')" + DNPM_AUTHUP_SECRET="$(generate_simple_password 'dnpm authup')" fi From bca63e82a9151272987521faca51284629d8fcf5 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 6 Feb 2025 15:43:37 +0100 Subject: [PATCH 14/19] fix: don't use return in transfairSetup For some reason the return not only exits transfairSetup, but also the bridgehead script --- modules/transfair-setup.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh index 720bb25..4b8091e 100755 --- a/modules/transfair-setup.sh +++ b/modules/transfair-setup.sh @@ -3,22 +3,20 @@ function transfairSetup() { if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then echo "Starting transfair." - else - return + OVERRIDE+=" -f ./modules/transfair-compose.yml" + if [ -n "$FHIR_INPUT_URL" ]; then + log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" + else + log INFO "TransFAIR input fhir store not set writing to internal blaze" + FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" + OVERRIDE+=" --profile transfair-input-blaze" + fi + if [ -n "$FHIR_REQUEST_URL" ]; then + log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" + else + log INFO "TransFAIR request fhir store not set writing to internal blaze" + FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" + OVERRIDE+=" --profile transfair-request-blaze" + fi fi - OVERRIDE+=" -f ./modules/transfair-compose.yml" - if [ -n "$FHIR_INPUT_URL" ]; then - log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" - else - log INFO "TransFAIR input fhir store not set writing to internal blaze" - FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" - OVERRIDE+=" --profile transfair-input-blaze" - fi - if [ -n "$FHIR_REQUEST_URL" ]; then - log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" - else - log INFO "TransFAIR request fhir store not set writing to internal blaze" - FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" - OVERRIDE+=" --profile transfair-request-blaze" - fi } From 8fe73a81238672693b30996c2be3eb78e233177e Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 7 Feb 2025 09:17:07 +0100 Subject: [PATCH 15/19] fix: support mode without ttp --- modules/transfair-compose.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 08d1a09..ec3f78a 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -4,17 +4,18 @@ services: image: docker.verbis.dkfz.de/cache/samply/transfair:latest container_name: bridgehead-transfair environment: - INSTITUTE_TTP_URL: "${INSTITUTE_TTP_URL}" - INSTITUTE_TTP_API_KEY: "${INSTITUTE_TTP_API_KEY}" - PROJECT_ID_SYSTEM: "${PROJECT_ID_SYSTEM}" - FHIR_REQUEST_URL: "${FHIR_REQUEST_URL}" - FHIR_INPUT_URL: "${FHIR_INPUT_URL}" - FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL:-http://blaze:8080}" - FHIR_REQUEST_CREDENTIALS: "${FHIR_REQUEST_CREDENTIALS}" - FHIR_INPUT_CREDENTIALS: "${FHIR_INPUT_CREDENTIALS}" - FHIR_OUTPUT_CREDENTIALS: "${FHIR_OUTPUT_CREDENTIALS}" - EXCHANGE_ID_SYSTEM: "${EXCHANGE_ID_SYSTEM:-SESSION_ID}" - DATABASE_URL: "sqlite://transfair/data_requests.sql?mode=rwc" + # NOTE: Those 3 variables need only to be passed if their set, otherwise transfair will complain about empty url values + - INSTITUTE_TTP_URL + - INSTITUTE_TTP_API_KEY + - PROJECT_ID_SYSTEM + - FHIR_REQUEST_URL=${FHIR_REQUEST_URL} + - FHIR_INPUT_URL=${FHIR_INPUT_URL} + - FHIR_OUTPUT_URL=${FHIR_OUTPUT_URL:-http://blaze:8080} + - FHIR_REQUEST_CREDENTIALS=${FHIR_REQUEST_CREDENTIALS} + - FHIR_INPUT_CREDENTIALS=${FHIR_INPUT_CREDENTIALS} + - FHIR_OUTPUT_CREDENTIALS=${FHIR_OUTPUT_CREDENTIALS} + - EXCHANGE_ID_SYSTEM=${EXCHANGE_ID_SYSTEM:-SESSION_ID} + - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc volumes: - /var/cache/bridgehead/${PROJECT}/transfair:/transfair @@ -46,4 +47,4 @@ services: volumes: transfair-input-blaze-data: - transfair-request-blaze-data: \ No newline at end of file + transfair-request-blaze-data: From 8384143387a48e84d8c5e2e8dcb171dbed9416a8 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 7 Feb 2025 09:17:17 +0100 Subject: [PATCH 16/19] fix: make transfair reach the internal blaze stores --- modules/transfair-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh index 4b8091e..58f7331 100755 --- a/modules/transfair-setup.sh +++ b/modules/transfair-setup.sh @@ -8,14 +8,14 @@ function transfairSetup() { log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" else log INFO "TransFAIR input fhir store not set writing to internal blaze" - FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" + FHIR_INPUT_URL="http://transfair-input-blaze:8080" OVERRIDE+=" --profile transfair-input-blaze" fi if [ -n "$FHIR_REQUEST_URL" ]; then log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" else log INFO "TransFAIR request fhir store not set writing to internal blaze" - FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" + FHIR_REQUEST_URL="http://transfair-requests-blaze:8080" OVERRIDE+=" --profile transfair-request-blaze" fi fi From c568a566515c65f4eb40a34b0e445cdf401a3528 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 7 Feb 2025 10:30:43 +0100 Subject: [PATCH 17/19] refactor: set transfair log to info --- modules/transfair-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index ec3f78a..9af09a6 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -16,6 +16,7 @@ services: - FHIR_OUTPUT_CREDENTIALS=${FHIR_OUTPUT_CREDENTIALS} - EXCHANGE_ID_SYSTEM=${EXCHANGE_ID_SYSTEM:-SESSION_ID} - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc + - RUST_LOG=${RUST_LOG:-info} volumes: - /var/cache/bridgehead/${PROJECT}/transfair:/transfair From 8000356b579d56865e470d648d9506cde560fc1a Mon Sep 17 00:00:00 2001 From: Torben Brenner <76154651+torbrenner@users.noreply.github.com> Date: Fri, 7 Feb 2025 11:21:43 +0100 Subject: [PATCH 18/19] docs: explicitly clone main branch (#269) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86bd1d9..0e8131a 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ Pay special attention to: Clone the bridgehead repository: ```shell sudo mkdir -p /srv/docker/ -sudo git clone https://github.com/samply/bridgehead.git /srv/docker/bridgehead +sudo git clone -b main https://github.com/samply/bridgehead.git /srv/docker/bridgehead ``` Then, run the installation script: From 8334fac84d59838f883e9430e3a1dfcdd2917afd Mon Sep 17 00:00:00 2001 From: Pierre Delpy <75260699+PierreDelpy@users.noreply.github.com> Date: Thu, 20 Feb 2025 13:48:10 +0100 Subject: [PATCH 19/19] fix: use correct obds2fhir-rest image --------- Co-authored-by: Pierre Delpy --- ccp/modules/obds2fhir-rest-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/obds2fhir-rest-compose.yml b/ccp/modules/obds2fhir-rest-compose.yml index 833580d..ec1737c 100644 --- a/ccp/modules/obds2fhir-rest-compose.yml +++ b/ccp/modules/obds2fhir-rest-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: obds2fhir-rest: container_name: bridgehead-obds2fhir-rest - image: docker.verbis.dkfz.de/ccp/obds2fhir-rest:main + image: docker.verbis.dkfz.de/samply/obds2fhir-rest:main environment: IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}