From 47364f999eee89275fb2fef7570e7f1da66d4b33 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 8 Jan 2025 15:27:00 +0000 Subject: [PATCH 1/8] wip: routine connector --- dhki/vars | 10 +++++++++- modules/transfair-compose.yml | 19 +++++++++++++++++++ modules/transfair-output-blaze-compose.yml | 17 +++++++++++++++++ modules/transfair-setup.sh | 13 +++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 modules/transfair-compose.yml create mode 100644 modules/transfair-output-blaze-compose.yml create mode 100755 modules/transfair-setup.sh diff --git a/dhki/vars b/dhki/vars index b728925..f7f7ecd 100644 --- a/dhki/vars +++ b/dhki/vars @@ -17,4 +17,12 @@ do done idManagementSetup -obds2fhirRestSetup \ No newline at end of file +obds2fhirRestSetup + +for module in modules/*.sh +do + log DEBUG "sourcing $module" + source $module +done + +routineConnectorSetup \ No newline at end of file diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml new file mode 100644 index 0000000..863ae4a --- /dev/null +++ b/modules/transfair-compose.yml @@ -0,0 +1,19 @@ + +services: + transfair: + image: samply/routine-connector:latest # TODO: Harbor image and new name ofc + container_name: bridgehead-transfair + environment: + INSTITUTE_TTP_URL: "${INSTITUTE_TTP_URL}" + INSTITUTE_TTP_API_KEY: "${INSTITUTE_TTP_API_KEY}" + PROJECT_ID_SYSTEM: "${PROJECT_ID_SYSTEM}" + FHIR_REQUEST_URL: "${FHIR_REQUEST_URL}" + FHIR_INPUT_URL: "${FHIR_INPUT_URL}" + FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL}" + FHIR_REQUEST_CREDENTIALS: "${FHIR_REQUEST_CREDENTIALS}" + FHIR_INPUT_CREDENTIALS: "${FHIR_INPUT_CREDENTIALS}" + FHIR_OUTPUT_CREDENTIALS: "${FHIR_OUTPUT_CREDENTIALS}" + EXCHANGE_ID_SYSTEM: "SESSION_ID" + DATABASE_URL: "sqlite://transfair/data_requests.sql?mode=rwc" + volumes: + - /var/cache/bridgehead/${PROJECT}/transfair:/transfair diff --git a/modules/transfair-output-blaze-compose.yml b/modules/transfair-output-blaze-compose.yml new file mode 100644 index 0000000..9f8583b --- /dev/null +++ b/modules/transfair-output-blaze-compose.yml @@ -0,0 +1,17 @@ + +services: + transfair-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-blaze + environment: + BASE_URL: "http://bridgehead-ccp-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} + CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-blaze-data:/app/data" + +volumes: + transfair-balze-data: \ No newline at end of file diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh new file mode 100755 index 0000000..6dff9c2 --- /dev/null +++ b/modules/transfair-setup.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e + +function transfairSetup() { + assertVarsNotEmpty INSTITUTE_TTP_URL INSTITUTE_TTP_API_KEY PROJECT_ID_SYSTEM FHIR_REQUEST_URL FHIR_INPUT_URL + OVERRIDE+=" -f ./modules/transfair-compose.yml" + if [ -n "$FHIR_OUTPUT_URL" ]; then + log INFO "TransFAIR output fhir store set to external $FHIR_OUTPUT_URL" + else + log INFO "TransFAIR output fhir store not set writing to internal blaze" + FHIR_OUTPUT_URL="http://transfair-blaze:8080" + OVERRIDE+=" -f ./modules/transfair-compose.yml" + fi +} From 2058a7a5c922515f3b6a38a1d23c3b2be3062c80 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 23 Jan 2025 12:06:43 +0000 Subject: [PATCH 2/8] update image url --- modules/transfair-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 863ae4a..2fa980f 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -1,7 +1,7 @@ services: transfair: - image: samply/routine-connector:latest # TODO: Harbor image and new name ofc + image: docker.verbis.dkfz.de/cache/samply/transfair:latest container_name: bridgehead-transfair environment: INSTITUTE_TTP_URL: "${INSTITUTE_TTP_URL}" From 139fcecabe2825ad3c253b4565baae92f4d01459 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 23 Jan 2025 12:42:59 +0000 Subject: [PATCH 3/8] redo transfair setup --- dhki/vars | 2 +- modules/transfair-compose.yml | 36 ++++++++++++++++++++-- modules/transfair-output-blaze-compose.yml | 17 ---------- modules/transfair-setup.sh | 23 ++++++++++---- 4 files changed, 52 insertions(+), 26 deletions(-) delete mode 100644 modules/transfair-output-blaze-compose.yml diff --git a/dhki/vars b/dhki/vars index f7f7ecd..d043dd2 100644 --- a/dhki/vars +++ b/dhki/vars @@ -25,4 +25,4 @@ do source $module done -routineConnectorSetup \ No newline at end of file +transfairSetup \ No newline at end of file diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 2fa980f..3927fd9 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -9,11 +9,43 @@ services: PROJECT_ID_SYSTEM: "${PROJECT_ID_SYSTEM}" FHIR_REQUEST_URL: "${FHIR_REQUEST_URL}" FHIR_INPUT_URL: "${FHIR_INPUT_URL}" - FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL}" + FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL:-http://blaze:8080}" FHIR_REQUEST_CREDENTIALS: "${FHIR_REQUEST_CREDENTIALS}" FHIR_INPUT_CREDENTIALS: "${FHIR_INPUT_CREDENTIALS}" FHIR_OUTPUT_CREDENTIALS: "${FHIR_OUTPUT_CREDENTIALS}" - EXCHANGE_ID_SYSTEM: "SESSION_ID" + EXCHANGE_ID_SYSTEM: "${EXCHANGE_ID_SYSTEM:-SESSION_ID}" DATABASE_URL: "sqlite://transfair/data_requests.sql?mode=rwc" volumes: - /var/cache/bridgehead/${PROJECT}/transfair:/transfair + + transfair-input-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-input-blaze + environment: + BASE_URL: "http://bridgehead-transfair-input-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} + CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-input-blaze-data:/app/data" + profiles: ["transfair-input-blaze"] + + transfair-request-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-requests-blaze + environment: + BASE_URL: "http://bridgehead-transfair-requests-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} + CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-request-blaze-data:/app/data" + profiles: ["transfair-request-blaze"] + +volumes: + transfair-input-blaze-data: + transfair-request-blaze-data: \ No newline at end of file diff --git a/modules/transfair-output-blaze-compose.yml b/modules/transfair-output-blaze-compose.yml deleted file mode 100644 index 9f8583b..0000000 --- a/modules/transfair-output-blaze-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ - -services: - transfair-blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 - container_name: bridgehead-transfair-blaze - environment: - BASE_URL: "http://bridgehead-ccp-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" - DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} - DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} - CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} - ENFORCE_REFERENTIAL_INTEGRITY: "false" - volumes: - - "transfair-blaze-data:/app/data" - -volumes: - transfair-balze-data: \ No newline at end of file diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh index 6dff9c2..720bb25 100755 --- a/modules/transfair-setup.sh +++ b/modules/transfair-setup.sh @@ -1,13 +1,24 @@ #!/bin/bash -e function transfairSetup() { - assertVarsNotEmpty INSTITUTE_TTP_URL INSTITUTE_TTP_API_KEY PROJECT_ID_SYSTEM FHIR_REQUEST_URL FHIR_INPUT_URL + if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then + echo "Starting transfair." + else + return + fi OVERRIDE+=" -f ./modules/transfair-compose.yml" - if [ -n "$FHIR_OUTPUT_URL" ]; then - log INFO "TransFAIR output fhir store set to external $FHIR_OUTPUT_URL" + if [ -n "$FHIR_INPUT_URL" ]; then + log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" else - log INFO "TransFAIR output fhir store not set writing to internal blaze" - FHIR_OUTPUT_URL="http://transfair-blaze:8080" - OVERRIDE+=" -f ./modules/transfair-compose.yml" + log INFO "TransFAIR input fhir store not set writing to internal blaze" + FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" + OVERRIDE+=" --profile transfair-input-blaze" + fi + if [ -n "$FHIR_REQUEST_URL" ]; then + log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" + else + log INFO "TransFAIR request fhir store not set writing to internal blaze" + FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" + OVERRIDE+=" --profile transfair-request-blaze" fi } From fa0d9fb8b4a47e11f36b37a6999685f20a44ba89 Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 24 Jan 2025 09:23:08 +0000 Subject: [PATCH 4/8] restrict additional blaze memory usage --- modules/transfair-compose.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 3927fd9..08d1a09 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -23,10 +23,9 @@ services: container_name: bridgehead-transfair-input-blaze environment: BASE_URL: "http://bridgehead-transfair-input-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" - DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} - DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} - CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + JAVA_TOOL_OPTIONS: "-Xmx1024m" + DB_BLOCK_CACHE_SIZE: 1024 + CQL_EXPR_CACHE_SIZE: 8 ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "transfair-input-blaze-data:/app/data" @@ -37,10 +36,9 @@ services: container_name: bridgehead-transfair-requests-blaze environment: BASE_URL: "http://bridgehead-transfair-requests-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" - DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} - DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} - CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} + JAVA_TOOL_OPTIONS: "-Xmx1024m" + DB_BLOCK_CACHE_SIZE: 1024 + CQL_EXPR_CACHE_SIZE: 8 ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "transfair-request-blaze-data:/app/data" From bca63e82a9151272987521faca51284629d8fcf5 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 6 Feb 2025 15:43:37 +0100 Subject: [PATCH 5/8] fix: don't use return in transfairSetup For some reason the return not only exits transfairSetup, but also the bridgehead script --- modules/transfair-setup.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh index 720bb25..4b8091e 100755 --- a/modules/transfair-setup.sh +++ b/modules/transfair-setup.sh @@ -3,22 +3,20 @@ function transfairSetup() { if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then echo "Starting transfair." - else - return + OVERRIDE+=" -f ./modules/transfair-compose.yml" + if [ -n "$FHIR_INPUT_URL" ]; then + log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" + else + log INFO "TransFAIR input fhir store not set writing to internal blaze" + FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" + OVERRIDE+=" --profile transfair-input-blaze" + fi + if [ -n "$FHIR_REQUEST_URL" ]; then + log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" + else + log INFO "TransFAIR request fhir store not set writing to internal blaze" + FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" + OVERRIDE+=" --profile transfair-request-blaze" + fi fi - OVERRIDE+=" -f ./modules/transfair-compose.yml" - if [ -n "$FHIR_INPUT_URL" ]; then - log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" - else - log INFO "TransFAIR input fhir store not set writing to internal blaze" - FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" - OVERRIDE+=" --profile transfair-input-blaze" - fi - if [ -n "$FHIR_REQUEST_URL" ]; then - log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" - else - log INFO "TransFAIR request fhir store not set writing to internal blaze" - FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" - OVERRIDE+=" --profile transfair-request-blaze" - fi } From 8fe73a81238672693b30996c2be3eb78e233177e Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 7 Feb 2025 09:17:07 +0100 Subject: [PATCH 6/8] fix: support mode without ttp --- modules/transfair-compose.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index 08d1a09..ec3f78a 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -4,17 +4,18 @@ services: image: docker.verbis.dkfz.de/cache/samply/transfair:latest container_name: bridgehead-transfair environment: - INSTITUTE_TTP_URL: "${INSTITUTE_TTP_URL}" - INSTITUTE_TTP_API_KEY: "${INSTITUTE_TTP_API_KEY}" - PROJECT_ID_SYSTEM: "${PROJECT_ID_SYSTEM}" - FHIR_REQUEST_URL: "${FHIR_REQUEST_URL}" - FHIR_INPUT_URL: "${FHIR_INPUT_URL}" - FHIR_OUTPUT_URL: "${FHIR_OUTPUT_URL:-http://blaze:8080}" - FHIR_REQUEST_CREDENTIALS: "${FHIR_REQUEST_CREDENTIALS}" - FHIR_INPUT_CREDENTIALS: "${FHIR_INPUT_CREDENTIALS}" - FHIR_OUTPUT_CREDENTIALS: "${FHIR_OUTPUT_CREDENTIALS}" - EXCHANGE_ID_SYSTEM: "${EXCHANGE_ID_SYSTEM:-SESSION_ID}" - DATABASE_URL: "sqlite://transfair/data_requests.sql?mode=rwc" + # NOTE: Those 3 variables need only to be passed if their set, otherwise transfair will complain about empty url values + - INSTITUTE_TTP_URL + - INSTITUTE_TTP_API_KEY + - PROJECT_ID_SYSTEM + - FHIR_REQUEST_URL=${FHIR_REQUEST_URL} + - FHIR_INPUT_URL=${FHIR_INPUT_URL} + - FHIR_OUTPUT_URL=${FHIR_OUTPUT_URL:-http://blaze:8080} + - FHIR_REQUEST_CREDENTIALS=${FHIR_REQUEST_CREDENTIALS} + - FHIR_INPUT_CREDENTIALS=${FHIR_INPUT_CREDENTIALS} + - FHIR_OUTPUT_CREDENTIALS=${FHIR_OUTPUT_CREDENTIALS} + - EXCHANGE_ID_SYSTEM=${EXCHANGE_ID_SYSTEM:-SESSION_ID} + - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc volumes: - /var/cache/bridgehead/${PROJECT}/transfair:/transfair @@ -46,4 +47,4 @@ services: volumes: transfair-input-blaze-data: - transfair-request-blaze-data: \ No newline at end of file + transfair-request-blaze-data: From 8384143387a48e84d8c5e2e8dcb171dbed9416a8 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 7 Feb 2025 09:17:17 +0100 Subject: [PATCH 7/8] fix: make transfair reach the internal blaze stores --- modules/transfair-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh index 4b8091e..58f7331 100755 --- a/modules/transfair-setup.sh +++ b/modules/transfair-setup.sh @@ -8,14 +8,14 @@ function transfairSetup() { log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" else log INFO "TransFAIR input fhir store not set writing to internal blaze" - FHIR_INPUT_URL="http://bridgehead-transfair-input-blaze:8080" + FHIR_INPUT_URL="http://transfair-input-blaze:8080" OVERRIDE+=" --profile transfair-input-blaze" fi if [ -n "$FHIR_REQUEST_URL" ]; then log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" else log INFO "TransFAIR request fhir store not set writing to internal blaze" - FHIR_REQUEST_URL="http://bridgehead-transfair-requests-blaze:8080" + FHIR_REQUEST_URL="http://transfair-requests-blaze:8080" OVERRIDE+=" --profile transfair-request-blaze" fi fi From c568a566515c65f4eb40a34b0e445cdf401a3528 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 7 Feb 2025 10:30:43 +0100 Subject: [PATCH 8/8] refactor: set transfair log to info --- modules/transfair-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml index ec3f78a..9af09a6 100644 --- a/modules/transfair-compose.yml +++ b/modules/transfair-compose.yml @@ -16,6 +16,7 @@ services: - FHIR_OUTPUT_CREDENTIALS=${FHIR_OUTPUT_CREDENTIALS} - EXCHANGE_ID_SYSTEM=${EXCHANGE_ID_SYSTEM:-SESSION_ID} - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc + - RUST_LOG=${RUST_LOG:-info} volumes: - /var/cache/bridgehead/${PROJECT}/transfair:/transfair