From 7d07c0623d9f95ac704c1cd7b9d91638abb90d41 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Tue, 20 Feb 2024 15:15:49 +0100 Subject: [PATCH 001/243] refactor: optimize memory usage of blaze --- bbmri/docker-compose.yml | 4 ++-- bridgehead | 1 + ccp/docker-compose.yml | 3 ++- lib/functions.sh | 10 ++++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 9bc05cc..dcd446c 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -8,8 +8,8 @@ services: container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx4g" - LOG_LEVEL: "debug" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "blaze-data:/app/data" diff --git a/bridgehead b/bridgehead index 31a838e..68e29e1 100755 --- a/bridgehead +++ b/bridgehead @@ -50,6 +50,7 @@ loadVars() { source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import" fi fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" + setBlazeMemoryCap [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index d92ccfb..823bcdf 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -6,7 +6,8 @@ services: container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx4g" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "blaze-data:/app/data" diff --git a/lib/functions.sh b/lib/functions.sh index 6a45d35..66fb701 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -155,6 +155,16 @@ setHostname() { fi } +# blaze memory cap should be approximately a quarter of the system memory +# the memory cap will be applied to both the java heap size and db clock cache +setBlazeMemoryCap() { + if [ -z "$BLAZE_MEMORY_CAP" ]; then + system_memory=$(grep MemTotal /proc/meminfo | grep -Po '\d+'); + system_memory_in_mb=$(("$system_memory"/1024)); + export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); + fi +} + # Takes 1) The Backup Directory Path 2) The name of the Service to be backuped # Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week createEncryptedPostgresBackup(){ From b5ef856f1283c3074a25add9eec116ec7ecd4578 Mon Sep 17 00:00:00 2001 From: Torben Brenner <76154651+torbrenner@users.noreply.github.com> Date: Fri, 23 Feb 2024 08:27:06 +0100 Subject: [PATCH 002/243] refactor: calculate memory using free Co-authored-by: Tobias Kussel --- lib/functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 66fb701..ab904d3 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -159,8 +159,7 @@ setHostname() { # the memory cap will be applied to both the java heap size and db clock cache setBlazeMemoryCap() { if [ -z "$BLAZE_MEMORY_CAP" ]; then - system_memory=$(grep MemTotal /proc/meminfo | grep -Po '\d+'); - system_memory_in_mb=$(("$system_memory"/1024)); + system_memory_in_mb=$(free -m | grep 'Mem:' | awk '{print $2}'); export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); fi } From 2e6edb61794d662c75cc308b3be6df89b54c9583 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 10 Feb 2023 12:41:26 +0100 Subject: [PATCH 003/243] Add Teiler UI and Teiler module --- .gitignore | 2 +- ccp/modules/teiler-compose.yml | 42 +++++ ccp/modules/teiler-setup.sh | 8 + ccp/modules/teiler-ui-compose.yml | 260 ++++++++++++++++++++++++++++++ ccp/modules/teiler-ui-setup.sh | 7 + ccp/vars | 3 +- 6 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 ccp/modules/teiler-compose.yml create mode 100644 ccp/modules/teiler-setup.sh create mode 100644 ccp/modules/teiler-ui-compose.yml create mode 100644 ccp/modules/teiler-ui-setup.sh diff --git a/.gitignore b/.gitignore index a884b44..655ca90 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ##Ignore site configuration .gitmodules site-config/* - +.idea ## Ignore site configuration */docker-compose.override.yml diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml new file mode 100644 index 0000000..b2c2966 --- /dev/null +++ b/ccp/modules/teiler-compose.yml @@ -0,0 +1,42 @@ +version: "3.7" + +services: + teiler: + image: docker.verbis.dkfz.de/dktk/dktk-teiler:latest + container_name: bridgehead-ccp-teiler + environment: + LOG_LEVEL: "INFO" + TEILER_API_KEY: "${TEILER_API_KEY}" + CROSS_ORIGINS: "https://${HOST}/ccp-teiler-root-config" + TEILER_DB_USER: "teiler" + TEILER_DB_PASSWORD: "${TEILER_DB_PASSWORD}" # Set in teiler-setup.sh + TEILER_DB_URL: "jdbc:postgresql://teiler-db:5432/teiler" + CLEAN_TEMP_FILES_CRON_EXPRESSION: "0 0 1 * * *" + TEMP_FILES_LIFETIME_IN_DAYS: "1" + CLEAN_WRITE_FILES_CRON_EXPRESSION: "0 0 2 * * *" + WRITE_FILES_LIFETIME_IN_DAYS: "30" + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_ccp.rule=PathPrefix(`/ccp-teiler`)" + - "traefik.http.services.teiler_ccp.loadbalancer.server.port=8092" + - "traefik.http.routers.teiler_ccp.tls=true" + - "traefik.http.routers.teiler_ccp.middlewares=auth" + volumes: + - "teiler:/app/teiler-files" + + teiler-db: + image: postgres:15.1-alpine + container_name: bridgehead-ccp-teiler-db + environment: + POSTGRES_USER: "teiler" + POSTGRES_PASSWORD: "${TEILER_DB_PASSWORD}" # Set in teiler-setup.sh + POSTGRES_DB: "teiler" + volumes: + - "teiler-db:/var/lib/postgresql/data" + + +volumes: + teiler-db: + name: "teiler-db" + teiler: + name: "teiler" diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh new file mode 100644 index 0000000..d651107 --- /dev/null +++ b/ccp/modules/teiler-setup.sh @@ -0,0 +1,8 @@ +function teilerSetup() { + if [ -n "$ENABLE_TEILER" ];then + log INFO "Teiler setup detected -- will start Teiler service." + OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" + fi + # TODO: Generate password in another way so that not all passwords are the same? + TEILER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +} diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml new file mode 100644 index 0000000..5bdc6b2 --- /dev/null +++ b/ccp/modules/teiler-ui-compose.yml @@ -0,0 +1,260 @@ +version: "3.7" + +############################################ Keycloak +login-db: + image: "postgres:15.1" + container_name: bridgehead-login-db + environment: + POSTGRES_USER: "keycloak" + POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + POSTGRES_DB: "keycloak" + volumes: + - "login-db:/var/lib/postgresql/data" + +login: + #image: "jboss/keycloak:16.1.1" + image: docker.verbis.dkfz.de/dktk/dktk-keycloak:latest + container_name: bridgehead-login + environment: + KEYCLOAK_USER: "admin" + KEYCLOAK_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" + DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + labels: + - "traefik.enable=true" + - "traefik.http.routers.login.rule=PathPrefix(`/login`)" + - "traefik.http.services.login.loadbalancer.server.port=8080" + - "traefik.http.routers.login.tls=true" + depends_on: + - login-db + +############################################ Teiler-UI +teiler-root-config: + image: samply/teiler-root-config:developer + container_name: bridgehead-teiler-root-config + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler-root-config`)" + - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" + - "traefik.http.routers.teiler_root_config_ccp.tls=true" + environment: + TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" + TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" + DEFAULT_LANGUAGE: "de" + +teiler-ui: + image: samply/teiler-ui:developer + container_name: bridgehead-teiler-ui +# ports: +# - 4200:80 + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_ui_ccp.rule=PathPrefix(`/ccp-teiler-ui`)" + - "traefik.http.services.teiler_ui_ccp.loadbalancer.server.port=80" + - "traefik.http.routers.teiler_ui_ccp.tls=true" + environment: + DEFAULT_LANGUAGE: "DE" + TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" + KEYCLOAK_URL: "https://${HOST}/login" + KEYCLOAK_REALM: "teiler-ui" + KEYCLOAK_CLIENT_ID: "teiler-ui" + TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" + TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" + TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" + TEILER_PROJECT: "${PROJECT}" + +teiler-core: + image: samply/teiler-core:developer + container_name: bridgehead-teiler + volumes: + - "/etc/bridgehead/ccp.conf:/app/ccp.conf" + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" + - "traefik.http.services.teiler_core_ccp.loadbalancer.server.port=8085" + - "traefik.http.routers.teiler_core_ccp.tls=true" + environment: + LOG_LEVEL: "INFO" + APPLICATION_PORT: "8085" + DEFAULT_LANGUAGE: "DE" + CONFIG_ENV_VAR_PATH: "/app/ccp.conf" + TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" + TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler-root-config" + TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" + TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" + TEILER_APP1_NAME: "config" +# TEILER_APP2_NAME: "quality-report" + TEILER_APP3_NAME: "keycloak" + TEILER_APP3_TITLE: "Keycloak" + TEILER_APP4_DESCRIPTION_EN: "Authentication site" + TEILER_APP4_DESCRIPTION_DE: "Authentifizierungsseite" + TEILER_APP4_SOURCEURL: "https://${HOST}/login/auth/" + TEILER_APP4_ROLES: "TEILER_ADMIN" + TEILER_APP4_ISACTIVATED: "true" + TEILER_APP4_ICONSOURCEURL: "https://upload.wikimedia.org/wikipedia/commons/2/29/Keycloak_Logo.png" + TEILER_APP4_ORDER: "5" + TEILER_APP4_ISEXTERNALLINK: "true" + TEILER_APP4_ISLOCAL: "true" +# TEILER_APP5_NAME: "pgadmin" +# TEILER_APP5_TITLE: "PgAdmin" +# TEILER_APP5_DESCRIPTION: "Database Management" +# TEILER_APP5_SOURCEURL: "http://localhost:5000" +# TEILER_APP5_ROLES: "TEILER_ADMIN" +# TEILER_APP5_ISACTIVATED: "true" +# TEILER_APP5_ICONSOURCEURL: "https://user-images.githubusercontent.com/24623425/36042969-f87531d4-0d8a-11e8-9dee-e87ab8c6a9e3.png" +# TEILER_APP5_ORDER: "6" +# TEILER_APP5_ISEXTERNALLINK: "true" +# TEILER_APP5_ISLOCAL: "true" + TEILER_APP6_NAME: "ldm" + TEILER_APP6_TITLE_EN: "Local data management" + TEILER_APP6_TITLE_DE: "Lokales Datenmanagement" + TEILER_APP6_DESCRIPTION_EN: "Local Data Management" + TEILER_APP6_DESCRIPTION_DE: "Lokales Datenmanagement" + TEILER_APP6_SOURCEURL: "${CENTRAXX_URL}" + TEILER_APP6_ROLES: "TEILER_PUBLIC" + TEILER_APP6_ISACTIVATED: "${IS_DKTK_SITE}" + TEILER_APP6_ICONCLASS: "bi bi-server" + TEILER_APP6_ORDER: "7" + TEILER_APP6_ISEXTERNALLINK: "true" + TEILER_APP6_ISLOCAL: "true" + TEILER_APP7_NAME: "id-manager" + TEILER_APP7_TITLE: "ID-Manager" + TEILER_APP7_DESCRIPTION: "ID Manager" + TEILER_APP7_SOURCEURL: "https://${HOST}/id-manager/index.html" + TEILER_APP7_ROLES: "TEILER_PUBLIC" + TEILER_APP7_ISACTIVATED: "true" + TEILER_APP7_ICONCLASS: "bi bi-person-bounding-box" + TEILER_APP7_ORDER: "8" + TEILER_APP7_ISEXTERNALLINK: "true" + TEILER_APP7_ISLOCAL: "true" + TEILER_APP8_NAME: "patient-list" + TEILER_APP8_TITLE_EN: "Patient List" + TEILER_APP8_TITLE_DE: "Patientenliste" + TEILER_APP8_DESCRIPTION_EN: "Patient List" + TEILER_APP8_DESCRIPTION_DE: "Patientenliste" + TEILER_APP8_SOURCEURL: "https://${HOST}/patientlist" + TEILER_APP8_ROLES: "TEILER_PUBLIC" + TEILER_APP8_ISACTIVATED: "true" + TEILER_APP8_ICONCLASS: "bi bi-person-rolodex" + TEILER_APP8_ORDER: "9" + TEILER_APP8_ISEXTERNALLINK: "true" + TEILER_APP8_ISLOCAL: "true" + TEILER_APP9_NAME: "project-pseudonymisation" + TEILER_APP9_TITLE_EN: "Project Pseudonymisation" + TEILER_APP9_TITLE_DE: "Projectpseudonymisierung" + TEILER_APP9_DESCRIPTION_EN: "Project Pseudonymisation" + TEILER_APP9_DESCRIPTION_DE: "Projectpseudonymisierung" + TEILER_APP9_SOURCEURL: "https://${HOST}/id-manager/html/projectSelection.html" + TEILER_APP9_ROLES: "TEILER_PUBLIC" + TEILER_APP9_ISACTIVATED: "true" + TEILER_APP9_ICONCLASS: "bi bi-person-lines-fill" + TEILER_APP9_ORDER: "10" + TEILER_APP9_ISEXTERNALLINK: "true" + TEILER_APP9_ISLOCAL: "true" + TEILER_APP10_NAME: "federated-search" + TEILER_APP10_TITLE: "Lens" + TEILER_APP10_DESCRIPTION_EN: "Federated Search" + TEILER_APP10_DESCRIPTION_DE: "Föderierte Suche" + TEILER_APP10_SOURCEURL: "https://demo.lens.samply.de/" + TEILER_APP10_ROLES: "TEILER_PUBLIC" + TEILER_APP10_ISACTIVATED: "true" + TEILER_APP10_ICONCLASS: "bi bi-search" + TEILER_APP10_ORDER: "13" + TEILER_APP10_ISEXTERNALLINK: "true" + TEILER_APP10_ISLOCAL: "false" + TEILER_APP11_NAME: "central-patient-list" + TEILER_APP11_TITLE_EN: "Central Patient List" + TEILER_APP11_TITLE_DE: "Zentrale Patientenliste" + TEILER_APP11_DESCRIPTION_EN: "Central Patient List" + TEILER_APP11_DESCRIPTION_DE: "Zentrale Patientenliste" + TEILER_APP11_SOURCEURL: "https://patientlist.ccp-it.dktk.dkfz.de/" + TEILER_APP11_ROLES: "TEILER_PUBLIC" + TEILER_APP11_ISACTIVATED: "true" + TEILER_APP11_ICONCLASS: "bi bi-person-rolodex" + TEILER_APP11_ORDER: "14" + TEILER_APP11_ISEXTERNALLINK: "true" + TEILER_APP11_ISLOCAL: "false" + TEILER_APP12_NAME: "central id-manager" + TEILER_APP12_TITLE_EN: "Central ID-Manager" + TEILER_APP12_TITLE_DE: "Zentraler ID-Manager" + TEILER_APP12_DESCRIPTION_EN: "Central ID Manager" + TEILER_APP12_DESCRIPTION_DE: "Zentraler ID-Manager" + TEILER_APP12_SOURCEURL: "https://dktk-kne.kgu.de/" + TEILER_APP12_ROLES: "TEILER_PUBLIC" + TEILER_APP12_ISACTIVATED: "true" + TEILER_APP12_ICONCLASS: "bi bi-person-bounding-box" + TEILER_APP12_ORDER: "15" + TEILER_APP12_ISEXTERNALLINK: "true" + TEILER_APP12_ISLOCAL: "false" + # TODO: Icinga to be replaced by Zabbix + TEILER_APP13_NAME: "monitoring" + TEILER_APP13_TITLE: "Icinga" + TEILER_APP13_DESCRIPTION: "Icinga Monitoring" + TEILER_APP13_SOURCEURL: "https://monitor.vmitro.de/icingaweb2/dashboard" + TEILER_APP13_ROLES: "TEILER_ADMIN" + TEILER_APP13_ISACTIVATED: "true" + TEILER_APP13_ICONSOURCEURL: "https://images.ctfassets.net/o7xu9whrs0u9/QmL67mCGdRQ8PBcuKHGnF/858c0aee95762f59d67b25073f9483c2/icinga-logo.png" + TEILER_APP13_ORDER: "16" + TEILER_APP13_ISEXTERNALLINK: "true" + TEILER_APP13_ISLOCAL: "false" +# TEILER_APP14_NAME: "function-tests" +# TEILER_APP15_NAME: "event-log" + TEILER_APP16_NAME: "active-inquiries" + TEILER_APP16_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP17_NAME: "archived-inquiries" + TEILER_APP17_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP18_NAME: "failed-inquiries" + TEILER_APP18_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP19_NAME: "inquiry" + TEILER_APP19_INMENU: "false" +# TEILER_APP20_NAME: "cbioportal" +# TEILER_APP20_TITLE: "cBioportal" +# TEILER_APP20_DESCRIPTION: "Interactive exploration of multidimensional cancer genomics data sets" +# TEILER_APP20_SOURCEURL: "http://localhost:8082" +# TEILER_APP20_ROLES: "TEILER_USER" +# TEILER_APP20_ISACTIVATED: "true" +# TEILER_APP20_ICONSOURCEURL: "https://docs.cbioportal.org/images/cbio-logo.png" +# TEILER_APP20_ORDER: "17" +# TEILER_APP20_ISEXTERNALLINK: "true" +# TEILER_APP20_ISLOCAL: "true" +# TEILER_APP21_NAME: "mtba-bp" +# TEILER_APP21_TITLE: "MTBA-BP" +# TEILER_APP21_DESCRIPTION: "MTBA Camunda Business Process" +# TEILER_APP21_SOURCEURL: "http://localhost:8480" +# TEILER_APP21_ROLES: "TEILER_ADMIN" +# TEILER_APP21_ISACTIVATED: "true" +# TEILER_APP21_ICONSOURCEURL: "https://camunda.com/wp-content/uploads/2020/05/logo-camunda-black.svg" +# TEILER_APP21_ORDER: "18" +# TEILER_APP21_ISEXTERNALLINK: "true" +# TEILER_APP21_ISLOCAL: "true" + TEILER_APP22_NAME: "dialog-quali" + TEILER_APP22_INMENU: "false" + TEILER_APP23_NAME: "dialog-uploads" + TEILER_APP23_INMENU: "false" + TEILER_APP24_NAME: "inquiry-dialog" + TEILER_APP24_INMENU: "false" + TEILER_APP25_NAME: "dialog-tests" + TEILER_APP25_INMENU: "false" +# TEILER_APP26_NAME: "opal" +# TEILER_APP26_TITLE: "Opal" +# TEILER_APP26_DESCRIPTION: "Opal is OBiBa’s core database application for biobanks." +# TEILER_APP26_SOURCEURL: "http://localhost:8880" +# TEILER_APP26_ROLES: "TEILER_USER" +# TEILER_APP26_ISACTIVATED: "true" +# TEILER_APP26_ICONSOURCEURL: "https://www.obiba.org/assets/themes/bootstrap/img/obiba-logo-small.png" +# TEILER_APP26_ORDER: "19" +# TEILER_APP26_ISEXTERNALLINK: "true" +# TEILER_APP26_ISLOCAL: "true" +# TEILER_APP27_NAME: "rstudio" +# TEILER_APP27_TITLE: "R Studio" +# TEILER_APP27_DESCRIPTION: "RStudio is an integrated development environment (IDE) for R and Python." +# TEILER_APP27_SOURCEURL: "http://localhost:8787" +# TEILER_APP27_ROLES: "TEILER_USER" +# TEILER_APP27_ISACTIVATED: "true" +# TEILER_APP27_ICONSOURCEURL: "https://rstudio.com/wp-content/uploads/2018/10/RStudio-Logo-Flat.png" +# TEILER_APP27_ORDER: "20" +# TEILER_APP27_ISEXTERNALLINK: "true" +# TEILER_APP27_ISLOCAL: "true" + +volumes: + login-db: + name: "login-db" diff --git a/ccp/modules/teiler-ui-setup.sh b/ccp/modules/teiler-ui-setup.sh new file mode 100644 index 0000000..c1f3933 --- /dev/null +++ b/ccp/modules/teiler-ui-setup.sh @@ -0,0 +1,7 @@ +function teilerUiSetup() { + if [ -n "$ENABLE_TEILER" ];then + log INFO "Teiler-UI setup detected -- will start Teiler-UI services." + OVERRIDE+=" -f ./$PROJECT/modules/teiler-ui-compose.yml" + fi + KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +} diff --git a/ccp/vars b/ccp/vars index 7cfb7db..b58710d 100644 --- a/ccp/vars +++ b/ccp/vars @@ -16,5 +16,6 @@ do done idManagementSetup +nngmSetup mtbaSetup -adt2fhirRestSetup \ No newline at end of file +adt2fhirRestSetup From 20e2b2a0ed0daa6055a795d25b47accc0a38a86d Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 10 Feb 2023 13:11:50 +0100 Subject: [PATCH 004/243] Add nngm and exliquid modules --- ccp/modules/exliquid-compose.yml | 0 ccp/modules/exliquid-setup.sh | 0 ccp/modules/mtba-setup.sh | 2 +- ccp/modules/teiler-setup.sh | 2 ++ ccp/modules/teiler-ui-setup.sh | 2 ++ 5 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 ccp/modules/exliquid-compose.yml create mode 100644 ccp/modules/exliquid-setup.sh diff --git a/ccp/modules/exliquid-compose.yml b/ccp/modules/exliquid-compose.yml new file mode 100644 index 0000000..e69de29 diff --git a/ccp/modules/exliquid-setup.sh b/ccp/modules/exliquid-setup.sh new file mode 100644 index 0000000..e69de29 diff --git a/ccp/modules/mtba-setup.sh b/ccp/modules/mtba-setup.sh index ac050e0..11af4ee 100644 --- a/ccp/modules/mtba-setup.sh +++ b/ccp/modules/mtba-setup.sh @@ -9,4 +9,4 @@ function mtbaSetup() { fi OVERRIDE+=" -f ./$PROJECT/modules/mtba-compose.yml" fi -} \ No newline at end of file +} diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index d651107..e4c4945 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + function teilerSetup() { if [ -n "$ENABLE_TEILER" ];then log INFO "Teiler setup detected -- will start Teiler service." diff --git a/ccp/modules/teiler-ui-setup.sh b/ccp/modules/teiler-ui-setup.sh index c1f3933..1ff6f3b 100644 --- a/ccp/modules/teiler-ui-setup.sh +++ b/ccp/modules/teiler-ui-setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + function teilerUiSetup() { if [ -n "$ENABLE_TEILER" ];then log INFO "Teiler-UI setup detected -- will start Teiler-UI services." From c916a357dc04037fda2d9c79b8d8d2ec6ebae774 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 10 Feb 2023 15:47:19 +0100 Subject: [PATCH 005/243] Change images of dktk-teiler and dktk-keycloak --- ccp/modules/teiler-compose.yml | 2 +- ccp/modules/teiler-ui-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index b2c2966..904e2ec 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: teiler: - image: docker.verbis.dkfz.de/dktk/dktk-teiler:latest + image: docker.verbis.dkfz.de/ccp/dktk-teiler:latest container_name: bridgehead-ccp-teiler environment: LOG_LEVEL: "INFO" diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 5bdc6b2..35317cc 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -13,7 +13,7 @@ login-db: login: #image: "jboss/keycloak:16.1.1" - image: docker.verbis.dkfz.de/dktk/dktk-keycloak:latest + image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest container_name: bridgehead-login environment: KEYCLOAK_USER: "admin" From 6340acdbe839e552fb0f1bc51e24be8b07be3e21 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 10 Feb 2023 16:04:42 +0100 Subject: [PATCH 006/243] Bugfix: services in teiler-ui-compose.yml --- ccp/modules/teiler-ui-compose.yml | 501 +++++++++++++++--------------- 1 file changed, 251 insertions(+), 250 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 35317cc..9f3a245 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -1,259 +1,260 @@ version: "3.7" -############################################ Keycloak -login-db: - image: "postgres:15.1" - container_name: bridgehead-login-db - environment: - POSTGRES_USER: "keycloak" - POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh - POSTGRES_DB: "keycloak" - volumes: - - "login-db:/var/lib/postgresql/data" +services: + ############################################ Keycloak + login-db: + image: "postgres:15.1" + container_name: bridgehead-login-db + environment: + POSTGRES_USER: "keycloak" + POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + POSTGRES_DB: "keycloak" + volumes: + - "login-db:/var/lib/postgresql/data" -login: - #image: "jboss/keycloak:16.1.1" - image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest - container_name: bridgehead-login - environment: - KEYCLOAK_USER: "admin" - KEYCLOAK_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" - DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh - labels: - - "traefik.enable=true" - - "traefik.http.routers.login.rule=PathPrefix(`/login`)" - - "traefik.http.services.login.loadbalancer.server.port=8080" - - "traefik.http.routers.login.tls=true" - depends_on: - - login-db + login: + #image: "jboss/keycloak:16.1.1" + image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest + container_name: bridgehead-login + environment: + KEYCLOAK_USER: "admin" + KEYCLOAK_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" + DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + labels: + - "traefik.enable=true" + - "traefik.http.routers.login.rule=PathPrefix(`/login`)" + - "traefik.http.services.login.loadbalancer.server.port=8080" + - "traefik.http.routers.login.tls=true" + depends_on: + - login-db -############################################ Teiler-UI -teiler-root-config: - image: samply/teiler-root-config:developer - container_name: bridgehead-teiler-root-config - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler-root-config`)" - - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" - - "traefik.http.routers.teiler_root_config_ccp.tls=true" - environment: - TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" - TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" - DEFAULT_LANGUAGE: "de" + ############################################ Teiler-UI + teiler-root-config: + image: samply/teiler-root-config:developer + container_name: bridgehead-teiler-root-config + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler-root-config`)" + - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" + - "traefik.http.routers.teiler_root_config_ccp.tls=true" + environment: + TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" + TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" + DEFAULT_LANGUAGE: "de" -teiler-ui: - image: samply/teiler-ui:developer - container_name: bridgehead-teiler-ui -# ports: -# - 4200:80 - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_ui_ccp.rule=PathPrefix(`/ccp-teiler-ui`)" - - "traefik.http.services.teiler_ui_ccp.loadbalancer.server.port=80" - - "traefik.http.routers.teiler_ui_ccp.tls=true" - environment: - DEFAULT_LANGUAGE: "DE" - TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" - KEYCLOAK_URL: "https://${HOST}/login" - KEYCLOAK_REALM: "teiler-ui" - KEYCLOAK_CLIENT_ID: "teiler-ui" - TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" - TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" - TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" - TEILER_PROJECT: "${PROJECT}" + teiler-ui: + image: samply/teiler-ui:developer + container_name: bridgehead-teiler-ui + # ports: + # - 4200:80 + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_ui_ccp.rule=PathPrefix(`/ccp-teiler-ui`)" + - "traefik.http.services.teiler_ui_ccp.loadbalancer.server.port=80" + - "traefik.http.routers.teiler_ui_ccp.tls=true" + environment: + DEFAULT_LANGUAGE: "DE" + TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" + KEYCLOAK_URL: "https://${HOST}/login" + KEYCLOAK_REALM: "teiler-ui" + KEYCLOAK_CLIENT_ID: "teiler-ui" + TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" + TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" + TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" + TEILER_PROJECT: "${PROJECT}" -teiler-core: - image: samply/teiler-core:developer - container_name: bridgehead-teiler - volumes: - - "/etc/bridgehead/ccp.conf:/app/ccp.conf" - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" - - "traefik.http.services.teiler_core_ccp.loadbalancer.server.port=8085" - - "traefik.http.routers.teiler_core_ccp.tls=true" - environment: - LOG_LEVEL: "INFO" - APPLICATION_PORT: "8085" - DEFAULT_LANGUAGE: "DE" - CONFIG_ENV_VAR_PATH: "/app/ccp.conf" - TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" - TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler-root-config" - TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" - TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" - TEILER_APP1_NAME: "config" -# TEILER_APP2_NAME: "quality-report" - TEILER_APP3_NAME: "keycloak" - TEILER_APP3_TITLE: "Keycloak" - TEILER_APP4_DESCRIPTION_EN: "Authentication site" - TEILER_APP4_DESCRIPTION_DE: "Authentifizierungsseite" - TEILER_APP4_SOURCEURL: "https://${HOST}/login/auth/" - TEILER_APP4_ROLES: "TEILER_ADMIN" - TEILER_APP4_ISACTIVATED: "true" - TEILER_APP4_ICONSOURCEURL: "https://upload.wikimedia.org/wikipedia/commons/2/29/Keycloak_Logo.png" - TEILER_APP4_ORDER: "5" - TEILER_APP4_ISEXTERNALLINK: "true" - TEILER_APP4_ISLOCAL: "true" -# TEILER_APP5_NAME: "pgadmin" -# TEILER_APP5_TITLE: "PgAdmin" -# TEILER_APP5_DESCRIPTION: "Database Management" -# TEILER_APP5_SOURCEURL: "http://localhost:5000" -# TEILER_APP5_ROLES: "TEILER_ADMIN" -# TEILER_APP5_ISACTIVATED: "true" -# TEILER_APP5_ICONSOURCEURL: "https://user-images.githubusercontent.com/24623425/36042969-f87531d4-0d8a-11e8-9dee-e87ab8c6a9e3.png" -# TEILER_APP5_ORDER: "6" -# TEILER_APP5_ISEXTERNALLINK: "true" -# TEILER_APP5_ISLOCAL: "true" - TEILER_APP6_NAME: "ldm" - TEILER_APP6_TITLE_EN: "Local data management" - TEILER_APP6_TITLE_DE: "Lokales Datenmanagement" - TEILER_APP6_DESCRIPTION_EN: "Local Data Management" - TEILER_APP6_DESCRIPTION_DE: "Lokales Datenmanagement" - TEILER_APP6_SOURCEURL: "${CENTRAXX_URL}" - TEILER_APP6_ROLES: "TEILER_PUBLIC" - TEILER_APP6_ISACTIVATED: "${IS_DKTK_SITE}" - TEILER_APP6_ICONCLASS: "bi bi-server" - TEILER_APP6_ORDER: "7" - TEILER_APP6_ISEXTERNALLINK: "true" - TEILER_APP6_ISLOCAL: "true" - TEILER_APP7_NAME: "id-manager" - TEILER_APP7_TITLE: "ID-Manager" - TEILER_APP7_DESCRIPTION: "ID Manager" - TEILER_APP7_SOURCEURL: "https://${HOST}/id-manager/index.html" - TEILER_APP7_ROLES: "TEILER_PUBLIC" - TEILER_APP7_ISACTIVATED: "true" - TEILER_APP7_ICONCLASS: "bi bi-person-bounding-box" - TEILER_APP7_ORDER: "8" - TEILER_APP7_ISEXTERNALLINK: "true" - TEILER_APP7_ISLOCAL: "true" - TEILER_APP8_NAME: "patient-list" - TEILER_APP8_TITLE_EN: "Patient List" - TEILER_APP8_TITLE_DE: "Patientenliste" - TEILER_APP8_DESCRIPTION_EN: "Patient List" - TEILER_APP8_DESCRIPTION_DE: "Patientenliste" - TEILER_APP8_SOURCEURL: "https://${HOST}/patientlist" - TEILER_APP8_ROLES: "TEILER_PUBLIC" - TEILER_APP8_ISACTIVATED: "true" - TEILER_APP8_ICONCLASS: "bi bi-person-rolodex" - TEILER_APP8_ORDER: "9" - TEILER_APP8_ISEXTERNALLINK: "true" - TEILER_APP8_ISLOCAL: "true" - TEILER_APP9_NAME: "project-pseudonymisation" - TEILER_APP9_TITLE_EN: "Project Pseudonymisation" - TEILER_APP9_TITLE_DE: "Projectpseudonymisierung" - TEILER_APP9_DESCRIPTION_EN: "Project Pseudonymisation" - TEILER_APP9_DESCRIPTION_DE: "Projectpseudonymisierung" - TEILER_APP9_SOURCEURL: "https://${HOST}/id-manager/html/projectSelection.html" - TEILER_APP9_ROLES: "TEILER_PUBLIC" - TEILER_APP9_ISACTIVATED: "true" - TEILER_APP9_ICONCLASS: "bi bi-person-lines-fill" - TEILER_APP9_ORDER: "10" - TEILER_APP9_ISEXTERNALLINK: "true" - TEILER_APP9_ISLOCAL: "true" - TEILER_APP10_NAME: "federated-search" - TEILER_APP10_TITLE: "Lens" - TEILER_APP10_DESCRIPTION_EN: "Federated Search" - TEILER_APP10_DESCRIPTION_DE: "Föderierte Suche" - TEILER_APP10_SOURCEURL: "https://demo.lens.samply.de/" - TEILER_APP10_ROLES: "TEILER_PUBLIC" - TEILER_APP10_ISACTIVATED: "true" - TEILER_APP10_ICONCLASS: "bi bi-search" - TEILER_APP10_ORDER: "13" - TEILER_APP10_ISEXTERNALLINK: "true" - TEILER_APP10_ISLOCAL: "false" - TEILER_APP11_NAME: "central-patient-list" - TEILER_APP11_TITLE_EN: "Central Patient List" - TEILER_APP11_TITLE_DE: "Zentrale Patientenliste" - TEILER_APP11_DESCRIPTION_EN: "Central Patient List" - TEILER_APP11_DESCRIPTION_DE: "Zentrale Patientenliste" - TEILER_APP11_SOURCEURL: "https://patientlist.ccp-it.dktk.dkfz.de/" - TEILER_APP11_ROLES: "TEILER_PUBLIC" - TEILER_APP11_ISACTIVATED: "true" - TEILER_APP11_ICONCLASS: "bi bi-person-rolodex" - TEILER_APP11_ORDER: "14" - TEILER_APP11_ISEXTERNALLINK: "true" - TEILER_APP11_ISLOCAL: "false" - TEILER_APP12_NAME: "central id-manager" - TEILER_APP12_TITLE_EN: "Central ID-Manager" - TEILER_APP12_TITLE_DE: "Zentraler ID-Manager" - TEILER_APP12_DESCRIPTION_EN: "Central ID Manager" - TEILER_APP12_DESCRIPTION_DE: "Zentraler ID-Manager" - TEILER_APP12_SOURCEURL: "https://dktk-kne.kgu.de/" - TEILER_APP12_ROLES: "TEILER_PUBLIC" - TEILER_APP12_ISACTIVATED: "true" - TEILER_APP12_ICONCLASS: "bi bi-person-bounding-box" - TEILER_APP12_ORDER: "15" - TEILER_APP12_ISEXTERNALLINK: "true" - TEILER_APP12_ISLOCAL: "false" - # TODO: Icinga to be replaced by Zabbix - TEILER_APP13_NAME: "monitoring" - TEILER_APP13_TITLE: "Icinga" - TEILER_APP13_DESCRIPTION: "Icinga Monitoring" - TEILER_APP13_SOURCEURL: "https://monitor.vmitro.de/icingaweb2/dashboard" - TEILER_APP13_ROLES: "TEILER_ADMIN" - TEILER_APP13_ISACTIVATED: "true" - TEILER_APP13_ICONSOURCEURL: "https://images.ctfassets.net/o7xu9whrs0u9/QmL67mCGdRQ8PBcuKHGnF/858c0aee95762f59d67b25073f9483c2/icinga-logo.png" - TEILER_APP13_ORDER: "16" - TEILER_APP13_ISEXTERNALLINK: "true" - TEILER_APP13_ISLOCAL: "false" -# TEILER_APP14_NAME: "function-tests" -# TEILER_APP15_NAME: "event-log" - TEILER_APP16_NAME: "active-inquiries" - TEILER_APP16_BACKENDURL: "https://${HOST}/ccp-teiler" - TEILER_APP17_NAME: "archived-inquiries" - TEILER_APP17_BACKENDURL: "https://${HOST}/ccp-teiler" - TEILER_APP18_NAME: "failed-inquiries" - TEILER_APP18_BACKENDURL: "https://${HOST}/ccp-teiler" - TEILER_APP19_NAME: "inquiry" - TEILER_APP19_INMENU: "false" -# TEILER_APP20_NAME: "cbioportal" -# TEILER_APP20_TITLE: "cBioportal" -# TEILER_APP20_DESCRIPTION: "Interactive exploration of multidimensional cancer genomics data sets" -# TEILER_APP20_SOURCEURL: "http://localhost:8082" -# TEILER_APP20_ROLES: "TEILER_USER" -# TEILER_APP20_ISACTIVATED: "true" -# TEILER_APP20_ICONSOURCEURL: "https://docs.cbioportal.org/images/cbio-logo.png" -# TEILER_APP20_ORDER: "17" -# TEILER_APP20_ISEXTERNALLINK: "true" -# TEILER_APP20_ISLOCAL: "true" -# TEILER_APP21_NAME: "mtba-bp" -# TEILER_APP21_TITLE: "MTBA-BP" -# TEILER_APP21_DESCRIPTION: "MTBA Camunda Business Process" -# TEILER_APP21_SOURCEURL: "http://localhost:8480" -# TEILER_APP21_ROLES: "TEILER_ADMIN" -# TEILER_APP21_ISACTIVATED: "true" -# TEILER_APP21_ICONSOURCEURL: "https://camunda.com/wp-content/uploads/2020/05/logo-camunda-black.svg" -# TEILER_APP21_ORDER: "18" -# TEILER_APP21_ISEXTERNALLINK: "true" -# TEILER_APP21_ISLOCAL: "true" - TEILER_APP22_NAME: "dialog-quali" - TEILER_APP22_INMENU: "false" - TEILER_APP23_NAME: "dialog-uploads" - TEILER_APP23_INMENU: "false" - TEILER_APP24_NAME: "inquiry-dialog" - TEILER_APP24_INMENU: "false" - TEILER_APP25_NAME: "dialog-tests" - TEILER_APP25_INMENU: "false" -# TEILER_APP26_NAME: "opal" -# TEILER_APP26_TITLE: "Opal" -# TEILER_APP26_DESCRIPTION: "Opal is OBiBa’s core database application for biobanks." -# TEILER_APP26_SOURCEURL: "http://localhost:8880" -# TEILER_APP26_ROLES: "TEILER_USER" -# TEILER_APP26_ISACTIVATED: "true" -# TEILER_APP26_ICONSOURCEURL: "https://www.obiba.org/assets/themes/bootstrap/img/obiba-logo-small.png" -# TEILER_APP26_ORDER: "19" -# TEILER_APP26_ISEXTERNALLINK: "true" -# TEILER_APP26_ISLOCAL: "true" -# TEILER_APP27_NAME: "rstudio" -# TEILER_APP27_TITLE: "R Studio" -# TEILER_APP27_DESCRIPTION: "RStudio is an integrated development environment (IDE) for R and Python." -# TEILER_APP27_SOURCEURL: "http://localhost:8787" -# TEILER_APP27_ROLES: "TEILER_USER" -# TEILER_APP27_ISACTIVATED: "true" -# TEILER_APP27_ICONSOURCEURL: "https://rstudio.com/wp-content/uploads/2018/10/RStudio-Logo-Flat.png" -# TEILER_APP27_ORDER: "20" -# TEILER_APP27_ISEXTERNALLINK: "true" -# TEILER_APP27_ISLOCAL: "true" + teiler-core: + image: samply/teiler-core:developer + container_name: bridgehead-teiler + volumes: + - "/etc/bridgehead/ccp.conf:/app/ccp.conf" + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" + - "traefik.http.services.teiler_core_ccp.loadbalancer.server.port=8085" + - "traefik.http.routers.teiler_core_ccp.tls=true" + environment: + LOG_LEVEL: "INFO" + APPLICATION_PORT: "8085" + DEFAULT_LANGUAGE: "DE" + CONFIG_ENV_VAR_PATH: "/app/ccp.conf" + TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" + TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler-root-config" + TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" + TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" + TEILER_APP1_NAME: "config" + # TEILER_APP2_NAME: "quality-report" + TEILER_APP3_NAME: "keycloak" + TEILER_APP3_TITLE: "Keycloak" + TEILER_APP4_DESCRIPTION_EN: "Authentication site" + TEILER_APP4_DESCRIPTION_DE: "Authentifizierungsseite" + TEILER_APP4_SOURCEURL: "https://${HOST}/login/auth/" + TEILER_APP4_ROLES: "TEILER_ADMIN" + TEILER_APP4_ISACTIVATED: "true" + TEILER_APP4_ICONSOURCEURL: "https://upload.wikimedia.org/wikipedia/commons/2/29/Keycloak_Logo.png" + TEILER_APP4_ORDER: "5" + TEILER_APP4_ISEXTERNALLINK: "true" + TEILER_APP4_ISLOCAL: "true" + # TEILER_APP5_NAME: "pgadmin" + # TEILER_APP5_TITLE: "PgAdmin" + # TEILER_APP5_DESCRIPTION: "Database Management" + # TEILER_APP5_SOURCEURL: "http://localhost:5000" + # TEILER_APP5_ROLES: "TEILER_ADMIN" + # TEILER_APP5_ISACTIVATED: "true" + # TEILER_APP5_ICONSOURCEURL: "https://user-images.githubusercontent.com/24623425/36042969-f87531d4-0d8a-11e8-9dee-e87ab8c6a9e3.png" + # TEILER_APP5_ORDER: "6" + # TEILER_APP5_ISEXTERNALLINK: "true" + # TEILER_APP5_ISLOCAL: "true" + TEILER_APP6_NAME: "ldm" + TEILER_APP6_TITLE_EN: "Local data management" + TEILER_APP6_TITLE_DE: "Lokales Datenmanagement" + TEILER_APP6_DESCRIPTION_EN: "Local Data Management" + TEILER_APP6_DESCRIPTION_DE: "Lokales Datenmanagement" + TEILER_APP6_SOURCEURL: "${CENTRAXX_URL}" + TEILER_APP6_ROLES: "TEILER_PUBLIC" + TEILER_APP6_ISACTIVATED: "${IS_DKTK_SITE}" + TEILER_APP6_ICONCLASS: "bi bi-server" + TEILER_APP6_ORDER: "7" + TEILER_APP6_ISEXTERNALLINK: "true" + TEILER_APP6_ISLOCAL: "true" + TEILER_APP7_NAME: "id-manager" + TEILER_APP7_TITLE: "ID-Manager" + TEILER_APP7_DESCRIPTION: "ID Manager" + TEILER_APP7_SOURCEURL: "https://${HOST}/id-manager/index.html" + TEILER_APP7_ROLES: "TEILER_PUBLIC" + TEILER_APP7_ISACTIVATED: "true" + TEILER_APP7_ICONCLASS: "bi bi-person-bounding-box" + TEILER_APP7_ORDER: "8" + TEILER_APP7_ISEXTERNALLINK: "true" + TEILER_APP7_ISLOCAL: "true" + TEILER_APP8_NAME: "patient-list" + TEILER_APP8_TITLE_EN: "Patient List" + TEILER_APP8_TITLE_DE: "Patientenliste" + TEILER_APP8_DESCRIPTION_EN: "Patient List" + TEILER_APP8_DESCRIPTION_DE: "Patientenliste" + TEILER_APP8_SOURCEURL: "https://${HOST}/patientlist" + TEILER_APP8_ROLES: "TEILER_PUBLIC" + TEILER_APP8_ISACTIVATED: "true" + TEILER_APP8_ICONCLASS: "bi bi-person-rolodex" + TEILER_APP8_ORDER: "9" + TEILER_APP8_ISEXTERNALLINK: "true" + TEILER_APP8_ISLOCAL: "true" + TEILER_APP9_NAME: "project-pseudonymisation" + TEILER_APP9_TITLE_EN: "Project Pseudonymisation" + TEILER_APP9_TITLE_DE: "Projectpseudonymisierung" + TEILER_APP9_DESCRIPTION_EN: "Project Pseudonymisation" + TEILER_APP9_DESCRIPTION_DE: "Projectpseudonymisierung" + TEILER_APP9_SOURCEURL: "https://${HOST}/id-manager/html/projectSelection.html" + TEILER_APP9_ROLES: "TEILER_PUBLIC" + TEILER_APP9_ISACTIVATED: "true" + TEILER_APP9_ICONCLASS: "bi bi-person-lines-fill" + TEILER_APP9_ORDER: "10" + TEILER_APP9_ISEXTERNALLINK: "true" + TEILER_APP9_ISLOCAL: "true" + TEILER_APP10_NAME: "federated-search" + TEILER_APP10_TITLE: "Lens" + TEILER_APP10_DESCRIPTION_EN: "Federated Search" + TEILER_APP10_DESCRIPTION_DE: "Föderierte Suche" + TEILER_APP10_SOURCEURL: "https://demo.lens.samply.de/" + TEILER_APP10_ROLES: "TEILER_PUBLIC" + TEILER_APP10_ISACTIVATED: "true" + TEILER_APP10_ICONCLASS: "bi bi-search" + TEILER_APP10_ORDER: "13" + TEILER_APP10_ISEXTERNALLINK: "true" + TEILER_APP10_ISLOCAL: "false" + TEILER_APP11_NAME: "central-patient-list" + TEILER_APP11_TITLE_EN: "Central Patient List" + TEILER_APP11_TITLE_DE: "Zentrale Patientenliste" + TEILER_APP11_DESCRIPTION_EN: "Central Patient List" + TEILER_APP11_DESCRIPTION_DE: "Zentrale Patientenliste" + TEILER_APP11_SOURCEURL: "https://patientlist.ccp-it.dktk.dkfz.de/" + TEILER_APP11_ROLES: "TEILER_PUBLIC" + TEILER_APP11_ISACTIVATED: "true" + TEILER_APP11_ICONCLASS: "bi bi-person-rolodex" + TEILER_APP11_ORDER: "14" + TEILER_APP11_ISEXTERNALLINK: "true" + TEILER_APP11_ISLOCAL: "false" + TEILER_APP12_NAME: "central id-manager" + TEILER_APP12_TITLE_EN: "Central ID-Manager" + TEILER_APP12_TITLE_DE: "Zentraler ID-Manager" + TEILER_APP12_DESCRIPTION_EN: "Central ID Manager" + TEILER_APP12_DESCRIPTION_DE: "Zentraler ID-Manager" + TEILER_APP12_SOURCEURL: "https://dktk-kne.kgu.de/" + TEILER_APP12_ROLES: "TEILER_PUBLIC" + TEILER_APP12_ISACTIVATED: "true" + TEILER_APP12_ICONCLASS: "bi bi-person-bounding-box" + TEILER_APP12_ORDER: "15" + TEILER_APP12_ISEXTERNALLINK: "true" + TEILER_APP12_ISLOCAL: "false" + # TODO: Icinga to be replaced by Zabbix + TEILER_APP13_NAME: "monitoring" + TEILER_APP13_TITLE: "Icinga" + TEILER_APP13_DESCRIPTION: "Icinga Monitoring" + TEILER_APP13_SOURCEURL: "https://monitor.vmitro.de/icingaweb2/dashboard" + TEILER_APP13_ROLES: "TEILER_ADMIN" + TEILER_APP13_ISACTIVATED: "true" + TEILER_APP13_ICONSOURCEURL: "https://images.ctfassets.net/o7xu9whrs0u9/QmL67mCGdRQ8PBcuKHGnF/858c0aee95762f59d67b25073f9483c2/icinga-logo.png" + TEILER_APP13_ORDER: "16" + TEILER_APP13_ISEXTERNALLINK: "true" + TEILER_APP13_ISLOCAL: "false" + # TEILER_APP14_NAME: "function-tests" + # TEILER_APP15_NAME: "event-log" + TEILER_APP16_NAME: "active-inquiries" + TEILER_APP16_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP17_NAME: "archived-inquiries" + TEILER_APP17_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP18_NAME: "failed-inquiries" + TEILER_APP18_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP19_NAME: "inquiry" + TEILER_APP19_INMENU: "false" + # TEILER_APP20_NAME: "cbioportal" + # TEILER_APP20_TITLE: "cBioportal" + # TEILER_APP20_DESCRIPTION: "Interactive exploration of multidimensional cancer genomics data sets" + # TEILER_APP20_SOURCEURL: "http://localhost:8082" + # TEILER_APP20_ROLES: "TEILER_USER" + # TEILER_APP20_ISACTIVATED: "true" + # TEILER_APP20_ICONSOURCEURL: "https://docs.cbioportal.org/images/cbio-logo.png" + # TEILER_APP20_ORDER: "17" + # TEILER_APP20_ISEXTERNALLINK: "true" + # TEILER_APP20_ISLOCAL: "true" + # TEILER_APP21_NAME: "mtba-bp" + # TEILER_APP21_TITLE: "MTBA-BP" + # TEILER_APP21_DESCRIPTION: "MTBA Camunda Business Process" + # TEILER_APP21_SOURCEURL: "http://localhost:8480" + # TEILER_APP21_ROLES: "TEILER_ADMIN" + # TEILER_APP21_ISACTIVATED: "true" + # TEILER_APP21_ICONSOURCEURL: "https://camunda.com/wp-content/uploads/2020/05/logo-camunda-black.svg" + # TEILER_APP21_ORDER: "18" + # TEILER_APP21_ISEXTERNALLINK: "true" + # TEILER_APP21_ISLOCAL: "true" + TEILER_APP22_NAME: "dialog-quali" + TEILER_APP22_INMENU: "false" + TEILER_APP23_NAME: "dialog-uploads" + TEILER_APP23_INMENU: "false" + TEILER_APP24_NAME: "inquiry-dialog" + TEILER_APP24_INMENU: "false" + TEILER_APP25_NAME: "dialog-tests" + TEILER_APP25_INMENU: "false" + # TEILER_APP26_NAME: "opal" + # TEILER_APP26_TITLE: "Opal" + # TEILER_APP26_DESCRIPTION: "Opal is OBiBa’s core database application for biobanks." + # TEILER_APP26_SOURCEURL: "http://localhost:8880" + # TEILER_APP26_ROLES: "TEILER_USER" + # TEILER_APP26_ISACTIVATED: "true" + # TEILER_APP26_ICONSOURCEURL: "https://www.obiba.org/assets/themes/bootstrap/img/obiba-logo-small.png" + # TEILER_APP26_ORDER: "19" + # TEILER_APP26_ISEXTERNALLINK: "true" + # TEILER_APP26_ISLOCAL: "true" + # TEILER_APP27_NAME: "rstudio" + # TEILER_APP27_TITLE: "R Studio" + # TEILER_APP27_DESCRIPTION: "RStudio is an integrated development environment (IDE) for R and Python." + # TEILER_APP27_SOURCEURL: "http://localhost:8787" + # TEILER_APP27_ROLES: "TEILER_USER" + # TEILER_APP27_ISACTIVATED: "true" + # TEILER_APP27_ICONSOURCEURL: "https://rstudio.com/wp-content/uploads/2018/10/RStudio-Logo-Flat.png" + # TEILER_APP27_ORDER: "20" + # TEILER_APP27_ISEXTERNALLINK: "true" + # TEILER_APP27_ISLOCAL: "true" volumes: login-db: From eb17d8c15936524e5c243011ce8239583a79dd06 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 13 Feb 2023 10:23:14 +0100 Subject: [PATCH 007/243] Configure login extern URLs --- ccp/modules/teiler-ui-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 9f3a245..a46e710 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -20,6 +20,8 @@ services: KEYCLOAK_USER: "admin" KEYCLOAK_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler-root-config" + LOGIN_EXTERN_URL: "https://${HOST}/login" labels: - "traefik.enable=true" - "traefik.http.routers.login.rule=PathPrefix(`/login`)" From 6626f860a2779dca4ee4d3b242daaf07034ca72d Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 13 Feb 2023 15:29:45 +0100 Subject: [PATCH 008/243] Rename teiler to exporter --- ccp/modules/exporter-compose.yml | 42 ++++++++++++++++++++++++++++++++ ccp/modules/exporter-setup.sh | 10 ++++++++ ccp/modules/teiler-compose.yml | 42 -------------------------------- ccp/modules/teiler-setup.sh | 10 -------- 4 files changed, 52 insertions(+), 52 deletions(-) create mode 100644 ccp/modules/exporter-compose.yml create mode 100644 ccp/modules/exporter-setup.sh delete mode 100644 ccp/modules/teiler-compose.yml delete mode 100644 ccp/modules/teiler-setup.sh diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml new file mode 100644 index 0000000..dc2e340 --- /dev/null +++ b/ccp/modules/exporter-compose.yml @@ -0,0 +1,42 @@ +version: "3.7" + +services: + exporter: + image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest + container_name: bridgehead-ccp-exporter + environment: + LOG_LEVEL: "INFO" + EXPORTER_API_KEY: "${EXPORTER_API_KEY}" + CROSS_ORIGINS: "https://${HOST}/ccp-teiler-root-config" + EXPORTER_DB_USER: "exporter" + EXPORTER_DB_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh + EXPORTER_DB_URL: "jdbc:postgresql://exporter-db:5432/exporter" + CLEAN_TEMP_FILES_CRON_EXPRESSION: "0 0 1 * * *" + TEMP_FILES_LIFETIME_IN_DAYS: "1" + CLEAN_WRITE_FILES_CRON_EXPRESSION: "0 0 2 * * *" + WRITE_FILES_LIFETIME_IN_DAYS: "30" + labels: + - "traefik.enable=true" + - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" + - "traefik.http.services.exporter_ccp.loadbalancer.server.port=8092" + - "traefik.http.routers.exporter_ccp.tls=true" + - "traefik.http.routers.exporter_ccp.middlewares=auth" + volumes: + - "exporter:/app/exporter-files" + + exporter-db: + image: postgres:15.1-alpine + container_name: bridgehead-ccp-exporter-db + environment: + POSTGRES_USER: "exporter" + POSTGRES_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh + POSTGRES_DB: "exporter" + volumes: + - "exporter-db:/var/lib/postgresql/data" + + +volumes: + exporter-db: + name: "exporter-db" + exporter: + name: "exporter" diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh new file mode 100644 index 0000000..090d23c --- /dev/null +++ b/ccp/modules/exporter-setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +function exporterSetup() { + if [ -n "$ENABLE_EXPORTER" ];then + log INFO "Exporter setup detected -- will start Exporter service." + OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" + fi + # TODO: Generate password in another way so that not all passwords are the same? + EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +} diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml deleted file mode 100644 index 904e2ec..0000000 --- a/ccp/modules/teiler-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: "3.7" - -services: - teiler: - image: docker.verbis.dkfz.de/ccp/dktk-teiler:latest - container_name: bridgehead-ccp-teiler - environment: - LOG_LEVEL: "INFO" - TEILER_API_KEY: "${TEILER_API_KEY}" - CROSS_ORIGINS: "https://${HOST}/ccp-teiler-root-config" - TEILER_DB_USER: "teiler" - TEILER_DB_PASSWORD: "${TEILER_DB_PASSWORD}" # Set in teiler-setup.sh - TEILER_DB_URL: "jdbc:postgresql://teiler-db:5432/teiler" - CLEAN_TEMP_FILES_CRON_EXPRESSION: "0 0 1 * * *" - TEMP_FILES_LIFETIME_IN_DAYS: "1" - CLEAN_WRITE_FILES_CRON_EXPRESSION: "0 0 2 * * *" - WRITE_FILES_LIFETIME_IN_DAYS: "30" - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_ccp.rule=PathPrefix(`/ccp-teiler`)" - - "traefik.http.services.teiler_ccp.loadbalancer.server.port=8092" - - "traefik.http.routers.teiler_ccp.tls=true" - - "traefik.http.routers.teiler_ccp.middlewares=auth" - volumes: - - "teiler:/app/teiler-files" - - teiler-db: - image: postgres:15.1-alpine - container_name: bridgehead-ccp-teiler-db - environment: - POSTGRES_USER: "teiler" - POSTGRES_PASSWORD: "${TEILER_DB_PASSWORD}" # Set in teiler-setup.sh - POSTGRES_DB: "teiler" - volumes: - - "teiler-db:/var/lib/postgresql/data" - - -volumes: - teiler-db: - name: "teiler-db" - teiler: - name: "teiler" diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh deleted file mode 100644 index e4c4945..0000000 --- a/ccp/modules/teiler-setup.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -function teilerSetup() { - if [ -n "$ENABLE_TEILER" ];then - log INFO "Teiler setup detected -- will start Teiler service." - OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" - fi - # TODO: Generate password in another way so that not all passwords are the same? - TEILER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" -} From 49be101165d5572b7e7b30a748ad9403fb100445 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 13 Feb 2023 15:35:35 +0100 Subject: [PATCH 009/243] Rename teiler to exporter (bugfix) --- ccp/modules/teiler-ui-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index a46e710..db08298 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -202,11 +202,11 @@ services: # TEILER_APP14_NAME: "function-tests" # TEILER_APP15_NAME: "event-log" TEILER_APP16_NAME: "active-inquiries" - TEILER_APP16_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP16_BACKENDURL: "https://${HOST}/ccp-exporter" TEILER_APP17_NAME: "archived-inquiries" - TEILER_APP17_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP17_BACKENDURL: "https://${HOST}/ccp-exporter" TEILER_APP18_NAME: "failed-inquiries" - TEILER_APP18_BACKENDURL: "https://${HOST}/ccp-teiler" + TEILER_APP18_BACKENDURL: "https://${HOST}/ccp-exporter" TEILER_APP19_NAME: "inquiry" TEILER_APP19_INMENU: "false" # TEILER_APP20_NAME: "cbioportal" From 38c7f3c24a56f98bd44e15533ca0f838a5df19f0 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 14 Feb 2023 10:20:19 +0100 Subject: [PATCH 010/243] beautiful config --- ccp/modules/exporter-setup.sh | 14 ++++++-------- ccp/modules/teiler-ui-setup.sh | 12 +++++------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index 090d23c..044116b 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -1,10 +1,8 @@ #!/bin/bash -function exporterSetup() { - if [ -n "$ENABLE_EXPORTER" ];then - log INFO "Exporter setup detected -- will start Exporter service." - OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" - fi - # TODO: Generate password in another way so that not all passwords are the same? - EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" -} +if [ -n "$ENABLE_EXPORTER" ];then + log INFO "Exporter setup detected -- will start Exporter service." + OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" +fi +# TODO: Generate password in another way so that not all passwords are the same? +EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" diff --git a/ccp/modules/teiler-ui-setup.sh b/ccp/modules/teiler-ui-setup.sh index 1ff6f3b..dc755ba 100644 --- a/ccp/modules/teiler-ui-setup.sh +++ b/ccp/modules/teiler-ui-setup.sh @@ -1,9 +1,7 @@ #!/bin/bash -function teilerUiSetup() { - if [ -n "$ENABLE_TEILER" ];then - log INFO "Teiler-UI setup detected -- will start Teiler-UI services." - OVERRIDE+=" -f ./$PROJECT/modules/teiler-ui-compose.yml" - fi - KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" -} +if [ "$ENABLE_TEILER" == true ];then + log INFO "Teiler-UI setup detected -- will start Teiler-UI services." + OVERRIDE+=" -f ./$PROJECT/modules/teiler-ui-compose.yml" +fi +KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" From c66dac988191bb0e194f7d3f3e5aa8557094a18b Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Mon, 20 Feb 2023 14:49:45 +0100 Subject: [PATCH 011/243] update keykloak config --- ccp/modules/teiler-ui-compose.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index db08298..1aba906 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -17,9 +17,12 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest container_name: bridgehead-login environment: - KEYCLOAK_USER: "admin" - KEYCLOAK_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" - DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + KEYCLOAK_ADMIN: "admin" + KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" + KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + KC_HOSTNAME_URL: "https://${HOST}/login" + KC_HOSTNAME_STRICT: "false" + KC_PROXY_ADDRESS_FORWARDING: "true" TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler-root-config" LOGIN_EXTERN_URL: "https://${HOST}/login" labels: From 8fe03a6cd2a35e302e5ff77756098b6b7c0b91fe Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 14 Feb 2023 15:46:57 +0100 Subject: [PATCH 012/243] Add original Keycloak config --- ccp/modules/teiler-ui-compose.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 1aba906..ba2ca27 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -14,17 +14,20 @@ services: login: #image: "jboss/keycloak:16.1.1" - image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest + image: docker.verbis.dkfz.de/ccp/dktk-keycloak:test container_name: bridgehead-login environment: KEYCLOAK_ADMIN: "admin" KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh - KC_HOSTNAME_URL: "https://${HOST}/login" - KC_HOSTNAME_STRICT: "false" - KC_PROXY_ADDRESS_FORWARDING: "true" + KC_HOSTNAME: "https://${HOST}/login" + #KEYCLOAK_USER: "admin" + #KEYCLOAK_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" + #DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler-root-config" LOGIN_EXTERN_URL: "https://${HOST}/login" + command: + - start-dev --import-realm labels: - "traefik.enable=true" - "traefik.http.routers.login.rule=PathPrefix(`/login`)" From efc04cea4fd6a4eebfbddb1347dfc2a38823c18c Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 15 Feb 2023 16:09:53 +0100 Subject: [PATCH 013/243] Update Teiler Core config --- ccp/modules/teiler-ui-compose.yml | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index ba2ca27..c4b5fa4 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -94,8 +94,8 @@ services: # TEILER_APP2_NAME: "quality-report" TEILER_APP3_NAME: "keycloak" TEILER_APP3_TITLE: "Keycloak" - TEILER_APP4_DESCRIPTION_EN: "Authentication site" - TEILER_APP4_DESCRIPTION_DE: "Authentifizierungsseite" + TEILER_APP4_EN_DESCRIPTION: "Authentication site" + TEILER_APP4_DE_DESCRIPTION: "Authentifizierungsseite" TEILER_APP4_SOURCEURL: "https://${HOST}/login/auth/" TEILER_APP4_ROLES: "TEILER_ADMIN" TEILER_APP4_ISACTIVATED: "true" @@ -114,10 +114,10 @@ services: # TEILER_APP5_ISEXTERNALLINK: "true" # TEILER_APP5_ISLOCAL: "true" TEILER_APP6_NAME: "ldm" - TEILER_APP6_TITLE_EN: "Local data management" - TEILER_APP6_TITLE_DE: "Lokales Datenmanagement" - TEILER_APP6_DESCRIPTION_EN: "Local Data Management" - TEILER_APP6_DESCRIPTION_DE: "Lokales Datenmanagement" + TEILER_APP6_EN_TITLE: "Local data management" + TEILER_APP6_DE_TITLE: "Lokales Datenmanagement" + TEILER_APP6_EN_DESCRIPTION: "Local Data Management" + TEILER_APP6_DE_DESCRIPTION: "Lokales Datenmanagement" TEILER_APP6_SOURCEURL: "${CENTRAXX_URL}" TEILER_APP6_ROLES: "TEILER_PUBLIC" TEILER_APP6_ISACTIVATED: "${IS_DKTK_SITE}" @@ -126,7 +126,7 @@ services: TEILER_APP6_ISEXTERNALLINK: "true" TEILER_APP6_ISLOCAL: "true" TEILER_APP7_NAME: "id-manager" - TEILER_APP7_TITLE: "ID-Manager" + TEILER_APP7_TITLE: "Magic PL" TEILER_APP7_DESCRIPTION: "ID Manager" TEILER_APP7_SOURCEURL: "https://${HOST}/id-manager/index.html" TEILER_APP7_ROLES: "TEILER_PUBLIC" @@ -136,10 +136,10 @@ services: TEILER_APP7_ISEXTERNALLINK: "true" TEILER_APP7_ISLOCAL: "true" TEILER_APP8_NAME: "patient-list" - TEILER_APP8_TITLE_EN: "Patient List" - TEILER_APP8_TITLE_DE: "Patientenliste" - TEILER_APP8_DESCRIPTION_EN: "Patient List" - TEILER_APP8_DESCRIPTION_DE: "Patientenliste" + TEILER_APP8_EN_TITLE: "Patient List" + TEILER_APP8_DE_TITLE: "Mainzelliste" + TEILER_APP8_EN_DESCRIPTION: "Patient List" + TEILER_APP8_DE_DESCRIPTION: "Patientenliste" TEILER_APP8_SOURCEURL: "https://${HOST}/patientlist" TEILER_APP8_ROLES: "TEILER_PUBLIC" TEILER_APP8_ISACTIVATED: "true" @@ -148,10 +148,10 @@ services: TEILER_APP8_ISEXTERNALLINK: "true" TEILER_APP8_ISLOCAL: "true" TEILER_APP9_NAME: "project-pseudonymisation" - TEILER_APP9_TITLE_EN: "Project Pseudonymisation" - TEILER_APP9_TITLE_DE: "Projectpseudonymisierung" - TEILER_APP9_DESCRIPTION_EN: "Project Pseudonymisation" - TEILER_APP9_DESCRIPTION_DE: "Projectpseudonymisierung" + TEILER_APP9_EN_TITLE: "Project Pseudonymisation" + TEILER_APP9_DE_TITLE: "Projectpseudonymisierung" + TEILER_APP9_EN_DESCRIPTION: "Project Pseudonymisation" + TEILER_APP9_DE_DESCRIPTION: "Projectpseudonymisierung" TEILER_APP9_SOURCEURL: "https://${HOST}/id-manager/html/projectSelection.html" TEILER_APP9_ROLES: "TEILER_PUBLIC" TEILER_APP9_ISACTIVATED: "true" @@ -161,8 +161,8 @@ services: TEILER_APP9_ISLOCAL: "true" TEILER_APP10_NAME: "federated-search" TEILER_APP10_TITLE: "Lens" - TEILER_APP10_DESCRIPTION_EN: "Federated Search" - TEILER_APP10_DESCRIPTION_DE: "Föderierte Suche" + TEILER_APP10_EN_DESCRIPTION: "Federated Search" + TEILER_APP10_DE_DESCRIPTION: "Föderierte Suche" TEILER_APP10_SOURCEURL: "https://demo.lens.samply.de/" TEILER_APP10_ROLES: "TEILER_PUBLIC" TEILER_APP10_ISACTIVATED: "true" @@ -171,10 +171,10 @@ services: TEILER_APP10_ISEXTERNALLINK: "true" TEILER_APP10_ISLOCAL: "false" TEILER_APP11_NAME: "central-patient-list" - TEILER_APP11_TITLE_EN: "Central Patient List" - TEILER_APP11_TITLE_DE: "Zentrale Patientenliste" - TEILER_APP11_DESCRIPTION_EN: "Central Patient List" - TEILER_APP11_DESCRIPTION_DE: "Zentrale Patientenliste" + TEILER_APP11_EN_TITLE: "Central Patient List" + TEILER_APP11_DE_TITLE: "Zentrale Mainzelliste" + TEILER_APP11_EN_DESCRIPTION: "Central Patient List" + TEILER_APP11_DE_DESCRIPTION: "Zentrale Patientenliste" TEILER_APP11_SOURCEURL: "https://patientlist.ccp-it.dktk.dkfz.de/" TEILER_APP11_ROLES: "TEILER_PUBLIC" TEILER_APP11_ISACTIVATED: "true" @@ -183,10 +183,10 @@ services: TEILER_APP11_ISEXTERNALLINK: "true" TEILER_APP11_ISLOCAL: "false" TEILER_APP12_NAME: "central id-manager" - TEILER_APP12_TITLE_EN: "Central ID-Manager" - TEILER_APP12_TITLE_DE: "Zentraler ID-Manager" - TEILER_APP12_DESCRIPTION_EN: "Central ID Manager" - TEILER_APP12_DESCRIPTION_DE: "Zentraler ID-Manager" + TEILER_APP12_EN_TITLE: "Central Magic PL" + TEILER_APP12_DE_TITLE: "Zentraler Magic PL" + TEILER_APP12_EN_DESCRIPTION: "Central ID Manager" + TEILER_APP12_DE_DESCRIPTION: "Zentraler ID-Manager" TEILER_APP12_SOURCEURL: "https://dktk-kne.kgu.de/" TEILER_APP12_ROLES: "TEILER_PUBLIC" TEILER_APP12_ISACTIVATED: "true" From 4d1a9bb7012a34509d7ddf295f3b64aa4e1691ad Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 21 Feb 2023 15:37:12 +0100 Subject: [PATCH 014/243] Add Endpoint for Teiler --- ccp/modules/teiler-ui-compose.yml | 68 +++++++++++++++---------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index c4b5fa4..a893639 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -14,20 +14,18 @@ services: login: #image: "jboss/keycloak:16.1.1" - image: docker.verbis.dkfz.de/ccp/dktk-keycloak:test + image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest container_name: bridgehead-login environment: KEYCLOAK_ADMIN: "admin" KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh - KC_HOSTNAME: "https://${HOST}/login" - #KEYCLOAK_USER: "admin" - #KEYCLOAK_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" - #DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh - TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler-root-config" - LOGIN_EXTERN_URL: "https://${HOST}/login" + KC_HOSTNAME_URL: "https://${HOST}/login" + KC_HOSTNAME_STRICT: "false" + KC_PROXY_ADDRESS_FORWARDING: "true" + TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler" command: - - start-dev --import-realm + - start-dev --import-realm --proxy edge --http-relative-path=/login labels: - "traefik.enable=true" - "traefik.http.routers.login.rule=PathPrefix(`/login`)" @@ -42,7 +40,7 @@ services: container_name: bridgehead-teiler-root-config labels: - "traefik.enable=true" - - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler-root-config`)" + - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler`)" - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" - "traefik.http.routers.teiler_root_config_ccp.tls=true" environment: @@ -87,15 +85,15 @@ services: DEFAULT_LANGUAGE: "DE" CONFIG_ENV_VAR_PATH: "/app/ccp.conf" TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" - TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler-root-config" + TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" TEILER_APP1_NAME: "config" # TEILER_APP2_NAME: "quality-report" TEILER_APP3_NAME: "keycloak" TEILER_APP3_TITLE: "Keycloak" - TEILER_APP4_EN_DESCRIPTION: "Authentication site" - TEILER_APP4_DE_DESCRIPTION: "Authentifizierungsseite" + TEILER_APP4_DESCRIPTION_EN: "Authentication site" + TEILER_APP4_DESCRIPTION_DE: "Authentifizierungsseite" TEILER_APP4_SOURCEURL: "https://${HOST}/login/auth/" TEILER_APP4_ROLES: "TEILER_ADMIN" TEILER_APP4_ISACTIVATED: "true" @@ -114,10 +112,10 @@ services: # TEILER_APP5_ISEXTERNALLINK: "true" # TEILER_APP5_ISLOCAL: "true" TEILER_APP6_NAME: "ldm" - TEILER_APP6_EN_TITLE: "Local data management" - TEILER_APP6_DE_TITLE: "Lokales Datenmanagement" - TEILER_APP6_EN_DESCRIPTION: "Local Data Management" - TEILER_APP6_DE_DESCRIPTION: "Lokales Datenmanagement" + TEILER_APP6_TITLE_EN: "Local data management" + TEILER_APP6_TITLE_DE: "Lokales Datenmanagement" + TEILER_APP6_DESCRIPTION_EN: "Local Data Management" + TEILER_APP6_DESCRIPTION_DE: "Lokales Datenmanagement" TEILER_APP6_SOURCEURL: "${CENTRAXX_URL}" TEILER_APP6_ROLES: "TEILER_PUBLIC" TEILER_APP6_ISACTIVATED: "${IS_DKTK_SITE}" @@ -126,7 +124,7 @@ services: TEILER_APP6_ISEXTERNALLINK: "true" TEILER_APP6_ISLOCAL: "true" TEILER_APP7_NAME: "id-manager" - TEILER_APP7_TITLE: "Magic PL" + TEILER_APP7_TITLE: "ID-Manager" TEILER_APP7_DESCRIPTION: "ID Manager" TEILER_APP7_SOURCEURL: "https://${HOST}/id-manager/index.html" TEILER_APP7_ROLES: "TEILER_PUBLIC" @@ -136,10 +134,10 @@ services: TEILER_APP7_ISEXTERNALLINK: "true" TEILER_APP7_ISLOCAL: "true" TEILER_APP8_NAME: "patient-list" - TEILER_APP8_EN_TITLE: "Patient List" - TEILER_APP8_DE_TITLE: "Mainzelliste" - TEILER_APP8_EN_DESCRIPTION: "Patient List" - TEILER_APP8_DE_DESCRIPTION: "Patientenliste" + TEILER_APP8_TITLE_EN: "Patient List" + TEILER_APP8_TITLE_DE: "Patientenliste" + TEILER_APP8_DESCRIPTION_EN: "Patient List" + TEILER_APP8_DESCRIPTION_DE: "Patientenliste" TEILER_APP8_SOURCEURL: "https://${HOST}/patientlist" TEILER_APP8_ROLES: "TEILER_PUBLIC" TEILER_APP8_ISACTIVATED: "true" @@ -148,10 +146,10 @@ services: TEILER_APP8_ISEXTERNALLINK: "true" TEILER_APP8_ISLOCAL: "true" TEILER_APP9_NAME: "project-pseudonymisation" - TEILER_APP9_EN_TITLE: "Project Pseudonymisation" - TEILER_APP9_DE_TITLE: "Projectpseudonymisierung" - TEILER_APP9_EN_DESCRIPTION: "Project Pseudonymisation" - TEILER_APP9_DE_DESCRIPTION: "Projectpseudonymisierung" + TEILER_APP9_TITLE_EN: "Project Pseudonymisation" + TEILER_APP9_TITLE_DE: "Projectpseudonymisierung" + TEILER_APP9_DESCRIPTION_EN: "Project Pseudonymisation" + TEILER_APP9_DESCRIPTION_DE: "Projectpseudonymisierung" TEILER_APP9_SOURCEURL: "https://${HOST}/id-manager/html/projectSelection.html" TEILER_APP9_ROLES: "TEILER_PUBLIC" TEILER_APP9_ISACTIVATED: "true" @@ -161,8 +159,8 @@ services: TEILER_APP9_ISLOCAL: "true" TEILER_APP10_NAME: "federated-search" TEILER_APP10_TITLE: "Lens" - TEILER_APP10_EN_DESCRIPTION: "Federated Search" - TEILER_APP10_DE_DESCRIPTION: "Föderierte Suche" + TEILER_APP10_DESCRIPTION_EN: "Federated Search" + TEILER_APP10_DESCRIPTION_DE: "Föderierte Suche" TEILER_APP10_SOURCEURL: "https://demo.lens.samply.de/" TEILER_APP10_ROLES: "TEILER_PUBLIC" TEILER_APP10_ISACTIVATED: "true" @@ -171,10 +169,10 @@ services: TEILER_APP10_ISEXTERNALLINK: "true" TEILER_APP10_ISLOCAL: "false" TEILER_APP11_NAME: "central-patient-list" - TEILER_APP11_EN_TITLE: "Central Patient List" - TEILER_APP11_DE_TITLE: "Zentrale Mainzelliste" - TEILER_APP11_EN_DESCRIPTION: "Central Patient List" - TEILER_APP11_DE_DESCRIPTION: "Zentrale Patientenliste" + TEILER_APP11_TITLE_EN: "Central Patient List" + TEILER_APP11_TITLE_DE: "Zentrale Patientenliste" + TEILER_APP11_DESCRIPTION_EN: "Central Patient List" + TEILER_APP11_DESCRIPTION_DE: "Zentrale Patientenliste" TEILER_APP11_SOURCEURL: "https://patientlist.ccp-it.dktk.dkfz.de/" TEILER_APP11_ROLES: "TEILER_PUBLIC" TEILER_APP11_ISACTIVATED: "true" @@ -183,10 +181,10 @@ services: TEILER_APP11_ISEXTERNALLINK: "true" TEILER_APP11_ISLOCAL: "false" TEILER_APP12_NAME: "central id-manager" - TEILER_APP12_EN_TITLE: "Central Magic PL" - TEILER_APP12_DE_TITLE: "Zentraler Magic PL" - TEILER_APP12_EN_DESCRIPTION: "Central ID Manager" - TEILER_APP12_DE_DESCRIPTION: "Zentraler ID-Manager" + TEILER_APP12_TITLE_EN: "Central ID-Manager" + TEILER_APP12_TITLE_DE: "Zentraler ID-Manager" + TEILER_APP12_DESCRIPTION_EN: "Central ID Manager" + TEILER_APP12_DESCRIPTION_DE: "Zentraler ID-Manager" TEILER_APP12_SOURCEURL: "https://dktk-kne.kgu.de/" TEILER_APP12_ROLES: "TEILER_PUBLIC" TEILER_APP12_ISACTIVATED: "true" From 17f52a7907df21823b3efc993a09fd138398bb43 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 21 Feb 2023 15:41:51 +0100 Subject: [PATCH 015/243] Add Teiler Core --- ccp/modules/teiler-ui-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index a893639..57cfbb1 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -71,7 +71,7 @@ services: teiler-core: image: samply/teiler-core:developer - container_name: bridgehead-teiler + container_name: bridgehead-teiler-core volumes: - "/etc/bridgehead/ccp.conf:/app/ccp.conf" labels: From c9b1975c9e47b75bce8cffa8b1aa5cf17a05b3ab Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 15 Mar 2023 13:52:36 +0100 Subject: [PATCH 016/243] Tidy teiler and mtba volumes --- ccp/modules/mtba-compose.yml | 3 --- ccp/modules/teiler-ui-compose.yml | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 60845f5..2f4bde9 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -31,6 +31,3 @@ services: # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! # TODO: Find a trigger to let mtba signal a restart for CBioPortal - -volumes: - mtba-data: diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 57cfbb1..bf7889f 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -10,10 +10,9 @@ services: POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh POSTGRES_DB: "keycloak" volumes: - - "login-db:/var/lib/postgresql/data" + - "bridgehead-login-db:/var/lib/postgresql/data" login: - #image: "jboss/keycloak:16.1.1" image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest container_name: bridgehead-login environment: @@ -263,5 +262,5 @@ services: # TEILER_APP27_ISLOCAL: "true" volumes: - login-db: - name: "login-db" + bridgehead-login-db: + name: "bridgehead-login-db" From 9299a201a6b017f0fac7321f666ef5f909a89e20 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 15 Mar 2023 15:43:14 +0100 Subject: [PATCH 017/243] Deactivate traffik for mtba --- ccp/modules/mtba-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 2f4bde9..a59f109 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -19,11 +19,11 @@ services: FILE_CHARSET: ${MTBA_FILE_CHARSET} FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} CSV_DELIMITER: ${MTBA_CSV_DELIMITER} - labels: - - "traefik.enable=true" - - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" - - "traefik.http.services.mtba.loadbalancer.server.port=80" - - "traefik.http.routers.mtba.tls=true" +# labels: +# - "traefik.enable=true" +# - "traefik.http.routers.mtba.rule=PathPrefix(`/mtba`)" +# - "traefik.http.services.mtba.loadbalancer.server.port=80" +# - "traefik.http.routers.mtba.tls=true" volumes: - /tmp/bridgehead/mtba/input:/app/input - /tmp/bridgehead/mtba/persist:/app/persist From e2d109558d30ab0fc7535e250266b23979e0da4d Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 15 Mar 2023 16:14:20 +0100 Subject: [PATCH 018/243] Add forward strategy to teiler-core --- ccp/modules/teiler-ui-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index bf7889f..b44cc07 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -69,7 +69,8 @@ services: TEILER_PROJECT: "${PROJECT}" teiler-core: - image: samply/teiler-core:developer + #image: samply/teiler-core:developer + image: teiler-core container_name: bridgehead-teiler-core volumes: - "/etc/bridgehead/ccp.conf:/app/ccp.conf" @@ -81,6 +82,8 @@ services: environment: LOG_LEVEL: "INFO" APPLICATION_PORT: "8085" + APPLICATION_FORWARD_STRATEGY: "framework" + APPLICATION_ADDRESS: "${HOST}" DEFAULT_LANGUAGE: "DE" CONFIG_ENV_VAR_PATH: "/app/ccp.conf" TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" From 3e0bf38018ded23b2b54b2ea42ad0c3573d276af Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 15 Mar 2023 17:04:50 +0100 Subject: [PATCH 019/243] Add forward strategy to teiler-core --- ccp/modules/teiler-ui-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index b44cc07..6fa28f8 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -82,7 +82,7 @@ services: environment: LOG_LEVEL: "INFO" APPLICATION_PORT: "8085" - APPLICATION_FORWARD_STRATEGY: "framework" + APPLICATION_FORWARD_STRATEGY: "native" APPLICATION_ADDRESS: "${HOST}" DEFAULT_LANGUAGE: "DE" CONFIG_ENV_VAR_PATH: "/app/ccp.conf" From c794508880ab240a1fd3d4756a38b4e495448e95 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 16 Mar 2023 10:19:07 +0100 Subject: [PATCH 020/243] Add stripprefix to teiler-core --- ccp/modules/teiler-ui-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 6fa28f8..4fb5497 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -73,12 +73,14 @@ services: image: teiler-core container_name: bridgehead-teiler-core volumes: - - "/etc/bridgehead/ccp.conf:/app/ccp.conf" + - "/etc/bridgehead/ccp.conf:/app/ccp.conf:ro" labels: - "traefik.enable=true" - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" - "traefik.http.services.teiler_core_ccp.loadbalancer.server.port=8085" - "traefik.http.routers.teiler_core_ccp.tls=true" + - "traefik.http.middlewares.teiler_core_ccp_strip.stripprefix.prefixes=/ccp-teiler-core" + - "traefik.http.routers.teiler_core_ccp.middlewares=teiler_core_ccp_strip" environment: LOG_LEVEL: "INFO" APPLICATION_PORT: "8085" From 0a17bbc81f717a1a3cc3b45723d83854308cde71 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 16 Mar 2023 10:35:17 +0100 Subject: [PATCH 021/243] Add stripprefix to teiler-ui --- ccp/modules/teiler-ui-compose.yml | 69 +++++++++++++++++-------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 4fb5497..18b05ee 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -35,28 +35,32 @@ services: ############################################ Teiler-UI teiler-root-config: - image: samply/teiler-root-config:developer + #image: samply/teiler-root-config:developer + image: teiler-root-config container_name: bridgehead-teiler-root-config labels: - "traefik.enable=true" - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler`)" - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" - "traefik.http.routers.teiler_root_config_ccp.tls=true" + - "traefik.http.middlewares.teiler_root_config_ccp_strip.stripprefix.prefixes=/ccp-teiler" + - "traefik.http.routers.teiler_root_config_ccp.middlewares=teiler_root_config_ccp_strip" environment: TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" DEFAULT_LANGUAGE: "de" teiler-ui: - image: samply/teiler-ui:developer + #image: samply/teiler-ui:developer + image: teiler-ui container_name: bridgehead-teiler-ui - # ports: - # - 4200:80 labels: - "traefik.enable=true" - "traefik.http.routers.teiler_ui_ccp.rule=PathPrefix(`/ccp-teiler-ui`)" - "traefik.http.services.teiler_ui_ccp.loadbalancer.server.port=80" - "traefik.http.routers.teiler_ui_ccp.tls=true" + - "traefik.http.middlewares.teiler_ui_ccp_strip.stripprefix.prefixes=/ccp-teiler-ui" + - "traefik.http.routers.teiler_ui_ccp.middlewares=teiler_ui_ccp_strip" environment: DEFAULT_LANGUAGE: "DE" TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" @@ -67,10 +71,12 @@ services: TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" TEILER_PROJECT: "${PROJECT}" + EXPORTER_API_KEY: "${EXPORTER_API_KEY}" + TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" + HTTP_RELATIVE_PATH: "/ccp-teiler" teiler-core: - #image: samply/teiler-core:developer - image: teiler-core + image: samply/teiler-core:developer container_name: bridgehead-teiler-core volumes: - "/etc/bridgehead/ccp.conf:/app/ccp.conf:ro" @@ -84,7 +90,6 @@ services: environment: LOG_LEVEL: "INFO" APPLICATION_PORT: "8085" - APPLICATION_FORWARD_STRATEGY: "native" APPLICATION_ADDRESS: "${HOST}" DEFAULT_LANGUAGE: "DE" CONFIG_ENV_VAR_PATH: "/app/ccp.conf" @@ -96,9 +101,9 @@ services: # TEILER_APP2_NAME: "quality-report" TEILER_APP3_NAME: "keycloak" TEILER_APP3_TITLE: "Keycloak" - TEILER_APP4_DESCRIPTION_EN: "Authentication site" - TEILER_APP4_DESCRIPTION_DE: "Authentifizierungsseite" - TEILER_APP4_SOURCEURL: "https://${HOST}/login/auth/" + TEILER_APP4_EN_DESCRIPTION: "Authentication site" + TEILER_APP4_DE_DESCRIPTION: "Authentifizierungsseite" + TEILER_APP4_SOURCEURL: "https://${HOST}/login/" TEILER_APP4_ROLES: "TEILER_ADMIN" TEILER_APP4_ISACTIVATED: "true" TEILER_APP4_ICONSOURCEURL: "https://upload.wikimedia.org/wikipedia/commons/2/29/Keycloak_Logo.png" @@ -116,10 +121,10 @@ services: # TEILER_APP5_ISEXTERNALLINK: "true" # TEILER_APP5_ISLOCAL: "true" TEILER_APP6_NAME: "ldm" - TEILER_APP6_TITLE_EN: "Local data management" - TEILER_APP6_TITLE_DE: "Lokales Datenmanagement" - TEILER_APP6_DESCRIPTION_EN: "Local Data Management" - TEILER_APP6_DESCRIPTION_DE: "Lokales Datenmanagement" + TEILER_APP6_EN_TITLE: "Local data management" + TEILER_APP6_DE_TITLE: "Lokales Datenmanagement" + TEILER_APP6_EN_DESCRIPTION: "Local Data Management" + TEILER_APP6_DE_DESCRIPTION: "Lokales Datenmanagement" TEILER_APP6_SOURCEURL: "${CENTRAXX_URL}" TEILER_APP6_ROLES: "TEILER_PUBLIC" TEILER_APP6_ISACTIVATED: "${IS_DKTK_SITE}" @@ -138,10 +143,10 @@ services: TEILER_APP7_ISEXTERNALLINK: "true" TEILER_APP7_ISLOCAL: "true" TEILER_APP8_NAME: "patient-list" - TEILER_APP8_TITLE_EN: "Patient List" - TEILER_APP8_TITLE_DE: "Patientenliste" - TEILER_APP8_DESCRIPTION_EN: "Patient List" - TEILER_APP8_DESCRIPTION_DE: "Patientenliste" + TEILER_APP8_EN_TITLE: "Patient List" + TEILER_APP8_DE_TITLE: "Patientenliste" + TEILER_APP8_EN_DESCRIPTION: "Patient List" + TEILER_APP8_DE_DESCRIPTION: "Patientenliste" TEILER_APP8_SOURCEURL: "https://${HOST}/patientlist" TEILER_APP8_ROLES: "TEILER_PUBLIC" TEILER_APP8_ISACTIVATED: "true" @@ -150,10 +155,10 @@ services: TEILER_APP8_ISEXTERNALLINK: "true" TEILER_APP8_ISLOCAL: "true" TEILER_APP9_NAME: "project-pseudonymisation" - TEILER_APP9_TITLE_EN: "Project Pseudonymisation" - TEILER_APP9_TITLE_DE: "Projectpseudonymisierung" - TEILER_APP9_DESCRIPTION_EN: "Project Pseudonymisation" - TEILER_APP9_DESCRIPTION_DE: "Projectpseudonymisierung" + TEILER_APP9_EN_TITLE: "Project Pseudonymisation" + TEILER_APP9_DE_TITLE: "Projectpseudonymisierung" + TEILER_APP9_EN_DESCRIPTION: "Project Pseudonymisation" + TEILER_APP9_DE_DESCRIPTION: "Projectpseudonymisierung" TEILER_APP9_SOURCEURL: "https://${HOST}/id-manager/html/projectSelection.html" TEILER_APP9_ROLES: "TEILER_PUBLIC" TEILER_APP9_ISACTIVATED: "true" @@ -163,8 +168,8 @@ services: TEILER_APP9_ISLOCAL: "true" TEILER_APP10_NAME: "federated-search" TEILER_APP10_TITLE: "Lens" - TEILER_APP10_DESCRIPTION_EN: "Federated Search" - TEILER_APP10_DESCRIPTION_DE: "Föderierte Suche" + TEILER_APP10_EN_DESCRIPTION: "Federated Search" + TEILER_APP10_DE_DESCRIPTION: "Föderierte Suche" TEILER_APP10_SOURCEURL: "https://demo.lens.samply.de/" TEILER_APP10_ROLES: "TEILER_PUBLIC" TEILER_APP10_ISACTIVATED: "true" @@ -173,10 +178,10 @@ services: TEILER_APP10_ISEXTERNALLINK: "true" TEILER_APP10_ISLOCAL: "false" TEILER_APP11_NAME: "central-patient-list" - TEILER_APP11_TITLE_EN: "Central Patient List" - TEILER_APP11_TITLE_DE: "Zentrale Patientenliste" - TEILER_APP11_DESCRIPTION_EN: "Central Patient List" - TEILER_APP11_DESCRIPTION_DE: "Zentrale Patientenliste" + TEILER_APP11_EN_TITLE: "Central Patient List" + TEILER_APP11_DE_TITLE: "Zentrale Patientenliste" + TEILER_APP11_EN_DESCRIPTION: "Central Patient List" + TEILER_APP11_DE_DESCRIPTION: "Zentrale Patientenliste" TEILER_APP11_SOURCEURL: "https://patientlist.ccp-it.dktk.dkfz.de/" TEILER_APP11_ROLES: "TEILER_PUBLIC" TEILER_APP11_ISACTIVATED: "true" @@ -185,10 +190,10 @@ services: TEILER_APP11_ISEXTERNALLINK: "true" TEILER_APP11_ISLOCAL: "false" TEILER_APP12_NAME: "central id-manager" - TEILER_APP12_TITLE_EN: "Central ID-Manager" - TEILER_APP12_TITLE_DE: "Zentraler ID-Manager" - TEILER_APP12_DESCRIPTION_EN: "Central ID Manager" - TEILER_APP12_DESCRIPTION_DE: "Zentraler ID-Manager" + TEILER_APP12_EN_TITLE: "Central ID-Manager" + TEILER_APP12_DE_TITLE: "Zentraler ID-Manager" + TEILER_APP12_EN_DESCRIPTION: "Central ID Manager" + TEILER_APP12_DE_DESCRIPTION: "Zentraler ID-Manager" TEILER_APP12_SOURCEURL: "https://dktk-kne.kgu.de/" TEILER_APP12_ROLES: "TEILER_PUBLIC" TEILER_APP12_ISACTIVATED: "true" From 4bbd2a15fee9eec3534f555a1d0107e614ef25d7 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 22 Mar 2023 13:13:04 +0100 Subject: [PATCH 022/243] Change volume names for teiler components --- ccp/modules/exporter-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index dc2e340..2b71d69 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -22,7 +22,7 @@ services: - "traefik.http.routers.exporter_ccp.tls=true" - "traefik.http.routers.exporter_ccp.middlewares=auth" volumes: - - "exporter:/app/exporter-files" + - "bridgehead-exporter:/app/exporter-files" exporter-db: image: postgres:15.1-alpine @@ -32,11 +32,11 @@ services: POSTGRES_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh POSTGRES_DB: "exporter" volumes: - - "exporter-db:/var/lib/postgresql/data" + - "bridgehead-exporter-db:/var/lib/postgresql/data" volumes: - exporter-db: - name: "exporter-db" - exporter: - name: "exporter" + bridgehead-exporter-db: + name: "bridgehead-exporter-db" + bridgehead-exporter: + name: "bridgehead-exporter" From 3a91259a8a3ca227c161b2d00b7b26b1f201e2e7 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 22 Mar 2023 14:22:10 +0100 Subject: [PATCH 023/243] Add keycloak teiler app to teiler-ui --- ccp/modules/teiler-ui-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 18b05ee..c1bf18d 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -99,8 +99,8 @@ services: TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" TEILER_APP1_NAME: "config" # TEILER_APP2_NAME: "quality-report" - TEILER_APP3_NAME: "keycloak" - TEILER_APP3_TITLE: "Keycloak" + TEILER_APP4_NAME: "keycloak" + TEILER_APP4_TITLE: "Keycloak" TEILER_APP4_EN_DESCRIPTION: "Authentication site" TEILER_APP4_DE_DESCRIPTION: "Authentifizierungsseite" TEILER_APP4_SOURCEURL: "https://${HOST}/login/" From fe07c63f36e43cc0549e65bae7288b2f3f5634d1 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 22 Mar 2023 14:41:07 +0100 Subject: [PATCH 024/243] Adapt teiler-ui to traefik --- ccp/modules/exporter-compose.yml | 6 +- ccp/modules/login-compose.yml | 38 +++++ ccp/modules/login-setup.sh | 7 + ccp/modules/login.md | 0 ccp/modules/teiler-ui-compose.yml | 224 ++---------------------------- ccp/modules/teiler-ui-setup.sh | 1 - 6 files changed, 59 insertions(+), 217 deletions(-) create mode 100644 ccp/modules/login-compose.yml create mode 100644 ccp/modules/login-setup.sh create mode 100644 ccp/modules/login.md diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 2b71d69..04ae29d 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -15,12 +15,16 @@ services: TEMP_FILES_LIFETIME_IN_DAYS: "1" CLEAN_WRITE_FILES_CRON_EXPRESSION: "0 0 2 * * *" WRITE_FILES_LIFETIME_IN_DAYS: "30" + HTTP_RELATIVE_PATH: "/ccp-exporter" labels: - "traefik.enable=true" - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" - "traefik.http.services.exporter_ccp.loadbalancer.server.port=8092" - "traefik.http.routers.exporter_ccp.tls=true" - - "traefik.http.routers.exporter_ccp.middlewares=auth" + # TODO: Add basic auth in Teiler UI to access exporter? + #- "traefik.http.routers.exporter_ccp.middlewares=auth" + - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" + - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" volumes: - "bridgehead-exporter:/app/exporter-files" diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml new file mode 100644 index 0000000..1832bc4 --- /dev/null +++ b/ccp/modules/login-compose.yml @@ -0,0 +1,38 @@ +version: "3.7" + +services: + ############################################ Keycloak + login-db: + image: postgres:15.1-alpine + container_name: bridgehead-login-db + environment: + POSTGRES_USER: "keycloak" + POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + POSTGRES_DB: "keycloak" + volumes: + - "bridgehead-login-db:/var/lib/postgresql/data" + + login: + image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest + container_name: bridgehead-login + environment: + KEYCLOAK_ADMIN: "admin" + KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" + KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + KC_HOSTNAME_URL: "https://${HOST}/login" + KC_HOSTNAME_STRICT: "false" + KC_PROXY_ADDRESS_FORWARDING: "true" + TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler" + command: + - start-dev --import-realm --proxy edge --http-relative-path=/login + labels: + - "traefik.enable=true" + - "traefik.http.routers.login.rule=PathPrefix(`/login`)" + - "traefik.http.services.login.loadbalancer.server.port=8080" + - "traefik.http.routers.login.tls=true" + depends_on: + - login-db + +volumes: + bridgehead-login-db: + name: "bridgehead-login-db" diff --git a/ccp/modules/login-setup.sh b/ccp/modules/login-setup.sh new file mode 100644 index 0000000..2432951 --- /dev/null +++ b/ccp/modules/login-setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ "$ENABLE_LOGIN" == true ];then + log INFO "Login setup detected -- will start Login services." + OVERRIDE+=" -f ./$PROJECT/modules/login-compose.yml" +fi +KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" diff --git a/ccp/modules/login.md b/ccp/modules/login.md new file mode 100644 index 0000000..e69de29 diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index c1bf18d..5e280a9 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -1,42 +1,9 @@ version: "3.7" services: - ############################################ Keycloak - login-db: - image: "postgres:15.1" - container_name: bridgehead-login-db - environment: - POSTGRES_USER: "keycloak" - POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh - POSTGRES_DB: "keycloak" - volumes: - - "bridgehead-login-db:/var/lib/postgresql/data" - login: - image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest - container_name: bridgehead-login - environment: - KEYCLOAK_ADMIN: "admin" - KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" - KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh - KC_HOSTNAME_URL: "https://${HOST}/login" - KC_HOSTNAME_STRICT: "false" - KC_PROXY_ADDRESS_FORWARDING: "true" - TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler" - command: - - start-dev --import-realm --proxy edge --http-relative-path=/login - labels: - - "traefik.enable=true" - - "traefik.http.routers.login.rule=PathPrefix(`/login`)" - - "traefik.http.services.login.loadbalancer.server.port=8080" - - "traefik.http.routers.login.tls=true" - depends_on: - - login-db - - ############################################ Teiler-UI teiler-root-config: - #image: samply/teiler-root-config:developer - image: teiler-root-config + image: docker.verbis.dkfz.de/cache/samply/teiler-root-config:develop container_name: bridgehead-teiler-root-config labels: - "traefik.enable=true" @@ -49,10 +16,10 @@ services: TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" DEFAULT_LANGUAGE: "de" + HTTP_RELATIVE_PATH: "/ccp-teiler" teiler-ui: - #image: samply/teiler-ui:developer - image: teiler-ui + image: docker.verbis.dkfz.de/cache/samply/teiler-ui:develop container_name: bridgehead-teiler-ui labels: - "traefik.enable=true" @@ -73,10 +40,11 @@ services: TEILER_PROJECT: "${PROJECT}" EXPORTER_API_KEY: "${EXPORTER_API_KEY}" TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" - HTTP_RELATIVE_PATH: "/ccp-teiler" + TEILER_UI_HTTP_RELATIVE_PATH: "/ccp-teiler-ui" + TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" teiler-core: - image: samply/teiler-core:developer + image: docker.verbis.dkfz.de/ccp/dktk-teiler-core:latest container_name: bridgehead-teiler-core volumes: - "/etc/bridgehead/ccp.conf:/app/ccp.conf:ro" @@ -94,183 +62,9 @@ services: DEFAULT_LANGUAGE: "DE" CONFIG_ENV_VAR_PATH: "/app/ccp.conf" TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" + TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" - TEILER_APP1_NAME: "config" - # TEILER_APP2_NAME: "quality-report" - TEILER_APP4_NAME: "keycloak" - TEILER_APP4_TITLE: "Keycloak" - TEILER_APP4_EN_DESCRIPTION: "Authentication site" - TEILER_APP4_DE_DESCRIPTION: "Authentifizierungsseite" - TEILER_APP4_SOURCEURL: "https://${HOST}/login/" - TEILER_APP4_ROLES: "TEILER_ADMIN" - TEILER_APP4_ISACTIVATED: "true" - TEILER_APP4_ICONSOURCEURL: "https://upload.wikimedia.org/wikipedia/commons/2/29/Keycloak_Logo.png" - TEILER_APP4_ORDER: "5" - TEILER_APP4_ISEXTERNALLINK: "true" - TEILER_APP4_ISLOCAL: "true" - # TEILER_APP5_NAME: "pgadmin" - # TEILER_APP5_TITLE: "PgAdmin" - # TEILER_APP5_DESCRIPTION: "Database Management" - # TEILER_APP5_SOURCEURL: "http://localhost:5000" - # TEILER_APP5_ROLES: "TEILER_ADMIN" - # TEILER_APP5_ISACTIVATED: "true" - # TEILER_APP5_ICONSOURCEURL: "https://user-images.githubusercontent.com/24623425/36042969-f87531d4-0d8a-11e8-9dee-e87ab8c6a9e3.png" - # TEILER_APP5_ORDER: "6" - # TEILER_APP5_ISEXTERNALLINK: "true" - # TEILER_APP5_ISLOCAL: "true" - TEILER_APP6_NAME: "ldm" - TEILER_APP6_EN_TITLE: "Local data management" - TEILER_APP6_DE_TITLE: "Lokales Datenmanagement" - TEILER_APP6_EN_DESCRIPTION: "Local Data Management" - TEILER_APP6_DE_DESCRIPTION: "Lokales Datenmanagement" - TEILER_APP6_SOURCEURL: "${CENTRAXX_URL}" - TEILER_APP6_ROLES: "TEILER_PUBLIC" - TEILER_APP6_ISACTIVATED: "${IS_DKTK_SITE}" - TEILER_APP6_ICONCLASS: "bi bi-server" - TEILER_APP6_ORDER: "7" - TEILER_APP6_ISEXTERNALLINK: "true" - TEILER_APP6_ISLOCAL: "true" - TEILER_APP7_NAME: "id-manager" - TEILER_APP7_TITLE: "ID-Manager" - TEILER_APP7_DESCRIPTION: "ID Manager" - TEILER_APP7_SOURCEURL: "https://${HOST}/id-manager/index.html" - TEILER_APP7_ROLES: "TEILER_PUBLIC" - TEILER_APP7_ISACTIVATED: "true" - TEILER_APP7_ICONCLASS: "bi bi-person-bounding-box" - TEILER_APP7_ORDER: "8" - TEILER_APP7_ISEXTERNALLINK: "true" - TEILER_APP7_ISLOCAL: "true" - TEILER_APP8_NAME: "patient-list" - TEILER_APP8_EN_TITLE: "Patient List" - TEILER_APP8_DE_TITLE: "Patientenliste" - TEILER_APP8_EN_DESCRIPTION: "Patient List" - TEILER_APP8_DE_DESCRIPTION: "Patientenliste" - TEILER_APP8_SOURCEURL: "https://${HOST}/patientlist" - TEILER_APP8_ROLES: "TEILER_PUBLIC" - TEILER_APP8_ISACTIVATED: "true" - TEILER_APP8_ICONCLASS: "bi bi-person-rolodex" - TEILER_APP8_ORDER: "9" - TEILER_APP8_ISEXTERNALLINK: "true" - TEILER_APP8_ISLOCAL: "true" - TEILER_APP9_NAME: "project-pseudonymisation" - TEILER_APP9_EN_TITLE: "Project Pseudonymisation" - TEILER_APP9_DE_TITLE: "Projectpseudonymisierung" - TEILER_APP9_EN_DESCRIPTION: "Project Pseudonymisation" - TEILER_APP9_DE_DESCRIPTION: "Projectpseudonymisierung" - TEILER_APP9_SOURCEURL: "https://${HOST}/id-manager/html/projectSelection.html" - TEILER_APP9_ROLES: "TEILER_PUBLIC" - TEILER_APP9_ISACTIVATED: "true" - TEILER_APP9_ICONCLASS: "bi bi-person-lines-fill" - TEILER_APP9_ORDER: "10" - TEILER_APP9_ISEXTERNALLINK: "true" - TEILER_APP9_ISLOCAL: "true" - TEILER_APP10_NAME: "federated-search" - TEILER_APP10_TITLE: "Lens" - TEILER_APP10_EN_DESCRIPTION: "Federated Search" - TEILER_APP10_DE_DESCRIPTION: "Föderierte Suche" - TEILER_APP10_SOURCEURL: "https://demo.lens.samply.de/" - TEILER_APP10_ROLES: "TEILER_PUBLIC" - TEILER_APP10_ISACTIVATED: "true" - TEILER_APP10_ICONCLASS: "bi bi-search" - TEILER_APP10_ORDER: "13" - TEILER_APP10_ISEXTERNALLINK: "true" - TEILER_APP10_ISLOCAL: "false" - TEILER_APP11_NAME: "central-patient-list" - TEILER_APP11_EN_TITLE: "Central Patient List" - TEILER_APP11_DE_TITLE: "Zentrale Patientenliste" - TEILER_APP11_EN_DESCRIPTION: "Central Patient List" - TEILER_APP11_DE_DESCRIPTION: "Zentrale Patientenliste" - TEILER_APP11_SOURCEURL: "https://patientlist.ccp-it.dktk.dkfz.de/" - TEILER_APP11_ROLES: "TEILER_PUBLIC" - TEILER_APP11_ISACTIVATED: "true" - TEILER_APP11_ICONCLASS: "bi bi-person-rolodex" - TEILER_APP11_ORDER: "14" - TEILER_APP11_ISEXTERNALLINK: "true" - TEILER_APP11_ISLOCAL: "false" - TEILER_APP12_NAME: "central id-manager" - TEILER_APP12_EN_TITLE: "Central ID-Manager" - TEILER_APP12_DE_TITLE: "Zentraler ID-Manager" - TEILER_APP12_EN_DESCRIPTION: "Central ID Manager" - TEILER_APP12_DE_DESCRIPTION: "Zentraler ID-Manager" - TEILER_APP12_SOURCEURL: "https://dktk-kne.kgu.de/" - TEILER_APP12_ROLES: "TEILER_PUBLIC" - TEILER_APP12_ISACTIVATED: "true" - TEILER_APP12_ICONCLASS: "bi bi-person-bounding-box" - TEILER_APP12_ORDER: "15" - TEILER_APP12_ISEXTERNALLINK: "true" - TEILER_APP12_ISLOCAL: "false" - # TODO: Icinga to be replaced by Zabbix - TEILER_APP13_NAME: "monitoring" - TEILER_APP13_TITLE: "Icinga" - TEILER_APP13_DESCRIPTION: "Icinga Monitoring" - TEILER_APP13_SOURCEURL: "https://monitor.vmitro.de/icingaweb2/dashboard" - TEILER_APP13_ROLES: "TEILER_ADMIN" - TEILER_APP13_ISACTIVATED: "true" - TEILER_APP13_ICONSOURCEURL: "https://images.ctfassets.net/o7xu9whrs0u9/QmL67mCGdRQ8PBcuKHGnF/858c0aee95762f59d67b25073f9483c2/icinga-logo.png" - TEILER_APP13_ORDER: "16" - TEILER_APP13_ISEXTERNALLINK: "true" - TEILER_APP13_ISLOCAL: "false" - # TEILER_APP14_NAME: "function-tests" - # TEILER_APP15_NAME: "event-log" - TEILER_APP16_NAME: "active-inquiries" - TEILER_APP16_BACKENDURL: "https://${HOST}/ccp-exporter" - TEILER_APP17_NAME: "archived-inquiries" - TEILER_APP17_BACKENDURL: "https://${HOST}/ccp-exporter" - TEILER_APP18_NAME: "failed-inquiries" - TEILER_APP18_BACKENDURL: "https://${HOST}/ccp-exporter" - TEILER_APP19_NAME: "inquiry" - TEILER_APP19_INMENU: "false" - # TEILER_APP20_NAME: "cbioportal" - # TEILER_APP20_TITLE: "cBioportal" - # TEILER_APP20_DESCRIPTION: "Interactive exploration of multidimensional cancer genomics data sets" - # TEILER_APP20_SOURCEURL: "http://localhost:8082" - # TEILER_APP20_ROLES: "TEILER_USER" - # TEILER_APP20_ISACTIVATED: "true" - # TEILER_APP20_ICONSOURCEURL: "https://docs.cbioportal.org/images/cbio-logo.png" - # TEILER_APP20_ORDER: "17" - # TEILER_APP20_ISEXTERNALLINK: "true" - # TEILER_APP20_ISLOCAL: "true" - # TEILER_APP21_NAME: "mtba-bp" - # TEILER_APP21_TITLE: "MTBA-BP" - # TEILER_APP21_DESCRIPTION: "MTBA Camunda Business Process" - # TEILER_APP21_SOURCEURL: "http://localhost:8480" - # TEILER_APP21_ROLES: "TEILER_ADMIN" - # TEILER_APP21_ISACTIVATED: "true" - # TEILER_APP21_ICONSOURCEURL: "https://camunda.com/wp-content/uploads/2020/05/logo-camunda-black.svg" - # TEILER_APP21_ORDER: "18" - # TEILER_APP21_ISEXTERNALLINK: "true" - # TEILER_APP21_ISLOCAL: "true" - TEILER_APP22_NAME: "dialog-quali" - TEILER_APP22_INMENU: "false" - TEILER_APP23_NAME: "dialog-uploads" - TEILER_APP23_INMENU: "false" - TEILER_APP24_NAME: "inquiry-dialog" - TEILER_APP24_INMENU: "false" - TEILER_APP25_NAME: "dialog-tests" - TEILER_APP25_INMENU: "false" - # TEILER_APP26_NAME: "opal" - # TEILER_APP26_TITLE: "Opal" - # TEILER_APP26_DESCRIPTION: "Opal is OBiBa’s core database application for biobanks." - # TEILER_APP26_SOURCEURL: "http://localhost:8880" - # TEILER_APP26_ROLES: "TEILER_USER" - # TEILER_APP26_ISACTIVATED: "true" - # TEILER_APP26_ICONSOURCEURL: "https://www.obiba.org/assets/themes/bootstrap/img/obiba-logo-small.png" - # TEILER_APP26_ORDER: "19" - # TEILER_APP26_ISEXTERNALLINK: "true" - # TEILER_APP26_ISLOCAL: "true" - # TEILER_APP27_NAME: "rstudio" - # TEILER_APP27_TITLE: "R Studio" - # TEILER_APP27_DESCRIPTION: "RStudio is an integrated development environment (IDE) for R and Python." - # TEILER_APP27_SOURCEURL: "http://localhost:8787" - # TEILER_APP27_ROLES: "TEILER_USER" - # TEILER_APP27_ISACTIVATED: "true" - # TEILER_APP27_ICONSOURCEURL: "https://rstudio.com/wp-content/uploads/2018/10/RStudio-Logo-Flat.png" - # TEILER_APP27_ORDER: "20" - # TEILER_APP27_ISEXTERNALLINK: "true" - # TEILER_APP27_ISLOCAL: "true" - -volumes: - bridgehead-login-db: - name: "bridgehead-login-db" + CENTRAX_URL: "${CENTRAXX_URL}" + IS_DKTK_SITE: "${IS_DKTK_SITE}" diff --git a/ccp/modules/teiler-ui-setup.sh b/ccp/modules/teiler-ui-setup.sh index dc755ba..793abdd 100644 --- a/ccp/modules/teiler-ui-setup.sh +++ b/ccp/modules/teiler-ui-setup.sh @@ -4,4 +4,3 @@ if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler-UI setup detected -- will start Teiler-UI services." OVERRIDE+=" -f ./$PROJECT/modules/teiler-ui-compose.yml" fi -KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" From 0cfe1d36175d2363fa736e43d6d55aea73bfa863 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 11 Apr 2023 11:02:14 +0200 Subject: [PATCH 025/243] Change salt string for exporter and login --- ccp/modules/exporter-setup.sh | 2 +- ccp/modules/login-setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index 044116b..323d8a4 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -5,4 +5,4 @@ if [ -n "$ENABLE_EXPORTER" ];then OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" fi # TODO: Generate password in another way so that not all passwords are the same? -EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" diff --git a/ccp/modules/login-setup.sh b/ccp/modules/login-setup.sh index 2432951..f3d0122 100644 --- a/ccp/modules/login-setup.sh +++ b/ccp/modules/login-setup.sh @@ -4,4 +4,4 @@ if [ "$ENABLE_LOGIN" == true ];then log INFO "Login setup detected -- will start Login services." OVERRIDE+=" -f ./$PROJECT/modules/login-compose.yml" fi -KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Keycloak. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" From 32de51eefb5a5e39b688628bf1e6b1a45f82ee08 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 11 Apr 2023 11:24:36 +0200 Subject: [PATCH 026/243] Merge id-management-setup with main --- ccp/modules/id-management-setup.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 1b347e7..2b1bbf0 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -51,3 +51,18 @@ function legacyIdMapping() { normalized_string=$(applySpecialCases "$uppercase_string"); echo "$normalized_string" | tr -d ' ' } + +if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then + log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." + OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" + + # Auto Generate local Passwords + PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" + + # Transform Seeds Configuration to pass it to the Mainzelliste Container + PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')" + + # Ensure old ids are working !!! + export IDMANAGEMENT_FRIENDLY_ID=$(legacyIdMapping "$SITE_ID") +fi From 72255e6211dca591fd9d3d7cee78700bef13170e Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 11 Apr 2023 15:57:12 +0200 Subject: [PATCH 027/243] Bugfix: cross origins of exporter --- ccp/modules/exporter-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 04ae29d..3fb0a86 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -7,7 +7,7 @@ services: environment: LOG_LEVEL: "INFO" EXPORTER_API_KEY: "${EXPORTER_API_KEY}" - CROSS_ORIGINS: "https://${HOST}/ccp-teiler-root-config" + CROSS_ORIGINS: "https://${HOST}" EXPORTER_DB_USER: "exporter" EXPORTER_DB_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh EXPORTER_DB_URL: "jdbc:postgresql://exporter-db:5432/exporter" @@ -16,6 +16,7 @@ services: CLEAN_WRITE_FILES_CRON_EXPRESSION: "0 0 2 * * *" WRITE_FILES_LIFETIME_IN_DAYS: "30" HTTP_RELATIVE_PATH: "/ccp-exporter" + HTTP_SERVLET_REQUEST_SCHEME: "https" labels: - "traefik.enable=true" - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" From 0b1e0474d74a21a5ac68a9fbb16d558ed07b9815 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 12 Apr 2023 09:46:35 +0200 Subject: [PATCH 028/243] Add DataSHIELD --- ccp/modules/datashield-compose.yml | 73 ++++++++++++++++++++++++++++++ ccp/modules/datashield-setup.sh | 7 +++ ccp/modules/datashield.md | 0 3 files changed, 80 insertions(+) create mode 100644 ccp/modules/datashield-compose.yml create mode 100644 ccp/modules/datashield-setup.sh create mode 100644 ccp/modules/datashield.md diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml new file mode 100644 index 0000000..2e6ad00 --- /dev/null +++ b/ccp/modules/datashield-compose.yml @@ -0,0 +1,73 @@ +version: "3.7" + +services: + ############################################ DataSHIELD Client (Rocker R-Studio) + bridgehead-rstudio: + image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest + #TODO: Connect with Keycloak: https://rocker-project.org/images/versioned/rstudio.html + environment: + USER: "ruser" + PASSWORD: "${RSTUDIO_PASSWORD}" + labels: + - "traefik.enable=true" + - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)" + - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787" + - "traefik.http.routers.rstudio_ccp.tls=true" + - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" + - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" + volumes: + - "bridgehead-dev-rstudio-config:/home/rstudio/.config/rstudio" + - "bridgehead-dev-rstudio-workspace:/home/rstudio/workspace" + + ############################################ DataSHIELD Server (Opal) + bridgehead-opal: + image: obiba/opal:4.5 + labels: + - "traefik.enable=true" + - "traefik.http.routers.opal_ccp.rule=PathPrefix(`/opal`)" + - "traefik.http.services.opal_ccp.loadbalancer.server.port=8443" + - "traefik.http.routers.opal_ccp.tls=true" + - "traefik.http.middlewares.opal_ccp_strip.stripprefix.prefixes=/opal" + - "traefik.http.routers.opal_ccp.middlewares=opal_ccp_strip" + links: + - bridgehead-opal-rserver + - bridgehead-opal-mongo + - bridgehead-opal-db + environment: + JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" + # OPAL_ADMINISTRATOR_USER: "administrator" + OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMINISTRATOR_PASSWORD}" + MONGO_HOST: "bridgehead-opal-mongo" + MONGO_PORT: "27017" + POSTGRESDATA_HOST: "bridgehead-opal-db" + POSTGRESDATA_DATABASE: "opal" + POSTGRESDATA_USER: "opal" + POSTGRESDATA_PASSWORD: "${OPAL_DB_PASSWORD}" + ROCK_HOSTS: "bridgehead-opal-rserver:8085" + volumes: + - "bridgehead-dev-opal:/srv" + + bridgehead-opal-mongo: # IDs + image: mongo:4.2 # TODO: Update mongo:6.0.4 + + bridgehead-opal-db: # Data + image: postgres:15.1-alpine + environment: + POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" + POSTGRES_USER: "opal" + POSTGRES_DB: "opal" + volumes: + - "bridgehead-opal-db:/var/lib/postgresql/data" + + bridgehead-opal-rserver: + image: datashield/rock-base:6.2-R4.2 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 + +volumes: + bridgehead-dev-rstudio-config: + name: "bridgehead-dev-rstudio-config" + bridgehead-dev-rstudio-workspace: + name: "bridgehead-dev-rstudio-workspace" + bridgehead-opal-db: + name: "bridgehead-opal-db" + bridgehead-dev-opal: + name: "bridgehead-dev-opal" diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh new file mode 100644 index 0000000..0bafe6b --- /dev/null +++ b/ccp/modules/datashield-setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ "$ENABLE_DATASHIELD" == true ];then + log INFO "DataSHIELD setup detected -- will start DataSHIELD services." + OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" +fi +OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" diff --git a/ccp/modules/datashield.md b/ccp/modules/datashield.md new file mode 100644 index 0000000..e69de29 From dfde7c18ffc653d4110b554c46f61aeb9ac156c5 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 12 Apr 2023 15:51:30 +0200 Subject: [PATCH 029/243] Experiment --- ccp/modules/datashield-compose.yml | 67 +++++++++++++++++++----------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 2e6ad00..9ba14ee 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -2,12 +2,14 @@ version: "3.7" services: ############################################ DataSHIELD Client (Rocker R-Studio) - bridgehead-rstudio: + rstudio: + container_name: bridgehead-rstudio image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest #TODO: Connect with Keycloak: https://rocker-project.org/images/versioned/rstudio.html environment: USER: "ruser" PASSWORD: "${RSTUDIO_PASSWORD}" + HTTP_RELATIVE_PATH: "/rstudio" labels: - "traefik.enable=true" - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)" @@ -16,41 +18,57 @@ services: - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" volumes: - - "bridgehead-dev-rstudio-config:/home/rstudio/.config/rstudio" - - "bridgehead-dev-rstudio-workspace:/home/rstudio/workspace" + #- "bridgehead-rstudio-config:/home/rstudio/.config/rstudio" + - "bridgehead-rstudio-workspace:/home/rstudio/workspace" ############################################ DataSHIELD Server (Opal) - bridgehead-opal: + opal: + container_name: bridgehead-opal image: obiba/opal:4.5 labels: - "traefik.enable=true" +# - "traefik.http.routers.opal_ccp.service=opal_ccp" - "traefik.http.routers.opal_ccp.rule=PathPrefix(`/opal`)" - - "traefik.http.services.opal_ccp.loadbalancer.server.port=8443" + - "traefik.http.services.opal_ccp.loadbalancer.server.port=8080" - "traefik.http.routers.opal_ccp.tls=true" - - "traefik.http.middlewares.opal_ccp_strip.stripprefix.prefixes=/opal" - - "traefik.http.routers.opal_ccp.middlewares=opal_ccp_strip" +# - "traefik.http.routers.opal_ccp.middlewares=opal_ccp_strip" +# - "traefik.http.middlewares.opal_ccp_strip.stripprefix.prefixes=/opal" + +# - "traefik.http.routers.opal_ccp2.service=opal_ccp" +# - "traefik.http.routers.opal_ccp2.rule=PathPrefix(`/ws`)" +# - "traefik.http.routers.opal_ccp2.tls=true" +# - "traefik.http.routers.opal_ccp2.middlewares=opal_ccp2_strip" +# - "traefik.http.middlewares.opal_ccp2_strip.stripprefix.prefixes=/ws" +# - "traefik.http.routers.opal_ccp2.middlewares=opal_ccp2_add" +# - "traefik.http.middlewares.opal_ccp2_add.addprefix.prefix=/opal/ws" + links: - - bridgehead-opal-rserver - - bridgehead-opal-mongo - - bridgehead-opal-db + - opal-rserver + - opal-mongo + - opal-db environment: JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" # OPAL_ADMINISTRATOR_USER: "administrator" OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMINISTRATOR_PASSWORD}" - MONGO_HOST: "bridgehead-opal-mongo" + MONGO_HOST: "opal-mongo" MONGO_PORT: "27017" - POSTGRESDATA_HOST: "bridgehead-opal-db" + POSTGRESDATA_HOST: "opal-db" POSTGRESDATA_DATABASE: "opal" POSTGRESDATA_USER: "opal" POSTGRESDATA_PASSWORD: "${OPAL_DB_PASSWORD}" - ROCK_HOSTS: "bridgehead-opal-rserver:8085" - volumes: - - "bridgehead-dev-opal:/srv" + ROCK_HOSTS: "opal-rserver:8085" +# APP_URL: "https://${HOST}/opal" +# APP_CONTEXT_PATH: "/opal" - bridgehead-opal-mongo: # IDs +# volumes: +# - "bridgehead-opal:/srv" + + opal-mongo: # IDs + container_name: bridgehead-opal-mongo image: mongo:4.2 # TODO: Update mongo:6.0.4 - bridgehead-opal-db: # Data + opal-db: # Data + container_name: bridgehead-opal-db image: postgres:15.1-alpine environment: POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" @@ -59,15 +77,16 @@ services: volumes: - "bridgehead-opal-db:/var/lib/postgresql/data" - bridgehead-opal-rserver: + opal-rserver: + container_name: bridgehead-opal-rserver image: datashield/rock-base:6.2-R4.2 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 volumes: - bridgehead-dev-rstudio-config: - name: "bridgehead-dev-rstudio-config" - bridgehead-dev-rstudio-workspace: - name: "bridgehead-dev-rstudio-workspace" +# bridgehead-rstudio-config: +# name: "bridgehead-rstudio-config" + bridgehead-rstudio-workspace: + name: "bridgehead-rstudio-workspace" bridgehead-opal-db: name: "bridgehead-opal-db" - bridgehead-dev-opal: - name: "bridgehead-dev-opal" + bridgehead-opal: + name: "bridgehead-opal" From bedc2ca6d01c9425ff8c95c75c8cd543d2cc3d2e Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 25 Apr 2023 11:10:35 +0200 Subject: [PATCH 030/243] Add beam connect to docekr-compose --- lib/beam-connect/central.json | 5 +++++ lib/beam-connect/local.json | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 lib/beam-connect/central.json create mode 100644 lib/beam-connect/local.json diff --git a/lib/beam-connect/central.json b/lib/beam-connect/central.json new file mode 100644 index 0000000..f9dc6c7 --- /dev/null +++ b/lib/beam-connect/central.json @@ -0,0 +1,5 @@ +{ + "sites": [ + // TODO: Add remote sites e.g: https://github.com/samply/beam-connect/blob/main/examples/example_central_test.json + ] +} diff --git a/lib/beam-connect/local.json b/lib/beam-connect/local.json new file mode 100644 index 0000000..4b59829 --- /dev/null +++ b/lib/beam-connect/local.json @@ -0,0 +1,3 @@ +[ + // TODO: Add local config e.g: https://github.com/samply/beam-connect/blob/main/examples/example_local_test.json +] From 68782d1c325b6117ef9636627c8751e3136e153d Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 25 Apr 2023 10:37:38 +0200 Subject: [PATCH 031/243] Experiment --- ccp/modules/exporter-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 3fb0a86..b15bf10 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -2,7 +2,8 @@ version: "3.7" services: exporter: - image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest + #image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest + image: dktk-exporter container_name: bridgehead-ccp-exporter environment: LOG_LEVEL: "INFO" From 325ae1d5741f6d512b4d086b579f0c67d9461cd5 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 25 Apr 2023 13:58:04 +0200 Subject: [PATCH 032/243] beam connect and move beam-connect config --- ccp/modules/datashield-compose.yml | 19 +++++++++++++++++++ ccp/modules/exporter-compose.yml | 4 ++-- lib/beam-connect/central.json | 5 ----- lib/beam-connect/local.json | 3 --- 4 files changed, 21 insertions(+), 10 deletions(-) delete mode 100644 lib/beam-connect/central.json delete mode 100644 lib/beam-connect/local.json diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 9ba14ee..d3e63f2 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -81,6 +81,25 @@ services: container_name: bridgehead-opal-rserver image: datashield/rock-base:6.2-R4.2 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 + beam-connect: + image: docker.verbis.dkfz.de/cache/samply/beam-connect:develop + container_name: bridgehead-datashield-connect + environment: + PROXY_URL: "http://beam-proxy:8081" + APP_ID: datashield-connect + PROXY_APIKEY: ${DATASHIELD_CONNECT_SECRET} + DISCOVERY_URL: "./map/central.json" + LOCAL_TARGETS_FILE: "./map/local.json" + depends_on: + - "beam-proxy" + volumes: + - /etc/bridgehead/datashield-connect/:/map:ro + + beam-proxy: + environment: + APP_2_ID: datashield-connect + APP_2_KEY: ${DATASHIELD_CONNECT_SECRET} + volumes: # bridgehead-rstudio-config: # name: "bridgehead-rstudio-config" diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index b15bf10..a4881d2 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -2,8 +2,8 @@ version: "3.7" services: exporter: - #image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest - image: dktk-exporter + image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest + #image: dktk-exporter container_name: bridgehead-ccp-exporter environment: LOG_LEVEL: "INFO" diff --git a/lib/beam-connect/central.json b/lib/beam-connect/central.json deleted file mode 100644 index f9dc6c7..0000000 --- a/lib/beam-connect/central.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sites": [ - // TODO: Add remote sites e.g: https://github.com/samply/beam-connect/blob/main/examples/example_central_test.json - ] -} diff --git a/lib/beam-connect/local.json b/lib/beam-connect/local.json deleted file mode 100644 index 4b59829..0000000 --- a/lib/beam-connect/local.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - // TODO: Add local config e.g: https://github.com/samply/beam-connect/blob/main/examples/example_local_test.json -] From b5ce1888422b11a116771019a012514817bd9784 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 25 Apr 2023 14:12:58 +0200 Subject: [PATCH 033/243] Fix beam connect app id --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index d3e63f2..8469fc3 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -86,7 +86,7 @@ services: container_name: bridgehead-datashield-connect environment: PROXY_URL: "http://beam-proxy:8081" - APP_ID: datashield-connect + APP_ID: datashield-connect.${SITE_ID}.${BROKER_ID} PROXY_APIKEY: ${DATASHIELD_CONNECT_SECRET} DISCOVERY_URL: "./map/central.json" LOCAL_TARGETS_FILE: "./map/local.json" From 996f53a16458f4ad84b061bedff241af9df991c3 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 25 Apr 2023 16:30:55 +0200 Subject: [PATCH 034/243] expose beam connect ports --- ccp/modules/datashield-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 8469fc3..0b43a1d 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -84,6 +84,8 @@ services: beam-connect: image: docker.verbis.dkfz.de/cache/samply/beam-connect:develop container_name: bridgehead-datashield-connect + ports: + - 8062:8062 environment: PROXY_URL: "http://beam-proxy:8081" APP_ID: datashield-connect.${SITE_ID}.${BROKER_ID} From 43ab59563cf1a22f95cd2ac6dc3d5a6b1b90b279 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 25 Apr 2023 17:29:05 +0200 Subject: [PATCH 035/243] Add Opal Password in Exporter --- ccp/modules/exporter-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index a4881d2..e4e226b 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -18,6 +18,7 @@ services: WRITE_FILES_LIFETIME_IN_DAYS: "30" HTTP_RELATIVE_PATH: "/ccp-exporter" HTTP_SERVLET_REQUEST_SCHEME: "https" + OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMINISTRATOR_PASSWORD}" labels: - "traefik.enable=true" - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" From 04cf5128b0938f057b89939e12af5a663ffadec1 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 26 Apr 2023 16:34:15 +0200 Subject: [PATCH 036/243] Remove mongo db --- ccp/modules/datashield-compose.yml | 34 ++--- ccp/modules/datashield-import-template.xml | 157 +++++++++++++++++++++ ccp/modules/exporter-compose.yml | 1 - 3 files changed, 165 insertions(+), 27 deletions(-) create mode 100644 ccp/modules/datashield-import-template.xml diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 0b43a1d..555a4f8 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -17,9 +17,9 @@ services: - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" - volumes: + #volumes: #- "bridgehead-rstudio-config:/home/rstudio/.config/rstudio" - - "bridgehead-rstudio-workspace:/home/rstudio/workspace" + #- "bridgehead-rstudio-workspace:/home/rstudio/workspace" ############################################ DataSHIELD Server (Opal) opal: @@ -27,45 +27,27 @@ services: image: obiba/opal:4.5 labels: - "traefik.enable=true" -# - "traefik.http.routers.opal_ccp.service=opal_ccp" - "traefik.http.routers.opal_ccp.rule=PathPrefix(`/opal`)" - "traefik.http.services.opal_ccp.loadbalancer.server.port=8080" - "traefik.http.routers.opal_ccp.tls=true" -# - "traefik.http.routers.opal_ccp.middlewares=opal_ccp_strip" -# - "traefik.http.middlewares.opal_ccp_strip.stripprefix.prefixes=/opal" - -# - "traefik.http.routers.opal_ccp2.service=opal_ccp" -# - "traefik.http.routers.opal_ccp2.rule=PathPrefix(`/ws`)" -# - "traefik.http.routers.opal_ccp2.tls=true" -# - "traefik.http.routers.opal_ccp2.middlewares=opal_ccp2_strip" -# - "traefik.http.middlewares.opal_ccp2_strip.stripprefix.prefixes=/ws" -# - "traefik.http.routers.opal_ccp2.middlewares=opal_ccp2_add" -# - "traefik.http.middlewares.opal_ccp2_add.addprefix.prefix=/opal/ws" - links: - opal-rserver - - opal-mongo - opal-db environment: JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" # OPAL_ADMINISTRATOR_USER: "administrator" OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMINISTRATOR_PASSWORD}" - MONGO_HOST: "opal-mongo" - MONGO_PORT: "27017" POSTGRESDATA_HOST: "opal-db" POSTGRESDATA_DATABASE: "opal" POSTGRESDATA_USER: "opal" POSTGRESDATA_PASSWORD: "${OPAL_DB_PASSWORD}" ROCK_HOSTS: "opal-rserver:8085" -# APP_URL: "https://${HOST}/opal" -# APP_CONTEXT_PATH: "/opal" + APP_URL: "https://${HOST}/opal" + APP_CONTEXT_PATH: "/opal" # volumes: # - "bridgehead-opal:/srv" - opal-mongo: # IDs - container_name: bridgehead-opal-mongo - image: mongo:4.2 # TODO: Update mongo:6.0.4 opal-db: # Data container_name: bridgehead-opal-db @@ -105,9 +87,9 @@ services: volumes: # bridgehead-rstudio-config: # name: "bridgehead-rstudio-config" - bridgehead-rstudio-workspace: - name: "bridgehead-rstudio-workspace" +# bridgehead-rstudio-workspace: +# name: "bridgehead-rstudio-workspace" bridgehead-opal-db: name: "bridgehead-opal-db" - bridgehead-opal: - name: "bridgehead-opal" +# bridgehead-opal: +# name: "bridgehead-opal" diff --git a/ccp/modules/datashield-import-template.xml b/ccp/modules/datashield-import-template.xml new file mode 100644 index 0000000..1de9c91 --- /dev/null +++ b/ccp/modules/datashield-import-template.xml @@ -0,0 +1,157 @@ + diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index e4e226b..6d48531 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -3,7 +3,6 @@ version: "3.7" services: exporter: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest - #image: dktk-exporter container_name: bridgehead-ccp-exporter environment: LOG_LEVEL: "INFO" From 5e376b17ad503efa642a2f8cb2fed2ee2cd2f5c5 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 27 Apr 2023 10:52:25 +0200 Subject: [PATCH 037/243] Remove unnecessary volumes --- ccp/modules/datashield-compose.yml | 20 ++++++++++---------- ccp/modules/exporter-compose.yml | 11 +++++------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 555a4f8..8504750 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -17,7 +17,7 @@ services: - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" - #volumes: + #volumes: #- "bridgehead-rstudio-config:/home/rstudio/.config/rstudio" #- "bridgehead-rstudio-workspace:/home/rstudio/workspace" @@ -35,7 +35,7 @@ services: - opal-db environment: JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" - # OPAL_ADMINISTRATOR_USER: "administrator" + # OPAL_ADMINISTRATOR_USER: "administrator" # Please, don't remove this line. It is informative. OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMINISTRATOR_PASSWORD}" POSTGRESDATA_HOST: "opal-db" POSTGRESDATA_DATABASE: "opal" @@ -45,8 +45,8 @@ services: APP_URL: "https://${HOST}/opal" APP_CONTEXT_PATH: "/opal" -# volumes: -# - "bridgehead-opal:/srv" + # volumes: + # - "bridgehead-opal:/srv" opal-db: # Data @@ -56,8 +56,8 @@ services: POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" POSTGRES_USER: "opal" POSTGRES_DB: "opal" - volumes: - - "bridgehead-opal-db:/var/lib/postgresql/data" + # volumes: + # - "bridgehead-opal-db:/var/lib/postgresql/data" opal-rserver: container_name: bridgehead-opal-rserver @@ -75,7 +75,7 @@ services: DISCOVERY_URL: "./map/central.json" LOCAL_TARGETS_FILE: "./map/local.json" depends_on: - - "beam-proxy" + - beam-proxy volumes: - /etc/bridgehead/datashield-connect/:/map:ro @@ -84,12 +84,12 @@ services: APP_2_ID: datashield-connect APP_2_KEY: ${DATASHIELD_CONNECT_SECRET} -volumes: +#volumes: # bridgehead-rstudio-config: # name: "bridgehead-rstudio-config" # bridgehead-rstudio-workspace: # name: "bridgehead-rstudio-workspace" - bridgehead-opal-db: - name: "bridgehead-opal-db" +# bridgehead-opal-db: +# name: "bridgehead-opal-db" # bridgehead-opal: # name: "bridgehead-opal" diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 6d48531..5a5597a 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -23,12 +23,10 @@ services: - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" - "traefik.http.services.exporter_ccp.loadbalancer.server.port=8092" - "traefik.http.routers.exporter_ccp.tls=true" - # TODO: Add basic auth in Teiler UI to access exporter? - #- "traefik.http.routers.exporter_ccp.middlewares=auth" - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" - volumes: - - "bridgehead-exporter:/app/exporter-files" +# volumes: +# - "bridgehead-exporter:/app/exporter-files" exporter-db: image: postgres:15.1-alpine @@ -38,11 +36,12 @@ services: POSTGRES_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh POSTGRES_DB: "exporter" volumes: + # TODO: Move to logging component in bridgehead - "bridgehead-exporter-db:/var/lib/postgresql/data" volumes: bridgehead-exporter-db: name: "bridgehead-exporter-db" - bridgehead-exporter: - name: "bridgehead-exporter" +# bridgehead-exporter: +# name: "bridgehead-exporter" From 687dbba3836636dd3b510a37b50daa7e619598eb Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 16 May 2023 16:40:22 +0200 Subject: [PATCH 038/243] Add opal certificate --- ccp/modules/datashield-compose.yml | 13 ++++++++++++- ccp/modules/datashield-setup.sh | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 8504750..e9e49d3 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -24,7 +24,7 @@ services: ############################################ DataSHIELD Server (Opal) opal: container_name: bridgehead-opal - image: obiba/opal:4.5 + image: docker.verbis.dkfz.de/ccp/dktk-opal:latest labels: - "traefik.enable=true" - "traefik.http.routers.opal_ccp.rule=PathPrefix(`/opal`)" @@ -44,6 +44,11 @@ services: ROCK_HOSTS: "opal-rserver:8085" APP_URL: "https://${HOST}/opal" APP_CONTEXT_PATH: "/opal" + OPAL_PRIVATE_KEY: "/run/secrets/opal-key.pem" + OPAL_CERTIFICATE: "/run/secrets/opal-cert.pem" + secrets: + - opal-cert.pem + - opal-key.pem # volumes: # - "bridgehead-opal:/srv" @@ -93,3 +98,9 @@ services: # name: "bridgehead-opal-db" # bridgehead-opal: # name: "bridgehead-opal" + +secrets: + opal-cert.pem: + file: /etc/bridgehead/traefik-tls/opal-cert.pem + opal-key.pem: + file: /etc/bridgehead/traefik-tls/opal-key.pem diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 0bafe6b..3573c8c 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -5,3 +5,6 @@ if [ "$ENABLE_DATASHIELD" == true ];then OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" fi OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" +if [ ! -e "/etc/bridgehead/traefik-tls/opal-cert.pem" ]; then + openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/bridgehead/traefik-tls/opal-key.pem -out /etc/bridgehead/traefik-tls/opal-cert.pem -days 3650 -subj "/CN=$HOST" +fi From e9e1ce5a6589622936e3fdd391de2e8b1f690543 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 16 May 2023 16:43:22 +0200 Subject: [PATCH 039/243] ccp.conf in teiler-core as secret --- ccp/modules/teiler-ui-compose.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 5e280a9..5a51c8e 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -46,8 +46,6 @@ services: teiler-core: image: docker.verbis.dkfz.de/ccp/dktk-teiler-core:latest container_name: bridgehead-teiler-core - volumes: - - "/etc/bridgehead/ccp.conf:/app/ccp.conf:ro" labels: - "traefik.enable=true" - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" @@ -60,7 +58,7 @@ services: APPLICATION_PORT: "8085" APPLICATION_ADDRESS: "${HOST}" DEFAULT_LANGUAGE: "DE" - CONFIG_ENV_VAR_PATH: "/app/ccp.conf" + CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" @@ -68,3 +66,9 @@ services: TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" CENTRAX_URL: "${CENTRAXX_URL}" IS_DKTK_SITE: "${IS_DKTK_SITE}" + secrets: + - ccp.conf + +secrets: + ccp.conf: + file: /etc/bridgehead/ccp.conf From f2f48869afc80d315d4afe0bfd56c92c8c880341 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 24 May 2023 08:50:37 +0000 Subject: [PATCH 040/243] Change cert permission and location --- ccp/modules/datashield-setup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 3573c8c..15c0359 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -5,6 +5,9 @@ if [ "$ENABLE_DATASHIELD" == true ];then OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" fi OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" -if [ ! -e "/etc/bridgehead/traefik-tls/opal-cert.pem" ]; then - openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/bridgehead/traefik-tls/opal-key.pem -out /etc/bridgehead/traefik-tls/opal-cert.pem -days 3650 -subj "/CN=$HOST" +if [ ! -e "/etc/bridgehead/trusted-ca-certs/opal-cert.pem" ]; then + openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/bridgehead/trusted-ca-certs/opal-key.pem -out /etc/bridgehead/trusted-ca-certs/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" + chmod g+r /etc/bridgehead/trusted-ca-certs/opal-key.pem + chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-key.pem + chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-cert.pem fi From 6438fc5f4e5f57c1217540eb0ff93b4153670e50 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 24 May 2023 08:51:12 +0000 Subject: [PATCH 041/243] Change beam-connect version and load opal cert --- ccp/modules/datashield-compose.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index e9e49d3..045a218 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -69,16 +69,19 @@ services: image: datashield/rock-base:6.2-R4.2 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 beam-connect: - image: docker.verbis.dkfz.de/cache/samply/beam-connect:develop + image: docker.verbis.dkfz.de/cache/samply/beam-connect:fix-connect container_name: bridgehead-datashield-connect ports: - 8062:8062 environment: PROXY_URL: "http://beam-proxy:8081" + TLS_CA_CERTIFICATES_DIR: /run/secrets APP_ID: datashield-connect.${SITE_ID}.${BROKER_ID} PROXY_APIKEY: ${DATASHIELD_CONNECT_SECRET} DISCOVERY_URL: "./map/central.json" LOCAL_TARGETS_FILE: "./map/local.json" + secrets: + - opal-cert.pem depends_on: - beam-proxy volumes: @@ -101,6 +104,7 @@ services: secrets: opal-cert.pem: - file: /etc/bridgehead/traefik-tls/opal-cert.pem + file: /etc/bridgehead/trusted-ca-certs/opal-cert.pem:ro opal-key.pem: - file: /etc/bridgehead/traefik-tls/opal-key.pem + file: /etc/bridgehead/trusted-ca-certs/opal-key.pem:ro + From bc239c0b02c38989ca381ef4f6007e18807fd635 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 24 May 2023 09:03:46 +0000 Subject: [PATCH 042/243] change to dockerhub image --- ccp/modules/datashield-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 045a218..0c1f8d4 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -69,7 +69,9 @@ services: image: datashield/rock-base:6.2-R4.2 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 beam-connect: - image: docker.verbis.dkfz.de/cache/samply/beam-connect:fix-connect + # We want to switch to this image if the changes from fix-connect are merged + # image: docker.verbis.dkfz.de/cache/samply/beam-connect:no-auth + image: samply/beam-connect:fix-connect container_name: bridgehead-datashield-connect ports: - 8062:8062 From 276f886546821393ff75d16474ca7a1e4923b1fa Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 24 May 2023 09:08:40 +0000 Subject: [PATCH 043/243] secrets are readonly by default --- ccp/modules/datashield-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 0c1f8d4..b6656af 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -106,7 +106,7 @@ services: secrets: opal-cert.pem: - file: /etc/bridgehead/trusted-ca-certs/opal-cert.pem:ro + file: /etc/bridgehead/trusted-ca-certs/opal-cert.pem opal-key.pem: - file: /etc/bridgehead/trusted-ca-certs/opal-key.pem:ro + file: /etc/bridgehead/trusted-ca-certs/opal-key.pem From 20c65336e6fc4c5aef0695a6e6426887591af036 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 25 May 2023 09:51:30 +0000 Subject: [PATCH 044/243] Switch to `no-auth` branch of beam-connect --- ccp/modules/datashield-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index b6656af..7518c6f 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -69,9 +69,7 @@ services: image: datashield/rock-base:6.2-R4.2 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 beam-connect: - # We want to switch to this image if the changes from fix-connect are merged - # image: docker.verbis.dkfz.de/cache/samply/beam-connect:no-auth - image: samply/beam-connect:fix-connect + image: docker.verbis.dkfz.de/cache/samply/beam-connect:no-auth container_name: bridgehead-datashield-connect ports: - 8062:8062 From 2d7d1d73b349474813cbc3f71e910e5d52cb7dc6 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 5 Jul 2023 13:14:25 +0200 Subject: [PATCH 045/243] Add reporter --- ccp/modules/exporter-compose.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 5a5597a..16ac2ff 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -39,6 +39,22 @@ services: # TODO: Move to logging component in bridgehead - "bridgehead-exporter-db:/var/lib/postgresql/data" + reporter: + image: docker.verbis.dkfz.de/ccp/dktk-reporter:latest + container_name: bridgehead-ccp-reporter + environment: + LOG_LEVEL: "INFO" + CROSS_ORIGINS: "https://${HOST}" + HTTP_RELATIVE_PATH: "/ccp-reporter" + EXPORTER_API_KEY: "${EXPORTER_API_KEY}" + EXPORTER_URL: "http://exporter:8092" + labels: + - "traefik.enable=true" + - "traefik.http.routers.reporter_ccp.rule=PathPrefix(`/ccp-reporter`)" + - "traefik.http.services.reporter_ccp.loadbalancer.server.port=8095" + - "traefik.http.routers.reporter_ccp.tls=true" + - "traefik.http.middlewares.reporter_ccp_strip.stripprefix.prefixes=/ccp-reporter" + - "traefik.http.routers.reporter_ccp.middlewares=reporter_ccp_strip" volumes: bridgehead-exporter-db: From 5148e3382d7a5c17b0f0af7636a9881be29d454e Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 6 Jul 2023 17:44:35 +0200 Subject: [PATCH 046/243] Add parameter LOG_FHIR_VALIDATION to exporter --- ccp/modules/exporter-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 16ac2ff..a64d5d4 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -48,6 +48,7 @@ services: HTTP_RELATIVE_PATH: "/ccp-reporter" EXPORTER_API_KEY: "${EXPORTER_API_KEY}" EXPORTER_URL: "http://exporter:8092" + LOG_FHIR_VALIDATION: "false" labels: - "traefik.enable=true" - "traefik.http.routers.reporter_ccp.rule=PathPrefix(`/ccp-reporter`)" From 50360d3f41a8c2b6a0cd6403a8f6a2145576e424 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 7 Jul 2023 09:52:35 +0200 Subject: [PATCH 047/243] update new broker --- ccp/modules/exliquid-compose.yml | 0 ccp/modules/exliquid-setup.sh | 0 ccp/modules/exporter-compose.yml | 4 ++++ ccp/modules/id-management-setup.sh | 15 --------------- ccp/modules/mtba-compose.yml | 13 ++++++++----- 5 files changed, 12 insertions(+), 20 deletions(-) delete mode 100644 ccp/modules/exliquid-compose.yml delete mode 100644 ccp/modules/exliquid-setup.sh diff --git a/ccp/modules/exliquid-compose.yml b/ccp/modules/exliquid-compose.yml deleted file mode 100644 index e69de29..0000000 diff --git a/ccp/modules/exliquid-setup.sh b/ccp/modules/exliquid-setup.sh deleted file mode 100644 index e69de29..0000000 diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index a64d5d4..5670142 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -49,6 +49,8 @@ services: EXPORTER_API_KEY: "${EXPORTER_API_KEY}" EXPORTER_URL: "http://exporter:8092" LOG_FHIR_VALIDATION: "false" +# volumes: +# - "bridgehead-reporter:/app/reports" labels: - "traefik.enable=true" - "traefik.http.routers.reporter_ccp.rule=PathPrefix(`/ccp-reporter`)" @@ -62,3 +64,5 @@ volumes: name: "bridgehead-exporter-db" # bridgehead-exporter: # name: "bridgehead-exporter" +# bridgehead-reporter: +# name: "bridgehead-reporter" diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 2b1bbf0..1b347e7 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -51,18 +51,3 @@ function legacyIdMapping() { normalized_string=$(applySpecialCases "$uppercase_string"); echo "$normalized_string" | tr -d ' ' } - -if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then - log INFO "id-management setup detected -- will start id-management (mainzelliste & magicpl)." - OVERRIDE+=" -f ./$PROJECT/modules/id-management-compose.yml" - - # Auto Generate local Passwords - PATIENTLIST_POSTGRES_PASSWORD="$(echo \"id-management-module-db-password-salt\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - IDMANAGER_LOCAL_PATIENTLIST_APIKEY="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" - - # Transform Seeds Configuration to pass it to the Mainzelliste Container - PATIENTLIST_SEEDS_TRANSFORMED="$(declare -p PATIENTLIST_SEEDS | tr -d '\"' | sed 's/\[/\[\"/g' | sed 's/\]/\"\]/g')" - - # Ensure old ids are working !!! - export IDMANAGEMENT_FRIENDLY_ID=$(legacyIdMapping "$SITE_ID") -fi diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index a59f109..60845f5 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -19,11 +19,11 @@ services: FILE_CHARSET: ${MTBA_FILE_CHARSET} FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} CSV_DELIMITER: ${MTBA_CSV_DELIMITER} -# labels: -# - "traefik.enable=true" -# - "traefik.http.routers.mtba.rule=PathPrefix(`/mtba`)" -# - "traefik.http.services.mtba.loadbalancer.server.port=80" -# - "traefik.http.routers.mtba.tls=true" + labels: + - "traefik.enable=true" + - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" + - "traefik.http.services.mtba.loadbalancer.server.port=80" + - "traefik.http.routers.mtba.tls=true" volumes: - /tmp/bridgehead/mtba/input:/app/input - /tmp/bridgehead/mtba/persist:/app/persist @@ -31,3 +31,6 @@ services: # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! # TODO: Find a trigger to let mtba signal a restart for CBioPortal + +volumes: + mtba-data: From c003999721f3d536f6ba40d29ca8d2d21be99162 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 17 Jul 2023 11:59:19 +0000 Subject: [PATCH 048/243] Migrate to new app key syntax --- ccp/modules/datashield-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 7518c6f..b12fed1 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -89,8 +89,7 @@ services: beam-proxy: environment: - APP_2_ID: datashield-connect - APP_2_KEY: ${DATASHIELD_CONNECT_SECRET} + APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET} #volumes: # bridgehead-rstudio-config: From 36ac8d41c8c2091b34fdfbd411f7bbba41cae2fa Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 18 Jul 2023 10:54:24 +0200 Subject: [PATCH 049/243] Add http scheme to exporter --- ccp/modules/exporter-compose.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 5670142..cbf38c0 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -25,8 +25,8 @@ services: - "traefik.http.routers.exporter_ccp.tls=true" - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" -# volumes: -# - "bridgehead-exporter:/app/exporter-files" + # volumes: + # - "bridgehead-exporter:/app/exporter-files" exporter-db: image: postgres:15.1-alpine @@ -49,8 +49,9 @@ services: EXPORTER_API_KEY: "${EXPORTER_API_KEY}" EXPORTER_URL: "http://exporter:8092" LOG_FHIR_VALIDATION: "false" -# volumes: -# - "bridgehead-reporter:/app/reports" + HTTP_SERVLET_REQUEST_SCHEME: "https" + # volumes: + # - "bridgehead-reporter:/app/reports" labels: - "traefik.enable=true" - "traefik.http.routers.reporter_ccp.rule=PathPrefix(`/ccp-reporter`)" From 09aa33c912eedfb84cd2f4ab5b37e2a3b1d2d576 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 19 Jul 2023 13:45:14 +0200 Subject: [PATCH 050/243] Generate passwords only if modules are enabled --- ccp/modules/datashield-setup.sh | 16 ++++++++-------- ccp/modules/exporter-setup.sh | 6 +++--- ccp/modules/login-setup.sh | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 15c0359..3ed0926 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -1,13 +1,13 @@ #!/bin/bash -if [ "$ENABLE_DATASHIELD" == true ];then +if [ "$ENABLE_DATASHIELD" == true ]; then log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" -fi -OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" -if [ ! -e "/etc/bridgehead/trusted-ca-certs/opal-cert.pem" ]; then - openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/bridgehead/trusted-ca-certs/opal-key.pem -out /etc/bridgehead/trusted-ca-certs/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" - chmod g+r /etc/bridgehead/trusted-ca-certs/opal-key.pem - chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-key.pem - chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-cert.pem + OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + if [ ! -e "/etc/bridgehead/trusted-ca-certs/opal-cert.pem" ]; then + openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/bridgehead/trusted-ca-certs/opal-key.pem -out /etc/bridgehead/trusted-ca-certs/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" + chmod g+r /etc/bridgehead/trusted-ca-certs/opal-key.pem + chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-key.pem + chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-cert.pem + fi fi diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index 323d8a4..0f05315 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -1,8 +1,8 @@ #!/bin/bash -if [ -n "$ENABLE_EXPORTER" ];then +if [ -n "$ENABLE_EXPORTER" ]; then log INFO "Exporter setup detected -- will start Exporter service." OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" + # TODO: Generate password in another way so that not all passwords are the same? + EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" fi -# TODO: Generate password in another way so that not all passwords are the same? -EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" diff --git a/ccp/modules/login-setup.sh b/ccp/modules/login-setup.sh index f3d0122..0ba0f98 100644 --- a/ccp/modules/login-setup.sh +++ b/ccp/modules/login-setup.sh @@ -1,7 +1,7 @@ #!/bin/bash -if [ "$ENABLE_LOGIN" == true ];then +if [ "$ENABLE_LOGIN" == true ]; then log INFO "Login setup detected -- will start Login services." OVERRIDE+=" -f ./$PROJECT/modules/login-compose.yml" + KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Keycloak. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" fi -KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Keycloak. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" From a1e76a61b876e51a8078d06698f6ab8025ad3a68 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 19 Jul 2023 14:01:17 +0200 Subject: [PATCH 051/243] Remove ports of beam-connect in datashield-compose.yml --- ccp/modules/datashield-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index b12fed1..63c0314 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -71,8 +71,6 @@ services: beam-connect: image: docker.verbis.dkfz.de/cache/samply/beam-connect:no-auth container_name: bridgehead-datashield-connect - ports: - - 8062:8062 environment: PROXY_URL: "http://beam-proxy:8081" TLS_CA_CERTIFICATES_DIR: /run/secrets @@ -106,4 +104,3 @@ secrets: file: /etc/bridgehead/trusted-ca-certs/opal-cert.pem opal-key.pem: file: /etc/bridgehead/trusted-ca-certs/opal-key.pem - From 0866cacc5afbad2b3310ed4cb46a625a1b544c7b Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 2 Aug 2023 14:38:25 +0200 Subject: [PATCH 052/243] User postgres if docker.verbis.dkfz.de --- ccp/modules/datashield-compose.yml | 2 +- ccp/modules/exporter-compose.yml | 2 +- ccp/modules/login-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 63c0314..785ee3f 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -56,7 +56,7 @@ services: opal-db: # Data container_name: bridgehead-opal-db - image: postgres:15.1-alpine + image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine environment: POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" POSTGRES_USER: "opal" diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index cbf38c0..382f20b 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -29,7 +29,7 @@ services: # - "bridgehead-exporter:/app/exporter-files" exporter-db: - image: postgres:15.1-alpine + image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine container_name: bridgehead-ccp-exporter-db environment: POSTGRES_USER: "exporter" diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 1832bc4..1f29084 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: ############################################ Keycloak login-db: - image: postgres:15.1-alpine + image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine container_name: bridgehead-login-db environment: POSTGRES_USER: "keycloak" From c8bafb2461c715892c3aefe5af669f74c3b1c9bb Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 3 Aug 2023 10:21:42 +0200 Subject: [PATCH 053/243] R-Server rock-base:6.3 --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 785ee3f..160e1c2 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -66,7 +66,7 @@ services: opal-rserver: container_name: bridgehead-opal-rserver - image: datashield/rock-base:6.2-R4.2 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 + image: docker.verbis.dkfz.de/cache/datashield/rockbase:6.3 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 beam-connect: image: docker.verbis.dkfz.de/cache/samply/beam-connect:no-auth From e182e2fbe684d3ef79cd4f8b899084376cc002b1 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 3 Aug 2023 10:35:41 +0200 Subject: [PATCH 054/243] Remove unnecessary version of docker-compose.override files --- bbmri/modules/dnpm-compose.yml | 0 ccp/modules/datashield-compose.yml | 2 -- ccp/modules/dnpm-compose.yml | 2 -- ccp/modules/exporter-compose.yml | 2 -- ccp/modules/id-management-compose.yml | 1 - ccp/modules/login-compose.yml | 2 -- ccp/modules/mtba-compose.yml | 2 -- ccp/modules/nngm-compose.yml | 3 --- ccp/modules/teiler-ui-compose.yml | 2 -- 9 files changed, 16 deletions(-) create mode 100644 bbmri/modules/dnpm-compose.yml diff --git a/bbmri/modules/dnpm-compose.yml b/bbmri/modules/dnpm-compose.yml new file mode 100644 index 0000000..e69de29 diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 160e1c2..be7ad8e 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: ############################################ DataSHIELD Client (Rocker R-Studio) rstudio: diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index c32426f..061e010 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: beam-proxy: environment: diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 382f20b..0673c17 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: exporter: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 11d45cb..9c72d76 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -1,4 +1,3 @@ -version: "3.7" services: id-manager: image: docker.verbis.dkfz.de/bridgehead/magicpl diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 1f29084..a23584e 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: ############################################ Keycloak login-db: diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 60845f5..c94e885 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: mtba: image: docker.verbis.dkfz.de/cache/samply/mtba:1.0.0 diff --git a/ccp/modules/nngm-compose.yml b/ccp/modules/nngm-compose.yml index e61532d..e52e26c 100644 --- a/ccp/modules/nngm-compose.yml +++ b/ccp/modules/nngm-compose.yml @@ -21,9 +21,6 @@ services: - "traefik.http.routers.connector.middlewares=connector_strip,auth-nngm" volumes: - nngm-rest:/var/log - traefik: labels: - "traefik.http.middlewares.auth-nngm.basicauth.users=${NNGM_AUTH}" - - diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 5a51c8e..92a98d8 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: teiler-root-config: From 43c45f0628616db15f513c47cf9db1941594e0c3 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 3 Aug 2023 15:48:22 +0200 Subject: [PATCH 055/243] Remove todo in rstudio --- ccp/modules/datashield-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index be7ad8e..3fcda9f 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -3,7 +3,6 @@ services: rstudio: container_name: bridgehead-rstudio image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest - #TODO: Connect with Keycloak: https://rocker-project.org/images/versioned/rstudio.html environment: USER: "ruser" PASSWORD: "${RSTUDIO_PASSWORD}" From 840096d1d50ee5bbf32f54e76326fe04c322760b Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 3 Aug 2023 15:58:19 +0200 Subject: [PATCH 056/243] Enable only if true --- ccp/modules/exporter-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index 0f05315..fed62e5 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ -n "$ENABLE_EXPORTER" ]; then +if [ "$ENABLE_EXPORTER" == true ]; then log INFO "Exporter setup detected -- will start Exporter service." OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" # TODO: Generate password in another way so that not all passwords are the same? From 73d969e3743379178dadfe1f399d4da81b676114 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:29:40 +0200 Subject: [PATCH 057/243] Use LDM_PASSWORD for all admin passwords --- ccp/modules/datashield-compose.yml | 4 ++-- ccp/modules/exporter-compose.yml | 2 +- ccp/modules/login-compose.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 3fcda9f..6f3a5af 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -4,7 +4,7 @@ services: container_name: bridgehead-rstudio image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: - USER: "ruser" + USER: "${PROJECT}" PASSWORD: "${RSTUDIO_PASSWORD}" HTTP_RELATIVE_PATH: "/rstudio" labels: @@ -33,7 +33,7 @@ services: environment: JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" # OPAL_ADMINISTRATOR_USER: "administrator" # Please, don't remove this line. It is informative. - OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMINISTRATOR_PASSWORD}" + OPAL_ADMINISTRATOR_PASSWORD: "${LDM_PASSWORD}" POSTGRESDATA_HOST: "opal-db" POSTGRESDATA_DATABASE: "opal" POSTGRESDATA_USER: "opal" diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 0673c17..1d96688 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -15,7 +15,7 @@ services: WRITE_FILES_LIFETIME_IN_DAYS: "30" HTTP_RELATIVE_PATH: "/ccp-exporter" HTTP_SERVLET_REQUEST_SCHEME: "https" - OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMINISTRATOR_PASSWORD}" + OPAL_ADMINISTRATOR_PASSWORD: "${LDM_PASSWORD}" labels: - "traefik.enable=true" - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index a23584e..1fbe7bb 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -15,7 +15,7 @@ services: container_name: bridgehead-login environment: KEYCLOAK_ADMIN: "admin" - KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD}" + KEYCLOAK_ADMIN_PASSWORD: "${LDM_PASSWORD}" KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh KC_HOSTNAME_URL: "https://${HOST}/login" KC_HOSTNAME_STRICT: "false" From 9b8331ed28783ae98140b39c95d7df251c9c7c71 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:30:19 +0200 Subject: [PATCH 058/243] Update ccp/modules/datashield-compose.yml --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 6f3a5af..440083c 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -5,7 +5,7 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: USER: "${PROJECT}" - PASSWORD: "${RSTUDIO_PASSWORD}" + PASSWORD: "${LDM_PASSWORD}" HTTP_RELATIVE_PATH: "/rstudio" labels: - "traefik.enable=true" From 44415369cc96e582618bce0efcf44c8de4556740 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:10:59 +0200 Subject: [PATCH 059/243] Update ccp/modules/datashield-compose.yml --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 440083c..f2ecba4 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -63,7 +63,7 @@ services: opal-rserver: container_name: bridgehead-opal-rserver - image: docker.verbis.dkfz.de/cache/datashield/rockbase:6.3 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 + image: docker.verbis.dkfz.de/cache/datashield/rock-base:6.3 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 beam-connect: image: docker.verbis.dkfz.de/cache/samply/beam-connect:no-auth From 50d28d293f56128a0f7b0424a3808726f3d1cb21 Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:19:06 +0200 Subject: [PATCH 060/243] Generate DATASHIELD_CONNECT_SECRET automatically --- ccp/modules/datashield-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 3ed0926..f5731af 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -4,6 +4,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ ! -e "/etc/bridgehead/trusted-ca-certs/opal-cert.pem" ]; then openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/bridgehead/trusted-ca-certs/opal-key.pem -out /etc/bridgehead/trusted-ca-certs/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" chmod g+r /etc/bridgehead/trusted-ca-certs/opal-key.pem From f3745b973aecf9d8c972ab28b47c84df56ee5ddd Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 12:23:29 +0200 Subject: [PATCH 061/243] User default user rstudio in rstudio --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index f2ecba4..f3c3f61 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -4,7 +4,7 @@ services: container_name: bridgehead-rstudio image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: - USER: "${PROJECT}" + #DEFAULT_USER: "rstudio" # Please, don't remove this line. It is informative. PASSWORD: "${LDM_PASSWORD}" HTTP_RELATIVE_PATH: "/rstudio" labels: From a6443a68574df188c5f9780f443818108d22acd4 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 12:23:29 +0200 Subject: [PATCH 062/243] Remove IS_DKTK_SITE --- ccp/modules/teiler-ui-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 92a98d8..4ffe16d 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -63,7 +63,6 @@ services: TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" CENTRAX_URL: "${CENTRAXX_URL}" - IS_DKTK_SITE: "${IS_DKTK_SITE}" secrets: - ccp.conf From 75c86b79e8569f60b50cc981ede7e0889dd4e362 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 16:29:44 +0200 Subject: [PATCH 063/243] Add Teiler Admin to Keycloak --- ccp/modules/login-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 1fbe7bb..d448900 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -16,6 +16,11 @@ services: environment: KEYCLOAK_ADMIN: "admin" KEYCLOAK_ADMIN_PASSWORD: "${LDM_PASSWORD}" + TEILER_ADMIN: "${PROJECT}" + TEILER_ADMIN_PASSWORD: "${LDM_PASSWORD}" + TEILER_ADMIN_FIRST_NAME: "${OPERATOR_FIRST_NAME}" + TEILER_ADMIN_LAST_NAME: "${OPERATOR_LAST_NAME}" + TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh KC_HOSTNAME_URL: "https://${HOST}/login" KC_HOSTNAME_STRICT: "false" From 10a362c237cf19ae14395bd1773429ed6f084b77 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 16:45:11 +0200 Subject: [PATCH 064/243] Add explanation why is the volume of exporter-db currently so important for us. --- ccp/modules/exporter-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 1d96688..7ea2936 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -34,7 +34,7 @@ services: POSTGRES_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh POSTGRES_DB: "exporter" volumes: - # TODO: Move to logging component in bridgehead + # Please, do NOT REMOVE this volume until we find a solution to save Lens-queries to be executed in the explorer. - "bridgehead-exporter-db:/var/lib/postgresql/data" reporter: From be9adcbfa212527c8a63fdf412fb10581d76e126 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 16:57:45 +0200 Subject: [PATCH 065/243] Remove clean temp files configuration of exporter --- ccp/modules/exporter-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 7ea2936..0f980c3 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -9,10 +9,6 @@ services: EXPORTER_DB_USER: "exporter" EXPORTER_DB_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh EXPORTER_DB_URL: "jdbc:postgresql://exporter-db:5432/exporter" - CLEAN_TEMP_FILES_CRON_EXPRESSION: "0 0 1 * * *" - TEMP_FILES_LIFETIME_IN_DAYS: "1" - CLEAN_WRITE_FILES_CRON_EXPRESSION: "0 0 2 * * *" - WRITE_FILES_LIFETIME_IN_DAYS: "30" HTTP_RELATIVE_PATH: "/ccp-exporter" HTTP_SERVLET_REQUEST_SCHEME: "https" OPAL_ADMINISTRATOR_PASSWORD: "${LDM_PASSWORD}" From c4c4f743d29c6151dd2cd44260646afc800bde4e Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 17:06:46 +0200 Subject: [PATCH 066/243] Remove updater cron of teiler-core --- ccp/modules/teiler-ui-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 4ffe16d..b819449 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -57,7 +57,6 @@ services: APPLICATION_ADDRESS: "${HOST}" DEFAULT_LANGUAGE: "DE" CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" - TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" From e7f6c0b1a055231eba8f3f8ade668c64462f443c Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 17:13:33 +0200 Subject: [PATCH 067/243] Add default language to ccp --- ccp/modules/teiler-ui-compose.yml | 6 +++--- ccp/vars | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index b819449..f12e1c5 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -13,7 +13,7 @@ services: environment: TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" - DEFAULT_LANGUAGE: "de" + DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE_LOWER_CASE}" HTTP_RELATIVE_PATH: "/ccp-teiler" teiler-ui: @@ -27,7 +27,7 @@ services: - "traefik.http.middlewares.teiler_ui_ccp_strip.stripprefix.prefixes=/ccp-teiler-ui" - "traefik.http.routers.teiler_ui_ccp.middlewares=teiler_ui_ccp_strip" environment: - DEFAULT_LANGUAGE: "DE" + DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" KEYCLOAK_URL: "https://${HOST}/login" KEYCLOAK_REALM: "teiler-ui" @@ -55,7 +55,7 @@ services: LOG_LEVEL: "INFO" APPLICATION_PORT: "8085" APPLICATION_ADDRESS: "${HOST}" - DEFAULT_LANGUAGE: "DE" + DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" diff --git a/ccp/vars b/ccp/vars index b58710d..ebc76e8 100644 --- a/ccp/vars +++ b/ccp/vars @@ -7,6 +7,8 @@ SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL +DEFAULT_LANGUAGE=DE +DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} for module in $PROJECT/modules/*.sh From d97ac5612605aa206e18ab205280869e46175677 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 18:06:32 +0200 Subject: [PATCH 068/243] Generate exporter api key automatically --- ccp/modules/datashield-compose.yml | 2 +- ccp/modules/exporter-compose.yml | 4 ++-- ccp/modules/exporter-setup.sh | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index f3c3f61..1e7341e 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -55,7 +55,7 @@ services: container_name: bridgehead-opal-db image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine environment: - POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" + POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" # Set in datashield-setup.sh POSTGRES_USER: "opal" POSTGRES_DB: "opal" # volumes: diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 0f980c3..56525b7 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -4,7 +4,7 @@ services: container_name: bridgehead-ccp-exporter environment: LOG_LEVEL: "INFO" - EXPORTER_API_KEY: "${EXPORTER_API_KEY}" + EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh CROSS_ORIGINS: "https://${HOST}" EXPORTER_DB_USER: "exporter" EXPORTER_DB_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh @@ -40,7 +40,7 @@ services: LOG_LEVEL: "INFO" CROSS_ORIGINS: "https://${HOST}" HTTP_RELATIVE_PATH: "/ccp-reporter" - EXPORTER_API_KEY: "${EXPORTER_API_KEY}" + EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh EXPORTER_URL: "http://exporter:8092" LOG_FHIR_VALIDATION: "false" HTTP_SERVLET_REQUEST_SCHEME: "https" diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index fed62e5..c109f9c 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -5,4 +5,5 @@ if [ "$ENABLE_EXPORTER" == true ]; then OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" # TODO: Generate password in another way so that not all passwords are the same? EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + EXPORTER_API_KEY="$(echo \"This is a salt string to generate one consistent API KEY for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 64)" fi From 7ed24f667da63401bf857e096e2cb27cb2ec694a Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 18:12:32 +0200 Subject: [PATCH 069/243] Export and QB Curl templates --- ccp/modules/export-and-qb.curl-templates | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 ccp/modules/export-and-qb.curl-templates diff --git a/ccp/modules/export-and-qb.curl-templates b/ccp/modules/export-and-qb.curl-templates new file mode 100644 index 0000000..0853ed9 --- /dev/null +++ b/ccp/modules/export-and-qb.curl-templates @@ -0,0 +1,6 @@ +# Full Excel Export +curl --location --request POST 'https://${HOST}/ccp-exporter/request?query=Patient&query-format=FHIR_QUERY&template-id=ccp&output-format=EXCEL' \ +--header 'x-api-key: ${EXPORT_API_KEY}' + +# QB +curl --location --request POST 'https://${HOST}/ccp-reporter/generate?template-id=ccp' From 26165232f0bd79de5fa3d88fc99e4f1907aba445 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 9 Aug 2023 18:15:30 +0200 Subject: [PATCH 070/243] Enable Login, Teiler and Exporter --- ccp/vars | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ccp/vars b/ccp/vars index ebc76e8..c106027 100644 --- a/ccp/vars +++ b/ccp/vars @@ -9,6 +9,10 @@ PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL DEFAULT_LANGUAGE=DE DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} +ENABLE_EXPORTER=true +ENABLE_LOGIN=true +ENABLE_TEILER=true +#ENABLE_DATASHIELD=true for module in $PROJECT/modules/*.sh From bb7451d8c32546c8f3b07e3180d874df8f5c3b07 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 13:55:02 +0200 Subject: [PATCH 071/243] Add JAVA_OPTS to reporter and exporter --- ccp/modules/exporter-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 56525b7..068dd20 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -3,6 +3,7 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest container_name: bridgehead-ccp-exporter environment: + JAVA_OPTS: "-Xms512m -Xmx1G" LOG_LEVEL: "INFO" EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh CROSS_ORIGINS: "https://${HOST}" @@ -37,6 +38,7 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-reporter:latest container_name: bridgehead-ccp-reporter environment: + JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" LOG_LEVEL: "INFO" CROSS_ORIGINS: "https://${HOST}" HTTP_RELATIVE_PATH: "/ccp-reporter" From 846e9c23a7f54096d43efd8523bebda99be50128 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 14:08:49 +0200 Subject: [PATCH 072/243] Add DataSHIELD module documentation --- ccp/modules/datashield.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ccp/modules/datashield.md b/ccp/modules/datashield.md index e69de29..33e0123 100644 --- a/ccp/modules/datashield.md +++ b/ccp/modules/datashield.md @@ -0,0 +1,26 @@ +# DataSHIELD +This module constitutes the infrastructure to run DataSHIELD within the bridghead. +For more information about DataSHIELD, please visit https://www.datashield.org/ + +## R-Studio +To connect to the different bridgeheads of the CCP through DataSHIELD, you can use your own R-Studio environment. +However, this R-Studio has already installed the DataSHIELD libraries and is integrated within the bridgehead. +This can save you some time for extra configuration of your R-Studio environment. + +## Opal +This is the core of DataSHIELD. It is made up of Opal, a Postgres database and an R-server. +For more information about Opal, please visit https://opaldoc.obiba.org + +### Opal +Opal is OBiBa’s core database application for biobanks. + +### Opal-DB +Opal requires a database to import the data for DataSHIELD. We use a Postgres instance as database. +The data is imported within the bridgehead through the exporter. + +### Opal-R-Server +R-Server to execute R scripts in DataSHIELD. + +## Beam +### Beam-Connect +### Beam-Proxy From adeaf433dc425c88130ce7854b135ac3e76b181b Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 14:28:43 +0200 Subject: [PATCH 073/243] Add Exporter module documentation --- ccp/modules/exporter.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ccp/modules/exporter.md diff --git a/ccp/modules/exporter.md b/ccp/modules/exporter.md new file mode 100644 index 0000000..24e81b0 --- /dev/null +++ b/ccp/modules/exporter.md @@ -0,0 +1,15 @@ +# Exporter and Reporter + + +## Exporter +The exporter is a REST API that exports the data of the different databases of the bridgehead in a set of tables. +It can accept different output formats as CSV, Excel, JSON or XML. It can also export data into Opal. + +## Exporter-DB +It is a database to save queries for its execution in the exporter. +The exporter manages also the different executions of the same query in through the database. + +## Reporter +This component is a plugin of the exporter that allows to create more complex Excel reports described in templates. +It is compatible with different template engines as Groovy, Thymeleaf,... +It is perfect to generate a document as our traditional CCP quality report. From e3b8a7369b6c9c85b5cda64044880f0c2242625b Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 14:53:11 +0200 Subject: [PATCH 074/243] Add login module documentation --- ccp/modules/login.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ccp/modules/login.md b/ccp/modules/login.md index e69de29..eee488c 100644 --- a/ccp/modules/login.md +++ b/ccp/modules/login.md @@ -0,0 +1,13 @@ +# Login +The login component is a local Keycloak instance. In the future will be replaced by the central keycloak instance +or maybe can be used to add local identity providers to the bridgehead or just to simplify the configuration of +the central keycloak instance for the integration of every new bridgehead. +The basic configuration of our Keycloak instance is contained in a small json file. + +### Teiler User +Currently, the local keycloak is used by the teiler. There is a basic admin user in the basic configuration of keycloak. +The user can be configured with the environment variables TEILER_ADMIN_XXX. + +## Login-DB +Keycloak requires a local database for its configuration. However, as we use an initial json configuration file, if no +local identity provider is configured nor any local user, theoretically we don't need a volume for the login. From b4805af0a11cbe0920869275bd8455bb4d1805d2 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 10 Aug 2023 13:12:09 +0000 Subject: [PATCH 075/243] Add some docs about beam-connect --- ccp/modules/datashield.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/datashield.md b/ccp/modules/datashield.md index 33e0123..aa2c4cb 100644 --- a/ccp/modules/datashield.md +++ b/ccp/modules/datashield.md @@ -23,4 +23,6 @@ R-Server to execute R scripts in DataSHIELD. ## Beam ### Beam-Connect +Beam-Connect is used to route http(s) traffic through beam to enable R-Studio to access data from other bridgeheads that have datashield enabled. ### Beam-Proxy +The usual beam proxy used for communication. From 957fa64ce9c3e630bbbb36379a27fe525d0a5914 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 18:18:50 +0200 Subject: [PATCH 076/243] Add teiler-ui module documentation --- ccp/modules/teiler-ui.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 ccp/modules/teiler-ui.md diff --git a/ccp/modules/teiler-ui.md b/ccp/modules/teiler-ui.md new file mode 100644 index 0000000..28817fb --- /dev/null +++ b/ccp/modules/teiler-ui.md @@ -0,0 +1,17 @@ +# Teiler +This module orchestrates the different microfrontends of the bridgehead as a single page application. + +## Teiler Root Config +Single SPA component that consists on the root HTML site of the single page application and a javascript code that +gets the information about the microfrontend calling the teiler core and is responsible for registering them. With the +resulting mapping, it can initialize, mount and unmount the required microfrontends on the fly. + +The microfrontends run independently in different containers and can be based on different frameworks (Angular, Vue, React,...) +This microfrontends can run as single alone but need an extension with Single-SPA (https://single-spa.js.org/docs/ecosystem). +There are also available three templates (Angular, Vue, React) to be directly extended to be used directly in the teiler. + +## Teiler UI +It consists on the main dashboard and a set of embedded services. + +## Teiler Core +In this component, the microfrontends are configured. From c52975f2045951b03eee6df50ea932f538b29a18 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 19:02:41 +0200 Subject: [PATCH 077/243] Add mtba module documentation --- ccp/modules/mtba.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 ccp/modules/mtba.md diff --git a/ccp/modules/mtba.md b/ccp/modules/mtba.md new file mode 100644 index 0000000..c8dff60 --- /dev/null +++ b/ccp/modules/mtba.md @@ -0,0 +1,8 @@ +# MTBA + +MTBA is referred to Molecular Tumor Board Alliance. + +In this module, the genetic data to import is stored in a directory (/tmp/bridgehead/mtba/input). A process checks +regularly if there are files in the directory. The files are pseudonomized when the IDAT is provided. The files are +combined with clinical data of the blaze and imported in cBioPortal. On the other hand, this files are also imported in +Blaze. From 5d8bec53c05575b204680b844fec1b16c1737d82 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 20:36:34 +0200 Subject: [PATCH 078/243] Bugfix: JAVA_OPTS for exporter --- ccp/modules/exporter-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 068dd20..c77cfd2 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -3,7 +3,7 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest container_name: bridgehead-ccp-exporter environment: - JAVA_OPTS: "-Xms512m -Xmx1G" + JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" LOG_LEVEL: "INFO" EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh CROSS_ORIGINS: "https://${HOST}" From 6cfb42dc9bdbe94d91488eb792b6121d5272976e Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 20:52:34 +0200 Subject: [PATCH 079/243] Comment on export and report volumes --- ccp/modules/exporter-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index c77cfd2..5b52269 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -20,6 +20,12 @@ services: - "traefik.http.routers.exporter_ccp.tls=true" - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" + + # In this initial development state of the bridgehead, we are trying to have so many volumes as possible. + # However, in the first executions in the CCP sites, this volume seems to be very important. An export is + # a process that can take several hours. There is a risk that the bridgehead restarts, losing the already + # created export. This is specially notorious while creating a report. + - # volumes: # - "bridgehead-exporter:/app/exporter-files" @@ -46,6 +52,12 @@ services: EXPORTER_URL: "http://exporter:8092" LOG_FHIR_VALIDATION: "false" HTTP_SERVLET_REQUEST_SCHEME: "https" + + # In this initial development state of the bridgehead, we are trying to have so many volumes as possible. + # However, in the first executions in the CCP sites, this volume seems to be very important. A report is + # a process that can take several hours, because it depends on the exporter. + # There is a risk that the bridgehead restarts, losing the already created export. + # volumes: # - "bridgehead-reporter:/app/reports" labels: From 839e7a45183115d04d93d2eebc50c2972897ed1c Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 20:54:15 +0200 Subject: [PATCH 080/243] Comment on datashield volume --- ccp/modules/datashield-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 1e7341e..bd01070 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -58,6 +58,8 @@ services: POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" # Set in datashield-setup.sh POSTGRES_USER: "opal" POSTGRES_DB: "opal" + # Please evaluate if this volume makes sense. An import can take several hours. If the bridgehead is restarted, + # the data should be reimported again. # volumes: # - "bridgehead-opal-db:/var/lib/postgresql/data" From 973b5828f6fb3deddf684b69b97d46b21c104105 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 21:41:03 +0200 Subject: [PATCH 081/243] Remove old comment of exporter-setup.sh --- ccp/modules/exporter-setup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index c109f9c..53e3773 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -3,7 +3,6 @@ if [ "$ENABLE_EXPORTER" == true ]; then log INFO "Exporter setup detected -- will start Exporter service." OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" - # TODO: Generate password in another way so that not all passwords are the same? EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" EXPORTER_API_KEY="$(echo \"This is a salt string to generate one consistent API KEY for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 64)" fi From f26a8f7a71ac3b75e9ed10a0cce492c0b2f31262 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 21:43:44 +0200 Subject: [PATCH 082/243] Fix comment in login-compose.yml --- ccp/modules/login-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index d448900..3267c5e 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -5,7 +5,7 @@ services: container_name: bridgehead-login-db environment: POSTGRES_USER: "keycloak" - POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in login-setup.sh POSTGRES_DB: "keycloak" volumes: - "bridgehead-login-db:/var/lib/postgresql/data" @@ -21,7 +21,7 @@ services: TEILER_ADMIN_FIRST_NAME: "${OPERATOR_FIRST_NAME}" TEILER_ADMIN_LAST_NAME: "${OPERATOR_LAST_NAME}" TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" - KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in teiler-setup.sh + KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in login-setup.sh KC_HOSTNAME_URL: "https://${HOST}/login" KC_HOSTNAME_STRICT: "false" KC_PROXY_ADDRESS_FORWARDING: "true" From 4b0b17424ff7492875a0a6ccca78321482d0c682 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 10 Aug 2023 21:50:52 +0200 Subject: [PATCH 083/243] Comment Keycloak volume --- ccp/modules/login-compose.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 3267c5e..4c5f40e 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -7,8 +7,10 @@ services: POSTGRES_USER: "keycloak" POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in login-setup.sh POSTGRES_DB: "keycloak" - volumes: - - "bridgehead-login-db:/var/lib/postgresql/data" +# Please leave this volume commented. We are in an early stage of the development and we need to evaluate +# if the sites need to add local users or if some manual changes are required for any reason. +# volumes: +# - "bridgehead-login-db:/var/lib/postgresql/data" login: image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest @@ -36,6 +38,6 @@ services: depends_on: - login-db -volumes: - bridgehead-login-db: - name: "bridgehead-login-db" +#volumes: +# bridgehead-login-db: +# name: "bridgehead-login-db" From 2b61775652d7b66b924b3fc86a6d5d493a068059 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 11 Aug 2023 15:06:54 +0200 Subject: [PATCH 084/243] Enable datashield --- ccp/vars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/vars b/ccp/vars index c106027..1b99a3c 100644 --- a/ccp/vars +++ b/ccp/vars @@ -12,7 +12,7 @@ DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} ENABLE_EXPORTER=true ENABLE_LOGIN=true ENABLE_TEILER=true -#ENABLE_DATASHIELD=true +ENABLE_DATASHIELD=true for module in $PROJECT/modules/*.sh From 765613b87fd0ca9848a9c268666fefe8f3f5cf5c Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 11 Aug 2023 15:07:42 +0200 Subject: [PATCH 085/243] Bugfix: MTBA path prefix --- ccp/modules/mtba-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index c94e885..e3c0d9b 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -19,7 +19,7 @@ services: CSV_DELIMITER: ${MTBA_CSV_DELIMITER} labels: - "traefik.enable=true" - - "traefik.http.routers.mtba.rule=PathPrefix(`/`)" + - "traefik.http.routers.mtba.rule=PathPrefix(`/mtba`)" - "traefik.http.services.mtba.loadbalancer.server.port=80" - "traefik.http.routers.mtba.tls=true" volumes: From 963144cc31ac76c256d84f24eaf5fe883c9add08 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 11 Aug 2023 15:08:08 +0200 Subject: [PATCH 086/243] Disable datashield --- ccp/vars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/vars b/ccp/vars index 1b99a3c..c106027 100644 --- a/ccp/vars +++ b/ccp/vars @@ -12,7 +12,7 @@ DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} ENABLE_EXPORTER=true ENABLE_LOGIN=true ENABLE_TEILER=true -ENABLE_DATASHIELD=true +#ENABLE_DATASHIELD=true for module in $PROJECT/modules/*.sh From cfc3c7c90e550451a8f6fc097c4f3685dd838cc8 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 11 Aug 2023 15:23:45 +0200 Subject: [PATCH 087/243] Bugfix: exporter --- ccp/modules/exporter-compose.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 5b52269..392ff18 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -21,11 +21,10 @@ services: - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" - # In this initial development state of the bridgehead, we are trying to have so many volumes as possible. - # However, in the first executions in the CCP sites, this volume seems to be very important. An export is - # a process that can take several hours. There is a risk that the bridgehead restarts, losing the already - # created export. This is specially notorious while creating a report. - - + # In this initial development state of the bridgehead, we are trying to have so many volumes as possible. + # However, in the first executions in the CCP sites, this volume seems to be very important. An export is + # a process that can take several hours. There is a risk that the bridgehead restarts, losing the already + # created export. This is specially notorious while creating a report. # volumes: # - "bridgehead-exporter:/app/exporter-files" From 8d38adc91ed6e5343219a810c8ff39fa2a9a9302 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 11 Aug 2023 17:22:34 +0200 Subject: [PATCH 088/243] Bugfix: mtba labels --- ccp/modules/mtba-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index e3c0d9b..c95f8ef 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -19,9 +19,11 @@ services: CSV_DELIMITER: ${MTBA_CSV_DELIMITER} labels: - "traefik.enable=true" - - "traefik.http.routers.mtba.rule=PathPrefix(`/mtba`)" - - "traefik.http.services.mtba.loadbalancer.server.port=80" - - "traefik.http.routers.mtba.tls=true" + - "traefik.http.routers.mtba_ccp.rule=PathPrefix(`/mtba`)" + - "traefik.http.services.mtba_ccp.loadbalancer.server.port=8480" + - "traefik.http.routers.mtba_ccp.tls=true" + - "traefik.http.middlewares.mtba_ccp_strip.stripprefix.prefixes=/mtba" + - "traefik.http.routers.mtba_ccp.middlewares=mtba_ccp_strip, auth" volumes: - /tmp/bridgehead/mtba/input:/app/input - /tmp/bridgehead/mtba/persist:/app/persist From ff1f7904ad35e51360922b64ba7201aa65cf4a5f Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 14 Aug 2023 11:17:44 +0200 Subject: [PATCH 089/243] Add forward proxy to teiler-core --- ccp/modules/teiler-ui-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index f12e1c5..09766f1 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -62,6 +62,7 @@ services: TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" CENTRAX_URL: "${CENTRAXX_URL}" + HTTP_PROXY: "http://forward_proxy:3128" secrets: - ccp.conf From 14aece46f78bd22eecf7a43373b7a70fd98c0717 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 14 Aug 2023 15:52:53 +0200 Subject: [PATCH 090/243] Add site to exporter and reporter --- ccp/modules/exporter-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 392ff18..5134f66 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -11,6 +11,7 @@ services: EXPORTER_DB_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh EXPORTER_DB_URL: "jdbc:postgresql://exporter-db:5432/exporter" HTTP_RELATIVE_PATH: "/ccp-exporter" + SITE: "${SITE_ID}" HTTP_SERVLET_REQUEST_SCHEME: "https" OPAL_ADMINISTRATOR_PASSWORD: "${LDM_PASSWORD}" labels: @@ -47,6 +48,7 @@ services: LOG_LEVEL: "INFO" CROSS_ORIGINS: "https://${HOST}" HTTP_RELATIVE_PATH: "/ccp-reporter" + SITE: "${SITE_ID}" EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh EXPORTER_URL: "http://exporter:8092" LOG_FHIR_VALIDATION: "false" From 2b3eabe95cf5f28734f614670fb6f8ff4e4cbbd2 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 17 Aug 2023 18:10:12 +0200 Subject: [PATCH 091/243] Rename Teiler Backend, Teiler Dashboard and Teiler Orchestrator --- ccp/modules/login-compose.yml | 2 +- ccp/modules/teiler-compose.yml | 71 +++++++++++++++++++++++++ ccp/modules/teiler-setup.sh | 6 +++ ccp/modules/teiler-ui-compose.yml | 71 ------------------------- ccp/modules/teiler-ui-setup.sh | 6 --- ccp/modules/{teiler-ui.md => teiler.md} | 8 +-- 6 files changed, 82 insertions(+), 82 deletions(-) create mode 100644 ccp/modules/teiler-compose.yml create mode 100644 ccp/modules/teiler-setup.sh delete mode 100644 ccp/modules/teiler-ui-compose.yml delete mode 100644 ccp/modules/teiler-ui-setup.sh rename ccp/modules/{teiler-ui.md => teiler.md} (88%) diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 4c5f40e..ec5688c 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -27,7 +27,7 @@ services: KC_HOSTNAME_URL: "https://${HOST}/login" KC_HOSTNAME_STRICT: "false" KC_PROXY_ADDRESS_FORWARDING: "true" - TEILER_ROOT_CONFIG_EXTERN_URL: "https://${HOST}/ccp-teiler" + TEILER_ORCHESTRATOR_EXTERN_URL: "https://${HOST}/ccp-teiler" command: - start-dev --import-realm --proxy edge --http-relative-path=/login labels: diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml new file mode 100644 index 0000000..01a5ce0 --- /dev/null +++ b/ccp/modules/teiler-compose.yml @@ -0,0 +1,71 @@ +services: + + teiler-orchestrator: + image: docker.verbis.dkfz.de/cache/samply/teiler-orchestrator:develop + container_name: bridgehead-teiler-orchestrator + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_orchestrator_ccp.rule=PathPrefix(`/ccp-teiler`)" + - "traefik.http.services.teiler_orchestrator_ccp.loadbalancer.server.port=9000" + - "traefik.http.routers.teiler_orchestrator_ccp.tls=true" + - "traefik.http.middlewares.teiler_orchestrator_ccp_strip.stripprefix.prefixes=/ccp-teiler" + - "traefik.http.routers.teiler_orchestrator_ccp.middlewares=teiler_orchestrator_ccp_strip" + environment: + TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend" + TEILER_DASHBOARD_URL: "https://${HOST}/ccp-teiler-dashboard" + DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE_LOWER_CASE}" + HTTP_RELATIVE_PATH: "/ccp-teiler" + + teiler-dashboard: + image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:develop + container_name: bridgehead-teiler-dashboard + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_dashboard_ccp.rule=PathPrefix(`/ccp-teiler-dashboard`)" + - "traefik.http.services.teiler_dashboard_ccp.loadbalancer.server.port=80" + - "traefik.http.routers.teiler_dashboard_ccp.tls=true" + - "traefik.http.middlewares.teiler_dashboard_ccp_strip.stripprefix.prefixes=/ccp-teiler-dashboard" + - "traefik.http.routers.teiler_dashboard_ccp.middlewares=teiler_dashboard_ccp_strip" + environment: + DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" + TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend" + KEYCLOAK_URL: "https://${HOST}/login" + KEYCLOAK_REALM: "teiler" + KEYCLOAK_CLIENT_ID: "teiler" + TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" + TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" + TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" + TEILER_PROJECT: "${PROJECT}" + EXPORTER_API_KEY: "${EXPORTER_API_KEY}" + TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler" + TEILER_DASHBOARD_HTTP_RELATIVE_PATH: "/ccp-teiler-dashboard" + TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler" + + teiler-backend: + image: docker.verbis.dkfz.de/ccp/dktk-teiler-backend:latest + container_name: bridgehead-teiler-backend + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_backend_ccp.rule=PathPrefix(`/ccp-teiler-backend`)" + - "traefik.http.services.teiler_backend_ccp.loadbalancer.server.port=8085" + - "traefik.http.routers.teiler_backend_ccp.tls=true" + - "traefik.http.middlewares.teiler_backend_ccp_strip.stripprefix.prefixes=/ccp-teiler-backend" + - "traefik.http.routers.teiler_backend_ccp.middlewares=teiler_backend_ccp_strip" + environment: + LOG_LEVEL: "INFO" + APPLICATION_PORT: "8085" + APPLICATION_ADDRESS: "${HOST}" + DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" + CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" + TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler" + TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler" + TEILER_DASHBOARD_DE_URL: "https://${HOST}/ccp-teiler-dashboard/de" + TEILER_DASHBOARD_EN_URL: "https://${HOST}/ccp-teiler-dashboard/en" + CENTRAX_URL: "${CENTRAXX_URL}" + HTTP_PROXY: "http://forward_proxy:3128" + secrets: + - ccp.conf + +secrets: + ccp.conf: + file: /etc/bridgehead/ccp.conf diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh new file mode 100644 index 0000000..ea92612 --- /dev/null +++ b/ccp/modules/teiler-setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ "$ENABLE_TEILER" == true ];then + log INFO "Teiler setup detected -- will start Teiler services." + OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" +fi diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml deleted file mode 100644 index 09766f1..0000000 --- a/ccp/modules/teiler-ui-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -services: - - teiler-root-config: - image: docker.verbis.dkfz.de/cache/samply/teiler-root-config:develop - container_name: bridgehead-teiler-root-config - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler`)" - - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" - - "traefik.http.routers.teiler_root_config_ccp.tls=true" - - "traefik.http.middlewares.teiler_root_config_ccp_strip.stripprefix.prefixes=/ccp-teiler" - - "traefik.http.routers.teiler_root_config_ccp.middlewares=teiler_root_config_ccp_strip" - environment: - TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" - TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" - DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE_LOWER_CASE}" - HTTP_RELATIVE_PATH: "/ccp-teiler" - - teiler-ui: - image: docker.verbis.dkfz.de/cache/samply/teiler-ui:develop - container_name: bridgehead-teiler-ui - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_ui_ccp.rule=PathPrefix(`/ccp-teiler-ui`)" - - "traefik.http.services.teiler_ui_ccp.loadbalancer.server.port=80" - - "traefik.http.routers.teiler_ui_ccp.tls=true" - - "traefik.http.middlewares.teiler_ui_ccp_strip.stripprefix.prefixes=/ccp-teiler-ui" - - "traefik.http.routers.teiler_ui_ccp.middlewares=teiler_ui_ccp_strip" - environment: - DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" - TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" - KEYCLOAK_URL: "https://${HOST}/login" - KEYCLOAK_REALM: "teiler-ui" - KEYCLOAK_CLIENT_ID: "teiler-ui" - TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" - TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" - TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" - TEILER_PROJECT: "${PROJECT}" - EXPORTER_API_KEY: "${EXPORTER_API_KEY}" - TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" - TEILER_UI_HTTP_RELATIVE_PATH: "/ccp-teiler-ui" - TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" - - teiler-core: - image: docker.verbis.dkfz.de/ccp/dktk-teiler-core:latest - container_name: bridgehead-teiler-core - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" - - "traefik.http.services.teiler_core_ccp.loadbalancer.server.port=8085" - - "traefik.http.routers.teiler_core_ccp.tls=true" - - "traefik.http.middlewares.teiler_core_ccp_strip.stripprefix.prefixes=/ccp-teiler-core" - - "traefik.http.routers.teiler_core_ccp.middlewares=teiler_core_ccp_strip" - environment: - LOG_LEVEL: "INFO" - APPLICATION_PORT: "8085" - APPLICATION_ADDRESS: "${HOST}" - DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" - CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" - TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" - TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" - TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" - TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" - CENTRAX_URL: "${CENTRAXX_URL}" - HTTP_PROXY: "http://forward_proxy:3128" - secrets: - - ccp.conf - -secrets: - ccp.conf: - file: /etc/bridgehead/ccp.conf diff --git a/ccp/modules/teiler-ui-setup.sh b/ccp/modules/teiler-ui-setup.sh deleted file mode 100644 index 793abdd..0000000 --- a/ccp/modules/teiler-ui-setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -if [ "$ENABLE_TEILER" == true ];then - log INFO "Teiler-UI setup detected -- will start Teiler-UI services." - OVERRIDE+=" -f ./$PROJECT/modules/teiler-ui-compose.yml" -fi diff --git a/ccp/modules/teiler-ui.md b/ccp/modules/teiler.md similarity index 88% rename from ccp/modules/teiler-ui.md rename to ccp/modules/teiler.md index 28817fb..fda0dbd 100644 --- a/ccp/modules/teiler-ui.md +++ b/ccp/modules/teiler.md @@ -1,17 +1,17 @@ # Teiler This module orchestrates the different microfrontends of the bridgehead as a single page application. -## Teiler Root Config +## Teiler Orchestrator Single SPA component that consists on the root HTML site of the single page application and a javascript code that -gets the information about the microfrontend calling the teiler core and is responsible for registering them. With the +gets the information about the microfrontend calling the teiler backend and is responsible for registering them. With the resulting mapping, it can initialize, mount and unmount the required microfrontends on the fly. The microfrontends run independently in different containers and can be based on different frameworks (Angular, Vue, React,...) This microfrontends can run as single alone but need an extension with Single-SPA (https://single-spa.js.org/docs/ecosystem). There are also available three templates (Angular, Vue, React) to be directly extended to be used directly in the teiler. -## Teiler UI +## Teiler Dashboard It consists on the main dashboard and a set of embedded services. -## Teiler Core +## Teiler Backend In this component, the microfrontends are configured. From 720783249d19d7760a56a711bcd15ad75f7dc6f0 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 4 Sep 2023 15:15:34 +0200 Subject: [PATCH 092/243] Bugfix: LDM_AUTH instead of LDM_PASSWORD --- ccp/modules/datashield-compose.yml | 4 ++-- ccp/modules/exporter-compose.yml | 2 +- ccp/modules/login-compose.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index bd01070..36c26ad 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -5,7 +5,7 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: #DEFAULT_USER: "rstudio" # Please, don't remove this line. It is informative. - PASSWORD: "${LDM_PASSWORD}" + PASSWORD: "${LDM_AUTH}" HTTP_RELATIVE_PATH: "/rstudio" labels: - "traefik.enable=true" @@ -33,7 +33,7 @@ services: environment: JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" # OPAL_ADMINISTRATOR_USER: "administrator" # Please, don't remove this line. It is informative. - OPAL_ADMINISTRATOR_PASSWORD: "${LDM_PASSWORD}" + OPAL_ADMINISTRATOR_PASSWORD: "${LDM_AUTH}" POSTGRESDATA_HOST: "opal-db" POSTGRESDATA_DATABASE: "opal" POSTGRESDATA_USER: "opal" diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 5134f66..81f3a78 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -13,7 +13,7 @@ services: HTTP_RELATIVE_PATH: "/ccp-exporter" SITE: "${SITE_ID}" HTTP_SERVLET_REQUEST_SCHEME: "https" - OPAL_ADMINISTRATOR_PASSWORD: "${LDM_PASSWORD}" + OPAL_ADMINISTRATOR_PASSWORD: "${LDM_AUTH}" labels: - "traefik.enable=true" - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index ec5688c..569d8f5 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -17,9 +17,9 @@ services: container_name: bridgehead-login environment: KEYCLOAK_ADMIN: "admin" - KEYCLOAK_ADMIN_PASSWORD: "${LDM_PASSWORD}" + KEYCLOAK_ADMIN_PASSWORD: "${LDM_AUTH}" TEILER_ADMIN: "${PROJECT}" - TEILER_ADMIN_PASSWORD: "${LDM_PASSWORD}" + TEILER_ADMIN_PASSWORD: "${LDM_AUTH}" TEILER_ADMIN_FIRST_NAME: "${OPERATOR_FIRST_NAME}" TEILER_ADMIN_LAST_NAME: "${OPERATOR_LAST_NAME}" TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" From ccf0b91f17ef3580fc9b86a2761ab1a9dd1f2ee6 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 4 Sep 2023 16:43:40 +0200 Subject: [PATCH 093/243] #!/bin/bash -e --- ccp/modules/datashield-setup.sh | 2 +- ccp/modules/dnpm-setup.sh | 2 +- ccp/modules/exporter-setup.sh | 2 +- ccp/modules/id-management-setup.sh | 2 +- ccp/modules/login-setup.sh | 2 +- ccp/modules/mtba-setup.sh | 2 +- ccp/modules/nngm-setup.sh | 2 +- ccp/modules/teiler-setup.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index f5731af..4948021 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e if [ "$ENABLE_DATASHIELD" == true ]; then log INFO "DataSHIELD setup detected -- will start DataSHIELD services." diff --git a/ccp/modules/dnpm-setup.sh b/ccp/modules/dnpm-setup.sh index 21d356c..9ca4767 100644 --- a/ccp/modules/dnpm-setup.sh +++ b/ccp/modules/dnpm-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e if [ -n "${ENABLE_DNPM}" ]; then log INFO "DNPM setup detected (Beam.Connect) -- will start Beam.Connect for DNPM." diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index 53e3773..e33978c 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e if [ "$ENABLE_EXPORTER" == true ]; then log INFO "Exporter setup detected -- will start Exporter service." diff --git a/ccp/modules/id-management-setup.sh b/ccp/modules/id-management-setup.sh index 1b347e7..3165956 100644 --- a/ccp/modules/id-management-setup.sh +++ b/ccp/modules/id-management-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e function idManagementSetup() { if [ -n "$IDMANAGER_UPLOAD_APIKEY" ]; then diff --git a/ccp/modules/login-setup.sh b/ccp/modules/login-setup.sh index 0ba0f98..5ead5d4 100644 --- a/ccp/modules/login-setup.sh +++ b/ccp/modules/login-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e if [ "$ENABLE_LOGIN" == true ]; then log INFO "Login setup detected -- will start Login services." diff --git a/ccp/modules/mtba-setup.sh b/ccp/modules/mtba-setup.sh index 11af4ee..53b4ce0 100644 --- a/ccp/modules/mtba-setup.sh +++ b/ccp/modules/mtba-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e function mtbaSetup() { if [ -n "$ENABLE_MTBA" ];then diff --git a/ccp/modules/nngm-setup.sh b/ccp/modules/nngm-setup.sh index 56be949..48fc45e 100644 --- a/ccp/modules/nngm-setup.sh +++ b/ccp/modules/nngm-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e if [ -n "$NNGM_CTS_APIKEY" ]; then log INFO "nNGM setup detected -- will start nNGM Connector." diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index ea92612..d1caebe 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler setup detected -- will start Teiler services." From 4aa8f0f3ba6c6193e91f14e50ca68355376c862f Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 11 Sep 2023 13:27:08 +0200 Subject: [PATCH 094/243] Bugfix: Add version in every docker compose file --- ccp/modules/datashield-compose.yml | 2 ++ ccp/modules/dnpm-compose.yml | 2 ++ ccp/modules/exporter-compose.yml | 2 ++ ccp/modules/id-management-compose.yml | 2 ++ ccp/modules/login-compose.yml | 2 ++ ccp/modules/mtba-compose.yml | 2 ++ ccp/modules/nngm-compose.yml | 1 + ccp/modules/teiler-compose.yml | 2 ++ 8 files changed, 15 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 36c26ad..cb322cb 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: ############################################ DataSHIELD Client (Rocker R-Studio) rstudio: diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index 061e010..c32426f 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: beam-proxy: environment: diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 81f3a78..4bf6669 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: exporter: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 9c72d76..2e922cb 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: id-manager: image: docker.verbis.dkfz.de/bridgehead/magicpl diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 569d8f5..5e6527b 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: ############################################ Keycloak login-db: diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index c95f8ef..eace283 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: mtba: image: docker.verbis.dkfz.de/cache/samply/mtba:1.0.0 diff --git a/ccp/modules/nngm-compose.yml b/ccp/modules/nngm-compose.yml index e52e26c..7ffa190 100644 --- a/ccp/modules/nngm-compose.yml +++ b/ccp/modules/nngm-compose.yml @@ -1,4 +1,5 @@ version: "3.7" + volumes: nngm-rest: diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index 01a5ce0..68f378c 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: teiler-orchestrator: From 862e452f3c3564ab163ae29703363396d24034c5 Mon Sep 17 00:00:00 2001 From: lablans Date: Fri, 15 Sep 2023 08:11:40 +0000 Subject: [PATCH 095/243] Cache opal in /var/cache/bridgehead --- ccp/modules/datashield-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index cb322cb..7ddc55c 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -62,8 +62,8 @@ services: POSTGRES_DB: "opal" # Please evaluate if this volume makes sense. An import can take several hours. If the bridgehead is restarted, # the data should be reimported again. - # volumes: - # - "bridgehead-opal-db:/var/lib/postgresql/data" + volumes: + - "/var/cache/bridgehead/opal-db:/var/lib/postgresql/data" opal-rserver: container_name: bridgehead-opal-rserver From b5ca5ea4a78fa329a586f762ec2afcc66c38a278 Mon Sep 17 00:00:00 2001 From: lablans Date: Fri, 15 Sep 2023 08:12:16 +0000 Subject: [PATCH 096/243] Autogenerate maps for Opal's beam-connect. To be completed by @Threated with a map-generator in the script. --- ccp/modules/datashield-compose.yml | 6 +++--- ccp/modules/datashield-setup.sh | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 7ddc55c..68576cc 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -84,7 +84,7 @@ services: depends_on: - beam-proxy volumes: - - /etc/bridgehead/datashield-connect/:/map:ro + - /tmp/bridgehead/opal-map/:/map/:ro beam-proxy: environment: @@ -102,6 +102,6 @@ services: secrets: opal-cert.pem: - file: /etc/bridgehead/trusted-ca-certs/opal-cert.pem + file: /tmp/bridgehead/opal-cert.pem opal-key.pem: - file: /etc/bridgehead/trusted-ca-certs/opal-key.pem + file: /tmp/bridgehead/opal-key.pem diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 4948021..8652431 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -5,10 +5,11 @@ if [ "$ENABLE_DATASHIELD" == true ]; then OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - if [ ! -e "/etc/bridgehead/trusted-ca-certs/opal-cert.pem" ]; then - openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/bridgehead/trusted-ca-certs/opal-key.pem -out /etc/bridgehead/trusted-ca-certs/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" - chmod g+r /etc/bridgehead/trusted-ca-certs/opal-key.pem - chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-key.pem - chown bridgehead:docker /etc/bridgehead/trusted-ca-certs/opal-cert.pem + if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then + mkdir -p /tmp/bridgehead/ + chown -R bridgehead:docker /tmp/bridgehead/ + openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" + chown -R bridgehead:docker /tmp/bridgehead/ + chmod g+r /tmp/bridgehead/opal-key.pem fi fi From f38d9f8c1989eba791bf0c2b57023a949f380756 Mon Sep 17 00:00:00 2001 From: lablans Date: Fri, 15 Sep 2023 08:14:12 +0000 Subject: [PATCH 097/243] Rework commented sections --- ccp/modules/datashield-compose.yml | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 68576cc..4a71a90 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -1,12 +1,11 @@ version: "3.7" services: - ############################################ DataSHIELD Client (Rocker R-Studio) rstudio: container_name: bridgehead-rstudio image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: - #DEFAULT_USER: "rstudio" # Please, don't remove this line. It is informative. + #DEFAULT_USER: "rstudio" # This line is kept for informational purposes PASSWORD: "${LDM_AUTH}" HTTP_RELATIVE_PATH: "/rstudio" labels: @@ -16,11 +15,7 @@ services: - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" - #volumes: - #- "bridgehead-rstudio-config:/home/rstudio/.config/rstudio" - #- "bridgehead-rstudio-workspace:/home/rstudio/workspace" - ############################################ DataSHIELD Server (Opal) opal: container_name: bridgehead-opal image: docker.verbis.dkfz.de/ccp/dktk-opal:latest @@ -34,7 +29,7 @@ services: - opal-db environment: JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" - # OPAL_ADMINISTRATOR_USER: "administrator" # Please, don't remove this line. It is informative. + # OPAL_ADMINISTRATOR_USER: "administrator" # This line is kept for informational purposes OPAL_ADMINISTRATOR_PASSWORD: "${LDM_AUTH}" POSTGRESDATA_HOST: "opal-db" POSTGRESDATA_DATABASE: "opal" @@ -49,19 +44,13 @@ services: - opal-cert.pem - opal-key.pem - # volumes: - # - "bridgehead-opal:/srv" - - - opal-db: # Data + opal-db: container_name: bridgehead-opal-db image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine environment: POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" # Set in datashield-setup.sh POSTGRES_USER: "opal" POSTGRES_DB: "opal" - # Please evaluate if this volume makes sense. An import can take several hours. If the bridgehead is restarted, - # the data should be reimported again. volumes: - "/var/cache/bridgehead/opal-db:/var/lib/postgresql/data" @@ -90,16 +79,6 @@ services: environment: APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET} -#volumes: -# bridgehead-rstudio-config: -# name: "bridgehead-rstudio-config" -# bridgehead-rstudio-workspace: -# name: "bridgehead-rstudio-workspace" -# bridgehead-opal-db: -# name: "bridgehead-opal-db" -# bridgehead-opal: -# name: "bridgehead-opal" - secrets: opal-cert.pem: file: /tmp/bridgehead/opal-cert.pem From c33726d3855a0e696763180ea0ad132d9cf3c75b Mon Sep 17 00:00:00 2001 From: lablans Date: Fri, 15 Sep 2023 08:19:10 +0000 Subject: [PATCH 098/243] Exporter cache --- ccp/modules/exporter-compose.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 4bf6669..3b935f7 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -23,13 +23,8 @@ services: - "traefik.http.routers.exporter_ccp.tls=true" - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" - - # In this initial development state of the bridgehead, we are trying to have so many volumes as possible. - # However, in the first executions in the CCP sites, this volume seems to be very important. An export is - # a process that can take several hours. There is a risk that the bridgehead restarts, losing the already - # created export. This is specially notorious while creating a report. - # volumes: - # - "bridgehead-exporter:/app/exporter-files" + volumes: + - "/var/cache/bridgehead/exporter-files:/app/exporter-files" exporter-db: image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine From 13a74e5dab78742ec09875232fec3c7ac28c169c Mon Sep 17 00:00:00 2001 From: lablans Date: Fri, 15 Sep 2023 08:20:24 +0000 Subject: [PATCH 099/243] Move exporter db to /var/cache/bridgehead --- ccp/modules/exporter-compose.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 3b935f7..20e31f5 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -34,8 +34,8 @@ services: POSTGRES_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh POSTGRES_DB: "exporter" volumes: - # Please, do NOT REMOVE this volume until we find a solution to save Lens-queries to be executed in the explorer. - - "bridgehead-exporter-db:/var/lib/postgresql/data" + # Consider removing this volume once we find a solution to save Lens-queries to be executed in the explorer. + - "/var/cache/bridgehead/exporter-db:/var/lib/postgresql/data" reporter: image: docker.verbis.dkfz.de/ccp/dktk-reporter:latest @@ -65,11 +65,3 @@ services: - "traefik.http.routers.reporter_ccp.tls=true" - "traefik.http.middlewares.reporter_ccp_strip.stripprefix.prefixes=/ccp-reporter" - "traefik.http.routers.reporter_ccp.middlewares=reporter_ccp_strip" - -volumes: - bridgehead-exporter-db: - name: "bridgehead-exporter-db" -# bridgehead-exporter: -# name: "bridgehead-exporter" -# bridgehead-reporter: -# name: "bridgehead-reporter" From 9cdcf2afb8dd9f6bf6710ade3611bca2f7e9ebb6 Mon Sep 17 00:00:00 2001 From: lablans Date: Fri, 15 Sep 2023 08:24:44 +0000 Subject: [PATCH 100/243] Rewrite comments --- ccp/modules/login-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 5e6527b..131e739 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,7 +1,7 @@ version: "3.7" services: - ############################################ Keycloak + login-db: image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine container_name: bridgehead-login-db @@ -9,8 +9,7 @@ services: POSTGRES_USER: "keycloak" POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in login-setup.sh POSTGRES_DB: "keycloak" -# Please leave this volume commented. We are in an early stage of the development and we need to evaluate -# if the sites need to add local users or if some manual changes are required for any reason. +# Consider removing this comment once we have collected experience in production. # volumes: # - "bridgehead-login-db:/var/lib/postgresql/data" @@ -40,6 +39,7 @@ services: depends_on: - login-db +# Consider removing this comment once we have collected experience in production. #volumes: # bridgehead-login-db: # name: "bridgehead-login-db" From 77145277def7d61223793484847cb8b2ce01df60 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 15 Sep 2023 15:16:46 +0200 Subject: [PATCH 101/243] Add ccp to /var/cache/bridgehead/* volumes --- ccp/modules/datashield-compose.yml | 2 +- ccp/modules/exporter-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 4a71a90..29da4c7 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -52,7 +52,7 @@ services: POSTGRES_USER: "opal" POSTGRES_DB: "opal" volumes: - - "/var/cache/bridgehead/opal-db:/var/lib/postgresql/data" + - "/var/cache/bridgehead/ccp/opal-db:/var/lib/postgresql/data" opal-rserver: container_name: bridgehead-opal-rserver diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 20e31f5..8110e87 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -24,7 +24,7 @@ services: - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" volumes: - - "/var/cache/bridgehead/exporter-files:/app/exporter-files" + - "/var/cache/bridgehead/ccp/exporter-files:/app/exporter-files" exporter-db: image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine @@ -35,7 +35,7 @@ services: POSTGRES_DB: "exporter" volumes: # Consider removing this volume once we find a solution to save Lens-queries to be executed in the explorer. - - "/var/cache/bridgehead/exporter-db:/var/lib/postgresql/data" + - "/var/cache/bridgehead/ccp/exporter-db:/var/lib/postgresql/data" reporter: image: docker.verbis.dkfz.de/ccp/dktk-reporter:latest From 5c7da0d40dcc7eba48c1b7d5d7dec4b53470429f Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 15 Sep 2023 09:45:28 +0000 Subject: [PATCH 102/243] Auto generate mappings --- ccp/modules/datashield-mappings.json | 17 +++++++++++++++++ ccp/modules/datashield-setup.sh | 8 ++++++++ 2 files changed, 25 insertions(+) create mode 100644 ccp/modules/datashield-mappings.json diff --git a/ccp/modules/datashield-mappings.json b/ccp/modules/datashield-mappings.json new file mode 100644 index 0000000..b6d3f37 --- /dev/null +++ b/ccp/modules/datashield-mappings.json @@ -0,0 +1,17 @@ + +{ + "sites": [ + { + "id": "$SITE_ID", + "name": "$SITE_ID", + "virtualhost": "opal-$SITE_ID", + "beamconnect": "datashield-connect.$SITE_ID.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "$SITE_ID", + "name": "$SITE_ID", + "virtualhost": "opal-$SITE_ID", + "beamconnect": "datashield-connect.$SITE_ID.broker.ccp-it.dktk.dkfz.de" + } + ] +} diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 8652431..c48a112 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -12,4 +12,12 @@ if [ "$ENABLE_DATASHIELD" == true ]; then chown -R bridgehead:docker /tmp/bridgehead/ chmod g+r /tmp/bridgehead/opal-key.pem fi + mkdir -p /tmp/bridgehead/opal-map + jq -n --argfile input ./$PROJECT/modules/datashield-mappings.json ' + { + "external": "opal-'"$SITE_ID"'", + "internal": "opal:8080", + "allowed": [$input.sites[].id | "datashield-connect.\(.).broker.ccp-it.dktk.dkfz.de"] + }' > /tmp/bridgehead/opal-map/local.json + cp ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json fi From 452946aa04bcb7d569653281178f7d787a377ffa Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 15 Sep 2023 11:17:07 +0000 Subject: [PATCH 103/243] Add all sites --- ccp/modules/datashield-mappings.json | 58 ++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/ccp/modules/datashield-mappings.json b/ccp/modules/datashield-mappings.json index b6d3f37..d902b8f 100644 --- a/ccp/modules/datashield-mappings.json +++ b/ccp/modules/datashield-mappings.json @@ -2,16 +2,58 @@ { "sites": [ { - "id": "$SITE_ID", - "name": "$SITE_ID", - "virtualhost": "opal-$SITE_ID", - "beamconnect": "datashield-connect.$SITE_ID.broker.ccp-it.dktk.dkfz.de" + "id": "berlin", + "name": "berlin", + "virtualhost": "opal-berlin", + "beamconnect": "datashield-connect.berlin.broker.ccp-it.dktk.dkfz.de" }, { - "id": "$SITE_ID", - "name": "$SITE_ID", - "virtualhost": "opal-$SITE_ID", - "beamconnect": "datashield-connect.$SITE_ID.broker.ccp-it.dktk.dkfz.de" + "id": "muenchen-lmu", + "name": "muenchen-lmu", + "virtualhost": "opal-muenchen-lmu", + "beamconnect": "datashield-connect.muenchen-lmu.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "dresden", + "name": "dresden", + "virtualhost": "opal-dresden", + "beamconnect": "datashield-connect.dresden.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "freiburg", + "name": "freiburg", + "virtualhost": "opal-freiburg", + "beamconnect": "datashield-connect.freiburg.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "muenchen-tum", + "name": "muenchen-tum", + "virtualhost": "opal-muenchen-tum", + "beamconnect": "datashield-connect.muenchen-tum.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "tuebingen", + "name": "tuebingen", + "virtualhost": "opal-tuebingen", + "beamconnect": "datashield-connect.tuebingen.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "mainz", + "name": "mainz", + "virtualhost": "opal-mainz", + "beamconnect": "datashield-connect.mainz.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "frankfurt", + "name": "frankfurt", + "virtualhost": "opal-frankfurt", + "beamconnect": "datashield-connect.frankfurt.broker.ccp-it.dktk.dkfz.de" + }, + { + "id": "essen", + "name": "essen", + "virtualhost": "opal-essen", + "beamconnect": "datashield-connect.essen.broker.ccp-it.dktk.dkfz.de" } ] } From e2f31b6eebc247f22927468e3c35394c1220211f Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 15 Sep 2023 14:09:06 +0000 Subject: [PATCH 104/243] Make sure copy works and the correct owner is set --- ccp/modules/datashield-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index c48a112..8421ea2 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -9,7 +9,6 @@ if [ "$ENABLE_DATASHIELD" == true ]; then mkdir -p /tmp/bridgehead/ chown -R bridgehead:docker /tmp/bridgehead/ openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" - chown -R bridgehead:docker /tmp/bridgehead/ chmod g+r /tmp/bridgehead/opal-key.pem fi mkdir -p /tmp/bridgehead/opal-map @@ -19,5 +18,6 @@ if [ "$ENABLE_DATASHIELD" == true ]; then "internal": "opal:8080", "allowed": [$input.sites[].id | "datashield-connect.\(.).broker.ccp-it.dktk.dkfz.de"] }' > /tmp/bridgehead/opal-map/local.json - cp ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json + cp -f ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json + chown -R bridgehead:docker /tmp/bridgehead/ fi From dcddbf2235c7ba2edf1d76f0f94fba4ff2164479 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 18 Sep 2023 12:02:36 +0200 Subject: [PATCH 105/243] Bugfix: Add version of docker-compose --- bbmri/modules/directory-sync-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbmri/modules/directory-sync-compose.yml b/bbmri/modules/directory-sync-compose.yml index 9776ecb..178f46c 100644 --- a/bbmri/modules/directory-sync-compose.yml +++ b/bbmri/modules/directory-sync-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: directory_sync_service: image: "docker.verbis.dkfz.de/cache/samply/directory_sync_service" From 3a6520a6687acdaa6616e41c2a0c68590d9aa3cb Mon Sep 17 00:00:00 2001 From: djuarezgf <46350150+djuarezgf@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:04:56 +0200 Subject: [PATCH 106/243] Update ccp/modules/mtba.md Co-authored-by: Martin Lablans <6804500+lablans@users.noreply.github.com> --- ccp/modules/mtba.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ccp/modules/mtba.md b/ccp/modules/mtba.md index c8dff60..400cb4d 100644 --- a/ccp/modules/mtba.md +++ b/ccp/modules/mtba.md @@ -1,6 +1,4 @@ -# MTBA - -MTBA is referred to Molecular Tumor Board Alliance. +# Molecular Tumor Board Alliance (MTBA) In this module, the genetic data to import is stored in a directory (/tmp/bridgehead/mtba/input). A process checks regularly if there are files in the directory. The files are pseudonomized when the IDAT is provided. The files are From 3dfc4cf57dc4ad6a4f558dbf448d83ad012aa01a Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 19 Sep 2023 14:21:15 +0200 Subject: [PATCH 107/243] Postgres 15.4 in datashield, exporter and login --- ccp/modules/datashield-compose.yml | 2 +- ccp/modules/exporter-compose.yml | 2 +- ccp/modules/login-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 29da4c7..5b0efde 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -46,7 +46,7 @@ services: opal-db: container_name: bridgehead-opal-db - image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine + image: docker.verbis.dkfz.de/cache/postgres:15.4-alpine environment: POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" # Set in datashield-setup.sh POSTGRES_USER: "opal" diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 8110e87..7fe19de 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -27,7 +27,7 @@ services: - "/var/cache/bridgehead/ccp/exporter-files:/app/exporter-files" exporter-db: - image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine + image: docker.verbis.dkfz.de/cache/postgres:15.4-alpine container_name: bridgehead-ccp-exporter-db environment: POSTGRES_USER: "exporter" diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 131e739..64b8918 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: login-db: - image: docker.verbis.dkfz.de/cache/postgres:15.1-alpine + image: docker.verbis.dkfz.de/cache/postgres:15.4-alpine container_name: bridgehead-login-db environment: POSTGRES_USER: "keycloak" From c8fc35576e1f09f67f673a7a0142d0c20a2fbf5a Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 19 Sep 2023 16:39:32 +0200 Subject: [PATCH 108/243] Bugfix: Exporter and Reporter /var/cache volumes --- ccp/modules/exporter-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 7fe19de..2b9b4e9 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -24,7 +24,7 @@ services: - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter" - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip" volumes: - - "/var/cache/bridgehead/ccp/exporter-files:/app/exporter-files" + - "/var/cache/bridgehead/ccp/exporter-files:/app/exporter-files/output" exporter-db: image: docker.verbis.dkfz.de/cache/postgres:15.4-alpine @@ -56,8 +56,8 @@ services: # a process that can take several hours, because it depends on the exporter. # There is a risk that the bridgehead restarts, losing the already created export. - # volumes: - # - "bridgehead-reporter:/app/reports" + volumes: + - "/var/cache/bridgehead/ccp/reporter-files:/app/reports" labels: - "traefik.enable=true" - "traefik.http.routers.reporter_ccp.rule=PathPrefix(`/ccp-reporter`)" From 2237562e6ee9793fa885b49bb50fc66da3ab3ffc Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Fri, 22 Sep 2023 10:24:10 +0200 Subject: [PATCH 109/243] Prevent anonymous volume creation --- ccp/modules/login-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 64b8918..47debb4 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -9,6 +9,8 @@ services: POSTGRES_USER: "keycloak" POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in login-setup.sh POSTGRES_DB: "keycloak" + tmpfs: + - /var/lib/postgresql/data # Consider removing this comment once we have collected experience in production. # volumes: # - "bridgehead-login-db:/var/lib/postgresql/data" From c1020c569a20ecf2d105d6ef83cfcb0e210979eb Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 22 Sep 2023 15:36:21 +0200 Subject: [PATCH 110/243] Bugfix: datashield local.json as array --- ccp/modules/datashield-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 8421ea2..3220c30 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -13,11 +13,11 @@ if [ "$ENABLE_DATASHIELD" == true ]; then fi mkdir -p /tmp/bridgehead/opal-map jq -n --argfile input ./$PROJECT/modules/datashield-mappings.json ' - { + [{ "external": "opal-'"$SITE_ID"'", "internal": "opal:8080", "allowed": [$input.sites[].id | "datashield-connect.\(.).broker.ccp-it.dktk.dkfz.de"] - }' > /tmp/bridgehead/opal-map/local.json + }]' > /tmp/bridgehead/opal-map/local.json cp -f ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json chown -R bridgehead:docker /tmp/bridgehead/ fi From 0039efa353df5b8fe24c2a8419a8c979ad4a1f12 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 22 Sep 2023 17:09:47 +0200 Subject: [PATCH 111/243] Add docu about login in teiler --- ccp/modules/teiler.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccp/modules/teiler.md b/ccp/modules/teiler.md index fda0dbd..51e94e4 100644 --- a/ccp/modules/teiler.md +++ b/ccp/modules/teiler.md @@ -11,7 +11,9 @@ This microfrontends can run as single alone but need an extension with Single-SP There are also available three templates (Angular, Vue, React) to be directly extended to be used directly in the teiler. ## Teiler Dashboard -It consists on the main dashboard and a set of embedded services. +It consists on the main dashboard and a set of embedded services. +### Login +user and password in ccp.local.conf ## Teiler Backend In this component, the microfrontends are configured. From 89c90d3aa0a98979c01011aee085151d37cd2e1a Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 26 Sep 2023 07:45:51 +0200 Subject: [PATCH 112/243] /var/cache for mtba --- ccp/modules/mtba-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index eace283..670d990 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -27,8 +27,8 @@ services: - "traefik.http.middlewares.mtba_ccp_strip.stripprefix.prefixes=/mtba" - "traefik.http.routers.mtba_ccp.middlewares=mtba_ccp_strip, auth" volumes: - - /tmp/bridgehead/mtba/input:/app/input - - /tmp/bridgehead/mtba/persist:/app/persist + - /var/cache/bridgehead/ccp/mtba/input:/app/input + - /var/cache/bridgehead/ccp/mtba/persist:/app/persist # TODO: Include CBioPortal in Deployment ... # NOTE: CBioPortal can't load data while the system is running. So after import of data bridgehead needs to be restarted! From ec6407414b734bcdca07610b6a7d9f623f4f9d63 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 10 Oct 2023 15:48:11 +0200 Subject: [PATCH 113/243] Update export template script: FHIR_QUERY to FHIR_PATH --- ccp/modules/export-and-qb.curl-templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/export-and-qb.curl-templates b/ccp/modules/export-and-qb.curl-templates index 0853ed9..739c5af 100644 --- a/ccp/modules/export-and-qb.curl-templates +++ b/ccp/modules/export-and-qb.curl-templates @@ -1,5 +1,5 @@ # Full Excel Export -curl --location --request POST 'https://${HOST}/ccp-exporter/request?query=Patient&query-format=FHIR_QUERY&template-id=ccp&output-format=EXCEL' \ +curl --location --request POST 'https://${HOST}/ccp-exporter/request?query=Patient&query-format=FHIR_PATH&template-id=ccp&output-format=EXCEL' \ --header 'x-api-key: ${EXPORT_API_KEY}' # QB From 058d1c83e619b084492035eda87852e727dcaf90 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 23 Oct 2023 12:33:56 +0000 Subject: [PATCH 114/243] Use newest version of `beam-connect` --- ccp/modules/datashield-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 5b0efde..d121e63 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -59,7 +59,7 @@ services: image: docker.verbis.dkfz.de/cache/datashield/rock-base:6.3 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 beam-connect: - image: docker.verbis.dkfz.de/cache/samply/beam-connect:no-auth + image: docker.verbis.dkfz.de/cache/samply/beam-connect:develop container_name: bridgehead-datashield-connect environment: PROXY_URL: "http://beam-proxy:8081" @@ -68,6 +68,7 @@ services: PROXY_APIKEY: ${DATASHIELD_CONNECT_SECRET} DISCOVERY_URL: "./map/central.json" LOCAL_TARGETS_FILE: "./map/local.json" + NO_AUTH: "true" secrets: - opal-cert.pem depends_on: From 876c4efa41a9543eeea5055762b7dfe65a34823d Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 24 Oct 2023 09:25:36 +0200 Subject: [PATCH 115/243] Make Opal use proxy server --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index d121e63..64a37fb 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -28,7 +28,7 @@ services: - opal-rserver - opal-db environment: - JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC" + JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC -Dhttps.proxyHost=${HTTPS_PROXY_HOST} -Dhttps.proxyPort=${HTTPS_PROXY_PORT} -Dhttps.proxyUser=${HTTPS_PROXY_USERNAME} -Dhttps.proxyPassword=${HTTPS_PROXY_PASSWORD}" # OPAL_ADMINISTRATOR_USER: "administrator" # This line is kept for informational purposes OPAL_ADMINISTRATOR_PASSWORD: "${LDM_AUTH}" POSTGRESDATA_HOST: "opal-db" From 77240ff92faa893f32361260db9b04ca73b8f400 Mon Sep 17 00:00:00 2001 From: lablans Date: Tue, 24 Oct 2023 08:33:15 +0000 Subject: [PATCH 116/243] Use Bridgehead's internal http proxy --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 64a37fb..ad5ce98 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -28,7 +28,7 @@ services: - opal-rserver - opal-db environment: - JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC -Dhttps.proxyHost=${HTTPS_PROXY_HOST} -Dhttps.proxyPort=${HTTPS_PROXY_PORT} -Dhttps.proxyUser=${HTTPS_PROXY_USERNAME} -Dhttps.proxyPassword=${HTTPS_PROXY_PASSWORD}" + JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC -Dhttps.proxyHost=forward_proxy -Dhttps.proxyPort=3128" # OPAL_ADMINISTRATOR_USER: "administrator" # This line is kept for informational purposes OPAL_ADMINISTRATOR_PASSWORD: "${LDM_AUTH}" POSTGRESDATA_HOST: "opal-db" From 178867cde7347fab2c16ca3d8554f84c570cb7b0 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Wed, 25 Oct 2023 12:33:56 +0200 Subject: [PATCH 117/243] Prevent creation of volumes --- ccp/modules/datashield-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index ad5ce98..d6bb477 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -43,6 +43,8 @@ services: secrets: - opal-cert.pem - opal-key.pem + tmpfs: + - /srv opal-db: container_name: bridgehead-opal-db @@ -57,6 +59,8 @@ services: opal-rserver: container_name: bridgehead-opal-rserver image: docker.verbis.dkfz.de/cache/datashield/rock-base:6.3 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 + tmpfs: + - /srv beam-connect: image: docker.verbis.dkfz.de/cache/samply/beam-connect:develop From a2c242583eadaa18b05e029cd7f41c8c8b1ce8e7 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 3 Nov 2023 17:03:15 +0100 Subject: [PATCH 118/243] Remove nngmSetup in vars --- ccp/vars | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/vars b/ccp/vars index c106027..f7c5d2b 100644 --- a/ccp/vars +++ b/ccp/vars @@ -22,6 +22,5 @@ do done idManagementSetup -nngmSetup mtbaSetup adt2fhirRestSetup From 8d4f487806756ae6079008a93a3effd24e1d23fa Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 29 Nov 2023 12:43:58 +0100 Subject: [PATCH 119/243] MTBA 1.0.0 --- ccp/modules/teiler-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index 68f378c..f0b0d60 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -42,6 +42,8 @@ services: TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler" TEILER_DASHBOARD_HTTP_RELATIVE_PATH: "/ccp-teiler-dashboard" TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler" + TEILER_USER: "TEILER_USER" + TEILER_ADMIN: "TEILER_ADMIN" teiler-backend: image: docker.verbis.dkfz.de/ccp/dktk-teiler-backend:latest From 90ee8d63f7adad81cffba6acca60d6a8535b8e07 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Fri, 9 Feb 2024 17:14:45 +0100 Subject: [PATCH 120/243] Externalize postgres version --- ccp/modules/datashield-compose.yml | 2 +- ccp/modules/exporter-compose.yml | 2 +- ccp/modules/id-management-compose.yml | 2 +- ccp/modules/login-compose.yml | 2 +- ccp/vars | 1 + 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index d6bb477..edca65c 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -48,7 +48,7 @@ services: opal-db: container_name: bridgehead-opal-db - image: docker.verbis.dkfz.de/cache/postgres:15.4-alpine + image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} environment: POSTGRES_PASSWORD: "${OPAL_DB_PASSWORD}" # Set in datashield-setup.sh POSTGRES_USER: "opal" diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 2b9b4e9..5922690 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -27,7 +27,7 @@ services: - "/var/cache/bridgehead/ccp/exporter-files:/app/exporter-files/output" exporter-db: - image: docker.verbis.dkfz.de/cache/postgres:15.4-alpine + image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} container_name: bridgehead-ccp-exporter-db environment: POSTGRES_USER: "exporter" diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 2e922cb..06d408f 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -44,7 +44,7 @@ services: - patientlist-db patientlist-db: - image: docker.verbis.dkfz.de/cache/postgres:15.6-alpine + image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} container_name: bridgehead-patientlist-db environment: POSTGRES_USER: "mainzelliste" diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 47debb4..787d4b2 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: login-db: - image: docker.verbis.dkfz.de/cache/postgres:15.4-alpine + image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} container_name: bridgehead-login-db environment: POSTGRES_USER: "keycloak" diff --git a/ccp/vars b/ccp/vars index f7c5d2b..8ca411d 100644 --- a/ccp/vars +++ b/ccp/vars @@ -14,6 +14,7 @@ ENABLE_LOGIN=true ENABLE_TEILER=true #ENABLE_DATASHIELD=true +POSTGRES_TAG=15.6-alpine for module in $PROJECT/modules/*.sh do From afb63306a8bd663689920303bd925f892d0fd227 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 3 Aug 2023 10:35:41 +0200 Subject: [PATCH 121/243] Remove unnecessary version of docker-compose.override files --- ccp/modules/datashield-compose.yml | 2 - ccp/modules/dnpm-compose.yml | 2 - ccp/modules/exporter-compose.yml | 2 - ccp/modules/login-compose.yml | 2 - ccp/modules/mtba-compose.yml | 2 - ccp/modules/teiler-ui-compose.yml | 72 ++++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 ccp/modules/teiler-ui-compose.yml diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index edca65c..bd486ea 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: rstudio: container_name: bridgehead-rstudio diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index c32426f..061e010 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: beam-proxy: environment: diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 5922690..adfce0b 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: exporter: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 787d4b2..db979b5 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: login-db: diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 670d990..c6cf474 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: mtba: image: docker.verbis.dkfz.de/cache/samply/mtba:1.0.0 diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml new file mode 100644 index 0000000..92a98d8 --- /dev/null +++ b/ccp/modules/teiler-ui-compose.yml @@ -0,0 +1,72 @@ +services: + + teiler-root-config: + image: docker.verbis.dkfz.de/cache/samply/teiler-root-config:develop + container_name: bridgehead-teiler-root-config + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler`)" + - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" + - "traefik.http.routers.teiler_root_config_ccp.tls=true" + - "traefik.http.middlewares.teiler_root_config_ccp_strip.stripprefix.prefixes=/ccp-teiler" + - "traefik.http.routers.teiler_root_config_ccp.middlewares=teiler_root_config_ccp_strip" + environment: + TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" + TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" + DEFAULT_LANGUAGE: "de" + HTTP_RELATIVE_PATH: "/ccp-teiler" + + teiler-ui: + image: docker.verbis.dkfz.de/cache/samply/teiler-ui:develop + container_name: bridgehead-teiler-ui + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_ui_ccp.rule=PathPrefix(`/ccp-teiler-ui`)" + - "traefik.http.services.teiler_ui_ccp.loadbalancer.server.port=80" + - "traefik.http.routers.teiler_ui_ccp.tls=true" + - "traefik.http.middlewares.teiler_ui_ccp_strip.stripprefix.prefixes=/ccp-teiler-ui" + - "traefik.http.routers.teiler_ui_ccp.middlewares=teiler_ui_ccp_strip" + environment: + DEFAULT_LANGUAGE: "DE" + TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" + KEYCLOAK_URL: "https://${HOST}/login" + KEYCLOAK_REALM: "teiler-ui" + KEYCLOAK_CLIENT_ID: "teiler-ui" + TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" + TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" + TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" + TEILER_PROJECT: "${PROJECT}" + EXPORTER_API_KEY: "${EXPORTER_API_KEY}" + TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" + TEILER_UI_HTTP_RELATIVE_PATH: "/ccp-teiler-ui" + TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" + + teiler-core: + image: docker.verbis.dkfz.de/ccp/dktk-teiler-core:latest + container_name: bridgehead-teiler-core + labels: + - "traefik.enable=true" + - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" + - "traefik.http.services.teiler_core_ccp.loadbalancer.server.port=8085" + - "traefik.http.routers.teiler_core_ccp.tls=true" + - "traefik.http.middlewares.teiler_core_ccp_strip.stripprefix.prefixes=/ccp-teiler-core" + - "traefik.http.routers.teiler_core_ccp.middlewares=teiler_core_ccp_strip" + environment: + LOG_LEVEL: "INFO" + APPLICATION_PORT: "8085" + APPLICATION_ADDRESS: "${HOST}" + DEFAULT_LANGUAGE: "DE" + CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" + TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" + TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" + TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" + TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" + TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" + CENTRAX_URL: "${CENTRAXX_URL}" + IS_DKTK_SITE: "${IS_DKTK_SITE}" + secrets: + - ccp.conf + +secrets: + ccp.conf: + file: /etc/bridgehead/ccp.conf From b87d746a20994b7658b89735fe7227fc1670cf19 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 3 Aug 2023 10:35:41 +0200 Subject: [PATCH 122/243] Remove unnecessary version of docker-compose.override files --- ccp/modules/teiler-ui-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 92a98d8..5a51c8e 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: teiler-root-config: From d3edb5e1439cdb6b9ee16539634ed584ddb7dccc Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 11 Sep 2023 13:27:08 +0200 Subject: [PATCH 123/243] Bugfix: Add version in every docker compose file --- ccp/modules/datashield-compose.yml | 2 ++ ccp/modules/dnpm-compose.yml | 2 ++ ccp/modules/exporter-compose.yml | 2 ++ ccp/modules/login-compose.yml | 2 ++ ccp/modules/mtba-compose.yml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index bd486ea..edca65c 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: rstudio: container_name: bridgehead-rstudio diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index 061e010..c32426f 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: beam-proxy: environment: diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index adfce0b..5922690 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: exporter: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index db979b5..787d4b2 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: login-db: diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index c6cf474..670d990 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: mtba: image: docker.verbis.dkfz.de/cache/samply/mtba:1.0.0 From 8e171b71de524df3fe2da232d9b62fd288ca0794 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 3 Aug 2023 10:35:41 +0200 Subject: [PATCH 124/243] Remove unnecessary version of docker-compose.override files --- ccp/modules/datashield-compose.yml | 2 -- ccp/modules/dnpm-compose.yml | 2 -- ccp/modules/exporter-compose.yml | 2 -- ccp/modules/login-compose.yml | 2 -- ccp/modules/mtba-compose.yml | 2 -- 5 files changed, 10 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index edca65c..bd486ea 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: rstudio: container_name: bridgehead-rstudio diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index c32426f..061e010 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: beam-proxy: environment: diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 5922690..adfce0b 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: exporter: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 787d4b2..db979b5 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: login-db: diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 670d990..c6cf474 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: mtba: image: docker.verbis.dkfz.de/cache/samply/mtba:1.0.0 From 3d136959e7d4c768d36b00680b27c9b8711ca542 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 11 Sep 2023 13:27:08 +0200 Subject: [PATCH 125/243] Bugfix: Add version in every docker compose file --- ccp/modules/datashield-compose.yml | 2 ++ ccp/modules/dnpm-compose.yml | 2 ++ ccp/modules/exporter-compose.yml | 2 ++ ccp/modules/login-compose.yml | 2 ++ ccp/modules/mtba-compose.yml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index bd486ea..edca65c 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: rstudio: container_name: bridgehead-rstudio diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index 061e010..c32426f 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: beam-proxy: environment: diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index adfce0b..5922690 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: exporter: image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index db979b5..787d4b2 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: login-db: diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index c6cf474..670d990 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -1,3 +1,5 @@ +version: "3.7" + services: mtba: image: docker.verbis.dkfz.de/cache/samply/mtba:1.0.0 From cec3dfe4cd742f6223ee4bda399237a1ef230837 Mon Sep 17 00:00:00 2001 From: janskiba Date: Tue, 7 Nov 2023 14:55:26 +0000 Subject: [PATCH 126/243] Add secret sync to the bridgehead --- bridgehead | 1 + lib/functions.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/bridgehead b/bridgehead index 31a838e..2740209 100755 --- a/bridgehead +++ b/bridgehead @@ -82,6 +82,7 @@ loadVars() { export FOCUS_TAG=main ;; esac + sync_secrets } case "$ACTION" in diff --git a/lib/functions.sh b/lib/functions.sh index 6a45d35..2bbf155 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -239,3 +239,35 @@ add_basic_auth_user() { log DEBUG "Saving clear text credentials in $FILE. If wanted, delete them manually." sed -i "/^$NAME/ s|$|\n# User: $USER\n# Password: $PASSWORD|" $FILE } + +SECRET_SYNC_ARGS=${SECRET_SYNC_ARGS:-""} +# First argument is the variable name that will be generated. +# Second argument is a comma seperated list of allowed redirect urls for the oidc client. +function generate_oidc_client() { + local delimiter=$'\x1E' + if [[ $SECRET_SYNC_ARGS == "" ]]; then + SECRET_SYNC_ARGS+="OIDC:$1:$2" + else + SECRET_SYNC_ARGS+="${delimiter}OIDC:$1:$2" + fi +} + +function sync_secrets() { + if [[ $SECRET_SYNC_ARGS == "" ]]; then + return + fi + # The oidc provider will need to be switched based on the project at some point I guess + docker run --rm \ + -v /var/cache/bridgehead/secrets:/usr/local/cache \ + -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \ + -v ./$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ + -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \ + -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \ + -e HTTPS_PROXY=$HTTPS_PROXY_FULL_URL \ + -e PROXY_ID=$PROXY_ID \ + -e BROKER_URL=$BROKER_URL \ + -e OIDC_PROVIDER=secret-sync.central.$BROKER_ID \ + -e SECRET_DEFINITIONS=$SECRET_SYNC_ARGS \ + docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest + source /var/cache/bridgehead/secrets/* +} From f854ab58ce6b2f7fd851963e438dd9025d6b827f Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 13 Nov 2023 16:22:23 +0000 Subject: [PATCH 127/243] Update to new secret-sync semantics --- lib/functions.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 2bbf155..d46353c 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -241,14 +241,27 @@ add_basic_auth_user() { } SECRET_SYNC_ARGS=${SECRET_SYNC_ARGS:-""} -# First argument is the variable name that will be generated. +# First argument is the variable name that will be generated it will not have a value. # Second argument is a comma seperated list of allowed redirect urls for the oidc client. -function generate_oidc_client() { +# The resulting client id will be $SITE_ID-public +function generate_public_oidc_client() { local delimiter=$'\x1E' if [[ $SECRET_SYNC_ARGS == "" ]]; then - SECRET_SYNC_ARGS+="OIDC:$1:$2" + SECRET_SYNC_ARGS+="OIDC:$1:public;$2" else - SECRET_SYNC_ARGS+="${delimiter}OIDC:$1:$2" + SECRET_SYNC_ARGS+="${delimiter}OIDC:$1:public;$2" + fi +} + +# First argument is the variable name that the client secret will be avalible at. +# Second argument is a comma seperated list of allowed redirect urls for the oidc client. +# The resulting client id will be $SITE_ID-private +function generate_private_oidc_client() { + local delimiter=$'\x1E' + if [[ $SECRET_SYNC_ARGS == "" ]]; then + SECRET_SYNC_ARGS+="OIDC:$1:private;$2" + else + SECRET_SYNC_ARGS+="${delimiter}OIDC:$1:private;$2" fi } From 41153199560b9814c27013c18f357aa3e26b6037 Mon Sep 17 00:00:00 2001 From: Jan Skiba Date: Thu, 16 Nov 2023 14:21:19 +0100 Subject: [PATCH 128/243] Setup hostname earlier --- bridgehead | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgehead b/bridgehead index 2740209..a1cb708 100755 --- a/bridgehead +++ b/bridgehead @@ -50,6 +50,7 @@ loadVars() { source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import" fi fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" + setHostname [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a @@ -64,7 +65,6 @@ loadVars() { OVERRIDE+=" -f ./$PROJECT/docker-compose.override.yml" fi detectCompose - setHostname setupProxy # Set some project-independent default values From 93a91326a2a9ae23f6b8e88bda94c6eebf75a5cb Mon Sep 17 00:00:00 2001 From: Jan Skiba Date: Thu, 16 Nov 2023 14:24:41 +0100 Subject: [PATCH 129/243] Make sure path exists --- lib/functions.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index d46353c..cc55643 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -269,9 +269,11 @@ function sync_secrets() { if [[ $SECRET_SYNC_ARGS == "" ]]; then return fi + mkdir -p /var/cache/bridgehead/secrets/ + touch /var/cache/bridgehead/secrets/oidc # The oidc provider will need to be switched based on the project at some point I guess docker run --rm \ - -v /var/cache/bridgehead/secrets:/usr/local/cache \ + -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \ -v ./$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \ @@ -279,7 +281,7 @@ function sync_secrets() { -e HTTPS_PROXY=$HTTPS_PROXY_FULL_URL \ -e PROXY_ID=$PROXY_ID \ -e BROKER_URL=$BROKER_URL \ - -e OIDC_PROVIDER=secret-sync.central.$BROKER_ID \ + -e OIDC_PROVIDER=secret-sync-central.oidc.$BROKER_ID \ -e SECRET_DEFINITIONS=$SECRET_SYNC_ARGS \ docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest source /var/cache/bridgehead/secrets/* From dc3d5496e137dc3569c4fc09f141844e29b6bb1c Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 17 Nov 2023 10:27:12 +0100 Subject: [PATCH 130/243] Integrate central Keycloak in Teiler --- ccp/modules/datashield-compose.yml | 14 +++++++++++--- ccp/modules/datashield-setup.sh | 6 +++++- ccp/modules/exporter-compose.yml | 2 +- ccp/modules/teiler-compose.yml | 11 ++++++----- ccp/modules/teiler-setup.sh | 1 + ccp/vars | 3 ++- lib/functions.sh | 10 ++++++++-- 7 files changed, 34 insertions(+), 13 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index edca65c..a126b1a 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -6,7 +6,8 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: #DEFAULT_USER: "rstudio" # This line is kept for informational purposes - PASSWORD: "${LDM_AUTH}" + #PASSWORD: "${LDM_AUTH}" + DISABLE_AUTH: "true" # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. HTTP_RELATIVE_PATH: "/rstudio" labels: - "traefik.enable=true" @@ -14,7 +15,7 @@ services: - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787" - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" + - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip,auth" opal: container_name: bridgehead-opal @@ -30,7 +31,7 @@ services: environment: JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC -Dhttps.proxyHost=forward_proxy -Dhttps.proxyPort=3128" # OPAL_ADMINISTRATOR_USER: "administrator" # This line is kept for informational purposes - OPAL_ADMINISTRATOR_PASSWORD: "${LDM_AUTH}" + OPAL_ADMINISTRATOR_PASSWORD: "${OPAL_ADMIN_PASSWORD}" POSTGRESDATA_HOST: "opal-db" POSTGRESDATA_DATABASE: "opal" POSTGRESDATA_USER: "opal" @@ -40,6 +41,13 @@ services: APP_CONTEXT_PATH: "/opal" OPAL_PRIVATE_KEY: "/run/secrets/opal-key.pem" OPAL_CERTIFICATE: "/run/secrets/opal-cert.pem" + KEYCLOAK_URL: "https://login.verbis.dkfz.de" + KEYCLOAK_REALM: "test-realm-01" + KEYCLOAK_CLIENT_ID: "${SITE_ID}-private" + KEYCLOAK_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}" + KEYCLOAK_ADMIN_GROUP: "${KEYCLOAK_ADMIN_GROUP}" + TOKEN_MANAGER_PASSWORD: "${TOKEN_MANAGER_OPAL_PASSWORD}" + EXPORTER_PASSWORD: "${EXPORTER_OPAL_PASSWORD}" secrets: - opal-cert.pem - opal-key.pem diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 3220c30..5f8fac4 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -3,7 +3,10 @@ if [ "$ENABLE_DATASHIELD" == true ]; then log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" - OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + EXPORTER_OPAL_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal DB. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + TOKEN_MANAGER_OPAL_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Token Manager in Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal DB. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + OPAL_ADMIN_PASSWORD="$(echo \"This is a salt string to generate one consistent admin password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ @@ -20,4 +23,5 @@ if [ "$ENABLE_DATASHIELD" == true ]; then }]' > /tmp/bridgehead/opal-map/local.json cp -f ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json chown -R bridgehead:docker /tmp/bridgehead/ + generate_private_oidc_client "OIDC_CLIENT_SECRET" "https://${HOST}/opal/*" fi diff --git a/ccp/modules/exporter-compose.yml b/ccp/modules/exporter-compose.yml index 5922690..d5eb227 100644 --- a/ccp/modules/exporter-compose.yml +++ b/ccp/modules/exporter-compose.yml @@ -15,7 +15,7 @@ services: HTTP_RELATIVE_PATH: "/ccp-exporter" SITE: "${SITE_ID}" HTTP_SERVLET_REQUEST_SCHEME: "https" - OPAL_ADMINISTRATOR_PASSWORD: "${LDM_AUTH}" + OPAL_PASSWORD: "${EXPORTER_OPAL_PASSWORD}" labels: - "traefik.enable=true" - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)" diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index f0b0d60..659c9e2 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -31,9 +31,10 @@ services: environment: DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend" - KEYCLOAK_URL: "https://${HOST}/login" - KEYCLOAK_REALM: "teiler" - KEYCLOAK_CLIENT_ID: "teiler" + KEYCLOAK_URL: "https://login.verbis.dkfz.de" + KEYCLOAK_REALM: "test-realm-01" + KEYCLOAK_CLIENT_ID: "${SITE_ID}-public" + KEYCLOAK_TOKEN_GROUP: "groups" TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" @@ -42,8 +43,8 @@ services: TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler" TEILER_DASHBOARD_HTTP_RELATIVE_PATH: "/ccp-teiler-dashboard" TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler" - TEILER_USER: "TEILER_USER" - TEILER_ADMIN: "TEILER_ADMIN" + TEILER_USER: "${KEYCLOAK_USER_GROUP}" + TEILER_ADMIN: "${KEYCLOAK_ADMIN_GROUP}" teiler-backend: image: docker.verbis.dkfz.de/ccp/dktk-teiler-backend:latest diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index d1caebe..e930a7e 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -3,4 +3,5 @@ if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler setup detected -- will start Teiler services." OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" + generate_public_oidc_client "OIDC_PUBLIC" "https://${HOST}/ccp-teiler/*" fi diff --git a/ccp/vars b/ccp/vars index 8ca411d..b133cf4 100644 --- a/ccp/vars +++ b/ccp/vars @@ -10,10 +10,11 @@ BROKER_URL_FOR_PREREQ=$BROKER_URL DEFAULT_LANGUAGE=DE DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} ENABLE_EXPORTER=true -ENABLE_LOGIN=true ENABLE_TEILER=true #ENABLE_DATASHIELD=true +KEYCLOAK_USER_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})" +KEYCLOAK_ADMIN_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})_Verwalter" POSTGRES_TAG=15.6-alpine for module in $PROJECT/modules/*.sh diff --git a/lib/functions.sh b/lib/functions.sh index cc55643..72bd8e8 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -275,14 +275,20 @@ function sync_secrets() { docker run --rm \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \ - -v ./$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ + -v /srv/docker/bridgehead/$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \ -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \ -e HTTPS_PROXY=$HTTPS_PROXY_FULL_URL \ -e PROXY_ID=$PROXY_ID \ -e BROKER_URL=$BROKER_URL \ - -e OIDC_PROVIDER=secret-sync-central.oidc.$BROKER_ID \ + -e OIDC_PROVIDER=secret-sync-central.dev-jan.$BROKER_ID \ -e SECRET_DEFINITIONS=$SECRET_SYNC_ARGS \ docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest source /var/cache/bridgehead/secrets/* } + +capitalize_first_letter() { + input="$1" + capitalized="$(tr '[:lower:]' '[:upper:]' <<< ${input:0:1})${input:1}" + echo "$capitalized" +} From 0015365d1b72697cd2f681520d82f7e82ef2afdd Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 20 Nov 2023 11:34:18 +0000 Subject: [PATCH 131/243] Generate addtional redirect url --- ccp/modules/teiler-setup.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index e930a7e..17b19bd 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -3,5 +3,10 @@ if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler setup detected -- will start Teiler services." OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" - generate_public_oidc_client "OIDC_PUBLIC" "https://${HOST}/ccp-teiler/*" + redirect_urls="https://${HOST}/ccp-teiler/*" + host_without_proxy="$(echo "$HOST" | cut -d '.' -f1)" + if [[ "$HOST" != "$host_without_proxy" ]]; then + redirect_urls+=",https://$host_without_proxy/ccp-teiler/*" + fi + generate_public_oidc_client "OIDC_PUBLIC" "$redirect_urls" fi From 3c8ec73ac341dbbf2203cd02f13c9373dec70804 Mon Sep 17 00:00:00 2001 From: janskiba Date: Tue, 21 Nov 2023 10:39:17 +0000 Subject: [PATCH 132/243] Update oidc provider to new url --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index 72bd8e8..e0367e2 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -281,7 +281,7 @@ function sync_secrets() { -e HTTPS_PROXY=$HTTPS_PROXY_FULL_URL \ -e PROXY_ID=$PROXY_ID \ -e BROKER_URL=$BROKER_URL \ - -e OIDC_PROVIDER=secret-sync-central.dev-jan.$BROKER_ID \ + -e OIDC_PROVIDER=secret-sync-central.oidc-client-enrollment.$BROKER_ID \ -e SECRET_DEFINITIONS=$SECRET_SYNC_ARGS \ docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest source /var/cache/bridgehead/secrets/* From bb076c5d5aa6bd90c1074dfdd4927d2fdd21b1a1 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 23 Nov 2023 10:38:50 +0100 Subject: [PATCH 133/243] Add function generate_redirect_urls --- ccp/modules/datashield-setup.sh | 2 +- ccp/modules/teiler-setup.sh | 7 +------ lib/functions.sh | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 5f8fac4..420a450 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -23,5 +23,5 @@ if [ "$ENABLE_DATASHIELD" == true ]; then }]' > /tmp/bridgehead/opal-map/local.json cp -f ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json chown -R bridgehead:docker /tmp/bridgehead/ - generate_private_oidc_client "OIDC_CLIENT_SECRET" "https://${HOST}/opal/*" + generate_private_oidc_client "OIDC_CLIENT_SECRET" "$(generate_redirect_urls '/opal/*')" fi diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index 17b19bd..0da6f6f 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -3,10 +3,5 @@ if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler setup detected -- will start Teiler services." OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" - redirect_urls="https://${HOST}/ccp-teiler/*" - host_without_proxy="$(echo "$HOST" | cut -d '.' -f1)" - if [[ "$HOST" != "$host_without_proxy" ]]; then - redirect_urls+=",https://$host_without_proxy/ccp-teiler/*" - fi - generate_public_oidc_client "OIDC_PUBLIC" "$redirect_urls" + generate_public_oidc_client "OIDC_PUBLIC" "$(generate_redirect_urls '/ccp-teiler/*')" fi diff --git a/lib/functions.sh b/lib/functions.sh index e0367e2..78317fd 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -292,3 +292,18 @@ capitalize_first_letter() { capitalized="$(tr '[:lower:]' '[:upper:]' <<< ${input:0:1})${input:1}" echo "$capitalized" } + +generate_redirect_urls(){ + local redirect_urls="https://${HOST}$1" + local host_without_proxy="$(echo "$HOST" | cut -d '.' -f1)" + local port="$(echo "$HOST" | rev | cut -d ':' -f1 | rev)" + if [ -z "${port}" ]; then + port="" + else + port=":$port" + fi + if [[ "$HOST" != "$host_without_proxy" ]]; then + redirect_urls+=",https://$host_without_proxy$port$1" + fi + echo "$redirect_urls" +} From 043e12b9854c292356cbce827577bb9dec12def8 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 23 Nov 2023 09:58:34 +0000 Subject: [PATCH 134/243] Remove port handeling when generating redirect url --- lib/functions.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 78317fd..0cb5aba 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -296,12 +296,6 @@ capitalize_first_letter() { generate_redirect_urls(){ local redirect_urls="https://${HOST}$1" local host_without_proxy="$(echo "$HOST" | cut -d '.' -f1)" - local port="$(echo "$HOST" | rev | cut -d ':' -f1 | rev)" - if [ -z "${port}" ]; then - port="" - else - port=":$port" - fi if [[ "$HOST" != "$host_without_proxy" ]]; then redirect_urls+=",https://$host_without_proxy$port$1" fi From 131b52f57b77ee0be127a4d1cb61fc43bc0f502d Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 23 Nov 2023 10:28:43 +0000 Subject: [PATCH 135/243] Account for ip address host values --- lib/functions.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 0cb5aba..0c93862 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -242,7 +242,7 @@ add_basic_auth_user() { SECRET_SYNC_ARGS=${SECRET_SYNC_ARGS:-""} # First argument is the variable name that will be generated it will not have a value. -# Second argument is a comma seperated list of allowed redirect urls for the oidc client. +# Second argument is a comma separated list of allowed redirect urls for the oidc client. # The resulting client id will be $SITE_ID-public function generate_public_oidc_client() { local delimiter=$'\x1E' @@ -253,8 +253,8 @@ function generate_public_oidc_client() { fi } -# First argument is the variable name that the client secret will be avalible at. -# Second argument is a comma seperated list of allowed redirect urls for the oidc client. +# First argument is the variable name that the client secret will be available at. +# Second argument is a comma separated list of allowed redirect urls for the oidc client. # The resulting client id will be $SITE_ID-private function generate_private_oidc_client() { local delimiter=$'\x1E' @@ -293,11 +293,15 @@ capitalize_first_letter() { echo "$capitalized" } +# Generate a string of ',' separated string of redirect urls relative to $HOST. +# $1 will be appended to the url +# If the host looks like dev-jan.inet.dkfz-heidelberg.de it will generate urls with dev-jan and the original $HOST as url Authorities generate_redirect_urls(){ local redirect_urls="https://${HOST}$1" local host_without_proxy="$(echo "$HOST" | cut -d '.' -f1)" - if [[ "$HOST" != "$host_without_proxy" ]]; then - redirect_urls+=",https://$host_without_proxy$port$1" + # Only append second url if its different and the host is not an ip address + if [[ "$HOST" != "$host_without_proxy" && ! "$HOST" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + redirect_urls+=",https://$host_without_proxy$1" fi echo "$redirect_urls" } From 9ebbf2ed9b42115ef11964b4b1da8bcf133237ba Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 23 Nov 2023 14:33:28 +0100 Subject: [PATCH 136/243] Bugfix: Export /var/cache/bridgehead/secrets as environment variables --- lib/functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/functions.sh b/lib/functions.sh index 0c93862..fe0ab67 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -284,7 +284,9 @@ function sync_secrets() { -e OIDC_PROVIDER=secret-sync-central.oidc-client-enrollment.$BROKER_ID \ -e SECRET_DEFINITIONS=$SECRET_SYNC_ARGS \ docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest + set -a # Export variables as environment variables source /var/cache/bridgehead/secrets/* + set +a # Export variables in the regular way } capitalize_first_letter() { From 163650f592da0e4fdf5e949ce18be951e551c51c Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 23 Nov 2023 15:54:44 +0100 Subject: [PATCH 137/243] Add generate_password function --- ccp/modules/datashield-setup.sh | 10 +++++----- ccp/modules/login-setup.sh | 2 +- lib/functions.sh | 13 +++++++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 420a450..3a964cd 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -3,10 +3,10 @@ if [ "$ENABLE_DATASHIELD" == true ]; then log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" - EXPORTER_OPAL_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal DB. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - TOKEN_MANAGER_OPAL_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Token Manager in Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal DB. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - OPAL_ADMIN_PASSWORD="$(echo \"This is a salt string to generate one consistent admin password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + EXPORTER_OPAL_PASSWORD="$(generate_password \"exporter in Opal\")" + TOKEN_MANAGER_OPAL_PASSWORD="$(generate_password \"Token Manager in Opal\")" + OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")" DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ @@ -20,7 +20,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then "external": "opal-'"$SITE_ID"'", "internal": "opal:8080", "allowed": [$input.sites[].id | "datashield-connect.\(.).broker.ccp-it.dktk.dkfz.de"] - }]' > /tmp/bridgehead/opal-map/local.json + }]' >/tmp/bridgehead/opal-map/local.json cp -f ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json chown -R bridgehead:docker /tmp/bridgehead/ generate_private_oidc_client "OIDC_CLIENT_SECRET" "$(generate_redirect_urls '/opal/*')" diff --git a/ccp/modules/login-setup.sh b/ccp/modules/login-setup.sh index 5ead5d4..1981b87 100644 --- a/ccp/modules/login-setup.sh +++ b/ccp/modules/login-setup.sh @@ -3,5 +3,5 @@ if [ "$ENABLE_LOGIN" == true ]; then log INFO "Login setup detected -- will start Login services." OVERRIDE+=" -f ./$PROJECT/modules/login-compose.yml" - KEYCLOAK_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Keycloak. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + KEYCLOAK_DB_PASSWORD="$(generate_password \"local Keycloak\")" fi diff --git a/lib/functions.sh b/lib/functions.sh index fe0ab67..1754767 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -307,3 +307,16 @@ generate_redirect_urls(){ fi echo "$redirect_urls" } + +generate_password(){ + local seed_text="$1" + local random_digit=$(openssl rand -hex 1 | head -c 1) + local random_upper=$(openssl rand -base64 3 | tr -dc 'A-Z' | head -c 1) + local random_lower=$(openssl rand -base64 3 | tr -dc 'a-z' | head -c 1) + local random_special=$(echo '@#$%^&+=' | fold -w1 | shuf -n1) + + local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret." + local main_password=$(echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" | base64 | head -c 26) + + echo "${main_password}${random_digit}${random_upper}${random_lower}${random_special}" +} From 8486abedd43f979595aad6e1247cb21028eaa5c4 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 23 Nov 2023 17:28:39 +0100 Subject: [PATCH 138/243] Add R-Studio Admin Password --- ccp/modules/datashield-compose.yml | 4 ++-- ccp/modules/datashield-setup.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index a126b1a..78ff12f 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -6,7 +6,7 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: #DEFAULT_USER: "rstudio" # This line is kept for informational purposes - #PASSWORD: "${LDM_AUTH}" + PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" DISABLE_AUTH: "true" # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. HTTP_RELATIVE_PATH: "/rstudio" labels: @@ -15,7 +15,7 @@ services: - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787" - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip,auth" + - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" opal: container_name: bridgehead-opal diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 3a964cd..e77e4c9 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -7,6 +7,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then TOKEN_MANAGER_OPAL_PASSWORD="$(generate_password \"Token Manager in Opal\")" OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")" + RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")" DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ From e32f484c31cbc00c6b2c4cce6d68ff1e403d62fc Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 24 Nov 2023 08:52:54 +0100 Subject: [PATCH 139/243] Add keycloak configuration --- ccp/modules/datashield-compose.yml | 6 +++--- ccp/modules/mtba-compose.yml | 1 + ccp/modules/teiler-compose.yml | 8 ++++---- ccp/vars | 6 ++++++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 78ff12f..bac8a07 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -41,9 +41,9 @@ services: APP_CONTEXT_PATH: "/opal" OPAL_PRIVATE_KEY: "/run/secrets/opal-key.pem" OPAL_CERTIFICATE: "/run/secrets/opal-cert.pem" - KEYCLOAK_URL: "https://login.verbis.dkfz.de" - KEYCLOAK_REALM: "test-realm-01" - KEYCLOAK_CLIENT_ID: "${SITE_ID}-private" + KEYCLOAK_URL: "${KEYCLOAK_URL}" + KEYCLOAK_REALM: "${KEYCLOAK_REALM}" + KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PRIVATE_CLIENT_ID}" KEYCLOAK_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}" KEYCLOAK_ADMIN_GROUP: "${KEYCLOAK_ADMIN_GROUP}" TOKEN_MANAGER_PASSWORD: "${TOKEN_MANAGER_OPAL_PASSWORD}" diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 670d990..b448378 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -19,6 +19,7 @@ services: FILE_CHARSET: ${MTBA_FILE_CHARSET} FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} CSV_DELIMITER: ${MTBA_CSV_DELIMITER} + HTTP_RELATIVE_PATH: "/mtba" labels: - "traefik.enable=true" - "traefik.http.routers.mtba_ccp.rule=PathPrefix(`/mtba`)" diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index 659c9e2..14b8633 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -31,10 +31,10 @@ services: environment: DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend" - KEYCLOAK_URL: "https://login.verbis.dkfz.de" - KEYCLOAK_REALM: "test-realm-01" - KEYCLOAK_CLIENT_ID: "${SITE_ID}-public" - KEYCLOAK_TOKEN_GROUP: "groups" + KEYCLOAK_URL: "${KEYCLOAK_URL}" + KEYCLOAK_REALM: "${KEYCLOAK_REALM}" + KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PUBLIC_CLIENT_ID}" + KEYCLOAK_TOKEN_GROUP: "${KEYCLOAK_TOKEN_GROUP}" TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" diff --git a/ccp/vars b/ccp/vars index b133cf4..5ca76a2 100644 --- a/ccp/vars +++ b/ccp/vars @@ -15,6 +15,12 @@ ENABLE_TEILER=true KEYCLOAK_USER_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})" KEYCLOAK_ADMIN_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})_Verwalter" +KEYCLOAK_PRIVATE_CLIENT_ID=${SITE_ID}-private +KEYCLOAK_PUBLIC_CLIENT_ID=${SITE_ID}-public +# TODO: Change Keycloak Realm to productive. "test-realm-01" is only for testing +KEYCLOAK_REALM="test-realm-01" +KEYCLOAK_URL="https://login.verbis.dkfz.de" +KEYCLOAK_TOKEN_GROUP="groups" POSTGRES_TAG=15.6-alpine for module in $PROJECT/modules/*.sh From 903ef0df9b32f4d7f126ef647f81bc1e08db6db1 Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 27 Nov 2023 19:39:16 +0100 Subject: [PATCH 140/243] Add Keycloak to MTBA --- bridgehead | 1 + ccp/modules/mtba-compose.yml | 7 +++++++ ccp/vars | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bridgehead b/bridgehead index a1cb708..4e25da7 100755 --- a/bridgehead +++ b/bridgehead @@ -41,6 +41,7 @@ case "$PROJECT" in ;; esac +# TODO: Please add proper documentation for variable priorities (1. secrets, 2. vars, 3. PROJECT.local.conf, 4. PROJECT.conf, 5. ??? loadVars() { # Load variables from /etc/bridgehead and /srv/docker/bridgehead set -a diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index b448378..fb7b19a 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -20,11 +20,18 @@ services: FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} CSV_DELIMITER: ${MTBA_CSV_DELIMITER} HTTP_RELATIVE_PATH: "/mtba" + KEYCLOAK_ADMIN_GROUP: "${KEYCLOAK_ADMIN_GROUP}" + KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PRIVATE_CLIENT_ID}" + KEYCLOAK_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}" + KEYCLOAK_REALM: "${KEYCLOAK_REALM}" + KEYCLOAK_URL: "${KEYCLOAK_URL}" + labels: - "traefik.enable=true" - "traefik.http.routers.mtba_ccp.rule=PathPrefix(`/mtba`)" - "traefik.http.services.mtba_ccp.loadbalancer.server.port=8480" - "traefik.http.routers.mtba_ccp.tls=true" + - "traefik.http.middlewares.mtba_ccp_strip.stripprefix.prefixes=/mtba" - "traefik.http.routers.mtba_ccp.middlewares=mtba_ccp_strip, auth" volumes: diff --git a/ccp/vars b/ccp/vars index 5ca76a2..b051cee 100644 --- a/ccp/vars +++ b/ccp/vars @@ -18,7 +18,7 @@ KEYCLOAK_ADMIN_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})_Verwalter" KEYCLOAK_PRIVATE_CLIENT_ID=${SITE_ID}-private KEYCLOAK_PUBLIC_CLIENT_ID=${SITE_ID}-public # TODO: Change Keycloak Realm to productive. "test-realm-01" is only for testing -KEYCLOAK_REALM="test-realm-01" +KEYCLOAK_REALM="${KEYCLOAK_REALM:-test-realm-01}" KEYCLOAK_URL="https://login.verbis.dkfz.de" KEYCLOAK_TOKEN_GROUP="groups" POSTGRES_TAG=15.6-alpine From ae965fddb307d1f52a4a4b01dd0199b35d46a70b Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 28 Nov 2023 16:50:36 +0100 Subject: [PATCH 141/243] Add proxy to R-Studio for loading R packages --- ccp/modules/datashield-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index bac8a07..14e9650 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -7,8 +7,10 @@ services: environment: #DEFAULT_USER: "rstudio" # This line is kept for informational purposes PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" - DISABLE_AUTH: "true" # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. + DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use + # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. HTTP_RELATIVE_PATH: "/rstudio" + ALL_PROXY: "http://forward_proxy:3128" # https://rocker-project.org/use/networking.html labels: - "traefik.enable=true" - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)" From f6965859fe89b57e40088a41eefc84df20182cea Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 28 Nov 2023 16:51:30 +0100 Subject: [PATCH 142/243] Add comment about PASSWORD and DISABLE_AUTH in R-Studio --- ccp/modules/datashield-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 14e9650..292f20c 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -6,7 +6,7 @@ services: image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest environment: #DEFAULT_USER: "rstudio" # This line is kept for informational purposes - PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" + PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" # It is required, even if the authentication is disabled DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. HTTP_RELATIVE_PATH: "/rstudio" @@ -17,7 +17,6 @@ services: - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787" - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - - "traefik.http.routers.rstudio_ccp.middlewares=rstudio_ccp_strip" opal: container_name: bridgehead-opal From 0cd4ededc70e8108541e24f704acfb17be3351f8 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 29 Nov 2023 09:29:18 +0100 Subject: [PATCH 143/243] Add oauth2_proxy --- ccp/docker-compose.yml | 44 ++++++++++++++++++++++++++++++ ccp/modules/datashield-compose.yml | 4 +-- ccp/modules/datashield-setup.sh | 1 + ccp/modules/teiler-compose.yml | 2 +- ccp/vars | 3 +- 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index d92ccfb..269ed2e 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -52,6 +52,50 @@ services: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - /srv/docker/bridgehead/ccp/root.crt.pem:/conf/root.crt.pem:ro + traefik: + labels: + - "traefik.http.middlewares.oidcAuth.forwardAuth.address=http://oauth2_proxy:4180/" + - "traefik.http.middlewares.oidcAuth.forwardAuth.trustForwardHeader=true" + - "traefik.http.middlewares.oidcAuth.forwardAuth.authResponseHeaders=X-Auth-Request-Access-Token,Authorization" + + + oauth2_proxy: + image: quay.io/oauth2-proxy/oauth2-proxy + container_name: bridgehead_oauth2_proxy + command: >- + --allowed-group=/${KEYCLOAK_USER_GROUP} + --oidc-groups-claim=${KEYCLOAK_GROUP_CLAIM} + --auth-logging=true + --whitelist-domain=${HOST} + --http-address="0.0.0.0:4180" + --reverse-proxy=true + --upstream="static://202" + --email-domain="*" + --cookie-name="_BRIDGEHEAD_oauth2" + --cookie-secret="${OAUTH2_PROXY_SECRET}" + --cookie-expire="12h" + --cookie-secure="true" + --cookie-httponly="true" + #OIDC settings + --provider="keycloak-oidc" + --provider-display-name="VerbIS Login" + --client-id="${KEYCLOAK_PRIVATE_CLIENT_ID}" + --client-secret="${OIDC_CLIENT_SECRET}" + --redirect-url="https://${HOST}/oauth2/callback" + --oidc-issuer-url="${KEYCLOAK_ISSUER_URL}" + --scope="openid email profile" + --code-challenge-method="S256" + --skip-provider-button=true + #X-Forwarded-Header settings - true/false depending on your needs + --pass-basic-auth=true + --pass-user-headers=false + --pass-access-token=false + labels: + - "traefik.enable=true" + - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`, `/oauth2/callback`)" + - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180" + - "traefik.http.routers.oauth2_proxy.tls=true" + volumes: blaze-data: diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 292f20c..611b39f 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -8,15 +8,15 @@ services: #DEFAULT_USER: "rstudio" # This line is kept for informational purposes PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" # It is required, even if the authentication is disabled DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use - # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. HTTP_RELATIVE_PATH: "/rstudio" ALL_PROXY: "http://forward_proxy:3128" # https://rocker-project.org/use/networking.html labels: - "traefik.enable=true" - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)" - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787" - - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" + - "traefik.http.routers.rstudio_ccp.tls=true" + - "traefik.http.routers.rstudio_ccp.middlewares=oidcAuth,rstudio_ccp_strip" opal: container_name: bridgehead-opal diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index e77e4c9..bd50a43 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -9,6 +9,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")" RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")" DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ chown -R bridgehead:docker /tmp/bridgehead/ diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index 14b8633..8266eca 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -34,7 +34,7 @@ services: KEYCLOAK_URL: "${KEYCLOAK_URL}" KEYCLOAK_REALM: "${KEYCLOAK_REALM}" KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PUBLIC_CLIENT_ID}" - KEYCLOAK_TOKEN_GROUP: "${KEYCLOAK_TOKEN_GROUP}" + KEYCLOAK_TOKEN_GROUP: "${KEYCLOAK_GROUP_CLAIM}" TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" diff --git a/ccp/vars b/ccp/vars index b051cee..62a8df3 100644 --- a/ccp/vars +++ b/ccp/vars @@ -20,7 +20,8 @@ KEYCLOAK_PUBLIC_CLIENT_ID=${SITE_ID}-public # TODO: Change Keycloak Realm to productive. "test-realm-01" is only for testing KEYCLOAK_REALM="${KEYCLOAK_REALM:-test-realm-01}" KEYCLOAK_URL="https://login.verbis.dkfz.de" -KEYCLOAK_TOKEN_GROUP="groups" +KEYCLOAK_ISSUER_URL="${KEYCLOAK_URL}/realms/${KEYCLOAK_REALM}" +KEYCLOAK_GROUP_CLAIM="groups" POSTGRES_TAG=15.6-alpine for module in $PROJECT/modules/*.sh From b44a208e08b9a6898cf83055b58511889fab997e Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 30 Nov 2023 13:46:08 +0000 Subject: [PATCH 144/243] Better redirect url handeling --- ccp/modules/datashield-setup.sh | 2 +- ccp/modules/teiler-setup.sh | 2 +- lib/functions.sh | 48 +++++++++++++++++++-------------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index bd50a43..e5625fa 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -25,5 +25,5 @@ if [ "$ENABLE_DATASHIELD" == true ]; then }]' >/tmp/bridgehead/opal-map/local.json cp -f ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json chown -R bridgehead:docker /tmp/bridgehead/ - generate_private_oidc_client "OIDC_CLIENT_SECRET" "$(generate_redirect_urls '/opal/*')" + add_private_oidc_redirect_url "/opal/*" fi diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index 0da6f6f..1e97079 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -3,5 +3,5 @@ if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler setup detected -- will start Teiler services." OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" - generate_public_oidc_client "OIDC_PUBLIC" "$(generate_redirect_urls '/ccp-teiler/*')" + add_public_oidc_redirect_url "/ccp-teiler/*" fi diff --git a/lib/functions.sh b/lib/functions.sh index 1754767..0039093 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -240,33 +240,41 @@ add_basic_auth_user() { sed -i "/^$NAME/ s|$|\n# User: $USER\n# Password: $PASSWORD|" $FILE } -SECRET_SYNC_ARGS=${SECRET_SYNC_ARGS:-""} -# First argument is the variable name that will be generated it will not have a value. -# Second argument is a comma separated list of allowed redirect urls for the oidc client. -# The resulting client id will be $SITE_ID-public -function generate_public_oidc_client() { - local delimiter=$'\x1E' - if [[ $SECRET_SYNC_ARGS == "" ]]; then - SECRET_SYNC_ARGS+="OIDC:$1:public;$2" +OIDC_PUBLIC_REDIRECT_URLS=${OIDC_PUBLIC_REDIRECT_URLS:-""} +OIDC_PRIVATE_REDIRECT_URLS=${OIDC_PRIVATE_REDIRECT_URLS:-""} + +# Add a redirect url to the public oidc client of the bridgehead +function add_public_oidc_redirect_url() { + if [[ $OIDC_PUBLIC_REDIRECT_URLS == "" ]]; then + OIDC_PUBLIC_REDIRECT_URLS+="$(generate_redirect_urls $1)" else - SECRET_SYNC_ARGS+="${delimiter}OIDC:$1:public;$2" + OIDC_PUBLIC_REDIRECT_URLS+=",$(generate_redirect_urls $1)" fi } -# First argument is the variable name that the client secret will be available at. -# Second argument is a comma separated list of allowed redirect urls for the oidc client. -# The resulting client id will be $SITE_ID-private -function generate_private_oidc_client() { - local delimiter=$'\x1E' - if [[ $SECRET_SYNC_ARGS == "" ]]; then - SECRET_SYNC_ARGS+="OIDC:$1:private;$2" +# Add a redirect url to the private oidc client of the bridgehead +function add_private_oidc_redirect_url() { + if [[ $OIDC_PRIVATE_REDIRECT_URLS == "" ]]; then + OIDC_PRIVATE_REDIRECT_URLS+="$(generate_redirect_urls $1)" else - SECRET_SYNC_ARGS+="${delimiter}OIDC:$1:private;$2" + OIDC_PRIVATE_REDIRECT_URLS+=",$(generate_redirect_urls $1)" fi } function sync_secrets() { - if [[ $SECRET_SYNC_ARGS == "" ]]; then + local delimiter=$'\x1E' + local secret_sync_args="" + if [[ $OIDC_PRIVATE_REDIRECT_URLS != "" ]]; then + secret_sync_args="OIDC:OIDC_CLIENT_SECRET:private;$OIDC_PRIVATE_REDIRECT_URLS" + fi + if [[ $OIDC_PRIVATE_REDIRECT_URLS != "" ]]; then + if [[ $secret_sync_args == "" ]]; then + secret_sync_args="OIDC:OIDC_PUBLIC:public;$OIDC_PUBLIC_REDIRECT_URLS" + else + secret_sync_args+="${delimiter}OIDC:$1:public;$OIDC_PUBLIC_REDIRECT_URLS" + fi + fi + if [[ $secret_sync_args == "" ]]; then return fi mkdir -p /var/cache/bridgehead/secrets/ @@ -282,7 +290,7 @@ function sync_secrets() { -e PROXY_ID=$PROXY_ID \ -e BROKER_URL=$BROKER_URL \ -e OIDC_PROVIDER=secret-sync-central.oidc-client-enrollment.$BROKER_ID \ - -e SECRET_DEFINITIONS=$SECRET_SYNC_ARGS \ + -e SECRET_DEFINITIONS=$secret_sync_args \ docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest set -a # Export variables as environment variables source /var/cache/bridgehead/secrets/* @@ -298,7 +306,7 @@ capitalize_first_letter() { # Generate a string of ',' separated string of redirect urls relative to $HOST. # $1 will be appended to the url # If the host looks like dev-jan.inet.dkfz-heidelberg.de it will generate urls with dev-jan and the original $HOST as url Authorities -generate_redirect_urls(){ +function generate_redirect_urls(){ local redirect_urls="https://${HOST}$1" local host_without_proxy="$(echo "$HOST" | cut -d '.' -f1)" # Only append second url if its different and the host is not an ip address From 5d4d0405ab889df8c24e76d4c270b9b8aa63f89c Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 30 Nov 2023 14:05:07 +0000 Subject: [PATCH 145/243] fix: public client generation --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index 0039093..507d323 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -271,7 +271,7 @@ function sync_secrets() { if [[ $secret_sync_args == "" ]]; then secret_sync_args="OIDC:OIDC_PUBLIC:public;$OIDC_PUBLIC_REDIRECT_URLS" else - secret_sync_args+="${delimiter}OIDC:$1:public;$OIDC_PUBLIC_REDIRECT_URLS" + secret_sync_args+="${delimiter}OIDC:OIDC_PUBLIC:public;$OIDC_PUBLIC_REDIRECT_URLS" fi fi if [[ $secret_sync_args == "" ]]; then From f9b26b695823282da9de90f33a89825a94f7c502 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 30 Nov 2023 17:25:32 +0100 Subject: [PATCH 146/243] Use develop branch for mtba --- ccp/modules/mtba-compose.yml | 4 +- ccp/modules/teiler-ui-compose.yml | 74 ------------------------------- 2 files changed, 1 insertion(+), 77 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index fb7b19a..290b846 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: mtba: - image: docker.verbis.dkfz.de/cache/samply/mtba:1.0.0 + image: docker.verbis.dkfz.de/cache/samply/mtba:develop container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 @@ -32,8 +32,6 @@ services: - "traefik.http.services.mtba_ccp.loadbalancer.server.port=8480" - "traefik.http.routers.mtba_ccp.tls=true" - - "traefik.http.middlewares.mtba_ccp_strip.stripprefix.prefixes=/mtba" - - "traefik.http.routers.mtba_ccp.middlewares=mtba_ccp_strip, auth" volumes: - /var/cache/bridgehead/ccp/mtba/input:/app/input - /var/cache/bridgehead/ccp/mtba/persist:/app/persist diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml index 5a51c8e..e69de29 100644 --- a/ccp/modules/teiler-ui-compose.yml +++ b/ccp/modules/teiler-ui-compose.yml @@ -1,74 +0,0 @@ -version: "3.7" - -services: - - teiler-root-config: - image: docker.verbis.dkfz.de/cache/samply/teiler-root-config:develop - container_name: bridgehead-teiler-root-config - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_root_config_ccp.rule=PathPrefix(`/ccp-teiler`)" - - "traefik.http.services.teiler_root_config_ccp.loadbalancer.server.port=9000" - - "traefik.http.routers.teiler_root_config_ccp.tls=true" - - "traefik.http.middlewares.teiler_root_config_ccp_strip.stripprefix.prefixes=/ccp-teiler" - - "traefik.http.routers.teiler_root_config_ccp.middlewares=teiler_root_config_ccp_strip" - environment: - TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" - TEILER_UI_URL: "https://${HOST}/ccp-teiler-ui" - DEFAULT_LANGUAGE: "de" - HTTP_RELATIVE_PATH: "/ccp-teiler" - - teiler-ui: - image: docker.verbis.dkfz.de/cache/samply/teiler-ui:develop - container_name: bridgehead-teiler-ui - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_ui_ccp.rule=PathPrefix(`/ccp-teiler-ui`)" - - "traefik.http.services.teiler_ui_ccp.loadbalancer.server.port=80" - - "traefik.http.routers.teiler_ui_ccp.tls=true" - - "traefik.http.middlewares.teiler_ui_ccp_strip.stripprefix.prefixes=/ccp-teiler-ui" - - "traefik.http.routers.teiler_ui_ccp.middlewares=teiler_ui_ccp_strip" - environment: - DEFAULT_LANGUAGE: "DE" - TEILER_CORE_URL: "https://${HOST}/ccp-teiler-core" - KEYCLOAK_URL: "https://${HOST}/login" - KEYCLOAK_REALM: "teiler-ui" - KEYCLOAK_CLIENT_ID: "teiler-ui" - TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" - TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" - TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" - TEILER_PROJECT: "${PROJECT}" - EXPORTER_API_KEY: "${EXPORTER_API_KEY}" - TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" - TEILER_UI_HTTP_RELATIVE_PATH: "/ccp-teiler-ui" - TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" - - teiler-core: - image: docker.verbis.dkfz.de/ccp/dktk-teiler-core:latest - container_name: bridgehead-teiler-core - labels: - - "traefik.enable=true" - - "traefik.http.routers.teiler_core_ccp.rule=PathPrefix(`/ccp-teiler-core`)" - - "traefik.http.services.teiler_core_ccp.loadbalancer.server.port=8085" - - "traefik.http.routers.teiler_core_ccp.tls=true" - - "traefik.http.middlewares.teiler_core_ccp_strip.stripprefix.prefixes=/ccp-teiler-core" - - "traefik.http.routers.teiler_core_ccp.middlewares=teiler_core_ccp_strip" - environment: - LOG_LEVEL: "INFO" - APPLICATION_PORT: "8085" - APPLICATION_ADDRESS: "${HOST}" - DEFAULT_LANGUAGE: "DE" - CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" - TEILER_CONFIG_UPDATER_CRON: "0 1 * * * *" - TEILER_ROOT_CONFIG_HTTP_RELATIVE_PATH: "/ccp-teiler" - TEILER_ROOT_CONFIG_URL: "https://${HOST}/ccp-teiler" - TEILER_UI_DE_URL: "https://${HOST}/ccp-teiler-ui/de" - TEILER_UI_EN_URL: "https://${HOST}/ccp-teiler-ui/en" - CENTRAX_URL: "${CENTRAXX_URL}" - IS_DKTK_SITE: "${IS_DKTK_SITE}" - secrets: - - ccp.conf - -secrets: - ccp.conf: - file: /etc/bridgehead/ccp.conf From 25ac4d2590c15a674fb0126429369d78ee0753fc Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 30 Nov 2023 17:32:51 +0100 Subject: [PATCH 147/243] mtba latest --- ccp/modules/mtba-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 290b846..8917f47 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: mtba: - image: docker.verbis.dkfz.de/cache/samply/mtba:develop + image: docker.verbis.dkfz.de/cache/samply/mtba:latest container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 From 0b2e64a2d5ff1897ca444cb54a3b2bf07a5b3477 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 30 Nov 2023 17:39:01 +0100 Subject: [PATCH 148/243] add /oauth2/callback and /mtba to Keycloak private client --- ccp/docker-compose.yml | 2 +- ccp/modules/mtba-setup.sh | 1 + ccp/vars | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 269ed2e..be2d358 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -81,7 +81,7 @@ services: --provider-display-name="VerbIS Login" --client-id="${KEYCLOAK_PRIVATE_CLIENT_ID}" --client-secret="${OIDC_CLIENT_SECRET}" - --redirect-url="https://${HOST}/oauth2/callback" + --redirect-url="https://${HOST}${OAUTH2_CALLBACK}" --oidc-issuer-url="${KEYCLOAK_ISSUER_URL}" --scope="openid email profile" --code-challenge-method="S256" diff --git a/ccp/modules/mtba-setup.sh b/ccp/modules/mtba-setup.sh index 53b4ce0..cdf0f31 100644 --- a/ccp/modules/mtba-setup.sh +++ b/ccp/modules/mtba-setup.sh @@ -8,5 +8,6 @@ function mtbaSetup() { exit 1; fi OVERRIDE+=" -f ./$PROJECT/modules/mtba-compose.yml" + add_private_oidc_redirect_url "/mtba/*" fi } diff --git a/ccp/vars b/ccp/vars index 62a8df3..94b79f0 100644 --- a/ccp/vars +++ b/ccp/vars @@ -22,6 +22,9 @@ KEYCLOAK_REALM="${KEYCLOAK_REALM:-test-realm-01}" KEYCLOAK_URL="https://login.verbis.dkfz.de" KEYCLOAK_ISSUER_URL="${KEYCLOAK_URL}/realms/${KEYCLOAK_REALM}" KEYCLOAK_GROUP_CLAIM="groups" +OAUTH2_CALLBACK=/oauth2/callback + +add_private_oidc_redirect_url "${OAUTH2_CALLBACK}" POSTGRES_TAG=15.6-alpine for module in $PROJECT/modules/*.sh From e411883d184b1a827ca3bae980a2d9c523db6dbf Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 30 Nov 2023 17:58:50 +0100 Subject: [PATCH 149/243] mtba develop --- ccp/modules/mtba-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 8917f47..f88c239 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -2,7 +2,8 @@ version: "3.7" services: mtba: - image: docker.verbis.dkfz.de/cache/samply/mtba:latest + #image: docker.verbis.dkfz.de/cache/samply/mtba:latest + image: samply/mtba:develop container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 From 28a612f2187f85bdfec43713ba25f9101ac6cea2 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 1 Dec 2023 08:58:36 +0100 Subject: [PATCH 150/243] add default template-ids of exporter and reporter --- ccp/modules/teiler-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index 8266eca..b28753f 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -45,6 +45,9 @@ services: TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler" TEILER_USER: "${KEYCLOAK_USER_GROUP}" TEILER_ADMIN: "${KEYCLOAK_ADMIN_GROUP}" + REPORTER_DEFAULT_TEMPLATE_ID: "ccp-qb" + EXPORTER_DEFAULT_TEMPLATE_ID: "ccp" + teiler-backend: image: docker.verbis.dkfz.de/ccp/dktk-teiler-backend:latest From 148e87341f5598177598f1cb72b2481892bdb142 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 1 Dec 2023 09:16:29 +0100 Subject: [PATCH 151/243] move OAUTH2_SECRET --- ccp/modules/datashield-setup.sh | 1 - ccp/vars | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index e5625fa..5262b6b 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -9,7 +9,6 @@ if [ "$ENABLE_DATASHIELD" == true ]; then OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")" RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")" DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ chown -R bridgehead:docker /tmp/bridgehead/ diff --git a/ccp/vars b/ccp/vars index 94b79f0..eb998d7 100644 --- a/ccp/vars +++ b/ccp/vars @@ -23,6 +23,7 @@ KEYCLOAK_URL="https://login.verbis.dkfz.de" KEYCLOAK_ISSUER_URL="${KEYCLOAK_URL}/realms/${KEYCLOAK_REALM}" KEYCLOAK_GROUP_CLAIM="groups" OAUTH2_CALLBACK=/oauth2/callback +OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" add_private_oidc_redirect_url "${OAUTH2_CALLBACK}" POSTGRES_TAG=15.6-alpine From 0a2dbb4b2d5e250c62671392a9c855d836076d1f Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 8 Dec 2023 11:50:06 +0000 Subject: [PATCH 152/243] fix: Restrict rstudio network access --- ccp/modules/datashield-compose.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 611b39f..501c666 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -17,6 +17,8 @@ services: - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.routers.rstudio_ccp.middlewares=oidcAuth,rstudio_ccp_strip" + networks: + - rstudio opal: container_name: bridgehead-opal @@ -88,6 +90,18 @@ services: - beam-proxy volumes: - /tmp/bridgehead/opal-map/:/map/:ro + networks: + - default + - rstudio + + traefik: + networks: + - default + - rstudio + forward_proxy: + networks: + - default + - rstudio beam-proxy: environment: @@ -98,3 +112,6 @@ secrets: file: /tmp/bridgehead/opal-cert.pem opal-key.pem: file: /tmp/bridgehead/opal-key.pem + +networks: + rstudio: From 371097377a2d8914eafae5e831c0ed6e049b25f2 Mon Sep 17 00:00:00 2001 From: janskiba Date: Tue, 12 Dec 2023 09:53:14 +0000 Subject: [PATCH 153/243] feat: Add token-manager to beam --- ccp/modules/datashield-compose.yml | 3 +++ ccp/modules/datashield-setup.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 501c666..55bda13 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -51,6 +51,8 @@ services: KEYCLOAK_ADMIN_GROUP: "${KEYCLOAK_ADMIN_GROUP}" TOKEN_MANAGER_PASSWORD: "${TOKEN_MANAGER_OPAL_PASSWORD}" EXPORTER_PASSWORD: "${EXPORTER_OPAL_PASSWORD}" + BEAM_APP_ID: token-manager.${PROXY_ID} + BEAM_SECRET: ${TOKEN_MANAGER_SECRET} secrets: - opal-cert.pem - opal-key.pem @@ -106,6 +108,7 @@ services: beam-proxy: environment: APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET} + APP_token-manager_KEY: ${TOKEN_MANAGER_SECRET} secrets: opal-cert.pem: diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 5262b6b..d9932c3 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -9,6 +9,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")" RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")" DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + TOKEN_MANAGER_SECRET="$(echo \"This is a salt string to generate one consistent password as the Token Manger secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ chown -R bridgehead:docker /tmp/bridgehead/ From 9f31e950a5c9375b2786f1fbc00e91a5eb24cc60 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 13 Dec 2023 11:01:25 +0000 Subject: [PATCH 154/243] fix: generate the right beam connect mappings --- ccp/modules/datashield-mappings.json | 70 +++++----------------------- ccp/modules/datashield-setup.sh | 18 ++++--- lib/functions.sh | 1 + 3 files changed, 23 insertions(+), 66 deletions(-) diff --git a/ccp/modules/datashield-mappings.json b/ccp/modules/datashield-mappings.json index d902b8f..a65d9d5 100644 --- a/ccp/modules/datashield-mappings.json +++ b/ccp/modules/datashield-mappings.json @@ -1,59 +1,11 @@ - -{ - "sites": [ - { - "id": "berlin", - "name": "berlin", - "virtualhost": "opal-berlin", - "beamconnect": "datashield-connect.berlin.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "muenchen-lmu", - "name": "muenchen-lmu", - "virtualhost": "opal-muenchen-lmu", - "beamconnect": "datashield-connect.muenchen-lmu.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "dresden", - "name": "dresden", - "virtualhost": "opal-dresden", - "beamconnect": "datashield-connect.dresden.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "freiburg", - "name": "freiburg", - "virtualhost": "opal-freiburg", - "beamconnect": "datashield-connect.freiburg.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "muenchen-tum", - "name": "muenchen-tum", - "virtualhost": "opal-muenchen-tum", - "beamconnect": "datashield-connect.muenchen-tum.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "tuebingen", - "name": "tuebingen", - "virtualhost": "opal-tuebingen", - "beamconnect": "datashield-connect.tuebingen.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "mainz", - "name": "mainz", - "virtualhost": "opal-mainz", - "beamconnect": "datashield-connect.mainz.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "frankfurt", - "name": "frankfurt", - "virtualhost": "opal-frankfurt", - "beamconnect": "datashield-connect.frankfurt.broker.ccp-it.dktk.dkfz.de" - }, - { - "id": "essen", - "name": "essen", - "virtualhost": "opal-essen", - "beamconnect": "datashield-connect.essen.broker.ccp-it.dktk.dkfz.de" - } - ] -} +[ + "berlin", + "muenchen-lmu", + "dresden", + "freiburg", + "muenchen-tum", + "tuebingen", + "mainz", + "frankfurt", + "essen" +] diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index d9932c3..bc1b1dc 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -17,13 +17,17 @@ if [ "$ENABLE_DATASHIELD" == true ]; then chmod g+r /tmp/bridgehead/opal-key.pem fi mkdir -p /tmp/bridgehead/opal-map - jq -n --argfile input ./$PROJECT/modules/datashield-mappings.json ' - [{ - "external": "opal-'"$SITE_ID"'", - "internal": "opal:8080", - "allowed": [$input.sites[].id | "datashield-connect.\(.).broker.ccp-it.dktk.dkfz.de"] - }]' >/tmp/bridgehead/opal-map/local.json - cp -f ./$PROJECT/modules/datashield-mappings.json /tmp/bridgehead/opal-map/central.json + jq -n '{"sites": input | map({ + "name": ., + "id": ., + "virtualhost": "opal-\(.):443", + "beamconnect": "datashield-connect.\(.).'"$BROKER_ID"'" + })}' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/central.json + jq -n '[{ + "external": "'"$SITE_ID"'", + "internal": "opal:8080", + "allowed": input | map("datashield-connect.\(.).'"$BROKER_ID"'") + }]' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/local.json chown -R bridgehead:docker /tmp/bridgehead/ add_private_oidc_redirect_url "/opal/*" fi diff --git a/lib/functions.sh b/lib/functions.sh index 507d323..b7d0fc4 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -279,6 +279,7 @@ function sync_secrets() { fi mkdir -p /var/cache/bridgehead/secrets/ touch /var/cache/bridgehead/secrets/oidc + chown -R bridgehead:docker /var/cache/bridgehead # The oidc provider will need to be switched based on the project at some point I guess docker run --rm \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ From b73ddc883cc7d0b2fdafbfe47482350d7924bd6e Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 13 Dec 2023 12:24:51 +0000 Subject: [PATCH 155/243] fix: Change permissions on new bridgehead dirs --- lib/functions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/functions.sh b/lib/functions.sh index b7d0fc4..548b912 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -132,6 +132,10 @@ assertVarsNotEmpty() { fixPermissions() { CHOWN=$(which chown) sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead + set +e + sudo $CHOWN -R --silent /var/cache/bridgehead + sudo $CHOWN -R --silent /tmp/bridgehead + set -e } source lib/monitoring.sh From 1edcdce5c637632ec7cfc2e454007173c53a7264 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 13 Dec 2023 13:17:16 +0000 Subject: [PATCH 156/243] fix: beam connect site renaming --- ccp/modules/datashield-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index bc1b1dc..46522ec 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -20,11 +20,11 @@ if [ "$ENABLE_DATASHIELD" == true ]; then jq -n '{"sites": input | map({ "name": ., "id": ., - "virtualhost": "opal-\(.):443", + "virtualhost": "\(.):443", "beamconnect": "datashield-connect.\(.).'"$BROKER_ID"'" })}' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/central.json jq -n '[{ - "external": "'"$SITE_ID"'", + "external": "'"$SITE_ID"':443", "internal": "opal:8080", "allowed": input | map("datashield-connect.\(.).'"$BROKER_ID"'") }]' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/local.json From b34f4f2a0f17500f876d3168e55a132171d54c20 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 13 Dec 2023 13:22:06 +0000 Subject: [PATCH 157/243] fix: chown syntax --- lib/functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 548b912..b27eeb1 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -133,8 +133,8 @@ fixPermissions() { CHOWN=$(which chown) sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead set +e - sudo $CHOWN -R --silent /var/cache/bridgehead - sudo $CHOWN -R --silent /tmp/bridgehead + sudo $CHOWN -R --silent bridgehead /var/cache/bridgehead + sudo $CHOWN -R --silent bridgehead /tmp/bridgehead set -e } From d3da4266101ea8e171e0ef6c232b403d198d00cc Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 13 Dec 2023 14:07:11 +0000 Subject: [PATCH 158/243] fix: opal ssl cert --- ccp/modules/datashield-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 46522ec..5142cb4 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -13,7 +13,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ chown -R bridgehead:docker /tmp/bridgehead/ - openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=${HOST:-opal}/C=DE" + openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=opal/C=DE" chmod g+r /tmp/bridgehead/opal-key.pem fi mkdir -p /tmp/bridgehead/opal-map @@ -25,7 +25,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then })}' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/central.json jq -n '[{ "external": "'"$SITE_ID"':443", - "internal": "opal:8080", + "internal": "opal:8443", "allowed": input | map("datashield-connect.\(.).'"$BROKER_ID"'") }]' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/local.json chown -R bridgehead:docker /tmp/bridgehead/ From 2a024e751d5659b843664c903617d5274742354b Mon Sep 17 00:00:00 2001 From: Jan <59206115+Threated@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:39:35 +0100 Subject: [PATCH 159/243] fix: only change permissions on related files --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index b27eeb1..7992276 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -283,7 +283,7 @@ function sync_secrets() { fi mkdir -p /var/cache/bridgehead/secrets/ touch /var/cache/bridgehead/secrets/oidc - chown -R bridgehead:docker /var/cache/bridgehead + chown -R bridgehead:docker /var/cache/bridgehead/secrets # The oidc provider will need to be switched based on the project at some point I guess docker run --rm \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ From fa141f8e8670263522543913bb0ffc6b4a5ea03c Mon Sep 17 00:00:00 2001 From: Jan <59206115+Threated@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:54:54 +0100 Subject: [PATCH 160/243] fix: undo permission changes on startup --- lib/functions.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 7992276..c53859b 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -132,10 +132,6 @@ assertVarsNotEmpty() { fixPermissions() { CHOWN=$(which chown) sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead - set +e - sudo $CHOWN -R --silent bridgehead /var/cache/bridgehead - sudo $CHOWN -R --silent bridgehead /tmp/bridgehead - set -e } source lib/monitoring.sh From 8e5ddc493c3522c6cfcfab09c5e2dfac93cbf7aa Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 13 Dec 2023 20:14:56 +0100 Subject: [PATCH 161/243] teiler-orchestrator and teiler-dashboard latest --- ccp/modules/teiler-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index b28753f..f9f7ab2 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: teiler-orchestrator: - image: docker.verbis.dkfz.de/cache/samply/teiler-orchestrator:develop + image: docker.verbis.dkfz.de/cache/samply/teiler-orchestrator:latest container_name: bridgehead-teiler-orchestrator labels: - "traefik.enable=true" @@ -19,7 +19,7 @@ services: HTTP_RELATIVE_PATH: "/ccp-teiler" teiler-dashboard: - image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:develop + image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:latest container_name: bridgehead-teiler-dashboard labels: - "traefik.enable=true" From f6dac7038ff58392b5a9605e88e835d533a146bb Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 13 Dec 2023 22:57:37 +0100 Subject: [PATCH 162/243] Only users of group DataSHIELD can use R-Studio --- ccp/docker-compose.yml | 38 ------------------------------ ccp/modules/datashield-compose.yml | 37 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index be2d358..c4610b6 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -59,44 +59,6 @@ services: - "traefik.http.middlewares.oidcAuth.forwardAuth.authResponseHeaders=X-Auth-Request-Access-Token,Authorization" - oauth2_proxy: - image: quay.io/oauth2-proxy/oauth2-proxy - container_name: bridgehead_oauth2_proxy - command: >- - --allowed-group=/${KEYCLOAK_USER_GROUP} - --oidc-groups-claim=${KEYCLOAK_GROUP_CLAIM} - --auth-logging=true - --whitelist-domain=${HOST} - --http-address="0.0.0.0:4180" - --reverse-proxy=true - --upstream="static://202" - --email-domain="*" - --cookie-name="_BRIDGEHEAD_oauth2" - --cookie-secret="${OAUTH2_PROXY_SECRET}" - --cookie-expire="12h" - --cookie-secure="true" - --cookie-httponly="true" - #OIDC settings - --provider="keycloak-oidc" - --provider-display-name="VerbIS Login" - --client-id="${KEYCLOAK_PRIVATE_CLIENT_ID}" - --client-secret="${OIDC_CLIENT_SECRET}" - --redirect-url="https://${HOST}${OAUTH2_CALLBACK}" - --oidc-issuer-url="${KEYCLOAK_ISSUER_URL}" - --scope="openid email profile" - --code-challenge-method="S256" - --skip-provider-button=true - #X-Forwarded-Header settings - true/false depending on your needs - --pass-basic-auth=true - --pass-user-headers=false - --pass-access-token=false - labels: - - "traefik.enable=true" - - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`, `/oauth2/callback`)" - - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180" - - "traefik.http.routers.oauth2_proxy.tls=true" - - volumes: blaze-data: diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 55bda13..780d049 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -110,6 +110,43 @@ services: APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET} APP_token-manager_KEY: ${TOKEN_MANAGER_SECRET} + oauth2_proxy: + image: quay.io/oauth2-proxy/oauth2-proxy + container_name: bridgehead_oauth2_proxy + command: >- + --allowed-group=/DataSHIELD + --oidc-groups-claim=${KEYCLOAK_GROUP_CLAIM} + --auth-logging=true + --whitelist-domain=${HOST} + --http-address="0.0.0.0:4180" + --reverse-proxy=true + --upstream="static://202" + --email-domain="*" + --cookie-name="_BRIDGEHEAD_oauth2" + --cookie-secret="${OAUTH2_PROXY_SECRET}" + --cookie-expire="12h" + --cookie-secure="true" + --cookie-httponly="true" + #OIDC settings + --provider="keycloak-oidc" + --provider-display-name="VerbIS Login" + --client-id="${KEYCLOAK_PRIVATE_CLIENT_ID}" + --client-secret="${OIDC_CLIENT_SECRET}" + --redirect-url="https://${HOST}${OAUTH2_CALLBACK}" + --oidc-issuer-url="${KEYCLOAK_ISSUER_URL}" + --scope="openid email profile" + --code-challenge-method="S256" + --skip-provider-button=true + #X-Forwarded-Header settings - true/false depending on your needs + --pass-basic-auth=true + --pass-user-headers=false + --pass-access-token=false + labels: + - "traefik.enable=true" + - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`, `/oauth2/callback`)" + - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180" + - "traefik.http.routers.oauth2_proxy.tls=true" + secrets: opal-cert.pem: file: /tmp/bridgehead/opal-cert.pem From 44d7b34834f4c3da5d1c77e6a090cd14d504b76f Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 13 Dec 2023 23:11:23 +0100 Subject: [PATCH 163/243] Use last version of mtba --- ccp/modules/mtba-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index f88c239..8917f47 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -2,8 +2,7 @@ version: "3.7" services: mtba: - #image: docker.verbis.dkfz.de/cache/samply/mtba:latest - image: samply/mtba:develop + image: docker.verbis.dkfz.de/cache/samply/mtba:latest container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 From 0793ea9fc6a29068f27548262b84b8fe87734471 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 13 Dec 2023 23:14:34 +0100 Subject: [PATCH 164/243] Use develop version of mtba --- ccp/modules/mtba-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 8917f47..9fcb74c 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -2,7 +2,8 @@ version: "3.7" services: mtba: - image: docker.verbis.dkfz.de/cache/samply/mtba:latest + #image: docker.verbis.dkfz.de/cache/samply/mtba:latest + image: docker.verbis.dkfz.de/cache/samply/mtba:develop container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 From 37f100dc01691653018cdfef7ee24b6da5a597ea Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 14 Dec 2023 00:08:41 +0100 Subject: [PATCH 165/243] Default values for MTBA --- ccp/modules/mtba-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 9fcb74c..f03532f 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -12,14 +12,14 @@ services: ID_MANAGER_API_KEY: ${IDMANAGER_UPLOAD_APIKEY} ID_MANAGER_PSEUDONYM_ID_TYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID ID_MANAGER_URL: http://id-manager:8080/id-manager - PATIENT_CSV_FIRST_NAME_HEADER: ${MTBA_PATIENT_CSV_FIRST_NAME_HEADER} - PATIENT_CSV_LAST_NAME_HEADER: ${MTBA_PATIENT_CSV_LAST_NAME_HEADER} - PATIENT_CSV_GENDER_HEADER: ${MTBA_PATIENT_CSV_GENDER_HEADER} - PATIENT_CSV_BIRTHDAY_HEADER: ${MTBA_PATIENT_CSV_BIRTHDAY_HEADER} + PATIENT_CSV_FIRST_NAME_HEADER: ${MTBA_PATIENT_CSV_FIRST_NAME_HEADER:-FIRST_NAME} + PATIENT_CSV_LAST_NAME_HEADER: ${MTBA_PATIENT_CSV_LAST_NAME_HEADER:-LAST_NAME} + PATIENT_CSV_GENDER_HEADER: ${MTBA_PATIENT_CSV_GENDER_HEADER:-GENDER} + PATIENT_CSV_BIRTHDAY_HEADER: ${MTBA_PATIENT_CSV_BIRTHDAY_HEADER:-BIRTHDAY} CBIOPORTAL_URL: http://cbioportal:8080 - FILE_CHARSET: ${MTBA_FILE_CHARSET} - FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE} - CSV_DELIMITER: ${MTBA_CSV_DELIMITER} + FILE_CHARSET: ${MTBA_FILE_CHARSET:-UTF-8} + FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE:-LF} + CSV_DELIMITER: ${MTBA_CSV_DELIMITER:-TAB} HTTP_RELATIVE_PATH: "/mtba" KEYCLOAK_ADMIN_GROUP: "${KEYCLOAK_ADMIN_GROUP}" KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PRIVATE_CLIENT_ID}" From 643e9e67a6edc552af77c0fb44e80d8e30364c9c Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 14 Dec 2023 14:04:42 +0100 Subject: [PATCH 166/243] Added: Enable MTBA and Enable DataSHIELD to Teiler Backend --- ccp/modules/teiler-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index f9f7ab2..40e394c 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -71,6 +71,8 @@ services: TEILER_DASHBOARD_EN_URL: "https://${HOST}/ccp-teiler-dashboard/en" CENTRAX_URL: "${CENTRAXX_URL}" HTTP_PROXY: "http://forward_proxy:3128" + ENABLE_MTBA: "${ENABLE_MTBA}" + ENABLE_DATASHIELD: "${ENABLE_DATASHIELD}" secrets: - ccp.conf From 977ad139f8225e3da71c60a2a33ddb47232dfdf1 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 14 Dec 2023 18:34:30 +0100 Subject: [PATCH 167/243] Added: allowed-groups --- ccp/modules/datashield-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 780d049..3b2da3e 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -110,6 +110,9 @@ services: APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET} APP_token-manager_KEY: ${TOKEN_MANAGER_SECRET} + # TODO: Allow users of group /DataSHIELD and KEYCLOAK_USER_GROUP at the same time: + # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/docs/configuration/overview/): + # --allowed-groups=/DataSHIELD,KEYCLOAK_USER_GROUP oauth2_proxy: image: quay.io/oauth2-proxy/oauth2-proxy container_name: bridgehead_oauth2_proxy From d62f5a404b72af701f4e306267744298dc9e7489 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 21 Dec 2023 08:28:47 +0000 Subject: [PATCH 168/243] Add central token manager beam id --- ccp/modules/datashield-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 3b2da3e..cb09b5d 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -53,6 +53,7 @@ services: EXPORTER_PASSWORD: "${EXPORTER_OPAL_PASSWORD}" BEAM_APP_ID: token-manager.${PROXY_ID} BEAM_SECRET: ${TOKEN_MANAGER_SECRET} + BEAM_DATASHIELD_PROXY: request-manager secrets: - opal-cert.pem - opal-key.pem From 2f04e51f96d2f1547a7f022a38cb73de3dec4ce0 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 21 Dec 2023 08:29:04 +0000 Subject: [PATCH 169/243] Add test sites --- ccp/modules/datashield-mappings.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-mappings.json b/ccp/modules/datashield-mappings.json index a65d9d5..7d8dad4 100644 --- a/ccp/modules/datashield-mappings.json +++ b/ccp/modules/datashield-mappings.json @@ -7,5 +7,7 @@ "tuebingen", "mainz", "frankfurt", - "essen" + "essen", + "dktk-datashield-test", + "dktk-test" ] From e54475f7044024e8778646709525f809d11590a5 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 21 Dec 2023 09:35:38 +0100 Subject: [PATCH 170/243] Added: volume for opal metadata db --- ccp/modules/datashield-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index cb09b5d..99d0883 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -54,6 +54,8 @@ services: BEAM_APP_ID: token-manager.${PROXY_ID} BEAM_SECRET: ${TOKEN_MANAGER_SECRET} BEAM_DATASHIELD_PROXY: request-manager + volumes: + - "/var/cache/bridgehead/ccp/opal-metadata-db:/srv/data/orientdb" # Opal metadata secrets: - opal-cert.pem - opal-key.pem @@ -68,7 +70,7 @@ services: POSTGRES_USER: "opal" POSTGRES_DB: "opal" volumes: - - "/var/cache/bridgehead/ccp/opal-db:/var/lib/postgresql/data" + - "/var/cache/bridgehead/ccp/opal-db:/var/lib/postgresql/data" # Opal project data (imported from exporter) opal-rserver: container_name: bridgehead-opal-rserver From 01efc6f9b9cca4c80e36d3c8b94ab3f7dfa59fd8 Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 21 Dec 2023 13:40:32 +0100 Subject: [PATCH 171/243] Added: volume for opal metadata db (II) --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 99d0883..40581b8 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -55,7 +55,7 @@ services: BEAM_SECRET: ${TOKEN_MANAGER_SECRET} BEAM_DATASHIELD_PROXY: request-manager volumes: - - "/var/cache/bridgehead/ccp/opal-metadata-db:/srv/data/orientdb" # Opal metadata + - "/var/cache/bridgehead/ccp/opal-metadata-db:/srv" # Opal metadata secrets: - opal-cert.pem - opal-key.pem From 935c45b74dd4b156634edc288a5fcf74dfbb24ca Mon Sep 17 00:00:00 2001 From: juarez Date: Thu, 21 Dec 2023 13:47:00 +0100 Subject: [PATCH 172/243] Added: volume for opal metadata db (III) --- ccp/modules/datashield-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 40581b8..fea2ff7 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -59,8 +59,6 @@ services: secrets: - opal-cert.pem - opal-key.pem - tmpfs: - - /srv opal-db: container_name: bridgehead-opal-db From f0a05b12ad19205cb015ccfef083f5c089be0252 Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 22 Dec 2023 10:41:07 +0000 Subject: [PATCH 173/243] fix: Generate stable passwords --- ccp/modules/datashield-setup.sh | 10 +++++----- lib/functions.sh | 26 +++++++++++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 5142cb4..604fcc8 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -3,13 +3,13 @@ if [ "$ENABLE_DATASHIELD" == true ]; then log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" - EXPORTER_OPAL_PASSWORD="$(generate_password \"exporter in Opal\")" - TOKEN_MANAGER_OPAL_PASSWORD="$(generate_password \"Token Manager in Opal\")" - OPAL_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for Opal. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + EXPORTER_OPAL_PASSWORD="$(generate_simple_password \"exporter in Opal\")" + TOKEN_MANAGER_OPAL_PASSWORD="$(generate_simple_password \"Token Manager in Opal\")" + OPAL_DB_PASSWORD="$(echo \"Opal DB\" | generate_simple_password)" OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")" RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")" - DATASHIELD_CONNECT_SECRET="$(echo \"This is a salt string to generate one consistent password as the DataShield Connect secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - TOKEN_MANAGER_SECRET="$(echo \"This is a salt string to generate one consistent password as the Token Manger secret. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + DATASHIELD_CONNECT_SECRET="$(echo \"DataShield Connect\" | generate_simple_password)" + TOKEN_MANAGER_SECRET="$(echo \"Token Manager\" | generate_simple_password)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ chown -R bridgehead:docker /tmp/bridgehead/ diff --git a/lib/functions.sh b/lib/functions.sh index c53859b..b89de60 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -317,15 +317,31 @@ function generate_redirect_urls(){ echo "$redirect_urls" } +# This password contains at least one special char, a random number and a random upper and lower case letter generate_password(){ local seed_text="$1" - local random_digit=$(openssl rand -hex 1 | head -c 1) - local random_upper=$(openssl rand -base64 3 | tr -dc 'A-Z' | head -c 1) - local random_lower=$(openssl rand -base64 3 | tr -dc 'a-z' | head -c 1) - local random_special=$(echo '@#$%^&+=' | fold -w1 | shuf -n1) + local seed_num=$(awk 'BEGIN{FS=""} NR==1{print $10}' /etc/bridgehead/pki/${SITE_ID}.priv.pem | od -An -tuC) + local nums="1234567890" + local n=$(echo "$seed_num" | awk '{print $1 % 10}') + local random_digit=${nums:$n:1} + local n=$(echo "$seed_num" | awk '{print $1 % 26}') + local upper="ABCDEFGHIJKLMNOPQRSTUVWXYZ" + local lower="abcdefghijklmnopqrstuvwxyz" + local random_upper=${upper:$n:1} + local random_lower=${lower:$n:1} + local n=$(echo "$seed_num" | awk '{print $1 % 8}') + local special='@#$%^&+=' + local random_special=${special:$n:1} local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret." - local main_password=$(echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" | base64 | head -c 26) + local main_password=$(echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26) echo "${main_password}${random_digit}${random_upper}${random_lower}${random_special}" } + +# This password only contains alphanumeric characters +generate_simple_password(){ + local seed_text="$1" + local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret." + echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/[+\/]/A/g' +} From c60c9fc4b48512b4f708b09d5c53302c901b8a24 Mon Sep 17 00:00:00 2001 From: janskiba Date: Fri, 22 Dec 2023 10:54:13 +0000 Subject: [PATCH 174/243] fix: Use strong pw for opal --- ccp/modules/datashield-setup.sh | 4 ++-- lib/functions.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 604fcc8..c600657 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -3,8 +3,8 @@ if [ "$ENABLE_DATASHIELD" == true ]; then log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" - EXPORTER_OPAL_PASSWORD="$(generate_simple_password \"exporter in Opal\")" - TOKEN_MANAGER_OPAL_PASSWORD="$(generate_simple_password \"Token Manager in Opal\")" + EXPORTER_OPAL_PASSWORD="$(generate_password \"exporter in Opal\")" + TOKEN_MANAGER_OPAL_PASSWORD="$(generate_password \"Token Manager in Opal\")" OPAL_DB_PASSWORD="$(echo \"Opal DB\" | generate_simple_password)" OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")" RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")" diff --git a/lib/functions.sh b/lib/functions.sh index b89de60..c098cf4 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -334,7 +334,7 @@ generate_password(){ local random_special=${special:$n:1} local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret." - local main_password=$(echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26) + local main_password=$(echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/\//A/g') echo "${main_password}${random_digit}${random_upper}${random_lower}${random_special}" } From 4e3cd6892246e205ec79343ab5356969be0f1134 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 22 Jan 2024 08:25:57 +0000 Subject: [PATCH 175/243] Only sync secrets on startup --- bridgehead | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgehead b/bridgehead index 4e25da7..49ad3a5 100755 --- a/bridgehead +++ b/bridgehead @@ -83,7 +83,6 @@ loadVars() { export FOCUS_TAG=main ;; esac - sync_secrets } case "$ACTION" in @@ -91,6 +90,7 @@ case "$ACTION" in loadVars hc_send log "Bridgehead $PROJECT startup: Checking requirements ..." checkRequirements + sync_secrets hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..." exec $COMPOSE -p $PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit ;; From 92a1f4bb59763f8d89ae0a0cb93aaffaf3240073 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 22 Jan 2024 13:47:25 +0000 Subject: [PATCH 176/243] Add `dsCCPhos` --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index fea2ff7..f777a01 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -72,7 +72,7 @@ services: opal-rserver: container_name: bridgehead-opal-rserver - image: docker.verbis.dkfz.de/cache/datashield/rock-base:6.3 # https://datashield.discourse.group/t/ds-aggregate-method-error/416/4 + image: docker.verbis.dkfz.de/ccp/dktk-rserver # datashield/rock-base + dsCCPhos tmpfs: - /srv From 01d3a38e1881a574ccaf07a1e131880eae8057ba Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 31 Jan 2024 09:21:19 +0000 Subject: [PATCH 177/243] refactor: Use jq from docker --- ccp/modules/datashield-setup.sh | 11 ++++++----- lib/functions.sh | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index c600657..464b0e1 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -17,17 +17,18 @@ if [ "$ENABLE_DATASHIELD" == true ]; then chmod g+r /tmp/bridgehead/opal-key.pem fi mkdir -p /tmp/bridgehead/opal-map - jq -n '{"sites": input | map({ + sites="$(cat ./$PROJECT/modules/datashield-mappings.json)" + echo "$sites" | docker_jq -n --args '{"sites": input | map({ "name": ., "id": ., "virtualhost": "\(.):443", "beamconnect": "datashield-connect.\(.).'"$BROKER_ID"'" - })}' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/central.json - jq -n '[{ + })}' $sites > /tmp/bridgehead/opal-map/central.json + echo "$sites" | docker_jq -n --args '[{ "external": "'"$SITE_ID"':443", "internal": "opal:8443", "allowed": input | map("datashield-connect.\(.).'"$BROKER_ID"'") - }]' ./$PROJECT/modules/datashield-mappings.json > /tmp/bridgehead/opal-map/local.json - chown -R bridgehead:docker /tmp/bridgehead/ + }]' > /tmp/bridgehead/opal-map/local.json + chown -R bridgehead:docker /tmp/bridgehead/* add_private_oidc_redirect_url "/opal/*" fi diff --git a/lib/functions.sh b/lib/functions.sh index c098cf4..b54ceec 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -345,3 +345,7 @@ generate_simple_password(){ local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret." echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/[+\/]/A/g' } + +docker_jq() { + docker run --rm -i docker.verbis.dkfz.de/cache/jqlang/jq:1.7 "$@" +} From 224c1472b2b9ff64b85a923fcb78d311709ec6e8 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 31 Jan 2024 14:23:14 +0000 Subject: [PATCH 178/243] fix: Correctly set file permissions --- ccp/modules/datashield-setup.sh | 8 +++++--- lib/functions.sh | 1 - lib/prepare-system.sh | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 464b0e1..9324305 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -12,9 +12,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then TOKEN_MANAGER_SECRET="$(echo \"Token Manager\" | generate_simple_password)" if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then mkdir -p /tmp/bridgehead/ - chown -R bridgehead:docker /tmp/bridgehead/ openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=opal/C=DE" - chmod g+r /tmp/bridgehead/opal-key.pem fi mkdir -p /tmp/bridgehead/opal-map sites="$(cat ./$PROJECT/modules/datashield-mappings.json)" @@ -29,6 +27,10 @@ if [ "$ENABLE_DATASHIELD" == true ]; then "internal": "opal:8443", "allowed": input | map("datashield-connect.\(.).'"$BROKER_ID"'") }]' > /tmp/bridgehead/opal-map/local.json - chown -R bridgehead:docker /tmp/bridgehead/* + if [ "$USER" == "root" ]; then + chown -R bridgehead:docker /tmp/bridgehead + chmod g+wr /tmp/bridgehead/opal-map/* + chmod g+r /tmp/bridgehead/opal-key.pem + fi add_private_oidc_redirect_url "/opal/*" fi diff --git a/lib/functions.sh b/lib/functions.sh index b54ceec..897eef2 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -279,7 +279,6 @@ function sync_secrets() { fi mkdir -p /var/cache/bridgehead/secrets/ touch /var/cache/bridgehead/secrets/oidc - chown -R bridgehead:docker /var/cache/bridgehead/secrets # The oidc provider will need to be switched based on the project at some point I guess docker run --rm \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index cd470b2..c43c0b1 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -88,7 +88,9 @@ elif [[ "$DEV_MODE" == "DEV" ]]; then git clone "$url" /etc/bridgehead fi -chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead +mkdir -p /tmp/bridgehead /var/cache/bridgehead +chown -R bridgehead:docker /etc/bridgehead /srv/docker/bridgehead /tmp/bridgehead /var/cache/bridgehead +chmod -R g+wr /var/cache/bridgehead /tmp/bridgehead log INFO "System preparation is completed and configuration is present." From 32ffb33ab161750f0952f6c01aca4132e06f5620 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 5 Feb 2024 08:31:00 +0000 Subject: [PATCH 179/243] fix: Only give writeable dirs the docker role --- lib/prepare-system.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index c43c0b1..156f7c8 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -88,8 +88,9 @@ elif [[ "$DEV_MODE" == "DEV" ]]; then git clone "$url" /etc/bridgehead fi +chown -R bridgehead /etc/bridgehead /srv/docker/bridgehead mkdir -p /tmp/bridgehead /var/cache/bridgehead -chown -R bridgehead:docker /etc/bridgehead /srv/docker/bridgehead /tmp/bridgehead /var/cache/bridgehead +chown -R bridgehead:docker /tmp/bridgehead /var/cache/bridgehead chmod -R g+wr /var/cache/bridgehead /tmp/bridgehead log INFO "System preparation is completed and configuration is present." From 51e8888fe1533d914d5da699d65ae2558ecfab9a Mon Sep 17 00:00:00 2001 From: janskiba Date: Tue, 6 Feb 2024 14:08:11 +0000 Subject: [PATCH 180/243] Use latest jq --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index 897eef2..ed2570b 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -346,5 +346,5 @@ generate_simple_password(){ } docker_jq() { - docker run --rm -i docker.verbis.dkfz.de/cache/jqlang/jq:1.7 "$@" + docker run --rm -i docker.verbis.dkfz.de/cache/jqlang/jq:latest "$@" } From af3e5231d88d40b3b6d9528e23cca0c655b712d0 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 6 Feb 2024 17:18:10 +0100 Subject: [PATCH 181/243] Added: Proxy to R-Studio oauth2-proxy --- ccp/modules/datashield-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index f777a01..e4e925b 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -112,7 +112,7 @@ services: APP_token-manager_KEY: ${TOKEN_MANAGER_SECRET} # TODO: Allow users of group /DataSHIELD and KEYCLOAK_USER_GROUP at the same time: - # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/docs/configuration/overview/): + # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/configuration/oauth_provider): # --allowed-groups=/DataSHIELD,KEYCLOAK_USER_GROUP oauth2_proxy: image: quay.io/oauth2-proxy/oauth2-proxy @@ -150,6 +150,9 @@ services: - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`, `/oauth2/callback`)" - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180" - "traefik.http.routers.oauth2_proxy.tls=true" + environment: + http_proxy: "http://forward_proxy:3128" + https_proxy: "http://forward_proxy:3128" secrets: opal-cert.pem: From 4a9427a1bd3b333475d13cb3eb58b34a707458de Mon Sep 17 00:00:00 2001 From: janskiba Date: Tue, 6 Feb 2024 16:18:23 +0000 Subject: [PATCH 182/243] fix: Use forward proxy for secret sync --- lib/functions.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index ed2570b..ee2cf2c 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -279,19 +279,23 @@ function sync_secrets() { fi mkdir -p /var/cache/bridgehead/secrets/ touch /var/cache/bridgehead/secrets/oidc + $COMPOSE -p secret_sync -f ./minimal/docker-compose.yml up -d forward_proxy # The oidc provider will need to be switched based on the project at some point I guess docker run --rm \ + --network secret_sync_default \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \ -v /srv/docker/bridgehead/$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \ -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \ - -e HTTPS_PROXY=$HTTPS_PROXY_FULL_URL \ + -e NO_PROXY=localhost,127.0.0.1 \ + -e ALL_PROXY=http://forward_proxy:3128 \ -e PROXY_ID=$PROXY_ID \ -e BROKER_URL=$BROKER_URL \ -e OIDC_PROVIDER=secret-sync-central.oidc-client-enrollment.$BROKER_ID \ -e SECRET_DEFINITIONS=$secret_sync_args \ docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest + $COMPOSE -p secret_sync -f ./minimal/docker-compose.yml down forward_proxy set -a # Export variables as environment variables source /var/cache/bridgehead/secrets/* set +a # Export variables in the regular way From b241feecdb8a1cc145a6cd769bc2578e4b6700b4 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 7 Feb 2024 14:08:00 +0000 Subject: [PATCH 183/243] fix: Pull oauth2 proxy from harbor --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index e4e925b..89a38d3 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -115,7 +115,7 @@ services: # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/configuration/oauth_provider): # --allowed-groups=/DataSHIELD,KEYCLOAK_USER_GROUP oauth2_proxy: - image: quay.io/oauth2-proxy/oauth2-proxy + image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:latest container_name: bridgehead_oauth2_proxy command: >- --allowed-group=/DataSHIELD From f3fa1ce712377bfede72b197023724e6f65f21db Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 7 Feb 2024 16:05:26 +0000 Subject: [PATCH 184/243] fix: secret sync account for minimal override --- lib/functions.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index ee2cf2c..7ec79ce 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -279,7 +279,11 @@ function sync_secrets() { fi mkdir -p /var/cache/bridgehead/secrets/ touch /var/cache/bridgehead/secrets/oidc - $COMPOSE -p secret_sync -f ./minimal/docker-compose.yml up -d forward_proxy + local override="" + if [ -f "minimal/docker-compose.override.yml" ]; then + override+=" -f ./minimal/docker-compose.override.yml" + fi + $COMPOSE -p secret_sync -f ./minimal/docker-compose.yml $override up -d forward_proxy # The oidc provider will need to be switched based on the project at some point I guess docker run --rm \ --network secret_sync_default \ From 64250d9d218f22e40d98092fe575add3775c921f Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 8 Feb 2024 13:38:37 +0000 Subject: [PATCH 185/243] refactor: Use beam proxy directly as proxy --- bridgehead | 2 +- lib/functions.sh | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/bridgehead b/bridgehead index 49ad3a5..b1ce678 100755 --- a/bridgehead +++ b/bridgehead @@ -90,7 +90,7 @@ case "$ACTION" in loadVars hc_send log "Bridgehead $PROJECT startup: Checking requirements ..." checkRequirements - sync_secrets + sync_secrets hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..." exec $COMPOSE -p $PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit ;; diff --git a/lib/functions.sh b/lib/functions.sh index 7ec79ce..c175fcf 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -279,27 +279,20 @@ function sync_secrets() { fi mkdir -p /var/cache/bridgehead/secrets/ touch /var/cache/bridgehead/secrets/oidc - local override="" - if [ -f "minimal/docker-compose.override.yml" ]; then - override+=" -f ./minimal/docker-compose.override.yml" - fi - $COMPOSE -p secret_sync -f ./minimal/docker-compose.yml $override up -d forward_proxy - # The oidc provider will need to be switched based on the project at some point I guess docker run --rm \ - --network secret_sync_default \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \ -v /srv/docker/bridgehead/$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \ -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \ -e NO_PROXY=localhost,127.0.0.1 \ - -e ALL_PROXY=http://forward_proxy:3128 \ + -e ALL_PROXY=$HTTPS_PROXY_FULL_URL \ -e PROXY_ID=$PROXY_ID \ -e BROKER_URL=$BROKER_URL \ -e OIDC_PROVIDER=secret-sync-central.oidc-client-enrollment.$BROKER_ID \ -e SECRET_DEFINITIONS=$secret_sync_args \ docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest - $COMPOSE -p secret_sync -f ./minimal/docker-compose.yml down forward_proxy + set -a # Export variables as environment variables source /var/cache/bridgehead/secrets/* set +a # Export variables in the regular way From 1995997ac203c306af61e1f71b8347ca3da0ecc7 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 8 Feb 2024 13:39:17 +0000 Subject: [PATCH 186/243] fix: Wait for forward proxy to start --- ccp/modules/datashield-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 89a38d3..2127ac2 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -153,6 +153,8 @@ services: environment: http_proxy: "http://forward_proxy:3128" https_proxy: "http://forward_proxy:3128" + depends_on: + - forward_proxy secrets: opal-cert.pem: From 97a558dd461d639f6182069cac6fcbf57aa0cc6a Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 13 Feb 2024 15:18:37 +0100 Subject: [PATCH 187/243] Removed:Login-compose --- ccp/modules/login-compose.yml | 47 ----------------------------------- ccp/modules/login-setup.sh | 7 ------ ccp/modules/login.md | 13 ---------- 3 files changed, 67 deletions(-) delete mode 100644 ccp/modules/login-setup.sh delete mode 100644 ccp/modules/login.md diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml index 787d4b2..e69de29 100644 --- a/ccp/modules/login-compose.yml +++ b/ccp/modules/login-compose.yml @@ -1,47 +0,0 @@ -version: "3.7" - -services: - - login-db: - image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG} - container_name: bridgehead-login-db - environment: - POSTGRES_USER: "keycloak" - POSTGRES_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in login-setup.sh - POSTGRES_DB: "keycloak" - tmpfs: - - /var/lib/postgresql/data -# Consider removing this comment once we have collected experience in production. -# volumes: -# - "bridgehead-login-db:/var/lib/postgresql/data" - - login: - image: docker.verbis.dkfz.de/ccp/dktk-keycloak:latest - container_name: bridgehead-login - environment: - KEYCLOAK_ADMIN: "admin" - KEYCLOAK_ADMIN_PASSWORD: "${LDM_AUTH}" - TEILER_ADMIN: "${PROJECT}" - TEILER_ADMIN_PASSWORD: "${LDM_AUTH}" - TEILER_ADMIN_FIRST_NAME: "${OPERATOR_FIRST_NAME}" - TEILER_ADMIN_LAST_NAME: "${OPERATOR_LAST_NAME}" - TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" - KC_DB_PASSWORD: "${KEYCLOAK_DB_PASSWORD}" # Set in login-setup.sh - KC_HOSTNAME_URL: "https://${HOST}/login" - KC_HOSTNAME_STRICT: "false" - KC_PROXY_ADDRESS_FORWARDING: "true" - TEILER_ORCHESTRATOR_EXTERN_URL: "https://${HOST}/ccp-teiler" - command: - - start-dev --import-realm --proxy edge --http-relative-path=/login - labels: - - "traefik.enable=true" - - "traefik.http.routers.login.rule=PathPrefix(`/login`)" - - "traefik.http.services.login.loadbalancer.server.port=8080" - - "traefik.http.routers.login.tls=true" - depends_on: - - login-db - -# Consider removing this comment once we have collected experience in production. -#volumes: -# bridgehead-login-db: -# name: "bridgehead-login-db" diff --git a/ccp/modules/login-setup.sh b/ccp/modules/login-setup.sh deleted file mode 100644 index 1981b87..0000000 --- a/ccp/modules/login-setup.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -e - -if [ "$ENABLE_LOGIN" == true ]; then - log INFO "Login setup detected -- will start Login services." - OVERRIDE+=" -f ./$PROJECT/modules/login-compose.yml" - KEYCLOAK_DB_PASSWORD="$(generate_password \"local Keycloak\")" -fi diff --git a/ccp/modules/login.md b/ccp/modules/login.md deleted file mode 100644 index eee488c..0000000 --- a/ccp/modules/login.md +++ /dev/null @@ -1,13 +0,0 @@ -# Login -The login component is a local Keycloak instance. In the future will be replaced by the central keycloak instance -or maybe can be used to add local identity providers to the bridgehead or just to simplify the configuration of -the central keycloak instance for the integration of every new bridgehead. -The basic configuration of our Keycloak instance is contained in a small json file. - -### Teiler User -Currently, the local keycloak is used by the teiler. There is a basic admin user in the basic configuration of keycloak. -The user can be configured with the environment variables TEILER_ADMIN_XXX. - -## Login-DB -Keycloak requires a local database for its configuration. However, as we use an initial json configuration file, if no -local identity provider is configured nor any local user, theoretically we don't need a volume for the login. From cea577bde58aac3ed2d7099a8d48b2202bc90011 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 14 Feb 2024 21:43:12 +0100 Subject: [PATCH 188/243] Removed: login-compose --- ccp/modules/login-compose.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ccp/modules/login-compose.yml diff --git a/ccp/modules/login-compose.yml b/ccp/modules/login-compose.yml deleted file mode 100644 index e69de29..0000000 From ef8866b94315d7e483cc4208769028abde531110 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 15 Feb 2024 13:13:52 +0000 Subject: [PATCH 189/243] fix: Start oauth proxy after forward_proxy is ready --- ccp/modules/datashield-compose.yml | 3 ++- minimal/docker-compose.yml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 2127ac2..db2760a 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -154,7 +154,8 @@ services: http_proxy: "http://forward_proxy:3128" https_proxy: "http://forward_proxy:3128" depends_on: - - forward_proxy + forward_proxy: + condition: service_healthy secrets: opal-cert.pem: diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index 9a43953..217f1b3 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -42,6 +42,8 @@ services: - /var/spool/squid volumes: - /etc/bridgehead/trusted-ca-certs:/docker/custom-certs/:ro + healthcheck: + test: ["CMD", "sleep", "1"] landing: container_name: bridgehead-landingpage From 2eb56e66c872a256a2e81ada4f0de16b545c3e79 Mon Sep 17 00:00:00 2001 From: juarez Date: Fri, 17 Nov 2023 10:27:12 +0100 Subject: [PATCH 190/243] Integrate central Keycloak in Teiler --- ccp/modules/datashield-compose.yml | 2 +- ccp/vars | 1 + lib/functions.sh | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index db2760a..e3e0d01 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -96,7 +96,7 @@ services: networks: - default - rstudio - + traefik: networks: - default diff --git a/ccp/vars b/ccp/vars index eb998d7..eb2a1c8 100644 --- a/ccp/vars +++ b/ccp/vars @@ -10,6 +10,7 @@ BROKER_URL_FOR_PREREQ=$BROKER_URL DEFAULT_LANGUAGE=DE DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} ENABLE_EXPORTER=true +ENABLE_LOGIN=true ENABLE_TEILER=true #ENABLE_DATASHIELD=true diff --git a/lib/functions.sh b/lib/functions.sh index c175fcf..d32bdbe 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -270,7 +270,7 @@ function sync_secrets() { if [[ $OIDC_PRIVATE_REDIRECT_URLS != "" ]]; then if [[ $secret_sync_args == "" ]]; then secret_sync_args="OIDC:OIDC_PUBLIC:public;$OIDC_PUBLIC_REDIRECT_URLS" - else + else secret_sync_args+="${delimiter}OIDC:OIDC_PUBLIC:public;$OIDC_PUBLIC_REDIRECT_URLS" fi fi @@ -282,7 +282,7 @@ function sync_secrets() { docker run --rm \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \ - -v /srv/docker/bridgehead/$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ + -v ./$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \ -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \ -e NO_PROXY=localhost,127.0.0.1 \ From 29d2bc04408a3cc1e6ff9c5abf343c1cf0e8e3be Mon Sep 17 00:00:00 2001 From: juarez Date: Mon, 27 Nov 2023 19:39:16 +0100 Subject: [PATCH 191/243] Add Keycloak to MTBA --- ccp/modules/mtba-compose.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index f03532f..3912bcb 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -21,12 +21,6 @@ services: FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE:-LF} CSV_DELIMITER: ${MTBA_CSV_DELIMITER:-TAB} HTTP_RELATIVE_PATH: "/mtba" - KEYCLOAK_ADMIN_GROUP: "${KEYCLOAK_ADMIN_GROUP}" - KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PRIVATE_CLIENT_ID}" - KEYCLOAK_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}" - KEYCLOAK_REALM: "${KEYCLOAK_REALM}" - KEYCLOAK_URL: "${KEYCLOAK_URL}" - labels: - "traefik.enable=true" - "traefik.http.routers.mtba_ccp.rule=PathPrefix(`/mtba`)" From 8a197ce5c7606e802f4615cb665a40ecd263a8d1 Mon Sep 17 00:00:00 2001 From: juarez Date: Wed, 29 Nov 2023 09:29:18 +0100 Subject: [PATCH 192/243] Add oauth2_proxy --- ccp/modules/datashield-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index e3e0d01..105c9ae 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -8,12 +8,14 @@ services: #DEFAULT_USER: "rstudio" # This line is kept for informational purposes PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" # It is required, even if the authentication is disabled DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use + # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. HTTP_RELATIVE_PATH: "/rstudio" ALL_PROXY: "http://forward_proxy:3128" # https://rocker-project.org/use/networking.html labels: - "traefik.enable=true" - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)" - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787" + - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.routers.rstudio_ccp.middlewares=oidcAuth,rstudio_ccp_strip" From 9a1860ccf9fb1d62be46150b6cde5c889d21b814 Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 13 Feb 2024 15:58:24 +0100 Subject: [PATCH 193/243] Removed: / from groups --- ccp/modules/datashield-compose.yml | 2 +- ccp/modules/mtba-compose.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 105c9ae..bc09e1f 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -120,7 +120,7 @@ services: image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:latest container_name: bridgehead_oauth2_proxy command: >- - --allowed-group=/DataSHIELD + --allowed-group=DataSHIELD --oidc-groups-claim=${KEYCLOAK_GROUP_CLAIM} --auth-logging=true --whitelist-domain=${HOST} diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 3912bcb..042bca1 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -3,7 +3,8 @@ version: "3.7" services: mtba: #image: docker.verbis.dkfz.de/cache/samply/mtba:latest - image: docker.verbis.dkfz.de/cache/samply/mtba:develop + #image: docker.verbis.dkfz.de/cache/samply/mtba:develop + image: samply/mtba:develop container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 From 19d0fefe94b147d9a0e8d891c2f5138d9b711f1a Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 13 Feb 2024 18:49:06 +0100 Subject: [PATCH 194/243] Changed: master realm --- ccp/vars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/vars b/ccp/vars index eb2a1c8..f4d70fe 100644 --- a/ccp/vars +++ b/ccp/vars @@ -18,8 +18,8 @@ KEYCLOAK_USER_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})" KEYCLOAK_ADMIN_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})_Verwalter" KEYCLOAK_PRIVATE_CLIENT_ID=${SITE_ID}-private KEYCLOAK_PUBLIC_CLIENT_ID=${SITE_ID}-public -# TODO: Change Keycloak Realm to productive. "test-realm-01" is only for testing -KEYCLOAK_REALM="${KEYCLOAK_REALM:-test-realm-01}" +# Use "test-realm-01" for testing +KEYCLOAK_REALM="${KEYCLOAK_REALM:-master}" KEYCLOAK_URL="https://login.verbis.dkfz.de" KEYCLOAK_ISSUER_URL="${KEYCLOAK_URL}/realms/${KEYCLOAK_REALM}" KEYCLOAK_GROUP_CLAIM="groups" From f72e7c77990081dd3ee64458c616daf6594e99aa Mon Sep 17 00:00:00 2001 From: juarez Date: Tue, 13 Feb 2024 18:54:26 +0100 Subject: [PATCH 195/243] Changed: replace keycloak with oidc --- ccp/modules/datashield-compose.yml | 22 +++++++++++----------- ccp/modules/mtba-compose.yml | 10 +++++++--- ccp/modules/teiler-compose.yml | 14 +++++++------- ccp/vars | 17 ++++++++--------- lib/functions.sh | 2 +- minimal/docker-compose.yml | 2 -- 6 files changed, 34 insertions(+), 33 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index bc09e1f..19a5e35 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -15,7 +15,6 @@ services: - "traefik.enable=true" - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)" - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787" - - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio" - "traefik.http.routers.rstudio_ccp.tls=true" - "traefik.http.routers.rstudio_ccp.middlewares=oidcAuth,rstudio_ccp_strip" @@ -46,11 +45,11 @@ services: APP_CONTEXT_PATH: "/opal" OPAL_PRIVATE_KEY: "/run/secrets/opal-key.pem" OPAL_CERTIFICATE: "/run/secrets/opal-cert.pem" - KEYCLOAK_URL: "${KEYCLOAK_URL}" - KEYCLOAK_REALM: "${KEYCLOAK_REALM}" - KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PRIVATE_CLIENT_ID}" - KEYCLOAK_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}" - KEYCLOAK_ADMIN_GROUP: "${KEYCLOAK_ADMIN_GROUP}" + OIDC_URL: "${OIDC_URL}" + OIDC_REALM: "${OIDC_REALM}" + OIDC_CLIENT_ID: "${OIDC_PRIVATE_CLIENT_ID}" + OIDC_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}" + OIDC_ADMIN_GROUP: "${OIDC_ADMIN_GROUP}" TOKEN_MANAGER_PASSWORD: "${TOKEN_MANAGER_OPAL_PASSWORD}" EXPORTER_PASSWORD: "${EXPORTER_OPAL_PASSWORD}" BEAM_APP_ID: token-manager.${PROXY_ID} @@ -113,15 +112,15 @@ services: APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET} APP_token-manager_KEY: ${TOKEN_MANAGER_SECRET} - # TODO: Allow users of group /DataSHIELD and KEYCLOAK_USER_GROUP at the same time: + # TODO: Allow users of group /DataSHIELD and OIDC_USER_GROUP at the same time: # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/configuration/oauth_provider): - # --allowed-groups=/DataSHIELD,KEYCLOAK_USER_GROUP + # --allowed-groups=/DataSHIELD,OIDC_USER_GROUP oauth2_proxy: image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:latest container_name: bridgehead_oauth2_proxy command: >- --allowed-group=DataSHIELD - --oidc-groups-claim=${KEYCLOAK_GROUP_CLAIM} + --oidc-groups-claim=${OIDC_GROUP_CLAIM} --auth-logging=true --whitelist-domain=${HOST} --http-address="0.0.0.0:4180" @@ -136,10 +135,10 @@ services: #OIDC settings --provider="keycloak-oidc" --provider-display-name="VerbIS Login" - --client-id="${KEYCLOAK_PRIVATE_CLIENT_ID}" + --client-id="${OIDC_PRIVATE_CLIENT_ID}" --client-secret="${OIDC_CLIENT_SECRET}" --redirect-url="https://${HOST}${OAUTH2_CALLBACK}" - --oidc-issuer-url="${KEYCLOAK_ISSUER_URL}" + --oidc-issuer-url="${OIDC_ISSUER_URL}" --scope="openid email profile" --code-challenge-method="S256" --skip-provider-button=true @@ -147,6 +146,7 @@ services: --pass-basic-auth=true --pass-user-headers=false --pass-access-token=false + labels: - "traefik.enable=true" - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`, `/oauth2/callback`)" diff --git a/ccp/modules/mtba-compose.yml b/ccp/modules/mtba-compose.yml index 042bca1..56bb015 100644 --- a/ccp/modules/mtba-compose.yml +++ b/ccp/modules/mtba-compose.yml @@ -2,9 +2,7 @@ version: "3.7" services: mtba: - #image: docker.verbis.dkfz.de/cache/samply/mtba:latest - #image: docker.verbis.dkfz.de/cache/samply/mtba:develop - image: samply/mtba:develop + image: docker.verbis.dkfz.de/cache/samply/mtba:develop container_name: bridgehead-mtba environment: BLAZE_STORE_URL: http://blaze:8080 @@ -22,6 +20,12 @@ services: FILE_END_OF_LINE: ${MTBA_FILE_END_OF_LINE:-LF} CSV_DELIMITER: ${MTBA_CSV_DELIMITER:-TAB} HTTP_RELATIVE_PATH: "/mtba" + OIDC_ADMIN_GROUP: "${OIDC_ADMIN_GROUP}" + OIDC_CLIENT_ID: "${OIDC_PRIVATE_CLIENT_ID}" + OIDC_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}" + OIDC_REALM: "${OIDC_REALM}" + OIDC_URL: "${OIDC_URL}" + labels: - "traefik.enable=true" - "traefik.http.routers.mtba_ccp.rule=PathPrefix(`/mtba`)" diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index 40e394c..a76f161 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -19,7 +19,7 @@ services: HTTP_RELATIVE_PATH: "/ccp-teiler" teiler-dashboard: - image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:latest + image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:develop container_name: bridgehead-teiler-dashboard labels: - "traefik.enable=true" @@ -31,10 +31,10 @@ services: environment: DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend" - KEYCLOAK_URL: "${KEYCLOAK_URL}" - KEYCLOAK_REALM: "${KEYCLOAK_REALM}" - KEYCLOAK_CLIENT_ID: "${KEYCLOAK_PUBLIC_CLIENT_ID}" - KEYCLOAK_TOKEN_GROUP: "${KEYCLOAK_GROUP_CLAIM}" + OIDC_URL: "${OIDC_URL}" + OIDC_REALM: "${OIDC_REALM}" + OIDC_CLIENT_ID: "${OIDC_PUBLIC_CLIENT_ID}" + OIDC_TOKEN_GROUP: "${OIDC_GROUP_CLAIM}" TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}" TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}" TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}" @@ -43,8 +43,8 @@ services: TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler" TEILER_DASHBOARD_HTTP_RELATIVE_PATH: "/ccp-teiler-dashboard" TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler" - TEILER_USER: "${KEYCLOAK_USER_GROUP}" - TEILER_ADMIN: "${KEYCLOAK_ADMIN_GROUP}" + TEILER_USER: "${OIDC_USER_GROUP}" + TEILER_ADMIN: "${OIDC_ADMIN_GROUP}" REPORTER_DEFAULT_TEMPLATE_ID: "ccp-qb" EXPORTER_DEFAULT_TEMPLATE_ID: "ccp" diff --git a/ccp/vars b/ccp/vars index f4d70fe..c1e9887 100644 --- a/ccp/vars +++ b/ccp/vars @@ -10,19 +10,18 @@ BROKER_URL_FOR_PREREQ=$BROKER_URL DEFAULT_LANGUAGE=DE DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} ENABLE_EXPORTER=true -ENABLE_LOGIN=true ENABLE_TEILER=true #ENABLE_DATASHIELD=true -KEYCLOAK_USER_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})" -KEYCLOAK_ADMIN_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})_Verwalter" -KEYCLOAK_PRIVATE_CLIENT_ID=${SITE_ID}-private -KEYCLOAK_PUBLIC_CLIENT_ID=${SITE_ID}-public +OIDC_USER_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})" +OIDC_ADMIN_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})_Verwalter" +OIDC_PRIVATE_CLIENT_ID=${SITE_ID}-private +OIDC_PUBLIC_CLIENT_ID=${SITE_ID}-public # Use "test-realm-01" for testing -KEYCLOAK_REALM="${KEYCLOAK_REALM:-master}" -KEYCLOAK_URL="https://login.verbis.dkfz.de" -KEYCLOAK_ISSUER_URL="${KEYCLOAK_URL}/realms/${KEYCLOAK_REALM}" -KEYCLOAK_GROUP_CLAIM="groups" +OIDC_REALM="${OIDC_REALM:-master}" +OIDC_URL="https://login.verbis.dkfz.de" +OIDC_ISSUER_URL="${OIDC_URL}/realms/${OIDC_REALM}" +OIDC_GROUP_CLAIM="groups" OAUTH2_CALLBACK=/oauth2/callback OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" diff --git a/lib/functions.sh b/lib/functions.sh index d32bdbe..fa2a144 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -282,7 +282,7 @@ function sync_secrets() { docker run --rm \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ -v $PRIVATEKEYFILENAME:/run/secrets/privkey.pem:ro \ - -v ./$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ + -v /srv/docker/bridgehead/$PROJECT/root.crt.pem:/run/secrets/root.crt.pem:ro \ -v /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro \ -e TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs \ -e NO_PROXY=localhost,127.0.0.1 \ diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index 217f1b3..9c761af 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -57,5 +57,3 @@ services: HOST: ${HOST} PROJECT: ${PROJECT} SITE_NAME: ${SITE_NAME} - - From 3e44dab9f212f2d536f7eb4d04ce9709a9e91be2 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 19 Feb 2024 08:26:53 +0000 Subject: [PATCH 196/243] chore: Remame datashield mappings to datashield sites --- ccp/modules/datashield-setup.sh | 2 +- ccp/modules/{datashield-mappings.json => datashield-sites.json} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ccp/modules/{datashield-mappings.json => datashield-sites.json} (100%) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 9324305..858d31f 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -15,7 +15,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=opal/C=DE" fi mkdir -p /tmp/bridgehead/opal-map - sites="$(cat ./$PROJECT/modules/datashield-mappings.json)" + sites="$(cat ./$PROJECT/modules/datashield-sites.json)" echo "$sites" | docker_jq -n --args '{"sites": input | map({ "name": ., "id": ., diff --git a/ccp/modules/datashield-mappings.json b/ccp/modules/datashield-sites.json similarity index 100% rename from ccp/modules/datashield-mappings.json rename to ccp/modules/datashield-sites.json From fb4da54297a8949f7c0783bc63b27c0cc31659dc Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 19 Feb 2024 08:30:49 +0000 Subject: [PATCH 197/243] chore: Add mannheim to datashield sites --- ccp/modules/datashield-sites.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccp/modules/datashield-sites.json b/ccp/modules/datashield-sites.json index 7d8dad4..07e2966 100644 --- a/ccp/modules/datashield-sites.json +++ b/ccp/modules/datashield-sites.json @@ -9,5 +9,6 @@ "frankfurt", "essen", "dktk-datashield-test", - "dktk-test" + "dktk-test", + "mannheim" ] From 74eb86f8af14f763342232912a5342bd4cf99257 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 21 Feb 2024 15:25:02 +0000 Subject: [PATCH 198/243] fix: Update permissions on update --- lib/functions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/functions.sh b/lib/functions.sh index fa2a144..d5c3a8c 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -132,6 +132,10 @@ assertVarsNotEmpty() { fixPermissions() { CHOWN=$(which chown) sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead + if [ -d "/tmp/bridgehead" ]; then # Used by datashield + sudo chown -R bridgehead:docker "/tmp/bridgehead" + if [ -d "/var/cache/bridgehead" ]; then # Used by the teiler + sudo chown -R bridgehead:docker "/var/cache/bridgehead" } source lib/monitoring.sh From db9692795af31617fb8c0dc756f69906a8e4f000 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 21 Feb 2024 15:28:51 +0000 Subject: [PATCH 199/243] fix: Fix if syntrax --- lib/functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/functions.sh b/lib/functions.sh index d5c3a8c..cc2f3ec 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -134,8 +134,10 @@ fixPermissions() { sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead if [ -d "/tmp/bridgehead" ]; then # Used by datashield sudo chown -R bridgehead:docker "/tmp/bridgehead" + fi if [ -d "/var/cache/bridgehead" ]; then # Used by the teiler sudo chown -R bridgehead:docker "/var/cache/bridgehead" + fi } source lib/monitoring.sh From 3777d4bf054622e9e7ada7041dc590cafb9d2d25 Mon Sep 17 00:00:00 2001 From: Torben Brenner <76154651+torbrenner@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:34:00 +0100 Subject: [PATCH 200/243] Add default value for BLAZE_MEMORY_CAP Co-authored-by: Tobias Kussel --- bbmri/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index dcd446c..378b64a 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -8,7 +8,7 @@ services: container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 823bcdf..a87754a 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -6,7 +6,7 @@ services: container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" - JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP}m" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: From b5c35211f625f1a8714685808fcb7fc37d89ef65 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 5 Mar 2024 14:58:06 +0000 Subject: [PATCH 201/243] Dont test clock skew and priv key for minimal bridgeheads --- lib/prerequisites.sh | 45 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 10166e0..235826a 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -67,29 +67,30 @@ log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..." source /etc/bridgehead/${PROJECT}.conf source ${PROJECT}/vars -set +e -SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" -RET=$? -set -e -if [ $RET -ne 0 ]; then - log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." - log WARN "Unable to check clock skew due to previous error." -else - log INFO "Checking clock skew ..." +if [ "${PROJECT}" != "minimal" ]; then + set +e + SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')" + RET=$? + set -e + if [ $RET -ne 0 ]; then + log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication." + log WARN "Unable to check clock skew due to previous error." + else + log INFO "Checking clock skew ..." - SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) - MYTIME=$(date +%s) - SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) - SKEW=$(echo $SKEW | awk -F- '{print $NF}') - SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." - if [ $SKEW -ge 300 ]; then - report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" - exit 1 - elif [ $SKEW -ge 60 ]; then - log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" - fi + SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s) + MYTIME=$(date +%s) + SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME)) + SKEW=$(echo $SKEW | awk -F- '{print $NF}') + SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd." + if [ $SKEW -ge 300 ]; then + report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT" + exit 1 + elif [ $SKEW -ge 60 ]; then + log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT" + fi + fi fi - checkPrivKey() { if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then log INFO "Success - private key found." @@ -100,7 +101,7 @@ checkPrivKey() { return 0 } -if [[ "$@" =~ "noprivkey" ]]; then +if [[ "$@" =~ "noprivkey" || "${PROJECT}" != "minimal" ]]; then log INFO "Skipping check for private key for now." else checkPrivKey || exit 1 From ea6441fbcbe8fc0510ef0bb70f7d57a4e511f9aa Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Fri, 8 Mar 2024 08:33:15 +0100 Subject: [PATCH 202/243] Updated ccp focus to 0.4.1 --- ccp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 5e26878..ac80334 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -19,7 +19,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" focus: - image: docker.verbis.dkfz.de/cache/samply/focus:0.4.0 + image: docker.verbis.dkfz.de/cache/samply/focus:0.4.1 container_name: bridgehead-focus environment: API_KEY: ${FOCUS_BEAM_SECRET_SHORT} From eeb17e7bfef300367787d8262583e8074d6c4acb Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 8 Mar 2024 13:33:30 +0100 Subject: [PATCH 203/243] feat: added optional resource cache cap --- bbmri/docker-compose.yml | 1 + ccp/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 378b64a..e48bd77 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -9,6 +9,7 @@ services: environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index a87754a..331ce0d 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -7,6 +7,7 @@ services: environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: From 7478d804dfd17eb00eb8ab22f9f695783a87672d Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 11 Mar 2024 10:34:05 +0000 Subject: [PATCH 204/243] refactor: Move vars to their setup files --- ccp/docker-compose.yml | 6 ------ ccp/modules/datashield-compose.yml | 4 ++++ ccp/modules/datashield-setup.sh | 3 +++ ccp/modules/teiler-setup.sh | 2 ++ ccp/vars | 5 ----- lib/functions.sh | 8 ++++---- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index e85e909..5e26878 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -52,12 +52,6 @@ services: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - /srv/docker/bridgehead/ccp/root.crt.pem:/conf/root.crt.pem:ro - traefik: - labels: - - "traefik.http.middlewares.oidcAuth.forwardAuth.address=http://oauth2_proxy:4180/" - - "traefik.http.middlewares.oidcAuth.forwardAuth.trustForwardHeader=true" - - "traefik.http.middlewares.oidcAuth.forwardAuth.authResponseHeaders=X-Auth-Request-Access-Token,Authorization" - volumes: blaze-data: diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 19a5e35..48f5276 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -99,6 +99,10 @@ services: - rstudio traefik: + labels: + - "traefik.http.middlewares.oidcAuth.forwardAuth.address=http://oauth2_proxy:4180/" + - "traefik.http.middlewares.oidcAuth.forwardAuth.trustForwardHeader=true" + - "traefik.http.middlewares.oidcAuth.forwardAuth.authResponseHeaders=X-Auth-Request-Access-Token,Authorization" networks: - default - rstudio diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 858d31f..7674ecf 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -1,6 +1,9 @@ #!/bin/bash -e if [ "$ENABLE_DATASHIELD" == true ]; then + # HACK: This only works because exporter-setup.sh and teiler-setup.sh are sourced after datashield-setup.sh + ENABLE_EXPORTER=true + ENABLE_TEILER=true log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" EXPORTER_OPAL_PASSWORD="$(generate_password \"exporter in Opal\")" diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index 1e97079..cc561d5 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -3,5 +3,7 @@ if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler setup detected -- will start Teiler services." OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" + DEFAULT_LANGUAGE=DE + DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} add_public_oidc_redirect_url "/ccp-teiler/*" fi diff --git a/ccp/vars b/ccp/vars index c1e9887..33f3e26 100644 --- a/ccp/vars +++ b/ccp/vars @@ -7,11 +7,6 @@ SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL -DEFAULT_LANGUAGE=DE -DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} -ENABLE_EXPORTER=true -ENABLE_TEILER=true -#ENABLE_DATASHIELD=true OIDC_USER_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})" OIDC_ADMIN_GROUP="DKTK_CCP_$(capitalize_first_letter ${SITE_ID})_Verwalter" diff --git a/lib/functions.sh b/lib/functions.sh index cc2f3ec..0e44a7f 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -133,11 +133,11 @@ fixPermissions() { CHOWN=$(which chown) sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead if [ -d "/tmp/bridgehead" ]; then # Used by datashield - sudo chown -R bridgehead:docker "/tmp/bridgehead" - fi + sudo $CHOWN -R bridgehead:docker "/tmp/bridgehead" + fi if [ -d "/var/cache/bridgehead" ]; then # Used by the teiler - sudo chown -R bridgehead:docker "/var/cache/bridgehead" - fi + sudo $CHOWN -R bridgehead:docker "/var/cache/bridgehead" + fi } source lib/monitoring.sh From e1e523f1acf09cdd11810bc6f551becf1f46665b Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Mon, 11 Mar 2024 09:58:30 +0100 Subject: [PATCH 205/243] refactor: tune configuration of blaze according to system memory --- bridgehead | 2 +- lib/functions.sh | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/bridgehead b/bridgehead index 68e29e1..62cb38e 100755 --- a/bridgehead +++ b/bridgehead @@ -50,7 +50,7 @@ loadVars() { source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import" fi fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" - setBlazeMemoryCap + optimizeBlazeMemoryUsage [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a diff --git a/lib/functions.sh b/lib/functions.sh index ab904d3..ecd330f 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -155,13 +155,26 @@ setHostname() { fi } -# blaze memory cap should be approximately a quarter of the system memory -# the memory cap will be applied to both the java heap size and db clock cache -setBlazeMemoryCap() { +# This function optimizes the usage of memory through blaze, according to the official performance tuning guide: +# https://github.com/samply/blaze/blob/master/docs/tuning-guide.md +# Short summary of the adjustments made: +# - set blaze memory cap to a quarter of the system memory +# - set db block cache size to a quarter of the system memory +# - limit resource count allowed in blaze to 1,25M per 4GB available system memory +optimizeBlazeMemoryUsage() { if [ -z "$BLAZE_MEMORY_CAP" ]; then system_memory_in_mb=$(free -m | grep 'Mem:' | awk '{print $2}'); export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); fi + if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then + available_system_memory_chuncks=$((BLAZE_MEMORY_CAP / 1000)) + if [ $available_system_memory_chuncks -eq 0 ]; then + log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower." + export BLAZE_RESOURCE_CACHE_CAP=128000; + else + export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chuncks * 312500)) + fi + fi } # Takes 1) The Backup Directory Path 2) The name of the Service to be backuped From 5a6322fcaaea66dbcf46be2209d4f7d761149508 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 14 Mar 2024 11:50:08 +0000 Subject: [PATCH 206/243] refactor: Move oauth2 proxy related things to datashield setup --- ccp/modules/datashield-setup.sh | 5 +++++ ccp/vars | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 7674ecf..4d7489c 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -4,6 +4,11 @@ if [ "$ENABLE_DATASHIELD" == true ]; then # HACK: This only works because exporter-setup.sh and teiler-setup.sh are sourced after datashield-setup.sh ENABLE_EXPORTER=true ENABLE_TEILER=true + + OAUTH2_CALLBACK=/oauth2/callback + OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" + add_private_oidc_redirect_url "${OAUTH2_CALLBACK}" + log INFO "DataSHIELD setup detected -- will start DataSHIELD services." OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml" EXPORTER_OPAL_PASSWORD="$(generate_password \"exporter in Opal\")" diff --git a/ccp/vars b/ccp/vars index 33f3e26..fa3f5a2 100644 --- a/ccp/vars +++ b/ccp/vars @@ -17,10 +17,7 @@ OIDC_REALM="${OIDC_REALM:-master}" OIDC_URL="https://login.verbis.dkfz.de" OIDC_ISSUER_URL="${OIDC_URL}/realms/${OIDC_REALM}" OIDC_GROUP_CLAIM="groups" -OAUTH2_CALLBACK=/oauth2/callback -OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" -add_private_oidc_redirect_url "${OAUTH2_CALLBACK}" POSTGRES_TAG=15.6-alpine for module in $PROJECT/modules/*.sh From 1f17fad3668890a06814193d6fe7959436e99034 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 14 Mar 2024 14:09:21 +0000 Subject: [PATCH 207/243] fix: Dont change ownership of all files under /tmp/bridgehead and /var/cache/bridgehead --- lib/functions.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 0e44a7f..ac2bafc 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -132,12 +132,6 @@ assertVarsNotEmpty() { fixPermissions() { CHOWN=$(which chown) sudo $CHOWN -R bridgehead /etc/bridgehead /srv/docker/bridgehead - if [ -d "/tmp/bridgehead" ]; then # Used by datashield - sudo $CHOWN -R bridgehead:docker "/tmp/bridgehead" - fi - if [ -d "/var/cache/bridgehead" ]; then # Used by the teiler - sudo $CHOWN -R bridgehead:docker "/var/cache/bridgehead" - fi } source lib/monitoring.sh @@ -283,7 +277,7 @@ function sync_secrets() { if [[ $secret_sync_args == "" ]]; then return fi - mkdir -p /var/cache/bridgehead/secrets/ + mkdir -p /var/cache/bridgehead/secrets/ || fail_and_report 1 "Failed to create '/var/cache/bridgehead/secrets/'. Please run sudo './bridgehead install $PROJECT' again." touch /var/cache/bridgehead/secrets/oidc docker run --rm \ -v /var/cache/bridgehead/secrets/oidc:/usr/local/cache \ From 6217e285902e64589289c293eaeda89d3fff2c2a Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 15 Mar 2024 11:48:25 +0100 Subject: [PATCH 208/243] fix: use always english output of free command --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index ecd330f..b4dc876 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -163,7 +163,7 @@ setHostname() { # - limit resource count allowed in blaze to 1,25M per 4GB available system memory optimizeBlazeMemoryUsage() { if [ -z "$BLAZE_MEMORY_CAP" ]; then - system_memory_in_mb=$(free -m | grep 'Mem:' | awk '{print $2}'); + system_memory_in_mb=$(LC_ALL=C free -m | grep 'Mem:' | awk '{print $2}'); export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); fi if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then From 6969a7a3bc6edea5457e5171b766f3221899f045 Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 12:02:53 +0100 Subject: [PATCH 209/243] Remove unnecessary comment --- bridgehead | 1 - 1 file changed, 1 deletion(-) diff --git a/bridgehead b/bridgehead index b1ce678..2706df8 100755 --- a/bridgehead +++ b/bridgehead @@ -41,7 +41,6 @@ case "$PROJECT" in ;; esac -# TODO: Please add proper documentation for variable priorities (1. secrets, 2. vars, 3. PROJECT.local.conf, 4. PROJECT.conf, 5. ??? loadVars() { # Load variables from /etc/bridgehead and /srv/docker/bridgehead set -a From ff06782234a4d836f196173b00f7a1e400e9c0d0 Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 12:04:04 +0100 Subject: [PATCH 210/243] Remove todo --- ccp/modules/datashield-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 48f5276..9237db7 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -8,7 +8,6 @@ services: #DEFAULT_USER: "rstudio" # This line is kept for informational purposes PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" # It is required, even if the authentication is disabled DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use - # TODO: Connect R-Studio with central Keycloak. Currently using Traefik authentication. HTTP_RELATIVE_PATH: "/rstudio" ALL_PROXY: "http://forward_proxy:3128" # https://rocker-project.org/use/networking.html labels: From 349027e9691bede131c443028e48dde9219e12cc Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 12:12:16 +0100 Subject: [PATCH 211/243] Rename oauth2_proxy docker service to oauth2-proxy --- ccp/modules/datashield-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 9237db7..968363d 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -99,7 +99,7 @@ services: traefik: labels: - - "traefik.http.middlewares.oidcAuth.forwardAuth.address=http://oauth2_proxy:4180/" + - "traefik.http.middlewares.oidcAuth.forwardAuth.address=http://oauth2-proxy:4180/" - "traefik.http.middlewares.oidcAuth.forwardAuth.trustForwardHeader=true" - "traefik.http.middlewares.oidcAuth.forwardAuth.authResponseHeaders=X-Auth-Request-Access-Token,Authorization" networks: @@ -118,9 +118,9 @@ services: # TODO: Allow users of group /DataSHIELD and OIDC_USER_GROUP at the same time: # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/configuration/oauth_provider): # --allowed-groups=/DataSHIELD,OIDC_USER_GROUP - oauth2_proxy: + oauth2-proxy: image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:latest - container_name: bridgehead_oauth2_proxy + container_name: bridgehead-oauth2proxy command: >- --allowed-group=DataSHIELD --oidc-groups-claim=${OIDC_GROUP_CLAIM} From 591d95e8db95a5b82df096a19b6c9e3ec52e7ec9 Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 12:13:09 +0100 Subject: [PATCH 212/243] Remove empty line --- ccp/modules/datashield-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 968363d..6426145 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -149,7 +149,6 @@ services: --pass-basic-auth=true --pass-user-headers=false --pass-access-token=false - labels: - "traefik.enable=true" - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`, `/oauth2/callback`)" From 8cb33c2ddc4d71b1d7ce38bfbe750f32390d52cf Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 12:18:19 +0100 Subject: [PATCH 213/243] Add warning if ENABLE_EXPORTER is not set or set to true --- ccp/modules/datashield-setup.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 4d7489c..5020f73 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -2,9 +2,9 @@ if [ "$ENABLE_DATASHIELD" == true ]; then # HACK: This only works because exporter-setup.sh and teiler-setup.sh are sourced after datashield-setup.sh - ENABLE_EXPORTER=true - ENABLE_TEILER=true - + if [ -z "${ENABLE_EXPORTER}" ] || [ "${ENABLE_EXPORTER}" != "true" ]; then + echo "The ENABLE_EXPORTER variable is either not set or not set to 'true'." + fi OAUTH2_CALLBACK=/oauth2/callback OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" add_private_oidc_redirect_url "${OAUTH2_CALLBACK}" @@ -29,12 +29,12 @@ if [ "$ENABLE_DATASHIELD" == true ]; then "id": ., "virtualhost": "\(.):443", "beamconnect": "datashield-connect.\(.).'"$BROKER_ID"'" - })}' $sites > /tmp/bridgehead/opal-map/central.json + })}' $sites >/tmp/bridgehead/opal-map/central.json echo "$sites" | docker_jq -n --args '[{ "external": "'"$SITE_ID"':443", "internal": "opal:8443", "allowed": input | map("datashield-connect.\(.).'"$BROKER_ID"'") - }]' > /tmp/bridgehead/opal-map/local.json + }]' >/tmp/bridgehead/opal-map/local.json if [ "$USER" == "root" ]; then chown -R bridgehead:docker /tmp/bridgehead chmod g+wr /tmp/bridgehead/opal-map/* From 3a8df378a6283fc0177131cfa07334af73c6ddbc Mon Sep 17 00:00:00 2001 From: djuarezgf <46350150+djuarezgf@users.noreply.github.com> Date: Mon, 18 Mar 2024 12:36:09 +0100 Subject: [PATCH 214/243] Update lib/functions.sh Co-authored-by: Tobias Kussel --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index ac2bafc..3d20729 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -334,7 +334,7 @@ generate_password(){ local random_special=${special:$n:1} local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret." - local main_password=$(echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/\//A/g') + local main_password=$(echo "${combined_text}" | sha1sum | openssl pkeyutl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/\//A/g') echo "${main_password}${random_digit}${random_upper}${random_lower}${random_special}" } From 3180d0fd769187bda1f9a44c2fd15310e5c20137 Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 12:44:34 +0100 Subject: [PATCH 215/243] Replace | openssl rsautl -sign with | sha1sum | openssl pkeyutl -sign --- ccp/modules/datashield-setup.sh | 2 +- ccp/modules/exporter-setup.sh | 4 ++-- lib/functions.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 5020f73..ed725ee 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -6,7 +6,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then echo "The ENABLE_EXPORTER variable is either not set or not set to 'true'." fi OAUTH2_CALLBACK=/oauth2/callback - OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" + OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" add_private_oidc_redirect_url "${OAUTH2_CALLBACK}" log INFO "DataSHIELD setup detected -- will start DataSHIELD services." diff --git a/ccp/modules/exporter-setup.sh b/ccp/modules/exporter-setup.sh index e33978c..9b947a6 100644 --- a/ccp/modules/exporter-setup.sh +++ b/ccp/modules/exporter-setup.sh @@ -3,6 +3,6 @@ if [ "$ENABLE_EXPORTER" == true ]; then log INFO "Exporter setup detected -- will start Exporter service." OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml" - EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" - EXPORTER_API_KEY="$(echo \"This is a salt string to generate one consistent API KEY for the exporter. It is not required to be secret.\" | openssl rsautl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 64)" + EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" + EXPORTER_API_KEY="$(echo \"This is a salt string to generate one consistent API KEY for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 64)" fi diff --git a/lib/functions.sh b/lib/functions.sh index 3d20729..1d6bfe9 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -343,7 +343,7 @@ generate_password(){ generate_simple_password(){ local seed_text="$1" local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret." - echo "${combined_text}" | openssl rsautl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/[+\/]/A/g' + echo "${combined_text}" | sha1sum | openssl pkeyutl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/[+\/]/A/g' } docker_jq() { From d8b9498ef915c27b8ab0d4b95e567132212cbd69 Mon Sep 17 00:00:00 2001 From: djuarezgf <46350150+djuarezgf@users.noreply.github.com> Date: Mon, 18 Mar 2024 12:45:46 +0100 Subject: [PATCH 216/243] Update minimal/docker-compose.yml Co-authored-by: Jan <59206115+Threated@users.noreply.github.com> --- minimal/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/minimal/docker-compose.yml b/minimal/docker-compose.yml index 9c761af..e9f53d6 100644 --- a/minimal/docker-compose.yml +++ b/minimal/docker-compose.yml @@ -43,6 +43,7 @@ services: volumes: - /etc/bridgehead/trusted-ca-certs:/docker/custom-certs/:ro healthcheck: + # Wait 1s before marking this service healthy. Required for the oauth2-proxy to talk to the OIDC provider on startup which will fail if the forward proxy is not started yet. test: ["CMD", "sleep", "1"] landing: From 480bbe04e75ad9453815a88ae6296862786f6f46 Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 16:47:40 +0100 Subject: [PATCH 217/243] Changed: TEILER_DEFAULT_LANGUAGE --- bbmri/modules/dnpm-compose.yml | 0 ccp/modules/teiler-compose.yml | 6 +++--- ccp/modules/teiler-setup.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 bbmri/modules/dnpm-compose.yml diff --git a/bbmri/modules/dnpm-compose.yml b/bbmri/modules/dnpm-compose.yml deleted file mode 100644 index e69de29..0000000 diff --git a/ccp/modules/teiler-compose.yml b/ccp/modules/teiler-compose.yml index a76f161..f415ee9 100644 --- a/ccp/modules/teiler-compose.yml +++ b/ccp/modules/teiler-compose.yml @@ -15,7 +15,7 @@ services: environment: TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend" TEILER_DASHBOARD_URL: "https://${HOST}/ccp-teiler-dashboard" - DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE_LOWER_CASE}" + DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE_LOWER_CASE}" HTTP_RELATIVE_PATH: "/ccp-teiler" teiler-dashboard: @@ -29,7 +29,7 @@ services: - "traefik.http.middlewares.teiler_dashboard_ccp_strip.stripprefix.prefixes=/ccp-teiler-dashboard" - "traefik.http.routers.teiler_dashboard_ccp.middlewares=teiler_dashboard_ccp_strip" environment: - DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" + DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE}" TEILER_BACKEND_URL: "https://${HOST}/ccp-teiler-backend" OIDC_URL: "${OIDC_URL}" OIDC_REALM: "${OIDC_REALM}" @@ -63,7 +63,7 @@ services: LOG_LEVEL: "INFO" APPLICATION_PORT: "8085" APPLICATION_ADDRESS: "${HOST}" - DEFAULT_LANGUAGE: "${DEFAULT_LANGUAGE}" + DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE}" CONFIG_ENV_VAR_PATH: "/run/secrets/ccp.conf" TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/ccp-teiler" TEILER_ORCHESTRATOR_URL: "https://${HOST}/ccp-teiler" diff --git a/ccp/modules/teiler-setup.sh b/ccp/modules/teiler-setup.sh index cc561d5..eed3f81 100644 --- a/ccp/modules/teiler-setup.sh +++ b/ccp/modules/teiler-setup.sh @@ -3,7 +3,7 @@ if [ "$ENABLE_TEILER" == true ];then log INFO "Teiler setup detected -- will start Teiler services." OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml" - DEFAULT_LANGUAGE=DE - DEFAULT_LANGUAGE_LOWER_CASE=${DEFAULT_LANGUAGE,,} + TEILER_DEFAULT_LANGUAGE=DE + TEILER_DEFAULT_LANGUAGE_LOWER_CASE=${TEILER_DEFAULT_LANGUAGE,,} add_public_oidc_redirect_url "/ccp-teiler/*" fi From adf8e35ba9c30e1722d6ce1676203c2606553009 Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Mon, 18 Mar 2024 19:22:10 +0100 Subject: [PATCH 218/243] Remove empty file (teiler-ui-compose.yml) --- ccp/modules/teiler-ui-compose.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ccp/modules/teiler-ui-compose.yml diff --git a/ccp/modules/teiler-ui-compose.yml b/ccp/modules/teiler-ui-compose.yml deleted file mode 100644 index e69de29..0000000 From 896b24be9bfcf1ebdf9f5433b4c835b7fd63c529 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 19 Mar 2024 08:45:50 +0000 Subject: [PATCH 219/243] Use bridgehead log functions in datashield setup --- ccp/modules/datashield-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index ed725ee..1c8ef48 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -3,7 +3,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then # HACK: This only works because exporter-setup.sh and teiler-setup.sh are sourced after datashield-setup.sh if [ -z "${ENABLE_EXPORTER}" ] || [ "${ENABLE_EXPORTER}" != "true" ]; then - echo "The ENABLE_EXPORTER variable is either not set or not set to 'true'." + log warn "The ENABLE_EXPORTER variable is either not set or not set to 'true'." fi OAUTH2_CALLBACK=/oauth2/callback OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" From ec6f9302a1498773f5f2782cc54d2d64da14ef65 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 19 Mar 2024 08:47:57 +0000 Subject: [PATCH 220/243] Fix spelling of log WARN --- ccp/modules/datashield-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-setup.sh b/ccp/modules/datashield-setup.sh index 1c8ef48..7a22050 100644 --- a/ccp/modules/datashield-setup.sh +++ b/ccp/modules/datashield-setup.sh @@ -3,7 +3,7 @@ if [ "$ENABLE_DATASHIELD" == true ]; then # HACK: This only works because exporter-setup.sh and teiler-setup.sh are sourced after datashield-setup.sh if [ -z "${ENABLE_EXPORTER}" ] || [ "${ENABLE_EXPORTER}" != "true" ]; then - log warn "The ENABLE_EXPORTER variable is either not set or not set to 'true'." + log WARN "The ENABLE_EXPORTER variable is either not set or not set to 'true'." fi OAUTH2_CALLBACK=/oauth2/callback OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)" From 70ad318b28353ffadf68aa8b6fdb1a237ed3a9c8 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Fri, 22 Mar 2024 13:59:42 +0100 Subject: [PATCH 221/243] Update focus to 0.4.2 --- ccp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 0407e07..1a1c117 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -21,7 +21,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" focus: - image: docker.verbis.dkfz.de/cache/samply/focus:0.4.1 + image: docker.verbis.dkfz.de/cache/samply/focus:0.4.2 container_name: bridgehead-focus environment: API_KEY: ${FOCUS_BEAM_SECRET_SHORT} From 2831fb9a228ce9b71b0e5df4ef1337f3f1bbfaf6 Mon Sep 17 00:00:00 2001 From: janskiba Date: Tue, 2 Apr 2024 14:36:23 +0200 Subject: [PATCH 222/243] fix: Make math work on bash 4.2 --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index f277368..1a566a8 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -164,7 +164,7 @@ setHostname() { optimizeBlazeMemoryUsage() { if [ -z "$BLAZE_MEMORY_CAP" ]; then system_memory_in_mb=$(LC_ALL=C free -m | grep 'Mem:' | awk '{print $2}'); - export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4)); + export BLAZE_MEMORY_CAP=$(($system_memory_in_mb/4)); fi if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then available_system_memory_chuncks=$((BLAZE_MEMORY_CAP / 1000)) From 94b2c29bc78641642259c4e550c4574e5799f394 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Mon, 15 Apr 2024 08:31:57 +0200 Subject: [PATCH 223/243] GBN maintenance mode --- bbmri/modules/gbn-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbmri/modules/gbn-setup.sh b/bbmri/modules/gbn-setup.sh index 5ab1325..296b8a1 100644 --- a/bbmri/modules/gbn-setup.sh +++ b/bbmri/modules/gbn-setup.sh @@ -2,7 +2,7 @@ if [ "${ENABLE_GBN}" == "true" ]; then log INFO "GBN setup detected -- will start services for German Biobank Node." - OVERRIDE+=" -f ./$PROJECT/modules/gbn-compose.yml" + # OVERRIDE+=" -f ./$PROJECT/modules/gbn-compose.yml" # The environment needs to be defined in /etc/bridgehead case "$ENVIRONMENT" in From 7418861e8c2d7bf116332a89853cda4ccf1a69a4 Mon Sep 17 00:00:00 2001 From: Pierre Delpy <75260699+PierreDelpy@users.noreply.github.com> Date: Mon, 15 Apr 2024 09:08:56 +0200 Subject: [PATCH 224/243] fix typo functions.sh --- lib/functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/functions.sh b/lib/functions.sh index 1a566a8..f5db927 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -167,12 +167,12 @@ optimizeBlazeMemoryUsage() { export BLAZE_MEMORY_CAP=$(($system_memory_in_mb/4)); fi if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then - available_system_memory_chuncks=$((BLAZE_MEMORY_CAP / 1000)) - if [ $available_system_memory_chuncks -eq 0 ]; then + available_system_memory_chunks=$((BLAZE_MEMORY_CAP / 1000)) + if [ $available_system_memory_chunks -eq 0 ]; then log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower." export BLAZE_RESOURCE_CACHE_CAP=128000; else - export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chuncks * 312500)) + export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chunks * 312500)) fi fi } From b767b3230f12779d6b76b1fed179de5b334b54f4 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Mon, 15 Apr 2024 10:13:16 +0200 Subject: [PATCH 225/243] update: dktk focus to 0.4.4 --- ccp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 1a1c117..c4a3b0f 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -21,7 +21,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" focus: - image: docker.verbis.dkfz.de/cache/samply/focus:0.4.2 + image: docker.verbis.dkfz.de/cache/samply/focus:0.4.4 container_name: bridgehead-focus environment: API_KEY: ${FOCUS_BEAM_SECRET_SHORT} From 0745eab7b5dfd77c78f44435b89a1ed203d3fde8 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 15 Apr 2024 13:12:17 +0200 Subject: [PATCH 226/243] fix: Kill stale secret-sync instances --- bridgehead | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bridgehead b/bridgehead index dc82d80..bde1e16 100755 --- a/bridgehead +++ b/bridgehead @@ -96,6 +96,8 @@ case "$ACTION" in ;; stop) loadVars + # Kill stale secret-sync instances if present + docker kill $(docker ps -q --filter ancestor=docker.verbis.dkfz.de/cache/samply/secret-sync-local) 2>/dev/null || true # HACK: This is temporarily to properly shut down false bridgehead instances (bridgehead-ccp instead ccp) $COMPOSE -p bridgehead-$PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE down exec $COMPOSE -p $PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE down From f9a9baf13d006394d3475cd094c7ab945fb63c17 Mon Sep 17 00:00:00 2001 From: janskiba Date: Mon, 15 Apr 2024 15:53:27 +0200 Subject: [PATCH 227/243] fix: Generate public oidc client when there is no private client --- lib/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/functions.sh b/lib/functions.sh index f5db927..b519369 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -289,7 +289,7 @@ function sync_secrets() { if [[ $OIDC_PRIVATE_REDIRECT_URLS != "" ]]; then secret_sync_args="OIDC:OIDC_CLIENT_SECRET:private;$OIDC_PRIVATE_REDIRECT_URLS" fi - if [[ $OIDC_PRIVATE_REDIRECT_URLS != "" ]]; then + if [[ $OIDC_PUBLIC_REDIRECT_URLS != "" ]]; then if [[ $secret_sync_args == "" ]]; then secret_sync_args="OIDC:OIDC_PUBLIC:public;$OIDC_PUBLIC_REDIRECT_URLS" else From 1afbf88a76ed190c082eb0f8335eb1d7f2db905c Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Tue, 16 Apr 2024 09:23:42 +0200 Subject: [PATCH 228/243] fix: use only bbmri broker --- bbmri/modules/gbn-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbmri/modules/gbn-setup.sh b/bbmri/modules/gbn-setup.sh index 296b8a1..d0a87af 100644 --- a/bbmri/modules/gbn-setup.sh +++ b/bbmri/modules/gbn-setup.sh @@ -21,7 +21,7 @@ if [ "${ENABLE_GBN}" == "true" ]; then ;; esac - GBN_BROKER_URL=https://${GBN_BROKER_ID} + #GBN_BROKER_URL=https://${GBN_BROKER_ID} GBN_PROXY_ID=${SITE_ID}.${GBN_BROKER_ID} GBN_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" GBN_SUPPORT_EMAIL=feedback@germanbiobanknode.de From fe3fc6204a3870b8f6268c87d6ad07d283af6d19 Mon Sep 17 00:00:00 2001 From: patrickskowronekdkfz <86347677+patrickskowronekdkfz@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:01:04 +0200 Subject: [PATCH 229/243] Revert "GBN maintenance mode" --- bbmri/modules/gbn-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/modules/gbn-setup.sh b/bbmri/modules/gbn-setup.sh index d0a87af..5ab1325 100644 --- a/bbmri/modules/gbn-setup.sh +++ b/bbmri/modules/gbn-setup.sh @@ -2,7 +2,7 @@ if [ "${ENABLE_GBN}" == "true" ]; then log INFO "GBN setup detected -- will start services for German Biobank Node." - # OVERRIDE+=" -f ./$PROJECT/modules/gbn-compose.yml" + OVERRIDE+=" -f ./$PROJECT/modules/gbn-compose.yml" # The environment needs to be defined in /etc/bridgehead case "$ENVIRONMENT" in @@ -21,7 +21,7 @@ if [ "${ENABLE_GBN}" == "true" ]; then ;; esac - #GBN_BROKER_URL=https://${GBN_BROKER_ID} + GBN_BROKER_URL=https://${GBN_BROKER_ID} GBN_PROXY_ID=${SITE_ID}.${GBN_BROKER_ID} GBN_FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" GBN_SUPPORT_EMAIL=feedback@germanbiobanknode.de From 29c2b5ef699b493bf5f46ccd1b6fa5cc2c9c71cc Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 26 Apr 2024 11:29:38 +0200 Subject: [PATCH 230/243] refactor: Ensure Mainzelliste returns SSL in Responses Before, the Mainzelliste would always use http instead of https then referring to it self in responses --- ccp/modules/id-management-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ccp/modules/id-management-compose.yml b/ccp/modules/id-management-compose.yml index 06d408f..61a4733 100644 --- a/ccp/modules/id-management-compose.yml +++ b/ccp/modules/id-management-compose.yml @@ -29,6 +29,7 @@ services: container_name: bridgehead-patientlist environment: - TOMCAT_REVERSEPROXY_FQDN=${HOST} + - TOMCAT_REVERSEPROXY_SSL=true - ML_SITE=${IDMANAGEMENT_FRIENDLY_ID} - ML_DB_PASS=${PATIENTLIST_POSTGRES_PASSWORD} - ML_API_KEY=${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} From 033da484d123c03bc9f4e69c0c571dca7711c1bd Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Tue, 21 May 2024 16:16:40 +0200 Subject: [PATCH 231/243] switch focus of ccp to tag --- ccp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index c4a3b0f..331ce0d 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -21,7 +21,7 @@ services: - "traefik.http.routers.blaze_ccp.tls=true" focus: - image: docker.verbis.dkfz.de/cache/samply/focus:0.4.4 + image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG} container_name: bridgehead-focus environment: API_KEY: ${FOCUS_BEAM_SECRET_SHORT} From a018104e0bdde6312277472d19bd2c68971a5e8c Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 5 Jun 2024 12:35:44 +0000 Subject: [PATCH 232/243] feat: Add logs command for journalctl and rename old one to docker-logs --- README.md | 7 ++++--- bridgehead | 5 +++++ lib/functions.sh | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 05038ae..d95a3b0 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ sudo systemctl [enable|disable] bridgehead@.service After starting the Bridgehead, you can watch the initialization process with the following command: ```shell -journalctl -u bridgehead@bbmri -f +./bridghead logs -f ``` if this exits with something similar to the following: @@ -220,8 +220,9 @@ docker ps There should be 6 - 10 Docker proceses. If there are fewer, then you know that something has gone wrong. To see what is going on, run: ```shell -journalctl -u bridgehead@bbmri -f +./bridghead logs -f ``` +This translates to a journalctl command so all the regular journalctl flags can be used. Once the Bridgehead has passed these checks, take a look at the landing page: @@ -235,7 +236,7 @@ You can either do this in a browser or with curl. If you visit the URL in the br curl -k https://localhost ``` -If you get errors when you do this, you need to use ```docker logs``` to examine your landing page container in order to determine what is going wrong. +If you get errors when you do this, you can inspect the logs of your landing page container in order to determine what is going wrong. To do this you can use `./bridgehead docker-logs landing -f` to follow the logs of the container. This transaltes to a docker compose logs command meaning all the ususal docker logs flags work. If you have chosen to take part in our monitoring program (by setting the ```MONITOR_APIKEY``` variable in the configuration), you will be informed by email when problems are detected in your Bridgehead. diff --git a/bridgehead b/bridgehead index bde1e16..85593b0 100755 --- a/bridgehead +++ b/bridgehead @@ -107,6 +107,11 @@ case "$ACTION" in exit $? ;; logs) + loadVars + shift 2 + exec journalctl -u bridgehead@$PROJECT -u bridgehead-update@$PROJECT -a $@ + ;; + docker-logs) loadVars shift 2 exec $COMPOSE -p $PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE logs -f $@ diff --git a/lib/functions.sh b/lib/functions.sh index b519369..5e69a04 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -53,7 +53,7 @@ checkOwner(){ } printUsage() { - echo "Usage: bridgehead start|stop|logs|is-running|update|install|uninstall|adduser|enroll PROJECTNAME" + echo "Usage: bridgehead start|stop|logs|docker-logs|is-running|update|install|uninstall|adduser|enroll PROJECTNAME" echo "PROJECTNAME should be one of ccp|bbmri" } From ec9df1feec212bd00e806083124cb2f4b6208955 Mon Sep 17 00:00:00 2001 From: Jan <59206115+Threated@users.noreply.github.com> Date: Wed, 5 Jun 2024 14:57:42 +0200 Subject: [PATCH 233/243] Update README.md Co-authored-by: patrickskowronekdkfz <86347677+patrickskowronekdkfz@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d95a3b0..bb332e4 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,7 @@ You can either do this in a browser or with curl. If you visit the URL in the br curl -k https://localhost ``` -If you get errors when you do this, you can inspect the logs of your landing page container in order to determine what is going wrong. To do this you can use `./bridgehead docker-logs landing -f` to follow the logs of the container. This transaltes to a docker compose logs command meaning all the ususal docker logs flags work. +Should the landing page not show anything, you can inspect the logs of the containers to determine what is going wrong. To do this you can use `./bridgehead docker-logs -f` to follow the logs of the container. This transaltes to a docker compose logs command meaning all the ususal docker logs flags work. If you have chosen to take part in our monitoring program (by setting the ```MONITOR_APIKEY``` variable in the configuration), you will be informed by email when problems are detected in your Bridgehead. From 4fc53c00bf96fc3dd159f30c5347d34bf15dd385 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Tue, 11 Jun 2024 08:41:35 +0200 Subject: [PATCH 234/243] Fix typo --- README.md | 4 ++-- ccp/modules/datashield.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bb332e4..c7a864b 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ sudo systemctl [enable|disable] bridgehead@.service After starting the Bridgehead, you can watch the initialization process with the following command: ```shell -./bridghead logs -f +/srv/docker/bridgehead/bridgehead logs -f ``` if this exits with something similar to the following: @@ -220,7 +220,7 @@ docker ps There should be 6 - 10 Docker proceses. If there are fewer, then you know that something has gone wrong. To see what is going on, run: ```shell -./bridghead logs -f +/srv/docker/bridgehead/bridgehead logs -f ``` This translates to a journalctl command so all the regular journalctl flags can be used. diff --git a/ccp/modules/datashield.md b/ccp/modules/datashield.md index aa2c4cb..4de5168 100644 --- a/ccp/modules/datashield.md +++ b/ccp/modules/datashield.md @@ -1,5 +1,5 @@ # DataSHIELD -This module constitutes the infrastructure to run DataSHIELD within the bridghead. +This module constitutes the infrastructure to run DataSHIELD within the bridgehead. For more information about DataSHIELD, please visit https://www.datashield.org/ ## R-Studio From e72c9969529b74a1399695f2d65fdf0e23e451b3 Mon Sep 17 00:00:00 2001 From: janskiba Date: Thu, 13 Jun 2024 07:29:54 +0000 Subject: [PATCH 235/243] feat: allow setting focus retry count and increase default --- bbmri/vars | 2 +- ccp/vars | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/vars b/bbmri/vars index d1362fb..248fbee 100644 --- a/bbmri/vars +++ b/bbmri/vars @@ -4,7 +4,7 @@ # Makes only sense for German Biobanks : ${ENABLE_GBN:=false} -FOCUS_RETRY_COUNT=32 +FOCUS_RETRY_COUNT=${FOCUS_RETRY_COUNT:-64} PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem for module in $PROJECT/modules/*.sh diff --git a/ccp/vars b/ccp/vars index fa3f5a2..027a093 100644 --- a/ccp/vars +++ b/ccp/vars @@ -2,7 +2,7 @@ BROKER_ID=broker.ccp-it.dktk.dkfz.de BROKER_URL=https://${BROKER_ID} PROXY_ID=${SITE_ID}.${BROKER_ID} FOCUS_BEAM_SECRET_SHORT="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)" -FOCUS_RETRY_COUNT=32 +FOCUS_RETRY_COUNT=${FOCUS_RETRY_COUNT:-64} SUPPORT_EMAIL=support-ccp@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem From 0db7df1440775552b119ba78f8a2ad897c0a4523 Mon Sep 17 00:00:00 2001 From: Torben Brenner <76154651+torbrenner@users.noreply.github.com> Date: Fri, 28 Jun 2024 13:57:30 +0200 Subject: [PATCH 236/243] Update docker-compose.yml --- ccp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 331ce0d..24999c9 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:latest + image: docker.verbis.dkfz.de/cache/samply/blaze:0.27 container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" From f7751b9d92eecfa37b8261ee0e130d3d7b8a412f Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 28 Jun 2024 14:29:56 +0200 Subject: [PATCH 237/243] fix: set blaze to version 0.28 The 0.28 release is not downgradeable, therefore switching again to 0.28 --- bbmri/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index e48bd77..ac8df45 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:latest + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index 24999c9..52e7eb5 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.27 + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" From 2ee8e0185a09de0109690ff1906c042bc4c900e9 Mon Sep 17 00:00:00 2001 From: Pierre Delpy Date: Mon, 1 Jul 2024 12:46:33 +0200 Subject: [PATCH 238/243] feature: upgrade to oBDS2FHIR --- ccp/modules/adt2fhir-rest-compose.yml | 18 ----------------- ccp/modules/mtba-setup.sh | 1 - ccp/modules/obds2fhir-rest-compose.yml | 20 +++++++++++++++++++ ...-rest-setup.sh => obds2fhir-rest-setup.sh} | 10 +++++----- ccp/vars | 2 +- 5 files changed, 26 insertions(+), 25 deletions(-) delete mode 100644 ccp/modules/adt2fhir-rest-compose.yml create mode 100644 ccp/modules/obds2fhir-rest-compose.yml rename ccp/modules/{adt2fhir-rest-setup.sh => obds2fhir-rest-setup.sh} (55%) diff --git a/ccp/modules/adt2fhir-rest-compose.yml b/ccp/modules/adt2fhir-rest-compose.yml deleted file mode 100644 index bba8163..0000000 --- a/ccp/modules/adt2fhir-rest-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3.7" - -services: - adt2fhir-rest: - container_name: bridgehead-adt2fhir-rest - image: docker.verbis.dkfz.de/ccp/adt2fhir-rest:main - environment: - IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID - MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} - SALT: ${LOCAL_SALT} - restart: always - labels: - - "traefik.enable=true" - - "traefik.http.routers.adt2fhir-rest.rule=PathPrefix(`/adt2fhir-rest`)" - - "traefik.http.middlewares.adt2fhir-rest_strip.stripprefix.prefixes=/adt2fhir-rest" - - "traefik.http.services.adt2fhir-rest.loadbalancer.server.port=8080" - - "traefik.http.routers.adt2fhir-rest.tls=true" - - "traefik.http.routers.adt2fhir-rest.middlewares=adt2fhir-rest_strip,auth" \ No newline at end of file diff --git a/ccp/modules/mtba-setup.sh b/ccp/modules/mtba-setup.sh index cdf0f31..d2acbe2 100644 --- a/ccp/modules/mtba-setup.sh +++ b/ccp/modules/mtba-setup.sh @@ -5,7 +5,6 @@ function mtbaSetup() { log INFO "MTBA setup detected -- will start MTBA Service and CBioPortal." if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log ERROR "Missing ID-Management Module! Fix this by setting up ID Management:" - exit 1; fi OVERRIDE+=" -f ./$PROJECT/modules/mtba-compose.yml" add_private_oidc_redirect_url "/mtba/*" diff --git a/ccp/modules/obds2fhir-rest-compose.yml b/ccp/modules/obds2fhir-rest-compose.yml new file mode 100644 index 0000000..3f1b0e9 --- /dev/null +++ b/ccp/modules/obds2fhir-rest-compose.yml @@ -0,0 +1,20 @@ +version: "3.7" + +services: + obds2fhir-rest: + container_name: bridgehead-obds2fhir-rest + image: docker.verbis.dkfz.de/ccp/obds2fhir-rest:main + environment: + IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID + MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} + SALT: ${LOCAL_SALT} + KEEP_INTERNAL_ID: ${KEEP_INTERNAL_ID:-false} + MAINZELLISTE_URL: ${PATIENTLIST_URL:-"http://patientlist:8080/patientlist"} + restart: always + labels: + - "traefik.enable=true" + - "traefik.http.routers.obds2fhir-rest.rule=PathPrefix(`/obds2fhir-rest`) || PathPrefix(`/adt2fhir-rest`)" + - "traefik.http.middlewares.obds2fhir-rest_strip.stripprefix.prefixes=/obds2fhir-rest,/adt2fhir-rest" + - "traefik.http.services.obds2fhir-rest.loadbalancer.server.port=8080" + - "traefik.http.routers.obds2fhir-rest.tls=true" + - "traefik.http.routers.obds2fhir-rest.middlewares=obds2fhir-rest_strip,auth" \ No newline at end of file diff --git a/ccp/modules/adt2fhir-rest-setup.sh b/ccp/modules/obds2fhir-rest-setup.sh similarity index 55% rename from ccp/modules/adt2fhir-rest-setup.sh rename to ccp/modules/obds2fhir-rest-setup.sh index 707d9c5..504bc86 100644 --- a/ccp/modules/adt2fhir-rest-setup.sh +++ b/ccp/modules/obds2fhir-rest-setup.sh @@ -1,13 +1,13 @@ #!/bin/bash -function adt2fhirRestSetup() { - if [ -n "$ENABLE_ADT2FHIR_REST" ]; then - log INFO "ADT2FHIR-REST setup detected -- will start adt2fhir-rest API." +function obds2fhirRestSetup() { + if [ -n "$ENABLE_OBDS2FHIR_REST" ]; then + log INFO "oBDS2FHIR-REST setup detected -- will start obds2fhir-rest module." if [ ! -n "$IDMANAGER_LOCAL_PATIENTLIST_APIKEY" ]; then log ERROR "Missing ID-Management Module! Fix this by setting up ID Management:" - exit 1; + PATIENTLIST_URL=" " fi - OVERRIDE+=" -f ./$PROJECT/modules/adt2fhir-rest-compose.yml" + OVERRIDE+=" -f ./$PROJECT/modules/obds2fhir-rest-compose.yml" LOCAL_SALT="$(echo \"local-random-salt\" | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" fi } diff --git a/ccp/vars b/ccp/vars index 027a093..080b134 100644 --- a/ccp/vars +++ b/ccp/vars @@ -28,4 +28,4 @@ done idManagementSetup mtbaSetup -adt2fhirRestSetup +obds2fhirRestSetup From 9e4bc214cec349f92b679cad137d53c3d4255b09 Mon Sep 17 00:00:00 2001 From: janskiba Date: Wed, 3 Jul 2024 13:01:02 +0000 Subject: [PATCH 239/243] fix: Fix traefik label for oauth2 redirect --- ccp/modules/datashield-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/datashield-compose.yml b/ccp/modules/datashield-compose.yml index 6426145..5e92db3 100644 --- a/ccp/modules/datashield-compose.yml +++ b/ccp/modules/datashield-compose.yml @@ -151,7 +151,7 @@ services: --pass-access-token=false labels: - "traefik.enable=true" - - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`, `/oauth2/callback`)" + - "traefik.http.routers.oauth2_proxy.rule=Host(`${HOST}`) && PathPrefix(`/oauth2`)" - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180" - "traefik.http.routers.oauth2_proxy.tls=true" environment: From 91dc31d0398d9a97a213c40260bb1a99e63dd7b8 Mon Sep 17 00:00:00 2001 From: Pierre Delpy Date: Mon, 8 Jul 2024 13:54:42 +0200 Subject: [PATCH 240/243] fix: use correct ID management flag for oBDS2FHIR --- ccp/modules/obds2fhir-rest-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccp/modules/obds2fhir-rest-setup.sh b/ccp/modules/obds2fhir-rest-setup.sh index 504bc86..677ea63 100644 --- a/ccp/modules/obds2fhir-rest-setup.sh +++ b/ccp/modules/obds2fhir-rest-setup.sh @@ -3,7 +3,7 @@ function obds2fhirRestSetup() { if [ -n "$ENABLE_OBDS2FHIR_REST" ]; then log INFO "oBDS2FHIR-REST setup detected -- will start obds2fhir-rest module." - if [ ! -n "$IDMANAGER_LOCAL_PATIENTLIST_APIKEY" ]; then + if [ ! -n "$IDMANAGER_UPLOAD_APIKEY" ]; then log ERROR "Missing ID-Management Module! Fix this by setting up ID Management:" PATIENTLIST_URL=" " fi From b36c9ae03e2c8ce982c52e38182ef4c0fcbd3daa Mon Sep 17 00:00:00 2001 From: Pierre Delpy <75260699+PierreDelpy@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:49:23 +0200 Subject: [PATCH 241/243] Fix patientlisturl in obds2fhir-rest-compose.yml --- ccp/modules/obds2fhir-rest-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccp/modules/obds2fhir-rest-compose.yml b/ccp/modules/obds2fhir-rest-compose.yml index 3f1b0e9..f201e23 100644 --- a/ccp/modules/obds2fhir-rest-compose.yml +++ b/ccp/modules/obds2fhir-rest-compose.yml @@ -9,7 +9,7 @@ services: MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} SALT: ${LOCAL_SALT} KEEP_INTERNAL_ID: ${KEEP_INTERNAL_ID:-false} - MAINZELLISTE_URL: ${PATIENTLIST_URL:-"http://patientlist:8080/patientlist"} + MAINZELLISTE_URL: ${PATIENTLIST_URL:-http://patientlist:8080/patientlist} restart: always labels: - "traefik.enable=true" @@ -17,4 +17,4 @@ services: - "traefik.http.middlewares.obds2fhir-rest_strip.stripprefix.prefixes=/obds2fhir-rest,/adt2fhir-rest" - "traefik.http.services.obds2fhir-rest.loadbalancer.server.port=8080" - "traefik.http.routers.obds2fhir-rest.tls=true" - - "traefik.http.routers.obds2fhir-rest.middlewares=obds2fhir-rest_strip,auth" \ No newline at end of file + - "traefik.http.routers.obds2fhir-rest.middlewares=obds2fhir-rest_strip,auth" From 6b4480c54bae41ebcf92b5584c58f263c6377c9e Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 17 Jul 2024 09:53:08 +0200 Subject: [PATCH 242/243] workaround: add second blaze --- ccp/modules/blaze-secondary-compose.yml | 27 +++++++++++++++++++++++++ ccp/modules/blaze-secondary-setup.sh | 11 ++++++++++ ccp/vars | 1 + 3 files changed, 39 insertions(+) create mode 100644 ccp/modules/blaze-secondary-compose.yml create mode 100644 ccp/modules/blaze-secondary-setup.sh diff --git a/ccp/modules/blaze-secondary-compose.yml b/ccp/modules/blaze-secondary-compose.yml new file mode 100644 index 0000000..f3f4752 --- /dev/null +++ b/ccp/modules/blaze-secondary-compose.yml @@ -0,0 +1,27 @@ +version: "3.7" + +services: + blaze-secondary: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-ccp-blaze-secondary + environment: + BASE_URL: "http://bridgehead-ccp-blaze-secondary:8080" + JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" + DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} + DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "blaze-secondary-data:/app/data" + labels: + - "traefik.enable=true" + - "traefik.http.routers.blaze-secondary_ccp.rule=PathPrefix(`/ccp-localdatamanagement-secondary`)" + - "traefik.http.middlewares.ccp_b-secondary_strip.stripprefix.prefixes=/ccp-localdatamanagement-secondary" + - "traefik.http.services.blaze-secondary_ccp.loadbalancer.server.port=8080" + - "traefik.http.routers.blaze-secondary_ccp.middlewares=ccp_b-secondary_strip,auth" + - "traefik.http.routers.blaze-secondary_ccp.tls=true" + obds2fhir-rest: + environment: + STORE_PATH: ${STORE_PATH:-http://blaze:8080/fhir} + +volumes: + blaze-secondary-data: \ No newline at end of file diff --git a/ccp/modules/blaze-secondary-setup.sh b/ccp/modules/blaze-secondary-setup.sh new file mode 100644 index 0000000..307da01 --- /dev/null +++ b/ccp/modules/blaze-secondary-setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +function blazeSecondarySetup() { + if [ -n "$ENABLE_SECONDARY_BLAZE" ]; then + log INFO "Secondary Blaze setup detected -- will start second blaze." + OVERRIDE+=" -f ./$PROJECT/modules/blaze-secondary-compose.yml" + #make oBDS2FHIR ignore ID-Management and replace target Blaze + PATIENTLIST_URL=" " + STORE_PATH="http://blaze-secondary:8080/fhir" + fi +} diff --git a/ccp/vars b/ccp/vars index 080b134..0900914 100644 --- a/ccp/vars +++ b/ccp/vars @@ -29,3 +29,4 @@ done idManagementSetup mtbaSetup obds2fhirRestSetup +blazeSecondarySetup \ No newline at end of file From 293810f2541bf104b6ba7ef26ea6d1aaf9949162 Mon Sep 17 00:00:00 2001 From: djuarezgf Date: Thu, 18 Jul 2024 09:54:16 +0200 Subject: [PATCH 243/243] Added: exporter with blaze-secondary --- ccp/modules/blaze-secondary-compose.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ccp/modules/blaze-secondary-compose.yml b/ccp/modules/blaze-secondary-compose.yml index f3f4752..b57bfbe 100644 --- a/ccp/modules/blaze-secondary-compose.yml +++ b/ccp/modules/blaze-secondary-compose.yml @@ -19,9 +19,14 @@ services: - "traefik.http.services.blaze-secondary_ccp.loadbalancer.server.port=8080" - "traefik.http.routers.blaze-secondary_ccp.middlewares=ccp_b-secondary_strip,auth" - "traefik.http.routers.blaze-secondary_ccp.tls=true" + obds2fhir-rest: environment: STORE_PATH: ${STORE_PATH:-http://blaze:8080/fhir} + exporter: + environment: + BLAZE_HOST: "blaze-secondary" + volumes: - blaze-secondary-data: \ No newline at end of file + blaze-secondary-data: