diff --git a/README.md b/README.md index b7e60ad..0e8131a 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ The following URLs need to be accessible (prefix with `https://`): * git.verbis.dkfz.de * To fetch docker images * docker.verbis.dkfz.de - * Official Docker, Inc. URLs (subject to change, see [official list](https://docs.docker.com/desktop/all)) + * Official Docker, Inc. URLs (subject to change, see [official list](https://docs.docker.com/desktop/setup/allow-list/)) * hub.docker.com * registry-1.docker.io * production.cloudflare.docker.com @@ -154,7 +154,7 @@ Pay special attention to: Clone the bridgehead repository: ```shell sudo mkdir -p /srv/docker/ -sudo git clone https://github.com/samply/bridgehead.git /srv/docker/bridgehead +sudo git clone -b main https://github.com/samply/bridgehead.git /srv/docker/bridgehead ``` Then, run the installation script: diff --git a/bridgehead b/bridgehead index d5d3a20..cbe7527 100755 --- a/bridgehead +++ b/bridgehead @@ -53,17 +53,44 @@ case "$PROJECT" in ;; esac +# Loads config variables and runs the projects setup script loadVars() { - # Load variables from /etc/bridgehead and /srv/docker/bridgehead set -a + # Source the project specific config file source /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "/etc/bridgehead/$PROJECT.conf not found" + # Source the project specific local config file if present + # This file is ignored by git as oposed to the regular config file as it contains private site information like etl auth data if [ -e /etc/bridgehead/$PROJECT.local.conf ]; then log INFO "Applying /etc/bridgehead/$PROJECT.local.conf" source /etc/bridgehead/$PROJECT.local.conf || fail_and_report 1 "Found /etc/bridgehead/$PROJECT.local.conf but failed to import" fi + # Set execution environment on main default to prod else test + if [[ -z "${ENVIRONMENT+x}" ]]; then + if [ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]; then + ENVIRONMENT="production" + else + ENVIRONMENT="test" + fi + fi + # Source the versions of the images components + case "$ENVIRONMENT" in + "production") + source ./versions/prod + ;; + "test") + source ./versions/test + ;; + *) + report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!" + source ./versions/prod + ;; + esac fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" setHostname optimizeBlazeMemoryUsage + # Run project specific setup if it exists + # This will ususally modiy the `OVERRIDE` to include all the compose files that the project depends on + # This is also where projects specify which modules to load [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars set +a @@ -79,26 +106,6 @@ loadVars() { fi detectCompose setupProxy - - # Set some project-independent default values - : ${ENVIRONMENT:=production} - export ENVIRONMENT - - case "$ENVIRONMENT" in - "production") - export FOCUS_TAG=main - export BEAM_TAG=main - ;; - "test") - export FOCUS_TAG=develop - export BEAM_TAG=develop - ;; - *) - report_error 7 "Environment \"$ENVIRONMENT\" is unknown. Assuming production. FIX THIS!" - export FOCUS_TAG=main - export BEAM_TAG=main - ;; - esac } case "$ACTION" in diff --git a/ccp/modules/dnpm-compose.yml b/ccp/modules/dnpm-compose.yml index c32426f..0ce7f74 100644 --- a/ccp/modules/dnpm-compose.yml +++ b/ccp/modules/dnpm-compose.yml @@ -13,7 +13,7 @@ services: PROXY_APIKEY: ${DNPM_BEAM_SECRET_SHORT} APP_ID: dnpm-connect.${PROXY_ID} DISCOVERY_URL: "./conf/central_targets.json" - LOCAL_TARGETS_FILE: "./conf/connect_targets.json" + LOCAL_TARGETS_FILE: "/conf/connect_targets.json" HTTP_PROXY: "http://forward_proxy:3128" HTTPS_PROXY: "http://forward_proxy:3128" NO_PROXY: beam-proxy,dnpm-backend,host.docker.internal${DNPM_ADDITIONAL_NO_PROXY} @@ -25,7 +25,7 @@ services: volumes: - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro - /etc/bridgehead/dnpm/local_targets.json:/conf/connect_targets.json:ro - - /etc/bridgehead/dnpm/central_targets.json:/conf/central_targets.json:ro + - /srv/docker/bridgehead/minimal/modules/dnpm-central-targets.json:/conf/central_targets.json:ro labels: - "traefik.enable=true" - "traefik.http.routers.dnpm-connect.rule=PathPrefix(`/dnpm-connect`)" diff --git a/ccp/modules/dnpm-node-compose.yml b/ccp/modules/dnpm-node-compose.yml index ee84d89..c1f7dde 100644 --- a/ccp/modules/dnpm-node-compose.yml +++ b/ccp/modules/dnpm-node-compose.yml @@ -1,34 +1,99 @@ version: "3.7" services: - dnpm-backend: - image: ghcr.io/kohlbacherlab/bwhc-backend:1.0-snapshot-broker-connector - container_name: bridgehead-dnpm-backend + dnpm-mysql: + image: mysql:9 + healthcheck: + test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ] + interval: 3s + timeout: 5s + retries: 5 environment: - - ZPM_SITE=${ZPM_SITE} - - N_RANDOM_FILES=${DNPM_SYNTH_NUM} + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD: ${DNPM_MYSQL_ROOT_PASSWORD} volumes: - - /etc/bridgehead/dnpm:/bwhc_config:ro - - ${DNPM_DATA_DIR}:/bwhc_data - labels: - - "traefik.enable=true" - - "traefik.http.routers.bwhc-backend.rule=PathPrefix(`/bwhc`)" - - "traefik.http.services.bwhc-backend.loadbalancer.server.port=9000" - - "traefik.http.routers.bwhc-backend.tls=true" + - /var/cache/bridgehead/dnpm/mysql:/var/lib/mysql - dnpm-frontend: - image: ghcr.io/kohlbacherlab/bwhc-frontend:2209 - container_name: bridgehead-dnpm-frontend - links: - - dnpm-backend + dnpm-authup: + image: authup/authup:latest + container_name: bridgehead-dnpm-authup + volumes: + - /var/cache/bridgehead/dnpm/authup:/usr/src/app/writable + depends_on: + dnpm-mysql: + condition: service_healthy + command: server/core start environment: - - NUXT_HOST=0.0.0.0 - - NUXT_PORT=8080 - - BACKEND_PROTOCOL=https - - BACKEND_HOSTNAME=$HOST - - BACKEND_PORT=443 + - PUBLIC_URL=https://${HOST}/auth/ + - AUTHORIZE_REDIRECT_URL=https://${HOST} + - ROBOT_ADMIN_ENABLED=true + - ROBOT_ADMIN_SECRET=${DNPM_AUTHUP_SECRET} + - ROBOT_ADMIN_SECRET_RESET=true + - DB_TYPE=mysql + - DB_HOST=dnpm-mysql + - DB_USERNAME=root + - DB_PASSWORD=${DNPM_MYSQL_ROOT_PASSWORD} + - DB_DATABASE=auth labels: - "traefik.enable=true" - - "traefik.http.routers.bwhc-frontend.rule=PathPrefix(`/`)" - - "traefik.http.services.bwhc-frontend.loadbalancer.server.port=8080" - - "traefik.http.routers.bwhc-frontend.tls=true" + - "traefik.http.middlewares.authup-strip.stripprefix.prefixes=/auth" + - "traefik.http.routers.dnpm-auth.middlewares=authup-strip" + - "traefik.http.routers.dnpm-auth.rule=PathPrefix(`/auth`)" + - "traefik.http.services.dnpm-auth.loadbalancer.server.port=3000" + - "traefik.http.routers.dnpm-auth.tls=true" + + dnpm-portal: + image: ghcr.io/dnpm-dip/portal:latest + container_name: bridgehead-dnpm-portal + environment: + - NUXT_API_URL=http://dnpm-backend:9000/ + - NUXT_PUBLIC_API_URL=https://${HOST}/api/ + - NUXT_AUTHUP_URL=http://dnpm-authup:3000/ + - NUXT_PUBLIC_AUTHUP_URL=https://${HOST}/auth/ + labels: + - "traefik.enable=true" + - "traefik.http.routers.dnpm-frontend.rule=PathPrefix(`/`)" + - "traefik.http.services.dnpm-frontend.loadbalancer.server.port=3000" + - "traefik.http.routers.dnpm-frontend.tls=true" + + dnpm-backend: + container_name: bridgehead-dnpm-backend + image: ghcr.io/dnpm-dip/backend:latest + environment: + - LOCAL_SITE=${ZPM_SITE}:${SITE_NAME} # Format: {Site-ID}:{Site-name}, e.g. UKT:Tübingen + - RD_RANDOM_DATA=${DNPM_SYNTH_NUM:--1} + - MTB_RANDOM_DATA=${DNPM_SYNTH_NUM:--1} + - HATEOAS_HOST=https://${HOST} + - CONNECTOR_TYPE=broker + - AUTHUP_URL=robot://system:${DNPM_AUTHUP_SECRET}@http://dnpm-authup:3000 + volumes: + - /etc/bridgehead/dnpm/config:/dnpm_config + - /var/cache/bridgehead/dnpm/backend-data:/dnpm_data + depends_on: + dnpm-authup: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.services.dnpm-backend.loadbalancer.server.port=9000" + # expose everything + - "traefik.http.routers.dnpm-backend.rule=PathPrefix(`/api`)" + - "traefik.http.routers.dnpm-backend.tls=true" + - "traefik.http.routers.dnpm-backend.service=dnpm-backend" + # except ETL + - "traefik.http.routers.dnpm-backend-etl.rule=PathRegexp(`^/api(/.*)?etl(/.*)?$`)" + - "traefik.http.routers.dnpm-backend-etl.tls=true" + - "traefik.http.routers.dnpm-backend-etl.service=dnpm-backend" + # this needs an ETL processor with support for basic auth + - "traefik.http.routers.dnpm-backend-etl.middlewares=auth" + # except peer-to-peer + - "traefik.http.routers.dnpm-backend-peer.rule=PathRegexp(`^/api(/.*)?/peer2peer(/.*)?$`)" + - "traefik.http.routers.dnpm-backend-peer.tls=true" + - "traefik.http.routers.dnpm-backend-peer.service=dnpm-backend" + - "traefik.http.routers.dnpm-backend-peer.middlewares=dnpm-backend-peer" + # this effectively denies all requests + # this is okay, because requests from peers don't go through Traefik + - "traefik.http.middlewares.dnpm-backend-peer.ipWhiteList.sourceRange=0.0.0.0/32" + + landing: + labels: + - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)" diff --git a/ccp/modules/dnpm-node-setup.sh b/ccp/modules/dnpm-node-setup.sh index bf8fd26..f3681b5 100644 --- a/ccp/modules/dnpm-node-setup.sh +++ b/ccp/modules/dnpm-node-setup.sh @@ -1,28 +1,16 @@ #!/bin/bash if [ -n "${ENABLE_DNPM_NODE}" ]; then - log INFO "DNPM setup detected (BwHC Node) -- will start BwHC node." + log INFO "DNPM setup detected -- will start DNPM:DIP node." OVERRIDE+=" -f ./$PROJECT/modules/dnpm-node-compose.yml" # Set variables required for BwHC Node. ZPM_SITE is assumed to be set in /etc/bridgehead/.conf - DNPM_APPLICATION_SECRET="$(echo \"This is a salt string to generate one consistent password for DNPM. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ -z "${ZPM_SITE+x}" ]; then log ERROR "Mandatory variable ZPM_SITE not defined!" exit 1 fi - if [ -z "${DNPM_DATA_DIR+x}" ]; then - log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" - exit 1 - fi - DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + mkdir -p /var/cache/bridgehead/dnpm/ || fail_and_report 1 "Failed to create '/var/cache/bridgehead/dnpm/'. Please run sudo './bridgehead install $PROJECT' again to fix the permissions." + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1} + DNPM_MYSQL_ROOT_PASSWORD="$(generate_simple_password 'dnpm mysql')" + DNPM_AUTHUP_SECRET="$(generate_simple_password 'dnpm authup')" fi diff --git a/ccp/modules/obds2fhir-rest-compose.yml b/ccp/modules/obds2fhir-rest-compose.yml index 833580d..ec1737c 100644 --- a/ccp/modules/obds2fhir-rest-compose.yml +++ b/ccp/modules/obds2fhir-rest-compose.yml @@ -3,7 +3,7 @@ version: "3.7" services: obds2fhir-rest: container_name: bridgehead-obds2fhir-rest - image: docker.verbis.dkfz.de/ccp/obds2fhir-rest:main + image: docker.verbis.dkfz.de/samply/obds2fhir-rest:main environment: IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY} diff --git a/dhki/vars b/dhki/vars index b728925..d043dd2 100644 --- a/dhki/vars +++ b/dhki/vars @@ -17,4 +17,12 @@ do done idManagementSetup -obds2fhirRestSetup \ No newline at end of file +obds2fhirRestSetup + +for module in modules/*.sh +do + log DEBUG "sourcing $module" + source $module +done + +transfairSetup \ No newline at end of file diff --git a/lib/functions.sh b/lib/functions.sh index 3fcae38..ffdc234 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -116,7 +116,7 @@ assertVarsNotEmpty() { MISSING_VARS="" for VAR in $@; do - if [ -z "${!VAR}" ]; then + if [ -z "${!VAR}" ]; then MISSING_VARS+="$VAR " fi done @@ -318,7 +318,7 @@ function sync_secrets() { docker.verbis.dkfz.de/cache/samply/secret-sync-local:latest set -a # Export variables as environment variables - source /var/cache/bridgehead/secrets/* + source /var/cache/bridgehead/secrets/oidc set +a # Export variables in the regular way } diff --git a/lib/gitlab-token-helper.sh b/lib/gitlab-token-helper.sh new file mode 100755 index 0000000..e618029 --- /dev/null +++ b/lib/gitlab-token-helper.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +[ "$1" = "get" ] || exit + +source /var/cache/bridgehead/secrets/gitlab_token + +# Any non-empty username works, only the token matters +cat << EOF +username=bk +password=$BRIDGEHEAD_CONFIG_REPO_TOKEN +EOF \ No newline at end of file diff --git a/lib/gitpassword.sh b/lib/gitpassword.sh deleted file mode 100755 index 17756d6..0000000 --- a/lib/gitpassword.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -if [ "$1" != "get" ]; then - echo "Usage: $0 get" - exit 1 -fi - -baseDir() { - # see https://stackoverflow.com/questions/59895 - SOURCE=${BASH_SOURCE[0]} - while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - DIR=$( cd -P "$( dirname "$SOURCE" )/.." >/dev/null 2>&1 && pwd ) - echo $DIR -} - -BASE=$(baseDir) -cd $BASE - -source lib/functions.sh - -assertVarsNotEmpty SITE_ID || fail_and_report 1 "gitpassword.sh failed: SITE_ID is empty." - -PARAMS="$(cat)" -GITHOST=$(echo "$PARAMS" | grep "^host=" | sed 's/host=\(.*\)/\1/g') - -fetchVarsFromVault GIT_PASSWORD - -if [ -z "${GIT_PASSWORD}" ]; then - fail_and_report 1 "gitpassword.sh failed: Git password not found." -fi - -cat <.conf - DNPM_APPLICATION_SECRET="$(echo \"This is a salt string to generate one consistent password for DNPM. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)" if [ -z "${ZPM_SITE+x}" ]; then log ERROR "Mandatory variable ZPM_SITE not defined!" exit 1 fi - if [ -z "${DNPM_DATA_DIR+x}" ]; then - log ERROR "Mandatory variable DNPM_DATA_DIR not defined!" - exit 1 - fi - DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0} - if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then - echo "Override of landing page url already in place" - else - echo "Adding override of landing page url" - if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then - echo -e ' landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - else - echo -e 'version: "3.7"\nservices:\n landing:\n labels:\n - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml - fi - fi + mkdir -p /var/cache/bridgehead/dnpm/ || fail_and_report 1 "Failed to create '/var/cache/bridgehead/dnpm/'. Please run sudo './bridgehead install $PROJECT' again to fix the permissions." + DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:--1} + DNPM_MYSQL_ROOT_PASSWORD="$(generate_simple_password 'dnpm mysql')" + DNPM_AUTHUP_SECRET="$(generate_simple_password 'dnpm authup')" fi diff --git a/modules/transfair-compose.yml b/modules/transfair-compose.yml new file mode 100644 index 0000000..9af09a6 --- /dev/null +++ b/modules/transfair-compose.yml @@ -0,0 +1,51 @@ + +services: + transfair: + image: docker.verbis.dkfz.de/cache/samply/transfair:latest + container_name: bridgehead-transfair + environment: + # NOTE: Those 3 variables need only to be passed if their set, otherwise transfair will complain about empty url values + - INSTITUTE_TTP_URL + - INSTITUTE_TTP_API_KEY + - PROJECT_ID_SYSTEM + - FHIR_REQUEST_URL=${FHIR_REQUEST_URL} + - FHIR_INPUT_URL=${FHIR_INPUT_URL} + - FHIR_OUTPUT_URL=${FHIR_OUTPUT_URL:-http://blaze:8080} + - FHIR_REQUEST_CREDENTIALS=${FHIR_REQUEST_CREDENTIALS} + - FHIR_INPUT_CREDENTIALS=${FHIR_INPUT_CREDENTIALS} + - FHIR_OUTPUT_CREDENTIALS=${FHIR_OUTPUT_CREDENTIALS} + - EXCHANGE_ID_SYSTEM=${EXCHANGE_ID_SYSTEM:-SESSION_ID} + - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc + - RUST_LOG=${RUST_LOG:-info} + volumes: + - /var/cache/bridgehead/${PROJECT}/transfair:/transfair + + transfair-input-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-input-blaze + environment: + BASE_URL: "http://bridgehead-transfair-input-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx1024m" + DB_BLOCK_CACHE_SIZE: 1024 + CQL_EXPR_CACHE_SIZE: 8 + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-input-blaze-data:/app/data" + profiles: ["transfair-input-blaze"] + + transfair-request-blaze: + image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + container_name: bridgehead-transfair-requests-blaze + environment: + BASE_URL: "http://bridgehead-transfair-requests-blaze:8080" + JAVA_TOOL_OPTIONS: "-Xmx1024m" + DB_BLOCK_CACHE_SIZE: 1024 + CQL_EXPR_CACHE_SIZE: 8 + ENFORCE_REFERENTIAL_INTEGRITY: "false" + volumes: + - "transfair-request-blaze-data:/app/data" + profiles: ["transfair-request-blaze"] + +volumes: + transfair-input-blaze-data: + transfair-request-blaze-data: diff --git a/modules/transfair-setup.sh b/modules/transfair-setup.sh new file mode 100755 index 0000000..58f7331 --- /dev/null +++ b/modules/transfair-setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash -e + +function transfairSetup() { + if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then + echo "Starting transfair." + OVERRIDE+=" -f ./modules/transfair-compose.yml" + if [ -n "$FHIR_INPUT_URL" ]; then + log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL" + else + log INFO "TransFAIR input fhir store not set writing to internal blaze" + FHIR_INPUT_URL="http://transfair-input-blaze:8080" + OVERRIDE+=" --profile transfair-input-blaze" + fi + if [ -n "$FHIR_REQUEST_URL" ]; then + log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL" + else + log INFO "TransFAIR request fhir store not set writing to internal blaze" + FHIR_REQUEST_URL="http://transfair-requests-blaze:8080" + OVERRIDE+=" --profile transfair-request-blaze" + fi + fi +} diff --git a/versions/prod b/versions/prod new file mode 100644 index 0000000..1dd754f --- /dev/null +++ b/versions/prod @@ -0,0 +1,2 @@ +FOCUS_TAG=main +BEAM_TAG=main \ No newline at end of file diff --git a/versions/test b/versions/test new file mode 100644 index 0000000..10ae062 --- /dev/null +++ b/versions/test @@ -0,0 +1,2 @@ +FOCUS_TAG=develop +BEAM_TAG=develop \ No newline at end of file