feat: remove local rstudio (#322)

* Added Exporter to CCE

* Add Teiler to CCE

* Add EXPORTER_USER to adduser function

bbmri/modules/exporter.md

@@ -1,15 +0,0 @@
-# Exporter and Reporter
-
-
-## Exporter
-The exporter is a REST API that exports the data of the different databases of the bridgehead in a set of tables.
-It can accept different output formats as CSV, Excel, JSON or XML. It can also export data into Opal.
-
-## Exporter-DB
-It is a database to save queries for its execution in the exporter. 
-The exporter manages also the different executions of the same query in through the database.
-
-## Reporter
-This component is a plugin of the exporter that allows to create more complex Excel reports described in templates.
-It is compatible with different template engines as Groovy, Thymeleaf,...
-It is perfect to generate a document as our traditional CCP quality report.

bbmri/modules/teiler.md

@@ -1,19 +0,0 @@
-# Teiler
-This module orchestrates the different microfrontends of the bridgehead as a single page application.  
-
-## Teiler Orchestrator
-Single SPA component that consists on the root HTML site of the single page application and a javascript code that
-gets the information about the microfrontend calling the teiler backend and is responsible for registering them. With the
-resulting mapping, it can initialize, mount and unmount the required microfrontends on the fly. 
-
-The microfrontends run independently in different containers and can be based on different frameworks (Angular, Vue, React,...)
-This microfrontends can run as single alone but need an extension with Single-SPA (https://single-spa.js.org/docs/ecosystem).
-There are also available three templates (Angular, Vue, React) to be directly extended to be used directly in the teiler.
-
-## Teiler Dashboard
-It consists on the main dashboard and a set of embedded services.
-### Login
-user and password in ccp.local.conf
-
-## Teiler Backend
-In this component, the microfrontends are configured.

bridgehead

@@ -162,7 +162,7 @@ case "$ACTION" in
 	adduser)
 		loadVars
 		log "INFO" "Adding encrypted credentials in /etc/bridgehead/$PROJECT.local.conf"
-    		read -p "Please choose the component (LDM_AUTH|NNGM_AUTH) you want to add a user to : " COMPONENT
+    		read -p "Please choose the component (LDM_AUTH|NNGM_AUTH|EXPORTER_USER) you want to add a user to : " COMPONENT
     		read -p "Please enter a username: " USER
     		read -s -p "Please enter a password (will not be echoed): "$'\n' PASSWORD
     		add_basic_auth_user $USER $PASSWORD $COMPONENT $PROJECT

cce/modules/exporter-compose.yml

@@ -0,0 +1,87 @@
+version: "3.7"
+
+services:
+
+  exporter:
+    image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest
+    container_name: bridgehead-cce-exporter
+    environment:
+      JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC"
+      LOG_LEVEL: "INFO"
+      EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh
+      CROSS_ORIGINS: "https://${HOST}"
+      EXPORTER_DB_USER: "exporter"
+      EXPORTER_DB_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh
+      EXPORTER_DB_URL: "jdbc:postgresql://exporter-db:5432/exporter"
+      HTTP_RELATIVE_PATH: "/cce-exporter"
+      SITE: "${SITE_ID}"
+      HTTP_SERVLET_REQUEST_SCHEME: "https"
+      OPAL_PASSWORD: "${EXPORTER_OPAL_PASSWORD}"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.exporter_cce.rule=PathPrefix(`/cce-exporter`)"
+      - "traefik.http.services.exporter_cce.loadbalancer.server.port=8092"
+      - "traefik.http.routers.exporter_cce.tls=true"
+      - "traefik.http.middlewares.exporter_cce_strip.stripprefix.prefixes=/cce-exporter"
+      - "traefik.http.routers.exporter_cce.middlewares=exporter_cce_strip"
+      # Main router
+      - "traefik.http.routers.exporter_cce.priority=20"
+
+      # API router
+      - "traefik.http.routers.exporter_cce_api.middlewares=exporter_cce_strip,exporter_auth"
+      - "traefik.http.routers.exporter_cce_api.rule=PathRegexp(`/cce-exporter/.+`)"
+      - "traefik.http.routers.exporter_cce_api.tls=true"
+      - "traefik.http.routers.exporter_cce_api.priority=25"
+
+      # Shared middlewares
+      - "traefik.http.middlewares.exporter_auth.basicauth.users=${EXPORTER_USER}"
+
+    volumes:
+      - "/var/cache/bridgehead/cce/exporter-files:/app/exporter-files/output"
+
+  exporter-db:
+    image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG}
+    container_name: bridgehead-cce-exporter-db
+    environment:
+      POSTGRES_USER: "exporter"
+      POSTGRES_PASSWORD: "${EXPORTER_DB_PASSWORD}" # Set in exporter-setup.sh
+      POSTGRES_DB: "exporter"
+    volumes:
+      # Consider removing this volume once we find a solution to save Lens-queries to be executed in the explorer.
+      - "/var/cache/bridgehead/cce/exporter-db:/var/lib/postgresql/data"
+
+  reporter:
+    image: docker.verbis.dkfz.de/ccp/dktk-reporter:latest
+    container_name: bridgehead-cce-reporter
+    environment:
+      JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC"
+      LOG_LEVEL: "INFO"
+      CROSS_ORIGINS: "https://${HOST}"
+      HTTP_RELATIVE_PATH: "/cce-reporter"
+      SITE: "${SITE_ID}"
+      EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh
+      EXPORTER_URL: "http://exporter:8092"
+      LOG_FHIR_VALIDATION: "false"
+      HTTP_SERVLET_REQUEST_SCHEME: "https"
+
+    # In this initial development state of the bridgehead, we are trying to have so many volumes as possible.
+    # However, in the first executions in the cce sites, this volume seems to be very important. A report is
+    # a process that can take several hours, because it depends on the exporter.
+    # There is a risk that the bridgehead restarts, losing the already created export.
+
+    volumes:
+      - "/var/cache/bridgehead/cce/reporter-files:/app/reports"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.reporter_cce.rule=PathPrefix(`/cce-reporter`)"
+      - "traefik.http.services.reporter_cce.loadbalancer.server.port=8095"
+      - "traefik.http.routers.reporter_cce.tls=true"
+      - "traefik.http.middlewares.reporter_cce_strip.stripprefix.prefixes=/cce-reporter"
+      - "traefik.http.routers.reporter_cce.middlewares=reporter_cce_strip"
+      - "traefik.http.routers.reporter_cce.priority=20"
+
+      - "traefik.http.routers.reporter_cce_api.middlewares=reporter_cce_strip,exporter_auth"
+      - "traefik.http.routers.reporter_cce_api.rule=PathRegexp(`/cce-reporter/.+`)"
+      - "traefik.http.routers.reporter_cce_api.tls=true"
+      - "traefik.http.routers.reporter_cce_api.priority=25"
+

cce/modules/exporter-setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+if [ "$ENABLE_EXPORTER" == true ]; then
+  log INFO "Exporter setup detected -- will start Exporter service."
+  OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml"
+  EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+  EXPORTER_API_KEY="$(echo \"This is a salt string to generate one consistent API KEY for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 64)"
+fi

cce/modules/lens-compose.yml

@@ -17,7 +17,6 @@ services:
       BEAM_PROXY_ID: ${SITE_ID}
       BEAM_BROKER_ID: ${BROKER_ID}
       BEAM_APP_ID: "focus"
-      PROJECT_METADATA: "cce_supervisors"
     depends_on:
       - "beam-proxy"
     labels:
@@ -30,4 +29,4 @@ services:
       - "traefik.http.routers.spot.rule=Host(`${HOST}`) && PathPrefix(`/backend`)"
       - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/backend"
       - "traefik.http.routers.spot.tls=true"
-      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot"
+      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth"

cce/modules/teiler-compose.yml

@@ -0,0 +1,69 @@
+version: "3.7"
+
+services:
+
+  teiler-orchestrator:
+    image: docker.verbis.dkfz.de/cache/samply/teiler-orchestrator:latest
+    container_name: bridgehead-teiler-orchestrator
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.teiler_orchestrator_cce.rule=PathPrefix(`/cce-teiler`)"
+      - "traefik.http.services.teiler_orchestrator_cce.loadbalancer.server.port=9000"
+      - "traefik.http.routers.teiler_orchestrator_cce.tls=true"
+      - "traefik.http.middlewares.teiler_orchestrator_cce_strip.stripprefix.prefixes=/cce-teiler"
+      - "traefik.http.routers.teiler_orchestrator_cce.middlewares=teiler_orchestrator_cce_strip"
+    environment:
+      TEILER_BACKEND_URL: "/cce-teiler-backend"
+      TEILER_DASHBOARD_URL: "/cce-teiler-dashboard"
+      DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE_LOWER_CASE}"
+      HTTP_RELATIVE_PATH: "/cce-teiler"
+
+  teiler-dashboard:
+    image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:develop
+    container_name: bridgehead-teiler-dashboard
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.teiler_dashboard_cce.rule=PathPrefix(`/cce-teiler-dashboard`)"
+      - "traefik.http.services.teiler_dashboard_cce.loadbalancer.server.port=80"
+      - "traefik.http.routers.teiler_dashboard_cce.tls=true"
+      - "traefik.http.middlewares.teiler_dashboard_cce_strip.stripprefix.prefixes=/cce-teiler-dashboard"
+      - "traefik.http.routers.teiler_dashboard_cce.middlewares=teiler_dashboard_cce_strip"
+    environment:
+      DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE}"
+      TEILER_BACKEND_URL: "/cce-teiler-backend"
+      TEILER_DASHBOARD_URL: "/cce-teiler-dashboard"
+      OIDC_URL: "${OIDC_URL}"
+      OIDC_CLIENT_ID: "${OIDC_PUBLIC_CLIENT_ID}"
+      OIDC_TOKEN_GROUP: "${OIDC_GROUP_CLAIM}"
+      TEILER_ADMIN_NAME: "${OPERATOR_FIRST_NAME} ${OPERATOR_LAST_NAME}"
+      TEILER_ADMIN_EMAIL: "${OPERATOR_EMAIL}"
+      TEILER_ADMIN_PHONE: "${OPERATOR_PHONE}"
+      TEILER_PROJECT: "${PROJECT}"
+      EXPORTER_API_KEY: "${EXPORTER_API_KEY}"
+      TEILER_ORCHESTRATOR_URL: "/cce-teiler"
+      TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/cce-teiler"
+      TEILER_USER: "${OIDC_USER_GROUP}"
+      TEILER_ADMIN: "${OIDC_ADMIN_GROUP}"
+      REPORTER_DEFAULT_TEMPLATE_ID: "cce-qb"
+      EXPORTER_DEFAULT_TEMPLATE_ID: "cce"
+
+
+  teiler-backend:
+    image: docker.verbis.dkfz.de/ccp/cce-teiler-backend:latest
+    container_name: bridgehead-teiler-backend
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.teiler_backend_cce.rule=PathPrefix(`/cce-teiler-backend`)"
+      - "traefik.http.services.teiler_backend_cce.loadbalancer.server.port=8085"
+      - "traefik.http.routers.teiler_backend_cce.tls=true"
+      - "traefik.http.middlewares.teiler_backend_cce_strip.stripprefix.prefixes=/cce-teiler-backend"
+      - "traefik.http.routers.teiler_backend_cce.middlewares=teiler_backend_cce_strip"
+    environment:
+      LOG_LEVEL: "INFO"
+      APPLICATION_PORT: "8085"
+      DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE}"
+      TEILER_ORCHESTRATOR_HTTP_RELATIVE_PATH: "/cce-teiler"
+      TEILER_ORCHESTRATOR_URL: "/cce-teiler"
+      TEILER_DASHBOARD_DE_URL: "/cce-teiler-dashboard/de"
+      TEILER_DASHBOARD_EN_URL: "/cce-teiler-dashboard/en"
+      HTTP_PROXY: "http://forward_proxy:3128"

cce/modules/teiler-setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+if [ "$ENABLE_TEILER" == true ];then
+  log INFO "Teiler setup detected -- will start Teiler services."
+  OVERRIDE+=" -f ./$PROJECT/modules/teiler-compose.yml"
+  TEILER_DEFAULT_LANGUAGE=EN
+  TEILER_DEFAULT_LANGUAGE_LOWER_CASE=${TEILER_DEFAULT_LANGUAGE,,}
+fi

ccp/modules/datashield-compose.yml

@@ -1,25 +1,6 @@
 version: "3.7"
 
 services:
-  rstudio:
-    container_name: bridgehead-rstudio
-    image: docker.verbis.dkfz.de/ccp/dktk-rstudio:latest
-    environment:
-      #DEFAULT_USER: "rstudio" # This line is kept for informational purposes
-      PASSWORD: "${RSTUDIO_ADMIN_PASSWORD}" # It is required, even if the authentication is disabled
-      DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use
-      HTTP_RELATIVE_PATH: "/rstudio"
-      ALL_PROXY: "http://forward_proxy:3128" # https://rocker-project.org/use/networking.html
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)"
-      - "traefik.http.services.rstudio_ccp.loadbalancer.server.port=8787"
-      - "traefik.http.middlewares.rstudio_ccp_strip.stripprefix.prefixes=/rstudio"
-      - "traefik.http.routers.rstudio_ccp.tls=true"
-      - "traefik.http.routers.rstudio_ccp.middlewares=oidcAuth,rstudio_ccp_strip"
-    networks:
-      - rstudio
-
   opal:
     container_name: bridgehead-opal
     image: docker.verbis.dkfz.de/ccp/dktk-opal:latest
@@ -92,79 +73,14 @@ services:
       - beam-proxy
     volumes:
       - /tmp/bridgehead/opal-map/:/map/:ro
-    networks:
-      - default
-      - rstudio
-
-  traefik:
-    labels:
-      - "traefik.http.middlewares.oidcAuth.forwardAuth.address=http://oauth2-proxy:4180/"
-      - "traefik.http.middlewares.oidcAuth.forwardAuth.trustForwardHeader=true"
-      - "traefik.http.middlewares.oidcAuth.forwardAuth.authResponseHeaders=X-Auth-Request-Access-Token,Authorization"
-    networks:
-      - default
-      - rstudio
-  forward_proxy:
-    networks:
-      - default
-      - rstudio
 
   beam-proxy:
     environment:
       APP_datashield-connect_KEY: ${DATASHIELD_CONNECT_SECRET}
       APP_token-manager_KEY: ${TOKEN_MANAGER_SECRET}
 
-  # TODO: Allow users of group /DataSHIELD and OIDC_USER_GROUP at the same time:
-  # Maybe a solution would be (https://oauth2-proxy.github.io/oauth2-proxy/configuration/oauth_provider):
-  # --allowed-groups=/DataSHIELD,OIDC_USER_GROUP
-  oauth2-proxy:
-    image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:latest
-    container_name: bridgehead-oauth2proxy
-    command: >-
-      --allowed-group=DataSHIELD
-      --oidc-groups-claim=${OIDC_GROUP_CLAIM}
-      --auth-logging=true
-      --whitelist-domain=${HOST}
-      --http-address="0.0.0.0:4180"
-      --reverse-proxy=true
-      --upstream="static://202"
-      --email-domain="*"
-      --cookie-name="_BRIDGEHEAD_oauth2"
-      --cookie-secret="${OAUTH2_PROXY_SECRET}"
-      --cookie-expire="12h"
-      --cookie-secure="true"
-      --cookie-httponly="true"
-      #OIDC settings
-      --provider="keycloak-oidc"
-      --provider-display-name="VerbIS Login"
-      --client-id="${OIDC_PRIVATE_CLIENT_ID}"
-      --client-secret="${OIDC_CLIENT_SECRET}"
-      --redirect-url="https://${HOST}${OAUTH2_CALLBACK}"
-      --oidc-issuer-url="${OIDC_URL}"
-      --scope="openid email profile"
-      --code-challenge-method="S256"
-      --skip-provider-button=true
-      #X-Forwarded-Header settings - true/false depending on your needs
-      --pass-basic-auth=true
-      --pass-user-headers=false
-      --pass-access-token=false
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.oauth2_proxy.rule=PathPrefix(`/oauth2`)"
-      - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180"
-      - "traefik.http.routers.oauth2_proxy.tls=true"
-    environment:
-      http_proxy: "http://forward_proxy:3128"
-      https_proxy: "http://forward_proxy:3128"
-    depends_on:
-      forward_proxy:
-        condition: service_healthy
-
 secrets:
   opal-cert.pem:
     file: /tmp/bridgehead/opal-cert.pem
   opal-key.pem:
     file: /tmp/bridgehead/opal-key.pem
-
-networks:
-  rstudio:

ccp/modules/datashield-setup.sh

@@ -5,17 +5,12 @@ if [ "$ENABLE_DATASHIELD" == true ]; then
   if [ -z "${ENABLE_EXPORTER}" ] || [ "${ENABLE_EXPORTER}" != "true" ]; then
     log WARN "The ENABLE_EXPORTER variable is either not set or not set to 'true'."
   fi
-  OAUTH2_CALLBACK=/oauth2/callback
-  OAUTH2_PROXY_SECRET="$(echo \"This is a salt string to generate one consistent encryption key for the oauth2_proxy. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 32)"
-  add_private_oidc_redirect_url "${OAUTH2_CALLBACK}"
-
   log INFO "DataSHIELD setup detected -- will start DataSHIELD services."
   OVERRIDE+=" -f ./$PROJECT/modules/datashield-compose.yml"
   EXPORTER_OPAL_PASSWORD="$(generate_password \"exporter in Opal\")"
   TOKEN_MANAGER_OPAL_PASSWORD="$(generate_password \"Token Manager in Opal\")"
   OPAL_DB_PASSWORD="$(echo \"Opal DB\" | generate_simple_password)"
   OPAL_ADMIN_PASSWORD="$(generate_password \"admin password for Opal\")"
-  RSTUDIO_ADMIN_PASSWORD="$(generate_password \"admin password for R-Studio\")"
   DATASHIELD_CONNECT_SECRET="$(echo \"DataShield Connect\" | generate_simple_password)"
   TOKEN_MANAGER_SECRET="$(echo \"Token Manager\" | generate_simple_password)"
   if [ ! -e /tmp/bridgehead/opal-cert.pem ]; then
@@ -23,18 +18,13 @@ if [ "$ENABLE_DATASHIELD" == true ]; then
     openssl req -x509 -newkey rsa:4096 -nodes -keyout /tmp/bridgehead/opal-key.pem -out /tmp/bridgehead/opal-cert.pem -days 3650 -subj "/CN=opal/C=DE"
   fi
   mkdir -p /tmp/bridgehead/opal-map
-  sites="$(cat ./$PROJECT/modules/datashield-sites.json)"
-  echo "$sites" | docker_jq -n --args '{"sites": input | map({
-    "name": .,
-    "id": .,
-    "virtualhost": "\(.):443",
-    "beamconnect": "datashield-connect.\(.).'"$BROKER_ID"'"
-  })}' $sites >/tmp/bridgehead/opal-map/central.json
-  echo "$sites" | docker_jq -n --args '[{
-    "external": "'"$SITE_ID"':443",
+  echo '{"sites": []}' >/tmp/bridgehead/opal-map/central.json
+  # Only allow connections from the central beam proxy that is used by all coder workspaces
+  echo '[{
+    "external": "'$SITE_ID':443",
     "internal": "opal:8443",
-    "allowed": input | map("\(.).'"$BROKER_ID"'")
-  }]' >/tmp/bridgehead/opal-map/local.json
+    "allowed": ["central-ds-orchestrator.'$BROKER_ID'"]
+  }]' > /tmp/bridgehead/opal-map/local.json
   if [ "$USER" == "root" ]; then
     chown -R bridgehead:docker /tmp/bridgehead
     chmod g+wr /tmp/bridgehead/opal-map/*

ccp/modules/datashield-sites.json

@@ -1,15 +0,0 @@
-[
-    "berlin",
-    "muenchen-lmu",
-    "dresden",
-    "freiburg",
-    "muenchen-tum",
-    "tuebingen",
-    "mainz",
-    "frankfurt",
-    "essen",
-    "dktk-datashield-test",
-    "dktk-test",
-    "mannheim",
-    "central-ds-orchestrator"
-]

itcc/modules/lens-compose.yml

@@ -17,7 +17,6 @@ services:
       BEAM_PROXY_ID: ${SITE_ID}
       BEAM_BROKER_ID: ${BROKER_ID}
       BEAM_APP_ID: "focus"
-      PROJECT_METADATA: "dktk_supervisors"
     depends_on:
       - "beam-proxy"
     labels:
@@ -30,4 +29,4 @@ services:
       - "traefik.http.routers.spot.rule=Host(`${HOST}`) && PathPrefix(`/backend`)"
       - "traefik.http.middlewares.stripprefix_spot.stripprefix.prefixes=/backend"
       - "traefik.http.routers.spot.tls=true"
-      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot"
+      - "traefik.http.routers.spot.middlewares=corsheaders2,stripprefix_spot,auth"

lib/functions.sh

@@ -436,7 +436,3 @@ generate_simple_password(){
   local combined_text="This is a salt string to generate one consistent password for ${seed_text}. It is not required to be secret."
   echo "${combined_text}" | sha1sum | openssl pkeyutl -sign -inkey "/etc/bridgehead/pki/${SITE_ID}.priv.pem" 2> /dev/null | base64 | head -c 26 | sed 's/[+\/]/A/g'
 }
-
-docker_jq() {
-    docker run --rm -i docker.verbis.dkfz.de/cache/jqlang/jq:latest "$@"
-}