refactor: explicitly set logging - to squash

Feat/ccp ppi

.github/scripts/rename_inactive_branches.py

@@ -0,0 +1,39 @@
+import os
+import requests
+from datetime import datetime, timedelta
+
+# Configuration
+GITHUB_TOKEN = os.getenv('GITHUB_TOKEN')
+REPO = 'samply/bridgehead'
+HEADERS = {'Authorization': f'token {GITHUB_TOKEN}', 'Accept': 'application/vnd.github.v3+json'}
+API_URL = f'https://api.github.com/repos/{REPO}/branches'
+INACTIVE_DAYS = 365
+CUTOFF_DATE = datetime.now() - timedelta(days=INACTIVE_DAYS)
+
+# Fetch all branches
+def get_branches():
+    response = requests.get(API_URL, headers=HEADERS)
+    response.raise_for_status()
+    return response.json() if response.status_code == 200 else []
+
+# Rename inactive branches
+def rename_branch(old_name, new_name):
+    rename_url = f'https://api.github.com/repos/{REPO}/branches/{old_name}/rename'
+    response = requests.post(rename_url, json={'new_name': new_name}, headers=HEADERS)
+    response.raise_for_status()
+    print(f"Renamed branch {old_name} to {new_name}" if response.status_code == 201 else f"Failed to rename {old_name}: {response.status_code}")
+
+# Check if the branch is inactive
+def is_inactive(commit_url):
+    last_commit_date = requests.get(commit_url, headers=HEADERS).json()['commit']['committer']['date']
+    return datetime.strptime(last_commit_date, '%Y-%m-%dT%H:%M:%SZ') < CUTOFF_DATE
+
+# Rename inactive branches
+def main():
+    for branch in get_branches():
+        if is_inactive(branch['commit']['url']):
+            #rename_branch(branch['name'], f"archived/{branch['name']}")
+            print(f"[LOG] Branch '{branch['name']}' is inactive and would be renamed to 'archived/{branch['name']}'")
+
+if __name__ == "__main__":
+    main()

.github/workflows/rename-inactive-branches.yml

@@ -0,0 +1,27 @@
+name: Cleanup - Rename Inactive Branches
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'  # Runs every Sunday at midnight
+
+jobs:
+  archive-stale-branches:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install Libraries
+        run: pip install requests
+
+      - name: Run Script to Rename Inactive Branches
+        run: |
+          python .github/scripts/rename_inactive_branches.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

bbmri/docker-compose.yml

@@ -10,7 +10,8 @@ services:
       BASE_URL: "http://bridgehead-bbmri-blaze:8080"
       JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
       DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
-      DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
+      DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP}
+      CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32}
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
     volumes:
       - "blaze-data:/app/data"

bbmri/modules/eric-compose.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   focus-eric:
-    image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG}-bbmri
     container_name: bridgehead-focus-eric
     environment:
       API_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT}

bbmri/modules/gbn-compose.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   focus-gbn:
-    image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG}-bbmri
     container_name: bridgehead-focus-gbn
     environment:
       API_KEY: ${GBN_FOCUS_BEAM_SECRET_SHORT}

ccp/docker-compose.yml

@@ -8,8 +8,10 @@ services:
       BASE_URL: "http://bridgehead-ccp-blaze:8080"
       JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
       DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
-      DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
+      DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP}
+      CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32}
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
+      LOG_LEVEL: ${LOG_LEVEL_BLAZE:-WARN}
     volumes:
       - "blaze-data:/app/data"
     labels:
@@ -21,7 +23,7 @@ services:
       - "traefik.http.routers.blaze_ccp.tls=true"
 
   focus:
-    image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/focus:${FOCUS_TAG}-dktk
     container_name: bridgehead-focus
     environment:
       API_KEY: ${FOCUS_BEAM_SECRET_SHORT}
@@ -32,6 +34,8 @@ services:
       RETRY_COUNT: ${FOCUS_RETRY_COUNT}
       EPSILON: 0.28
       QUERIES_TO_CACHE: '/queries_to_cache.conf'
+      ENDPOINT_TYPE: ${FOCUS_ENDPOINT_TYPE:-blaze}
+      RUST_LOG: ${LOG_LEVEL_FOCUS:-WARN}
     volumes:
       - /srv/docker/bridgehead/ccp/queries_to_cache.conf:/queries_to_cache.conf
     depends_on:
@@ -49,6 +53,7 @@ services:
       ALL_PROXY: http://forward_proxy:3128
       TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
       ROOTCERT_FILE: /conf/root.crt.pem
+      RUST_LOG: ${LOG_LEVEL_FOCUS:-WARN}
     secrets:
       - proxy.pem
     depends_on:
@@ -57,7 +62,6 @@ services:
       - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
       - /srv/docker/bridgehead/ccp/root.crt.pem:/conf/root.crt.pem:ro
 
-
 volumes:
   blaze-data:
 

ccp/modules/blaze-secondary-compose.yml

@@ -10,6 +10,7 @@ services:
       DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
       DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
+      LOG_LEVEL: ${LOG_LEVEL_BLAZE:-WARN}
     volumes:
       - "blaze-secondary-data:/app/data"
     labels:

ccp/modules/datashield-compose.yml

@@ -10,6 +10,7 @@ services:
       DISABLE_AUTH: "true" # https://rocker-project.org/images/versioned/rstudio.html#how-to-use
       HTTP_RELATIVE_PATH: "/rstudio"
       ALL_PROXY: "http://forward_proxy:3128" # https://rocker-project.org/use/networking.html
+      LOG_LEVEL: ${LOG_LEVEL_RSTUDIO:-WARN}
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.rstudio_ccp.rule=PathPrefix(`/rstudio`)"
@@ -54,6 +55,7 @@ services:
       BEAM_APP_ID: token-manager.${PROXY_ID}
       BEAM_SECRET: ${TOKEN_MANAGER_SECRET}
       BEAM_DATASHIELD_PROXY: request-manager
+      LOG_LEVEL: ${LOG_LEVEL_OPAL:-WARN}
     volumes:
       - "/var/cache/bridgehead/ccp/opal-metadata-db:/srv" # Opal metadata
     secrets:
@@ -75,6 +77,8 @@ services:
     image: docker.verbis.dkfz.de/ccp/dktk-rserver # datashield/rock-base + dsCCPhos
     tmpfs:
       - /srv
+    environment:
+      LOG_LEVEL: ${LOG_LEVEL_OPAL:-WARN}
 
   beam-connect:
     image: docker.verbis.dkfz.de/cache/samply/beam-connect:develop
@@ -87,6 +91,7 @@ services:
       DISCOVERY_URL: "./map/central.json"
       LOCAL_TARGETS_FILE: "./map/local.json"
       NO_AUTH: "true"
+      RUST_LOG: ${LOG_LEVEL_BEAMCONNECT:-WARN}
     secrets:
       - opal-cert.pem
     depends_on:
@@ -121,38 +126,42 @@ services:
   oauth2-proxy:
     image: docker.verbis.dkfz.de/cache/oauth2-proxy/oauth2-proxy:latest
     container_name: bridgehead-oauth2proxy
-    environment:
-      - http_proxy=http://forward_proxy:3128
-      - https_proxy=http://forward_proxy:3128
-      - OAUTH2_PROXY_ALLOWED_GROUPS=DataSHIELD
-      - OAUTH2_PROXY_OIDC_GROUPS_CLAIM=${OIDC_GROUP_CLAIM}
-      - OAUTH2_PROXY_WHITELIST_DOMAIN=${HOST}
-      - OAUTH2_PROXY_HTTP_ADDRESS=:4180
-      - OAUTH2_PROXY_REVERSE_PROXY=true
-      - OAUTH2_PROXY_UPSTREAMS=static://202
-      - OAUTH2_PROXY_EMAIL_DOMAINS=*
-      - OAUTH2_PROXY_COOKIE_NAME=_BRIDGEHEAD_oauth2
-      - OAUTH2_PROXY_COOKIE_SECRET=${OAUTH2_PROXY_SECRET}
-      - OAUTH2_PROXY_COOKIE_EXPIRE=12h
+    command: >-
+      --allowed-group=DataSHIELD
+      --oidc-groups-claim=${OIDC_GROUP_CLAIM}
+      --auth-logging=true
+      --whitelist-domain=${HOST}
+      --http-address="0.0.0.0:4180"
+      --reverse-proxy=true
+      --upstream="static://202"
+      --email-domain="*"
+      --cookie-name="_BRIDGEHEAD_oauth2"
+      --cookie-secret="${OAUTH2_PROXY_SECRET}"
+      --cookie-expire="12h"
+      --cookie-secure="true"
+      --cookie-httponly="true"
       #OIDC settings
-      - OAUTH2_PROXY_PROVIDER=keycloak-oidc
-      - OAUTH2_PROXY_PROVIDER_DISPLAY_NAME="VerbIS Login"
-      - OAUTH2_PROXY_CLIENT_ID=${OIDC_PRIVATE_CLIENT_ID}
-      - OAUTH2_PROXY_CLIENT_SECRET=${OIDC_CLIENT_SECRET}
-      - OAUTH2_PROXY_REDIRECT_URL="https://${HOST}${OAUTH2_CALLBACK}"
-      - OAUTH2_PROXY_OIDC_ISSUER_URL=${OIDC_ISSUER_URL}
-      - OAUTH2_PROXY_SCOPE=openid profile email
-      - OAUTH2_PROXY_CODE_CHALLENGE_METHOD=true
-      - OAUTH2_PROXY_SKIP_PROVIDER_BUTTON=true
+      --provider="keycloak-oidc"
+      --provider-display-name="VerbIS Login"
+      --client-id="${OIDC_PRIVATE_CLIENT_ID}"
+      --client-secret="${OIDC_CLIENT_SECRET}"
+      --redirect-url="https://${HOST}${OAUTH2_CALLBACK}"
+      --oidc-issuer-url="${OIDC_ISSUER_URL}"
+      --scope="openid email profile"
+      --code-challenge-method="S256"
+      --skip-provider-button=true
       #X-Forwarded-Header settings - true/false depending on your needs
-      - OAUTH2_PROXY_PASS_BASIC_AUTH=true
-      - OAUTH2_PROXY_PASS_USER_HEADERS=false
-      - OAUTH2_PROXY_ACCESS_TOKEN=false
+      --pass-basic-auth=true
+      --pass-user-headers=false
+      --pass-access-token=false
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.oauth2_proxy.rule=PathPrefix(`/oauth2`)"
       - "traefik.http.services.oauth2_proxy.loadbalancer.server.port=4180"
       - "traefik.http.routers.oauth2_proxy.tls=true"
+    environment:
+      http_proxy: "http://forward_proxy:3128"
+      https_proxy: "http://forward_proxy:3128"
     depends_on:
       forward_proxy:
         condition: service_healthy

ccp/modules/dnpm-compose.yml

@@ -17,7 +17,7 @@ services:
       HTTP_PROXY: "http://forward_proxy:3128"
       HTTPS_PROXY: "http://forward_proxy:3128"
       NO_PROXY: beam-proxy,dnpm-backend,host.docker.internal${DNPM_ADDITIONAL_NO_PROXY}
-      RUST_LOG: ${RUST_LOG:-info}
+      RUST_LOG: ${LOG_LEVEL_BEAMCONNECTDNPM:-WARN}
       NO_AUTH: "true"
       TLS_CA_CERTIFICATES_DIR: ./conf/trusted-ca-certs
     extra_hosts:

ccp/modules/exporter-compose.yml

@@ -6,7 +6,6 @@ services:
     container_name: bridgehead-ccp-exporter
     environment:
       JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC"
-      LOG_LEVEL: "INFO"
       EXPORTER_API_KEY: "${EXPORTER_API_KEY}" # Set in exporter-setup.sh
       CROSS_ORIGINS: "https://${HOST}"
       EXPORTER_DB_USER: "exporter"
@@ -16,6 +15,7 @@ services:
       SITE: "${SITE_ID}"
       HTTP_SERVLET_REQUEST_SCHEME: "https"
       OPAL_PASSWORD: "${EXPORTER_OPAL_PASSWORD}"
+      LOG_LEVEL: ${LOG_LEVEL_EXPORTER:-WARN}
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)"
@@ -42,7 +42,6 @@ services:
     container_name: bridgehead-ccp-reporter
     environment:
       JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC"
-      LOG_LEVEL: "INFO"
       CROSS_ORIGINS: "https://${HOST}"
       HTTP_RELATIVE_PATH: "/ccp-reporter"
       SITE: "${SITE_ID}"
@@ -50,6 +49,7 @@ services:
       EXPORTER_URL: "http://exporter:8092"
       LOG_FHIR_VALIDATION: "false"
       HTTP_SERVLET_REQUEST_SCHEME: "https"
+      LOG_LEVEL: ${LOG_LEVEL_REPORTER:-WARN}
 
     # In this initial development state of the bridgehead, we are trying to have so many volumes as possible.
     # However, in the first executions in the CCP sites, this volume seems to be very important. A report is
@@ -65,3 +65,8 @@ services:
       - "traefik.http.routers.reporter_ccp.tls=true"
       - "traefik.http.middlewares.reporter_ccp_strip.stripprefix.prefixes=/ccp-reporter"
       - "traefik.http.routers.reporter_ccp.middlewares=reporter_ccp_strip"
+
+  focus:
+    environment:
+      EXPORTER_URL: "http://exporter:8092"
+      AUTH_HEADER: "${EXPORTER_API_KEY}"

ccp/modules/fhir2sql-compose.yml

@@ -23,3 +23,7 @@ services:
       POSTGRES_DB: "dashboard"
     volumes:
       - "/var/cache/bridgehead/ccp/dashboard-db:/var/lib/postgresql/data"
+
+  focus:
+    environment:
+      POSTGRES_CONNECTION_STRING: "postgresql://dashboard:${DASHBOARD_DB_PASSWORD}@dashboard-db/dashboard"

ccp/modules/fhir2sql-setup.sh

@@ -4,4 +4,5 @@ if [ "$ENABLE_FHIR2SQL" == true ]; then
   log INFO "Dashboard setup detected -- will start Dashboard backend and FHIR2SQL service."
   OVERRIDE+=" -f ./$PROJECT/modules/fhir2sql-compose.yml"
   DASHBOARD_DB_PASSWORD="$(generate_simple_password 'fhir2sql')"
+  FOCUS_ENDPOINT_TYPE="blaze-and-sql"
 fi

ccp/modules/id-management-compose.yml

@@ -14,6 +14,7 @@ services:
       MAGICPL_CONNECTOR_APIKEY: ${IDMANAGER_READ_APIKEY}
       MAGICPL_CENTRAL_PATIENTLIST_APIKEY: ${IDMANAGER_CENTRAL_PATIENTLIST_APIKEY}
       MAGICPL_CONTROLNUMBERGENERATOR_APIKEY: ${IDMANAGER_CONTROLNUMBERGENERATOR_APIKEY}
+      ML_LOG_LEVEL: ${LOG_LEVEL_IDMANAGER:-WARN}
     depends_on:
       - patientlist
       - traefik-forward-auth
@@ -44,6 +45,8 @@ services:
       - ML_UPLOAD_API_KEY=${IDMANAGER_UPLOAD_APIKEY}
       # Add Variables from /etc/patientlist-id-generators.env
       - PATIENTLIST_SEEDS_TRANSFORMED
+      - ML_LOG_LEVEL=${LOG_LEVEL_PATIENTLIST:-WARN}
+      #TODO confirm LOG_LEVEL
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)"
@@ -101,5 +104,12 @@ services:
       forward_proxy:
         condition: service_healthy
 
+  ccp-patient-project-identificator:
+    image: samply/ccp-patient-project-identificator
+    container_name: bridgehead-ccp-patient-project-identificator
+    environment:
+      MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}
+      SITE_NAME: ${SITE_NAME}
+
 volumes:
   patientlist-db-data:

ccp/modules/mtba-compose.yml

@@ -25,6 +25,7 @@ services:
       OIDC_CLIENT_SECRET: "${OIDC_CLIENT_SECRET}"
       OIDC_REALM: "${OIDC_REALM}"
       OIDC_URL: "${OIDC_URL}"
+      LOG_LEVEL: ${LOG_LEVEL_MTBA:-WARN}
 
     labels:
       - "traefik.enable=true"

ccp/modules/nngm-compose.yml

@@ -12,6 +12,7 @@ services:
       CTS_API_KEY: ${NNGM_CTS_APIKEY}
       CRYPT_KEY: ${NNGM_CRYPTKEY}
       #CTS_MAGICPL_SITE: ${SITE_ID}TODO
+      LOG_LEVEL: ${LOG_LEVEL_NNGM:-WARN}
     restart: always
     labels:
       - "traefik.enable=true"

ccp/modules/obds2fhir-rest-compose.yml

@@ -10,6 +10,7 @@ services:
       SALT: ${LOCAL_SALT}
       KEEP_INTERNAL_ID: ${KEEP_INTERNAL_ID:-false}
       MAINZELLISTE_URL: ${PATIENTLIST_URL:-http://patientlist:8080/patientlist}
+      LOG_LEVEL: ${LOG_LEVEL_REPORTER:-WARN}
     restart: always
     labels:
       - "traefik.enable=true"

ccp/modules/teiler-compose.yml

@@ -17,6 +17,7 @@ services:
       TEILER_DASHBOARD_URL: "https://${HOST}/ccp-teiler-dashboard"
       DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE_LOWER_CASE}"
       HTTP_RELATIVE_PATH: "/ccp-teiler"
+      LOG_LEVEL: ${LOG_LEVEL_TEILER:-WARN}
 
   teiler-dashboard:
     image: docker.verbis.dkfz.de/cache/samply/teiler-dashboard:develop
@@ -47,6 +48,7 @@ services:
       TEILER_ADMIN: "${OIDC_ADMIN_GROUP}"
       REPORTER_DEFAULT_TEMPLATE_ID: "ccp-qb"
       EXPORTER_DEFAULT_TEMPLATE_ID: "ccp"
+      LOG_LEVEL: ${LOG_LEVEL_TEILER:-WARN}
 
 
   teiler-backend:
@@ -60,7 +62,6 @@ services:
       - "traefik.http.middlewares.teiler_backend_ccp_strip.stripprefix.prefixes=/ccp-teiler-backend"
       - "traefik.http.routers.teiler_backend_ccp.middlewares=teiler_backend_ccp_strip"
     environment:
-      LOG_LEVEL: "INFO"
       APPLICATION_PORT: "8085"
       APPLICATION_ADDRESS: "${HOST}"
       DEFAULT_LANGUAGE: "${TEILER_DEFAULT_LANGUAGE}"
@@ -73,6 +74,7 @@ services:
       HTTP_PROXY: "http://forward_proxy:3128"
       ENABLE_MTBA: "${ENABLE_MTBA}"
       ENABLE_DATASHIELD: "${ENABLE_DATASHIELD}"
+      LOG_LEVEL: ${LOG_LEVEL_TEILER:-WARN}
     secrets:
       - ccp.conf
 

kr/docker-compose.yml

@@ -1,6 +1,10 @@
 version: "3.7"
 
 services:
+  landing:
+    deploy:
+      replicas: 0 #deactivate landing page
+
   blaze:
     image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
     container_name: bridgehead-kr-blaze

kr/modules/lens-compose.yml

@@ -1,6 +1,8 @@
 version: "3.7"
 services:
   landing:
+    deploy:
+      replicas: 1 #reactivate if lens is in use
     container_name: lens_federated-search
     image: docker.verbis.dkfz.de/ccp/lens:${SITE_ID}
     labels:

lib/functions.sh

@@ -171,8 +171,10 @@ optimizeBlazeMemoryUsage() {
 		if [ $available_system_memory_chunks -eq 0 ]; then
 			log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower."
 			export BLAZE_RESOURCE_CACHE_CAP=128000;
+			export BLAZE_CQL_CACHE_CAP=32;
 		else
 			export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chunks * 312500))
+			export BLAZE_CQL_CACHE_CAP=$((($system_memory_in_mb/4)/16));
 		fi
 	fi
 }