Feature/ml itcc (#380)

* sites moved to etc itcc.comf

* mainzelliste test

* volume

* fix image

* db name chaged

* ingest component test

* clean up

* maizelliste env db

* test server ip

* beam sockets

* teswt socket and task

* secure db and ml

* refactor and patient endpoints

* partner id

* new refactor and encription key generation

* db deleted

* keyset to var/

itcc/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/blaze:latest
     container_name: bridgehead-itcc-blaze
     environment:
       BASE_URL: "http://bridgehead-itcc-blaze:8080"
@@ -32,7 +32,7 @@ services:
       BEAM_PROXY_URL: http://beam-proxy:8081
       RETRY_COUNT: ${FOCUS_RETRY_COUNT}
       EPSILON: 0.28
-      QUERIES_TO_CACHE: '/queries_to_cache.conf'
+      QUERIES_TO_CACHE: "/queries_to_cache.conf"
       ENDPOINT_TYPE: ${FOCUS_ENDPOINT_TYPE:-blaze}
     volumes:
       - /srv/docker/bridgehead/itcc/queries_to_cache.conf:/queries_to_cache.conf:ro
@@ -41,12 +41,13 @@ services:
       - "blaze"
 
   beam-proxy:
-    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG}
+    image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop-sockets
     container_name: bridgehead-beam-proxy
     environment:
       BROKER_URL: ${BROKER_URL}
       PROXY_ID: ${PROXY_ID}
       APP_focus_KEY: ${FOCUS_BEAM_SECRET_SHORT}
+      APP_omics-endpoint_KEY: ${FOCUS_BEAM_SECRET_SHORT}
       PRIVKEY_FILE: /run/secrets/proxy.pem
       ALL_PROXY: http://forward_proxy:3128
       TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs
@@ -59,7 +60,6 @@ services:
       - /etc/bridgehead/trusted-ca-certs:/conf/trusted-ca-certs:ro
       - /srv/docker/bridgehead/itcc/root.crt.pem:/conf/root.crt.pem:ro
 
-
 volumes:
   blaze-data:
 

itcc/modules/itcc-omics-ingest.sh

@@ -3,4 +3,25 @@
 if [ -n "$ENABLE_OMICS" ];then
   OVERRIDE+=" -f ./$PROJECT/modules/itcc-omics-ingest.yaml"
   GENERATE_API_KEY="$(generate_simple_password 'omics')"
+  PATIENTLIST_POSTGRES_PASSWORD=="$(generate_simple_password 'mainzelliste')"
+  KEYSET=/var/bridgehead/mainzelliste/keyset_siv.json
+  if [ ! -f "$KEYSET" ]; then
+    mkdir -p "$(dirname "$KEYSET")"
+    KEY_ID=$(($(openssl rand -hex 4 | sed 's/^/0x/') & 0x7FFFFFFF))
+    VALUE=$({ printf '\x12\x40'; openssl rand 64; } | base64 | tr -d '\n')
+    jq -n --argjson id "$KEY_ID" --arg value "$VALUE" '{
+      primaryKeyId: $id,
+      key: [{
+        keyData: {
+          typeUrl: "type.googleapis.com/google.crypto.tink.AesSivKey",
+          value: $value,
+          keyMaterialType: "SYMMETRIC"
+        },
+        status: "ENABLED",
+        keyId: $id,
+        outputPrefixType: "TINK"
+      }]
+    }' > "$KEYSET"
+    chmod 600 "$KEYSET"
+  fi
 fi

itcc/modules/itcc-omics-ingest.yaml

@@ -1,14 +1,69 @@
 services:
   omics-endpoint:
-    image: ghcr.io/samply/itcc-omics-ingest:main
+    image: samply/itcc-omics-ingest:main
     environment:
-      - API_KEY=${GENERATE_API_KEY}
-    volumes:
-      - /var/cache/bridgehead/omics/data:/data/uploads
+      API_KEY: ${GENERATE_API_KEY}
+      BEAM_APP_ID_LONG: omics-endpoint.${PROXY_ID}
+      BEAM_SECRET: ${FOCUS_BEAM_SECRET_SHORT}
+      DWH_SOCKET_ID: ${DWH_SOCKET_ID}
+      DWH_TASK_ID: ${DWH_TASK_ID}
+      PARTNER_ID: ${SITE_ID}
+      ML_API_KEY: ${GENERATE_API_KEY}
     labels:
-      - "traefik.http.routers.omics.rule=Host(`${HOST}`) && PathPrefix(`/api/omics`)"
+      - "traefik.http.routers.omics.rule=Host(`${HOST}`) &&
+        PathPrefix(`/api/upload`)"
       - "traefik.enable=true"
       - "traefik.http.services.omics.loadbalancer.server.port=6080"
       - "traefik.http.routers.omics.tls=true"
       - "traefik.http.middlewares.omics-stripprefix.stripprefix.prefixes=/api"
       - "traefik.http.routers.omics.middlewares=omics-stripprefix"
+
+  patientlist-db:
+    image: postgres:${POSTGRES_TAG}
+    container_name: bridgehead-patientlist-db
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: mainzelliste
+      POSTGRES_USER: ${ML_DB_USER}
+      POSTGRES_PASSWORD: ${PATIENTLIST_POSTGRES_PASSWORD}
+    volumes:
+      - "patientlist-db-data:/var/lib/postgresql/data"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${ML_DB_USER} -d mainzelliste"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  patientlist:
+    image: medicalinformatics/mainzelliste:latest
+    container_name: bridgehead-patientlist
+    restart: unless-stopped
+    depends_on:
+      patientlist-db:
+        condition: service_healthy
+    environment:
+      ML_API_KEY: ${GENERATE_API_KEY}
+      ML_DB_HOST: patientlist-db
+      ML_DB_PORT: "5432"
+      ML_DB_NAME: mainzelliste
+      ML_DB_USER: ${ML_DB_USER}
+      ML_DB_PASS: ${PATIENTLIST_POSTGRES_PASSWORD}
+      ML_DB_DRIVER: org.postgresql.Driver
+      ML_DB_TYPE: postgresql
+      ML_LOG_LEVEL: INFO
+      ML_ALLOWEDREMOTEADDRESSES: "127.0.0.1,::1,172.16.0.0/12"
+
+    secrets:
+      - mainzelliste.docker.conf
+      - source: symmetric_key
+        target: /etc/resources/keys/symmetric_key.json
+
+volumes:
+  patientlist-db-data:
+secrets:
+  mainzelliste.docker.conf:
+    file: /etc/bridgehead/mainzelliste/mainzelliste.docker.conf
+
+  symmetric_key:
+    file: /var/bridgehead/mainzelliste/keyset_siv.json

itcc/vars

@@ -7,6 +7,9 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de
 PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem
 BROKER_URL_FOR_PREREQ=$BROKER_URL
 PUBLIC_ENVIRONMENT=prod
+DWH_SOCKET_ID=socket.itcc-datalake.${BROKER_ID}
+DWH_TASK_ID=task.itcc-datalake.${BROKER_ID}
+ML_DB_USER=mainzelliste
 
 for module in $PROJECT/modules/*.sh
 do