Added more detail to metadata feedback README

Most of the things added during testing were not necessary and they were
removed. This had the additional advantage that many files are now identical
to their equivalents in the develop branch, making the diff more manageable.

README.md

@@ -1,4 +1,3 @@
-# Just Testing the Branch Rules
 # Bridgehead
 
 The Bridgehead is a secure, low-effort solution to connect your research institution to a federated research network. It bundles interoperable, open-source software components into a turnkey package for installation on one of your secure servers. The Bridgehead is pre-configured with sane defaults, centrally monitored and with an absolute minimum of "moving parts" on your side, making it an extremely low-maintenance gateway to data sharing.
@@ -24,6 +23,7 @@ This repository is the starting point for any information and tools you will nee
     - [File structure](#file-structure)
     - [BBMRI-ERIC Directory entry needed](#bbmri-eric-directory-entry-needed)
     - [Loading data](#loading-data)
+    - [Metadata feedback](#metadata-feedback)
 4. [Things you should know](#things-you-should-know)
     - [Auto-Updates](#auto-updates)
     - [Auto-Backups](#auto-backups)
@@ -155,7 +155,8 @@ Pay special attention to:
 Clone the bridgehead repository:
 ```shell
 sudo mkdir -p /srv/docker/
-sudo git clone -b main https://github.com/samply/bridgehead.git /srv/docker/bridgehead
+sudo git clone https://github.com/samply/bridgehead.git /srv/docker/bridgehead
+sudo git checkout metadata_fb # Only needed if you want to use metadata feedback
 ```
 
 Then, run the installation script:
@@ -348,6 +349,26 @@ Normally, you will need to build your own ETL to feed the Bridgehead. However, t
 
 You can find the profiles for generating FHIR in [Simplifier](https://simplifier.net/bbmri.de/~resources?category=Profile).
 
+### Metadata feedback
+
+The Bridgehead comes with a tool that allows you to associate metadata with samples. Multiple arbitrary text strings are allowed. A typical use case would be publications based on research using a sample. Here, one could lay down the DOI of the publication in the sample.
+
+Full details of the system can be found [here](https://github.com/samply/feedback-deployment). To avail yourself of this feature, you need to
+
+- Use the bbmri project.
+- work with the ```metadata_fb``` branch of the Bridgehead repository.
+- Build the feedback-agent Docker container (more details [here](https://github.com/samply/feedback-agent/)).
+- Build the feedback-agent-ui Docker container (more details [here](https://github.com/samply/feedback-agent-ui/)).
+
+The following extra environment variables need to be added to your ```/etc/bridgehead/bbmri.conf``` file:
+
+``` code
+ENABLE_EXPORTER=true
+ENABLE_FEEDBACK_AGENT=true
+FEEDBACK_HUB_URL=<URL for central feedback hub backend API>
+FOCUS_RETRY_COUNT=256
+```
+
 ## Things you should know
 
 ### Auto-Updates

bbmri/modules/eric-compose.yml

@@ -22,6 +22,7 @@ services:
       BROKER_URL: ${ERIC_BROKER_URL}
       PROXY_ID: ${ERIC_PROXY_ID}
       APP_focus_KEY: ${ERIC_FOCUS_BEAM_SECRET_SHORT}
+      APP_feedback-agent_KEY: ${FEEDBACK_AGENT_BEAM_SECRET}
       PRIVKEY_FILE: /run/secrets/proxy.pem
       ALL_PROXY: http://forward_proxy:3128
       TLS_CA_CERTIFICATES_DIR: /conf/trusted-ca-certs

bbmri/modules/exporter-compose.yml

@@ -0,0 +1,67 @@
+version: "3.7"
+
+services:
+  exporter:
+    image: docker.verbis.dkfz.de/ccp/dktk-exporter:latest
+    container_name: bridgehead-ccp-exporter
+    environment:
+      JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC"
+      LOG_LEVEL: "INFO"
+      EXPORTER_API_KEY: "${EXPORTER_API_KEY}"
+      CROSS_ORIGINS: "https://${HOST}"
+      EXPORTER_DB_USER: "exporter"
+      EXPORTER_DB_PASSWORD: "${EXPORTER_DB_PASSWORD}"
+      EXPORTER_DB_URL: "jdbc:postgresql://exporter-db:5432/exporter"
+      HTTP_RELATIVE_PATH: "/ccp-exporter"
+      SITE: "${SITE_ID}"
+      HTTP_SERVLET_REQUEST_SCHEME: "https"
+      OPAL_PASSWORD: "${EXPORTER_OPAL_PASSWORD}"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.exporter_ccp.rule=PathPrefix(`/ccp-exporter`)"
+      - "traefik.http.services.exporter_ccp.loadbalancer.server.port=8092"
+      - "traefik.http.routers.exporter_ccp.tls=true"
+      - "traefik.http.middlewares.exporter_ccp_strip.stripprefix.prefixes=/ccp-exporter"
+      - "traefik.http.routers.exporter_ccp.middlewares=exporter_ccp_strip"
+    volumes:
+      - "/var/cache/bridgehead/ccp/exporter-files:/app/exporter-files/output"
+
+  exporter-db:
+    image: docker.verbis.dkfz.de/cache/postgres:${POSTGRES_TAG}
+    container_name: bridgehead-ccp-exporter-db
+    environment:
+      POSTGRES_USER: "exporter"
+      POSTGRES_PASSWORD: "${EXPORTER_DB_PASSWORD}"
+      POSTGRES_DB: "exporter"
+    volumes:
+      # Consider removing this volume once we find a solution to save Lens-queries to be executed in the explorer.
+      - "/var/cache/bridgehead/ccp/exporter-db:/var/lib/postgresql/data"
+
+  reporter:
+    image: docker.verbis.dkfz.de/ccp/dktk-reporter:latest
+    container_name: bridgehead-ccp-reporter
+    environment:
+      JAVA_OPTS: "-Xms1G -Xmx8G -XX:+UseG1GC"
+      LOG_LEVEL: "INFO"
+      CROSS_ORIGINS: "https://${HOST}"
+      HTTP_RELATIVE_PATH: "/ccp-reporter"
+      SITE: "${SITE_ID}"
+      EXPORTER_API_KEY: "${EXPORTER_API_KEY}"
+      EXPORTER_URL: "http://exporter:8092"
+      LOG_FHIR_VALIDATION: "false"
+      HTTP_SERVLET_REQUEST_SCHEME: "https"
+
+    # In this initial development state of the bridgehead, we are trying to have so many volumes as possible.
+    # However, in the first executions in the CCP sites, this volume seems to be very important. A report is
+    # a process that can take several hours, because it depends on the exporter.
+    # There is a risk that the bridgehead restarts, losing the already created export.
+
+    volumes:
+      - "/var/cache/bridgehead/ccp/reporter-files:/app/reports"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.reporter_ccp.rule=PathPrefix(`/ccp-reporter`)"
+      - "traefik.http.services.reporter_ccp.loadbalancer.server.port=8095"
+      - "traefik.http.routers.reporter_ccp.tls=true"
+      - "traefik.http.middlewares.reporter_ccp_strip.stripprefix.prefixes=/ccp-reporter"
+      - "traefik.http.routers.reporter_ccp.middlewares=reporter_ccp_strip"

bbmri/modules/exporter-setup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+if [ "$ENABLE_EXPORTER" == true ]; then
+  log INFO "Exporter setup detected -- will start Exporter service."
+  OVERRIDE+=" -f ./$PROJECT/modules/exporter-compose.yml"
+  EXPORTER_DB_PASSWORD="$(echo \"This is a salt string to generate one consistent password for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 30)"
+  EXPORTER_API_KEY="$(echo \"This is a salt string to generate one consistent API KEY for the exporter. It is not required to be secret.\" | sha1sum | openssl pkeyutl -sign -inkey /etc/bridgehead/pki/${SITE_ID}.priv.pem | base64 | head -c 64)"
+  POSTGRES_TAG=15.6-alpine
+fi

bbmri/modules/exporter.md

@@ -0,0 +1,15 @@
+# Exporter and Reporter
+
+
+## Exporter
+The exporter is a REST API that exports the data of the different databases of the bridgehead in a set of tables.
+It can accept different output formats as CSV, Excel, JSON or XML. It can also export data into Opal.
+
+## Exporter-DB
+It is a database to save queries for its execution in the exporter. 
+The exporter manages also the different executions of the same query in through the database.
+
+## Reporter
+This component is a plugin of the exporter that allows to create more complex Excel reports described in templates.
+It is compatible with different template engines as Groovy, Thymeleaf,...
+It is perfect to generate a document as our traditional CCP quality report.

bbmri/modules/feedback-agent-compose.yml

@@ -0,0 +1,59 @@
+version: "3.7"
+
+services:
+  feedback-agent-ui:
+    image: "samply/feedback-agent-ui"
+    environment:
+      - VUE_APP_EXPORTER_URL=https://localhost/ccp-exporter
+      - VUE_APP_FB_BACKEND_URL=http://localhost:8072
+    labels:
+     - traefik.enable=true
+     # HTTPS
+     - traefik.http.routers.feedback_agent_ui_ccp_https.rule=PathPrefix(`/ccp-feedback-agent-ui`)
+     - traefik.http.services.feedback_agent_ui_ccp_https.loadbalancer.server.port=8096
+     - traefik.http.routers.feedback_agent_ui_ccp_https.entrypoints=websecure
+     - traefik.http.routers.feedback_agent_ui_ccp_https.tls=true
+  feedback-agent:
+    image: "samply/feedback-agent"
+    environment:
+      - SPRING_DATASOURCE_URL=jdbc:postgresql://feedback-agent-db:5432/compose-postgres
+      - SPRING_DATASOURCE_USERNAME=compose-postgres
+      - SPRING_DATASOURCE_PASSWORD=${FEEDBACK_AGENT_DB_PASSWORD}
+      - SPRING_JPA_HIBERNATE_DDL_AUTO=update
+      - BEAM_PROXY_URI=http://beam-proxy-eric:8081
+      - FEEDBACK_HUB_URL=${FEEDBACK_HUB_URL}
+      - BLAZE_BASE_URL=http://blaze:8080/fhir
+      - FEEDBACK_AGENT_SECRET=${FEEDBACK_AGENT_BEAM_SECRET}
+      - FEEDBACK_AGENT_BEAM_ID=feedback-agent.${ERIC_PROXY_ID}
+      - FEEDBACK_HUB_BEAM_ID=feedback-hub.feedback-central.${ERIC_BROKER_ID}
+      - EXPORTER_API_KEY=${EXPORTER_API_KEY}
+      - CORS_ALLOWED_ORIGINS="https://${HOST}
+    networks:
+      # Only needed for local testing.
+      - feedback
+      - default
+    labels:
+     - traefik.enable=true
+     # HTTPS
+     - traefik.http.routers.feedback_agent_ccp_https.rule=PathPrefix(`/ccp-feedback-agent`)
+     - traefik.http.services.feedback_agent_ccp_https.loadbalancer.server.port=8072
+     - traefik.http.routers.feedback_agent_ccp_https.entrypoints=websecure
+     - traefik.http.middlewares.feedback_agent_ccp_https_strip.stripprefix.prefixes=/ccp-feedback-agent
+     - traefik.http.routers.feedback_agent_ccp_https.middlewares=feedback_agent_ccp_https_strip
+     - traefik.http.routers.feedback_agent_ccp_https.tls=true
+  feedback-agent-db:
+    image: 'postgres:13.1-alpine'
+    container_name: feedback-agent-db
+    environment:
+      - POSTGRES_USER=compose-postgres
+      - POSTGRES_PASSWORD=${FEEDBACK_AGENT_DB_PASSWORD}
+
+# This is needed when you run both agent and hub locally in a test
+# environment. Not necessary in production, though it probably won't
+# cause any problems.
+networks:
+  # Network to connect agent and hub.
+  feedback:
+    name: feedback
+    driver: bridge
+

bbmri/modules/feedback-agent-setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ "$ENABLE_FEEDBACK_AGENT" == true ]; then
+  OVERRIDE+=" -f ./$PROJECT/modules/feedback-agent-compose.yml"
+  FEEDBACK_AGENT_BEAM_SECRET="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
+  FEEDBACK_AGENT_DB_PASSWORD="$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20)"
+fi
+

bbmri/modules/feedback-agent.md

@@ -0,0 +1,6 @@
+# Metadata feedback agent
+
+This component can be used to choose the sample to be associated
+with a given piece of metadata (generally the ID of a publication
+relating to research done with the sample).
+

ccp/modules/obds2fhir-rest-compose.yml

@@ -3,7 +3,7 @@ version: "3.7"
 services:
   obds2fhir-rest:
     container_name: bridgehead-obds2fhir-rest
-    image: docker.verbis.dkfz.de/samply/obds2fhir-rest:main
+    image: docker.verbis.dkfz.de/ccp/obds2fhir-rest:main
     environment:
       IDTYPE: BK_${IDMANAGEMENT_FRIENDLY_ID}_L-ID
       MAINZELLISTE_APIKEY: ${IDMANAGER_LOCAL_PATIENTLIST_APIKEY}

dhki/vars

@@ -17,12 +17,4 @@ do
 done
 
 idManagementSetup
-obds2fhirRestSetup
+obds2fhirRestSetup
-
-for module in modules/*.sh
-do
-    log DEBUG "sourcing $module"
-    source $module
-done
-
-transfairSetup

modules/transfair-compose.yml

@@ -1,51 +0,0 @@
-
-services:
-  transfair:
-    image: docker.verbis.dkfz.de/cache/samply/transfair:latest
-    container_name: bridgehead-transfair
-    environment:
-      # NOTE: Those 3 variables need only to be passed if their set, otherwise transfair will complain about empty url values
-      - INSTITUTE_TTP_URL
-      - INSTITUTE_TTP_API_KEY
-      - PROJECT_ID_SYSTEM
-      - FHIR_REQUEST_URL=${FHIR_REQUEST_URL}
-      - FHIR_INPUT_URL=${FHIR_INPUT_URL}
-      - FHIR_OUTPUT_URL=${FHIR_OUTPUT_URL:-http://blaze:8080}
-      - FHIR_REQUEST_CREDENTIALS=${FHIR_REQUEST_CREDENTIALS}
-      - FHIR_INPUT_CREDENTIALS=${FHIR_INPUT_CREDENTIALS}
-      - FHIR_OUTPUT_CREDENTIALS=${FHIR_OUTPUT_CREDENTIALS}
-      - EXCHANGE_ID_SYSTEM=${EXCHANGE_ID_SYSTEM:-SESSION_ID}
-      - DATABASE_URL=sqlite://transfair/data_requests.sql?mode=rwc
-      - RUST_LOG=${RUST_LOG:-info}
-    volumes:
-      - /var/cache/bridgehead/${PROJECT}/transfair:/transfair
-
-  transfair-input-blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
-    container_name: bridgehead-transfair-input-blaze
-    environment:
-      BASE_URL: "http://bridgehead-transfair-input-blaze:8080"
-      JAVA_TOOL_OPTIONS: "-Xmx1024m"
-      DB_BLOCK_CACHE_SIZE: 1024
-      CQL_EXPR_CACHE_SIZE: 8
-      ENFORCE_REFERENTIAL_INTEGRITY: "false"
-    volumes:
-      - "transfair-input-blaze-data:/app/data"
-    profiles: ["transfair-input-blaze"]
-
-  transfair-request-blaze:
-    image: docker.verbis.dkfz.de/cache/samply/blaze:0.28
-    container_name: bridgehead-transfair-requests-blaze
-    environment:
-      BASE_URL: "http://bridgehead-transfair-requests-blaze:8080"
-      JAVA_TOOL_OPTIONS: "-Xmx1024m"
-      DB_BLOCK_CACHE_SIZE: 1024
-      CQL_EXPR_CACHE_SIZE: 8
-      ENFORCE_REFERENTIAL_INTEGRITY: "false"
-    volumes:
-      - "transfair-request-blaze-data:/app/data"
-    profiles: ["transfair-request-blaze"]
-
-volumes:
-  transfair-input-blaze-data:
-  transfair-request-blaze-data:

modules/transfair-setup.sh

@@ -1,22 +0,0 @@
-#!/bin/bash -e
-
-function transfairSetup() {
-    if [[ -n "$INSTITUTE_TTP_URL" || -n "$EXCHANGE_ID_SYSTEM" ]]; then
-        echo "Starting transfair."
-	    OVERRIDE+=" -f ./modules/transfair-compose.yml"
-	    if [ -n "$FHIR_INPUT_URL" ]; then
-		    log INFO "TransFAIR input fhir store set to external $FHIR_INPUT_URL"
-	    else
-		    log INFO "TransFAIR input fhir store not set writing to internal blaze"
-		    FHIR_INPUT_URL="http://transfair-input-blaze:8080"
-		    OVERRIDE+=" --profile transfair-input-blaze"
-	    fi
-	    if [ -n "$FHIR_REQUEST_URL" ]; then
-		    log INFO "TransFAIR request fhir store set to external $FHIR_REQUEST_URL"
-	    else
-		    log INFO "TransFAIR request fhir store not set writing to internal blaze"
-		    FHIR_REQUEST_URL="http://transfair-requests-blaze:8080"
-		    OVERRIDE+=" --profile transfair-request-blaze"
-	    fi
-    fi
-}