From c683bb76bc4d4e21db4c4bf0cd1af0f32a9326ed Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Tue, 21 Dec 2021 09:12:34 +0100 Subject: [PATCH 001/215] WIP: Merged docker files and install script from bk-dep and c4 docker --- c4/docker-compose.yml | 177 +++++++ convenience/bridgehead-update@.service | 10 + convenience/bridgehead-update@.timer | 8 + convenience/bridgehead@.service | 23 + convenience/traefik.service | 19 + dktk/docker-compose.yml | 144 ++++++ gbn/README.md | 334 +++++++++++++ gbn/docker-compose.yml | 61 +++ install-bridgehead.sh | 40 ++ internal-configuration/bridgehead-common.xml | 13 + .../centralSearchPublicKey.der | Bin 0 -> 294 bytes internal-configuration/magicpl.xml | 444 ++++++++++++++++++ internal-configuration/mainzelliste.conf | 227 +++++++++ internal-configuration/nginx.conf | 37 ++ internal-configuration/proxy.xml | 21 + prerequisites.sh | 49 ++ 16 files changed, 1607 insertions(+) create mode 100644 c4/docker-compose.yml create mode 100644 convenience/bridgehead-update@.service create mode 100644 convenience/bridgehead-update@.timer create mode 100644 convenience/bridgehead@.service create mode 100644 convenience/traefik.service create mode 100644 dktk/docker-compose.yml create mode 100644 gbn/README.md create mode 100644 gbn/docker-compose.yml create mode 100644 install-bridgehead.sh create mode 100644 internal-configuration/bridgehead-common.xml create mode 100644 internal-configuration/centralSearchPublicKey.der create mode 100644 internal-configuration/magicpl.xml create mode 100644 internal-configuration/mainzelliste.conf create mode 100644 internal-configuration/nginx.conf create mode 100644 internal-configuration/proxy.xml create mode 100644 prerequisites.sh diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml new file mode 100644 index 0000000..8c4e7fd --- /dev/null +++ b/c4/docker-compose.yml @@ -0,0 +1,177 @@ +version: "3.7" +volumes: + connector_db_data: + connector_logs: + patientlist_db_data: + patientlist_logs: + idmanager_logs: + store_db_data: + store_logs: +secrets: + mainzelliste.docker.conf: + file: ./internal-configuration/mainzelliste.conf + magicpl.docker.xml: + file: ./internal-configuration/magicpl.xml + dktk_bridgehead_info.docker.xml: + file: ./internal-configuration/bridgehead-common.xml + centralSearchPublicKey: + file: ./internal-configuration/centralSearchPublicKey.der + proxy.docker.xml: + file: ./internal-configuration/proxy.xml + +services: + connector: + container_name: c4_connector + image: samply/share-client:${PROJECT}-develop + environment: + SITE: ${SITE} + SITEID: ${SITEID} + CENTRAL_SEARCH: ${CCP_CENTRALSEARCH_URL} + DECENTRAL_SEARCH: ${CCP_DECENTRALSEARCH_URL} + MDR_URL: ${CCP_MDR_URL} + MONITOR_URL: ${CCP_MONITOR_URL} + SHARE_URL: "${PROTOCOL}://${HOST}:${PORT}" + ID_MANAGER_URL: ${ID_MANAGER_URL} + PROJECTPSEUDONYMISATION_URL: ${PROJECTPSEUDONYMISATION_URL} + PATIENTLIST_URL: ${PATIENTLIST_URL} + STORE_URL: ${LDM_URL} + POSTGRES_HOST: ${CONNECTOR_DB_HOST} + POSTGRES_PORT: 5432 + POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} + POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} + POSTGRES_PASS: ${C4_CONNECTOR_POSTGRES_PASSWORD} + DEPLOYMENT_CONTEXT: ${COMPOSE_PROJECT_NAME}-connector + HTTP_PROXY_URL: ${HTTP_PROXY_URL} + HTTP_PROXY_USERNAME: ${HTTP_PROXY_USERNAME} + HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} + HTTPS_PROXY_URL: ${HTTPS_PROXY_URL} + HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USERNAME} + HTTPS_PROXY_PASSWORD: ${HTTPS_PROXY_PASSWORD} + TZ: Europe/Berlin + volumes: + - "connector_logs:/usr/local/tomcat/logs" + depends_on: + - connector_db + restart: always + extra_hosts: + - "host.docker.internal:host-gateway" + - "${HOST}:${HOSTIP}" + + connector_db: + image: postgres:10.17 + environment: + POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} + POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} + POSTGRES_PASSWORD: ${C4_CONNECTOR_POSTGRES_PASSWORD} + TZ: Europe/Berlin + volumes: + - "connector_db_data:/var/lib/postgresql/data" + restart: always + extra_hosts: + - "host.docker.internal:host-gateway" + - "${HOST}:${HOSTIP}" + + ## ID-Management + idmanager: + container_name: idmanager + image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/ID-Manager + TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} + TOMCAT_REVERSEPROXY_PORT: ${PORT} + MAGICPL_MAINZELLISTE_URL: http://patientlist:8080/Patientlist + MAGICPL_MAINZELLISTE_API_KEY: ${LOCAL_IDMANAGER_MAINZELLISTE_APIKEY} + MAGICPL_API_KEY: ${LOCAL_IDMANAGER_LDM_APIKEY} + MAGICPL_API_KEY_CONNECTOR: ${LOCAL_IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_PASSPHRASE: notUsedInThisConfigurationButMandatory + ### Configuration for communication with central identity management + MAGICPL_MAINZELLISTE_CENTRAL_URL: ${CCP_PATIENTLIST_URL} + MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${CCP_PATIENTLISTE_APIKEY} + MAGICPL_CENTRAL_URL: ${CCP_CONTROLLNUMBERGENERATOR_URL} + MAGICPL_CENTRAL_API_KEY: ${CCP_CONTROLLNUMBERGENERATOR_APIKEY} + MAGICPL_OIDC_PROVIDER: ${CCP_OIDC_PROVIDER_URL}/oauth2 + MAGICPL_OIDC_CLIENT_ID: ${CCP_OIDC_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${CCP_OIDC_CLIENT_SECRET} + MAGICPL_SITE: ${SITEID} + MAGICPL_LOG_LEVEL: info + TZ: Europe/Berlin + volumes: + - "idmanager_logs:/usr/local/tomcat/logs" + secrets: + - magicpl.docker.xml + - dktk_bridgehead_info.docker.xml + - proxy.docker.xml + depends_on: + - patientlist + + patientlist: + container_name: patientlist + image: medicalinformatics/mainzelliste:develop + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/Patientlist + TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} + TOMCAT_REVERSEPROXY_PORT: ${PORT} + ML_DB_DRIVER: org.postgresql.Driver + ML_DB_TYPE: postgresql + ML_DB_HOST: patientlist_db + ML_DB_PORT: 5432 + ML_DB_NAME: ${ML_DB_NAME} + ML_DB_USER: ${ML_DB_USER} + ML_DB_PASS: ${LOCAL_PATIENTLIST_DBPASS} + ML_API_KEY: ${LOCAL_IDMANAGER_MAINZELLISTE_APIKEY} + ML_LOG_LEVEL: warning + ML_SITE: ${SITEID} + TZ: Europe/Berlin + env_file: + - ./site-config/patientlist.env + volumes: + - "patientlist_logs:/usr/local/tomcat/logs" + secrets: + - mainzelliste.docker.conf + - centralSearchPublicKey + depends_on: + - patientlist_db + + patientlist_db: + container_name: patientlist_db + image: postgres:13.1-alpine + environment: + POSTGRES_DB: ${ML_DB_NAME} + POSTGRES_USER: ${ML_DB_USER} + POSTGRES_PASSWORD: ${C4_LOCAL_PATIENTLIST_DBPASS} + TZ: Europe/Berlin + volumes: + - "patientlist_db_data:/var/lib/postgresql/data" + networks: + - idmanagement +store: + image: docker.verbis.dkfz.de/ccp/samply.store:release-5.1.2 + environment: + MDR_URL: ${CCP_MDR_URL} + MDR_NAMESPACE: adt,dktk,marker + MDR_VALIDATION: "false" + POSTGRES_HOST: store_db + POSTGRES_PORT: 5432 + POSTGRES_DB: samplystore + POSTGRES_USER: samplystore + POSTGRES_PASSWORD: ${C4_SAMPLY_STORE_PASS} + TZ: Europe/Berlin + volumes: + - "store_logs:/usr/local/tomcat/logs" + depends_on: + - store_db + restart: always + + store_db: + image: postgres:10.17 + command: postgres -c datestyle='iso, dmy' + environment: + POSTGRES_HOST: store_db + POSTGRES_PORT: 5432 + POSTGRES_DB: samplystore + POSTGRES_USER: samplystore + POSTGRES_PASSWORD: ${C4_SAMPLY_STORE_PASS} + TZ: Europe/Berlin + volumes: + - "store_db_data:/var/lib/postgresql/data" + restart: always diff --git a/convenience/bridgehead-update@.service b/convenience/bridgehead-update@.service new file mode 100644 index 0000000..b568076 --- /dev/null +++ b/convenience/bridgehead-update@.service @@ -0,0 +1,10 @@ +[Unit] +Description=Bridgehead (%i) Update Service + +[Service] +Type=oneshot +WorkingDirectory=/srv/docker/bridgehead/ +ExecStart=/bin/bash -c "/srv/docker/bridgehead/update-bridgehead.sh %i" + +[Install] +WantedBy=multi-user.target diff --git a/convenience/bridgehead-update@.timer b/convenience/bridgehead-update@.timer new file mode 100644 index 0000000..e09ad87 --- /dev/null +++ b/convenience/bridgehead-update@.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Nightly Updates of Bridgehead (%i) + +[Timer] +OnCalendar=*-*-* 03:00:00 + +[Install] +WantedBy=basic.target diff --git a/convenience/bridgehead@.service b/convenience/bridgehead@.service new file mode 100644 index 0000000..dd86c45 --- /dev/null +++ b/convenience/bridgehead@.service @@ -0,0 +1,23 @@ +[Unit] +Description=Bridgehead (%i) Service +Requires=traefik.service +After=traefik.service + +[Service] +Restart=always +RestartSec=30 + +WorkingDirectory=/srv/docker/bridgehead + +ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/%i.env down +ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/%i.env rm +ExecStartPre=CONNECTOR_POSTGRES_PASSWORD=%i_CONNECTOR_POSTGRES_PASSWORD +ExecStartPre=export CONNECTOR_POSTGRES_PASSWORD +ExecStart=/usr/local/bin/docker-compose --env-file site-config/%i.env up +ExecStart=CONNECTOR_POSTGRES_PASSWORD= +ExecStart=export CONNECTOR_POSTGRES_PASSWORD + +ExecStop=/usr/local/bin/docker-compose --env-file site-config/%i.env down + +[Install] +WantedBy=multi-user.target diff --git a/convenience/traefik.service b/convenience/traefik.service new file mode 100644 index 0000000..8c52ee3 --- /dev/null +++ b/convenience/traefik.service @@ -0,0 +1,19 @@ +[Unit] +Description=Traefik Service +Requires=docker.service +After=docker.service + +[Service] +Restart=always +RestartSec=30 + +WorkingDirectory=/srv/docker/bridgehead + +ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/start.env down +ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/start.env rm +ExecStart=/usr/local/bin/docker-compose --env-file site-config/start.env up + +ExecStop=/usr/local/bin/docker-compose --env-file site-config/start.env down + +[Install] +WantedBy=multi-user.target diff --git a/dktk/docker-compose.yml b/dktk/docker-compose.yml new file mode 100644 index 0000000..bc57de6 --- /dev/null +++ b/dktk/docker-compose.yml @@ -0,0 +1,144 @@ +version: "3.7" +volumes: + connector_db_data: + connector_logs: + patientlist_db_data: + patientlist_logs: + idmanager_logs: +secrets: + mainzelliste.docker.conf: + file: ../internal-configuration/mainzelliste.conf + magicpl.docker.xml: + file: ../internal-configuration/magicpl.xml + dktk_bridgehead_info.docker.xml: + file: ../internal-configuration/bridgehead-common.xml + centralSearchPublicKey: + file: ../internal-configuration/centralSearchPublicKey.der + proxy.docker.xml: + file: ../internal-configuration/proxy.xml + +services: + connector: + container_name: dktk_connector + image: samply/share-client:dktk-develop + environment: + SITE: ${SITE} + SITEID: ${SITEID} + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/${COMPOSE_PROJECT_NAME}-connector + CENTRAL_SEARCH: ${CCP_CENTRALSEARCH_URL} + DECENTRAL_SEARCH: ${CCP_DECENTRALSEARCH_URL} + MDR_URL: ${CCP_MDR_URL} + MONITOR_URL: ${CCP_MONITOR_URL} + SHARE_URL: "${PROTOCOL}://${HOST}:${PORT}" + ID_MANAGER_URL: ${ID_MANAGER_URL} + PROJECTPSEUDONYMISATION_URL: ${PROJECTPSEUDONYMISATION_URL} + PATIENTLIST_URL: ${PATIENTLIST_URL} + STORE_URL: ${LDM_URL} + POSTGRES_HOST: ${CONNECTOR_DB_HOST} + POSTGRES_PORT: 5432 + POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} + POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} + POSTGRES_PASS: ${DKTK_CONNECTOR_POSTGRES_PASSWORD} + DEPLOYMENT_CONTEXT: ${COMPOSE_PROJECT_NAME}-connector + HTTP_PROXY_URL: ${HTTP_PROXY_URL} + HTTP_PROXY_USERNAME: ${HTTP_PROXY_USERNAME} + HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} + HTTPS_PROXY_URL: ${HTTPS_PROXY_URL} + HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USERNAME} + HTTPS_PROXY_PASSWORD: ${HTTPS_PROXY_PASSWORD} + TZ: Europe/Berlin + volumes: + - "connector_logs:/usr/local/tomcat/logs" + depends_on: + - connector_db + restart: always + extra_hosts: + - "host.docker.internal:host-gateway" + - "${HOST}:${HOSTIP}" + + connector_db: + image: postgres:10.17 + environment: + POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} + POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} + POSTGRES_PASSWORD: ${DKTK_CONNECTOR_POSTGRES_PASSWORD} + TZ: Europe/Berlin + volumes: + - "connector_db_data:/var/lib/postgresql/data" + restart: always + extra_hosts: + - "host.docker.internal:host-gateway" + - "${HOST}:${HOSTIP}" + + ## ID-Management + idmanager: + container_name: idmanager + image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/ID-Manager + TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} + TOMCAT_REVERSEPROXY_PORT: ${PORT} + MAGICPL_MAINZELLISTE_URL: http://patientlist:8080/Patientlist + MAGICPL_MAINZELLISTE_API_KEY: ${LOCAL_IDMANAGER_MAINZELLISTE_APIKEY} + MAGICPL_API_KEY: ${LOCAL_IDMANAGER_LDM_APIKEY} + MAGICPL_API_KEY_CONNECTOR: ${LOCAL_IDMANAGER_CONNECTOR_APIKEY} + MAGICPL_PASSPHRASE: notUsedInThisConfigurationButMandatory + ### Configuration for communication with central identity management + MAGICPL_MAINZELLISTE_CENTRAL_URL: ${CCP_PATIENTLIST_URL} + MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${CCP_PATIENTLISTE_APIKEY} + MAGICPL_CENTRAL_URL: ${CCP_CONTROLLNUMBERGENERATOR_URL} + MAGICPL_CENTRAL_API_KEY: ${CCP_CONTROLLNUMBERGENERATOR_APIKEY} + MAGICPL_OIDC_PROVIDER: ${CCP_OIDC_PROVIDER_URL}/oauth2 + MAGICPL_OIDC_CLIENT_ID: ${CCP_OIDC_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${CCP_OIDC_CLIENT_SECRET} + MAGICPL_SITE: ${SITEID} + MAGICPL_LOG_LEVEL: info + TZ: Europe/Berlin + volumes: + - "idmanager_logs:/usr/local/tomcat/logs" + secrets: + - magicpl.docker.xml + - dktk_bridgehead_info.docker.xml + - proxy.docker.xml + depends_on: + - patientlist + + patientlist: + container_name: patientlist + image: medicalinformatics/mainzelliste:develop + environment: + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/Patientlist + TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} + TOMCAT_REVERSEPROXY_PORT: ${PORT} + ML_DB_DRIVER: org.postgresql.Driver + ML_DB_TYPE: postgresql + ML_DB_HOST: patientlist_db + ML_DB_PORT: 5432 + ML_DB_NAME: ${ML_DB_NAME} + ML_DB_USER: ${ML_DB_USER} + ML_DB_PASS: ${DKTK_PATIENTLIST_DBPASS} + ML_API_KEY: ${DKTK_IDMANAGER_MAINZELLISTE_APIKEY} + ML_LOG_LEVEL: warning + ML_SITE: ${SITEID} + TZ: Europe/Berlin + env_file: + - ./site-config/patientlist.env + volumes: + - "patientlist_logs:/usr/local/tomcat/logs" + secrets: + - mainzelliste.docker.conf + - centralSearchPublicKey + depends_on: + - patientlist_db + + patientlist_db: + container_name: patientlist_db + image: postgres:13.1-alpine + environment: + POSTGRES_DB: ${ML_DB_NAME} + POSTGRES_USER: ${ML_DB_USER} + POSTGRES_PASSWORD: ${DKTK_PATIENTLIST_DBPASS} + TZ: Europe/Berlin + volumes: + - "patientlist_db_data:/var/lib/postgresql/data" + diff --git a/gbn/README.md b/gbn/README.md new file mode 100644 index 0000000..a7073a7 --- /dev/null +++ b/gbn/README.md @@ -0,0 +1,334 @@ +# Bridgehead Deployment + + +## Goal +Allow the Sample Locator to search for patients and samples in your biobanks, giving researchers easy access to your resources. + + +## Quick start +If you simply want to set up a test installation, without exploring all of the possibilities offered by the Bridgehead, then the sections you need to look at are: +* [Starting a Bridgehead](#starting-a-bridgehead) +* [Register with a Sample Locator](#register-with-a-sample-locator) +* [Checking your newly installed Bridgehead](#checking-your-newly-installed-bridgehead) + + +## Background +The **Sample Locator** is a tool that allows researchers to make searches for samples over a large number of geographically distributed biobanks. Each biobank runs a so-called **Bridgehead** at its site, which makes it visible to the Sample Locator. The Bridgehead is designed to give a high degree of protection to patient data. Additionally, a tool called the [Negotiator][negotiator] puts you in complete control over which samples and which data are delivered to which researcher. + +You will most likely want to make your biobanks visible via the [publicly accessible Sample Locator][sl], but the possibility also exists to install your own Sample Locator for your site or organization, see the GitHub pages for [the server][sl-server-src] and [the GUI][sl-ui-src]. + +The Bridgehead has two primary components: +* The **Blaze Store**. This is a highly responsive FHIR data store, which you will need to fill with your data via an ETL chain. +* The **Connector**. This is the communication portal to the Sample Locator, with specially designed features that make it possible to run it behind a corporate firewall without making any compromises on security. + +This document will show you how to: +* Install the components making up the Bridgehead. +* Register your Bridgehead with the Sample Locator, so that researchers can start searching your resources. + + +## Requirements +For data protection concept, server requirements, validation or import instructions, see [the list of general requirements][requirements]. + + +## Starting a Bridgehead +The file `docker-compose.yml` contains the the minimum settings needed for installing and starting a Bridgehead on your computer. This Bridgehead should run straight out of the box. However, you may wish to modify this file, e.g. in order to: +* Enable a corporate proxy (see below). +* Set an alternative Sample Locator URL. +* Change the admin credentials for the Connector. + +To start a Bridgehead on your computer, you will need to follow the following steps: + +* [Install Docker][docker] and [git][git]and test with: + +```sh +docker run hello-world +git --version +``` + +* Download this repository: + +```sh +git clone https://github.com/samply/bridgehead-deployment +cd bridgehead-deployment +``` + +* Launch the Bridgehead with the following command: + +```sh +docker-compose up -d +``` + +* First test of the installation: check to see if there is a Connector running on port 8082: + +```sh +curl localhost:8082 | grep Welcome +``` + +* If you need to stop the Bridgehead, from within this directory: + +```sh +docker-compose down +``` + +## Port usage +Once you have started the Bridgehead, the following components will be visible to you via ports on localhost: +* Blaze Store: port 8080 +* Connector admin: port 8082 + +## Connector Administration +The Connector administration panel allows you to set many of the parameters regulating your Bridgehead. Most especially, it is the place where you can register your site with the Sample Locator. To access this page, proceed as follows: + +* Open the Connector page: http://localhost:8082 +* In the "Local components" box, click the "Samply Share" button. +* A new page will be opened, where you will need to log in using the administrator credentials (admin/adminpass by default). +* After log in, you will be taken to the administration dashboard, allowing you to configure the Connector. +* If this is the first time you have logged in as an administrator, you are strongly recommended to set a more secure password! You can use the "Users" button on the dashboard to do this. + +Note: your browser must be running on the same machine as the Connector for "localhost" URLs to work. + +### Register with a Directory +The [Directory][directory] is a BBMRI project that aims to catalog all biobanks in Europe and beyond. Each biobank is given its own unique ID and the Directory maintains counts of the number of donors and the number of samples held at each biobank. You are strongly encouraged to register with the Directory, because this opens the door to further services, such as the [Negotiator][negotiator]. + +Generally, you should register with the BBMRI national node for the country where your biobank is based. You can find a list of contacts for the national nodes [here](http://www.bbmri-eric.eu/national-nodes/). If your country is not in this list, or you have any questions, please contact the [BBMRI helpdesk](mailto:directory@helpdesk.bbmri-eric.eu). If your biobank is for COVID samples, you can also take advantage of an accelerated registration process [here](https://docs.google.com/forms/d/e/1FAIpQLSdIFfxADikGUf1GA0M16J0HQfc2NHJ55M_E47TXahju5BlFIQ). + +Your national node will give you detailed instructions for registering, but for your information, here are the basic steps: + +* Log in to the Directory for your country. +* Add your biobank and enter its details, including contact information for a person involved in running the biobank. +* You will need to create at least one collection. +* Note the biobank ID and the collection ID that you have created - these will be needed when you register with the Locator (see below). + +### Register with a Locator +* Go to the registration page http://localhost:8082/admin/broker_list.xhtml. +* To register with a Locator, enter the following values in the three fields under "Join new Searchbroker": + * "Address": Depends on which Locator you want to register with: + * `https://locator.bbmri-eric.eu/broker/`: BBMRI Locator production service (European). + * `http://147.251.124.125:8088/broker/`: BBMRI Locator test service (European). + * `https://samplelocator.bbmri.de/broker/`: GBA Sample Locator production service (German). + * `https://samplelocator.test.bbmri.de/broker/`: GBA Sample Locator test service (German). + * "Your email address": this is the email to which the registration token will be returned. + * "Automatic reply": Set this to be `Total Size` +* Click "Join" to start the registration process. +* You should now have a list containing exactly one broker. You will notice that the "Status" box is empty. +* Send an email to `feedback@germanbiobanknode.de` and let us know which of our Sample Locators you would like to register to. Please include the biobank ID and the collection ID from your Directory registration, if you have these available. +* We will send you a registration token per email. +* You will then re-open the Connector and enter the token into the "Status" box. +* You should send us an email to let us know that you have done this. +* We will then complete the registration process +* We will email you to let you know that your biobank is now visible in the Sample Locator. + +If you are a Sample Locator administrator, you will need to understand the [registration process](./SampleLocatorRegistration.md). Normal bridgehead admins do not need to worry about this. + +### Monitoring +You are strongly encouraged to set up an automated monitoring of your new Bridgehead. This will periodically test the Bridgehead in various ways, and (if you wish) will also send you emails if problems are detected. It helps you to become aware of problems before users do, and also gives you the information you need to track down the source of the problems. To activate monitoring, perform the following steps: + +* Open the Connector administration dashboard in your browser, see [Admin](#connector-administration) for details. +* Click the "Configuration" button. +* ![grafik](https://user-images.githubusercontent.com/86475306/142425285-977e5649-7f2e-44db-8da0-ee5e28b5e91b.png) +* Scroll to the section "Reporting to central services". +* Click on all of the services in this section so that they have the status "ON". +* ![grafik](https://user-images.githubusercontent.com/86475306/142425378-e1b68f13-df7a-4f23-978e-121184611586.png) +* **Important:** Scroll to the bottom of the page and click the "Save" button. +* ![grafik](https://user-images.githubusercontent.com/86475306/142425417-68a28059-37e0-48a3-bb1e-1bf29a39ccfc.png) +* Return to the dashboard, and click the button "Scheduled Tasks". +* ![grafik](https://user-images.githubusercontent.com/86475306/142425447-a662257a-d556-4795-aa0b-89f7699ba1e4.png) +* Scroll down to the box labelled "ReportToMonitoringJob". For newer Versions of the bridgehead there this job is separated into "ReportToMonitoringJobShortFrequence" and "ReportToMonitoringJobLongFrequence". +* Click the button "Run now". This switches the monitoring on. If you have the newer Version of the bridgeheads please run both jobs. +* ![grafik](https://user-images.githubusercontent.com/86475306/142425487-6d297779-28c1-44b7-b2c4-dcf2ede24eb9.png) +* If you want to receive emails when the monitoring service detects problems with your Bridgehead, please send a list of email addresses for the people to be notified to: `feedback@germanbiobanknode.de`. + +You are now done! + +### Troubleshooting +To get detailled information about Connector problems, you need to use the Docker logging facility: + +* Log into the server where the Connector is running. You need a command line login. +* Discover the container ID of the Connector. First run "docker ps". Look in the list of results. The relevant line contains the image name "samply/share-client". +* Execute the following command: "docker logs \" +* The last 100 lines of the log are relevant. Maybe you will see the problem there right away. Otherwise, send the log-selection to us. + +### User +* To enable a user to access the connector, a new user can be created under http://localhost:8082/admin/user_list.xhtml. +This user has the possibility to view incoming queries + +### Jobs +* The connector uses [Quartz Jobs](http://www.quartz-scheduler.org/) to do things like collect the queries from the searchbroker or execute the queries. +The full list of jobs can be viewed under the job page http://localhost:8082/admin/job_list.xhtml. + +### Tests +* The Connector connectivity checks can be found under the test page http://localhost:8082/admin/tests.xhtml. + +## Checking your newly installed Bridgehead +We will load some test data and then run a query to see if it shows up. + +First, install [bbmri-fhir-gen][bbmri-fhir-gen]. Run the following command: + +```sh +mkdir TestData +bbmri-fhir-gen TestData -n 10 +``` + +This will generate test data for 10 patients, and place it in the directory `TestData`. + +Next, install [blazectl][blazectl]. Run the following commands: + +```sh +blazectl --server http://localhost:8080/fhir upload TestData +blazectl --server http://localhost:8080/fhir count-resources +``` + +If both of them complete successfully, it means that the test data has successfully been uploaded to the Blaze Store. + +Open the [Sample Locator][sl] and hit the "SEND" button. You may need to wait for a minute before all results are returned. Click the "Login" link to log in via the academic institution where you work (AAI). You should now see a list of the biobanks known to the Sample Locator. + +If your biobank is present, and it contains non-zero counts of patients and samples, then your installation was successful. + +If you wish to remove the test data, you can do so by simply deleting the Docker volume for the Blaze Store database: + +```sh +docker-compose down +docker volume rm store-db-data +``` + +## Manual installation +The installation described here uses Docker, meaning that you don't have to worry about configuring or installing the Bridgehead components - Docker does it all for you. If you do not wish to use Docker, you can install all of the software directly on your machine, as follows: + +* Install the [Blaze Store][man-store] +* Install the [Connector][man-connector] +* Register with the Sample Locator (see above) + + +Source code for components deployed by `docker-compose.yml`: + +* [Store][store-src] +* [Connector][connector-src] + + +## Optional configuration: + +#### Proxy example +Add environments variables in `docker-compose.yml` (remove user and password environments if not available): +"http://proxy.example.de:8080", user "testUser", password "testPassword" + + version: '3.4' + services: + store: + container_name: "store" + image: "samply/blaze:0.11" + environment: + BASE_URL: "http://store:8080" + JAVA_TOOL_OPTIONS: "-Xmx4g" + PROXY_HOST: "http://proxy.example.de" + PROXY_PORT: "8080" + PROXY_USER: "testUser" + PROXY_PASSWORD: "testPassword" + networks: + - "samply" + + ....... + + connector: + container_name: "connector" + image: "samply/connector:7.0.0" + environment: + POSTGRES_HOST: "connector-db" + POSTGRES_DB: "samply.connector" + POSTGRES_USER: "samply" + POSTGRES_PASS: "samply" + STORE_URL: "http://store:8080/fhir" + QUERY_LANGUAGE: "CQL" + MDR_URL: "https://mdr.germanbiobanknode.de/v3/api/mdr" + HTTP_PROXY: "http://proxy.example.de:8080" + PROXY_USER: "testUser" + PROXY_PASS: "testPassword + networks: + - "samply" + + ....... + + + +#### General information on Docker environment variables used in the Bridgehead + +* [Store][env-store] +* [Connector][env-connector] + + +## Notes +* If you see database connection errors of Store or Connector, open a second terminal and run `docker-compose stop` followed by `docker-compose start`. Database connection problems should only occur at the first start because the store and the connector doesn't wait for the databases to be ready. Both try to connect at startup which might be to early. + +* If one needs only one of the the Bridgehead components, they can be started individually: + +```sh +docker-compose up store -d +docker-compose up connector -d +``` + +* To shut down all services (but keep databases): + +```sh +docker-compose down +``` + +* To delete databases as well (destroy before): + +```sh +docker volume rm store-db-data +docker volume rm connector-db-data +``` + +* To see all executed queries, create a [new user][connector-user], logout and login with this normal user. + +* To set Store-Basic-Auth-credentials in Connector (as default `Lokales Datenmanagement` with dummy values was generated) + * Login at [Connector-UI][connector-login] (default usr=admin, pwd=adminpass) + * Open [credentials page][connector-credentials] + - Delete all instances of`Lokales Datenmanagement` + - for "Ziel" select `Lokales Datenmanagement`, provide decrypted CREDENTIALS in "Benutzername" and "Passwort", select "Zugangsdaten hinzufügen" + +* If you would like to read about the experiences of a team in Brno who have installed the Bridgehead and a local Sample Locator instance, take a look at [SL-BH_deploy](SL-BH_deploy). + +## Useful Links +* [FHIR Quality Reporting Authoring UI][quality-ui-github] +* [How to join Sample Locator][join-sl] +* [Samply code repositories][samply] + +## License + +Copyright 2019 - 2021 The Samply Community + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +[sl]: +[sl-ui-src]: +[sl-server-src]: +[negotiator]: +[directory]: +[bbmri]: +[docker]: +[git]: + +[connector-user]: +[connector-login]: +[connector-credentials]: + +[requirements]: + +[man-store]: +[env-store]: +[env-connector]: + +[bbmri-fhir-gen]: +[blazectl]: + +[man-connector]: + +[store-src]: +[connector-src]: + +[quality-ui-github]: +[join-sl]: +[samply]: diff --git a/gbn/docker-compose.yml b/gbn/docker-compose.yml new file mode 100644 index 0000000..abba8de --- /dev/null +++ b/gbn/docker-compose.yml @@ -0,0 +1,61 @@ +version: '3.4' +services: + + store: + container_name: "store" + image: "samply/blaze:0.15" + environment: + BASE_URL: "http://store:8080" + JAVA_TOOL_OPTIONS: "-Xmx4g" + networks: + - "samply" + volumes: + - "store-db-data:/app/data" + ports: + - "8080:8080" + restart: "always" + + connector: + container_name: "connector" + image: "samply/share-client:gbn-7" + environment: + POSTGRES_HOST: "connector-db" + POSTGRES_DB: "samply.connector" + POSTGRES_USER: "samply" + POSTGRES_PASS: "samply" + STORE_URL: "http://store:8080/fhir" + QUERY_LANGUAGE: "CQL" + MDR_URL: "https://mdr.germanbiobanknode.de/v3/api/mdr" + networks: + - "samply" + - "connector-db" + ports: + - "8082:8080" + depends_on: + - "connector-db" + restart: "always" + + connector-db: + container_name: "connector-db" + image: "postgres:9.6" + environment: + POSTGRES_USER: "samply" + POSTGRES_PASSWORD: "samply" + POSTGRES_DB: "samply.connector" + networks: + - "connector-db" + volumes: + - "connector-db-data:/var/lib/postgresql/data" + restart: "always" + +networks: + samply: + driver: "bridge" + connector-db: + driver: "bridge" + +volumes: + store-db-data: + name: "store-db-data" + connector-db-data: + name: "connector-db-data" diff --git a/install-bridgehead.sh b/install-bridgehead.sh new file mode 100644 index 0000000..2ada9c6 --- /dev/null +++ b/install-bridgehead.sh @@ -0,0 +1,40 @@ +#!/bin/bash +### Note: Currently not complete, needs some features before useable for production + +./prerequisites.sh +source site.conf + +echo "Installing bridgehead" + +cd /etc/systemd/system/ + +echo "Installing bridgehead\@.service in systemd ..." +sudo cp /srv/docker/bridgehead/convenience/bridgehead\@.service ./ +echo "Installing bridgehead\@.update.service in systemd ..." +sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.service ./ +sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.timer ./ + +echo "Loading the bridgehead and traefik service definitions in systemd" +sudo systemctl daemon-reload + + +echo "Starting Project ${project} " + if [ ! -f "/etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf" ]; then + echo "Can't find local configuration file for bridgehead@${project} service. Please ensure that the file /etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf exists" + continue + fi + + sudo systemctl is-active --quiet bridgehead@"${project}" + if [ ! $? -eq 0 ]; then + echo "Starting bridgehead@${project} service ..." + sudo systemctl start bridgehead@"${project}" + echo "Enabling autostart of bridgehead@${project}.service" + sudo systemctl enable bridgehead@"${project}" + echo "Enabling nightly updates for bridgehead@${project}.service ..." + sudo systemctl enable --now bridgehead-update@"${project}".timer + fi +done + +# Switch back to execution directory; +cd - +# TODO: Configuration of the different modules diff --git a/internal-configuration/bridgehead-common.xml b/internal-configuration/bridgehead-common.xml new file mode 100644 index 0000000..c210e42 --- /dev/null +++ b/internal-configuration/bridgehead-common.xml @@ -0,0 +1,13 @@ + + + + Brückenkopf + DKTK Brückenkopf Teststandort + MAGICPL_SITE + NOT_USED + string +NOT_USED + NOT_USED + string + QUERY + diff --git a/internal-configuration/centralSearchPublicKey.der b/internal-configuration/centralSearchPublicKey.der new file mode 100644 index 0000000000000000000000000000000000000000..8f3e74cb1f1a3780313895517e190d465636b49c GIT binary patch literal 294 zcmV+>0ondAf&n5h4F(A+hDe6@4FLfG1potr0S^E$f&mHwf&l>l@nLuEK#1N)Tv!M;ba;%H@9S5r3T59 zZ_11@v(S6%IBCaiW85SP_OXWRX>sPubk){4 + + + + + + + + + +]> + + + + getId + + + + + + + + + + + + + getRequestedId + + + + + + + + + + + + + + + consented + + + + + + getDktkIdWithIdat + + + + + + + + + + + + + + getIdToken + + + ¢ralKNEUrl;/paths/getIdToken + POST + ¢ralKNEApiKey; + BK_&site;_L-ID + locallyUniqueId + + + + + + + + + + de.mainzelliste.paths.processor.PathClient + + + getDktkId + + ¢ralPatientListUrl; + ¢ralPatientListApiKey; + 3.2 + + + + + + + + + de.mainzelliste.paths.processor.MainzellisteClient + + + mapIdStringToExternalId + + ids->extIDs + + + + + + + + de.mainzelliste.paths.processor.InputMapperProcessor + + + getLocalTKTExternalId + + &localPatientListUrl; + &localPatientlistApiKey; + plain + + + + + + + + + + + + + de.mainzelliste.paths.processor.MainzellisteKNTicketClient + + + generateLocalIdExternalId + + &localPatientListUrl; + &localPatientlistApiKey; + 3.2 + BK_&site;_L-ID + + + + + + + + + + de.mainzelliste.paths.processor.MainzellisteClient + + + + + + getLocalIdWithIDAT + + + + + + + + + + + getTKT + + &localPatientListUrl; + &localPatientlistApiKey; + plain + + + + + + + + + + + de.mainzelliste.paths.processor.MainzellisteKNTicketClient + + + generateLocalId + + &localPatientListUrl; + &localPatientlistApiKey; + 3.2 + BK_&site;_L-ID + + + + + + + + + de.mainzelliste.paths.processor.MainzellisteClient + + + + + + + + readPatients + + + + + + + + + + DecideLocalOrCentralRequest + + + + + + + + + + + + MDS_\*_G-ID + resultIds + + + + + readPatientsCentrally + + + + + + + + + + createReadTokenCentral + + ¢ralPatientListUrl; + ¢ralPatientListApiKey; + true + + + + + + + + + + de.mainzelliste.paths.processor.MainzellisteReadTokenClient + + + resolveReadTokenCentral + + ¢ralPatientListUrl; + ¢ralPatientListApiKey; + + + + + + + + de.mainzelliste.paths.processor.MainzellisteReadClient + + + + + + readPatientsLocally + + + + + + + + + + createReadToken + + &localPatientListUrl; + &localPatientlistApiKey; + true + + + + + + + + + + de.mainzelliste.paths.processor.MainzellisteReadTokenClient + + + resolveReadToken + + &localPatientListUrl; + &localPatientlistApiKey; + + + + + + + + de.mainzelliste.paths.processor.MainzellisteReadClient + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + getId + + + + + DKTK000001950_&site;_L-ID + DKTK000001950_&site;_G-ID + + + + + DKTK000001950_USER + + + getId + + + + + DKTK000001951_&site;_L-ID + DKTK000001951_&site;_G-ID + + + + + DKTK000001951_USER + + + getId + + + + + DKTK000001985_&site;_L-ID + DKTK000001985_&site;_G-ID + + + + + DKTK000001985_USER + + + getId + + + + + DKTK000001986_&site;_L-ID + DKTK000001986_&site;_G-ID + + + + + DKTK000001986_USER + + + getId + + + + + DKTK999999999_&site;_L-ID + DKTK999999999_&site;_G-ID + + + + + DKTK999999999_USER + + + getId + + + + + DKTK000002089_&site;_L-ID + DKTK000002089_&site;_G-ID + + + + + DKTK000002089_USER + + + getId + &magicplApiKey; + + + readPatients + &magicplApiKeyConnector; + + + diff --git a/internal-configuration/mainzelliste.conf b/internal-configuration/mainzelliste.conf new file mode 100644 index 0000000..9a5e99c --- /dev/null +++ b/internal-configuration/mainzelliste.conf @@ -0,0 +1,227 @@ +# For a comprehensive documentation of configuration parameters see the configuration manual +# ("Konfigurationshandbuch"). + +# debug mode: If set to true, token handling is disabled (i.e. no tokens are necessary). +debug = false + +# Name of this mainzelliste distribution instance. +dist = Lokale Patientenliste der CCP-IT am Standort ML_SITE + +# Database setup +db.driver = ML_DB_DRIVER +db.url = jdbc:ML_DB_TYPE://ML_DB_HOST:ML_DB_PORT/ML_DB_NAME +db.username = ML_DB_USER +db.password = ML_DB_PASS + +# Logging +# By default, logs are written to stdout +gcp.audittrail = true + +# Field definitions +field.Vorname.type = PlainTextField +field.Nachname.type = PlainTextField +field.Fruehere_Namen.type = PlainTextField +field.Geburtstag.type = IntegerField +field.Geburtsmonat.type = IntegerField +field.Geburtsjahr.type = IntegerField +field.Staatsangehoerigkeit.type = PlainTextField +field.Geschlecht.type = PlainTextField +field.locallyUniqueId.type = PlainTextField + +# Server-side validation +validator.field.Vorname.required = true +validator.field.Nachname.required = true +validator.field.Geburtstag.required = true +validator.field.Geburtsmonat.required = true +validator.field.Geburtsjahr.required = true + +validator.field.Vorname.format = [A-Za-zäÄöÖüÜß\\.\\-' ]*[A-Za-zäÄöÖüÜß]+[A-Za-zäÄöÖüÜß\\.\\-' ]* +validator.field.Nachname.format = [A-Za-zäÄöÖüÜß\\.\\-' ]*[A-Za-zäÄöÖüÜß]+[A-Za-zäÄöÖüÜß\\.\\-' ]* +validator.field.Fruehere_Namen.format = [A-Za-zäÄöÖüÜß\\.\\-' ]*[A-Za-zäÄöÖüÜß]+[A-Za-zäÄöÖüÜß\\.\\-' ]* + +# Date validation: +validator.date.0.fields = Geburtstag, Geburtsmonat, Geburtsjahr +validator.date.0.format = ddMMyyyy + +# Exchange groups +# An exchange group denotes a set of fields whose values are considered interchangeable. +# The matcher (currently only EpilinkMatcher) compares the fields in each exchange group to +# all of its permutations and uses the best matching combination for final decision. +exchangeGroup.0 = Vorname, Nachname, Fruehere_Namen + +# Field transformations +# +# Transformations for a field {fieldname} are defined by +# field.{fieldname}.transformers = {transformerlist}, where +# transformerlist is a comma-separated list of names of subclasses of FieldTransformer. +field.Vorname.transformers = StringNormalizer, FirstNameDecomposer +field.Nachname.transformers = StringNormalizer, GermanLastNameDecomposer +field.Fruehere_Namen.transformers = StringNormalizer, GermanLastNameDecomposer + +# field comparators +field.Vorname.comparator = NGramComparator +field.Nachname.comparator = NGramComparator +field.Fruehere_Namen.comparator = NGramComparator + +field.Geburtstag.comparator = BinaryFieldComparator +field.Geburtsmonat.comparator = BinaryFieldComparator +field.Geburtsjahr.comparator = BinaryFieldComparator + +field.Staatsangehoerigkeit.comparator = BinaryFieldComparator +field.Geschlecht.comparator = BinaryFieldComparator + +# Matcher definition +# Property matcher defines the class name of the matcher to use, which must be a subclass +# of Matcher. Currently only EpilinkMatcher and NullMatcher are supported. +matcher = EpilinkMatcher + +# Config for Epilink Matcher +# +# matcher.epilink.{fieldname}.frequency: Defines the assumed mean frequency of values +# for field {fieldname}. This corresponds to the u-probability in the Fellegi-Sunter-Model +# and can be estimated by the reciprocal of the number of distinct values (i.e. the +# frequency of "month" is 1 / 12). +matcher.epilink.Vorname.frequency = 0.000235 +matcher.epilink.Nachname.frequency = 0.0000271 +matcher.epilink.Fruehere_Namen.frequency = 0.0000271 + +# 1 / 30 (approximate average number of days in a month) +matcher.epilink.Geburtstag.frequency = 0.0333 +# 1 / 12 +matcher.epilink.Geburtsmonat.frequency = 0.0833 +# example value: birth years for an age range of 35 years (40 - 75). +matcher.epilink.Geburtsjahr.frequency = 0.0286 + +# Berechnungsgrundlage: 193 anerkannte + 13 umstrittene Staaten (Quelle: Wikipedia) -> ca. 1 / 200 +matcher.epilink.Staatsangehoerigkeit.frequency = 0.005 +matcher.epilink.Geschlecht.frequency = 0.5 + +# matcher.epilink.{fieldname}.error_rate defines the assumed error_rate for +# field {fieldname} (1 - m-probability) +# +# The supplied number stem from the evaluation of a German cancer registry +matcher.epilink.Vorname.errorRate = 0.01 +matcher.epilink.Nachname.errorRate = 0.008 +matcher.epilink.Fruehere_Namen.errorRate = 0.008 +matcher.epilink.Geburtstag.errorRate = 0.005 +matcher.epilink.Geburtsmonat.errorRate = 0.002 +matcher.epilink.Geburtsjahr.errorRate = 0.004 +matcher.epilink.Staatsangehoerigkeit.errorRate = 0.04 +matcher.epilink.Geschlecht.errorRate = 0.04 + +# matcher.epilink.threshold.match: Defines the minimum weight (in the interval [0,1]) +# for which a pair of records is considered a definite match. +# +# matcher.epilink.threshold_non_match: Defines the weight (in the interval [0,1]) +# below which a pair of records is considered a definite non-match. Must be less than or equal +# to matcher.epilink.threshold_match. +# +# Record pairs with a weight w with threshold_non_match <= w < threshold_match are +# considered possible matches and lead to a tentative PID. +matcher.epilink.threshold_match = 0.95 +matcher.epilink.threshold_non_match = 0.65 + +# Blocking +# The blocking strategy choice depends on the field types after the transformation. +# If compound fields are passed to the blocker, whether created by a field transformation or as input fields, +# the type of the subfields is the one that is relevant for the blocking. +# If exchange groups are used, only one of the fieldnames should be given here. The blocker is applied to +# the other fields as well. + +# Use Soundex for PlainText fields +blocking.soundex.type = Soundex +blocking.soundex.fields = Vorname, Nachname, Fruehere_Namen + +# Use FieldEquality e.g. for birthday based blocking +# This can be combined with Soundex blocking. +blocking.dob.type = FieldEquality +blocking.dob.fields = Geburtstag, Geburtsmonat, Geburtsjahr + +# Servers (apiKey and permission definition) +servers.0.apiKey = ML_API_KEY +servers.0.permissions = createSession;showSessionIds;deleteAllPatients;createToken;tt_addPatient;tt_readPatients;tt_editPatient;readConfiguration +# TODO: Reduce to bridgeheads ip +servers.0.allowedRemoteAdresses = 0.0.0.0/0 + +# Allowed format of the callback +callback.allowedFormat = https://.* +callback.allowSelfsigned = false + +# IdGenerators +idgenerators = BK_ML_SITE_L-ID, MDS_ML_SITE_L-ID, BK_ML_SITE_G-ID, locallyUniqueId, DKTK000001985_ML_SITE_L-ID, DKTK000001985_ML_SITE_G-ID, DKTK000001986_ML_SITE_L-ID, DKTK000001986_ML_SITE_G-ID, DKTK000001950_ML_SITE_L-ID, DKTK000001950_ML_SITE_G-ID, DKTK000001951_ML_SITE_L-ID, DKTK000001951_ML_SITE_G-ID, DKTK999999999_ML_SITE_L-ID, DKTK999999999_ML_SITE_G-ID, DKTK000002089_ML_SITE_L-ID, DKTK000002089_ML_SITE_G-ID + +idgenerators.eagerGeneration = false + +# Configuration of the local and global BK Pseudonymes +idgenerator.BK_ML_SITE_L-ID = PIDGenerator +idgenerator.BK_ML_SITE_L-ID.k1 = ML_BK_IDGENERATOR_RANDOM_1 +idgenerator.BK_ML_SITE_L-ID.k2 = ML_BK_IDGENERATOR_RANDOM_2 +idgenerator.BK_ML_SITE_L-ID.k3 = ML_BK_IDGENERATOR_RANDOM_3 + +idgenerator.BK_ML_SITE_G-ID = ExternalIDGenerator + +# The unique id from the local datawarehouse +idgenerator.locallyUniqueId = ExternalIDGenerator + +# Configuration for the local mds-id +idgenerator.MDS_ML_SITE_L-ID = PIDGenerator +idgenerator.MDS_ML_SITE_L-ID.k1 = ML_MDS_IDGENERATOR_RANDOM_1 +idgenerator.MDS_ML_SITE_L-ID.k2 = ML_MDS_IDGENERATOR_RANDOM_2 +idgenerator.MDS_ML_SITE_L-ID.k3 = ML_MDS_IDGENERATOR_RANDOM_3 +idgenerator.MDS_ML_SITE_L-ID.eager = * +idgenerator.MDS_ML_SITE_L-ID.exportEncryption = mdsIdEncryption +# Encryption for export of mds +crypto.key.mdsIdPublicKey.type = RSA_PUBLIC +crypto.key.mdsIdPublicKey.uri = file:///run/secrets/centralSearchPublicKey +crypto.encryption.mdsIdEncryption.key = mdsIdPublicKey +crypto.encryption.mdsIdEncryption.type = RSA_ENCRYPT + +# Configuration of the study specific id-generators + +# Configuration for project DKTK000001985 +idgenerator.DKTK000001985_ML_SITE_L-ID = PIDGenerator +idgenerator.DKTK000001985_ML_SITE_L-ID.k1 = ML_DKTK000001985_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001985_ML_SITE_L-ID.k2 = ML_DKTK000001985_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001985_ML_SITE_L-ID.k3 = ML_DKTK000001985_IDGENERATOR_RANDOM_1 + +idgenerator.DKTK000001985_ML_SITE_G-ID = ExternalIDGenerator + +# Configuration for project DKTK000001986 +idgenerator.DKTK000001986_ML_SITE_L-ID = PIDGenerator +idgenerator.DKTK000001986_ML_SITE_L-ID.k1 = ML_DKTK000001986_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001986_ML_SITE_L-ID.k2 = ML_DKTK000001986_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001986_ML_SITE_L-ID.k3 = ML_DKTK000001986_IDGENERATOR_RANDOM_1 + +idgenerator.DKTK000001986_ML_SITE_G-ID = ExternalIDGenerator + +# Configuration for project DKTK000001950 +idgenerator.DKTK000001950_ML_SITE_L-ID = PIDGenerator +idgenerator.DKTK000001950_ML_SITE_L-ID.k1 = ML_DKTK000001950_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001950_ML_SITE_L-ID.k2 = ML_DKTK000001950_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001950_ML_SITE_L-ID.k3 = ML_DKTK000001950_IDGENERATOR_RANDOM_1 + +idgenerator.DKTK000001950_ML_SITE_G-ID = ExternalIDGenerator + +# Configuration for project DKTK000001951 +idgenerator.DKTK000001951_ML_SITE_L-ID = PIDGenerator +idgenerator.DKTK000001951_ML_SITE_L-ID.k1 = ML_DKTK000001951_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001951_ML_SITE_L-ID.k2 = ML_DKTK000001951_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000001951_ML_SITE_L-ID.k3 = ML_DKTK000001951_IDGENERATOR_RANDOM_1 + +idgenerator.DKTK000001951_ML_SITE_G-ID = ExternalIDGenerator + +# Configuration for project DKTK999999999 +idgenerator.DKTK999999999_ML_SITE_L-ID = PIDGenerator +idgenerator.DKTK999999999_ML_SITE_L-ID.k1 = ML_DKTK999999999_IDGENERATOR_RANDOM_1 +idgenerator.DKTK999999999_ML_SITE_L-ID.k2 = ML_DKTK999999999_IDGENERATOR_RANDOM_1 +idgenerator.DKTK999999999_ML_SITE_L-ID.k3 = ML_DKTK999999999_IDGENERATOR_RANDOM_1 + +idgenerator.DKTK999999999_ML_SITE_G-ID = ExternalIDGenerator + +# Configuration for project DKTK000002089 +idgenerator.DKTK000002089_ML_SITE_L-ID = PIDGenerator +idgenerator.DKTK000002089_ML_SITE_L-ID.k1 = ML_DKTK000002089_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000002089_ML_SITE_L-ID.k2 = ML_DKTK000002089_IDGENERATOR_RANDOM_1 +idgenerator.DKTK000002089_ML_SITE_L-ID.k3 = ML_DKTK000002089_IDGENERATOR_RANDOM_1 + +idgenerator.DKTK000002089_ML_SITE_G-ID = ExternalIDGenerator diff --git a/internal-configuration/nginx.conf b/internal-configuration/nginx.conf new file mode 100644 index 0000000..c225894 --- /dev/null +++ b/internal-configuration/nginx.conf @@ -0,0 +1,37 @@ +# Konfiguration for the bridgehead + server { + + # this is the internal Docker DNS, cache only for 30s + resolver 127.0.0.11 valid=30s; + + listen 80; + listen [::]:80; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $proxy_host; + proxy_set_header X-Forwarded-Proto $scheme; + + ### Defining the urls like this, will stop nginx from failing on startup if one service isn't started. + set $internal_connector_url http://connector:8080; + set $internal_idmanager_url http://idmanager:8080; + set $internal_patientlist_url http://patientlist:8080; + set $internal_ldm_url http://ldm:8080; + + location / { + proxy_pass $internal_connector_url; + } + + location /ID-Manager/ { + proxy_pass $internal_idmanager_url; + } + + location /Patientlist/ { + proxy_pass $internal_patientlist_url; + } + + location /LocalDatamanagement/ { + proxy_pass $internal_ldm_url; + } + } diff --git a/internal-configuration/proxy.xml b/internal-configuration/proxy.xml new file mode 100644 index 0000000..7c0ba23 --- /dev/null +++ b/internal-configuration/proxy.xml @@ -0,0 +1,21 @@ + + + + + HTTP_PROXY_URL + HTTP_PROXY_USERNAME + HTTP_PROXY_PASSWORD + + + HTTPS_PROXY_URL + HTTPS_PROXY_USERNAME + HTTPS_PROXY_PASSWORD + + + NO_PROXY_HOSTS + + + + \ No newline at end of file diff --git a/prerequisites.sh b/prerequisites.sh new file mode 100644 index 0000000..91d8a50 --- /dev/null +++ b/prerequisites.sh @@ -0,0 +1,49 @@ +#!/bin/bash + + +## Check if user is a su +echo "Welcome to the starting a bridgehead. We will get your instance up and running in no time" +echo "First we will check if all prerequisites are met ..." +prerequisites="git docker docker-compose" +for prerequisite in $prerequisites; do + $prerequisite --version 2>&1 + is_available=$? + if [ $is_available -gt 0 ]; then + log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" + exit 79 + fi + # TODO: Check for specific version +done +echo "All prerequisites are installed. Now we can download the bridgehead from https://code.mitro.dkfz.de/scm/docker/c4.bk.docker.git." + +#check if site.conf is created +if [ ! -f site.conf ]; then + echo "Please create your specific site.conf file from the site.dev.conf" + exit +fi + +#Load site specific variables +source site.conf + +if [ -z "$site_name" ]; then + echo "Site name is empty,please enter site name:" + read -r site_name + site_name_lowercase=$(echo "$site_name" | tr '[:upper:]' '[:lower:]') +fi + +## Download submodule +if [ ! -d "site-config" ]; then + echo "Please set up the site-config folder. Instruction are in the readme." + exit +else + echo "Site configuration is already loaded" +fi + +#Check if a project is selected +if [ -z "$project" ]; then + echo "No project selected! Please add a Project in your local site.conf." + exit +fi + +# Check for each project +echo "All prerequisites meet! All systems are ready to go!" From ed2f2ad590577cababe7e10f2f1cd6daa4ad09ec Mon Sep 17 00:00:00 2001 From: root Date: Tue, 21 Dec 2021 13:48:28 +0100 Subject: [PATCH 002/215] Added files for running a gbn project --- .gitignore | 10 ++++ convenience/bridgehead@.service | 15 ++--- convenience/traefik.service | 19 ------ gbn/docker-compose.yml | 4 +- gbn/gbn.env.dev | 2 + install-bridgehead.sh | 79 ++++++++++++------------ prerequisites.sh | 103 +++++++++++++++++--------------- site.dev.conf | 59 ++++++++++++++++++ 8 files changed, 170 insertions(+), 121 deletions(-) create mode 100644 .gitignore delete mode 100644 convenience/traefik.service create mode 100644 gbn/gbn.env.dev mode change 100644 => 100755 install-bridgehead.sh mode change 100644 => 100755 prerequisites.sh create mode 100644 site.dev.conf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d1362ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +##Ignore site configuration +.gitsubmodules +site-config + +## Ignore site configuration +config/**/* +!config/**/*.default +docker-compose.override.yml +site-specific.conf + diff --git a/convenience/bridgehead@.service b/convenience/bridgehead@.service index dd86c45..b4b5c9f 100644 --- a/convenience/bridgehead@.service +++ b/convenience/bridgehead@.service @@ -1,23 +1,16 @@ [Unit] Description=Bridgehead (%i) Service -Requires=traefik.service -After=traefik.service [Service] Restart=always RestartSec=30 -WorkingDirectory=/srv/docker/bridgehead +WorkingDirectory=/srv/docker/bridgehead/%i -ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/%i.env down -ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/%i.env rm -ExecStartPre=CONNECTOR_POSTGRES_PASSWORD=%i_CONNECTOR_POSTGRES_PASSWORD -ExecStartPre=export CONNECTOR_POSTGRES_PASSWORD -ExecStart=/usr/local/bin/docker-compose --env-file site-config/%i.env up -ExecStart=CONNECTOR_POSTGRES_PASSWORD= -ExecStart=export CONNECTOR_POSTGRES_PASSWORD +ExecStartPre=/usr/local/bin/docker-compose --env-file ../site-config/%i.env down +ExecStart=/usr/local/bin/docker-compose --env-file ../site-config/%i.env up -ExecStop=/usr/local/bin/docker-compose --env-file site-config/%i.env down +ExecStop=/usr/local/bin/docker-compose --env-file ../site-config/%i.env down [Install] WantedBy=multi-user.target diff --git a/convenience/traefik.service b/convenience/traefik.service deleted file mode 100644 index 8c52ee3..0000000 --- a/convenience/traefik.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Traefik Service -Requires=docker.service -After=docker.service - -[Service] -Restart=always -RestartSec=30 - -WorkingDirectory=/srv/docker/bridgehead - -ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/start.env down -ExecStartPre=/usr/local/bin/docker-compose --env-file site-config/start.env rm -ExecStart=/usr/local/bin/docker-compose --env-file site-config/start.env up - -ExecStop=/usr/local/bin/docker-compose --env-file site-config/start.env down - -[Install] -WantedBy=multi-user.target diff --git a/gbn/docker-compose.yml b/gbn/docker-compose.yml index abba8de..dbcf044 100644 --- a/gbn/docker-compose.yml +++ b/gbn/docker-compose.yml @@ -22,7 +22,7 @@ services: POSTGRES_HOST: "connector-db" POSTGRES_DB: "samply.connector" POSTGRES_USER: "samply" - POSTGRES_PASS: "samply" + POSTGRES_PASS: ${GBN_CONNECTOR_POSTGRES_PASS} STORE_URL: "http://store:8080/fhir" QUERY_LANGUAGE: "CQL" MDR_URL: "https://mdr.germanbiobanknode.de/v3/api/mdr" @@ -37,7 +37,7 @@ services: connector-db: container_name: "connector-db" - image: "postgres:9.6" + image: "postgres:10.17" environment: POSTGRES_USER: "samply" POSTGRES_PASSWORD: "samply" diff --git a/gbn/gbn.env.dev b/gbn/gbn.env.dev new file mode 100644 index 0000000..767700c --- /dev/null +++ b/gbn/gbn.env.dev @@ -0,0 +1,2 @@ +GBN_CONNECTOR_POSTGRES_PASS=samply + diff --git a/install-bridgehead.sh b/install-bridgehead.sh old mode 100644 new mode 100755 index 2ada9c6..1bbd7fe --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -1,40 +1,39 @@ -#!/bin/bash -### Note: Currently not complete, needs some features before useable for production - -./prerequisites.sh -source site.conf - -echo "Installing bridgehead" - -cd /etc/systemd/system/ - -echo "Installing bridgehead\@.service in systemd ..." -sudo cp /srv/docker/bridgehead/convenience/bridgehead\@.service ./ -echo "Installing bridgehead\@.update.service in systemd ..." -sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.service ./ -sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.timer ./ - -echo "Loading the bridgehead and traefik service definitions in systemd" -sudo systemctl daemon-reload - - -echo "Starting Project ${project} " - if [ ! -f "/etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf" ]; then - echo "Can't find local configuration file for bridgehead@${project} service. Please ensure that the file /etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf exists" - continue - fi - - sudo systemctl is-active --quiet bridgehead@"${project}" - if [ ! $? -eq 0 ]; then - echo "Starting bridgehead@${project} service ..." - sudo systemctl start bridgehead@"${project}" - echo "Enabling autostart of bridgehead@${project}.service" - sudo systemctl enable bridgehead@"${project}" - echo "Enabling nightly updates for bridgehead@${project}.service ..." - sudo systemctl enable --now bridgehead-update@"${project}".timer - fi -done - -# Switch back to execution directory; -cd - -# TODO: Configuration of the different modules +#!/bin/bash +### Note: Currently not complete, needs some features before useable for production + +./prerequisites.sh +source site.conf + +echo "Installing bridgehead" + +cd /etc/systemd/system/ + +echo "Installing bridgehead\@.service in systemd ..." +sudo cp /srv/docker/bridgehead/convenience/bridgehead\@.service ./ +echo "Installing bridgehead\@.update.service in systemd ..." +sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.service ./ +sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.timer ./ + +echo "Loading the bridgehead and traefik service definitions in systemd" +sudo systemctl daemon-reload + + +echo "Starting Project ${project} " + if [ ! -f "/etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf" ]; then + echo "Can't find local configuration file for bridgehead@${project} service. Please ensure that the file /etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf exists" + continue + fi + + sudo systemctl is-active --quiet bridgehead@"${project}" + if [ ! $? -eq 0 ]; then + echo "Starting bridgehead@${project} service ..." + sudo systemctl start bridgehead@"${project}" + echo "Enabling autostart of bridgehead@${project}.service" + sudo systemctl enable bridgehead@"${project}" + echo "Enabling nightly updates for bridgehead@${project}.service ..." + sudo systemctl enable --now bridgehead-update@"${project}".timer + fi + +# Switch back to execution directory; +cd - +# TODO: Configuration of the different modules diff --git a/prerequisites.sh b/prerequisites.sh old mode 100644 new mode 100755 index 91d8a50..438df45 --- a/prerequisites.sh +++ b/prerequisites.sh @@ -1,49 +1,54 @@ -#!/bin/bash - - -## Check if user is a su -echo "Welcome to the starting a bridgehead. We will get your instance up and running in no time" -echo "First we will check if all prerequisites are met ..." -prerequisites="git docker docker-compose" -for prerequisite in $prerequisites; do - $prerequisite --version 2>&1 - is_available=$? - if [ $is_available -gt 0 ]; then - log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" - exit 79 - fi - # TODO: Check for specific version -done -echo "All prerequisites are installed. Now we can download the bridgehead from https://code.mitro.dkfz.de/scm/docker/c4.bk.docker.git." - -#check if site.conf is created -if [ ! -f site.conf ]; then - echo "Please create your specific site.conf file from the site.dev.conf" - exit -fi - -#Load site specific variables -source site.conf - -if [ -z "$site_name" ]; then - echo "Site name is empty,please enter site name:" - read -r site_name - site_name_lowercase=$(echo "$site_name" | tr '[:upper:]' '[:lower:]') -fi - -## Download submodule -if [ ! -d "site-config" ]; then - echo "Please set up the site-config folder. Instruction are in the readme." - exit -else - echo "Site configuration is already loaded" -fi - -#Check if a project is selected -if [ -z "$project" ]; then - echo "No project selected! Please add a Project in your local site.conf." - exit -fi - -# Check for each project -echo "All prerequisites meet! All systems are ready to go!" +#!/bin/bash + + +## Check if user is a su +echo "Welcome to the starting a bridgehead. We will get your instance up and running in no time" +echo "First we will check if all prerequisites are met ..." +prerequisites="git docker docker-compose" +for prerequisite in $prerequisites; do + $prerequisite --version 2>&1 + is_available=$? + if [ $is_available -gt 0 ]; then + log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" + exit 79 + fi + # TODO: Check for specific version +done + +echo "Checking site.conf" + +#check if site.conf is created +if [ ! -f site.conf ]; then + echo "Please create your specific site.conf file from the site.dev.conf" + exit +fi + +#Load site specific variables +source site.conf + +if [ -z "$site_name" ]; then + echo "Please set site_name" +fi + +echo "Checking site-config module" + +## Download submodule +if [ ! -d "site-config" ]; then + echo "Please set up the site-config folder. Instruction are in the readme." + exit +else + echo "Site configuration is already loaded" +fi + +#Check if a project is selected +if [ -z "$project" ]; then + echo "No project selected! Please add a Project in your local site.conf." + exit +fi + +#check if project env is present +if [ -d "site-config/${project}.env" ]; then + echo "Please copy the tempalte from ${project} and put it in the site-config folder" +fi + +echo "All prerequisites meet! All systems are ready to go!" diff --git a/site.dev.conf b/site.dev.conf new file mode 100644 index 0000000..216e367 --- /dev/null +++ b/site.dev.conf @@ -0,0 +1,59 @@ +#!/bin/bash +### This is the configuration file for secrets, only your site should know +### Because the deployment via systemd is not possible in the current wsl environment, +### developers need to export the necessary variables. +### You can copy this file to site-specific.conf and run source site-specific.conf. +### After this docker-compose up should use the environment variables defined in this file. + +### Connector configuration +## The password the for your connector database +export dktk_CONNECTOR_POSTGRES_PASSWORD= +export c4_CONNECTOR_POSTGRES_PASSWORD= +export gbn_CONNECTOR_POSTGRES_PASSWORD= + +### Local Datamanagenment configuration if necessary +## Supply this, if you use samplystore +# export LDM_DATABASE_PASSWORD=this-should-only-be-used-by-developers-ldm-database-password; + +### ID-Management configuration +## provided by DKFZ with cooperation in Mainz +export CCP_PATIENTLISTE_APIKEY= +## provided by DKFZ with cooperation in Frankfurt +export CCP_CONTROLLNUMBERGENERATOR_APIKEY= +export LOCAL_IDMANAGER_LDM_APIKEY= +## Additional new ID-Manager Configuration +export LOCAL_IDMANAGER_MAINZELLISTE_APIKEY= +export LOCAL_IDMANAGER_CONNECTOR_APIKEY= +export LOCAL_PATIENTLIST_DBPASS= + +export dktk_CCP_OIDC_CLIENT_SECRET= +export c4_CCP_OIDC_CLIENT_SECRET= +### Mail Server username and password +# export MAIL_USER= +# export MAIL_PASSWORD= + +### Monitoring +## by default, the bridgehead will report it's state to DKFZ monitoring. Change here to opt-out. +# export MONITOR_OPTOUT=true + +### nNGM Configuration, only necessary if your site takes part in nNGM +## The cts login data, that healex provided you +# export NNGM_CTS_USER= +# export NNGM_CTS_PASSWORD= +## The idmanagement authentication data, that is provided by dkfz dataprotection office +# export NNGM_MAGICPL_APIKEY= +# export NNGM_MAINZELLISTE_APIKEY= + +### Workaround for local developers network environment +## Note: MSYS_NO_PATHCONV is needed, because git bash and msys2 will otherwise convert /etc/hosts to a windows path +#export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); +#export HOST=$(hostname) + +export HOSTIP= +export HOST= + +export site_name= +### Write the Project you want to start with the brigdehead +##Exmaple project=gbn +project= + From 71355348707b89fc33b3572f82d8f79cc2f88cd0 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Wed, 22 Dec 2021 13:10:33 +0100 Subject: [PATCH 003/215] Add uninstall sysxtemd units --- uninstall-bridgehead.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 uninstall-bridgehead.sh diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh new file mode 100644 index 0000000..628e8ed --- /dev/null +++ b/uninstall-bridgehead.sh @@ -0,0 +1,6 @@ +echo "Stoping systemd services and removing bridgehead" + +source site.conf + +systemctl stop bridgehead@"${project}".service +systemctl stop bridgehead-update@"${project}".timer From 2cf8fe282989daaaefc058c25fc5eacb060888d8 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Wed, 22 Dec 2021 13:15:42 +0100 Subject: [PATCH 004/215] Add update systemd job --- update-bridgehead.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 update-bridgehead.sh diff --git a/update-bridgehead.sh b/update-bridgehead.sh new file mode 100644 index 0000000..3f6e7a7 --- /dev/null +++ b/update-bridgehead.sh @@ -0,0 +1,42 @@ +#!/bin/bash +service="bridgehead" +# Creates logs +log() { + echo "$(date +'%Y-%m-%d %T')" "$1:" "$2" +} + +log "INFO" "Checking for updates of $service" +# check prerequisites +prerequisites="git docker docker-compose" +for prerequisite in $prerequisites; do + $prerequisite --version 2>&1 + is_available=$? + if [ $is_available -gt 0 ]; then + log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" + exit 79 + fi +done +# check if updates are available +old_git_hash="$(git rev-parse --verify HEAD)" +git fetch 2>&1 +git pull 2>&1 +new_git_hash="$(git rev-parse --verify HEAD)" +git_updated="false" +if [ "$old_git_hash" != "$new_git_hash" ]; then + log "INFO" "Pulled new changes from origin" + git_updated="true" +fi +docker_updated="false" +for image in $(docker ps --filter "name=$service" --format {{.Image}}); do + log "INFO" "Checking for Updates of Image: $image" + if docker pull $image | grep "Downloaded newer image"; then + log "INFO" "$image updated." + docker_updated="true" + fi +done +if [ $git_updated = "true" ] || [ $docker_updated = "true" ]; then + log "INFO" "Due to previous updates now restarting $service@$1" + systemctl restart "$service@$1.service" +fi +log "INFO" "checking updates finished" +exit 0 From 4f406a96c89f2e641c4ea6f73a3801171d92a832 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 22 Dec 2021 13:18:41 +0100 Subject: [PATCH 005/215] Reformated files with unix coding --- uninstall-bridgehead.sh | 12 +++--- update-bridgehead.sh | 84 ++++++++++++++++++++--------------------- 2 files changed, 48 insertions(+), 48 deletions(-) mode change 100644 => 100755 uninstall-bridgehead.sh mode change 100644 => 100755 update-bridgehead.sh diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh old mode 100644 new mode 100755 index 628e8ed..6425f49 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -1,6 +1,6 @@ -echo "Stoping systemd services and removing bridgehead" - -source site.conf - -systemctl stop bridgehead@"${project}".service -systemctl stop bridgehead-update@"${project}".timer +echo "Stoping systemd services and removing bridgehead" + +source site.conf + +systemctl stop bridgehead@"${project}".service +systemctl stop bridgehead-update@"${project}".timer diff --git a/update-bridgehead.sh b/update-bridgehead.sh old mode 100644 new mode 100755 index 3f6e7a7..11b1393 --- a/update-bridgehead.sh +++ b/update-bridgehead.sh @@ -1,42 +1,42 @@ -#!/bin/bash -service="bridgehead" -# Creates logs -log() { - echo "$(date +'%Y-%m-%d %T')" "$1:" "$2" -} - -log "INFO" "Checking for updates of $service" -# check prerequisites -prerequisites="git docker docker-compose" -for prerequisite in $prerequisites; do - $prerequisite --version 2>&1 - is_available=$? - if [ $is_available -gt 0 ]; then - log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" - exit 79 - fi -done -# check if updates are available -old_git_hash="$(git rev-parse --verify HEAD)" -git fetch 2>&1 -git pull 2>&1 -new_git_hash="$(git rev-parse --verify HEAD)" -git_updated="false" -if [ "$old_git_hash" != "$new_git_hash" ]; then - log "INFO" "Pulled new changes from origin" - git_updated="true" -fi -docker_updated="false" -for image in $(docker ps --filter "name=$service" --format {{.Image}}); do - log "INFO" "Checking for Updates of Image: $image" - if docker pull $image | grep "Downloaded newer image"; then - log "INFO" "$image updated." - docker_updated="true" - fi -done -if [ $git_updated = "true" ] || [ $docker_updated = "true" ]; then - log "INFO" "Due to previous updates now restarting $service@$1" - systemctl restart "$service@$1.service" -fi -log "INFO" "checking updates finished" -exit 0 +#!/bin/bash +service="bridgehead" +# Creates logs +log() { + echo "$(date +'%Y-%m-%d %T')" "$1:" "$2" +} + +log "INFO" "Checking for updates of $service" +# check prerequisites +prerequisites="git docker docker-compose" +for prerequisite in $prerequisites; do + $prerequisite --version 2>&1 + is_available=$? + if [ $is_available -gt 0 ]; then + log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" + exit 79 + fi +done +# check if updates are available +old_git_hash="$(git rev-parse --verify HEAD)" +git fetch 2>&1 +git pull 2>&1 +new_git_hash="$(git rev-parse --verify HEAD)" +git_updated="false" +if [ "$old_git_hash" != "$new_git_hash" ]; then + log "INFO" "Pulled new changes from origin" + git_updated="true" +fi +docker_updated="false" +for image in $(docker ps --filter "name=$service" --format {{.Image}}); do + log "INFO" "Checking for Updates of Image: $image" + if docker pull $image | grep "Downloaded newer image"; then + log "INFO" "$image updated." + docker_updated="true" + fi +done +if [ $git_updated = "true" ] || [ $docker_updated = "true" ]; then + log "INFO" "Due to previous updates now restarting $service@$1" + systemctl restart "$service@$1.service" +fi +log "INFO" "checking updates finished" +exit 0 From f9f72cbe9aea505e14e29ce9696a9327807726b8 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 23 Dec 2021 14:54:35 +0100 Subject: [PATCH 006/215] Added readme and improved systemd scripts --- README.md | 22 +++++++++++++++++- c4/docker-compose.yml | 49 +++++------------------------------------ dktk/docker-compose.yml | 9 ++++---- gbn/docker-compose.yml | 6 ++--- install-bridgehead.sh | 4 ++-- uninstall-bridgehead.sh | 8 +++++++ 6 files changed, 45 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 4967df5..e8ebc44 100644 --- a/README.md +++ b/README.md @@ -1 +1,21 @@ -# bridgehead \ No newline at end of file +# bridgehead + +This repository contains all information and tools to deploy a bridgehead. If you have any questions about deploying a bridgehead, please contact us. + +There are some prerequisites, which need to be meet befor starting a bridgehead. + +The first step is to copy the site.conf file. It contains some configuration and secrets for your bridgehead. + +With cp site.dev.conf site.conf you can clone the template. You need to set the project accoriding to the which bridgehead you want to start. It's either a GBN/BBMRI-ERIC, DKTK or C4 Bridgehead. + +The next step is creating a configuration for your bridghead. We can provide you a configuration git repository for bridgehead, just ask us. When you have a DKTK or C4 Bridgehead you propably need some configuration from us anyways. + +git submodule add -f https://"$git_username":"$git_access_token"@code.mitro.dkfz.de/scm/bd/"$site_name_lowercase"-config.git ./site-config + +If you want to manage the configuration your self you need to copy a env file from the respective project folder into a site-config folder. + +Step 3 is determind your prefered start method. You can can just start the docker container with start-bridgehead and stop it. Alternatively, we advice to use the install script. This script add a systemd service to your system which starts the bridgehead, it also stops and starts the system in the event of a reboot. Also it contains a service for automatic updating your bridgehead. It will check at 3:00 am if there are any updates and will apply them. + +Step 4 is checking your bridgehead. + + diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index 8c4e7fd..ea62c67 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -21,8 +21,8 @@ secrets: services: connector: - container_name: c4_connector - image: samply/share-client:${PROJECT}-develop + container_name: bridgehead_c4_connector + image: "samply/share-client:c4-7" environment: SITE: ${SITE} SITEID: ${SITEID} @@ -58,6 +58,7 @@ services: - "${HOST}:${HOSTIP}" connector_db: + container_name: bridgehead_connector_db image: postgres:10.17 environment: POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} @@ -73,7 +74,7 @@ services: ## ID-Management idmanager: - container_name: idmanager + container_name: bridehead_idmanager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST}/ID-Manager @@ -104,47 +105,8 @@ services: depends_on: - patientlist - patientlist: - container_name: patientlist - image: medicalinformatics/mainzelliste:develop - environment: - TOMCAT_REVERSEPROXY_FQDN: ${HOST}/Patientlist - TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} - TOMCAT_REVERSEPROXY_PORT: ${PORT} - ML_DB_DRIVER: org.postgresql.Driver - ML_DB_TYPE: postgresql - ML_DB_HOST: patientlist_db - ML_DB_PORT: 5432 - ML_DB_NAME: ${ML_DB_NAME} - ML_DB_USER: ${ML_DB_USER} - ML_DB_PASS: ${LOCAL_PATIENTLIST_DBPASS} - ML_API_KEY: ${LOCAL_IDMANAGER_MAINZELLISTE_APIKEY} - ML_LOG_LEVEL: warning - ML_SITE: ${SITEID} - TZ: Europe/Berlin - env_file: - - ./site-config/patientlist.env - volumes: - - "patientlist_logs:/usr/local/tomcat/logs" - secrets: - - mainzelliste.docker.conf - - centralSearchPublicKey - depends_on: - - patientlist_db - - patientlist_db: - container_name: patientlist_db - image: postgres:13.1-alpine - environment: - POSTGRES_DB: ${ML_DB_NAME} - POSTGRES_USER: ${ML_DB_USER} - POSTGRES_PASSWORD: ${C4_LOCAL_PATIENTLIST_DBPASS} - TZ: Europe/Berlin - volumes: - - "patientlist_db_data:/var/lib/postgresql/data" - networks: - - idmanagement store: + container_name: bridgehead_store image: docker.verbis.dkfz.de/ccp/samply.store:release-5.1.2 environment: MDR_URL: ${CCP_MDR_URL} @@ -163,6 +125,7 @@ store: restart: always store_db: + container_name: bridgehead_store_db image: postgres:10.17 command: postgres -c datestyle='iso, dmy' environment: diff --git a/dktk/docker-compose.yml b/dktk/docker-compose.yml index bc57de6..cd379a2 100644 --- a/dktk/docker-compose.yml +++ b/dktk/docker-compose.yml @@ -19,7 +19,7 @@ secrets: services: connector: - container_name: dktk_connector + container_name: bridgehead_dktk_connector image: samply/share-client:dktk-develop environment: SITE: ${SITE} @@ -57,6 +57,7 @@ services: - "${HOST}:${HOSTIP}" connector_db: + container_name: bridgead_connector_db image: postgres:10.17 environment: POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} @@ -72,7 +73,7 @@ services: ## ID-Management idmanager: - container_name: idmanager + container_name: bridgehead_idmanager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST}/ID-Manager @@ -104,7 +105,7 @@ services: - patientlist patientlist: - container_name: patientlist + container_name: bridgehead_patientlist image: medicalinformatics/mainzelliste:develop environment: TOMCAT_REVERSEPROXY_FQDN: ${HOST}/Patientlist @@ -132,7 +133,7 @@ services: - patientlist_db patientlist_db: - container_name: patientlist_db + container_name: bridgehead_patientlist_db image: postgres:13.1-alpine environment: POSTGRES_DB: ${ML_DB_NAME} diff --git a/gbn/docker-compose.yml b/gbn/docker-compose.yml index dbcf044..e97961b 100644 --- a/gbn/docker-compose.yml +++ b/gbn/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.4' services: store: - container_name: "store" + container_name: "bridgehead_blaze_store" image: "samply/blaze:0.15" environment: BASE_URL: "http://store:8080" @@ -16,7 +16,7 @@ services: restart: "always" connector: - container_name: "connector" + container_name: "bridgehead_connector" image: "samply/share-client:gbn-7" environment: POSTGRES_HOST: "connector-db" @@ -36,7 +36,7 @@ services: restart: "always" connector-db: - container_name: "connector-db" + container_name: "bridgehead_connector-db" image: "postgres:10.17" environment: POSTGRES_USER: "samply" diff --git a/install-bridgehead.sh b/install-bridgehead.sh index 1bbd7fe..0324a60 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -14,14 +14,14 @@ echo "Installing bridgehead\@.update.service in systemd ..." sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.service ./ sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.timer ./ -echo "Loading the bridgehead and traefik service definitions in systemd" +echo "Loading the bridgehead definitions in systemd" sudo systemctl daemon-reload echo "Starting Project ${project} " if [ ! -f "/etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf" ]; then echo "Can't find local configuration file for bridgehead@${project} service. Please ensure that the file /etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf exists" - continue + exit fi sudo systemctl is-active --quiet bridgehead@"${project}" diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh index 6425f49..a8f7b78 100755 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -4,3 +4,11 @@ source site.conf systemctl stop bridgehead@"${project}".service systemctl stop bridgehead-update@"${project}".timer +systemctl stop bridgehead-update@"${project}".service + +cd /etc/systemd/system/ +rm bridgehead\@.service +rm bridgehead-update\@.timer +rm bridgehead-update\@.service + +cd - From f8fde8197e90c5d1a671ae645fbb61125cf87298 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Mon, 27 Dec 2021 11:16:27 +0100 Subject: [PATCH 007/215] Updated Readme and c4 docker file --- LICENSE | 402 +++++++++++++++++++++--------------------- README.md | 23 ++- c4/docker-compose.yml | 20 +-- gbn/gbn.env.dev | 64 ++++++- 4 files changed, 296 insertions(+), 213 deletions(-) diff --git a/LICENSE b/LICENSE index 261eeb9..29f81d8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,201 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index e8ebc44..fae934e 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,33 @@ This repository contains all information and tools to deploy a bridgehead. If you have any questions about deploying a bridgehead, please contact us. -There are some prerequisites, which need to be meet befor starting a bridgehead. +There are some prerequisites, which need to be meet befor starting a bridgehead. If you runnig a Windows or Mac OS maschine you should read starting a bridgehead. If you running a Linux maschine you can do start and install a bridgehead. + +## Starting + + +## Install + +Create the dir /src/docker and clone the repository. The first step is to copy the site.conf file. It contains some configuration and secrets for your bridgehead. With cp site.dev.conf site.conf you can clone the template. You need to set the project accoriding to the which bridgehead you want to start. It's either a GBN/BBMRI-ERIC, DKTK or C4 Bridgehead. +In this file you can put important varibales + +### DKTK + +### GBA/BBMRI-ERIC + +### C4 + +The following vairbales need to be set + +C4_SAMPLY_STORE_PASS +C4_CONNECTOR_POSTGRES_PASSWORD + + The next step is creating a configuration for your bridghead. We can provide you a configuration git repository for bridgehead, just ask us. When you have a DKTK or C4 Bridgehead you propably need some configuration from us anyways. git submodule add -f https://"$git_username":"$git_access_token"@code.mitro.dkfz.de/scm/bd/"$site_name_lowercase"-config.git ./site-config diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index ea62c67..dd60c32 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -1,12 +1,12 @@ version: "3.7" volumes: - connector_db_data: - connector_logs: - patientlist_db_data: - patientlist_logs: - idmanager_logs: - store_db_data: - store_logs: + connector_db_data: null + connector_logs: null + patientlist_db_data: null + patientlist_logs: null + idmanager_logs: null + store_db_data: null + store_logs: null secrets: mainzelliste.docker.conf: file: ./internal-configuration/mainzelliste.conf @@ -20,7 +20,7 @@ secrets: file: ./internal-configuration/proxy.xml services: - connector: + connector: container_name: bridgehead_c4_connector image: "samply/share-client:c4-7" environment: @@ -56,7 +56,7 @@ services: extra_hosts: - "host.docker.internal:host-gateway" - "${HOST}:${HOSTIP}" - + connector_db: container_name: bridgehead_connector_db image: postgres:10.17 @@ -105,7 +105,7 @@ services: depends_on: - patientlist -store: + store: container_name: bridgehead_store image: docker.verbis.dkfz.de/ccp/samply.store:release-5.1.2 environment: diff --git a/gbn/gbn.env.dev b/gbn/gbn.env.dev index 767700c..04cc72e 100644 --- a/gbn/gbn.env.dev +++ b/gbn/gbn.env.dev @@ -1,2 +1,64 @@ -GBN_CONNECTOR_POSTGRES_PASS=samply +SITE= +SITEID= +TESTPROD= +project=gbn + +### Bridgehead Operator +# users may see this contact data for support purposes +OPERATOR_FIRST_NAME= +OPERATOR_LAST_NAME= +OPERATOR_EMAIL= +OPERATOR_PHONE= + +### Proxy configuration +## ex.: http://my-proxy-host:my-proxy-port +HTTP_PROXY_HOST= +HTTP_PROXY_USER= +HTTP_PROXY_PASSWORD= +HTTPS_PROXY_HOST= +HTTPS_PROXY_USER= +HTTPS_PROXY_PASSWORD= + +HTTP_PROXY= +HTTPS_PROXY= +http_proxy= +https_proxy= + +NO_PROXY= +no_proxy= + +### SMTP Server +## ex.: mailhost.intern.klinik.de +# MAIL_HOST= +# MAIL_PORT=25 +## ex.: no-reply@bridgehead.intern.klinik.de +# MAIL_FROM_ADDRESS= +# MAIL_FROM_NAME=Bridgehead Searchbroker + +## Connector +CONNECTOR_POSTGRES_HOST= +CONNECTOR_POSTGRES_PORT= +CONNECTOR_POSTGRES_DB= +CONNECTOR_POSTGRES_USER= +CONNECTOR_SHARE_URL= +CONNECTOR_ENABLE_METRICS= +CONNECTOR_MONITOR_INTERVAL= +CONNECTOR_UPDATE_SERVER= +CONNECTOR_QUERY_LANGUAGE= + +CATALINA_OPTS= +MDR_URL= +CENTRAL_SEARCH= +DECENTRAL_SEARCH= +MONITOR_URL= +MONITOR_OPTOUT= + +LOG_LEVEL= + +feature_BBMRI_DIRECTORY_SYNC=false +feature_DKTK_CENTRAL_SEARCH=false +feature_NNGM_CTS=false + +LDM_URL= +ID_MANAGER_URL= \ No newline at end of file From 37d3d3f2036071ac302e16f5b54b055701475530 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Mon, 27 Dec 2021 13:16:11 +0100 Subject: [PATCH 008/215] Fixed missing values for c4 --- c4/docker-compose.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index dd60c32..d97e379 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -74,27 +74,15 @@ services: ## ID-Management idmanager: - container_name: bridehead_idmanager + container_name: bridgehead_idmanager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 environment: - TOMCAT_REVERSEPROXY_FQDN: ${HOST}/ID-Manager - TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} - TOMCAT_REVERSEPROXY_PORT: ${PORT} - MAGICPL_MAINZELLISTE_URL: http://patientlist:8080/Patientlist - MAGICPL_MAINZELLISTE_API_KEY: ${LOCAL_IDMANAGER_MAINZELLISTE_APIKEY} - MAGICPL_API_KEY: ${LOCAL_IDMANAGER_LDM_APIKEY} - MAGICPL_API_KEY_CONNECTOR: ${LOCAL_IDMANAGER_CONNECTOR_APIKEY} - MAGICPL_PASSPHRASE: notUsedInThisConfigurationButMandatory - ### Configuration for communication with central identity management MAGICPL_MAINZELLISTE_CENTRAL_URL: ${CCP_PATIENTLIST_URL} - MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${CCP_PATIENTLISTE_APIKEY} MAGICPL_CENTRAL_URL: ${CCP_CONTROLLNUMBERGENERATOR_URL} - MAGICPL_CENTRAL_API_KEY: ${CCP_CONTROLLNUMBERGENERATOR_APIKEY} - MAGICPL_OIDC_PROVIDER: ${CCP_OIDC_PROVIDER_URL}/oauth2 - MAGICPL_OIDC_CLIENT_ID: ${CCP_OIDC_CLIENT_ID} - MAGICPL_OIDC_CLIENT_SECRET: ${CCP_OIDC_CLIENT_SECRET} MAGICPL_SITE: ${SITEID} MAGICPL_LOG_LEVEL: info + GLOBAL_ID: DKTK + TZ: Europe/Berlin volumes: - "idmanager_logs:/usr/local/tomcat/logs" @@ -102,8 +90,6 @@ services: - magicpl.docker.xml - dktk_bridgehead_info.docker.xml - proxy.docker.xml - depends_on: - - patientlist store: container_name: bridgehead_store From 3bece857178f07b8a55b55e184d716cfe61162d3 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Mon, 27 Dec 2021 13:24:17 +0100 Subject: [PATCH 009/215] Added start and stop bridgehead scripts --- start-bridgehead.sh | 15 +++++++++++++++ stop-bridgehead.sh | 9 +++++++++ 2 files changed, 24 insertions(+) create mode 100644 start-bridgehead.sh create mode 100644 stop-bridgehead.sh diff --git a/start-bridgehead.sh b/start-bridgehead.sh new file mode 100644 index 0000000..124be38 --- /dev/null +++ b/start-bridgehead.sh @@ -0,0 +1,15 @@ +#!/bin/bash +### Note: Currently not complete, needs some features before useable for production + +./prerequisites.sh +source site.conf + +echo "Starting bridgehead" + +cd ${project} + +docker-compose --env-file ../site-config/${project}.env up + +cd .. + +echo "The bridgehead should be in online in a few seconds" diff --git a/stop-bridgehead.sh b/stop-bridgehead.sh new file mode 100644 index 0000000..6d39480 --- /dev/null +++ b/stop-bridgehead.sh @@ -0,0 +1,9 @@ +echo "Stoping bridgehead" + +source site.conf + +cd ${project} + +docker-compose --env-file ../site-config/{project}.env down + +cd .. From c23c1a8d3d62aeb4c1ec9e53d0b55cc9aabcebe7 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 27 Dec 2021 14:41:54 +0100 Subject: [PATCH 010/215] Fixed start and stop and c4 deployment --- c4/docker-compose.yml | 35 ++++++++++++++++++----------------- start-bridgehead.sh | 30 +++++++++++++++--------------- stop-bridgehead.sh | 18 +++++++++--------- 3 files changed, 42 insertions(+), 41 deletions(-) mode change 100644 => 100755 start-bridgehead.sh mode change 100644 => 100755 stop-bridgehead.sh diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index d97e379..0e649bc 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -9,15 +9,15 @@ volumes: store_logs: null secrets: mainzelliste.docker.conf: - file: ./internal-configuration/mainzelliste.conf + file: ../internal-configuration/mainzelliste.conf magicpl.docker.xml: - file: ./internal-configuration/magicpl.xml + file: ../internal-configuration/magicpl.xml dktk_bridgehead_info.docker.xml: - file: ./internal-configuration/bridgehead-common.xml + file: ../internal-configuration/bridgehead-common.xml centralSearchPublicKey: - file: ./internal-configuration/centralSearchPublicKey.der + file: ../internal-configuration/centralSearchPublicKey.der proxy.docker.xml: - file: ./internal-configuration/proxy.xml + file: ../internal-configuration/proxy.xml services: connector: @@ -28,7 +28,7 @@ services: SITEID: ${SITEID} CENTRAL_SEARCH: ${CCP_CENTRALSEARCH_URL} DECENTRAL_SEARCH: ${CCP_DECENTRALSEARCH_URL} - MDR_URL: ${CCP_MDR_URL} + MDR_URL: ${MDR_URL} MONITOR_URL: ${CCP_MONITOR_URL} SHARE_URL: "${PROTOCOL}://${HOST}:${PORT}" ID_MANAGER_URL: ${ID_MANAGER_URL} @@ -36,7 +36,7 @@ services: PATIENTLIST_URL: ${PATIENTLIST_URL} STORE_URL: ${LDM_URL} POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_PORT: 5432 + POSTGRES_PORT: ${CONNECTOR_DB_PORT} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASS: ${C4_CONNECTOR_POSTGRES_PASSWORD} @@ -52,11 +52,11 @@ services: - "connector_logs:/usr/local/tomcat/logs" depends_on: - connector_db + ports: + - "8082:8080" + - "65496:65395" restart: always - extra_hosts: - - "host.docker.internal:host-gateway" - - "${HOST}:${HOSTIP}" - + connector_db: container_name: bridgehead_connector_db image: postgres:10.17 @@ -68,24 +68,22 @@ services: volumes: - "connector_db_data:/var/lib/postgresql/data" restart: always - extra_hosts: - - "host.docker.internal:host-gateway" - - "${HOST}:${HOSTIP}" ## ID-Management idmanager: container_name: bridgehead_idmanager - image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 + image: docker.verbis.dkfz.de/ccp/idmanager:deploy-c4 environment: MAGICPL_MAINZELLISTE_CENTRAL_URL: ${CCP_PATIENTLIST_URL} MAGICPL_CENTRAL_URL: ${CCP_CONTROLLNUMBERGENERATOR_URL} MAGICPL_SITE: ${SITEID} MAGICPL_LOG_LEVEL: info GLOBAL_ID: DKTK - TZ: Europe/Berlin volumes: - "idmanager_logs:/usr/local/tomcat/logs" + ports: + - "8084:8080" secrets: - magicpl.docker.xml - dktk_bridgehead_info.docker.xml @@ -98,7 +96,7 @@ services: MDR_URL: ${CCP_MDR_URL} MDR_NAMESPACE: adt,dktk,marker MDR_VALIDATION: "false" - POSTGRES_HOST: store_db + POSTGRES_HOST: bridgehead_store_db POSTGRES_PORT: 5432 POSTGRES_DB: samplystore POSTGRES_USER: samplystore @@ -108,6 +106,9 @@ services: - "store_logs:/usr/local/tomcat/logs" depends_on: - store_db + ports: + - "8083:8080" + - "65495:65395" restart: always store_db: diff --git a/start-bridgehead.sh b/start-bridgehead.sh old mode 100644 new mode 100755 index 124be38..de34193 --- a/start-bridgehead.sh +++ b/start-bridgehead.sh @@ -1,15 +1,15 @@ -#!/bin/bash -### Note: Currently not complete, needs some features before useable for production - -./prerequisites.sh -source site.conf - -echo "Starting bridgehead" - -cd ${project} - -docker-compose --env-file ../site-config/${project}.env up - -cd .. - -echo "The bridgehead should be in online in a few seconds" +#!/bin/bash +### Note: Currently not complete, needs some features before useable for production + +./prerequisites.sh +source site.conf + +echo "Starting bridgehead" + +cd ${project} + +docker-compose --env-file ../site-config/${project}.env up -d + +cd .. + +echo "The bridgehead should be in online in a few seconds" diff --git a/stop-bridgehead.sh b/stop-bridgehead.sh old mode 100644 new mode 100755 index 6d39480..8ab4eff --- a/stop-bridgehead.sh +++ b/stop-bridgehead.sh @@ -1,9 +1,9 @@ -echo "Stoping bridgehead" - -source site.conf - -cd ${project} - -docker-compose --env-file ../site-config/{project}.env down - -cd .. +echo "Stoping bridgehead" + +source site.conf + +cd ${project} + +docker-compose --env-file ../site-config/${project}.env down + +cd .. From ca7e3a641a2a24ca59fa87a15e5c7e495d13cb1c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Jan 2022 15:26:34 +0100 Subject: [PATCH 011/215] Added setting bridgehead path to enviroment --- c4/docker-compose.yml | 102 +++++++++++++++++++++++++++++----------- dktk/docker-compose.yml | 96 ++++++++++++++++++------------------- gbn/docker-compose.yml | 14 +++--- install-bridgehead.sh | 19 ++++++-- start-bridgehead.sh | 5 +- uninstall-bridgehead.sh | 3 ++ 6 files changed, 149 insertions(+), 90 deletions(-) diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index 0e649bc..5952c20 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -28,7 +28,7 @@ services: SITEID: ${SITEID} CENTRAL_SEARCH: ${CCP_CENTRALSEARCH_URL} DECENTRAL_SEARCH: ${CCP_DECENTRALSEARCH_URL} - MDR_URL: ${MDR_URL} + MDR_URL: ${CCP_MDR_URL} MONITOR_URL: ${CCP_MONITOR_URL} SHARE_URL: "${PROTOCOL}://${HOST}:${PORT}" ID_MANAGER_URL: ${ID_MANAGER_URL} @@ -39,8 +39,7 @@ services: POSTGRES_PORT: ${CONNECTOR_DB_PORT} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} - POSTGRES_PASS: ${C4_CONNECTOR_POSTGRES_PASSWORD} - DEPLOYMENT_CONTEXT: ${COMPOSE_PROJECT_NAME}-connector + POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} HTTP_PROXY_URL: ${HTTP_PROXY_URL} HTTP_PROXY_USERNAME: ${HTTP_PROXY_USERNAME} HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} @@ -53,54 +52,104 @@ services: depends_on: - connector_db ports: - - "8082:8080" - - "65496:65395" + - "8080:8080" restart: always connector_db: container_name: bridgehead_connector_db image: postgres:10.17 environment: + POSTGRES_HOST: ${CONNECTOR_POSTGRES_HOST} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} - POSTGRES_PASSWORD: ${C4_CONNECTOR_POSTGRES_PASSWORD} + POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} TZ: Europe/Berlin volumes: - "connector_db_data:/var/lib/postgresql/data" restart: always - ## ID-Management + idmanager: container_name: bridgehead_idmanager - image: docker.verbis.dkfz.de/ccp/idmanager:deploy-c4 + image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 environment: - MAGICPL_MAINZELLISTE_CENTRAL_URL: ${CCP_PATIENTLIST_URL} - MAGICPL_CENTRAL_URL: ${CCP_CONTROLLNUMBERGENERATOR_URL} - MAGICPL_SITE: ${SITEID} + MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} + MAGICPL_MAINZELLISTE_API_KEY: ${MAGICPL_MAINZELLISTE_API_KEY} + MAGICPL_API_KEY: ${MAGICPL_API_KEY} + MAGICPL_API_KEY_CONNECTOR: ${MAGICPL_API_KEY_CONNECTOR} + MAGICPL_PASSPHRASE: ${MAGICPL_PASSPHRASE} + MAGICPL_MAINZELLISTE_CENTRAL_URL: ${MAGICPL_MAINZELLISTE_CENTRAL_URL} + MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${MAGICPL_MAINZELLISTE_CENTRAL_API_KEY} + MAGICPL_CENTRAL_URL: ${MAGICPL_CENTRAL_URL} + MAGICPL_CENTRAL_API_KEY: ${MAGICPL_CENTRAL_API_KEY} + MAGICPL_OIDC_PROVIDER: ${MAGICPL_OIDC_PROVIDER} + MAGICPL_OIDC_CLIENT_ID: ${MAGICPL_OIDC_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${MAGICPL_OIDC_CLIENT_SECRET} + MAGICPL_SITE: adt MAGICPL_LOG_LEVEL: info - GLOBAL_ID: DKTK TZ: Europe/Berlin volumes: - "idmanager_logs:/usr/local/tomcat/logs" - ports: - - "8084:8080" secrets: - magicpl.docker.xml - dktk_bridgehead_info.docker.xml - proxy.docker.xml + depends_on: + - patientlist + ports: + - "8085:8080" + + patientlist: + container_name: bridgehead_patientlist + image: medicalinformatics/mainzelliste:develop + environment: + ML_DB_DRIVER: org.postgresql.Driver + ML_DB_TYPE: postgresql + ML_DB_HOST: ${ML_DB_HOST} + ML_DB_PORT: ${ML_DB_PORT} + ML_DB_NAME: ${ML_DB_NAME} + ML_DB_USER: ${ML_DB_USER} + ML_DB_PASS: ${ML_DB_PASS} + ML_API_KEY: ${ML_API_KEY} + ML_LOG_LEVEL: warning + ML_SITE: adt + TZ: Europe/Berlin + env_file: + - ./site-config/patientlist.env + volumes: + - "patientlist_logs:/usr/local/tomcat/logs" + secrets: + - mainzelliste.docker.conf + - centralSearchPublicKey + depends_on: + - patientlist_db + ports: + - "8086:8080" + + patientlist_db: + container_name: bridgehead_patientlist_db + image: postgres:13.1-alpine + environment: + POSTGRES_HOST: ${ML_DB_HOST} + POSTGRES_DB: ${ML_DB_NAME} + POSTGRES_USER: ${ML_DB_USER} + POSTGRES_PASSWORD: ${ML_DB_PASS} + TZ: Europe/Berlin + volumes: + - "patientlist_db_data:/var/lib/postgresql/data" store: container_name: bridgehead_store image: docker.verbis.dkfz.de/ccp/samply.store:release-5.1.2 environment: MDR_URL: ${CCP_MDR_URL} - MDR_NAMESPACE: adt,dktk,marker - MDR_VALIDATION: "false" - POSTGRES_HOST: bridgehead_store_db - POSTGRES_PORT: 5432 - POSTGRES_DB: samplystore - POSTGRES_USER: samplystore - POSTGRES_PASSWORD: ${C4_SAMPLY_STORE_PASS} + MDR_NAMESPACE: ${MDR_NAMESPACE} + MDR_VALIDATION: ${MDR_VALIDATION} + POSTGRES_HOST: ${STORE_POSTGRES_HOST} + POSTGRES_PORT: ${STORE_POSTGRES_PORT} + POSTGRES_DB: ${STORE_POSTGRES_DB} + POSTGRES_USER: ${STORE_POSTGRES_USER} + POSTGRES_PASSWORD: ${STORE_POSTGRES_PASS} TZ: Europe/Berlin volumes: - "store_logs:/usr/local/tomcat/logs" @@ -108,7 +157,6 @@ services: - store_db ports: - "8083:8080" - - "65495:65395" restart: always store_db: @@ -116,11 +164,11 @@ services: image: postgres:10.17 command: postgres -c datestyle='iso, dmy' environment: - POSTGRES_HOST: store_db - POSTGRES_PORT: 5432 - POSTGRES_DB: samplystore - POSTGRES_USER: samplystore - POSTGRES_PASSWORD: ${C4_SAMPLY_STORE_PASS} + POSTGRES_HOST: ${STORE_POSTGRES_HOST} + POSTGRES_PORT: ${STORE_POSTGRES_PORT} + POSTGRES_DB: ${STORE_POSTGRES_DB} + POSTGRES_USER: ${STORE_POSTGRES_USER} + POSTGRES_PASSWORD: ${STORE_POSTGRES_PASS} TZ: Europe/Berlin volumes: - "store_db_data:/var/lib/postgresql/data" diff --git a/dktk/docker-compose.yml b/dktk/docker-compose.yml index cd379a2..0bd5bde 100644 --- a/dktk/docker-compose.yml +++ b/dktk/docker-compose.yml @@ -1,10 +1,12 @@ version: "3.7" volumes: - connector_db_data: - connector_logs: - patientlist_db_data: - patientlist_logs: - idmanager_logs: + connector_db_data: null + connector_logs: null + patientlist_db_data: null + patientlist_logs: null + idmanager_logs: null + store_db_data: null + store_logs: null secrets: mainzelliste.docker.conf: file: ../internal-configuration/mainzelliste.conf @@ -19,12 +21,11 @@ secrets: services: connector: - container_name: bridgehead_dktk_connector - image: samply/share-client:dktk-develop + container_name: bridgehead_c4_connector + image: "samply/share-client:c4-7" environment: SITE: ${SITE} SITEID: ${SITEID} - TOMCAT_REVERSEPROXY_FQDN: ${HOST}/${COMPOSE_PROJECT_NAME}-connector CENTRAL_SEARCH: ${CCP_CENTRALSEARCH_URL} DECENTRAL_SEARCH: ${CCP_DECENTRALSEARCH_URL} MDR_URL: ${CCP_MDR_URL} @@ -35,11 +36,10 @@ services: PATIENTLIST_URL: ${PATIENTLIST_URL} STORE_URL: ${LDM_URL} POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_PORT: 5432 + POSTGRES_PORT: ${CONNECTOR_DB_PORT} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} - POSTGRES_PASS: ${DKTK_CONNECTOR_POSTGRES_PASSWORD} - DEPLOYMENT_CONTEXT: ${COMPOSE_PROJECT_NAME}-connector + POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} HTTP_PROXY_URL: ${HTTP_PROXY_URL} HTTP_PROXY_USERNAME: ${HTTP_PROXY_USERNAME} HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} @@ -51,48 +51,41 @@ services: - "connector_logs:/usr/local/tomcat/logs" depends_on: - connector_db + ports: + - "8080:8080" restart: always - extra_hosts: - - "host.docker.internal:host-gateway" - - "${HOST}:${HOSTIP}" - + connector_db: - container_name: bridgead_connector_db + container_name: bridgehead_connector_db image: postgres:10.17 environment: + POSTGRES_HOST: ${CONNECTOR_POSTGRES_HOST} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} - POSTGRES_PASSWORD: ${DKTK_CONNECTOR_POSTGRES_PASSWORD} + POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} TZ: Europe/Berlin volumes: - "connector_db_data:/var/lib/postgresql/data" restart: always - extra_hosts: - - "host.docker.internal:host-gateway" - - "${HOST}:${HOSTIP}" - ## ID-Management + idmanager: container_name: bridgehead_idmanager - image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 + image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 environment: - TOMCAT_REVERSEPROXY_FQDN: ${HOST}/ID-Manager - TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} - TOMCAT_REVERSEPROXY_PORT: ${PORT} - MAGICPL_MAINZELLISTE_URL: http://patientlist:8080/Patientlist - MAGICPL_MAINZELLISTE_API_KEY: ${LOCAL_IDMANAGER_MAINZELLISTE_APIKEY} - MAGICPL_API_KEY: ${LOCAL_IDMANAGER_LDM_APIKEY} - MAGICPL_API_KEY_CONNECTOR: ${LOCAL_IDMANAGER_CONNECTOR_APIKEY} - MAGICPL_PASSPHRASE: notUsedInThisConfigurationButMandatory - ### Configuration for communication with central identity management - MAGICPL_MAINZELLISTE_CENTRAL_URL: ${CCP_PATIENTLIST_URL} - MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${CCP_PATIENTLISTE_APIKEY} - MAGICPL_CENTRAL_URL: ${CCP_CONTROLLNUMBERGENERATOR_URL} - MAGICPL_CENTRAL_API_KEY: ${CCP_CONTROLLNUMBERGENERATOR_APIKEY} - MAGICPL_OIDC_PROVIDER: ${CCP_OIDC_PROVIDER_URL}/oauth2 - MAGICPL_OIDC_CLIENT_ID: ${CCP_OIDC_CLIENT_ID} - MAGICPL_OIDC_CLIENT_SECRET: ${CCP_OIDC_CLIENT_SECRET} - MAGICPL_SITE: ${SITEID} + MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} + MAGICPL_MAINZELLISTE_API_KEY: ${MAGICPL_MAINZELLISTE_API_KEY} + MAGICPL_API_KEY: ${MAGICPL_API_KEY} + MAGICPL_API_KEY_CONNECTOR: ${MAGICPL_API_KEY_CONNECTOR} + MAGICPL_PASSPHRASE: ${MAGICPL_PASSPHRASE} + MAGICPL_MAINZELLISTE_CENTRAL_URL: ${MAGICPL_MAINZELLISTE_CENTRAL_URL} + MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${MAGICPL_MAINZELLISTE_CENTRAL_API_KEY} + MAGICPL_CENTRAL_URL: ${MAGICPL_CENTRAL_URL} + MAGICPL_CENTRAL_API_KEY: ${MAGICPL_CENTRAL_API_KEY} + MAGICPL_OIDC_PROVIDER: ${MAGICPL_OIDC_PROVIDER} + MAGICPL_OIDC_CLIENT_ID: ${MAGICPL_OIDC_CLIENT_ID} + MAGICPL_OIDC_CLIENT_SECRET: ${MAGICPL_OIDC_CLIENT_SECRET} + MAGICPL_SITE: adt MAGICPL_LOG_LEVEL: info TZ: Europe/Berlin volumes: @@ -103,24 +96,23 @@ services: - proxy.docker.xml depends_on: - patientlist - + ports: + - "8085:8080" + patientlist: container_name: bridgehead_patientlist image: medicalinformatics/mainzelliste:develop environment: - TOMCAT_REVERSEPROXY_FQDN: ${HOST}/Patientlist - TOMCAT_REVERSEPROXY_SCHEME: ${PROTOCOL} - TOMCAT_REVERSEPROXY_PORT: ${PORT} ML_DB_DRIVER: org.postgresql.Driver ML_DB_TYPE: postgresql - ML_DB_HOST: patientlist_db - ML_DB_PORT: 5432 + ML_DB_HOST: ${ML_DB_HOST} + ML_DB_PORT: ${ML_DB_PORT} ML_DB_NAME: ${ML_DB_NAME} ML_DB_USER: ${ML_DB_USER} - ML_DB_PASS: ${DKTK_PATIENTLIST_DBPASS} - ML_API_KEY: ${DKTK_IDMANAGER_MAINZELLISTE_APIKEY} + ML_DB_PASS: ${ML_DB_PASS} + ML_API_KEY: ${ML_API_KEY} ML_LOG_LEVEL: warning - ML_SITE: ${SITEID} + ML_SITE: adt TZ: Europe/Berlin env_file: - ./site-config/patientlist.env @@ -131,15 +123,17 @@ services: - centralSearchPublicKey depends_on: - patientlist_db + ports: + - "8086:8080" patientlist_db: container_name: bridgehead_patientlist_db image: postgres:13.1-alpine environment: + POSTGRES_HOST: ${ML_DB_HOST} POSTGRES_DB: ${ML_DB_NAME} POSTGRES_USER: ${ML_DB_USER} - POSTGRES_PASSWORD: ${DKTK_PATIENTLIST_DBPASS} + POSTGRES_PASSWORD: ${ML_DB_PASS} TZ: Europe/Berlin volumes: - - "patientlist_db_data:/var/lib/postgresql/data" - + - "patientlist_db_data:/var/lib/postgresql/data" \ No newline at end of file diff --git a/gbn/docker-compose.yml b/gbn/docker-compose.yml index e97961b..81d761e 100644 --- a/gbn/docker-compose.yml +++ b/gbn/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: "bridgehead_blaze_store" image: "samply/blaze:0.15" environment: - BASE_URL: "http://store:8080" + BASE_URL: "http://bridgehead_blzae_store:8080" JAVA_TOOL_OPTIONS: "-Xmx4g" networks: - "samply" @@ -16,14 +16,14 @@ services: restart: "always" connector: - container_name: "bridgehead_connector" + container_name: "bridgehead_gbn_connector" image: "samply/share-client:gbn-7" environment: - POSTGRES_HOST: "connector-db" + POSTGRES_HOST: "bridgehead_connector_db" POSTGRES_DB: "samply.connector" POSTGRES_USER: "samply" - POSTGRES_PASS: ${GBN_CONNECTOR_POSTGRES_PASS} - STORE_URL: "http://store:8080/fhir" + POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} + STORE_URL: "http://bridgehead_store:8080/fhir" QUERY_LANGUAGE: "CQL" MDR_URL: "https://mdr.germanbiobanknode.de/v3/api/mdr" networks: @@ -36,11 +36,11 @@ services: restart: "always" connector-db: - container_name: "bridgehead_connector-db" + container_name: "bridgehead_connector_db" image: "postgres:10.17" environment: POSTGRES_USER: "samply" - POSTGRES_PASSWORD: "samply" + POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} POSTGRES_DB: "samply.connector" networks: - "connector-db" diff --git a/install-bridgehead.sh b/install-bridgehead.sh index 0324a60..e4bf6f8 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -1,18 +1,29 @@ #!/bin/bash ### Note: Currently not complete, needs some features before useable for production -./prerequisites.sh +if ! ./prerequisites.sh; then + echo "Prerequisites failed, exiting" + exiting +fi source site.conf echo "Installing bridgehead" +if ! grep -E 'BRIDGEHEAD_PATH=' /etc/environment; then + echo "BRIDGEHEAD_PATH=${PWD}" >> /etc/environment + echo "Please reboot the system to properly set the enviroment" + exit +fi + +sed -i -e "s|\BRIDGEHEAD_PATH=.*|\BRIDGEHEAD_PATH=${PWD}|" environment + cd /etc/systemd/system/ echo "Installing bridgehead\@.service in systemd ..." -sudo cp /srv/docker/bridgehead/convenience/bridgehead\@.service ./ +sudo cp ${BRIDGEHEAD_PATH}/convenience/bridgehead\@.service ./ echo "Installing bridgehead\@.update.service in systemd ..." -sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.service ./ -sudo cp /srv/docker/bridgehead/convenience/bridgehead-update\@.timer ./ +sudo cp ${BRIDGEHEAD_PATH}/convenience/bridgehead-update\@.service ./ +sudo cp ${BRIDGEHEAD_PATH}/convenience/bridgehead-update\@.timer ./ echo "Loading the bridgehead definitions in systemd" sudo systemctl daemon-reload diff --git a/start-bridgehead.sh b/start-bridgehead.sh index de34193..2e552b4 100755 --- a/start-bridgehead.sh +++ b/start-bridgehead.sh @@ -1,7 +1,10 @@ #!/bin/bash ### Note: Currently not complete, needs some features before useable for production -./prerequisites.sh +if ! ./prerequisites.sh; then + echo "Prerequisites failed, exiting" + exiting +fi source site.conf echo "Starting bridgehead" diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh index a8f7b78..900221f 100755 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -6,6 +6,9 @@ systemctl stop bridgehead@"${project}".service systemctl stop bridgehead-update@"${project}".timer systemctl stop bridgehead-update@"${project}".service +sed -i -e "s|BRIDGEHEAD_PATH=.*||" /etc/environment + + cd /etc/systemd/system/ rm bridgehead\@.service rm bridgehead-update\@.timer From 3709c0dbf7602e97478729285a0b88fffe31e47b Mon Sep 17 00:00:00 2001 From: root Date: Sat, 8 Jan 2022 09:46:45 +0100 Subject: [PATCH 012/215] Some more adjustments for the new deployment --- c4/docker-compose.yml | 67 ++++++++++++++++++++++++++++------------- dktk/docker-compose.yml | 57 +++++++++++++++++++++++------------ install-bridgehead.sh | 4 +-- start-bridgehead.sh | 2 +- uninstall-bridgehead.sh | 1 - 5 files changed, 85 insertions(+), 46 deletions(-) diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index 5952c20..56259aa 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -1,12 +1,12 @@ version: "3.7" volumes: - connector_db_data: null - connector_logs: null - patientlist_db_data: null - patientlist_logs: null - idmanager_logs: null - store_db_data: null - store_logs: null + connector_db_data: + connector_logs: + patientlist_db_data: + patientlist_logs: + idmanager_logs: + store_db_data: + store_logs: secrets: mainzelliste.docker.conf: file: ../internal-configuration/mainzelliste.conf @@ -36,16 +36,18 @@ services: PATIENTLIST_URL: ${PATIENTLIST_URL} STORE_URL: ${LDM_URL} POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_PORT: ${CONNECTOR_DB_PORT} + POSTGRES_PORT: ${CONNECTOR_POSTGRES_PORT} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} - HTTP_PROXY_URL: ${HTTP_PROXY_URL} - HTTP_PROXY_USERNAME: ${HTTP_PROXY_USERNAME} + HTTP_PROXY: ${HTTP_PROXY_URL} + HTTPS_PROXY: ${HTTP_PROXY_URL} + HTTP_PROXY_USERNAME: ${HTTP_PROXY_USER} HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} HTTPS_PROXY_URL: ${HTTPS_PROXY_URL} - HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USERNAME} + HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USER} HTTPS_PROXY_PASSWORD: ${HTTPS_PROXY_PASSWORD} + NO_PROXY: ${NO_PROXY} TZ: Europe/Berlin volumes: - "connector_logs:/usr/local/tomcat/logs" @@ -54,12 +56,15 @@ services: ports: - "8080:8080" restart: always - + networks: + - "samply" + - "connector-db" + connector_db: container_name: bridgehead_connector_db image: postgres:10.17 environment: - POSTGRES_HOST: ${CONNECTOR_POSTGRES_HOST} + POSTGRES_HOST: ${CONNECTOR_DB_HOST} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} @@ -67,10 +72,14 @@ services: volumes: - "connector_db_data:/var/lib/postgresql/data" restart: always + networks: + - "connector-db" + ports: + - "8088:5432" idmanager: - container_name: bridgehead_idmanager + container_name: bridgehead-idmanager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 environment: MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} @@ -98,9 +107,11 @@ services: - patientlist ports: - "8085:8080" - + networks: + - "samply" + patientlist: - container_name: bridgehead_patientlist + container_name: bridgehead-patientlist image: medicalinformatics/mainzelliste:develop environment: ML_DB_DRIVER: org.postgresql.Driver @@ -115,7 +126,7 @@ services: ML_SITE: adt TZ: Europe/Berlin env_file: - - ./site-config/patientlist.env + - ../site-config/patientlist.env volumes: - "patientlist_logs:/usr/local/tomcat/logs" secrets: @@ -125,21 +136,24 @@ services: - patientlist_db ports: - "8086:8080" + networks: + - "samply" patientlist_db: container_name: bridgehead_patientlist_db image: postgres:13.1-alpine environment: - POSTGRES_HOST: ${ML_DB_HOST} POSTGRES_DB: ${ML_DB_NAME} POSTGRES_USER: ${ML_DB_USER} POSTGRES_PASSWORD: ${ML_DB_PASS} TZ: Europe/Berlin volumes: - "patientlist_db_data:/var/lib/postgresql/data" + networks: + - "samply" store: - container_name: bridgehead_store + container_name: bridgeheadstore image: docker.verbis.dkfz.de/ccp/samply.store:release-5.1.2 environment: MDR_URL: ${CCP_MDR_URL} @@ -158,13 +172,14 @@ services: ports: - "8083:8080" restart: always + networks: + - "samply" store_db: container_name: bridgehead_store_db - image: postgres:10.17 + image: postgres:9.5-alpine command: postgres -c datestyle='iso, dmy' environment: - POSTGRES_HOST: ${STORE_POSTGRES_HOST} POSTGRES_PORT: ${STORE_POSTGRES_PORT} POSTGRES_DB: ${STORE_POSTGRES_DB} POSTGRES_USER: ${STORE_POSTGRES_USER} @@ -173,3 +188,13 @@ services: volumes: - "store_db_data:/var/lib/postgresql/data" restart: always + networks: + - "samply" + ports: + - "8084:5432" + +networks: + samply: + driver: "bridge" + connector-db: + driver: "bridge" \ No newline at end of file diff --git a/dktk/docker-compose.yml b/dktk/docker-compose.yml index 0bd5bde..b26c6b2 100644 --- a/dktk/docker-compose.yml +++ b/dktk/docker-compose.yml @@ -1,12 +1,12 @@ version: "3.7" volumes: - connector_db_data: null - connector_logs: null - patientlist_db_data: null - patientlist_logs: null - idmanager_logs: null - store_db_data: null - store_logs: null + connector_db_data: + connector_logs: + patientlist_db_data: + patientlist_logs: + idmanager_logs: + store_db_data: + store_logs: secrets: mainzelliste.docker.conf: file: ../internal-configuration/mainzelliste.conf @@ -21,8 +21,8 @@ secrets: services: connector: - container_name: bridgehead_c4_connector - image: "samply/share-client:c4-7" + container_name: bridgehead_dktk_connector + image: "samply/share-client:dktk-7" environment: SITE: ${SITE} SITEID: ${SITEID} @@ -36,16 +36,17 @@ services: PATIENTLIST_URL: ${PATIENTLIST_URL} STORE_URL: ${LDM_URL} POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_PORT: ${CONNECTOR_DB_PORT} + POSTGRES_PORT: ${CONNECTOR_POSTGRES_PORT} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} HTTP_PROXY_URL: ${HTTP_PROXY_URL} - HTTP_PROXY_USERNAME: ${HTTP_PROXY_USERNAME} + HTTP_PROXY_USERNAME: ${HTTP_PROXY_USER} HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} HTTPS_PROXY_URL: ${HTTPS_PROXY_URL} - HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USERNAME} + HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USER} HTTPS_PROXY_PASSWORD: ${HTTPS_PROXY_PASSWORD} + NO_PROXY: ${NO_PROXY} TZ: Europe/Berlin volumes: - "connector_logs:/usr/local/tomcat/logs" @@ -54,12 +55,15 @@ services: ports: - "8080:8080" restart: always - + networks: + - "samply" + - "connector-db" + connector_db: container_name: bridgehead_connector_db image: postgres:10.17 environment: - POSTGRES_HOST: ${CONNECTOR_POSTGRES_HOST} + POSTGRES_HOST: ${CONNECTOR_DB_HOST} POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} @@ -67,10 +71,12 @@ services: volumes: - "connector_db_data:/var/lib/postgresql/data" restart: always + networks: + - "connector-db" idmanager: - container_name: bridgehead_idmanager + container_name: bridgehead-idmanager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 environment: MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} @@ -98,9 +104,11 @@ services: - patientlist ports: - "8085:8080" - + networks: + - "samply" + patientlist: - container_name: bridgehead_patientlist + container_name: bridgehead-patientlist image: medicalinformatics/mainzelliste:develop environment: ML_DB_DRIVER: org.postgresql.Driver @@ -115,7 +123,7 @@ services: ML_SITE: adt TZ: Europe/Berlin env_file: - - ./site-config/patientlist.env + - ../site-config/patientlist.env volumes: - "patientlist_logs:/usr/local/tomcat/logs" secrets: @@ -125,15 +133,24 @@ services: - patientlist_db ports: - "8086:8080" + networks: + - "samply" patientlist_db: container_name: bridgehead_patientlist_db image: postgres:13.1-alpine environment: - POSTGRES_HOST: ${ML_DB_HOST} POSTGRES_DB: ${ML_DB_NAME} POSTGRES_USER: ${ML_DB_USER} POSTGRES_PASSWORD: ${ML_DB_PASS} TZ: Europe/Berlin volumes: - - "patientlist_db_data:/var/lib/postgresql/data" \ No newline at end of file + - "patientlist_db_data:/var/lib/postgresql/data" + networks: + - "samply" + +networks: + samply: + driver: "bridge" + connector-db: + driver: "bridge" \ No newline at end of file diff --git a/install-bridgehead.sh b/install-bridgehead.sh index e4bf6f8..adf5b13 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -9,14 +9,12 @@ source site.conf echo "Installing bridgehead" -if ! grep -E 'BRIDGEHEAD_PATH=' /etc/environment; then +if [ -z "$BRIDGEHEAD_PATH" ] ; then echo "BRIDGEHEAD_PATH=${PWD}" >> /etc/environment echo "Please reboot the system to properly set the enviroment" exit fi -sed -i -e "s|\BRIDGEHEAD_PATH=.*|\BRIDGEHEAD_PATH=${PWD}|" environment - cd /etc/systemd/system/ echo "Installing bridgehead\@.service in systemd ..." diff --git a/start-bridgehead.sh b/start-bridgehead.sh index 2e552b4..6634dd6 100755 --- a/start-bridgehead.sh +++ b/start-bridgehead.sh @@ -3,7 +3,7 @@ if ! ./prerequisites.sh; then echo "Prerequisites failed, exiting" - exiting + exit fi source site.conf diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh index 900221f..380313b 100755 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -8,7 +8,6 @@ systemctl stop bridgehead-update@"${project}".service sed -i -e "s|BRIDGEHEAD_PATH=.*||" /etc/environment - cd /etc/systemd/system/ rm bridgehead\@.service rm bridgehead-update\@.timer From 33b2c80aa85fffdf650dac1cf9594250de31965f Mon Sep 17 00:00:00 2001 From: root Date: Mon, 10 Jan 2022 14:29:12 +0100 Subject: [PATCH 013/215] C4 psuedo fix --- c4/docker-compose.yml | 4 ++-- install-bridgehead.sh | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index 56259aa..43a3169 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -94,7 +94,7 @@ services: MAGICPL_OIDC_PROVIDER: ${MAGICPL_OIDC_PROVIDER} MAGICPL_OIDC_CLIENT_ID: ${MAGICPL_OIDC_CLIENT_ID} MAGICPL_OIDC_CLIENT_SECRET: ${MAGICPL_OIDC_CLIENT_SECRET} - MAGICPL_SITE: adt + MAGICPL_SITE: ${SITEID} MAGICPL_LOG_LEVEL: info TZ: Europe/Berlin volumes: @@ -123,7 +123,7 @@ services: ML_DB_PASS: ${ML_DB_PASS} ML_API_KEY: ${ML_API_KEY} ML_LOG_LEVEL: warning - ML_SITE: adt + ML_SITE: ${SITEID} TZ: Europe/Berlin env_file: - ../site-config/patientlist.env diff --git a/install-bridgehead.sh b/install-bridgehead.sh index adf5b13..fb74ec0 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -15,13 +15,14 @@ if [ -z "$BRIDGEHEAD_PATH" ] ; then exit fi -cd /etc/systemd/system/ +_systemd_path=/etc/systemd/system/ + echo "Installing bridgehead\@.service in systemd ..." -sudo cp ${BRIDGEHEAD_PATH}/convenience/bridgehead\@.service ./ +sudo cp convenience/bridgehead\@.service $_systemd_path echo "Installing bridgehead\@.update.service in systemd ..." -sudo cp ${BRIDGEHEAD_PATH}/convenience/bridgehead-update\@.service ./ -sudo cp ${BRIDGEHEAD_PATH}/convenience/bridgehead-update\@.timer ./ +sudo cp convenience/bridgehead-update\@.service $_systemd_path +sudo cp convenience/bridgehead-update\@.timer $_systemd_path echo "Loading the bridgehead definitions in systemd" sudo systemctl daemon-reload From 77773a099048575afb587350ba494ef1a75817e1 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Mon, 10 Jan 2022 14:32:40 +0100 Subject: [PATCH 014/215] Updated Readme --- README.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index fae934e..432235d 100644 --- a/README.md +++ b/README.md @@ -2,41 +2,43 @@ This repository contains all information and tools to deploy a bridgehead. If you have any questions about deploying a bridgehead, please contact us. -There are some prerequisites, which need to be meet befor starting a bridgehead. If you runnig a Windows or Mac OS maschine you should read starting a bridgehead. If you running a Linux maschine you can do start and install a bridgehead. - -## Starting +There are some prerequisites, which need to be meet befor starting a bridgehead. If you runnig a Windows or Mac OS maschine you should read starting a bridgehead. If you running a Linux maschine you can start or install a bridgehead. -## Install +## Setup -Create the dir /src/docker and clone the repository. +Clone this repository to /srv/docker -The first step is to copy the site.conf file. It contains some configuration and secrets for your bridgehead. +The first step is to copy the site.conf . It contains some configuration and secrets for your bridgehead. With cp site.dev.conf site.conf you can clone the template. You need to set the project accoriding to the which bridgehead you want to start. It's either a GBN/BBMRI-ERIC, DKTK or C4 Bridgehead. -In this file you can put important varibales +Each Project needs it own .env file where all the settings are located. Each Project has a template for it in there respective folder. We offer you to setup the file with and also to manage it. + ### DKTK +For DKTK set in the site.conf the project to "dkkt". Also you need to set many settings in the env file. For the API keys for the psuenomisation you need to contact the Mainzelliste Team. + ### GBA/BBMRI-ERIC +For an GBN/BBMRI-ERIC deployment set the project to gbn. When you already deployed a bridgehead you can reuse the env file for it. + ### C4 -The following vairbales need to be set +For C4 project it is similar to DKTK. Set the ldm_base_url in the configuration table to null. -C4_SAMPLY_STORE_PASS -C4_CONNECTOR_POSTGRES_PASSWORD +### Git repository - -The next step is creating a configuration for your bridghead. We can provide you a configuration git repository for bridgehead, just ask us. When you have a DKTK or C4 Bridgehead you propably need some configuration from us anyways. +If you already have a git config repositpory you can clone it with git submodule add -f https://"$git_username":"$git_access_token"@code.mitro.dkfz.de/scm/bd/"$site_name_lowercase"-config.git ./site-config -If you want to manage the configuration your self you need to copy a env file from the respective project folder into a site-config folder. +## Starting your bridgehead -Step 3 is determind your prefered start method. You can can just start the docker container with start-bridgehead and stop it. Alternatively, we advice to use the install script. This script add a systemd service to your system which starts the bridgehead, it also stops and starts the system in the event of a reboot. Also it contains a service for automatic updating your bridgehead. It will check at 3:00 am if there are any updates and will apply them. +There two methods to start the bridgehead. For Windows, Linux and Mac OS you can use the start-bridgehead.sh to deploy it wit docker-compose. If will also check some other setting of your system. -Step 4 is checking your bridgehead. +The second methods is using the systemd management tool you start, stop and update your bridgehead. +Just run the install-bridgehead and thats it. From 44398c50887aa5af43ac53bae0628ff1bdbc5c7b Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 14:45:52 +0100 Subject: [PATCH 015/215] Move prerequisites.sh to lib folder --- install-bridgehead.sh | 2 +- prerequisites.sh => lib/prerequisites.sh | 0 update-bridgehead.sh | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) rename prerequisites.sh => lib/prerequisites.sh (100%) diff --git a/install-bridgehead.sh b/install-bridgehead.sh index fb74ec0..849e6c0 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -1,7 +1,7 @@ #!/bin/bash ### Note: Currently not complete, needs some features before useable for production -if ! ./prerequisites.sh; then +if ! ./lib/prerequisites.sh; then echo "Prerequisites failed, exiting" exiting fi diff --git a/prerequisites.sh b/lib/prerequisites.sh similarity index 100% rename from prerequisites.sh rename to lib/prerequisites.sh diff --git a/update-bridgehead.sh b/update-bridgehead.sh index 11b1393..9c3290b 100755 --- a/update-bridgehead.sh +++ b/update-bridgehead.sh @@ -7,6 +7,7 @@ log() { log "INFO" "Checking for updates of $service" # check prerequisites +##TODO: Move to lib/prereq.sh prerequisites="git docker docker-compose" for prerequisite in $prerequisites; do $prerequisite --version 2>&1 From c00759f093665ad3a09c131011d0e51a506f2200 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 10 Jan 2022 14:54:09 +0100 Subject: [PATCH 016/215] removed old file --- site.dev.conf | 53 +++------------------------------------------------ 1 file changed, 3 insertions(+), 50 deletions(-) diff --git a/site.dev.conf b/site.dev.conf index 216e367..e39140b 100644 --- a/site.dev.conf +++ b/site.dev.conf @@ -1,59 +1,12 @@ #!/bin/bash ### This is the configuration file for secrets, only your site should know -### Because the deployment via systemd is not possible in the current wsl environment, -### developers need to export the necessary variables. -### You can copy this file to site-specific.conf and run source site-specific.conf. -### After this docker-compose up should use the environment variables defined in this file. -### Connector configuration -## The password the for your connector database -export dktk_CONNECTOR_POSTGRES_PASSWORD= -export c4_CONNECTOR_POSTGRES_PASSWORD= -export gbn_CONNECTOR_POSTGRES_PASSWORD= - -### Local Datamanagenment configuration if necessary -## Supply this, if you use samplystore -# export LDM_DATABASE_PASSWORD=this-should-only-be-used-by-developers-ldm-database-password; - -### ID-Management configuration -## provided by DKFZ with cooperation in Mainz -export CCP_PATIENTLISTE_APIKEY= -## provided by DKFZ with cooperation in Frankfurt -export CCP_CONTROLLNUMBERGENERATOR_APIKEY= -export LOCAL_IDMANAGER_LDM_APIKEY= -## Additional new ID-Manager Configuration -export LOCAL_IDMANAGER_MAINZELLISTE_APIKEY= -export LOCAL_IDMANAGER_CONNECTOR_APIKEY= -export LOCAL_PATIENTLIST_DBPASS= - -export dktk_CCP_OIDC_CLIENT_SECRET= -export c4_CCP_OIDC_CLIENT_SECRET= -### Mail Server username and password -# export MAIL_USER= -# export MAIL_PASSWORD= - -### Monitoring -## by default, the bridgehead will report it's state to DKFZ monitoring. Change here to opt-out. -# export MONITOR_OPTOUT=true - -### nNGM Configuration, only necessary if your site takes part in nNGM -## The cts login data, that healex provided you -# export NNGM_CTS_USER= -# export NNGM_CTS_PASSWORD= -## The idmanagement authentication data, that is provided by dkfz dataprotection office -# export NNGM_MAGICPL_APIKEY= -# export NNGM_MAINZELLISTE_APIKEY= - -### Workaround for local developers network environment -## Note: MSYS_NO_PATHCONV is needed, because git bash and msys2 will otherwise convert /etc/hosts to a windows path -#export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); -#export HOST=$(hostname) - -export HOSTIP= +##Setting Network properties +export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); export HOST= export site_name= ### Write the Project you want to start with the brigdehead ##Exmaple project=gbn -project= +project=c4 From 1658e7357b5f52fc58219cf9159be0616dea2503 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 15:11:35 +0100 Subject: [PATCH 017/215] Correctly ignore git submodules --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d1362ce..6bd96d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ ##Ignore site configuration -.gitsubmodules +.gitmodules site-config ## Ignore site configuration From 28fad0fa908682084fbf21e1b042f5f9fa9eec12 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 15:11:57 +0100 Subject: [PATCH 018/215] Fix prereq exit --- install-bridgehead.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-bridgehead.sh b/install-bridgehead.sh index 849e6c0..b8b694a 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -3,7 +3,7 @@ if ! ./lib/prerequisites.sh; then echo "Prerequisites failed, exiting" - exiting + exit 1 fi source site.conf From 3bab2f6dfd3196a58ce2558e7647717ac1c8215c Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 15:12:29 +0100 Subject: [PATCH 019/215] Correctly ignore site-specific config --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6bd96d4..8960758 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,5 @@ site-config config/**/* !config/**/*.default docker-compose.override.yml -site-specific.conf +site.conf From 75b33690f6644d1afbb180b500311b293258b5fb Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 15:17:27 +0100 Subject: [PATCH 020/215] Remove BRIDGEHEAD_PATH env variable --- install-bridgehead.sh | 6 ------ uninstall-bridgehead.sh | 2 -- 2 files changed, 8 deletions(-) diff --git a/install-bridgehead.sh b/install-bridgehead.sh index b8b694a..df070c7 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -9,12 +9,6 @@ source site.conf echo "Installing bridgehead" -if [ -z "$BRIDGEHEAD_PATH" ] ; then - echo "BRIDGEHEAD_PATH=${PWD}" >> /etc/environment - echo "Please reboot the system to properly set the enviroment" - exit -fi - _systemd_path=/etc/systemd/system/ diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh index 380313b..a8f7b78 100755 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -6,8 +6,6 @@ systemctl stop bridgehead@"${project}".service systemctl stop bridgehead-update@"${project}".timer systemctl stop bridgehead-update@"${project}".service -sed -i -e "s|BRIDGEHEAD_PATH=.*||" /etc/environment - cd /etc/systemd/system/ rm bridgehead\@.service rm bridgehead-update\@.timer From 123ce65c825a62b276fd0d58e00d69a543b326b3 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 15:31:07 +0100 Subject: [PATCH 021/215] Don't do sudo --- install-bridgehead.sh | 25 +++++++++++++++---------- lib/functions.sh | 12 ++++++++++++ 2 files changed, 27 insertions(+), 10 deletions(-) create mode 100755 lib/functions.sh diff --git a/install-bridgehead.sh b/install-bridgehead.sh index df070c7..54f4568 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -1,6 +1,10 @@ #!/bin/bash ### Note: Currently not complete, needs some features before useable for production +source lib/functions.sh + +exitIfNotRoot + if ! ./lib/prerequisites.sh; then echo "Prerequisites failed, exiting" exit 1 @@ -12,14 +16,15 @@ echo "Installing bridgehead" _systemd_path=/etc/systemd/system/ -echo "Installing bridgehead\@.service in systemd ..." -sudo cp convenience/bridgehead\@.service $_systemd_path -echo "Installing bridgehead\@.update.service in systemd ..." -sudo cp convenience/bridgehead-update\@.service $_systemd_path -sudo cp convenience/bridgehead-update\@.timer $_systemd_path +echo "Installing systemd units ..." +cp -v \ + convenience/bridgehead\@.service \ + convenience/bridgehead-update\@.service \ + convenience/bridgehead-update\@.timer \ + $_systemd_path echo "Loading the bridgehead definitions in systemd" -sudo systemctl daemon-reload +systemctl daemon-reload echo "Starting Project ${project} " @@ -28,14 +33,14 @@ echo "Starting Project ${project} " exit fi - sudo systemctl is-active --quiet bridgehead@"${project}" + systemctl is-active --quiet bridgehead@"${project}" if [ ! $? -eq 0 ]; then echo "Starting bridgehead@${project} service ..." - sudo systemctl start bridgehead@"${project}" + systemctl start bridgehead@"${project}" echo "Enabling autostart of bridgehead@${project}.service" - sudo systemctl enable bridgehead@"${project}" + systemctl enable bridgehead@"${project}" echo "Enabling nightly updates for bridgehead@${project}.service ..." - sudo systemctl enable --now bridgehead-update@"${project}".timer + systemctl enable --now bridgehead-update@"${project}".timer fi # Switch back to execution directory; diff --git a/lib/functions.sh b/lib/functions.sh new file mode 100755 index 0000000..4dc30f5 --- /dev/null +++ b/lib/functions.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +exitIfNotRoot() { + if [ "$EUID" -ne 0 ]; then + echo "Please run as root" + exit 1 + fi +} + +log() { + echo "$(date +'%Y-%m-%d %T')" "$1:" "$2" +} From 99a8b5038ec304baa8dc6abf8e322fc16ef9e084 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 15:41:07 +0100 Subject: [PATCH 022/215] Make sure docker-compose is found --- convenience/bridgehead@.service | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/convenience/bridgehead@.service b/convenience/bridgehead@.service index b4b5c9f..630d623 100644 --- a/convenience/bridgehead@.service +++ b/convenience/bridgehead@.service @@ -7,10 +7,9 @@ RestartSec=30 WorkingDirectory=/srv/docker/bridgehead/%i -ExecStartPre=/usr/local/bin/docker-compose --env-file ../site-config/%i.env down -ExecStart=/usr/local/bin/docker-compose --env-file ../site-config/%i.env up - -ExecStop=/usr/local/bin/docker-compose --env-file ../site-config/%i.env down +ExecStartPre=docker-compose --env-file ../site-config/%i.env down +ExecStart=docker-compose --env-file ../site-config/%i.env up +ExecStop=docker-compose --env-file ../site-config/%i.env down [Install] WantedBy=multi-user.target From 0ecf38cabbe2e22ee11421eb45f3a04f02953391 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 15:41:38 +0100 Subject: [PATCH 023/215] Correct typo --- lib/prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 438df45..65ec9e6 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -51,4 +51,4 @@ if [ -d "site-config/${project}.env" ]; then echo "Please copy the tempalte from ${project} and put it in the site-config folder" fi -echo "All prerequisites meet! All systems are ready to go!" +echo "All prerequisites are met!" From 0d7769fa7f059c591aaf7fc46b47459e5a59da7c Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 16:31:48 +0100 Subject: [PATCH 024/215] Several fixes to shell scripts --- install-bridgehead.sh | 31 ++++++++++--------------------- start-bridgehead.sh | 8 +++++--- stop-bridgehead.sh | 5 ++++- uninstall-bridgehead.sh | 17 +++++++---------- update-bridgehead.sh | 6 ++---- 5 files changed, 28 insertions(+), 39 deletions(-) diff --git a/install-bridgehead.sh b/install-bridgehead.sh index 54f4568..bd9afea 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -11,38 +11,27 @@ if ! ./lib/prerequisites.sh; then fi source site.conf -echo "Installing bridgehead" - _systemd_path=/etc/systemd/system/ -echo "Installing systemd units ..." +echo -e "\nInstalling systemd units ..." cp -v \ - convenience/bridgehead\@.service \ - convenience/bridgehead-update\@.service \ - convenience/bridgehead-update\@.timer \ - $_systemd_path + convenience/bridgehead\@.service \ + convenience/bridgehead-update\@.service \ + convenience/bridgehead-update\@.timer \ + $_systemd_path -echo "Loading the bridgehead definitions in systemd" systemctl daemon-reload +echo -echo "Starting Project ${project} " - if [ ! -f "/etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf" ]; then - echo "Can't find local configuration file for bridgehead@${project} service. Please ensure that the file /etc/systemd/system/bridgehead@${project}.service.d/bridgehead.conf exists" - exit - fi - - systemctl is-active --quiet bridgehead@"${project}" - if [ ! $? -eq 0 ]; then - echo "Starting bridgehead@${project} service ..." - systemctl start bridgehead@"${project}" +if ! systemctl is-active --quiet bridgehead@"${project}"; then echo "Enabling autostart of bridgehead@${project}.service" systemctl enable bridgehead@"${project}" echo "Enabling nightly updates for bridgehead@${project}.service ..." systemctl enable --now bridgehead-update@"${project}".timer - fi +fi + +echo -e "\nDone - now start your bridgehead by running\n\tsystemctl start bridgehead@${project}.service\nor by rebooting your machine." -# Switch back to execution directory; -cd - # TODO: Configuration of the different modules diff --git a/start-bridgehead.sh b/start-bridgehead.sh index 6634dd6..3d4f2ef 100755 --- a/start-bridgehead.sh +++ b/start-bridgehead.sh @@ -1,13 +1,15 @@ #!/bin/bash ### Note: Currently not complete, needs some features before useable for production +source lib/functions.sh + if ! ./prerequisites.sh; then - echo "Prerequisites failed, exiting" + log "Prerequisites failed, exiting" exit fi source site.conf -echo "Starting bridgehead" +log "Starting bridgehead" cd ${project} @@ -15,4 +17,4 @@ docker-compose --env-file ../site-config/${project}.env up -d cd .. -echo "The bridgehead should be in online in a few seconds" +log "The bridgehead should be in online in a few seconds" diff --git a/stop-bridgehead.sh b/stop-bridgehead.sh index 8ab4eff..1d6798e 100755 --- a/stop-bridgehead.sh +++ b/stop-bridgehead.sh @@ -1,7 +1,10 @@ -echo "Stoping bridgehead" +#!/bin/bash -e +source lib/functions.sh source site.conf +log "Stopping bridgehead" + cd ${project} docker-compose --env-file ../site-config/${project}.env down diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh index a8f7b78..d56cb1a 100755 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -1,14 +1,11 @@ -echo "Stoping systemd services and removing bridgehead" +#!/bin/bash -e source site.conf +source lib/functions.sh -systemctl stop bridgehead@"${project}".service -systemctl stop bridgehead-update@"${project}".timer -systemctl stop bridgehead-update@"${project}".service +echo "Stopping systemd services and removing bridgehead ..." -cd /etc/systemd/system/ -rm bridgehead\@.service -rm bridgehead-update\@.timer -rm bridgehead-update\@.service - -cd - +for i in bridgehead\@.service bridgehead-update\@.timer bridgehead-update\@.service; do + systemctl disable $i --now + rm -v /etc/systemd/system/$i +done diff --git a/update-bridgehead.sh b/update-bridgehead.sh index 9c3290b..2af6156 100755 --- a/update-bridgehead.sh +++ b/update-bridgehead.sh @@ -1,9 +1,7 @@ #!/bin/bash service="bridgehead" -# Creates logs -log() { - echo "$(date +'%Y-%m-%d %T')" "$1:" "$2" -} + +source lib/functions.sh log "INFO" "Checking for updates of $service" # check prerequisites From 31ab3ddef2fc4782e4d7ff114a4e7e9d8d1e098a Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 16:36:54 +0100 Subject: [PATCH 025/215] Fix uninstall-bridgehead.sh --- uninstall-bridgehead.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh index d56cb1a..b5c3f91 100755 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -5,7 +5,8 @@ source lib/functions.sh echo "Stopping systemd services and removing bridgehead ..." -for i in bridgehead\@.service bridgehead-update\@.timer bridgehead-update\@.service; do - systemctl disable $i --now - rm -v /etc/systemd/system/$i -done +systemctl disable --now bridgehead@${project}.service +systemctl disable --now bridgehead-update@${project}.timer +systemctl disable --now bridgehead-update@${project}.service + +rm -v /etc/systemd/system/{bridgehead\@.service,bridgehead-update\@.timer,bridgehead-update\@.service} From bcf1ba2ce14723f3b415674ab83c4518f77a2671 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 16:39:46 +0100 Subject: [PATCH 026/215] Make systemctl disable more robust --- uninstall-bridgehead.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uninstall-bridgehead.sh b/uninstall-bridgehead.sh index b5c3f91..877ca01 100755 --- a/uninstall-bridgehead.sh +++ b/uninstall-bridgehead.sh @@ -5,8 +5,6 @@ source lib/functions.sh echo "Stopping systemd services and removing bridgehead ..." -systemctl disable --now bridgehead@${project}.service -systemctl disable --now bridgehead-update@${project}.timer -systemctl disable --now bridgehead-update@${project}.service +systemctl disable --now bridgehead@${project}.service bridgehead-update@${project}.timer bridgehead-update@${project}.service rm -v /etc/systemd/system/{bridgehead\@.service,bridgehead-update\@.timer,bridgehead-update\@.service} From 43bdb238491369aa24e05d7789147f3141eeee5d Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Mon, 10 Jan 2022 16:42:13 +0100 Subject: [PATCH 027/215] Rename "convenience" systemd units --- install-bridgehead.sh | 6 +++--- {convenience => lib/systemd}/bridgehead-update@.service | 0 {convenience => lib/systemd}/bridgehead-update@.timer | 0 {convenience => lib/systemd}/bridgehead@.service | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename {convenience => lib/systemd}/bridgehead-update@.service (100%) rename {convenience => lib/systemd}/bridgehead-update@.timer (100%) rename {convenience => lib/systemd}/bridgehead@.service (100%) diff --git a/install-bridgehead.sh b/install-bridgehead.sh index bd9afea..24c86cf 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -16,9 +16,9 @@ _systemd_path=/etc/systemd/system/ echo -e "\nInstalling systemd units ..." cp -v \ - convenience/bridgehead\@.service \ - convenience/bridgehead-update\@.service \ - convenience/bridgehead-update\@.timer \ + lib/systemd/bridgehead\@.service \ + lib/systemd/bridgehead-update\@.service \ + lib/systemd/bridgehead-update\@.timer \ $_systemd_path systemctl daemon-reload diff --git a/convenience/bridgehead-update@.service b/lib/systemd/bridgehead-update@.service similarity index 100% rename from convenience/bridgehead-update@.service rename to lib/systemd/bridgehead-update@.service diff --git a/convenience/bridgehead-update@.timer b/lib/systemd/bridgehead-update@.timer similarity index 100% rename from convenience/bridgehead-update@.timer rename to lib/systemd/bridgehead-update@.timer diff --git a/convenience/bridgehead@.service b/lib/systemd/bridgehead@.service similarity index 100% rename from convenience/bridgehead@.service rename to lib/systemd/bridgehead@.service From ce89382a32f727058deaca7b5fa355b8131873b4 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 11 Jan 2022 13:48:02 +0100 Subject: [PATCH 028/215] Add which docker-compose to locate the correct path for systemd --- lib/systemd/bridgehead@.service | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/systemd/bridgehead@.service b/lib/systemd/bridgehead@.service index 630d623..8563e4f 100644 --- a/lib/systemd/bridgehead@.service +++ b/lib/systemd/bridgehead@.service @@ -5,11 +5,11 @@ Description=Bridgehead (%i) Service Restart=always RestartSec=30 -WorkingDirectory=/srv/docker/bridgehead/%i +WorkingDirectory=/srv/docker/bridgehead/ -ExecStartPre=docker-compose --env-file ../site-config/%i.env down -ExecStart=docker-compose --env-file ../site-config/%i.env up -ExecStop=docker-compose --env-file ../site-config/%i.env down +ExecStartPre=/bin/bash -c '`which docker-compose` -f %i/docker-compose.yml --env-file site-config/%i.env down' +ExecStart=/bin/bash -c '`which docker-compose` -f %i/docker-compose.yml --env-file site-config/%i.env up' +ExecStop=/bin/bash -c '`which docker-compose` -f %i/docker-compose.yml --env-file site-config/%i.env down' [Install] WantedBy=multi-user.target From 9f304d8fedac2ada10ba6e585bb301fed3f56994 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 11 Jan 2022 13:49:36 +0100 Subject: [PATCH 029/215] remove switching directories to run docker-compose --- start-bridgehead.sh | 8 ++------ stop-bridgehead.sh | 5 +---- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/start-bridgehead.sh b/start-bridgehead.sh index 3d4f2ef..0e7272d 100755 --- a/start-bridgehead.sh +++ b/start-bridgehead.sh @@ -3,7 +3,7 @@ source lib/functions.sh -if ! ./prerequisites.sh; then +if ! lib/prerequisites.sh; then log "Prerequisites failed, exiting" exit fi @@ -11,10 +11,6 @@ source site.conf log "Starting bridgehead" -cd ${project} - -docker-compose --env-file ../site-config/${project}.env up -d - -cd .. +docker-compose -f ${project}/docker-compose.yml --env-file site-config/${project}.env up -d log "The bridgehead should be in online in a few seconds" diff --git a/stop-bridgehead.sh b/stop-bridgehead.sh index 1d6798e..91b7d8d 100755 --- a/stop-bridgehead.sh +++ b/stop-bridgehead.sh @@ -5,8 +5,5 @@ source site.conf log "Stopping bridgehead" -cd ${project} +docker-compose -f ${project}/docker-compose.yml --env-file site-config/${project}.env down -docker-compose --env-file ../site-config/${project}.env down - -cd .. From 1649d0b6240b752a12c4fb7466c9e6e0d9b21b7e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 11 Jan 2022 13:52:20 +0100 Subject: [PATCH 030/215] Uniform container_names --- gbn/docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gbn/docker-compose.yml b/gbn/docker-compose.yml index 81d761e..e2f6bad 100644 --- a/gbn/docker-compose.yml +++ b/gbn/docker-compose.yml @@ -2,10 +2,10 @@ version: '3.4' services: store: - container_name: "bridgehead_blaze_store" + container_name: "bridgehead_gbn_blaze_store" image: "samply/blaze:0.15" environment: - BASE_URL: "http://bridgehead_blzae_store:8080" + BASE_URL: "http://bridgehead_gbn_blaze_store:8080" JAVA_TOOL_OPTIONS: "-Xmx4g" networks: - "samply" @@ -19,7 +19,7 @@ services: container_name: "bridgehead_gbn_connector" image: "samply/share-client:gbn-7" environment: - POSTGRES_HOST: "bridgehead_connector_db" + POSTGRES_HOST: "bridgehead_gbn_connector_db" POSTGRES_DB: "samply.connector" POSTGRES_USER: "samply" POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} @@ -36,7 +36,7 @@ services: restart: "always" connector-db: - container_name: "bridgehead_connector_db" + container_name: "bridgehead_gbn_connector_db" image: "postgres:10.17" environment: POSTGRES_USER: "samply" From 42d69cef4e397149f1d1b2cbb0ae37466af3dcb1 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 11 Jan 2022 14:40:16 +0100 Subject: [PATCH 031/215] Uniform dktk and c4 docker-compose files --- c4/docker-compose.yml | 12 ++++++------ dktk/docker-compose.yml | 8 ++++---- install-bridgehead.sh | 15 ++++++--------- update-bridgehead.sh | 16 ++++++---------- 4 files changed, 22 insertions(+), 29 deletions(-) diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index 43a3169..584dfb5 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -61,7 +61,7 @@ services: - "connector-db" connector_db: - container_name: bridgehead_connector_db + container_name: bridgehead_c4_connector_db image: postgres:10.17 environment: POSTGRES_HOST: ${CONNECTOR_DB_HOST} @@ -79,7 +79,7 @@ services: idmanager: - container_name: bridgehead-idmanager + container_name: bridgehead_c4_idmanager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 environment: MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} @@ -111,7 +111,7 @@ services: - "samply" patientlist: - container_name: bridgehead-patientlist + container_name: bridgehead_c4_patientlist image: medicalinformatics/mainzelliste:develop environment: ML_DB_DRIVER: org.postgresql.Driver @@ -140,7 +140,7 @@ services: - "samply" patientlist_db: - container_name: bridgehead_patientlist_db + container_name: bridgehead_c4_patientlist_db image: postgres:13.1-alpine environment: POSTGRES_DB: ${ML_DB_NAME} @@ -153,7 +153,7 @@ services: - "samply" store: - container_name: bridgeheadstore + container_name: bridgehead_c4_store image: docker.verbis.dkfz.de/ccp/samply.store:release-5.1.2 environment: MDR_URL: ${CCP_MDR_URL} @@ -176,7 +176,7 @@ services: - "samply" store_db: - container_name: bridgehead_store_db + container_name: bridgehead_c4_store_db image: postgres:9.5-alpine command: postgres -c datestyle='iso, dmy' environment: diff --git a/dktk/docker-compose.yml b/dktk/docker-compose.yml index b26c6b2..c8ef8fd 100644 --- a/dktk/docker-compose.yml +++ b/dktk/docker-compose.yml @@ -60,7 +60,7 @@ services: - "connector-db" connector_db: - container_name: bridgehead_connector_db + container_name: bridgehead_dktk_connector_db image: postgres:10.17 environment: POSTGRES_HOST: ${CONNECTOR_DB_HOST} @@ -76,7 +76,7 @@ services: idmanager: - container_name: bridgehead-idmanager + container_name: bridgehead_dktk_idmanager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 environment: MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} @@ -108,7 +108,7 @@ services: - "samply" patientlist: - container_name: bridgehead-patientlist + container_name: bridgehead_dktk_patientlist image: medicalinformatics/mainzelliste:develop environment: ML_DB_DRIVER: org.postgresql.Driver @@ -137,7 +137,7 @@ services: - "samply" patientlist_db: - container_name: bridgehead_patientlist_db + container_name: bridgehead_dktk_patientlist_db image: postgres:13.1-alpine environment: POSTGRES_DB: ${ML_DB_NAME} diff --git a/install-bridgehead.sh b/install-bridgehead.sh index 24c86cf..d360453 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -6,32 +6,29 @@ source lib/functions.sh exitIfNotRoot if ! ./lib/prerequisites.sh; then - echo "Prerequisites failed, exiting" + log "Prerequisites failed, exiting" exit 1 fi source site.conf -_systemd_path=/etc/systemd/system/ - - -echo -e "\nInstalling systemd units ..." +log -e "\nInstalling systemd units ..." cp -v \ lib/systemd/bridgehead\@.service \ lib/systemd/bridgehead-update\@.service \ lib/systemd/bridgehead-update\@.timer \ - $_systemd_path + /etc/systemd/system/ systemctl daemon-reload echo if ! systemctl is-active --quiet bridgehead@"${project}"; then - echo "Enabling autostart of bridgehead@${project}.service" + log "Enabling autostart of bridgehead@${project}.service" systemctl enable bridgehead@"${project}" - echo "Enabling nightly updates for bridgehead@${project}.service ..." + log "Enabling nightly updates for bridgehead@${project}.service ..." systemctl enable --now bridgehead-update@"${project}".timer fi -echo -e "\nDone - now start your bridgehead by running\n\tsystemctl start bridgehead@${project}.service\nor by rebooting your machine." +log -e "\nDone - now start your bridgehead by running\n\tsystemctl start bridgehead@${project}.service\nor by rebooting your machine." # TODO: Configuration of the different modules diff --git a/update-bridgehead.sh b/update-bridgehead.sh index 2af6156..b00e376 100755 --- a/update-bridgehead.sh +++ b/update-bridgehead.sh @@ -3,18 +3,14 @@ service="bridgehead" source lib/functions.sh +if ! ./prerequisites.sh; then + log "Prerequisites failed, exiting" + exit +fi + log "INFO" "Checking for updates of $service" # check prerequisites -##TODO: Move to lib/prereq.sh -prerequisites="git docker docker-compose" -for prerequisite in $prerequisites; do - $prerequisite --version 2>&1 - is_available=$? - if [ $is_available -gt 0 ]; then - log "ERROR" "Prerequisite not fulfilled - $prerequisite is not available!" - exit 79 - fi -done + # check if updates are available old_git_hash="$(git rev-parse --verify HEAD)" git fetch 2>&1 From 735787067042662cdb0828c9134989d7e28c885e Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Jan 2022 10:52:25 +0100 Subject: [PATCH 032/215] Manges changes and moving vars to the correct file --- c4/internal.env | 47 +++++++++++++++++++++++++++++++++ dktk/docker-compose.yml | 57 ++++++++++++----------------------------- dktk/internal.env | 56 ++++++++++++++++++++++++++++++++++++++++ gbn/docker-compose.yml | 41 ++++++++++++++--------------- gbn/internal.env | 26 +++++++++++++++++++ install-bridgehead.sh | 8 +++--- site.dev.conf | 10 +++++++- update-bridgehead.sh | 2 +- 8 files changed, 181 insertions(+), 66 deletions(-) create mode 100644 c4/internal.env create mode 100644 dktk/internal.env create mode 100644 gbn/internal.env diff --git a/c4/internal.env b/c4/internal.env new file mode 100644 index 0000000..f0c0daa --- /dev/null +++ b/c4/internal.env @@ -0,0 +1,47 @@ +SITE=bridgehead_test +SITEID=BRIDGEHEAD_TEST + +CONNECTOR_SHARE_URL="http://${HOST}:8080" +CONNECTOR_ENABLE_METRICS=false +CONNECTOR_MONITOR_INTERVAL= +CONNECTOR_UPDATE_SERVER= + +CONNECTOR_POSTGRES_PORT=5432 +CONNECTOR_POSTGRES_DB=samply.connector +CONNECTOR_POSTGRES_USER=samply.connector + +HTTP_PROXY_HOST=${PROXY_URL} +HTTP_PROXY_USER= +HTTP_PROXY_PASSWORD= +HTTPS_PROXY_HOST=${PROXY_URL} +HTTPS_PROXY_USER= +HTTPS_PROXY_PASSWORD= +HTTP_PROXY=${PROXY_URL} +HTTPS_PROXY=${PROXY_URL} +http_proxy=${PROXY_URL} +https_proxy=${PROXY_URL} + +CCP_CENTRALSEARCH_URL=https://centralsearch-test.dktk.dkfz.de/ +CCP_DECENTRALSEARCH_URL=https://decentralsearch-test.ccp-it.dktk.dkfz.de/ + +CCP_MDR_URL=https://mdr.ccp-it.dktk.dkfz.de/v3/api/mdr +CCP_MONITOR_URL= +MONITOR_OPTOUT= + +## nNGM +#NNGM_URL=http://bridgeheadstore:8080 +#NNGM_PROFILE=http://uk-koeln.de/fhir/StructureDefinition/Patient/nNGM/pseudonymisiert +#NNGM_MAINZELLISTE_URL=https://test.verbis.dkfz.de/mpl + +##MDR +MDR_NAMESPACE=adt,dktk,marker +#MDR_MAP= +MDR_VALIDATION=false + +ML_DB_NAME=mainzelliste +ML_DB_PORT=5432 +ML_DB_USER=mainzelliste + +CENTRAL_CONTROL_NUMBER_GENERATPR_URL=http://e260-serv-03/central/api +GLOBAL_ID=DKTK +MAINZELLISTE_URL=https://patientlist-test.ccpit.dktk.dkfz.de/mainzelliste \ No newline at end of file diff --git a/dktk/docker-compose.yml b/dktk/docker-compose.yml index c8ef8fd..defaea7 100644 --- a/dktk/docker-compose.yml +++ b/dktk/docker-compose.yml @@ -24,30 +24,9 @@ services: container_name: bridgehead_dktk_connector image: "samply/share-client:dktk-7" environment: - SITE: ${SITE} - SITEID: ${SITEID} - CENTRAL_SEARCH: ${CCP_CENTRALSEARCH_URL} - DECENTRAL_SEARCH: ${CCP_DECENTRALSEARCH_URL} - MDR_URL: ${CCP_MDR_URL} - MONITOR_URL: ${CCP_MONITOR_URL} - SHARE_URL: "${PROTOCOL}://${HOST}:${PORT}" - ID_MANAGER_URL: ${ID_MANAGER_URL} - PROJECTPSEUDONYMISATION_URL: ${PROJECTPSEUDONYMISATION_URL} - PATIENTLIST_URL: ${PATIENTLIST_URL} - STORE_URL: ${LDM_URL} + STORE_URL: ${STORE_URL} POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_PORT: ${CONNECTOR_POSTGRES_PORT} - POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} - POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} - HTTP_PROXY_URL: ${HTTP_PROXY_URL} - HTTP_PROXY_USERNAME: ${HTTP_PROXY_USER} - HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} - HTTPS_PROXY_URL: ${HTTPS_PROXY_URL} - HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USER} - HTTPS_PROXY_PASSWORD: ${HTTPS_PROXY_PASSWORD} - NO_PROXY: ${NO_PROXY} - TZ: Europe/Berlin volumes: - "connector_logs:/usr/local/tomcat/logs" depends_on: @@ -55,6 +34,9 @@ services: ports: - "8080:8080" restart: always + env_file: + ###TODO: Move into Image + - internal.env networks: - "samply" - "connector-db" @@ -64,20 +46,20 @@ services: image: postgres:10.17 environment: POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} - POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} - TZ: Europe/Berlin volumes: - "connector_db_data:/var/lib/postgresql/data" restart: always + env_file: + ###TODO: Move into Image + - internal.env networks: - "connector-db" idmanager: container_name: bridgehead_dktk_idmanager - image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 + image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 environment: MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} MAGICPL_MAINZELLISTE_API_KEY: ${MAGICPL_MAINZELLISTE_API_KEY} @@ -91,8 +73,7 @@ services: MAGICPL_OIDC_PROVIDER: ${MAGICPL_OIDC_PROVIDER} MAGICPL_OIDC_CLIENT_ID: ${MAGICPL_OIDC_CLIENT_ID} MAGICPL_OIDC_CLIENT_SECRET: ${MAGICPL_OIDC_CLIENT_SECRET} - MAGICPL_SITE: adt - MAGICPL_LOG_LEVEL: info + TZ: Europe/Berlin volumes: - "idmanager_logs:/usr/local/tomcat/logs" @@ -102,6 +83,9 @@ services: - proxy.docker.xml depends_on: - patientlist + env_file: + ###TODO: Move into Image + - internal.env ports: - "8085:8080" networks: @@ -111,19 +95,12 @@ services: container_name: bridgehead_dktk_patientlist image: medicalinformatics/mainzelliste:develop environment: - ML_DB_DRIVER: org.postgresql.Driver - ML_DB_TYPE: postgresql - ML_DB_HOST: ${ML_DB_HOST} - ML_DB_PORT: ${ML_DB_PORT} - ML_DB_NAME: ${ML_DB_NAME} - ML_DB_USER: ${ML_DB_USER} + ML_DB_HOST: bridgehead_dktk_patientlist_db ML_DB_PASS: ${ML_DB_PASS} ML_API_KEY: ${ML_API_KEY} - ML_LOG_LEVEL: warning - ML_SITE: adt - TZ: Europe/Berlin env_file: - ../site-config/patientlist.env + - internal.env volumes: - "patientlist_logs:/usr/local/tomcat/logs" secrets: @@ -140,12 +117,12 @@ services: container_name: bridgehead_dktk_patientlist_db image: postgres:13.1-alpine environment: - POSTGRES_DB: ${ML_DB_NAME} - POSTGRES_USER: ${ML_DB_USER} POSTGRES_PASSWORD: ${ML_DB_PASS} - TZ: Europe/Berlin volumes: - "patientlist_db_data:/var/lib/postgresql/data" + env_file: + ###TODO: Move into Image + - internal.env networks: - "samply" diff --git a/dktk/internal.env b/dktk/internal.env new file mode 100644 index 0000000..94b7261 --- /dev/null +++ b/dktk/internal.env @@ -0,0 +1,56 @@ +SITE=bridgehead_dktk_test +SITEID=BRIDGEHEAD_DKTK_TEST + +CONNECTOR_SHARE_URL="http://${HOST}:8080" +CONNECTOR_ENABLE_METRICS=false +CONNECTOR_MONITOR_INTERVAL= +CONNECTOR_UPDATE_SERVER= + +POSTGRES_PORT=5432 +POSTGRES_DB=samply.connector +POSTGRES_USER=samply.connector + +HTTP_PROXY_HOST=${PROXY_URL} +HTTP_PROXY_USER= +HTTP_PROXY_PASSWORD= +HTTPS_PROXY_HOST=${PROXY_URL} +HTTPS_PROXY_USER= +HTTPS_PROXY_PASSWORD= +HTTP_PROXY=${PROXY_URL} +HTTPS_PROXY=${PROXY_URL} +http_proxy=${PROXY_URL} +https_proxy=${PROXY_URL} + +CCP_CENTRALSEARCH_URL=https://centralsearch-test.dktk.dkfz.de/ +CCP_DECENTRALSEARCH_URL=https://decentralsearch-test.ccp-it.dktk.dkfz.de/ + +CCP_MDR_URL=https://mdr.ccp-it.dktk.dkfz.de/v3/api/mdr +CCP_MONITOR_URL= +MONITOR_OPTOUT= + +## nNGM +#NNGM_URL=http://bridgeheadstore:8080 +#NNGM_PROFILE=http://uk-koeln.de/fhir/StructureDefinition/Patient/nNGM/pseudonymisiert +#NNGM_MAINZELLISTE_URL=https://test.verbis.dkfz.de/mpl + +##MDR +MDR_NAMESPACE=adt,dktk,marker +#MDR_MAP= +MDR_VALIDATION=false + +ML_DB_NAME=samply.connector +ML_DB_PORT=5432 +ML_DB_USER=samply.connector + +CENTRAL_CONTROL_NUMBER_GENERATPR_URL=http://e260-serv-03/central/api +GLOBAL_ID=DKTK +MAINZELLISTE_URL=https://patientlist-test.ccpit.dktk.dkfz.de/mainzelliste +ML_DB_DRIVER=org.postgresql.Driver +ML_DB_TYPE=postgresql + +ML_LOG_LEVEL=warning +ML_SITE=BRIDGEHEAD_DKTK_TEST +TZ=Europe/Berlin + +MAGICPL_SITE=adt +MAGICPL_LOG_LEVEL=info \ No newline at end of file diff --git a/gbn/docker-compose.yml b/gbn/docker-compose.yml index e2f6bad..684e693 100644 --- a/gbn/docker-compose.yml +++ b/gbn/docker-compose.yml @@ -1,6 +1,17 @@ version: '3.4' -services: +networks: + samply: + driver: "bridge" + connector-db: + driver: "bridge" +volumes: + store-db-data: + name: "store-db-data" + connector-db-data: + name: "connector-db-data" + +services: store: container_name: "bridgehead_gbn_blaze_store" image: "samply/blaze:0.15" @@ -20,12 +31,12 @@ services: image: "samply/share-client:gbn-7" environment: POSTGRES_HOST: "bridgehead_gbn_connector_db" - POSTGRES_DB: "samply.connector" - POSTGRES_USER: "samply" POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} - STORE_URL: "http://bridgehead_store:8080/fhir" - QUERY_LANGUAGE: "CQL" - MDR_URL: "https://mdr.germanbiobanknode.de/v3/api/mdr" + POSTGRES_PORT: 5432 + STORE_URL: "http://bridgehead_gbn_blaze_store:8080/fhir" + env_file: + ###TODO: Move into Image + - internal.env networks: - "samply" - "connector-db" @@ -39,23 +50,13 @@ services: container_name: "bridgehead_gbn_connector_db" image: "postgres:10.17" environment: - POSTGRES_USER: "samply" POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} - POSTGRES_DB: "samply.connector" + POSTGRES_PORT: 5432 networks: - "connector-db" volumes: - "connector-db-data:/var/lib/postgresql/data" restart: "always" - -networks: - samply: - driver: "bridge" - connector-db: - driver: "bridge" - -volumes: - store-db-data: - name: "store-db-data" - connector-db-data: - name: "connector-db-data" + env_file: + ###TODO: Move into Image + - internal.env \ No newline at end of file diff --git a/gbn/internal.env b/gbn/internal.env new file mode 100644 index 0000000..0fa1a6f --- /dev/null +++ b/gbn/internal.env @@ -0,0 +1,26 @@ +POSTGRES_DB=samply.connector +POSTGRES_USER=samply +QUERY_LANGUAGE=CQL +CONNECTOR_SHARE_URL="http://${HOST}:8080" +MDR_URL="https://mdr.germanbiobanknode.de/v3/api/mdr" +MONITOR_URL=https://ccpit.dktk.dkfz.de/dktk.monitor/rest/info +no_proxy=bridgehead_gbn_connector,bridgehead_gbn_blaze_store,bridgehead_gbn_connector_db +CONNECTOR_MONITOR_INTERVAL= +CONNECTOR_UPDATE_SERVER= +feature_BBMRI_DIRECTORY_SYNC=false +feature_DKTK_CENTRAL_SEARCH=false +feature_NNGM_CTS=false +LOG_LEVEL=info + +HTTP_PROXY_HOST=${PROXY_URL} +HTTP_PROXY_USER= +HTTP_PROXY_PASSWORD= +HTTPS_PROXY_HOST=${PROXY_URL} +HTTPS_PROXY_USER= +HTTPS_PROXY_PASSWORD= +HTTP_PROXY=${PROXY_URL} +HTTPS_PROXY=${PROXY_URL} +http_proxy=${PROXY_URL} +https_proxy=${PROXY_URL} + +CONNECTOR_ENABLE_METRICS=false diff --git a/install-bridgehead.sh b/install-bridgehead.sh index d360453..7a6793c 100755 --- a/install-bridgehead.sh +++ b/install-bridgehead.sh @@ -11,7 +11,7 @@ if ! ./lib/prerequisites.sh; then fi source site.conf -log -e "\nInstalling systemd units ..." +echo -e "\nInstalling systemd units ..." cp -v \ lib/systemd/bridgehead\@.service \ lib/systemd/bridgehead-update\@.service \ @@ -23,12 +23,12 @@ systemctl daemon-reload echo if ! systemctl is-active --quiet bridgehead@"${project}"; then - log "Enabling autostart of bridgehead@${project}.service" + echo "Enabling autostart of bridgehead@${project}.service" systemctl enable bridgehead@"${project}" - log "Enabling nightly updates for bridgehead@${project}.service ..." + echo "Enabling nightly updates for bridgehead@${project}.service ..." systemctl enable --now bridgehead-update@"${project}".timer fi -log -e "\nDone - now start your bridgehead by running\n\tsystemctl start bridgehead@${project}.service\nor by rebooting your machine." +echo -e "\nDone - now start your bridgehead by running\n\tsystemctl start bridgehead@${project}.service\nor by rebooting your machine." # TODO: Configuration of the different modules diff --git a/site.dev.conf b/site.dev.conf index e39140b..662c644 100644 --- a/site.dev.conf +++ b/site.dev.conf @@ -4,9 +4,17 @@ ##Setting Network properties export HOSTIP=$(MSYS_NO_PATHCONV=1 docker run --rm --add-host=host.docker.internal:host-gateway ubuntu cat /etc/hosts | grep 'host.docker.internal' | awk '{print $1}'); export HOST= +export PROXY_URL= export site_name= ### Write the Project you want to start with the brigdehead ##Exmaple project=gbn -project=c4 +project= +###secrets +## Connector Password +export CONNECTOR_POSTGRES_PASS= + +##ID-Manager +#This needs to be set if the project is either c4 or dktk +export \ No newline at end of file diff --git a/update-bridgehead.sh b/update-bridgehead.sh index b00e376..fdd9a9f 100755 --- a/update-bridgehead.sh +++ b/update-bridgehead.sh @@ -3,7 +3,7 @@ service="bridgehead" source lib/functions.sh -if ! ./prerequisites.sh; then +if ! lib/prerequisites.sh; then log "Prerequisites failed, exiting" exit fi From 31525571e14e75ce5c2e3d74f80b6d9753051b58 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 21 Jan 2022 09:32:21 +0100 Subject: [PATCH 033/215] Make C4 and Dktk work --- c4/docker-compose.yml | 197 ++++++++++++++---------------------- c4/internal.env | 18 +++- dktk/docker-compose.yml | 84 ++++++++++------ dktk/internal.env | 9 +- gbn/docker-compose.yml | 58 ++++++++--- gbn/internal.env | 2 + landing/index.html | 216 ++++++++++++++++++++++++++++++++++++++++ stop-bridgehead.sh | 1 - 8 files changed, 409 insertions(+), 176 deletions(-) create mode 100644 landing/index.html diff --git a/c4/docker-compose.yml b/c4/docker-compose.yml index 584dfb5..0359daa 100644 --- a/c4/docker-compose.yml +++ b/c4/docker-compose.yml @@ -20,181 +20,136 @@ secrets: file: ../internal-configuration/proxy.xml services: + traefik: + container_name: bridgehead_traefik + image: traefik:2.4 + command: + - --api.insecure=true + - --entrypoints.web.address=:80 + - --entrypoints.web-secure.address=:443 + - --providers.docker=true + environment: + http_proxy: "" + HTTP_PROXY: "" + https_proxy: "" + HTTPS_PROXY: "" + ports: + - 80:80 + - 443:443 + - 8080:8080 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + + landing: + container_name: bridgehead_landingpage + image: nginx:stable + volumes: + - ../landing/:/usr/share/nginx/html + labels: + - "traefik.enable=true" + - "traefik.http.routers.landing.rule=PathPrefix(`/`)" + - "traefik.http.services.landing.loadbalancer.server.port=80" + connector: container_name: bridgehead_c4_connector - image: "samply/share-client:c4-7" + image: docker.verbis.dkfz.de/ccp/connector:3 environment: - SITE: ${SITE} - SITEID: ${SITEID} - CENTRAL_SEARCH: ${CCP_CENTRALSEARCH_URL} - DECENTRAL_SEARCH: ${CCP_DECENTRALSEARCH_URL} - MDR_URL: ${CCP_MDR_URL} - MONITOR_URL: ${CCP_MONITOR_URL} - SHARE_URL: "${PROTOCOL}://${HOST}:${PORT}" - ID_MANAGER_URL: ${ID_MANAGER_URL} - PROJECTPSEUDONYMISATION_URL: ${PROJECTPSEUDONYMISATION_URL} - PATIENTLIST_URL: ${PATIENTLIST_URL} - STORE_URL: ${LDM_URL} + STORE_URL: http://bridgehead_c4_store:8080 + LDM_URL: http://bridgehead_c4_store:8080 POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_PORT: ${CONNECTOR_POSTGRES_PORT} - POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} - POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} - POSTGRES_PASS: ${CONNECTOR_POSTGRES_PASS} - HTTP_PROXY: ${HTTP_PROXY_URL} - HTTPS_PROXY: ${HTTP_PROXY_URL} - HTTP_PROXY_USERNAME: ${HTTP_PROXY_USER} - HTTP_PROXY_PASSWORD: ${HTTP_PROXY_PASSWORD} - HTTPS_PROXY_URL: ${HTTPS_PROXY_URL} - HTTPS_PROXY_USERNAME: ${HTTPS_PROXY_USER} - HTTPS_PROXY_PASSWORD: ${HTTPS_PROXY_PASSWORD} - NO_PROXY: ${NO_PROXY} - TZ: Europe/Berlin + POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} + NNGM_PASSWORD: ${NNGM_PASSWORD} + NNGM_MAINZELLISTE_APIKEY: ${NNGM_MAINZELLISTE_APIKEY} volumes: - "connector_logs:/usr/local/tomcat/logs" + labels: + - "traefik.enable=true" + - "traefik.http.routers.c4_connector.rule=PathPrefix(`/c4-connector`)" + - "traefik.http.services.c4_connector.loadbalancer.server.port=8080" depends_on: - connector_db - ports: - - "8080:8080" restart: always - networks: - - "samply" - - "connector-db" + env_file: + ###TODO: Move into Image + - internal.env connector_db: container_name: bridgehead_c4_connector_db image: postgres:10.17 environment: POSTGRES_HOST: ${CONNECTOR_DB_HOST} - POSTGRES_DB: ${CONNECTOR_POSTGRES_DB} - POSTGRES_USER: ${CONNECTOR_POSTGRES_USER} POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} - TZ: Europe/Berlin volumes: - "connector_db_data:/var/lib/postgresql/data" restart: always - networks: - - "connector-db" - ports: - - "8088:5432" - + env_file: + ###TODO: Move into Image + - internal.env idmanager: - container_name: bridgehead_c4_idmanager - image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:0.2.0-RC23 + container_name: bridgehead_idmanager + image: docker.verbis.dkfz.de/ccp/idmanager:deploy-c4 environment: - MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/id-manager MAGICPL_MAINZELLISTE_API_KEY: ${MAGICPL_MAINZELLISTE_API_KEY} MAGICPL_API_KEY: ${MAGICPL_API_KEY} MAGICPL_API_KEY_CONNECTOR: ${MAGICPL_API_KEY_CONNECTOR} MAGICPL_PASSPHRASE: ${MAGICPL_PASSPHRASE} - MAGICPL_MAINZELLISTE_CENTRAL_URL: ${MAGICPL_MAINZELLISTE_CENTRAL_URL} MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${MAGICPL_MAINZELLISTE_CENTRAL_API_KEY} - MAGICPL_CENTRAL_URL: ${MAGICPL_CENTRAL_URL} MAGICPL_CENTRAL_API_KEY: ${MAGICPL_CENTRAL_API_KEY} - MAGICPL_OIDC_PROVIDER: ${MAGICPL_OIDC_PROVIDER} MAGICPL_OIDC_CLIENT_ID: ${MAGICPL_OIDC_CLIENT_ID} MAGICPL_OIDC_CLIENT_SECRET: ${MAGICPL_OIDC_CLIENT_SECRET} - MAGICPL_SITE: ${SITEID} - MAGICPL_LOG_LEVEL: info - TZ: Europe/Berlin volumes: - "idmanager_logs:/usr/local/tomcat/logs" secrets: - magicpl.docker.xml - dktk_bridgehead_info.docker.xml - proxy.docker.xml - depends_on: - - patientlist - ports: - - "8085:8080" - networks: - - "samply" - - patientlist: - container_name: bridgehead_c4_patientlist - image: medicalinformatics/mainzelliste:develop - environment: - ML_DB_DRIVER: org.postgresql.Driver - ML_DB_TYPE: postgresql - ML_DB_HOST: ${ML_DB_HOST} - ML_DB_PORT: ${ML_DB_PORT} - ML_DB_NAME: ${ML_DB_NAME} - ML_DB_USER: ${ML_DB_USER} - ML_DB_PASS: ${ML_DB_PASS} - ML_API_KEY: ${ML_API_KEY} - ML_LOG_LEVEL: warning - ML_SITE: ${SITEID} - TZ: Europe/Berlin + labels: + - "traefik.http.routers.idmanager.rule=PathPrefix(`/id-manager`)" + - "traefik.http.middlewares.idmanager_strip.stripprefix.prefixes=/id-mana\ + ger" + - "traefik.http.routers.idmanager.middlewares=idmanager_strip" + - "traefik.http.services.idmanager.loadbalancer.server.port=8080" env_file: - - ../site-config/patientlist.env - volumes: - - "patientlist_logs:/usr/local/tomcat/logs" - secrets: - - mainzelliste.docker.conf - - centralSearchPublicKey - depends_on: - - patientlist_db - ports: - - "8086:8080" - networks: - - "samply" - - patientlist_db: - container_name: bridgehead_c4_patientlist_db - image: postgres:13.1-alpine - environment: - POSTGRES_DB: ${ML_DB_NAME} - POSTGRES_USER: ${ML_DB_USER} - POSTGRES_PASSWORD: ${ML_DB_PASS} - TZ: Europe/Berlin - volumes: - - "patientlist_db_data:/var/lib/postgresql/data" - networks: - - "samply" + ###TODO: Move into Image + - internal.env store: container_name: bridgehead_c4_store image: docker.verbis.dkfz.de/ccp/samply.store:release-5.1.2 environment: - MDR_URL: ${CCP_MDR_URL} - MDR_NAMESPACE: ${MDR_NAMESPACE} - MDR_VALIDATION: ${MDR_VALIDATION} - POSTGRES_HOST: ${STORE_POSTGRES_HOST} - POSTGRES_PORT: ${STORE_POSTGRES_PORT} - POSTGRES_DB: ${STORE_POSTGRES_DB} - POSTGRES_USER: ${STORE_POSTGRES_USER} - POSTGRES_PASSWORD: ${STORE_POSTGRES_PASS} + POSTGRES_HOST: bridgehead_c4_store_db + POSTGRES_PORT: 5432 + POSTGRES_DB: samplystore + POSTGRES_USER: samplystore + POSTGRES_PASSWORD: samplystore TZ: Europe/Berlin volumes: - "store_logs:/usr/local/tomcat/logs" + labels: + - "traefik.enable=true" + - "traefik.http.routers.store_c4.rule=PathPrefix(`/c4-loc\ + aldatamanagement`)" depends_on: - store_db - ports: - - "8083:8080" restart: always - networks: - - "samply" + env_file: + ###TODO: Move into Image + - internal.env store_db: container_name: bridgehead_c4_store_db image: postgres:9.5-alpine command: postgres -c datestyle='iso, dmy' environment: - POSTGRES_PORT: ${STORE_POSTGRES_PORT} - POSTGRES_DB: ${STORE_POSTGRES_DB} - POSTGRES_USER: ${STORE_POSTGRES_USER} - POSTGRES_PASSWORD: ${STORE_POSTGRES_PASS} - TZ: Europe/Berlin + POSTGRES_PORT: 5432 + POSTGRES_DB: samplystore + POSTGRES_USER: samplystore + POSTGRES_PASSWORD: samplystore volumes: - "store_db_data:/var/lib/postgresql/data" restart: always - networks: - - "samply" - ports: - - "8084:5432" - -networks: - samply: - driver: "bridge" - connector-db: - driver: "bridge" \ No newline at end of file + env_file: + ###TODO: Move into Image + - internal.env \ No newline at end of file diff --git a/c4/internal.env b/c4/internal.env index f0c0daa..edb04ef 100644 --- a/c4/internal.env +++ b/c4/internal.env @@ -1,14 +1,17 @@ SITE=bridgehead_test SITEID=BRIDGEHEAD_TEST +TOMCAT_REVERSEPROXY_FQDN=${HOST}/c4-connector +DEPLOYMENT_CONTEXT=c4-connector + CONNECTOR_SHARE_URL="http://${HOST}:8080" CONNECTOR_ENABLE_METRICS=false CONNECTOR_MONITOR_INTERVAL= CONNECTOR_UPDATE_SERVER= -CONNECTOR_POSTGRES_PORT=5432 -CONNECTOR_POSTGRES_DB=samply.connector -CONNECTOR_POSTGRES_USER=samply.connector +POSTGRES_PORT=5432 +POSTGRES_DB=samply.connector +POSTGRES_USER=samply.connector HTTP_PROXY_HOST=${PROXY_URL} HTTP_PROXY_USER= @@ -22,9 +25,12 @@ http_proxy=${PROXY_URL} https_proxy=${PROXY_URL} CCP_CENTRALSEARCH_URL=https://centralsearch-test.dktk.dkfz.de/ +CENTRAL_SEARCH=https://centralsearch-test.dktk.dkfz.de/ CCP_DECENTRALSEARCH_URL=https://decentralsearch-test.ccp-it.dktk.dkfz.de/ +DECENTRAL_SEARCH=https://decentralsearch-test.ccp-it.dktk.dkfz.de/ CCP_MDR_URL=https://mdr.ccp-it.dktk.dkfz.de/v3/api/mdr +MDR_URL=https://mdr.ccp-it.dktk.dkfz.de/v3/api/mdr CCP_MONITOR_URL= MONITOR_OPTOUT= @@ -44,4 +50,8 @@ ML_DB_USER=mainzelliste CENTRAL_CONTROL_NUMBER_GENERATPR_URL=http://e260-serv-03/central/api GLOBAL_ID=DKTK -MAINZELLISTE_URL=https://patientlist-test.ccpit.dktk.dkfz.de/mainzelliste \ No newline at end of file +MAINZELLISTE_URL=https://patientlist-test.ccpit.dktk.dkfz.de/mainzelliste + +NNGM_URL=https://test.anforderung.nngm.de/trial/nNGMv09/import-fb +NNGM_PROFILE=http://uk-koeln.de/fhir/StructureDefinition/Patient/nNGM/pseudonymisiert +NNGM_MAINZELLISTE_URL=https://test.verbis.dkfz.de/mpl diff --git a/dktk/docker-compose.yml b/dktk/docker-compose.yml index defaea7..c8b748a 100644 --- a/dktk/docker-compose.yml +++ b/dktk/docker-compose.yml @@ -20,6 +20,36 @@ secrets: file: ../internal-configuration/proxy.xml services: + traefik: + container_name: bridgehead_traefik + image: traefik:2.4 + command: + - --api.insecure=true + - --entrypoints.web.address=:80 + - --entrypoints.web-secure.address=:443 + - --providers.docker=true + environment: + http_proxy: "" + HTTP_PROXY: "" + https_proxy: "" + HTTPS_PROXY: "" + ports: + - 80:80 + - 443:443 + - 8080:8080 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + + landing: + container_name: bridgehead_landingpage + image: nginx:stable + volumes: + - ../landing/:/usr/share/nginx/html + labels: + - "traefik.enable=true" + - "traefik.http.routers.landing.rule=PathPrefix(`/`)" + - "traefik.http.services.landing.loadbalancer.server.port=80" + connector: container_name: bridgehead_dktk_connector image: "samply/share-client:dktk-7" @@ -31,15 +61,14 @@ services: - "connector_logs:/usr/local/tomcat/logs" depends_on: - connector_db - ports: - - "8080:8080" restart: always + labels: + - "traefik.enable=true" + - "traefik.http.routers.dktk_connector.rule=PathPrefix(`/dktk-connector`)" + - "traefik.http.services.dktk_connector.loadbalancer.server.port=8080" env_file: ###TODO: Move into Image - internal.env - networks: - - "samply" - - "connector-db" connector_db: container_name: bridgehead_dktk_connector_db @@ -53,27 +82,20 @@ services: env_file: ###TODO: Move into Image - internal.env - networks: - - "connector-db" - idmanager: - container_name: bridgehead_dktk_idmanager + container_name: bridgehead_id-manager image: docker.verbis.dkfz.de/pseudonymisierung/magicpl:release-0.2.0 environment: - MAGICPL_MAINZELLISTE_URL: ${MAGICPL_MAINZELLISTE_URL} + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/id-manager MAGICPL_MAINZELLISTE_API_KEY: ${MAGICPL_MAINZELLISTE_API_KEY} MAGICPL_API_KEY: ${MAGICPL_API_KEY} MAGICPL_API_KEY_CONNECTOR: ${MAGICPL_API_KEY_CONNECTOR} MAGICPL_PASSPHRASE: ${MAGICPL_PASSPHRASE} - MAGICPL_MAINZELLISTE_CENTRAL_URL: ${MAGICPL_MAINZELLISTE_CENTRAL_URL} MAGICPL_MAINZELLISTE_CENTRAL_API_KEY: ${MAGICPL_MAINZELLISTE_CENTRAL_API_KEY} - MAGICPL_CENTRAL_URL: ${MAGICPL_CENTRAL_URL} MAGICPL_CENTRAL_API_KEY: ${MAGICPL_CENTRAL_API_KEY} - MAGICPL_OIDC_PROVIDER: ${MAGICPL_OIDC_PROVIDER} MAGICPL_OIDC_CLIENT_ID: ${MAGICPL_OIDC_CLIENT_ID} MAGICPL_OIDC_CLIENT_SECRET: ${MAGICPL_OIDC_CLIENT_SECRET} - TZ: Europe/Berlin volumes: - "idmanager_logs:/usr/local/tomcat/logs" @@ -83,21 +105,23 @@ services: - proxy.docker.xml depends_on: - patientlist + labels: + - "traefik.http.routers.idmanager.rule=PathPrefix(`/id-manager`)" + - "traefik.http.middlewares.idmanager_strip.stripprefix.prefixes=/id-mana\ + ger" + - "traefik.http.routers.idmanager.middlewares=idmanager_strip" + - "traefik.http.services.idmanager.loadbalancer.server.port=8080" env_file: ###TODO: Move into Image - internal.env - ports: - - "8085:8080" - networks: - - "samply" patientlist: - container_name: bridgehead_dktk_patientlist + container_name: bridgehead_patientlist image: medicalinformatics/mainzelliste:develop environment: - ML_DB_HOST: bridgehead_dktk_patientlist_db ML_DB_PASS: ${ML_DB_PASS} ML_API_KEY: ${ML_API_KEY} + TOMCAT_REVERSEPROXY_FQDN: ${HOST}/patientlist env_file: - ../site-config/patientlist.env - internal.env @@ -106,12 +130,14 @@ services: secrets: - mainzelliste.docker.conf - centralSearchPublicKey + labels: + - "traefik.http.routers.patientlist.rule=PathPrefix(`/patientlist`)" + - "traefik.http.middlewares.patientlist_strip.stripprefix.prefixes=/patie\ + ntlist" + - "traefik.http.routers.patientlist.middlewares=patientlist_strip" + - "traefik.http.services.patientlist.loadbalancer.server.port=8080" depends_on: - patientlist_db - ports: - - "8086:8080" - networks: - - "samply" patientlist_db: container_name: bridgehead_dktk_patientlist_db @@ -122,12 +148,4 @@ services: - "patientlist_db_data:/var/lib/postgresql/data" env_file: ###TODO: Move into Image - - internal.env - networks: - - "samply" - -networks: - samply: - driver: "bridge" - connector-db: - driver: "bridge" \ No newline at end of file + - internal.env \ No newline at end of file diff --git a/dktk/internal.env b/dktk/internal.env index 94b7261..c2cab6d 100644 --- a/dktk/internal.env +++ b/dktk/internal.env @@ -6,6 +6,9 @@ CONNECTOR_ENABLE_METRICS=false CONNECTOR_MONITOR_INTERVAL= CONNECTOR_UPDATE_SERVER= +TOMCAT_REVERSEPROXY_FQDN=${HOST}/dktk-connector +DEPLOYMENT_CONTEXT=dktk-connector + POSTGRES_PORT=5432 POSTGRES_DB=samply.connector POSTGRES_USER=samply.connector @@ -38,6 +41,7 @@ MDR_NAMESPACE=adt,dktk,marker #MDR_MAP= MDR_VALIDATION=false +ML_DB_HOST=bridgehead_dktk_patientlist_db ML_DB_NAME=samply.connector ML_DB_PORT=5432 ML_DB_USER=samply.connector @@ -53,4 +57,7 @@ ML_SITE=BRIDGEHEAD_DKTK_TEST TZ=Europe/Berlin MAGICPL_SITE=adt -MAGICPL_LOG_LEVEL=info \ No newline at end of file +MAGICPL_LOG_LEVEL=info +MAGICPL_MAINZELLISTE_CENTRAL_URL=http://e260-serv-03/central/mainzelliste +MAGICPL_CENTRAL_URL=http://e260-serv-03/central/api +MAGICPL_OIDC_PROVIDER=https://auth-test.ccp-it.dktk.dkfz.de diff --git a/gbn/docker-compose.yml b/gbn/docker-compose.yml index 684e693..e3b7b54 100644 --- a/gbn/docker-compose.yml +++ b/gbn/docker-compose.yml @@ -1,9 +1,4 @@ version: '3.4' -networks: - samply: - driver: "bridge" - connector-db: - driver: "bridge" volumes: store-db-data: @@ -12,18 +7,52 @@ volumes: name: "connector-db-data" services: + traefik: + container_name: bridgehead_traefik + image: traefik:2.4 + command: + - --api.insecure=true + - --entrypoints.web.address=:80 + - --entrypoints.web-secure.address=:443 + - --providers.docker=true + environment: + http_proxy: "" + HTTP_PROXY: "" + https_proxy: "" + HTTPS_PROXY: "" + ports: + - 80:80 + - 443:443 + - 8080:8080 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + + landing: + container_name: bridgehead_landingpage + image: nginx:stable + volumes: + - ../landing/:/usr/share/nginx/html + labels: + - "traefik.enable=true" + - "traefik.http.routers.landing.rule=PathPrefix(`/`)" + - "traefik.http.services.landing.loadbalancer.server.port=80" + store: container_name: "bridgehead_gbn_blaze_store" image: "samply/blaze:0.15" environment: BASE_URL: "http://bridgehead_gbn_blaze_store:8080" JAVA_TOOL_OPTIONS: "-Xmx4g" - networks: - - "samply" volumes: - "store-db-data:/app/data" - ports: - - "8080:8080" + labels: + - "traefik.enable=true" + - "traefik.http.middlewares.test-auth.basicauth.users=user:$$apr1$$RjhTX8\ + ji$$3ToBWBch1K4RBnvp86TuH/" + - "traefik.http.routers.blaze_gbn.rule=PathPrefix(`/gbn-localdatamanagement`)" + - "traefik.http.middlewares.gbn_b_strip.stripprefix.prefixes=/gbn-localdatamanagement" + - "traefik.http.services.blaze_gbn.loadbalancer.server.port=8080" + - "traefik.http.routers.blaze_gbn.middlewares=gbn_b_strip,test-auth" restart: "always" connector: @@ -37,11 +66,10 @@ services: env_file: ###TODO: Move into Image - internal.env - networks: - - "samply" - - "connector-db" - ports: - - "8082:8080" + labels: + - "traefik.enable=true" + - "traefik.http.routers.gbn_connector.rule=PathPrefix(`/gbn-connector`)" + - "traefik.http.services.gbn_connector.loadbalancer.server.port=8080" depends_on: - "connector-db" restart: "always" @@ -52,8 +80,6 @@ services: environment: POSTGRES_PASSWORD: ${CONNECTOR_POSTGRES_PASS} POSTGRES_PORT: 5432 - networks: - - "connector-db" volumes: - "connector-db-data:/var/lib/postgresql/data" restart: "always" diff --git a/gbn/internal.env b/gbn/internal.env index 0fa1a6f..57806f0 100644 --- a/gbn/internal.env +++ b/gbn/internal.env @@ -5,6 +5,8 @@ CONNECTOR_SHARE_URL="http://${HOST}:8080" MDR_URL="https://mdr.germanbiobanknode.de/v3/api/mdr" MONITOR_URL=https://ccpit.dktk.dkfz.de/dktk.monitor/rest/info no_proxy=bridgehead_gbn_connector,bridgehead_gbn_blaze_store,bridgehead_gbn_connector_db +TOMCAT_REVERSEPROXY_FQDN=${HOST}/gbn-connector +DEPLOYMENT_CONTEXT=gbn-connector CONNECTOR_MONITOR_INTERVAL= CONNECTOR_UPDATE_SERVER= feature_BBMRI_DIRECTORY_SYNC=false diff --git a/landing/index.html b/landing/index.html new file mode 100644 index 0000000..bc3fad7 --- /dev/null +++ b/landing/index.html @@ -0,0 +1,216 @@ + + + + + + + Bridgehead Overview + + + + + + + + + + +
+ There is a componente update for your bridgehead. Please run the updater +
+ +
+ Your Bridgehead was updated! +
+ + diff --git a/lib/setup-bridgehead-units.sh b/lib/setup-bridgehead-units.sh index d70adcd..cd37a6c 100755 --- a/lib/setup-bridgehead-units.sh +++ b/lib/setup-bridgehead-units.sh @@ -1,9 +1,7 @@ #!/bin/bash -### Note: Currently not complete, needs some features before useable for production source lib/functions.sh -<<<<<<< HEAD exitIfNotRoot if [ $# -eq 0 ]; then @@ -18,8 +16,6 @@ fi export project=$1 -======= ->>>>>>> 290fe5459d7399ff23a2a8db067c1728858350d5 if ! ./lib/prerequisites.sh; then log "Prerequisites failed, exiting" exit 1 From d36ad9b4e3cf5c90a26e3a5a580d6a35551f1c21 Mon Sep 17 00:00:00 2001 From: Patrick Skowronek Date: Thu, 5 May 2022 14:17:57 +0200 Subject: [PATCH 078/215] Fix some more merge conflicts and some improvements --- .gitignore | 2 +- README.md | 16 ----------- landing/index.html | 64 -------------------------------------------- lib/generate.sh | 3 +-- lib/prerequisites.sh | 12 +++++++++ start-bridgehead.sh | 2 ++ 6 files changed, 16 insertions(+), 83 deletions(-) delete mode 100644 landing/index.html diff --git a/.gitignore b/.gitignore index 2d9a08b..f289dbd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ config/**/* !config/**/*.default docker-compose.override.yml site.conf -lading/* +landing/* certs/* diff --git a/README.md b/README.md index 4feaa10..9fced0e 100644 --- a/README.md +++ b/README.md @@ -147,26 +147,10 @@ sudo mkdir -p /srv/docker/; sudo git clone https://github.com/samply/bridgehead.git /srv/docker/bridgehead; ``` -<<<<<<< HEAD adduser --no-create-home --disabled-login --ingroup docker --gecos "" bridgehead useradd -M -g docker -N -s /sbin/nologin bridgehead chown bridghead /srv/docker/bridgehead/ -R -======= -The next step is to create a user for the bridgehead service - -``` shell -#!/bin/bash - -mkdir /srv/docker && cd /srv/docker - -adduser --no-create-home --disabled-login --ingroup docker --gecos "" bridgehead -useradd -M -g docker -N -s /sbin/nologin bridgehead - -chown 777 /srv/docker/bridgehead bridgehead -sudo chown bridgehead /srv/docker/bridgehead/ -``` ->>>>>>> version-1 Next, you need to configure a set of variables, specific for your site with not so high security concerns. You can visit the configuration template at [GitHub](https://github.com/samply/bridgehead-config). You can download the repositories contents and add them to the "bridgehead-config" directory. diff --git a/landing/index.html b/landing/index.html deleted file mode 100644 index c652299..0000000 --- a/landing/index.html +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - Bridgehead Overview - - - - - - - - - -