From 0d1f425df0755c6b573d27169238a2601ff933eb Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Mon, 18 Mar 2024 11:03:50 +0100 Subject: [PATCH] Auto-build focus and transfair This auto-build functionality is a workaround for a bug, described in README. The README now has additional documentation for EHDS2. The root certificate files have been expanded to contain certificates for both the ECDC and the DKFZ setup. --- README.md | 19 +++++++++ bbmri/modules/ehds2-compose.yml | 12 ++++-- bbmri/modules/ehds2.root.crt.pem | 24 +++++++++++ bbmri/modules/ehds2.test.root.crt.pem | 24 +++++++++++ bridgehead | 9 ++++ lib/functions.sh | 60 +++++++++++++++++++++++++++ 6 files changed, 145 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c0c8954..a6bf032 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ This repository is the starting point for any information and tools you will nee - [Hardware](#hardware) - [Software](#software) - [Network](#network) + - [EHDS2/ECDC](#ehds2-ecdc) 2. [Deployment](#deployment) - [Site name](#site-name) - [Projects](#projects) @@ -92,6 +93,24 @@ The following URLs need to be accessible (prefix with `https://`): > 📝 Ubuntu's pre-installed uncomplicated firewall (ufw) is known to conflict with Docker, more info [here](https://github.com/chaifeng/ufw-docker). +### EHDS2/ECDC + +ECDC data should be provided as a CSV file and placed in the directory /srv/docker/ecdc/data. The Bridgehead can be started without data, but obviously, any searches run from a Locator will return zero results for this site if you do that. Note that an empty data directory will automatically be inserted on the first start of the Bridgehead if you don't set one up yourself. + +To get the right Beam certificate for your setup, you will need to edit the following files and comment/uncomment as appropriate: + +- bbmri/modules/ehds2.root.crt.pem +- bbmri/modules/ehds2.test.root.crt.pem + +When you first start the Bridgehead, it will clone two extra repositories into /srv/docker, namely, focus and transfair. It will automatically build local images of these repositories for you. These components have the following functionality that has been customized for ECDC: + +- *focus.* This component is responsible for completing the CQL that is used for running queries against the Blaze FHIR store. It uses a set of templates for doing this. Extra templates have been written for the ECDC use case. +- *transfair.* This is an ETL component. It takes the CSV data that you provide, converts it to FHIR, and loads it to Blaze. + +These images will normally be rebuilt every time you restart the Bridgehead. This is a workaround to fix a bug: if you don't rebuild these images for every start, then legacy versions will be used and you will lose the new ECDC functionality. The reason for this is still under investigation. + +Note that the /srv/docker/ecdc/data directory is also used as the home for a "lock" file, which will be created if your data has been successfully converted to FHIR by transfair. As long as this file exists, transfair will not be run again when you restart the Bridgehead. Hence, if you add or modify data, you will need to delete this file by hand before restarting the Bridgehead. + ## Deployment ### Site name diff --git a/bbmri/modules/ehds2-compose.yml b/bbmri/modules/ehds2-compose.yml index a0c2eb5..6475199 100644 --- a/bbmri/modules/ehds2-compose.yml +++ b/bbmri/modules/ehds2-compose.yml @@ -46,11 +46,17 @@ services: #WRITE_BUNDLES_TO_FILE: "true" AMR_FILE_PATH: "/app/data" restart: on-failure - command: sh -c "sleep 60 && rm -rf /app/test/* && java -jar transFAIR.jar && tail -f /dev/null" + #command: sh -c "sleep 60 && rm -rf /app/test/* && ([ ! -f "/app/lock" && ] && java -jar transFAIR.jar && touch "/app/lock) && tail -f /dev/null" + #command: sh -c "sleep 60 && rm -rf /app/test/* && java -jar transFAIR.jar && tail -f /dev/null" + command: sh -c "echo listing /app/data && ls -la /app/data && /app/launch.sh" #command: sh -c "rm -rf /app/test/* && java -jar transFAIR.jar" volumes: - - /home/gerhard/Projects/EHDS2/PrototypeSpring2024/test/:/app/test/ - - /home/gerhard/Projects/EHDS2/PrototypeSpring2024/Data/:/app/data/ + #- /home/gerhard/Projects/EHDS2/PrototypeSpring2024/test/:/app/test/ + #- /home/gerhard/Projects/EHDS2/PrototypeSpring2024/Data/:/app/data/ + #- ../ecdc/test:/app/test/ + #- ../ecdc/data:/app/data/ + - ../../ecdc/test:/app/test/ + - ../../ecdc/data:/app/data/ # Report on the data pushed to Blaze by TransFAIR blazectl: diff --git a/bbmri/modules/ehds2.root.crt.pem b/bbmri/modules/ehds2.root.crt.pem index eae0d4d..9f8a275 100644 --- a/bbmri/modules/ehds2.root.crt.pem +++ b/bbmri/modules/ehds2.root.crt.pem @@ -1,3 +1,4 @@ +# DKFZ certificate -----BEGIN CERTIFICATE----- MIIDNTCCAh2gAwIBAgIUMy/n0zFRihhVR3aAD54LumzeYdwwDQYJKoZIhvcNAQEL BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjIxMDI1MDczNTA4WhcNMzIx @@ -18,3 +19,26 @@ rx2Adit76eZu/MJoJNzl1r8MPxLqyAie3KcIU54A+UMozLrWEQP/TyOyWZdjUjJt cBYgkKJTjwdRhc+ehI3kFo7b/a/Z/jl9szKsAPHozMixSi8lGnsYwN80oqeRvT7h wcMUK+igv3/K -----END CERTIFICATE----- + +# EHDS2/ECDC certificate +#-----BEGIN CERTIFICATE----- +#MIIDNTCCAh2gAwIBAgIUamQJB/Uevzf3+qrDQOqA8mwkwX8wDQYJKoZIhvcNAQEL +#BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjQwMjA3MTQ1MTQxWhcNMzQw +#MjA0MTQ1MjExWjAWMRQwEgYDVQQDEwtCcm9rZXItUm9vdDCCASIwDQYJKoZIhvcN +#AQEBBQADggEPADCCAQoCggEBAKvI2XT0os20vG7d8FjRuMq9mALcQgLZvjAbQZbm +#JyeVxRQP5HsSrBlBwaZWgmXWFfIPlwsEsTAUsCneAl4ZL3gQ6831nbmmpgThOEVy +#rGBsbL6qlEWWplD/AGpcdEbAuDmuMSO9tKR6iRm+acTHJcXqsDm2HUUS7U+ManWY +#S0dG10beJE2eVtEv5/fnFlVSn72VgAVgXVl7UA5E77zXK9GUAyQ6Oo7MET9zifZ/ +#EifeSoEvLk+U7HRLgQbKz2ti6ABh0wnK7Ec2DjekMBXyeQz0tR+8dS53s5pqti7c +#SUQMHmL4jKxvfQr7fr5wbJpFDu37Vif9GdND9WSDRcufuM8CAwEAAaN7MHkwDgYD +#VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFKMF6QgDV4cV +#WyQZn3EHKLUumXzuMB8GA1UdIwQYMBaAFKMF6QgDV4cVWyQZn3EHKLUumXzuMBYG +#A1UdEQQPMA2CC0Jyb2tlci1Sb290MA0GCSqGSIb3DQEBCwUAA4IBAQAhhE0ht7Sp +#inKLJSVZrdFHbAbRXnYcUz/3oyy/uw0YvF0dJAHz2sgbtCLMHzKU7NwotaEqZedA +#I7688nI42aHggIkwa1CzmFo9ybQ2afgrYNeF+TcTQI6G+fFpeRT5I+FvA9hUI3r4 +#Kp63Si7cpQS5uiHs90sv2SNMedo2Gf9VyJKB7NOReQJWsrw32ITi4QLNKHcHciKm +#cpi/OqbR1YTJRoUrcC655G7bSKENzccdTWo8kBJgYLVbe+FohqgheqPqWpdH2zoR +#OxOwviutfJTNvDyhXu8FrDc5tr9f5VjXv1hK2szYuvggwbybUBB9bLsx+fV79oZe +#3zR0j5YwB28G +#-----END CERTIFICATE----- + diff --git a/bbmri/modules/ehds2.test.root.crt.pem b/bbmri/modules/ehds2.test.root.crt.pem index 1db431b..1bac208 100644 --- a/bbmri/modules/ehds2.test.root.crt.pem +++ b/bbmri/modules/ehds2.test.root.crt.pem @@ -1,3 +1,4 @@ +# DKFZ certificate -----BEGIN CERTIFICATE----- MIIDNTCCAh2gAwIBAgIUJ0g7k2vrdAwNTU38S1/mU8NO26MwDQYJKoZIhvcNAQEL BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjMwNzEwMTIyMzQxWhcNMzMw @@ -18,3 +19,26 @@ UNe/254bc0vqHEPT6VI/86c7qAmk1xR0RUfrnKAEqZtUeuoj2fe1L/6yOB16fxt5 3V3oim7EO6eZCTjDo9fU5DaFiqSMe7WVdr03Na0cWet60XKRH/xaiC6gMWdHWcbh vZdXnV1qjlM2 -----END CERTIFICATE----- + +# EHDS2/ECDC certificate +#-----BEGIN CERTIFICATE----- +#MIIDNTCCAh2gAwIBAgIUamQJB/Uevzf3+qrDQOqA8mwkwX8wDQYJKoZIhvcNAQEL +#BQAwFjEUMBIGA1UEAxMLQnJva2VyLVJvb3QwHhcNMjQwMjA3MTQ1MTQxWhcNMzQw +#MjA0MTQ1MjExWjAWMRQwEgYDVQQDEwtCcm9rZXItUm9vdDCCASIwDQYJKoZIhvcN +#AQEBBQADggEPADCCAQoCggEBAKvI2XT0os20vG7d8FjRuMq9mALcQgLZvjAbQZbm +#JyeVxRQP5HsSrBlBwaZWgmXWFfIPlwsEsTAUsCneAl4ZL3gQ6831nbmmpgThOEVy +#rGBsbL6qlEWWplD/AGpcdEbAuDmuMSO9tKR6iRm+acTHJcXqsDm2HUUS7U+ManWY +#S0dG10beJE2eVtEv5/fnFlVSn72VgAVgXVl7UA5E77zXK9GUAyQ6Oo7MET9zifZ/ +#EifeSoEvLk+U7HRLgQbKz2ti6ABh0wnK7Ec2DjekMBXyeQz0tR+8dS53s5pqti7c +#SUQMHmL4jKxvfQr7fr5wbJpFDu37Vif9GdND9WSDRcufuM8CAwEAAaN7MHkwDgYD +#VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFKMF6QgDV4cV +#WyQZn3EHKLUumXzuMB8GA1UdIwQYMBaAFKMF6QgDV4cVWyQZn3EHKLUumXzuMBYG +#A1UdEQQPMA2CC0Jyb2tlci1Sb290MA0GCSqGSIb3DQEBCwUAA4IBAQAhhE0ht7Sp +#inKLJSVZrdFHbAbRXnYcUz/3oyy/uw0YvF0dJAHz2sgbtCLMHzKU7NwotaEqZedA +#I7688nI42aHggIkwa1CzmFo9ybQ2afgrYNeF+TcTQI6G+fFpeRT5I+FvA9hUI3r4 +#Kp63Si7cpQS5uiHs90sv2SNMedo2Gf9VyJKB7NOReQJWsrw32ITi4QLNKHcHciKm +#cpi/OqbR1YTJRoUrcC655G7bSKENzccdTWo8kBJgYLVbe+FohqgheqPqWpdH2zoR +#OxOwviutfJTNvDyhXu8FrDc5tr9f5VjXv1hK2szYuvggwbybUBB9bLsx+fV79oZe +#3zR0j5YwB28G +#-----END CERTIFICATE----- + diff --git a/bridgehead b/bridgehead index 8922174..1b897b7 100755 --- a/bridgehead +++ b/bridgehead @@ -89,6 +89,15 @@ case "$ACTION" in loadVars hc_send log "Bridgehead $PROJECT startup: Checking requirements ..." checkRequirements + # Local versions of focus and transfair are needed by EHDS2 + clone_focus_if_nonexistent ${BASE}/.. + build_focus ${BASE}/.. + clone_transfair_if_nonexistent ${BASE}/.. + build_transfair ${BASE}/.. + # Location for input data and results for EHDS2 + mkdir -p ${BASE}/../ecdc/test + mkdir -p ${BASE}/../ecdc/data + chown -R bridgehead ${BASE}/../ecdc hc_send log "Bridgehead $PROJECT startup: Requirements checked out. Now starting bridgehead ..." exec $COMPOSE -p $PROJECT -f ./minimal/docker-compose.yml -f ./$PROJECT/docker-compose.yml $OVERRIDE up --abort-on-container-exit ;; diff --git a/lib/functions.sh b/lib/functions.sh index 4d2bb2f..f84e791 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -239,3 +239,63 @@ add_basic_auth_user() { log DEBUG "Saving clear text credentials in $FILE. If wanted, delete them manually." sed -i "/^$NAME/ s|$|\n# User: $USER\n# Password: $PASSWORD|" $FILE } + +function clone_repo_if_nonexistent() { + local repo_url="$1" # First argument: Repository URL + local target_dir="$2" # Second argument: Target directory + local branch_name="$3" # Third argument: Branch name + + echo Repo directory: $target_dir + + # Check if the target directory exists + if [ ! -d "$target_dir" ]; then + echo "Directory '$target_dir' does not exist. Cloning the repository..." + # Clone the repository + git clone "$repo_url" "$target_dir" + fi + + # Change to the cloned directory + cd "$target_dir" + + # Checkout the specified branch + git checkout "$branch_name" + echo "Checked out branch '$branch_name'." + + cd - +} + +function clone_transfair_if_nonexistent() { + local base_dir="$1" + + clone_repo_if_nonexistent https://github.com/samply/transFAIR.git $base_dir/transfair main_ecdc_amt_prototype +} + +function clone_focus_if_nonexistent() { + local base_dir="$1" + + clone_repo_if_nonexistent https://github.com/samply/focus.git $base_dir/focus ehds2 +} + + +function build_transfair() { + local base_dir="$1" + + # We only take the touble to build transfair if: + # + # 1. There is no data lock file (which means that no ETL has yet been run) and + # 2. There is data available. + if [ ! -f "../ecdc/data/lock" ] && [ ! -z "$(ls -A ../ecdc/data)" ]; then + cd $base_dir/transfair + docker build --progress=plain -t samply/transfair --no-cache . + cd - + fi +} + +function build_focus() { + local base_dir="$1" + + cd $base_dir/focus + docker build --progress=plain -f DockerfileWithBuild -t samply/focus --no-cache . + cd - +} +