diff --git a/apps/cli/executables/wf_framework/sh/ingest-request.sh b/apps/cli/executables/wf_framework/sh/ingest-request.sh index c1a6426bbee9e3e853f22074cd7d433952a485e5..4c5f3b9c2a97207367333c28d87fee5efa1bafc7 100644 --- a/apps/cli/executables/wf_framework/sh/ingest-request.sh +++ b/apps/cli/executables/wf_framework/sh/ingest-request.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# +# For use ***outside*** of docker containers - needs to be easily accessible to DAs and such +# set -o errexit -o nounset @@ -28,7 +31,7 @@ case $option in ;; esac -WORKFLOW_SERVICE=$(pycapo -q edu.nrao.workspaces.WorkflowSettings.serviceUrl) +WORKFLOW_SERVICE=$(capo -q edu.nrao.workspaces.WorkflowSettings.serviceUrl) if [ "$action" = "ingest_cal" ]; then curl -X POST $WORKFLOW_SERVICE/workflows/std_calibration/requests/$2/ingest diff --git a/schema/versions/d6d6c2ffaa66_add_memory_to_ingestion_workflows.py b/schema/versions/d6d6c2ffaa66_add_memory_to_ingestion_workflows.py new file mode 100644 index 0000000000000000000000000000000000000000..e0f56835f2747332a1f42e607878bcb774ea664e --- /dev/null +++ b/schema/versions/d6d6c2ffaa66_add_memory_to_ingestion_workflows.py @@ -0,0 +1,122 @@ +"""add memory to ingestion workflows + +Revision ID: d6d6c2ffaa66 +Revises: d295036a7c72 +Create Date: 2021-08-04 14:16:40.092084 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "d6d6c2ffaa66" +down_revision = "d295036a7c72" +branch_labels = None +depends_on = None + + +cal_content = """executable = ingest_cal.sh +arguments = metadata.json + +output = ingest.out +error = ingest.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/calibration-table-collector.sh, ./metadata.json + +request_memory = {{ramInGb}} +getenv = True +environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1" + +queue + +""" + +img_content = """executable = ingest_image.sh +arguments = metadata.json + +output = ingest.out +error = ingest.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/image-product-collector.sh, ./metadata.json, ./aux_image_metadata.json + +request_memory = {{ramInGb}} +getenv = True +environment = "CAPO_PATH=/home/casa/capo" + +queue + +""" + +old_cal = """executable = ingest_cal.sh +arguments = metadata.json + +output = ingest.out +error = ingest.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/calibration-table-collector.sh, ./metadata.json + +request_memory = 1G +getenv = True +environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1" + +queue + +""" + +old_img = """executable = ingest_image.sh +arguments = metadata.json + +output = ingest.out +error = ingest.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/image-product-collector.sh, ./metadata.json, ./aux_image_metadata.json + +getenv = True +environment = "CAPO_PATH=/home/casa/capo" + +queue + +""" + + +def upgrade(): + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{cal_content}' WHERE filename='ingest_cal.condor' + """ + ) + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{img_content}' WHERE filename='ingest_image.condor' + """ + ) + + +def downgrade(): + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{old_cal}' WHERE filename='ingest_cal.condor' + """ + ) + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{old_img}' WHERE filename='ingest_image.condor' + """ + ) diff --git a/services/workflow/Dockerfile b/services/workflow/Dockerfile index 286efb01c482ba250bd0b609d6fb48c0611fab1c..f0910f86c4425126bf6ea143afa95bab6c54b121 100644 --- a/services/workflow/Dockerfile +++ b/services/workflow/Dockerfile @@ -9,7 +9,7 @@ ENV ENV=${env} USER root # HTCondor install -RUN apt update && apt install -y wget gnupg sudo curl +RUN apt update && apt install -y wget gnupg sudo RUN wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | sudo apt-key add - RUN echo "deb http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" > /etc/apt/sources.list.d/htcondor.list RUN echo "deb-src http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list diff --git a/services/workflow/Dockerfile.local b/services/workflow/Dockerfile.local index 57e0d31ff6a2b30fb7a0a266a86c442ec6001bf6..beebb37b5d7832cce4d85c00bc6248ee238dda95 100644 --- a/services/workflow/Dockerfile.local +++ b/services/workflow/Dockerfile.local @@ -14,7 +14,7 @@ RUN python setup.py develop --user USER root # HTCondor install -RUN apt update && apt install -y wget gnupg sudo curl +RUN apt update && apt install -y wget gnupg sudo RUN wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | sudo apt-key add - RUN echo "deb http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" > /etc/apt/sources.list.d/htcondor.list RUN echo "deb-src http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list