From 80f0942026c96736b0e2896ec71232602f48e4da Mon Sep 17 00:00:00 2001
From: chausman <chausman@nrao.edu>
Date: Wed, 4 Aug 2021 15:04:02 -0600
Subject: [PATCH] tweaking settings and ingestion script
---
.../wf_framework/sh/ingest-request.sh | 5 +-
...faa66_add_memory_to_ingestion_workflows.py | 122 ++++++++++++++++++
2 files changed, 126 insertions(+), 1 deletion(-)
create mode 100644 schema/versions/d6d6c2ffaa66_add_memory_to_ingestion_workflows.py
diff --git a/apps/cli/executables/wf_framework/sh/ingest-request.sh b/apps/cli/executables/wf_framework/sh/ingest-request.sh
index c1a6426bb..4c5f3b9c2 100644
--- a/apps/cli/executables/wf_framework/sh/ingest-request.sh
+++ b/apps/cli/executables/wf_framework/sh/ingest-request.sh
@@ -1,4 +1,7 @@
#!/usr/bin/env bash
+#
+# For use ***outside*** of docker containers - needs to be easily accessible to DAs and such
+#
set -o errexit -o nounset
@@ -28,7 +31,7 @@ case $option in
;;
esac
-WORKFLOW_SERVICE=$(pycapo -q edu.nrao.workspaces.WorkflowSettings.serviceUrl)
+WORKFLOW_SERVICE=$(capo -q edu.nrao.workspaces.WorkflowSettings.serviceUrl)
if [ "$action" = "ingest_cal" ]; then
curl -X POST $WORKFLOW_SERVICE/workflows/std_calibration/requests/$2/ingest
diff --git a/schema/versions/d6d6c2ffaa66_add_memory_to_ingestion_workflows.py b/schema/versions/d6d6c2ffaa66_add_memory_to_ingestion_workflows.py
new file mode 100644
index 000000000..37b30ec3c
--- /dev/null
+++ b/schema/versions/d6d6c2ffaa66_add_memory_to_ingestion_workflows.py
@@ -0,0 +1,122 @@
+"""add memory to ingestion workflows
+
+Revision ID: d6d6c2ffaa66
+Revises: d295036a7c72
+Create Date: 2021-08-04 14:16:40.092084
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "d6d6c2ffaa66"
+down_revision = "d295036a7c72"
+branch_labels = None
+depends_on = None
+
+
+cal_content = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/calibration-table-collector.sh, ./metadata.json
+
+request_memory = {{ramInGB}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1"
+
+queue
+
+"""
+
+img_content = """executable = ingest_image.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/image-product-collector.sh, ./metadata.json, ./aux_image_metadata.json
+
+request_memory = {{ramInGB}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+old_cal = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/calibration-table-collector.sh, ./metadata.json
+
+request_memory = 1G
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1"
+
+queue
+
+"""
+
+old_img = """executable = ingest_image.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/image-product-collector.sh, ./metadata.json, ./aux_image_metadata.json
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+
+def upgrade():
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{cal_content}' WHERE filename='ingest_cal.condor'
+ """
+ )
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{img_content}' WHERE filename='ingest_image.condor'
+ """
+ )
+
+
+def downgrade():
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{old_cal}' WHERE filename='ingest_cal.condor'
+ """
+ )
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{old_img}' WHERE filename='ingest_image.condor'
+ """
+ )
--
GitLab