From 7eb4813bde25795a45ee79fd8fd6c965907b7747 Mon Sep 17 00:00:00 2001
From: chausman <chausman@nrao.edu>
Date: Mon, 12 Sep 2022 12:14:54 -0600
Subject: [PATCH] add weblog caching to vlass cals
---
...38d2307_add_vlass_calibration_templates.py | 44 +++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/shared/workspaces/alembic/versions/5dd6938d2307_add_vlass_calibration_templates.py b/shared/workspaces/alembic/versions/5dd6938d2307_add_vlass_calibration_templates.py
index eadf16dc6..d4f99fdb9 100644
--- a/shared/workspaces/alembic/versions/5dd6938d2307_add_vlass_calibration_templates.py
+++ b/shared/workspaces/alembic/versions/5dd6938d2307_add_vlass_calibration_templates.py
@@ -22,7 +22,11 @@ VARS FETCH jobname="$(JOB)"
JOB ENVOY vlass_calibration_envoy.condor
VARS ENVOY jobname="$(JOB)"
+JOB CONVEY vlass_calibration_convey.condor
+VARS CONVEY jobname="$(JOB)"
+
PARENT FETCH CHILD ENVOY
+PARENT ENVOY CHILD CONVEY
"""
fetch_condor = """executable = vlass_calibration_fetch.sh
@@ -101,6 +105,34 @@ set -o errexit
./casa_envoy --vlass-cal $1 $2
"""
+convey_condor = """executable = vlass_calibration_convey.sh
+arguments = metadata.json
+
+output = calibration_convey.out
+error = calibration_convey.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/conveyor, nraorsync://$(SPOOL_DIR)/metadata.json, nraorsync://$(SPOOL_DIR)/working, nraorsync://$(SPOOL_DIR)/rawdata, nraorsync://$(SPOOL_DIR)/products
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+
+queue
+"""
+
+convey_script = """#!/bin/sh
+set -o errexit
+
+export HOME=$TMPDIR
+
+./conveyor --vlass $1
+"""
+
def upgrade():
op.execute(
@@ -138,6 +170,18 @@ def upgrade():
VALUES ('vlass_calibration_envoy.sh', E'{envoy_script}', 'vlass_calibration')
"""
)
+ op.execute(
+ f"""
+ INSERT INTO workflow_templates (filename, content, workflow_name)
+ VALUES ('vlass_calibration_convey.condor', E'{convey_condor}', 'vlass_calibration')
+ """
+ )
+ op.execute(
+ f"""
+ INSERT INTO workflow_templates (filename, content, workflow_name)
+ VALUES ('vlass_calibration_convey.sh', E'{convey_script}', 'vlass_calibration')
+ """
+ )
def downgrade():
--
GitLab