initial vlass calibration templates

shared/workspaces/alembic/versions/5dd6938d2307_add_vlass_calibration_templates.py

@@ -6,7 +6,6 @@ Create Date: 2022-09-12 10:01:22.749122
 
 """
 from alembic import op
-import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
@@ -22,7 +21,11 @@ VARS FETCH jobname="$(JOB)"
 JOB ENVOY vlass_calibration_envoy.condor
 VARS ENVOY jobname="$(JOB)"
 
+JOB CONVEY vlass_calibration_convey.condor
+VARS CONVEY jobname="$(JOB)"
+
 PARENT FETCH CHILD ENVOY
+PARENT ENVOY CHILD CONVEY
 """
 
 fetch_condor = """executable = vlass_calibration_fetch.sh
@@ -71,7 +74,7 @@ log = condor.log
 VLASS_DIR = {{data_location}}
 SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
 should_transfer_files = yes
-transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products, nraorsync://$(VLASS_DIR)/{{profile}}.properties, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
 when_to_transfer_output = ON_EXIT
 transfer_output_files = .job.ad
 +nrao_output_files = "working products"
@@ -80,8 +83,11 @@ output_destination = nraorsync://$(VLASS_DIR)
 
 request_memory = 80G
 getenv = True
-environment = "CAPO_PATH=."
 
+{{^remote}}
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+{{/remote}}
 {{#remote}}
 requirements = (VLASS == True)
 +partition = "VLASS"
@@ -95,12 +101,44 @@ queue
 envoy_script = """#!/bin/sh
 
 export HOME=$TMPDIR
+{{#remote}}
+export CAPO_PATH=.
+export CAPO_PROFILE={{profile}}
+{{/remote}}
 
 set -o errexit
 
 ./casa_envoy --vlass-cal $1 $2
 """
 
+convey_condor = """executable = vlass_calibration_convey.sh
+arguments = metadata.json
+
+output = calibration_convey.out
+error = calibration_convey.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/conveyor, nraorsync://$(SPOOL_DIR)/metadata.json, nraorsync://$(SPOOL_DIR)/working, nraorsync://$(SPOOL_DIR)/rawdata, nraorsync://$(SPOOL_DIR)/products
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+
+queue
+"""
+
+convey_script = """#!/bin/sh
+set -o errexit
+
+export HOME=$TMPDIR
+
+./conveyor --vlass $1
+"""
+
 
 def upgrade():
     op.execute(
@@ -138,6 +176,18 @@ def upgrade():
         VALUES ('vlass_calibration_envoy.sh', E'{envoy_script}', 'vlass_calibration') 
         """
     )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_convey.condor', E'{convey_condor}', 'vlass_calibration') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_convey.sh', E'{convey_script}', 'vlass_calibration') 
+        """
+    )
 
 
 def downgrade():