initial vlass calibration templates

shared/workspaces/alembic/versions/5dd6938d2307_add_vlass_calibration_templates.py

+"""add vlass calibration templates
+
+Revision ID: 5dd6938d2307
+Revises: 1bf29ab04f27
+Create Date: 2022-09-12 10:01:22.749122
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "5dd6938d2307"
+down_revision = "1bf29ab04f27"
+branch_labels = None
+depends_on = None
+
+
+dag = """JOB FETCH vlass_calibration_fetch.condor
+VARS FETCH jobname="$(JOB)"
+
+JOB ENVOY vlass_calibration_envoy.condor
+VARS ENVOY jobname="$(JOB)"
+
+JOB CONVEY vlass_calibration_convey.condor
+VARS CONVEY jobname="$(JOB)"
+
+PARENT FETCH CHILD ENVOY
+PARENT ENVOY CHILD CONVEY
+"""
+
+fetch_condor = """executable = vlass_calibration_fetch.sh
+arguments = {{product_locator}}
+
+output = calibration_fetch.out
+error = calibration_fetch.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/productfetcher, nraorsync://$(SPOOL_DIR)/rawdata
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
++nrao_output_files = "rawdata"
+output_destination = nraorsync://$(SPOOL_DIR)
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+"""
+
+fetch_script = """#!/bin/sh
+set -o errexit
+
+export HOME=$TMPDIR
+
+chmod 770 .
+cd rawdata/
+../productfetcher --product-locator $1 
+cd ../
+"""
+
+envoy_condor = """executable = vlass_calibration_envoy.sh
+arguments = metadata.json PPR.xml
+
+output = calibration_envoy.out
+error = calibration_envoy.err
+log = condor.log
+
+VLASS_DIR = {{data_location}}
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
++nrao_output_files = "working products"
+output_destination = nraorsync://$(VLASS_DIR)
++WantIOProxy = True
+
+request_memory = 80G
+getenv = True
+
+{{^remote}}
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+{{/remote}}
+{{#remote}}
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True)
+{{/remote}}
+
+queue
+
+"""
+
+envoy_script = """#!/bin/sh
+
+export HOME=$TMPDIR
+{{#remote}}
+export CAPO_PATH=.
+export CAPO_PROFILE={{profile}}
+{{/remote}}
+
+set -o errexit
+
+./casa_envoy --vlass-cal $1 $2
+"""
+
+convey_condor = """executable = vlass_calibration_convey.sh
+arguments = metadata.json
+
+output = calibration_convey.out
+error = calibration_convey.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/conveyor, nraorsync://$(SPOOL_DIR)/metadata.json, nraorsync://$(SPOOL_DIR)/working, nraorsync://$(SPOOL_DIR)/rawdata, nraorsync://$(SPOOL_DIR)/products
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+
+queue
+"""
+
+convey_script = """#!/bin/sh
+set -o errexit
+
+export HOME=$TMPDIR
+
+./conveyor --vlass $1
+"""
+
+
+def upgrade():
+    op.execute(
+        """
+        INSERT INTO workflows (workflow_name, uses_casa) VALUES ('vlass_calibration', true);
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration.dag', E'{dag}', 'vlass_calibration') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_fetch.condor', E'{fetch_condor}', 'vlass_calibration') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_fetch.sh', E'{fetch_script}', 'vlass_calibration') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_envoy.condor', E'{envoy_condor}', 'vlass_calibration') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_envoy.sh', E'{envoy_script}', 'vlass_calibration') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_convey.condor', E'{convey_condor}', 'vlass_calibration') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('vlass_calibration_convey.sh', E'{convey_script}', 'vlass_calibration') 
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        """
+        DELETE FROM workflow_templates WHERE workflow_name = 'vlass_calibration'
+        """
+    )
+    op.execute(
+        """
+        DELETE FROM workflows WHERE workflow_name = 'vlass_calibration'
+        """
+    )