update non-dag capabilities to use file transfer plugin

schema/versions/2d19b2a6e999_update_seci_carta_for_transfer_plugin.py

+"""update seci for transfer plugin
+
+Revision ID: 2d19b2a6e999
+Revises: 78c9a733483f
+Create Date: 2021-11-04 11:25:16.153179
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "2d19b2a6e999"
+down_revision = "78c9a733483f"
+branch_labels = None
+depends_on = None
+
+seci_condor = """executable = vlass_seci.sh
+arguments = metadata.json PPR.xml
+
+output = seci.out
+error = seci.err
+log = condor.log
+
+VLASS_DIR = {{data_location}}
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/framework.sh, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products, nraorsync://$(VLASS_DIR)/{{profile}}.properties, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
++nrao_output_files = "rawdata working products"
+output_destination = nraorsync://$(VLASS_DIR)
++WantIOProxy = True
+
+request_memory = 50G
+getenv = True
+environment = "CAPO_PATH=."
+
+{{#remote}}
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True)
+{{/remote}}
+
+queue
+
+"""
+
+carta_condor = """executable = carta.sh
+arguments = {{source_dir}} {{user_email}} {{request_id}} {{notify_ready}}
+
+output = carta.out
+error = carta.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{spool_dir}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/framework.sh, nraorsync://$(SBIN_PATH)/productfetcher, nraorsync://$(SBIN_PATH)/carta_envoy{{#remote-cv}}, nraorsync://$(SPOOL_DIR)/{{profile}}.properties{{/remote-cv}}
++WantIOProxy = True
+
+request_memory = 16G
+getenv = True
+{{^remote-cv}}
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+{{/remote-cv}}
+{{#remote-cv}}
+requirements = HasLustre == True && (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+queue
+
+"""
+
+
+old_seci_condor = """executable = vlass_seci.sh
+arguments = metadata.json PPR.xml
+
+output = seci.out
+error = seci.err
+log = condor.log
+
+VLASS_DIR = {{data_location}}
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/vela, $(SBIN_PATH)/casa_envoy, ./working, ./rawdata, ./products, {{profile}}.properties, ./PPR.xml, ./metadata.json
+transfer_output_files = rawdata, working, products
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=."
+
+{{#remote}}
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True)
+{{/remote}}
+
+queue
+"""
+
+old_carta_condor = """executable = carta.sh
+arguments = {{source_dir}} {{user_email}} {{request_id}} {{notify_ready}}
+
+output = carta.out
+error = carta.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/carta_envoy{{#remote-cv}}, {{profile}}.properties{{/remote-cv}}
+{{#need_data}}transfer_output_files = rawdata{{/need_data}}
+
+request_memory = {{ramInGb}}
+getenv = True
+{{^remote-cv}}
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+{{/remote-cv}}
+{{#remote-cv}}
+requirements = HasLustre == True && (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+queue
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{seci_condor}' WHERE filename='vlass_seci.condor'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{carta_condor}' WHERE filename='carta.condor'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{old_seci_condor}' WHERE filename='vlass_seci.condor'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{old_carta_condor}' WHERE filename='carta.condor'
+        """
+    )

schema/versions/3d0591e1b28e_update_ingestion_for_transfer_plugin.py

+"""update ingestion for transfer plugin
+
+Revision ID: 3d0591e1b28e
+Revises: 2d19b2a6e999
+Create Date: 2021-11-04 11:51:09.385687
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "3d0591e1b28e"
+down_revision = "2d19b2a6e999"
+branch_labels = None
+depends_on = None
+
+ingest_cal_condor = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/conveyor, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/calibration-table-collector.sh, nraorsync://{{spool_dir}}/metadata.json
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1"
+
+requirements = HasLustre == True
+
+queue
+
+"""
+
+ingest_img_condor = """executable = ingest_image.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{spool_dir}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/conveyor, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/image-product-collector.sh, nraorsync://$(SPOOL_DIR)/metadata.json, nraorsync://$(SPOOL_DIR)/aux_image_metadata.json
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+"""
+
+old_cal = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/calibration-table-collector.sh, ./metadata.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1"
+
+requirements = HasLustre == True
+
+queue
+"""
+
+old_img = """executable = ingest_image.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/image-product-collector.sh, ./metadata.json, ./aux_image_metadata.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{ingest_cal_condor}' WHERE filename='ingest_cal.condor'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{ingest_img_condor}' WHERE filename='ingest_image.condor'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{old_cal}' WHERE filename='ingest_cal.condor'
+        """
+    )
+    op.execute(
+        f"""
+            UPDATE workflow_templates
+            SET content=E'{old_img}' WHERE filename='ingest_image.condor'
+            """
+    )

schema/versions/78c9a733483f_update_null_download_for_transfer_plugin.py

+"""update null download for transfer plugin
+
+Revision ID: 78c9a733483f
+Revises: 5f2672e2233d
+Create Date: 2021-11-04 11:08:04.638814
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "78c9a733483f"
+down_revision = "5f2672e2233d"
+branch_labels = None
+depends_on = None
+
+null_script = """#!/bin/sh
+./null $*
+echo "I am a transfer test!" > null.txt
+"""
+
+null_condor = """executable = null.sh
+arguments = {{arguments}}
+
+output = null.out
+error = null.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/null
+
+transfer_output_files = .job.ad
++nrao_output_files = "null.txt"
+output_destination = nraorsync://{{spool_dir}}
++WantIOProxy = True
+{{#remote-cv}}
+
+#run on a condor node in CV-NAASC
+requirements = (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+{{#remote}}
+#run on a NMT condor node. (naming consistent with vlass-seci)
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True)
+{{/remote}}
+
+queue
+"""
+
+download_condor = """
+executable = test_download.sh
+arguments = {{product_locator}} {{request_id}}
+
+output = test_download.out
+error = test_download.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/productfetcher, nraorsync://$(SBIN_PATH)/deliver{{#remote-cv}}, nraorsync://{{spool_dir}}/{{profile}}.properties{{/remote-cv}}
+
++WantIOProxy = True
+{{^remote-cv}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+{{/remote-cv}}
+{{#remote-cv}}
+environment = "CAPO_PATH=. CAPO_PROFILE={{profile}}"
+
+requirements = HasLustre == True && (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+queue
+
+"""
+
+
+old_null_script = """#!/bin/sh
+./null $*
+"""
+
+old_null_condor = """executable = null.sh
+arguments = {{arguments}}
+
+output = null.out
+error = null.err
+log = condor.log
+should_transfer_files = yes
+transfer_input_files = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin/null
+
+{{#remote-cv}}
+#run on a condor node in CV-NAASC
+requirements = (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+{{#remote}}
+#run on a NMT condor node. (naming consistent with vlass-seci)
+requirements = (HasLustre =!= True)
+{{/remote}}
+
+queue
+
+"""
+
+old_download_condor = """executable = test_download.sh
+arguments = {{product_locator}} {{request_id}}
+
+output = test_download.out
+error = test_download.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/deliver{{#remote-cv}}, {{profile}}.properties{{/remote-cv}}
+
+{{^remote-cv}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+{{/remote-cv}}
+{{#remote-cv}}
+environment = "CAPO_PATH=. CAPO_PROFILE={{profile}}"
+
+requirements = HasLustre == True && (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+queue
+
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{null_script}' WHERE filename='null.sh'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{null_condor}' WHERE filename='null.condor'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{download_condor}' WHERE filename='test_download.condor'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{old_null_script}' WHERE filename='null.sh'
+        """
+    )
+    op.execute(
+        f"""
+            UPDATE workflow_templates
+            SET content=E'{old_null_condor}' WHERE filename='null.condor'
+            """
+    )
+    op.execute(
+        f"""
+            UPDATE workflow_templates
+            SET content=E'{old_download_condor}' WHERE filename='test_download.condor'
+            """
+    )