Ws 1082: WS side testing fixes (and framework.sh removal while I'm at it)

shared/workspaces/alembic/versions/bcc9821d3073_fix_fetch_no_framework_sh_required.py

+# Copyright (C) 2022 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+"""fix fetch (no framework.sh required)
+
+Revision ID: bcc9821d3073
+Revises: 0b36465f0e09
+Create Date: 2022-07-01 11:27:59.175009
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "bcc9821d3073"
+down_revision = "0b36465f0e09"
+branch_labels = None
+depends_on = None
+
+old_fetch_sh = """#!/bin/sh
+set -o errexit
+
+export HOME=$TMPDIR
+
+./framework.sh -d .
+chmod 770 .
+cd rawdata/
+../productfetcher --product-locator $1
+cd ../
+
+"""
+old_fetch_condor = """executable = vlass_coarse_fetch.sh
+arguments = {{product_locator}}
+
+output = vlass_coarse_fetch.out
+error = vlass_coarse_fetch.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/framework.sh, nraorsync://$(SBIN_PATH)/productfetcher, nraorsync://{{rawdata_dir}}
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
++nrao_output_files = "working rawdata products"
+output_destination = nraorsync://$(SPOOL_DIR)
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+"""
+
+new_fetch_sh = """#!/bin/sh
+set -o errexit
+
+export HOME=$TMPDIR
+
+chmod 770 .
+cd rawdata/
+../productfetcher --product-locator $1
+cd ../
+
+
+"""
+new_fetch_condor = """executable = vlass_coarse_fetch.sh
+arguments = {{product_locator}}
+
+output = vlass_coarse_fetch.out
+error = vlass_coarse_fetch.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/productfetcher, nraorsync://$(SPOOL_DIR)/rawdata
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
++nrao_output_files = "rawdata"
+output_destination = nraorsync://$(SPOOL_DIR)
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+"""
+
+old_envoy_condor = """executable = vlass_coarse.sh
+arguments = metadata.json PPR.xml
+
+output = vlass_coarse_envoy.out
+error = vlass_coarse_env0y.err
+log = condor.log
+
+VLASS_DIR = {{data_location}}
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/framework.sh, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products, nraorsync://$(VLASS_DIR)/{{profile}}.properties, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
++nrao_output_files = "rawdata working products"
+output_destination = nraorsync://$(VLASS_DIR)
++WantIOProxy = True
+
+request_memory = 50G
+request_disk = 850G
+getenv = True
+environment = "CAPO_PATH=."
+
+{{#remote}}
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True)
+{{/remote}}
+
+queue
+
+"""
+
+new_envoy_condor = """executable = vlass_coarse_envoy.sh
+arguments = metadata.json PPR.xml
+
+output = vlass_coarse_envoy.out
+error = vlass_coarse_envoy.err
+log = condor.log
+
+VLASS_DIR = {{data_location}}
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/framework.sh, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products, nraorsync://$(VLASS_DIR)/{{profile}}.properties, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
++nrao_output_files = "rawdata working products"
+output_destination = nraorsync://$(VLASS_DIR)
++WantIOProxy = True
+
+request_memory = 50G
+request_disk = 850G
+getenv = True
+environment = "CAPO_PATH=."
+
+{{#remote}}
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True)
+{{/remote}}
+
+queue
+
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+    UPDATE workflow_templates SET content = E'{new_fetch_sh}' WHERE workflow_name = 'vlass_coarse'
+    AND  filename='vlass_coarse_fetch.sh'
+    """
+    )
+
+    op.execute(
+        f"""
+    UPDATE workflow_templates SET content = E'{new_fetch_condor}' WHERE workflow_name = 'vlass_coarse'
+    AND  filename='vlass_coarse_fetch.condor'
+    """
+    )
+
+    op.execute(
+        f"""
+    UPDATE workflow_templates SET content = E'{new_envoy_condor}' WHERE workflow_name = 'vlass_coarse'
+    AND  filename='vlass_coarse_envoy.condor'
+    """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+    UPDATE workflow_templates SET content = E'{old_fetch_sh}' WHERE workflow_name = 'vlass_coarse'
+    AND  filename='vlass_coarse_fetch.sh'
+    """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates SET content = E'{old_fetch_condor}' WHERE workflow_name = 'vlass_coarse'
+        AND  filename='vlass_coarse_fetch.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates SET content = E'{old_envoy_condor}' WHERE workflow_name = 'vlass_coarse'
+        AND  filename='vlass_coarse_envoy.condor'
+        """
+    )