Corrected pims_split templates

Fixed split.condor for HTC10 by removing the quotes around an already quoted variable. I didn't find any other instances of this happening in other template files.

Also brought over update_stage commands into split.sh that disappeared sometime since 2.8.1.

shared/workspaces/alembic/versions/d992b379d8a7_htc10_workflow_fixes.py

+"""htc10 workflow fixes
+
+Revision ID: d992b379d8a7
+Revises: 36591dc3f14d
+Create Date: 2024-02-29 12:37:02.423565
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd992b379d8a7'
+down_revision = '36591dc3f14d'
+branch_labels = None
+depends_on = None
+
+# Remove quotes around string variables that have already been quoted
+new_split_condor = """executable = split.sh
+arguments = "$(split_dir)"{{#quicklook_url}} "$(quicklook_url)"{{/quicklook_url}}
+
+output = tiles/$(split_dir)/split.out
+error = tiles/$(split_dir)/split.err
+log = condor.log
+
+SPLIT_DIR = $(split_dir)
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{spool_dir}}
+PIMS_PATH = /lustre/aoc/cluster/pipeline/vlass_{{vlass_env}}/workspaces/bin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(PIMS_PATH)/pimscache, nraorsync://$(SPOOL_DIR)/metadata.json, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/PPR.xml, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/working, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/rawdata, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/products
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
+output_destination = nraorsync://$(SPOOL_DIR)
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+request_memory = 24G
+request_disk = 20G
+
+requirements = (VLASS == True) && (HasLustre == True)
++partition = "VLASS"
+
+queue
+
+"""
+
+old_split_condor = """executable = split.sh
+arguments = "$(split_dir)"{{#quicklook_url}} "$(quicklook_url)"{{/quicklook_url}}
+
+output = tiles/$(split_dir)/split.out
+error = tiles/$(split_dir)/split.err
+log = condor.log
+
+SPLIT_DIR = "$(split_dir)"
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{spool_dir}}
+PIMS_PATH = /lustre/aoc/cluster/pipeline/vlass_{{vlass_env}}/workspaces/bin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(PIMS_PATH)/pimscache, nraorsync://$(SPOOL_DIR)/metadata.json, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/PPR.xml, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/working, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/rawdata, nraorsync://$(SPOOL_DIR)/tiles/$(SPLIT_DIR)/products
+when_to_transfer_output = ON_EXIT
+transfer_output_files = .job.ad
+output_destination = nraorsync://$(SPOOL_DIR)
++WantIOProxy = True
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+request_memory = 24G
+request_disk = 20G
+
+requirements = (VLASS == True) && (HasLustre == True)
++partition = "VLASS"
+
+queue
+
+"""
+
+# Put back in the update_stage calls that disappeared since 2.8.1
+new_split_sh = """#!/bin/sh
+export HOME=$TMPDIR
+TILE=$(echo $1 | cut -d "/" -f 1)
+PHCENTER=$(echo $1 | cut -d "/" -f 2)
+STAGE=$1
+
+# Get the measurement set path
+{{^existing_restore}}
+MS={{data_location}}/working/*.ms
+{{/existing_restore}}
+{{#existing_restore}}
+MS={{existing_restore}}
+{{/existing_restore}}
+
+./update_stage SPLIT_${STAGE}
+
+# Link it in the splits rawdata
+ln -s $MS rawdata/
+
+# failed_splits.txt needs to be present even if its empty for pims_analyzer
+touch {{data_location}}/failed_splits.txt
+
+# Run CASA
+./casa_envoy --split metadata.json PPR.xml
+
+# Populate cache
+./pimscache cp -c {{vlass_product}} -t $TILE -p $PHCENTER working/*_split.ms
+
+# If pimscache call failed, output the failed split to a file for pims_analyzer
+if [[ $? -ne 0 ]] ; then
+    echo "${TILE}.${PHCENTER}" >> {{data_location}}/failed_splits.txt
+fi
+
+# Run quicklook if second parameter was given
+if ! [[ -z "$2" ]]; then
+    curl --request PUT --header "Content-Length: 0" $2
+fi
+
+./update_stage SPLIT_${STAGE} --complete
+"""
+
+old_split_sh = """#!/bin/sh
+export HOME=$TMPDIR
+TILE=$(echo $1 | cut -d "/" -f 1)
+PHCENTER=$(echo $1 | cut -d "/" -f 2)
+
+# Get the measurement set path
+{{^existing_restore}}
+MS={{data_location}}/working/*.ms
+{{/existing_restore}}
+{{#existing_restore}}
+MS={{existing_restore}}
+{{/existing_restore}}
+
+# Link it in the splits rawdata
+ln -s $MS rawdata/
+
+# failed_splits.txt needs to be present even if its empty for pims_analyzer
+touch {{data_location}}/failed_splits.txt
+
+# Run CASA
+./casa_envoy --split metadata.json PPR.xml
+
+# Populate cache
+./pimscache cp -c {{vlass_product}} -t $TILE -p $PHCENTER working/*_split.ms
+
+# If pimscache call failed, output the failed split to a file for pims_analyzer
+if [[ $? -ne 0 ]] ; then
+    echo "${TILE}.${PHCENTER}" >> {{data_location}}/failed_splits.txt
+fi
+
+# Run quicklook if second parameter was given
+if ! [[ -z "$2" ]]; then
+    curl --request PUT --header "Content-Length: 0" $2
+fi
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{new_split_condor}'
+        WHERE filename = 'split.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{new_split_sh}'
+        WHERE filename = 'split.sh'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{old_split_condor}'
+        WHERE filename = 'split.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{old_split_sh}'
+        WHERE filename = 'split.sh'
+        """
+    )