diff --git a/shared/workspaces/alembic/versions/e8e6d54d8444_add_pimscache_split_workflow.py b/shared/workspaces/alembic/versions/e8e6d54d8444_add_pimscache_split_workflow.py index f8d28dbe28849cb1a2f34b4ec9d06fa7608127e2..a8159c37ed68b7525f1abd5bac2a9664c87c51d4 100644 --- a/shared/workspaces/alembic/versions/e8e6d54d8444_add_pimscache_split_workflow.py +++ b/shared/workspaces/alembic/versions/e8e6d54d8444_add_pimscache_split_workflow.py @@ -16,7 +16,7 @@ branch_labels = None depends_on = None # Main DAG for the workflow -pims_split_dag = """JOB RESTORE restore_cms.condor +pims_split_dag = """JOB RESTORE restore.condor {#splits} JOB {{.}} split_workflow.condor @@ -30,23 +30,28 @@ PARENT SPLIT CHILD FINISH """ # Restore job -restore_cms_condor = """executable = restore_cms.sh -arguments = {{product_locator}} {{cal_locator}} metadata.json PPR.xml +restore_condor = """executable = restore.sh +arguments = {{product_locator}} {{cal_locator}} -output = restore_cms.out -error = restore_cms.err +output = restore.out +error = restore.err log = condor.log SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin SPOOL_DIR = {{spool_dir}} +VLASS_DIR = {{data_location}} should_transfer_files = yes -transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/productfetcher, nraorsync://$(SBIN_PATH)/casa_envoy - +{{^existingRestore}} +transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraosync://$(VLASS_DIR)/rawdata, nraosync://$(VLASS_DIR)/products, nraosync://$(VLASS_DIR)/working, nraosync://$(VLASS_DIR)/metadata.json, nraosync://$(VLASS_DIR)/PPR.xml, nraorsync://$(SBIN_PATH)/productfetcher, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela transfer_output_files = .job.ad -+nrao_output_files = "rawdata" ++nrao_output_files = "rawdata working products" output_destination = nraorsync://$(SPOOL_DIR) +WantIOProxy = True +{{/existingRestore}} +{{#existingRestore}} +transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraosync://$(SBIN_PATH)/null +{{/existingRestore}} getenv = True environment = "CAPO_PATH=/home/casa/capo" @@ -57,16 +62,17 @@ queue """ -restore_cms_sh = """#!/bin/sh +restore_sh = """#!/bin/sh set -o errexit -cd {{data_location}} - {{^existingRestore}} cd rawdata/ ../productfetcher --product-locator $1 --product-locator $2 cd ../ -./casa_envoy --restore -c $3 $4 +./casa_envoy --restore -c metadata.json PPR.xml +{{/existingRestore}} +{{#existingRestore}} +./null $* {{/existingRestore}} """ @@ -80,7 +86,7 @@ VLASS_DIR = {{data_location}} SPOOL_DIR = {{spool_dir}} PIMS_PATH = /lustre/aoc/cluster/pipeline/vlass_{{vlass_env}}/workspaces/bin should_transfer_files = yes -transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(PIMS_PATH)/pimscache, nraosync://$(VLASS_DIR)/metadata.json, nraosync://$(VLASS_DIR)/working nraosync://$(VLASS_DIR)/tiles/$(SPLIT_DIR) +transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(PIMS_PATH)/pimscache, nraosync://$(VLASS_DIR)/metadata.json, nraosync://$(VLASS_DIR)/tiles/$(SPLIT_DIR) transfer_output_files = .job.ad output_destination = nraorsync://$(SPOOL_DIR) +WantIOProxy = True @@ -106,7 +112,7 @@ PHCENTER=$(echo $1 | cut -d '/' -f 2) # Get the measurement set path {{^existingRestore}} -MS={{data_location}}/*.ms +MS={{data_location}}/working/*.ms {{/existingRestore}} {{#existingRestore}} MS={{existingRestore}} @@ -173,14 +179,14 @@ def upgrade(): op.execute( f""" INSERT INTO workflow_templates (filename, content, workflow_name) - VALUES ('restore_cms.condor', E'{restore_cms_condor}', 'pims_split') + VALUES ('restore.condor', E'{restore_condor}', 'pims_split') """ ) op.execute( f""" INSERT INTO workflow_templates (filename, content, workflow_name) - VALUES ('restore_cms.sh', E'{restore_cms_sh}', 'pims_split') + VALUES ('restore.sh', E'{restore_sh}', 'pims_split') """ )