WS-570: verify reimaging works (needs dev testing)

schema/versions/1d653467ede6_.py

+"""empty message
+
+Revision ID: 1d653467ede6
+Revises: 5aafb3c0172e, 90a2d72f0d1c
+Create Date: 2021-08-05 16:17:55.368019
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1d653467ede6'
+down_revision = ('5aafb3c0172e', '90a2d72f0d1c')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

schema/versions/90a2d72f0d1c_add_version_file_transfer_to_condor.py

+"""add version file transfer to condor
+
+Revision ID: 90a2d72f0d1c
+Revises: d6d6c2ffaa66
+Create Date: 2021-08-05 11:25:17.750929
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "90a2d72f0d1c"
+down_revision = "d6d6c2ffaa66"
+branch_labels = None
+depends_on = None
+
+restore_cms_condor = """executable = restore_cms.sh
+arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml
+
+output = restore.out
+error = restore.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json{{files_to_transfer}}
+transfer_output_files = working, rawdata, products, delivery.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+std_cal_condor = """executable = std_calibration.sh
+arguments = {{product_locator}} {{request_id}} metadata.json PPR.xml
+
+output = calibration.out
+error = calibration.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/conveyor, ./PPR.xml, ./metadata.json{{files_to_transfer}}
+# results are delivered to QA area by conveyor script, no need to transfer
+# transfer_output_files = working, rawdata, products
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+std_cms_condor = """executable = std_cms_imaging.sh
+arguments = metadata.json PPR.xml
+
+output = imaging.out
+error = imaging.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+CMS_PATH = {{cms_path}}
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/conveyor, ./PPR.xml, ./metadata.json, $(CMS_PATH)/{{sdmId}}.ms{{files_to_transfer}}
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+std_restore_img_condor = """executable = std_restore_imaging.sh
+arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml
+
+output = restore-img.out
+error = restore-img.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json{{files_to_transfer}}
+transfer_output_files = working, rawdata, products, delivery.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+
+old_restore = """executable = restore_cms.sh
+arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml
+
+output = restore.out
+error = restore.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json
+transfer_output_files = working, rawdata, products, delivery.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+old_std_cal = """executable = std_calibration.sh
+arguments = {{product_locator}} {{request_id}} metadata.json PPR.xml
+
+output = calibration.out
+error = calibration.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/conveyor, ./PPR.xml, ./metadata.json
+# results are delivered to QA area by conveyor script, no need to transfer
+# transfer_output_files = working, rawdata, products
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+old_std_cms = """executable = std_cms_imaging.sh
+arguments = metadata.json PPR.xml
+
+output = imaging.out
+error = imaging.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+CMS_PATH = {{cms_path}}
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/conveyor, ./PPR.xml, ./metadata.json, $(CMS_PATH)/{{sdmId}}.ms
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+old_restore_img = """executable = std_restore_imaging.sh
+arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml
+
+output = restore-img.out
+error = restore-img.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json
+transfer_output_files = working, rawdata, products, delivery.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{restore_cms_condor}' WHERE filename='restore_cms.condor'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{std_cal_condor}' WHERE filename='std_calibration.condor'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{std_cms_condor}' WHERE filename='std_cms_imaging.condor'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{std_restore_img_condor}' WHERE filename='std_restore_imaging.condor'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{old_restore}' WHERE filename='restore_cms.condor'
+        """
+    )
+    op.execute(
+        f"""
+            UPDATE workflow_templates
+            SET content=E'{old_std_cal}' WHERE filename='std_calibration.condor'
+            """
+    )
+    op.execute(
+        f"""
+            UPDATE workflow_templates
+            SET content=E'{old_std_cms}' WHERE filename='std_cms_imaging.condor'
+            """
+    )
+    op.execute(
+        f"""
+            UPDATE workflow_templates
+            SET content=E'{old_restore_img}' WHERE filename='std_restore_imaging.condor'
+            """
+    )

shared/workspaces/workspaces/capability/services/capability_engine.py

@@ -66,14 +66,18 @@ class CapabilityEngine(CapabilityEngineIF):
         :param workflow_args:
         :param files:
         """
+        transfer_files_string = self.get_transfer_files_property(files)
+
+        args_list = {**workflow_args, "files_to_transfer": transfer_files_string}
+
         logger.info(
             "creating workflow request with name %s, args %s (and arg type %s)",
             workflow_name,
-            workflow_args,
-            type(workflow_args),
+            args_list,
+            type(args_list),
         )
         workflow_request = self.workflow_service.create_workflow_request(
-            workflow=workflow_name, argument=workflow_args
+            workflow=workflow_name, argument=args_list
         )
 
         logger.info(
@@ -100,3 +104,8 @@ class CapabilityEngine(CapabilityEngineIF):
     def announce_qa_ready(self, execution_id: int):
         execution = self.capability_info.lookup_execution(execution_id)
         self.workflow_service.announce_qa_ready(execution.current_workflow_request_id)
+
+    @staticmethod
+    def get_transfer_files_property(files: List[AbstractFile]):
+        separator = ", ./"
+        return separator + separator.join(file.filename for file in files)