Restore imaging capability definitions and templates

schema/versions/6508afd4da68_.py

+"""empty message
+
+Revision ID: 6508afd4da68
+Revises: 74c70c6e1f08, a70e3e5d5bf3
+Create Date: 2021-07-15 11:35:27.196165
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6508afd4da68'
+down_revision = ('74c70c6e1f08', 'a70e3e5d5bf3')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

schema/versions/74c70c6e1f08_add_restore_imaging_capability.py

+"""add restore imaging capability
+
+Revision ID: 74c70c6e1f08
+Revises: 7200d0d19938
+Create Date: 2021-07-13 10:41:13.527209
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "74c70c6e1f08"
+down_revision = "7200d0d19938"
+branch_labels = None
+depends_on = None
+
+
+condor_content = """executable = std_restore_imaging.sh
+arguments = {{product_locator}} {{cal_product_locator}} {{request_id}} metadata.json PPR.xml
+
+output = restore.out
+error = restore.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json
+transfer_output_files = working, rawdata, products
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+"""
+
+script_content = """#!/bin/sh
+
+export HOME=$TMPDIR
+
+set -o errexit
+./framework.sh -d .
+chmod 770 .
+cd rawdata/
+../productfetcher --product-locator $1 --product-locator $2
+cd ../
+./casa_envoy --integrated -i $4 $5
+./deliver -p ./products --prefix $3
+
+"""
+
+metadata_content = """{
+  "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"]
+  "workflowName": "std_restore_imaging",
+  "systemId": "{{request_id}}",
+  "creationTime": "{{created_at}}",
+  "productLocator": "{{product_locator}}",
+  "calProductLocator": "{{cal_locator}}",
+  "projectMetadata": {
+    "projectCode": "{{projectCode}}",
+    "title": "{{title}}",
+    "startTime": "{{startTime}}",
+    "observer": "{{observer}}"
+  },
+  "destinationDirectory": "{{root_directory}}/{{relative_path}}"
+}
+    
+"""
+
+ppr_content = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<ns2:SciPipeRequest xmlns:ns2="Common/pipelinescience/SciPipeRequest">
+    <ProjectSummary>
+        <ProposalCode>VLA/null</ProposalCode>
+        <Observatory>NRAO</Observatory>
+        <Telescope>VLA</Telescope>
+        <ProcessingSite>Socorro</ProcessingSite>
+        <Operator>vlapipe</Operator>
+        <Mode>SCIENCE</Mode>
+        <Version>NGRH-ALMA-10_8</Version>
+        <CreationTime>{{created_at}}</CreationTime>
+    </ProjectSummary>
+    <ProjectStructure>TBD</ProjectStructure>
+    <ProcessingRequests>
+        <RootDirectory>{{root_directory}}</RootDirectory>
+        <ProcessingRequest>
+            <ProcessingIntents>
+                <Intents>
+                    <Keyword>VLA_INTERFEROMETRY_STANDARD_OBSERVING_MODE</Keyword>
+                    <Value>Undefined</Value>
+                </Intents>
+            </ProcessingIntents>
+            <ProcessingProcedure>
+                <ProcedureTitle>Workspaces Restore Imaging</ProcedureTitle>
+                <ProcessingCommand>
+                    <Command>hifv_restoredata</Command>
+                    <ParameterSet/>
+                </ProcessingCommand>
+                <ProcessingCommand>
+                    <Command>hifv_statwt</Command>
+                    <ParameterSet/>
+                </ProcessingCommand>
+                <ProcessingCommand>
+                    <Command xmlns="">breakpoint</Command>
+                    <ParameterSet>
+                    </ParameterSet>
+                </ProcessingCommand>
+                <ProcessingCommand>
+                    <Command xmlns="">hif_mstransform</Command>
+                    <ParameterSet>
+                    </ParameterSet>
+                </ProcessingCommand>
+                <ProcessingCommand>
+                    <Command xmlns="">hif_checkproductsize</Command>
+                    <ParameterSet>
+                        <Parameter>
+                            <Keyword xmlns="">maximsize</Keyword>
+                            <Value xmlns="">16384</Value>
+                        </Parameter>
+                    </ParameterSet>
+                </ProcessingCommand>
+                <ProcessingCommand>
+                    <Command xmlns="">hif_makeimlist</Command>
+                    <ParameterSet>
+                        <Parameter>
+                            <Keyword xmlns="">specmode</Keyword>
+                            <Value xmlns="">cont</Value>
+                        </Parameter>
+                    </ParameterSet>
+                </ProcessingCommand>
+                <ProcessingCommand>
+                    <Command xmlns="">hif_makeimages</Command>
+                    <ParameterSet>
+                        <Parameter>
+                            <Keyword xmlns="">hm_masking</Keyword>
+                            <Value xmlns="">none</Value>
+                        </Parameter>
+                        <Parameter>
+                            <Keyword xmlns="">hm_cyclefactor</Keyword>
+                            <Value xmlns="">3.0</Value>
+                        </Parameter>
+                    </ParameterSet>
+                </ProcessingCommand>
+                <ProcessingCommand>
+                    <Command xmlns="">hifv_exportdata</Command>
+                    <ParameterSet>
+                        <Parameter>
+                            <Keyword xmlns="">imaging_products_only</Keyword>
+                            <Value xmlns="">True</Value>
+                        </Parameter>
+                    </ParameterSet>
+                </ProcessingCommand>
+            </ProcessingProcedure>
+            <DataSet>
+                <RelativePath>{{relative_path}}</RelativePath>
+                <SdmIdentifier>{{sdmId}}</SdmIdentifier>
+                <DataType>asdm</DataType>
+            </DataSet>
+        </ProcessingRequest>0
+    </ProcessingRequests>
+    <ResultsProcessing>
+        <ArchiveResults>false</ArchiveResults>
+        <CleanUpDisk>false</CleanUpDisk>
+        <UpdateProjectLifeCycle>false</UpdateProjectLifeCycle>
+        <NotifyOperatorWhenDone>false</NotifyOperatorWhenDone>
+        <SDMall>false</SDMall>
+        <SDMonly>false</SDMonly>
+        <PipelineOperatorAddress>Unknown</PipelineOperatorAddress>
+    </ResultsProcessing>
+</ns2:SciPipeRequest>
+
+"""
+
+fix_restore_metadata = """{
+    "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"],
+    "workflowName": "restore_cms",
+    "systemId": "{{request_id}}",
+    "creationTime": "{{created_at}}",
+    "productLocator": "{{product_locator}}",
+    "calProductLocator": "{{cal_locator}}",
+    "projectMetadata": {
+        "projectCode": "{{projectCode}}",
+        "title": "{{title}}",
+        "startTime": "{{startTime}}",
+        "observer": "{{observer}}"
+    },
+    "destinationDirectory": "{{root_directory}}/{{relative_path}}"
+}
+
+"""
+
+fix_restore_script = """#!/bin/sh
+
+export HOME=$TMPDIR
+
+set -o errexit
+./framework.sh -d .
+chmod 770 .
+cd rawdata/
+../productfetcher --product-locator $1 --product-locator $2
+cd ../
+./casa_envoy --restore -c $4 $5
+cp ./working/{{sdmId}}.ms ./products/
+./deliver -p ./products --prefix $3
+
+"""
+
+
+def upgrade():
+    restore_image_steps = """prepare-and-run-workflow std_restore_imaging
+await-workflow
+await-qa
+    """
+
+    op.execute(
+        f"""
+        INSERT INTO capabilities (capability_name, capability_steps, max_jobs) 
+        VALUES ('std_restore_imaging', '{restore_image_steps}', 20) 
+        """
+    )
+    op.execute(
+        """
+        INSERT INTO workflows (workflow_name) VALUES ('std_restore_imaging')
+        """
+    )
+
+    op.execute(
+        f"""
+            INSERT INTO workflow_templates (filename, content, workflow_name) 
+            VALUES ('std_restore_imaging.condor', E'{condor_content}', 'std_restore_imaging')
+            """
+    )
+    op.execute(
+        f"""
+            INSERT INTO workflow_templates (filename, content, workflow_name) 
+            VALUES ('std_restore_imaging.sh', E'{script_content}', 'std_restore_imaging')
+            """
+    )
+    op.execute(
+        f"""
+            INSERT INTO workflow_templates (filename, content, workflow_name) 
+            VALUES ('metadata.json', E'{metadata_content}', 'std_restore_imaging')
+            """
+    )
+    op.execute(
+        f"""
+            INSERT INTO workflow_templates (filename, content, workflow_name) 
+            VALUES ('PPR.xml', E'{ppr_content}', 'std_restore_imaging')
+            """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{fix_restore_metadata}' WHERE filename='metadata.json' AND workflow_name='restore_cms'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{fix_restore_script}' WHERE filename='restore_cms.sh'
+        """
+    )
+
+
+def downgrade():
+
+    op.execute(
+        """
+        DELETE FROM capabilities WHERE capability_name = 'std_restore_imaging'
+        """
+    )
+
+    op.execute(
+        """
+        DELETE FROM workflows WHERE workflow_name = 'std_restore_imaging'
+        """
+    )
+
+    op.execute(
+        """
+        DELETE FROM workflow_templates WHERE workflow_name = 'std_restore_imaging'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{old_restore_metadata}' WHERE filename='metadata.json' AND workflow_name='restore_cms'
+        """
+    )
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{old_restore_script}' WHERE filename='restore_cms.sh'
+        """
+    )
+
+
+old_restore_script = """#!/bin/sh
+
+export HOME=$TMPDIR
+
+set -o errexit
+./framework.sh -d .
+chmod 770 .
+cd rawdata/
+../productfetcher --product-locator $1 $2
+cd ../
+./casa_envoy --restore -c $4 $5
+cp ./working/{{sdmId}}.ms ./products/
+./deliver -p ./products --prefix $3
+
+"""
+
+
+old_restore_metadata = """{
+  "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"]
+  "workflowName": "std_calibration",
+  "systemId": "{{request_id}}",
+  "creationTime": "{{created_at}}",
+  "productLocator": "{{product_locator}}",
+  "calProductLocator": "{{cal_locator}}",
+  "projectMetadata": {
+    "projectCode": "{{projectCode}}",
+    "title": "{{title}}",
+    "startTime": "{{startTime}}",
+    "observer": "{{observer}}"
+  },
+  "destinationDirectory": "{{root_directory}}/{{relative_path}}"
+}
+    
+"""