diff --git a/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py b/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py
index 3f333db98135759d0d213f7d1bb186e193536845..ae04798697bcff5765004d76c6653a51adf8f02f 100644
--- a/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py
+++ b/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py
@@ -109,10 +109,11 @@ class CalibrationLauncher(LauncherIF):
self.metadata = get_abs_file(parameters.get("metadata"))
def launch_casa(self):
- if self.parameters["product_type"] == "restore":
- check_input = self.check_restorable()
- else:
- check_input = self.check_calibratable()
+ check_input = (
+ self.check_restorable()
+ if self.parameters["product_type"] == "restore"
+ else self.check_calibratable()
+ )
if check_input:
self.prepare_for_casa()
@@ -174,10 +175,11 @@ class ImagingLauncher(LauncherIF):
self.metadata = get_abs_file(parameters.get("metadata"))
def launch_casa(self):
- if self.parameters["product_type"] == "integrated":
- check_input = self.check_cal_and_imageable()
- else:
- check_input = self.check_imageable()
+ check_input = (
+ self.check_cal_and_imageable()
+ if self.parameters["product_type"] == "integrated"
+ else self.check_imageable()
+ )
if check_input:
self.run_audit(self.parameters)
diff --git a/apps/cli/executables/pexable/delivery/delivery/finder.py b/apps/cli/executables/pexable/delivery/delivery/finder.py
index 71029040418ee924153f40b38d1d696a960249b4..151599be91d7d17a8374299560ab14c3a16351b6 100644
--- a/apps/cli/executables/pexable/delivery/delivery/finder.py
+++ b/apps/cli/executables/pexable/delivery/delivery/finder.py
@@ -8,7 +8,7 @@ import os
import pathlib
from typing import Iterator
-from .products import ExecutionBlock, SpooledProduct, CalibratedMS
+from .products import ExecutionBlock, SpooledProduct, CalibratedMS, Image, Weblog
class ProductFinder(abc.ABC):
@@ -45,8 +45,12 @@ class HeuristicProductFinder(ProductFinder):
:return: List of products
"""
- for subdir in self.path.glob("*"):
- if subdir.is_dir() and subdir.name.endswith(".ms"):
- yield CalibratedMS(subdir)
- elif subdir.is_dir() and not subdir.name.startswith("tmp"):
- yield ExecutionBlock(subdir)
+ for item in self.path.glob("*"):
+ if item.is_dir() and item.name.endswith(".ms"):
+ yield CalibratedMS(item)
+ elif item.is_dir() and not item.name.startswith("tmp"):
+ yield ExecutionBlock(item)
+ elif item.is_file() and item.name.endswith(".fits"):
+ yield Image(item)
+ elif item.is_file() and item.name.startswith("weblog"):
+ yield Weblog(item)
diff --git a/apps/cli/executables/pexable/delivery/delivery/products.py b/apps/cli/executables/pexable/delivery/delivery/products.py
index 7fdfb59d1659ec1da0704a8be7a165fe216c5898..becca310ec3546220e0c1686269c7b080527233e 100644
--- a/apps/cli/executables/pexable/delivery/delivery/products.py
+++ b/apps/cli/executables/pexable/delivery/delivery/products.py
@@ -170,9 +170,30 @@ class Calibration(SpooledProduct):
return f"calibration"
-# class Image(SpooledProduct):
-# def deliver_to(self, destination: Destination):
-# raise NotImplementedError
+class Image(SpooledProduct):
+ @property
+ def img_name(self):
+ return self.path.absolute().name
+
+ def deliver_to(self, destination: Destination):
+ destination.add_file(self.path, self.img_name)
+
+ def __str__(self):
+ return f"Image {self.img_name}"
+
+
+class Weblog(SpooledProduct):
+ @property
+ def file_name(self):
+ return self.path.absolute().name
+
+ def deliver_to(self, destination: Destination):
+ destination.add_file(self.path, self.file_name)
+
+ def __str__(self):
+ return f"weblog"
+
+
#
#
# class ProcessingRequest(SpooledProduct):
diff --git a/schema/versions/c857908b6acb_correcting_restore_imaging.py b/schema/versions/c857908b6acb_correcting_restore_imaging.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c32c3cc89a0b0b50062c24a3640e885149fbd95
--- /dev/null
+++ b/schema/versions/c857908b6acb_correcting_restore_imaging.py
@@ -0,0 +1,330 @@
+"""correcting restore imaging
+
+Revision ID: c857908b6acb
+Revises: 7b284596590c
+Create Date: 2021-07-21 09:35:52.735903
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "c857908b6acb"
+down_revision = "7b284596590c"
+branch_labels = None
+depends_on = None
+
+condor = """executable = std_restore_imaging.sh
+arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml
+
+output = restore-img.out
+error = restore-img.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json
+transfer_output_files = working, rawdata, products, delivery.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+
+"""
+
+ppr = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<ns2:SciPipeRequest xmlns:ns2="Common/pipelinescience/SciPipeRequest">
+ <ProjectSummary>
+ <ProposalCode>VLA/{{projectCode}}</ProposalCode>
+ <Observatory>NRAO</Observatory>
+ <Telescope>VLA</Telescope>
+ <ProcessingSite>Socorro</ProcessingSite>
+ <Operator>vlapipe</Operator>
+ <Mode>SCIENCE</Mode>
+ <Version>NGRH-ALMA-10_8</Version>
+ <CreationTime>{{created_at}}</CreationTime>
+ </ProjectSummary>
+ <ProjectStructure>TBD</ProjectStructure>
+ <ProcessingRequests>
+ <RootDirectory>{{root_directory}}</RootDirectory>
+ <ProcessingRequest>
+ <ProcessingIntents/>
+ <ProcessingProcedure>
+ <ProcedureTitle>Workspaces Restore Imaging</ProcedureTitle>
+ <ProcessingCommand>
+ <Command>hifv_restoredata</Command>
+ <ParameterSet/>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command>hifv_statwt</Command>
+ <ParameterSet/>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">breakpoint</Command>
+ <ParameterSet>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_mstransform</Command>
+ <ParameterSet>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_checkproductsize</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">maximsize</Keyword>
+ <Value xmlns="">16384</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_makeimlist</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">specmode</Keyword>
+ <Value xmlns="">cont</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_makeimages</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">hm_masking</Keyword>
+ <Value xmlns="">none</Value>
+ </Parameter>
+ <Parameter>
+ <Keyword xmlns="">hm_cyclefactor</Keyword>
+ <Value xmlns="">3.0</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hifv_exportdata</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">imaging_products_only</Keyword>
+ <Value xmlns="">True</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ </ProcessingProcedure>
+ <DataSet>
+ <RelativePath>{{relative_path}}</RelativePath>
+ <SdmIdentifier>{{sdmId}}</SdmIdentifier>
+ <DataType>asdm</DataType>
+ </DataSet>
+ </ProcessingRequest>
+ </ProcessingRequests>
+</ns2:SciPipeRequest>
+
+"""
+
+metadata = """{
+ "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"],
+ "workflowName": "std_restore_imaging",
+ "systemId": "{{request_id}}",
+ "creationTime": "{{created_at}}",
+ "productLocator": "{{product_locator}}",
+ "calProductLocator": "{{cal_locator}}",
+ "projectMetadata": {
+ "projectCode": "{{projectCode}}",
+ "title": "{{title}}",
+ "telescope": "{{telescope}}",
+ "startTime": "{{startTime}}",
+ "observer": "{{observer}}"
+ },
+ "destinationDirectory": "{{root_directory}}/{{relative_path}}"
+}
+
+"""
+
+
+old_condor = """executable = std_restore_imaging.sh
+arguments = {{product_locator}} {{cal_product_locator}} {{request_id}} metadata.json PPR.xml
+
+output = restore.out
+error = restore.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json
+transfer_output_files = working, rawdata, products, delivery.json
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+
+"""
+
+old_ppr = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<ns2:SciPipeRequest xmlns:ns2="Common/pipelinescience/SciPipeRequest">
+ <ProjectSummary>
+ <ProposalCode>VLA/null</ProposalCode>
+ <Observatory>NRAO</Observatory>
+ <Telescope>VLA</Telescope>
+ <ProcessingSite>Socorro</ProcessingSite>
+ <Operator>vlapipe</Operator>
+ <Mode>SCIENCE</Mode>
+ <Version>NGRH-ALMA-10_8</Version>
+ <CreationTime>{{created_at}}</CreationTime>
+ </ProjectSummary>
+ <ProjectStructure>TBD</ProjectStructure>
+ <ProcessingRequests>
+ <RootDirectory>{{root_directory}}</RootDirectory>
+ <ProcessingRequest>
+ <ProcessingIntents>
+ <Intents>
+ <Keyword>VLA_INTERFEROMETRY_STANDARD_OBSERVING_MODE</Keyword>
+ <Value>Undefined</Value>
+ </Intents>
+ </ProcessingIntents>
+ <ProcessingProcedure>
+ <ProcedureTitle>Workspaces Restore Imaging</ProcedureTitle>
+ <ProcessingCommand>
+ <Command>hifv_restoredata</Command>
+ <ParameterSet/>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command>hifv_statwt</Command>
+ <ParameterSet/>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">breakpoint</Command>
+ <ParameterSet>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_mstransform</Command>
+ <ParameterSet>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_checkproductsize</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">maximsize</Keyword>
+ <Value xmlns="">16384</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_makeimlist</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">specmode</Keyword>
+ <Value xmlns="">cont</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hif_makeimages</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">hm_masking</Keyword>
+ <Value xmlns="">none</Value>
+ </Parameter>
+ <Parameter>
+ <Keyword xmlns="">hm_cyclefactor</Keyword>
+ <Value xmlns="">3.0</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ <ProcessingCommand>
+ <Command xmlns="">hifv_exportdata</Command>
+ <ParameterSet>
+ <Parameter>
+ <Keyword xmlns="">imaging_products_only</Keyword>
+ <Value xmlns="">True</Value>
+ </Parameter>
+ </ParameterSet>
+ </ProcessingCommand>
+ </ProcessingProcedure>
+ <DataSet>
+ <RelativePath>{{relative_path}}</RelativePath>
+ <SdmIdentifier>{{sdmId}}</SdmIdentifier>
+ <DataType>asdm</DataType>
+ </DataSet>
+ </ProcessingRequest>0
+ </ProcessingRequests>
+ <ResultsProcessing>
+ <ArchiveResults>false</ArchiveResults>
+ <CleanUpDisk>false</CleanUpDisk>
+ <UpdateProjectLifeCycle>false</UpdateProjectLifeCycle>
+ <NotifyOperatorWhenDone>false</NotifyOperatorWhenDone>
+ <SDMall>false</SDMall>
+ <SDMonly>false</SDMonly>
+ <PipelineOperatorAddress>Unknown</PipelineOperatorAddress>
+ </ResultsProcessing>
+</ns2:SciPipeRequest>
+
+"""
+
+old_metadata = """{
+ "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"]
+ "workflowName": "std_restore_imaging",
+ "systemId": "{{request_id}}",
+ "creationTime": "{{created_at}}",
+ "productLocator": "{{product_locator}}",
+ "calProductLocator": "{{cal_locator}}",
+ "projectMetadata": {
+ "projectCode": "{{projectCode}}",
+ "title": "{{title}}",
+ "startTime": "{{startTime}}",
+ "observer": "{{observer}}"
+ },
+ "destinationDirectory": "{{root_directory}}/{{relative_path}}"
+}
+
+"""
+
+
+def upgrade():
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{condor}' WHERE filename='std_restore_imaging.condor'
+ """
+ )
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{ppr}' WHERE filename='PPR.xml' AND workflow_name='std_restore_imaging'
+ """
+ )
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{metadata}' WHERE filename='metadata.json' AND workflow_name='std_restore_imaging'
+ """
+ )
+
+
+def downgrade():
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{old_condor}' WHERE filename='std_restore_imaging.condor'
+ """
+ )
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{old_ppr}' WHERE filename='PPR.xml' AND workflow_name='std_restore_imaging'
+ """
+ )
+ op.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=E'{old_metadata}' WHERE filename='metadata.json' AND workflow_name='std_restore_imaging'
+ """
+ )