Skip to content
Snippets Groups Projects
Commit b230d3c5 authored by Charlotte Hausman's avatar Charlotte Hausman
Browse files

Post Apocalypse Round 5 - restore & image

parent 4615cf61
No related branches found
No related tags found
1 merge request!362Post Apocalypse Round 5 - restore & image
Pipeline #2275 passed
Stage: run-schema
Stage: cache-build
Stage: build
Stage: unit-test
Stage: test-coverage
Stage: push
Stage: deploy-coverage-page
Stage: deploy
Stage: e2e-test
Stage: .post
Hide whitespace changes
Inline Side-by-side
apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py
+ 10
− 8
View file @ b230d3c5
......@@ -109,10 +109,11 @@ class CalibrationLauncher(LauncherIF):
self.metadata = get_abs_file(parameters.get("metadata"))
def launch_casa(self):
if self.parameters["product_type"] == "restore":
check_input = self.check_restorable()
else:
check_input = self.check_calibratable()
check_input = (
self.check_restorable()
if self.parameters["product_type"] == "restore"
else self.check_calibratable()
)
if check_input:
self.prepare_for_casa()
......@@ -174,10 +175,11 @@ class ImagingLauncher(LauncherIF):
self.metadata = get_abs_file(parameters.get("metadata"))
def launch_casa(self):
if self.parameters["product_type"] == "integrated":
check_input = self.check_cal_and_imageable()
else:
check_input = self.check_imageable()
check_input = (
self.check_cal_and_imageable()
if self.parameters["product_type"] == "integrated"
else self.check_imageable()
)
if check_input:
self.run_audit(self.parameters)
......
apps/cli/executables/pexable/delivery/delivery/finder.py
+ 10
− 6
View file @ b230d3c5
......@@ -8,7 +8,7 @@ import os
import pathlib
from typing import Iterator
from .products import ExecutionBlock, SpooledProduct, CalibratedMS
from .products import ExecutionBlock, SpooledProduct, CalibratedMS, Image, Weblog
class ProductFinder(abc.ABC):
......@@ -45,8 +45,12 @@ class HeuristicProductFinder(ProductFinder):
:return: List of products
"""
for subdir in self.path.glob("*"):
if subdir.is_dir() and subdir.name.endswith(".ms"):
yield CalibratedMS(subdir)
elif subdir.is_dir() and not subdir.name.startswith("tmp"):
yield ExecutionBlock(subdir)
for item in self.path.glob("*"):
if item.is_dir() and item.name.endswith(".ms"):
yield CalibratedMS(item)
elif item.is_dir() and not item.name.startswith("tmp"):
yield ExecutionBlock(item)
elif item.is_file() and item.name.endswith(".fits"):
yield Image(item)
elif item.is_file() and item.name.startswith("weblog"):
yield Weblog(item)
apps/cli/executables/pexable/delivery/delivery/products.py
+ 24
− 3
View file @ b230d3c5
......@@ -170,9 +170,30 @@ class Calibration(SpooledProduct):
return f"calibration"
# class Image(SpooledProduct):
# def deliver_to(self, destination: Destination):
# raise NotImplementedError
class Image(SpooledProduct):
@property
def img_name(self):
return self.path.absolute().name
def deliver_to(self, destination: Destination):
destination.add_file(self.path, self.img_name)
def __str__(self):
return f"Image {self.img_name}"
class Weblog(SpooledProduct):
@property
def file_name(self):
return self.path.absolute().name
def deliver_to(self, destination: Destination):
destination.add_file(self.path, self.file_name)
def __str__(self):
return f"weblog"
#
#
# class ProcessingRequest(SpooledProduct):
......
schema/versions/c857908b6acb_correcting_restore_imaging.py 0 → 100644
+ 330
− 0
View file @ b230d3c5
"""correcting restore imaging
Revision ID: c857908b6acb
Revises: 7b284596590c
Create Date: 2021-07-21 09:35:52.735903
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = "c857908b6acb"
down_revision = "7b284596590c"
branch_labels = None
depends_on = None
condor = """executable = std_restore_imaging.sh
arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml
output = restore-img.out
error = restore-img.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
should_transfer_files = yes
transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json
transfer_output_files = working, rawdata, products, delivery.json
request_memory = {{ramInGb}}
getenv = True
environment = "CAPO_PATH=/home/casa/capo"
queue
"""
ppr = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ns2:SciPipeRequest xmlns:ns2="Common/pipelinescience/SciPipeRequest">
<ProjectSummary>
<ProposalCode>VLA/{{projectCode}}</ProposalCode>
<Observatory>NRAO</Observatory>
<Telescope>VLA</Telescope>
<ProcessingSite>Socorro</ProcessingSite>
<Operator>vlapipe</Operator>
<Mode>SCIENCE</Mode>
<Version>NGRH-ALMA-10_8</Version>
<CreationTime>{{created_at}}</CreationTime>
</ProjectSummary>
<ProjectStructure>TBD</ProjectStructure>
<ProcessingRequests>
<RootDirectory>{{root_directory}}</RootDirectory>
<ProcessingRequest>
<ProcessingIntents/>
<ProcessingProcedure>
<ProcedureTitle>Workspaces Restore Imaging</ProcedureTitle>
<ProcessingCommand>
<Command>hifv_restoredata</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_statwt</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">breakpoint</Command>
<ParameterSet>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_mstransform</Command>
<ParameterSet>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_checkproductsize</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">maximsize</Keyword>
<Value xmlns="">16384</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_makeimlist</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">specmode</Keyword>
<Value xmlns="">cont</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_makeimages</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">hm_masking</Keyword>
<Value xmlns="">none</Value>
</Parameter>
<Parameter>
<Keyword xmlns="">hm_cyclefactor</Keyword>
<Value xmlns="">3.0</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hifv_exportdata</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">imaging_products_only</Keyword>
<Value xmlns="">True</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
</ProcessingProcedure>
<DataSet>
<RelativePath>{{relative_path}}</RelativePath>
<SdmIdentifier>{{sdmId}}</SdmIdentifier>
<DataType>asdm</DataType>
</DataSet>
</ProcessingRequest>
</ProcessingRequests>
</ns2:SciPipeRequest>
"""
metadata = """{
"fileSetIds": ["{{sdmId}}", "{{calSdmId}}"],
"workflowName": "std_restore_imaging",
"systemId": "{{request_id}}",
"creationTime": "{{created_at}}",
"productLocator": "{{product_locator}}",
"calProductLocator": "{{cal_locator}}",
"projectMetadata": {
"projectCode": "{{projectCode}}",
"title": "{{title}}",
"telescope": "{{telescope}}",
"startTime": "{{startTime}}",
"observer": "{{observer}}"
},
"destinationDirectory": "{{root_directory}}/{{relative_path}}"
}
"""
old_condor = """executable = std_restore_imaging.sh
arguments = {{product_locator}} {{cal_product_locator}} {{request_id}} metadata.json PPR.xml
output = restore.out
error = restore.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
should_transfer_files = yes
transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json
transfer_output_files = working, rawdata, products, delivery.json
request_memory = {{ramInGb}}
getenv = True
environment = "CAPO_PATH=/home/casa/capo"
queue
"""
old_ppr = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ns2:SciPipeRequest xmlns:ns2="Common/pipelinescience/SciPipeRequest">
<ProjectSummary>
<ProposalCode>VLA/null</ProposalCode>
<Observatory>NRAO</Observatory>
<Telescope>VLA</Telescope>
<ProcessingSite>Socorro</ProcessingSite>
<Operator>vlapipe</Operator>
<Mode>SCIENCE</Mode>
<Version>NGRH-ALMA-10_8</Version>
<CreationTime>{{created_at}}</CreationTime>
</ProjectSummary>
<ProjectStructure>TBD</ProjectStructure>
<ProcessingRequests>
<RootDirectory>{{root_directory}}</RootDirectory>
<ProcessingRequest>
<ProcessingIntents>
<Intents>
<Keyword>VLA_INTERFEROMETRY_STANDARD_OBSERVING_MODE</Keyword>
<Value>Undefined</Value>
</Intents>
</ProcessingIntents>
<ProcessingProcedure>
<ProcedureTitle>Workspaces Restore Imaging</ProcedureTitle>
<ProcessingCommand>
<Command>hifv_restoredata</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_statwt</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">breakpoint</Command>
<ParameterSet>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_mstransform</Command>
<ParameterSet>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_checkproductsize</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">maximsize</Keyword>
<Value xmlns="">16384</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_makeimlist</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">specmode</Keyword>
<Value xmlns="">cont</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hif_makeimages</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">hm_masking</Keyword>
<Value xmlns="">none</Value>
</Parameter>
<Parameter>
<Keyword xmlns="">hm_cyclefactor</Keyword>
<Value xmlns="">3.0</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command xmlns="">hifv_exportdata</Command>
<ParameterSet>
<Parameter>
<Keyword xmlns="">imaging_products_only</Keyword>
<Value xmlns="">True</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
</ProcessingProcedure>
<DataSet>
<RelativePath>{{relative_path}}</RelativePath>
<SdmIdentifier>{{sdmId}}</SdmIdentifier>
<DataType>asdm</DataType>
</DataSet>
</ProcessingRequest>0
</ProcessingRequests>
<ResultsProcessing>
<ArchiveResults>false</ArchiveResults>
<CleanUpDisk>false</CleanUpDisk>
<UpdateProjectLifeCycle>false</UpdateProjectLifeCycle>
<NotifyOperatorWhenDone>false</NotifyOperatorWhenDone>
<SDMall>false</SDMall>
<SDMonly>false</SDMonly>
<PipelineOperatorAddress>Unknown</PipelineOperatorAddress>
</ResultsProcessing>
</ns2:SciPipeRequest>
"""
old_metadata = """{
"fileSetIds": ["{{sdmId}}", "{{calSdmId}}"]
"workflowName": "std_restore_imaging",
"systemId": "{{request_id}}",
"creationTime": "{{created_at}}",
"productLocator": "{{product_locator}}",
"calProductLocator": "{{cal_locator}}",
"projectMetadata": {
"projectCode": "{{projectCode}}",
"title": "{{title}}",
"startTime": "{{startTime}}",
"observer": "{{observer}}"
},
"destinationDirectory": "{{root_directory}}/{{relative_path}}"
}
"""
def upgrade():
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{condor}' WHERE filename='std_restore_imaging.condor'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{ppr}' WHERE filename='PPR.xml' AND workflow_name='std_restore_imaging'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{metadata}' WHERE filename='metadata.json' AND workflow_name='std_restore_imaging'
"""
)
def downgrade():
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{old_condor}' WHERE filename='std_restore_imaging.condor'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{old_ppr}' WHERE filename='PPR.xml' AND workflow_name='std_restore_imaging'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{old_metadata}' WHERE filename='metadata.json' AND workflow_name='std_restore_imaging'
"""
)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment