Skip to content
Snippets Groups Projects
Commit 6c1cac6b authored by Charlotte Hausman's avatar Charlotte Hausman
Browse files

Merge branch 'ws-2116-fix-numerical-path' into '2.8.3-DEVELOPMENT' 

add formatting to relative path correction for numerical values

See merge request !1583
parents 5eaee333 46159c81
No related branches found
No related tags found
2 merge requests!1607merge 2.8.3 to main,!1583add formatting to relative path correction for numerical values
Pipeline #14366 passed
Stage: pull-db
Stage: build-packages
Stage: push-packages
Stage: deploy
Hide whitespace changes
Inline Side-by-side
Showing
with 349 additions and 3 deletions
apps/cli/executables/pexable/casa_envoy/casa_envoy/auditor.py
+ 14
− 2
View file @ 6c1cac6b
......@@ -146,6 +146,17 @@ class AuditFiles(AuditorIF):
else:
return True
@staticmethod
def format_relative_path(relative_path: str) -> str:
"""
Occasionally (very very very occasionally) the relative path might be numerical only.
CASA hates this and falls over (see WS-2116). If the relative path is numerical enclose it in quotes
:param relative_path: the relative path to check
:return: relative path string, corrected if necessary
"""
return relative_path if not relative_path.isdigit() else "'" + str(relative_path) + "'"
def correct_for_condor(self, ppr: AbstractTextFile) -> AbstractTextFile:
"""
Correct PPR.xml for HTCondor directory and path
......@@ -158,8 +169,7 @@ class AuditFiles(AuditorIF):
parsed_xml = get_xml_content(ppr)
parsed_xml.find("RootDirectory").string = self.settings["rootDirectory"]
parsed_xml.find("RelativePath").string = self.settings["processingDirectory"]
parsed_xml.find("RelativePath").string = self.format_relative_path(self.settings["processingDirectory"])
strip_declaration = str(parsed_xml).split("\n")[1:]
fixed_declaration = "".join(strip_declaration)
......@@ -177,6 +187,8 @@ class AuditFiles(AuditorIF):
with open(ppr.filename, "w") as file:
file.write(corrected_content.decode())
# make sure we also update the currently used content. needed for field validation
ppr.content = corrected_content.decode()
os.chdir("../")
return ppr
......
apps/cli/executables/pexable/casa_envoy/test/input_files/PPR-numerical-path-correct.xml 0 → 100644
+ 155
− 0
View file @ 6c1cac6b
<?xml version='1.0' encoding='UTF-8' standalone='yes'?>
<ns2:SciPipeRequest xmlns:ns2="Common/pipelinescience/SciPipeRequest">
<ProjectSummary>
<ProposalCode>VLA/null</ProposalCode>
<Observatory>NRAO</Observatory>
<Telescope>VLA</Telescope>
<ProcessingSite>Socorro</ProcessingSite>
<Operator>vlapipe</Operator>
<Mode>SCIENCE</Mode>
<Version>NGRH-ALMA-10_8</Version>
<CreationTime>2021-05-06T17:34:36</CreationTime>
</ProjectSummary>
<ProjectStructure>TBD</ProjectStructure>
<ProcessingRequests>
<RootDirectory>/tmp/workspaces_tmp</RootDirectory>
<ProcessingRequest>
<ProcessingIntents>
<Intents>
<Keyword>VLA_INTERFEROMETRY_STANDARD_OBSERVING_MODE</Keyword>
<Value>Undefined</Value>
</Intents>
</ProcessingIntents>
<ProcessingProcedure>
<ProcedureTitle>hifv_cal</ProcedureTitle>
<ProcessingCommand>
<Command>hifv_importdata</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_hanning</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_flagdata</Command>
<ParameterSet>
<Parameter>
<Keyword>hm_tbuff</Keyword>
<Value>1.5int</Value>
</Parameter>
<Parameter>
<Keyword>intents</Keyword>
<Value>*POINTING*,*FOCUS*,*ATMOSPHERE*,*SIDEBAND_RATIO*, *UNKNOWN*, *SYSTEM_CONFIGURATION*, *UNSPECIFIED#UNSPECIFIED*</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_vlasetjy</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_priorcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_testBPdcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_checkflag</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_semiFinalBPdcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_checkflag</Command>
<ParameterSet>
<Parameter>
<Keyword>checkflagmode</Keyword>
<Value>semi</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_semiFinalBPdcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_solint</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_fluxboot2</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_finalcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_applycals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_targetflag</Command>
<ParameterSet>
<Parameter>
<Keyword>intents</Keyword>
<Value>*CALIBRATE*,*TARGET*</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_statwt</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_plotsummary</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hif_makeimlist</Command>
<ParameterSet>
<Parameter>
<Keyword>intent</Keyword>
<Value>PHASE,BANDPASS</Value>
</Parameter>
<Parameter>
<Keyword>specmode</Keyword>
<Value>cont</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hif_makeimages</Command>
<ParameterSet>
<Parameter>
<Keyword>hm_masking</Keyword>
<Value>none</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_exportdata</Command>
<ParameterSet/>
</ProcessingCommand>
</ProcessingProcedure>
<DataSet>
<RelativePath>'1234567'</RelativePath>
<SdmIdentifier>brain_001.58099.678886747686</SdmIdentifier>
<DataType>asdm</DataType>
</DataSet>
</ProcessingRequest>
</ProcessingRequests>
<ResultsProcessing>
<ArchiveResults>false</ArchiveResults>
<CleanUpDisk>false</CleanUpDisk>
<UpdateProjectLifeCycle>false</UpdateProjectLifeCycle>
<NotifyOperatorWhenDone>false</NotifyOperatorWhenDone>
<SDMall>false</SDMall>
<SDMonly>false</SDMonly>
<PipelineOperatorAddress>Unknown</PipelineOperatorAddress>
</ResultsProcessing>
</ns2:SciPipeRequest>
apps/cli/executables/pexable/casa_envoy/test/input_files/PPR-numerical-path.xml 0 → 100644
+ 155
− 0
View file @ 6c1cac6b
<?xml version='1.0' encoding='UTF-8' standalone='yes'?>
<ns2:SciPipeRequest xmlns:ns2="Common/pipelinescience/SciPipeRequest">
<ProjectSummary>
<ProposalCode>VLA/null</ProposalCode>
<Observatory>NRAO</Observatory>
<Telescope>VLA</Telescope>
<ProcessingSite>Socorro</ProcessingSite>
<Operator>vlapipe</Operator>
<Mode>SCIENCE</Mode>
<Version>NGRH-ALMA-10_8</Version>
<CreationTime>2021-05-06T17:34:36</CreationTime>
</ProjectSummary>
<ProjectStructure>TBD</ProjectStructure>
<ProcessingRequests>
<RootDirectory>/tmp/workspaces_tmp</RootDirectory>
<ProcessingRequest>
<ProcessingIntents>
<Intents>
<Keyword>VLA_INTERFEROMETRY_STANDARD_OBSERVING_MODE</Keyword>
<Value>Undefined</Value>
</Intents>
</ProcessingIntents>
<ProcessingProcedure>
<ProcedureTitle>hifv_cal</ProcedureTitle>
<ProcessingCommand>
<Command>hifv_importdata</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_hanning</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_flagdata</Command>
<ParameterSet>
<Parameter>
<Keyword>hm_tbuff</Keyword>
<Value>1.5int</Value>
</Parameter>
<Parameter>
<Keyword>intents</Keyword>
<Value>*POINTING*,*FOCUS*,*ATMOSPHERE*,*SIDEBAND_RATIO*, *UNKNOWN*, *SYSTEM_CONFIGURATION*, *UNSPECIFIED#UNSPECIFIED*</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_vlasetjy</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_priorcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_testBPdcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_checkflag</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_semiFinalBPdcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_checkflag</Command>
<ParameterSet>
<Parameter>
<Keyword>checkflagmode</Keyword>
<Value>semi</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_semiFinalBPdcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_solint</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_fluxboot2</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_finalcals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_applycals</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_targetflag</Command>
<ParameterSet>
<Parameter>
<Keyword>intents</Keyword>
<Value>*CALIBRATE*,*TARGET*</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_statwt</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_plotsummary</Command>
<ParameterSet/>
</ProcessingCommand>
<ProcessingCommand>
<Command>hif_makeimlist</Command>
<ParameterSet>
<Parameter>
<Keyword>intent</Keyword>
<Value>PHASE,BANDPASS</Value>
</Parameter>
<Parameter>
<Keyword>specmode</Keyword>
<Value>cont</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hif_makeimages</Command>
<ParameterSet>
<Parameter>
<Keyword>hm_masking</Keyword>
<Value>none</Value>
</Parameter>
</ParameterSet>
</ProcessingCommand>
<ProcessingCommand>
<Command>hifv_exportdata</Command>
<ParameterSet/>
</ProcessingCommand>
</ProcessingProcedure>
<DataSet>
<RelativePath>1234567</RelativePath>
<SdmIdentifier>brain_001.58099.678886747686</SdmIdentifier>
<DataType>asdm</DataType>
</DataSet>
</ProcessingRequest>
</ProcessingRequests>
<ResultsProcessing>
<ArchiveResults>false</ArchiveResults>
<CleanUpDisk>false</CleanUpDisk>
<UpdateProjectLifeCycle>false</UpdateProjectLifeCycle>
<NotifyOperatorWhenDone>false</NotifyOperatorWhenDone>
<SDMall>false</SDMall>
<SDMonly>false</SDMonly>
<PipelineOperatorAddress>Unknown</PipelineOperatorAddress>
</ResultsProcessing>
</ns2:SciPipeRequest>
apps/cli/executables/pexable/casa_envoy/test/test_auditor.py
+ 21
− 1
View file @ 6c1cac6b
......@@ -41,6 +41,10 @@ cal_settings = {
"product_type": "standard-cal",
}
test_ppr = AbstractTextFile(filename="test/input_files/PPR.xml", content=Path("test/input_files/PPR.xml").read_text())
test_ppr_numerical = AbstractTextFile(
filename="test/input_files/PPR-numerical-path.xml",
content=Path("test/input_files/PPR-numerical-path.xml").read_text(),
)
test_cal_metadata = AbstractTextFile(
filename="test/input_files/test.json", content=Path("test/input_files/test.json").read_text()
)
......@@ -182,12 +186,28 @@ class TestAuditFiles:
result2F = audit_files.check_required_fields(test_ppr, fields2F)
assert result2F is False
@patch("builtins.open")
@patch("os.chdir")
@patch("shutil.copy")
def test_correct_for_condor(self, mock_copy, mock_os):
def test_correct_for_condor(self, mock_copy, mock_os, mock_open):
ppr = AuditFiles([test_cal_metadata, test_ppr], cal_settings).correct_for_condor(ppr=test_ppr)
assert mock_open.call_count == 1 # would this have written out as expected if not mocked?
assert ppr.filename == "test/input_files/PPR.xml"
@patch("builtins.open")
@patch("os.chdir")
@patch("shutil.copy")
def test_correct_for_condor_numerical_path(self, mock_copy, mock_os, mock_open):
cal_settings_num = cal_settings.copy()
cal_settings_num["ppr"] = "test/input_files/PPR-numerical-path.xml"
cal_settings_num["processingDirectory"] = "1234567"
ppr = AuditFiles([test_cal_metadata, test_ppr_numerical], cal_settings_num).correct_for_condor(
ppr=test_ppr_numerical
)
correct_content = Path("test/input_files/PPR-numerical-path-correct.xml").read_text()
assert mock_open.call_count == 1 # would this have written out as expected if not mocked?
assert ppr.content == correct_content
def test_audit(self):
with patch("casa_envoy.auditor.AuditFiles.correct_for_condor") as condor:
result = AuditFiles([test_cal_metadata, test_ppr], cal_settings).audit()
......
config/pex-watcher/watcher.py
+ 4
− 0
View file @ 6c1cac6b
......@@ -31,6 +31,10 @@ class PexHandler(FileSystemEventHandler):
event_cache = {}
def on_modified(self, event):
print(event)
if event.is_directory or event.src_path.endswith(".py~"):
return
seconds = int(time.time())
key = (seconds, event.src_path)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment