From c28c09c5fe26bb49bf011db59030e4f57a0f906f Mon Sep 17 00:00:00 2001
From: Janet Goldstein <jgoldste@nrao.edu>
Date: Wed, 4 Aug 2021 14:55:15 +0000
Subject: [PATCH] WS-601: Fixed broken EVLA CAL test
---
.../ingest_envoy/ingestion_manifest.py | 19 +++++++++---
.../ingest_envoy/manifest_components.py | 15 +++++++---
.../test/test_evla_cal_manifest.py | 29 ++++++++-----------
.../ingest_envoy/test/test_image_manifest.py | 4 +--
.../test_manifest_builder_entry_points.py | 2 ++
5 files changed, 41 insertions(+), 28 deletions(-)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index 1aded5b16..04e49a137 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -85,21 +85,30 @@ class ManifestIF(ManifestComponentIF):
params = ManifestParameters(
telescope=self.telescope,
reingest=False,
- ngas_ingest=False,
+ ngas_ingest=True,
calibrate=False,
staging_source_dir=self.staging_source_dir,
additional_metadata=additional_metadata,
)
- else:
+ self.additional_metadata = additional_metadata
+
+ elif self.sp_type == ScienceProductType.IMAGE:
params = ManifestParameters(
telescope=self.telescope,
reingest=False,
- ngas_ingest=False,
+ ngas_ingest=True,
calibrate=False,
staging_source_dir=self.staging_source_dir,
)
- self.additional_metadata = additional_metadata
+ else:
+ params = ManifestParameters(
+ telescope=self.telescope,
+ reingest=False,
+ ngas_ingest=True,
+ staging_source_dir=self.staging_source_dir,
+ )
+
return params
@abc.abstractmethod
@@ -136,6 +145,8 @@ class ManifestIF(ManifestComponentIF):
class IngestionManifestBuilder:
"""Builds ingestion manifest and associated files from files in ingestion_path"""
+ # TODO?: We are making assumptions about the values of "reingest", "ngas_ingest",
+ # and "calibrate"; is this valid?
def __init__(
self,
staging_source_dir: Path,
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
index bd4c542d1..19f56a88b 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
@@ -106,22 +106,27 @@ class InputGroup(ManifestComponentIF):
class ManifestParameters(ManifestComponentIF):
- """Represents "parameters" section of ingestion manifest"""
+ """Represents "parameters" section of ingestion manifest.
+ ASSUMPTIONS:
+ * EVLA CAL manifest has no "calibrate" parameter
+ * "ngas_ingest" is always True (per our testing examples)
+ """
def __init__(
self,
telescope: Telescope,
reingest: bool,
ngas_ingest: bool,
- calibrate: bool,
staging_source_dir: Path,
additional_metadata: AbstractTextFile = None,
+ calibrate: bool = None,
):
self.telescope = telescope
self.reingest = reingest
self.ngas_ingest = ngas_ingest
- self.calibrate = calibrate
+ if calibrate is not None:
+ self.calibrate = calibrate
self.staging_source_dir = staging_source_dir
self.additional_metadata = additional_metadata
@@ -141,15 +146,17 @@ class ManifestParameters(ManifestComponentIF):
return False
def to_json(self) -> JSON:
+
json_dict = {
ParamsKey.TELESCOPE.value: self.telescope.value,
# The ingestion manifest must have "true" and "false"
# rather than "True" and "False"
ParamsKey.REINGEST.value: "true" if self.reingest else "false",
ParamsKey.NGAS_INGEST.value: "true" if self.ngas_ingest else "false",
- ParamsKey.CALIBRATE.value: "true" if self.calibrate else "false",
ParamsKey.INGESTION_PATH.value: str(self.staging_source_dir),
}
+ if hasattr(self, "calibrate"):
+ json_dict[ParamsKey.CALIBRATE.value] = "true" if self.calibrate else "false"
if self.additional_metadata:
json_dict[ParamsKey.ADDITIONAL_METADATA.value] = str(self.additional_metadata)
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
index 81d122d09..b1e2ede31 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
@@ -75,7 +75,8 @@ def test_filters_cal_input_files(ingest_path: Path):
assert manifest.locator == locator
params = manifest.parameters
- assert not params.reingest and not params.ngas_ingest and not params.calibrate
+ assert params.reingest == False and params.ngas_ingest == True
+ assert not hasattr(params, "calibrate")
input_group = manifest.input_group
assert len(input_group.science_products) == 1
@@ -100,7 +101,6 @@ def test_filters_cal_input_files(ingest_path: Path):
shutil.rmtree(ingest_path)
-@pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation")
def test_writes_expected_evla_cal_output_files(ingest_path: Path):
"""
Did the manifest builder produce the manifest file, the weblog, and the science product tar?
@@ -319,21 +319,16 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
IngestionManifestKey.INGESTION_PATH.value
] = expected_dir_name
- # TODO 2021-08-03am NEXT MR : get this to work (and don't worry about the JSON per se)
- # expected_params = expected_json["parameters"]
- # actual_params = manifest.parameters.to_json()
- # actual_params["ingestion_path"] = expected_params["ingestion_path"]
- # print(f"\n>>> ACTUAL: {actual_params}")
- # print(f">>> EXPECTED: {actual_params}")
- # for key, exp_val in expected_json["parameters"].items():
- # act_val = actual_params[key]
- # if isinstance(exp_val, bool):
- # exp_val_str = "true" if exp_val else "false"
- # assert act_val == exp_val_str
- # else:
- # assert actual_params[key] == exp_val
- # assert actual_params == expected_params
- # assert manifest.input_group.to_json() == expected_json["input_group"]
+ expected_params = expected_json["parameters"]
+ actual_params = manifest.parameters.to_json()
+
+ expected_reingest = "true" if expected_params["reingest"] else "false"
+ assert actual_params["reingest"] == expected_reingest
+ expected_ngas_ingest = "true" if expected_params["ngas_ingest"] else "false"
+ assert actual_params["ngas_ingest"] == expected_ngas_ingest
+
+ assert "calibrate" not in actual_params.keys()
+ assert manifest.input_group.to_json() == expected_json["input_group"]
expected_outgroup = expected_json["output_group"]
expected_osp = expected_outgroup["science_products"]
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
index e13b51298..4fa5d1e58 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
@@ -8,7 +8,6 @@ from typing import List
import json
import shutil
import tarfile
-import pytest
from ingest_envoy.ingestion_manifest import IngestionManifestBuilder
from ingest_envoy.schema import AbstractTextFile
@@ -331,7 +330,6 @@ def test_writes_expected_output_files(ingest_path: Path):
assert len(artifacts) == 1
files_accounted_for.append(artifacts[0])
- unique_files = set(files_accounted_for)
assert len(set(files_accounted_for)) == len(ingestion_files) - 1
shutil.rmtree(ingest_path)
@@ -354,7 +352,7 @@ def manifest_parameters(ingest_path: Path) -> ManifestParameters:
params = ManifestParameters(
telescope=Telescope.EVLA,
reingest=False,
- ngas_ingest=False,
+ ngas_ingest=True,
calibrate=False,
staging_source_dir=ingest_path,
additional_metadata=AbstractTextFile(filename=addl_md.filename, content=""),
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py
index 34d7777e2..96bb67fa0 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py
@@ -5,6 +5,7 @@ import sys
from pathlib import Path
# pylint: disable=E0401, E0402, R1721, W0611, W0621
+import shutil
from ingest_envoy.ingestion_manifest import (
IngestionManifest,
@@ -105,3 +106,4 @@ def test_entry_point_for_image(ingest_path: Path):
print(f">>> {file.name} present after manifest build")
assert len(ingestion_files_after) == expected_file_count_before + 2
+ shutil.rmtree(ingest_path)
--
GitLab