diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py
index 30e4095b1ad5220952209b930b7e41386ea5ee07..a541afe450e471d6ab7aaf3e8d2e7909397c39d6 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py
@@ -13,15 +13,15 @@ def main():
:return:
"""
- if len(sys.argv) < 2:
- raise ValueError("directory containing files to be ingested must be specified")
- ingestion_type, locator, staging_dir = sys.argv[1], sys.argv[2], sys.argv[3]
+ # parser handles command-line args; no validation needed here
+ staging_dir, ingestion_type, locator = sys.argv[1], sys.argv[2], sys.argv[3]
+
+ # nor here
source_dir = Path(staging_dir)
- if not source_dir.is_dir():
- raise FileNotFoundError(f"directory {source_dir} not found")
# ValueError will be thrown if ingestion_type isn't a known ScienceProductType
ingestion_type = ScienceProductType.from_str(ingestion_type)
+ # Launch the manifest builder
IngestionManifest(source_dir, ingestion_type, locator).create()
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index e26dfd8dfd736fc7f046d2e59b1d8ef7af2b9889..fee031d5de309aa24fa63c94f947be77aab1f79d 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -15,7 +15,7 @@ logger.addHandler(logging.StreamHandler(sys.stdout))
class IngestionManifest:
- """needed for ingestion-launching iterface"""
+ """needed for ingestion-launching interface"""
def __init__(self, staging_source_dir: str, ingestion_type: str, locator: str):
self.ingest_path = Path(staging_source_dir)
@@ -29,5 +29,11 @@ class IngestionManifest:
:return:
"""
- writer = EvlaCalIngestionManifestWriter(self.sp_type, self.ingest_path)
+
+ if self.sp_type != ScienceProductType.EVLA_CAL:
+ return NotImplementedError(
+ f"Don't yet know how to handle {self.sp_type.value} science product"
+ )
+
+ writer = EvlaCalIngestionManifestWriter(self.ingest_path)
writer.write_evla_cal_manifest(self.locator)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_writer.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_writer.py
index 576d826b98e5afcf39001c63fdca63302371368b..0329aa15db9b8639e8dfad3a791ffa1d23ab7540 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_writer.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_writer.py
@@ -16,7 +16,6 @@ from .utilities import (
MANIFEST_NAME_EXT,
ARTIFACT_NAME,
ARTIFACT_EXT,
- ScienceProductType,
WEBLOG,
SCIENCE_PRODUCT_PATTERN,
EvlaCalInputScienceProduct,
@@ -33,10 +32,8 @@ logger.addHandler(logging.StreamHandler(sys.stdout))
class EvlaCalIngestionManifestWriter:
"""For building ingestion manifests"""
- # This one does NOT accept a manifest as a parameter.
-
- def __init__(self, sp_type: ScienceProductType, ingest_path: Path):
- self.sp_type = sp_type
+ # (science product type is always EVLA_CAL)
+ def __init__(self, ingest_path: Path):
self.ingest_path = ingest_path
self.input_group = self.output_group = self.infiles = None
@@ -53,37 +50,45 @@ class EvlaCalIngestionManifestWriter:
self.output_group = self._make_evla_cal_output_group()
# Pull out the manifest content and stringify it
- manifest_content = json.dumps(self.content())
+ manifest_content = json.dumps(self.content(), indent=4)
manifest_filename = self.manifest_filename()
# Write the manifest to the staging area
- staging_manifest = self.ingest_path / manifest_filename
+ staging_manifest = Path(self.ingest_path / manifest_filename)
with open(staging_manifest, "w") as out:
- out.write(manifest_content)
+ out.write(f"{manifest_content}\n")
# Get all the files we'll need
addl_ingestion_files = self.find_additional_ingestion_files()
+ # Write the artifacts tar.
+ artifacts_tar = self.write_ingestion_artifacts_tar(staging_manifest, addl_ingestion_files)
+ addl_ingestion_files.append(artifacts_tar)
+
# return a Path explicitly; LocalPath won't work
return Path(staging_manifest), addl_ingestion_files
- @staticmethod
def write_ingestion_artifacts_tar(
- ingestion_location: Path, ingestion_files: List[Path]
+ self, manifest_file: Path, ingestion_files: List[Path]
) -> Path:
"""
Take the list of files and build a tar for inclusion into the archive.
This happens in the staging area for ingestion.
+ The EVLA CAL tar will contain just the manifest
- :param ingestion_location: path to ingestion location (probably the spool directory)
+ :param manifest_file: the ingestion manifest
:param ingestion_files: all the files needed for ingestion
:return: a .tar archive of the ingestion artifacts
"""
- ingestion_artifacts_fn = ingestion_location / "ingestion_artifacts.tar"
+ ingestion_artifacts_fn = self.ingest_path / "ingestion_artifacts.tar"
with tarfile.open(ingestion_artifacts_fn, "w") as ingestion_artifacts_tar:
for file in ingestion_files:
ingestion_artifacts_tar.add(file)
+ # include the manifest
+ if manifest_file not in ingestion_files:
+ ingestion_artifacts_tar.add(manifest_file)
+
return ingestion_artifacts_fn
def _make_evla_cal_output_group(self):
@@ -102,23 +107,28 @@ class EvlaCalIngestionManifestWriter:
return EvlaCalOutputGroup(science_product, None)
- def content(self) -> Dict:
+ def content(self) -> Dict[str, str]:
"""
Accessor for manifest content
:return: manifest as dict
"""
- if self.sp_type != ScienceProductType.EVLA_CAL:
- raise NotImplementedError(f"{self.sp_type}")
- return dict(
- input_group=repr(self.input_group),
- output_group=repr(self.output_group),
- # associate_group=repr(self.associate_group), # TODO when we need it
- ingestion_path=repr(self.ingest_path),
- science_products=repr(self.input_group.science_products),
- ancillary_products=repr(self.output_group.ancillary_products),
+ params = {
+ "reingest": "false",
+ "ngas_ingest": "false",
+ "calibrate": "false",
+ "ingestion_path": str(self.ingest_path),
+ }
+
+ json_out = dict(
+ parameters=params,
+ input_group=self.input_group.__json__(),
+ output_group=self.output_group.__json__(),
+ # associate_group=self.associate_group.__json__(), # TODO when we need it
+ ingestion_path=str(self.ingest_path),
)
+ return json_out
def _find_science_product_tar(self) -> Path:
"""
@@ -126,30 +136,24 @@ class EvlaCalIngestionManifestWriter:
:return:
"""
- if self.sp_type == ScienceProductType.EVLA_CAL:
- files = [file for file in self.ingest_path.iterdir() if file.is_file]
- for file in files:
- if re.match(SCIENCE_PRODUCT_PATTERN, file.name):
- return file
-
- raise FileNotFoundError(f"no science product found at {self.ingest_path}")
+ files = [file for file in self.ingest_path.iterdir() if file.is_file]
+ for file in files:
+ if re.match(SCIENCE_PRODUCT_PATTERN, file.name):
+ return file
- raise NotImplementedError(f"{self.sp_type}")
+ raise FileNotFoundError(f"no science product found at {self.ingest_path}")
@staticmethod
def format_timestamp(datetime: DateTime) -> str:
"""
- Format the current time as follows:
- input format:
- 2021-07-01T13:49:17.237119+00:00
- desired output format as yyyy_MM_dd_'T'HH_mm_ss.SSS:
- 2021_07_01'T'13_49_17.237
+ Format the current time as
+ 2021_07_01T13_49_17.237
- :param datetime: current pendulum timestamp
+ :param datetime: current timestamp
:return: timestamp suitable for ingestion manifest filename
"""
- return datetime.format("YYYY_MM_DD'T'hh_mm_ss.SSS")
+ return datetime.format("YYYY_MM_DDThh_mm_ss.SSS")
@staticmethod
def manifest_filename() -> str:
@@ -176,14 +180,13 @@ class EvlaCalIngestionManifestWriter:
def find_additional_ingestion_files(self) -> List[Path]:
"""
- Gather the files required for ingestion
+ Gather the files required for ingestion.
+ There won't be any for EVLA CAL ingestion; this is just a placeholder.
:return: ingestion inputs
"""
- if self.sp_type == ScienceProductType.EVLA_CAL:
- return []
- raise NotImplementedError(f"Don't yet know how to write manifest for {self.sp_type}")
+ return []
# TODO: we'll have extra information for other ingestion types
# coll_files = aux_files = []
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
index 7456dd31649964339d184c4fa36c717c27f25d83..cad2a31dcfe9f645953129235dcf6c1911dd52f7 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
@@ -1,15 +1,16 @@
""" Objects pertaining to the various ingestion manifests """
+
from __future__ import annotations
import json
import re
import tarfile
from enum import Enum
-
-# pylint: disable=E0401, R0903, R1721
from pathlib import Path
from typing import List, Dict
+# pylint: disable=E0401, R0903, R1721
+
MANIFEST_NAME_BASE = "ingestion_manifest_"
MANIFEST_NAME_EXT = ".json"
ARTIFACT_NAME = "ingestion_artifacts_"
@@ -78,21 +79,27 @@ class ScienceProductType(Enum):
return f'"{str(self.value)}"'
@staticmethod
- def from_str(spt_str: str) -> ScienceProductType:
+ def from_str(sp_type_in) -> ScienceProductType:
"""
- In comes a string. Out comes the corresponding ScienceProductType, if any
+ In comes a string; out comes the corresponding ScienceProductType, if any.
+ Or maybe it's already a ScienceProductType, in which case we can just return it.
- :param spt_str: a string that "should" represent a ScienceProductType
+ :param sp_type_in: a string that "should" represent a ScienceProductType
:return:
"""
+ if isinstance(sp_type_in, ScienceProductType):
+ return sp_type_in
+
for spt in ScienceProductType:
- if spt.value == spt_str:
+ if spt.value == sp_type_in:
return spt
- raise ValueError(f"unrecognized ScienceProductType: {spt_str}")
+ raise ValueError(
+ f"unrecognized ScienceProductType: {sp_type_in}; it's a {type(sp_type_in)}"
+ )
-class AncillaryProductType:
+class AncillaryProductType(Enum):
"""The various types of ancillary products we'll encounter"""
INGESTION_ARTIFACTS = "ingestion_artifacts"
@@ -182,7 +189,10 @@ class EvlaCalInputGroup:
"""
json_out = self.__dict__
sps = json_out["science_products"]
- json_out["science_products"] = [sps[0].__json__()]
+ sci_prod = sps[0]
+ sp_str = sci_prod if isinstance(sci_prod, str) else sci_prod.__json__()
+ json_out["science_products"] = f"[{sp_str}]"
+
return json_out
@@ -225,22 +235,20 @@ class EvlaCalOutputGroup:
:return: JSONified OutputGroup
"""
- return dict(self.__dict__)
- # json_out = self.__dict__
- # ap_json = self.ancillary_products[0].__json__()
- # json_out["ancillary_products"] = ap_json
- #
- # sp_json = self.science_products[0].__json__()
- # json_out["science_products"] = sp_json
- #
- # sp_repr = repr(json_out)
- # pprint(f">>> json_out:\n'{json_out}'")
- #
- # return repr(json_out)
+
+ json_out = self.__dict__
+ anc_prod = self.ancillary_products[0]
+ ap_str = anc_prod if isinstance(anc_prod, str) else anc_prod.__json__()
+ json_out[IngestionManifestKey.ANCILLARY_PRODUCTS.value] = f"[{ap_str}]"
+ sci_prod = self.science_products[0]
+ sp_str = sci_prod if isinstance(sci_prod, str) else sci_prod.__json__()
+ json_out[IngestionManifestKey.SCIENCE_PRODUCTS.value] = f"[{sp_str}]"
+
+ return json_out
class EvlaCalIngestionManifest:
- """this is JUST the ingestion manifest JSON"""
+ """TODO: this is JUST the ingestion manifest JSON, not a bespoke object"""
def __init__(self, ingestion_path: Path, spl: str):
"""
@@ -258,8 +266,8 @@ class EvlaCalIngestionManifest:
def __str__(self):
params = self._make_params_section()
- input_group = repr(self.input_group)
- output_group = repr(self.output_group)
+ input_group = self.input_group.__json__()
+ output_group = self.output_group.__json__()
return f"{params}\n{input_group}\n{output_group}"
def _make_params_section(self) -> str:
@@ -303,3 +311,13 @@ class EvlaCalIngestionManifest:
for file in self.infiles:
if re.match(SCIENCE_PRODUCT_PATTERN, file.name):
return file
+
+
+class IngestionManifestKey(Enum):
+ """Sections we expect to see in a manifest"""
+
+ INPUT_GROUP = "input_group"
+ OUTPUT_GROUP = "output_group"
+ INGESTION_PATH = "ingestion_path"
+ SCIENCE_PRODUCTS = "science_products"
+ ANCILLARY_PRODUCTS = "ancillary_products"
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/conftest.py b/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
index 817a482dffcfa405266cf408374119719e869861..77f21016eea84a4a09a42ba968916d9e3fd38a20 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
@@ -2,7 +2,6 @@
# pylint: disable=E0401, R1721
-from enum import Enum
from pathlib import Path
from typing import List
@@ -63,13 +62,3 @@ def populate_fake_ingest_path(staging_dir: Path) -> List[Path]:
files.append(path)
return files
-
-
-class IngestionManifestKey(Enum):
- """Sections we expect to see in a manifest"""
-
- INPUT_GROUP = "input_group"
- OUPUT_GROUP = "output_group"
- INGESTION_PATH = "ingestion_path"
- SCIENCE_PRODUCTS = "science_products"
- ANCILLARY_PRODUCTS = "ancillary_products"
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_alma_ing_manifests.py b/apps/cli/executables/pexable/ingest_envoy/test/test_alma_ing_manifests.py
index da37adf8fda3820ae08758060f0ffd4b9a4f3e49..bd018d7241f7368e352c580241e8b845a36c94f2 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_alma_ing_manifests.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_alma_ing_manifests.py
@@ -1,5 +1,7 @@
""" Test for the various types of ALMA ingestion manifests """
+# pylint: disable=E0401
+
import pytest
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
index 76d75c4d43d5d57b6329e3795643a14cfde1817f..4c27e1c60f4e0ca6b244345d08644e0439df31ac 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
@@ -7,7 +7,7 @@ import shutil
import sys
from pathlib import Path
-# pylint: disable=E0402
+# pylint: disable=E0401, E0402, R0914
from ingest_envoy.ingestion_manifest import IngestionManifest
from ingest_envoy.ingestion_manifest_writer import (
@@ -21,8 +21,14 @@ from ingest_envoy.utilities import (
EvlaCalOutputScienceProduct,
Weblog,
EvlaCalIngestionManifest,
+ IngestionManifestKey,
+)
+from .conftest import (
+ find_example_manifest,
+ populate_fake_ingest_path,
+ WANTED_FILENAMES,
+ UNWANTED,
)
-from .conftest import find_example_manifest, populate_fake_ingest_path, WANTED_FILENAMES, UNWANTED
logger = logging.getLogger(IngestionManifest.__name__)
logger.setLevel(logging.INFO)
@@ -64,7 +70,7 @@ def test_creates_empty_evla_cal_manifest(ingest_path: Path):
ingest_path.rmdir()
-def test_creates_expected_evla_cal_manifest(ingest_path: Path):
+def test_creates_expected_manifest(ingest_path: Path):
"""
Make sure we create the manifest we expect.
@@ -102,9 +108,35 @@ def test_creates_expected_evla_cal_manifest(ingest_path: Path):
shutil.rmtree(ingest_path)
+def test_writes_real_manifest_to_file(ingest_path: Path):
+ """
+ We should get a pretty, formatted, human-readable JSON text file
+
+ :param ingest_path: the staging dir
+ :return:
+ """
+
+ populate_fake_ingest_path(ingest_path)
+
+ writer = EvlaCalIngestionManifestWriter(ingest_path=ingest_path)
+ locator = "uid://evla/calibration/my_devastating_observation"
+ manifest_file, _ = writer.write_evla_cal_manifest(locator)
+
+ with open(manifest_file, "r") as mf_in:
+ manifest_content = dict(json.load(mf_in).items())
+
+ assert len(manifest_content.keys()) >= len(IngestionManifestKey) - 1
+ for key in ["parameters", "input_group", "output_group"]:
+ assert key in manifest_content.keys()
+
+ shutil.rmtree(ingest_path)
+
+
def test_builds_expected_manifest_filename(ingest_path: Path):
"""
- We expect the manifest to be named like "ingestion_manifest_2019_07_30_T13_03_00.936.json"
+ We expect the manifest to be named like
+
+ ingestion_manifest_2019_07_30_T13_03_00.936.json
:param ingest_path: ingestion location
:return:
@@ -122,7 +154,7 @@ def test_builds_expected_manifest_filename(ingest_path: Path):
timestamp = filename.replace(MANIFEST_NAME_BASE, "").replace(MANIFEST_NAME_EXT, "")
# we should have gotten year, month, day, hours, minutes, seconds to 3 decimal places
- assert re.match(r"\d{4}_\d{2}_\d{2}'T'\d{2}_\d{2}_\d{2}\.\d{0,3}", timestamp)
+ assert re.match(r"\d{4}_\d{2}_\d{2}T\d{2}_\d{2}_\d{2}\.\d{0,3}", timestamp)
shutil.rmtree(ingest_path)
@@ -150,16 +182,17 @@ def test_writes_expected_output_files(ingest_path: Path):
ip_in = mf_json["parameters"]["ingestion_path"]
assert ip_in == "/home/mchammer/evla/parallel-prod"
- writer = EvlaCalIngestionManifestWriter(
- sp_type=ScienceProductType.EVLA_CAL, ingest_path=ingest_path
- )
+ writer = EvlaCalIngestionManifestWriter(ingest_path=ingest_path)
manifest_file, more_ingestion_files = writer.write_evla_cal_manifest(ig_sp["locator"])
- assert len(more_ingestion_files) == 0
assert manifest_file.exists()
+ for file in more_ingestion_files:
+ assert file.exists()
+ assert len(more_ingestion_files) == 1
- artifacts_file = Path(ingest_path / writer.artifact_filename())
- # for EVLA CAL ingestion, we're -not- expecting this
- assert not artifacts_file.exists()
+ # make sure that one file is the artifacts tar
+ file = more_ingestion_files[0]
+ assert file.exists()
+ assert file.name.startswith("ingestion_artifacts") and file.name.endswith(".tar")
shutil.rmtree(ingest_path)
@@ -168,6 +201,7 @@ def test_filters_cal_input_files(ingest_path: Path):
"""
We'll be getting calibration products from a directory under
/lustre/aoc/cluster/pipeline/{CAPO_PROFILE}/workspaces/staging
+
Make sure we take -only- the files to be ingested.
:param ingest_path: our temporary dir
@@ -175,25 +209,60 @@ def test_filters_cal_input_files(ingest_path: Path):
"""
populate_fake_ingest_path(ingest_path)
- writer = EvlaCalIngestionManifestWriter(
- sp_type=ScienceProductType.EVLA_CAL, ingest_path=ingest_path
- )
+ writer = EvlaCalIngestionManifestWriter(ingest_path=ingest_path)
locator = "uid://evla/calibration/im_a_one-touch_espresso_machine"
- writer.write_evla_cal_manifest(locator)
-
- ig_json = writer.input_group.__json__()
- assert len(ig_json["science_products"]) == 1
- ig_sp = ig_json["science_products"][0]
- assert ig_sp["type"] == ScienceProductType.EVLA_CAL.value
- assert ig_sp["locator"] == locator
-
- og_json = writer.output_group.__json__()
- for sp_out in og_json["science_products"]:
- filename = Path(sp_out.filename).name
+ manifest_file, _ = writer.write_evla_cal_manifest(locator)
+
+ with open(manifest_file, "r") as mf_in:
+ manifest_content = dict(json.load(mf_in).items())
+ input_group = manifest_content[IngestionManifestKey.INPUT_GROUP.value]
+ assert isinstance(input_group, dict)
+ assert len(input_group) == 1
+
+ for val in input_group.values():
+ sci_prod = val.replace("'", '"', len(val))
+ sps = json.loads(sci_prod)
+
+ assert isinstance(sps, list)
+ assert len(sps) == 1
+ sci_prod = sps[0]
+ assert sci_prod["type"] == ScienceProductType.EVLA_CAL.value
+ assert sci_prod["locator"] == locator
+
+ output_group = manifest_content[IngestionManifestKey.OUTPUT_GROUP.value]
+ assert isinstance(output_group, dict)
+ assert len(output_group) == 2
+
+ for key, val in output_group.items():
+ if key == IngestionManifestKey.SCIENCE_PRODUCTS.value:
+ sci_prod = val.replace("'", '"', len(val))
+ sps = json.loads(sci_prod)
+ assert len(sps) == 1
+ else:
+ assert key == IngestionManifestKey.ANCILLARY_PRODUCTS.value
+ anc_prod = val.replace("'", '"', len(val))
+ aps = json.loads(anc_prod)
+ assert len(aps) == 1
+
+ for sci_prod in sps:
+ filename = Path(sci_prod["filename"]).name
file = Path(ingest_path / filename)
assert file.exists()
assert file.parent == ingest_path
assert filename in WANTED_FILENAMES
assert filename not in UNWANTED
+ anc_prod_dict1 = aps[0]
+ for key, val in anc_prod_dict1.items():
+ anc_prod_dict2 = val
+
+ assert isinstance(anc_prod_dict2, dict)
+ for key, val in anc_prod_dict2.items():
+ if key == "type":
+ assert val == "weblog"
+ else:
+ assert key == "filename"
+ file = Path(ingest_path / val)
+ assert file.exists()
+
shutil.rmtree(ingest_path)
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py b/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
index ac69491ca76d45b5a03e9e26b45eea56f6e8752a..fc043286d6880653b80d718174a8c2d6a57c5876 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
@@ -11,8 +11,8 @@ from pathlib import Path
import pytest
from ingest_envoy.ingestion_manifest import IngestionManifest
-from ingest_envoy.utilities import ScienceProductType
-from .conftest import populate_fake_ingest_path, WANTED_FILENAMES, UNWANTED, IngestionManifestKey
+from ingest_envoy.utilities import ScienceProductType, IngestionManifestKey
+from .conftest import populate_fake_ingest_path, WANTED_FILENAMES, UNWANTED
logger = logging.getLogger(IngestionManifest.__name__)
logger.setLevel(logging.INFO)
@@ -28,36 +28,51 @@ def test_entry_point_for_evla_cal(ingest_path: Path):
"""
populate_fake_ingest_path(ingest_path)
manifest = IngestionManifest(
- ingest_path, ScienceProductType.EVLA_CAL.value, "uid://evla/calibration/meeniemyniemoe"
+ str(ingest_path), ScienceProductType.EVLA_CAL.value, "uid://evla/calibration/meeniemyniemoe"
)
manifest.create()
- manifest_file = [file for file in ingest_path.iterdir() if file.name.endswith(".json")][0]
- assert manifest_file.exists()
+
files = [file for file in ingest_path.iterdir()]
- assert len(files) == len(WANTED_FILENAMES) + len(UNWANTED) + 1
+ manifest_file = [file for file in files if file.name.endswith(".json")][0]
+ assert manifest_file.exists()
- # make sure manifest_file contains an IngestionManifest
+ assert len(files) == len(WANTED_FILENAMES) + len(UNWANTED) + 2
+ # make sure manifest_file contains an IngestionManifest
with open(manifest_file, "r") as out:
manifest_content = dict(json.load(out).items())
- assert len(manifest_content.keys()) >= len(IngestionManifestKey)
- for key in IngestionManifestKey:
- assert key.value in manifest_content.keys()
+ for key in ["parameters", "input_group", "output_group", "ingestion_path"]:
+ assert key in manifest_content.keys()
+
+ input_group = manifest_content[IngestionManifestKey.INPUT_GROUP.value]
+ assert isinstance(input_group, dict)
+ assert len(input_group) == 1
- assert "EvlaCalInputGroup object" in manifest_content[IngestionManifestKey.INPUT_GROUP.value]
- assert "EvlaCalOutputGroup object" in manifest_content[IngestionManifestKey.OUPUT_GROUP.value]
- assert "PosixPath" in manifest_content[IngestionManifestKey.INGESTION_PATH.value]
+ output_group = manifest_content[IngestionManifestKey.OUTPUT_GROUP.value]
+ assert isinstance(output_group, dict)
+ for key, val in output_group.items():
+ val = val.replace("'", '"', len(val))
+ sci_prods = json.loads(val)
- science_products = manifest_content[IngestionManifestKey.SCIENCE_PRODUCTS.value]
- assert science_products[0] == "["
- assert science_products[-1] == "]"
- assert "EvlaCalInputScienceProduct object" in science_products
+ assert isinstance(sci_prods, list)
+ assert len(sci_prods) == 1
- ancillary_products = manifest_content[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
+ ancillary_products = output_group[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
assert ancillary_products[0] == "["
assert ancillary_products[-1] == "]"
- assert "Weblog object" in ancillary_products
+ assert "weblog" in ancillary_products
+ assert "type" in ancillary_products
+ assert "filename" in ancillary_products
+
+ a_prods = json.loads(ancillary_products.replace("'", '"', len(ancillary_products)))
+ assert isinstance(a_prods, list)
+ a_prods = a_prods[0]
+ assert isinstance(a_prods, dict)
+ for key, val in a_prods.items():
+ assert isinstance(val, dict)
+ a_prods = val
+ assert len(a_prods) == 2
shutil.rmtree(ingest_path)
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_vlba_manifests.py b/apps/cli/executables/pexable/ingest_envoy/test/test_vlba_manifests.py
index 16cd3707f3a9f17edfe5620b8d6eb574d70e6a57..3da5fffee3f5d7eb3c6857a4228bee41d222b9bc 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_vlba_manifests.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_vlba_manifests.py
@@ -1,5 +1,7 @@
""" Tests for VLBA product ingestion manifests """
+# pylint: disable=E0401
+
import pytest