diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index cd364c5b541ab5be53c0bcc98a4ffd3d687877ad..e6257091bbf5bef9650e28595b4a433960a067c0 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -190,7 +190,7 @@ class IngestionManifestBuilder:
manifest_file = manifest.write()
artifacts_tar = self.write_ingestion_artifacts_tar()
artifacts_ap = AncillaryProduct(
- AncillaryProductType.INGESTION_ARTIFACTS, filename=str(artifacts_tar)
+ AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
)
manifest.output_group.ancillary_products = [artifacts_ap]
@@ -215,7 +215,11 @@ class IngestionManifestBuilder:
manifest_file = manifest.write()
- self.write_ingestion_artifacts_tar()
+ artifacts_tar = self.write_ingestion_artifacts_tar()
+ artifacts_ap = AncillaryProduct(
+ type=AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
+ )
+ manifest.output_group.ancillary_products.append(artifacts_ap)
return manifest, manifest_file
@@ -257,7 +261,7 @@ class IngestionManifestBuilder:
"""
products_finder = ImageIngestionProductsFinder(self.staging_source_dir)
- science_products = products_finder.science_products
+ science_products = products_finder.output_science_products
ancillary_products = products_finder.ancillary_products
return OutputGroup(science_products=science_products, ancillary_products=ancillary_products)
@@ -273,12 +277,11 @@ class IngestionManifestBuilder:
timestamp = format_timestamp(current_time)
return f"{INGESTION_ARTIFACTS_NAME}{timestamp}{TARFILE_EXT}"
- def write_ingestion_artifacts_tar(self) -> Path:
+ def write_ingestion_artifacts_tar(self) -> tarfile.TarFile:
"""
Take the list of files and build a tar for inclusion into the archive.
This happens in the staging area for ingestion.
The EVLA CAL tar will contain just the manifest.
- For image ingestion, this must be done -after- the pipeline artifacts tar is built.
:return: a .tar archive of the ingestion artifacts
"""
@@ -290,10 +293,9 @@ class IngestionManifestBuilder:
ingestion_artifacts_tar.add(file)
# include the manifest
- if manifest_file not in self.files_found:
- ingestion_artifacts_tar.add(manifest_file)
+ ingestion_artifacts_tar.add(manifest_file)
- return ing_tar
+ return ingestion_artifacts_tar
class IngestionManifest(ManifestIF):
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py
index 0f1a1972a784cf94d14100aecc28bdfb9fb7b4b6..aff7a3e0768f03d9102ca4f17abba11050bcdc3b 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py
@@ -2,9 +2,15 @@
from pathlib import Path
from typing import List
-from ingest_envoy.manifest_components import OutputScienceProduct, AncillaryProduct, WEBLOG_FILENAME
+from ingest_envoy.manifest_components import (
+ OutputScienceProduct,
+ AncillaryProduct,
+ WEBLOG_FILENAME,
+ TARFILE_EXT,
+)
from ingest_envoy.utilities import AncillaryProductType
+
# pylint: disable=R1721
class ImageIngestionProductsFinder:
"""Finds ancillary science products and other ancillary products needed for image ingestion"""
@@ -12,7 +18,7 @@ class ImageIngestionProductsFinder:
def __init__(self, staging_source_dir: Path):
self.staging_source_dir = staging_source_dir
self.files_found = [file for file in self.staging_source_dir.iterdir()]
- self.science_products = self._find_output_science_products()
+ self.output_science_products = self._find_output_science_products()
self.ancillary_products = self._find_other_ancillary_products()
def _find_output_science_products(self) -> List[OutputScienceProduct]:
@@ -70,22 +76,17 @@ class ImageIngestionProductsFinder:
except ValueError as exc:
raise FileNotFoundError(f"No weblog found in {self.staging_source_dir}") from exc
- try:
- pipeline_artifacts_tar = [
- file
- for file in self.files_found
- if file.name.endswith("tar") and "ingestion_artifacts" not in file.name
- ][0]
- ancillary_products.append(
- AncillaryProduct(
- type=AncillaryProductType.PIPELINE_ARTIFACTS,
- filename=str(pipeline_artifacts_tar),
- )
+ ancillary_files = [Path(ap.filename) for ap in ancillary_products]
+
+ # find the pipeline artifacts tar
+ for file in self.files_found:
+ if file not in ancillary_files and self._is_ancillary_image_product(file):
+ pipeline_artifacts_tar = file
+ if pipeline_artifacts_tar:
+ pip_ap = AncillaryProduct(
+ type=AncillaryProductType.PIPELINE_ARTIFACTS, filename=str(pipeline_artifacts_tar)
)
- except ValueError as exc:
- raise FileNotFoundError(
- f"No pipeline artifacts found in {self.staging_source_dir}"
- ) from exc
+ ancillary_products.append(pip_ap)
return ancillary_products
@@ -107,24 +108,36 @@ class ImageIngestionProductsFinder:
)
return AncillaryProduct(type=AncillaryProductType.QUICKLOOK_IMAGE, filename=filename)
- @staticmethod
- def is_casa_product(file: Path) -> bool:
+ def _get_product_root(self):
"""
- Was this file created by CASA? If so, and it's not the weblog,
- we'll include it in the pipeline artifacts tar.
+ There should be a common anchor for the science products and ancillary products
+ of an ingestion manifest. In our example, it's
+ VLASS2.1.ql.T08t09.J055438_113000.10.2048.v1.
- :param file: some file found in the ingestion staging dir
- :return: whether it's a CASA byproduct
+ :return:
+ """
+ files = [file for file in self.staging_source_dir.iterdir()]
+ for file in files:
+ # Okay, this is kludgy, but it works with our example manifest
+ # and currently that's all we have to go on. -- JLG, 2021-07-30
+ if file.name.endswith(".fits"):
+ parts = file.name.split(".v")
+ return parts[0]
+ raise ValueError(
+ f"Unable to determine the product root from the output science products"
+ f" in {self.staging_source_dir}"
+ )
+ def _is_ancillary_image_product(self, file: Path) -> bool:
"""
- if file.name.startswith("casa_"):
- return True
+ Is this the kind of ancillary product that will go in the "ancillary_products"
+ section of the manifest (and -not- an ancillary product belonging to
+ a science product?
- if "_band" in file.name and file.name.endswith(".fits"):
+ :param file: some file found in the ingestion staging dir
+ :return: whether it's an ancillary product
+ """
+ if file.name == WEBLOG_FILENAME or "ingestion_artifacts" in file.name:
return True
-
- return file.name in [
- "pipeline_aquareport.xml",
- "unknown.auxproducts.tgz",
- "unknown.pipeline_manifest.xml",
- ]
+ product_root = self._get_product_root()
+ return file.name.startswith(product_root) and file.name.endswith(TARFILE_EXT)
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
index 925dc1f054d2cb8d17dcdd2dbb11e7f132d2e414..60e14afd0477b35892f7816ae38193c109a90615 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
@@ -3,9 +3,11 @@
from pathlib import Path
from typing import List
-# pylint: disable=C0103, E0401, E0402, R1721
+# pylint: disable=C0103, E0401, E0402, R0914, R1721, W0621
import json
+import shutil
+import tarfile
import pytest
from ingest_envoy.ingestion_manifest import IngestionManifestBuilder
@@ -13,7 +15,12 @@ from ingest_envoy.schema import AbstractTextFile
from ingest_envoy.utilities import Telescope, AncillaryProductType, ScienceProductType
# ingest_path is NOT unused. IJ is dumb.
-from .conftest import ingest_path, populate_fake_image_ingest_path
+from .conftest import (
+ ingest_path,
+ populate_fake_image_ingest_path,
+ IMG_MANIFEST_INPUT_FILENAMES,
+ CASA_BYPRODUCTS,
+)
from ingest_envoy.manifest_components import (
WEBLOG_FILENAME,
@@ -24,16 +31,17 @@ from ingest_envoy.manifest_components import (
AncillaryProduct,
InputGroup,
InputScienceProduct,
+ MANIFEST_FILENAME,
)
-def test_parameters_json_well_formed():
+def test_parameters_json_well_formed(ingest_path: Path):
"""
Make sure we get expected ManifestParameters for an image.
:return:
"""
- params = manifest_parameters()
+ params = manifest_parameters(ingest_path)
params_json = params.to_json()
# Only difference from EVLA CAL manifest is additional_metadata
@@ -42,29 +50,33 @@ def test_parameters_json_well_formed():
# gotta be able to dump it; test will fail if dump runs into trouble
json.dumps(params_json)
+ shutil.rmtree(ingest_path)
+
-def test_output_science_prods_built_properly():
+def test_output_science_prods_built_properly(ingest_path: Path):
"""
Output science products for an image ingestion contain ancillary products.
Let's make sure we can build one of these OSPs.
:return:
"""
- ap = rms_ancillary_prod()
- osp = osp_ql_with_ancillary(ap)
+ aps = rms_ancillary_prods(ingest_path)
+ osp = osp_ql_with_ancillaries(ingest_path, aps)
+ # check the output science product
assert osp.type == AncillaryProductType.QUICKLOOK_IMAGE
- assert (
- osp.filename
- == "VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
+ assert osp.filename.endswith(
+ "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
)
- assert len(osp.ancillary_products) == 1
+ # check the ancillary products belonging to the science product
+ assert len(osp.ancillary_products) == 2
osp_ap = osp.ancillary_products[0]
- assert osp_ap == ap
- ap_json = ap.to_json()
+ assert osp_ap == aps[0]
+ aps_json = [ap.to_json() for ap in aps]
# if we can't dump it, we know it's no good
- ap_str = json.dumps(ap_json)
+ ap_str = json.dumps(aps_json)
+ # confirm the output science products "look" right
osp_json = osp.to_json()
assert osp_json["type"] == osp.type.value
assert osp_json["filename"] == osp.filename
@@ -72,15 +84,43 @@ def test_output_science_prods_built_properly():
op_str = json.dumps(osp_json)
assert ap_str in op_str
+ shutil.rmtree(ingest_path)
+
+
+def test_output_group_made_correctly(ingest_path: Path):
+ """
+ Does the output group produced by the manifest builder contain everything it should?
+
+ :param ingest_path: staging source dir
+ :return:
+ """
+
+ # make some fake science-product-associated ancillary products
+ sp_aps = rms_ancillary_prods(ingest_path)
+ # make an output science product with 'em
+ osp = osp_ql_with_ancillaries(ingest_path, sp_aps)
+
+ # make the other ancillary products
+ aps = separate_aps(ingest_path)
-def test_output_group_json_well_formed():
+ # make the output group using them....
+ output_group = OutputGroup(science_products=[osp], ancillary_products=aps)
+ # ... and confirm it contains what we expected.
+ assert output_group.science_products[0] == osp
+ assert output_group.ancillary_products == aps
+
+ shutil.rmtree(ingest_path)
+
+
+def test_output_group_json_well_formed(ingest_path: Path):
"""
- In addition to the ancillary_products section of the image ingestion manifest,
- output science products may themselves contain ancillary products.
+ Does the output group produced by the manifest builder "look right"?
+
+ :param ingest_path: staging source dir
:return:
"""
- output_group = build_output_group()
+ output_group = build_output_group(ingest_path)
expected_og_json = output_group.to_json()
sps = output_group.science_products
@@ -91,24 +131,63 @@ def test_output_group_json_well_formed():
assert len(sp.ancillary_products) > 0
expected_sp_json = [sp.to_json() for sp in sps]
- other_aps = separate_aps()
+ other_aps = separate_aps(ingest_path)
expected_aps_json = [ap.to_json() for ap in other_aps]
+ # if any of the JSON isn't well formed, raise a stink....
og_dump = json.dumps(expected_og_json)
aps_dump = json.dumps(expected_aps_json)
sps_dump = json.dumps(expected_sp_json)
+ # ... and make sure each of the dumps contains what it should.
assert sps_dump in og_dump
assert aps_dump in og_dump
assert aps_dump not in sps_dump
+ shutil.rmtree(ingest_path)
-@pytest.mark.skip("TODO")
-def test_makes_correct_ingest_artifacts_tar():
+
+def test_ingestion_artifacts_tar_correct(ingest_path: Path):
"""
- Can we build the correct ingestion_artifacts tar for each implemented ingestion type?
+ Do we build the correct ingestion_artifacts tar for for standard imaging ingestion?
:return:
"""
+ # fill the ingestion path with fake files
+ populate_fake_image_ingest_path(ingest_path)
+
+ locator = "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
+ addl_md = AbstractTextFile(filename="image_metadata_2021_05_21_T10_17_19.180.json", content="")
+
+ # manifest maker, manifest maker, make me a manifest
+ IngestionManifestBuilder(
+ staging_source_dir=ingest_path,
+ sp_type=ScienceProductType.IMAGE.value,
+ locator=locator,
+ telescope=Telescope.EVLA.value,
+ additional_metadata=addl_md,
+ ).build()
+
+ # get ingestion artifacts
+ artifacts_file = [file for file in ingest_path.glob("ingestion_artifacts*.tar")][0]
+ with tarfile.open(artifacts_file, "r") as tar:
+ # confirm that contains has as many files as we expect...
+ members = tar.getmembers()
+ assert len(members) == len(IMG_MANIFEST_INPUT_FILENAMES) + len(CASA_BYPRODUCTS) + 2
+
+ # ...and that they include the manifest and CASA byproducts
+ filenames = [Path(tar_info.name).name for tar_info in members]
+ manifest_found = False
+ casa_byproducts = []
+ for filename in filenames:
+ if filename == MANIFEST_FILENAME:
+ manifest_found = True
+ elif filename in CASA_BYPRODUCTS:
+ casa_byproducts.append(filename)
+
+ assert manifest_found
+ assert len(casa_byproducts) == len(CASA_BYPRODUCTS)
+
+ shutil.rmtree(ingest_path)
def test_creates_expected_manifest(ingest_path: Path):
@@ -122,7 +201,10 @@ def test_creates_expected_manifest(ingest_path: Path):
populate_fake_image_ingest_path(ingest_path)
locator = "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
- addl_md = AbstractTextFile(filename="image_metadata_2021_05_21_T10_17_19.180.json", content="")
+ addl_md_file = ingest_path / "image_metadata_2021_05_21_T10_17_19.180.json"
+ addl_md = AbstractTextFile(filename=str(addl_md_file), content="")
+
+ # build us a manifest in the ingest_path using this locator and additional metadata
builder = IngestionManifestBuilder(
staging_source_dir=ingest_path,
sp_type=ScienceProductType.IMAGE.value,
@@ -132,80 +214,67 @@ def test_creates_expected_manifest(ingest_path: Path):
)
manifest, _ = builder.build()
- expected_params = manifest_parameters()
- expected_params.staging_source_dir = ingest_path
- assert manifest.parameters == expected_params
- assert manifest.input_group == InputGroup(science_products=[InputScienceProduct(locator)])
-
- ql_rms_ap = (
- ingest_path
- / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.rms.subim.fits"
- )
- sp_ap1 = AncillaryProduct(
- AncillaryProductType.QUICKLOOK_RMS_IMAGE,
- filename=str(ql_rms_ap),
- )
-
- thumb_ap = (
- ingest_path
- / "VLASS2.1.ql.T08t09.J055438_113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.png"
- )
- sp_ap2 = AncillaryProduct(
- type=AncillaryProductType.THUMBNAIL_IMG,
- filename=str(thumb_ap),
- )
-
- # make a quicklook image science product with a quicklook_rms and a thumbnail
- ql_ap = (
- ingest_path
- / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
- )
- sci_prod = OutputScienceProduct(
- type=AncillaryProductType.QUICKLOOK_IMAGE,
- filename=str(ql_ap),
- ancillary_products=[sp_ap1, sp_ap2],
- )
-
- weblog_path = ingest_path / WEBLOG_FILENAME
- ap1 = AncillaryProduct(
- type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=str(weblog_path)
- )
-
- pl_af = ingest_path / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.tar"
- ap2 = AncillaryProduct(
- type=AncillaryProductType.PIPELINE_ARTIFACTS,
- filename=str(pl_af),
- )
-
- ing_if = ingest_path / "ingestion_artifacts_2021_05_21_T10_17_19.275.tar"
- ap3 = AncillaryProduct(
- type=AncillaryProductType.INGESTION_ARTIFACTS,
- filename=str(ing_if),
- )
+ expected_params = manifest_parameters(ingest_path)
+ actual_params = manifest.parameters
+ assert actual_params == expected_params
- expected_output_group = OutputGroup(
- science_products=[sci_prod], ancillary_products=[ap1, ap2, ap3]
- )
+ assert manifest.input_group == InputGroup(science_products=[InputScienceProduct(locator)])
- # make sure expected values are -expected- expected values
- expected_sci_prods = expected_output_group.science_products
- assert len(expected_sci_prods) == 1
- expected_sp_aps = expected_sci_prods[0].ancillary_products
- assert len(expected_sp_aps) == 2
- expected_aps = expected_output_group.ancillary_products
- assert len(expected_aps) == 3
+ # make the output group the manifest should have
+ osp = osp_ql_with_ancillaries(ingest_path, rms_ancillary_prods(ingest_path))
+ aps = separate_aps(ingest_path)
+ expected_output_group = OutputGroup(science_products=[osp], ancillary_products=aps)
+ # the output groups won't be -exactly- the same since the ingestion artifacts
+ # filenames will differ, but we can compare the parts of them
+ # that should match
actual_output_group = manifest.output_group
actual_sci_prods = actual_output_group.science_products
- assert actual_sci_prods == expected_sci_prods
-
- expected_sp_aps = actual_sci_prods[0].ancillary_products
- assert len(expected_sp_aps) == len(expected_sp_aps)
+ assert actual_sci_prods == expected_output_group.science_products
- assert actual_output_group.science_products == expected_output_group.science_products
- # TODO: make these assertions pass
- # assert actual_output_group.ancillary_products == expected_output_group.ancillary_products
- # assert actual_output_group == expected_output_group
+ expected_aps = expected_output_group.ancillary_products
+ actual_aps = actual_output_group.ancillary_products
+
+ assert len(actual_aps) == len(expected_aps)
+
+ # compare the weblogs
+ expected_weblog = [
+ ap for ap in expected_aps if ap.type == AncillaryProductType.PIPELINE_WEBLOG_TYPE
+ ][0]
+ actual_weblog = [
+ ap for ap in actual_aps if ap.type == AncillaryProductType.PIPELINE_WEBLOG_TYPE
+ ][0]
+ assert actual_weblog == expected_weblog
+
+ # compare pipeline artifacts
+ expected_pip_art = [
+ ap for ap in expected_aps if ap.type == AncillaryProductType.PIPELINE_ARTIFACTS
+ ][0]
+ actual_pip_art = [
+ ap for ap in actual_aps if ap.type == AncillaryProductType.PIPELINE_ARTIFACTS
+ ][0]
+ assert actual_pip_art == expected_pip_art
+ expected_tar = Path(expected_pip_art.filename)
+ actual_tar = Path(actual_pip_art.filename)
+ assert actual_tar.exists()
+ assert actual_tar == expected_tar
+
+ # the ingestion artifacts files won't have exactly the same name, because the filename
+ # is created on the fly using the current timestamp, but they should be identical otherwise
+ expected_ing_art = [
+ ap for ap in expected_aps if ap.type == AncillaryProductType.INGESTION_ARTIFACTS
+ ][0]
+ assert "ingestion_artifacts" in expected_ing_art.filename
+ actual_ing_art = [
+ ap for ap in actual_aps if ap.type == AncillaryProductType.INGESTION_ARTIFACTS
+ ][0]
+ assert "ingestion_artifacts" in actual_ing_art.filename
+ expected_tar = Path(expected_ing_art.filename)
+ actual_tar = Path(actual_ing_art.filename)
+ assert actual_tar.exists()
+ assert actual_tar.parent == expected_tar.parent
+
+ shutil.rmtree(ingest_path)
@pytest.mark.skip("TODO")
@@ -216,6 +285,10 @@ def test_writes_expected_output_files(ingest_path: Path):
:param ingest_path:
:return:
"""
+ # TODO:
+ raise NotImplementedError
+
+ shutil.rmtree(ingest_path)
# -----------------------------
@@ -223,81 +296,104 @@ def test_writes_expected_output_files(ingest_path: Path):
# -----------------------------
-def manifest_parameters() -> ManifestParameters:
+def manifest_parameters(ingest_path: Path) -> ManifestParameters:
"""
Build a ManifestParameters for our tests
:return: the manifest parameters we're expecting
"""
- return ManifestParameters(
+ addl_md_path = ingest_path / "image_metadata_2021_05_21_T10_17_19.180.json"
+ addl_md = AbstractTextFile(filename=str(addl_md_path), content="")
+
+ params = ManifestParameters(
telescope=Telescope.EVLA,
reingest=False,
ngas_ingest=False,
calibrate=False,
- staging_source_dir=Path(
- "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/"
- + "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1"
- ),
- additional_metadata=AbstractTextFile(
- filename="image_metadata_2021_05_21_T10_17_19.180.json", content=""
- ),
+ staging_source_dir=ingest_path,
+ additional_metadata=AbstractTextFile(filename=addl_md.filename, content=""),
)
+ return params
+
-def rms_ancillary_prod() -> AncillaryProduct:
+def rms_ancillary_prods(staging_source_dir: Path) -> List[AncillaryProduct]:
"""
- Build an AncillaryProduct for our tests
+ Build an AncillaryProduct to use in a test.
:return: the product we're expecting
"""
- return AncillaryProduct(
+ file1 = (
+ staging_source_dir
+ / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.rms.subim.fits"
+ )
+ ap1 = AncillaryProduct(
type=AncillaryProductType.QUICKLOOK_RMS_IMAGE,
- filename="VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image"
- + ".pbcor.tt0.rms.subim.fits",
+ filename=str(file1),
+ )
+ file2 = (
+ staging_source_dir
+ / "VLASS2.1.ql.T08t09.J055438_113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.png"
+ )
+ ap2 = AncillaryProduct(
+ type=AncillaryProductType.THUMBNAIL_IMG,
+ filename=str(file2),
)
+ return [ap1, ap2]
-def osp_ql_with_ancillary(ap: AncillaryProduct) -> OutputScienceProduct:
+
+def osp_ql_with_ancillaries(
+ staging_source_dir: Path, aps: List[AncillaryProduct]
+) -> OutputScienceProduct:
"""
- Build an OutputScienceProduct for our tests
+ Build an OutputScienceProduct for a test
:return: the product we're expecting
"""
+
+ sp_file = (
+ staging_source_dir
+ / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
+ )
return OutputScienceProduct(
type=AncillaryProductType(AncillaryProductType.QUICKLOOK_IMAGE),
- filename="VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits",
- ancillary_products=[ap],
+ filename=str(sp_file),
+ ancillary_products=aps,
)
-def separate_aps() -> List[AncillaryProduct]:
+def separate_aps(staging_source_dir: Path) -> List[AncillaryProduct]:
"""
Build a list of AncillaryProducts for our tests
:return: the products we're expecting
"""
- ap1 = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=WEBLOG_FILENAME)
+ file1 = staging_source_dir / WEBLOG_FILENAME
+ ap1 = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=str(file1))
+
+ file2 = staging_source_dir / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.tar"
ap2 = AncillaryProduct(
type=AncillaryProductType.PIPELINE_ARTIFACTS,
- filename="uid____EVLA_ingestion_artifacts_b1ab328d-200e-4da4-86bf-514773f31e2b.tar",
+ filename=str(file2),
)
+ file3 = staging_source_dir / "ingestion_artifacts_2021_05_21_T10_17_19.275.tar"
ap3 = AncillaryProduct(
type=AncillaryProductType.INGESTION_ARTIFACTS,
- filename="ingestion_artifacts_2019_07_30_T13_03_00.936.tar",
+ filename=str(file3),
)
return [ap1, ap2, ap3]
-def build_output_group() -> OutputGroup:
+def build_output_group(staging_source_dir: Path) -> OutputGroup:
"""
Build an OutputGroup for our tests
:return: the output group we're expecting
"""
- ap = rms_ancillary_prod()
- osp = osp_ql_with_ancillary(ap)
+ osp = osp_ql_with_ancillaries(staging_source_dir, rms_ancillary_prods(staging_source_dir))
- other_aps = separate_aps()
+ other_aps = separate_aps(staging_source_dir)
ap_list = other_aps
return OutputGroup(science_products=[osp], ancillary_products=ap_list)
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py
similarity index 89%
rename from apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
rename to apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py
index 65fd4118b2ee5db0cfc18f1b71298ae94941749d..34d7777e282eb0e641ac800b1041665ca7945a53 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py
@@ -96,4 +96,12 @@ def test_entry_point_for_image(ingest_path: Path):
# ...and everything we started with, plus a new ingestion artifacts tar
ingestion_files_after = [file for file in ingest_path.iterdir()]
+ if len(ingestion_files_after) != expected_file_count_before + 2:
+ for file in ingestion_files_before:
+ if file not in ingestion_files_after:
+ print(f">>> {file.name} missing after manifest build")
+ for file in ingestion_files_after:
+ if file not in ingestion_files_before:
+ print(f">>> {file.name} present after manifest build")
+
assert len(ingestion_files_after) == expected_file_count_before + 2
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
index 5273f8aff690d51e6df04c548162e7fd4c0c200a..b1934831e493bed32a3d5403e638eddc7f0e9a82 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
@@ -98,7 +98,7 @@ def test_filters_cal_input_files(ingest_path: Path):
@pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation")
-def test_writes_expected_output_files(ingest_path: Path):
+def test_writes_expected_evla_cal_output_files(ingest_path: Path):
"""
Did the manifest builder produce the manifest file, the weblog, and the science product tar?
@@ -219,6 +219,15 @@ def test_output_group_well_formed():
assert actual_json == expected_json
+@pytest.mark.skip("TODO")
+def test_ingestion_artifacts_tar_correct():
+ """
+ TODO
+ :return:
+ """
+ raise NotImplementedError
+
+
@pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation")
def test_evla_cal_manifest_matches_example(ingest_path: Path):
"""