WS-601: Fix broken ingestion manifest builder tests and implement remaining TODOs

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py

@@ -190,7 +190,7 @@ class IngestionManifestBuilder:
         manifest_file = manifest.write()
         artifacts_tar = self.write_ingestion_artifacts_tar()
         artifacts_ap = AncillaryProduct(
-            AncillaryProductType.INGESTION_ARTIFACTS, filename=str(artifacts_tar)
+            AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
         )
 
         manifest.output_group.ancillary_products = [artifacts_ap]
@@ -215,7 +215,11 @@ class IngestionManifestBuilder:
 
         manifest_file = manifest.write()
 
-        self.write_ingestion_artifacts_tar()
+        artifacts_tar = self.write_ingestion_artifacts_tar()
+        artifacts_ap = AncillaryProduct(
+            type=AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
+        )
+        manifest.output_group.ancillary_products.append(artifacts_ap)
 
         return manifest, manifest_file
 
@@ -257,7 +261,7 @@ class IngestionManifestBuilder:
         """
 
         products_finder = ImageIngestionProductsFinder(self.staging_source_dir)
-        science_products = products_finder.science_products
+        science_products = products_finder.output_science_products
         ancillary_products = products_finder.ancillary_products
 
         return OutputGroup(science_products=science_products, ancillary_products=ancillary_products)
@@ -273,12 +277,11 @@ class IngestionManifestBuilder:
         timestamp = format_timestamp(current_time)
         return f"{INGESTION_ARTIFACTS_NAME}{timestamp}{TARFILE_EXT}"
 
-    def write_ingestion_artifacts_tar(self) -> Path:
+    def write_ingestion_artifacts_tar(self) -> tarfile.TarFile:
         """
         Take the list of files and build a tar for inclusion into the archive.
         This happens in the staging area for ingestion.
         The EVLA CAL tar will contain just the manifest.
-        For image ingestion, this must be done -after- the pipeline artifacts tar is built.
 
         :return: a .tar archive of the ingestion artifacts
         """
@@ -290,10 +293,9 @@ class IngestionManifestBuilder:
                 ingestion_artifacts_tar.add(file)
 
             # include the manifest
-            if manifest_file not in self.files_found:
-                ingestion_artifacts_tar.add(manifest_file)
+            ingestion_artifacts_tar.add(manifest_file)
 
-        return ing_tar
+        return ingestion_artifacts_tar
 
 
 class IngestionManifest(ManifestIF):

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py

@@ -2,9 +2,15 @@
 from pathlib import Path
 from typing import List
 
-from ingest_envoy.manifest_components import OutputScienceProduct, AncillaryProduct, WEBLOG_FILENAME
+from ingest_envoy.manifest_components import (
+    OutputScienceProduct,
+    AncillaryProduct,
+    WEBLOG_FILENAME,
+    TARFILE_EXT,
+)
 from ingest_envoy.utilities import AncillaryProductType
 
+
 # pylint: disable=R1721
 class ImageIngestionProductsFinder:
     """Finds ancillary science products and other ancillary products needed for image ingestion"""
@@ -12,7 +18,7 @@ class ImageIngestionProductsFinder:
     def __init__(self, staging_source_dir: Path):
         self.staging_source_dir = staging_source_dir
         self.files_found = [file for file in self.staging_source_dir.iterdir()]
-        self.science_products = self._find_output_science_products()
+        self.output_science_products = self._find_output_science_products()
         self.ancillary_products = self._find_other_ancillary_products()
 
     def _find_output_science_products(self) -> List[OutputScienceProduct]:
@@ -70,22 +76,17 @@ class ImageIngestionProductsFinder:
         except ValueError as exc:
             raise FileNotFoundError(f"No weblog found in {self.staging_source_dir}") from exc
 
-        try:
-            pipeline_artifacts_tar = [
-                file
-                for file in self.files_found
-                if file.name.endswith("tar") and "ingestion_artifacts" not in file.name
-            ][0]
-            ancillary_products.append(
-                AncillaryProduct(
-                    type=AncillaryProductType.PIPELINE_ARTIFACTS,
-                    filename=str(pipeline_artifacts_tar),
-                )
+        ancillary_files = [Path(ap.filename) for ap in ancillary_products]
+
+        # find the pipeline artifacts tar
+        for file in self.files_found:
+            if file not in ancillary_files and self._is_ancillary_image_product(file):
+                pipeline_artifacts_tar = file
+        if pipeline_artifacts_tar:
+            pip_ap = AncillaryProduct(
+                type=AncillaryProductType.PIPELINE_ARTIFACTS, filename=str(pipeline_artifacts_tar)
             )
-        except ValueError as exc:
-            raise FileNotFoundError(
-                f"No pipeline artifacts found in {self.staging_source_dir}"
-            ) from exc
+            ancillary_products.append(pip_ap)
 
         return ancillary_products
 
@@ -107,24 +108,36 @@ class ImageIngestionProductsFinder:
                 )
             return AncillaryProduct(type=AncillaryProductType.QUICKLOOK_IMAGE, filename=filename)
 
-    @staticmethod
-    def is_casa_product(file: Path) -> bool:
+    def _get_product_root(self):
         """
-        Was this file created by CASA? If so, and it's not the weblog,
-        we'll include it in the pipeline artifacts tar.
+        There should be a common anchor for the science products and ancillary products
+        of an ingestion manifest. In our example, it's
+        VLASS2.1.ql.T08t09.J055438_113000.10.2048.v1.
 
-        :param file: some file found in the ingestion staging dir
-        :return: whether it's a CASA byproduct
+        :return:
+        """
+        files = [file for file in self.staging_source_dir.iterdir()]
+        for file in files:
+            # Okay, this is kludgy, but it works with our example manifest
+            # and currently that's all we have to go on. -- JLG, 2021-07-30
+            if file.name.endswith(".fits"):
+                parts = file.name.split(".v")
+                return parts[0]
+        raise ValueError(
+            f"Unable to determine the product root from the output science products"
+            f" in {self.staging_source_dir}"
+        )
 
+    def _is_ancillary_image_product(self, file: Path) -> bool:
         """
-        if file.name.startswith("casa_"):
-            return True
+        Is this the kind of ancillary product that will go in the "ancillary_products"
+        section of the manifest (and -not- an ancillary product belonging to
+        a science product?
 
-        if "_band" in file.name and file.name.endswith(".fits"):
+        :param file: some file found in the ingestion staging dir
+        :return: whether it's an ancillary product
+        """
+        if file.name == WEBLOG_FILENAME or "ingestion_artifacts" in file.name:
             return True
-
-        return file.name in [
-            "pipeline_aquareport.xml",
-            "unknown.auxproducts.tgz",
-            "unknown.pipeline_manifest.xml",
-        ]
+        product_root = self._get_product_root()
+        return file.name.startswith(product_root) and file.name.endswith(TARFILE_EXT)

apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py

@@ -3,9 +3,11 @@
 from pathlib import Path
 from typing import List
 
-# pylint: disable=C0103, E0401, E0402, R1721
+# pylint: disable=C0103, E0401, E0402, R0914, R1721, W0621
 
 import json
+import shutil
+import tarfile
 import pytest
 
 from ingest_envoy.ingestion_manifest import IngestionManifestBuilder
@@ -13,7 +15,12 @@ from ingest_envoy.schema import AbstractTextFile
 from ingest_envoy.utilities import Telescope, AncillaryProductType, ScienceProductType
 
 # ingest_path is NOT unused. IJ is dumb.
-from .conftest import ingest_path, populate_fake_image_ingest_path
+from .conftest import (
+    ingest_path,
+    populate_fake_image_ingest_path,
+    IMG_MANIFEST_INPUT_FILENAMES,
+    CASA_BYPRODUCTS,
+)
 
 from ingest_envoy.manifest_components import (
     WEBLOG_FILENAME,
@@ -24,16 +31,17 @@ from ingest_envoy.manifest_components import (
     AncillaryProduct,
     InputGroup,
     InputScienceProduct,
+    MANIFEST_FILENAME,
 )
 
 
-def test_parameters_json_well_formed():
+def test_parameters_json_well_formed(ingest_path: Path):
     """
     Make sure we get expected ManifestParameters for an image.
 
     :return:
     """
-    params = manifest_parameters()
+    params = manifest_parameters(ingest_path)
 
     params_json = params.to_json()
     # Only difference from EVLA CAL manifest is additional_metadata
@@ -42,29 +50,33 @@ def test_parameters_json_well_formed():
     # gotta be able to dump it; test will fail if dump runs into trouble
     json.dumps(params_json)
 
+    shutil.rmtree(ingest_path)
+
 
-def test_output_science_prods_built_properly():
+def test_output_science_prods_built_properly(ingest_path: Path):
     """
     Output science products for an image ingestion contain ancillary products.
     Let's make sure we can build one of these OSPs.
     :return:
     """
-    ap = rms_ancillary_prod()
-    osp = osp_ql_with_ancillary(ap)
+    aps = rms_ancillary_prods(ingest_path)
+    osp = osp_ql_with_ancillaries(ingest_path, aps)
 
+    # check the output science product
     assert osp.type == AncillaryProductType.QUICKLOOK_IMAGE
-    assert (
-        osp.filename
-        == "VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
+    assert osp.filename.endswith(
+        "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
     )
-    assert len(osp.ancillary_products) == 1
 
+    # check the ancillary products belonging to the science product
+    assert len(osp.ancillary_products) == 2
     osp_ap = osp.ancillary_products[0]
-    assert osp_ap == ap
-    ap_json = ap.to_json()
+    assert osp_ap == aps[0]
+    aps_json = [ap.to_json() for ap in aps]
     # if we can't dump it, we know it's no good
-    ap_str = json.dumps(ap_json)
+    ap_str = json.dumps(aps_json)
 
+    # confirm the output science products "look" right
     osp_json = osp.to_json()
     assert osp_json["type"] == osp.type.value
     assert osp_json["filename"] == osp.filename
@@ -72,15 +84,43 @@ def test_output_science_prods_built_properly():
     op_str = json.dumps(osp_json)
     assert ap_str in op_str
 
+    shutil.rmtree(ingest_path)
+
+
+def test_output_group_made_correctly(ingest_path: Path):
+    """
+    Does the output group produced by the manifest builder contain everything it should?
+
+    :param ingest_path: staging source dir
+    :return:
+    """
+
+    # make some fake science-product-associated ancillary products
+    sp_aps = rms_ancillary_prods(ingest_path)
+    # make an output science product with 'em
+    osp = osp_ql_with_ancillaries(ingest_path, sp_aps)
+
+    # make the other ancillary products
+    aps = separate_aps(ingest_path)
 
-def test_output_group_json_well_formed():
+    # make the output group using them....
+    output_group = OutputGroup(science_products=[osp], ancillary_products=aps)
+    # ... and confirm it contains what we expected.
+    assert output_group.science_products[0] == osp
+    assert output_group.ancillary_products == aps
+
+    shutil.rmtree(ingest_path)
+
+
+def test_output_group_json_well_formed(ingest_path: Path):
     """
-    In addition to the ancillary_products section of the image ingestion manifest,
-    output science products may themselves contain ancillary products.
+    Does the output group produced by the manifest builder "look right"?
+
+    :param ingest_path: staging source dir
     :return:
     """
 
-    output_group = build_output_group()
+    output_group = build_output_group(ingest_path)
     expected_og_json = output_group.to_json()
 
     sps = output_group.science_products
@@ -91,24 +131,63 @@ def test_output_group_json_well_formed():
         assert len(sp.ancillary_products) > 0
     expected_sp_json = [sp.to_json() for sp in sps]
 
-    other_aps = separate_aps()
+    other_aps = separate_aps(ingest_path)
     expected_aps_json = [ap.to_json() for ap in other_aps]
 
+    # if any of the JSON isn't well formed, raise a stink....
     og_dump = json.dumps(expected_og_json)
     aps_dump = json.dumps(expected_aps_json)
     sps_dump = json.dumps(expected_sp_json)
+    # ... and make sure each of the dumps contains what it should.
     assert sps_dump in og_dump
     assert aps_dump in og_dump
     assert aps_dump not in sps_dump
 
+    shutil.rmtree(ingest_path)
 
-@pytest.mark.skip("TODO")
-def test_makes_correct_ingest_artifacts_tar():
+
+def test_ingestion_artifacts_tar_correct(ingest_path: Path):
     """
-    Can we build the correct ingestion_artifacts tar for each implemented ingestion type?
+    Do we build the correct ingestion_artifacts tar for for standard imaging ingestion?
 
     :return:
     """
+    # fill the ingestion path with fake files
+    populate_fake_image_ingest_path(ingest_path)
+
+    locator = "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
+    addl_md = AbstractTextFile(filename="image_metadata_2021_05_21_T10_17_19.180.json", content="")
+
+    # manifest maker, manifest maker, make me a manifest
+    IngestionManifestBuilder(
+        staging_source_dir=ingest_path,
+        sp_type=ScienceProductType.IMAGE.value,
+        locator=locator,
+        telescope=Telescope.EVLA.value,
+        additional_metadata=addl_md,
+    ).build()
+
+    # get ingestion artifacts
+    artifacts_file = [file for file in ingest_path.glob("ingestion_artifacts*.tar")][0]
+    with tarfile.open(artifacts_file, "r") as tar:
+        # confirm that contains has as many files as we expect...
+        members = tar.getmembers()
+        assert len(members) == len(IMG_MANIFEST_INPUT_FILENAMES) + len(CASA_BYPRODUCTS) + 2
+
+    # ...and that they include the manifest and CASA byproducts
+    filenames = [Path(tar_info.name).name for tar_info in members]
+    manifest_found = False
+    casa_byproducts = []
+    for filename in filenames:
+        if filename == MANIFEST_FILENAME:
+            manifest_found = True
+        elif filename in CASA_BYPRODUCTS:
+            casa_byproducts.append(filename)
+
+    assert manifest_found
+    assert len(casa_byproducts) == len(CASA_BYPRODUCTS)
+
+    shutil.rmtree(ingest_path)
 
 
 def test_creates_expected_manifest(ingest_path: Path):
@@ -122,7 +201,10 @@ def test_creates_expected_manifest(ingest_path: Path):
     populate_fake_image_ingest_path(ingest_path)
 
     locator = "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
-    addl_md = AbstractTextFile(filename="image_metadata_2021_05_21_T10_17_19.180.json", content="")
+    addl_md_file = ingest_path / "image_metadata_2021_05_21_T10_17_19.180.json"
+    addl_md = AbstractTextFile(filename=str(addl_md_file), content="")
+
+    # build us a manifest in the ingest_path using this locator and additional metadata
     builder = IngestionManifestBuilder(
         staging_source_dir=ingest_path,
         sp_type=ScienceProductType.IMAGE.value,
@@ -132,80 +214,67 @@ def test_creates_expected_manifest(ingest_path: Path):
     )
     manifest, _ = builder.build()
 
-    expected_params = manifest_parameters()
-    expected_params.staging_source_dir = ingest_path
-    assert manifest.parameters == expected_params
-    assert manifest.input_group == InputGroup(science_products=[InputScienceProduct(locator)])
-
-    ql_rms_ap = (
-        ingest_path
-        / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.rms.subim.fits"
-    )
-    sp_ap1 = AncillaryProduct(
-        AncillaryProductType.QUICKLOOK_RMS_IMAGE,
-        filename=str(ql_rms_ap),
-    )
-
-    thumb_ap = (
-        ingest_path
-        / "VLASS2.1.ql.T08t09.J055438_113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.png"
-    )
-    sp_ap2 = AncillaryProduct(
-        type=AncillaryProductType.THUMBNAIL_IMG,
-        filename=str(thumb_ap),
-    )
-
-    # make a quicklook image science product with a quicklook_rms and a thumbnail
-    ql_ap = (
-        ingest_path
-        / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
-    )
-    sci_prod = OutputScienceProduct(
-        type=AncillaryProductType.QUICKLOOK_IMAGE,
-        filename=str(ql_ap),
-        ancillary_products=[sp_ap1, sp_ap2],
-    )
-
-    weblog_path = ingest_path / WEBLOG_FILENAME
-    ap1 = AncillaryProduct(
-        type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=str(weblog_path)
-    )
-
-    pl_af = ingest_path / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.tar"
-    ap2 = AncillaryProduct(
-        type=AncillaryProductType.PIPELINE_ARTIFACTS,
-        filename=str(pl_af),
-    )
-
-    ing_if = ingest_path / "ingestion_artifacts_2021_05_21_T10_17_19.275.tar"
-    ap3 = AncillaryProduct(
-        type=AncillaryProductType.INGESTION_ARTIFACTS,
-        filename=str(ing_if),
-    )
+    expected_params = manifest_parameters(ingest_path)
+    actual_params = manifest.parameters
+    assert actual_params == expected_params
 
-    expected_output_group = OutputGroup(
-        science_products=[sci_prod], ancillary_products=[ap1, ap2, ap3]
-    )
+    assert manifest.input_group == InputGroup(science_products=[InputScienceProduct(locator)])
 
-    # make sure expected values are -expected- expected values
-    expected_sci_prods = expected_output_group.science_products
-    assert len(expected_sci_prods) == 1
-    expected_sp_aps = expected_sci_prods[0].ancillary_products
-    assert len(expected_sp_aps) == 2
-    expected_aps = expected_output_group.ancillary_products
-    assert len(expected_aps) == 3
+    # make the output group the manifest should have
+    osp = osp_ql_with_ancillaries(ingest_path, rms_ancillary_prods(ingest_path))
+    aps = separate_aps(ingest_path)
+    expected_output_group = OutputGroup(science_products=[osp], ancillary_products=aps)
 
+    # the output groups won't be -exactly- the same since the ingestion artifacts
+    # filenames will differ, but we can compare the parts of them
+    # that should match
     actual_output_group = manifest.output_group
     actual_sci_prods = actual_output_group.science_products
-    assert actual_sci_prods == expected_sci_prods
-
-    expected_sp_aps = actual_sci_prods[0].ancillary_products
-    assert len(expected_sp_aps) == len(expected_sp_aps)
+    assert actual_sci_prods == expected_output_group.science_products
 
-    assert actual_output_group.science_products == expected_output_group.science_products
-    # TODO: make these assertions pass
-    # assert actual_output_group.ancillary_products == expected_output_group.ancillary_products
-    # assert actual_output_group == expected_output_group
+    expected_aps = expected_output_group.ancillary_products
+    actual_aps = actual_output_group.ancillary_products
+
+    assert len(actual_aps) == len(expected_aps)
+
+    # compare the weblogs
+    expected_weblog = [
+        ap for ap in expected_aps if ap.type == AncillaryProductType.PIPELINE_WEBLOG_TYPE
+    ][0]
+    actual_weblog = [
+        ap for ap in actual_aps if ap.type == AncillaryProductType.PIPELINE_WEBLOG_TYPE
+    ][0]
+    assert actual_weblog == expected_weblog
+
+    # compare pipeline artifacts
+    expected_pip_art = [
+        ap for ap in expected_aps if ap.type == AncillaryProductType.PIPELINE_ARTIFACTS
+    ][0]
+    actual_pip_art = [
+        ap for ap in actual_aps if ap.type == AncillaryProductType.PIPELINE_ARTIFACTS
+    ][0]
+    assert actual_pip_art == expected_pip_art
+    expected_tar = Path(expected_pip_art.filename)
+    actual_tar = Path(actual_pip_art.filename)
+    assert actual_tar.exists()
+    assert actual_tar == expected_tar
+
+    # the ingestion artifacts files won't have exactly the same name, because the filename
+    # is created on the fly using the current timestamp, but they should be identical otherwise
+    expected_ing_art = [
+        ap for ap in expected_aps if ap.type == AncillaryProductType.INGESTION_ARTIFACTS
+    ][0]
+    assert "ingestion_artifacts" in expected_ing_art.filename
+    actual_ing_art = [
+        ap for ap in actual_aps if ap.type == AncillaryProductType.INGESTION_ARTIFACTS
+    ][0]
+    assert "ingestion_artifacts" in actual_ing_art.filename
+    expected_tar = Path(expected_ing_art.filename)
+    actual_tar = Path(actual_ing_art.filename)
+    assert actual_tar.exists()
+    assert actual_tar.parent == expected_tar.parent
+
+    shutil.rmtree(ingest_path)
 
 
 @pytest.mark.skip("TODO")
@@ -216,6 +285,10 @@ def test_writes_expected_output_files(ingest_path: Path):
     :param ingest_path:
     :return:
     """
+    # TODO:
+    raise NotImplementedError
+
+    shutil.rmtree(ingest_path)
 
 
 # -----------------------------
@@ -223,81 +296,104 @@ def test_writes_expected_output_files(ingest_path: Path):
 # -----------------------------
 
 
-def manifest_parameters() -> ManifestParameters:
+def manifest_parameters(ingest_path: Path) -> ManifestParameters:
     """
     Build a ManifestParameters for our tests
 
     :return: the manifest parameters we're expecting
     """
-    return ManifestParameters(
+    addl_md_path = ingest_path / "image_metadata_2021_05_21_T10_17_19.180.json"
+    addl_md = AbstractTextFile(filename=str(addl_md_path), content="")
+
+    params = ManifestParameters(
         telescope=Telescope.EVLA,
         reingest=False,
         ngas_ingest=False,
         calibrate=False,
-        staging_source_dir=Path(
-            "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/"
-            + "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1"
-        ),
-        additional_metadata=AbstractTextFile(
-            filename="image_metadata_2021_05_21_T10_17_19.180.json", content=""
-        ),
+        staging_source_dir=ingest_path,
+        additional_metadata=AbstractTextFile(filename=addl_md.filename, content=""),
     )
 
+    return params
+
 
-def rms_ancillary_prod() -> AncillaryProduct:
+def rms_ancillary_prods(staging_source_dir: Path) -> List[AncillaryProduct]:
     """
-    Build an AncillaryProduct for our tests
+    Build an AncillaryProduct to use in a test.
 
     :return: the product we're expecting
     """
-    return AncillaryProduct(
+    file1 = (
+        staging_source_dir
+        / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.rms.subim.fits"
+    )
+    ap1 = AncillaryProduct(
         type=AncillaryProductType.QUICKLOOK_RMS_IMAGE,
-        filename="VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image"
-        + ".pbcor.tt0.rms.subim.fits",
+        filename=str(file1),
+    )
+    file2 = (
+        staging_source_dir
+        / "VLASS2.1.ql.T08t09.J055438_113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.png"
+    )
+    ap2 = AncillaryProduct(
+        type=AncillaryProductType.THUMBNAIL_IMG,
+        filename=str(file2),
     )
 
+    return [ap1, ap2]
 
-def osp_ql_with_ancillary(ap: AncillaryProduct) -> OutputScienceProduct:
+
+def osp_ql_with_ancillaries(
+    staging_source_dir: Path, aps: List[AncillaryProduct]
+) -> OutputScienceProduct:
     """
-    Build an OutputScienceProduct for our tests
+    Build an OutputScienceProduct for a test
 
     :return: the product we're expecting
     """
+
+    sp_file = (
+        staging_source_dir
+        / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
+    )
     return OutputScienceProduct(
         type=AncillaryProductType(AncillaryProductType.QUICKLOOK_IMAGE),
-        filename="VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits",
-        ancillary_products=[ap],
+        filename=str(sp_file),
+        ancillary_products=aps,
     )
 
 
-def separate_aps() -> List[AncillaryProduct]:
+def separate_aps(staging_source_dir: Path) -> List[AncillaryProduct]:
     """
     Build a list of AncillaryProducts for our tests
 
     :return: the products we're expecting
     """
-    ap1 = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=WEBLOG_FILENAME)
+    file1 = staging_source_dir / WEBLOG_FILENAME
+    ap1 = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=str(file1))
+
+    file2 = staging_source_dir / "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.tar"
     ap2 = AncillaryProduct(
         type=AncillaryProductType.PIPELINE_ARTIFACTS,
-        filename="uid____EVLA_ingestion_artifacts_b1ab328d-200e-4da4-86bf-514773f31e2b.tar",
+        filename=str(file2),
     )
+    file3 = staging_source_dir / "ingestion_artifacts_2021_05_21_T10_17_19.275.tar"
     ap3 = AncillaryProduct(
         type=AncillaryProductType.INGESTION_ARTIFACTS,
-        filename="ingestion_artifacts_2019_07_30_T13_03_00.936.tar",
+        filename=str(file3),
     )
     return [ap1, ap2, ap3]
 
 
-def build_output_group() -> OutputGroup:
+def build_output_group(staging_source_dir: Path) -> OutputGroup:
     """
     Build an OutputGroup for our tests
 
     :return: the output group we're expecting
     """
-    ap = rms_ancillary_prod()
-    osp = osp_ql_with_ancillary(ap)
+    osp = osp_ql_with_ancillaries(staging_source_dir, rms_ancillary_prods(staging_source_dir))
 
-    other_aps = separate_aps()
+    other_aps = separate_aps(staging_source_dir)
     ap_list = other_aps
 
     return OutputGroup(science_products=[osp], ancillary_products=ap_list)

apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py → apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py

@@ -96,4 +96,12 @@ def test_entry_point_for_image(ingest_path: Path):
 
     # ...and everything we started with, plus a new ingestion artifacts tar
     ingestion_files_after = [file for file in ingest_path.iterdir()]
+    if len(ingestion_files_after) != expected_file_count_before + 2:
+        for file in ingestion_files_before:
+            if file not in ingestion_files_after:
+                print(f">>> {file.name} missing after manifest build")
+        for file in ingestion_files_after:
+            if file not in ingestion_files_before:
+                print(f">>> {file.name} present after manifest build")
+
     assert len(ingestion_files_after) == expected_file_count_before + 2

apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py

@@ -98,7 +98,7 @@ def test_filters_cal_input_files(ingest_path: Path):
 
 
 @pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation")
-def test_writes_expected_output_files(ingest_path: Path):
+def test_writes_expected_evla_cal_output_files(ingest_path: Path):
     """
     Did the manifest builder produce the manifest file, the weblog, and the science product tar?
 
@@ -219,6 +219,15 @@ def test_output_group_well_formed():
     assert actual_json == expected_json
 
 
+@pytest.mark.skip("TODO")
+def test_ingestion_artifacts_tar_correct():
+    """
+    TODO
+    :return:
+    """
+    raise NotImplementedError
+
+
 @pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation")
 def test_evla_cal_manifest_matches_example(ingest_path: Path):
     """