WS-601: more fixing of broken tests and implementation of TODO tests

* EVLA CAL manifest example updated
* manifest filename now `ingestion-manifest.json`
* EVLA CAL ingestion artifacts file contains only the manifest
* 1500 new grey hairs added to Janet's head

TODO next MR:
* yet more fixing of broken tests and implementation of TODO tests*

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py

@@ -290,7 +290,8 @@ class IngestionManifestBuilder:
         ing_tar = self.staging_source_dir / self.build_artifacts_filename()
         with tarfile.open(ing_tar, "w") as ingestion_artifacts_tar:
             for file in self.files_found:
-                ingestion_artifacts_tar.add(file)
+                if self.sp_type == ScienceProductType.IMAGE:
+                    ingestion_artifacts_tar.add(file)
 
             # include the manifest
             ingestion_artifacts_tar.add(manifest_file)

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py

@@ -9,7 +9,7 @@ from typing import Union, List, Dict
 from ingest_envoy.schema import AbstractTextFile
 from ingest_envoy.utilities import ScienceProductType, Telescope, AncillaryProductType
 
-MANIFEST_FILENAME = "ingestion_manifest.json"
+MANIFEST_FILENAME = "ingestion-manifest.json"
 INGESTION_ARTIFACTS_NAME = "ingestion_artifacts_"
 TARFILE_EXT = ".tar"
 WEBLOG_FILENAME = "weblog.tgz"

apps/cli/executables/pexable/ingest_envoy/test/conftest.py

@@ -9,7 +9,7 @@ import pytest
 
 from ingest_envoy.manifest_components import WEBLOG_FILENAME
 
-EVLA_CAL_WANTED_FILENAMES = ["my_science_products.tar", WEBLOG_FILENAME]
+EVLA_CAL_INPUT_FILENAMES = ["20A-346_2021_07_23_T13_37_08.376.tar", WEBLOG_FILENAME]
 UNWANTED = ["ignore_me.fits", "just_a_lotta_nothing", "uninteresting_metadata.xml"]
 
 IMG_MANIFEST_INPUT_FILENAMES = [
@@ -80,7 +80,7 @@ def populate_fake_evla_cal_ingest_path(staging_dir: Path) -> List[Path]:
     """
 
     files = []
-    filenames = [filename for filename in EVLA_CAL_WANTED_FILENAMES]
+    filenames = [filename for filename in EVLA_CAL_INPUT_FILENAMES]
     for filename in UNWANTED:
         filenames.append(filename)
 

apps/cli/executables/pexable/ingest_envoy/test/examples/_16B_069_cal_manifest.json

@@ -27,4 +27,4 @@
       }
     ]
   }
-}
+}
\ No newline at end of file

apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_EVLA_Cal.json → apps/cli/executables/pexable/ingest_envoy/test/examples/evla_cal_manifest_2021-08-02.json

 {
   "parameters": {
     "reingest": false,
-    "ngas_ingest": false,
+    "ngas_ingest": true,
     "telescope": "EVLA",
-    "ingestion_path": "/lustre/yea/and/here/we/go"
+    "ingestion_path": "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/20A-346_2021_07_23_T13_37_08.376"
   },
   "input_group": {
     "science_products": [
       {
-        "locator": "uid://evla/execblock/fjdsakljfkdlsajfkldsa"
+        "locator": "uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d"
       }
     ]
   },
@@ -16,17 +16,17 @@
     "science_products": [
       {
         "type": "calibration",
-        "filename": "XYZ-abc+TMN.O00.tar"
+        "filename": "20A-346_2021_07_23_T13_37_08.376.tar"
       }
     ],
     "ancillary_products": [
       {
         "type": "pipeline_weblog",
-        "filename": "qrs.weblog.tgz"
+        "filename": "weblog.tgz"
       },
       {
         "type": "ingestion_artifacts",
-        "filename": "ingestion_artifacts_2019_07_25_T15_43_33.144.tar"
+        "filename": "ingestion_artifacts_2021_07_27_T17_35_11.463.tar"
       }
     ]
   }

apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py

@@ -32,6 +32,7 @@ from ingest_envoy.manifest_components import (
     InputGroup,
     InputScienceProduct,
     MANIFEST_FILENAME,
+    TARFILE_EXT,
 )
 
 
@@ -277,7 +278,6 @@ def test_creates_expected_manifest(ingest_path: Path):
     shutil.rmtree(ingest_path)
 
 
-@pytest.mark.skip("TODO")
 def test_writes_expected_output_files(ingest_path: Path):
     """
     Did the image ingestion manifest builder produce the output file(s) we expect?
@@ -285,8 +285,54 @@ def test_writes_expected_output_files(ingest_path: Path):
     :param ingest_path:
     :return:
     """
-    # TODO:
-    raise NotImplementedError
+    populate_fake_image_ingest_path(ingest_path)
+    manifest, manifest_file = IngestionManifestBuilder(
+        telescope=Telescope.EVLA.value,
+        staging_source_dir=ingest_path,
+        locator="uid://evla/image/kiss-me-Im_Elvish",
+        sp_type=ScienceProductType.IMAGE.value,
+    ).build()
+    assert manifest_file
+    assert manifest_file.name == MANIFEST_FILENAME
+    assert manifest
+
+    ingestion_files = [file for file in ingest_path.iterdir()]
+
+    # In addition to the manifest, the science products tar and the ingestion artifact,
+    # we expect to find the additional metadata .json, two .fits image files, a thumbnail.png,
+    # CASA byproducts, and the random tar we tossed in
+    assert len(ingestion_files) == 13
+    files_accounted_for = []
+
+    fits = [file for file in ingestion_files if file.name.endswith(".fits")]
+    assert len(fits) == 2
+    for file in fits:
+        files_accounted_for.append(file)
+
+    thumbnails = [file for file in ingestion_files if file.name.endswith(".png")]
+    assert len(thumbnails) == 1
+    files_accounted_for.append(thumbnails[0])
+
+    casa_byproducts = [file for file in ingestion_files if file.name in CASA_BYPRODUCTS]
+    assert len(casa_byproducts) == len(CASA_BYPRODUCTS)
+    for file in casa_byproducts:
+        files_accounted_for.append(file)
+
+    manifests = [file for file in ingestion_files if file.name == MANIFEST_FILENAME]
+    assert len(manifests) == 1
+    files_accounted_for.append(manifests[0])
+
+    inputs = [file for file in ingestion_files if file.name in IMG_MANIFEST_INPUT_FILENAMES]
+    assert len(inputs) == len(IMG_MANIFEST_INPUT_FILENAMES)
+    for file in inputs:
+        files_accounted_for.append(file)
+
+    artifacts = [file for file in ingestion_files if is_ingestion_artifact(file)]
+    assert len(artifacts) == 1
+    files_accounted_for.append(artifacts[0])
+
+    unique_files = set(files_accounted_for)
+    assert len(set(files_accounted_for)) == len(ingestion_files) - 1
 
     shutil.rmtree(ingest_path)
 
@@ -397,3 +443,13 @@ def build_output_group(staging_source_dir: Path) -> OutputGroup:
     ap_list = other_aps
 
     return OutputGroup(science_products=[osp], ancillary_products=ap_list)
+
+
+def is_ingestion_artifact(file: Path) -> bool:
+    """
+    We ought to have an ingestion artifact named like "ingestion_artifacts_*.tar"
+
+    :param file: some tar we found in staging source dir
+    :return:
+    """
+    return file.name.startswith("ingestion_artifacts") and file.name.endswith(TARFILE_EXT)

apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py

@@ -8,6 +8,7 @@ from pathlib import Path
 
 # pylint: disable=E0401, E0402, R1721, W0621
 import pytest
+import tarfile
 
 from ingest_envoy.ingestion_manifest import (
     IngestionManifestBuilder,
@@ -24,6 +25,7 @@ from ingest_envoy.manifest_components import (
     TARFILE_EXT,
     INGESTION_ARTIFACTS_NAME,
     WEBLOG_FILENAME,
+    MANIFEST_FILENAME,
 )
 from ingest_envoy.utilities import (
     ScienceProductType,
@@ -37,7 +39,7 @@ from ingest_envoy.utilities import (
 from .conftest import (
     ingest_path,
     populate_fake_evla_cal_ingest_path,
-    EVLA_CAL_WANTED_FILENAMES,
+    EVLA_CAL_INPUT_FILENAMES,
     UNWANTED,
     find_example_manifest,
 )
@@ -82,7 +84,7 @@ def test_filters_cal_input_files(ingest_path: Path):
     assert len(output_group.science_products) == 1
     assert len(output_group.ancillary_products) == 1
     for product in output_group.ancillary_products:
-        if product.filename not in EVLA_CAL_WANTED_FILENAMES:
+        if product.filename not in EVLA_CAL_INPUT_FILENAMES:
             assert product.filename.startswith(
                 INGESTION_ARTIFACTS_NAME
             ) and product.filename.endswith(TARFILE_EXT)
@@ -91,7 +93,7 @@ def test_filters_cal_input_files(ingest_path: Path):
     sp_out = output_group.science_products[0]
     assert sp_out.type == ScienceProductType.EVLA_CAL
 
-    assert sp_out.filename in EVLA_CAL_WANTED_FILENAMES
+    assert sp_out.filename in EVLA_CAL_INPUT_FILENAMES
     assert sp_out.filename not in UNWANTED
 
     shutil.rmtree(ingest_path)
@@ -219,13 +221,34 @@ def test_output_group_well_formed():
     assert actual_json == expected_json
 
 
-@pytest.mark.skip("TODO")
-def test_ingestion_artifacts_tar_correct():
+def test_ingestion_artifacts_tar_correct(ingest_path: Path):
     """
-    TODO
+    Do we build the correct ingestion_artifacts tar for for standard imaging ingestion?
+
+    :param ingest_path: the ingestion staging dir
     :return:
     """
-    raise NotImplementedError
+
+    populate_fake_evla_cal_ingest_path(ingest_path)
+    locator = "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
+    # if you build it, they will come
+    IngestionManifestBuilder(
+        staging_source_dir=ingest_path,
+        sp_type=ScienceProductType.EVLA_CAL.value,
+        locator=locator,
+        telescope=Telescope.EVLA.value,
+    ).build()
+
+    # get ingestion artifacts
+    artifacts_file = [file for file in ingest_path.glob("ingestion_artifacts*.tar")][0]
+    with tarfile.open(artifacts_file, "r") as tar:
+        # confirm that contains has as many files as we expect...
+        members = tar.getmembers()
+
+    assert len(members) == 1
+    member = members[0]
+    mf_path = Path(member.name)
+    assert mf_path.name == MANIFEST_FILENAME
 
 
 @pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation")
@@ -268,16 +291,3 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
     assert actual_json == expected_json
 
     shutil.rmtree(ingest_path)
-
-
-def populate_ingest_path_for_manifest_evla_cal_example(ingest_path: Path):
-    """
-    Create fake input files to match EVLA CAL manifest example
-
-    :param ingest_path:
-    :return:
-    """
-    weblog_file = ingest_path / "weblog.tgz"
-    weblog_file.touch()
-    cal_file = ingest_path / "XYZ-abc+TMN.O00.tar"
-    cal_file.touch()