diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index e6257091bbf5bef9650e28595b4a433960a067c0..be55c55da105ef07a406c5aae9bec594995c9c6f 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -290,7 +290,8 @@ class IngestionManifestBuilder:
ing_tar = self.staging_source_dir / self.build_artifacts_filename()
with tarfile.open(ing_tar, "w") as ingestion_artifacts_tar:
for file in self.files_found:
- ingestion_artifacts_tar.add(file)
+ if self.sp_type == ScienceProductType.IMAGE:
+ ingestion_artifacts_tar.add(file)
# include the manifest
ingestion_artifacts_tar.add(manifest_file)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
index 37a930a0d919a51f4a2403933eb34eadedf298be..96b281fe5b36143a1e025ad152e0bf1e82ac755e 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
@@ -9,7 +9,7 @@ from typing import Union, List, Dict
from ingest_envoy.schema import AbstractTextFile
from ingest_envoy.utilities import ScienceProductType, Telescope, AncillaryProductType
-MANIFEST_FILENAME = "ingestion_manifest.json"
+MANIFEST_FILENAME = "ingestion-manifest.json"
INGESTION_ARTIFACTS_NAME = "ingestion_artifacts_"
TARFILE_EXT = ".tar"
WEBLOG_FILENAME = "weblog.tgz"
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/conftest.py b/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
index 2c7c51381d84d702b8c8ee2a4d64bb79ddc102ac..18e79977401a0164fe831135e5fc6c2b1c0833bd 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
@@ -9,7 +9,7 @@ import pytest
from ingest_envoy.manifest_components import WEBLOG_FILENAME
-EVLA_CAL_WANTED_FILENAMES = ["my_science_products.tar", WEBLOG_FILENAME]
+EVLA_CAL_INPUT_FILENAMES = ["20A-346_2021_07_23_T13_37_08.376.tar", WEBLOG_FILENAME]
UNWANTED = ["ignore_me.fits", "just_a_lotta_nothing", "uninteresting_metadata.xml"]
IMG_MANIFEST_INPUT_FILENAMES = [
@@ -80,7 +80,7 @@ def populate_fake_evla_cal_ingest_path(staging_dir: Path) -> List[Path]:
"""
files = []
- filenames = [filename for filename in EVLA_CAL_WANTED_FILENAMES]
+ filenames = [filename for filename in EVLA_CAL_INPUT_FILENAMES]
for filename in UNWANTED:
filenames.append(filename)
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/_16B_069_cal_manifest.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/_16B_069_cal_manifest.json
index 849c8021972d718e1f4ffa9ba85fb5d74c61181f..6a4fa9bf3d17301624c947234d19ddef591a6de5 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/examples/_16B_069_cal_manifest.json
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/_16B_069_cal_manifest.json
@@ -27,4 +27,4 @@
}
]
}
-}
+}
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_EVLA_Cal.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_cal_manifest_2021-08-02.json
similarity index 50%
rename from apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_EVLA_Cal.json
rename to apps/cli/executables/pexable/ingest_envoy/test/examples/evla_cal_manifest_2021-08-02.json
index 43d5150387c77361a31ef58fdad34695ba98c955..c87a865bc8a0610e3ef1686f3b8c4668c425532f 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_EVLA_Cal.json
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_cal_manifest_2021-08-02.json
@@ -1,14 +1,14 @@
{
"parameters": {
"reingest": false,
- "ngas_ingest": false,
+ "ngas_ingest": true,
"telescope": "EVLA",
- "ingestion_path": "/lustre/yea/and/here/we/go"
+ "ingestion_path": "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/20A-346_2021_07_23_T13_37_08.376"
},
"input_group": {
"science_products": [
{
- "locator": "uid://evla/execblock/fjdsakljfkdlsajfkldsa"
+ "locator": "uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d"
}
]
},
@@ -16,17 +16,17 @@
"science_products": [
{
"type": "calibration",
- "filename": "XYZ-abc+TMN.O00.tar"
+ "filename": "20A-346_2021_07_23_T13_37_08.376.tar"
}
],
"ancillary_products": [
{
"type": "pipeline_weblog",
- "filename": "qrs.weblog.tgz"
+ "filename": "weblog.tgz"
},
{
"type": "ingestion_artifacts",
- "filename": "ingestion_artifacts_2019_07_25_T15_43_33.144.tar"
+ "filename": "ingestion_artifacts_2021_07_27_T17_35_11.463.tar"
}
]
}
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
index 60e14afd0477b35892f7816ae38193c109a90615..e13b512984871e515c77b779ecf2071196ff6270 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
@@ -32,6 +32,7 @@ from ingest_envoy.manifest_components import (
InputGroup,
InputScienceProduct,
MANIFEST_FILENAME,
+ TARFILE_EXT,
)
@@ -277,7 +278,6 @@ def test_creates_expected_manifest(ingest_path: Path):
shutil.rmtree(ingest_path)
-@pytest.mark.skip("TODO")
def test_writes_expected_output_files(ingest_path: Path):
"""
Did the image ingestion manifest builder produce the output file(s) we expect?
@@ -285,8 +285,54 @@ def test_writes_expected_output_files(ingest_path: Path):
:param ingest_path:
:return:
"""
- # TODO:
- raise NotImplementedError
+ populate_fake_image_ingest_path(ingest_path)
+ manifest, manifest_file = IngestionManifestBuilder(
+ telescope=Telescope.EVLA.value,
+ staging_source_dir=ingest_path,
+ locator="uid://evla/image/kiss-me-Im_Elvish",
+ sp_type=ScienceProductType.IMAGE.value,
+ ).build()
+ assert manifest_file
+ assert manifest_file.name == MANIFEST_FILENAME
+ assert manifest
+
+ ingestion_files = [file for file in ingest_path.iterdir()]
+
+ # In addition to the manifest, the science products tar and the ingestion artifact,
+ # we expect to find the additional metadata .json, two .fits image files, a thumbnail.png,
+ # CASA byproducts, and the random tar we tossed in
+ assert len(ingestion_files) == 13
+ files_accounted_for = []
+
+ fits = [file for file in ingestion_files if file.name.endswith(".fits")]
+ assert len(fits) == 2
+ for file in fits:
+ files_accounted_for.append(file)
+
+ thumbnails = [file for file in ingestion_files if file.name.endswith(".png")]
+ assert len(thumbnails) == 1
+ files_accounted_for.append(thumbnails[0])
+
+ casa_byproducts = [file for file in ingestion_files if file.name in CASA_BYPRODUCTS]
+ assert len(casa_byproducts) == len(CASA_BYPRODUCTS)
+ for file in casa_byproducts:
+ files_accounted_for.append(file)
+
+ manifests = [file for file in ingestion_files if file.name == MANIFEST_FILENAME]
+ assert len(manifests) == 1
+ files_accounted_for.append(manifests[0])
+
+ inputs = [file for file in ingestion_files if file.name in IMG_MANIFEST_INPUT_FILENAMES]
+ assert len(inputs) == len(IMG_MANIFEST_INPUT_FILENAMES)
+ for file in inputs:
+ files_accounted_for.append(file)
+
+ artifacts = [file for file in ingestion_files if is_ingestion_artifact(file)]
+ assert len(artifacts) == 1
+ files_accounted_for.append(artifacts[0])
+
+ unique_files = set(files_accounted_for)
+ assert len(set(files_accounted_for)) == len(ingestion_files) - 1
shutil.rmtree(ingest_path)
@@ -397,3 +443,13 @@ def build_output_group(staging_source_dir: Path) -> OutputGroup:
ap_list = other_aps
return OutputGroup(science_products=[osp], ancillary_products=ap_list)
+
+
+def is_ingestion_artifact(file: Path) -> bool:
+ """
+ We ought to have an ingestion artifact named like "ingestion_artifacts_*.tar"
+
+ :param file: some tar we found in staging source dir
+ :return:
+ """
+ return file.name.startswith("ingestion_artifacts") and file.name.endswith(TARFILE_EXT)
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
index b1934831e493bed32a3d5403e638eddc7f0e9a82..20ce3736169ac6c84e82093a23e8ee1a560ba087 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
@@ -8,6 +8,7 @@ from pathlib import Path
# pylint: disable=E0401, E0402, R1721, W0621
import pytest
+import tarfile
from ingest_envoy.ingestion_manifest import (
IngestionManifestBuilder,
@@ -24,6 +25,7 @@ from ingest_envoy.manifest_components import (
TARFILE_EXT,
INGESTION_ARTIFACTS_NAME,
WEBLOG_FILENAME,
+ MANIFEST_FILENAME,
)
from ingest_envoy.utilities import (
ScienceProductType,
@@ -37,7 +39,7 @@ from ingest_envoy.utilities import (
from .conftest import (
ingest_path,
populate_fake_evla_cal_ingest_path,
- EVLA_CAL_WANTED_FILENAMES,
+ EVLA_CAL_INPUT_FILENAMES,
UNWANTED,
find_example_manifest,
)
@@ -82,7 +84,7 @@ def test_filters_cal_input_files(ingest_path: Path):
assert len(output_group.science_products) == 1
assert len(output_group.ancillary_products) == 1
for product in output_group.ancillary_products:
- if product.filename not in EVLA_CAL_WANTED_FILENAMES:
+ if product.filename not in EVLA_CAL_INPUT_FILENAMES:
assert product.filename.startswith(
INGESTION_ARTIFACTS_NAME
) and product.filename.endswith(TARFILE_EXT)
@@ -91,7 +93,7 @@ def test_filters_cal_input_files(ingest_path: Path):
sp_out = output_group.science_products[0]
assert sp_out.type == ScienceProductType.EVLA_CAL
- assert sp_out.filename in EVLA_CAL_WANTED_FILENAMES
+ assert sp_out.filename in EVLA_CAL_INPUT_FILENAMES
assert sp_out.filename not in UNWANTED
shutil.rmtree(ingest_path)
@@ -219,13 +221,34 @@ def test_output_group_well_formed():
assert actual_json == expected_json
-@pytest.mark.skip("TODO")
-def test_ingestion_artifacts_tar_correct():
+def test_ingestion_artifacts_tar_correct(ingest_path: Path):
"""
- TODO
+ Do we build the correct ingestion_artifacts tar for for standard imaging ingestion?
+
+ :param ingest_path: the ingestion staging dir
:return:
"""
- raise NotImplementedError
+
+ populate_fake_evla_cal_ingest_path(ingest_path)
+ locator = "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
+ # if you build it, they will come
+ IngestionManifestBuilder(
+ staging_source_dir=ingest_path,
+ sp_type=ScienceProductType.EVLA_CAL.value,
+ locator=locator,
+ telescope=Telescope.EVLA.value,
+ ).build()
+
+ # get ingestion artifacts
+ artifacts_file = [file for file in ingest_path.glob("ingestion_artifacts*.tar")][0]
+ with tarfile.open(artifacts_file, "r") as tar:
+ # confirm that contains has as many files as we expect...
+ members = tar.getmembers()
+
+ assert len(members) == 1
+ member = members[0]
+ mf_path = Path(member.name)
+ assert mf_path.name == MANIFEST_FILENAME
@pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation")
@@ -268,16 +291,3 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
assert actual_json == expected_json
shutil.rmtree(ingest_path)
-
-
-def populate_ingest_path_for_manifest_evla_cal_example(ingest_path: Path):
- """
- Create fake input files to match EVLA CAL manifest example
-
- :param ingest_path:
- :return:
- """
- weblog_file = ingest_path / "weblog.tgz"
- weblog_file.touch()
- cal_file = ingest_path / "XYZ-abc+TMN.O00.tar"
- cal_file.touch()