diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index 8741f0081a7e3741749555cbe4147dc5f25eed56..27969f93639da9cafa4b9a8bee85b81c048f03da 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -2,7 +2,6 @@
import abc
import json
import logging
-import re
import sys
import tarfile
from pathlib import Path
@@ -26,9 +25,10 @@ from ingest_envoy.manifest_components import (
OutputScienceProduct,
AncillaryProduct,
OutputGroup,
- SCIENCE_PRODUCT_PATTERN,
MANIFEST_FILENAME,
+ ParamsKey,
)
+from ingest_envoy.schema import AbstractTextFile
from ingest_envoy.utilities import (
ScienceProductType,
Telescope,
@@ -57,6 +57,8 @@ class ManifestIF(ManifestComponentIF):
input_group: InputGroup,
# every manifest has at least one output group
output_group: OutputGroup,
+ # image manifest has this:
+ additional_metadata: AbstractTextFile = None,
):
self.staging_source_dir = staging_source_dir
self.sp_type = sp_type
@@ -65,10 +67,39 @@ class ManifestIF(ManifestComponentIF):
self.output_group = output_group
self.telescope = telescope
- self.parameters = self.build_ingest_parameters()
+ self.parameters = self.build_ingest_parameters(additional_metadata)
self.files_found = [file for file in self.staging_source_dir.iterdir()]
+ def build_ingest_parameters(self, additional_metadata: AbstractTextFile):
+ """
+ Make the "parameters" section of the manifest
+
+ :return:
+ """
+ if self.sp_type not in [ScienceProductType.EVLA_CAL, ScienceProductType.IMAGE]:
+ raise NotImplementedError()
+
+ if additional_metadata:
+ params = ManifestParameters(
+ telescope=self.telescope,
+ reingest=False,
+ ngas_ingest=False,
+ calibrate=False,
+ staging_source_dir=self.staging_source_dir,
+ additional_metadata=additional_metadata,
+ )
+ else:
+ params = ManifestParameters(
+ telescope=self.telescope,
+ reingest=False,
+ ngas_ingest=False,
+ calibrate=False,
+ staging_source_dir=self.staging_source_dir,
+ )
+
+ return params
+
@abc.abstractmethod
def write(self):
"""
@@ -82,7 +113,11 @@ class ManifestIF(ManifestComponentIF):
def __eq__(self, other):
if isinstance(other, IngestionManifest):
- return other.input_group == self.input_group and other.output_group == self.output_group
+ return (
+ other.parameters == self.parameters
+ and other.input_group == self.input_group
+ and other.output_group == self.output_group
+ )
return False
@@ -102,14 +137,16 @@ class IngestionManifestBuilder:
def __init__(
self,
staging_source_dir: Path,
- sp_type: str,
+ sp_type: ScienceProductType,
locator: str,
telescope: Telescope,
+ additional_metadata: AbstractTextFile = None,
):
self.telescope = telescope
self.staging_source_dir = staging_source_dir
self.sp_type = ScienceProductType(sp_type)
self.locator = locator
+ self.additional_metadata = additional_metadata
self.files_found = [file for file in staging_source_dir.iterdir()]
if len(self.files_found) == 0:
raise IngestionManifestException(f"No ingestion files found at {staging_source_dir}")
@@ -134,6 +171,8 @@ class IngestionManifestBuilder:
input_group=self._build_input_group(),
output_group=self._build_output_group(),
)
+ if self.additional_metadata:
+ manifest.parameters.additional_metadata = self.additional_metadata
manifest_file = manifest.write()
@@ -170,15 +209,34 @@ class IngestionManifestBuilder:
# find ancillary products, if any
ancillary_products = self._find_ancillary_products()
- # N.B. this is NOT done for EVLA CAL manifest, but keep code for future use
- # tar_filename = self.build_artifacts_filename()
- # artifacts_ap = AncillaryProduct(
- # type=AncillaryProductType.PIPELINE_ARTIFACTS, filename=tar_filename
- # )
- # ancillary_products.append(artifacts_ap)
-
return OutputGroup(self._define_output_science_products(), ancillary_products)
+ def _build_ancillary_product(self, file: Path) -> AncillaryProduct:
+ """
+ If this file is required for ingestion manifest creation,
+ create an ancillary product from it.
+
+ :param file: file found in staging dir
+ :return: ancillary product represented by this file, if any
+ """
+ if file.name == WEBLOG_FILENAME:
+ return AncillaryProduct(
+ type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=str(file)
+ )
+
+ if AncillaryProductType.PIPELINE_ARTIFACTS.value in file.name:
+ return AncillaryProduct(
+ type=AncillaryProductType.PIPELINE_ARTIFACTS, filename=str(file)
+ )
+
+ if AncillaryProductType.INGESTION_ARTIFACTS.value in file.name:
+ return AncillaryProduct(
+ type=AncillaryProductType.INGESTION_ARTIFACTS, filename=str(file)
+ )
+
+ # this is not an ancillary product
+ return None
+
@staticmethod
def build_artifacts_filename() -> str:
"""
@@ -219,6 +277,7 @@ class IngestionManifestBuilder:
"""
ancillary_products = []
+
# if there's a weblog in here, grab it
maybe_weblogs = [file for file in self.files_found if file.name == WEBLOG_FILENAME]
if len(maybe_weblogs) > 0:
@@ -228,9 +287,11 @@ class IngestionManifestBuilder:
)
ancillary_products.append(weblog_ap)
- more_aps = self._find_additional_ingestion_files()
- if len(more_aps) > 0:
- ancillary_products.append(more_aps)
+ additional_files = self._find_additional_ingestion_files()
+ for file in additional_files:
+ maybe_ap = self._build_ancillary_product(file)
+ if maybe_ap and maybe_ap not in ancillary_products:
+ ancillary_products.append(maybe_ap)
return ancillary_products
@@ -245,6 +306,9 @@ class IngestionManifestBuilder:
# there won't be any others
return []
+ if self.sp_type == ScienceProductType.IMAGE:
+ return [file for file in self.staging_source_dir.iterdir()]
+
# TODO when the time comes: we'll have extra information for other ingestion types;
# see archive-metaproject
raise NotImplementedError
@@ -253,26 +317,8 @@ class IngestionManifestBuilder:
class IngestionManifest(ManifestIF):
"""write ingestion manifest to file"""
- def build_ingest_parameters(self):
- """
- Make the "parameters" section of the manifest
-
- :return:
- """
- if self.sp_type != ScienceProductType.EVLA_CAL:
- raise NotImplementedError()
-
- return ManifestParameters(
- telescope=self.telescope,
- reingest=False,
- ngas_ingest=False,
- calibrate=False,
- staging_source_dir=self.staging_source_dir,
- )
-
- # @property
def ingestion_path(self) -> Path:
- return self.parameters.ingestion_path
+ return self.parameters.staging_source_dir
def write(self) -> Path:
"""
@@ -300,7 +346,7 @@ class IngestionManifest(ManifestIF):
me_dict = dict(self.__dict__)
to_return = {
- IngestionManifestKey.PARAMETERS.value: self.build_ingest_parameters().to_json(),
+ IngestionManifestKey.PARAMETERS.value: self.parameters.to_json(),
IngestionManifestKey.INPUT_GROUP.value: me_dict[
IngestionManifestKey.INPUT_GROUP.value
].to_json(),
@@ -308,22 +354,13 @@ class IngestionManifest(ManifestIF):
IngestionManifestKey.OUTPUT_GROUP.value
].to_json(),
}
+ if self.parameters.additional_metadata:
+ to_return[ParamsKey.ADDITIONAL_METADATA.value] = str(
+ self.parameters.additional_metadata
+ )
return to_return
- def _find_science_product_tar(self) -> Path:
- """
- A calibration ingestion staging dir should have ONE science product tar; ignore any others
-
- :return:
- """
- files = [file for file in self.staging_source_dir.iterdir() if file.is_file]
- for file in files:
- if re.match(SCIENCE_PRODUCT_PATTERN, file.name):
- return file
-
- raise FileNotFoundError(f"no science product found at {self.staging_source_dir}")
-
def format_timestamp(datetime: DateTime) -> str:
"""
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
index d5f59e84bdf871ae1d92c4e68fcc5db8d7fffa01..66e2236e79662b2e3bce93f9e7eef81f67c05db2 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
@@ -6,6 +6,7 @@ from enum import Enum
from pathlib import Path
from typing import Union, List, Dict
+from ingest_envoy.schema import AbstractTextFile
from ingest_envoy.utilities import ScienceProductType, Telescope, AncillaryProductType
MANIFEST_FILENAME = "ingestion_manifest.json"
@@ -36,6 +37,7 @@ class ParamsKey(Enum):
NGAS_INGEST = "ngas_ingest"
CALIBRATE = "calibrate"
INGESTION_PATH = IngestionManifestKey.INGESTION_PATH.value
+ ADDITIONAL_METADATA = "additional_metadata"
class ManifestComponentIF(abc.ABC):
@@ -113,12 +115,14 @@ class ManifestParameters(ManifestComponentIF):
ngas_ingest: bool,
calibrate: bool,
staging_source_dir: Path,
+ additional_metadata: AbstractTextFile = None,
):
self.telescope = telescope
self.reingest = reingest
self.ngas_ingest = ngas_ingest
self.calibrate = calibrate
self.staging_source_dir = staging_source_dir
+ self.additional_metadata = additional_metadata
def __eq__(self, other):
if isinstance(other, ManifestParameters):
@@ -128,35 +132,24 @@ class ManifestParameters(ManifestComponentIF):
and other.ngas_ingest == self.ngas_ingest
and other.calibrate == self.calibrate
and other.staging_source_dir == self.staging_source_dir
+ and other.additional_metadata.filename == self.additional_metadata.filename
+ and other.additional_metadata.content == self.additional_metadata.content
)
return False
def to_json(self) -> JSON:
- return {
- ParamsKey.TELESCOPE.value: self.telescope,
+ json_dict = {
+ ParamsKey.TELESCOPE.value: self.telescope.value,
ParamsKey.REINGEST.value: str(self.reingest).lower(),
ParamsKey.NGAS_INGEST.value: str(self.ngas_ingest).lower(),
ParamsKey.CALIBRATE.value: str(self.calibrate).lower(),
ParamsKey.INGESTION_PATH.value: str(self.staging_source_dir),
}
+ if self.additional_metadata:
+ json_dict[ParamsKey.ADDITIONAL_METADATA.value] = str(self.additional_metadata)
-
-class OutputScienceProduct(ManifestComponentIF):
- """Generic science product contained in manifest output group"""
-
- def __init__(self, type: ScienceProductType, filename: str):
- self.type = type
- self.filename = filename
-
- def __eq__(self, other):
- if isinstance(other, OutputScienceProduct):
- return other.type == self.type and other.filename == self.filename
-
- return False
-
- def to_json(self) -> JSON:
- return {"type": self.type.value, "filename": self.filename}
+ return json_dict
class AncillaryProduct(ManifestComponentIF):
@@ -208,6 +201,37 @@ class AncillaryProduct(ManifestComponentIF):
return clean_dict
+class OutputScienceProduct(ManifestComponentIF):
+ """Generic science product contained in manifest output group"""
+
+ def __init__(
+ self,
+ type: ScienceProductType,
+ filename: str,
+ ancillary_products: List[AncillaryProduct] = None,
+ ):
+ self.type = type
+ self.filename = filename
+ self.ancillary_products = ancillary_products
+
+ def __eq__(self, other):
+ if isinstance(other, OutputScienceProduct):
+ return (
+ other.type == self.type
+ and other.filename == self.filename
+ and other.ancillary_products == self.ancillary_products
+ )
+
+ return False
+
+ def to_json(self) -> JSON:
+ json_dict = {"type": self.type.value, "filename": self.filename}
+ if self.ancillary_products:
+ aps_json = [ap.to_json() for ap in self.ancillary_products]
+ json_dict[IngestionManifestKey.ANCILLARY_PRODUCTS.value] = aps_json
+ return json_dict
+
+
class OutputGroup(ManifestComponentIF):
"""Generic ingestion manifest output group"""
@@ -242,11 +266,10 @@ class OutputGroup(ManifestComponentIF):
aps = me_dict[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
if aps:
- aps = [ap.to_json() for ap in aps]
- me_dict[IngestionManifestKey.ANCILLARY_PRODUCTS.value] = aps
+ ap_jsons = [ap.to_json() for ap in aps]
+ me_dict[IngestionManifestKey.ANCILLARY_PRODUCTS.value] = ap_jsons
return me_dict
- # return {IngestionManifestKey.OUTPUT_GROUP.value: me_dict}
class Weblog:
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
index 4e4f578f3742c747508c7d6e6168862fec0f8174..14ed7fb7b0b9f770afa0063620c04866b9a039dc 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
@@ -42,7 +42,8 @@ class AncillaryProductType(Enum):
# our default FITS type
FITS = "fits_image"
- VLASS_QUICKLOOK = "quicklook_rms_image"
+ QUICKLOOK_IMAGE = "quicklook_image"
+ QUICKLOOK_RMS_IMAGE = "quicklook_rms_image"
AUDI_FITS_MASK = "clean_mask"
AUDI_PB_FITS = "primary_beam"
ALPHA_FITS = "spectral_index"
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_VLASS_QUICKLOOK.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_VLASS_QUICKLOOK.json
index 2e2dd550aa06b94867b7e84a06887d865779d757..aaee0c33a01dbe93243dcb1a2c3d8f5a3d5e745e 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_VLASS_QUICKLOOK.json
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/Manifest_VLASS_QUICKLOOK.json
@@ -1,15 +1,15 @@
{
"parameters": {
"reingest": false,
- "ngas_ingest": false,
+ "ngas_ingest": true,
"telescope": "EVLA",
- "ingestion_path": "/this/is/my/fake/path",
- "additional_metadata": "T00t99.json"
+ "ingestion_path": "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1",
+ "additional_metadata": "image_metadata_2021_05_21_T10_17_19.180.json"
},
"input_group": {
"science_products": [
{
- "locator": "uid://evla/calibration/ABCDE-FGHIJKLMN-OPQRSTUVW-XYZ"
+ "locator": "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
}
]
},
@@ -17,11 +17,15 @@
"science_products": [
{
"type": "quicklook_image",
- "filename": "foo_bar-car.pbcor.tt5.subim.fits",
+ "filename": "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits",
"ancillary_products": [
{
"type": "quicklook_rms_image",
- "filename": "foo_bar-car.pbcor.tt5.rms.subim.fits"
+ "filename": "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.I.iter1.image.pbcor.tt0.rms.subim.fits"
+ },
+ {
+ "type": "thumbnail_image",
+ "filename": "VLASS2.1.ql.T08t09.J055438_113000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.png"
}
]
}
@@ -32,13 +36,13 @@
"filename": "weblog.tgz"
},
{
- "type": "pipeline_auxillary_tar",
- "filename": "ABC+DEFG_HIJK-LMNO.tar"
+ "type": "pipeline_artifacts",
+ "filename": "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1.tar"
},
{
"type": "ingestion_artifacts",
- "filename": "ingestion_artifacts_2019_07_30_T13_03_00.936.tar"
+ "filename": "ingestion_artifacts_2021_05_21_T10_17_19.275.tar"
}
]
}
-}
+}
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2f8bdbfa435a10f0a6b84bd81efbd4c9fa5149d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_image_manifest.py
@@ -0,0 +1,262 @@
+""" Tests for building image ingestion manifest """
+
+from pathlib import Path
+from typing import List
+
+# pylint: disable=C0103, E0401, E0402, R1721
+
+import json
+import pytest
+
+from ingest_envoy.ingestion_manifest import IngestionManifestBuilder
+from ingest_envoy.schema import AbstractTextFile
+from ingest_envoy.utilities import Telescope, AncillaryProductType, ScienceProductType
+
+# NOT unused. IJ is dumb. KEEP THIS.
+from .conftest import ingest_path
+
+from ingest_envoy.manifest_components import (
+ WEBLOG_FILENAME,
+ ManifestParameters,
+ ParamsKey,
+ OutputGroup,
+ OutputScienceProduct,
+ AncillaryProduct,
+ InputGroup,
+ InputScienceProduct,
+)
+
+IMG_MANIFEST_FILENAMES = [
+ "image_metadata_2021_05_21_T10_17_19.180.json",
+ "VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits",
+ "VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.rms.subim.fits",
+ WEBLOG_FILENAME,
+ "uid____EVLA_ingestion_artifacts_b1ab328d-200e-4da4-86bf-514773f31e2b.tar",
+ "ingestion_artifacts_2019_07_30_T13_03_00.936.tar",
+]
+OTHER_FILENAMES = [
+ "unknown.pipeline_manifest.xml",
+ "unknown.aux_products.tgz",
+ "casa_commands.log",
+ "casa_pipescript.py",
+ "totally_random_tar.tar",
+]
+
+
+def test_parameters_json_well_formed():
+ """
+ Make sure we get expected ManifestParameters for an image.
+
+ :return:
+ """
+ params = manifest_parameters()
+
+ params_json = params.to_json()
+ # Only difference from EVLA CAL manifest is additional_metadata
+ assert params_json[ParamsKey.ADDITIONAL_METADATA.value] == str(params.additional_metadata)
+
+ # gotta be able to dump it; test will fail if dump runs into trouble
+ json.dumps(params_json)
+
+
+def test_output_science_prods_built_properly():
+ """
+ Output science products for an image ingestion contain ancillary products.
+ Let's make sure we can build one of these OSPs.
+ :return:
+ """
+ ap = rms_ancillary_prod()
+ osp = osp_ql_with_ancillary(ap)
+
+ assert osp.type == AncillaryProductType.QUICKLOOK_IMAGE
+ assert (
+ osp.filename
+ == "VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits"
+ )
+ assert len(osp.ancillary_products) == 1
+
+ osp_ap = osp.ancillary_products[0]
+ assert osp_ap == ap
+ ap_json = ap.to_json()
+ # if we can't dump it, we know it's no good
+ ap_str = json.dumps(ap_json)
+
+ osp_json = osp.to_json()
+ assert osp_json["type"] == osp.type.value
+ assert osp_json["filename"] == osp.filename
+
+ op_str = json.dumps(osp_json)
+ assert ap_str in op_str
+
+
+def test_output_group_json_well_formed():
+ """
+ In addition to the ancillary_products section of the image ingestion manifest,
+ output science products may themselves contain ancillary products.
+ :return:
+ """
+
+ output_group = build_output_group()
+ expected_og_json = output_group.to_json()
+
+ sps = output_group.science_products
+ expected_sp_json = [sp.to_json() for sp in sps]
+
+ other_aps = separate_aps()
+ expected_aps_json = [ap.to_json() for ap in other_aps]
+
+ og_dump = json.dumps(expected_og_json)
+ aps_dump = json.dumps(expected_aps_json)
+ sps_dump = json.dumps(expected_sp_json)
+ assert sps_dump in og_dump
+ assert aps_dump in og_dump
+ assert aps_dump not in sps_dump
+
+
+def test_creates_expected_manifest(ingest_path: Path):
+ """
+ Did the image ingestion manifest builder make the manifest we expected?
+
+ :return:
+ """
+
+ # fill the ingestion path with fake files
+ populate_fake_ingest_path(ingest_path)
+ locator = "uid://evla/calibration/3dfa528b-9870-46c9-a200-131dbac701cc"
+ addl_md = AbstractTextFile(filename="image_metadata_2021_05_21_T10_17_19.180.json", content="")
+ builder = IngestionManifestBuilder(
+ staging_source_dir=ingest_path,
+ sp_type=ScienceProductType.IMAGE,
+ locator=locator,
+ telescope=Telescope.EVLA,
+ additional_metadata=addl_md,
+ )
+ manifest, _ = builder.build()
+ expected_params = manifest_parameters()
+ expected_params.staging_source_dir = ingest_path
+ assert manifest.parameters == expected_params
+ assert manifest.input_group == InputGroup(science_products=[InputScienceProduct(locator)])
+
+
+@pytest.mark.skip("TODO")
+def test_filters_irrelevant_files(ingest_path: Path):
+ """
+ The image ingestion manifest should contain no references to additional files in a directory
+ that aren't needed for the manifest.
+
+ :param ingest_path: the staging directory
+ :return:
+ """
+ raise NotImplementedError
+
+
+@pytest.mark.skip("TODO")
+def test_writes_expected_output_files(ingest_path: Path):
+ """
+ Did the image ingestion manifest builder produce the output file(s) we expect?
+
+ :param ingest_path:
+ :return:
+ """
+
+
+# -----------------------------
+# U T I L I T I E S
+# -----------------------------
+
+
+def manifest_parameters() -> ManifestParameters:
+ """
+ Build a ManifestParameters for our tests
+
+ :return: the manifest parameters we're expecting
+ """
+ return ManifestParameters(
+ telescope=Telescope.EVLA,
+ reingest=False,
+ ngas_ingest=False,
+ calibrate=False,
+ staging_source_dir=Path(
+ "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/"
+ + "VLASS2.1.ql.T08t09.J055438-113000.10.2048.v1"
+ ),
+ additional_metadata=AbstractTextFile(
+ filename="image_metadata_2021_05_21_T10_17_19.180.json", content=""
+ ),
+ )
+
+
+def rms_ancillary_prod() -> AncillaryProduct:
+ """
+ Build an AncillaryProduct for our tests
+
+ :return: the product we're expecting
+ """
+ return AncillaryProduct(
+ type=AncillaryProductType.QUICKLOOK_RMS_IMAGE,
+ filename="VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image"
+ + ".pbcor.tt0.rms.subim.fits",
+ )
+
+
+def osp_ql_with_ancillary(ap: AncillaryProduct) -> OutputScienceProduct:
+ """
+ Build an OutputScienceProduct for our tests
+
+ :return: the product we're expecting
+ """
+ return OutputScienceProduct(
+ type=AncillaryProductType(AncillaryProductType.QUICKLOOK_IMAGE),
+ filename="VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.tt0.subim.fits",
+ ancillary_products=[ap],
+ )
+
+
+def separate_aps() -> List[AncillaryProduct]:
+ """
+ Build a list of AncillaryProducts for our tests
+
+ :return: the products we're expecting
+ """
+ ap1 = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=WEBLOG_FILENAME)
+ ap2 = AncillaryProduct(
+ type=AncillaryProductType.PIPELINE_ARTIFACTS,
+ filename="uid____EVLA_ingestion_artifacts_b1ab328d-200e-4da4-86bf-514773f31e2b.tar",
+ )
+ ap3 = AncillaryProduct(
+ type=AncillaryProductType.INGESTION_ARTIFACTS,
+ filename="ingestion_artifacts_2019_07_30_T13_03_00.936.tar",
+ )
+ return [ap1, ap2, ap3]
+
+
+def build_output_group() -> OutputGroup:
+ """
+ Build an OutputGroup for our tests
+
+ :return: the output group we're expecting
+ """
+ ap = rms_ancillary_prod()
+ osp = osp_ql_with_ancillary(ap)
+
+ other_aps = separate_aps()
+ ap_list = other_aps
+
+ return OutputGroup(science_products=[osp], ancillary_products=ap_list)
+
+
+def populate_fake_ingest_path(staging_dir: Path) -> List[Path]:
+ """
+ Create a directory containing fake image products, plus other stuff
+ that we -don't- want to ingest.
+
+ :param staging_dir: our temporary dir
+ :return:
+ """
+ for filename in IMG_MANIFEST_FILENAMES:
+ file = staging_dir / filename
+ file.touch()
+ for filename in OTHER_FILENAMES:
+ file = staging_dir / filename
+ file.touch()
+ return [file for file in staging_dir.iterdir()]
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
index 61d16291a315663bcec4be5740c23396fe52d625..aceee54558e0eba09fda45e3db5c166ff36b2c6f 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py
@@ -1,8 +1,5 @@
""" Tests for generic and EVLA CAL manifests """
-# TODO when the time comes: tests of ingestion manifests for each remaining ingestion type
-# as they're implemented, in separate modules so they don't get too unwieldy
-
import json
import logging
import shutil
@@ -64,7 +61,7 @@ def test_filters_cal_input_files(ingest_path: Path):
populate_fake_evla_cal_ingest_path(ingest_path)
locator = "uid://evla/calibration/twinkle-twinkle-little-quasar"
manifest, _ = IngestionManifestBuilder(
- telescope=Telescope.EVLA.value,
+ telescope=Telescope.EVLA,
staging_source_dir=ingest_path,
sp_type=ScienceProductType.EVLA_CAL.value,
locator=locator,
@@ -107,7 +104,7 @@ def test_writes_expected_output_files(ingest_path: Path):
"""
populate_fake_evla_cal_ingest_path(ingest_path)
manifest_file, manifest = IngestionManifestBuilder(
- telescope=Telescope.EVLA.value,
+ telescope=Telescope.EVLA,
staging_source_dir=ingest_path,
locator="uid://evla/calibration/fee-fi-fo-fum-acdf23",
sp_type=ScienceProductType.EVLA_CAL.value,
@@ -134,7 +131,7 @@ def test_params_json_well_formed():
:return:
"""
params = ManifestParameters(
- telescope=Telescope.EVLA.value,
+ telescope=Telescope.EVLA,
reingest=False,
ngas_ingest=False,
calibrate=False,
@@ -242,8 +239,8 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
builder = IngestionManifestBuilder(
staging_source_dir=ingest_path,
- telescope=Telescope.EVLA.value,
- sp_type=ScienceProductType.EVLA_CAL.value,
+ telescope=Telescope.EVLA,
+ sp_type=ScienceProductType.EVLA_CAL,
locator="uid://evla/execblock/48ba4c9d-d7c7-4a8f-9803-1115cd52459b",
)
_, manifest_file = builder.build()
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py b/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
index fca7fa84435c3b2b8375e1033cfe1a4467867b72..dad73870ddbf659d43f791a2e9a9709bc6e0728a 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_miscellaneous_manifests.py
@@ -38,7 +38,7 @@ def test_entry_point_for_evla_cal(ingest_path: Path):
assert sp_tar
builder = IngestionManifestBuilder(
- telescope=Telescope.EVLA.value,
+ telescope=Telescope.EVLA,
locator=locator,
sp_type=ScienceProductType.EVLA_CAL.value,
staging_source_dir=ingest_path,
@@ -60,43 +60,20 @@ def test_entry_point_for_evla_cal(ingest_path: Path):
assert len(artifact_tars) == 1
-@pytest.mark.skip("TODO: test_builds_realfast_sdm_manifest")
-def test_builds_realfast_sdm_manifest():
+@pytest.mark.skip("TODO: test_builds_image_manifest")
+def test_builds_image_manifest(ingest_path: Path):
"""
- Have we built a well-formed RealFast product ingestion manifest?
+ TODO NEXT 2021-07-28am
:return:
"""
raise NotImplementedError
-@pytest.mark.skip("TODO: test_builds_vlass_quicklook_manifest")
-def test_builds_vlass_quicklook_manifest():
+@pytest.mark.skip("TODO: test_entry_point_for_image")
+def test_entry_point_for_image(ingest_path: Path):
"""
- Have we built a well-formed VLASS quicklook product ingestion manifest?
-
- :return:
- """
- raise NotImplementedError
-
-
-@pytest.mark.skip("TODO: test_builds_gbt_manifest")
-def test_builds_gbt_manifest():
- """
- Not yet in use, but sooner or later we'll need to create manifests
- for ingestion of GBT products.
-
- :return:
- """
- raise NotImplementedError
-
-
-@pytest.mark.skip("TODO: test_builds_lvla_eb_manifest")
-def test_builds_lvla_eb_manifest():
- """
- Not yet in use, but one of these days we'll need to create manifests
- for ingestion of LVLA execution blocks.
-
+ TODO
:return:
"""
raise NotImplementedError