WS-543: Addressed issues found in calibration ingestion testing

• hyphens in manifest text are all changed to underscores
• InputScienceProduct is simplified
• weblog type is now pipeline_weblog
• manifest filename is just ingestion_manifest.json

TODO:

• additional test to duplicate manifest content of the 16B-069 example manifest

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py

@@ -14,11 +14,9 @@ import pendulum
 from pendulum import DateTime
 
 from ingest_envoy.manifest_components import (
-    MANIFEST_NAME_BASE,
-    MANIFEST_NAME_EXT,
     ARTIFACT_NAME,
-    ARTIFACT_EXT,
-    WEBLOG,
+    TARFILE_EXT,
+    WEBLOG_FILENAME,
     JSON,
     IngestionManifestKey,
     ManifestComponentIF,
@@ -29,6 +27,7 @@ from ingest_envoy.manifest_components import (
     AncillaryProduct,
     OutputGroup,
     SCIENCE_PRODUCT_PATTERN,
+    MANIFEST_FILENAME,
 )
 from ingest_envoy.utilities import (
     ScienceProductType,
@@ -161,7 +160,7 @@ class IngestionManifestBuilder:
         # N.B. this is sufficient for most types of ingestion,
         # but ALMA CALs will have multiple EB SPs, identified only by locator,
         # and VLBAs have no input group at all.
-        sp_in = InputScienceProduct(sp_type=self.sp_type, locator=self.locator)
+        sp_in = InputScienceProduct(locator=self.locator)
 
         return InputGroup([sp_in])
 
@@ -197,7 +196,7 @@ class IngestionManifestBuilder:
         """
         current_time = pendulum.now()
         timestamp = format_timestamp(current_time)
-        return f"{ARTIFACT_NAME}{timestamp}{ARTIFACT_EXT}"
+        return f"{ARTIFACT_NAME}{timestamp}{TARFILE_EXT}"
 
     def write_ingestion_artifacts_tar(self) -> Path:
         """
@@ -229,11 +228,11 @@ class IngestionManifestBuilder:
 
         ancillary_products = []
         # if there's a weblog in here, grab it
-        maybe_weblogs = [file for file in self.files_found if file.name.endswith(WEBLOG)]
+        maybe_weblogs = [file for file in self.files_found if file.name == WEBLOG_FILENAME]
         if len(maybe_weblogs) > 0:
             weblog = maybe_weblogs[0]
             weblog_ap = AncillaryProduct(
-                type=AncillaryProductType.PIPELINE_WEBLOG, filename=weblog.name
+                type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=weblog.name
             )
             ancillary_products.append(weblog_ap)
 
@@ -287,7 +286,7 @@ class IngestionManifest(ManifestIF):
         # N.B. this is sufficient for most types of ingestion,
         # but ALMA CALs will have multiple EB SPs, identified only by locator,
         # and VLBAs have no input group at all.
-        sp_in = InputScienceProduct(sp_type=self.sp_type, locator=self.locator)
+        sp_in = InputScienceProduct(locator=self.locator)
         return InputGroup([sp_in])
 
     def _build_output_group(self) -> OutputGroup:
@@ -301,13 +300,12 @@ class IngestionManifest(ManifestIF):
 
         # find ancillary products, if any
         ancillary_products = self._find_ancillary_products()
-        weblog = Path(self.ingestion_path / WEBLOG)
+        weblog = Path(self.ingestion_path / WEBLOG_FILENAME)
         if weblog.exists():
-            ancillary_products.append(AncillaryProduct(type=WEBLOG, filename=str(weblog)))
+            ancillary_products.append(AncillaryProduct(type=WEBLOG_FILENAME, filename=str(weblog)))
 
         return OutputGroup(sps_out)
 
-    # @property
     def ingestion_path(self) -> Path:
         return self.parameters.ingestion_path
 
@@ -318,7 +316,7 @@ class IngestionManifest(ManifestIF):
         :return:
         """
 
-        output_path = self.staging_source_dir / build_manifest_filename()
+        output_path = self.staging_source_dir / MANIFEST_FILENAME
 
         to_write = json.dumps(self.to_json(), indent=4)
         with open(output_path, "w") as out:
@@ -394,17 +392,6 @@ def format_timestamp(datetime: DateTime) -> str:
     return datetime.format("YYYY_MM_DDThh_mm_ss.SSS")
 
 
-def build_manifest_filename() -> str:
-    """
-    Build unique manifest filename in standard format.
-
-    :return: the filename
-    """
-    current_time = pendulum.now()
-    timestamp = format_timestamp(current_time)
-    return f"{MANIFEST_NAME_BASE}{timestamp}{MANIFEST_NAME_EXT}"
-
-
 def find_manifest(ingestion_path: Path) -> Path:
     """
     Find the ingestion manifest at this ingestion path.
@@ -412,8 +399,7 @@ def find_manifest(ingestion_path: Path) -> Path:
     :param ingestion_path: home of ingestion files
     :return:
     """
-    for file in ingestion_path.iterdir():
-        if file.name.startswith(MANIFEST_NAME_BASE) and file.name.endswith(MANIFEST_NAME_EXT):
-            return file
+    for json_file in ingestion_path.glob(MANIFEST_FILENAME):
+        return json_file
 
     raise FileNotFoundError(f"No ingestion manifest found at {ingestion_path}")