From fb27c776d83f247135e87f3d4101f1f3901fafd9 Mon Sep 17 00:00:00 2001
From: "Janet L. Goldstein" <jgoldste@nrao.edu>
Date: Thu, 22 Jul 2021 14:37:54 -0600
Subject: [PATCH] Post Apocalypse Round 6 - Ingestion
---
.../ingest_envoy/ingestion_manifest.py | 33 +++++++++++++------
1 file changed, 23 insertions(+), 10 deletions(-)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index 32557c016..04eaf3dbc 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -14,9 +14,11 @@ import pendulum
from pendulum import DateTime
from ingest_envoy.manifest_components import (
+ MANIFEST_NAME_BASE,
+ MANIFEST_NAME_EXT,
ARTIFACT_NAME,
- TARFILE_EXT,
- WEBLOG_FILENAME,
+ ARTIFACT_EXT,
+ WEBLOG,
JSON,
IngestionManifestKey,
ManifestComponentIF,
@@ -27,7 +29,6 @@ from ingest_envoy.manifest_components import (
AncillaryProduct,
OutputGroup,
SCIENCE_PRODUCT_PATTERN,
- MANIFEST_FILENAME,
)
from ingest_envoy.utilities import (
ScienceProductType,
@@ -151,7 +152,7 @@ class IngestionManifestBuilder:
# N.B. this is sufficient for most types of ingestion,
# but ALMA CALs will have multiple EB SPs, identified only by locator,
# and VLBAs have no input group at all.
- sp_in = InputScienceProduct(locator=self.locator)
+ sp_in = InputScienceProduct(sp_type=self.sp_type, locator=self.locator)
return InputGroup([sp_in])
@@ -187,7 +188,7 @@ class IngestionManifestBuilder:
"""
current_time = pendulum.now()
timestamp = format_timestamp(current_time)
- return f"{ARTIFACT_NAME}{timestamp}{TARFILE_EXT}"
+ return f"{ARTIFACT_NAME}{timestamp}{ARTIFACT_EXT}"
def write_ingestion_artifacts_tar(self) -> Path:
"""
@@ -219,11 +220,11 @@ class IngestionManifestBuilder:
ancillary_products = []
# if there's a weblog in here, grab it
- maybe_weblogs = [file for file in self.files_found if file.name == WEBLOG_FILENAME]
+ maybe_weblogs = [file for file in self.files_found if file.name.endswith(WEBLOG)]
if len(maybe_weblogs) > 0:
weblog = maybe_weblogs[0]
weblog_ap = AncillaryProduct(
- type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=weblog.name
+ type=AncillaryProductType.PIPELINE_WEBLOG, filename=weblog.name
)
ancillary_products.append(weblog_ap)
@@ -280,7 +281,7 @@ class IngestionManifest(ManifestIF):
:return:
"""
- output_path = self.staging_source_dir / MANIFEST_FILENAME
+ output_path = self.staging_source_dir / build_manifest_filename()
to_write = json.dumps(self.to_json(), indent=4)
with open(output_path, "w") as out:
@@ -335,6 +336,17 @@ def format_timestamp(datetime: DateTime) -> str:
return datetime.format("YYYY_MM_DDThh_mm_ss.SSS")
+def build_manifest_filename() -> str:
+ """
+ Build unique manifest filename in standard format.
+
+ :return: the filename
+ """
+ current_time = pendulum.now()
+ timestamp = format_timestamp(current_time)
+ return f"{MANIFEST_NAME_BASE}{timestamp}{MANIFEST_NAME_EXT}"
+
+
def find_manifest(ingestion_path: Path) -> Path:
"""
Find the ingestion manifest at this ingestion path.
@@ -342,7 +354,8 @@ def find_manifest(ingestion_path: Path) -> Path:
:param ingestion_path: home of ingestion files
:return:
"""
- for json_file in ingestion_path.glob(MANIFEST_FILENAME):
- return json_file
+ for file in ingestion_path.iterdir():
+ if file.name.startswith(MANIFEST_NAME_BASE) and file.name.endswith(MANIFEST_NAME_EXT):
+ return file
raise FileNotFoundError(f"No ingestion manifest found at {ingestion_path}")
--
GitLab