diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py index 32557c0162ea08fbe41c1cc5b0bcb186d565d0c4..04eaf3dbccf1feec8f9f4291e3116bed07d12065 100644 --- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py +++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py @@ -14,9 +14,11 @@ import pendulum from pendulum import DateTime from ingest_envoy.manifest_components import ( + MANIFEST_NAME_BASE, + MANIFEST_NAME_EXT, ARTIFACT_NAME, - TARFILE_EXT, - WEBLOG_FILENAME, + ARTIFACT_EXT, + WEBLOG, JSON, IngestionManifestKey, ManifestComponentIF, @@ -27,7 +29,6 @@ from ingest_envoy.manifest_components import ( AncillaryProduct, OutputGroup, SCIENCE_PRODUCT_PATTERN, - MANIFEST_FILENAME, ) from ingest_envoy.utilities import ( ScienceProductType, @@ -151,7 +152,7 @@ class IngestionManifestBuilder: # N.B. this is sufficient for most types of ingestion, # but ALMA CALs will have multiple EB SPs, identified only by locator, # and VLBAs have no input group at all. - sp_in = InputScienceProduct(locator=self.locator) + sp_in = InputScienceProduct(sp_type=self.sp_type, locator=self.locator) return InputGroup([sp_in]) @@ -187,7 +188,7 @@ class IngestionManifestBuilder: """ current_time = pendulum.now() timestamp = format_timestamp(current_time) - return f"{ARTIFACT_NAME}{timestamp}{TARFILE_EXT}" + return f"{ARTIFACT_NAME}{timestamp}{ARTIFACT_EXT}" def write_ingestion_artifacts_tar(self) -> Path: """ @@ -219,11 +220,11 @@ class IngestionManifestBuilder: ancillary_products = [] # if there's a weblog in here, grab it - maybe_weblogs = [file for file in self.files_found if file.name == WEBLOG_FILENAME] + maybe_weblogs = [file for file in self.files_found if file.name.endswith(WEBLOG)] if len(maybe_weblogs) > 0: weblog = maybe_weblogs[0] weblog_ap = AncillaryProduct( - type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=weblog.name + type=AncillaryProductType.PIPELINE_WEBLOG, filename=weblog.name ) ancillary_products.append(weblog_ap) @@ -280,7 +281,7 @@ class IngestionManifest(ManifestIF): :return: """ - output_path = self.staging_source_dir / MANIFEST_FILENAME + output_path = self.staging_source_dir / build_manifest_filename() to_write = json.dumps(self.to_json(), indent=4) with open(output_path, "w") as out: @@ -335,6 +336,17 @@ def format_timestamp(datetime: DateTime) -> str: return datetime.format("YYYY_MM_DDThh_mm_ss.SSS") +def build_manifest_filename() -> str: + """ + Build unique manifest filename in standard format. + + :return: the filename + """ + current_time = pendulum.now() + timestamp = format_timestamp(current_time) + return f"{MANIFEST_NAME_BASE}{timestamp}{MANIFEST_NAME_EXT}" + + def find_manifest(ingestion_path: Path) -> Path: """ Find the ingestion manifest at this ingestion path. @@ -342,7 +354,8 @@ def find_manifest(ingestion_path: Path) -> Path: :param ingestion_path: home of ingestion files :return: """ - for json_file in ingestion_path.glob(MANIFEST_FILENAME): - return json_file + for file in ingestion_path.iterdir(): + if file.name.startswith(MANIFEST_NAME_BASE) and file.name.endswith(MANIFEST_NAME_EXT): + return file raise FileNotFoundError(f"No ingestion manifest found at {ingestion_path}")