From 5fc93c416a013e9a400d84b0a414565c60dfc5c1 Mon Sep 17 00:00:00 2001
From: chausman <chausman@nrao.edu>
Date: Tue, 12 Mar 2024 10:38:16 -0600
Subject: [PATCH] fix tests
---
.../ingest_envoy/ingestion_manifest.py | 24 ++++++++++++++++---
.../pexable/ingest_envoy/test/conftest.py | 8 ++++---
.../test/test_curator_manifest.py | 18 +++++++-------
3 files changed, 34 insertions(+), 16 deletions(-)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index 16a495d75..7c4639f2d 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -440,12 +440,18 @@ class IngestionManifestBuilder:
destination_dir=self.manifest_destination_dir,
)
- artifacts_ap = self._build_artifacts_product()
+ if is_full_curation:
+ # find the existing artifacts tar file name
+ artifacts_ap = self._find_existing_record(self.files_found)
+ else:
+ # build new artifacts tar
+ artifacts_ap = self._build_artifacts_product()
+
if artifacts_ap not in manifest.output_group.ancillary_products:
manifest.output_group.ancillary_products.append(artifacts_ap)
manifest_file = manifest.write()
- if not self.curation_type:
- # curator doesn't need a new artifacts tar, skip it.
+ if not is_full_curation:
+ # we're running initial ingestion, make a new artifacts tar
self.write_ingestion_artifacts_tar(self.manifest_destination_dir / artifacts_ap.filename)
return manifest, manifest_file
@@ -550,6 +556,18 @@ class IngestionManifestBuilder:
AncillaryProductType.INGESTION_ARTIFACTS, IngestionManifestBuilder._build_artifacts_filename()
)
+ @staticmethod
+ def _find_existing_record(available_files: list) -> AncillaryProduct:
+ """
+ Find an existing artifacts tar for curation
+
+ :param available_files: existing files list
+ :return: Ancillary Product of existing tar
+ """
+ for file in available_files:
+ if INGESTION_ARTIFACTS_NAME in file.name:
+ return AncillaryProduct(AncillaryProductType.INGESTION_ARTIFACTS, file.name)
+
def write_ingestion_artifacts_tar(self, artifacts_path: Path) -> tarfile.TarFile:
"""
Take the list of files and write a tar file for inclusion in the archive.
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/conftest.py b/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
index 82bab5b5f..fc111f3ae 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/conftest.py
@@ -244,6 +244,9 @@ def populate_fake_tmpx_ratuqh_ingest_path(
fake_files_to_create.append(OUTPUT_SCIENCE_PRODUCT_K.filename)
fake_files_to_create.append(OUTPUT_SCIENCE_PRODUCT_X.filename)
+ if is_for_curation_test:
+ fake_files_to_create.append(INGESTION_AF_ANCILLARY.filename)
+
files = []
for filename in fake_files_to_create:
file = staging_source_dir / filename
@@ -321,9 +324,8 @@ def find_ingestion_artifacts_tar(staging_source_dir: Path):
if file.name.startswith(AncillaryProductType.INGESTION_ARTIFACTS.value) and file.name.endswith(TARFILE_EXT)
]
if len(ing_artifacts_tars) == 0:
- # curation currently has no need of artifacts tars
- # assertions handled in calling test
- return ing_artifacts_tars
+ # we're testing curation, no new tar in directory
+ return None
# otherwise ensure there is only 1 tar file present
assert len(ing_artifacts_tars) == 1
return ing_artifacts_tars[0]
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_curator_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_curator_manifest.py
index 3809f09d9..bcfc6bdd3 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_curator_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_curator_manifest.py
@@ -299,16 +299,14 @@ def test_manifest_full_curation_image(use_file_list: bool, ingest_path: Path, al
assert weblog_candidates[0].type == AncillaryProductType.PIPELINE_WEBLOG
ingest_artifacts_tar = find_ingestion_artifacts_tar(manifest_destination_dir)
- assert len(ingest_artifacts_tar) == 0
- # assert ingest_artifacts_candidates[0].type == AncillaryProductType.INGESTION_ARTIFACTS
- # """
- # The ingestion_artifacts tar should contain ONLY the ingestion manifest
- # """
- # with tarfile.open(ingest_artifacts_tar, "r") as tar:
- # members = tar.getmembers()
- # assert len(members) == 1
- # for member in members:
- # assert member.name.endswith(CURATOR_MANIFEST_FILENAME)
+ if use_file_list:
+ # there is not a file on disk, but we know it exists for this product
+ assert ingest_artifacts_tar is None
+ else:
+ # We are running with a real directory and there should be an artifacts tar available
+ ingest_artifacts_candidates = [ap for ap in aps if ap.filename == ingest_artifacts_tar.name]
+ assert len(ingest_artifacts_candidates) == 1
+ assert ingest_artifacts_candidates[0].type == AncillaryProductType.INGESTION_ARTIFACTS
# Inspect the manifest's JSON dict
"""
--
GitLab