WS-601: ancillary products are now added to EVLA CAL manifest

• bug fix: ancillary products are now added to EVLA CAL manifest
• kludges repaired

TODO next MR:

• yet more fixing of broken tests and implementation of TODO tests

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py

@@ -26,6 +26,7 @@ from ingest_envoy.manifest_components import (
     OutputGroup,
     MANIFEST_FILENAME,
     ParamsKey,
+    WEBLOG_FILENAME,
 )
 from ingest_envoy.schema import AbstractTextFile
 from ingest_envoy.std_img_manifest_utils import ImageIngestionProductsFinder
@@ -187,13 +188,25 @@ class IngestionManifestBuilder:
             input_group=self._build_input_group(),
             output_group=self._build_evla_cal_output_group(),
         )
-        manifest_file = manifest.write()
-        artifacts_tar = self.write_ingestion_artifacts_tar()
+
+        # We can't create the ingestion artifacts tar quite yet,
+        # because it will contain the yet-to-be-written manifest itself
+        # (required for ingestion, evidently)
+        artifacts_file = self.staging_source_dir / self._build_artifacts_filename()
         artifacts_ap = AncillaryProduct(
-            AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
+            AncillaryProductType.INGESTION_ARTIFACTS, filename=str(artifacts_file)
+        )
+
+        if not manifest.output_group.ancillary_products:
+            manifest.output_group.ancillary_products = []
+        weblog_ap = AncillaryProduct(
+            type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=WEBLOG_FILENAME
         )
+        manifest.output_group.ancillary_products.append(weblog_ap)
+        manifest.output_group.ancillary_products.append(artifacts_ap)
 
-        manifest.output_group.ancillary_products = [artifacts_ap]
+        manifest_file = manifest.write()
+        self.write_ingestion_artifacts_tar(artifacts_file)
 
         return manifest, manifest_file
 
@@ -215,11 +228,12 @@ class IngestionManifestBuilder:
 
         manifest_file = manifest.write()
 
-        artifacts_tar = self.write_ingestion_artifacts_tar()
+        artifacts_file = self.staging_source_dir / self._build_artifacts_filename()
         artifacts_ap = AncillaryProduct(
-            type=AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
+            type=AncillaryProductType.INGESTION_ARTIFACTS, filename=str(artifacts_file)
         )
         manifest.output_group.ancillary_products.append(artifacts_ap)
+        self.write_ingestion_artifacts_tar(artifacts_file)
 
         return manifest, manifest_file
 
@@ -267,7 +281,7 @@ class IngestionManifestBuilder:
         return OutputGroup(science_products=science_products, ancillary_products=ancillary_products)
 
     @staticmethod
-    def build_artifacts_filename() -> str:
+    def _build_artifacts_filename() -> str:
         """
         Build unique manifest filename in standard format.
 
@@ -277,7 +291,7 @@ class IngestionManifestBuilder:
         timestamp = format_timestamp(current_time)
         return f"{INGESTION_ARTIFACTS_NAME}{timestamp}{TARFILE_EXT}"
 
-    def write_ingestion_artifacts_tar(self) -> tarfile.TarFile:
+    def write_ingestion_artifacts_tar(self, artifacts_path: Path) -> tarfile.TarFile:
         """
         Take the list of files and build a tar for inclusion into the archive.
         This happens in the staging area for ingestion.
@@ -286,14 +300,14 @@ class IngestionManifestBuilder:
         :return: a .tar archive of the ingestion artifacts
         """
 
-        manifest_file = find_manifest(self.staging_source_dir)
-        ing_tar = self.staging_source_dir / self.build_artifacts_filename()
-        with tarfile.open(ing_tar, "w") as ingestion_artifacts_tar:
+        with tarfile.open(artifacts_path, "w") as ingestion_artifacts_tar:
             for file in self.files_found:
                 if self.sp_type == ScienceProductType.IMAGE:
                     ingestion_artifacts_tar.add(file)
 
-            # include the manifest
+            # The manifest file itself is considered an ingestion artifact.
+            # (It's turtles all the way down.)
+            manifest_file = self.staging_source_dir / MANIFEST_FILENAME
             ingestion_artifacts_tar.add(manifest_file)
 
         return ingestion_artifacts_tar