WS-795: don't add stuff more than once

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py

@@ -242,7 +242,8 @@ class IngestionManifestBuilder:
         # (required for ingestion, evidently)
         artifacts_filename = self._build_artifacts_filename()
         artifacts_ap = AncillaryProduct(AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_filename)
-        manifest.output_group.ancillary_products.append(artifacts_ap)
+        if artifacts_ap not in manifest.output_group.ancillary_products:
+            manifest.output_group.ancillary_products.append(artifacts_ap)
 
         if not manifest.output_group.ancillary_products:
             manifest.output_group.ancillary_products = []
@@ -250,13 +251,6 @@ class IngestionManifestBuilder:
         if weblog_ap not in manifest.output_group.ancillary_products:
             manifest.output_group.ancillary_products.append(weblog_ap)
 
-        # # If this isn't version 1, there should be an initial weblog from v1
-        # init_weblog = self._find_init_weblog_if_any()
-        # if init_weblog:
-        #     init_weblog_ap = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE,
-        #                                       filename=INIT_WEBLOG_FILENAME)
-        #     manifest.output_group.ancillary_products.append(init_weblog_ap)
-
         manifest_file = manifest.write()
         artifacts_file = self.staging_source_dir / artifacts_filename
         self.write_ingestion_artifacts_tar(artifacts_file)
@@ -295,7 +289,8 @@ class IngestionManifestBuilder:
 
         artifacts_file = self.staging_source_dir / self._build_artifacts_filename()
         artifacts_ap = AncillaryProduct(type=AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_file.name)
-        manifest.output_group.ancillary_products.append(artifacts_ap)
+        if artifacts_ap not in manifest.output_group.ancillary_products:
+            manifest.output_group.ancillary_products.append(artifacts_ap)
         manifest_file = manifest.write()
         self.write_ingestion_artifacts_tar(artifacts_file)
 

apps/cli/executables/pexable/ingest_envoy/test/examples/evla_cal_final_manifest.json

@@ -8,7 +8,7 @@
   "input_group": {
     "science_products": [
       {
-        "locator": "uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d"
+        "locator": "uid://evla/execblock/some-nonsense-not-in-db"
       }
     ]
   },

apps/cli/executables/pexable/ingest_envoy/test/examples/evla_cal_manifest_2021-08-02.json

@@ -8,7 +8,7 @@
   "input_group": {
     "science_products": [
       {
-        "locator": "uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d"
+        "locator": "uid://evla/execblock/i-just-play-an-eb-on-teevee"
       }
     ]
   },

apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py

@@ -150,6 +150,7 @@ def test_writes_final_evla_cal_output(ingest_path: Path):
     :param ingest_path: ingestion location
     :return:
     """
+    # TODO: implement this as soon as duplicate-weblog bug is fixed
     raise NotImplementedError
 
 
@@ -323,7 +324,7 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
         staging_source_dir=ingest_path,
         telescope=Telescope.EVLA.value,
         sp_type=ScienceProductType.EVLA_CAL.value,
-        locator="uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d",
+        locator="uid://evla/execblock/i-just-play-an-eb-on-teevee",
     )
     manifest, manifest_file = builder.build()
 
@@ -359,12 +360,59 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
     shutil.rmtree(ingest_path)
 
 
+def test_evla_cal_final_manifest_finds_init_weblog(ingest_path: Path):
+    """
+    Given a staging dir that contains an initial -and- a final weblog,
+    we expect both to be included in the ingestion manifest.
+
+    :param ingest_path:  ingestion staging dir
+    :return:
+    """
+    # populate ingestion path with fake files for manifest builder to find
+    populate_fake_final_evla_cal_ingest_path(ingest_path)
+
+    weblogs = []
+    for file in [file for file in ingest_path.iterdir() if file.is_file()]:
+        if "weblog.tgz" in file.name:
+            weblogs.append(file.name)
+
+    # be sure both files we expect are there...
+    assert WEBLOG_FILENAME in weblogs and INIT_WEBLOG_FILENAME in weblogs
+    # and each of them only once
+    assert len(weblogs) == 2
+
+    builder = IngestionManifestBuilder(
+        staging_source_dir=ingest_path,
+        telescope=Telescope.EVLA.value,
+        sp_type=ScienceProductType.EVLA_CAL.value,
+        locator="uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d",
+    )
+    manifest, manifest_file = builder.build()
+
+    with open(manifest_file, "r") as infile:
+        actual_json = dict(json.load(infile).items())
+        og = actual_json["output_group"]
+        anc_prods = og["ancillary_products"]
+        assert len(anc_prods) >= 2
+
+        weblog_filenames = []
+        for ap in anc_prods:
+            logger.info(f"{ap}")
+            assert "filename" in ap.keys()
+            if "weblog" in ap["filename"]:
+                weblog_filenames.append(ap["filename"])
+
+        assert WEBLOG_FILENAME in weblog_filenames and INIT_WEBLOG_FILENAME in weblog_filenames
+        assert len(weblog_filenames) == 2
+
+
 def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
     """
     Given the correct parameters, manifest that matches evla_cal_final_manifest
     should be generated; i.e., ancillary)products should contain initial_weblog.tgz
     in addition to weblog.tgz
 
+    :param ingest_path: ingestion staging dir
     :return:
     """
 
@@ -380,7 +428,7 @@ def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
         staging_source_dir=ingest_path,
         telescope=Telescope.EVLA.value,
         sp_type=ScienceProductType.EVLA_CAL.value,
-        locator="uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d",
+        locator="uid://evla/execblock/some-nonsense-not-in-db",
     )
     manifest, manifest_file = builder.build()
 
@@ -403,8 +451,8 @@ def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
     expected_outgroup = expected_json["output_group"]
     expected_osp = expected_outgroup["science_products"]
     actual_osp = manifest.output_group.science_products
-    assert len(actual_osp) == len(expected_osp)
-    logger.info(">>> SCIENCE PRODUCTS:")
+    assert len(actual_osp) == len(expected_osp) == 1
+
     for sp in actual_osp:
         logger.info(f"{sp.filename}")
         if sp.ancillary_products is not None:
@@ -414,7 +462,7 @@ def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
     expected_aps = expected_outgroup["ancillary_products"]
     assert len(expected_aps) == 3
     actual_aps = manifest.output_group.ancillary_products
-    logger.info(">>> ACTUAL ANCILLARY PRODUCTS")
+    logger.info(">>> ancillary products")
     for ap in actual_aps:
         logger.info(f"{ap.filename}: {ap.type}, group with {ap.group_with}")
     assert len(actual_aps) == len(expected_aps)