WS-795: laying groundwork for "final" std calib. ingestion manifest

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py

@@ -30,6 +30,7 @@ MANIFEST_FILENAME = "ingestion-manifest.json"
 INGESTION_ARTIFACTS_NAME = "ingestion_artifacts_"
 TARFILE_EXT = ".tar"
 WEBLOG_FILENAME = "weblog.tgz"
+INIT_WEBLOG_FILENAME = "initial_weblog.tgz"
 SCIENCE_PRODUCT_PATTERN = re.compile("[a-zA-Z0-9._\\-+]*\\.tar")
 JSON = Union[int, float, str, List["JSON"], Dict[str, "JSON"]]
 
@@ -197,7 +198,7 @@ class AncillaryProduct(ManifestComponentIF):
 
         # make this an ancillary to a particular science product (assumes locator string)
         if science_associate:
-            self.science_associate = science_associate  # TODO, possibly: enum?
+            self.science_associate = science_associate
         # make this an ancillary to the group of a science product (assumes locator string)
         if group_with:
             self.group_with = group_with
@@ -256,7 +257,7 @@ class OutputScienceProduct(ManifestComponentIF):
         return False
 
     def __str__(self):
-        return f"{Path(self.filename).name}: {self.type.value}, " f"{len(self.ancillary_products)} ancillary products"
+        return f"{Path(self.filename).name}: {self.type.value}, {len(self.ancillary_products)} ancillary products"
 
     def to_json(self) -> JSON:
         json_dict = {"type": self.type.value, "filename": self.filename}

apps/cli/executables/pexable/ingest_envoy/test/conftest.py

@@ -24,6 +24,7 @@ from typing import List
 
 import pytest
 from ingest_envoy.manifest_components import (
+    INIT_WEBLOG_FILENAME,
     WEBLOG_FILENAME,
     AncillaryProduct,
     OutputScienceProduct,
@@ -88,6 +89,20 @@ def populate_fake_evla_cal_ingest_path(staging_dir: Path) -> List[Path]:
     return files
 
 
+def populate_fake_final_evla_cal_ingest_path(staging_dir: Path) -> List[Path]:
+    """
+    Create a directory containing fake calibration products, plus other stuff
+    that we -don't- want to ingest, PLUS an initial weblog
+
+    :param staging_dir: our temporary dir
+    :return:
+    """
+    files = populate_fake_evla_cal_ingest_path(staging_dir)
+    init_weblog = staging_dir / "initial_weblog.tgz"
+    files.append(init_weblog)
+    return files
+
+
 # -----------------------------
 #    Image manifest test data
 # -----------------------------
@@ -112,6 +127,7 @@ OUTPUT_SCIENCE_PRODUCT = OutputScienceProduct(
 
 # input files
 WEBLOG_ANCILLARY = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=WEBLOG_FILENAME)
+INIT_WEBLOG_ANCILLARY = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=INIT_WEBLOG_FILENAME)
 PIPELINE_AF_ANCILLARY = AncillaryProduct(
     type=AncillaryProductType.PIPELINE_ARTIFACTS,
     filename="pipeline_artifacts_2021_08_04T15_46_02.tar",
@@ -129,7 +145,7 @@ STAGING_DIR_FILES = [
 ]
 
 
-def populate_fake_tmpx_ratuqh_ingest_path(staging_source_dir: Path) -> List[Path]:
+def populate_fake_tmpx_ratuqh_ingest_path(staging_source_dir: Path, is_final: bool = False) -> List[Path]:
     """
     make a bunch of fake files that should result in the example manifest
 
@@ -142,6 +158,8 @@ def populate_fake_tmpx_ratuqh_ingest_path(staging_source_dir: Path) -> List[Path
 
     fake_files_to_create.append(PIPELINE_AF_ANCILLARY.filename)
     fake_files_to_create.append(WEBLOG_ANCILLARY.filename)
+    if is_final:
+        fake_files_to_create.append(INIT_WEBLOG_ANCILLARY.filename)
 
     fake_files_to_create.append(OUTPUT_SCIENCE_PRODUCT.filename)
 

apps/cli/executables/pexable/ingest_envoy/test/examples/evla_cal_final_manifest.json

+{
+  "parameters": {
+    "reingest": false,
+    "ngas_ingest": true,
+    "telescope": "EVLA",
+    "ingestion_path": "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/20A-346_2021_07_23_T13_37_08.376"
+  },
+  "input_group": {
+    "science_products": [
+      {
+        "locator": "uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d"
+      }
+    ]
+  },
+  "output_group": {
+    "science_products": [
+      {
+        "type": "calibration",
+        "filename": "20A-346_2021_07_23_T13_37_08.376.tar"
+      }
+    ],
+    "ancillary_products": [
+      {
+        "type": "pipeline_weblog",
+        "filename": "weblog.tgz"
+      },
+      {
+        "type": "pipeline_weblog",
+        "filename": "initial_weblog.tgz"
+      },
+      {
+        "type": "ingestion_artifacts",
+        "filename": "ingestion_artifacts_2021_07_27_T17_35_11.463.tar"
+      }
+    ]
+  }
+}

apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py

@@ -27,6 +27,7 @@ from pathlib import Path
 # pylint: disable=E0401, E0402, R1721, W0621
 from unittest.mock import patch
 
+import pytest
 from ingest_envoy.ingestion_manifest import IngestionManifestBuilder, find_manifest
 from ingest_envoy.manifest_components import (
     INGESTION_ARTIFACTS_NAME,
@@ -49,6 +50,7 @@ from .conftest import (
     find_example_manifest,
     ingest_path,
     populate_fake_evla_cal_ingest_path,
+    populate_fake_final_evla_cal_ingest_path,
 )
 
 # pylint: disable=E0401, E1120
@@ -137,6 +139,19 @@ def test_writes_expected_evla_cal_output_files(ingest_path: Path):
     shutil.rmtree(ingest_path)
 
 
+@pytest.mark.skip("TODO")
+def test_writes_final_evla_cal_output(ingest_path: Path):
+    """
+    For a second or subsequent version of a standard calibration,
+    id the manifest builder produce the manifest file, the science product tar,
+    the weblog, -and- the initial weblog?
+
+    :param ingest_path: ingestion location
+    :return:
+    """
+    raise NotImplementedError
+
+
 def test_params_json_well_formed():
     """
     Make sure our ManifestParameters makes nice JSON
@@ -337,3 +352,63 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
     assert len(actual_aps) == len(expected_aps) == 2
 
     shutil.rmtree(ingest_path)
+
+
+@pytest.mark.skip("TODO")
+def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
+    """
+    Given the correct parameters, manifest that matches evla_cal_final_manifest
+    should be generated
+    TODO: make this test pass
+
+    :return:
+    """
+
+    expected_dir_name = "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/20A-346_2021_07_23_T13_37_08.376"
+    example = find_example_manifest("evla_cal_final_manifest")
+    with open(example, "r") as infile:
+        expected_json = dict(json.load(infile).items())
+
+    # populate ingestion path with fake files for manifest builder to find
+    populate_fake_final_evla_cal_ingest_path(ingest_path)
+
+    builder = IngestionManifestBuilder(
+        staging_source_dir=ingest_path,
+        telescope=Telescope.EVLA.value,
+        sp_type=ScienceProductType.EVLA_CAL.value,
+        locator="uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d",
+    )
+    manifest, manifest_file = builder.build()
+
+    with open(manifest_file, "r") as infile:
+        actual_json = dict(json.load(infile).items())
+
+    actual_json[IngestionManifestKey.PARAMETERS.value][IngestionManifestKey.INGESTION_PATH.value] = expected_dir_name
+
+    expected_params = expected_json["parameters"]
+    actual_params = manifest.parameters.to_json()
+
+    expected_reingest = "true" if expected_params["reingest"] else "false"
+    assert actual_params["reingest"] == expected_reingest
+    expected_ngas_ingest = "true" if expected_params["ngas_ingest"] else "false"
+    assert actual_params["ngas_ingest"] == expected_ngas_ingest
+
+    assert "calibrate" not in actual_params.keys()
+    assert manifest.input_group.to_json() == expected_json["input_group"]
+
+    expected_outgroup = expected_json["output_group"]
+    expected_osp = expected_outgroup["science_products"]
+    actual_osp = manifest.output_group.science_products
+    assert len(actual_osp) == len(expected_osp)
+
+    expected_aps = expected_outgroup["ancillary_products"]
+    actual_aps = manifest.output_group.ancillary_products
+    assert len(actual_aps) == len(expected_aps) == 3
+
+    found_count = 0
+    for ap in actual_aps:
+        if ap.filename == "weblog.tgz" or ap.filename == "initial_weblog.tgz":
+            found_count += 1
+    assert found_count == 2
+
+    shutil.rmtree(ingest_path)