Added failing tests for creating full-curation manifest

Added passing test for ReingestGroup's dump-ability

apps/cli/executables/pexable/ingest_envoy/test/conftest.py

@@ -19,6 +19,7 @@
 
 # pylint: disable=E0401, R1721
 
+from os import mkdir
 from pathlib import Path
 from typing import List
 
@@ -121,16 +122,17 @@ CLEAN_MASK_ANCILLARY = AncillaryProduct(
 ALPHA_ERROR = AncillaryProduct(
     type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.K_band.cont.I.alpha.error.fits"
 )
-ALPHA = AncillaryProduct(
-    type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.K_band.cont.I.alpha.fits"
-)
-TT0 = AncillaryProduct(
-    type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.K_band.cont.I.tt0.fits"
-)
-TT1 = AncillaryProduct(
-    type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.K_band.cont.I.tt1.fits"
-)
-OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_K = [PRIMARY_BEAM_ANCILLARY, CLEAN_MASK_ANCILLARY, ALPHA_ERROR, ALPHA, TT0, TT1]
+ALPHA = AncillaryProduct(type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.K_band.cont.I.alpha.fits")
+TT0 = AncillaryProduct(type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.K_band.cont.I.tt0.fits")
+TT1 = AncillaryProduct(type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.K_band.cont.I.tt1.fits")
+OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_K = [
+    PRIMARY_BEAM_ANCILLARY,
+    CLEAN_MASK_ANCILLARY,
+    ALPHA_ERROR,
+    ALPHA,
+    TT0,
+    TT1,
+]
 
 OUTPUT_SCIENCE_PRODUCT_K = OutputScienceProduct(
     product_type=AncillaryProductType.FITS,
@@ -147,16 +149,17 @@ CLEAN_MASK_ANCILLARY = AncillaryProduct(
 ALPHA_ERROR = AncillaryProduct(
     type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.X_band.cont.I.alpha.error.fits"
 )
-ALPHA = AncillaryProduct(
-    type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.X_band.cont.I.alpha.fits"
-)
-TT0 = AncillaryProduct(
-    type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.X_band.cont.I.tt0.fits"
-)
-TT1 = AncillaryProduct(
-    type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.X_band.cont.I.tt1.fits"
-)
-OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_X = [PRIMARY_BEAM_ANCILLARY, CLEAN_MASK_ANCILLARY, ALPHA_ERROR, ALPHA, TT0, TT1]
+ALPHA = AncillaryProduct(type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.X_band.cont.I.alpha.fits")
+TT0 = AncillaryProduct(type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.X_band.cont.I.tt0.fits")
+TT1 = AncillaryProduct(type=AncillaryProductType.FITS, filename="oussid.J1522+3934_sci.X_band.cont.I.tt1.fits")
+OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_X = [
+    PRIMARY_BEAM_ANCILLARY,
+    CLEAN_MASK_ANCILLARY,
+    ALPHA_ERROR,
+    ALPHA,
+    TT0,
+    TT1,
+]
 
 OUTPUT_SCIENCE_PRODUCT_X = OutputScienceProduct(
     product_type=AncillaryProductType.FITS,
@@ -217,3 +220,55 @@ def populate_fake_tmpx_ratuqh_ingest_path(staging_source_dir: Path, is_final: bo
     assert len(files) == len(fake_files_to_create)
 
     return files
+
+
+# -----------------------------
+# SDM manifest test data
+# -----------------------------
+SDM_FILE_LIST = [
+    "Antenna.xml",
+    "ASDM.xml",
+    "CalData.xml",
+    "CalDevice.xml",
+    "CalPointing.xml",
+    "CalReduction.xml",
+    "ConfigDescription.xml",
+    "CorrelatorMode.xml",
+    "DataDescription.xml",
+    "Doppler.xml",
+    "Ephemeris.xml",
+    "ExecBlock.xml",
+    "Feed.xml",
+    "Field.xml",
+    "Flag.xml",
+    "Main.xml",
+    "Pointing.xml",
+    "PointingModel.xml",
+    "Polarization.xml",
+    "Processor.xml",
+    "Receiver.xml",
+    "SBSummary.xml",
+    "Scan.xml",
+    "Source.xml",
+    "SpectralWindow.xml",
+    "State.xml",
+    "Station.xml",
+    "Subscan.xml",
+    "SwitchCycle.xml",
+    "SysCal.xml",
+    "SysPower.bin",
+    "Weather.xml",
+]
+# From file ./examples/full_curation_evla_eb_manifest.json
+EVLA_EB_NAME = "19A-001.sb1234567890.eb233423545632.54321.894327984569"
+EVLA_EB_LOCATOR = "uid://I/am/a/locator"
+
+
+def populate_fake_evla_eb_curator_source_path(staging_dir: Path) -> list[Path]:
+    eb_dir = staging_dir / EVLA_EB_NAME
+    eb_dir.mkdir()
+    for sdm_filename in SDM_FILE_LIST:
+        sdm_file_path = eb_dir / sdm_filename
+        sdm_file_path.touch()
+    # Only really care about the directory for the manifest
+    return [eb_dir]

apps/cli/executables/pexable/ingest_envoy/test/examples/full_curation_evla_eb_manifest.json

+{
+  "parameters": {
+    "ngas_ingest": false,
+    "telescope": "EVLA",
+    "ingestion_path": "/lustre/aoc/cluster/pipeline/dsoc-prod/workspaces/spool/tmp8gfknlo9/19A-001/observation.54321.894327984569"
+  },
+  "reingest": {
+    "locator": "uid://I/am/a/locator",
+    "type": "execution_block"
+  },
+  "output_group": {
+    "science_products": [
+      {
+        "type": "execution_block",
+        "filename": "19A-001.sb1234567890.eb233423545632.54321.894327984569"
+      }
+    ],
+    "ancillary_products": [
+      {
+        "type": "ingestion_artifacts",
+        "filename": "ingestion_artifacts_2019_07_26_T10_49_44.890.tar"
+      }
+    ]
+  }
+}

apps/cli/executables/pexable/ingest_envoy/test/test_curator_manifest.py

+# Copyright (C) 2023 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+#
+import json
+import shutil
+import tarfile
+from pathlib import Path
+
+from conftest import (
+    EVLA_EB_LOCATOR,
+    EVLA_EB_NAME,
+    find_example_manifest,
+    populate_fake_evla_eb_curator_source_path,
+)
+from ingest_envoy.ingestion_manifest import IngestionManifestBuilder
+from ingest_envoy.manifest_components import (
+    CURATOR_MANIFEST_FILENAME,
+    INGESTION_ARTIFACTS_NAME,
+    IngestionManifestKey,
+    ParamsKey,
+    ReingestGroup,
+)
+from ingest_envoy.utilities import (
+    AncillaryProductType,
+    CuratorType,
+    ScienceProductType,
+    Telescope,
+)
+
+
+def test_manifest_full_curation_eb_manifest_metadata(ingest_path: Path):
+    """Test manifest creation
+    :param ingest_path: directory where curator's sources will be
+    """
+    eb_files = populate_fake_evla_eb_curator_source_path(ingest_path)
+
+    try:
+        assert len(eb_files) == 1
+
+        actual_manifest, _ = IngestionManifestBuilder(
+            telescope=Telescope.EVLA.value,
+            staging_source_dir=None,
+            sp_type=ScienceProductType.EXEC_BLOCK.value,
+            locator=EVLA_EB_LOCATOR,
+            filename=EVLA_EB_NAME,
+            curate=(CuratorType.FULL, str(ingest_path), None),
+        ).build()
+
+        params = actual_manifest.parameters
+        assert params.ngas_ingest is False
+        assert params.telescope == Telescope.EVLA
+        assert params.staging_source_dir == ingest_path
+        assert not hasattr(params, "calibrate")
+
+        assert not actual_manifest.input_group
+
+        output_group = actual_manifest.output_group
+        assert output_group
+        assert len(output_group.science_products) == 1
+        assert output_group.science_products[0].filename == EVLA_EB_NAME
+        assert len(output_group.ancillary_products) == 1
+        assert INGESTION_ARTIFACTS_NAME in output_group.ancillary_products[0].filename
+
+        reingest = actual_manifest.reingest
+        assert reingest
+        assert reingest.targets == None
+        assert reingest.locator == EVLA_EB_LOCATOR
+        assert reingest.product_type == ScienceProductType.EXEC_BLOCK
+    finally:
+        shutil.rmtree(ingest_path)
+
+
+def test_manifest_full_curation_eb_manifest_file(ingest_path: Path):
+    """Test serialization of manifest as well as its creation
+    :param ingest_path: directory where curator's sources will be
+    """
+
+    populate_fake_evla_eb_curator_source_path(ingest_path)
+
+    try:
+        expected_manifest_file = find_example_manifest("full_curation_evla_eb_manifest")
+        with open(expected_manifest_file) as f:
+            expected_manifest = json.load(f)
+
+        actual_manifest, actual_manifest_file = IngestionManifestBuilder(
+            telescope=Telescope.EVLA.value,
+            staging_source_dir=None,
+            sp_type=ScienceProductType.EXEC_BLOCK.value,
+            locator=EVLA_EB_LOCATOR,
+            filename=EVLA_EB_NAME,
+            curate=(CuratorType.FULL, str(ingest_path), None),
+        ).build()
+        with open(actual_manifest_file) as f:
+            actual_manifest_deser = json.load(f)
+
+        assert actual_manifest.to_dict() == actual_manifest_deser
+
+        # ingestion_path depends on the ingest_path fixture, so ignore it
+        expected_params = expected_manifest["parameters"]
+        actual_params = actual_manifest_deser["parameters"]
+        assert actual_params["ngas_ingest"] == expected_params["ngas_ingest"]
+        assert not actual_manifest_deser.get("input_group") and not expected_manifest.get("input_group")
+
+        expected_outgroup = expected_manifest["output_group"]
+        expected_osp = expected_outgroup["science_products"]
+        actual_outgroup = actual_manifest_deser["output_group"]
+        actual_osp = actual_outgroup["science_products"]
+        assert actual_osp == expected_osp
+
+        # the ingestion_artifacts' filename depends on current time, so ignore it
+        expected_aps = expected_outgroup["ancillary_products"]
+        actual_aps = actual_outgroup["ancillary_products"]
+        assert len(expected_aps) == 1
+        assert len(expected_aps) == len(actual_aps)
+        assert expected_aps[0]["type"] == AncillaryProductType.INGESTION_ARTIFACTS.value
+        assert expected_aps[0]["type"] == actual_aps[0]["type"]
+
+        # get ingestion artifacts
+        artifacts_file = [file for file in ingest_path.glob("ingestion_artifacts*.tar")][0]
+        with tarfile.open(artifacts_file, "r") as tar:
+            # confirm that contains has as many files as we expect...
+            members = tar.getmembers()
+
+        assert len(members) == 1
+        member = members[0]
+        mf_path = Path(member.name)
+        assert mf_path.name == CURATOR_MANIFEST_FILENAME
+    finally:
+        # Clean up
+        shutil.rmtree(ingest_path)
+
+
+def test_reingest_block_json_well_formed_with_targets():
+    """
+    Make sure our ReingestGroup makes nice JSON when given targets
+    """
+    reingest = ReingestGroup(EVLA_EB_LOCATOR, ScienceProductType.EXEC_BLOCK, ["subscans.ra", "subscans.dec"])
+    reingest_json = reingest.to_dict()
+    # if we can dump it, it's good
+    json.dumps(reingest_json)
+
+
+def test_reingest_block_json_well_formed_no_targets():
+    """
+    Make sure our ReingestGroup makes nice JSON without targets.
+
+    Curator requires the "targets" field to be None, not [], in order to trigger full curation.
+    """
+    reingest = ReingestGroup(EVLA_EB_LOCATOR, ScienceProductType.EXEC_BLOCK, None)
+    reingest_json = reingest.to_dict()
+    # if we can dump it, it's good
+    json.dumps(reingest_json)