Skip to content
Snippets Groups Projects
Commit e284eb71 authored by Janet Goldstein's avatar Janet Goldstein
Browse files

WS-795: laying groundwork for "final" std calib. ingestion manifest

parent 274637a9
No related branches found
Tags end-of-sprint-24
1 merge request!654WS-795: laying groundwork for "final" std calib. ingestion manifest
Pipeline #3671 passed
Pipeline: workspaces

#3672

    ......@@ -30,6 +30,7 @@ MANIFEST_FILENAME = "ingestion-manifest.json"
    INGESTION_ARTIFACTS_NAME = "ingestion_artifacts_"
    TARFILE_EXT = ".tar"
    WEBLOG_FILENAME = "weblog.tgz"
    INIT_WEBLOG_FILENAME = "initial_weblog.tgz"
    SCIENCE_PRODUCT_PATTERN = re.compile("[a-zA-Z0-9._\\-+]*\\.tar")
    JSON = Union[int, float, str, List["JSON"], Dict[str, "JSON"]]
    ......@@ -197,7 +198,7 @@ class AncillaryProduct(ManifestComponentIF):
    # make this an ancillary to a particular science product (assumes locator string)
    if science_associate:
    self.science_associate = science_associate # TODO, possibly: enum?
    self.science_associate = science_associate
    # make this an ancillary to the group of a science product (assumes locator string)
    if group_with:
    self.group_with = group_with
    ......@@ -256,7 +257,7 @@ class OutputScienceProduct(ManifestComponentIF):
    return False
    def __str__(self):
    return f"{Path(self.filename).name}: {self.type.value}, " f"{len(self.ancillary_products)} ancillary products"
    return f"{Path(self.filename).name}: {self.type.value}, {len(self.ancillary_products)} ancillary products"
    def to_json(self) -> JSON:
    json_dict = {"type": self.type.value, "filename": self.filename}
    ......
    ......@@ -24,6 +24,7 @@ from typing import List
    import pytest
    from ingest_envoy.manifest_components import (
    INIT_WEBLOG_FILENAME,
    WEBLOG_FILENAME,
    AncillaryProduct,
    OutputScienceProduct,
    ......@@ -88,6 +89,20 @@ def populate_fake_evla_cal_ingest_path(staging_dir: Path) -> List[Path]:
    return files
    def populate_fake_final_evla_cal_ingest_path(staging_dir: Path) -> List[Path]:
    """
    Create a directory containing fake calibration products, plus other stuff
    that we -don't- want to ingest, PLUS an initial weblog
    :param staging_dir: our temporary dir
    :return:
    """
    files = populate_fake_evla_cal_ingest_path(staging_dir)
    init_weblog = staging_dir / "initial_weblog.tgz"
    files.append(init_weblog)
    return files
    # -----------------------------
    # Image manifest test data
    # -----------------------------
    ......@@ -112,6 +127,7 @@ OUTPUT_SCIENCE_PRODUCT = OutputScienceProduct(
    # input files
    WEBLOG_ANCILLARY = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=WEBLOG_FILENAME)
    INIT_WEBLOG_ANCILLARY = AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=INIT_WEBLOG_FILENAME)
    PIPELINE_AF_ANCILLARY = AncillaryProduct(
    type=AncillaryProductType.PIPELINE_ARTIFACTS,
    filename="pipeline_artifacts_2021_08_04T15_46_02.tar",
    ......@@ -129,7 +145,7 @@ STAGING_DIR_FILES = [
    ]
    def populate_fake_tmpx_ratuqh_ingest_path(staging_source_dir: Path) -> List[Path]:
    def populate_fake_tmpx_ratuqh_ingest_path(staging_source_dir: Path, is_final: bool = False) -> List[Path]:
    """
    make a bunch of fake files that should result in the example manifest
    ......@@ -142,6 +158,8 @@ def populate_fake_tmpx_ratuqh_ingest_path(staging_source_dir: Path) -> List[Path
    fake_files_to_create.append(PIPELINE_AF_ANCILLARY.filename)
    fake_files_to_create.append(WEBLOG_ANCILLARY.filename)
    if is_final:
    fake_files_to_create.append(INIT_WEBLOG_ANCILLARY.filename)
    fake_files_to_create.append(OUTPUT_SCIENCE_PRODUCT.filename)
    ......
    {
    "parameters": {
    "reingest": false,
    "ngas_ingest": true,
    "telescope": "EVLA",
    "ingestion_path": "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/20A-346_2021_07_23_T13_37_08.376"
    },
    "input_group": {
    "science_products": [
    {
    "locator": "uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d"
    }
    ]
    },
    "output_group": {
    "science_products": [
    {
    "type": "calibration",
    "filename": "20A-346_2021_07_23_T13_37_08.376.tar"
    }
    ],
    "ancillary_products": [
    {
    "type": "pipeline_weblog",
    "filename": "weblog.tgz"
    },
    {
    "type": "pipeline_weblog",
    "filename": "initial_weblog.tgz"
    },
    {
    "type": "ingestion_artifacts",
    "filename": "ingestion_artifacts_2021_07_27_T17_35_11.463.tar"
    }
    ]
    }
    }
    ......@@ -27,6 +27,7 @@ from pathlib import Path
    # pylint: disable=E0401, E0402, R1721, W0621
    from unittest.mock import patch
    import pytest
    from ingest_envoy.ingestion_manifest import IngestionManifestBuilder, find_manifest
    from ingest_envoy.manifest_components import (
    INGESTION_ARTIFACTS_NAME,
    ......@@ -49,6 +50,7 @@ from .conftest import (
    find_example_manifest,
    ingest_path,
    populate_fake_evla_cal_ingest_path,
    populate_fake_final_evla_cal_ingest_path,
    )
    # pylint: disable=E0401, E1120
    ......@@ -137,6 +139,19 @@ def test_writes_expected_evla_cal_output_files(ingest_path: Path):
    shutil.rmtree(ingest_path)
    @pytest.mark.skip("TODO")
    def test_writes_final_evla_cal_output(ingest_path: Path):
    """
    For a second or subsequent version of a standard calibration,
    id the manifest builder produce the manifest file, the science product tar,
    the weblog, -and- the initial weblog?
    :param ingest_path: ingestion location
    :return:
    """
    raise NotImplementedError
    def test_params_json_well_formed():
    """
    Make sure our ManifestParameters makes nice JSON
    ......@@ -337,3 +352,63 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
    assert len(actual_aps) == len(expected_aps) == 2
    shutil.rmtree(ingest_path)
    @pytest.mark.skip("TODO")
    def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
    """
    Given the correct parameters, manifest that matches evla_cal_final_manifest
    should be generated
    TODO: make this test pass
    :return:
    """
    expected_dir_name = "/lustre/aoc/cluster/pipeline/dsoc-prod/stage_products/20A-346_2021_07_23_T13_37_08.376"
    example = find_example_manifest("evla_cal_final_manifest")
    with open(example, "r") as infile:
    expected_json = dict(json.load(infile).items())
    # populate ingestion path with fake files for manifest builder to find
    populate_fake_final_evla_cal_ingest_path(ingest_path)
    builder = IngestionManifestBuilder(
    staging_source_dir=ingest_path,
    telescope=Telescope.EVLA.value,
    sp_type=ScienceProductType.EVLA_CAL.value,
    locator="uid://evla/execblock/50bb85af-ce52-49d8-b9d8-9221bfce939d",
    )
    manifest, manifest_file = builder.build()
    with open(manifest_file, "r") as infile:
    actual_json = dict(json.load(infile).items())
    actual_json[IngestionManifestKey.PARAMETERS.value][IngestionManifestKey.INGESTION_PATH.value] = expected_dir_name
    expected_params = expected_json["parameters"]
    actual_params = manifest.parameters.to_json()
    expected_reingest = "true" if expected_params["reingest"] else "false"
    assert actual_params["reingest"] == expected_reingest
    expected_ngas_ingest = "true" if expected_params["ngas_ingest"] else "false"
    assert actual_params["ngas_ingest"] == expected_ngas_ingest
    assert "calibrate" not in actual_params.keys()
    assert manifest.input_group.to_json() == expected_json["input_group"]
    expected_outgroup = expected_json["output_group"]
    expected_osp = expected_outgroup["science_products"]
    actual_osp = manifest.output_group.science_products
    assert len(actual_osp) == len(expected_osp)
    expected_aps = expected_outgroup["ancillary_products"]
    actual_aps = manifest.output_group.ancillary_products
    assert len(actual_aps) == len(expected_aps) == 3
    found_count = 0
    for ap in actual_aps:
    if ap.filename == "weblog.tgz" or ap.filename == "initial_weblog.tgz":
    found_count += 1
    assert found_count == 2
    shutil.rmtree(ingest_path)
    0% Loading or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment