From 42a81a28f970aaf6b28379624ef010a4b623c3c5 Mon Sep 17 00:00:00 2001
From: Janet Goldstein <jgoldste@nrao.edu>
Date: Tue, 29 Jun 2021 15:36:03 +0000
Subject: [PATCH] WS-507: Yet another rebase
---
.../ingest_envoy/ingest_envoy/classes.py | 49 --------
.../ingest_envoy/ingestion_manifest.py | 83 ++++++++++++++
.../ingestion_manifest_builder.py | 22 ++++
.../ingest_envoy/ingest_envoy/utilities.py | 105 ++++++++++++++++++
.../examples/evla_calibration_manifest.json | 30 +++++
.../test/examples/evla_eb_manifest.json | 22 ++++
.../test/examples/image_set_manifest.json | 60 ++++++++++
.../test/examples/quicklook_manifest.json | 40 +++++++
.../test/examples/vlass_catalog_manifest.json | 15 +++
.../test/test_evla_ing_manifests.py | 13 ++-
10 files changed, 389 insertions(+), 50 deletions(-)
delete mode 100644 apps/cli/executables/pexable/ingest_envoy/ingest_envoy/classes.py
create mode 100644 apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
create mode 100644 apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_builder.py
create mode 100644 apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
create mode 100644 apps/cli/executables/pexable/ingest_envoy/test/examples/evla_calibration_manifest.json
create mode 100644 apps/cli/executables/pexable/ingest_envoy/test/examples/evla_eb_manifest.json
create mode 100644 apps/cli/executables/pexable/ingest_envoy/test/examples/image_set_manifest.json
create mode 100644 apps/cli/executables/pexable/ingest_envoy/test/examples/quicklook_manifest.json
create mode 100644 apps/cli/executables/pexable/ingest_envoy/test/examples/vlass_catalog_manifest.json
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/classes.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/classes.py
deleted file mode 100644
index a9ae76670..000000000
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/classes.py
+++ /dev/null
@@ -1,49 +0,0 @@
-""" Objects pertaining to the various ingestion manifests """
-
-from enum import Enum
-
-
-class Telescope(Enum):
- """Codifying the names of our telescopes, because Janet can't abide magic strings"""
-
- VLA = 1
- EVLA = 2
- ALMA = 3
- VLBA = 4
- GBT = 5
- NONE = 6
-
-
-class IngestionType(Enum):
- """Types of ingestion we'll have to do"""
-
- # ALMA products
- ALMA_SDM = Telescope.ALMA
- ALMA_CAL = Telescope.ALMA
- ALMA_AUDI = Telescope.ALMA
-
- # EVLA products
- EVLA_SDM = Telescope.EVLA
- EVLA_BDF = Telescope.EVLA
- EVLA_CAL = Telescope.EVLA
-
- # RealFast projects
- REALFAST_SDM = Telescope.EVLA
-
- # VLASS projects
- VLASS_QUICKLOOK = Telescope.EVLA
-
- # VLBA ingestion. (IDI and UVFITS products are treated the same.)
- VLBA_FITS = Telescope.VLBA
-
- # Coming Real Soon Now: VLBA Mark 4 product ingestion
- VLBA_MARK4 = Telescope.VLBA
-
- # Also coming Real Soon: GBT execution block ingestion
- GBT_EB = Telescope.GBT
-
- # Hot on its heels: LVLA execution block ingestion
- LVLA_EB = Telescope.VLA
-
- # When we just don't know what we're dealing with
- UNKNOWN = Telescope.NONE
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
new file mode 100644
index 000000000..dc3865179
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -0,0 +1,83 @@
+""" The ingestion manifest """
+
+from pathlib import Path
+
+from .utilities import (
+ Telescope,
+ ScienceProduct,
+)
+
+
+class Parameters:
+ """a manifest's various input parameters"""
+
+ def __init__(
+ self,
+ telescope: Telescope,
+ ingestion_path: Path,
+ additional_metadata: str,
+ collection_metadata: str,
+ reingest: bool = False,
+ ngas_ingest: bool = False,
+ ):
+ self.telescope = telescope
+ self.ingestion_path = ingestion_path
+ self.additional_metadata = additional_metadata
+ self.collection_metadata = collection_metadata
+ self.reingest = reingest
+ self.ngas_ingest = ngas_ingest
+
+
+class InputGroup:
+ """
+ This represents the starting point for processing which generated a science product.
+
+ There is not always an input group for every output group (rawdata, for instance).
+
+ Initial assumption: Input groups consist only of science products.
+ """
+
+ def __init__(self):
+ self.science_products = []
+
+
+class IngestionManifest:
+ """Represents JSON layout of ingestion information, encompassing several potential scenarios.
+ see ingest_envoy/test/examples, nicked from https://open-confluence.nrao.edu/x/roPCAQ
+ """
+
+ def __init__(self, parameters: Parameters):
+ self.parameters = parameters
+
+ # to be an InputGroup
+ self.input_group = None
+ # to be an OutputGroup
+ self.output_group = None
+ # to be an AssociateGroup (not required?)
+ self.associate_group = None
+ self.science_products = []
+ self.ancillary_products = []
+
+
+class OutputGroup:
+ """Represents result of data processing"""
+
+ def __init__(self):
+ self.science_products = []
+ self.ancillary_products = []
+
+
+class AssociateGroup:
+ """
+ A representation of Science Products which are not part of the same Input or Output groups
+ but are still fundamentally linked. Created for RealFast project, to link the RealFast
+ specific execution block & image to the execution block during which a transient was
+ discovered.
+
+ Associate groups also, by definition, include any science product(s) within the output
+ group to be ingested. The new locators generated at ingestion time will be added to any
+ which compose an associate group in the manifest.
+ """
+
+ def __init__(self):
+ self.science_products = []
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_builder.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_builder.py
new file mode 100644
index 000000000..fcce3d452
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest_builder.py
@@ -0,0 +1,22 @@
+"""Build an ingestion manifest file"""
+
+from .ingestion_manifest import (
+ Parameters,
+ IngestionManifest,
+)
+
+
+class IngestionManifestBuilder:
+ """Uses supplied parameters to build ingestion manifest files
+ for the various types of ingestion"""
+
+ def __init__(self, parameters: Parameters):
+ self.parameters = parameters
+
+ def build(self) -> IngestionManifest:
+ """
+ Create the ingestion manifest indicated by the parameters.
+
+ :return: the ingestion manifest constructed from the parameters
+ """
+ raise NotImplementedError
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
new file mode 100644
index 000000000..09366f128
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
@@ -0,0 +1,105 @@
+""" Objects pertaining to the various ingestion manifests """
+
+from enum import Enum
+
+
+class Telescope(Enum):
+ """Codifying the names of our telescopes, because Janet can't abide magic strings"""
+
+ VLA = 1
+ EVLA = 2
+ ALMA = 3
+ VLBA = 4
+ GBT = 5
+ NONE = 6
+
+
+class IngestionType(Enum):
+ """Types of ingestion we'll have to do"""
+
+ # ALMA products
+ ALMA_SDM = Telescope.ALMA
+ ALMA_CAL = Telescope.ALMA
+ ALMA_AUDI = Telescope.ALMA
+
+ # EVLA products
+ EVLA_SDM = Telescope.EVLA
+ EVLA_BDF = Telescope.EVLA
+ EVLA_CAL = Telescope.EVLA
+
+ # RealFast projects
+ REALFAST_SDM = Telescope.EVLA
+
+ # VLASS projects
+ VLASS_QUICKLOOK = Telescope.EVLA
+
+ # VLBA ingestion. (IDI and UVFITS products are treated the same.)
+ VLBA_FITS = Telescope.VLBA
+
+ # Coming Real Soon Now: VLBA Mark 4 product ingestion
+ VLBA_MARK4 = Telescope.VLBA
+
+ # Also coming Real Soon: GBT execution block ingestion
+ GBT_EB = Telescope.GBT
+
+ # Hot on its heels: LVLA execution block ingestion
+ LVLA_EB = Telescope.VLA
+
+ # When we just don't know what we're dealing with
+ UNKNOWN = Telescope.NONE
+
+
+class ScienceProductType(Enum):
+ """Canonical collection of ingestible types of science products"""
+
+ EXEC_BLOCK = "execution_block"
+ CAL = "calibration"
+ CATALOG = "catalog"
+ IMAGE = "image"
+
+
+class ScienceProduct:
+ """Represents a science product in an ingestion manifest"""
+
+ def __init__(self, sp_type: ScienceProductType, filename: str, locator: str, group_with: str):
+ self.sp_type = sp_type
+ self.filename = filename
+ # product locator, used for input groups; locator for a known science product
+ self.locator = locator
+ # for "late" science products; they get added to an existing output group
+ self.group_with = group_with
+
+
+class AncillaryProductType:
+ """The various types of ancillary products we'll encounter"""
+
+ INGESTION_ARTIFACTS = "ingestion_artifacts"
+ PIPELINE_ARTIFACTS = "pipeline_artifacts"
+ PIPELINE_WEBLOG = "pipeline_weblog"
+ LOG = "log_file"
+
+ ### Images ###
+
+ # our default FITS type
+ FITS = "fits_image"
+
+ VLASS_QUICKLOOK = "quicklook_rms_image"
+ AUDI_FITS_MASK = "clean_mask"
+ AUDI_PB_FITS = "primary_beam"
+ ALPHA_FITS = "spectral_index"
+ CANDIDATE_IMG = "candidate_image"
+ THUMBNAIL_IMG = "thumbnail_image"
+
+
+class AncillaryProduct:
+ """Represents an ancillary product in an ingestion manifest"""
+
+ def __init__(
+ self, type: AncillaryProductType, filename: str, science_associate: str, group_with: str
+ ):
+ self.type = type
+ self.filename = filename
+ # make this an ancillary to a particular science product (assumes locator string)
+ self.science_associate = science_associate # TODO: enum?
+ # make this an ancillary to the group of a science product (assumes locator string)
+ self.group_with = group_with
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_calibration_manifest.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_calibration_manifest.json
new file mode 100644
index 000000000..87a5dd84c
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_calibration_manifest.json
@@ -0,0 +1,30 @@
+{
+ "parameters": {
+ "reingest": "false",
+ "ngas-ingest": "false",
+ "calibrate": "false",
+ "ingestion_path": "/lustre/...../"
+ },
+ "input-group": {
+ "science_products": [
+ {
+ "type": "execution-block",
+ "locator": "......"
+ }
+ ]
+ },
+ "output-group": {
+ "science_products": [
+ {
+ "type": "calibration",
+ "filename": "19A-321_2019......tar"
+ }
+ ],
+ "ancillary_products": [
+ {
+ "type": "weblog",
+ "filename": "weblog.tgz"
+ }
+ ]
+ }
+}
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_eb_manifest.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_eb_manifest.json
new file mode 100644
index 000000000..0a2ad890c
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/evla_eb_manifest.json
@@ -0,0 +1,22 @@
+{
+ "parameters": {
+ "reingest": false,
+ "ngas_ingest": false,
+ "telescope": "EVLA",
+ "ingestion_path": "/home/mchammer/evla/parallel-prod"
+ },
+ "output_group": {
+ "science_products": [
+ {
+ "type": "execution_block",
+ "filename": "X_osro_000.59368.65423814815"
+ }
+ ],
+ "ancillary_products": [
+ {
+ "type": "ingestion_artifacts",
+ "filename": "ingestion_artifacts_2021_06_03_T15_52_35.031.tar"
+ }
+ ]
+ }
+}
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/image_set_manifest.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/image_set_manifest.json
new file mode 100644
index 000000000..8d27af401
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/image_set_manifest.json
@@ -0,0 +1,60 @@
+{
+ "parameters": {
+ "reingest": "false",
+ "ngas-ingest": "false",
+ "calibrate": "false",
+ "ingestion_path": "/lustre/.."
+ },
+ "input-group": {
+ "science_products": [
+ {
+ "type": "calibration",
+ "locator": "...."
+ }
+ ]
+ },
+ "output-group": {
+ "science_products": [
+ {
+ "type": "image",
+ "filename": "image1.fits",
+ "ancillary-products": [
+ {
+ "type": "image_mask",
+ "filename": "image1.mask.fits"
+ }
+ ]
+ },
+ {
+ "type": "image",
+ "filename": "image2.fits",
+ "ancillary-products": [
+ {
+ "type": "image_mask",
+ "filename": "image2.mask.fits"
+ }
+ ]
+ },
+ {
+ "type": "image",
+ "filename": "image3.fits",
+ "ancillary-products": [
+ {
+ "type": "image_mask",
+ "filename": "image3.mask.fits"
+ }
+ ]
+ }
+ ],
+ "ancillary-products": [
+ {
+ "type": "weblog",
+ "filename": "weblog.tgz"
+ },
+ {
+ "type": "tar",
+ "filename": "imaging_extras.tar"
+ }
+ ]
+ }
+}
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/quicklook_manifest.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/quicklook_manifest.json
new file mode 100644
index 000000000..72a30aec2
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/quicklook_manifest.json
@@ -0,0 +1,40 @@
+{
+ "parameters": {
+ "reingest": "false",
+ "ngas-ingest": "false",
+ "calibrate": "false",
+ "ingestion_path": "/lustre/aoc/cluster/pipeline/vlass_auto/cache/quicklook/VLASS1.1/T29t04/VLASS1.1.ql.T29t04.J094850+743000.10.2048.v5"
+ },
+ "input-group": {
+ "science_products": [
+ {
+ "type": "calibration",
+ "locator": "uid://evla/calibration/a47c2e78-4f4e-4516-ab95-8bbb4057e9bb"
+ },
+ {
+ "type": "execution_block",
+ "locator": "uid://evla/execblock/52dd9e10-63fb-4fa8-b6ff-fcf6240b97f4"
+ }
+ ]
+ },
+ "output-group": {
+ "science_products": [
+ {
+ "type": "quicklook_image",
+ "filename": "VLASS1.1.ql.T29t04.J094850+743000.10.2048.v5.I.iter1.image.pbcor.tt0.subim.fits",
+ "ancillary-products": [
+ {
+ "type": "quicklook_rms",
+ "filename": "VLASS1.1.ql.T29t04.J094850+743000.10.2048.v5.I.iter1.image.pbcor.tt0.rms.subim.fits"
+ }
+ ]
+ }
+ ],
+ "ancillary-products": [
+ {
+ "type": "tar",
+ "filename": "VLASS1.1.ql.T29t04.J094850+743000.10.2048.v5.tar"
+ }
+ ]
+ }
+}
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/examples/vlass_catalog_manifest.json b/apps/cli/executables/pexable/ingest_envoy/test/examples/vlass_catalog_manifest.json
new file mode 100644
index 000000000..5306a3d5d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest_envoy/test/examples/vlass_catalog_manifest.json
@@ -0,0 +1,15 @@
+{
+ "parameters": {
+ "reingest": "false",
+ "ngas-ingest": "false",
+ "calibrate": "false",
+ "ingestion_path": "/lustre/...../"
+ },
+ "science-products": [
+ {
+ "type": "vlass_catalog",
+ "filename": "imaging_catalog_example.cat",
+ "group_with": ".... (locator for an image)"
+ }
+ ]
+}
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_ing_manifests.py b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_ing_manifests.py
index 5d6bff8d5..319074aba 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_ing_manifests.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_ing_manifests.py
@@ -1,7 +1,12 @@
""" Test for the various types of EVLA ingestion manifests """
+from pathlib import Path
import pytest
+from ..ingest_envoy.ingestion_manifest import Parameters
+from ..ingest_envoy.ingestion_manifest_builder import IngestionManifestBuilder
+from apps.cli.executables.pexable.ingest_envoy.ingest_envoy.utilities import Telescope
+
@pytest.mark.skip("TODO: test_builds_evla_sdm_manifest")
def test_builds_evla_sdm_manifest():
@@ -10,7 +15,13 @@ def test_builds_evla_sdm_manifest():
:return:
"""
- raise NotImplementedError
+ parameters = Parameters(Telescope.EVLA, Path("/path/TODO"), None, None)
+ manifest = IngestionManifestBuilder(parameters).build()
+ assert manifest.input_group is not None
+ assert manifest.output_group is not None
+ assert len(manifest.science_products) > 0
+ assert len(manifest.ancillary_products) > 0
+ # TODO: tests for manifest parameters
@pytest.mark.skip("TODO: test_builds_evla_bdf_manifest")
--
GitLab