Disable NGAS ingest via a capo setting

This sets ngas_ingest in the ingestion-manifest.json to the capo property archive-ingestion.ingestNGAS.

With this parameter set to false in the manifest, NGAS ingestion will be skipped by this line in manifestpersist.py:

    if ngas_ingest and len(ngas_files_to_ingest) > 0:
        ngas_archive(ngas_files_to_ingest, ingestion_path, fileset)

Example manifest when the capo setting archive-ingestion.ingestNGAS is true:

{
    "parameters": {
        "telescope": "VLBA",
        "reingest": "false",
        "ngas_ingest": "true",
        "ingestion_path": "/lustre/aoc/cluster/pipeline/docker/workspaces/staging/UG004M.ug004m"
    },
    "output_group": {
        "science_products": [
            ...
        ],
        "ancillary_products": [
            ...
        ]
    }
}

Example manifest when the capo setting archive-ingestion.ingestNGAS is false:

{
    "parameters": {
        "telescope": "VLBA",
        "reingest": "false",
        "ngas_ingest": "false",
        "ingestion_path": "/lustre/aoc/cluster/pipeline/docker/workspaces/staging/UG004M.ug004m"
    },
    "output_group": {
        "science_products": [
            ...
        ],
        "ancillary_products": [
            ...
        ]
    }
}

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py

@@ -21,6 +21,7 @@ import logging
 import sys
 import tarfile
 from pathlib import Path
+from pycapo import CapoConfig
 
 # pylint: disable=C0301, E0401, R0903, R1721
 from typing import Tuple
@@ -87,6 +88,12 @@ class IngestionManifest(ManifestComponentIF):
         self.output_group = output_group
         self.telescope = telescope
 
+        # Check if NGAS ingestion should be disabled for all manifests in this environment
+        try:
+            self.ngas_ingest = not CapoConfig().getboolean("archive-ingestion.skipNGAS")
+        except (KeyError, ValueError):
+            self.ngas_ingest = True
+
         self.parameters = self.build_ingest_parameters(additional_metadata)
         self.files_found = [file for file in self.staging_source_dir.iterdir()]
 
@@ -108,7 +115,7 @@ class IngestionManifest(ManifestComponentIF):
             params = ManifestParameters(
                 telescope=self.telescope,
                 reingest=False,
-                ngas_ingest=True,
+                ngas_ingest=self.ngas_ingest,
                 calibrate=False,
                 staging_source_dir=self.staging_source_dir,
                 additional_metadata=additional_metadata,
@@ -118,7 +125,7 @@ class IngestionManifest(ManifestComponentIF):
             params = ManifestParameters(
                 telescope=self.telescope,
                 reingest=False,
-                ngas_ingest=True,
+                ngas_ingest=self.ngas_ingest,
                 calibrate=False,
                 staging_source_dir=self.staging_source_dir,
             )
@@ -127,7 +134,7 @@ class IngestionManifest(ManifestComponentIF):
             params = ManifestParameters(
                 telescope=self.telescope,
                 reingest=False,
-                ngas_ingest=True,
+                ngas_ingest=self.ngas_ingest,
                 staging_source_dir=self.staging_source_dir,
             )
 

[Stephan Witz]

I wonder if it shouldn't just explode if it can't find the property.

[Daniel Nemergut]

Think there's a case to have this be toggleable on prod? I could require the property to be set on all environments.

[Daniel Nemergut]

I like it that way, thanks for the suggestion.