From c6420a51e379e4c0a4d8fd8fe616fba54d13bf58 Mon Sep 17 00:00:00 2001
From: chausman <chausman@nrao.edu>
Date: Tue, 28 May 2024 15:08:58 -0600
Subject: [PATCH] - add selfcal tar as image group ancillary when present -
updates for vela local testing file forging
---
.../ingest_envoy/ingest_envoy/launchers.py | 5 ++---
.../ingest_envoy/manifest_components.py | 1 +
.../ingest_envoy/std_img_manifest_utils.py | 9 +++++++++
.../ingest_envoy/ingest_envoy/utilities.py | 3 +++
.../cli/executables/pexable/vela/vela/forger.py | 17 +++++++++++++++++
.../image-product-collector.sh | 11 +++++++++--
6 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
index 1e967c330..bb2238ae9 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
@@ -162,7 +162,7 @@ class IngestCalibrationLauncher(LauncherIF):
telescope = self.parameters["telescope"]
IngestionManifestBuilder(
- staging_source_dir=Path(self.staging_source_dir),
+ manifest_source_dir=Path(self.staging_source_dir),
sp_type=self.sci_product_type,
locator=spl,
telescope=telescope,
@@ -290,9 +290,8 @@ class IngestObservationLauncher(LauncherIF):
self.logger.info("Creating ingestion manifest...")
telescope = self.parameters["telescope"]
filename = self.parameters["filename"]
-
IngestionManifestBuilder(
- staging_source_dir=Path(self.staging_source_dir),
+ manifest_source_dir=Path(self.staging_source_dir),
sp_type=self.sci_product_type,
locator=None,
telescope=telescope,
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
index 6710efd3b..62713b74c 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
@@ -35,6 +35,7 @@ WEBLOG_FILENAME = "weblog.tgz"
INIT_WEBLOG_FILENAME = "initial_weblog.tgz"
REIMAGING_FILENAME = "reimaging_resources.tgz"
PARAMETER_FILENAME = "SEIP_parameter.list"
+SELFCAL_AUX_FILENAME = "selfcal.auxproducts.tgz"
SCIENCE_PRODUCT_PATTERN = re.compile("[a-zA-Z0-9._\\-+]*\\.tar")
JSON = Union[int, float, str, List["JSON"], Dict[str, "JSON"]]
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py
index 4dbc3468a..65701bb2e 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py
@@ -24,6 +24,7 @@ from typing import List
from ingest_envoy.manifest_components import (
PARAMETER_FILENAME,
REIMAGING_FILENAME,
+ SELFCAL_AUX_FILENAME,
TARFILE_EXT,
WEBLOG_FILENAME,
AncillaryProduct,
@@ -130,6 +131,7 @@ class ImageIngestionProductsFinder:
except IndexError as err:
raise FileNotFoundError(f"WARNING: No pipeline artifacts found in {self.files_found}") from err
+ # for images both the weblog and pipeline artifacts must exist, these from the group ancillary base
ancillary_products = [
AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG, filename=WEBLOG_FILENAME),
AncillaryProduct(type=AncillaryProductType.PIPELINE_ARTIFACTS, filename=pipeline_artifacts_tar.name),
@@ -144,6 +146,13 @@ class ImageIngestionProductsFinder:
AncillaryProduct(type=AncillaryProductType.PARAMETER_LIST, filename=PARAMETER_FILENAME)
)
+ # if selfcal completed successfully, ingest the resulting auxproducts tar
+ selfcal_auxproducts_tar = [file for file in self.files_found if file.name.endswith(SELFCAL_AUX_FILENAME)]
+ if selfcal_auxproducts_tar:
+ ancillary_products.append(
+ AncillaryProduct(type=AncillaryProductType.AUX_PRODUCTS, filename=selfcal_auxproducts_tar[0].name)
+ )
+
return ancillary_products
@staticmethod
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
index 93717f60d..34c620a44 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py
@@ -97,6 +97,9 @@ class AncillaryProductType(Enum):
COMPONENT_LIST = "component_list"
OBSERVATION_DETAILS = "observation_details"
+ # Image extras
+ AUX_PRODUCTS = "pipeline_aux_products"
+
class InvalidLocatorException(Exception):
"""Throw this if we're fed a bad science product locator"""
diff --git a/apps/cli/executables/pexable/vela/vela/forger.py b/apps/cli/executables/pexable/vela/vela/forger.py
index 03cdbaff5..8bb372040 100644
--- a/apps/cli/executables/pexable/vela/vela/forger.py
+++ b/apps/cli/executables/pexable/vela/vela/forger.py
@@ -118,13 +118,30 @@ class VelaProduct:
os.chdir("../")
def img_products(self, parameters: dict):
+ """
+ Forge a variety of image files for ingestion testing
+
+ :param parameters:
+ :return:
+ """
path = Path.cwd() / "products"
os.chdir(path)
+ # pbcor.tt0 is the primary science product
+ pbcor_filename = "oussid.1-93305_sci.L_band.cont.I.pbcor.tt0.fits"
+ selfcal_pbcor = "oussid.1-93305_sci.S_band.cont.selfcal.I.pbcor.tt0.fits"
pb_filename = "oussid.1-93305_sci.L_band.cont.I.pb.tt0.fits"
cont_filename = "oussid.1-93305_sci.L_band.cont.I.tt0.fits"
mask_filename = "oussid.1-93305_sci.L_band.cont.I.mask.fits"
+ fits = open(pbcor_filename, "x")
+ fits.write("I am a primary science image.")
+ fits.close()
+
+ fits = open(selfcal_pbcor, "x")
+ fits.write("I am a selfcal image.")
+ fits.close()
+
fits = open(pb_filename, "x")
fits.write("I am a primary beam image.")
fits.close()
diff --git a/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh b/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh
index 1790a3d65..21fa1945f 100644
--- a/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh
+++ b/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh
@@ -88,6 +88,13 @@ fi
# NOTE: This fails if we have no weblog, previously we allowed that
# possibility.
+SELFCAL=$(ls -t ${SOURCE_DIR}/*selfcal.auxproducts.tgz)
+
+if [[ -n "$SELFCAL" ]]
+then
+ /bin/cp -f ${SOURCE_DIR}/*selfcal.auxproducts.tgz ${STAGE_DIR}/${WORKFLOW_DIR}
+fi
+
#
# Prepare the artifacts file, we'll collect everything to be ingested in a directory.
#
@@ -98,9 +105,9 @@ then
fi
# tar a subset of files into a tar archive in the storage path
-# Skipped: FITS files, Weblog (zipped and unzipped)
+# Skipped: FITS files, Weblog (zipped and unzipped), selfcal auxproducts
#
-tar --exclude=\*.fits --exclude=pipeline-\* --exclude=*weblog.tgz -C${SOURCE_DIR} -cvf ${STAGE_DIR}/${WORKFLOW_DIR}/${FILENAME}.tar .
+tar --exclude=\*.fits --exclude=pipeline-\* --exclude=*weblog.tgz --exclude=*selfcal.auxproducts.tgz -C${SOURCE_DIR} -cvf ${STAGE_DIR}/${WORKFLOW_DIR}/${FILENAME}.tar .
if [[ $? -ne 0 ]]
then
echo "Creation of pipeline artifacts tar file failed, exiting"
--
GitLab