- add selfcal tar as image group ancillary when present

- updates for vela local testing file forging

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py

@@ -35,6 +35,7 @@ WEBLOG_FILENAME = "weblog.tgz"
 INIT_WEBLOG_FILENAME = "initial_weblog.tgz"
 REIMAGING_FILENAME = "reimaging_resources.tgz"
 PARAMETER_FILENAME = "SEIP_parameter.list"
+SELFCAL_AUX_FILENAME = "selfcal.auxproducts.tgz"
 SCIENCE_PRODUCT_PATTERN = re.compile("[a-zA-Z0-9._\\-+]*\\.tar")
 JSON = Union[int, float, str, List["JSON"], Dict[str, "JSON"]]
 

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py

@@ -24,6 +24,7 @@ from typing import List
 from ingest_envoy.manifest_components import (
     PARAMETER_FILENAME,
     REIMAGING_FILENAME,
+    SELFCAL_AUX_FILENAME,
     TARFILE_EXT,
     WEBLOG_FILENAME,
     AncillaryProduct,
@@ -130,6 +131,7 @@ class ImageIngestionProductsFinder:
         except IndexError as err:
             raise FileNotFoundError(f"WARNING: No pipeline artifacts found in {self.files_found}") from err
 
+        # for images both the weblog and pipeline artifacts must exist, these from the group ancillary base
         ancillary_products = [
             AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG, filename=WEBLOG_FILENAME),
             AncillaryProduct(type=AncillaryProductType.PIPELINE_ARTIFACTS, filename=pipeline_artifacts_tar.name),
@@ -144,6 +146,13 @@ class ImageIngestionProductsFinder:
                 AncillaryProduct(type=AncillaryProductType.PARAMETER_LIST, filename=PARAMETER_FILENAME)
             )
 
+        # if selfcal completed successfully, ingest the resulting auxproducts tar
+        selfcal_auxproducts_tar = [file for file in self.files_found if file.name.endswith(SELFCAL_AUX_FILENAME)]
+        if selfcal_auxproducts_tar:
+            ancillary_products.append(
+                AncillaryProduct(type=AncillaryProductType.AUX_PRODUCTS, filename=selfcal_auxproducts_tar[0].name)
+            )
+
         return ancillary_products
 
     @staticmethod

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py

@@ -97,6 +97,9 @@ class AncillaryProductType(Enum):
     COMPONENT_LIST = "component_list"
     OBSERVATION_DETAILS = "observation_details"
 
+    # Image extras
+    AUX_PRODUCTS = "pipeline_aux_products"
+
 
 class InvalidLocatorException(Exception):
     """Throw this if we're fed a bad science product locator"""

apps/cli/executables/pexable/vela/vela/forger.py

@@ -118,13 +118,30 @@ class VelaProduct:
         os.chdir("../")
 
     def img_products(self, parameters: dict):
+        """
+        Forge a variety of image files for ingestion testing
+
+        :param parameters:
+        :return:
+        """
         path = Path.cwd() / "products"
         os.chdir(path)
 
+        # pbcor.tt0 is the primary science product
+        pbcor_filename = "oussid.1-93305_sci.L_band.cont.I.pbcor.tt0.fits"
+        selfcal_pbcor = "oussid.1-93305_sci.S_band.cont.selfcal.I.pbcor.tt0.fits"
         pb_filename = "oussid.1-93305_sci.L_band.cont.I.pb.tt0.fits"
         cont_filename = "oussid.1-93305_sci.L_band.cont.I.tt0.fits"
         mask_filename = "oussid.1-93305_sci.L_band.cont.I.mask.fits"
 
+        fits = open(pbcor_filename, "x")
+        fits.write("I am a primary science image.")
+        fits.close()
+
+        fits = open(selfcal_pbcor, "x")
+        fits.write("I am a selfcal image.")
+        fits.close()
+
         fits = open(pb_filename, "x")
         fits.write("I am a primary beam image.")
         fits.close()

apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh

@@ -88,6 +88,13 @@ fi
 # NOTE: This fails if we have no weblog, previously we allowed that
 #       possibility.
 
+SELFCAL=$(ls -t ${SOURCE_DIR}/*selfcal.auxproducts.tgz)
+
+if [[ -n "$SELFCAL" ]]
+then
+  /bin/cp -f ${SOURCE_DIR}/*selfcal.auxproducts.tgz ${STAGE_DIR}/${WORKFLOW_DIR}
+fi
+
 #
 # Prepare the artifacts file, we'll collect everything to be ingested in a directory.
 #
@@ -98,9 +105,9 @@ then
 fi
 
 # tar a subset of files into a tar archive in the storage path
-#   Skipped: FITS files, Weblog (zipped and unzipped)
+#   Skipped: FITS files, Weblog (zipped and unzipped), selfcal auxproducts
 #
-tar --exclude=\*.fits --exclude=pipeline-\* --exclude=*weblog.tgz -C${SOURCE_DIR} -cvf ${STAGE_DIR}/${WORKFLOW_DIR}/${FILENAME}.tar .
+tar --exclude=\*.fits --exclude=pipeline-\* --exclude=*weblog.tgz --exclude=*selfcal.auxproducts.tgz -C${SOURCE_DIR} -cvf ${STAGE_DIR}/${WORKFLOW_DIR}/${FILENAME}.tar .
 if [[ $? -ne 0 ]]
 then
     echo "Creation of pipeline artifacts tar file failed, exiting"