diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py index 6710efd3b94506d936f8720c6b34562ea4aa93ac..62713b74ccf6104ed91f4af39fa2cfa0d0f094c6 100644 --- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py +++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py @@ -35,6 +35,7 @@ WEBLOG_FILENAME = "weblog.tgz" INIT_WEBLOG_FILENAME = "initial_weblog.tgz" REIMAGING_FILENAME = "reimaging_resources.tgz" PARAMETER_FILENAME = "SEIP_parameter.list" +SELFCAL_AUX_FILENAME = "selfcal.auxproducts.tgz" SCIENCE_PRODUCT_PATTERN = re.compile("[a-zA-Z0-9._\\-+]*\\.tar") JSON = Union[int, float, str, List["JSON"], Dict[str, "JSON"]] diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py index 4dbc3468a784a499e3cb67240d6a2867a93dab13..65701bb2e3fe236f3377d967dd5be31f610f994d 100644 --- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py +++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/std_img_manifest_utils.py @@ -24,6 +24,7 @@ from typing import List from ingest_envoy.manifest_components import ( PARAMETER_FILENAME, REIMAGING_FILENAME, + SELFCAL_AUX_FILENAME, TARFILE_EXT, WEBLOG_FILENAME, AncillaryProduct, @@ -130,6 +131,7 @@ class ImageIngestionProductsFinder: except IndexError as err: raise FileNotFoundError(f"WARNING: No pipeline artifacts found in {self.files_found}") from err + # for images both the weblog and pipeline artifacts must exist, these from the group ancillary base ancillary_products = [ AncillaryProduct(type=AncillaryProductType.PIPELINE_WEBLOG, filename=WEBLOG_FILENAME), AncillaryProduct(type=AncillaryProductType.PIPELINE_ARTIFACTS, filename=pipeline_artifacts_tar.name), @@ -144,6 +146,13 @@ class ImageIngestionProductsFinder: AncillaryProduct(type=AncillaryProductType.PARAMETER_LIST, filename=PARAMETER_FILENAME) ) + # if selfcal completed successfully, ingest the resulting auxproducts tar + selfcal_auxproducts_tar = [file for file in self.files_found if file.name.endswith(SELFCAL_AUX_FILENAME)] + if selfcal_auxproducts_tar: + ancillary_products.append( + AncillaryProduct(type=AncillaryProductType.AUX_PRODUCTS, filename=selfcal_auxproducts_tar[0].name) + ) + return ancillary_products @staticmethod diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py index 93717f60dcc123b3803674202c8c06072f9156e1..34c620a44da747d47ad5fcedf70606e0c8f43201 100644 --- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py +++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py @@ -97,6 +97,9 @@ class AncillaryProductType(Enum): COMPONENT_LIST = "component_list" OBSERVATION_DETAILS = "observation_details" + # Image extras + AUX_PRODUCTS = "pipeline_aux_products" + class InvalidLocatorException(Exception): """Throw this if we're fed a bad science product locator""" diff --git a/apps/cli/executables/pexable/vela/vela/forger.py b/apps/cli/executables/pexable/vela/vela/forger.py index 03cdbaff52ff413b320ad51ab0635ec5a7b64676..8bb37204038fba0fa2708237126ef624c12e4bb7 100644 --- a/apps/cli/executables/pexable/vela/vela/forger.py +++ b/apps/cli/executables/pexable/vela/vela/forger.py @@ -118,13 +118,30 @@ class VelaProduct: os.chdir("../") def img_products(self, parameters: dict): + """ + Forge a variety of image files for ingestion testing + + :param parameters: + :return: + """ path = Path.cwd() / "products" os.chdir(path) + # pbcor.tt0 is the primary science product + pbcor_filename = "oussid.1-93305_sci.L_band.cont.I.pbcor.tt0.fits" + selfcal_pbcor = "oussid.1-93305_sci.S_band.cont.selfcal.I.pbcor.tt0.fits" pb_filename = "oussid.1-93305_sci.L_band.cont.I.pb.tt0.fits" cont_filename = "oussid.1-93305_sci.L_band.cont.I.tt0.fits" mask_filename = "oussid.1-93305_sci.L_band.cont.I.mask.fits" + fits = open(pbcor_filename, "x") + fits.write("I am a primary science image.") + fits.close() + + fits = open(selfcal_pbcor, "x") + fits.write("I am a selfcal image.") + fits.close() + fits = open(pb_filename, "x") fits.write("I am a primary beam image.") fits.close() diff --git a/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh b/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh index 1790a3d65099cbaa0576be39d479c85b2aa7af6b..21fa1945fbcf9a51a423ceaed89ab1c14dfa088e 100644 --- a/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh +++ b/apps/cli/executables/wf_framework/ingest_requirements/image-product-collector.sh @@ -88,6 +88,13 @@ fi # NOTE: This fails if we have no weblog, previously we allowed that # possibility. +SELFCAL=$(ls -t ${SOURCE_DIR}/*selfcal.auxproducts.tgz) + +if [[ -n "$SELFCAL" ]] +then + /bin/cp -f ${SOURCE_DIR}/*selfcal.auxproducts.tgz ${STAGE_DIR}/${WORKFLOW_DIR} +fi + # # Prepare the artifacts file, we'll collect everything to be ingested in a directory. # @@ -98,9 +105,9 @@ then fi # tar a subset of files into a tar archive in the storage path -# Skipped: FITS files, Weblog (zipped and unzipped) +# Skipped: FITS files, Weblog (zipped and unzipped), selfcal auxproducts # -tar --exclude=\*.fits --exclude=pipeline-\* --exclude=*weblog.tgz -C${SOURCE_DIR} -cvf ${STAGE_DIR}/${WORKFLOW_DIR}/${FILENAME}.tar . +tar --exclude=\*.fits --exclude=pipeline-\* --exclude=*weblog.tgz --exclude=*selfcal.auxproducts.tgz -C${SOURCE_DIR} -cvf ${STAGE_DIR}/${WORKFLOW_DIR}/${FILENAME}.tar . if [[ $? -ne 0 ]] then echo "Creation of pipeline artifacts tar file failed, exiting"