diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py index b0a8d78f5703ba5e5e3e215fd691be1cb0713e09..49c33c0704ef316fc9b7aaf23a8147be039e5f7c 100644 --- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py +++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py @@ -15,7 +15,9 @@ # # You should have received a copy of the GNU General Public License # along with Workspaces. If not, see <https://www.gnu.org/licenses/>. +import glob import logging +import os import shutil import subprocess @@ -67,6 +69,9 @@ class ImageCollector(CollectorIF): staging_dir = self.parameters["staging_area"] tar_name = self.create_artifacts_name() + # rename image files to allow for duplicate ingestion + self.rename_images() + # run script collector = subprocess.run( ["./image-product-collector.sh", workflow_dir, staging_dir, tar_name], @@ -89,6 +94,27 @@ class ImageCollector(CollectorIF): date = self.parameters["processingStart"] return name + date + def rename_images(self): + """ + Rename images to allow for duplicate ingestion + + :return: + """ + self.logger.info("Renaming images...") + sdm = self.parameters["sdmId"] + ebMjd = sdm.split(".")[-2] + "." + sdm.split(".")[-1] + + prefix = self.parameters["project"] + ".MJD" + ebMjd + + path = self.parameters["spoolDir"] + "/products" + + images = glob.glob(path + "/*.fits") + + for image in images: + newname = image.replace("oussid", prefix) + self.logger.info(f"Renaming {image} to {newname}") + os.rename(image, newname) + class SECICollector(CollectorIF): """Collect SECI Image Products for Ingestion""" diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/solicitor.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/solicitor.py index cc96d8da982fe437410b2b11e1d5b536c7569168..1bcf260d245138fca36a32571bcafa2be1621003 100644 --- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/solicitor.py +++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/solicitor.py @@ -143,6 +143,7 @@ class Solicitor: """ gen = { + "sdmId": self.metadata["fileSetIds"], "requestId": self.metadata["systemId"], "telescope": self.metadata["projectMetadata"]["telescope"], # all, needed by manifest generator "project": self.metadata["projectMetadata"]["projectCode"], # needed for post ingestion messaging @@ -169,7 +170,6 @@ class Solicitor: general = self.get_general_params() cal = { - "sdmId": self.metadata["fileSetIds"], # calibration only "spl": self.metadata["productLocator"], # calibration only } @@ -183,7 +183,10 @@ class Solicitor: """ general = self.get_general_params() - img = {"calSpl": self.metadata["calProductLocator"]} # image only + img = { + "calSpl": self.metadata["calProductLocator"], # image only + "spoolDir": self.metadata["destinationDirectory"], + } return {**general, **img} diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_collectors.py b/apps/cli/executables/pexable/ingest_envoy/test/test_collectors.py index 15fb90d3ee2d408bc6f5d3c99173fda9d105f89e..938bef1d3e19cb6c559134a93f9f87fa2c59582f 100644 --- a/apps/cli/executables/pexable/ingest_envoy/test/test_collectors.py +++ b/apps/cli/executables/pexable/ingest_envoy/test/test_collectors.py @@ -35,6 +35,8 @@ parameters = { "calSpl": "uid://evla/calibration/c78ccfd6-fe4e-43c6-a5c5-70e5bcfde16b", "staging_area": "/lustre/aoc/cluster/pipeline/docker/workspaces/staging", "storage_area": "/lustre/aoc/cluster/pipeline/docker/workspaces/storage", + "project": "Operations", + "spoolDir": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool/tmprb1se376", } image_collector = ImageCollector(parameters=parameters) @@ -56,7 +58,10 @@ def test_aux_file_to_staging(): class TestImageCollector: @patch("shutil.copy2") @patch("subprocess.run") - def test_collect_image_products(self, mock_run, mock_copy): + @patch("glob.glob") + @patch("os.rename") + @patch("os.replace") + def test_collect_image_products(self, mock_replace, mock_rename, mock_glob, mock_run, mock_copy): mock_run.return_value.returncode = 0 image_collector.collect_products() assert mock_run.call_count == 1 diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_solicitor.py b/apps/cli/executables/pexable/ingest_envoy/test/test_solicitor.py index 1152db17586eb9e331ea5868a341b08404d4c3ae..e2b6a53462e912a557f53fba9ac9889427261075 100644 --- a/apps/cli/executables/pexable/ingest_envoy/test/test_solicitor.py +++ b/apps/cli/executables/pexable/ingest_envoy/test/test_solicitor.py @@ -94,6 +94,7 @@ class TestSolicitor: def test_solicit_parameters_image(self, image_solicitor: Solicitor): initial_version_dir = "iamthefirst" metadata = { + "sdmId": "brain_000.58099.67095825232", "telescope": "EVLA", "workflowName": "std_cms_imaging", "processingStart": "2021_07_29T14_26_31", @@ -103,6 +104,7 @@ class TestSolicitor: "calSpl": "uid://evla/calibration/c78ccfd6-fe4e-43c6-a5c5-70e5bcfde16b", "project": "Operations", "requestId": "12", + "spoolDir": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool/tmprb1se376", } with patch("ingest_envoy.solicitor.Solicitor.solicit_initial_directory_name", return_value=initial_version_dir): parameters = image_solicitor.solicit_parameters()