Rename images

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py

@@ -15,7 +15,9 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+import glob
 import logging
+import os
 import shutil
 import subprocess
 
@@ -67,6 +69,9 @@ class ImageCollector(CollectorIF):
         staging_dir = self.parameters["staging_area"]
         tar_name = self.create_artifacts_name()
 
+        # rename image files to allow for duplicate ingestion
+        self.rename_images()
+
         # run script
         collector = subprocess.run(
             ["./image-product-collector.sh", workflow_dir, staging_dir, tar_name],
@@ -89,6 +94,27 @@ class ImageCollector(CollectorIF):
         date = self.parameters["processingStart"]
         return name + date
 
+    def rename_images(self):
+        """
+        Rename images to allow for duplicate ingestion
+
+        :return:
+        """
+        self.logger.info("Renaming images...")
+        sdm = self.parameters["sdmId"]
+        ebMjd = sdm.split(".")[-2] + "." + sdm.split(".")[-1]
+
+        prefix = self.parameters["project"] + ".MJD" + ebMjd
+
+        path = self.parameters["spoolDir"] + "/products"
+
+        images = glob.glob(path + "/*.fits")
+
+        for image in images:
+            newname = image.replace("oussid", prefix)
+            self.logger.info(f"Renaming {image} to {newname}")
+            os.rename(image, newname)
+
 
 class SECICollector(CollectorIF):
     """Collect SECI Image Products for Ingestion"""

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/solicitor.py

@@ -143,6 +143,7 @@ class Solicitor:
         """
 
         gen = {
+            "sdmId": self.metadata["fileSetIds"],
             "requestId": self.metadata["systemId"],
             "telescope": self.metadata["projectMetadata"]["telescope"],  # all, needed by manifest generator
             "project": self.metadata["projectMetadata"]["projectCode"],  # needed for post ingestion messaging
@@ -169,7 +170,6 @@ class Solicitor:
         general = self.get_general_params()
 
         cal = {
-            "sdmId": self.metadata["fileSetIds"],  # calibration only
             "spl": self.metadata["productLocator"],  # calibration only
         }
 
@@ -183,7 +183,10 @@ class Solicitor:
         """
         general = self.get_general_params()
 
-        img = {"calSpl": self.metadata["calProductLocator"]}  # image only
+        img = {
+            "calSpl": self.metadata["calProductLocator"],  # image only
+            "spoolDir": self.metadata["destinationDirectory"],
+        }
 
         return {**general, **img}
 

apps/cli/executables/pexable/ingest_envoy/test/test_collectors.py

@@ -35,6 +35,8 @@ parameters = {
     "calSpl": "uid://evla/calibration/c78ccfd6-fe4e-43c6-a5c5-70e5bcfde16b",
     "staging_area": "/lustre/aoc/cluster/pipeline/docker/workspaces/staging",
     "storage_area": "/lustre/aoc/cluster/pipeline/docker/workspaces/storage",
+    "project": "Operations",
+    "spoolDir": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool/tmprb1se376",
 }
 
 image_collector = ImageCollector(parameters=parameters)
@@ -56,7 +58,10 @@ def test_aux_file_to_staging():
 class TestImageCollector:
     @patch("shutil.copy2")
     @patch("subprocess.run")
-    def test_collect_image_products(self, mock_run, mock_copy):
+    @patch("glob.glob")
+    @patch("os.rename")
+    @patch("os.replace")
+    def test_collect_image_products(self, mock_replace, mock_rename, mock_glob, mock_run, mock_copy):
         mock_run.return_value.returncode = 0
         image_collector.collect_products()
         assert mock_run.call_count == 1

apps/cli/executables/pexable/ingest_envoy/test/test_solicitor.py

@@ -94,6 +94,7 @@ class TestSolicitor:
     def test_solicit_parameters_image(self, image_solicitor: Solicitor):
         initial_version_dir = "iamthefirst"
         metadata = {
+            "sdmId": "brain_000.58099.67095825232",
             "telescope": "EVLA",
             "workflowName": "std_cms_imaging",
             "processingStart": "2021_07_29T14_26_31",
@@ -103,6 +104,7 @@ class TestSolicitor:
             "calSpl": "uid://evla/calibration/c78ccfd6-fe4e-43c6-a5c5-70e5bcfde16b",
             "project": "Operations",
             "requestId": "12",
+            "spoolDir": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool/tmprb1se376",
         }
         with patch("ingest_envoy.solicitor.Solicitor.solicit_initial_directory_name", return_value=initial_version_dir):
             parameters = image_solicitor.solicit_parameters()