From b036a9f8f24c3f0b4af3699be37492f9faa2dcf3 Mon Sep 17 00:00:00 2001
From: Daniel Nemergut <dnemergu@nrao.edu>
Date: Tue, 31 Oct 2023 10:37:09 -0400
Subject: [PATCH] Using the sbin location in capo to run the collectors and
ingest scripts for the image, cal, seci, and curator workflows
---
.../pexable/ingest_envoy/ingest_envoy/collectors.py | 6 ++++--
.../pexable/ingest_envoy/ingest_envoy/launchers.py | 13 +++++++------
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py
index 6a9a98fe7..cbb227cf9 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py
@@ -67,6 +67,7 @@ class ImageCollector(CollectorIF):
self.logger.info("Collecting image products for staging...")
workflow_dir = self.parameters["workflowDir"]
staging_dir = self.parameters["staging_area"]
+ sbin_dir = self.parameters["script_location"]
tar_name = self.create_artifacts_name()
# rename image files to allow for duplicate ingestion
@@ -74,7 +75,7 @@ class ImageCollector(CollectorIF):
# run script
collector = subprocess.run(
- ["./image-product-collector.sh", workflow_dir, staging_dir, tar_name],
+ [f"{sbin_dir}/image-product-collector.sh", workflow_dir, staging_dir, tar_name],
stdout=sys.stdout,
stderr=sys.stderr,
)
@@ -147,10 +148,11 @@ class SECICollector(CollectorIF):
cache_path = self.parameters["sourcePath"]
workflow_dir = self.parameters["workflowDir"]
staging_dir = self.parameters["staging_area"]
+ sbin_dir = self.parameters["script_location"]
tar_name = self.create_artifacts_name()
collector = subprocess.run(
- ["./seci-product-collector.sh", cache_path, workflow_dir, staging_dir, tar_name],
+ [f"{sbin_dir}/seci-product-collector.sh", cache_path, workflow_dir, staging_dir, tar_name],
stdout=sys.stdout,
stderr=sys.stderr,
)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
index 8407f2795..774e732de 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
@@ -43,7 +43,6 @@ def trigger_ingest(real_ingest: bool, staging_dir: str, bin_dir: str = ".") -> i
:return: return code
"""
if real_ingest:
- # TODO: This should be cleaned up when cal and image ingests don't transfer files into condor and use sbin
ingest_process = subprocess.run(
[f"{bin_dir}/ingest", "--json", "-p", staging_dir],
stdout=sys.stdout,
@@ -96,7 +95,7 @@ class IngestCalibrationLauncher(LauncherIF):
self.prepare_for_launch()
self.logger.info("Running ingest!")
- return trigger_ingest(self.parameters["useIngest"], self.staging_source_dir)
+ return trigger_ingest(self.parameters["useIngest"], self.staging_source_dir, self.parameters["script_location"])
def prepare_for_launch(self):
"""
@@ -122,12 +121,14 @@ class IngestCalibrationLauncher(LauncherIF):
"""
self.logger.info("Collecting calibration tables for staging...")
workflow_dir = self.parameters["workflowDir"]
+ sbin_dir = self.parameters["script_location"]
sdm_id = self.parameters["sdmId"]
cal_processing_datetime = self.parameters["processingStart"]
+
# run script
if self.parameters["multiVersion"] is False:
collector = subprocess.run(
- ["./calibration-table-collector.sh", workflow_dir, sdm_id, cal_processing_datetime],
+ [f"{sbin_dir}/calibration-table-collector.sh", workflow_dir, sdm_id, cal_processing_datetime],
stdout=sys.stdout,
stderr=sys.stderr,
)
@@ -136,7 +137,7 @@ class IngestCalibrationLauncher(LauncherIF):
collector = subprocess.run(
[
- "./calibration-table-collector.sh",
+ f"{sbin_dir}/calibration-table-collector.sh",
workflow_dir,
sdm_id,
cal_processing_datetime,
@@ -185,7 +186,7 @@ class IngestImageLauncher(LauncherIF):
self.prepare_for_launch()
self.logger.info("Running ingest!")
- return trigger_ingest(self.parameters["useIngest"], self.staging_source_dir)
+ return trigger_ingest(self.parameters["useIngest"], self.staging_source_dir, self.parameters["script_location"])
def prepare_for_launch(self):
"""
@@ -344,4 +345,4 @@ class CuratorLauncher(LauncherIF):
self.prepare_for_launch()
self.logger.info("Running curator!")
- return trigger_curator(self.parameters["useIngest"])
+ return trigger_curator(self.parameters["useIngest"], self.parameters["script_location"])
--
GitLab