From 9a4c04e63f6fb1ba4348c96758038b1fba10bc27 Mon Sep 17 00:00:00 2001
From: Daniel Nemergut <dnemergu@nrao.edu>
Date: Wed, 18 Oct 2023 00:27:15 -0400
Subject: [PATCH] Fixing the collector and ingest script paths to use the sbin
path defined in capo since they're no longer copied into the container
---
.../pexable/ingest_envoy/ingest_envoy/collectors.py | 3 ++-
.../pexable/ingest_envoy/ingest_envoy/ingest.py | 2 ++
.../pexable/ingest_envoy/ingest_envoy/launchers.py | 8 +++++---
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py
index a9545581c..067223884 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/collectors.py
@@ -177,10 +177,11 @@ class ObservationCollector(CollectorIF):
source_path = self.parameters["sourcePath"]
workflow_dir = self.parameters["workflowDir"]
staging_dir = self.parameters["staging_area"]
+ sbin_dir = self.parameters["script_location"]
# run script
collector = subprocess.run(
- ["./observation-product-collector.sh", source_path, workflow_dir, staging_dir],
+ [f".{sbin_dir}/observation-product-collector.sh", source_path, workflow_dir, staging_dir],
stdout=sys.stdout,
stderr=sys.stderr,
)
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py
index 3962883fc..b0b33f269 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py
@@ -56,6 +56,7 @@ def _get_settings(
:return: a dict containing required settings to run the provided ingestion type
"""
ingestion_settings = CapoConfig().settings("edu.nrao.workspaces.IngestionSettings")
+ script_location = CapoConfig().settings("edu.nrao.workspaces.ProcessingSettings").scriptLocation
capability_url = CapoConfig().settings("edu.nrao.workspaces.CapabilitySettings").externalServiceUrl
workflow_url = CapoConfig().settings("edu.nrao.workspaces.WorkflowSettings").serviceUrl
@@ -70,6 +71,7 @@ def _get_settings(
parameters["staging_area"] = ingestion_settings.stagingDirectory
parameters["storage_area"] = ingestion_settings.storageDirectory
parameters["useIngest"] = strtobool(ingestion_settings.useIngest)
+ parameters["script_location"] = script_location
parameters["workflowUrl"] = workflow_url
return parameters
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
index b871bf64e..617e8a219 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/launchers.py
@@ -33,17 +33,19 @@ from ingest_envoy.schema import AbstractTextFile
from ingest_envoy.utilities import IngestType, VLASSIngestType
-def trigger_ingest(real_ingest: bool, staging_dir: str) -> int:
+def trigger_ingest(real_ingest: bool, staging_dir: str, bin_dir: str = "") -> int:
"""
Run ingest
:param real_ingest: real ingestion vs. testing flag
:param staging_dir: staging directory to ingest from
+ :param bin_dir: directory containing the ingestion script
:return: return code
"""
if real_ingest:
+ # TODO: This should be cleaned up when cal and image ingests don't transfer files into condor and use sbin
ingest_process = subprocess.run(
- ["./ingest", "--json", "-p", staging_dir],
+ [f".{bin_dir}/ingest", "--json", "-p", staging_dir],
stdout=sys.stdout,
stderr=sys.stderr,
)
@@ -229,7 +231,7 @@ class IngestObservationLauncher(IngestLauncherIF):
self.prepare_for_ingest()
self.logger.info("Running ingest!")
- return trigger_ingest(self.parameters["useIngest"], self.staging_source_dir)
+ return trigger_ingest(self.parameters["useIngest"], self.staging_source_dir, self.parameters["script_location"])
def prepare_for_ingest(self):
"""
--
GitLab