From 76a28798fb5019f1c80710d79f642c5004795c59 Mon Sep 17 00:00:00 2001
From: Charlotte Hausman <chausman@nrao.edu>
Date: Thu, 17 Feb 2022 12:27:38 -0500
Subject: [PATCH] fix for file transfer issue - on main
---
.../casa_envoy/casa_envoy/foundation.py | 40 +++++++++++++++++++
.../casa_envoy/casa_envoy/launchers.py | 5 ++-
.../pexable/casa_envoy/test/test_launchers.py | 5 ++-
.../capability/services/execution_manager.py | 16 ++++++--
.../workflow/services/workflow_service.py | 2 +-
5 files changed, 61 insertions(+), 7 deletions(-)
diff --git a/apps/cli/executables/pexable/casa_envoy/casa_envoy/foundation.py b/apps/cli/executables/pexable/casa_envoy/casa_envoy/foundation.py
index 20a1ea48c..e1484b362 100644
--- a/apps/cli/executables/pexable/casa_envoy/casa_envoy/foundation.py
+++ b/apps/cli/executables/pexable/casa_envoy/casa_envoy/foundation.py
@@ -15,11 +15,14 @@
#
# You should have received a copy of the GNU General Public License
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
+
"""
Classes and methods for laying the data location foundation for various types of CASA processing
"""
import logging
import os
+import shutil
+
import sys
import tarfile
from distutils.dir_util import copy_tree
@@ -29,7 +32,43 @@ from casa_envoy.interfaces import FoundationIF
from casa_envoy.schema import AbstractTextFile
+class GeneralFoundation(FoundationIF):
+ """
+ Class to set up the general data foundation for anything running CASA.
+ - Special attention to user provided file locations, must be placed in working directory.
+ """
+
+ def __init__(self, parameters: dict, metadata: AbstractTextFile):
+ self.logger = logging.getLogger("casa_envoy")
+ self.parent_path = parameters["parent_path"]
+ self.working_dir = self.parent_path / "working"
+ self.metadata = metadata
+
+ def data_foundation(self):
+ """
+ All user provided files should be placed in the 'working' directory inorder for CASA to pick them up.
+
+ :return:
+ """
+ self.logger.info("LAYING DATA FOUNDATION...")
+
+ if "providedFiles" in self.metadata.content:
+ provided_files = (
+ self.metadata.content["providedFiles"].replace("[", "").replace("'", "").replace("]", "").split(",")
+ )
+
+ # move provided files from parent directory into working directory for CASA
+ for file in provided_files:
+ shutil.copy(file.strip(), self.working_dir)
+
+ self.logger.info("DATA FOUNDATION COMPLETE!")
+
+
class RestoreFoundation(FoundationIF):
+ """
+ Class to set up the data foundation for restores
+ """
+
def __init__(self, parameters: dict, metadata: AbstractTextFile):
self.logger = logging.getLogger("casa_envoy")
self.parameters = parameters
@@ -42,6 +81,7 @@ class RestoreFoundation(FoundationIF):
After download, all data is in rawdata and the calibrations tables are contained
in a tar file. We need to extract all the calibration tables to the products directory
for CASA processing.
+
:return:
"""
diff --git a/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py b/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py
index bc6c497be..e5312659c 100644
--- a/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py
+++ b/apps/cli/executables/pexable/casa_envoy/casa_envoy/launchers.py
@@ -25,7 +25,7 @@ import sys
from typing import Dict
from casa_envoy.auditor import AuditDirectories, AuditFiles
-from casa_envoy.foundation import RestoreFoundation
+from casa_envoy.foundation import RestoreFoundation, GeneralFoundation
from casa_envoy.interfaces import LauncherIF
from casa_envoy.schema import AbstractTextFile
@@ -143,6 +143,7 @@ class CalibrationLauncher(LauncherIF):
def prepare_for_casa(self):
# Ensure all data is in the required locations for CASA processing (This is not always rawdata!)
+ GeneralFoundation(self.parameters, self.metadata).data_foundation()
if self.parameters["product_type"] == "restore":
RestoreFoundation(self.parameters, self.metadata).data_foundation()
@@ -197,6 +198,8 @@ class ImagingLauncher(LauncherIF):
if check_input:
wf_name = self.metadata.content["workflowName"]
+ GeneralFoundation(self.parameters, self.metadata).data_foundation()
+
if "restore" in wf_name:
RestoreFoundation(self.parameters, self.metadata).data_foundation()
diff --git a/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py b/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py
index c25bfb455..b97379b1b 100644
--- a/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py
+++ b/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py
@@ -16,7 +16,8 @@
# You should have received a copy of the GNU General Public License
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
import os
-from unittest.mock import mock_open, patch
+from pathlib import Path
+from unittest.mock import patch
from casa_envoy.launchers import CalibrationLauncher, CasaLauncher, ImagingLauncher
@@ -28,6 +29,7 @@ cal_parameters = {
"metadata": "test/input_files/test.json",
"ppr": "test/input_files/PPR.xml",
"product_type": "standard-cal",
+ "parent_path": Path("I/am/a/path"),
}
img_parameters = {
"useCasa": False,
@@ -37,6 +39,7 @@ img_parameters = {
"metadata": "test/input_files/image-metadata.json",
"ppr": "test/input_files/cmsimage-PPR.xml",
"product_type": "standard-img",
+ "parent_path": Path("I/am/a/path"),
}
restore_parameters = {
diff --git a/shared/workspaces/workspaces/capability/services/execution_manager.py b/shared/workspaces/workspaces/capability/services/execution_manager.py
index 8ffe3e6e8..eb88a707d 100644
--- a/shared/workspaces/workspaces/capability/services/execution_manager.py
+++ b/shared/workspaces/workspaces/capability/services/execution_manager.py
@@ -158,8 +158,8 @@ class ExecutionManager(ExecutionManagerIF):
:param files:
"""
logger.info("Submitting workflow request")
- transfer_files_string = self._get_transfer_files_property(files)
- args_list = {**workflow_args, "files_to_transfer": transfer_files_string}
+ transfer_files_string, transfer_files_list = self._get_transfer_files_properties(files)
+ args_list = {**workflow_args, "files_to_transfer": transfer_files_string, "provided_files": transfer_files_list}
execution = self.capability_info.lookup_execution(execution_id)
@@ -299,6 +299,14 @@ class ExecutionManager(ExecutionManagerIF):
self._error_execution(execution)
@staticmethod
- def _get_transfer_files_property(files: List[AbstractFile]):
+ def _get_transfer_files_properties(files: List[AbstractFile]):
+ name_list = []
separator = ", ./"
- return separator + separator.join(file.filename for file in files)
+ # needed for template rendering
+ if files:
+ for file in files:
+ name_list.append(file.filename)
+
+ return [separator + separator.join(file.filename for file in files), name_list]
+ else:
+ return ["", []]
diff --git a/shared/workspaces/workspaces/workflow/services/workflow_service.py b/shared/workspaces/workspaces/workflow/services/workflow_service.py
index e4103d69f..067dbd1b8 100644
--- a/shared/workspaces/workspaces/workflow/services/workflow_service.py
+++ b/shared/workspaces/workspaces/workflow/services/workflow_service.py
@@ -700,9 +700,9 @@ class WorkflowMessageHandler:
subject = message["subject"]
wf_req_id = subject["workflow_request_id"]
wf_request = self.info.lookup_workflow_request(wf_req_id)
- subject["execution_wf_id"] = wf_request.argument["parent_wf_request_id"]
if "ingest" in wf_request.workflow_name:
+ subject["execution_wf_id"] = wf_request.argument["parent_wf_request_id"]
logger.info("SENDING INGESTION FAILED MESSAGE!")
ingestion_failed_msg = WorkflowMessageArchitect(previous_info=subject).compose_message("ingestion_failed")
--
GitLab