From e69d09eceeb9a09d8a5ce48153b0312f968033cd Mon Sep 17 00:00:00 2001
From: Charlotte Hausman <chausman@nrao.edu>
Date: Thu, 13 Oct 2022 18:23:41 -0400
Subject: [PATCH] make workflow setup account for case where previous condor
files and logs exit in processing directory
---
docker.properties | 2 +-
.../workspaces/capability/schema.py | 4 +--
.../workflow/services/workflow_service.py | 27 +++++++++++++++++++
3 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/docker.properties b/docker.properties
index ae4ec668b..cc9bb27a3 100644
--- a/docker.properties
+++ b/docker.properties
@@ -24,7 +24,7 @@ edu.nrao.workspaces.CapabilitySettings.externalServiceUrl = http://capability:34
edu.nrao.workspaces.ProcessingSettings.useCasa = false
edu.nrao.workspaces.ProcessingSettings.rootDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/spool
edu.nrao.workspaces.ProcessingSettings.scriptLocation = /lustre/aoc/cluster/pipeline/docker/workspaces/sbin
-edu.nrao.workspaces.ProcessingSettings.ramInGb = 0.2G
+edu.nrao.workspaces.ProcessingSettings.ramInGb = 0.21G
edu.nrao.workspaces.ProcessingSettings.CasaVersion.vlass = /home/casa/packages/pipeline/casa-6.1.3-3-pipeline-2021.1.1.32
edu.nrao.archive.workflow.config.CasaVersions.homeForReprocessing = /home/casa/packages/pipeline/current
diff --git a/shared/workspaces/workspaces/capability/schema.py b/shared/workspaces/workspaces/capability/schema.py
index 4622f5ff4..1e9f13402 100644
--- a/shared/workspaces/workspaces/capability/schema.py
+++ b/shared/workspaces/workspaces/capability/schema.py
@@ -197,7 +197,7 @@ class SingleQAStateMachine(StateMachine):
SendMessage(arguments="execution_complete"),
SendNotification(arguments=json.dumps({"template": "workflow-complete", "send_to_qa": True})),
SendNotification(
- arguments=json.dumps({"template": workflow_name + "_complete", "send_to_qa": True})
+ arguments=json.dumps({"template": workflow_name + "_complete", "needs_contacts": True})
),
],
("Error", "ingestion-failed"): [
@@ -272,7 +272,7 @@ class DoubleQAStateMachine(StateMachine):
SendMessage(arguments="execution_complete"),
SendNotification(arguments=json.dumps({"template": "workflow-complete", "send_to_qa": True})),
SendNotification(
- arguments=json.dumps({"template": workflow_name + "_complete", "send_to_qa": True})
+ arguments=json.dumps({"template": workflow_name + "_complete", "needs_contacts": True})
),
],
("Error", "ingestion-failed"): [
diff --git a/shared/workspaces/workspaces/workflow/services/workflow_service.py b/shared/workspaces/workspaces/workflow/services/workflow_service.py
index f4ebaae71..56be6352e 100644
--- a/shared/workspaces/workspaces/workflow/services/workflow_service.py
+++ b/shared/workspaces/workspaces/workflow/services/workflow_service.py
@@ -16,6 +16,8 @@
# You should have received a copy of the GNU General Public License
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
""" This is the workflow service. """
+import glob
+import itertools
# pylint: disable=E0401, E0402, W1203
@@ -23,6 +25,7 @@ import json
import logging
import os
import pathlib
+import shutil
import stat
import subprocess
from pathlib import Path
@@ -375,6 +378,7 @@ class WorkflowService(WorkflowServiceIF):
workflow_files = self._determine_usable_files(request, templated_files)
# prepare files for condor execution
+ self._check_for_old_condor_files(temp_folder)
self._prepare_files_for_condor(workflow_files, temp_folder)
self.info.save_request(request)
@@ -544,6 +548,7 @@ class WorkflowService(WorkflowServiceIF):
:param files: a dictionary of filename -> content
:return: a Path
"""
+
# 1. spool each of the temp files to tmp directory
for file in files:
path = temp_folder / file.filename
@@ -555,6 +560,28 @@ class WorkflowService(WorkflowServiceIF):
logger.info("Making %s executable", file)
file.chmod(file.stat().st_mode | stat.S_IEXEC)
+ @staticmethod
+ def _check_for_old_condor_files(dir_path: Path):
+ """
+ It is possible that a follow-on workflow will use an existing, previously used, directory, and this directory
+ might contain previously used condor submit files and their associated output. Move these aside so they don't
+ interfere with the new workflow execution
+ Example: VLASS Image Caching post SECI or Quicklook
+
+ :param dir_path: path of the directory that will be submitted to condor
+ :return:
+ """
+ extensions = ["*.condor", ".dag", ".log"]
+
+ old_files = itertools.chain.from_iterable(dir_path.glob(pattern) for pattern in extensions)
+ if len(list(old_files)) > 0:
+ logger.info("Shifting old condor files aside...")
+ aside = dir_path / "old_condor"
+ aside.mkdir(0o777, parents=False, exist_ok=True)
+ for file in old_files:
+ shutil.move(file, aside)
+ logger.info("Finished shifting old condor files.")
+
def _execute_prepared(self, folder: Path, run_cv: bool) -> [Path, bool]:
"""
Execute HTCondor using the named folder as the source of the files.
--
GitLab