From 5614d9a2f0dedc7468a55582b782dad791cee7d3 Mon Sep 17 00:00:00 2001
From: Daniel Nemergut <dnemergu@nrao.edu>
Date: Thu, 18 Jul 2024 13:27:28 -0400
Subject: [PATCH] Added flags for processing in CV to the restore_cms templates
---
.../versions/175c5c2accf8_alma_restores.py | 71 +++++++++++++++++++
.../restore_cms/deliver_condor_2.8.4.txt | 15 ++++
.../restore_cms/deliver_condor_2.8.5.txt | 23 ++++++
.../restore_cms/envoy_condor_2.8.5.txt | 23 ++++++
.../restore_cms/fetch_condor_2.8.4.txt | 15 ++++
.../restore_cms/fetch_condor_2.8.5.txt | 22 ++++++
6 files changed, 169 insertions(+)
create mode 100644 shared/workspaces/alembic/versions/175c5c2accf8_alma_restores.py
create mode 100644 shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.4.txt
create mode 100644 shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.5.txt
create mode 100644 shared/workspaces/alembic/versions/templates/restore_cms/envoy_condor_2.8.5.txt
create mode 100644 shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.4.txt
create mode 100644 shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.5.txt
diff --git a/shared/workspaces/alembic/versions/175c5c2accf8_alma_restores.py b/shared/workspaces/alembic/versions/175c5c2accf8_alma_restores.py
new file mode 100644
index 000000000..e758b57f9
--- /dev/null
+++ b/shared/workspaces/alembic/versions/175c5c2accf8_alma_restores.py
@@ -0,0 +1,71 @@
+# Copyright (C) 2023 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
+#
+"""alma restores
+
+Revision ID: 175c5c2accf8
+Revises: 985ca1bdec62
+Create Date: 2024-07-18 09:58:07.092540
+
+"""
+from pathlib import Path
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "175c5c2accf8"
+down_revision = "985ca1bdec62"
+branch_labels = None
+depends_on = None
+
+workflow = "restore_cms"
+old_suffix = "_condor_2.8.4.txt"
+new_suffix = "_condor_2.8.5.txt"
+
+
+def set_content(filename: str) -> bytes:
+ return (Path.cwd() / "versions" / "templates" / workflow / filename).read_text().encode()
+
+
+def upgrade():
+ conn = op.get_bind()
+
+ # Update the restore_cms condor templates to handle processing in CV
+ for task in ["fetch", "envoy", "deliver"]:
+ conn.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=%s
+ WHERE filename=E'{workflow}_{task}.condor' AND workflow_name=E'{workflow}'
+ """,
+ set_content(task + new_suffix),
+ )
+
+
+def downgrade():
+ conn = op.get_bind()
+
+ for task in ["fetch", "envoy", "deliver"]:
+ conn.execute(
+ f"""
+ UPDATE workflow_templates
+ SET content=%s
+ WHERE filename=E'{workflow}_{task}.condor' AND workflow_name=E'{workflow}'
+ """,
+ set_content(task + old_suffix),
+ )
diff --git a/shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.4.txt b/shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.4.txt
new file mode 100644
index 000000000..8851f159d
--- /dev/null
+++ b/shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.4.txt
@@ -0,0 +1,15 @@
+executable = restore_cms_deliver.sh
+arguments = {{request_id}} {{spool_dir}}
+
+output = deliver.out
+error = deliver.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+
+queue
diff --git a/shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.5.txt b/shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.5.txt
new file mode 100644
index 000000000..3c48fcddd
--- /dev/null
+++ b/shared/workspaces/alembic/versions/templates/restore_cms/deliver_condor_2.8.5.txt
@@ -0,0 +1,23 @@
+executable = restore_cms_deliver.sh
+arguments = {{request_id}} {{spool_dir}}
+
+output = deliver.out
+error = deliver.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+request_memory = {{ramInGb}}
+
+{{^remote-cv}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+{{/remote-cv}}
+{{#remote-cv}}
+environment = "CAPO_PATH=. CAPO_PROFILE={{profile}} LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_21_1"
+requirements = HasLustre == True && (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+queue
diff --git a/shared/workspaces/alembic/versions/templates/restore_cms/envoy_condor_2.8.5.txt b/shared/workspaces/alembic/versions/templates/restore_cms/envoy_condor_2.8.5.txt
new file mode 100644
index 000000000..a9c240c65
--- /dev/null
+++ b/shared/workspaces/alembic/versions/templates/restore_cms/envoy_condor_2.8.5.txt
@@ -0,0 +1,23 @@
+executable = restore_cms_envoy.sh
+arguments = metadata.json PPR.xml
+
+output = envoy.out
+error = envoy.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+request_memory = {{ramInGb}}
+
+{{^remote-cv}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+{{/remote-cv}}
+{{#remote-cv}}
+environment = "CAPO_PATH=. CAPO_PROFILE={{profile}} LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_21_1"
+requirements = HasLustre == True && (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+queue
diff --git a/shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.4.txt b/shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.4.txt
new file mode 100644
index 000000000..3baf461f4
--- /dev/null
+++ b/shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.4.txt
@@ -0,0 +1,15 @@
+executable = restore_cms_fetch.sh
+arguments = {{product_locator}} {{cal_locator}}
+
+output = fetch.out
+error = fetch.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
diff --git a/shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.5.txt b/shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.5.txt
new file mode 100644
index 000000000..b8cdb05b5
--- /dev/null
+++ b/shared/workspaces/alembic/versions/templates/restore_cms/fetch_condor_2.8.5.txt
@@ -0,0 +1,22 @@
+executable = restore_cms_fetch.sh
+arguments = {{product_locator}} {{cal_locator}}
+
+output = fetch.out
+error = fetch.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+
+{{^remote-cv}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = HasLustre == True
+{{/remote-cv}}
+{{#remote-cv}}
+environment = "CAPO_PATH=. CAPO_PROFILE={{profile}} LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_21_1"
+requirements = HasLustre == True && (CVPOST == True)
++partition = "CVPOST"
+{{/remote-cv}}
+
+queue
--
GitLab