Merge branch 'ws-1904-add-radial-modification-to-workflow' into '2.8.2-DEVELOPMENT'

WS-1904: Add RADIAL configuration to workflow container and quicklook template

See merge request !1418

config/htcondor/submit/condor_ssh_config

+# This allows for the -J option to tunnel through the schedd (radialhead)
+IdentityFile condor_transfer
+# This prevents ssh from prompting to accept a host figerprint
+StrictHostKeyChecking no
+EOM

services/workflow/Dockerfile

@@ -93,6 +93,9 @@ COPY --chown=vlapipe:vlapipe ./services/workflow ./services/workflow
 WORKDIR /code/services/workflow
 RUN poetry install
 
+# Needed for nraorsync to work with the RADIAL cluster
+COPY /config/htcondor/submit/condor_ssh_config /home/vlapipe/.ssh/condor_ssh_config
+
 FROM pex-base as prod
 ARG WS_VERSION=unknown-version
 ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local"

shared/workspaces/alembic/versions/28b6a6bfb73c_add_radial_config_to_htcondor_templates.py

+# Copyright (C) 2023 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+#
+"""add radial config to htcondor templates
+
+Revision ID: 28b6a6bfb73c
+Revises: 1f32110d170d
+Create Date: 2023-08-11 14:18:53.827274
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "28b6a6bfb73c"
+down_revision = "1f32110d170d"
+branch_labels = None
+depends_on = None
+
+old_vlass_ql_envoy_condor = """executable = vlass_ql_envoy.sh
+arguments = metadata.json PPR.xml {{request_id}}
+
+output = envoy.out
+error = envoy.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+VLASS_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json{{files_to_transfer}}
+transfer_output_files = .job.ad
++nrao_output_files = "working products"
+when_to_transfer_output = ON_EXIT
+output_destination = nraorsync://$(VLASS_DIR)
++WantIOProxy = True
+
+request_memory = 31G
+request_disk = 100G
+getenv = True
+
+{{^remote}}
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = (VLASS == True) && (HasLustre == True)
++partition = "VLASS"
+{{/remote}}
+{{#remote}}
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True) + (HasLustre =!= True)
+{{/remote}}
+
+queue
+"""
+
+new_vlass_ql_envoy_condor = """executable = vlass_ql_envoy.sh
+arguments = metadata.json PPR.xml {{request_id}}
+
+output = envoy.out
+error = envoy.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+VLASS_DIR = {{data_location}}
+should_transfer_files = yes
+transfer_input_files = {{#radial}}$ENV(HOME)/.ssh/condor_ssh_config, {{/radial}}$ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json{{files_to_transfer}}
+transfer_output_files = .job.ad
++nrao_output_files = "working products"
+when_to_transfer_output = ON_EXIT
+output_destination = nraorsync://$(VLASS_DIR)
++WantIOProxy = True
+{{#radial}}
+universe = grid
+grid_resource = condor radialhead.nrao.radial.local radialhead.nrao.radial.local
++remote_jobuniverse = 5
++remote_requirements = True
++remote_ShouldTransferFiles = "YES"
++remote_WhenToTransferOutput = "ON_EXIT"
+{{/radial}}
+
+request_memory = 31G
+request_disk = 100G
+getenv = True
+
+{{^remote}}
+environment = "CAPO_PATH=/home/casa/capo"
+requirements = (VLASS == True) && (HasLustre == True)
++partition = "VLASS"
+{{/remote}}
+{{#remote}}
+requirements = (VLASS == True)
++partition = "VLASS"
+Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True) + (HasLustre =!= True)
+{{/remote}}
+
+queue
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{new_vlass_ql_envoy_condor} WHERE filename='vlass_ql_envoy.condor'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{old_vlass_ql_envoy_condor} WHERE filename='vlass_ql_envoy.condor'
+        """
+    )