Skip to content
Snippets Groups Projects
Commit 08de3081 authored by Nathan Bockisch's avatar Nathan Bockisch
Browse files

WS-1904: Add RADIAL configuration to workflow container and quicklook template

parent 839dbdae
No related branches found
Tags 2.5.0-rc2
2 merge requests!1452Merge 2.8.2 to main,!1418WS-1904: Add RADIAL configuration to workflow container and quicklook template
Pipeline #11016 passed
# This allows for the -J option to tunnel through the schedd (radialhead)
IdentityFile condor_transfer
# This prevents ssh from prompting to accept a host figerprint
StrictHostKeyChecking no
EOM
......@@ -93,6 +93,9 @@ COPY --chown=vlapipe:vlapipe ./services/workflow ./services/workflow
WORKDIR /code/services/workflow
RUN poetry install
# Needed for nraorsync to work with the RADIAL cluster
COPY /config/htcondor/submit/condor_ssh_config /home/vlapipe/.ssh/condor_ssh_config
FROM pex-base as prod
ARG WS_VERSION=unknown-version
ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local"
......
# Copyright (C) 2023 Associated Universities, Inc. Washington DC, USA.
#
# This file is part of NRAO Workspaces.
#
# Workspaces is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Workspaces is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
#
"""add radial config to htcondor templates
Revision ID: 28b6a6bfb73c
Revises: 1f32110d170d
Create Date: 2023-08-11 14:18:53.827274
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "28b6a6bfb73c"
down_revision = "1f32110d170d"
branch_labels = None
depends_on = None
old_vlass_ql_envoy_condor = """executable = vlass_ql_envoy.sh
arguments = metadata.json PPR.xml {{request_id}}
output = envoy.out
error = envoy.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
VLASS_DIR = {{data_location}}
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json{{files_to_transfer}}
transfer_output_files = .job.ad
+nrao_output_files = "working products"
when_to_transfer_output = ON_EXIT
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
request_memory = 31G
request_disk = 100G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True) + (HasLustre =!= True)
{{/remote}}
queue
"""
new_vlass_ql_envoy_condor = """executable = vlass_ql_envoy.sh
arguments = metadata.json PPR.xml {{request_id}}
output = envoy.out
error = envoy.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
VLASS_DIR = {{data_location}}
should_transfer_files = yes
transfer_input_files = {{^radial}}$ENV(HOME)/.ssh/condor_ssh_config, {{/radial}}$ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json{{files_to_transfer}}
transfer_output_files = .job.ad
+nrao_output_files = "working products"
when_to_transfer_output = ON_EXIT
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
{{^radial}}
universe = grid
grid_resource = condor radialhead.nrao.radial.local radialhead.nrao.radial.local
+remote_jobuniverse = 5
+remote_requirements = True
+remote_ShouldTransferFiles = "YES"
+remote_WhenToTransferOutput = "ON_EXIT"
{{/radial}}
request_memory = 31G
request_disk = 100G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True) + (HasLustre =!= True)
{{/remote}}
queue
"""
def upgrade():
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{new_vlass_ql_envoy_condor} WHERE filename='vlass_ql_envoy.condor'
"""
)
def downgrade():
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{old_vlass_ql_envoy_condor} WHERE filename='vlass_ql_envoy.condor'
"""
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment