Skip to content
Snippets Groups Projects
Commit cee3a364 authored by Charlotte Hausman's avatar Charlotte Hausman
Browse files

alembic version merge

parent 5dd31121
No related branches found
No related tags found
2 merge requests!1607merge 2.8.3 to main,!1581catch up with 2.8.2.3
Pipeline #14322 passed
Stage: pull-db
Stage: build-packages
Stage: build
Stage: unit-test
Stage: .post
Hide whitespace changes
Inline Side-by-side
shared/workspaces/alembic/versions/12d0499d232d_fix_vlass_seci_envoy_condor.py 0 → 100644
+ 314
− 0
View file @ cee3a364
# Copyright (C) 2023 Associated Universities, Inc. Washington DC, USA.
#
# This file is part of NRAO Workspaces.
#
# Workspaces is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Workspaces is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
#
"""fix vlass_seci_envoy.condor
Revision ID: 12d0499d232d
Revises: 1c435b5d7a8d
Create Date: 2024-02-07 11:25:02.570523
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "12d0499d232d"
down_revision = "1c435b5d7a8d"
branch_labels = None
depends_on = None
old_vlass_seci_envoy_condor = """executable = vlass_seci_envoy.sh
arguments = metadata.json PPR.xml
output = envoy.out
error = envoy.err
log = condor.log
VLASS_DIR = {{data_location}}
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
when_to_transfer_output = ON_EXIT
transfer_output_files = .job.ad
+nrao_output_files = "working products"
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
request_memory = 50G
request_disk = 200G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True) + (HasLustre =!= True)
{{/remote}}
queue
"""
new_vlass_seci_envoy_condor = """executable = vlass_seci_envoy.sh
arguments = metadata.json PPR.xml
output = envoy.out
error = envoy.err
log = condor.log
VLASS_DIR = {{data_location}}
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
when_to_transfer_output = ON_EXIT
transfer_output_files = .job.ad
+nrao_output_files = "working products"
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
request_memory = 50G
request_disk = 200G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True) && (HasLustre =!= True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) && (TARGET.VLASSTEST =!= True)
{{/remote}}
queue
"""
old_vlass_ql_envoy_condor = """executable = vlass_ql_envoy.sh
arguments = metadata.json PPR.xml {{request_id}}
output = envoy.out
error = envoy.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
VLASS_DIR = {{data_location}}
should_transfer_files = yes
transfer_input_files = {{#radial}}$ENV(HOME)/.ssh/condor_ssh_config, {{/radial}}$ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json{{files_to_transfer}}
transfer_output_files = .job.ad
+nrao_output_files = "working products"
when_to_transfer_output = ON_EXIT
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
{{#radial}}
universe = grid
grid_resource = condor radialhead.nrao.radial.local radialhead.nrao.radial.local
+remote_jobuniverse = 5
+remote_requirements = True
+remote_ShouldTransferFiles = "YES"
+remote_WhenToTransferOutput = "ON_EXIT"
{{/radial}}
request_memory = 31G
request_disk = 100G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True) + (HasLustre =!= True)
{{/remote}}
queue
"""
new_vlass_ql_envoy_condor = """executable = vlass_ql_envoy.sh
arguments = metadata.json PPR.xml {{request_id}}
output = envoy.out
error = envoy.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
VLASS_DIR = {{data_location}}
should_transfer_files = yes
transfer_input_files = {{#radial}}$ENV(HOME)/.ssh/condor_ssh_config, {{/radial}}$ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json{{files_to_transfer}}
transfer_output_files = .job.ad
+nrao_output_files = "working products"
when_to_transfer_output = ON_EXIT
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
{{#radial}}
universe = grid
grid_resource = condor radialhead.nrao.radial.local radialhead.nrao.radial.local
+remote_jobuniverse = 5
+remote_requirements = True
+remote_ShouldTransferFiles = "YES"
+remote_WhenToTransferOutput = "ON_EXIT"
{{/radial}}
request_memory = 31G
request_disk = 100G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True) && (HasLustre =!= True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) && (TARGET.VLASSTEST =!= True)
{{/remote}}
queue
"""
old_vlass_coarse_envoy_condor = """executable = vlass_coarse_envoy.sh
arguments = metadata.json PPR.xml {{requested_parallel}}
output = envoy.out
error = envoy.err
log = condor.log
VLASS_DIR = {{data_location}}
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
VLASS_BIN = /lustre/aoc/cluster/pipeline/vlass_{{vlass_env}}/workspaces/bin
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_BIN)/planescraper, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
when_to_transfer_output = ON_EXIT
transfer_output_files = .job.ad
+nrao_output_files = "working products planes.json"
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
request_cpus = {{requested_parallel}}
request_memory = 150G
request_disk = 950G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) + (TARGET.VLASSTEST =!= True) + (HasLustre =!= True)
{{/remote}}
queue
"""
new_vlass_coarse_envoy_condor = """executable = vlass_coarse_envoy.sh
arguments = metadata.json PPR.xml {{requested_parallel}}
output = envoy.out
error = envoy.err
log = condor.log
VLASS_DIR = {{data_location}}
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
VLASS_BIN = /lustre/aoc/cluster/pipeline/vlass_{{vlass_env}}/workspaces/bin
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/.matplotlib, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/update_stage, nraorsync://$(SBIN_PATH)/vela, nraorsync://$(SBIN_PATH)/casa_envoy, nraorsync://$(VLASS_BIN)/planescraper, nraorsync://$(VLASS_DIR)/working, nraorsync://$(VLASS_DIR)/rawdata, nraorsync://$(VLASS_DIR)/products{{#remote}}, nraorsync://$(VLASS_DIR)/{{profile}}.properties{{/remote}}, nraorsync://$(VLASS_DIR)/PPR.xml, nraorsync://$(VLASS_DIR)/metadata.json
when_to_transfer_output = ON_EXIT
transfer_output_files = .job.ad
+nrao_output_files = "working products planes.json"
output_destination = nraorsync://$(VLASS_DIR)
+WantIOProxy = True
request_cpus = {{requested_parallel}}
request_memory = 150G
request_disk = 950G
getenv = True
{{^remote}}
environment = "CAPO_PATH=/home/casa/capo"
requirements = (VLASS == True) && (HasLustre == True)
+partition = "VLASS"
{{/remote}}
{{#remote}}
requirements = (VLASS == True) && (HasLustre =!= True)
+partition = "VLASS"
Rank = (TARGET.VLASS == True) && (TARGET.VLASSTEST =!= True)
{{/remote}}
queue
"""
def upgrade():
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{new_vlass_seci_envoy_condor}'
WHERE filename='vlass_seci_envoy.condor'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{new_vlass_ql_envoy_condor}'
WHERE filename='vlass_ql_envoy.condor'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{new_vlass_coarse_envoy_condor}'
WHERE filename='vlass_coarse_envoy.condor'
"""
)
def downgrade():
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{old_vlass_seci_envoy_condor}'
WHERE filename='vlass_seci_envoy.condor'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{old_vlass_ql_envoy_condor}'
WHERE filename='vlass_ql_envoy.condor'
"""
)
op.execute(
f"""
UPDATE workflow_templates
SET content=E'{old_vlass_coarse_envoy_condor}'
WHERE filename='vlass_coarse_envoy.condor'
"""
)
shared/workspaces/alembic/versions/2a8ba99a1250_.py 0 → 100644
+ 24
− 0
View file @ cee3a364
"""empty message
Revision ID: 2a8ba99a1250
Revises: 12d0499d232d, f49af22c5c0e
Create Date: 2024-02-21 16:04:33.999785
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2a8ba99a1250'
down_revision = ('12d0499d232d', 'f49af22c5c0e')
branch_labels = None
depends_on = None
def upgrade():
pass
def downgrade():
pass
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment