Skip to content
Snippets Groups Projects
Commit 89d32369 authored by Daniel Nemergut's avatar Daniel Nemergut
Browse files

The GMVA ingest tool is intended toe run manually so we can use the ingest_obs...

The GMVA ingest tool is intended toe run manually so we can use the ingest_obs workflow for GMVA ingestion
parent 52424801
No related branches found
No related tags found
2 merge requests!1452Merge 2.8.2 to main,!1429WS-1808 VLBA and GMVA ingest workflows
Pipeline #11362 passed
"""add ingest gmva workflow
Revision ID: 7c17edbcf541
Revises: 491102d56809
Create Date: 2023-08-29 23:53:59.341314
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '7c17edbcf541'
down_revision = '491102d56809'
branch_labels = None
depends_on = None
wf_name = "ingest_gmva"
ingest_gmva_condor = """executable = ingest_gmva.sh
arguments = metadata.json {{data_location}} {{directory_name}}
output = ingest.out
error = ingest.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
SPOOL_DIR = {{spool_dir}}
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/gmva_ingester, ./metadata.json
+WantIOProxy = True
request_memory = {{ramInGb}}
getenv = True
environment = "CAPO_PATH=/home/casa/capo"
requirements = HasLustre == True
queue
"""
ingest_gmva_sh = """#!/bin/sh
set -o errexit
./gmva_ingester $3
./ingest_envoy --observation $1 $2
"""
metadata_json = """{
"telescope": "{{telescope}}",
"data_location": "{{data_location}}",
"projectMetadata": {
"telescope": "{{telescope}}",
"projectCode": "{{projectCode}}"
}
}
"""
def upgrade():
op.execute(
f"""
INSERT INTO workflows (workflow_name) VALUES (E'{wf_name}')
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('ingest_gmva.condor', E'{ingest_gmva_condor}', E'{wf_name}')
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('ingest_gmva.sh', E'{ingest_gmva_sh}', E'{wf_name}')
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('metadata.json', E'{metadata_json}', E'{wf_name}')
"""
)
def downgrade():
op.execute(
f"""
DELETE FROM workflow_templates WHERE workflow_name = E'{wf_name}'
"""
)
op.execute(
f"""
DELETE FROM workflows WHERE workflow_name = E'{wf_name}'
"""
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment