Skip to content
Snippets Groups Projects
Commit 42c67f67 authored by Daniel Nemergut's avatar Daniel Nemergut Committed by Charlotte Hausman
Browse files

(Don't migrate) made new curator workflow, input files are still incomplete

parent 09360686
No related branches found
No related tags found
3 merge requests!1571catch 2.8.2.3 up with main,!15592.8.2.2-DEVELOPMENT into main,!1551Cherry pick partial curation for patch
"""add curator workflow
Revision ID: 3eae1178cace
Revises: 61cbcd1d83f7
Create Date: 2023-10-12 11:25:34.165586
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3eae1178cace'
down_revision = '61cbcd1d83f7'
branch_labels = None
depends_on = None
wf_name = "curator"
# TODO: What input files?
curator_condor = """executable = curator.sh
arguments = metadata.json {{curator_type}}
output = curator.out
error = curator.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
SPOOL_DIR = {{spool_dir}}
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/observation-product-collector.sh, ./metadata.json
+WantIOProxy = True
request_memory = {{ramInGb}}
getenv = True
environment = "CAPO_PATH=/home/casa/capo"
requirements = HasLustre == True
queue
"""
curator_sh = """#!/bin/sh
set -o errexit
./ingest_envoy curate --$2 $1
"""
metadata_json = """{
"product_locator": "{{product_locator}}",
"data_location": "{{data_location}}",
"product_type": "{{product_type}}",
"target_list": ["{{target_list}}"],
"projectMetadata": {
"telescope": "{{telescope}}",
"projectCode": "{{project_code}}"
}
}
"""
def upgrade():
op.execute(
f"""
INSERT INTO workflows (workflow_name, requires_lustre) VALUES (E'{wf_name}', true)
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('curator.condor', E'{curator_condor}', E'{wf_name}')
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('curator.sh', E'{curator_sh}', E'{wf_name}')
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('metadata.json', E'{metadata_json}', E'{wf_name}')
"""
)
def downgrade():
op.execute(
f"""
DELETE FROM workflow_templates WHERE workflow_name = E'{wf_name}'
"""
)
op.execute(
f"""
DELETE FROM workflows WHERE workflow_name = E'{wf_name}'
"""
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment