Skip to content
Snippets Groups Projects
Commit f81eb3b1 authored by Daniel Nemergut's avatar Daniel Nemergut
Browse files

(Don't migrate) made new curator workflow, input files are still incomplete

parent 9e7d864f
No related branches found
No related tags found
2 merge requests!1605Merge 2.8.2.3 work to main,!1493WS-1893 curator workflow
"""add curator workflow
Revision ID: 3eae1178cace
Revises: 61cbcd1d83f7
Create Date: 2023-10-12 11:25:34.165586
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3eae1178cace'
down_revision = '61cbcd1d83f7'
branch_labels = None
depends_on = None
wf_name = "curator"
# TODO: What input files?
curator_condor = """executable = curator.sh
arguments = metadata.json {{curator_type}}
output = curator.out
error = curator.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
SPOOL_DIR = {{spool_dir}}
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/observation-product-collector.sh, ./metadata.json
+WantIOProxy = True
request_memory = {{ramInGb}}
getenv = True
environment = "CAPO_PATH=/home/casa/capo"
requirements = HasLustre == True
queue
"""
curator_sh = """#!/bin/sh
set -o errexit
./ingest_envoy curate --$2 $1
"""
metadata_json = """{
"product_locator": "{{product_locator}}",
"data_location": "{{data_location}}",
"product_type": "{{product_type}}",
"target_list": ["{{target_list}}"],
"projectMetadata": {
"telescope": "{{telescope}}",
"projectCode": "{{project_code}}"
}
}
"""
def upgrade():
op.execute(
f"""
INSERT INTO workflows (workflow_name, requires_lustre) VALUES (E'{wf_name}', true)
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('curator.condor', E'{curator_condor}', E'{wf_name}')
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('curator.sh', E'{curator_sh}', E'{wf_name}')
"""
)
op.execute(
f"""
INSERT INTO workflow_templates (filename, content, workflow_name)
VALUES ('metadata.json', E'{metadata_json}', E'{wf_name}')
"""
)
def downgrade():
op.execute(
f"""
DELETE FROM workflow_templates WHERE workflow_name = E'{wf_name}'
"""
)
op.execute(
f"""
DELETE FROM workflows WHERE workflow_name = E'{wf_name}'
"""
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment