Skip to content
Snippets Groups Projects
Commit 8833de56 authored by Daniel Nemergut's avatar Daniel Nemergut
Browse files

No longer transferring files into the container, running from lustre directly

parent 547994a5
No related branches found
No related tags found
1 merge request!1480Ingest obs run on lustre
Pipeline #12288 passed
This commit is part of merge request !1480. Comments created here will be created in the context of that merge request.
......@@ -24,16 +24,11 @@ output = ingest.out
error = ingest.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
SPOOL_DIR = {{spool_dir}}
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/observation-product-collector.sh, ./metadata.json
should_transfer_files = NO
+WantIOProxy = True
request_memory = {{ramInGb}}
getenv = True
environment = "CAPO_PATH=/home/casa/capo"
requirements = HasLustre == True
queue
......@@ -43,7 +38,9 @@ queue
ingest_obs_sh = """#!/bin/sh
set -o errexit
./ingest_envoy --observation $1 $2
SBIN_PATH=/lustre/aoc/cluster/pipeline/$CAPO_PROFILE/workspaces/sbin
${SBIN_PATH}/ingest_envoy --observation $1 $2
"""
......
......@@ -18,7 +18,9 @@ depends_on = None
old_ingest_obs_sh = """#!/bin/sh
set -o errexit
./ingest_envoy --observation $1 $2
SBIN_PATH=/lustre/aoc/cluster/pipeline/$CAPO_PROFILE/workspaces/sbin
${SBIN_PATH}/ingest_envoy --observation $1 $2
"""
......@@ -30,9 +32,11 @@ running=$2.running
ingested=$2.ingested
failed=$2.failed
SBIN_PATH=/lustre/aoc/cluster/pipeline/$CAPO_PROFILE/workspaces/sbin
mv $2 $running
./ingest_envoy --observation $1 $running
${SBIN_PATH}/ingest_envoy --observation $1 $running
status=$?
[ $status -eq 0 ] && mv $running $ingested || mv $running $failed
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment