Skip to content
Snippets Groups Projects

Mark4 ingest obs run on lustre

Merged Daniel Nemergut requested to merge mark4_ingest_on_lustre into main
2 files
+ 13
12
Compare changes
  • Side-by-side
  • Inline
Files
2
@@ -24,16 +24,11 @@ output = ingest.out
error = ingest.err
log = condor.log
SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
SPOOL_DIR = {{spool_dir}}
should_transfer_files = yes
transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo
should_transfer_files = NO
+WantIOProxy = True
request_memory = {{ramInGb}}
getenv = True
environment = "CAPO_PATH=/home/casa/capo"
requirements = HasLustre == True
queue
@@ -43,13 +38,15 @@ queue
ingest_mk_four_obs_sh = """#!/bin/sh
set -o errexit
SBIN_PATH=/lustre/aoc/cluster/pipeline/$CAPO_PROFILE/workspaces/sbin
# Get NGAS hosts and set up variables to randomly select one
NGASHOSTSTR=$(./pycapo -q archive-ingestion.NGASHosts)
NGASHOSTSTR=$($SBIN_PATH/pycapo -q archive-ingestion.NGASHosts)
NGASHOSTARR=(`/bin/echo ${NGASHOSTSTR}`) # Put the space-delimited host list into an array
NGASHOSTLEN=${#NGASHOSTARR[@]}
# Copy from the difx area to the Workspaces staging area
WSSTAGINGDIR=$(./pycapo -q edu.nrao.workspaces.IngestionSettings.stagingDirectory)
WSSTAGINGDIR=$($SBIN_PATH/pycapo -q edu.nrao.workspaces.IngestionSettings.stagingDirectory)
/bin/cp -r $1 $WSSTAGINGDIR
OBSDIR=$(/bin/basename $1)
Loading