Small comment about implementation detail

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/utilities.py

@@ -75,6 +75,8 @@ class ScienceProduct:
 
     @classmethod
     def init_from_filename(cls, filename: str) -> ScienceProduct:
+        # TODO: Determine how we want ScienceProduct objects to be initialized. From filename? or from metadata
+        #  like above?
         raise NotImplementedError
 
     def get_sp_pattern(self):

apps/cli/executables/pexable/ingest_envoy/sh/calibration-table-collector.sh

+#!/usr/bin/env bash
+#
+#  A replacement for the old qaarchive script used by the
+#  data analysts.  This script will tar & zip the pipeline
+#  weblog into WEBLOG.tar.gz and then collect everything but
+#  FITS files in the products directory from a CIPL run into
+#  a single tar file (naming convention TBD) that is created
+#  in a storage directory for ingestion.
+#
+#  Arguments:
+#       1: Working Directory in qa2 to be worked upon
+#       2: Filename (if different from the above)
+#
+#
+#Basics: Path modification
+set -o errexit -o nounset -o xtrace
+
+#
+# command line argument
+#
+# The tar file will be named after the working directory it came from
+# which preserves the processing time information.
+#
+SUBDIRECTORY=$1;shift
+PROFILE=$1;shift
+
+
+
+# Get the qa2, spool, and staging paths from CAPO
+SPOOL_DIR=$(capo -P ${PROFILE} -q edu.nrao.archive.workflow.config.CiplWorkflowSettings.spoolDirectory)
+STAGE_DIR=$(capo -P ${PROFILE} -q edu.nrao.archive.workflow.config.CiplWorkflowSettings.stageDirectory)
+STORE_DIR=$(capo -P ${PROFILE} -q edu.nrao.archive.workflow.config.CiplWorkflowSettings.storageDirectory)
+
+#Yet More VLASS Specialness
+if [[ ${PROFILE} != vlass* ]]
+then
+    QA2_DIR=$(capo -P ${PROFILE} -q edu.nrao.archive.workflow.config.CiplWorkflowSettings.qaDirectory)
+    FILENAME=${SUBDIRECTORY}
+else
+    # For VLASS, they don't want the data moved between qa2/ and spool/
+    QA2_DIR=${SPOOL_DIR}
+    # They also provide an extra layer of directory within the filename.
+    IFS='/' # redefine the character on which to split
+    read -ra COMPONENTS <<< "${SUBDIRECTORY}"
+    IFS=' ' # reset to default after
+    # We get:  calibration/VLASS1.1_stuff --> FILENAME becomes VLASS1.1_stuff  (in line with CIPL)
+    FILENAME=${COMPONENTS[1]}
+fi
+
+# Get the weblog caching directory from CAPO
+WEBLOG_CACHE=$(capo -P ${PROFILE} -q edu.nrao.archive.workflow.config.CiplWorkflowSettings.cacheWeblogDirectory)
+
+#
+# For the ability to reproduce results, we'll want the PPR.xml file.  Ensure it is
+# included in the products/ directory:
+#
+# TODO: Check for base_dir/products/*.pprequest.xml. If it exists, do nothing. If not, use base_dir/PPR.xml
+if [ ! -e ${QA2_DIR}/${SUBDIRECTORY}/products/PPR.xml ]
+then
+    cp ${QA2_DIR}/${SUBDIRECTORY}/working/PPR.xml ${QA2_DIR}/${SUBDIRECTORY}/products/PPR.xml
+fi
+
+#
+# The VLASS project wants the flux.csv file.  Check if it's here, if not, check for it in
+# the working directory parallel to this one.  Don't fail if we can't find it, however (so
+# we minimize the disruption to the CIPL system).
+#
+if [ ! -e ${QA2_DIR}/${SUBDIRECTORY}/products/flux.csv ]
+then
+    if [ -e ${QA2_DIR}/${SUBDIRECTORY}/working/flux.csv ]
+    then
+        cp ${QA2_DIR}/${SUBDIRECTORY}/working/flux.csv ${QA2_DIR}/${SUBDIRECTORY}/products/flux.csv
+    else
+        echo "No flux.csv file found here or in parallel working directory.  Continuing."
+    fi
+fi
+
+
+#
+# Both the pipeline-YYYMMDDTHHMMSS directory and weblog.tgz should exist.  We prefer the
+# directory (in case of updates/edits), but fall back on the tgz file.
+#
+# Check that they're both home, as we expect
+WEBLOG_DIR=$(ls -t ${QA2_DIR}/${SUBDIRECTORY}/products | grep pipeline- | head -1)
+if [ -n "$WEBLOG_DIR" ]
+then
+    # if weblog.tgz exists, we want to remove it
+    if [ -e ${QA2_DIR}/${SUBDIRECTORY}/products/weblog.tgz ]
+    then
+        rm -f ${QA2_DIR}/${SUBDIRECTORY}/products/weblog.tgz
+    fi
+
+    # Tar & Zip the weblog
+    tar -C${QA2_DIR}/${SUBDIRECTORY}/products -czf ${QA2_DIR}/${SUBDIRECTORY}/products/weblog.tgz ${WEBLOG_DIR}
+
+    if [ $? -ne 0 ]
+    then
+        echo "Creation of weblog.tgz failed, exiting"
+        exit -1
+    fi
+else
+    # no weblog directory.  If there's no weblog.tgz file, there's an issue: Issue a warning
+    if [ ! -e ${QA2_DIR}/${SUBDIRECTORY}/products/weblog.tgz ]
+    then
+        echo "Neither weblog.tgz or the weblog directory exist, continuing"
+    fi
+fi
+
+
+
+#
+# Sanity checks: create a staging subdirectory for this cal, and if the file already exists, remove it.
+#
+mkdir -p ${STAGE_DIR}/${SUBDIRECTORY}
+if [ -e ${STAGE_DIR}/${SUBDIRECTORY}/${FILENAME}.tar ]
+then
+    echo "Calibration Tar File Already Exists!  Removing the file for recreation"
+    #We could rename them with a version ...
+    #FILENAME=${SUBDIRECTORY}.$(ls -1 ${STAGE_DIR}/${SUBDIRECTORY} | wc -l)
+    # if we rename it... how do we tell the workflow?
+    /bin/rm -f ${STAGE_DIR}/${SUBDIRECTORY}/${FILENAME}.tar
+fi
+
+#
+# tar all non-fits and non-weblog-related files into a tar archive in the storage path
+# SSA-6115:  Don't exclude the weblog.tgz: Users and DAs prefer it bundled in.
+#
+tar --exclude=\*.fits --exclude=pipeline-\* -C${QA2_DIR}/${SUBDIRECTORY} -cvf ${STAGE_DIR}/${SUBDIRECTORY}/${FILENAME}.tar products
+if [ $? -ne 0 ]
+then
+    echo "Creation of main tar file failed, exiting"
+    exit -1
+fi
+
+#
+# Copy the weblog over, for ingestion as an ancillary file
+#
+/bin/cp -f ${QA2_DIR}/${SUBDIRECTORY}/products/weblog.tgz ${STAGE_DIR}/${SUBDIRECTORY}
+if [ $? -ne 0 ]
+then
+    echo "Copy of the weblog to staging location failed.  Exiting."
+    exit -1
+fi
+
+#
+# To stay consistent with current working methods: Copy from STAGE_DIR to STORE_DIR
+#
+cp ${STAGE_DIR}/${SUBDIRECTORY}/${FILENAME}.tar ${STORE_DIR}
+if [ $? -ne 0 ]
+then
+    # If something goes wrong, make some noise, but continue on.
+    echo "Failed to copy the calibration to ${STORE_DIR}, continuing."
+    $? = 0
+fi
+
+# Move subdirectories to the /spool/ copy of this directory
+# if it exists, otherwise, just move what we have to /spool/
+#
+# If this is an ingestion for VLASS, don't move anything.
+#
+if [[ ${PROFILE} != vlass* ]]
+then
+    if [ -e ${SPOOL_DIR}/${SUBDIRECTORY} ]
+    then
+        # Our base directory with the outputlogs is there, move our subdirectories back
+        /bin/mv -f ${QA2_DIR}/${SUBDIRECTORY}/products ${SPOOL_DIR}/${SUBDIRECTORY}
+        /bin/mv -f ${QA2_DIR}/${SUBDIRECTORY}/rawdata ${SPOOL_DIR}/${SUBDIRECTORY}
+        /bin/mv -f ${QA2_DIR}/${SUBDIRECTORY}/working ${SPOOL_DIR}/${SUBDIRECTORY}
+        # Cleanup the QA2 area
+        /bin/rm -rf ${QA2_DIR}/${SUBDIRECTORY}
+    else
+        #if no old directory exists, just move the whole thing back
+        /bin/mv -f ${QA2_DIR}/${SUBDIRECTORY} ${SPOOL_DIR}
+    fi
+fi