Wrote migration to fix mark4-ingestion directory renaming

shared/workspaces/alembic/versions/dc7a1598b40b_fix_mark4_renaming.py

+"""fix mark4 renaming
+
+Revision ID: dc7a1598b40b
+Revises: ddd1912dd17b
+Create Date: 2023-12-04 11:44:43.741026
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "dc7a1598b40b"
+down_revision = "ddd1912dd17b"
+branch_labels = None
+depends_on = None
+
+old_ingest_mk_four_obs_sh = """#!/bin/sh
+
+ingested=$1.ingested
+failed=$1.failed
+
+SBIN_PATH=/lustre/aoc/cluster/pipeline/$CAPO_PROFILE/workspaces/sbin
+
+# Are we running live NGAS ingestion or testing only? check what is feeding into the manifests
+ISLIVE=$($SBIN_PATH/pycapo -q archive-ingestion.ingestNGAS)
+
+# Get NGAS hosts and set up variables to randomly select one
+NGASHOSTSTR=$($SBIN_PATH/pycapo -q archive-ingestion.NGASHosts)
+NGASHOSTARR=(`/bin/echo ${NGASHOSTSTR}`) # Put the space-delimited host list into an array
+NGASHOSTLEN=${#NGASHOSTARR[@]}
+
+# Copy from the difx area to the Workspaces staging area
+WSSTAGINGDIR=$($SBIN_PATH/pycapo -q edu.nrao.workspaces.IngestionSettings.stagingDirectory)
+/bin/cp -r $1 $WSSTAGINGDIR
+
+OBSDIR=$(/bin/basename $1)
+/bin/chmod -R 750 $WSSTAGINGDIR/$OBSDIR # Make sure NGAS has permissions to ingest the files
+cd $WSSTAGINGDIR/$OBSDIR
+
+for FILE in *; do
+    # Pick random NGAS host to distribute the ingestion load
+    NGASINDEX=$(($RANDOM % $NGASHOSTLEN))
+    NGASHOST=${NGASHOSTARR[$NGASINDEX]}
+
+    FULLPATH=$(/bin/readlink -f $FILE)
+    NGASCMD="${NGASHOST}ARCHIVE?filename=file://${FULLPATH}"
+
+    if [[ $ISLIVE = true ]]; then
+        INGESTOUT=$(/bin/curl $NGASCMD)
+        if echo $INGESTOUT | grep -i "error"; then
+            echo "Failed to ingest ${FILE}"
+            mv $1 $failed
+            exit 1
+        fi
+    else
+        echo "Test Only! ${NGASCMD}"
+    fi
+done
+
+mv $1 $ingested
+"""
+
+new_ingest_mk_four_obs_sh = """#!/bin/sh
+
+ingested=$1.ingested
+failed=$1.failed
+DID_SUCCEED=true
+
+SBIN_PATH=/lustre/aoc/cluster/pipeline/$CAPO_PROFILE/workspaces/sbin
+
+# Tell if any (non-`if`-related, non-||\\'d, non-&&\\'d) command exited with non-0
+trap \\'DID_SUCCEED=false\\' ERR
+
+# Are we running live NGAS ingestion or testing only? check what is feeding into the manifests
+ISLIVE=$($SBIN_PATH/pycapo -q archive-ingestion.ingestNGAS.observation)
+# Don\\'t consider the script to have failed if the above CAPO setting isn\\'t found
+DID_SUCCEED=true
+
+# Get NGAS hosts and set up variables to randomly select one
+NGASHOSTSTR=$($SBIN_PATH/pycapo -q archive-ingestion.NGASHosts)
+NGASHOSTARR=(`/bin/echo ${NGASHOSTSTR}`) # Put the space-delimited host list into an array
+NGASHOSTLEN=${#NGASHOSTARR[@]}
+
+if [[ $DID_SUCCEED = true ]]; then
+    # Copy from the difx area to the Workspaces staging area
+    WSSTAGINGDIR=$($SBIN_PATH/pycapo -q edu.nrao.workspaces.IngestionSettings.stagingDirectory)
+    /bin/cp -r $1 $WSSTAGINGDIR
+
+    OBSDIR=$(/bin/basename $1)
+    /bin/chmod -R 750 $WSSTAGINGDIR/$OBSDIR # Make sure NGAS has permissions to ingest the files
+    cd $WSSTAGINGDIR/$OBSDIR
+
+    if $DID_SUCCEED; then
+        for FILE in *; do
+            # Pick random NGAS host to distribute the ingestion load
+            NGASINDEX=$(($RANDOM % $NGASHOSTLEN))
+            NGASHOST=${NGASHOSTARR[$NGASINDEX]}
+
+            FULLPATH=$(/bin/readlink -f $FILE)
+            NGASCMD="${NGASHOST}ARCHIVE?filename=file://${FULLPATH}"
+
+            if [[ $ISLIVE = true ]]; then
+                INGESTOUT=$(/bin/curl $NGASCMD)
+                    if echo $INGESTOUT | grep -i "error"; then
+                        echo "Failed to ingest ${FILE}"
+                        DID_SUCCEED=false
+                        break
+                    fi
+            else
+                echo "Test Only! ${NGASCMD}"
+            fi
+        done
+    fi
+fi
+
+if $DID_SUCCEED; then
+    mv $1 $ingested && \
+    exit 0
+else
+    mv $1 $failed
+    exit 1
+fi
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{new_ingest_mk_four_obs_sh}'
+        WHERE filename='ingest_mk_four_obs.sh'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content=E'{old_ingest_mk_four_obs_sh}'
+        WHERE filename='ingest_mk_four_obs.sh'
+        """
+    )