Lustre locking the remaining ingest workflows to be consistent

shared/workspaces/alembic/versions/8aa23551a0fa_lustre_lock_ingests.py

+"""lustre lock ingests
+
+Revision ID: 8aa23551a0fa
+Revises: 5939146da7bb
+Create Date: 2023-10-26 22:09:52.624117
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8aa23551a0fa'
+down_revision = '5939146da7bb'
+branch_labels = None
+depends_on = None
+
+ingest_cal_condor_old = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/conveyor, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/calibration-table-collector.sh, nraorsync://{{spool_dir}}/metadata.json
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1"
+
+requirements = HasLustre == True
+
+queue
+
+"""
+
+ingest_cal_sh_old = """#!/bin/sh
+set -o errexit
+
+./conveyor --retrieve $1
+./ingest_envoy --calibration $1
+
+"""
+
+ingest_cal_condor_new = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1"
+
+requirements = HasLustre == True
+
+queue
+
+"""
+
+ingest_cal_sh_new = """#!/bin/sh
+set -o errexit
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+
+${SBIN_PATH}/conveyor --retrieve $1
+${SBIN_PATH}/ingest_envoy --calibration $1
+
+"""
+
+
+ingest_image_condor_old = """executable = ingest_image.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{spool_dir}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/conveyor, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/image-product-collector.sh, nraorsync://$(SPOOL_DIR)/metadata.json, nraorsync://$(SPOOL_DIR)/aux_image_metadata.json
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+
+"""
+
+ingest_image_sh_old = """#!/bin/sh
+set -o errexit
+
+./conveyor --retrieve-img $1
+./ingest_envoy --image $1
+
+"""
+
+ingest_image_condor_new = """executable = ingest_image.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+
+"""
+
+ingest_image_sh_new = """#!/bin/sh
+set -o errexit
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+
+${SBIN_PATH}/conveyor --retrieve-img $1
+${SBIN_PATH}/ingest_envoy --image $1
+
+"""
+
+
+ingest_seci_condor_old = """executable = ingest_seci.sh
+arguments = {{cal_spl}} {{cache_path}}
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+SPOOL_DIR = {{spool_dir}}
+should_transfer_files = yes
+transfer_input_files = $ENV(HOME)/.ssh/condor_transfer, nraorsync://$(SBIN_PATH)/pycapo, nraorsync://$(SBIN_PATH)/ingest_envoy, nraorsync://$(SBIN_PATH)/ingest, nraorsync://$(SBIN_PATH)/seci-product-collector.sh, nraorsync://$(SPOOL_DIR)/aux_image_metadata.json
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+
+"""
+
+ingest_seci_sh_old = """#!/bin/sh
+set -o errexit
+
+./ingest_envoy --seci $1 $2
+
+"""
+
+ingest_seci_condor_new = """executable = ingest_seci.sh
+arguments = {{cal_spl}} {{cache_path}}
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+should_transfer_files = NO
++WantIOProxy = True
+
+request_memory = {{ramInGb}}
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+requirements = HasLustre == True
+
+queue
+
+"""
+
+ingest_seci_sh_new = """#!/bin/sh
+set -o errexit
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+
+${SBIN_PATH}/ingest_envoy --seci $1 $2
+
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_cal_condor_new}'
+        WHERE filename = 'ingest_cal.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_cal_sh_new}'
+        WHERE filename = 'ingest_cal.sh'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_image_condor_new}'
+        WHERE filename = 'ingest_image.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_image_sh_new}'
+        WHERE filename = 'ingest_image.sh'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_seci_condor_new}'
+        WHERE filename = 'ingest_seci.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_seci_sh_new}'
+        WHERE filename = 'ingest_seci.sh'
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_cal_condor_old}'
+        WHERE filename = 'ingest_cal.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_cal_sh_old}'
+        WHERE filename = 'ingest_cal.sh'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_image_condor_old}'
+        WHERE filename = 'ingest_image.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_image_sh_old}'
+        WHERE filename = 'ingest_image.sh'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_seci_condor_old}'
+        WHERE filename = 'ingest_seci.condor'
+        """
+    )
+
+    op.execute(
+        f"""
+        UPDATE workflow_templates
+        SET content = E'{ingest_seci_sh_old}'
+        WHERE filename = 'ingest_seci.sh'
+        """
+    )