WS-528: Fix HTCondor environment for `ingest`

apps/cli/executables/wf_framework/sh/ingest-request.sh

@@ -28,7 +28,7 @@ case $option in
   ;;
 esac
 
-WORKFLOW_SERVICE=$(capo -q edu.nrao.workspaces.WorkflowSettings.serviceUrl)
+WORKFLOW_SERVICE=$(pycapo -q edu.nrao.workspaces.WorkflowSettings.serviceUrl)
 
 if [ "$action" = "ingest_cal" ]; then
     curl -X POST $WORKFLOW_SERVICE/workflows/std_calibration/requests/$2/ingest

docker-compose.local.yml

@@ -74,11 +74,9 @@ services:
       context: ./schema/
       dockerfile: Dockerfile.local
     command: ["./bin/run-migrations.sh", "dsoc-dev"]
-    depends_on:
-      - db
     volumes:
       - ./schema:/code/schema
-      - ~/.capo:/home/casa/capo
+      - ~/.capo:/home/ssa/capo
     profiles:
       - schema-dev
 

docs/source/known-issues.rst

@@ -83,10 +83,16 @@ Database
 Pipeline
 --------
 
--  Update the end-to-end test container to see how detailed we can be
+-  Update the end-to-end test container with new tests for all new pages/features
 
 -  Prevent `cleanup` stage from deleting tagged images when multiple pipelines are running; this issue causes `push` stage to fail
 
+-  Raise concurrent pipeline job limit (currently 6)
+
+-  Rule for canceling pipelines on a MR if a newer pipeline starts running
+
+-  "Testing on dev" flag that we can set to prevent pipelines (or maybe just deploy step?) from running on main (they'll just queue up and run when the flag is unset)
+
 Code Tweaks
 -----------
 

schema/versions/d295036a7c72_update_calibration_ingestion_oracle_and_.py

+"""update calibration ingestion oracle and mysql
+
+Revision ID: d295036a7c72
+Revises: f0f6d7be45e3
+Create Date: 2021-07-30 14:56:51.201331
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d295036a7c72"
+down_revision = "f0f6d7be45e3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    condor_content = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/calibration-table-collector.sh, ./metadata.json
+
+request_memory = 1G
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo CFLAGS=-I/usr/include/mysql LDFLAGS=-L/usr/lib64/mysql ORACLE_HOME=/home/ssa/share/oracle/instantclient_12_1 LD_LIBRARY_PATH=/home/ssa/share/oracle/instantclient_12_1 PATH=/bin:/usr/bin:$(PATH)/home/ssa/share/oracle/instantclient_12_1"
+
+queue
+    """
+    op.execute(
+        """
+        DELETE FROM workflow_templates WHERE filename='ingest_cal.condor'
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('ingest_cal.condor', E'{condor_content}', 'ingest_cal') 
+        """
+    )
+
+
+def downgrade():
+    old_condor_content = """executable = ingest_cal.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/calibration-table-collector.sh, ./metadata.json
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+    """
+    op.execute(
+        """
+        DELETE FROM workflow_templates WHERE filename='ingest_cal.condor'
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('ingest_cal.condor', E'{old_condor_content}', 'ingest_cal') 
+        """
+    )

services/workflow/Dockerfile

@@ -9,7 +9,7 @@ ENV ENV=${env}
 USER root
 
 # HTCondor install
-RUN apt update && apt install -y wget gnupg sudo
+RUN apt update && apt install -y wget gnupg sudo curl
 RUN wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | sudo apt-key add -
 RUN echo "deb http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" > /etc/apt/sources.list.d/htcondor.list
 RUN echo "deb-src http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list

services/workflow/Dockerfile.local

@@ -14,7 +14,7 @@ RUN python setup.py develop --user
 USER root
 
 # HTCondor install
-RUN apt update && apt install -y wget gnupg sudo
+RUN apt update && apt install -y wget gnupg sudo curl
 RUN wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | sudo apt-key add -
 RUN echo "deb http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" > /etc/apt/sources.list.d/htcondor.list
 RUN echo "deb-src http://research.cs.wisc.edu/htcondor/debian/8.8/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list