ci/build.template.yml

 # CI Build Template
 .build:
-    variables:
-        CAPO_BUILD_ARG: "docker"
     script:
         - echo "Building branch or tag -- ${IMAGE_TAG}"
         - NAME="${REGISTRY_URL}/${PROJECT_NAME}/${SERVICE_NAME}"
-        - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile.dev . --build-arg capo_env=${CAPO_BUILD_ARG} --build-arg TAGNAME=${CI_COMMIT_SHORT_SHA}
+        - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile . --build-arg env=${DEPLOY_ENV} --build-arg TAGNAME=${CI_COMMIT_SHORT_SHA}
         - docker tag ${NAME}:${CI_COMMIT_SHORT_SHA} ${NAME}:${IMAGE_TAG}
         - echo "TAG=${IMAGE_TAG}" >> build.env
     artifacts:
@@ -15,16 +13,19 @@
         - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
           variables:
             IMAGE_TAG: $CI_COMMIT_BRANCH
+            DEPLOY_ENV: "dev"
         - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
           variables:
             IMAGE_TAG: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+            DEPLOY_ENV: "dev"
+        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
         - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
           variables:
             IMAGE_TAG: $CI_COMMIT_TAG
             # override DEPLOY_ENV
             DEPLOY_ENV: "test"
-        # - if: '$CI_COMMIT_TAG =~ /^test_[0-9]+\.[0-9]+.[0-9]+$/'
-        #   variables:
-        #     IMAGE_TAG: $CI_COMMIT_TAG
-        #     # override DEPLOY_ENV
-        #     DEPLOY_ENV: "test"
\ No newline at end of file
+        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
+          variables:
+            IMAGE_TAG: $CI_COMMIT_TAG
+            # override DEPLOY_ENV
+            DEPLOY_ENV: "prod"

ci/push.template.yml

@@ -12,7 +12,7 @@
         - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
           variables:
             IMAGE_TAG: $CI_COMMIT_TAG
-        # - if: '$CI_COMMIT_TAG =~ /^test_[0-9]+\.[0-9]+.[0-9]+$/'
-        #   variables:
-        #     IMAGE_TAG: $CI_COMMIT_TAG
+        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
+          variables:
+            IMAGE_TAG: $CI_COMMIT_TAG
     dependencies: []
\ No newline at end of file

ci/unit-test.template.yml

@@ -10,4 +10,10 @@
     rules:
         - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
         - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
+          variables:
+            IMAGE_TAG: $CI_COMMIT_TAG
+        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
+          variables:
+            IMAGE_TAG: $CI_COMMIT_TAG
     dependencies: []

config/htcondor/submit/99-workspaces-submit.prod.conf

+CONDOR_HOST = nmpost-cm.aoc.nrao.edu
+
+use ROLE : Submit
+
+ALLOW_WRITE = *.aoc.nrao.edu
+ALLOW_DAEMON = $(ALLOW_WRITE)
+
+# one or more submit hosts has an external IP
+PRIVATE_NETWORK_NAME = nrao.edu
+
+# while not in production yet
+CONDOR_ADMIN = krowe@nrao.edu
+
+# send email if job errors
+JOB_DEFAULT_NOTIFICATION = ERROR
+
+# Differentiate from other pools if needed
+PoolName = "nmpost"
+
+# Set UID domain
+UID_DOMAIN = aoc.nrao.edu

development.md

@@ -22,24 +22,24 @@ command again unless you delete this image or modify the `Dockerfile.base` Docke
 ### Starting Workspaces
 To start Workspaces as foreground process, run:
 ```sh
-docker-compose up
+docker compose -f docker-compose.local.yml up
 ```
 
 To start Workspaces as background process, run:
 ```sh
-docker-compose up -d
+docker compose -f docker-compose.local.yml up -d
 ```
 
 ### Stopping Workspaces
 To stop Workspaces, run:
 ```sh
-docker-compose down
+docker compose -f docker-compose.local.yml down
 ```
 
 ### Starting Individual Services
 To start Workspaces as foreground process, run:
 ```sh
-docker-compose up <service_name>
+docker compose -f docker-compose.local.yml up <service_name>
 ```
 
 ### List Containers
@@ -71,7 +71,7 @@ docker exec -it <container_name> bash
 
 Using Docker Compose:
 ```sh
-docker-compose exec <service_name> bash
+docker compose -f docker-compose.local.yml exec <service_name> bash
 ```
 
 ### Run a command on a Running Container
@@ -82,5 +82,5 @@ docker exec <container_name> pwd
 
 Using Docker Compose:
 ```sh
-docker-compose exec <service_name> pwd
+docker compose -f docker-compose.local.yml exec <service_name> pwd
 ```

docker-compose.dev.yml

-version: '3.8'
-services:
-
-  workflow:
-    image: ssa-containers.aoc.nrao.edu/workspaces/workflow:${TAG}
-    networks:
-      - host
-    secrets:
-      - source: dsoc_${ENV}_secrets
-        target: dsoc-${ENV}.properties
-    environment:
-      CAPO_PATH: /run/secrets
-      CAPO_PROFILE: dsoc-${ENV}
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.node_env==${ENV}"
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-        window: 5s
-      update_config:
-        parallelism: 2
-        delay: 5s
-        order: stop-first
-        failure_action: rollback
-      rollback_config:
-        parallelism: 0
-        order: stop-first
-    volumes:
-      - /lustre/aoc/cluster/pipeline/dsoc-${ENV}/downloads/nrao:/lustre/aoc/cluster/pipeline/dsoc-${ENV}/downloads/nrao
-      - /lustre/aoc/cluster/pipeline/dsoc-${ENV}/workspaces:/lustre/aoc/cluster/pipeline/dsoc-${ENV}/workspaces
-      - /home/ssa/bin/python3.8:/home/ssa/bin/python3.8
-      - /home/casa/capo:/home/casa/capo
-
-  capability:
-    image: ssa-containers.aoc.nrao.edu/workspaces/capability:${TAG}
-    ports:
-      - target: 3457
-        published: 3457
-        protocol: tcp
-        mode: host
-    secrets:
-      - source: dsoc_${ENV}_secrets
-        target: dsoc-${ENV}.properties
-    environment:
-      CAPO_PATH: /run/secrets
-      CAPO_PROFILE: dsoc-${ENV}
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.node_env==${ENV}"
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-        window: 5s
-      update_config:
-        parallelism: 2
-        delay: 5s
-        order: stop-first
-        failure_action: rollback
-      rollback_config:
-        parallelism: 0
-        order: stop-first
-
-  notification:
-    image: ssa-containers.aoc.nrao.edu/workspaces/notification:${TAG}
-    ports:
-      - target: 3458
-        published: 3458
-        protocol: tcp
-        mode: host
-    secrets:
-      - source: dsoc_${ENV}_secrets
-        target: dsoc-${ENV}.properties
-    environment:
-      CAPO_PATH: /run/secrets
-      CAPO_PROFILE: dsoc-${ENV}
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.node_env==${ENV}"
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-        window: 5s
-      update_config:
-        parallelism: 2
-        delay: 5s
-        order: stop-first
-        failure_action: rollback
-      rollback_config:
-        parallelism: 0
-        order: stop-first
-
-  web:
-    image: ssa-containers.aoc.nrao.edu/workspaces/web:${TAG}
-    ports:
-      - target: 80
-        published: 4444
-        protocol: tcp
-        mode: host
-    environment:
-      DL_HOST: ${DL_HOST}
-      ENV_HOST: ${ENV_HOST}
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.node_env==${ENV}"
-      restart_policy:
-        condition: on-failure
-        max_attempts: 3
-        window: 5s
-      update_config:
-        parallelism: 2
-        delay: 5s
-        order: stop-first
-        failure_action: rollback
-      rollback_config:
-        parallelism: 0
-        order: stop-first
-
-
-secrets:
-  dsoc_ENV_secrets:
-    external: true
-
-networks:
-  host:
-    external: true
-    name: host

docker-compose.local.yml

+version: '3.8'
+services:
+  nginx:
+    image: nginx:1.19.7-alpine
+    restart: always
+    ports:
+      - "4444:80"
+    depends_on:
+      - frontend
+    command: nginx -g "daemon off;"
+    volumes: 
+      - ./apps/web/ws-nginx.local.conf:/etc/nginx/conf.d/default.conf
+      - ./lustre/:/lustre/aoc/cluster/pipeline/docker/workspaces
+      - ./delivery_root:/tmp/delivery_root
+  db:
+    image: ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces
+    restart: always
+    ports:
+      - "54322:5432"
+    environment:
+      POSTGRES_USER: archive
+      POSTGRES_PASSWORD: docker
+    command: postgres -c listen_addresses=*
+
+  amqp:
+    image: rabbitmq:3.8-management
+    restart: always
+    ports:
+      - "15672:15672"
+
+  condor-cm:
+    build:
+      context: .
+      dockerfile: ./config/htcondor/cm/Dockerfile.local
+    ports:
+      - 9618
+
+  condor-execute:
+    build:
+      context: .
+      dockerfile: ./config/htcondor/execute/Dockerfile.local
+    volumes:
+      - ./delivery_root:/tmp/delivery_root
+      - ./lustre/aoc/cluster/pipeline/docker/workspaces:/lustre/aoc/cluster/pipeline/docker/workspaces
+
+  ngas:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.ngas
+    depends_on:
+      - db
+    ports:
+      - "7777:7777"
+    volumes:
+      - ./local_ngas_root:/home/ngas/NGAS
+
+  schema:
+    build:
+      context: ./schema/
+      dockerfile: Dockerfile.local
+    depends_on:
+      - db
+    volumes:
+      - ./schema:/code/schema
+      - ./docker.properties:/home/ssa/capo/docker.properties
+
+  # Used for doing a DB migration on dsoc-dev
+  # Uncomment this and run the command `docker compose -f docker-compose.local.yml up schema-dev` and the migration will run
+#  schema-dev:
+#    build:
+#      context: ./schema/
+#      dockerfile: Dockerfile.local
+#    command: ["./bin/run-migrations.sh", "dsoc-dev"]
+#    depends_on:
+#      - db
+#    volumes:
+#      - ./schema:/code/schema
+#      - ~/.capo:/home/casa/capo
+
+  workflow:
+    build:
+      context: .
+      dockerfile: ./services/workflow/Dockerfile.local
+    ports:
+      - "3456:3456"
+      - 9618
+    depends_on:
+      - schema
+      - amqp
+    volumes:
+      - ./services/workflow:/code
+      - /code/ssa_workflow.egg-info
+      - ./shared:/packages/shared
+      - ./apps/cli:/packages/apps/cli
+      - ./testing:/packages/testing
+      - ./lustre/aoc/cluster/pipeline/docker/workspaces:/lustre/aoc/cluster/pipeline/docker/workspaces
+      - ./delivery_root:/tmp/delivery_root
+      - ~/.capo:/home/ssa/capo
+      - ./docker.properties:/home/ssa/capo/docker.properties
+      - /packages/shared/workspaces/ssa_workspaces.egg-info
+      - /packages/shared/schema/ssa_schema.egg-info
+      - /packages/shared/messaging/ssa_messaging.egg-info
+      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+      - /packages/apps/cli/executables/pexable/vela/ssa_vela.egg-info
+      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
+
+  capability:
+    build:
+      context: .
+      dockerfile: ./services/capability/Dockerfile.local
+    ports:
+      - "3457:3457"
+    depends_on:
+      - schema
+      - workflow
+      - notification
+      - amqp
+    volumes:
+      - ./docker.properties:/home/ssa/capo/docker.properties
+      - ./services/capability:/code
+      - ./shared:/packages/shared
+      - ./apps/cli:/packages/apps/cli
+      - ./testing:/packages/testing
+      - /code/ssa_capability.egg-info
+      - /packages/shared/workspaces/ssa_workspaces.egg-info
+      - /packages/shared/schema/ssa_schema.egg-info
+      - /packages/shared/messaging/ssa_messaging.egg-info
+      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+      - /packages/apps/cli/executables/pexable/vela/ssa_vela.egg-info
+      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
+
+  notification:
+    build:
+      context: .
+      dockerfile: ./services/notification/Dockerfile.local
+    ports:
+      - "3458:3458"
+    depends_on:
+      - schema
+      - amqp
+    volumes:
+      - ./docker.properties:/home/ssa/capo/docker.properties
+      - ./services/notification:/code
+      - /code/ssa_notification.egg-info
+      - ./shared:/packages/shared
+      - ./apps/cli:/packages/apps/cli
+      - ./testing:/packages/testing
+      - /packages/shared/workspaces/ssa_workspaces.egg-info
+      - /packages/shared/schema/ssa_schema.egg-info
+      - /packages/shared/messaging/ssa_messaging.egg-info
+      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+      - /packages/apps/cli/executables/pexable/vela/ssa_vela.egg-info
+      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
+
+  frontend:
+    build:
+      context: .
+      dockerfile: ./apps/web/Dockerfile.local
+    init: true
+    ports:
+      - "4200:4200"
+    depends_on:
+      - capability
+      - workflow
+    volumes:
+      - ./apps/web:/code
+      - /code/node_modules

docker-compose.yml

 version: '3.8'
 services:
-  nginx:
-    image: nginx:1.19.7-alpine
-    restart: always
-    ports:
-      - "4444:80"
-    depends_on:
-      - frontend
-    command: nginx -g "daemon off;"
-    volumes: 
-      - ./apps/web/ws-nginx.local.conf:/etc/nginx/conf.d/default.conf
-      - ./lustre/:/lustre/aoc/cluster/pipeline/docker/workspaces
-      - ./delivery_root:/tmp/delivery_root
-  db:
-    image: ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces
-    restart: always
-    ports:
-      - "54322:5432"
-    environment:
-      POSTGRES_USER: archive
-      POSTGRES_PASSWORD: docker
-    command: postgres -c listen_addresses=*
-
-  amqp:
-    image: rabbitmq:3.8-management
-    restart: always
-    ports:
-      - "15672:15672"
-
-  condor-cm:
-    build:
-      context: .
-      dockerfile: ./config/htcondor/cm/Dockerfile.local
-    ports:
-      - 9618
-
-  condor-execute:
-    build:
-      context: .
-      dockerfile: ./config/htcondor/execute/Dockerfile.local
-    volumes:
-      - ./delivery_root:/tmp/delivery_root
-      - ./lustre/aoc/cluster/pipeline/docker/workspaces:/lustre/aoc/cluster/pipeline/docker/workspaces
-
-  ngas:
-    build:
-      context: .
-      dockerfile: ./Dockerfile.ngas
-    depends_on:
-      - db
-    ports:
-      - "7777:7777"
-    volumes:
-      - ./local_ngas_root:/home/ngas/NGAS
-
-  schema:
-    build:
-      context: ./schema/
-      dockerfile: Dockerfile.local
-    depends_on:
-      - db
-    volumes:
-      - ./schema:/code/schema
-      - ./docker.properties:/home/ssa/capo/docker.properties
-
-  # Used for doing a DB migration on dsoc-dev
-  # Uncomment this and run the command `docker compose up schema-dev` and the migration will run
-#  schema-dev:
-#    build:
-#      context: ./schema/
-#      dockerfile: Dockerfile.local
-#    command: ["./bin/run-migrations.sh", "dsoc-dev"]
-#    depends_on:
-#      - db
-#    volumes:
-#      - ./schema:/code/schema
-#      - ~/.capo:/home/casa/capo
 
   workflow:
-    build:
-      context: .
-      dockerfile: ./services/workflow/Dockerfile.local
-    ports:
-      - "3456:3456"
-      - 9618
-    depends_on:
-      - schema
-      - amqp
+    image: ssa-containers.aoc.nrao.edu/workspaces/workflow:${TAG}
+    networks:
+      - host
+    secrets:
+      - source: dsoc_${ENV}_secrets
+        target: dsoc-${ENV}.properties
+    environment:
+      CAPO_PATH: /run/secrets
+      CAPO_PROFILE: dsoc-${ENV}
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.node_env==${ENV}"
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+        window: 5s
+      update_config:
+        parallelism: 2
+        delay: 5s
+        order: stop-first
+        failure_action: rollback
+      rollback_config:
+        parallelism: 0
+        order: stop-first
     volumes:
-      - ./services/workflow:/code
-      - /code/ssa_workflow.egg-info
-      - ./shared:/packages/shared
-      - ./apps/cli:/packages/apps/cli
-      - ./testing:/packages/testing
-      - ./lustre/aoc/cluster/pipeline/docker/workspaces:/lustre/aoc/cluster/pipeline/docker/workspaces
-      - ./delivery_root:/tmp/delivery_root
-      - ~/.capo:/home/ssa/capo
-      - ./docker.properties:/home/ssa/capo/docker.properties
-      - /packages/shared/workspaces/ssa_workspaces.egg-info
-      - /packages/shared/schema/ssa_schema.egg-info
-      - /packages/shared/messaging/ssa_messaging.egg-info
-      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
-      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
-      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
-      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
-      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
-      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
-      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
-      - /packages/apps/cli/executables/pexable/vela/ssa_vela.egg-info
-      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
-      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
+      - /lustre/aoc/cluster/pipeline/dsoc-${ENV}/downloads/nrao:/lustre/aoc/cluster/pipeline/dsoc-${ENV}/downloads/nrao
+      - /lustre/aoc/cluster/pipeline/dsoc-${ENV}/workspaces:/lustre/aoc/cluster/pipeline/dsoc-${ENV}/workspaces
+      - /home/ssa/bin/python3.8:/home/ssa/bin/python3.8
+      - /home/casa/capo:/home/casa/capo
 
   capability:
-    build:
-      context: .
-      dockerfile: ./services/capability/Dockerfile.local
+    image: ssa-containers.aoc.nrao.edu/workspaces/capability:${TAG}
     ports:
-      - "3457:3457"
-    depends_on:
-      - schema
-      - workflow
-      - notification
-      - amqp
-    volumes:
-      - ./docker.properties:/home/ssa/capo/docker.properties
-      - ./services/capability:/code
-      - ./shared:/packages/shared
-      - ./apps/cli:/packages/apps/cli
-      - ./testing:/packages/testing
-      - /code/ssa_capability.egg-info
-      - /packages/shared/workspaces/ssa_workspaces.egg-info
-      - /packages/shared/schema/ssa_schema.egg-info
-      - /packages/shared/messaging/ssa_messaging.egg-info
-      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
-      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
-      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
-      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
-      - /packages/apps/cli/executables/pexable/vela/ssa_vela.egg-info
-      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
-      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
-      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
-      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
-      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
+      - target: 3457
+        published: 3457
+        protocol: tcp
+        mode: host
+    secrets:
+      - source: dsoc_${ENV}_secrets
+        target: dsoc-${ENV}.properties
+    environment:
+      CAPO_PATH: /run/secrets
+      CAPO_PROFILE: dsoc-${ENV}
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.node_env==${ENV}"
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+        window: 5s
+      update_config:
+        parallelism: 2
+        delay: 5s
+        order: stop-first
+        failure_action: rollback
+      rollback_config:
+        parallelism: 0
+        order: stop-first
 
   notification:
-    build:
-      context: .
-      dockerfile: ./services/notification/Dockerfile.local
+    image: ssa-containers.aoc.nrao.edu/workspaces/notification:${TAG}
     ports:
-      - "3458:3458"
-    depends_on:
-      - schema
-      - amqp
-    volumes:
-      - ./docker.properties:/home/ssa/capo/docker.properties
-      - ./services/notification:/code
-      - /code/ssa_notification.egg-info
-      - ./shared:/packages/shared
-      - ./apps/cli:/packages/apps/cli
-      - ./testing:/packages/testing
-      - /packages/shared/workspaces/ssa_workspaces.egg-info
-      - /packages/shared/schema/ssa_schema.egg-info
-      - /packages/shared/messaging/ssa_messaging.egg-info
-      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
-      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
-      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
-      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
-      - /packages/apps/cli/executables/pexable/vela/ssa_vela.egg-info
-      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
-      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
-      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
-      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
-      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
+      - target: 3458
+        published: 3458
+        protocol: tcp
+        mode: host
+    secrets:
+      - source: dsoc_${ENV}_secrets
+        target: dsoc-${ENV}.properties
+    environment:
+      CAPO_PATH: /run/secrets
+      CAPO_PROFILE: dsoc-${ENV}
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.node_env==${ENV}"
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+        window: 5s
+      update_config:
+        parallelism: 2
+        delay: 5s
+        order: stop-first
+        failure_action: rollback
+      rollback_config:
+        parallelism: 0
+        order: stop-first
 
-  frontend:
-    build:
-      context: .
-      dockerfile: ./apps/web/Dockerfile.local
-    init: true
+  web:
+    image: ssa-containers.aoc.nrao.edu/workspaces/web:${TAG}
     ports:
-      - "4200:4200"
-    depends_on:
-      - capability
-      - workflow
-    volumes:
-      - ./apps/web:/code
-      - /code/node_modules
+      - target: 80
+        published: 4444
+        protocol: tcp
+        mode: host
+    environment:
+      DL_HOST: ${DL_HOST}
+      ENV_HOST: ${ENV_HOST}
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.node_env==${ENV}"
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+        window: 5s
+      update_config:
+        parallelism: 2
+        delay: 5s
+        order: stop-first
+        failure_action: rollback
+      rollback_config:
+        parallelism: 0
+        order: stop-first
+
+
+secrets:
+  dsoc_ENV_secrets:
+    external: true
+
+networks:
+  host:
+    external: true
+    name: host

docker.properties

@@ -12,19 +12,19 @@ edu.nrao.archive.configuration.AmqpServer.port     = 5672
 #
 # Workspaces settings
 #
-edu.nrao.archive.workspaces.CapabilitySettings.serviceUrl = http://capability:3457
-edu.nrao.archive.workspaces.WorkflowSettings.serviceUrl = http://workflow:3456
-edu.nrao.archive.workspaces.NotificationSettings.serviceUrl = http://notification:3458
-edu.nrao.archive.workspaces.UISettings.serviceUrl = http://localhost:4444/workspaces
-
+edu.nrao.workspaces.CapabilitySettings.serviceUrl = http://capability:3457
+edu.nrao.workspaces.WorkflowSettings.serviceUrl = http://workflow:3456
+edu.nrao.workspaces.NotificationSettings.serviceUrl = http://notification:3458
+edu.nrao.workspaces.UISettings.serviceUrl = http://localhost:4444/workspaces
 
 #
 # Processing Settings
 #
-edu.nrao.archive.workspaces.ProcessingSettings.useCasa = false
-edu.nrao.archive.workspaces.ProcessingSettings.rootDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/spool
-edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation = /packages/apps/cli/executables/wf_framework/sh
-edu.nrao.archive.workspaces.ProcessingSettings.ramInGb = 1G
+edu.nrao.workspaces.ProcessingSettings.useCasa = false
+edu.nrao.workspaces.ProcessingSettings.rootDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/spool
+edu.nrao.workspaces.ProcessingSettings.scriptLocation = /lustre/aoc/cluster/pipeline/docker/workspaces/sbin
+edu.nrao.workspaces.ProcessingSettings.ramInGb = 1G
+edu.nrao.workspaces.ProcessingSettings.autoGenerateStandardCals = False
 
 edu.nrao.archive.workflow.config.CasaVersions.homeForReprocessing = /home/casa/packages/pipeline/current
 
@@ -34,37 +34,31 @@ edu.nrao.archive.workflow.config.CasaVersions.homeForReprocessing = /home/casa/p
 edu.nrao.archive.workflow.config.DeliverySettings.nraoDownloadDirectory = /tmp/delivery_root
 edu.nrao.archive.workflow.config.DeliverySettings.nraoDownloadUrl = http://localhost:4444/dl
 
-
 #
 # Ingestion Settings
 #
-edu.nrao.archive.workspaces.IngestionSettings.stagingDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/staging
-edu.nrao.archive.workspaces.IngestionSettings.storageDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/storage
+edu.nrao.workspaces.IngestionSettings.stagingDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/staging
+edu.nrao.workspaces.IngestionSettings.storageDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/storage
 
 #
 # Standard Calibration Settings
 #
-edu.nrao.archive.workspaces.DeliverySettings.ciplDelivery =  /lustre/aoc/cluster/pipeline/docker/workspaces/qa2
-edu.nrao.archive.workspaces.DeliverySettings.cacheWeblogDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/cache/weblog
+edu.nrao.workspaces.DeliverySettings.standardCalibrationDelivery =  /lustre/aoc/cluster/pipeline/docker/workspaces/qa2
+edu.nrao.workspaces.DeliverySettings.cacheWeblogDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/cache/weblog
 
 #
 # Standard Imaging Settings
 #
-edu.nrao.archive.workspaces.DeliverySettings.standardImageDelivery = /lustre/aoc/cluster/pipeline/docker/workspaces/image-qa
-
+edu.nrao.workspaces.DeliverySettings.standardImageDelivery = /lustre/aoc/cluster/pipeline/docker/workspaces/image-qa
 
 #
-# Data fetcher settings
+# Product Fetcher settings
 #
-edu.nrao.archive.workflow.config.DataFetcherSettings.ramInGb = 16
-edu.nrao.archive.workflow.config.DataFetcherSettings.clusterTimeout = 01:00:00:00
-edu.nrao.archive.datafetcher.DataFetcherSettings.locatorServiceUrlPrefix = https://webtest.aoc.nrao.edu/archive-services/location?locator
-edu.nrao.archive.datafetcher.DataFetcherSettings.defaultThreadsPerHost = 4
-edu.nrao.archive.datafetcher.DataFetcherSettings.executionSite = local_test
-edu.nrao.archive.datafetcher.DataFetcherSettings.downloadDirectory=/tmp/
-edu.nrao.archive.workflow.config.RequestHandlerSettings.downloadDirectory=/tmp/
+edu.nrao.workspaces.ProductFetcherSettings.locatorServiceUrlPrefix = https://webtest.aoc.nrao.edu/archive-service/location?locator
+edu.nrao.workspaces.ProductFetcherSettings.defaultThreadsPerHost = 4
+edu.nrao.workspaces.ProductFetcherSettings.executionSite = local_test
 
 #
 # Notification Settings
 #
-edu.nrao.archive.workspaces.NotificationSettings.analystEmail = workspaces-analysts@listmgr.nrao.edu
+edu.nrao.workspaces.NotificationSettings.analystEmail = workspaces-analysts-test@listmgr.nrao.edu

docs/source/confluence/dockerized-setup.rst

@@ -36,7 +36,7 @@ make clean
 
 make build
 
-docker-compose up
+docker compose -f docker-compose.local.yml up
 
 make setup # in a different terminal window
 
@@ -119,7 +119,7 @@ needing to access them via Docker Desktop or terminal.
 
            |image2|
 
-Now you should be able to run 'docker-compose up' by simply selecting
+Now you should be able to run 'docker compose up' by simply selecting
 this configuration and clicking the green run arrow next to the drop
 down menu.
 
@@ -151,7 +151,7 @@ create one now.
 You should now be able to debug workspaces! To use this do the
 following:
 
-1. Make sure you've run docker-compose up and that your containers are
+1. Make sure you've run docker compose up and that your containers are
    running
 
 2. Set breakpoints at the lines you wish to investigate.
@@ -168,7 +168,7 @@ relaunch the debugger to pick it up.
 
 Also: Once you are done debugging, the debugger shuts down the attached
 container on exit. You will need to restart the container via
-docker-compose up <service>
+docker compose -f docker-compose.local.yml up <service>
 
 Debugging Workspaces Tests
 ^^^^^^^^^^^^^^^^^^^^^^^^^^

schema/README.md

@@ -8,4 +8,4 @@ where `$PROFILE` is `local` or `nmtest` or `nmprod` or whatever.
 
 To upgrade in docker, run:
 
-    docker-compose up schema
\ No newline at end of file
+    docker compose -f docker-compose.local.yml up schema

schema/versions/291edd508557_add_std_calibration_script.py

@@ -11,21 +11,24 @@ from pycapo import CapoConfig
 
 
 # revision identifiers, used by Alembic.
-revision = '291edd508557'
-down_revision = '16b40772e51a'
+revision = "291edd508557"
+down_revision = "16b40772e51a"
 branch_labels = None
 depends_on = None
 
 
 def get_script_location() -> str:
-    return CapoConfig().settings("edu.nrao.archive.workspaces.ProcessingSettings").scriptLocation
+    return CapoConfig().settings("edu.nrao.workspaces.ProcessingSettings").scriptLocation
 
 
 def upgrade():
     path = get_script_location()
-    script_body = """#!/bin/sh
+    script_body = (
+        """#!/bin/sh
 
-    export PATH=$PATH:""" + path + """
+    export PATH=$PATH:"""
+        + path
+        + """
 
     framework.sh -d .
     cd rawdata/
@@ -34,11 +37,14 @@ def upgrade():
     casa_envoy --standard-cal $3 $4
     deliver -r ./products --prefix $2
     """
+    )
 
     op.execute(
         """
         INSERT INTO workflow_templates (filename, content, workflow_name)
-        VALUES ('std_calibration.sh', E'""" + script_body + """' , 'std_calibration')
+        VALUES ('std_calibration.sh', E'"""
+        + script_body
+        + """' , 'std_calibration')
         """
     )
 

schema/versions/2b6e18877c8e_update_std_calibration_workflow_.py

@@ -36,7 +36,7 @@ queue
     )
 
     script_contents = """#!/bin/sh
-SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
    
 export PATH=$PATH:${SCRIPTS_DIR}
 
@@ -58,7 +58,7 @@ def downgrade():
         UPDATE workflow_templates
         SET content = 
         E'#!/bin/sh
-        SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+        SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
            
         export PATH=$PATH:${SCRIPTS_DIR}
         

schema/versions/7d53b022c5b3_std_calibration_add_chmod_call_before_.py

@@ -56,7 +56,7 @@ queue
 
 def downgrade():
     script_contents = """#!/bin/sh
-SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
    
 export PATH=$PATH:${SCRIPTS_DIR}
 

schema/versions/dcbfdfafe16c_add_ingest_img_workflow.py

+"""add ingest_img workflow
+
+Revision ID: dcbfdfafe16c
+Revises: f2e524e1e04d
+Create Date: 2021-07-27 15:38:06.960178
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "dcbfdfafe16c"
+down_revision = "f2e524e1e04d"
+branch_labels = None
+depends_on = None
+
+condor_content = """executable = ingest_image.sh
+arguments = metadata.json
+
+output = ingest.out
+error = ingest.err
+log = condor.log
+
+SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin
+should_transfer_files = yes
+transfer_input_files = $(SBIN_PATH)/pycapo, $(SBIN_PATH)/conveyor, $(SBIN_PATH)/ingest_envoy, $(SBIN_PATH)/ingest, $(SBIN_PATH)/image-product-collector.sh, ./metadata.json
+
+getenv = True
+environment = "CAPO_PATH=/home/casa/capo"
+
+queue
+
+
+"""
+
+script_content = """#!/bin/sh
+set -o errexit
+
+./conveyor --retrieve-img $1
+./ingest_envoy --image $1
+
+"""
+
+
+def upgrade():
+    op.execute(
+        f"""
+        INSERT INTO workflows (workflow_name)
+        VALUES ('ingest_image')
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('ingest_image.condor', E'{condor_content}', 'ingest_image') 
+        """
+    )
+    op.execute(
+        f"""
+        INSERT INTO workflow_templates (filename, content, workflow_name) 
+        VALUES ('ingest_image.sh', E'{script_content}', 'ingest_image') 
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        """
+        DELETE FROM workflows WHERE workflow_name='ingest_image'
+        """
+    )
+    op.execute(
+        """
+        DELETE FROM workflow_templates WHERE workflow_name='ingest_image'
+        """
+    )

schema/versions/e4452ab33088_correcting_calibration_script.py

@@ -10,8 +10,8 @@ import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = 'e4452ab33088'
-down_revision = '291edd508557'
+revision = "e4452ab33088"
+down_revision = "291edd508557"
 branch_labels = None
 depends_on = None
 
@@ -22,7 +22,7 @@ def upgrade():
         UPDATE workflow_templates
         SET content = 
         E'#!/bin/sh
-        SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+        SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
            
         export PATH=$PATH:${SCRIPTS_DIR}
         
@@ -33,7 +33,8 @@ def upgrade():
         casa_envoy --standard-cal $3 $4
         deliver-cipl.sh ${PWD##*/}'
         WHERE filename = 'std_calibration.sh'
-        """)
+        """
+    )
     pass
 
 

services/capability/Dockerfile.dev → services/capability/Dockerfile

@@ -20,8 +20,8 @@ ENV PATH "${PATH}:/home/vlapipe/.local/bin"
 
 RUN python setup.py develop --user
 
-# Set Capo 
-ARG capo_env=dsoc-dev
-ENV CAPO_PROFILE $capo_env
+# Set Capo for build stage
+# Gets reset to proper environment's profile in the deploy stage
+ENV CAPO_PROFILE docker
 
 CMD ["pserve", "--reload", "development.ini"]

services/notification/Dockerfile.dev → services/notification/Dockerfile

@@ -20,8 +20,8 @@ ENV PATH "${PATH}:/home/vlapipe/.local/bin"
 
 RUN python setup.py develop --user
 
-# Set Capo 
-ARG capo_env=dsoc-dev
-ENV CAPO_PROFILE $capo_env
+# Set Capo for build stage
+# Gets reset to proper environment's profile in the deploy stage
+ENV CAPO_PROFILE docker
 
 CMD ["pserve", "--reload", "development.ini"]

services/workflow/Dockerfile.dev → services/workflow/Dockerfile

@@ -2,6 +2,10 @@
 ARG TAGNAME="tmp"
 FROM cache:${TAGNAME}
 
+# Build arg that sets environment; sets to "dev" if no build arg is given
+ARG env=dev
+ENV ENV=${env}
+
 USER root
 
 # HTCondor install
@@ -14,7 +18,7 @@ RUN sudo apt install -y htcondor
 
 # HTCondor setup
 # Copy over HTCondor submit node config
-COPY ./config/htcondor/submit/99-workspaces-submit.dev.conf /etc/condor/config.d/99-workspaces-submit.dev.conf
+COPY ./config/htcondor/submit/99-workspaces-submit.${ENV}.conf /etc/condor/config.d/99-workspaces-submit.${ENV}.conf
 
 # Change working directory to /code
 WORKDIR /code
@@ -34,10 +38,9 @@ ENV PATH "${PATH}:/home/vlapipe/.local/bin"
 
 RUN python setup.py develop --user
 
-# Set Capo
-ARG capo_env=dsoc-dev
-ENV CAPO_PROFILE $capo_env
-ENV PATH $PATH:/lustre/aoc/pipeline/$CAPO_PROFILE/workflow
+# Set Capo for build stage
+# Gets reset to proper environment's profile in the deploy stage
+ENV CAPO_PROFILE docker
 
 USER root
 

services/workflow/bin/boot-condor-and-workflow.sh

@@ -23,8 +23,8 @@ SPOOL_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/spool
 QA_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/qa2
 WEBLOG_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/cache
 IMAGE_QA_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/image-qa
-STAGING_DIR=lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/staging
-STORAGE_DIR=lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/storage
+STAGING_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/staging
+STORAGE_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/storage
 mkdir -p "$SPOOL_DIR"
 mkdir -p "$QA_DIR"
 mkdir -p "$WEBLOG_DIR"
@@ -34,9 +34,7 @@ mkdir -p "$STAGING_DIR"
 mkdir -p "$STORAGE_DIR"
 
 # Copy wf_framework shell scripts to workflow dir
-cp /packages/apps/cli/executables/wf_framework/sh/framework.sh "$WORKFLOW_DIR"
-cp /packages/apps/cli/executables/wf_framework/sh/calibration-table-collector.sh "$WORKFLOW_DIR"
-cp /packages/apps/cli/executables/wf_framework/sh/ingest-request.sh "$WORKFLOW_DIR"
+cp -a /packages/apps/cli/executables/wf_framework/sh/. "$WORKFLOW_DIR"
 cp -R /packages/apps/cli/executables/wf_framework/casa_requirements/.matplotlib "$WORKFLOW_DIR"