From c4b1e32d5aac22ed9bfb1f96f765d36b441b6a6b Mon Sep 17 00:00:00 2001
From: Andrew Kapuscinski <akapusci@nrao.edu>
Date: Wed, 9 Jun 2021 23:34:58 +0000
Subject: [PATCH] Ws 422 cache image
---
.dockerignore | 14 ++++++
.gitlab-ci.yml | 8 +++-
Dockerfile.base | 10 ++---
Dockerfile.cache | 16 +++++++
Makefile | 8 +++-
apps/cli/utilities/system_mediator/setup.py | 2 +-
ci/build.template.yml | 2 +-
ci/cleanup.template.yml | 2 +-
ci/unit-test.template.yml | 4 +-
config/htcondor/execute/boot-execute.sh | 2 +-
docker-compose.yml | 43 +++++++++++++++++++
requirements.txt | 12 ++++++
services/capability/Dockerfile.dev | 23 ++--------
services/capability/Dockerfile.local | 12 ++----
services/capability/bin/run-tests.sh | 4 +-
services/capability/requirements.txt | 12 ------
services/capability/setup.py | 3 +-
services/notification/Dockerfile.dev | 22 ++--------
services/notification/Dockerfile.local | 12 ++----
services/notification/bin/run-tests.sh | 4 +-
services/notification/requirements.txt | 12 ------
services/notification/setup.py | 2 +-
services/workflow/Dockerfile.dev | 20 +++------
services/workflow/Dockerfile.local | 14 +++---
.../workflow/bin/boot-condor-and-workflow.sh | 9 +---
services/workflow/bin/run-tests.sh | 4 +-
services/workflow/requirements.txt | 16 -------
services/workflow/setup.py | 2 +-
shared/schema/setup.py | 2 +-
shared/workspaces/setup.py | 2 +-
testing/requirements.txt | 7 ---
31 files changed, 146 insertions(+), 159 deletions(-)
create mode 100644 .dockerignore
create mode 100644 Dockerfile.cache
create mode 100644 requirements.txt
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 000000000..e0fbca6c1
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,14 @@
+# Version Control
+.git
+.gitmodules
+
+# Intellij
+.idea
+
+.DS_Store
+.gitlab-ci.yml
+/tmp
+/delivery_root
+.coverage
+.coveragerc
+/docs
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index af01c65f4..777997ed3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,6 +2,7 @@ stages:
- build-base
- push-base
- run-schema
+ - cache-build
- build
- unit-test
- test-coverage
@@ -84,13 +85,18 @@ run schema:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always
+# Cache
+build cache:
+ stage: cache-build
+ script:
+ - docker build -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache .
+
# Build Stages
build workflow:
stage: build
variables:
SERVICE_NAME: "workflow"
PATH_PREFIX: "services/"
- USER: "vlapipe"
extends: .build
build capability:
diff --git a/Dockerfile.base b/Dockerfile.base
index 164083675..90e9aa5ce 100644
--- a/Dockerfile.base
+++ b/Dockerfile.base
@@ -1,8 +1,8 @@
FROM python:3.8-slim
# Get postgres/mysql development stuff in the image
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
+RUN apt update \
+ && apt install -y --no-install-recommends \
gcc \
libmariadb-dev-compat \
libpq-dev \
@@ -12,7 +12,7 @@ RUN apt-get update \
# - CAPO_PROFILE will be overridden for Dev, Test, and Prod
ENV PIP_NO_CACHE_DIR false
ENV CAPO_PROFILE docker
-ENV CAPO_PATH /home/casa/capo
+ENV CAPO_PATH /home/ssa/capo
# Create vlapipe group
RUN addgroup --gid 6000 vlapipe && \
@@ -22,8 +22,8 @@ RUN addgroup --gid 6000 vlapipe && \
# Switch to appuser
USER vlapipe
-# Change working directory to /home/casa/capo
-WORKDIR /home/casa/capo
+# Change working directory to /home/ssa/capo
+WORKDIR /home/ssa/capo
# set ownership of docker.properties to vlapipe and the vlapipe group
COPY --chown=vlapipe:vlapipe docker.properties docker.properties
diff --git a/Dockerfile.cache b/Dockerfile.cache
new file mode 100644
index 000000000..432882922
--- /dev/null
+++ b/Dockerfile.cache
@@ -0,0 +1,16 @@
+FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+
+WORKDIR /code/
+RUN chown vlapipe . && chgrp vlapipe .
+
+USER vlapipe
+
+WORKDIR /packages/
+COPY --chown=vlapipe:vlapipe ./shared ./shared
+COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli
+COPY --chown=vlapipe:vlapipe ./testing ./testing
+
+ENV PATH "${PATH}:/home/vlapipe/.local/bin"
+# package installation
+COPY --chown=vlapipe:vlapipe ./requirements.txt ./requirements.txt
+RUN pip install --user -r requirements.txt
diff --git a/Makefile b/Makefile
index 36bf31c0f..682012ff5 100644
--- a/Makefile
+++ b/Makefile
@@ -46,13 +46,17 @@ docker-base: db
docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/base:workspaces -f Dockerfile.base .
docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/base:nodejs-14 -f apps/web/Dockerfile.base .
+# Build cache image
+cache:
+ docker build --no-cache -t cache:tmp -f Dockerfile.cache .
+
# Build DB image
db:
docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces -f ./ci/psql/Dockerfile.db .
# Build docker images
-build: docker-base db
- docker-compose build --no-cache
+build: docker-base db cache
+ docker compose build --no-cache
# Generate HTML coverage report
coverage:
diff --git a/apps/cli/utilities/system_mediator/setup.py b/apps/cli/utilities/system_mediator/setup.py
index e43b924cf..cbc7309ec 100644
--- a/apps/cli/utilities/system_mediator/setup.py
+++ b/apps/cli/utilities/system_mediator/setup.py
@@ -11,7 +11,7 @@ README = Path("README.md").read_text()
requires = [
"ssa-workspaces",
"ssa-messaging",
- "sqlalchemy",
+ "sqlalchemy==1.3.23",
"pycapo"
]
diff --git a/ci/build.template.yml b/ci/build.template.yml
index 07d6e3936..451c786fd 100644
--- a/ci/build.template.yml
+++ b/ci/build.template.yml
@@ -5,7 +5,7 @@
script:
- echo "Building branch or tag -- ${IMAGE_TAG}"
- NAME="${REGISTRY_URL}/${PROJECT_NAME}/${SERVICE_NAME}"
- - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile.dev . --build-arg capo_env=${CAPO_BUILD_ARG}
+ - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile.dev . --build-arg capo_env=${CAPO_BUILD_ARG} --build-arg TAGNAME=${CI_COMMIT_SHORT_SHA}
- docker tag ${NAME}:${CI_COMMIT_SHORT_SHA} ${NAME}:${IMAGE_TAG}
- echo "TAG=${IMAGE_TAG}" >> build.env
artifacts:
diff --git a/ci/cleanup.template.yml b/ci/cleanup.template.yml
index fd020b1f0..a36d24d93 100644
--- a/ci/cleanup.template.yml
+++ b/ci/cleanup.template.yml
@@ -6,7 +6,7 @@
printf "%s\n" "- Removing Images -" \
"${NAME}:${TAG}" \
"${NAME}:${CI_COMMIT_SHORT_SHA}"
- - docker image rm --force "${NAME}:${TAG}" "${NAME}:${CI_COMMIT_SHORT_SHA}"
+ - docker image rm --force "${NAME}:${TAG}" "${NAME}:${CI_COMMIT_SHORT_SHA}" "cache:${CI_COMMIT_SHORT_SHA}"
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_MESSAGE =~ /\A(?i)-debug/'
when: never
diff --git a/ci/unit-test.template.yml b/ci/unit-test.template.yml
index 20a5139bc..f824a459c 100644
--- a/ci/unit-test.template.yml
+++ b/ci/unit-test.template.yml
@@ -2,8 +2,8 @@
image: ${REGISTRY_URL}/${PROJECT_NAME}/${SERVICE_NAME}:${CI_COMMIT_SHORT_SHA}
script:
- |
- ([ ! -z ${USER} ] && su vlapipe -c "cd /code && ./bin/run-tests.sh -b") || cd /code && ./bin/run-tests.sh -b
- - mv .coverage ${CI_PROJECT_DIR}/.coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA}
+ ([ $(whoami) = "root" ] && su vlapipe -c "cd /code && ./bin/run-tests.sh -b") || (cd /code && ./bin/run-tests.sh -b)
+ - mv /code/.coverage ${CI_PROJECT_DIR}/.coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA}
artifacts:
paths:
- .coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA}
diff --git a/config/htcondor/execute/boot-execute.sh b/config/htcondor/execute/boot-execute.sh
index 13d66ca12..f92a7d514 100755
--- a/config/htcondor/execute/boot-execute.sh
+++ b/config/htcondor/execute/boot-execute.sh
@@ -14,7 +14,7 @@ yum install -y libffi-devel
cd /opt
wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz
tar xzf Python-$PYTHON_VERSION.tgz && cd Python-$PYTHON_VERSION
-./configure --enable-optimizations # --exec-prefix=/home/ssa --prefix=/home/ssa
+./configure --enable-optimizations
make altinstall
cd ..
rm -rf Python-$PYTHON_VERSION.tgz
diff --git a/docker-compose.yml b/docker-compose.yml
index 9db296a29..c2055be71 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -50,6 +50,7 @@ services:
- db
volumes:
- ./schema:/code/schema
+ - ./docker.properties:/home/ssa/capo/docker.properties
# Used for doing a DB migration on dsoc-dev
# Uncomment this and run the command `docker compose up schema-dev` and the migration will run
@@ -76,12 +77,26 @@ services:
- amqp
volumes:
- ./services/workflow:/code
+ - /code/ssa_workflow.egg-info
- ./shared:/packages/shared
- ./apps/cli:/packages/apps/cli
- ./testing:/packages/testing
- ./tmp/:/tmp/workspaces_tmp
- ./delivery_root:/tmp/delivery_root
- ~/.capo:/home/ssa/capo
+ - ./docker.properties:/home/ssa/capo/docker.properties
+ - /packages/shared/workspaces/ssa_workspaces.egg-info
+ - /packages/shared/schema/ssa_schema.egg-info
+ - /packages/shared/messaging/ssa_messaging.egg-info
+ - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+ - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+ - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+ - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+ - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+ - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+ - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+ - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+ - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
capability:
build:
@@ -95,10 +110,24 @@ services:
- notification
- amqp
volumes:
+ - ./docker.properties:/home/ssa/capo/docker.properties
- ./services/capability:/code
- ./shared:/packages/shared
- ./apps/cli:/packages/apps/cli
- ./testing:/packages/testing
+ - /code/ssa_capability.egg-info
+ - /packages/shared/workspaces/ssa_workspaces.egg-info
+ - /packages/shared/schema/ssa_schema.egg-info
+ - /packages/shared/messaging/ssa_messaging.egg-info
+ - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+ - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+ - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+ - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+ - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+ - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+ - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+ - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+ - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
notification:
build:
@@ -110,10 +139,24 @@ services:
- schema
- amqp
volumes:
+ - ./docker.properties:/home/ssa/capo/docker.properties
- ./services/notification:/code
+ - /code/ssa_notification.egg-info
- ./shared:/packages/shared
- ./apps/cli:/packages/apps/cli
- ./testing:/packages/testing
+ - /packages/shared/workspaces/ssa_workspaces.egg-info
+ - /packages/shared/schema/ssa_schema.egg-info
+ - /packages/shared/messaging/ssa_messaging.egg-info
+ - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+ - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+ - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+ - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+ - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+ - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+ - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+ - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+ - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info
frontend:
build:
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 000000000..2ef14fab6
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,12 @@
+-e ../packages/shared/schema
+-e ../packages/shared/messaging
+-e ../packages/shared/workspaces
+-e ../packages/apps/cli/utilities/wf_monitor
+-e ../packages/apps/cli/utilities/ws_metrics
+-e ../packages/apps/cli/utilities/aat_wrest
+-e ../packages/apps/cli/utilities/system_mediator
+-e ../packages/apps/cli/executables/pexable/productfetcher
+-e ../packages/apps/cli/executables/pexable/delivery
+-e ../packages/apps/cli/executables/pexable/null
+-e ../packages/apps/cli/executables/pexable/casa_envoy
+-e ../packages/apps/cli/executables/tmpdir_eraser
diff --git a/services/capability/Dockerfile.dev b/services/capability/Dockerfile.dev
index 637350fb3..ae5e77d8d 100644
--- a/services/capability/Dockerfile.dev
+++ b/services/capability/Dockerfile.dev
@@ -1,5 +1,6 @@
# This is nrao:capability
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+ARG TAGNAME="tmp"
+FROM cache:${TAGNAME}
# Change working directory to /code
WORKDIR /code
@@ -14,28 +15,10 @@ USER vlapipe
# set ownership of content to vlapipe and the vlapipe group
COPY --chown=vlapipe:vlapipe ./services/capability ./
-# Change working directory to /packages
-WORKDIR /packages/
-
-#
-# USER root
-# RUN mkdir ./apps && chown vlapipe ./apps && chgrp vlapipe ./apps
-# USER vlapipe
-
-# Copy shared, apps/cli/, and testing directories to /packages
-# set ownership of content to vlapipe and the vlapipe group
-COPY --chown=vlapipe:vlapipe ./shared ./shared
-COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli
-COPY --chown=vlapipe:vlapipe ./testing ./testing
-
-# Change working directory to /code
-WORKDIR /code
-
ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local"
ENV PATH "${PATH}:/home/vlapipe/.local/bin"
-RUN pip install --user -r requirements.txt \
- && python setup.py develop --install-dir="~/.local"
+RUN python setup.py develop --user
# Set Capo
ARG capo_env=dsoc-dev
diff --git a/services/capability/Dockerfile.local b/services/capability/Dockerfile.local
index 94c864f1f..9428ddc6a 100644
--- a/services/capability/Dockerfile.local
+++ b/services/capability/Dockerfile.local
@@ -1,20 +1,14 @@
# This is nrao:capability
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+FROM cache:tmp
# Get capability code into the image
WORKDIR /code
-COPY ./services/capability ./
-
-WORKDIR /packages/
-COPY ./shared ./shared
-COPY ./apps/cli ./apps/cli
-COPY ./testing ./testing
+COPY --chown=vlapipe:vlapipe ./services/capability ./
# Python library installation
WORKDIR /code/
-RUN pip install -r requirements.txt \
- && python setup.py develop
+RUN python setup.py develop --user
# Set Capo
ARG capo_env=docker
diff --git a/services/capability/bin/run-tests.sh b/services/capability/bin/run-tests.sh
index cd2382c5b..e8b1f26c4 100755
--- a/services/capability/bin/run-tests.sh
+++ b/services/capability/bin/run-tests.sh
@@ -14,7 +14,9 @@ skip_arg="--skip-empty"
# Install testing requirements
init () {
- pip install --user -r /packages/testing/requirements.txt
+ pushd /packages/testing
+ python setup.py develop --user
+ popd
# if exists, remove old .coverage file
if [[ -e .coverage ]]; then
diff --git a/services/capability/requirements.txt b/services/capability/requirements.txt
index 61883a3b6..b9fe525a7 100644
--- a/services/capability/requirements.txt
+++ b/services/capability/requirements.txt
@@ -7,15 +7,3 @@
-e ../packages/apps/cli/utilities/wf_monitor
-e ../packages/apps/cli/utilities/ws_metrics
-e ../packages/testing
-
-behave == 1.2.6
-pycapo == 0.3.0
-pyramid == 1.10
-pyramid_beaker == 0.8
-pyramid_debugtoolbar == 4.5
-pyramid_retry == 2.1.1
-pyramid_tm == 2.2.1
-requests == 2.23
-sqlalchemy == 1.3.23
-waitress == 1.4
-zope.sqlalchemy == 1.1
diff --git a/services/capability/setup.py b/services/capability/setup.py
index 51eb84303..a2ecdc7c5 100644
--- a/services/capability/setup.py
+++ b/services/capability/setup.py
@@ -39,12 +39,13 @@ requires = [
"pendulum",
"pycapo",
"pyramid",
+ "pyramid_retry",
"pyramid_beaker",
"pyramid_debugtoolbar",
"pyramid_tm",
"requests",
"ssa-schema",
- "sqlalchemy",
+ "sqlalchemy==1.3.23",
"waitress",
"ssa-workspaces",
"zope.sqlalchemy",
diff --git a/services/notification/Dockerfile.dev b/services/notification/Dockerfile.dev
index c0a77a785..12e7470b2 100644
--- a/services/notification/Dockerfile.dev
+++ b/services/notification/Dockerfile.dev
@@ -1,5 +1,6 @@
# This is nrao:notification
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+ARG TAGNAME="tmp"
+FROM cache:${TAGNAME}
# Change working directory to /code
WORKDIR /code
@@ -14,27 +15,10 @@ USER vlapipe
# set ownership of content to vlapipe and the vlapipe group
COPY --chown=vlapipe:vlapipe ./services/notification ./
-# Change working directory to /packages
-WORKDIR /packages/
-
-# USER root
-# RUN mkdir ./apps && chown vlapipe ./apps && chgrp vlapipe ./apps
-# USER vlapipe
-
-# Copy shared, apps/cli/, and testing directories to /packages
-# set ownership of content to vlapipe and the vlapipe group
-COPY --chown=vlapipe:vlapipe ./shared ./shared
-COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli
-COPY --chown=vlapipe:vlapipe ./testing ./testing
-
-# Change working directory to /code
-WORKDIR /code
-
ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local"
ENV PATH "${PATH}:/home/vlapipe/.local/bin"
-RUN pip install --user -r requirements.txt \
- && python setup.py develop --install-dir="~/.local"
+RUN python setup.py develop --user
# Set Capo
ARG capo_env=dsoc-dev
diff --git a/services/notification/Dockerfile.local b/services/notification/Dockerfile.local
index 4e551c404..64b7e1871 100644
--- a/services/notification/Dockerfile.local
+++ b/services/notification/Dockerfile.local
@@ -1,20 +1,14 @@
# This is nrao:notification
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+FROM cache:tmp
# Get workflow code into the image
WORKDIR /code
-COPY ./services/notification ./
-
-WORKDIR /packages/
-COPY ./shared ./shared
-COPY ./apps/cli ./apps/cli
-COPY ./testing ./testing
+COPY --chown=vlapipe:vlapipe ./services/notification ./
# Python library installation
WORKDIR /code/
-RUN pip install -r requirements.txt \
- && python setup.py develop
+RUN python setup.py develop --user
# Set Capo
ARG capo_env=docker
diff --git a/services/notification/bin/run-tests.sh b/services/notification/bin/run-tests.sh
index 3aa678d47..9d70c8bb4 100755
--- a/services/notification/bin/run-tests.sh
+++ b/services/notification/bin/run-tests.sh
@@ -14,7 +14,9 @@ skip_arg="--skip-empty"
# Install testing requirements
init () {
- pip install --user -r /packages/testing/requirements.txt
+ pushd /packages/testing
+ python setup.py develop --user
+ popd
# if exists, remove old .coverage file
if [[ -e .coverage ]]; then
diff --git a/services/notification/requirements.txt b/services/notification/requirements.txt
index 9c8503e2f..3cf7b96d6 100644
--- a/services/notification/requirements.txt
+++ b/services/notification/requirements.txt
@@ -8,15 +8,3 @@
-e ../packages/apps/cli/utilities/ws_metrics
-e ../packages/apps/cli/executables/pexable/null
-e ../packages/testing
-
-pycapo == 0.3.0
-pyramid == 1.10
-pyramid_beaker == 0.8
-pyramid_debugtoolbar == 4.5
-pyramid_retry == 2.1.1
-pyramid_tm == 2.2.1
-pytest == 5.4.3
-requests == 2.23
-sqlalchemy == 1.3.23
-waitress == 1.4
-zope.sqlalchemy == 1.1
diff --git a/services/notification/setup.py b/services/notification/setup.py
index 42c784308..9ff7d3d4f 100644
--- a/services/notification/setup.py
+++ b/services/notification/setup.py
@@ -46,7 +46,7 @@ requires = [
"pyramid_retry",
"requests",
"ssa-schema",
- "sqlalchemy",
+ "sqlalchemy==1.3.23",
"waitress",
"ssa-workspaces",
"zope.sqlalchemy",
diff --git a/services/workflow/Dockerfile.dev b/services/workflow/Dockerfile.dev
index bccbdc703..2ac9111f2 100644
--- a/services/workflow/Dockerfile.dev
+++ b/services/workflow/Dockerfile.dev
@@ -1,5 +1,8 @@
# This is nrao:workflow
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+ARG TAGNAME="tmp"
+FROM cache:${TAGNAME}
+
+USER root
# HTCondor install
RUN apt update && apt install -y wget gnupg sudo
@@ -26,23 +29,10 @@ USER vlapipe
# set ownership of content to vlapipe and the vlapipe group
COPY --chown=vlapipe:vlapipe ./services/workflow ./
-# Change working directory to /packages
-WORKDIR /packages/
-
-# Copy shared, apps/cli/, and testing directories to /packages
-# set ownership of content to vlapipe and the vlapipe group id
-COPY --chown=vlapipe:vlapipe ./shared ./shared
-COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli
-COPY --chown=vlapipe:vlapipe ./testing ./testing
-
-# Change working directory to /code
-WORKDIR /code
-
ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local"
ENV PATH "${PATH}:/home/vlapipe/.local/bin"
-RUN pip install --user -r requirements.txt \
- && python setup.py develop --install-dir="~/.local"
+RUN python setup.py develop --user
# Set Capo
ARG capo_env=dsoc-dev
diff --git a/services/workflow/Dockerfile.local b/services/workflow/Dockerfile.local
index bbb890cb7..beebb37b5 100644
--- a/services/workflow/Dockerfile.local
+++ b/services/workflow/Dockerfile.local
@@ -1,19 +1,17 @@
# Workflow service layer
# This is nrao:workflow
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+FROM cache:tmp
# Get workflow code into the image
WORKDIR /code
-COPY ./services/workflow ./
-
-WORKDIR /packages/
-COPY ./shared ./shared
-COPY ./apps/cli ./apps/cli
-COPY ./testing ./testing
+COPY --chown=vlapipe:vlapipe ./services/workflow ./
# Python library installation
WORKDIR /code/
-RUN pip install -r requirements.txt && python setup.py develop
+USER vlapipe
+RUN python setup.py develop --user
+
+USER root
# HTCondor install
RUN apt update && apt install -y wget gnupg sudo
diff --git a/services/workflow/bin/boot-condor-and-workflow.sh b/services/workflow/bin/boot-condor-and-workflow.sh
index 83ad8bbbc..63a3a8de5 100755
--- a/services/workflow/bin/boot-condor-and-workflow.sh
+++ b/services/workflow/bin/boot-condor-and-workflow.sh
@@ -29,11 +29,4 @@ chown root:vlapipe "$WORKFLOW_DIR"/*
chmod 777 "$WORKFLOW_DIR"
chmod 777 "$WORKFLOW_DIR"/*
-if [ "$CAPO_PROFILE" = "dsoc-dev" ] || [ "$CAPO_PROFILE" = "dsoc-test" ]
-then
- # If on dev or test, boot workflow as vlapipe
- su vlapipe -c "/code/bin/boot-workflow.sh"
-else
- # If local, just boot as root
- /code/bin/boot-workflow.sh
-fi
+su vlapipe -c "/code/bin/boot-workflow.sh"
diff --git a/services/workflow/bin/run-tests.sh b/services/workflow/bin/run-tests.sh
index 64cdba99e..782730312 100755
--- a/services/workflow/bin/run-tests.sh
+++ b/services/workflow/bin/run-tests.sh
@@ -14,10 +14,8 @@ skip_arg="--skip-empty"
# Install testing requirements
init () {
- pip install --user -r /packages/testing/requirements.txt
-
pushd /packages/testing
- python setup.py develop
+ python setup.py develop --user
popd
# if exists, remove old .coverage file
diff --git a/services/workflow/requirements.txt b/services/workflow/requirements.txt
index c01957e46..93ce39163 100644
--- a/services/workflow/requirements.txt
+++ b/services/workflow/requirements.txt
@@ -14,19 +14,3 @@
-e ../packages/apps/cli/executables/pexable/null
-e ../packages/apps/cli/executables/vela
-e ../packages/testing
-
-psycopg2 >= 2.8.5,<3.0
-pycapo == 0.3.1
-py-dag == 3.0.1
-pyramid == 1.10
-pyramid_beaker == 0.8
-pyramid_debugtoolbar == 4.5
-pyramid_retry == 2.1.1
-pyramid_tm == 2.2.1
-pytest == 5.4.3
-pytest-resource-path == 1.2.1
-requests == 2.23
-requests_mock == 1.9.2
-sqlalchemy == 1.3.23
-waitress == 1.4
-zope.sqlalchemy == 1.1
diff --git a/services/workflow/setup.py b/services/workflow/setup.py
index 99a230821..8ce99dec1 100644
--- a/services/workflow/setup.py
+++ b/services/workflow/setup.py
@@ -46,7 +46,7 @@ requires = [
"pyramid_retry",
"requests",
"ssa-schema",
- "sqlalchemy",
+ "sqlalchemy==1.3.23",
"waitress",
"ssa-workspaces",
"zope.sqlalchemy",
diff --git a/shared/schema/setup.py b/shared/schema/setup.py
index 6f335c8e5..704ac4043 100644
--- a/shared/schema/setup.py
+++ b/shared/schema/setup.py
@@ -19,7 +19,7 @@ setup(
license="GPL",
install_requires=[
"pendulum==2.1.2",
- "sqlalchemy",
+ "sqlalchemy==1.3.23",
"pycapo",
"psycopg2",
"mysqlclient",
diff --git a/shared/workspaces/setup.py b/shared/workspaces/setup.py
index f052090f9..925b39c42 100644
--- a/shared/workspaces/setup.py
+++ b/shared/workspaces/setup.py
@@ -12,7 +12,7 @@ requires = [
"pycapo",
"marshmallow",
"ssa-schema",
- "sqlalchemy",
+ "sqlalchemy==1.3.23",
"cx-Oracle",
"chevron",
"requests",
diff --git a/testing/requirements.txt b/testing/requirements.txt
index 7710bfb3e..753927e0c 100644
--- a/testing/requirements.txt
+++ b/testing/requirements.txt
@@ -13,10 +13,3 @@
-e ../packages/apps/cli/executables/pexable/null
-e ../packages/apps/cli/executables/pexable/casa_envoy
-e ../packages/apps/cli/executables/tmpdir_eraser
-
-pytest>=5.4,<6.0
-pendulum==2.1.2
-pytest-mock==3.3.1
-behave==1.2.6
-pytest-cov==2.11
-mock_alchemy==0.2.1
--
GitLab