Ws 422 cache image

c4b1e32d · Andrew Kapuscinski · 2b17c9cd · c4b1e32d · c4b1e32d · c4b1e32d
Commit c4b1e32d authored 3 years ago by Andrew Kapuscinski
--- a/.dockerignore
+++ b/.dockerignore
+# Version Control
+.git
+.gitmodules
+
+# Intellij
+.idea
+
+.DS_Store
+.gitlab-ci.yml
+/tmp
+/delivery_root
+.coverage
+.coveragerc
+/docs
\ No newline at end of file
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,6 +2,7 @@ stages:
    - build-base
    - push-base
    - run-schema
+    - cache-build
    - build
    - unit-test
    - test-coverage
@@ -84,13 +85,18 @@ run schema:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
          when: always

+# Cache
+build cache:
+    stage: cache-build
+    script:
+        - docker build -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache .
+
 # Build Stages
 build workflow:
    stage: build
    variables:
        SERVICE_NAME: "workflow"
        PATH_PREFIX: "services/"
-        USER: "vlapipe"
    extends: .build

 build capability:

--- a/Dockerfile.base
+++ b/Dockerfile.base
 FROM python:3.8-slim

 # Get postgres/mysql development stuff in the image
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \ 
+RUN apt update \
+    && apt install -y --no-install-recommends \
    gcc \
    libmariadb-dev-compat \
    libpq-dev \
@@ -12,7 +12,7 @@ RUN apt-get update \
 # - CAPO_PROFILE will be overridden for Dev, Test, and Prod
 ENV PIP_NO_CACHE_DIR false
 ENV CAPO_PROFILE docker
-ENV CAPO_PATH /home/casa/capo
+ENV CAPO_PATH /home/ssa/capo

 # Create vlapipe group
 RUN addgroup --gid 6000 vlapipe && \
@@ -22,8 +22,8 @@ RUN addgroup --gid 6000 vlapipe && \
 # Switch to appuser
 USER vlapipe

-# Change working directory to /home/casa/capo
-WORKDIR /home/casa/capo
+# Change working directory to /home/ssa/capo
+WORKDIR /home/ssa/capo

 # set ownership of docker.properties to vlapipe and the vlapipe group
 COPY --chown=vlapipe:vlapipe docker.properties docker.properties

--- a/Dockerfile.cache
+++ b/Dockerfile.cache
+FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+
+WORKDIR /code/
+RUN chown vlapipe . && chgrp vlapipe .
+
+USER vlapipe
+
+WORKDIR /packages/
+COPY --chown=vlapipe:vlapipe ./shared ./shared
+COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli
+COPY --chown=vlapipe:vlapipe ./testing ./testing
+
+ENV PATH "${PATH}:/home/vlapipe/.local/bin"
+# package installation
+COPY --chown=vlapipe:vlapipe ./requirements.txt ./requirements.txt
+RUN pip install --user -r requirements.txt
--- a/Makefile
+++ b/Makefile
@@ -46,13 +46,17 @@ docker-base: db
 	docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/base:workspaces -f Dockerfile.base .
 	docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/base:nodejs-14 -f apps/web/Dockerfile.base .

+# Build cache image
+cache:
+	docker build --no-cache -t cache:tmp -f Dockerfile.cache .
+
 # Build DB image
 db:
 	docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces -f ./ci/psql/Dockerfile.db .

 # Build docker images
-build: docker-base db
-	docker-compose build --no-cache
+build: docker-base db cache
+	docker compose build --no-cache

 # Generate HTML coverage report
 coverage:

--- a/apps/cli/utilities/system_mediator/setup.py
+++ b/apps/cli/utilities/system_mediator/setup.py
@@ -11,7 +11,7 @@ README = Path("README.md").read_text()
 requires = [
    "ssa-workspaces",
    "ssa-messaging",
-    "sqlalchemy",
+    "sqlalchemy==1.3.23",
    "pycapo"
 ]


--- a/ci/build.template.yml
+++ b/ci/build.template.yml
@@ -5,7 +5,7 @@
    script:
        - echo "Building branch or tag -- ${IMAGE_TAG}"
        - NAME="${REGISTRY_URL}/${PROJECT_NAME}/${SERVICE_NAME}"
-        - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile.dev . --build-arg capo_env=${CAPO_BUILD_ARG}
+        - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile.dev . --build-arg capo_env=${CAPO_BUILD_ARG} --build-arg TAGNAME=${CI_COMMIT_SHORT_SHA}
        - docker tag ${NAME}:${CI_COMMIT_SHORT_SHA} ${NAME}:${IMAGE_TAG}
        - echo "TAG=${IMAGE_TAG}" >> build.env
    artifacts:

--- a/ci/cleanup.template.yml
+++ b/ci/cleanup.template.yml
@@ -6,7 +6,7 @@
            printf "%s\n" "- Removing Images -" \
            "${NAME}:${TAG}" \
            "${NAME}:${CI_COMMIT_SHORT_SHA}"
-        - docker image rm --force "${NAME}:${TAG}" "${NAME}:${CI_COMMIT_SHORT_SHA}"
+        - docker image rm --force "${NAME}:${TAG}" "${NAME}:${CI_COMMIT_SHORT_SHA}" "cache:${CI_COMMIT_SHORT_SHA}"
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_MESSAGE =~ /\A(?i)-debug/'
          when: never

--- a/ci/unit-test.template.yml
+++ b/ci/unit-test.template.yml
@@ -2,8 +2,8 @@
    image: ${REGISTRY_URL}/${PROJECT_NAME}/${SERVICE_NAME}:${CI_COMMIT_SHORT_SHA}
    script:
        - |
-            ([ ! -z ${USER} ] && su vlapipe -c "cd /code && ./bin/run-tests.sh -b") || cd /code && ./bin/run-tests.sh -b
-        - mv .coverage ${CI_PROJECT_DIR}/.coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA}
+            ([ $(whoami) = "root" ] && su vlapipe -c "cd /code && ./bin/run-tests.sh -b") || (cd /code && ./bin/run-tests.sh -b)
+        - mv /code/.coverage ${CI_PROJECT_DIR}/.coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA}
    artifacts:
        paths:
            - .coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA}

--- a/config/htcondor/execute/boot-execute.sh
+++ b/config/htcondor/execute/boot-execute.sh
@@ -14,7 +14,7 @@ yum install -y libffi-devel
 cd /opt
 wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz
 tar xzf Python-$PYTHON_VERSION.tgz && cd Python-$PYTHON_VERSION
-./configure --enable-optimizations # --exec-prefix=/home/ssa --prefix=/home/ssa
+./configure --enable-optimizations
 make altinstall
 cd ..
 rm -rf Python-$PYTHON_VERSION.tgz

--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -50,6 +50,7 @@ services:
      - db
    volumes:
      - ./schema:/code/schema
+      - ./docker.properties:/home/ssa/capo/docker.properties

  # Used for doing a DB migration on dsoc-dev
  # Uncomment this and run the command `docker compose up schema-dev` and the migration will run
@@ -76,12 +77,26 @@ services:
      - amqp
    volumes:
      - ./services/workflow:/code
+      - /code/ssa_workflow.egg-info
      - ./shared:/packages/shared
      - ./apps/cli:/packages/apps/cli
      - ./testing:/packages/testing
      - ./tmp/:/tmp/workspaces_tmp
      - ./delivery_root:/tmp/delivery_root
      - ~/.capo:/home/ssa/capo
+      - ./docker.properties:/home/ssa/capo/docker.properties
+      - /packages/shared/workspaces/ssa_workspaces.egg-info
+      - /packages/shared/schema/ssa_schema.egg-info
+      - /packages/shared/messaging/ssa_messaging.egg-info
+      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info

  capability:
    build:
@@ -95,10 +110,24 @@ services:
      - notification
      - amqp
    volumes:
+      - ./docker.properties:/home/ssa/capo/docker.properties
      - ./services/capability:/code
      - ./shared:/packages/shared
      - ./apps/cli:/packages/apps/cli
      - ./testing:/packages/testing
+      - /code/ssa_capability.egg-info
+      - /packages/shared/workspaces/ssa_workspaces.egg-info
+      - /packages/shared/schema/ssa_schema.egg-info
+      - /packages/shared/messaging/ssa_messaging.egg-info
+      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info

  notification:
    build:
@@ -110,10 +139,24 @@ services:
      - schema
      - amqp
    volumes:
+      - ./docker.properties:/home/ssa/capo/docker.properties
      - ./services/notification:/code
+      - /code/ssa_notification.egg-info
      - ./shared:/packages/shared
      - ./apps/cli:/packages/apps/cli
      - ./testing:/packages/testing
+      - /packages/shared/workspaces/ssa_workspaces.egg-info
+      - /packages/shared/schema/ssa_schema.egg-info
+      - /packages/shared/messaging/ssa_messaging.egg-info
+      - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info
+      - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info
+      - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info
+      - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info
+      - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info
+      - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info
+      - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info
+      - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info
+      - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info

  frontend:
    build:

--- a/requirements.txt
+++ b/requirements.txt
+-e ../packages/shared/schema
+-e ../packages/shared/messaging
+-e ../packages/shared/workspaces
+-e ../packages/apps/cli/utilities/wf_monitor
+-e ../packages/apps/cli/utilities/ws_metrics
+-e ../packages/apps/cli/utilities/aat_wrest
+-e ../packages/apps/cli/utilities/system_mediator
+-e ../packages/apps/cli/executables/pexable/productfetcher
+-e ../packages/apps/cli/executables/pexable/delivery
+-e ../packages/apps/cli/executables/pexable/null
+-e ../packages/apps/cli/executables/pexable/casa_envoy
+-e ../packages/apps/cli/executables/tmpdir_eraser
--- a/services/capability/Dockerfile.dev
+++ b/services/capability/Dockerfile.dev
 # This is nrao:capability
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+ARG TAGNAME="tmp"
+FROM cache:${TAGNAME}

 # Change working directory to /code
 WORKDIR /code
@@ -14,28 +15,10 @@ USER vlapipe
 # set ownership of content to vlapipe and the vlapipe group
 COPY --chown=vlapipe:vlapipe ./services/capability ./

-# Change working directory to /packages
-WORKDIR /packages/
-
-# 
-# USER root
-# RUN mkdir ./apps && chown vlapipe ./apps && chgrp vlapipe ./apps
-# USER vlapipe
-
-# Copy shared, apps/cli/, and testing directories to /packages
-# set ownership of content to vlapipe and the vlapipe group
-COPY --chown=vlapipe:vlapipe ./shared ./shared
-COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli
-COPY --chown=vlapipe:vlapipe ./testing ./testing
-
-# Change working directory to /code
-WORKDIR /code
-
 ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local"
 ENV PATH "${PATH}:/home/vlapipe/.local/bin"

-RUN pip install --user -r requirements.txt \
-    && python setup.py develop --install-dir="~/.local"
+RUN python setup.py develop --user

 # Set Capo 
 ARG capo_env=dsoc-dev

--- a/services/capability/Dockerfile.local
+++ b/services/capability/Dockerfile.local
 # This is nrao:capability
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+FROM cache:tmp

 # Get capability code into the image
 WORKDIR /code

-COPY ./services/capability ./
-
-WORKDIR /packages/
-COPY ./shared ./shared
-COPY ./apps/cli ./apps/cli
-COPY ./testing ./testing
+COPY --chown=vlapipe:vlapipe ./services/capability ./

 # Python library installation
 WORKDIR /code/
-RUN pip install -r requirements.txt \
-    && python setup.py develop
+RUN python setup.py develop --user

 # Set Capo 
 ARG capo_env=docker

--- a/services/capability/bin/run-tests.sh
+++ b/services/capability/bin/run-tests.sh
@@ -14,7 +14,9 @@ skip_arg="--skip-empty"

 # Install testing requirements
 init () {
-  pip install --user -r /packages/testing/requirements.txt
+  pushd /packages/testing
+  python setup.py develop --user
+  popd

 #  if exists, remove old .coverage file
  if [[ -e .coverage ]]; then

--- a/services/capability/requirements.txt
+++ b/services/capability/requirements.txt
@@ -7,15 +7,3 @@
 -e ../packages/apps/cli/utilities/wf_monitor
 -e ../packages/apps/cli/utilities/ws_metrics
 -e ../packages/testing
-
-behave == 1.2.6
-pycapo == 0.3.0
-pyramid == 1.10
-pyramid_beaker == 0.8
-pyramid_debugtoolbar == 4.5
-pyramid_retry == 2.1.1
-pyramid_tm == 2.2.1
-requests == 2.23
-sqlalchemy == 1.3.23
-waitress == 1.4
-zope.sqlalchemy == 1.1
--- a/services/capability/setup.py
+++ b/services/capability/setup.py
@@ -39,12 +39,13 @@ requires = [
    "pendulum",
    "pycapo",
    "pyramid",
+    "pyramid_retry",
    "pyramid_beaker",
    "pyramid_debugtoolbar",
    "pyramid_tm",
    "requests",
    "ssa-schema",
-    "sqlalchemy",
+    "sqlalchemy==1.3.23",
    "waitress",
    "ssa-workspaces",
    "zope.sqlalchemy",

--- a/services/notification/Dockerfile.dev
+++ b/services/notification/Dockerfile.dev
 # This is nrao:notification
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+ARG TAGNAME="tmp"
+FROM cache:${TAGNAME}

 # Change working directory to /code
 WORKDIR /code
@@ -14,27 +15,10 @@ USER vlapipe
 # set ownership of content to vlapipe and the vlapipe group
 COPY --chown=vlapipe:vlapipe ./services/notification ./

-# Change working directory to /packages
-WORKDIR /packages/
-
-# USER root
-# RUN mkdir ./apps && chown vlapipe ./apps && chgrp vlapipe ./apps
-# USER vlapipe
-
-# Copy shared, apps/cli/, and testing directories to /packages
-# set ownership of content to vlapipe and the vlapipe group
-COPY --chown=vlapipe:vlapipe ./shared ./shared
-COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli
-COPY --chown=vlapipe:vlapipe ./testing ./testing
-
-# Change working directory to /code
-WORKDIR /code
-
 ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local"
 ENV PATH "${PATH}:/home/vlapipe/.local/bin"

-RUN pip install --user -r requirements.txt \
-    && python setup.py develop --install-dir="~/.local"
+RUN python setup.py develop --user

 # Set Capo 
 ARG capo_env=dsoc-dev

--- a/services/notification/Dockerfile.local
+++ b/services/notification/Dockerfile.local
 # This is nrao:notification
-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
+FROM cache:tmp

 # Get workflow code into the image
 WORKDIR /code

-COPY ./services/notification ./
-
-WORKDIR /packages/
-COPY ./shared ./shared
-COPY ./apps/cli ./apps/cli
-COPY ./testing ./testing
+COPY --chown=vlapipe:vlapipe ./services/notification ./

 # Python library installation
 WORKDIR /code/
-RUN pip install -r requirements.txt \
-    && python setup.py develop
+RUN python setup.py develop --user

 # Set Capo 
 ARG capo_env=docker

--- a/services/notification/bin/run-tests.sh
+++ b/services/notification/bin/run-tests.sh
@@ -14,7 +14,9 @@ skip_arg="--skip-empty"

 # Install testing requirements
 init () {
-  pip install --user -r /packages/testing/requirements.txt
+  pushd /packages/testing
+  python setup.py develop --user
+  popd

 #  if exists, remove old .coverage file
  if [[ -e .coverage ]]; then