diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..e0fbca6c1be196a931272aaaeec4ff643eb56cc9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Version Control +.git +.gitmodules + +# Intellij +.idea + +.DS_Store +.gitlab-ci.yml +/tmp +/delivery_root +.coverage +.coveragerc +/docs \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af01c65f4e7645a976b5c6c92e4268c8692231a0..777997ed36e5ad836f52b86cdcf87db821b57620 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ stages: - build-base - push-base - run-schema + - cache-build - build - unit-test - test-coverage @@ -84,13 +85,18 @@ run schema: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: always +# Cache +build cache: + stage: cache-build + script: + - docker build -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache . + # Build Stages build workflow: stage: build variables: SERVICE_NAME: "workflow" PATH_PREFIX: "services/" - USER: "vlapipe" extends: .build build capability: diff --git a/Dockerfile.base b/Dockerfile.base index 1640836752acdf8226657c853249cb6126fb63db..90e9aa5ce9e20d10cff3baf687d0f939868faa7f 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,8 +1,8 @@ FROM python:3.8-slim # Get postgres/mysql development stuff in the image -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +RUN apt update \ + && apt install -y --no-install-recommends \ gcc \ libmariadb-dev-compat \ libpq-dev \ @@ -12,7 +12,7 @@ RUN apt-get update \ # - CAPO_PROFILE will be overridden for Dev, Test, and Prod ENV PIP_NO_CACHE_DIR false ENV CAPO_PROFILE docker -ENV CAPO_PATH /home/casa/capo +ENV CAPO_PATH /home/ssa/capo # Create vlapipe group RUN addgroup --gid 6000 vlapipe && \ @@ -22,8 +22,8 @@ RUN addgroup --gid 6000 vlapipe && \ # Switch to appuser USER vlapipe -# Change working directory to /home/casa/capo -WORKDIR /home/casa/capo +# Change working directory to /home/ssa/capo +WORKDIR /home/ssa/capo # set ownership of docker.properties to vlapipe and the vlapipe group COPY --chown=vlapipe:vlapipe docker.properties docker.properties diff --git a/Dockerfile.cache b/Dockerfile.cache new file mode 100644 index 0000000000000000000000000000000000000000..4328829229b39f43e596fc4c84913bddda2245d2 --- /dev/null +++ b/Dockerfile.cache @@ -0,0 +1,16 @@ +FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces + +WORKDIR /code/ +RUN chown vlapipe . && chgrp vlapipe . + +USER vlapipe + +WORKDIR /packages/ +COPY --chown=vlapipe:vlapipe ./shared ./shared +COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli +COPY --chown=vlapipe:vlapipe ./testing ./testing + +ENV PATH "${PATH}:/home/vlapipe/.local/bin" +# package installation +COPY --chown=vlapipe:vlapipe ./requirements.txt ./requirements.txt +RUN pip install --user -r requirements.txt diff --git a/Makefile b/Makefile index 36bf31c0f1f8f1cde24cbdd643154d52576421db..682012ff54d20309739bf8578092a34a02bfe799 100644 --- a/Makefile +++ b/Makefile @@ -46,13 +46,17 @@ docker-base: db docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/base:workspaces -f Dockerfile.base . docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/base:nodejs-14 -f apps/web/Dockerfile.base . +# Build cache image +cache: + docker build --no-cache -t cache:tmp -f Dockerfile.cache . + # Build DB image db: docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces -f ./ci/psql/Dockerfile.db . # Build docker images -build: docker-base db - docker-compose build --no-cache +build: docker-base db cache + docker compose build --no-cache # Generate HTML coverage report coverage: diff --git a/apps/cli/utilities/system_mediator/setup.py b/apps/cli/utilities/system_mediator/setup.py index e43b924cfe2ba065117c17fb048663d143cbaa9e..cbc7309ec1a7eb754a4ba79d083c009b5c44e1b4 100644 --- a/apps/cli/utilities/system_mediator/setup.py +++ b/apps/cli/utilities/system_mediator/setup.py @@ -11,7 +11,7 @@ README = Path("README.md").read_text() requires = [ "ssa-workspaces", "ssa-messaging", - "sqlalchemy", + "sqlalchemy==1.3.23", "pycapo" ] diff --git a/ci/build.template.yml b/ci/build.template.yml index 07d6e3936d7c5a2bd9eba3a425e31f27aabe7c47..451c786fd2f6edce5d6739225d9903c280e59806 100644 --- a/ci/build.template.yml +++ b/ci/build.template.yml @@ -5,7 +5,7 @@ script: - echo "Building branch or tag -- ${IMAGE_TAG}" - NAME="${REGISTRY_URL}/${PROJECT_NAME}/${SERVICE_NAME}" - - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile.dev . --build-arg capo_env=${CAPO_BUILD_ARG} + - docker build -t ${NAME}:${CI_COMMIT_SHORT_SHA} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile.dev . --build-arg capo_env=${CAPO_BUILD_ARG} --build-arg TAGNAME=${CI_COMMIT_SHORT_SHA} - docker tag ${NAME}:${CI_COMMIT_SHORT_SHA} ${NAME}:${IMAGE_TAG} - echo "TAG=${IMAGE_TAG}" >> build.env artifacts: diff --git a/ci/cleanup.template.yml b/ci/cleanup.template.yml index fd020b1f0db05b9bd96ebfa5b7a10e498cd80359..a36d24d9363945dd88dc3f0f82e61de7ffceaf07 100644 --- a/ci/cleanup.template.yml +++ b/ci/cleanup.template.yml @@ -6,7 +6,7 @@ printf "%s\n" "- Removing Images -" \ "${NAME}:${TAG}" \ "${NAME}:${CI_COMMIT_SHORT_SHA}" - - docker image rm --force "${NAME}:${TAG}" "${NAME}:${CI_COMMIT_SHORT_SHA}" + - docker image rm --force "${NAME}:${TAG}" "${NAME}:${CI_COMMIT_SHORT_SHA}" "cache:${CI_COMMIT_SHORT_SHA}" rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_MESSAGE =~ /\A(?i)-debug/' when: never diff --git a/ci/unit-test.template.yml b/ci/unit-test.template.yml index 20a5139bc87f357aa037a6308c15beb9d2fc2dd1..f824a459cf6bf93bff44092612d4de96bbba3717 100644 --- a/ci/unit-test.template.yml +++ b/ci/unit-test.template.yml @@ -2,8 +2,8 @@ image: ${REGISTRY_URL}/${PROJECT_NAME}/${SERVICE_NAME}:${CI_COMMIT_SHORT_SHA} script: - | - ([ ! -z ${USER} ] && su vlapipe -c "cd /code && ./bin/run-tests.sh -b") || cd /code && ./bin/run-tests.sh -b - - mv .coverage ${CI_PROJECT_DIR}/.coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA} + ([ $(whoami) = "root" ] && su vlapipe -c "cd /code && ./bin/run-tests.sh -b") || (cd /code && ./bin/run-tests.sh -b) + - mv /code/.coverage ${CI_PROJECT_DIR}/.coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA} artifacts: paths: - .coverage.${SERVICE_NAME}.${CI_COMMIT_SHORT_SHA} diff --git a/config/htcondor/execute/boot-execute.sh b/config/htcondor/execute/boot-execute.sh index 13d66ca12ab6a49e7465df7c7ac7092674c9f544..f92a7d514344f20037b0bc84c5f94cb2b4dec38e 100755 --- a/config/htcondor/execute/boot-execute.sh +++ b/config/htcondor/execute/boot-execute.sh @@ -14,7 +14,7 @@ yum install -y libffi-devel cd /opt wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz tar xzf Python-$PYTHON_VERSION.tgz && cd Python-$PYTHON_VERSION -./configure --enable-optimizations # --exec-prefix=/home/ssa --prefix=/home/ssa +./configure --enable-optimizations make altinstall cd .. rm -rf Python-$PYTHON_VERSION.tgz diff --git a/docker-compose.yml b/docker-compose.yml index 9db296a298de54d3b23036aa14d8e6177a16d7a3..c2055be71caaa33acb49b5d83cbc7fa34635e561 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: - db volumes: - ./schema:/code/schema + - ./docker.properties:/home/ssa/capo/docker.properties # Used for doing a DB migration on dsoc-dev # Uncomment this and run the command `docker compose up schema-dev` and the migration will run @@ -76,12 +77,26 @@ services: - amqp volumes: - ./services/workflow:/code + - /code/ssa_workflow.egg-info - ./shared:/packages/shared - ./apps/cli:/packages/apps/cli - ./testing:/packages/testing - ./tmp/:/tmp/workspaces_tmp - ./delivery_root:/tmp/delivery_root - ~/.capo:/home/ssa/capo + - ./docker.properties:/home/ssa/capo/docker.properties + - /packages/shared/workspaces/ssa_workspaces.egg-info + - /packages/shared/schema/ssa_schema.egg-info + - /packages/shared/messaging/ssa_messaging.egg-info + - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info + - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info + - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info + - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info + - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info + - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info + - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info + - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info + - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info capability: build: @@ -95,10 +110,24 @@ services: - notification - amqp volumes: + - ./docker.properties:/home/ssa/capo/docker.properties - ./services/capability:/code - ./shared:/packages/shared - ./apps/cli:/packages/apps/cli - ./testing:/packages/testing + - /code/ssa_capability.egg-info + - /packages/shared/workspaces/ssa_workspaces.egg-info + - /packages/shared/schema/ssa_schema.egg-info + - /packages/shared/messaging/ssa_messaging.egg-info + - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info + - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info + - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info + - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info + - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info + - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info + - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info + - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info + - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info notification: build: @@ -110,10 +139,24 @@ services: - schema - amqp volumes: + - ./docker.properties:/home/ssa/capo/docker.properties - ./services/notification:/code + - /code/ssa_notification.egg-info - ./shared:/packages/shared - ./apps/cli:/packages/apps/cli - ./testing:/packages/testing + - /packages/shared/workspaces/ssa_workspaces.egg-info + - /packages/shared/schema/ssa_schema.egg-info + - /packages/shared/messaging/ssa_messaging.egg-info + - /packages/apps/cli/utilities/wf_monitor/ssa_wf_monitor.egg-info + - /packages/apps/cli/utilities/system_mediator/ssa_system_mediator.egg-info + - /packages/apps/cli/utilities/ws_metrics/ssa_ws_metrics.egg-info + - /packages/apps/cli/utilities/aat_wrest/ssa_aat_wrest.egg-info + - /packages/apps/cli/executables/tmpdir_eraser/ssa_tmpdir_eraser.egg-info + - /packages/apps/cli/executables/pexable/productfetcher/ssa_productfetcher.egg-info + - /packages/apps/cli/executables/pexable/casa_envoy/ssa_casa_envoy.egg-info + - /packages/apps/cli/executables/pexable/delivery/ssa_delivery.egg-info + - /packages/apps/cli/executables/pexable/null/ssa_null.egg-info frontend: build: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2ef14fab6c587b5674b248bb6d195357f1091376 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +-e ../packages/shared/schema +-e ../packages/shared/messaging +-e ../packages/shared/workspaces +-e ../packages/apps/cli/utilities/wf_monitor +-e ../packages/apps/cli/utilities/ws_metrics +-e ../packages/apps/cli/utilities/aat_wrest +-e ../packages/apps/cli/utilities/system_mediator +-e ../packages/apps/cli/executables/pexable/productfetcher +-e ../packages/apps/cli/executables/pexable/delivery +-e ../packages/apps/cli/executables/pexable/null +-e ../packages/apps/cli/executables/pexable/casa_envoy +-e ../packages/apps/cli/executables/tmpdir_eraser diff --git a/services/capability/Dockerfile.dev b/services/capability/Dockerfile.dev index 637350fb320a65bcfa76f722795ad7699a85d90e..ae5e77d8d32d31c6e2e2ee801fe892dd81da2313 100644 --- a/services/capability/Dockerfile.dev +++ b/services/capability/Dockerfile.dev @@ -1,5 +1,6 @@ # This is nrao:capability -FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces +ARG TAGNAME="tmp" +FROM cache:${TAGNAME} # Change working directory to /code WORKDIR /code @@ -14,28 +15,10 @@ USER vlapipe # set ownership of content to vlapipe and the vlapipe group COPY --chown=vlapipe:vlapipe ./services/capability ./ -# Change working directory to /packages -WORKDIR /packages/ - -# -# USER root -# RUN mkdir ./apps && chown vlapipe ./apps && chgrp vlapipe ./apps -# USER vlapipe - -# Copy shared, apps/cli/, and testing directories to /packages -# set ownership of content to vlapipe and the vlapipe group -COPY --chown=vlapipe:vlapipe ./shared ./shared -COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli -COPY --chown=vlapipe:vlapipe ./testing ./testing - -# Change working directory to /code -WORKDIR /code - ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local" ENV PATH "${PATH}:/home/vlapipe/.local/bin" -RUN pip install --user -r requirements.txt \ - && python setup.py develop --install-dir="~/.local" +RUN python setup.py develop --user # Set Capo ARG capo_env=dsoc-dev diff --git a/services/capability/Dockerfile.local b/services/capability/Dockerfile.local index 94c864f1f43c59a0008f6a2fe6a83409781116d7..9428ddc6a12996a1b6419c686494e4898dec8237 100644 --- a/services/capability/Dockerfile.local +++ b/services/capability/Dockerfile.local @@ -1,20 +1,14 @@ # This is nrao:capability -FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces +FROM cache:tmp # Get capability code into the image WORKDIR /code -COPY ./services/capability ./ - -WORKDIR /packages/ -COPY ./shared ./shared -COPY ./apps/cli ./apps/cli -COPY ./testing ./testing +COPY --chown=vlapipe:vlapipe ./services/capability ./ # Python library installation WORKDIR /code/ -RUN pip install -r requirements.txt \ - && python setup.py develop +RUN python setup.py develop --user # Set Capo ARG capo_env=docker diff --git a/services/capability/bin/run-tests.sh b/services/capability/bin/run-tests.sh index cd2382c5b872cc7879559772627523baa748bec1..e8b1f26c43d26d4c9464f6c2269ebbe1d4fde94f 100755 --- a/services/capability/bin/run-tests.sh +++ b/services/capability/bin/run-tests.sh @@ -14,7 +14,9 @@ skip_arg="--skip-empty" # Install testing requirements init () { - pip install --user -r /packages/testing/requirements.txt + pushd /packages/testing + python setup.py develop --user + popd # if exists, remove old .coverage file if [[ -e .coverage ]]; then diff --git a/services/capability/requirements.txt b/services/capability/requirements.txt index 61883a3b688dc68a7102fa590785d5805fa6c343..b9fe525a7394fe99892048ad6eeb147a797741d0 100644 --- a/services/capability/requirements.txt +++ b/services/capability/requirements.txt @@ -7,15 +7,3 @@ -e ../packages/apps/cli/utilities/wf_monitor -e ../packages/apps/cli/utilities/ws_metrics -e ../packages/testing - -behave == 1.2.6 -pycapo == 0.3.0 -pyramid == 1.10 -pyramid_beaker == 0.8 -pyramid_debugtoolbar == 4.5 -pyramid_retry == 2.1.1 -pyramid_tm == 2.2.1 -requests == 2.23 -sqlalchemy == 1.3.23 -waitress == 1.4 -zope.sqlalchemy == 1.1 diff --git a/services/capability/setup.py b/services/capability/setup.py index 51eb84303cbd57f9af675a5138e82aac29ed7965..a2ecdc7c5f79fe0425ac2757ab689e6e11a07f52 100644 --- a/services/capability/setup.py +++ b/services/capability/setup.py @@ -39,12 +39,13 @@ requires = [ "pendulum", "pycapo", "pyramid", + "pyramid_retry", "pyramid_beaker", "pyramid_debugtoolbar", "pyramid_tm", "requests", "ssa-schema", - "sqlalchemy", + "sqlalchemy==1.3.23", "waitress", "ssa-workspaces", "zope.sqlalchemy", diff --git a/services/notification/Dockerfile.dev b/services/notification/Dockerfile.dev index c0a77a785007248e968dfba80696daf545c3162d..12e7470b2986ec82dd98cc9ddca91760c02f3ec7 100644 --- a/services/notification/Dockerfile.dev +++ b/services/notification/Dockerfile.dev @@ -1,5 +1,6 @@ # This is nrao:notification -FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces +ARG TAGNAME="tmp" +FROM cache:${TAGNAME} # Change working directory to /code WORKDIR /code @@ -14,27 +15,10 @@ USER vlapipe # set ownership of content to vlapipe and the vlapipe group COPY --chown=vlapipe:vlapipe ./services/notification ./ -# Change working directory to /packages -WORKDIR /packages/ - -# USER root -# RUN mkdir ./apps && chown vlapipe ./apps && chgrp vlapipe ./apps -# USER vlapipe - -# Copy shared, apps/cli/, and testing directories to /packages -# set ownership of content to vlapipe and the vlapipe group -COPY --chown=vlapipe:vlapipe ./shared ./shared -COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli -COPY --chown=vlapipe:vlapipe ./testing ./testing - -# Change working directory to /code -WORKDIR /code - ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local" ENV PATH "${PATH}:/home/vlapipe/.local/bin" -RUN pip install --user -r requirements.txt \ - && python setup.py develop --install-dir="~/.local" +RUN python setup.py develop --user # Set Capo ARG capo_env=dsoc-dev diff --git a/services/notification/Dockerfile.local b/services/notification/Dockerfile.local index 4e551c4046e26c88a8cfb50abd4a03ec4641ad14..64b7e18719d9b1ce41dc09a8866b82645805a0a5 100644 --- a/services/notification/Dockerfile.local +++ b/services/notification/Dockerfile.local @@ -1,20 +1,14 @@ # This is nrao:notification -FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces +FROM cache:tmp # Get workflow code into the image WORKDIR /code -COPY ./services/notification ./ - -WORKDIR /packages/ -COPY ./shared ./shared -COPY ./apps/cli ./apps/cli -COPY ./testing ./testing +COPY --chown=vlapipe:vlapipe ./services/notification ./ # Python library installation WORKDIR /code/ -RUN pip install -r requirements.txt \ - && python setup.py develop +RUN python setup.py develop --user # Set Capo ARG capo_env=docker diff --git a/services/notification/bin/run-tests.sh b/services/notification/bin/run-tests.sh index 3aa678d473bf8c6d58d9802d83465e7d1aadb164..9d70c8bb4908cd7ea41595d64fa6862b20a6af59 100755 --- a/services/notification/bin/run-tests.sh +++ b/services/notification/bin/run-tests.sh @@ -14,7 +14,9 @@ skip_arg="--skip-empty" # Install testing requirements init () { - pip install --user -r /packages/testing/requirements.txt + pushd /packages/testing + python setup.py develop --user + popd # if exists, remove old .coverage file if [[ -e .coverage ]]; then diff --git a/services/notification/requirements.txt b/services/notification/requirements.txt index 9c8503e2f3ffb08c20346f77eda7268530f987b2..3cf7b96d64b9427cf3a09530c23f4f91e14c5818 100644 --- a/services/notification/requirements.txt +++ b/services/notification/requirements.txt @@ -8,15 +8,3 @@ -e ../packages/apps/cli/utilities/ws_metrics -e ../packages/apps/cli/executables/pexable/null -e ../packages/testing - -pycapo == 0.3.0 -pyramid == 1.10 -pyramid_beaker == 0.8 -pyramid_debugtoolbar == 4.5 -pyramid_retry == 2.1.1 -pyramid_tm == 2.2.1 -pytest == 5.4.3 -requests == 2.23 -sqlalchemy == 1.3.23 -waitress == 1.4 -zope.sqlalchemy == 1.1 diff --git a/services/notification/setup.py b/services/notification/setup.py index 42c784308a28ca1665274075f5c1eb50f04e01d3..9ff7d3d4fb1257ec6ffffca7b693f287cc50b825 100644 --- a/services/notification/setup.py +++ b/services/notification/setup.py @@ -46,7 +46,7 @@ requires = [ "pyramid_retry", "requests", "ssa-schema", - "sqlalchemy", + "sqlalchemy==1.3.23", "waitress", "ssa-workspaces", "zope.sqlalchemy", diff --git a/services/workflow/Dockerfile.dev b/services/workflow/Dockerfile.dev index bccbdc7033dafc1a4f31a736551bf9097570c836..2ac9111f25066f14f46d3d32b59abf07fb5f722b 100644 --- a/services/workflow/Dockerfile.dev +++ b/services/workflow/Dockerfile.dev @@ -1,5 +1,8 @@ # This is nrao:workflow -FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces +ARG TAGNAME="tmp" +FROM cache:${TAGNAME} + +USER root # HTCondor install RUN apt update && apt install -y wget gnupg sudo @@ -26,23 +29,10 @@ USER vlapipe # set ownership of content to vlapipe and the vlapipe group COPY --chown=vlapipe:vlapipe ./services/workflow ./ -# Change working directory to /packages -WORKDIR /packages/ - -# Copy shared, apps/cli/, and testing directories to /packages -# set ownership of content to vlapipe and the vlapipe group id -COPY --chown=vlapipe:vlapipe ./shared ./shared -COPY --chown=vlapipe:vlapipe ./apps/cli ./apps/cli -COPY --chown=vlapipe:vlapipe ./testing ./testing - -# Change working directory to /code -WORKDIR /code - ENV PYTHONPATH "${PYTHONPATH}:/home/vlapipe/.local" ENV PATH "${PATH}:/home/vlapipe/.local/bin" -RUN pip install --user -r requirements.txt \ - && python setup.py develop --install-dir="~/.local" +RUN python setup.py develop --user # Set Capo ARG capo_env=dsoc-dev diff --git a/services/workflow/Dockerfile.local b/services/workflow/Dockerfile.local index bbb890cb752c0e0907cfe97ec010ba8f5f861b26..beebb37b5d7832cce4d85c00bc6248ee238dda95 100644 --- a/services/workflow/Dockerfile.local +++ b/services/workflow/Dockerfile.local @@ -1,19 +1,17 @@ # Workflow service layer # This is nrao:workflow -FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces +FROM cache:tmp # Get workflow code into the image WORKDIR /code -COPY ./services/workflow ./ - -WORKDIR /packages/ -COPY ./shared ./shared -COPY ./apps/cli ./apps/cli -COPY ./testing ./testing +COPY --chown=vlapipe:vlapipe ./services/workflow ./ # Python library installation WORKDIR /code/ -RUN pip install -r requirements.txt && python setup.py develop +USER vlapipe +RUN python setup.py develop --user + +USER root # HTCondor install RUN apt update && apt install -y wget gnupg sudo diff --git a/services/workflow/bin/boot-condor-and-workflow.sh b/services/workflow/bin/boot-condor-and-workflow.sh index 83ad8bbbcf72e72cf07f80e1db81df8d7eb640ad..63a3a8de57bd7cbee98ebab9233bf95473077339 100755 --- a/services/workflow/bin/boot-condor-and-workflow.sh +++ b/services/workflow/bin/boot-condor-and-workflow.sh @@ -29,11 +29,4 @@ chown root:vlapipe "$WORKFLOW_DIR"/* chmod 777 "$WORKFLOW_DIR" chmod 777 "$WORKFLOW_DIR"/* -if [ "$CAPO_PROFILE" = "dsoc-dev" ] || [ "$CAPO_PROFILE" = "dsoc-test" ] -then - # If on dev or test, boot workflow as vlapipe - su vlapipe -c "/code/bin/boot-workflow.sh" -else - # If local, just boot as root - /code/bin/boot-workflow.sh -fi +su vlapipe -c "/code/bin/boot-workflow.sh" diff --git a/services/workflow/bin/run-tests.sh b/services/workflow/bin/run-tests.sh index 64cdba99eee61f616900d6bac2d3236dabc510b2..78273031218fafa7e0a227467edd87739a8a5f21 100755 --- a/services/workflow/bin/run-tests.sh +++ b/services/workflow/bin/run-tests.sh @@ -14,10 +14,8 @@ skip_arg="--skip-empty" # Install testing requirements init () { - pip install --user -r /packages/testing/requirements.txt - pushd /packages/testing - python setup.py develop + python setup.py develop --user popd # if exists, remove old .coverage file diff --git a/services/workflow/requirements.txt b/services/workflow/requirements.txt index c01957e4663e9590cc039f8434af3be41423bdfa..93ce39163d6f76f39e3ad62a6928579eddf2d2b5 100644 --- a/services/workflow/requirements.txt +++ b/services/workflow/requirements.txt @@ -14,19 +14,3 @@ -e ../packages/apps/cli/executables/pexable/null -e ../packages/apps/cli/executables/vela -e ../packages/testing - -psycopg2 >= 2.8.5,<3.0 -pycapo == 0.3.1 -py-dag == 3.0.1 -pyramid == 1.10 -pyramid_beaker == 0.8 -pyramid_debugtoolbar == 4.5 -pyramid_retry == 2.1.1 -pyramid_tm == 2.2.1 -pytest == 5.4.3 -pytest-resource-path == 1.2.1 -requests == 2.23 -requests_mock == 1.9.2 -sqlalchemy == 1.3.23 -waitress == 1.4 -zope.sqlalchemy == 1.1 diff --git a/services/workflow/setup.py b/services/workflow/setup.py index 99a230821cb2714d70ac8eb3d8807eaaf8365443..8ce99dec1bc9f7cc786d7e5f5581d82ff36d2545 100644 --- a/services/workflow/setup.py +++ b/services/workflow/setup.py @@ -46,7 +46,7 @@ requires = [ "pyramid_retry", "requests", "ssa-schema", - "sqlalchemy", + "sqlalchemy==1.3.23", "waitress", "ssa-workspaces", "zope.sqlalchemy", diff --git a/shared/schema/setup.py b/shared/schema/setup.py index 6f335c8e5038880665837047e65d9a8a73a6396d..704ac404362f17ca27f57a25b8a8fde059139a92 100644 --- a/shared/schema/setup.py +++ b/shared/schema/setup.py @@ -19,7 +19,7 @@ setup( license="GPL", install_requires=[ "pendulum==2.1.2", - "sqlalchemy", + "sqlalchemy==1.3.23", "pycapo", "psycopg2", "mysqlclient", diff --git a/shared/workspaces/setup.py b/shared/workspaces/setup.py index f052090f99e8e6fdb19dc944e9cb292b9fa4f34d..925b39c427daa5f06d85e853c2a7b03ed193f2ee 100644 --- a/shared/workspaces/setup.py +++ b/shared/workspaces/setup.py @@ -12,7 +12,7 @@ requires = [ "pycapo", "marshmallow", "ssa-schema", - "sqlalchemy", + "sqlalchemy==1.3.23", "cx-Oracle", "chevron", "requests", diff --git a/testing/requirements.txt b/testing/requirements.txt index 7710bfb3e4469ff975de9cebe2ef82d363851214..753927e0c491082b82a876548ff2258269f0d62f 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -13,10 +13,3 @@ -e ../packages/apps/cli/executables/pexable/null -e ../packages/apps/cli/executables/pexable/casa_envoy -e ../packages/apps/cli/executables/tmpdir_eraser - -pytest>=5.4,<6.0 -pendulum==2.1.2 -pytest-mock==3.3.1 -behave==1.2.6 -pytest-cov==2.11 -mock_alchemy==0.2.1