.coveragerc

+[paths]
+source =
+	./
+	/code/
+	/packages/
+	/code/packages
\ No newline at end of file

.gitignore

@@ -65,7 +65,7 @@ services/capability/capability.log
 services/capability/workflow.log
 services/workflow/workflow.log
 services/notification/notification.log
-**/.coverage
+**/.coverage*
 **/htmlcov
 **/coverage.xml
 **/coverage.json

.gitlab-ci.yml

@@ -117,24 +117,20 @@ unit test dev notification:
         - build dev notification
 
 
-.unit test coverage:
+unit test coverage:
     stage: test-coverage
     image: python:3.8-slim
     before_script:
-        - apt update && apt install make
         - pip install pytest pytest-cov
     script:
-        - make create-coverage-config
-        - coverage combine
+        - coverage combine --append
         - coverage report
-        - coverage html
         - coverage xml
     artifacts:
         reports:
             cobertura: coverage.xml
         paths:
             - coverage.xml
-            - htmlcov/
     dependencies:
       - unit test dev workflow
       - unit test dev capability
@@ -171,18 +167,21 @@ clean build workflow:
     variables:
         SERVICE_NAME: "workflow"
     extends: .cleanup
+    allow_failure: true
 
 clean build capability:
     stage: .post
     variables:
         SERVICE_NAME: "capability"
     extends: .cleanup
+    allow_failure: true
 
 clean build notification:
     stage: .post
     variables:
         SERVICE_NAME: "notification"
     extends: .cleanup
+    allow_failure: true
 
 # Deploy Stages
 # Disabled until GL Pages are set up

Makefile

 SHELL := /bin/bash
 
+.PHONY: check-build test-dev test dev setup alembic-update docker-base build coverage clean
+
 all: dev check-build
 
 # Check if local code will pass CI build
-.PHONY: check-build
 check-build: docker-base docker-dev-images-locally test-dev
 
 # Run tests on Dockerfile.dev images
-.PHONY: test-dev
 test-dev:
 	docker run nrao:workflow ./bin/run-tests.sh
 	docker run nrao:capability ./bin/run-tests.sh
 	docker run nrao:notification ./bin/run-tests.sh
 
 # Run tests on Dockerfile.local containers
-.PHONY: test
 test:
 	docker exec workspaces_workflow_1 ./bin/run-tests.sh
 	docker exec workspaces_capability_1 ./bin/run-tests.sh
 	docker exec workspaces_notification_1 ./bin/run-tests.sh
 
 # Setup local development environment
-.PHONY: dev
 dev: docker-base
 
-.PHONY: setup
 setup:
 	docker exec workspaces_capability_1 /bin/bash -c 'python -m pip install -r requirements.txt'
 	docker exec workspaces_capability_1 /bin/bash -c 'python -m pip install -e .'
+	docker exec workspaces_workflow_1 /bin/bash -c 'python -m pip install -r requirements.txt'
 	docker exec workspaces_workflow_1 /bin/bash -c 'python -m pip install -e .'
+	docker exec workspaces_notification_1 /bin/bash -c 'python -m pip install -r requirements.txt'
 	docker exec workspaces_notification_1 /bin/bash -c 'python -m pip install -e .'
 	docker-compose restart
 
-.PHONY: alembic-update
 alembic-update:
 	cd schema; \
 	env CAPO_PROFILE=local alembic upgrade head
 
 # Build images from Dockerfile.dev
-.PHONY: docker-dev-images-locally
 docker-dev-images-locally:
 	docker build -t nrao:workflow -f services/workflow/Dockerfile.local . --build-arg capo_env=docker
 	docker build -t nrao:capability -f services/capability/Dockerfile.local . --build-arg capo_env=docker
 	docker build -t nrao:notification -f services/notification/Dockerfile.local . --build-arg capo_env=docker
 
 # Build base image
-.PHONY: docker-base
 docker-base:
 	docker build -t marconi.aoc.nrao.edu/ops/base:workspaces -f Dockerfile.base .
 
+# Build docker images
+build: docker-base
+	docker-compose build --no-cache
+
+# Generate HTML coverage report
+coverage:
+	docker exec workspaces_workflow_1 ./bin/run-tests.sh -b
+	cp services/workflow/.coverage ./.coverage.wf 
+	docker exec workspaces_capability_1 ./bin/run-tests.sh -b
+	cp services/capability/.coverage ./.coverage.cap
+	docker exec workspaces_notification_1 ./bin/run-tests.sh -b
+	cp services/notification/.coverage ./.coverage.no
+	coverage combine --append
+	coverage html
 
-# CI HELPERS
-# create .coveragerc
-.PHONY: create-coverage-config
-create-coverage-config:
-	./ci/helpers/add-coverage-rc.sh
+# Clean up environment
+clean:
+	# Clean up Docker environment
+	docker-compose down
+	-docker images -aq | xargs docker rmi
\ No newline at end of file
+	docker system prune --volumes -af
+	docker volume prune -f
+	# Delete Python cache directories
+	find . \( -name "*.egg-info" -o -name ".pytest_cache" \) | xargs rm -r

ci/helpers/add-coverage-rc.sh

-#!/bin/bash
-
-# This is a helper script to properly format the coverage reports in our CI pipeline
-# It adds a .coveragerc config file to the project root directory
-# The .coveragerc files is needed for coverage to properly generate HTML and XML reports in CI
-
-# This is to be ran by our CI build server only
-
-# Coveragerc reference https://coverage.readthedocs.io/en/latest/config.html#paths
-
-# Create .coveragerc in project root
-touch .coveragerc
-
-# printf formatting:
-    # |<string>
-    # |<string>
-    # |<tab><string>
-    # |<tab><string>
-    # |<tab><string>
-printf "%s\n%s\n\t%s\n\t%s\n\t%s\n\t%s" \
-"[paths]" \
-"source =" \
-"./" \
-"/code/src" \
-"/code" >> .coveragerc

docker-compose.yml

@@ -35,9 +35,9 @@ services:
       - schema
     volumes:
       - ./services/workflow:/code
-      - ./shared:/code/src/shared
-      - ./apps:/code/src/apps
-      - ./testing:/code/src/testing
+      - ./shared:/packages/shared
+      - ./apps:/packages/apps
+      - ./testing:/packages/testing
 
   capability:
     build:
@@ -51,9 +51,9 @@ services:
       - notification
     volumes:
       - ./services/capability:/code
-      - ./shared:/code/src/shared
-      - ./apps:/code/src/apps
-      - ./testing:/code/src/testing
+      - ./shared:/packages/shared
+      - ./apps:/packages/apps
+      - ./testing:/packages/testing
 
   notification:
     build:
@@ -65,9 +65,9 @@ services:
       - schema
     volumes:
       - ./services/notification:/code
-      - ./shared:/code/src/shared
-      - ./apps:/code/src/apps
-      - ./testing:/code/src/testing
+      - ./shared:/packages/shared
+      - ./apps:/packages/apps
+      - ./testing:/packages/testing
 
   frontend:
     build:

services/capability/Dockerfile.dev

@@ -5,11 +5,15 @@ FROM marconi.aoc.nrao.edu/ops/base:workspaces
 WORKDIR /code
 
 COPY ./services/capability ./
-COPY ./shared ./src/shared
-COPY ./apps/cli ./src/apps/cli
-COPY ./testing ./src/testing
+
+WORKDIR /packages/
+
+COPY ./shared ./shared
+COPY ./apps/cli ./apps/cli
+COPY ./testing ./testing
 
 # Python library installation
+WORKDIR /code
 RUN pip install -r requirements.txt \
     && python setup.py develop
 

services/capability/Dockerfile.local

@@ -5,11 +5,14 @@ FROM marconi.aoc.nrao.edu/ops/base:workspaces
 WORKDIR /code
 
 COPY ./services/capability ./
-COPY ./shared ./src/shared
-COPY ./apps/cli ./src/apps/cli
-COPY ./testing ./src/testing
+
+WORKDIR /packages/
+COPY ./shared ./shared
+COPY ./apps/cli ./apps/cli
+COPY ./testing ./testing
 
 # Python library installation
+WORKDIR /code/
 RUN pip install -r requirements.txt \
     && python setup.py develop
 

services/capability/bin/config/.coveragerc

 [paths]
 source =
-    ./services/capability/test
-    /code/test
+    ./services/capability/
+    /code/
\ No newline at end of file

services/capability/bin/run-tests.sh

@@ -14,9 +14,7 @@ skip_arg="--skip-empty"
 
 # Install testing requirements
 init () {
-  cd src/testing || exit
-  pip install -r requirements.txt
-  cd /code || exit
+  pip install -r /packages/testing/requirements.txt
 
 #  if exists, remove old .coverage file
   if [[ -e .coverage ]]; then
@@ -28,7 +26,7 @@ init () {
 run_tests () {
   # Look for lines in requirements.txt starting with "-e"
   # to find paths to packages containings tests to be executed.
-  pkgs=($(sed -n "s|-e ./||p" requirements.txt | grep -v testing | tr "\n" " "))
+  pkgs=($(sed -n "s|-e ..||p" requirements.txt | grep -v testing | tr "\n" " "))
   pkgs+=("$path_to_test")
 
   for pkg in "${pkgs[@]}"
@@ -44,7 +42,7 @@ run_tests () {
 }
 
 pytestWithCoverage () {
-  coverage run --parallel-mode -m pytest
+  coverage run --parallel-mode --branch -m pytest
   if [ "$1" == "b" ]; then
     cd /code/ && coverage combine --rcfile="/code/bin/config/.coveragerc" --append "$pkg" && cd "$pkg"
   else

services/capability/requirements.txt

 # This file is intended to support the Dockerfile.base, not for actual development
 # DO NOT MODIFY THIS FILE, THIS IS FOR DOCKER ONLY!
 
--e ./src/shared/schema
--e ./src/shared/channels
--e ./src/shared/workspaces
--e ./src/apps/cli/utilities/wf_monitor
--e ./src/testing
+-e ../packages/shared/schema
+-e ../packages/shared/channels
+-e ../packages/shared/workspaces
+-e ../packages/apps/cli/utilities/wf_monitor
+-e ../packages/testing
 
 behave == 1.2.6
 pycapo == 0.3.0

services/notification/Dockerfile.dev

@@ -5,11 +5,15 @@ FROM marconi.aoc.nrao.edu/ops/base:workspaces
 WORKDIR /code
 
 COPY ./services/notification ./
-COPY ./shared ./src/shared
-COPY ./apps/cli ./src/apps/cli
-COPY ./testing ./src/testing
+
+WORKDIR /packages/
+
+COPY ./shared ./shared
+COPY ./apps/cli ./apps/cli
+COPY ./testing ./testing
 
 # Python library installation
+WORKDIR /code
 RUN pip install -r requirements.txt \
     && python setup.py develop
 

services/notification/Dockerfile.local

@@ -5,11 +5,14 @@ FROM marconi.aoc.nrao.edu/ops/base:workspaces
 WORKDIR /code
 
 COPY ./services/notification ./
-COPY ./shared ./src/shared
-COPY ./apps/cli ./src/apps/cli
-COPY ./testing ./src/testing
+
+WORKDIR /packages/
+COPY ./shared ./shared
+COPY ./apps/cli ./apps/cli
+COPY ./testing ./testing
 
 # Python library installation
+WORKDIR /code/
 RUN pip install -r requirements.txt \
     && python setup.py develop
 

services/notification/bin/config/.coveragerc

 [paths]
 source =
-    ./services/notification/test
-    /code/test
+    ./services/notification/
+    /code/
\ No newline at end of file

services/notification/bin/run-tests.sh

@@ -14,9 +14,7 @@ skip_arg="--skip-empty"
 
 # Install testing requirements
 init () {
-  cd src/testing || exit
-  pip install -r requirements.txt
-  cd /code || exit
+  pip install -r /packages/testing/requirements.txt
 
 #  if exists, remove old .coverage file
   if [[ -e .coverage ]]; then
@@ -28,7 +26,7 @@ init () {
 run_tests () {
   # Look for lines in requirements.txt starting with "-e"
   # to find paths to packages containings tests to be executed.
-  pkgs=($(sed -n "s|-e ./||p" requirements.txt | grep -v testing | tr "\n" " "))
+  pkgs=($(sed -n "s|-e ..||p" requirements.txt | grep -v testing | tr "\n" " "))
   pkgs+=("$path_to_test")
 
   for pkg in "${pkgs[@]}"
@@ -44,7 +42,7 @@ run_tests () {
 }
 
 pytestWithCoverage () {
-  coverage run --parallel-mode -m pytest
+  coverage run --parallel-mode --branch -m pytest
   if [ "$1" == "b" ]; then
     cd /code/ && coverage combine --rcfile="/code/bin/config/.coveragerc" --append "$pkg" && cd "$pkg"
   else

services/notification/requirements.txt

 # This file is intended to support the Dockerfile.base, not for actual development
 # DO NOT MODIFY THIS FILE, THIS IS FOR DOCKER ONLY!
 
--e ./src/shared/schema
--e ./src/shared/channels
--e ./src/shared/workspaces
--e ./src/apps/cli/utilities/wf_monitor
--e ./src/apps/cli/executables/null
--e ./src/apps/cli/executables/vulture
--e ./src/testing
+-e ../packages/shared/schema
+-e ../packages/shared/channels
+-e ../packages/shared/workspaces
+-e ../packages/apps/cli/utilities/wf_monitor
+-e ../packages/apps/cli/executables/null
+-e ../packages/apps/cli/executables/vulture
+-e ../packages/testing
 
 pycapo == 0.3.0
 pyramid == 1.10

services/workflow/Dockerfile.dev

@@ -5,11 +5,15 @@ FROM marconi.aoc.nrao.edu/ops/base:workspaces
 WORKDIR /code
 
 COPY ./services/workflow ./
-COPY ./shared ./src/shared
-COPY ./apps/cli ./src/apps/cli
-COPY ./testing ./src/testing
+
+WORKDIR /packages/
+
+COPY ./shared ./shared
+COPY ./apps/cli ./apps/cli
+COPY ./testing ./testing
 
 # Python library installation
+WORKDIR /code
 RUN pip install -r requirements.txt \
     && python setup.py develop
 

services/workflow/Dockerfile.local

@@ -5,11 +5,14 @@ FROM marconi.aoc.nrao.edu/ops/base:workspaces
 WORKDIR /code
 
 COPY ./services/workflow ./
-COPY ./shared ./src/shared
-COPY ./apps/cli ./src/apps/cli
-COPY ./testing ./src/testing
+
+WORKDIR /packages/
+COPY ./shared ./shared
+COPY ./apps/cli ./apps/cli
+COPY ./testing ./testing
 
 # Python library installation
+WORKDIR /code/
 RUN pip install -r requirements.txt \
     && python setup.py develop
 

services/workflow/bin/config/.coveragerc

 [paths]
 source =
-    ./services/workflow/test
-    /code/test
+    ./services/workflow/
+    /code/
\ No newline at end of file

services/workflow/bin/run-tests.sh

@@ -14,9 +14,7 @@ skip_arg="--skip-empty"
 
 # Install testing requirements
 init () {
-  cd src/testing || exit
-  pip install -r requirements.txt
-  cd /code || exit
+  pip install -r /packages/testing/requirements.txt
 
 #  if exists, remove old .coverage file
   if [[ -e .coverage ]]; then
@@ -28,7 +26,7 @@ init () {
 run_tests () {
   # Look for lines in requirements.txt starting with "-e"
   # to find paths to packages containings tests to be executed.
-  pkgs=($(sed -n "s|-e ./||p" requirements.txt | grep -v testing | tr "\n" " "))
+  pkgs=($(sed -n "s|-e ..||p" requirements.txt | grep -v testing | tr "\n" " "))
   pkgs+=("$path_to_test")
 
   for pkg in "${pkgs[@]}"
@@ -44,7 +42,7 @@ run_tests () {
 }
 
 pytestWithCoverage () {
-  coverage run --parallel-mode -m pytest
+  coverage run --parallel-mode --branch -m pytest
   if [ "$1" == "b" ]; then
     cd /code/ && coverage combine --rcfile="/code/bin/config/.coveragerc" --append "$pkg" && cd "$pkg"
   else