Story 1636 - Merge Compose Files.

.env

+# Infrastructure Related
+BASE_REGISTRY_URL="brooks.aoc.nrao.edu:5000/ssa-docker/workspaces"   # the url for the project's docker registry
+BASE_IMAGE_TAG=prod   # TODO: remove once cache image removed
+CACHE_IMAGE_TAG=prod  # TODO: remove once cache image removed
+ENV=prod   # the default environment to build for (one of: dev, test, local, prod)
+TAG=prod   # the tag name to pull images from when building
+ENV_HOST="ws.nrao.edu"
+DL_HOST="https://dl-dsoc.nrao.edu"
+NG_APP_WS_VERSION=${ENV}
+WS_VERSION=unknown-version
+
+# CAPO Environment Properties
+CAPO_PATH=/home/casa/capo
+CAPO_PROFILE=dsoc-prod
\ No newline at end of file

.gitlab-ci.yml

@@ -27,6 +27,7 @@ variables:
     POSTGRES_DB: archive
     POSTGRES_USER: "archive"
     POSTGRES_PASSWORD: "docker"
+    BASE_REGISTRY_URL: "brooks.aoc.nrao.edu:5000/ssa-docker/workspaces"
 
 image: docker:19.03.12
 
@@ -50,26 +51,18 @@ build base image:
     interruptible: true
     stage: build-base
     script:
-        - docker build --no-cache -t ${REGISTRY_URL}/ops/base:${PROJECT_NAME} -f Dockerfile.base .
-        - docker tag ${REGISTRY_URL}/ops/base:${PROJECT_NAME} ${REGISTRY_URL}/ops/base:${CI_COMMIT_SHORT_SHA}
-    rules:
-        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-          changes:
-            - Dockerfile.base
-            - docker.properties
+        - docker build --no-cache -t ${BASE_REGISTRY_URL}/base:${CI_COMMIT_SHORT_SHA} -f Dockerfile.base .
+        - docker build -t ${BASE_REGISTRY_URL}/web/base:${CI_COMMIT_SHORT_SHA} -f apps/web/Dockerfile.base .
 
 # Push Base Image Stage
 push base image:
     interruptible: true
     stage: push-base
     script:
-        - echo "$HARBOR_PASSWORD" | docker login -u "$HARBOR_USER" --password-stdin $REGISTRY_URL
-        - docker push ${REGISTRY_URL}/ops/base:${PROJECT_NAME}
-    rules:
-        - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
-          changes:
-            - Dockerfile.base
-            - docker.properties
+        - echo "$REGISTRY_URL_PASS" | docker login --username "$REGISTRY_URL_USER" --password-stdin $BASE_REGISTRY_URL
+        - docker push ${BASE_REGISTRY_URL}/base:${CI_COMMIT_SHORT_SHA}
+        - docker push ${BASE_REGISTRY_URL}/web/base:${CI_COMMIT_SHORT_SHA}
+        - docker pull ${BASE_REGISTRY_URL}/db:workspaces
 
 build-pex-base image:
   interruptible: true
@@ -97,7 +90,8 @@ build cache:
         # Enable Git submodules https://docs.gitlab.com/ee/ci/git_submodules.html#use-git-submodules-in-cicd-jobs
         GIT_SUBMODULE_STRATEGY: recursive
     script:
-        - docker build --no-cache -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache . --build-arg WS_VERSION=${VERSION}
+        - echo "$REGISTRY_URL_PASS" | docker login --username "$REGISTRY_URL_USER" --password-stdin $BASE_REGISTRY_URL
+        - docker build --no-cache -t ${BASE_REGISTRY_URL}/cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache . --build-arg WS_VERSION=${VERSION} --build-arg BASE_IMAGE_TAG=${CI_COMMIT_SHORT_SHA} --build-arg BASE_REGISTRY_URL
     rules:
         - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
           variables:
@@ -107,7 +101,6 @@ build cache:
           variables:
             IMAGE_TAG: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
             VERSION: 0.0.1+$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
         - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
           variables:
             IMAGE_TAG: $CI_COMMIT_TAG
@@ -185,7 +178,7 @@ unit test notification:
         - build notification
 
 # Generate Coverage reports
-unit test coverage:
+.unit test coverage:
     interruptible: true
     stage: test-coverage
     image: python:3.10-slim
@@ -269,7 +262,7 @@ clean build web:
     allow_failure: true
 
 # Deploy Stages
-pages:
+.pages:
     interruptible: true
     stage: deploy-coverage-page
     image: python:3.10-slim
@@ -386,7 +379,7 @@ deploy:
         # This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:"
         - sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
         - sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
-        - ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST docker stack deploy --compose-file docker-compose.yml workspaces-${DEPLOY_ENV}
+        - ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST BASE_REGISTRY_URL=$BASE_REGISTRY_URL docker stack deploy --compose-file docker-compose.yml workspaces-${DEPLOY_ENV} -e TAG=${IMAGE_TAG}
     rules:
         - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
           variables:

Dockerfile.cache

-FROM ssa-containers.aoc.nrao.edu/ops/base:workspaces
-
+ARG BASE_IMAGE_TAG=dev
 ARG WS_VERSION=unknown-version
-ENV WS_VERSION=${WS_VERSION}
+ARG BASE_REGISTRY_URL
+FROM ${BASE_REGISTRY_URL}/base:${BASE_IMAGE_TAG}
 
+ENV WS_VERSION=${WS_VERSION}
 WORKDIR /code/
 RUN chown vlapipe . && chgrp vlapipe .
 

Makefile

@@ -45,9 +45,9 @@ alembic-update:
 
 # Build images from Dockerfile
 docker-dev-images-locally:
-	docker build -t nrao:workflow -f services/workflow/Dockerfile.local . --build-arg capo_env=docker
-	docker build -t nrao:capability -f services/capability/Dockerfile.local . --build-arg capo_env=docker
-	docker build -t nrao:notification -f services/notification/Dockerfile.local . --build-arg capo_env=docker
+	docker build -t nrao:workflow -f services/workflow/Dockerfile . --build-arg ENV=local --build-arg CACHE_IMAGE_TAG=local
+	docker build -t nrao:capability -f services/capability/Dockerfile . --build-arg ENV=local --build-arg CACHE_IMAGE_TAG=local
+	docker build -t nrao:notification -f services/notification/Dockerfile . --build-arg ENV=local --build-arg CACHE_IMAGE_TAG=local
 
 # Build base image
 docker-base: db
@@ -55,8 +55,8 @@ docker-base: db
 	docker build --no-cache -t ssa-containers.aoc.nrao.edu/ops/base:nodejs-14 -f apps/web/Dockerfile.base .
 
 # Build cache image
-cache:
-	 docker build --no-cache -t cache:tmp -f Dockerfile.cache . --build-arg WS_VERSION=0.0.0+local
+cache local:
+	 docker build --no-cache -t cache:local -f Dockerfile.cache . --build-arg WS_VERSION=0.0.0+local
 
 # Build DB image
 db:
@@ -100,3 +100,11 @@ docs: godocs
 
 godocs:
 	./build_go_docs.sh
+
+
+build base images:
+    docker build -t local/base:local -f Dockerfile.base .
+    docker build -t local/cache:local -f Dockerfile.cache . --build-arg BASE_IMAGE_TAG=local --build-arg BASE_REGISTRY_URL=localz
+    docker build -t workflow:local -f services/workflow/Dockerfile . --build-arg ENV=local --build-arg CACHE_IMAGE_TAG=local --target dev
+    docker build -t workflow:local -f services/workflow/Dockerfile . --build-arg ENV=local --build-arg CACHE_IMAGE_TAG=local --target dev
+    docker build -t workflow:local -f services/workflow/Dockerfile . --build-arg ENV=local --build-arg CACHE_IMAGE_TAG=local --target dev

apps/cli/executables/pexable/carta_envoy/carta_envoy/__init__.py

@@ -15,3 +15,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Workspaces system for launching CARTA for viewing images
+"""
+__version__ = "2.9.0rc1"

apps/cli/executables/pexable/carta_envoy/carta_envoy/_version.py

-#
-# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
-#
-# This file is part of NRAO Workspaces.
-#
-# Workspaces is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Workspaces is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
-""" Version information for this package, don't put anything else here. """
-___version___ = "4.0.0a1.dev1"

apps/cli/executables/pexable/carta_envoy/pyproject.toml

+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "ssa-carta-envoy"
+authors = [{ name = "SSA Team", email = "dms-ssa@nrao.edu" }]
+readme = "README.md"
+license = { file = "LICENSE" }
+classifiers = ["License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)"]
+dynamic = ["version", "description"]
+dependencies = ["pycapo", "redis", "requests", "pendulum", "pex==2.1.119"]
+
+[project.urls]
+Home = "https://ssa.gitlab-pages.nrao.edu/workspaces"
+
+[tool.flit.module]
+name = "carta_envoy"
\ No newline at end of file

apps/cli/executables/pexable/carta_envoy/setup.py

-#!/usr/bin/python
-#
-# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
-#
-# This file is part of NRAO Workspaces.
-#
-# Workspaces is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Workspaces is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
-# -*- coding: utf-8 -*-
-
-from pathlib import Path
-
-from setuptools import find_packages, setup
-
-VERSION = open("carta_envoy/_version.py").readlines()[-1].split()[-1].strip("\"'")
-README = Path("README.md").read_text()
-
-requires = ["pycapo", "redis", "requests", "pendulum", "pex==2.1.119"]
-
-setup(
-    name="ssa-" + Path().absolute().name,
-    version=VERSION,
-    description="Workspaces system for launching CARTA for viewing images",
-    long_description=README,
-    author="NRAO SSA Team",
-    author_email="dms-ssa@nrao.edu",
-    url="TBD",
-    license="GPL",
-    install_requires=requires,
-    keywords=[],
-    packages=find_packages(),
-    classifiers=["Programming Language :: Python :: 3.10"],
-    entry_points={"console_scripts": ["carta_envoy = carta_envoy.carta:main"]},
-)

apps/cli/executables/pexable/casa_envoy/casa_envoy/__init__.py

@@ -15,3 +15,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Workspaces CASA functionality bridge
+"""
+__version__ = "2.9.0rc1"

apps/cli/executables/pexable/casa_envoy/casa_envoy/_version.py

-#
-# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
-#
-# This file is part of NRAO Workspaces.
-#
-# Workspaces is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Workspaces is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
-""" Version information for this package, don't put anything else here. """
-___version___ = "4.0.0a1.dev1"

apps/cli/executables/pexable/casa_envoy/pyproject.toml

+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "ssa-casa-envoy"
+authors = [{ name = "SSA Team", email = "dms-ssa@nrao.edu" }]
+readme = "README.md"
+license = { file = "LICENSE" }
+classifiers = ["License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)"]
+dynamic = ["version", "description"]
+dependencies = ["pycapo", "bs4", "lxml", "prettierfier", "pex==2.1.119"]
+
+[project.urls]
+Home = "https://ssa.gitlab-pages.nrao.edu/workspaces"
+
+[tool.flit.module]
+name = "casa_envoy"
+
+[project.scripts]
+casa_envoy = "casa_envoy.palaver:main"

apps/cli/executables/pexable/casa_envoy/setup.py

-#!/usr/bin/python
-#
-# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
-#
-# This file is part of NRAO Workspaces.
-#
-# Workspaces is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Workspaces is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
-# -*- coding: utf-8 -*-
-
-from pathlib import Path
-
-from setuptools import find_packages, setup
-
-VERSION = open("casa_envoy/_version.py").readlines()[-1].split()[-1].strip("\"'")
-README = Path("README.md").read_text()
-
-requires = ["pycapo", "bs4", "lxml", "prettierfier", "pex==2.1.119"]
-
-setup(
-    name="ssa-" + Path().absolute().name,
-    version=VERSION,
-    description="Workspaces CASA functionality bridge",
-    long_description=README,
-    author="NRAO SSA Team",
-    author_email="dms-ssa@nrao.edu",
-    url="TBD",
-    license="GPL",
-    install_requires=requires,
-    keywords=[],
-    packages=find_packages(),
-    classifiers=["Programming Language :: Python :: 3.10"],
-    entry_points={"console_scripts": ["casa_envoy = casa_envoy.palaver:main"]},
-)

apps/cli/executables/pexable/conveyor/conveyor/__init__.py

@@ -15,3 +15,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Conveyor
+"""
+__version__ = "2.9.0rc1"

apps/cli/executables/pexable/conveyor/pyproject.toml

+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "ssa-conveyor"
+authors = [{ name = "SSA Team", email = "dms-ssa@nrao.edu" }]
+readme = "README.md"
+license = { file = "LICENSE" }
+classifiers = ["License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)"]
+dynamic = ["version", "description"]
+dependencies = ["pycapo==0.3.1", "requests", "pex==2.1.119"]
+
+[project.urls]
+Home = "https://ssa.gitlab-pages.nrao.edu/workspaces"
+
+[tool.flit.module]
+name = "conveyor"
+
+[project.scripts]
+conveyor = "conveyor.conveyor:main"

apps/cli/executables/pexable/conveyor/setup.py

-#!/usr/bin/python
-#
-# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
-#
-# This file is part of NRAO Workspaces.
-#
-# Workspaces is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Workspaces is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
-# -*- coding: utf-8 -*-
-
-from pathlib import Path
-
-from setuptools import find_packages, setup
-
-VERSION = open("conveyor/_version.py").readlines()[-1].split()[-1].strip("\"'")
-README = Path("README.md").read_text()
-
-requires = ["pycapo==0.3.1", "requests", "pex==2.1.119"]
-
-setup(
-    name="ssa-" + Path().absolute().name,
-    version=VERSION,
-    description="NRAO Workspaces Standard Capability Delivery/Post QA System",
-    long_description=README,
-    author="NRAO SSA Team",
-    author_email="dms-ssa@nrao.edu",
-    url="TBD",
-    license="GPL",
-    install_requires=requires,
-    tests_require=["pytest>=5.4,<6.0", "pytest-resource-path"],
-    keywords=[],
-    packages=find_packages(),
-    classifiers=["Programming Language :: Python :: 3.10"],
-    entry_points={"console_scripts": ["conveyor = conveyor.conveyor:main"]},
-)

apps/cli/executables/pexable/deliver/delivery/__init__.py

@@ -15,3 +15,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+"""
+Workspaces data delivery module
+"""
+__version__ = "2.9.0rc1"