Newer
Older

Andrew Kapuscinski
committed
- build-base
- push-base

Andrew Kapuscinski
committed
- build
- unit-test

Andrew Kapuscinski
committed
- push

Andrew Kapuscinski
committed
- deploy

Andrew Kapuscinski
committed
- e2e-test

Andrew Kapuscinski
committed
# Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/
GIT_DEPTH: 10
# Workspaces default variables

Andrew Kapuscinski
committed
DEPLOY_ENV: "dev"

Andrew Kapuscinski
committed
DL_HOST: https://dl-nrao.aoc.nrao.edu
# Postgres Service Variables
POSTGRES_DB: archive
POSTGRES_USER: "archive"
POSTGRES_PASSWORD: "docker"
image: docker:19.03.12

Andrew Kapuscinski
committed
workflow:
rules:
- if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/
when: never

Andrew Kapuscinski
committed
- if: $CI_MERGE_REQUEST_IID
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# Import Templates
include:
- '/ci/build.template.yml'
- '/ci/push.template.yml'
- '/ci/cleanup.template.yml'
- '/ci/unit-test.template.yml'

Andrew Kapuscinski
committed
# Build Base Image

Andrew Kapuscinski
committed
build base image:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build-base
script:
- docker build -t ${REGISTRY_URL}/ops/base:${PROJECT_NAME} -f Dockerfile.base .
- docker tag ${REGISTRY_URL}/ops/base:${PROJECT_NAME} ${REGISTRY_URL}/ops/base:${CI_COMMIT_SHORT_SHA}

Andrew Kapuscinski
committed
rules:

Charlotte Hausman
committed
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

Andrew Kapuscinski
committed
changes:

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
# Push Base Image Stage
push base image:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: push-base
script:
- echo "$HARBOR_PASSWORD" | docker login -u "$HARBOR_USER" --password-stdin $REGISTRY_URL
- docker push ${REGISTRY_URL}/ops/base:${PROJECT_NAME}

Andrew Kapuscinski
committed
rules:

Charlotte Hausman
committed
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

Andrew Kapuscinski
committed
changes:

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
variables:
# For building pycapo pex
# Enable Git submodules https://docs.gitlab.com/ee/ci/git_submodules.html#use-git-submodules-in-cicd-jobs
GIT_SUBMODULE_STRATEGY: recursive
script:
- docker build -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache .

Andrew Kapuscinski
committed
build workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "workflow"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "capability"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "notification"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build web:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "web"
PATH_PREFIX: "apps/"
extends: .build
## Test Stages ##
# Unit Tests

Andrew Kapuscinski
committed
unit test workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "workflow"
extends: .unit-test

Andrew Kapuscinski
committed
- build workflow

Andrew Kapuscinski
committed
unit test capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "capability"
extends: .unit-test

Andrew Kapuscinski
committed
- build capability

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
unit test notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "notification"
extends: .unit-test

Andrew Kapuscinski
committed
- build notification
# Generate Coverage reports

Andrew Kapuscinski
committed
unit test coverage:

Andrew Kapuscinski
committed
interruptible: true
- pip install pytest pytest-cov

Andrew Kapuscinski
committed
- coverage combine --append
- coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
artifacts:
reports:
cobertura: coverage.xml
paths:
- coverage.xml

Andrew Kapuscinski
committed
- unit test workflow
- unit test capability
- unit test notification
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

Andrew Kapuscinski
committed
# E2E Tests
e2e:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: e2e-test
image: docker/compose:1.27.4
# CI Postgres Service
services:
- name: ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces
alias: db

Andrew Kapuscinski
committed
variables:

Andrew Kapuscinski
committed
before_script:

Andrew Kapuscinski
committed
- docker build -t base-build:${CI_COMMIT_SHORT_SHA} --target base-build -f apps/web/Dockerfile --build-arg env=dev .
- docker build -t e2e:${CI_COMMIT_SHORT_SHA} -f apps/web/Dockerfile.ci . --build-arg TAGNAME=${CI_COMMIT_SHORT_SHA}

Andrew Kapuscinski
committed
script:
# setting env variables with .env https://docs.docker.com/compose/environment-variables/
- echo "ENV=$DEPLOY_ENV" >> .env
- echo "TAG=$CI_COMMIT_SHORT_SHA" >> .env
- echo "DL_HOST=$DL_HOST" >> .env

Andrew Kapuscinski
committed
# set ENV_HOST to workflow because the workflow container isn't using host networking
- echo "ENV_HOST=$ENV_HOST" >> .env

Andrew Kapuscinski
committed
- docker-compose -f docker-compose.ci.yml -p ws-${CI_COMMIT_SHORT_SHA} up -d capability workflow notification web

Andrew Kapuscinski
committed
- docker-compose -f docker-compose.ci.yml -p ws-${CI_COMMIT_SHORT_SHA} run e2e

Andrew Kapuscinski
committed
after_script:
# log the containers for visibility into why e2e tests failed

Andrew Kapuscinski
committed
- docker logs ws-${CI_COMMIT_SHORT_SHA}_capability_1
- docker logs ws-${CI_COMMIT_SHORT_SHA}_workflow_1
- docker logs ws-${CI_COMMIT_SHORT_SHA}_web_1

Andrew Kapuscinski
committed
- COMPOSE_PROJECT_NAME=ws-${CI_COMMIT_SHORT_SHA} docker-compose -f docker-compose.ci.yml down

Andrew Kapuscinski
committed
- docker image rm -f base-build:${CI_COMMIT_SHORT_SHA}

Andrew Kapuscinski
committed
- docker image rm -f e2e:${CI_COMMIT_SHORT_SHA}
dependencies: []

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
push workflow:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "workflow"
extends: .push

Andrew Kapuscinski
committed
push capability:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "capability"
extends: .push

Andrew Kapuscinski
committed
push notification:
stage: push
variables:
SERVICE_NAME: "notification"
extends: .push

Andrew Kapuscinski
committed
push web:
stage: push
variables:
SERVICE_NAME: "web"
extends: .push
# UI tests coming soon!
# needs:
# - unit test dev ui
stage: .post
variables:
SERVICE_NAME: "workflow"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
stage: .post
variables:
SERVICE_NAME: "capability"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
clean build notification:
stage: .post
variables:
SERVICE_NAME: "notification"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
clean build web:
stage: .post
variables:
SERVICE_NAME: "web"
extends: .cleanup
allow_failure: true

Andrew Kapuscinski
committed
interruptible: true
stage: deploy-coverage-page
image: python:3.8-slim
dependencies:
- unit test coverage
before_script:
- pip install -r docs/requirements.txt
- apt update
- apt install make
- mv htmlcov public/htmlcov
- cd docs && make html && mv _build/html/* ../public/
artifacts:
paths:
- public
expire_in: 2 weeks
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

Andrew Kapuscinski
committed
deploy:
stage: deploy
# Docker doesn't allow variable interpolation when declaring Docker Secret names
# This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:"
- sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST docker stack deploy --compose-file docker-compose.yml workspaces-${DEPLOY_ENV}
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

Andrew Kapuscinski
committed
variables:
IMAGE_TAG: ${CI_DEFAULT_BRANCH}
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'

Andrew Kapuscinski
committed
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "test"
# override DL_HOST
DL_HOST: https://dl-dsoc-test.nrao.edu

Andrew Kapuscinski
committed
# override ENV_HOST

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "prod"
# override DL_HOST
DL_HOST: https://dl-dsoc.nrao.edu
# override ENV_HOST
ENV_HOST: ws.nrao.edu

Andrew Kapuscinski
committed
# e2e:
# stage: e2e-test
# image: trion/ng-cli-karma:11.2.7
# script:
# - cd apps/web/
# - npm install
# - ng e2e --configuration=${DEPLOY_ENV}
# rules:
# - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
# variables:
# # override DEPLOY_ENV
# DEPLOY_ENV: "test"
# when: manual
# allow_failure: true