Newer
Older

Andrew Kapuscinski
committed
- build-base
- push-base

Andrew Kapuscinski
committed
- build
- unit-test

Andrew Kapuscinski
committed
- push

Andrew Kapuscinski
committed
- deploy
# - e2e-test # e2e tests disabled

Andrew Kapuscinski
committed
# Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/
GIT_DEPTH: 10
# Workspaces default variables

Andrew Kapuscinski
committed
DEPLOY_ENV: "dev"

Andrew Kapuscinski
committed
DL_HOST: https://dl-nrao.aoc.nrao.edu
# Postgres Service Variables
POSTGRES_DB: archive
POSTGRES_USER: "archive"
POSTGRES_PASSWORD: "docker"
image: docker:19.03.12

Andrew Kapuscinski
committed
workflow:
rules:
- if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/
when: never

Andrew Kapuscinski
committed
- if: $CI_MERGE_REQUEST_IID
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# Import Templates
include:
- '/ci/build.template.yml'
- '/ci/push.template.yml'
- '/ci/cleanup.template.yml'
- '/ci/unit-test.template.yml'

Andrew Kapuscinski
committed
# Build Base Image

Andrew Kapuscinski
committed
build base image:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build-base
script:
- docker build -t ${REGISTRY_URL}/ops/base:${PROJECT_NAME} -f Dockerfile.base .
- docker tag ${REGISTRY_URL}/ops/base:${PROJECT_NAME} ${REGISTRY_URL}/ops/base:${CI_COMMIT_SHORT_SHA}

Andrew Kapuscinski
committed
rules:

Charlotte Hausman
committed
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

Andrew Kapuscinski
committed
changes:

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
# Push Base Image Stage
push base image:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: push-base
script:
- echo "$HARBOR_PASSWORD" | docker login -u "$HARBOR_USER" --password-stdin $REGISTRY_URL
- docker push ${REGISTRY_URL}/ops/base:${PROJECT_NAME}

Andrew Kapuscinski
committed
rules:

Charlotte Hausman
committed
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

Andrew Kapuscinski
committed
changes:

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
variables:
# For building pycapo pex
# Enable Git submodules https://docs.gitlab.com/ee/ci/git_submodules.html#use-git-submodules-in-cicd-jobs
GIT_SUBMODULE_STRATEGY: recursive
- docker build -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache . --build-arg WS_VERSION=${IMAGE_TAG}
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
IMAGE_TAG: $CI_COMMIT_BRANCH
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
variables:
IMAGE_TAG: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG

Andrew Kapuscinski
committed
build workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "workflow"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "capability"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "notification"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build web:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "web"
PATH_PREFIX: "apps/"
extends: .build
## Test Stages ##
# Unit Tests

Andrew Kapuscinski
committed
unit test workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "workflow"
extends: .unit-test

Andrew Kapuscinski
committed
- build workflow

Andrew Kapuscinski
committed
unit test capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "capability"
extends: .unit-test

Andrew Kapuscinski
committed
- build capability

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
unit test notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "notification"
extends: .unit-test

Andrew Kapuscinski
committed
- build notification
# Generate Coverage reports

Andrew Kapuscinski
committed
unit test coverage:

Andrew Kapuscinski
committed
interruptible: true
- pip install pytest pytest-cov

Andrew Kapuscinski
committed
- coverage combine --append
- coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
artifacts:
reports:
cobertura: coverage.xml
paths:
- coverage.xml

Andrew Kapuscinski
committed
- unit test workflow
- unit test capability
- unit test notification
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

Andrew Kapuscinski
committed
# E2E Tests

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: e2e-test
image: docker/compose:1.27.4
# CI Postgres Service
services:
- name: ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces
alias: db

Andrew Kapuscinski
committed
variables:

Andrew Kapuscinski
committed
before_script:

Andrew Kapuscinski
committed
- docker build -t base-build:${CI_COMMIT_SHORT_SHA} --target base-build -f apps/web/Dockerfile --build-arg env=dev .
- docker build -t e2e:${CI_COMMIT_SHORT_SHA} -f apps/web/Dockerfile.ci . --build-arg TAGNAME=${CI_COMMIT_SHORT_SHA}

Andrew Kapuscinski
committed
script:
# setting env variables with .env https://docs.docker.com/compose/environment-variables/
- echo "ENV=$DEPLOY_ENV" >> .env
- echo "TAG=$CI_COMMIT_SHORT_SHA" >> .env
- echo "DL_HOST=$DL_HOST" >> .env

Andrew Kapuscinski
committed
# set ENV_HOST to workflow because the workflow container isn't using host networking
- echo "ENV_HOST=$ENV_HOST" >> .env

Andrew Kapuscinski
committed
- docker-compose -f docker-compose.ci.yml -p ws-${CI_COMMIT_SHORT_SHA} up -d capability workflow notification web

Andrew Kapuscinski
committed
- docker-compose -f docker-compose.ci.yml -p ws-${CI_COMMIT_SHORT_SHA} run e2e

Andrew Kapuscinski
committed
after_script:
# log the containers for visibility into why e2e tests failed

Andrew Kapuscinski
committed
- docker logs ws-${CI_COMMIT_SHORT_SHA}_capability_1
- docker logs ws-${CI_COMMIT_SHORT_SHA}_workflow_1
- docker logs ws-${CI_COMMIT_SHORT_SHA}_web_1

Andrew Kapuscinski
committed
- COMPOSE_PROJECT_NAME=ws-${CI_COMMIT_SHORT_SHA} docker-compose -f docker-compose.ci.yml down

Andrew Kapuscinski
committed
- docker image rm -f base-build:${CI_COMMIT_SHORT_SHA}

Andrew Kapuscinski
committed
- docker image rm -f e2e:${CI_COMMIT_SHORT_SHA}
dependencies: []

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
push workflow:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "workflow"
extends: .push

Andrew Kapuscinski
committed
push capability:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "capability"
extends: .push

Andrew Kapuscinski
committed
push notification:
stage: push
variables:
SERVICE_NAME: "notification"
extends: .push

Andrew Kapuscinski
committed
push web:
stage: push
variables:
SERVICE_NAME: "web"
extends: .push
# UI tests coming soon!
# needs:
# - unit test dev ui
stage: .post
variables:
SERVICE_NAME: "workflow"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
stage: .post
variables:
SERVICE_NAME: "capability"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
clean build notification:
stage: .post
variables:
SERVICE_NAME: "notification"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
clean build web:
stage: .post
variables:
SERVICE_NAME: "web"
extends: .cleanup
allow_failure: true

Andrew Kapuscinski
committed
interruptible: true
stage: deploy-coverage-page
image: python:3.8-slim
dependencies:
- unit test coverage
before_script:
- pip install -r docs/requirements.txt
- apt update
- apt install make
- mv htmlcov public/htmlcov
- cd docs && make html && mv _build/html/* ../public/
artifacts:
paths:
- public
expire_in: 2 weeks
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# Generate PEX builder yaml for child pipeline
pex generate yaml:
stage: generate-yaml
image: python:3.8-slim
before_script:
- apt update && apt install -y git
- pip install pyyaml
script:
- DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-yaml.py
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
changes:
- apps/cli/executables/pexable/**/*

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
artifacts:
paths:
- generated-pex-build-pipeline.yml
# Trigger child pipeline based on generated PEX builder yaml
pex child pipeline:
stage: trigger
trigger:
include:
- artifact: generated-pex-build-pipeline.yml
job: pex generate yaml
strategy: depend
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
changes:
- apps/cli/executables/pexable/**/*

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
# Generate go builder yaml for child pipeline
go generate yaml:
stage: generate-yaml
image: python:3.8-slim
before_script:
- apt update && apt install -y git
- pip install pyyaml
script:
- DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-go-yaml.py
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
changes:
- apps/cli/executables/go/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
artifacts:
paths:
- generated-go-build-pipeline.yml
# Trigger child pipeline based on generated go builder yaml
go child pipeline:
stage: trigger
trigger:
include:
- artifact: generated-go-build-pipeline.yml
job: go generate yaml
strategy: depend
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
changes:
- apps/cli/executables/go/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"

Andrew Kapuscinski
committed
deploy:
stage: deploy
# Docker doesn't allow variable interpolation when declaring Docker Secret names
# This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:"
- sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST docker stack deploy --compose-file docker-compose.yml workspaces-${DEPLOY_ENV}
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

Andrew Kapuscinski
committed
variables:
IMAGE_TAG: ${CI_DEFAULT_BRANCH}

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'

Andrew Kapuscinski
committed
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "test"
# override DL_HOST
DL_HOST: https://dl-dsoc-test.nrao.edu

Andrew Kapuscinski
committed
# override ENV_HOST

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "prod"
# override DL_HOST
DL_HOST: https://dl-dsoc.nrao.edu
# override ENV_HOST
ENV_HOST: ws.nrao.edu

Andrew Kapuscinski
committed
# e2e:
# stage: e2e-test
# image: trion/ng-cli-karma:11.2.7
# script:
# - cd apps/web/
# - npm install
# - ng e2e --configuration=${DEPLOY_ENV}
# rules:
# - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
# variables:
# # override DEPLOY_ENV
# DEPLOY_ENV: "test"
# when: manual
# allow_failure: true