Newer
Older

Andrew Kapuscinski
committed
- build-base
- push-base

Andrew Kapuscinski
committed
- build

Andrew Kapuscinski
committed
- unit-test

Andrew Kapuscinski
committed
- push
- generate-pex-yaml
- pex-trigger
- generate-go-yaml
- go-trigger

Andrew Kapuscinski
committed
- deploy

Andrew Kapuscinski
committed
# Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/
GIT_DEPTH: 10
# Workspaces default variables

Andrew Kapuscinski
committed
DEPLOY_ENV: "dev"

Andrew Kapuscinski
committed
DL_HOST: https://dl-nrao.aoc.nrao.edu
# Postgres Service Variables
POSTGRES_DB: archive
POSTGRES_USER: "archive"
POSTGRES_PASSWORD: "docker"
image: docker:19.03.12

Andrew Kapuscinski
committed
workflow:
rules:
- if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/
when: never

Andrew Kapuscinski
committed
- if: $CI_MERGE_REQUEST_IID
- if: $CI_COMMIT_TAG
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

Andrew Kapuscinski
committed
# Import Templates
include:
- '/ci/build.template.yml'
- '/ci/push.template.yml'
- '/ci/cleanup.template.yml'
- '/ci/unit-test.template.yml'

Andrew Kapuscinski
committed
# Build Base Image

Andrew Kapuscinski
committed
build base image:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build-base
script:
- docker build --no-cache -t ${REGISTRY_URL}/ops/base:${PROJECT_NAME} -f Dockerfile.base .
- docker tag ${REGISTRY_URL}/ops/base:${PROJECT_NAME} ${REGISTRY_URL}/ops/base:${CI_COMMIT_SHORT_SHA}

Andrew Kapuscinski
committed
rules:

Charlotte Hausman
committed
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

Andrew Kapuscinski
committed
changes:

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
# Push Base Image Stage
push base image:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: push-base
script:
- echo "$HARBOR_PASSWORD" | docker login -u "$HARBOR_USER" --password-stdin $REGISTRY_URL
- docker push ${REGISTRY_URL}/ops/base:${PROJECT_NAME}

Andrew Kapuscinski
committed
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

Andrew Kapuscinski
committed
changes:

Andrew Kapuscinski
committed
build-pex-base image:
interruptible: true
stage: build-pex-base
script:
- docker build --no-cache -t nrao_pex_base_3_10 -f ./ci/bin/Dockerfile.pex .
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELPMENT/'
changes:
- apps/cli/executables/pexable/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
variables:
# For building pycapo pex
# Enable Git submodules https://docs.gitlab.com/ee/ci/git_submodules.html#use-git-submodules-in-cicd-jobs
GIT_SUBMODULE_STRATEGY: recursive
- docker build --no-cache -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache . --build-arg WS_VERSION=${VERSION}
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
variables:
IMAGE_TAG: $CI_COMMIT_BRANCH
VERSION: 0.0.1+$CI_COMMIT_BRANCH
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
variables:
IMAGE_TAG: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
VERSION: 0.0.1+$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
VERSION: 0.0.2+$CI_COMMIT_TAG
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
VERSION: $CI_COMMIT_TAG
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
VERSION: $CI_COMMIT_TAG

Andrew Kapuscinski
committed
build workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "workflow"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "capability"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "notification"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build web:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "web"
PATH_PREFIX: "apps/"
extends: .build
## Test Stages ##
# Unit Tests

Andrew Kapuscinski
committed
unit test workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "workflow"
extends: .unit-test

Andrew Kapuscinski
committed
- build workflow

Andrew Kapuscinski
committed
unit test capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "capability"
extends: .unit-test

Andrew Kapuscinski
committed
- build capability

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
unit test notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "notification"
extends: .unit-test

Andrew Kapuscinski
committed
- build notification
# Generate Coverage reports

Andrew Kapuscinski
committed
unit test coverage:

Andrew Kapuscinski
committed
interruptible: true
- pip install pytest pytest-cov

Andrew Kapuscinski
committed
- coverage combine --append
- coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
coverage_report:
coverage_format: cobertura
path: coverage.xml

Charlotte Hausman
committed
paths:
- htmlcov/

Andrew Kapuscinski
committed
- unit test workflow
- unit test capability
- unit test notification
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

Andrew Kapuscinski
committed
push workflow:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "workflow"
extends: .push

Andrew Kapuscinski
committed
push capability:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "capability"
extends: .push

Andrew Kapuscinski
committed
push notification:
stage: push
variables:
SERVICE_NAME: "notification"
extends: .push

Andrew Kapuscinski
committed
push web:
stage: push
variables:
SERVICE_NAME: "web"
extends: .push
# UI tests coming soon!
# needs:
# - unit test dev ui
stage: .post
variables:
SERVICE_NAME: "workflow"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
stage: .post
variables:
SERVICE_NAME: "capability"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
clean build notification:
stage: .post
variables:
SERVICE_NAME: "notification"
extends: .cleanup

Andrew Kapuscinski
committed
allow_failure: true
clean build web:
stage: .post
variables:
SERVICE_NAME: "web"
extends: .cleanup
allow_failure: true

Andrew Kapuscinski
committed
interruptible: true
dependencies:
- unit test coverage
before_script:
- pip install -r docs/requirements.txt
- apt update
- apt install make
- mv htmlcov public/htmlcov
- cd docs && make html && mv _build/html/* ../public/
artifacts:
paths:
- public
expire_in: 2 weeks
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
# Generate PEX builder yaml for child pipeline
pex generate yaml:
before_script:
- apt update && apt install -y git
- pip install pyyaml
script:
- DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-yaml.py
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/pexable/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
artifacts:
paths:
- generated-pex-build-pipeline.yml
# Trigger child pipeline based on generated PEX builder yaml
stage: pex-trigger
trigger:
include:
- artifact: generated-pex-build-pipeline.yml
job: pex generate yaml
strategy: depend
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/pexable/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
# Generate go builder yaml for child pipeline
before_script:
- apt update && apt install -y git
- pip install pyyaml
script:
- DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-go-yaml.py
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/go/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
artifacts:
paths:
- generated-go-build-pipeline.yml
# Trigger child pipeline based on generated go builder yaml
stage: go-trigger
trigger:
include:
- artifact: generated-go-build-pipeline.yml
job: go generate yaml
strategy: depend
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/go/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"

Andrew Kapuscinski
committed
deploy:
stage: deploy
# Docker doesn't allow variable interpolation when declaring Docker Secret names
# This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:"
- sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST docker stack deploy --compose-file docker-compose.yml workspaces-${DEPLOY_ENV}
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

Andrew Kapuscinski
committed
variables:

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'

Andrew Kapuscinski
committed
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "test"
# override DL_HOST
DL_HOST: https://dl-dsoc-test.nrao.edu

Andrew Kapuscinski
committed
# override ENV_HOST

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "prod"
# override DL_HOST
DL_HOST: https://dl-dsoc.nrao.edu
# override ENV_HOST
ENV_HOST: ws.nrao.edu