Newer
Older

Daniel Lyons
committed
- build-packages

Andrew Kapuscinski
committed
- build
- unit-test

Andrew Kapuscinski
committed
- push

Andrew Kapuscinski
committed
- deploy

Andrew Kapuscinski
committed
# Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/
GIT_DEPTH: 10
# Workspaces default variables

Andrew Kapuscinski
committed
DEPLOY_ENV: "dev"

Andrew Kapuscinski
committed
DL_HOST: https://dl-nrao.aoc.nrao.edu
# Postgres Service Variables
POSTGRES_DB: archive
POSTGRES_USER: "archive"
POSTGRES_PASSWORD: "docker"
image: docker:19.03.12

Andrew Kapuscinski
committed
workflow:
rules:
- if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/
when: never

Andrew Kapuscinski
committed
- if: $CI_MERGE_REQUEST_IID
- if: $CI_COMMIT_TAG
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

Andrew Kapuscinski
committed
# Import Templates
include:
- '/ci/build.template.yml'
- '/ci/push.template.yml'
- '/ci/cleanup.template.yml'
- '/ci/unit-test.template.yml'
- '/ci/package-build.template.yml'
# Unit testing steps require a specific database image to be available; this step downloads it

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
script:
- echo "$HARBOR_PASSWORD" | docker login --username "$HARBOR_USER" --password-stdin "$REGISTRY_URL"
- docker pull ${REGISTRY_URL}/workspaces/db:workspaces

Andrew Kapuscinski
committed
###############################################
# Build Python Packages
###############################################
build carta envoy:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/carta_envoy"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/carta_envoy/**/*"
build casa envoy:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/casa_envoy"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/casa_envoy/**/*"
build core sampler:
interruptible: true

Daniel Lyons
committed
stage: build-packages
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/utilities/core_sampler"
only:
changes:
- "apps/cli/utilities/core_sampler/**/*"
build conveyor:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/conveyor"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/conveyor/**/*"
build deliver:

Daniel Lyons
committed
stage: build-packages

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/deliver"
only:
changes:
- "apps/cli/executables/pexable/deliver/**/*"
build ingest envoy:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/ingest_envoy"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/ingest_envoy/**/*"
build mediator:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/mediator"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/mediator/**/*"
build null:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/null"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/null/**/*"
build productfetcher:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/productfetcher"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/productfetcher/**/*"
build update stage:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/update_stage"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/update_stage/**/*"
build vela:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/vela"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/vela/**/*"
build wf inspector:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/wf_inspector"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/wf_inspector/**/*"
build ws annihilator:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/ws_annihilator"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/ws_annihilator/**/*"
build ws metrics:

Brittany Faciane
committed
interruptible: true

Daniel Lyons
committed
stage: build-packages

Brittany Faciane
committed
variables:

Daniel Lyons
committed
PACKAGE_PATH: "apps/cli/executables/pexable/ws_metrics"

Brittany Faciane
committed
only:
changes:
- "apps/cli/executables/pexable/ws_metrics/**/*"
###############################################
# Build Service and Web Images
###############################################

Andrew Kapuscinski
committed
build workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "workflow"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "capability"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "notification"
PATH_PREFIX: "services/"

Andrew Kapuscinski
committed
build web:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: build
variables:
SERVICE_NAME: "web"
PATH_PREFIX: "apps/"
extends: .build
###############################################
# Test Stages for Services
###############################################

Andrew Kapuscinski
committed
unit test workflow:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "workflow"
extends: .unit-test

Andrew Kapuscinski
committed
unit test capability:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "capability"
extends: .unit-test

Andrew Kapuscinski
committed

Andrew Kapuscinski
committed
unit test notification:

Andrew Kapuscinski
committed
interruptible: true

Andrew Kapuscinski
committed
stage: unit-test
variables:
SERVICE_NAME: "notification"
###############################################
# Create Coverage Report
###############################################

Andrew Kapuscinski
committed
interruptible: true
- pip install pytest pytest-cov

Andrew Kapuscinski
committed
- coverage combine --append
- coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
- coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
coverage_report:
coverage_format: cobertura
path: coverage.xml

Charlotte Hausman
committed
paths:
- htmlcov/

Andrew Kapuscinski
committed
- unit test workflow
- unit test capability
- unit test notification
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
###############################################
# Push Service and Web Images to Registry
###############################################

Andrew Kapuscinski
committed
push workflow:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "workflow"

Andrew Kapuscinski
committed
extends: .push

Andrew Kapuscinski
committed
push capability:
stage: push

Andrew Kapuscinski
committed
variables:
SERVICE_NAME: "capability"

Andrew Kapuscinski
committed
extends: .push

Andrew Kapuscinski
committed
push notification:
stage: push
variables:
SERVICE_NAME: "notification"

Andrew Kapuscinski
committed
push web:
stage: push
variables:
SERVICE_NAME: "web"
extends: .push
# UI tests coming soon!
# needs:
# - unit test dev ui
###############################################
# Clean Pipeline of Service and Web Images
###############################################
stage: .post
variables:
SERVICE_NAME: "workflow"

Andrew Kapuscinski
committed
allow_failure: true
stage: .post
variables:
SERVICE_NAME: "capability"

Andrew Kapuscinski
committed
allow_failure: true
clean build notification:
stage: .post
variables:
SERVICE_NAME: "notification"

Andrew Kapuscinski
committed
allow_failure: true
clean build web:
stage: .post
variables:
SERVICE_NAME: "web"
extends: .cleanup
allow_failure: true
###############################################
# Deployment Stages
###############################################

Andrew Kapuscinski
committed
interruptible: true
dependencies:
- unit test coverage
before_script:
- pip install -r docs/requirements.txt
- apt update
- apt install make
- mv htmlcov public/htmlcov
- cd docs && make html && mv _build/html/* ../public/
artifacts:
paths:
- public
expire_in: 2 weeks
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
before_script:
- apt update && apt install -y git
- pip install pyyaml
script:
- DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-go-yaml.py
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/go/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
artifacts:
paths:
- generated-go-build-pipeline.yml
# Trigger child pipeline based on generated go builder yaml
stage: go-trigger
trigger:
include:
- artifact: generated-go-build-pipeline.yml
job: go generate yaml
strategy: depend
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/go/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"

Andrew Kapuscinski
committed
stage: deploy
# Docker doesn't allow variable interpolation when declaring Docker Secret names
# This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:"
- sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
- ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST BASE_REGISTRY_URL=$REGISTRY_URL docker stack deploy --compose-file docker-compose.yml --with-registry-auth workspaces-${DEPLOY_ENV}
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

Andrew Kapuscinski
committed
variables:
IMAGE_TAG: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
# override DEPLOY_ENV
DEPLOY_ENV: "dev"
# override DL_HOST
# override ENV_HOST
ENV_HOST: ws-dev.nrao.edu

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'

Andrew Kapuscinski
committed
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "test"
# override DL_HOST
DL_HOST: https://dl-dsoc-test.nrao.edu

Andrew Kapuscinski
committed
# override ENV_HOST

Andrew Kapuscinski
committed
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
IMAGE_TAG: $CI_COMMIT_TAG
# override DEPLOY_ENV
DEPLOY_ENV: "prod"
# override DL_HOST
DL_HOST: https://dl-dsoc.nrao.edu
# override ENV_HOST
ENV_HOST: ws.nrao.edu