.gitlab-ci.yml

stages:
    - build-base
    - push-base
    - cache-build
    - build
    - build-pex-base
    - unit-test
    - test-coverage
    - push
    - deploy-coverage-page
    - generate-pex-yaml
    - pex-trigger
    - generate-go-yaml
    - go-trigger
    - deploy
    - .post

variables:
    # Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/
    GIT_DEPTH: 10
    # Workspaces default variables
    PROJECT_NAME: "workspaces"
    DEPLOY_ENV: "dev"
    DL_HOST: https://dl-nrao.aoc.nrao.edu
    ENV_HOST: ws-dev.nrao.edu
    # Postgres Service Variables
    POSTGRES_DB: archive
    POSTGRES_USER: "archive"
    POSTGRES_PASSWORD: "docker"

image: docker:19.03.12

workflow:
  rules:
    - if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/
      when: never
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

# Import Templates
include:
    - '/ci/build.template.yml'
    - '/ci/push.template.yml'
    - '/ci/cleanup.template.yml'
    - '/ci/unit-test.template.yml'

# Build Base Image
build base image:
    interruptible: true
    stage: build-base
    script:
        - docker build --no-cache -t ${REGISTRY_URL}/ops/base:${PROJECT_NAME} -f Dockerfile.base .
        - docker tag ${REGISTRY_URL}/ops/base:${PROJECT_NAME} ${REGISTRY_URL}/ops/base:${CI_COMMIT_SHORT_SHA}
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
          changes:
            - Dockerfile.base
            - docker.properties

# Push Base Image Stage
push base image:
    interruptible: true
    stage: push-base
    script:
        - echo "$HARBOR_PASSWORD" | docker login -u "$HARBOR_USER" --password-stdin $REGISTRY_URL
        - docker push ${REGISTRY_URL}/ops/base:${PROJECT_NAME}
    rules:
        - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
          changes:
            - Dockerfile.base
            - docker.properties

build-pex-base image:
  interruptible: true
  stage: build-pex-base
  script:
    - docker build --no-cache -t nrao_pex_base_3_10 -f ./ci/bin/Dockerfile.pex .
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELPMENT/'
      changes:
        - apps/cli/executables/pexable/**/*
    - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
      variables:
        # override DEPLOY_ENV
        DEPLOY_ENV: "test"
    - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
      variables:
        DEPLOY_ENV: "prod"

# Cache
build cache:
    interruptible: true
    stage: cache-build
    variables:
        # For building pycapo pex
        # Enable Git submodules https://docs.gitlab.com/ee/ci/git_submodules.html#use-git-submodules-in-cicd-jobs
        GIT_SUBMODULE_STRATEGY: recursive
    script:
        - docker build --no-cache -t cache:${CI_COMMIT_SHORT_SHA} -f Dockerfile.cache . --build-arg WS_VERSION=${VERSION}
    rules:
        - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
          variables:
            IMAGE_TAG: $CI_COMMIT_BRANCH
            VERSION: 0.0.1+$CI_COMMIT_BRANCH
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
          variables:
            IMAGE_TAG: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
            VERSION: 0.0.1+$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            VERSION: 0.0.2+$CI_COMMIT_TAG
        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            VERSION: $CI_COMMIT_TAG
        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            VERSION: $CI_COMMIT_TAG

# Build Stages
build workflow:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "workflow"
        PATH_PREFIX: "services/"
    extends: .build

build capability:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "capability"
        PATH_PREFIX: "services/"
    extends: .build

build notification:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "notification"
        PATH_PREFIX: "services/"
    extends: .build

build web:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "web"
        PATH_PREFIX: "apps/"
    extends: .build

## Test Stages ##

# Unit Tests
unit test workflow:
    interruptible: true
    stage: unit-test
    variables:
        SERVICE_NAME: "workflow"
    extends: .unit-test
    needs:
        - build workflow

unit test capability:
    interruptible: true
    stage: unit-test
    variables:
        SERVICE_NAME: "capability"
    extends: .unit-test
    needs:
        - build capability

unit test notification:
    interruptible: true
    stage: unit-test
    variables:
        SERVICE_NAME: "notification"
    extends: .unit-test
    needs:
        - build notification

# Generate Coverage reports
unit test coverage:
    interruptible: true
    stage: test-coverage
    image: python:3.10-slim
    before_script:
        - pip install pytest pytest-cov
    script:
        - coverage combine --append
        - coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
        - coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
        - coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
    artifacts:
        reports:
          coverage_report:
            coverage_format: cobertura
            path: coverage.xml
        paths:
          - htmlcov/
    dependencies:
      - unit test workflow
      - unit test capability
      - unit test notification
    rules:
        - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

# Push Stages
push workflow:
    stage: push
    variables:
        SERVICE_NAME: "workflow"
    extends: .push

push capability:
    stage: push
    variables:
        SERVICE_NAME: "capability"
    extends: .push

push notification:
    stage: push
    variables:
        SERVICE_NAME: "notification"
    extends: .push

push web:
    stage: push
    variables:
        SERVICE_NAME: "web"
    extends: .push
    # UI tests coming soon!
    # needs:
    #     - unit test dev ui

# Cleanup
clean build workflow:
    stage: .post
    variables:
        SERVICE_NAME: "workflow"
    extends: .cleanup
    allow_failure: true

clean build capability:
    stage: .post
    variables:
        SERVICE_NAME: "capability"
    extends: .cleanup
    allow_failure: true

clean build notification:
    stage: .post
    variables:
        SERVICE_NAME: "notification"
    extends: .cleanup
    allow_failure: true

clean build web:
    stage: .post
    variables:
        SERVICE_NAME: "web"
    extends: .cleanup
    allow_failure: true

# Deploy Stages
pages:
    interruptible: true
    stage: deploy-coverage-page
    image: python:3.10-slim
    dependencies:
        - unit test coverage
    before_script:
      - pip install -r docs/requirements.txt
      - apt update
      - apt install make
    script:
        - mkdir public
        - mv htmlcov public/htmlcov
        - cd docs && make html && mv _build/html/* ../public/
    artifacts:
        paths:
        - public
        expire_in: 2 weeks
    rules:
      - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

# Generate PEX builder yaml for child pipeline
pex generate yaml:
   stage: generate-pex-yaml
   image: python:3.10-slim
   before_script:
       - apt update && apt install -y git
       - pip install pyyaml
   script:
       - DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-yaml.py
   rules:
       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
         changes:
           - apps/cli/executables/pexable/**/*
       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
         variables:
           # override DEPLOY_ENV
           DEPLOY_ENV: "test"
       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
         variables:
           DEPLOY_ENV: "prod"
   artifacts:
       paths:
           - generated-pex-build-pipeline.yml

 # Trigger child pipeline based on generated PEX builder yaml
pex child pipeline:
   stage: pex-trigger
   trigger:
       include:
           - artifact: generated-pex-build-pipeline.yml
             job: pex generate yaml
       strategy: depend
   rules:
       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
         changes:
           - apps/cli/executables/pexable/**/*
       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
         variables:
           # override DEPLOY_ENV
           DEPLOY_ENV: "test"
       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
         variables:
           DEPLOY_ENV: "prod"

 # Generate go builder yaml for child pipeline
go generate yaml:
   stage: generate-go-yaml
   image: python:3.10-slim
   before_script:
       - apt update && apt install -y git
       - pip install pyyaml
   script:
       - DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-go-yaml.py
   rules:
       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
         changes:
           - apps/cli/executables/go/**/*
       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
         variables:
           # override DEPLOY_ENV
           DEPLOY_ENV: "test"
       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
         variables:
           DEPLOY_ENV: "prod"
   artifacts:
       paths:
           - generated-go-build-pipeline.yml

 # Trigger child pipeline based on generated go builder yaml
go child pipeline:
   stage: go-trigger
   trigger:
       include:
           - artifact: generated-go-build-pipeline.yml
             job: go generate yaml
       strategy: depend
   rules:
       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
         changes:
           - apps/cli/executables/go/**/*
       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
         variables:
           # override DEPLOY_ENV
           DEPLOY_ENV: "test"
       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
         variables:
           DEPLOY_ENV: "prod"

# Development
deploy:
    stage: deploy
    script:
        # Docker doesn't allow variable interpolation when declaring Docker Secret names
        # This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:"
        - sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
        - sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
        - ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST docker stack deploy --compose-file docker-compose.yml workspaces-${DEPLOY_ENV}
    rules:
        - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
          variables:
            IMAGE_TAG: ${CI_COMMIT_BRANCH}
        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            # override DEPLOY_ENV
            DEPLOY_ENV: "test"
            # override DL_HOST
            DL_HOST: https://dl-dsoc-test.nrao.edu
            # override ENV_HOST
            ENV_HOST: ws-test.nrao.edu
        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            # override DEPLOY_ENV
            DEPLOY_ENV: "prod"
            # override DL_HOST
            DL_HOST: https://dl-dsoc.nrao.edu
            # override ENV_HOST
            ENV_HOST: ws.nrao.edu