.gitlab-ci.yml

stages:
    - pull-db
    - build-packages
    - build
    - unit-test
    - test-coverage
    - push
    - push-packages
    - deploy-coverage-page
    - generate-go-yaml
    - go-trigger
    - deploy
    - .post

variables:
    # Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/
    GIT_DEPTH: 10
    # Workspaces default variables
    PROJECT_NAME: "workspaces"
    DEPLOY_ENV: "dev"
    DL_HOST: https://dl-nrao.aoc.nrao.edu
    ENV_HOST: ws-dev.nrao.edu
    # Postgres Service Variables
    POSTGRES_DB: archive
    POSTGRES_USER: "archive"
    POSTGRES_PASSWORD: "docker"

image: docker:19.03.12

workflow:
  rules:
    - if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/
      when: never
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

# Import Templates
include:
    - '/ci/build.template.yml'
    - '/ci/push.template.yml'
    - '/ci/cleanup.template.yml'
    - '/ci/unit-test.template.yml'
    - '/ci/package-build.template.yml'
    - '/ci/push-package.template.yml'


# Unit testing steps require a specific database image to be available; this step downloads it
pull db image:
    interruptible: true
    stage: pull-db
    script:
        - echo "$HARBOR_PASSWORD" | docker login --username "$HARBOR_USER" --password-stdin "$REGISTRY_URL"
        - docker pull ${REGISTRY_URL}/workspaces/db:workspaces

###############################################
# Build Python Packages
###############################################
build carta envoy:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/carta_envoy"
  extends: .build-package

build casa envoy:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/casa_envoy"
  extends: .build-package

build core sampler:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/utilities/core_sampler"
  extends: .build-package

build conveyor:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/conveyor"
  extends: .build-package

build deliver:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/deliver"
  extends: .build-package

build ingest envoy:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/ingest_envoy"
  extends: .build-package

build mediator:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/mediator"
  extends: .build-package

build null:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/null"
  extends: .build-package

build productfetcher:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/productfetcher"
  extends: .build-package

build update stage:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/update_stage"
  extends: .build-package

build vela:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/vela"
  extends: .build-package

build wf inspector:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/wf_inspector"
  extends: .build-package

build ws annihilator:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/ws_annihilator"
  extends: .build-package

build ws metrics:
  interruptible: true
  stage: build-packages
  variables:
    PACKAGE_PATH: "apps/cli/executables/pexable/ws_metrics"
  extends: .build-package



###############################################
# Build Service and Web Images
###############################################
build workflow:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "workflow"
        PATH_PREFIX: "services/"
    extends: .build

build capability:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "capability"
        PATH_PREFIX: "services/"
    extends: .build

build notification:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "notification"
        PATH_PREFIX: "services/"
    extends: .build

build web:
    interruptible: true
    stage: build
    variables:
        SERVICE_NAME: "web"
        PATH_PREFIX: "apps/"
    extends: .build


###############################################
# Test Stages for Services
###############################################
unit test workflow:
    interruptible: true
    stage: unit-test
    variables:
        SERVICE_NAME: "workflow"
        PATH_PREFIX: "services/"
    extends: .unit-test

unit test capability:
    interruptible: true
    stage: unit-test
    variables:
        SERVICE_NAME: "capability"
        PATH_PREFIX: "services/"
    extends: .unit-test

unit test notification:
    interruptible: true
    stage: unit-test
    variables:
        SERVICE_NAME: "notification"
        PATH_PREFIX: "services/"
    extends: .unit-test


###############################################
# Create Coverage Report
###############################################
.unit test coverage:
    interruptible: true
    stage: test-coverage
    image: python:3.10-slim
    before_script:
        - pip install pytest pytest-cov
    script:
        - coverage combine --append
        - coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
        - coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
        - coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty
    artifacts:
        reports:
          coverage_report:
            coverage_format: cobertura
            path: coverage.xml
        paths:
          - htmlcov/
    dependencies:
      - unit test workflow
      - unit test capability
      - unit test notification
    rules:
        - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'


###############################################
# Push Service and Web Images to Registry
###############################################
push workflow:
    stage: push
    variables:
        SERVICE_NAME: "workflow"
        PATH_PREFIX: "services/"
    extends: .push

push capability:
    stage: push
    variables:
        SERVICE_NAME: "capability"
        PATH_PREFIX: "services/"
    extends: .push

push notification:
    stage: push
    variables:
        SERVICE_NAME: "notification"
        PATH_PREFIX: "services/"
    extends: .push

push web:
    stage: push
    variables:
        SERVICE_NAME: "web"
        PATH_PREFIX: "apps/"
    extends: .push
    # UI tests coming soon!
    # needs:
    #     - unit test dev ui

###############################################
# Push Packages
###############################################
push carta-envoy:
  stage: push-packages
  variables:
    PACKAGE_NAME: "carta_envoy"
    PIP_NAME: "carta-envoy"
  extends: .push-packages

push casa-envoy:
  stage: push-packages
  variables:
    PACKAGE_NAME: "casa_envoy"
    PIP_NAME: "casa-envoy"
  extends: .push-packages

push conveyor:
  stage: push-packages
  variables:
    PACKAGE_NAME: "conveyor"
    PIP_NAME: "conveyor"
  extends: .push-packages

push deliver:
  stage: push-packages
  variables:
    PACKAGE_NAME: "deliver"
    PIP_NAME: "ssa_deliver"
  extends: .push-packages

push ingest-envoy:
  stage: push-packages
  variables:
    PACKAGE_NAME: "ingest_envoy"
    PIP_NAME: "ingest-envoy"
  extends: .push-packages

push mediator:
  stage: push-packages
  variables:
    PACKAGE_NAME: "mediator"
    PIP_NAME: "ssa_mediator"
  extends: .push-packages

push null:
  stage: push-packages
  variables:
    PACKAGE_NAME: "null"
    PIP_NAME: "ssa_null"
  extends: .push-packages

push productfetcher:
  stage: push-packages
  variables:
    PACKAGE_NAME: "productfetcher"
    PIP_NAME: "productfetcher"
  extends: .push-packages

push update-stage:
  stage: push-packages
  variables:
    PACKAGE_NAME: "update_stage"
    PIP_NAME: "ssa-update-stage"
  extends: .push-packages

push vela:
  stage: push-packages
  variables:
    PACKAGE_NAME: "vela"
    PIP_NAME: "ssa-vela"
  extends: .push-packages

push wf-inspector:
  stage: push-packages
  variables:
    PACKAGE_NAME: "wf_inspector"
    PIP_NAME: "wf-inspector"
  extends: .push-packages

push ws-annihilator:
  stage: push-packages
  variables:
    PACKAGE_NAME: "ws_annihilator"
    PIP_NAME: "ws-annihilator"
  extends: .push-packages

push ws-metrics:
  stage: push-packages
  variables:
    PACKAGE_NAME: "ws_metrics"
    PIP_NAME: "ws-metrics"
  extends: .push-packages

###############################################
# Clean Pipeline of Service and Web Images
###############################################
clean build workflow:
    stage: .post
    variables:
        SERVICE_NAME: "workflow"
        PATH_PREFIX: "services/"
    extends: .cleanup
    allow_failure: true

clean build capability:
    stage: .post
    variables:
        SERVICE_NAME: "capability"
        PATH_PREFIX: "services/"
    extends: .cleanup
    allow_failure: true

clean build notification:
    stage: .post
    variables:
        SERVICE_NAME: "notification"
        PATH_PREFIX: "services/"
    extends: .cleanup
    allow_failure: true

clean build web:
    stage: .post
    variables:
        SERVICE_NAME: "web"
        PATH_PREFIX: "apps/"
    extends: .cleanup
    allow_failure: true


###############################################
# Deployment Stages
###############################################
.pages:
    interruptible: true
    stage: deploy-coverage-page
    image: python:3.10-slim
    dependencies:
        - unit test coverage
    before_script:
      - pip install -r docs/requirements.txt
      - apt update
      - apt install make
    script:
        - mkdir public
        - mv htmlcov public/htmlcov
        - cd docs && make html && mv _build/html/* ../public/
    artifacts:
        paths:
        - public
        expire_in: 2 weeks
    rules:
      - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'

 # Generate go builder yaml for child pipeline
go generate yaml:
   stage: generate-go-yaml
   image: python:3.10-slim
   before_script:
       - apt update && apt install -y git
       - pip install pyyaml
   script:
       - DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-go-yaml.py
   rules:
       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
         changes:
           - apps/cli/executables/go/**/*
       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
         variables:
           # override DEPLOY_ENV
           DEPLOY_ENV: "test"
       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
         variables:
           DEPLOY_ENV: "prod"
   artifacts:
       paths:
           - generated-go-build-pipeline.yml

 # Trigger child pipeline based on generated go builder yaml
go child pipeline:
   stage: go-trigger
   trigger:
       include:
           - artifact: generated-go-build-pipeline.yml
             job: go generate yaml
       strategy: depend
   rules:
       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
         changes:
           - apps/cli/executables/go/**/*
       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
         variables:
           # override DEPLOY_ENV
           DEPLOY_ENV: "test"
       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
         variables:
           DEPLOY_ENV: "prod"

# Development
deploy:
    stage: deploy
    script:
        # Docker doesn't allow variable interpolation when declaring Docker Secret names
        # This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:"
        - sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
        - sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml
        - ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST WS_VERSION=$VERSION BASE_REGISTRY_URL=$REGISTRY_URL docker stack deploy --compose-file docker-compose.yml --with-registry-auth workspaces-${DEPLOY_ENV}
    rules:
        - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
          variables:
            IMAGE_TAG: ${CI_COMMIT_BRANCH}
            # override DEPLOY_ENV
            DEPLOY_ENV: "dev"
            # override DL_HOST
            DL_HOST: https://dl-nrao.nrao.edu
            # override ENV_HOST
            ENV_HOST: ws-dev.nrao.edu
            VERSION: 0.0.1+$CI_COMMIT_BRANCH
        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            # override DEPLOY_ENV
            DEPLOY_ENV: "test"
            # override DL_HOST
            DL_HOST: https://dl-dsoc-test.nrao.edu
            # override ENV_HOST
            ENV_HOST: ws-test.nrao.edu
            VERSION: 0.0.2+$CI_COMMIT_TAG
        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            # override DEPLOY_ENV
            DEPLOY_ENV: "test"
            # override DL_HOST
            DL_HOST: https://dl-dsoc-test.nrao.edu
            # override ENV_HOST
            ENV_HOST: ws-test.nrao.edu
            VERSION: $CI_COMMIT_TAG
        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
          variables:
            IMAGE_TAG: $CI_COMMIT_TAG
            # override DEPLOY_ENV
            DEPLOY_ENV: "prod"
            # override DL_HOST
            DL_HOST: https://dl-dsoc.nrao.edu
            # override ENV_HOST
            ENV_HOST: ws.nrao.edu
            VERSION: $CI_COMMIT_TAG