stages: - pull-db - build-packages - build - unit-test - test-coverage - push - push-packages - deploy-coverage-page - generate-go-yaml - go-trigger - deploy - .post variables: # Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/ GIT_DEPTH: 10 # Workspaces default variables PROJECT_NAME: "workspaces" DEPLOY_ENV: "dev" DL_HOST: https://dl-nrao.aoc.nrao.edu ENV_HOST: ws-dev.nrao.edu # Postgres Service Variables POSTGRES_DB: archive POSTGRES_USER: "archive" POSTGRES_PASSWORD: "docker" RELEASE_VERSION: 2.8.2.3rc1 image: docker:19.03.12 workflow: rules: - if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/ when: never - if: $CI_MERGE_REQUEST_IID - if: $CI_COMMIT_TAG - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9](\.[0-9])*)-DEVELOPMENT/' # Import Templates include: - '/ci/build.template.yml' - '/ci/push.template.yml' - '/ci/cleanup.template.yml' - '/ci/unit-test.template.yml' - '/ci/package-build.template.yml' - '/ci/push-package.template.yml' # Unit testing steps require a specific database image to be available; this step downloads it pull db image: interruptible: true stage: pull-db script: - echo "$HARBOR_PASSWORD" | docker login --username "$HARBOR_USER" --password-stdin "$REGISTRY_URL" - docker pull ${REGISTRY_URL}/workspaces/db:workspaces ############################################### # Build Python Packages ############################################### build carta envoy: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/carta_envoy" extends: .build-package build casa envoy: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/casa_envoy" extends: .build-package build core sampler: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/utilities/core_sampler" extends: .build-package build conveyor: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/conveyor" extends: .build-package build data annotator: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/data_annotator" extends: .build-package build deliver: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/deliver" extends: .build-package build ingest envoy: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/ingest_envoy" extends: .build-package build mediator: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/mediator" extends: .build-package build null: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/null" extends: .build-package build productfetcher: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/productfetcher" extends: .build-package build update stage: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/update_stage" extends: .build-package build vela: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/vela" extends: .build-package build wf inspector: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/wf_inspector" extends: .build-package build ws annihilator: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/ws_annihilator" extends: .build-package build ws metrics: interruptible: true stage: build-packages variables: PACKAGE_PATH: "apps/cli/executables/pexable/ws_metrics" extends: .build-package ############################################### # Build Service and Web Images ############################################### build workflow: interruptible: true stage: build variables: SERVICE_NAME: "workflow" PATH_PREFIX: "services/" extends: .build build capability: interruptible: true stage: build variables: SERVICE_NAME: "capability" PATH_PREFIX: "services/" extends: .build build notification: interruptible: true stage: build variables: SERVICE_NAME: "notification" PATH_PREFIX: "services/" extends: .build build web: interruptible: true stage: build variables: SERVICE_NAME: "web" PATH_PREFIX: "apps/" extends: .build ############################################### # Test Stages for Services ############################################### unit test workflow: interruptible: true stage: unit-test variables: SERVICE_NAME: "workflow" PATH_PREFIX: "services/" extends: .unit-test unit test capability: interruptible: true stage: unit-test variables: SERVICE_NAME: "capability" PATH_PREFIX: "services/" extends: .unit-test unit test notification: interruptible: true stage: unit-test variables: SERVICE_NAME: "notification" PATH_PREFIX: "services/" extends: .unit-test ############################################### # Create Coverage Report ############################################### unit test coverage: interruptible: true stage: test-coverage image: python:3.10-slim before_script: - pip install pytest pytest-cov script: - coverage combine --append - coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty - coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty - coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty artifacts: reports: coverage_report: coverage_format: cobertura path: coverage.xml paths: - htmlcov/ dependencies: - unit test workflow - unit test capability - unit test notification rules: # for testing only! # - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/' variables: # override DEPLOY_ENV DEPLOY_ENV: "test" - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/' variables: DEPLOY_ENV: "prod" ############################################### # Push Service and Web Images to Registry ############################################### push workflow: stage: push variables: SERVICE_NAME: "workflow" PATH_PREFIX: "services/" extends: .push push capability: stage: push variables: SERVICE_NAME: "capability" PATH_PREFIX: "services/" extends: .push push notification: stage: push variables: SERVICE_NAME: "notification" PATH_PREFIX: "services/" extends: .push push web: stage: push variables: SERVICE_NAME: "web" PATH_PREFIX: "apps/" extends: .push # UI tests coming soon! # needs: # - unit test dev ui ############################################### # Push Packages ############################################### push carta-envoy: stage: push-packages variables: PACKAGE_NAME: "carta_envoy" PIP_NAME: "carta-envoy" extends: .push-packages push casa-envoy: stage: push-packages variables: PACKAGE_NAME: "casa_envoy" PIP_NAME: "casa-envoy" extends: .push-packages push conveyor: stage: push-packages variables: PACKAGE_NAME: "conveyor" PIP_NAME: "conveyor" extends: .push-packages push data-annotator: stage: push-packages variables: PACKAGE_NAME: "data_annotator" PIP_NAME: "data-annotator" extends: .push-packages push deliver: stage: push-packages variables: PACKAGE_NAME: "deliver" PIP_NAME: "ssa_deliver" extends: .push-packages push ingest-envoy: stage: push-packages variables: PACKAGE_NAME: "ingest_envoy" PIP_NAME: "ingest-envoy" extends: .push-packages push mediator: stage: push-packages variables: PACKAGE_NAME: "mediator" PIP_NAME: "ssa_mediator" extends: .push-packages push null: stage: push-packages variables: PACKAGE_NAME: "null" PIP_NAME: "ssa_null" extends: .push-packages push productfetcher: stage: push-packages variables: PACKAGE_NAME: "productfetcher" PIP_NAME: "productfetcher" extends: .push-packages push update-stage: stage: push-packages variables: PACKAGE_NAME: "update_stage" PIP_NAME: "ssa-update-stage" extends: .push-packages push vela: stage: push-packages variables: PACKAGE_NAME: "vela" PIP_NAME: "ssa-vela" extends: .push-packages push wf-inspector: stage: push-packages variables: PACKAGE_NAME: "wf_inspector" PIP_NAME: "wf-inspector" extends: .push-packages push ws-annihilator: stage: push-packages variables: PACKAGE_NAME: "ws_annihilator" PIP_NAME: "ws-annihilator" extends: .push-packages push ws-metrics: stage: push-packages variables: PACKAGE_NAME: "ws_metrics" PIP_NAME: "ws-metrics" extends: .push-packages ############################################### # Clean Pipeline of Service and Web Images ############################################### clean build workflow: stage: .post variables: SERVICE_NAME: "workflow" PATH_PREFIX: "services/" extends: .cleanup allow_failure: true clean build capability: stage: .post variables: SERVICE_NAME: "capability" PATH_PREFIX: "services/" extends: .cleanup allow_failure: true clean build notification: stage: .post variables: SERVICE_NAME: "notification" PATH_PREFIX: "services/" extends: .cleanup allow_failure: true clean build web: stage: .post variables: SERVICE_NAME: "web" PATH_PREFIX: "apps/" extends: .cleanup allow_failure: true ############################################### # Deployment Stages ############################################### pages: interruptible: true stage: deploy-coverage-page image: python:3.10-slim dependencies: - unit test coverage before_script: - pip install -r docs/requirements.txt - apt update - apt install make script: - ls -al docs - mkdir -p public - mv htmlcov public/htmlcov - cd docs && make html && mv _build/html/* ../public/ artifacts: paths: - public expire_in: 2 weeks rules: # For testing only! # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9](\.[0-9])*)-DEVELOPMENT/' # - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/' variables: # override DEPLOY_ENV DEPLOY_ENV: "test" - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/' variables: DEPLOY_ENV: "prod" # Generate go builder yaml for child pipeline go generate yaml: stage: generate-go-yaml image: python:3.10-slim before_script: - apt update && apt install -y git - pip install pyyaml script: - DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-go-yaml.py rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9](\.[0-9])*)-DEVELOPMENT/' variables: DEPLOY_ENV: "dev" changes: - apps/cli/executables/go/**/* - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/' variables: # override DEPLOY_ENV DEPLOY_ENV: "test" - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/' variables: DEPLOY_ENV: "prod" artifacts: paths: - generated-go-build-pipeline.yml # Trigger child pipeline based on generated go builder yaml go child pipeline: stage: go-trigger trigger: include: - artifact: generated-go-build-pipeline.yml job: go generate yaml strategy: depend rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9](\.[0-9])*)-DEVELOPMENT/' allow_failure: true variables: DEPLOY_ENV: "dev" changes: - apps/cli/executables/go/**/* - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/' variables: # override DEPLOY_ENV DEPLOY_ENV: "test" - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/' variables: DEPLOY_ENV: "prod" # Development deploy: stage: deploy script: # Docker doesn't allow variable interpolation when declaring Docker Secret names # This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:" - sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml - sed -i "s/naasc_ENV_secrets:/naasc_${DEPLOY_ENV}_secrets:/g" docker-compose.yml - ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST WS_VERSION=$VERSION docker stack deploy --compose-file docker-compose.yml --with-registry-auth workspaces-${DEPLOY_ENV} rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9](\.[0-9])*)-DEVELOPMENT/' variables: IMAGE_TAG: ${CI_COMMIT_BRANCH} # override DEPLOY_ENV DEPLOY_ENV: "dev" # override DL_HOST DL_HOST: https://dl-nrao.nrao.edu # override ENV_HOST ENV_HOST: ws-dev.nrao.edu VERSION: 0.0.1+$CI_COMMIT_BRANCH - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/' variables: IMAGE_TAG: $CI_COMMIT_TAG # override DEPLOY_ENV DEPLOY_ENV: "test" # override DL_HOST DL_HOST: https://dl-dsoc-test.nrao.edu # override ENV_HOST ENV_HOST: ws-test.nrao.edu VERSION: 0.0.2+$CI_COMMIT_TAG - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/' variables: IMAGE_TAG: $CI_COMMIT_TAG # override DEPLOY_ENV DEPLOY_ENV: "test" # override DL_HOST DL_HOST: https://dl-dsoc-test.nrao.edu # override ENV_HOST ENV_HOST: ws-test.nrao.edu VERSION: $CI_COMMIT_TAG - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/' variables: IMAGE_TAG: $CI_COMMIT_TAG # override DEPLOY_ENV DEPLOY_ENV: "prod" # override DL_HOST DL_HOST: https://dl-dsoc.nrao.edu # override ENV_HOST ENV_HOST: ws.nrao.edu VERSION: $CI_COMMIT_TAG