stages: - build-base - push-base - run-schema - build - unit-test - test-coverage - push - deploy-coverage-page - deploy - e2e-test - .post # - deploy-test variables: # Gitlab optimization https://docs.gitlab.com/ee/ci/large_repositories/ GIT_DEPTH: 10 # Workspaces default variables PROJECT_NAME: "workspaces" DEPLOY_ENV: "dev" DL_HOST: https://dl-nrao.aoc.nrao.edu ENV_HOST: ws-dev.nrao.edu # Postgres Service Variables POSTGRES_DB: archive POSTGRES_USER: "archive" POSTGRES_PASSWORD: "docker" # CI Postgres Service services: - name: ssa-containers.aoc.nrao.edu/ops/ci/db:workspaces alias: db image: docker:19.03.12 workflow: rules: - if: $CI_MERGE_REQUEST_TITLE =~ /^WIP:|^Draft:/ when: never - if: $CI_MERGE_REQUEST_IID - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Import Templates include: - '/ci/build.template.yml' - '/ci/push.template.yml' - '/ci/cleanup.template.yml' - '/ci/unit-test.template.yml' # Build Base Image build base image: stage: build-base script: - docker build -t ${REGISTRY_URL}/ops/base:${PROJECT_NAME} -f Dockerfile.base . - docker tag ${REGISTRY_URL}/ops/base:${PROJECT_NAME} ${REGISTRY_URL}/ops/base:${CI_COMMIT_SHORT_SHA} rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' changes: - Dockerfile.base - docker.properties # Push Base Image Stage push base image: stage: push-base script: - echo "$HARBOR_PASSWORD" | docker login -u "$HARBOR_USER" --password-stdin $REGISTRY_URL - docker push ${REGISTRY_URL}/ops/base:${PROJECT_NAME} rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' changes: - Dockerfile.base - docker.properties # Run Schema run schema: stage: run-schema image: ${REGISTRY_URL}/ops/base:${PROJECT_NAME} script: - export PGPASSWORD=$POSTGRES_PASSWORD - cd schema && ./bin/run-migrations.sh "docker" rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: always - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: always # Build Stages build workflow: stage: build variables: SERVICE_NAME: "workflow" PATH_PREFIX: "services/" USER: "vlapipe" extends: .build build capability: stage: build variables: SERVICE_NAME: "capability" PATH_PREFIX: "services/" extends: .build build notification: stage: build variables: SERVICE_NAME: "notification" PATH_PREFIX: "services/" extends: .build build web: stage: build variables: SERVICE_NAME: "web" PATH_PREFIX: "apps/" extends: .build ## Test Stages ## # Unit Tests unit test workflow: stage: unit-test variables: SERVICE_NAME: "workflow" extends: .unit-test needs: - build workflow unit test capability: stage: unit-test variables: SERVICE_NAME: "capability" extends: .unit-test needs: - build capability unit test notification: stage: unit-test variables: SERVICE_NAME: "notification" extends: .unit-test needs: - build notification # Generate Coverage reports unit test coverage: stage: test-coverage image: python:3.8-slim before_script: - pip install pytest pytest-cov script: - coverage combine --append - coverage report -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty - coverage xml -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty - coverage html -i --omit="**/test_*.py,**/_version.py,**/conftest.py,**/*interfaces.py" --skip-empty artifacts: reports: cobertura: coverage.xml paths: - coverage.xml - htmlcov/ dependencies: - unit test workflow - unit test capability - unit test notification rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' # E2E Tests e2e: stage: e2e-test image: docker/compose:1.27.4 variables: ENV_HOST: workflow before_script: - docker build -t base-build:ci --target base-build -f apps/web/Dockerfile.dev . - docker build -t e2e:${CI_COMMIT_SHORT_SHA} -f apps/web/Dockerfile.ci . script: # setting env variables with .env https://docs.docker.com/compose/environment-variables/ - echo "ENV=$DEPLOY_ENV" >> .env - echo "TAG=$CI_COMMIT_SHORT_SHA" >> .env - echo "DL_HOST=$DL_HOST" >> .env # set ENV_HOST to workflow because the workflow container isn't using host networking - echo "ENV_HOST=$ENV_HOST" >> .env - docker-compose -f docker-compose.ci.yml up -d capability workflow notification web - sleep 10 - docker-compose -f docker-compose.ci.yml run e2e after_script: # log the containers for visibility into why e2e tests failed - docker logs workspaces_capability_1 - docker logs workspaces_workflow_1 - docker logs workspaces_web_1 - docker-compose -f docker-compose.ci.yml down - docker image rm -f base-build:ci - docker image rm -f e2e:${CI_COMMIT_SHORT_SHA} dependencies: [] retry: 2 # Push Stages push workflow: stage: push variables: SERVICE_NAME: "workflow" extends: .push push capability: stage: push variables: SERVICE_NAME: "capability" extends: .push push notification: stage: push variables: SERVICE_NAME: "notification" extends: .push push web: stage: push variables: SERVICE_NAME: "web" extends: .push # UI tests coming soon! # needs: # - unit test dev ui # Cleanup clean build workflow: stage: .post variables: SERVICE_NAME: "workflow" extends: .cleanup allow_failure: true clean build capability: stage: .post variables: SERVICE_NAME: "capability" extends: .cleanup allow_failure: true clean build notification: stage: .post variables: SERVICE_NAME: "notification" extends: .cleanup allow_failure: true clean build web: stage: .post variables: SERVICE_NAME: "web" extends: .cleanup allow_failure: true # Deploy Stages pages: stage: deploy-coverage-page image: python:3.8-slim dependencies: - unit test coverage before_script: - pip install -r docs/requirements.txt - apt update - apt install make script: - mkdir public - mv htmlcov public/htmlcov - cd docs && make html && mv _build/html/* ../public/ artifacts: paths: - public expire_in: 2 weeks rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Development deploy: stage: deploy script: # Docker doesn't allow variable interpolation when declaring Docker Secret names # This sed command finds and replaces "dsoc_ENV_secrets:" with "dsoc_${DEPLOY_ENV}_secrets:" - sed -i "s/dsoc_ENV_secrets:/dsoc_${DEPLOY_ENV}_secrets:/g" docker-compose.dev.yml - ENV=$DEPLOY_ENV TAG=$IMAGE_TAG DL_HOST=$DL_HOST ENV_HOST=$ENV_HOST docker stack deploy --compose-file docker-compose.dev.yml workspaces-${DEPLOY_ENV} rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH variables: IMAGE_TAG: ${CI_DEFAULT_BRANCH} - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/' variables: IMAGE_TAG: $CI_COMMIT_TAG # override DEPLOY_ENV DEPLOY_ENV: "test" # override DL_HOST DL_HOST: https://dl-dsoc-test.nrao.edu # override ENV_HOST ENV_HOST: ws-test.nrao.edu # - if: '$CI_COMMIT_TAG =~ /^test_[0-9]+\.[0-9]+.[0-9]+$/' # variables: # IMAGE_TAG: $CI_COMMIT_TAG # # override DEPLOY_ENV # DEPLOY_ENV: "test" # e2e: # stage: e2e-test # image: trion/ng-cli-karma:11.2.7 # script: # - cd apps/web/ # - npm install # - ng e2e --configuration=${DEPLOY_ENV} # rules: # - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/' # variables: # # override DEPLOY_ENV # DEPLOY_ENV: "test" # when: manual # allow_failure: true