Skip to content
Snippets Groups Projects
Commit 4835d913 authored by Brittany Faciane's avatar Brittany Faciane
Browse files

WS-1657: Add rest of pexes to pipeline and remove old pex building steps.

parent 5efb134b
No related branches found
Tags 2.5.0-rc2
2 merge requests!1452Merge 2.8.2 to main,!1305WS-1657: Add rest of pexes to pipeline and remove old pex building steps.
Pipeline #9564 passed
Stage: pull-db
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 131
− 65
View file @ 4835d913
......@@ -2,13 +2,10 @@ stages:
- pull-db
- build-pexes
- build
- build-pex-base
- unit-test
- test-coverage
- push
- deploy-coverage-page
- generate-pex-yaml
- pex-trigger
- generate-go-yaml
- go-trigger
- deploy
......@@ -55,27 +52,39 @@ pull db image:
- echo "$REGISTRY_URL_PASS" | docker login --username "$REGISTRY_URL_USER" --password-stdin $BASE_REGISTRY_URL
- docker pull ${BASE_REGISTRY_URL}/db:workspaces
build-pex-base image:
interruptible: true
stage: build-pex-base
script:
- docker build --no-cache -t nrao_pex_base_3_10 -f ./ci/bin/Dockerfile.pex .
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELPMENT/'
changes:
- apps/cli/executables/pexable/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
###############################################
# Build Pexes
###############################################
build pex carta envoy:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/carta_envoy"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/carta_envoy/**/*"
build pex casa envoy:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/casa_envoy"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/casa_envoy/**/*"
build pex conveyor:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/conveyor"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/conveyor/**/*"
build pex deliver:
interruptible: true
stage: build-pexes
......@@ -86,6 +95,107 @@ build pex deliver:
changes:
- "apps/cli/executables/pexable/deliver/**/*"
build pex ingest:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/ingest"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/ingest/**/*"
build pex ingest envoy:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/ingest_envoy"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/ingest_envoy/**/*"
build pex mediator:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/mediator"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/mediator/**/*"
build pex null:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/null"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/null/**/*"
build pex productfetcher:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/productfetcher"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/productfetcher/**/*"
build pex update stage:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/update_stage"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/update_stage/**/*"
build pex vela:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/vela"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/vela/**/*"
build pex wf inspector:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/wf_inspector"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/wf_inspector/**/*"
build pex ws annihilator:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/ws_annihilator"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/ws_annihilator/**/*"
build pex ws metrics:
interruptible: true
stage: build-pexes
variables:
PEX_PATH: "apps/cli/executables/pexable/ws_metrics"
extends: .build-pexes
only:
changes:
- "apps/cli/executables/pexable/ws_metrics/**/*"
###############################################
# Build Service and Web Images
......@@ -276,50 +386,6 @@ clean build web:
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
# Generate PEX builder yaml for child pipeline
pex generate yaml:
stage: generate-pex-yaml
image: python:3.10-slim
before_script:
- apt update && apt install -y git
- pip install pyyaml
script:
- DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-yaml.py
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/pexable/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
artifacts:
paths:
- generated-pex-build-pipeline.yml
# Trigger child pipeline based on generated PEX builder yaml
pex child pipeline:
stage: pex-trigger
trigger:
include:
- artifact: generated-pex-build-pipeline.yml
job: pex generate yaml
strategy: depend
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
changes:
- apps/cli/executables/pexable/**/*
- if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
variables:
# override DEPLOY_ENV
DEPLOY_ENV: "test"
- if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
variables:
DEPLOY_ENV: "prod"
# Generate go builder yaml for child pipeline
go generate yaml:
stage: generate-go-yaml
......
ci/bin/Dockerfile.pex deleted 100644 → 0
+ 0
− 17
View file @ 5efb134b
FROM centos/python-38-centos7:20210726-fad62e9
USER root
Run mkdir /ci
WORKDIR /ci
COPY ./ci/bin /ci
RUN /ci/install-python310.sh
RUN python3.10 -m pip install pex
# Add Python install location to the path (can't set in boot script)
ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/opt/Python-3.10.9
ENV PATH $PATH:/home/ssa/bin
ci/bin/build-pexables.sh deleted 100755 → 0
+ 0
− 50
View file @ 5efb134b
#!/bin/bash
# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
#
# This file is part of NRAO Workspaces.
#
# Workspaces is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Workspaces is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
#
# Set failfast
set -e
set -o pipefail
BUILD_DIR=/builds/ssa/workspaces/pexes
[[ -d $BUILD_DIR ]] || mkdir -p $BUILD_DIR
cd apps/cli/executables/pexable/
for pexable in "$@"
do
if [ $pexable == "ingest" ]; then
echo "Skipping $pexable"
continue
fi
# if directory doesn't exist, skip
if [ ! -d "$pexable" ]; then
echo "Directory $pexable doesn't exist skipping"
continue
fi
cd "$pexable"
if [ -e setup.py ]; then
until python3.10 setup.py bdist_pex --bdist-all --bdist-dir="$BUILD_DIR" --pex-args="--python-shebang /home/ssa/bin/python3.10"; do
echo "PEX build failed. Retrying."; sleep 2;
done
else
echo "PEX build impossible in $PWD because there is no setup.py file"
fi
cd ..
done
ci/bin/generate-yaml.py deleted 100755 → 0
+ 0
− 140
View file @ 5efb134b
#!/usr/bin/env python3
#
# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
#
# This file is part of NRAO Workspaces.
#
# Workspaces is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Workspaces is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
import os
import re
import subprocess
import sys
# Write gitlab-ci workflow rule for generated yaml
def write_global_rule(**kwargs):
global_rule = """
workflow:
rules:
- if: {rule}
"""
with open("generated-pex-build-pipeline.yml", "a") as yfile:
yfile.write(global_rule.format(**kwargs))
# Write build job to generated yaml
def write_build_config(**kwargs):
template = """
pex-{pex_name}-build:
image: nrao_pex_base_3_10
variables:
GIT_SUBMODULE_STRATEGY: recursive
script:
- echo "Building PEX - {pex_name}"
- ./ci/bin/build-pexables.sh {pex_name}
- |
curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file pexes/{pex_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{pex_name}/0.0.1/{pex_name}"
rules:
- if: {build_rule}
{changes_rule}
"""
with open("generated-pex-build-pipeline.yml", "a") as yfile:
yfile.write(template.format(**kwargs))
# Write release job to generated yaml
def write_release_config(**kwargs):
template = """
pex-{pex_name}-release:
image: python:3.10-slim
needs: ["pex-{pex_name}-build"]
before_script:
- mkdir -p ~/.ssh
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval $(ssh-agent -s)
- chmod 700 ~/.ssh
- echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa
- '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config'
script:
- echo "Releasing PEX to sbin area - {pex_name}"
- |
RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\
curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{pex_name}/0.0.1/{pex_name}' --output {pex_name} && \\
chmod 755 {pex_name}"
- B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g')
- ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash"
rules:
- if: {build_rule}
{changes_rule}
allow_failure: true
"""
with open("generated-pex-build-pipeline.yml", "a") as yfile:
yfile.write(template.format(**kwargs))
# Get list of all pexables
def get_list_of_pexables():
pex_changes = os.listdir("./apps/cli/executables/pexable")
pex_changes.remove("ingest")
return pex_changes
def main(argv):
deploy_env = os.environ["DEPLOY_ENV"]
pex_changes = []
rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/"
changes_rule = ""
if deploy_env == "dev":
commit_sha = os.environ["CI_COMMIT_SHA"]
# Get list of files that have changed from commit SHA
# git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA
sp = subprocess.run(
["git", "diff-tree", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"],
stdout=subprocess.PIPE,
universal_newlines=True,
)
# Of those changes, make a list of changes to pexables
# and remove duplicates
pex_changes = list(dict.fromkeys(re.findall("/pexable/(.*?)/", sp.stdout)))
# remove ingest
if "ingest" in pex_changes:
pex_changes.remove("ingest")
write_global_rule(rule=rule)
changes_rule = """changes:
- apps/cli/executables/pexable/**/*"""
else:
if deploy_env == "test":
rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/"
if deploy_env == "prod":
rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/"
write_global_rule(rule=rule)
pex_changes = get_list_of_pexables()
print(f"{pex_changes}")
for pexable in pex_changes:
write_build_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule)
write_release_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule, env=deploy_env)
if __name__ == "__main__":
main(sys.argv[1:])
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment