#!/usr/bin/env python3 # # Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA. # # This file is part of NRAO Workspaces. # # Workspaces is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Workspaces is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Workspaces. If not, see <https://www.gnu.org/licenses/>. import os import re import subprocess import sys # Write gitlab-ci workflow rule for generated yaml def write_global_rule(**kwargs): global_rule = """ workflow: rules: - if: {rule} """ with open("generated-pex-build-pipeline.yml", "a") as yfile: yfile.write(global_rule.format(**kwargs)) # Write build job to generated yaml def write_build_config(**kwargs): template = """ pex-{pex_name}-build: image: nrao_pex_base_3_10 variables: GIT_SUBMODULE_STRATEGY: recursive script: - echo "Building PEX - {pex_name}" - ./ci/bin/build-pexables.sh {pex_name} - | curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file pexes/{pex_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{pex_name}/0.0.1/{pex_name}" rules: - if: {build_rule} {changes_rule} """ with open("generated-pex-build-pipeline.yml", "a") as yfile: yfile.write(template.format(**kwargs)) # Write release job to generated yaml def write_release_config(**kwargs): template = """ pex-{pex_name}-release: image: python:3.10-slim needs: ["pex-{pex_name}-build"] before_script: - mkdir -p ~/.ssh - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval $(ssh-agent -s) - chmod 700 ~/.ssh - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config' script: - echo "Releasing PEX to sbin area - {pex_name}" - | RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\ curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{pex_name}/0.0.1/{pex_name}' --output {pex_name} && \\ chmod 755 {pex_name}" - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g') - ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash" rules: - if: {build_rule} {changes_rule} allow_failure: true """ with open("generated-pex-build-pipeline.yml", "a") as yfile: yfile.write(template.format(**kwargs)) # Get list of all pexables def get_list_of_pexables(): pex_changes = os.listdir("./apps/cli/executables/pexable") pex_changes.remove("ingest") return pex_changes def main(argv): deploy_env = os.environ["DEPLOY_ENV"] pex_changes = [] rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/" changes_rule = "" if deploy_env == "dev": commit_sha = os.environ["CI_COMMIT_SHA"] # Get list of files that have changed from commit SHA # git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA sp = subprocess.run( ["git", "diff-tree", "-m", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"], stdout=subprocess.PIPE, universal_newlines=True, ) # Of those changes, make a list of changes to pexables # and remove duplicates pex_changes = list(dict.fromkeys(re.findall("/pexable/(.*?)/", sp.stdout))) # remove ingest if "ingest" in pex_changes: pex_changes.remove("ingest") write_global_rule(rule=rule) changes_rule = """changes: - apps/cli/executables/pexable/**/*""" else: if deploy_env == "test": rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+\.[0.9]+-rc[0-9]+/" if deploy_env == "prod": rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/" write_global_rule(rule=rule) pex_changes = get_list_of_pexables() print(f"{pex_changes}") for pexable in pex_changes: write_build_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule) write_release_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule, env=deploy_env) if __name__ == "__main__": main(sys.argv[1:])