#!/usr/bin/env python3 # # Copyright (C) 2022 Associated Universities, Inc. Washington DC, USA. # # This file is part of NRAO Workspaces. # # Workspaces is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Workspaces is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Workspaces. If not, see <https://www.gnu.org/licenses/>. import os import re import subprocess import sys # Write gitlab-ci workflow rule for generated yaml def write_global_rule(**kwargs): global_rule = """ workflow: rules: - if: {rule} """ with open("generated-go-build-pipeline.yml", "a") as yfile: yfile.write(global_rule.format(**kwargs)) # Write build job to generated yaml def write_build_config(**kwargs): template = """ go-{go_name}-build: image: golang:1.18.0-alpine3.15 variables: GIT_SUBMODULE_STRATEGY: recursive before_script: - apk add git curl script: - echo "Building module - {go_name}" - /builds/ssa/workspaces/ci/bin/build-go.sh {go_name} - pwd - ls go/ - ls - | curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file go/{go_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{go_name}/0.0.1/{go_name}" rules: - if: {build_rule} {changes_rule} """ with open("generated-go-build-pipeline.yml", "a") as yfile: yfile.write(template.format(**kwargs)) # Write release job to generated yaml def write_release_config(**kwargs): template = """ go-{go_name}-release: image: golang:1.18.1-bullseye needs: ["go-{go_name}-build"] before_script: - mkdir -p ~/.ssh - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval $(ssh-agent -s) - chmod 700 ~/.ssh - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config' script: - echo "Releasing module to sbin area - {go_name}" - | RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\ curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{go_name}/0.0.1/{go_name}' --output {go_name} && \\ chmod 755 {go_name}" - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g') - ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash" rules: - if: {build_rule} {changes_rule} allow_failure: true """ with open("generated-go-build-pipeline.yml", "a") as yfile: yfile.write(template.format(**kwargs)) # Get list of all pexables def get_list_of_go_modules(): go_changes = os.listdir("./apps/cli/executables/go") return go_changes def main(argv): deploy_env = os.environ["DEPLOY_ENV"] go_changes = [] rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/" changes_rule = "" if deploy_env == "dev": commit_sha = os.environ["CI_COMMIT_SHA"] # Get list of files that have changed from commit SHA # git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA sp = subprocess.run( ["git", "diff-tree", "-m", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"], stdout=subprocess.PIPE, universal_newlines=True, ) # Of those changes, make a list of changes to go modules # and remove duplicates go_changes = list(dict.fromkeys(re.findall("/go/(.*?)/", sp.stdout))) # remove ingest if "ingest" in go_changes: go_changes.remove("ingest") write_global_rule(rule=rule) changes_rule = """changes: - apps/cli/executables/go/**/*""" else: if deploy_env == "test": rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+\.[0.9]+-rc[0-9]+/" if deploy_env == "prod": rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/" write_global_rule(rule=rule) go_changes = get_list_of_go_modules() print(f"{go_changes}") for module in go_changes: write_build_config(go_name=module, build_rule=rule, changes_rule=changes_rule) write_release_config(go_name=module, build_rule=rule, changes_rule=changes_rule, env=deploy_env) if __name__ == "__main__": main(sys.argv[1:])