generate-go-yaml.py

#!/usr/bin/env python3
#
# Copyright (C) 2022 Associated Universities, Inc. Washington DC, USA.
#
# This file is part of NRAO Workspaces.
#
# Workspaces is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Workspaces is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.


import os
import re
import subprocess
import sys


# Write gitlab-ci workflow rule for generated yaml
def write_global_rule(**kwargs):
    global_rule = """
workflow:
  rules:
    - if: {rule}
"""
    with open("generated-go-build-pipeline.yml", "a") as yfile:
        yfile.write(global_rule.format(**kwargs))


# Write build job to generated yaml
def write_build_config(**kwargs):
    template = """
go-{go_name}-build:
    image: golang:1.18.0-alpine3.15
    variables:
        GIT_SUBMODULE_STRATEGY: recursive
    before_script:
        - apk add git curl
    script:
        - echo "Building module - {go_name}"
        - /builds/ssa/workspaces/ci/bin/build-go.sh {go_name}
        - pwd
        - ls go/
        - ls
        - |
            curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file go/{go_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{go_name}/0.0.1/{go_name}"
    rules:
        - if: {build_rule}
          {changes_rule}
"""
    with open("generated-go-build-pipeline.yml", "a") as yfile:
        yfile.write(template.format(**kwargs))


# Write release job to generated yaml
def write_release_config(**kwargs):
    template = """
go-{go_name}-release:
    image: golang:1.18.1-bullseye
    needs: ["go-{go_name}-build"]
    before_script:
        - mkdir -p ~/.ssh
        - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
        - eval $(ssh-agent -s)
        - chmod 700 ~/.ssh
        - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa
        - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config'
    script:
        - echo "Releasing module to sbin area - {go_name}"
        - |
            RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\
                curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{go_name}/0.0.1/{go_name}' --output {go_name} && \\
                chmod 755 {go_name}"
        - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g')
        - ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash"
    rules:
        - if: {build_rule}
          {changes_rule}
    allow_failure: true
"""
    with open("generated-go-build-pipeline.yml", "a") as yfile:
        yfile.write(template.format(**kwargs))


# Get list of all pexables
def get_list_of_go_modules():
    go_changes = os.listdir("./apps/cli/executables/go")
    return go_changes


def main(argv):
    deploy_env = os.environ["DEPLOY_ENV"]

    go_changes = []
    rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/"
    changes_rule = ""

    if deploy_env == "dev":
        commit_sha = os.environ["CI_COMMIT_SHA"]

        # Get list of files that have changed from commit SHA
        # git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA
        sp = subprocess.run(
            ["git", "diff-tree", "-m", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"],
            stdout=subprocess.PIPE,
            universal_newlines=True,
        )

        # Of those changes, make a list of changes to go modules
        # and remove duplicates
        go_changes = list(dict.fromkeys(re.findall("/go/(.*?)/", sp.stdout)))

        # remove ingest
        if "ingest" in go_changes:
            go_changes.remove("ingest")

        write_global_rule(rule=rule)
        changes_rule = """changes:
            - apps/cli/executables/go/**/*"""
    else:
        if deploy_env == "test":
            rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+\.[0.9]+-rc[0-9]+/"
        if deploy_env == "prod":
            rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/"

        write_global_rule(rule=rule)
        go_changes = get_list_of_go_modules()

    print(f"{go_changes}")

    for module in go_changes:
        write_build_config(go_name=module, build_rule=rule, changes_rule=changes_rule)
        write_release_config(go_name=module, build_rule=rule, changes_rule=changes_rule, env=deploy_env)


if __name__ == "__main__":
    main(sys.argv[1:])