generate-yaml.py

#!/usr/bin/env python3
#
# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
#
# This file is part of NRAO Workspaces.
#
# Workspaces is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Workspaces is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.


import os
import re
import subprocess
import sys


# Write gitlab-ci workflow rule for generated yaml
def write_global_rule(**kwargs):
    global_rule = """
workflow:
  rules:
    - if: {rule}
"""
    with open("generated-pex-build-pipeline.yml", "a") as yfile:
        yfile.write(global_rule.format(**kwargs))


# Write build job to generated yaml
def write_build_config(**kwargs):
    template = """
pex-{pex_name}-build:
    image: nrao_pex_base_3_10
    variables:
        GIT_SUBMODULE_STRATEGY: recursive
    script:
        - echo "Building PEX - {pex_name}"
        - ./ci/bin/build-pexables.sh {pex_name}
        - |
            curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file pexes/{pex_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{pex_name}/0.0.1/{pex_name}"
    rules:
        - if: {build_rule}
          {changes_rule}
"""
    with open("generated-pex-build-pipeline.yml", "a") as yfile:
        yfile.write(template.format(**kwargs))


# Write release job to generated yaml
def write_release_config(**kwargs):
    template = """
pex-{pex_name}-release:
    image: python:3.10-slim
    needs: ["pex-{pex_name}-build"]
    before_script:
        - mkdir -p ~/.ssh
        - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
        - eval $(ssh-agent -s)
        - chmod 700 ~/.ssh
        - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa
        - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config'
    script:
        - echo "Releasing PEX to sbin area - {pex_name}"
        - |
            RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\
                curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{pex_name}/0.0.1/{pex_name}' --output {pex_name} && \\
                chmod 755 {pex_name}"
        - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g')
        - ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash"
    rules:
        - if: {build_rule}
          {changes_rule}
    allow_failure: true
"""
    with open("generated-pex-build-pipeline.yml", "a") as yfile:
        yfile.write(template.format(**kwargs))


# Get list of all pexables
def get_list_of_pexables():
    pex_changes = os.listdir("./apps/cli/executables/pexable")
    pex_changes.remove("ingest")
    return pex_changes


def main(argv):
    deploy_env = os.environ["DEPLOY_ENV"]

    pex_changes = []
    rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/"
    changes_rule = ""

    if deploy_env == "dev":
        commit_sha = os.environ["CI_COMMIT_SHA"]

        # Get list of files that have changed from commit SHA
        # git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA
        sp = subprocess.run(
            ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"],
            stdout=subprocess.PIPE,
            universal_newlines=True,
        )
        # Of those changes, make a list of changes to pexables
        # and remove duplicates
        pex_changes = list(dict.fromkeys(re.findall("/pexable/(.*?)/", sp.stdout)))

        # remove ingest
        if "ingest" in pex_changes:
            pex_changes.remove("ingest")

        write_global_rule(rule=rule)
        changes_rule = """changes:
            - apps/cli/executables/pexable/**/*"""
    else:
        if deploy_env == "test":
            rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/"
        if deploy_env == "prod":
            rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/"

        write_global_rule(rule=rule)
        pex_changes = get_list_of_pexables()

    print(f"{pex_changes}")

    for pexable in pex_changes:
        write_build_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule)
        write_release_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule, env=deploy_env)


if __name__ == "__main__":
    main(sys.argv[1:])