Added yaml and sh scripts to build go modules via CI

.gitlab-ci.yml

@@ -329,6 +329,50 @@ pex child pipeline:
           variables:
             DEPLOY_ENV: "prod"
 
+# Generate go builder yaml for child pipeline
+go generate yaml:
+    stage: generate-yaml
+    image: python:3.8-slim
+    before_script:
+        - apt update && apt install -y git
+        - pip install pyyaml
+    script:
+        - DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-go-yaml.py
+    rules:
+        - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+          changes:
+            - apps/cli/executables/pexable/**/*
+        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
+          variables:
+            # override DEPLOY_ENV
+            DEPLOY_ENV: "test"
+        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
+          variables:
+            DEPLOY_ENV: "prod"
+    artifacts:
+        paths:
+            - generated-pex-build-pipeline.yml
+
+# Trigger child pipeline based on generated go builder yaml
+go child pipeline:
+    stage: trigger
+    trigger:
+        include:
+            - artifact: generated-go-build-pipeline.yml
+              job: go generate yaml
+        strategy: depend
+    rules:
+        - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+          changes:
+            - apps/cli/executables/go/**/*
+        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
+          variables:
+            # override DEPLOY_ENV
+            DEPLOY_ENV: "test"
+        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
+          variables:
+            DEPLOY_ENV: "prod"
+
 # Development
 deploy:
     stage: deploy

apps/cli/executables/go/mod_analyst/go.mod

+module ssa/mod_analyst
+
+go 1.18
+
+require (
+	github.com/lib/pq v1.10.5 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+)

apps/cli/executables/go/mod_analyst/go.sum

+github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
+github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
+github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=

apps/cli/executables/go/mod_analyst/main.go

+/*
+ * Copyright (C) 2022 Associated Universities, Inc. Washington DC, USA.
+ *
+ * This file is part of NRAO Workspaces.
+ *
+ * Workspaces is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Workspaces is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package main
+
+import (
+	"flag"
+	"os"
+	"ssa/mod_analyst/pkg/db"
+)
+
+func main() {
+	// user args
+	var user_info db.UserInfo
+	var is_remove bool
+
+	// DB connection args
+	var db_info db.DbInfo
+
+	// Get user properties
+	flag.BoolVar(&user_info.Is_aod, "aod", false, "When adding a user, makes them part of the AOD group (by default they are added to the DA group)")
+	flag.StringVar(&user_info.Name, "name", "", "The full name of the DA/AOD to be added, use quotes for the first and last name (e.g. -name 'First Last')")
+	flag.StringVar(&user_info.Email, "email", "", "The email of the DA/AOD to be added")
+	flag.BoolVar(&is_remove, "rm", false, "Remove the user from the database")
+
+	// Get DB connection args
+	flag.BoolVar(&db_info.Is_ssl, "ssl", false, "Use ssl for the database connection")
+	flag.IntVar(&db_info.Port, "port", 5432, "The port for the database")
+	flag.StringVar(&db_info.Dbname, "dbname", "archive", "The name of the database to connect to")
+	flag.StringVar(&db_info.Host, "host", os.Getenv("HOSTNAME"), "The host where the database is located")
+	flag.StringVar(&db_info.Property_path, "prop", "/home/casa/capo/${CAPO_PROFILE}.properties", "Path to the properties file with the login details for the database")
+	flag.Parse()
+
+	// Make sure name is given
+	if len(user_info.Name) == 0 {
+		flag.Usage()
+		return
+	}
+
+	if is_remove {
+		db.RemoveUser(user_info, db_info)
+	} else {
+		db.AddUser(user_info, db_info)
+	}
+}

apps/cli/executables/go/mod_analyst/pkg/db/get_db.go

+/*
+ * Copyright (C) 2022 Associated Universities, Inc. Washington DC, USA.
+ *
+ * This file is part of NRAO Workspaces.
+ *
+ * Workspaces is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Workspaces is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package db
+
+import (
+	"github.com/magiconair/properties"
+)
+
+type DbInfo struct {
+	Host          string
+	Port          int
+	User          string
+	Password      string
+	Dbname        string
+	Property_path string
+	Is_ssl        bool
+}
+
+/**
+ * Pull the login information for the database from a properties file
+ *
+ * @param db_info a DbInfo type with information to connect to the database
+ * @return a DbInfo type holding db_info's data with the User and Password
+ * fields populated
+ **/
+func getDbLoginFromProperties(db_info DbInfo) DbInfo {
+	prop := properties.MustLoadFile(db_info.Property_path, properties.UTF8)
+
+	db_info.User = prop.GetString("metadataDatabase.jdbcUsername", "archive")
+	db_info.Password = prop.GetString("metadataDatabase.jdbcPassword", "docker")
+
+	return db_info
+}

apps/cli/executables/go/mod_analyst/pkg/db/mod_db.go

+/*
+ * Copyright (C) 2022 Associated Universities, Inc. Washington DC, USA.
+ *
+ * This file is part of NRAO Workspaces.
+ *
+ * Workspaces is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Workspaces is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package db
+
+import (
+	"database/sql"
+	"fmt"
+	_ "github.com/lib/pq"
+)
+
+type UserInfo struct {
+	Name   string
+	Email  string
+	Is_aod bool
+}
+
+/**
+ * Check if an error happened and panic on it if so
+ *
+ * @param err An error object to report
+ **/
+func checkError(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+/**
+ * Establish a connection to the database
+ *
+ * @param db_info A DbInfo type with information to connect to the database
+ * @return A DB object with an open connection to the database
+ **/
+func getConnection(db_info DbInfo) *sql.DB {
+	// Get db info and build string to get connection
+	db_info = getDbLoginFromProperties(db_info)
+
+	ssl_mode := "disable"
+	if db_info.Is_ssl {
+		ssl_mode = "require"
+	}
+	conn_info := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", db_info.Host, db_info.Port, db_info.User, db_info.Password, db_info.Dbname, ssl_mode)
+
+	db, err := sql.Open("postgres", conn_info)
+	checkError(err)
+
+	err = db.Ping()
+	checkError(err)
+
+	return db
+}
+
+/**
+ * Add a user to the qa_staff table in the database
+ *
+ * @param name a string with the name of the person to be added
+ * @param is_aod a bool to check if the user is an AOD or not
+ * @param db_info a DbInfo type with information to connect to the database
+ **/
+func AddUser(user_info UserInfo, db_info DbInfo) {
+	// Get a connection to the database
+	db := getConnection(db_info)
+
+	defer db.Close()
+
+	group := "DA"
+	if user_info.Is_aod {
+		group = "AOD"
+	}
+
+	// Add the user
+	insertStatement := `insert into "qa_staff"("user_name", "group", "available", "email") values($1, $2, $3, $4)`
+	_, err := db.Exec(insertStatement, user_info.Name, group, true, user_info.Email)
+	checkError(err)
+
+	fmt.Println("Added " + user_info.Name + " to qa_staff")
+}
+
+/**
+ * Remove a user from the qa_staff table in the database
+ *
+ * @param name a string with the name of the person to be removed
+ * @param db_info a DbInfo type with information to connect to the database
+ **/
+func RemoveUser(user_info UserInfo, db_info DbInfo) {
+	// Get a connection to the database
+	db := getConnection(db_info)
+
+	defer db.Close()
+
+	// Remove the user
+	deleteStatement := `delete from "qa_staff" where "user_name"=$1`
+	_, err := db.Exec(deleteStatement, user_info.Name)
+	checkError(err)
+
+	fmt.Println("Removed " + user_info.Name + " from qa_staff")
+}

ci/bin/build-go.sh

+#!/bin/bash
+# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Set failfast
+set -e
+set -o pipefail
+
+BUILD_DIR=/builds/ssa/workspaces/go
+[[ -d $BUILD_DIR ]] ||  mkdir -p $BUILD_DIR
+
+cd  apps/cli/executables/go/
+for module in "$@"
+do
+  if [ $module == "ingest" ]; then
+    echo "Skipping $module"
+    continue
+  fi
+
+  # if directory doesn't exist, skip
+  if [ ! -d "$module" ]; then
+    echo "Directory $module doesn't exist skipping"
+    continue
+  fi
+
+  cd  "$module"
+  # if [ -e setup.py ]; then
+  #   until python3 setup.py bdist_pex --bdist-all --bdist-dir="$BUILD_DIR" --pex-args="--python-shebang /home/ssa/bin/python3.8"; do
+  #     echo "PEX build failed. Retrying."; sleep 2;
+  #   done
+  # else
+  #   echo "PEX build impossible in $PWD because there is no setup.py file"
+  # fi
+  # cd ..
+  go build
+done

ci/bin/generate-go-yaml.py

+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import os
+import re
+import subprocess
+import sys
+from distutils.util import strtobool
+
+import yaml
+
+# Write gitlab-ci workflow rule for generated yaml
+def write_global_rule(**kwargs):
+    global_rule = """
+workflow:
+  rules:
+    - if: {rule}
+"""
+    with open("generated-go-build-pipeline.yml", "a") as yfile:
+        yfile.write(global_rule.format(**kwargs))
+
+
+# Write build job to generated yaml
+def write_build_config(**kwargs):
+    template = """
+go-{go_name}-build:
+    image: golang:1.18.0-alpine3.15
+    variables:
+        GIT_SUBMODULE_STRATEGY: recursive
+    script:
+        - echo "Building module - {go_name}"
+        - ./ci/bin/build-go.sh {go_name}
+        - |
+            curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file go/{go_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{go_name}/0.0.1/{go_name}"
+    rules:
+        - if: {build_rule}
+          {changes_rule}
+"""
+    with open("generated-go-build-pipeline.yml", "a") as yfile:
+        yfile.write(template.format(**kwargs))
+
+
+# Write release job to generated yaml
+def write_release_config(**kwargs):
+    template = """
+go-{go_name}-release:
+    image: golang:1.18.1-bullseye
+    needs: ["go-{go_name}-build"]
+    before_script:
+        - mkdir -p ~/.ssh
+        - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+        - eval $(ssh-agent -s)
+        - chmod 700 ~/.ssh
+        - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa
+        - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config'
+    script:
+        - echo "Releasing module to sbin area - {go_name}"
+        - |
+            RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\
+                curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{go_name}/0.0.1/{go_name}' --output {go_name} && \\
+                chmod 755 {go_name}"
+        - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g')
+        - ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash"
+    rules:
+        - if: {build_rule}
+          {changes_rule}
+    allow_failure: true
+"""
+    with open("generated-go-build-pipeline.yml", "a") as yfile:
+        yfile.write(template.format(**kwargs))
+
+
+# Get list of all pexables
+def get_list_of_go_modules():
+    go_changes = os.listdir("./apps/cli/executables/go")
+    go_changes.remove("ingest")
+    return go_changes
+
+
+def main(argv):
+    deploy_env = os.environ["DEPLOY_ENV"]
+
+    go_changes = []
+    rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
+    changes_rule = ""
+
+    if deploy_env == "dev":
+        commit_sha = os.environ["CI_COMMIT_SHA"]
+
+        # Get list of files that have changed from commit SHA
+        # git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA
+        sp = subprocess.run(
+            ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"],
+            stdout=subprocess.PIPE,
+            universal_newlines=True,
+        )
+
+        # Of those changes, make a list of changes to go modules
+        # and remove duplicates
+        go_changes = list(dict.fromkeys(re.findall("/go/(.*?)/", sp.stdout)))
+
+        # remove ingest
+        if "ingest" in go_changes:
+            go_changes.remove("ingest")
+
+        write_global_rule(rule=rule)
+        changes_rule = """changes:
+            - apps/cli/executables/go/**/*"""
+    else:
+        if deploy_env == "test":
+            rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/"
+        if deploy_env == "prod":
+            rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/"
+
+        write_global_rule(rule=rule)
+        go_changes = get_list_of_go_modules()
+
+    print(f"{go_changes}")
+
+    for module in go_changes:
+        write_build_config(go_name=module, build_rule=rule, changes_rule=changes_rule)
+        write_release_config(go_name=module, build_rule=rule, changes_rule=changes_rule, env=deploy_env)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])