WS-1657: Add rest of pexes to pipeline and remove old pex building steps.

.gitlab-ci.yml

@@ -2,13 +2,10 @@ stages:
     - pull-db
     - build-pexes
     - build
-    - build-pex-base
     - unit-test
     - test-coverage
     - push
     - deploy-coverage-page
-    - generate-pex-yaml
-    - pex-trigger
     - generate-go-yaml
     - go-trigger
     - deploy
@@ -55,27 +52,39 @@ pull db image:
         - echo "$REGISTRY_URL_PASS" | docker login --username "$REGISTRY_URL_USER" --password-stdin $BASE_REGISTRY_URL
         - docker pull ${BASE_REGISTRY_URL}/db:workspaces
 
-build-pex-base image:
-  interruptible: true
-  stage: build-pex-base
-  script:
-    - docker build --no-cache -t nrao_pex_base_3_10 -f ./ci/bin/Dockerfile.pex .
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELPMENT/'
-      changes:
-        - apps/cli/executables/pexable/**/*
-    - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
-      variables:
-        # override DEPLOY_ENV
-        DEPLOY_ENV: "test"
-    - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
-      variables:
-        DEPLOY_ENV: "prod"
-
-
 ###############################################
 # Build Pexes
 ###############################################
+build pex carta envoy:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/carta_envoy"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/carta_envoy/**/*"
+
+build pex casa envoy:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/casa_envoy"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/casa_envoy/**/*"
+
+build pex conveyor:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/conveyor"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/conveyor/**/*"
+
 build pex deliver:
   interruptible: true
   stage: build-pexes
@@ -86,6 +95,107 @@ build pex deliver:
     changes:
       - "apps/cli/executables/pexable/deliver/**/*"
 
+build pex ingest:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/ingest"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/ingest/**/*"
+
+build pex ingest envoy:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/ingest_envoy"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/ingest_envoy/**/*"
+
+build pex mediator:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/mediator"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/mediator/**/*"
+
+build pex null:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/null"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/null/**/*"
+
+build pex productfetcher:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/productfetcher"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/productfetcher/**/*"
+
+build pex update stage:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/update_stage"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/update_stage/**/*"
+
+build pex vela:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/vela"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/vela/**/*"
+
+build pex wf inspector:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/wf_inspector"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/wf_inspector/**/*"
+
+build pex ws annihilator:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/ws_annihilator"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/ws_annihilator/**/*"
+
+build pex ws metrics:
+  interruptible: true
+  stage: build-pexes
+  variables:
+    PEX_PATH: "apps/cli/executables/pexable/ws_metrics"
+  extends: .build-pexes
+  only:
+    changes:
+      - "apps/cli/executables/pexable/ws_metrics/**/*"
+
+
 
 ###############################################
 # Build Service and Web Images
@@ -276,50 +386,6 @@ clean build web:
     rules:
       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
 
-# Generate PEX builder yaml for child pipeline
-pex generate yaml:
-   stage: generate-pex-yaml
-   image: python:3.10-slim
-   before_script:
-       - apt update && apt install -y git
-       - pip install pyyaml
-   script:
-       - DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-yaml.py
-   rules:
-       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
-         changes:
-           - apps/cli/executables/pexable/**/*
-       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
-         variables:
-           # override DEPLOY_ENV
-           DEPLOY_ENV: "test"
-       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
-         variables:
-           DEPLOY_ENV: "prod"
-   artifacts:
-       paths:
-           - generated-pex-build-pipeline.yml
-
- # Trigger child pipeline based on generated PEX builder yaml
-pex child pipeline:
-   stage: pex-trigger
-   trigger:
-       include:
-           - artifact: generated-pex-build-pipeline.yml
-             job: pex generate yaml
-       strategy: depend
-   rules:
-       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/'
-         changes:
-           - apps/cli/executables/pexable/**/*
-       - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/'
-         variables:
-           # override DEPLOY_ENV
-           DEPLOY_ENV: "test"
-       - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
-         variables:
-           DEPLOY_ENV: "prod"
-
  # Generate go builder yaml for child pipeline
 go generate yaml:
    stage: generate-go-yaml

ci/bin/Dockerfile.pex

-FROM centos/python-38-centos7:20210726-fad62e9
-
-USER root
-
-Run mkdir /ci
-
-WORKDIR /ci
-
-COPY ./ci/bin /ci
-
-RUN /ci/install-python310.sh
-RUN python3.10 -m pip install pex
-
-# Add Python install location to the path (can't set in boot script)
-ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/opt/Python-3.10.9
-ENV PATH $PATH:/home/ssa/bin
-

ci/bin/build-pexables.sh

-#!/bin/bash
-# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
-#
-# This file is part of NRAO Workspaces.
-#
-# Workspaces is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Workspaces is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
-#
-
-# Set failfast
-set -e
-set -o pipefail
-
-BUILD_DIR=/builds/ssa/workspaces/pexes
-[[ -d $BUILD_DIR ]] ||  mkdir -p $BUILD_DIR
-
-cd  apps/cli/executables/pexable/
-for pexable in "$@"
-do
-  if [ $pexable == "ingest" ]; then
-    echo "Skipping $pexable"
-    continue
-  fi
-
-  # if directory doesn't exist, skip
-  if [ ! -d "$pexable" ]; then
-    echo "Directory $pexable doesn't exist skipping"
-    continue
-  fi
-
-  cd  "$pexable"
-  if [ -e setup.py ]; then
-    until python3.10 setup.py bdist_pex --bdist-all --bdist-dir="$BUILD_DIR" --pex-args="--python-shebang /home/ssa/bin/python3.10"; do
-      echo "PEX build failed. Retrying."; sleep 2;
-    done
-  else
-    echo "PEX build impossible in $PWD because there is no setup.py file"
-  fi
-  cd ..
-done

ci/bin/generate-yaml.py

-#!/usr/bin/env python3
-#
-# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
-#
-# This file is part of NRAO Workspaces.
-#
-# Workspaces is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Workspaces is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
-
-
-import os
-import re
-import subprocess
-import sys
-
-
-# Write gitlab-ci workflow rule for generated yaml
-def write_global_rule(**kwargs):
-    global_rule = """
-workflow:
-  rules:
-    - if: {rule}
-"""
-    with open("generated-pex-build-pipeline.yml", "a") as yfile:
-        yfile.write(global_rule.format(**kwargs))
-
-
-# Write build job to generated yaml
-def write_build_config(**kwargs):
-    template = """
-pex-{pex_name}-build:
-    image: nrao_pex_base_3_10
-    variables:
-        GIT_SUBMODULE_STRATEGY: recursive
-    script:
-        - echo "Building PEX - {pex_name}"
-        - ./ci/bin/build-pexables.sh {pex_name}
-        - |
-            curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file pexes/{pex_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{pex_name}/0.0.1/{pex_name}"
-    rules:
-        - if: {build_rule}
-          {changes_rule}
-"""
-    with open("generated-pex-build-pipeline.yml", "a") as yfile:
-        yfile.write(template.format(**kwargs))
-
-
-# Write release job to generated yaml
-def write_release_config(**kwargs):
-    template = """
-pex-{pex_name}-release:
-    image: python:3.10-slim
-    needs: ["pex-{pex_name}-build"]
-    before_script:
-        - mkdir -p ~/.ssh
-        - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
-        - eval $(ssh-agent -s)
-        - chmod 700 ~/.ssh
-        - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa
-        - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config'
-    script:
-        - echo "Releasing PEX to sbin area - {pex_name}"
-        - |
-            RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\
-                curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{pex_name}/0.0.1/{pex_name}' --output {pex_name} && \\
-                chmod 755 {pex_name}"
-        - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g')
-        - ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash"
-    rules:
-        - if: {build_rule}
-          {changes_rule}
-    allow_failure: true
-"""
-    with open("generated-pex-build-pipeline.yml", "a") as yfile:
-        yfile.write(template.format(**kwargs))
-
-
-# Get list of all pexables
-def get_list_of_pexables():
-    pex_changes = os.listdir("./apps/cli/executables/pexable")
-    pex_changes.remove("ingest")
-    return pex_changes
-
-
-def main(argv):
-    deploy_env = os.environ["DEPLOY_ENV"]
-
-    pex_changes = []
-    rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/"
-    changes_rule = ""
-
-    if deploy_env == "dev":
-        commit_sha = os.environ["CI_COMMIT_SHA"]
-
-        # Get list of files that have changed from commit SHA
-        # git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA
-        sp = subprocess.run(
-            ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"],
-            stdout=subprocess.PIPE,
-            universal_newlines=True,
-        )
-        # Of those changes, make a list of changes to pexables
-        # and remove duplicates
-        pex_changes = list(dict.fromkeys(re.findall("/pexable/(.*?)/", sp.stdout)))
-
-        # remove ingest
-        if "ingest" in pex_changes:
-            pex_changes.remove("ingest")
-
-        write_global_rule(rule=rule)
-        changes_rule = """changes:
-            - apps/cli/executables/pexable/**/*"""
-    else:
-        if deploy_env == "test":
-            rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/ || $CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+/"
-        if deploy_env == "prod":
-            rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/"
-
-        write_global_rule(rule=rule)
-        pex_changes = get_list_of_pexables()
-
-    print(f"{pex_changes}")
-
-    for pexable in pex_changes:
-        write_build_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule)
-        write_release_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule, env=deploy_env)
-
-
-if __name__ == "__main__":
-    main(sys.argv[1:])