WS-758 simplify pex building ci

.gitignore

@@ -76,6 +76,7 @@ package-lock.json
 services/**/**/apps
 services/**/**/shared
 services/**/**/testing
+config/pex-watcher/apps/**/**
 tmp
 delivery_root
 lustre/

.gitlab-ci.yml

@@ -7,6 +7,8 @@ stages:
     - test-coverage
     - push
     - deploy-coverage-page
+    - generate-yaml
+    - trigger
     - deploy
     - e2e-test
     - .post
@@ -283,6 +285,50 @@ pages:
     rules:
       - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
+# Generate PEX builder yaml for child pipeline
+pex generate yaml:
+    stage: generate-yaml
+    image: python:3.8-slim
+    before_script:
+        - apt update && apt install -y git
+        - pip install pyyaml
+    script:
+        - DEPLOY_ENV=${DEPLOY_ENV} ./ci/bin/generate-yaml.py
+    rules:
+        - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+          changes:
+            - apps/cli/executables/pexable/**/*
+        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
+          variables:
+            # override DEPLOY_ENV
+            DEPLOY_ENV: "test"
+        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
+          variables:
+            DEPLOY_ENV: "prod"
+    artifacts:
+        paths:
+            - generated-pex-build-pipeline.yml
+
+# Trigger child pipeline based on generated PEX builder yaml
+pex child pipeline:
+    stage: trigger
+    trigger:
+        include:
+            - artifact: generated-pex-build-pipeline.yml
+              job: pex generate yaml
+        strategy: depend
+    rules:
+        - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+          changes:
+            - apps/cli/executables/pexable/**/*
+        - if: '$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/'
+          variables:
+            # override DEPLOY_ENV
+            DEPLOY_ENV: "test"
+        - if: '$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/'
+          variables:
+            DEPLOY_ENV: "prod"
+
 # Development
 deploy:
     stage: deploy

ci/bin/build-pexables.sh

+#!/bin/bash
+# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Set failfast
+set -e
+set -o pipefail
+
+BUILD_DIR=/builds/ssa/workspaces/pexes
+[[ -d $BUILD_DIR ]] ||  mkdir -p $BUILD_DIR
+
+cd  apps/cli/executables/pexable/
+for pexable in "$@"
+do
+  if [ $pexable == "ingest" ]; then
+    echo "Skipping $pexable"
+    continue
+  fi
+  cd  "$pexable"
+  if [ -e setup.py ]; then
+    until python3 setup.py bdist_pex --bdist-all --bdist-dir="$BUILD_DIR" --pex-args="--python-shebang /home/ssa/bin/python3.8"; do
+      echo "PEX build failed. Retrying."; sleep 2;
+    done
+  else
+    echo "PEX build impossible in $PWD because there is no setup.py file"
+  fi
+  cd ..
+done

ci/bin/generate-yaml.py

+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import os
+import re
+import subprocess
+import sys
+from distutils.util import strtobool
+
+import yaml
+
+# Write gitlab-ci workflow rule for generated yaml
+def write_global_rule(**kwargs):
+    global_rule = """
+workflow:
+  rules:
+    - if: {rule}
+"""
+    with open("generated-pex-build-pipeline.yml", "a") as yfile:
+        yfile.write(global_rule.format(**kwargs))
+
+
+# Write build job to generated yaml
+def write_build_config(**kwargs):
+    template = """
+pex-{pex_name}-build:
+    image: centos/python-38-centos7:20210726-fad62e9
+    variables:
+        GIT_SUBMODULE_STRATEGY: recursive
+    before_script:
+        - pip install pex
+    script:
+        - echo "Building PEX - {pex_name}"
+        - ./ci/bin/build-pexables.sh {pex_name}
+        - |
+            curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file pexes/{pex_name} "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/{pex_name}/0.0.1/{pex_name}"
+    rules:
+        - if: {build_rule}
+          {changes_rule}
+"""
+    with open("generated-pex-build-pipeline.yml", "a") as yfile:
+        yfile.write(template.format(**kwargs))
+
+
+# Write release job to generated yaml
+def write_release_config(**kwargs):
+    template = """
+pex-{pex_name}-release:
+    image: python:3.8-slim
+    needs: ["pex-{pex_name}-build"]
+    before_script:
+        - mkdir -p ~/.ssh
+        - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+        - eval $(ssh-agent -s)
+        - chmod 700 ~/.ssh
+        - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa
+        - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config'
+    script:
+        - echo "Releasing PEX to sbin area - {pex_name}"
+        - |
+            RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-{env}/workspaces/sbin/ && \\
+                curl --header 'JOB-TOKEN: ${{CI_JOB_TOKEN}}' '${{CI_API_V4_URL}}/projects/${{CI_PROJECT_ID}}/packages/generic/{pex_name}/0.0.1/{pex_name}' --output {pex_name} && \\
+                chmod 755 {pex_name}"
+        - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\\n//g')
+        - ssh -A shipman.aoc.nrao.edu "echo ${{B64CMD}} | base64 -d | bash"
+    rules:
+        - if: {build_rule}
+          {changes_rule}
+"""
+    with open("generated-pex-build-pipeline.yml", "a") as yfile:
+        yfile.write(template.format(**kwargs))
+
+
+# Get list of all pexables
+def get_list_of_pexables():
+    pex_changes = os.listdir("./apps/cli/executables/pexable")
+    pex_changes.remove("ingest")
+    return pex_changes
+
+
+def main(argv):
+    deploy_env = os.environ["DEPLOY_ENV"]
+
+    pex_changes = []
+    rule = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
+    changes_rule = ""
+
+    if deploy_env == "dev":
+        commit_sha = os.environ["CI_COMMIT_SHA"]
+
+        # Get list of files that have changed from commit SHA
+        # git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA
+        sp = subprocess.run(
+            ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", f"{commit_sha}"],
+            stdout=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        # Of those changes, make a list of changes to pexables
+        # and remove duplicates
+        pex_changes = list(dict.fromkeys(re.findall("/pexable/(.*?)/", sp.stdout)))
+
+        write_global_rule(rule=rule)
+        changes_rule = """changes:
+            - apps/cli/executables/pexable/**/*"""
+    else:
+        if deploy_env == "test":
+            rule = "$CI_COMMIT_TAG =~ /^end-of-sprint-[0-9]+/"
+        if deploy_env == "prod":
+            rule = "$CI_COMMIT_TAG =~ /[0-9]+\.[0-9]+\.[0-9]+$/"
+
+        write_global_rule(rule=rule)
+        pex_changes = get_list_of_pexables()
+
+    print(f"{pex_changes}")
+
+    for pexable in pex_changes:
+        write_build_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule)
+        write_release_config(pex_name=pexable, build_rule=rule, changes_rule=changes_rule, env=deploy_env)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

config/pex-watcher/Dockerfile.pex

+FROM centos/python-38-centos7:20210726-fad62e9
+
+USER root
+
+RUN pip install pex watchdog
+
+WORKDIR /config
+
+COPY ./config/pex-watcher /config
+
+CMD [ "/config/watcher.py" ]

config/pex-watcher/bin/local-build-pexables.sh

+#!/bin/bash
+#
+# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Set failfast
+set -e
+set -o pipefail
+
+BUILD_DIR=/lustre/aoc/cluster/pipeline/docker/workspaces/sbin
+[[ -d $BUILD_DIR ]] ||  mkdir -p $BUILD_DIR
+
+cd  apps/cli/executables/pexable/
+for pexable in "$@"
+do
+  if [ $pexable == "ingest" ]; then
+    echo "Skipping $pexable"
+    continue
+  fi
+  cd  "$pexable"
+  if [ -e setup.py ]; then
+    pex_build_cmd="python3 setup.py bdist_pex --bdist-all --bdist-dir=\"$BUILD_DIR\" --pex-args=\"--python-shebang /home/ssa/bin/python3.8\""
+    build_attempts=0
+    until eval "$pex_build_cmd" || [ $build_attempts -gt 3 ] ; do
+      ((build_attempts++))
+      echo "PEX build failed. Attempt: $build_attempts. Retrying."; sleep 2;
+    done
+  else
+    echo "PEX build impossible in $PWD because there is no setup.py file"
+  fi
+  cd ..
+done

config/pex-watcher/watcher.py

+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+import subprocess
+import sys
+import time
+
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
+
+
+class PexHandler(FileSystemEventHandler):
+    event_cache = {}
+
+    def on_modified(self, event):
+        seconds = int(time.time())
+        key = (seconds, event.src_path)
+
+        # check for identical events
+        if key in self.event_cache:
+            return
+
+        # don't handle events if they happen quickly after another event
+        if self.event_cache.__len__() != 0:
+            previous_entry = list(self.event_cache)[-1]
+            elapsed_time = seconds - previous_entry[0]
+            print(f"elapsed_time between events: {elapsed_time}")
+            if elapsed_time < 10:
+                print(f"Too little time between events, Skipping...")
+                return
+
+        self.event_cache[key] = True
+
+        # Only build pex if changes are made to files that end in ".py"
+        if event.src_path.endswith(".py"):
+            pexable = re.search("/pexable/(.*?)/", event.src_path).group(1)
+            print(f"******************** Building pex - {pexable} ********************")
+            # run local-build-pexables.sh
+            sp = subprocess.run(
+                ["./bin/local-build-pexables.sh", f"{pexable}"],
+                stdout=subprocess.PIPE,
+                universal_newlines=True,
+            )
+            print(f"{sp.stdout}")
+            print(f"*********************** {pexable} finished ***********************")
+        print("Modification event handled.")
+
+
+if __name__ == "__main__":
+    path = sys.argv[1] if len(sys.argv) > 1 else "./apps/cli/executables/pexable/"
+    event_handler = PexHandler()
+    observer = Observer()
+    observer.schedule(event_handler, path, recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    finally:
+        observer.stop()
+        observer.join()

docker-compose.local.yml

@@ -46,7 +46,15 @@ services:
       - ./lustre/aoc/cluster/pipeline/docker/workspaces:/lustre/aoc/cluster/pipeline/docker/workspaces
       - ./lustre/aoc/cluster/pipeline/vlass_docker:/lustre/aoc/cluster/pipeline/vlass_docker
 
-  # To enable NGAS container, run: `docker compose -f docker-compose.local.yml --profile ngas up`
+  pex-watcher:
+    build:
+      context: .
+      dockerfile: ./config/pex-watcher/Dockerfile.pex
+    volumes:
+      - ./config/pex-watcher:/config
+      - ./apps/cli/executables/pexable/:/config/apps/cli/executables/pexable/
+      - ./lustre/aoc/cluster/pipeline/docker/workspaces/sbin:/lustre/aoc/cluster/pipeline/docker/workspaces/sbin
+
   ngas:
     build:
       context: .

services/workflow/bin/boot-workflow.sh

@@ -21,25 +21,6 @@
 set -e
 set -o pipefail
 
-WORKFLOW_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/sbin
-
-# Build PEX files
-cd /packages/apps/cli/executables/pexable
-for d in * ; do
-  cd "${d}"
-
-  # Try building PEX file until it succeeds
-  if [ -e setup.py ]; then
-    until python3 setup.py bdist_pex --bdist-all --bdist-dir="$WORKFLOW_DIR" --pex-args="--python-shebang /home/ssa/bin/python3.8"; do
-      echo "PEX build failed. Retrying."; sleep 2;
-    done
-  else
-    echo "PEX build impossible in $PWD because there is no setup.py file"
-  fi
-
-  cd ..
-done
-
 # Return to base /code directory
 cd /code