Pipeline Testing: Fixing pex installation.

.env

@@ -10,6 +10,7 @@ DL_HOST="https://dl-dsoc.nrao.edu"
 NG_APP_WS_VERSION=${ENV}
 WS_VERSION=unknown-version
 LOCAL_OR_SERVER_PEX=local-pex  # determines if pexes are built from scratch or pulled from the registry (one of: local-pex, server-pex)
+API_TOKEN=glpat-vPamXXk4PZPe8GQAzmY2
 
 # CAPO Environment Properties
 CAPO_PATH=/home/casa/capo

.gitlab-ci.yml

@@ -5,6 +5,7 @@ stages:
     - unit-test
     - test-coverage
     - push
+    - push-packages
     - deploy-coverage-page
     - generate-go-yaml
     - go-trigger
@@ -41,6 +42,7 @@ include:
     - '/ci/cleanup.template.yml'
     - '/ci/unit-test.template.yml'
     - '/ci/package-build.template.yml'
+    - '/ci/push-package.template.yml'
 
 
 # Unit testing steps require a specific database image to be available; this step downloads it
@@ -324,6 +326,100 @@ push web:
     # needs:
     #     - unit test dev ui
 
+###############################################
+# Push Packages
+###############################################
+push carta-envoy:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "carta_envoy"
+    PIP_NAME: "carta-envoy"
+  extends: .push-packages
+
+push casa-envoy:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "casa_envoy"
+    PIP_NAME: "casa-envoy"
+  extends: .push-packages
+
+push conveyor:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "conveyor"
+    PIP_NAME: "conveyor"
+  extends: .push-packages
+
+push deliver:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "deliver"
+    PIP_NAME: "deliver"
+  extends: .push-packages
+
+push ingest-envoy:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "ingest_envoy"
+    PIP_NAME: "ingest-envoy"
+  extends: .push-packages
+
+push mediator:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "mediator"
+    PIP_NAME: "mediator"
+  extends: .push-packages
+
+push null:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "null"
+    PIP_NAME: "null"
+  extends: .push-packages
+
+push productfetcher:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "productfetcher"
+    PIP_NAME: "productfetcher"
+  extends: .push-packages
+
+push update-stage:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "update-stage"
+    PIP_NAME: "update_stage"
+  extends: .push-packages
+
+push vela:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "vela"
+    PIP_NAME: "vela"
+  extends: .push-packages
+
+push wf-inspector:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "wf_inspector"
+    PIP_NAME: "wf-inspector"
+  extends: .push-packages
+
+push ws-annihilator:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "ws_annihilator"
+    PIP_NAME: "ws-annihilator"
+  extends: .push-packages
+
+push ws-metrics:
+  stage: push-packages
+  variables:
+    PACKAGE_NAME: "ws_metrics"
+    PIP_NAME: "ws-metrics"
+  extends: .push-packages
+
 ###############################################
 # Clean Pipeline of Service and Web Images
 ###############################################

ci/build.template.yml

@@ -3,7 +3,8 @@
     script:
         - echo "Building branch or tag -- ${IMAGE_TAG}"
         - NAME="${REGISTRY_URL}/workspaces/${SERVICE_NAME}"
-        - docker build -t ${NAME}:${IMAGE_TAG} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile . --build-arg DEPLOY_ENV=${DEPLOY_ENV} --build-arg WS_VERSION=${VERSION} --build-arg CAPO_PROFILE=dsoc-${DEPLOY_ENV} --target prod
+        - docker build -t ${NAME}:${IMAGE_TAG} -f ${PATH_PREFIX}${SERVICE_NAME}/Dockerfile . --build-arg DEPLOY_ENV=${DEPLOY_ENV} --build-arg WS_VERSION=${VERSION}
+             --build-arg CAPO_PROFILE=dsoc-${DEPLOY_ENV} --build-arg API_TOKEN=glpat-vPamXXk4PZPe8GQAzmY2 --target prod
         - echo "TAG=${IMAGE_TAG}" >> build.env
     artifacts:
         reports:

ci/push-package.template.yml

+# CI Build Template
+.push-packages:
+    image: python:3.10-slim
+    before_script:
+      - mkdir -p ~/.ssh
+      - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+      - eval $(ssh-agent -s)
+      - chmod 700 ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_rsa
+      - '[[ -f /.dockerenv ]] && echo -e "Host *\\n\\tStrictHostKeyChecking no\\n\\n" > ~/.ssh/config'
+    script:
+      - echo "Releasing PEX to sbin area - ${PACKAGE_NAME}"
+      - |
+        RELEASE_CMD="cd /lustre/aoc/cluster/pipeline/dsoc-dev/workspaces/sbin/ && \
+            pex ${PIP_NAME} -c ${PACKAGE_NAME} -o ./${PACKAGE_NAME} -i https://gitlab.nrao.edu/api/v4/projects/621/packages/pypi/simple"
+      - B64CMD=$(echo "$RELEASE_CMD" | base64 | sed ':a;N;$!ba;s/\n//g')
+      - ssh -A shipman.aoc.nrao.edu "echo ${B64CMD} | base64 -d | bash"
+    rules:
+      - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /(^[0-9]\.[0-9]|^[0-9]\.[0-9]\.[0-9])-DEVELOPMENT/ || $CI_PIPELINE_SOURCE == "merge_request_event"'
+        changes:
+          - apps/cli/executables/pexable/${PACKAGE_NAME}/**/*
\ No newline at end of file

docker-compose.yml

@@ -14,6 +14,7 @@ services:
         - WS_VERSION=${WS_VERSION}
         - LOCAL_OR_SERVER_PEX="server-pex"
         - DEPLOY_ENV=${ENV}
+        - API_TOKEN=${API_TOKEN}
 #    ports:
 #      - target: 3456
 #        published: 3456

services/workflow/Dockerfile

@@ -6,6 +6,7 @@ ARG DEPLOY_ENV
 ENV DEPLOY_ENV=${DEPLOY_ENV}
 ARG ENV_HOST
 ENV ENV_HOST=${ENV_HOST}
+ARG API_TOKEN
 
 # Environment variables
 ENV PIP_NO_CACHE_DIR false
@@ -58,9 +59,9 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION=${WS_VERSION} pip install --user -r test-requ
 FROM base as server-pex
 USER root
 
-COPY --chown=vlapipe:vlapipe ./services/workflow/requirements.txt ./requirements.txt
+COPY --chown=vlapipe:vlapipe ./services/workflow/gitlab-requirements.txt ./requirements.txt
 COPY --chown=vlapipe:vlapipe ./test-requirements.txt ./test-requirements.txt
-RUN SETUPTOOLS_SCM_PRETEND_VERSION=${WS_VERSION} pip install --user -r requirements.txt
+RUN SETUPTOOLS_SCM_PRETEND_VERSION=${WS_VERSION} pip install --user -r requirements.txt --extra-index-url https://${API_TOKEN}@gitlab.nrao.edu/api/v4/projects/621/packages/pypi/simple
 RUN SETUPTOOLS_SCM_PRETEND_VERSION=${WS_VERSION} pip install --user -r test-requirements.txt
 
 #COPY --chown=vlapipe:vlapipe ./services/workflow/bin/install-pexes.sh ./install-pexes.sh

services/workflow/bin/install-pexes.sh

-pexes='[
-          {"name":"carta_envoy", "version":"2.8.2rc1"},
-          {"name":"casa_envoy", "version":"2.8.2rc1"},
-          {"name":"conveyor", "version":"2.8.2rc1"},
-          {"name":"deliver", "version":"2.8.2rc1"},
-          {"name":"ingest", "version":"2.8.2rc1"},
-          {"name":"ingest_envoy", "version":"2.8.2rc1"},
-          {"name":"mediator", "version":"2.8.2rc1"},
-          {"name":"null", "version":"2.8.2rc1"},
-          {"name":"productfetcher", "version":"2.8.2rc1"},
-          {"name":"update_stage", "version":"2.8.2rc1"},
-          {"name":"vela", "version":"2.8.2rc1"},
-          {"name":"wf_inspector", "version":"2.8.2rc1"},
-          {"name":"ws_annihilator", "version":"2.8.2rc1"},
-          {"name":"ws_metrics", "version":"2.8.2rc1"}
-        ]'
-
-BUILD_DIR=/lustre/aoc/cluster/pipeline/dsoc-dev/workspaces/sbin
-[[ -d $BUILD_DIR ]] ||  mkdir -p "$BUILD_DIR"
-
-for row in $(echo "${pexes}" | jq -r '.[] | @base64'); do
-    _jq() {
-     echo "${row}" | base64 --decode | jq -r "${1}"
-    }
-   curl --header "PRIVATE-TOKEN: glpat-vPamXXk4PZPe8GQAzmY2" "https://gitlab.nrao.edu/api/v4/projects/621/packages/pypi/$(_jq '.name')/$(_jq '.version')/$(_jq '.name')-$(_jq '.version').tar" --output "$(_jq '.name')-$(_jq '.version').tar"
-   tar -zxf "$(_jq '.name')-$(_jq '.version').tar.gz"
-   cd "$(_jq '.name')-$(_jq '.version')" || return
-   if [ -e pyproject.toml ]; then
-       until pex . -c "$(_jq '.name')" -o "$BUILD_DIR/$(_jq '.name')" --python-shebang /home/ssa/bin/python3.10 ; do
-         echo "PEX build failed. Retrying."; sleep 2;
-       done
-     else
-       echo "PEX build impossible in $PWD because there is no pyproject.toml file"
-     fi
-     cd ..
-#   pip3 install "$(_jq '.name')-$(_jq '.version')-py3-none-any.whl"
-done

services/workflow/gitlab-requirements.txt

+# Pex requirements pulled from gitlab
+carta-envoy==2.8.2rc1
+casa-envoy==2.8.2rc1
+conveyor==2.8.2rc1
+deliver==2.8.2rc1
+ingest-envoy==2.8.2rc1
+mediator==2.8.2rc1
+null==2.8.2rc1
+productfetcher==2.8.2rc1
+update-stage==2.8.2rc1
+vela==2.8.2rc1
+wf-inspector==2.8.2rc1
+ws-annihilator==2.8.2rc1
+ws-metrics==2.8.2rc1
\ No newline at end of file