From 142bb1904f6ebd321e965abe63fa9e8caf10fc65 Mon Sep 17 00:00:00 2001
From: Charlotte Hausman <chausman@nrao.edu>
Date: Mon, 12 Jul 2021 19:48:06 +0000
Subject: [PATCH] WS-474: REST calls and CLI for calibration ingestion
---
.../wf_framework/sh/ingest-request.sh | 31 ++++++++++++++++++
services/workflow/workflow/server.py | 32 +++++++++++++++++--
.../workflow/services/workflow_service.py | 11 ++++++-
3 files changed, 70 insertions(+), 4 deletions(-)
create mode 100644 apps/cli/executables/wf_framework/sh/ingest-request.sh
diff --git a/apps/cli/executables/wf_framework/sh/ingest-request.sh b/apps/cli/executables/wf_framework/sh/ingest-request.sh
new file mode 100644
index 000000000..61da16eee
--- /dev/null
+++ b/apps/cli/executables/wf_framework/sh/ingest-request.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset
+
+function usage {
+ echo "Usage: ingest-request [-c] [workflow_request_id]
+
+ This script sets off a Workspaces ingestion workflow of the specified type,
+ given by the chosen option, for the provided workflow request id.
+
+ Options:
+ -c, --calibration run the ingest_cal workflow for the specified request
+ -h, --help display this help and exit
+ "
+}
+
+option=$(echo "$1" | tr A-Z a-z)
+case $option in
+ --calibration|-c)
+ action="ingest_cal"
+ ;;
+ --help|-h)
+ usage
+ ;;
+esac
+
+WORKFLOW_SERVICE=$(capo -q edu.nrao.archive.workspaces.WorkflowSettings.serviceUrl)
+
+if [ "$action" = "ingest_cal" ]; then
+ curl -X POST $WORKFLOW_SERVICE/workflows/std_calibration/requests/$2/ingest
+fi
diff --git a/services/workflow/workflow/server.py b/services/workflow/workflow/server.py
index 681e71bc5..a10f179d8 100644
--- a/services/workflow/workflow/server.py
+++ b/services/workflow/workflow/server.py
@@ -238,6 +238,29 @@ class WorkflowRequestRestService:
return self.request.workflows.announce_qa_ready(self.request.matchdict["request_id"])
+ @view_config(request_method="POST", route_name="ingest_workflow_result")
+ def ingest(self):
+ """
+ Ingest specified workflow request's results into NGAS and archive
+ :return:
+ """
+ print(f"Ingesting results for workflow request {self.request.context}")
+
+ # 1. retrieve metadata.json for workflow request
+ self.request.matchdict["filename"] = "metadata.json"
+ file = lookup_file(request=self.request)
+
+ # 2. create ingestion workflow request
+ ingest_type = "ingest_cal" if "calibration" in self.request.matchdict["name"] else "ingest"
+ ingest_request = self.request.info.create_workflow_request(ingest_type)
+ # 3. attach metadata.json to ingestion wf request
+ self.request.workflows.attach_file_to_request(
+ request=ingest_request, filename=file.filename, content=file.content
+ )
+ # 4. submit ingestion workflow request
+ self.request.workflows.execute(ingest_request)
+
+
@view_defaults(route_name="workflow_request_files", renderer="json")
class WorkflowFilesRestService:
"""
@@ -417,12 +440,15 @@ def main(global_config, **settings):
"/workflows/{name}/requests/{request_id}/submit",
factory=lookup_request,
)
+ config.add_route(
+ "ingest_workflow_result",
+ "/workflows/{name}/requests/{request_id}/ingest",
+ factory=lookup_request,
+ )
# yes I know this doesn't match pattern, it's a stopgap.
config.add_route(
- "announce_qa_ready",
- "/workflows/requests/{request_id}/qa",
- factory=lookup_request
+ "announce_qa_ready", "/workflows/requests/{request_id}/qa", factory=lookup_request
)
config.include("pyramid_beaker")
config.scan(".")
diff --git a/shared/workspaces/workspaces/workflow/services/workflow_service.py b/shared/workspaces/workspaces/workflow/services/workflow_service.py
index edf468b2c..91cbd67a9 100644
--- a/shared/workspaces/workspaces/workflow/services/workflow_service.py
+++ b/shared/workspaces/workspaces/workflow/services/workflow_service.py
@@ -98,12 +98,21 @@ class WorkflowServiceRESTClient(WorkflowServiceIF):
Announce results are available for QA
THIS IS A WORKAROUND PENDING THE IMPLEMENTATION OF THE QA SYSTEM!
<insert unhappy tears here>
- :param workflow_request: completed workflow request
:param workflow_request_id: id of completed workflow
:return:
"""
requests.post(f"{self.url}/workflows/requests/{workflow_request_id}/qa")
+ def ingest(self, request: WorkflowRequestIF):
+ """
+ Ingest results for previously run workflow into NGAS and archive
+ :param request: completed workflow request to ingest
+ :return:
+ """
+ requests.post(
+ f"{self.url}/workflows/{request.workflow_name}/requests/{request.workflow_request_id}/ingest"
+ )
+
class WorkflowService(WorkflowServiceIF):
"""
--
GitLab