diff --git a/docker.properties b/docker.properties
index 707cd55e955a81f5fb4ef1a99b38d7a9d46ba043..ce42fc956dd4123376b6a16d6cdb33658ef6b9be 100644
--- a/docker.properties
+++ b/docker.properties
@@ -85,9 +85,6 @@ edu.nrao.carta.reverseProxyHost = black-abyss.nrao.edu
edu.nrao.carta.redisPort = 6397
edu.nrao.carta.redisPassword = password
-# Sentry.io Settings
-edu.nrao.workspaces.SentrySettings.sentry_key = local
-
# WS Annihilator Settings
edu.nrao.workspaces.AnnihilatorSettings.keepSpoolForDays = 10
edu.nrao.workspaces.AnnihilatorSettings.keepStagingForDays = 10
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 838cc87d57c226f5462725b51ec48e831d27624f..1d13d23e3f48f21086bcc671a90f580711ee4a71 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -36,7 +36,6 @@ prettierfier
pytest
requests
scp
-sentry_sdk
simplejson
sqlalchemy==1.4.46
tqdm
diff --git a/services/capability/capability/server.py b/services/capability/capability/server.py
index 841edd50990a6b044f13956cccd35b53d2dd7fbc..9bf8e35a418623fa387011687010fcce20c56f8e 100644
--- a/services/capability/capability/server.py
+++ b/services/capability/capability/server.py
@@ -17,7 +17,6 @@
# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
import time
-import sentry_sdk
import sqlalchemy.orm
import transaction
import zope.sqlalchemy
@@ -30,7 +29,6 @@ from pyramid.renderers import JSONP
from pyramid.request import Request
from pyramid.response import Response
from pyramid_beaker import BeakerSessionFactoryConfig, session_factory_from_settings
-from sentry_sdk.integrations.pyramid import PyramidIntegration
from workspaces.capability.services.capability_info import CapabilityInfo
from workspaces.capability.services.capability_service import (
@@ -46,6 +44,7 @@ from workspaces.workflow.services.workflow_service import WorkflowServiceRESTCli
from .utility.notificationsender import TransactionalNotificationSender
+
# Copied from here: https://stackoverflow.com/questions/21107057/pyramid-cors-for-ajax-requests
def add_cors_headers_response_callback(event: NewRequest):
"""
@@ -84,7 +83,9 @@ def add_cors_headers_response_callback(event: NewRequest):
# ---------------------------------------------------------
-def add_services(config: Configurator, session_factory: BeakerSessionFactoryConfig) -> Configurator:
+def add_services(
+ config: Configurator, session_factory: BeakerSessionFactoryConfig
+) -> Configurator:
"""
Add capability info, capability service, and workflow service to Pyramid request configuration
@@ -95,10 +96,17 @@ def add_services(config: Configurator, session_factory: BeakerSessionFactoryConf
# # these objects should exist across request boundaries
workflow_rest_client = WorkflowServiceRESTClient()
notification_rest_client = TransactionalNotificationSender(transaction.manager)
- capability_info = CapabilityInfo(get_tm_session(session_factory, transaction.manager))
- state_machine_info = StateMachineInfo(get_tm_session(session_factory, transaction.manager))
+ capability_info = CapabilityInfo(
+ get_tm_session(session_factory, transaction.manager)
+ )
+ state_machine_info = StateMachineInfo(
+ get_tm_session(session_factory, transaction.manager)
+ )
capability_service = CapabilityService(
- capability_info, state_machine_info, workflow_rest_client, notification_rest_client
+ capability_info,
+ state_machine_info,
+ workflow_rest_client,
+ notification_rest_client,
)
archive_service = ArchiveService()
@@ -128,13 +136,19 @@ def add_services(config: Configurator, session_factory: BeakerSessionFactoryConf
)
# make capability launcher available for use in Pyramid
- config.add_request_method(create_capability_launcher, "capability_launcher", reify=True)
+ config.add_request_method(
+ create_capability_launcher, "capability_launcher", reify=True
+ )
# make capability queue starter available for use in Pyramid
- config.add_request_method(create_queue_restarter, "capability_queue_restarter", reify=True)
+ config.add_request_method(
+ create_queue_restarter, "capability_queue_restarter", reify=True
+ )
# make capability canceller available for use in Pyramid
- config.add_request_method(create_capability_canceller, "capability_canceller", reify=True)
+ config.add_request_method(
+ create_capability_canceller, "capability_canceller", reify=True
+ )
# make workflow_info available for use in Pyramid
config.add_request_method(
@@ -144,15 +158,21 @@ def add_services(config: Configurator, session_factory: BeakerSessionFactoryConf
reify=True,
)
- config.add_request_method(lambda request: notification_rest_client, "notification_service", reify=True)
+ config.add_request_method(
+ lambda request: notification_rest_client, "notification_service", reify=True
+ )
# make archive_service available for use in Pyramid
- config.add_request_method(lambda request: archive_service, "archive_service", reify=True)
+ config.add_request_method(
+ lambda request: archive_service, "archive_service", reify=True
+ )
return config
def lookup_request(request):
- return request.capability_info.lookup_capability_request(request.matchdict["request_id"])
+ return request.capability_info.lookup_capability_request(
+ request.matchdict["request_id"]
+ )
# ---------------------------------------------------------
@@ -163,7 +183,8 @@ def lookup_request(request):
def get_tm_session(
- session_factory: BeakerSessionFactoryConfig, transaction_manager: transaction.TransactionManager
+ session_factory: BeakerSessionFactoryConfig,
+ transaction_manager: transaction.TransactionManager,
) -> sqlalchemy.orm.Session:
"""
Enable Zope's transaction manager on our session
@@ -183,7 +204,9 @@ def get_tm_session(
# ---------------------------------------------------------
# Prometheus logging
-http_requests = Summary("http_request_timing", "HTTP Requests", ["status_code", "route_name"])
+http_requests = Summary(
+ "http_request_timing", "HTTP Requests", ["status_code", "route_name"]
+)
def prometheus_route_timing_factory(handler, registry):
@@ -193,9 +216,9 @@ def prometheus_route_timing_factory(handler, registry):
response = handler(request)
end = time.time()
if request.matched_route:
- http_requests.labels(status_code=response.status_code, route_name=request.matched_route.name).observe(
- end - start
- )
+ http_requests.labels(
+ status_code=response.status_code, route_name=request.matched_route.name
+ ).observe(end - start)
return response
return prometheus_route_timer
@@ -211,7 +234,11 @@ class QueueReportCollector:
self.capability_info = capability_info
def collect(self):
- c = CounterMetricFamily("capability_queue", "Queue status for each capability", labels=["capability", "status"])
+ c = CounterMetricFamily(
+ "capability_queue",
+ "Queue status for each capability",
+ labels=["capability", "status"],
+ )
for report in self.capability_info.report():
c.add_metric([report.capability, "waiting"], report.waiting)
@@ -228,18 +255,6 @@ class QueueReportCollector:
def main(global_config, **settings):
with Configurator(settings=settings) as config:
- sentry_key = CapoConfig().settings("edu.nrao.workspaces.SentrySettings").sentry_key
-
- if sentry_key != "local":
- sentry_sdk.init(
- dsn=sentry_key,
- integrations=[PyramidIntegration()],
- # Set traces_sample_rate to 1.0 to capture 100%
- # of transactions for performance monitoring.
- # We recommend adjusting this value in production.
- traces_sample_rate=1.0,
- )
-
# Helpers
config.add_subscriber(add_cors_headers_response_callback, NewRequest)
diff --git a/services/capability/poetry.lock b/services/capability/poetry.lock
index 98ed222f6bb244065c7f5f5d7c4c1d1be0265e64..0569ac4ff6a222c7607492dde558360e2dfb664a 100644
--- a/services/capability/poetry.lock
+++ b/services/capability/poetry.lock
@@ -556,6 +556,16 @@ files = [
{file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
{file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
{file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
{file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
{file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
{file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},
@@ -1083,39 +1093,6 @@ urllib3 = ">=1.21.1,<3"
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
-[[package]]
-name = "sentry-sdk"
-version = "1.5.10"
-description = "Python client for Sentry (https://sentry.io)"
-optional = false
-python-versions = "*"
-files = [
- {file = "sentry-sdk-1.5.10.tar.gz", hash = "sha256:0a9eb20a84f4c17c08c57488d59fdad18669db71ebecb28fb0721423a33535f9"},
- {file = "sentry_sdk-1.5.10-py2.py3-none-any.whl", hash = "sha256:972c8fe9318a415b5cf35f687f568321472ef94b36806407c370ce9c88a67f2e"},
-]
-
-[package.dependencies]
-certifi = "*"
-urllib3 = ">=1.10.0"
-
-[package.extras]
-aiohttp = ["aiohttp (>=3.5)"]
-beam = ["apache-beam (>=2.12)"]
-bottle = ["bottle (>=0.12.13)"]
-celery = ["celery (>=3)"]
-chalice = ["chalice (>=1.16.0)"]
-django = ["django (>=1.8)"]
-falcon = ["falcon (>=1.4)"]
-flask = ["blinker (>=1.1)", "flask (>=0.11)"]
-httpx = ["httpx (>=0.16.0)"]
-pure-eval = ["asttokens", "executing", "pure-eval"]
-pyspark = ["pyspark (>=2.4.4)"]
-quart = ["blinker (>=1.1)", "quart (>=0.16.1)"]
-rq = ["rq (>=0.6)"]
-sanic = ["sanic (>=0.8)"]
-sqlalchemy = ["sqlalchemy (>=1.2)"]
-tornado = ["tornado (>=5)"]
-
[[package]]
name = "setuptools"
version = "68.0.0"
@@ -1447,4 +1424,4 @@ test = ["zope.testing"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
-content-hash = "0330bcb9dc2a78fd69b90a5f52bb4c6256c58929d77315c05fb162cc19199081"
+content-hash = "4fc1eea6d18754c3d0bf8ffa6af19a70dfbd34779b7ecd95543d019a0c7460b1"
diff --git a/services/capability/pyproject.toml b/services/capability/pyproject.toml
index bb89abc402cdd2a37c604d91383a69b7635ec61a..f2d42d06e65b3ba9c887b96231f393160f13dfab 100644
--- a/services/capability/pyproject.toml
+++ b/services/capability/pyproject.toml
@@ -24,7 +24,6 @@ sqlalchemy = "1.4.47"
waitress = "^2.1.2"
"zope.sqlalchemy" = "2.0"
immutable-views = "^0.6.1"
-sentry-sdk = "1.5.10"
prometheus-client = "0.4.1"
workspaces = {path = "../../shared/workspaces"}
messaging = {path = "../../shared/messaging"}
diff --git a/services/workflow/poetry.lock b/services/workflow/poetry.lock
index 2fadcbbc9b483d5beacd8c4822c9cff8956a3da2..ec7d48f14ead2ba68e2f3b0780f5fab3fb1f4543 100644
--- a/services/workflow/poetry.lock
+++ b/services/workflow/poetry.lock
@@ -421,6 +421,16 @@ files = [
{file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
{file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
{file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
+ {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
{file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
{file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
{file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},
@@ -848,38 +858,6 @@ urllib3 = ">=1.21.1,<3"
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
-[[package]]
-name = "sentry-sdk"
-version = "1.5.0"
-description = "Python client for Sentry (https://sentry.io)"
-optional = false
-python-versions = "*"
-files = [
- {file = "sentry-sdk-1.5.0.tar.gz", hash = "sha256:789a11a87ca02491896e121efdd64e8fd93327b69e8f2f7d42f03e2569648e88"},
- {file = "sentry_sdk-1.5.0-py2.py3-none-any.whl", hash = "sha256:0db297ab32e095705c20f742c3a5dac62fe15c4318681884053d0898e5abb2f6"},
-]
-
-[package.dependencies]
-certifi = "*"
-urllib3 = ">=1.10.0"
-
-[package.extras]
-aiohttp = ["aiohttp (>=3.5)"]
-beam = ["apache-beam (>=2.12)"]
-bottle = ["bottle (>=0.12.13)"]
-celery = ["celery (>=3)"]
-chalice = ["chalice (>=1.16.0)"]
-django = ["django (>=1.8)"]
-falcon = ["falcon (>=1.4)"]
-flask = ["blinker (>=1.1)", "flask (>=0.11)"]
-httpx = ["httpx (>=0.16.0)"]
-pure-eval = ["asttokens", "executing", "pure-eval"]
-pyspark = ["pyspark (>=2.4.4)"]
-rq = ["rq (>=0.6)"]
-sanic = ["sanic (>=0.8)"]
-sqlalchemy = ["sqlalchemy (>=1.2)"]
-tornado = ["tornado (>=5)"]
-
[[package]]
name = "setuptools"
version = "68.0.0"
@@ -1211,4 +1189,4 @@ test = ["zope.testing"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
-content-hash = "2d9b5aac63d792b67046ca1fe2eca5a472e8379858a798e364fe4a451336dd07"
+content-hash = "d7ad3ca6f57737f1b18e350b15663cb08907fdd65d41353f02018850d517caec"
diff --git a/services/workflow/pyproject.toml b/services/workflow/pyproject.toml
index 32c4e1039cc8996fd8da194872d984e901cda663..15f28d33c86c354eecbbb5cd432e33914201e0a4 100644
--- a/services/workflow/pyproject.toml
+++ b/services/workflow/pyproject.toml
@@ -20,7 +20,6 @@ workspaces = {path = "../../shared/workspaces"}
messaging = {path = "../../shared/messaging"}
"zope.sqlalchemy" = "2.0"
immutable-views = "^0.6.1"
-sentry-sdk = "1.5.0"
prometheus-client = "0.4.1"
psycopg2 = "^2.9.6"
diff --git a/services/workflow/requirements.txt b/services/workflow/requirements.txt
index 0bebaf31b506db9b75cfdca97c63badaf79dd8c5..ec6b80199d368fc65ab34a61b842dd961647c229 100644
--- a/services/workflow/requirements.txt
+++ b/services/workflow/requirements.txt
@@ -1,5 +1,3 @@
-# -e ../code/shared/messaging
-# -e ../code/shared/workspaces
-e ../code/apps/cli/utilities/wf_monitor
-e ../code/apps/cli/utilities/aat_wrest
-e ../code/apps/cli/utilities/contacts_wrest
diff --git a/services/workflow/workflow/server.py b/services/workflow/workflow/server.py
index 594bfb8ae202c0fb24c447aca30a21bbcd04e1ee..937807a3a6d9d01061c73c2b65fc3fc05dadf66e 100644
--- a/services/workflow/workflow/server.py
+++ b/services/workflow/workflow/server.py
@@ -26,7 +26,6 @@ from json import JSONDecodeError
from pathlib import Path
import prometheus_client
-import sentry_sdk
import transaction
import zope.sqlalchemy
from pycapo import CapoConfig
@@ -37,7 +36,6 @@ from pyramid.response import FileResponse, Response
from pyramid.view import view_config, view_defaults
from pyramid.wsgi import wsgiapp
from pyramid_beaker import session_factory_from_settings
-from sentry_sdk.integrations.pyramid import PyramidIntegration
from workspaces.system.schema import get_engine, get_session_factory
from workspaces.workflow.schema import Workflow, WorkflowRequest, WorkflowRequestFile
@@ -153,7 +151,9 @@ class WorkflowWorkingDirRestService:
:return: None
"""
- requested_workflow = self.request.info.lookup_workflow_request(self.request.matchdict["request_id"])
+ requested_workflow = self.request.info.lookup_workflow_request(
+ self.request.matchdict["request_id"]
+ )
results_path = requested_workflow.results_dir
parent_paths = [requested_workflow.results_dir]
@@ -171,11 +171,15 @@ class WorkflowWorkingDirRestService:
@view_config(request_method="GET", route_name="serve_weblog", renderer="json")
def serve_weblog(self):
- requested_workflow = self.request.info.lookup_workflow_request(self.request.matchdict["request_id"])
+ requested_workflow = self.request.info.lookup_workflow_request(
+ self.request.matchdict["request_id"]
+ )
results_path = Path(requested_workflow.results_dir)
index_path_list = list(results_path.glob("products/pipeline-*/html/index.html"))
- failed_weblog_path_list = list(results_path.glob("working/pipeline-*/html/index.html"))
+ failed_weblog_path_list = list(
+ results_path.glob("working/pipeline-*/html/index.html")
+ )
if index_path_list and len(index_path_list) == 1:
index_path = index_path_list[0]
elif failed_weblog_path_list and len(failed_weblog_path_list) == 1:
@@ -195,10 +199,14 @@ class WorkflowWorkingDirRestService:
@view_config(request_method="GET", route_name="get_qa_notes", renderer="json")
def get_qa_notes(self):
- requested_workflow = self.request.info.lookup_workflow_request(self.request.matchdict["request_id"])
+ requested_workflow = self.request.info.lookup_workflow_request(
+ self.request.matchdict["request_id"]
+ )
results_path = Path(requested_workflow.results_dir)
- qa_notes_list = list(results_path.glob("products/pipeline-*/html/qa_notes.html"))
+ qa_notes_list = list(
+ results_path.glob("products/pipeline-*/html/qa_notes.html")
+ )
if qa_notes_list and len(qa_notes_list) == 1:
qa_notes_path = qa_notes_list[0]
else:
@@ -213,14 +221,20 @@ class WorkflowWorkingDirRestService:
with open(qa_notes_path, "r") as qa_notes:
qa_notes_text = qa_notes.read()
- return Response(status_int=http.HTTPStatus.OK, json_body={"resp": f"{qa_notes_text}"})
+ return Response(
+ status_int=http.HTTPStatus.OK, json_body={"resp": f"{qa_notes_text}"}
+ )
@view_config(request_method="POST", route_name="get_qa_notes", renderer="json")
def save_qa_notes(self):
- requested_workflow = self.request.info.lookup_workflow_request(self.request.matchdict["request_id"])
+ requested_workflow = self.request.info.lookup_workflow_request(
+ self.request.matchdict["request_id"]
+ )
results_path = Path(requested_workflow.results_dir)
- qa_notes_list = list(results_path.glob("products/pipeline-*/html/qa_notes.html"))
+ qa_notes_list = list(
+ results_path.glob("products/pipeline-*/html/qa_notes.html")
+ )
if qa_notes_list and len(qa_notes_list) == 1:
qa_notes_path = qa_notes_list[0]
else:
@@ -235,15 +249,21 @@ class WorkflowWorkingDirRestService:
# sanitize input before writing/persisting
# \\u0000 is an invalid character that is incompatible with postgres json columns
# from StackOverflow: https://stackoverflow.com/questions/63092267/how-to-handle-api-responsesjson-containing-x00-or-u0000-in-its-data-and-s
- edits = json.loads(json.dumps(self.request.json_body["edits"])).replace("\\u0000", "")
+ edits = json.loads(json.dumps(self.request.json_body["edits"])).replace(
+ "\\u0000", ""
+ )
qa_notes.write(edits)
return Response(
status_int=http.HTTPStatus.OK,
- json_body={"resp": f"Edits made to QA notes file in workflow {self.request.matchdict['request_id']}."},
+ json_body={
+ "resp": f"Edits made to QA notes file in workflow {self.request.matchdict['request_id']}."
+ },
)
- @view_config(request_method="GET", route_name="serve_carta_wrapper", renderer="json")
+ @view_config(
+ request_method="GET", route_name="serve_carta_wrapper", renderer="json"
+ )
def serve_carta_wrapper(self):
"""
Dish up some HTML containing the CARTA URL in a frame.
@@ -251,9 +271,13 @@ class WorkflowWorkingDirRestService:
:return:
"""
- path = Path(f"/lustre/aoc/cluster/pipeline/docker/workspaces/html/{self.request.matchdict['request_id']}")
+ path = Path(
+ f"/lustre/aoc/cluster/pipeline/docker/workspaces/html/{self.request.matchdict['request_id']}"
+ )
carta_html_file = list(path.iterdir())[0]
- return FileResponse(carta_html_file, request=self.request, content_type="text/html")
+ return FileResponse(
+ carta_html_file, request=self.request, content_type="text/html"
+ )
@view_config(route_name="get_healthcheck", renderer="json")
def get_healthcheck(self) -> Response:
@@ -263,7 +287,10 @@ class WorkflowWorkingDirRestService:
:return:
"""
return Response(
- status=http.HTTPStatus.OK, json_body={"healthcheck": f"Workflow service returned {http.HTTPStatus.OK}"}
+ status=http.HTTPStatus.OK,
+ json_body={
+ "healthcheck": f"Workflow service returned {http.HTTPStatus.OK}"
+ },
)
def generate_working_directory_dict(self, results_path, parent_paths) -> dict:
@@ -290,7 +317,13 @@ class WorkflowWorkingDirRestService:
# check if url needs a slash to divide paths
divider = ("/", "")[self.request.current_route_url().endswith("/")]
- content_key.update({key.name: {"url": self.request.current_route_url() + divider + key.name}})
+ content_key.update(
+ {
+ key.name: {
+ "url": self.request.current_route_url() + divider + key.name
+ }
+ }
+ )
# add full path for content
content_key[key.name].update({"full_path": key.absolute().__str__()})
@@ -306,7 +339,9 @@ class WorkflowWorkingDirRestService:
# if it is a directory, create a json object
workdir_json = json.dumps(workdir_dict, indent=2)
# create response with the json object as the body
- response = Response(body=workdir_json, request=self.request, content_type="text/json")
+ response = Response(
+ body=workdir_json, request=self.request, content_type="text/json"
+ )
else:
# if it is not a directory, serve the static file
response = FileResponse(
@@ -317,11 +352,15 @@ class WorkflowWorkingDirRestService:
def generate_url_from_path(self, root_path, results_path):
current_url = self.request.current_route_url()
- return current_url.replace("/weblog", "/dir") + root_path.replace(results_path, "")
+ return current_url.replace("/weblog", "/dir") + root_path.replace(
+ results_path, ""
+ )
def generate_qa_notes_path(self, root_path, results_path):
current_url = self.request.current_route_url()
- return current_url.replace("/qa_notes", "/dir") + root_path.replace(results_path, "")
+ return current_url.replace("/qa_notes", "/dir") + root_path.replace(
+ results_path, ""
+ )
@view_defaults(route_name="workflow_request", renderer="json")
@@ -353,7 +392,9 @@ class WorkflowRequestRestService:
# Most common case: Empty body for simple requests, continue with an empty dict
argument_body = {}
- return self.request.info.create_workflow_request(self.request.context.workflow_name, argument_body)
+ return self.request.info.create_workflow_request(
+ self.request.context.workflow_name, argument_body
+ )
@view_config(request_method="POST", route_name="submit_workflow_request")
def submit_workflow(self):
@@ -393,7 +434,11 @@ class WorkflowRequestRestService:
file = lookup_file(request=self.request)
# 2. create ingestion workflow request
- ingest_type = "ingest_cal" if "calibration" in self.request.matchdict["name"] else "ingest_image"
+ ingest_type = (
+ "ingest_cal"
+ if "calibration" in self.request.matchdict["name"]
+ else "ingest_image"
+ )
ingest_request = self.request.info.create_workflow_request(
workflow=ingest_type,
argument={"parent_wf_request_id": self.request.matchdict["request_id"]},
@@ -419,7 +464,9 @@ class WorkflowRequestRestService:
argument=self.request.json_body,
)
self.request.workflows.execute(ingestion_request)
- return Response(status_code=http.HTTPStatus.OK, json_body=ingestion_request.__json__())
+ return Response(
+ status_code=http.HTTPStatus.OK, json_body=ingestion_request.__json__()
+ )
@view_config(request_method="POST", route_name="abort_workflow_request")
def abort(self):
@@ -500,10 +547,16 @@ class WorkflowRequestRestService:
"""
body = self.request.json_body
- identifier = int(body["request_id"]) if "request_id" in body else body["project_code"]
+ identifier = (
+ int(body["request_id"]) if "request_id" in body else body["project_code"]
+ )
msg_type = self.request.matchdict["msg_type"]
- additional = body["project_code"] if "project_code" in body and identifier != body["project_code"] else None
+ additional = (
+ body["project_code"]
+ if "project_code" in body and identifier != body["project_code"]
+ else None
+ )
self.request.workflows.message_archive(identifier, msg_type, additional)
return Response(
@@ -538,16 +591,22 @@ class WorkflowRequestRestService:
request_id = self.request.matchdict["request_id"]
self.request.workflows.send_forced_fail(request_id)
- @view_config(request_method="GET", route_name="workflow_request_htcondor_id", renderer="json")
+ @view_config(
+ request_method="GET", route_name="workflow_request_htcondor_id", renderer="json"
+ )
def get_request_htcondor_id(self):
"""
Pyramid view that gives back the HTCondor job ID for a given workflow request
:return: HTTP response with HTCondor job ID in the body
"""
- requested_workflow = self.request.info.lookup_workflow_request(self.request.matchdict["request_id"])
+ requested_workflow = self.request.info.lookup_workflow_request(
+ self.request.matchdict["request_id"]
+ )
- return Response(json_body={"htcondor_job_id": str(requested_workflow.htcondor_job_id)})
+ return Response(
+ json_body={"htcondor_job_id": str(requested_workflow.htcondor_job_id)}
+ )
@view_config(request_method="GET", route_name="list_stale_requests")
def get_stale_requests(self):
@@ -688,20 +747,23 @@ def get_tm_session(session_factory, transaction_manager):
http_requests = None
+
def prometheus_route_timing_factory(handler, registry):
# if timing support is enabled, return a wrapper
global http_requests
# Prometheus logging
- http_requests = prometheus_client.Summary("http_request_timing", "HTTP Requests", ["status_code", "route_name"])
+ http_requests = prometheus_client.Summary(
+ "http_request_timing", "HTTP Requests", ["status_code", "route_name"]
+ )
def prometheus_route_timer(request):
start = time.time()
response = handler(request)
end = time.time()
if request.matched_route:
- http_requests.labels(status_code=response.status_code, route_name=request.matched_route.name).observe(
- end - start
- )
+ http_requests.labels(
+ status_code=response.status_code, route_name=request.matched_route.name
+ ).observe(end - start)
return response
return prometheus_route_timer
@@ -716,18 +778,6 @@ def prometheus_route_timing_factory(handler, registry):
def main(global_config, **settings):
with Configurator(settings=settings) as config:
- sentry_key = CapoConfig().settings("edu.nrao.workspaces.SentrySettings").sentry_key
-
- if sentry_key != "local":
- sentry_sdk.init(
- dsn=sentry_key,
- integrations=[PyramidIntegration()],
- # Set traces_sample_rate to 1.0 to capture 100%
- # of transactions for performance monitoring.
- # We recommend adjusting this value in production.
- traces_sample_rate=1.0,
- )
-
session_factory = session_factory_from_settings(settings)
config.set_session_factory(session_factory)
config.add_renderer("jsonp", JSONP(param_name="callback"))
@@ -745,7 +795,9 @@ def main(global_config, **settings):
# we need to build a workflow_info here for the message handler, but
# we won't use it anywhere else, we will make new ones per-request
- workflow_info = WorkflowInfo(get_tm_session(session_factory, transaction.manager))
+ workflow_info = WorkflowInfo(
+ get_tm_session(session_factory, transaction.manager)
+ )
message_handler = WorkflowMessageHandler(workflow_info)
workflow_recover = MonitorRecover(workflow_info)
@@ -757,7 +809,9 @@ def main(global_config, **settings):
# make workflow_service available for use in Pyramid
config.add_request_method(
- lambda r: WorkflowService(r.info, message_handler, workflow_recover), "workflows", reify=True
+ lambda r: WorkflowService(r.info, message_handler, workflow_recover),
+ "workflows",
+ reify=True,
)
# GET /workflows <- list of workflows
diff --git a/test-requirements.txt b/test-requirements.txt
index 8166c668c422882fc82f2bef451a44c50735f20e..29fba01387cc84a517a9933eafe706e153e5ac4f 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -17,7 +17,6 @@ pyramid-tm==2.5
immutable_views
hypothesis
kombu
-sentry_sdk
zope-sqlalchemy==2.0
-e ./shared/messaging
-e ./shared/workspaces