From d283a58188c0d0267ef5c4ff78ed1b071e673343 Mon Sep 17 00:00:00 2001
From: Daniel K Lyons <dlyons@nrao.edu>
Date: Tue, 13 Sep 2022 15:30:54 -0600
Subject: [PATCH] Retry the validation as well as the fetch
---
.../productfetcher/productfetcher/fetchers.py | 15 +----
.../productfetcher/interfaces.py | 4 +-
.../productfetcher/productfetcher/retry.py | 57 +++++++++++++++++++
3 files changed, 63 insertions(+), 13 deletions(-)
create mode 100644 apps/cli/executables/pexable/productfetcher/productfetcher/retry.py
diff --git a/apps/cli/executables/pexable/productfetcher/productfetcher/fetchers.py b/apps/cli/executables/pexable/productfetcher/productfetcher/fetchers.py
index 33785253f..4f6d7af51 100644
--- a/apps/cli/executables/pexable/productfetcher/productfetcher/fetchers.py
+++ b/apps/cli/executables/pexable/productfetcher/productfetcher/fetchers.py
@@ -41,6 +41,7 @@ from .exceptions import (
)
from .interfaces import FetchProgressReporter, FileFetcher, LocatedFile
from .locations import NgasFile, OracleXml
+from .retry import retry
# pylint: disable=E0401, E0402, W0221
@@ -245,18 +246,8 @@ class RetryableFileFetcher(FileFetcherDecorator):
super().__init__(underlying)
self.retries = retries
- def do_fetch(self, attempt=1) -> Path:
- try:
- return self.underlying.do_fetch()
- except RetryableFetchError as r_err:
- if attempt < self.retries:
- # sleep for 2, 4, 8 seconds between attempts
- time.sleep(2**attempt)
- return self.do_fetch(attempt + 1)
- else:
- # let's annotate the error with how many times we tried
- r_err.retries = attempt
- raise r_err
+ def do_fetch(self) -> Path:
+ return retry(self.underlying.do_fetch, [2**attempt for attempt in range(1, 4)])
def __str__(self):
return f"{self.underlying} (with up to {self.retries} retries)"
diff --git a/apps/cli/executables/pexable/productfetcher/productfetcher/interfaces.py b/apps/cli/executables/pexable/productfetcher/productfetcher/interfaces.py
index f26c6358c..1ae735eb2 100644
--- a/apps/cli/executables/pexable/productfetcher/productfetcher/interfaces.py
+++ b/apps/cli/executables/pexable/productfetcher/productfetcher/interfaces.py
@@ -42,6 +42,8 @@ from abc import ABC, abstractmethod
from pathlib import Path
from typing import List, NamedTuple, Union
+from .retry import retry
+
class LocatedFile(abc.ABC):
"""
@@ -170,7 +172,7 @@ class FileFetcher(ABC):
# now validate with our validators, if we fetched a file
if result is not None:
for validator in self.validators:
- validator.validate(result)
+ retry(lambda: validator.validate(result), [1, 4, 9])
# if we made it here, we have a result to report
reporter.complete(self.file, result)
diff --git a/apps/cli/executables/pexable/productfetcher/productfetcher/retry.py b/apps/cli/executables/pexable/productfetcher/productfetcher/retry.py
new file mode 100644
index 000000000..97fc0c2ce
--- /dev/null
+++ b/apps/cli/executables/pexable/productfetcher/productfetcher/retry.py
@@ -0,0 +1,57 @@
+#
+# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
+#
+# This file is part of NRAO Workspaces.
+#
+# Workspaces is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Workspaces is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Workspaces. If not, see <https://www.gnu.org/licenses/>.
+import time
+from typing import Callable, List, TypeVar
+
+T = TypeVar("T")
+
+
+def retry(f: Callable[[], T], backoff_times: List[int] = None, retry_number=1) -> T:
+ """
+ Calls f() and returns its result. However, if f() throws an exception, we will retry.
+ Each entry in backoff_times is a length of time (in seconds) to sleep using `time.sleep`.
+ So if the backoff_times = [1, 4, 9], we will sleep 1 second, then 4 seconds, then 9 seconds between retries.
+
+ Also, if an exception should be raised on the last attempt, we will attempt to add a "retries" attribute to
+ it with the number of retries that were attempted.
+
+ :param f: the function to call
+ :param backoff_times: a list of times to wait
+ :param retry_number: the current "retry" (used only for the retries attribute)
+ :return: whatever the result of `f` is
+ """
+ exception = None
+ try:
+ return f()
+ except Exception as ex:
+ if backoff_times:
+ # if we have, say, [1, 4, 9] in our list of backoff times,
+ # we will delay for 1 second and then have [4, 9] remaining backoff times
+ delay, *remaining_backoff_times = backoff_times
+
+ # sleep
+ time.sleep(delay)
+
+ return retry(f, remaining_backoff_times, retry_number + 1)
+ else:
+ # if this exception has a slot for "retry count" we should set it
+ if hasattr(ex, "retries"):
+ ex.retries = retry_number
+
+ # now raise the exception
+ raise exception
--
GitLab