Skip to content
Snippets Groups Projects

Retry the validation as well as the fetch

Merged Daniel Lyons requested to merge product-fetcher-retry-file-validation into main
3 files
+ 63
13
Compare changes
  • Side-by-side
  • Inline
Files
3
@@ -41,6 +41,7 @@ from .exceptions import (
)
from .interfaces import FetchProgressReporter, FileFetcher, LocatedFile
from .locations import NgasFile, OracleXml
from .retry import retry
# pylint: disable=E0401, E0402, W0221
@@ -245,18 +246,8 @@ class RetryableFileFetcher(FileFetcherDecorator):
super().__init__(underlying)
self.retries = retries
def do_fetch(self, attempt=1) -> Path:
try:
return self.underlying.do_fetch()
except RetryableFetchError as r_err:
if attempt < self.retries:
# sleep for 2, 4, 8 seconds between attempts
time.sleep(2**attempt)
return self.do_fetch(attempt + 1)
else:
# let's annotate the error with how many times we tried
r_err.retries = attempt
raise r_err
def do_fetch(self) -> Path:
return retry(self.underlying.do_fetch, [2**attempt for attempt in range(1, 4)])
def __str__(self):
return f"{self.underlying} (with up to {self.retries} retries)"
Loading