Skip to content
Snippets Groups Projects

remove system exits from datafetcher tests

Merged Janet Goldstein requested to merge WS-179-4-remove-sys-exits into main
All threads resolved!
1 file
+ 3
6
Compare changes
  • Side-by-side
  • Inline
@@ -3,15 +3,12 @@
""" Implementations of assorted product fetchers """
import copy
import sys
from argparse import Namespace
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from pprint import pprint
from typing import Dict
from datafetcher.errors import NGASServiceErrorException
from datafetcher.return_codes import ReturnCode
from datafetcher.errors import NGASServiceErrorException, NGASFetchError
from .file_retrievers import NGASFileRetriever
from .utilities import FlexLogger
@@ -134,7 +131,6 @@ class ParallelFetcher(BaseFetcher):
def fetch_bucket(self, bucket):
""" Grab the files in this bucket """
file_sizes = [file["size"] for file in bucket["files"]]
pprint(f"retrieving files {file_sizes} from {bucket['server']}")
self._LOG.debug(
f"{bucket['retrieve_method']} "
f"{len(bucket['files'])} files from "
@@ -149,10 +145,8 @@ class ParallelFetcher(BaseFetcher):
def run(self):
""" Fetch all the files for the product locator """
print(f"running {self.__class__.__name__}")
if self.args.dry_run:
self._LOG.debug("This is a dry run; files will not be fetched")
return 0
with ThreadPoolExecutor() as executor:
results = executor.map(self.fetch_bucket, self.bucketized_files)
@@ -166,14 +160,8 @@ class ParallelFetcher(BaseFetcher):
f"{self.num_files_expected} files expected, "
f"but only {self.num_files_retrieved} retrieved"
)
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
# successful retrieval
print("returning")
return 0
except (FileExistsError, NGASServiceErrorException) as exc:
self._LOG.error(f"{exc}")
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
except (FileExistsError, NGASFetchError, NGASServiceErrorException):
raise
except AttributeError:
# (This error sometimes gets thrown after all files
# actually -have- been retrieved. I blame the NGAS API.)
@@ -191,13 +179,4 @@ class ParallelFetcher(BaseFetcher):
f"{self.num_files_expected} files expected, but only "
f"{self.num_files_retrieved} found"
)
self._exit_with_error(ReturnCode.CATASTROPHIC_REQUEST_ERROR)
return 0
except Exception as exc:
self._LOG.error(f"{exc}")
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
def _exit_with_error(self, return_code: ReturnCode):
sys.exit(return_code.value["code"])
raise NGASFetchError
Loading