Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ssa/workspaces
1 result
Show changes
Commits on Source (2)
Hide whitespace changes
Inline Side-by-side
Showing
with 644 additions and 790 deletions
apps/cli/executables/datafetcher/datafetcher/datafetcher.py
View file @ 71b97f8f
...@@ -2,24 +2,21 @@ ...@@ -2,24 +2,21 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" Module for the command line interface to data-fetcher. """ """ Module for the command line interface to data-fetcher. """
import logging import logging
import sys
from argparse import Namespace from argparse import Namespace
from pathlib import Path from pathlib import Path
# pylint: disable=C0103, E0402, E0611, R0902, R0903, W0703, W1203
from datafetcher.errors import MissingSettingsException, NoProfileException from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.project_fetcher import ParallelFetcher from datafetcher.project_fetcher import ParallelFetcher
from datafetcher.return_codes import ReturnCode
from .locations_report import LocationsReport from .locations_report import LocationsReport
from .utilities import get_arg_parser, get_capo_settings, FlexLogger, path_is_accessible from .utilities import get_arg_parser, get_capo_settings, path_is_accessible
# pylint: disable=C0103, W1203, R0902, R0903
_APPLICATION_NAME = "datafetcher" _APPLICATION_NAME = "datafetcher"
# pylint: disable=C0103, W0703 logger = logging.getLogger(__name__)
class DataFetcher: class DataFetcher:
...@@ -48,60 +45,53 @@ class DataFetcher: ...@@ -48,60 +45,53 @@ class DataFetcher:
def __init__(self, args: Namespace, df_capo_settings: dict): def __init__(self, args: Namespace, df_capo_settings: dict):
self.usage = self._build_usage_message() self.usage = self._build_usage_message()
if args is None or df_capo_settings is None: if args is None or df_capo_settings is None:
self._exit_with_error(ReturnCode.MISSING_SETTING) raise MissingSettingsException()
self.args = args self.args = args
self.settings = df_capo_settings self.settings = df_capo_settings
self.verbose = self.args.verbose try:
self.verbose = self.args.verbose
except AttributeError:
# we don't care; --verbose will be dropped later in WS-179
pass
# required arguments # required arguments
self.profile = args.profile
if self.profile is None:
raise NoProfileException()
self.output_dir = args.output_dir self.output_dir = args.output_dir
if self.output_dir is None: if self.output_dir is None:
self._exit_with_error(ReturnCode.MISSING_SETTING) raise MissingSettingsException("output directory option is missing")
output_dir = Path(self.output_dir) output_dir = Path(self.output_dir)
if not output_dir.is_dir() or not path_is_accessible(output_dir): if not output_dir.is_dir() or not path_is_accessible(output_dir):
logging.error(f"output location {self.output_dir} inaccessible " f"or not found") raise MissingSettingsException(
self._exit_with_error(ReturnCode.MISSING_SETTING) f"output location {self.output_dir} inaccessible or not found"
)
self._LOG = FlexLogger(self.__class__.__name__, self.output_dir, self.verbose)
if args.location_file is not None: if args.location_file is not None:
if args.product_locator is not None: if args.product_locator is not None:
self._LOG.error("required: location file OR product locator " "-- not both") raise MissingSettingsException(
self._exit_with_error(ReturnCode.MISSING_SETTING) "required: location file OR product locator -- not both"
)
self.location_file = args.location_file self.location_file = args.location_file
elif args.product_locator is not None: elif args.product_locator is not None:
self.product_locator = args.product_locator self.product_locator = args.product_locator
else: else:
self._LOG.error("you must specify either a location file or a " "product locator") raise MissingSettingsException(
self._exit_with_error(ReturnCode.MISSING_SETTING) "you must specify either a location file or a product locator"
)
self.profile = args.profile
if self.profile is None:
self._exit_with_error(ReturnCode.MISSING_PROFILE)
# optional arguments # optional arguments
self.is_dry = args.dry_run self.is_dry = args.dry_run
self.force = args.force self.force = args.force
self.verbose = args.verbose or False
try: try:
self.locations_report = self._get_locations() self.verbose = args.verbose or False
self.servers_report = self.locations_report.servers_report except AttributeError:
except SystemExit as exc: # verbose is going away soon
self._LOG.error(f"{exc}") self.verbose = False
if args.location_file:
self._exit_with_error(ReturnCode.CANNOT_OPEN_LOCATION_FILE) self.locations_report = self._get_locations()
else: self.servers_report = self.locations_report.servers_report
self._exit_with_error(ReturnCode.PRODUCT_LOCATOR_NOT_FOUND)
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected {type(exc)} during init: {exc}")
raise
def _exit_with_error(self, return_code: ReturnCode):
print(self.usage)
sys.exit(return_code.value["code"])
@staticmethod @staticmethod
def _build_usage_message() -> str: def _build_usage_message() -> str:
...@@ -110,9 +100,6 @@ class DataFetcher: ...@@ -110,9 +100,6 @@ class DataFetcher:
(--product-locator PRODUCT_LOCATOR | --location-file LOCATION_FILE) (--product-locator PRODUCT_LOCATOR | --location-file LOCATION_FILE)
[--dry-run] [--output-dir OUTPUT_DIR] [-v, --verbose] [--dry-run] [--output-dir OUTPUT_DIR] [-v, --verbose]
[--profile PROFILE]\n""" [--profile PROFILE]\n"""
usage_str += "\n\t\tReturn codes:\n"
for return_code in ReturnCode:
usage_str += f"\n\t\t\t{return_code.value['code']}: " f"{return_code.value['text']}"
return usage_str return usage_str
def run(self): def run(self):
...@@ -121,36 +108,12 @@ class DataFetcher: ...@@ -121,36 +108,12 @@ class DataFetcher:
:return: :return:
""" """
try: fetcher = ParallelFetcher(self.args, self.settings, self.servers_report)
fetcher = ParallelFetcher(self.args, self.settings, self._LOG, self.servers_report) fetcher.run()
return fetcher.run()
except SystemExit as exc:
self._LOG.error(f"{exc}")
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected exception during run: {exc}")
raise
def _get_locations(self): def _get_locations(self):
try: capo_settings = get_capo_settings(self.profile)
capo_settings = get_capo_settings(self.profile) return LocationsReport(self.args, capo_settings)
except MissingSettingsException as exc:
# if a setting couldn't be found, we can be pretty sure that
# the profile's no good
raise NoProfileException from exc
try:
return LocationsReport(self._LOG, self.args, capo_settings)
except Exception as exc:
self._LOG.error(f"{exc}")
if hasattr(self, "location_file"):
self._exit_with_error(ReturnCode.CANNOT_OPEN_LOCATION_FILE)
elif hasattr(self, "product_locator"):
self._exit_with_error(ReturnCode.PRODUCT_LOCATOR_NOT_FOUND)
else:
# should never happen; we've already checked
self._exit_with_error(ReturnCode.MISSING_SETTING)
def main(): def main():
...@@ -158,18 +121,12 @@ def main(): ...@@ -158,18 +121,12 @@ def main():
from the command line. from the command line.
""" """
parser = get_arg_parser() logging.basicConfig(level=logging.DEBUG)
try:
args = parser.parse_args() args = get_arg_parser().parse_args()
except Exception as exc:
logging.error(f"{exc}")
return exc.value
except SystemExit as exc:
logging.error(f"{exc}")
return exc.value.code
settings = get_capo_settings(args.profile) settings = get_capo_settings(args.profile)
datafetcher = DataFetcher(args, settings) DataFetcher(args, settings).run()
return datafetcher.run()
if __name__ == "__main__": if __name__ == "__main__":
......
apps/cli/executables/datafetcher/datafetcher/errors.py
View file @ 71b97f8f
""" Custom error definitions for data-fetcher """ """ Custom error definitions for data-fetcher """
import logging import logging
import sys
import traceback
from enum import Enum
_LOG = logging.getLogger(__name__) _LOG = logging.getLogger(__name__)
# pylint: disable=W1202 # pylint: disable=W1202
class Errors(Enum): errors = [
""" (1, "NoProfileException", "no CAPO profile provided"),
Assorted constants and functions involving errors (2, "MissingSettingsException", "missing required setting"),
(
""" 3,
"LocationServiceTimeoutException",
NO_PROFILE = 1 "request to locator service timed out",
MISSING_SETTING = 2 ),
LOCATION_SERVICE_TIMEOUT = 3 (
LOCATION_SERVICE_REDIRECTS = 4 4,
LOCATION_SERVICE_ERROR = 5 "LocationServiceRedirectsException",
NO_LOCATOR = 6 "too many redirects on locator service",
FILE_ERROR = 7 ),
NGAS_SERVICE_TIMEOUT = 8 (
NGAS_SERVICE_REDIRECTS = 9 5,
NGAS_SERVICE_ERROR = 10 "LocationServiceErrorException",
SIZE_MISMATCH = 11 "catastrophic error on locator service",
FILE_EXISTS_ERROR = 12 ),
FILE_NOT_FOUND_ERROR = 13 (6, "NoLocatorException", "product locator not found"),
(7, "FileErrorException", "not able to open specified location file"),
(8, "NGASServiceTimeoutException", "request to NGAS timed out"),
class NoProfileException(Exception): (
""" throw this if Capo profile can't be determined""" 9,
"NGASServiceRedirectsException",
"too many redirects on NGAS service",
class MissingSettingsException(Exception): ),
""" throw this if a required setting isn't found in command-line args """ (10, "NGASServiceErrorException", "catastrophic error on NGAS service"),
(11, "SizeMismatchException", "retrieved file not expected size"),
(12, "FileExistsError", "specified location file exists"),
class LocationServiceTimeoutException(Exception): (13, "FileNotFoundError", "target directory or file not found"),
""" throw this if the location service takes too long to return """ (14, "NGASFetchError", "trouble retrieving files from NGAS"),
]
class LocationServiceRedirectsException(Exception):
"""throw this if the location service class DataFetcherException(Exception):
complains about too many redirects code: int
""" message: str
class LocationServiceErrorException(Exception): def define_datafetcher_exception(code, name, message):
""" throw this if the location service falls over """ result = type(name, (DataFetcherException,), {})
result.code, result.message = code, message
class NGASServiceTimeoutException(Exception):
""" throw this if the NGAS service time out """
class NGASServiceRedirectsException(Exception):
""" throw this if the NGAS service complains about too many redirects """
class NGASServiceErrorException(Exception):
""" throw this if the NGAS service falls over """
class NoLocatorException(Exception):
"""throw this if no product locator could be determined from
command-line arguments
"""
class FileErrorException(Exception):
"""throw this if file retriever can't access or create the output
directory
"""
class SizeMismatchException(Exception):
"""throw this if the size of the file retrieved
doesn't match expected size
"""
TERMINAL_ERRORS = {
Errors.NO_PROFILE: "no CAPO profile provided",
Errors.MISSING_SETTING: "missing required setting",
Errors.LOCATION_SERVICE_TIMEOUT: "request to locator service timed out",
Errors.LOCATION_SERVICE_REDIRECTS: "too many redirects on locator service",
Errors.LOCATION_SERVICE_ERROR: "catastrophic error on locator service",
Errors.NO_LOCATOR: "product locator not found",
Errors.FILE_ERROR: "not able to open specified location file",
Errors.FILE_EXISTS_ERROR: "specified location file exists",
Errors.NGAS_SERVICE_TIMEOUT: "request to NGAS timed out",
Errors.NGAS_SERVICE_REDIRECTS: "too many redirects on NGAS service",
Errors.NGAS_SERVICE_ERROR: "catastrophic error on NGAS service",
Errors.SIZE_MISMATCH: "retrieved file not expected size",
Errors.FILE_NOT_FOUND_ERROR: "target directory or file not found",
}
def get_error_descriptions():
""" user-friendly reporting of errors """
result = "Return Codes:\n"
for error in Errors:
result += "\t{}: {}\n".format(error.value, TERMINAL_ERRORS[error])
return result return result
def terminal_error(errno): # Define the exceptions
"""report error, then throw in the towel""" NoProfileException = define_datafetcher_exception(*errors[0])
if errno in TERMINAL_ERRORS: MissingSettingsException = define_datafetcher_exception(*errors[1])
_LOG.error(TERMINAL_ERRORS[errno]) LocationServiceTimeoutException = define_datafetcher_exception(*errors[2])
else: LocationServiceRedirectsException = define_datafetcher_exception(*errors[3])
_LOG.error("unspecified error {}".format(errno)) LocationServiceErrorException = define_datafetcher_exception(*errors[4])
NoLocatorException = define_datafetcher_exception(*errors[5])
sys.exit(errno.value) FileErrorException = define_datafetcher_exception(*errors[6])
NGASServiceTimeoutException = define_datafetcher_exception(*errors[7])
NGASServiceRedirectsException = define_datafetcher_exception(*errors[8])
def exception_to_error(exception): NGASServiceErrorException = define_datafetcher_exception(*errors[9])
"""translate an exception to one of our custom errors""" SizeMismatchException = define_datafetcher_exception(*errors[10])
switcher = { FileExistsError = define_datafetcher_exception(*errors[11])
"NoProfileException": Errors.NO_PROFILE, FileNotFoundError = define_datafetcher_exception(*errors[12])
"MissingSettingsException": Errors.MISSING_SETTING, NGASFetchError = define_datafetcher_exception(*errors[13])
"LocationServiceTimeoutException": Errors.LOCATION_SERVICE_TIMEOUT,
"LocationServiceRedirectsException": Errors.LOCATION_SERVICE_REDIRECTS,
"LocationServiceErrorException": Errors.LOCATION_SERVICE_ERROR, TERMINAL_ERROR_CODES = "Return Codes:\n"
"NoLocatorException": Errors.NO_LOCATOR, for error_code, exception_name, message in errors:
"FileErrorException": Errors.FILE_ERROR, TERMINAL_ERROR_CODES += f"\t{error_code}: {message}\n"
"FileExistsError": Errors.FILE_EXISTS_ERROR,
"NGASServiceTimeoutException": Errors.NGAS_SERVICE_TIMEOUT,
"NGASServiceRedirectsException": Errors.NGAS_SERVICE_REDIRECTS,
"NGASServiceErrorException": Errors.NGAS_SERVICE_ERROR,
"SizeMismatchException": Errors.SIZE_MISMATCH,
"FileNotFoundError": Errors.FILE_NOT_FOUND_ERROR,
}
return switcher.get(exception.__class__.__name__)
def terminal_exception(exception):
"""Report this exception, then throw in the towel.
should be used by DataFetcher -ONLY-
"""
errorno = exception_to_error(exception)
_LOG.debug(traceback.format_exc())
_LOG.error(str(exception))
sys.exit(errorno.value)
apps/cli/executables/datafetcher/datafetcher/file_retrievers.py
View file @ 71b97f8f
...@@ -4,10 +4,13 @@ ...@@ -4,10 +4,13 @@
Implementations of assorted file retrievers. Implementations of assorted file retrievers.
""" """
import http import http
import logging
import os import os
from argparse import Namespace from argparse import Namespace
from pathlib import Path from pathlib import Path
# pylint: disable=C0103, E0401, E0402, R0903, R0914, W1202, W1203
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
...@@ -17,21 +20,21 @@ from .errors import ( ...@@ -17,21 +20,21 @@ from .errors import (
FileErrorException, FileErrorException,
MissingSettingsException, MissingSettingsException,
) )
from .utilities import RetrievalMode, Retryer, MAX_TRIES, SLEEP_INTERVAL_SECONDS, FlexLogger from .utilities import RetrievalMode, Retryer, MAX_TRIES, SLEEP_INTERVAL_SECONDS
_DIRECT_COPY_PLUGIN = "ngamsDirectCopyDppi" _DIRECT_COPY_PLUGIN = "ngamsDirectCopyDppi"
_STREAMING_CHUNK_SIZE = 8192 _STREAMING_CHUNK_SIZE = 8192
# pylint: disable=C0103, R0903, R0914 logger = logging.getLogger(__name__)
class NGASFileRetriever: class NGASFileRetriever:
"""Responsible for getting a file out of NGAS """Responsible for getting a file out of NGAS
and saving it to the requested location. and saving it to the requested location.
""" """
def __init__(self, args: Namespace, logger: FlexLogger): def __init__(self, args: Namespace):
self.output_dir = args.output_dir self.output_dir = args.output_dir
self._LOG = logger
self.logfile = self._LOG.logfile
self.dry_run = args.dry_run self.dry_run = args.dry_run
self.force_overwrite = args.force self.force_overwrite = args.force
self.fetch_attempted = False self.fetch_attempted = False
...@@ -56,7 +59,7 @@ class NGASFileRetriever: ...@@ -56,7 +59,7 @@ class NGASFileRetriever:
func = ( func = (
self._copying_fetch if retrieve_method == RetrievalMode.COPY else self._streaming_fetch self._copying_fetch if retrieve_method == RetrievalMode.COPY else self._streaming_fetch
) )
retryer = Retryer(func, MAX_TRIES, SLEEP_INTERVAL_SECONDS, self._LOG) retryer = Retryer(func, MAX_TRIES, SLEEP_INTERVAL_SECONDS)
try: try:
retryer.retry(download_url, destination, file_spec) retryer.retry(download_url, destination, file_spec)
finally: finally:
...@@ -113,7 +116,7 @@ class NGASFileRetriever: ...@@ -113,7 +116,7 @@ class NGASFileRetriever:
:param file_spec: the file specification of that file :param file_spec: the file specification of that file
""" """
if not self.dry_run: if not self.dry_run:
self._LOG.debug(f"verifying fetch of {destination}") logger.debug(f"verifying fetch of {destination}")
if not destination.exists(): if not destination.exists():
raise NGASServiceErrorException(f"file not delivered to {destination}") raise NGASServiceErrorException(f"file not delivered to {destination}")
if file_spec["size"] != os.path.getsize(destination): if file_spec["size"] != os.path.getsize(destination):
...@@ -122,9 +125,9 @@ class NGASFileRetriever: ...@@ -122,9 +125,9 @@ class NGASFileRetriever:
f"expected {file_spec['size']}, " f"expected {file_spec['size']}, "
f"got {os.path.getsize(destination)}" f"got {os.path.getsize(destination)}"
) )
self._LOG.debug("\tlooks good; sizes match") logger.debug("\tlooks good; sizes match")
else: else:
self._LOG.debug("(This was a dry run; no files should have been fetched)") logger.debug("(This was a dry run; no files should have been fetched)")
def _copying_fetch(self, args: list): def _copying_fetch(self, args: list):
"""Pull a file out of NGAS via the direct copy plugin. """Pull a file out of NGAS via the direct copy plugin.
...@@ -138,7 +141,7 @@ class NGASFileRetriever: ...@@ -138,7 +141,7 @@ class NGASFileRetriever:
"processingPars": "outfile=" + str(destination), "processingPars": "outfile=" + str(destination),
"file_version": file_spec["version"], "file_version": file_spec["version"],
} }
self._LOG.debug( logger.debug(
"attempting copying download:\nurl: {}\ndestination: {}".format( "attempting copying download:\nurl: {}\ndestination: {}".format(
download_url, destination download_url, destination
) )
...@@ -148,7 +151,7 @@ class NGASFileRetriever: ...@@ -148,7 +151,7 @@ class NGASFileRetriever:
response = session.get(download_url, params=params) response = session.get(download_url, params=params)
if response.status_code != http.HTTPStatus.OK: if response.status_code != http.HTTPStatus.OK:
self._LOG.error("NGAS does not like this request:\n{}".format(response.url)) logger.error("NGAS does not like this request:\n{}".format(response.url))
soup = BeautifulSoup(response.text, features="lxml") soup = BeautifulSoup(response.text, features="lxml")
ngams_status = soup.NgamsStatus.Status ngams_status = soup.NgamsStatus.Status
message = ngams_status.get("Message") message = ngams_status.get("Message")
...@@ -163,7 +166,7 @@ class NGASFileRetriever: ...@@ -163,7 +166,7 @@ class NGASFileRetriever:
) )
else: else:
self._LOG.debug( logger.debug(
f"if this were not a dry run, we would have been copying " f"if this were not a dry run, we would have been copying "
f'{file_spec["relative_path"]}' f'{file_spec["relative_path"]}'
) )
...@@ -180,7 +183,7 @@ class NGASFileRetriever: ...@@ -180,7 +183,7 @@ class NGASFileRetriever:
download_url, destination, file_spec = args download_url, destination, file_spec = args
params = {"file_id": file_spec["ngas_file_id"], "file_version": file_spec["version"]} params = {"file_id": file_spec["ngas_file_id"], "file_version": file_spec["version"]}
self._LOG.debug( logger.debug(
"attempting streaming download:\nurl: {}\ndestination: {}".format( "attempting streaming download:\nurl: {}\ndestination: {}".format(
download_url, destination download_url, destination
) )
...@@ -191,15 +194,12 @@ class NGASFileRetriever: ...@@ -191,15 +194,12 @@ class NGASFileRetriever:
try: try:
response = session.get(download_url, params=params, stream=True) response = session.get(download_url, params=params, stream=True)
with open(destination, "wb") as file_to_write: with open(destination, "wb") as file_to_write:
# a word to the wise: DO NOT try to assign chunk size
# to a variable or to make it a constant. This will
# result in an incomplete download.
for chunk in response.iter_content(chunk_size=_STREAMING_CHUNK_SIZE): for chunk in response.iter_content(chunk_size=_STREAMING_CHUNK_SIZE):
file_to_write.write(chunk) file_to_write.write(chunk)
expected_size = file_spec["size"] expected_size = file_spec["size"]
actual_size = os.path.getsize(destination) actual_size = os.path.getsize(destination)
if actual_size == 0: if actual_size == 0:
raise FileErrorException(f"{Path(destination).name} " f"was not retrieved") raise FileErrorException(f"{Path(destination).name} was not retrieved")
if actual_size != expected_size: if actual_size != expected_size:
raise SizeMismatchException( raise SizeMismatchException(
f"expected {Path(destination).name} " f"expected {Path(destination).name} "
...@@ -212,10 +212,10 @@ class NGASFileRetriever: ...@@ -212,10 +212,10 @@ class NGASFileRetriever:
f"problem connecting with {download_url}" f"problem connecting with {download_url}"
) from c_err ) from c_err
except AttributeError as a_err: except AttributeError as a_err:
self._LOG.warning(f"possible problem streaming: {a_err}") logger.warning(f"possible problem streaming: {a_err}")
if response.status_code != http.HTTPStatus.OK: if response.status_code != http.HTTPStatus.OK:
self._LOG.error("NGAS does not like this request:\n{}".format(response.url)) logger.error("NGAS does not like this request:\n{}".format(response.url))
soup = BeautifulSoup(response.text, "lxml-xml") soup = BeautifulSoup(response.text, "lxml-xml")
ngams_status = soup.NgamsStatus.Status ngams_status = soup.NgamsStatus.Status
message = ngams_status.get("Message") message = ngams_status.get("Message")
...@@ -229,10 +229,10 @@ class NGASFileRetriever: ...@@ -229,10 +229,10 @@ class NGASFileRetriever:
} }
) )
self._LOG.debug("retrieved {} from {}".format(destination, response.url)) logger.debug(f"retrieved {destination} from {response.url}")
else: else:
self._LOG.debug( logger.debug(
f"if this were not a dry run, we would have been streaming " f"if this were not a dry run, we would have been streaming "
f'{file_spec["relative_path"]}' f'{file_spec["relative_path"]}'
) )
......
apps/cli/executables/datafetcher/datafetcher/locations_report.py
View file @ 71b97f8f
...@@ -6,14 +6,18 @@ ...@@ -6,14 +6,18 @@
""" """
# pylint: disable=C0103, C0301, E0401, E0402, R0902, R0903, R0914, W0703, W1202, W1203
import copy import copy
import http import http
import json import json
import logging
from argparse import Namespace from argparse import Namespace
from typing import Dict from typing import Dict
import requests import requests
from . import errors
from .errors import ( from .errors import (
LocationServiceTimeoutException, LocationServiceTimeoutException,
LocationServiceRedirectsException, LocationServiceRedirectsException,
...@@ -21,27 +25,29 @@ from .errors import ( ...@@ -21,27 +25,29 @@ from .errors import (
NoLocatorException, NoLocatorException,
MissingSettingsException, MissingSettingsException,
) )
from .utilities import Cluster, RetrievalMode, validate_file_spec, FlexLogger from .utilities import Cluster, RetrievalMode, validate_file_spec
# pylint: disable=C0103, R0902, R0903, R0914, W0703 logger = logging.getLogger(__name__)
class LocationsReport: class LocationsReport:
""" Builds a location report """ """ Builds a location report """
def __init__(self, logger: FlexLogger, args: Namespace, settings: Dict): def __init__(self, args: Namespace, settings: Dict):
self._LOG = logger
self.logfile = logger.logfile try:
self._verbose = args and args.verbose self.verbose = args.verbose or False
if self._verbose: except AttributeError:
logger.set_verbose(True) # doesn't matter; verbose is going away soon
self.verbose = False
self._capture_and_validate_input(args, settings) self._capture_and_validate_input(args, settings)
self._run() self._run()
def _capture_and_validate_input(self, args, settings): def _capture_and_validate_input(self, args, settings):
if args is None: if args is None:
raise MissingSettingsException( raise MissingSettingsException(
"arguments (locator and/or report file, destination) " "are required" "arguments (locator and/or report file, destination) are required"
) )
self.args = args self.args = args
if settings is None: if settings is None:
...@@ -139,15 +145,14 @@ class LocationsReport: ...@@ -139,15 +145,14 @@ class LocationsReport:
to pull in the location report. to pull in the location report.
""" """
self._LOG.debug(f'fetching files from report "{self.location_file}"') logger.debug(f'fetching files from report "{self.location_file}"')
# try: # try:
try: try:
with open(self.location_file) as to_read: with open(self.location_file) as to_read:
result = json.load(to_read) result = json.load(to_read)
return result return result
except FileNotFoundError as err: except FileNotFoundError as err:
self._LOG.error(f"{err}") raise errors.FileNotFoundError() from err
raise
def _get_location_report_from_service(self): def _get_location_report_from_service(self):
"""Use 'requests' to fetch the location report from the locator service. """Use 'requests' to fetch the location report from the locator service.
...@@ -156,7 +161,7 @@ class LocationsReport: ...@@ -156,7 +161,7 @@ class LocationsReport:
""" """
url = self.settings["locator_service_url"] url = self.settings["locator_service_url"]
self._LOG.debug("fetching report from {} for {}".format(url, self.product_locator)) logger.debug("fetching report from {} for {}".format(url, self.product_locator))
# this is needed to prevent SSL errors when tests are run # this is needed to prevent SSL errors when tests are run
# inside a Docker container # inside a Docker container
...@@ -172,8 +177,6 @@ class LocationsReport: ...@@ -172,8 +177,6 @@ class LocationsReport:
raise LocationServiceRedirectsException() from exc_re raise LocationServiceRedirectsException() from exc_re
except requests.exceptions.RequestException as ex: except requests.exceptions.RequestException as ex:
raise LocationServiceErrorException(ex) from ex raise LocationServiceErrorException(ex) from ex
except Exception as exc:
self._LOG.error(f"{exc}")
if response.status_code == http.HTTPStatus.OK: if response.status_code == http.HTTPStatus.OK:
return response.json() return response.json()
......
apps/cli/executables/datafetcher/datafetcher/locations_report_refactor.py
View file @ 71b97f8f
...@@ -30,8 +30,7 @@ from .utilities import Cluster, RetrievalMode, validate_file_spec, get_arg_parse ...@@ -30,8 +30,7 @@ from .utilities import Cluster, RetrievalMode, validate_file_spec, get_arg_parse
# pylint: disable=C0103, R0902, R0903, R0914, W0703, W1203 # pylint: disable=C0103, R0902, R0903, R0914, W0703, W1203
_LOG = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
REQUIRED_SETTINGS = { REQUIRED_SETTINGS = {
"EDU.NRAO.ARCHIVE.DATAFETCHER.DATAFETCHERSETTINGS.LOCATORSERVICEURLPREFIX": "locator_service_url", "EDU.NRAO.ARCHIVE.DATAFETCHER.DATAFETCHERSETTINGS.LOCATORSERVICEURLPREFIX": "locator_service_url",
...@@ -79,7 +78,7 @@ class LocationsReportRefactor: ...@@ -79,7 +78,7 @@ class LocationsReportRefactor:
return self._add_retrieve_method_field(result) return self._add_retrieve_method_field(result)
except JSONDecodeError as js_err: except JSONDecodeError as js_err:
_LOG.error(f"Unable to parse {self.location_file}") logger.error(f"Unable to parse {self.location_file}")
raise ReportFileParseException from js_err raise ReportFileParseException from js_err
def _get_location_report_from_file(self) -> Dict[str, str]: def _get_location_report_from_file(self) -> Dict[str, str]:
...@@ -88,19 +87,19 @@ class LocationsReportRefactor: ...@@ -88,19 +87,19 @@ class LocationsReportRefactor:
:return: :return:
""" """
_LOG.debug(f'fetching files from report "{self.location_file}"') logger.debug(f'fetching files from report "{self.location_file}"')
try: try:
with open(self.location_file) as to_read: with open(self.location_file) as to_read:
result = json.load(to_read) result = json.load(to_read)
return result return result
except FileNotFoundError as err: except FileNotFoundError as err:
_LOG.error(f"{err}") logger.error(f"{err}")
raise raise
except JSONDecodeError as js_err: except JSONDecodeError as js_err:
_LOG.error(f"Unable to parse {self.location_file}") logger.error(f"Unable to parse {self.location_file}")
raise ReportFileParseException from js_err raise ReportFileParseException from js_err
except Exception as exc: except Exception as exc:
_LOG.error(f"Problem getting location report from f{self.location_file}: {exc}") logger.error(f"Problem getting location report from f{self.location_file}: {exc}")
raise raise
def _get_location_report_from_service(self): def _get_location_report_from_service(self):
...@@ -112,7 +111,7 @@ class LocationsReportRefactor: ...@@ -112,7 +111,7 @@ class LocationsReportRefactor:
"edu.nrao.archive.datafetcher.DataFetcherSettings.locatorServiceUrlPrefix" "edu.nrao.archive.datafetcher.DataFetcherSettings.locatorServiceUrlPrefix"
) )
_LOG.debug(f"fetching report from {url} for {self.product_locator}") logger.debug(f"fetching report from {url} for {self.product_locator}")
# this is needed to prevent SSL errors when tests are run # this is needed to prevent SSL errors when tests are run
# inside a Docker container # inside a Docker container
...@@ -129,7 +128,7 @@ class LocationsReportRefactor: ...@@ -129,7 +128,7 @@ class LocationsReportRefactor:
except requests.exceptions.RequestException as ex: except requests.exceptions.RequestException as ex:
raise LocationServiceErrorException(ex) from ex raise LocationServiceErrorException(ex) from ex
except Exception as exc: except Exception as exc:
_LOG.error(f"{exc}") logger.error(f"{exc}")
if response.status_code == http.HTTPStatus.OK: if response.status_code == http.HTTPStatus.OK:
return response.json() return response.json()
......
apps/cli/executables/datafetcher/datafetcher/project_fetcher.py
View file @ 71b97f8f
...@@ -3,50 +3,46 @@ ...@@ -3,50 +3,46 @@
""" Implementations of assorted product fetchers """ """ Implementations of assorted product fetchers """
import copy import copy
import sys import logging
from argparse import Namespace from argparse import Namespace
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor
from pathlib import Path from pathlib import Path
from pprint import pprint
from typing import Dict from typing import Dict
from datafetcher.errors import NGASServiceErrorException # pylint: disable=C0103, E0402, R0201, R0902, R0903, W0703, W1202, W1203
from datafetcher.return_codes import ReturnCode
from datafetcher.errors import NGASFetchError
from .file_retrievers import NGASFileRetriever from .file_retrievers import NGASFileRetriever
from .utilities import FlexLogger
# pylint: disable=C0103, R0201, R0902, R0903, W0703 logger = logging.getLogger(__name__)
class BaseFetcher: class BaseFetcher:
""" This is a base class for fetchers. """ """ This is a base class for fetchers. """
def __init__( def __init__(self, args: Namespace, df_capo_settings: dict, servers_report: dict):
self, args: Namespace, df_capo_settings: dict, logger: FlexLogger, servers_report: dict
):
self.args = args self.args = args
self.output_dir = self.args.output_dir self.output_dir = self.args.output_dir
self._LOG = logger
self.force_overwrite = args.force self.force_overwrite = args.force
self.dry_run = args.dry_run self.dry_run = args.dry_run
self.servers_report = servers_report self.servers_report = servers_report
self.settings = df_capo_settings self.settings = df_capo_settings
self.ngas_retriever = NGASFileRetriever(self.args, self._LOG) self.ngas_retriever = NGASFileRetriever(self.args)
self.retrieved = [] self.retrieved = []
self.num_files_retrieved = 0 self.num_files_retrieved = 0
def retrieve_files(self, server, retrieve_method, file_specs): def retrieve_files(self, server, retrieve_method, file_specs):
""" This is the part where we actually fetch the files. """ """ This is the part where we actually fetch the files. """
retriever = NGASFileRetriever(self.args, self._LOG) retriever = NGASFileRetriever(self.args)
num_files = len(file_specs) num_files = len(file_specs)
count = 0 count = 0
for file_spec in file_specs: for file_spec in file_specs:
count += 1 count += 1
self._LOG.debug( logger.debug(
f">>> retrieving {file_spec['relative_path']} " f">>> retrieving {file_spec['relative_path']} "
f"({file_spec['size']} bytes, " f"({file_spec['size']} bytes, "
f"no. {count} of {num_files})...." f"no. {count} of {num_files})...."
...@@ -62,17 +58,15 @@ class SerialFetcher(BaseFetcher): ...@@ -62,17 +58,15 @@ class SerialFetcher(BaseFetcher):
""" """
def __init__( def __init__(self, args: Namespace, df_capo_settings: Dict, servers_report: Dict):
self, args: Namespace, df_capo_settings: Dict, logger: FlexLogger, servers_report: Dict super().__init__(args, df_capo_settings, servers_report)
):
super().__init__(args, df_capo_settings, logger, servers_report)
def run(self): def run(self):
""" fetch 'em """ """ fetch 'em """
self._LOG.debug("writing to {}".format(self.output_dir)) logger.debug("writing to {}".format(self.output_dir))
self._LOG.debug("dry run: {}".format(self.dry_run)) logger.debug("dry run: {}".format(self.dry_run))
self._LOG.debug(f"force overwrite: {self.force_overwrite}") logger.debug(f"force overwrite: {self.force_overwrite}")
for server in self.servers_report: for server in self.servers_report:
self.retrieve_files( self.retrieve_files(
server, server,
...@@ -84,10 +78,8 @@ class SerialFetcher(BaseFetcher): ...@@ -84,10 +78,8 @@ class SerialFetcher(BaseFetcher):
class ParallelFetcher(BaseFetcher): class ParallelFetcher(BaseFetcher):
""" Pull the files out in parallel; try to be clever about it. """ """ Pull the files out in parallel; try to be clever about it. """
def __init__( def __init__(self, args: Namespace, df_capo_settings: dict, servers_report: dict):
self, args: Namespace, df_capo_settings: dict, logger: FlexLogger, servers_report: dict super().__init__(args, df_capo_settings, servers_report)
):
super().__init__(args, df_capo_settings, logger, servers_report)
self.num_files_expected = self._count_files_expected() self.num_files_expected = self._count_files_expected()
self.bucketized_files = self._bucketize_files() self.bucketized_files = self._bucketize_files()
...@@ -134,8 +126,7 @@ class ParallelFetcher(BaseFetcher): ...@@ -134,8 +126,7 @@ class ParallelFetcher(BaseFetcher):
def fetch_bucket(self, bucket): def fetch_bucket(self, bucket):
""" Grab the files in this bucket """ """ Grab the files in this bucket """
file_sizes = [file["size"] for file in bucket["files"]] file_sizes = [file["size"] for file in bucket["files"]]
pprint(f"retrieving files {file_sizes} from {bucket['server']}") logger.debug(
self._LOG.debug(
f"{bucket['retrieve_method']} " f"{bucket['retrieve_method']} "
f"{len(bucket['files'])} files from " f"{len(bucket['files'])} files from "
f"{bucket['server']}...." f"{bucket['server']}...."
...@@ -149,10 +140,8 @@ class ParallelFetcher(BaseFetcher): ...@@ -149,10 +140,8 @@ class ParallelFetcher(BaseFetcher):
def run(self): def run(self):
""" Fetch all the files for the product locator """ """ Fetch all the files for the product locator """
print(f"running {self.__class__.__name__}")
if self.args.dry_run: if self.args.dry_run:
self._LOG.debug("This is a dry run; files will not be fetched") logger.debug("This is a dry run; files will not be fetched")
return 0
with ThreadPoolExecutor() as executor: with ThreadPoolExecutor() as executor:
results = executor.map(self.fetch_bucket, self.bucketized_files) results = executor.map(self.fetch_bucket, self.bucketized_files)
...@@ -162,21 +151,13 @@ class ParallelFetcher(BaseFetcher): ...@@ -162,21 +151,13 @@ class ParallelFetcher(BaseFetcher):
print(f"result has arrived: {result}") print(f"result has arrived: {result}")
self.num_files_retrieved += result self.num_files_retrieved += result
if self.num_files_retrieved != self.num_files_expected: if self.num_files_retrieved != self.num_files_expected:
self._LOG.error( logger.error(
f"{self.num_files_expected} files expected, " f"{self.num_files_expected} files expected, "
f"but only {self.num_files_retrieved} retrieved" f"but only {self.num_files_retrieved} retrieved"
) )
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
# successful retrieval
print("returning")
return 0
except (FileExistsError, NGASServiceErrorException) as exc:
self._LOG.error(f"{exc}")
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
except AttributeError: except AttributeError:
# (This error sometimes gets thrown after all files # (This error sometimes gets thrown after all files
# actually -have- been retrieved. I blame the NGAS API.) # actually -have- been retrieved. I blame the NGAS API. - JLG)
output_path = Path(self.args.output_dir) output_path = Path(self.args.output_dir)
files = [ files = [
...@@ -187,17 +168,8 @@ class ParallelFetcher(BaseFetcher): ...@@ -187,17 +168,8 @@ class ParallelFetcher(BaseFetcher):
and not str(file).endswith(".log") and not str(file).endswith(".log")
] ]
if len(files) < self.num_files_expected: if len(files) < self.num_files_expected:
self._LOG.error( logger.error(
f"{self.num_files_expected} files expected, but only " f"{self.num_files_expected} files expected, but only "
f"{self.num_files_retrieved} found" f"{self.num_files_retrieved} found"
) )
self._exit_with_error(ReturnCode.CATASTROPHIC_REQUEST_ERROR) raise NGASFetchError
return 0
except Exception as exc:
self._LOG.error(f"{exc}")
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
def _exit_with_error(self, return_code: ReturnCode):
sys.exit(return_code.value["code"])
apps/cli/executables/datafetcher/datafetcher/return_codes.py
View file @ 71b97f8f
""" data-fetcher return codes as specified in README and usage string """
from enum import Enum
class ReturnCode(Enum):
""" Canonical data-fetcher exit code values """
SUCCESS = {"code": 0, "text": "success"}
MISSING_PROFILE = {"code": 1, "text": "no CAPO profile provided"}
MISSING_SETTING = {"code": 2, "text": "missing required setting"}
LOCATOR_SERVICE_TIMEOUT = {"code": 3, "text": "request to locator service timed out"}
TOO_MANY_SERVICE_REDIRECTS = {"code": 4, "text": "too many redirects on locator service"}
CATASTROPHIC_REQUEST_ERROR = {"code": 5, "text": "catastrophic error on request service"}
PRODUCT_LOCATOR_NOT_FOUND = {"code": 6, "text": "product locator not found"}
CANNOT_OPEN_LOCATION_FILE = {"code": 7, "text": "not able to open specified location file"}
NGAS_FETCH_ERROR = {"code": 8, "text": "error fetching file from NGAS server"}
SIZE_MISMATCH = {"code": 9, "text": "retrieved file not expected size"}
apps/cli/executables/datafetcher/datafetcher/utilities.py
View file @ 71b97f8f
...@@ -8,28 +8,26 @@ ...@@ -8,28 +8,26 @@
""" """
import argparse import argparse
from enum import Enum
import logging import logging
import os import os
import pathlib import pathlib
from enum import Enum import time
from time import time
from typing import Callable from typing import Callable
# pylint:disable=C0301, C0303, C0415, E0401, E0402, R0903, W0212, W1202, W0404, W0621, W1203
import psycopg2 as pg import psycopg2 as pg
from pycapo import CapoConfig from pycapo import CapoConfig
from .errors import ( from .errors import (
get_error_descriptions,
NoProfileException, NoProfileException,
MissingSettingsException, MissingSettingsException,
NGASServiceErrorException, NGASServiceErrorException,
SizeMismatchException, SizeMismatchException,
TERMINAL_ERROR_CODES,
) )
# pylint:disable=C0303, C0415, R0903, W0212, W0404, W0621, W1203
LOG_FORMAT = "%(module)s.%(funcName)s, %(lineno)d: %(message)s" LOG_FORMAT = "%(module)s.%(funcName)s, %(lineno)d: %(message)s"
MAX_TRIES = 10 MAX_TRIES = 10
...@@ -50,7 +48,7 @@ SERVER_SPEC_KEYS = ["server", "location", "cluster", "retrieve_method"] ...@@ -50,7 +48,7 @@ SERVER_SPEC_KEYS = ["server", "location", "cluster", "retrieve_method"]
_PROLOGUE = """Retrieve a product (a science product or an ancillary product) _PROLOGUE = """Retrieve a product (a science product or an ancillary product)
from the NRAO archive, either by specifying the product's locator or by from the NRAO archive, either by specifying the product's locator or by
providing the path to a product locator report.""" providing the path to a product locator report."""
_EPILOGUE = get_error_descriptions() _EPILOGUE = TERMINAL_ERROR_CODES
# This is a dictionary of required CAPO settings # This is a dictionary of required CAPO settings
# and the attribute names we'll store them as. # and the attribute names we'll store them as.
...@@ -61,6 +59,8 @@ REQUIRED_SETTINGS = { ...@@ -61,6 +59,8 @@ REQUIRED_SETTINGS = {
"EDU.NRAO.ARCHIVE.WORKFLOW.CONFIG.REQUESTHANDLERSETTINGS.DOWNLOADDIRECTORY": "download_dir", "EDU.NRAO.ARCHIVE.WORKFLOW.CONFIG.REQUESTHANDLERSETTINGS.DOWNLOADDIRECTORY": "download_dir",
} }
logger = logging.getLogger(__name__)
def path_is_accessible(path: pathlib.Path) -> bool: def path_is_accessible(path: pathlib.Path) -> bool:
""" """
...@@ -88,7 +88,9 @@ def get_arg_parser(): ...@@ -88,7 +88,9 @@ def get_arg_parser():
cwd = pathlib.Path().absolute() cwd = pathlib.Path().absolute()
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=_PROLOGUE, epilog=_EPILOGUE, formatter_class=argparse.RawTextHelpFormatter description=_PROLOGUE,
epilog=_EPILOGUE,
formatter_class=argparse.RawTextHelpFormatter,
) )
# Can't find a way of clearing the action groups # Can't find a way of clearing the action groups
# without hitting an internal attribute. # without hitting an internal attribute.
...@@ -121,14 +123,14 @@ def get_arg_parser(): ...@@ -121,14 +123,14 @@ def get_arg_parser():
default=cwd, default=cwd,
help="output directory, default current dir", help="output directory, default current dir",
) )
optional_group.add_argument( # optional_group.add_argument(
"--verbose", # "--verbose",
action="store_true", # action="store_true",
required=False, # required=False,
dest="verbose", # dest="verbose",
default=False, # default=False,
help="make a lot of noise", # help="make a lot of noise",
) # )
optional_group.add_argument( optional_group.add_argument(
"--force", "--force",
action="store_true", action="store_true",
...@@ -160,19 +162,19 @@ def get_capo_settings(profile: str): ...@@ -160,19 +162,19 @@ def get_capo_settings(profile: str):
:param profile: the profile to use :param profile: the profile to use
:return: a bunch of settings :return: a bunch of settings
""" """
result = dict() if not profile:
if profile is None:
raise NoProfileException("CAPO_PROFILE required; none provided") raise NoProfileException("CAPO_PROFILE required; none provided")
capo = CapoConfig(profile=profile) capo = CapoConfig(profile=profile)
result = dict()
for setting in REQUIRED_SETTINGS: for setting in REQUIRED_SETTINGS:
setting = setting.upper() setting = setting.upper()
try: try:
value = capo[setting] result[REQUIRED_SETTINGS[setting]] = capo[setting]
except KeyError as k_err: except KeyError as k_err:
raise MissingSettingsException( raise MissingSettingsException(
'missing required setting "{}"'.format(setting) f'missing required setting "{setting}" with profile "{profile}"'
) from k_err ) from k_err
result[REQUIRED_SETTINGS[setting]] = value
return result return result
...@@ -226,9 +228,7 @@ class ProductLocatorLookup: ...@@ -226,9 +228,7 @@ class ProductLocatorLookup:
password=self.credentials["jdbcPassword"], password=self.credentials["jdbcPassword"],
) as conn: ) as conn:
cursor = conn.cursor() cursor = conn.cursor()
sql = ( sql = "SELECT science_product_locator FROM science_products WHERE external_name=%s"
"SELECT science_product_locator " "FROM science_products " "WHERE external_name=%s"
)
cursor.execute( cursor.execute(
sql, sql,
(external_name,), (external_name,),
...@@ -237,75 +237,12 @@ class ProductLocatorLookup: ...@@ -237,75 +237,12 @@ class ProductLocatorLookup:
return product_locator[0] return product_locator[0]
class FlexLogger:
"""
This class wraps a logger, adding the ability to specify
logging level as warning or debug based on the "verbose" flag
"""
def __init__(self, class_name: str, output_dir: pathlib.Path, verbose=False):
"""
Set up logging.
:param class_name: class to be logged
:param output_dir: where log is to be written
:param verbose: if true, additional output
"""
if class_name is None:
raise MissingSettingsException("class name is required")
log_pathname = f"{output_dir}/{class_name}_{str(time())}.log"
try:
self.logfile = pathlib.Path(output_dir, log_pathname)
self.logger = logging.getLogger(str(self.logfile))
self.verbose = verbose
handler = logging.FileHandler(self.logfile)
formatter = logging.Formatter(LOG_FORMAT)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
except (PermissionError, FileNotFoundError) as err:
self.logger.error(f"Problem creating logger for {class_name}: {err}")
raise
level = logging.DEBUG if self.verbose else logging.WARN
self.logger.setLevel(level)
def set_verbose(self, verbose: bool) -> None:
"""
Specify verbose logging:
True == debug and above
False == warnings and above
:param verbose:
:return:
"""
self.verbose = verbose
def debug(self, message: str):
""" log a debug message """
if self.verbose:
self.logger.debug(message)
def warning(self, message: str):
""" log a warning message """
self.logger.warning(message)
def error(self, message: str):
""" log an error """
self.logger.error(message)
class Retryer: class Retryer:
""" """
Retry executing a function, or die trying Retry executing a function, or die trying
""" """
def __init__(self, func: Callable, max_tries: int, sleep_interval: int, logger: FlexLogger): def __init__(self, func: Callable, max_tries: int, sleep_interval: int):
self.func = func self.func = func
self.num_tries = 0 self.num_tries = 0
self.max_tries = max_tries self.max_tries = max_tries
...@@ -320,9 +257,6 @@ class Retryer: ...@@ -320,9 +257,6 @@ class Retryer:
:param args: :param args:
:return: :return:
""" """
import time
while self.num_tries < self.max_tries and not self.complete: while self.num_tries < self.max_tries and not self.complete:
self.num_tries += 1 self.num_tries += 1
......
apps/cli/executables/datafetcher/test/df_pytest_arg_utils.py 0 → 100644
View file @ 71b97f8f
""" Utilities for passing arguments to datafetcher in tests """
import os
from pathlib import Path
from typing import List
from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.utilities import get_arg_parser, path_is_accessible
def parse_args(args: List[str]):
"""
Wrapper around parser.parse_args() so in the event of a foo
an exception is thrown rather than sys.exe
:param args:
:return:
"""
confirm_complete_args(args)
get_arg_parser().parse_args(args)
def confirm_complete_args(args: List[str]):
"""
Let's scrutinize the args -before- calling parse_args()
so we can differentiate among errors.
:param args:
:return:
"""
# we must have a profile
assert "--profile" in args or "CAPO_PROFILE" in os.environ
if "--profile" not in args and "CAPO_PROFILE" in os.environ:
raise NoProfileException("Capo profile is required.")
# we must have an output dir....
assert "--output-dir" in args
# ... and it must exist
args_iter = iter(args)
args_dict = dict(zip(args_iter, args_iter))
output_dir_arg = args_dict["--output-dir"]
output_dir = Path(output_dir_arg)
if not output_dir.exists():
# did they just forget to specify it?
if output_dir_arg.startswith("--"):
raise MissingSettingsException("--output-dir is required")
raise FileNotFoundError(f"output dir '{output_dir_arg}' not found")
if not path_is_accessible(output_dir):
raise FileNotFoundError(f"permission denied on {output_dir}")
# we must have EITHER a product locator OR a locations file....
if "--product-locator" not in args_dict.keys() and "--location-file" not in args_dict.keys():
raise MissingSettingsException("either product locator or locations file required")
if "--product-locator" in args_dict.keys() and "--location-file" in args_dict.keys():
raise MissingSettingsException("product locator OR locations file required -- not both")
if "--product-locator" in args:
product_locator = args_dict["--product-locator"]
assert product_locator
else:
locations_file = args_dict["--location-file"]
if not Path(locations_file).exists():
raise FileNotFoundError(f"{locations_file} not found")
apps/cli/executables/datafetcher/test/df_pytest_utils.py
View file @ 71b97f8f
...@@ -4,45 +4,25 @@ ...@@ -4,45 +4,25 @@
""" Various conveniences for use and re-use in test cases """ """ Various conveniences for use and re-use in test cases """
import json import json
import logging
import os import os
import sys import sys
import tempfile import tempfile
from pathlib import Path from pathlib import Path
from typing import List, Dict
sys.path.insert(0, str(Path(".").absolute()))
sys.path.insert(0, str(Path("..").absolute()))
# TODO: Some Fine Day: this duplicates same function in package tester.
# CAVEAT PROGRAMMOR: attempts to centralize it have resulted in tears.
def get_project_root() -> Path:
"""
Get the root of this project.
:return:
"""
my_path = Path(__file__)
path = my_path
while not path.name.endswith("workspaces") and not path.name.endswith("packages"):
path = path.parent
return path
import pytest import pytest
from pycapo import CapoConfig from pycapo import CapoConfig
# pylint: disable=C0115, C0116, C0200, R0902, R0903, R0914, R1721, W0212, W0613, W0621, W0703, W1203 # pylint: disable=C0115, C0116, C0200, R0902, R0903, R0914, R1721, W0212, W0613,
sys.path.insert(0, str(get_project_root())) # pylint: disable=W0621, W0703, W1203
from shared.workspaces.test.test_data.utilities import (
from .df_testdata_utils import (
get_locations_report, get_locations_report,
get_test_data_dir, get_test_data_dir,
) )
from datafetcher.datafetcher import DataFetcher from datafetcher.datafetcher import DataFetcher
from datafetcher.return_codes import ReturnCode
from datafetcher.errors import MissingSettingsException, NoProfileException from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.locations_report import LocationsReport from datafetcher.locations_report import LocationsReport
from datafetcher.utilities import ( from datafetcher.utilities import (
...@@ -54,8 +34,9 @@ from datafetcher.utilities import ( ...@@ -54,8 +34,9 @@ from datafetcher.utilities import (
) )
TEST_PROFILE = "docker" TEST_PROFILE = "docker"
MISSING_SETTING = ReturnCode.MISSING_SETTING.value["code"] # set this to False when debugging one or more tests
MISSING_PROFILE = ReturnCode.MISSING_PROFILE.value["code"] # so as not to have to sit thru every test;
# comment out the target test(s)' "@pytest.skip"
RUN_ALL = True RUN_ALL = True
LOCATION_REPORTS = { LOCATION_REPORTS = {
...@@ -259,7 +240,7 @@ def make_tempdir() -> Path: ...@@ -259,7 +240,7 @@ def make_tempdir() -> Path:
umask = os.umask(0o000) umask = os.umask(0o000)
top_level = tempfile.mkdtemp(prefix="datafetcher_test_", dir="/var/tmp") top_level = tempfile.mkdtemp(prefix="datafetcher_test_", dir="/var/tmp")
os.umask(umask) os.umask(umask)
yield top_level return top_level
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
...@@ -322,41 +303,27 @@ def launch_datafetcher(args: list, df_capo_settings: dict) -> int: ...@@ -322,41 +303,27 @@ def launch_datafetcher(args: list, df_capo_settings: dict) -> int:
""" """
if args is None or len(args) == 0: if args is None or len(args) == 0:
return MISSING_SETTING raise MissingSettingsException
try:
namespace = evaluate_args_and_capo(args, df_capo_settings)
fetcher = DataFetcher(namespace, df_capo_settings)
return fetcher.run()
except SystemExit as exc:
if hasattr(exc, "value"):
return exc.value.code if hasattr(exc.value, "code") else exc.value
if hasattr(exc, "code"):
return exc.code
raise namespace = evaluate_args_and_capo(args, df_capo_settings)
except (KeyError, NoProfileException) as exc: datafetcher = DataFetcher(namespace, df_capo_settings)
logging.error(f"{exc}") datafetcher.run()
return MISSING_PROFILE
except Exception as exc:
pytest.fail(f"{exc}")
def evaluate_args_and_capo(args: list, capo_settings: dict): def evaluate_args_and_capo(args: List[str], capo_settings: Dict[str, str]):
if args is None or len(args) == 0: if args is None or len(args) == 0:
sys.exit(MISSING_SETTING) raise MissingSettingsException
profile = get_profile_from_args(args) profile = get_profile_from_args(args)
if profile is None: if profile is None:
profile = capo_settings["profile"] profile = capo_settings["profile"]
if profile is None: if profile is None:
sys.exit(MISSING_PROFILE) raise NoProfileException
else: else:
args["profile"] = profile args["profile"] = profile
namespace = get_arg_parser().parse_args(args) return get_arg_parser().parse_args(args)
return namespace
def get_profile_from_args(args: list) -> str: def get_profile_from_args(args: list) -> str:
......
apps/cli/executables/datafetcher/test/df_testdata_utils.py 0 → 100644
View file @ 71b97f8f
""" Helper functions for download product testing """
# NB: this is a copy of the file in shared/workspaces/test/test_data
# Directly accessing that file seems to be troublesome for the test suite. -DKL
import json
import sys
from enum import Enum
from pathlib import Path
from typing import Dict, Tuple
DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
DATE_FORMAT = "%Y-%m-%d"
def get_report_file(basename: str):
"""
Get a locations file from our collection in test_data,
given a basename (.json filename w/o extension)
:param basename:
:return:
"""
test_data_dir = get_test_data_dir()
for file in test_data_dir.glob(basename.upper() + ".json"):
# (there should be only one for this basename)
return file
return None
def get_exec_block_details_from_loc_report(prefix: str, exec_blocks: list):
"""
Fetch and read locations report indicated by basename;
for filegroup IDs of exec blocks, return file info dict
and total size of files
:param prefix:
:param exec_blocks:
:return:
"""
file_info = dict()
total_size = 0
for exec_block in exec_blocks:
basename = prefix + str(exec_block.filegroup_id)
locations_report = None
try:
locations_report = get_locations_report(basename)
except FileNotFoundError:
# special case: GBT product
if basename.startswith("AGBT17B_044"):
locations_report = get_locations_report("AGBT17B_044_02")
total_size += locations_report["aggregate_size"]
for file_spec in locations_report["files"]:
filename = file_spec["ngas_file_id"]
size = file_spec["size"]
file_info[filename] = size
return file_info, total_size
def get_test_data_dir():
""" where's our test data? """
top_level_subdirs = sys.path
shared_ws_src = None
for pathname in top_level_subdirs:
if "shared/workspaces" in pathname:
shared_ws_src = pathname
break
shared_wksp = Path(shared_ws_src).parent
# test data will be a few levels under shared_wksp
for item in shared_wksp.rglob("location_files"):
assert item.is_dir()
return item
return None
def get_file_info_from_json_file(json_filename: str) -> dict:
""" Pluck file information from a .json location file """
to_read = None
for file in Path.cwd().rglob(json_filename):
to_read = file
break
assert to_read is not None
with open(to_read, "r") as content:
file_info = json.loads(content.read())
return file_info
def get_file_info_from_loc_report(locations_info: Dict[str, str]) -> Tuple:
"""
Grab the data we'll need to retrieve the files.
:param locations_info: data from locations report
:return:
"""
file_info = dict()
total_size = 0
total_size += locations_info["aggregate_size"]
for file_spec in locations_info["files"]:
filename = file_spec["ngas_file_id"]
size = file_spec["size"]
file_info[filename] = size
return file_info, total_size
def get_locations_report(basename: str) -> Dict[str, str]:
"""
Get a locations report from a file in test_data.
:param basename: locations report filename w/o exension
:return:
"""
report_path = get_report_file(basename)
if report_path is not None:
with open(report_path, "r") as content:
locations_info = json.loads(content.read())
return locations_info
raise FileNotFoundError(f'{basename.upper() + ".json"} not found')
class Deliverable(Enum):
""" Type of product to be delivered: SDM, BDF, TAR, etc. """
SDM = "SDM"
BDF = "BDF"
MS = "MS"
CMS = "CMS"
IMG = "IMG"
TAR = "TAR"
# VLBA
IDIFITS = "IDIFITS"
@staticmethod
def from_str(key: str):
for dtype in Deliverable:
if dtype.value == key:
return dtype
return None
class DeliverableProduct:
""" Wraps a deliverable and its locations report info. """
def __init__(self, dtype: Deliverable, file_info: dict):
self.type = dtype
self.file_info = file_info
apps/cli/executables/datafetcher/test/fake_archive_service.py
View file @ 71b97f8f
...@@ -7,8 +7,8 @@ from pathlib import Path ...@@ -7,8 +7,8 @@ from pathlib import Path
from datafetcher.locations_report_refactor import LocationsReportRefactor, UnknownLocatorException from datafetcher.locations_report_refactor import LocationsReportRefactor, UnknownLocatorException
# pylint: disable=C0301, R0903 # pylint: disable=C0301, R0903
_LOG = logging.getLogger(__name__) _LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
PROFILE = "docker" PROFILE = "docker"
......
apps/cli/executables/datafetcher/test/mock_data_fetcher.py
View file @ 71b97f8f
""" for testing the attempt to copy rather than stream files """ """ for testing the attempt to copy rather than stream files """
import logging
import sys
from argparse import Namespace from argparse import Namespace
# pylint: disable=C0103, C0301, E0401, E0402, R0201, R0902, R0903, W0621
from datafetcher.locations_report import LocationsReport from datafetcher.locations_report import LocationsReport
from datafetcher.project_fetcher import ParallelFetcher from datafetcher.project_fetcher import ParallelFetcher
from datafetcher.return_codes import ReturnCode from datafetcher.utilities import get_capo_settings, ExecutionSite
from datafetcher.utilities import get_capo_settings, ExecutionSite, FlexLogger from datafetcher.errors import MissingSettingsException
from .df_pytest_utils import TEST_PROFILE from .df_pytest_utils import TEST_PROFILE
# pylint: disable=C0103, R0201, R0902, R0903, W0621 logger = logging.getLogger(__name__)
class MockProdDataFetcher: class MockProdDataFetcher:
...@@ -18,29 +19,15 @@ class MockProdDataFetcher: ...@@ -18,29 +19,15 @@ class MockProdDataFetcher:
def __init__(self, args: Namespace, settings: dict): def __init__(self, args: Namespace, settings: dict):
if args is None or settings is None: if args is None or settings is None:
self._exit_with_error(ReturnCode.MISSING_SETTING) raise MissingSettingsException()
self.args = args self.args = args
self.settings = settings self.settings = settings
self.verbose = self.args.verbose
self.output_dir = args.output_dir self.output_dir = args.output_dir
self.profile = args.profile self.profile = args.profile
self._LOG = FlexLogger(self.__class__.__name__, self.output_dir, self.verbose) self.locations_report = self._get_locations()
self.servers_report = self.locations_report.servers_report
try:
self.locations_report = self._get_locations()
self.servers_report = self.locations_report.servers_report
except SystemExit:
if args.location_file:
self._exit_with_error(ReturnCode.CANNOT_OPEN_LOCATION_FILE)
else:
self._exit_with_error(ReturnCode.PRODUCT_LOCATOR_NOT_FOUND)
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected {type(exc)} during init: {exc}")
raise
def _get_locations(self): def _get_locations(self):
""" """
...@@ -52,7 +39,7 @@ class MockProdDataFetcher: ...@@ -52,7 +39,7 @@ class MockProdDataFetcher:
capo_settings = get_capo_settings(TEST_PROFILE) capo_settings = get_capo_settings(TEST_PROFILE)
capo_settings["execution_site"] = ExecutionSite.DSOC.value capo_settings["execution_site"] = ExecutionSite.DSOC.value
return LocationsReport(self._LOG, self.args, capo_settings) return LocationsReport(self.args, capo_settings)
def run(self): def run(self):
""" """
...@@ -60,14 +47,4 @@ class MockProdDataFetcher: ...@@ -60,14 +47,4 @@ class MockProdDataFetcher:
:return: :return:
""" """
try: return ParallelFetcher(self.args, self.settings, self.servers_report).run()
return ParallelFetcher(self.args, self.settings, self._LOG, self.servers_report).run()
except SystemExit as exc:
self._LOG.error(f"{exc}")
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected exception during run: {exc}")
raise
def _exit_with_error(self, return_code: ReturnCode):
sys.exit(return_code.value["code"])
apps/cli/executables/datafetcher/test/test_df_function.py
View file @ 71b97f8f
...@@ -2,19 +2,19 @@ ...@@ -2,19 +2,19 @@
from pathlib import Path from pathlib import Path
# pylint: disable=E0401, E0402, W0511
import pytest import pytest
import sys
sys.path.insert(0, str(Path(".").absolute()))
from .df_pytest_utils import get_project_root
project_root = get_project_root()
sys.path.insert(0, str(project_root))
from datafetcher.errors import (
NGASFetchError,
MissingSettingsException,
NGASServiceErrorException,
)
# pylint: disable=C0115, C0116, C0200, C0415, R0801, R0902, R0903, R0914, R1721, W0212, W0611, W0613, W0621, W0703, W1203 # pylint: disable=C0115, C0116, C0200, C0415, R0801, R0902, R0903, R0914, R1721,
# pylint: disable=W0212, W0611, W0613, W0621, W0703, W1203
from datafetcher.datafetcher import DataFetcher, ReturnCode from datafetcher.datafetcher import DataFetcher
from datafetcher.utilities import ( from datafetcher.utilities import (
get_arg_parser, get_arg_parser,
ProductLocatorLookup, ProductLocatorLookup,
...@@ -23,6 +23,9 @@ from datafetcher.utilities import ( ...@@ -23,6 +23,9 @@ from datafetcher.utilities import (
Cluster, Cluster,
) )
from .test_df_return_status import try_to_launch_df
# N.B. these are all in use; SonarLint just doesn't get it # N.B. these are all in use; SonarLint just doesn't get it
from .df_pytest_utils import ( from .df_pytest_utils import (
TEST_PROFILE, TEST_PROFILE,
...@@ -35,15 +38,14 @@ from .df_pytest_utils import ( ...@@ -35,15 +38,14 @@ from .df_pytest_utils import (
make_tempdir, make_tempdir,
RUN_ALL, RUN_ALL,
confirm_retrieve_mode_copy, confirm_retrieve_mode_copy,
get_mini_exec_block,
) )
_LOCATION_FILENAME = "locations.json" _LOCATION_FILENAME = "locations.json"
_ASDM_XML = "ASDM.xml" _ASDM_XML = "ASDM.xml"
_EB_EXTERNAL_NAME = "sysstartS.58955.83384832176" _EB_EXTERNAL_NAME = "sysstartS.58955.83384832176"
# set this to False when debugging one or more tests # TODO: log this in WS-179-3
# so as not to have to sit thru every test;
# comment out the target test(s)' @pytest.skip
print(f">>> RUNNING ALL TESTS: {RUN_ALL}") print(f">>> RUNNING ALL TESTS: {RUN_ALL}")
...@@ -57,16 +59,6 @@ def test_settings_setup(settings): ...@@ -57,16 +59,6 @@ def test_settings_setup(settings):
assert isinstance(settings.test_data, dict) assert isinstance(settings.test_data, dict)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_usage_statement_makes_sense():
""" Ensure that the datafetcher's "usage" statement is as we expect """
usage = DataFetcher._build_usage_message()
assert usage.startswith("Usage:")
lines = usage.split("\n")
assert len(lines) >= len(ReturnCode) + 1
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_nothing_retrieved_if_dry_locator(make_tempdir, settings): def test_nothing_retrieved_if_dry_locator(make_tempdir, settings):
""" Simulates dry run with product locator """ """ Simulates dry run with product locator """
...@@ -80,8 +72,7 @@ def test_nothing_retrieved_if_dry_locator(make_tempdir, settings): ...@@ -80,8 +72,7 @@ def test_nothing_retrieved_if_dry_locator(make_tempdir, settings):
TEST_PROFILE, TEST_PROFILE,
"--dry-run", "--dry-run",
] ]
return_code = launch_datafetcher(args, settings.capo_settings) launch_datafetcher(args, settings.capo_settings)
assert return_code == 0
tempdir_files = Path(make_tempdir).iterdir() tempdir_files = Path(make_tempdir).iterdir()
for file in tempdir_files: for file in tempdir_files:
if not str(file).endswith(".log") and not str(file).endswith(".json"): if not str(file).endswith(".log") and not str(file).endswith(".json"):
...@@ -111,7 +102,7 @@ def test_nothing_retrieved_if_dry_file(make_tempdir, settings): ...@@ -111,7 +102,7 @@ def test_nothing_retrieved_if_dry_file(make_tempdir, settings):
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_force_flag_overwrites_existing_file(make_tempdir, settings): def test_force_flag_overwrites_existing_file(make_tempdir, capo_settings):
top_level = Path(make_tempdir) top_level = Path(make_tempdir)
location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME) location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME)
dest_dir = Path(top_level, _EB_EXTERNAL_NAME) dest_dir = Path(top_level, _EB_EXTERNAL_NAME)
...@@ -131,19 +122,11 @@ def test_force_flag_overwrites_existing_file(make_tempdir, settings): ...@@ -131,19 +122,11 @@ def test_force_flag_overwrites_existing_file(make_tempdir, settings):
str(top_level), str(top_level),
"--force", "--force",
] ]
try: launch_datafetcher(args, capo_settings)
launch_datafetcher(args, settings.capo_settings)
except SystemExit as ex:
pytest.fail(f"{ex}")
raise
except Exception as exc:
pytest.fail(f"{exc}")
raise
sizes = dict()
# go thru destination directory recursively # go thru destination directory recursively
# and get everybody's size # and get everybody's size
sizes = dict()
for file in dest_dir.rglob("*"): for file in dest_dir.rglob("*"):
sizes[str(file)] = file.stat().st_size sizes[str(file)] = file.stat().st_size
assert len(sizes) == 37 assert len(sizes) == 37
...@@ -152,16 +135,24 @@ def test_force_flag_overwrites_existing_file(make_tempdir, settings): ...@@ -152,16 +135,24 @@ def test_force_flag_overwrites_existing_file(make_tempdir, settings):
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_no_overwrite_without_force(make_tempdir, settings): def test_no_overwrite_without_force(make_tempdir, capo_settings):
top_level = Path(make_tempdir) top_level = Path(make_tempdir)
location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME) location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME)
locations_dict = get_mini_exec_block()
files = locations_dict["files"]
num_files_expected = 37
assert len(files) == num_files_expected
dest_dir = Path(top_level, _EB_EXTERNAL_NAME) dest_dir = Path(top_level, _EB_EXTERNAL_NAME)
dest_dir.mkdir(parents=True, exist_ok=True) dest_dir.mkdir(parents=True, exist_ok=True)
# make a fake file to be overwritten # make a fake file to be overwritten
fake_file = dest_dir / _ASDM_XML fake_file = dest_dir / _ASDM_XML
if fake_file.exists():
fake_file.unlink()
with open(fake_file, "w") as to_write: with open(fake_file, "w") as to_write:
to_write.write("dang! what a kick in the rubber parts!") to_write.write("dang! what a kick in the rubber parts!")
assert fake_file.stat().st_size == 38
args = [ args = [
"--location-file", "--location-file",
...@@ -172,91 +163,38 @@ def test_no_overwrite_without_force(make_tempdir, settings): ...@@ -172,91 +163,38 @@ def test_no_overwrite_without_force(make_tempdir, settings):
str(top_level), str(top_level),
] ]
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(FileExistsError):
assert return_code == ReturnCode.NGAS_FETCH_ERROR.value["code"] try_to_launch_df(capo_settings, args)
sizes = dict()
for file in dest_dir.rglob("*"):
sizes[str(file)] = file.stat().st_size
assert len(sizes) < 37
fake_size = fake_file.stat().st_size
assert fake_size == 38
retrieved = [
@pytest.mark.skip("verbose mode goes away in WS-179 and isn't in use now") file for file in top_level.rglob("*") if file.is_file() and not str(file).endswith(".json")
def test_more_output_when_verbose(make_tempdir, settings):
top_level = Path(make_tempdir)
location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME)
args = [
"--location-file",
str(location_file),
"--profile",
TEST_PROFILE,
"--output-dir",
str(make_tempdir),
"--verbose",
] ]
num_not_too_big_files_expected = 28
return_code = launch_datafetcher(args, settings.capo_settings) assert len(retrieved) == num_not_too_big_files_expected
assert return_code == ReturnCode.SUCCESS.value["code"]
num_files_expected = 37
retrieved = [file for file in top_level.rglob("*") if file.is_file()]
assert num_files_expected == len(retrieved) - 2
verbose_logfile = None
for file in retrieved:
if str(file).endswith(".log"):
verbose_logfile = file
break
assert verbose_logfile is not None
verbose_log_size = verbose_logfile.stat().st_size
assert verbose_log_size > 0
# get rid of all the files we downloaded, plus the log # get rid of all the files we downloaded, plus the log
deleted = [file.unlink() for file in retrieved if not str(file).endswith(".json")] deleted = [
assert len(deleted) >= num_files_expected file.unlink() for file in retrieved if file.is_file() and not str(file).endswith(".json")
# same download, but without verbose logging
args = [
"--location-file",
str(location_file),
"--profile",
TEST_PROFILE,
"--output-dir",
str(make_tempdir),
] ]
return_code = launch_datafetcher(args, settings.capo_settings) assert len(deleted) >= num_not_too_big_files_expected
assert return_code == ReturnCode.SUCCESS.value["code"]
retrieved = [file for file in top_level.rglob("*")]
assert len(retrieved) == num_files_expected + 3
logfile = None
for file in retrieved:
if str(file).endswith(".log"):
logfile = file
break
assert logfile is not None
logsize = logfile.stat().st_size
# successful download, non-verbose logging,
# should result in zero-size log file
assert logsize == 0
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings): def test_copy_attempt_throws_fetch_error(make_tempdir, settings):
"""We set profile to dsoc-prod here so as to force the DF """
We set profile to dsoc-prod here so as to force the DF
to try to copy rather than stream to try to copy rather than stream
:param make_tempdir: temp directory fixture
:param settings: Capo settings for datafetcher
:return:
""" """
# N.B. can't do this import with the rest of the imports up top, # N.B. can't do this import with the rest of the imports up top,
# because test_df_return_codes not yet initialized # because test_df_return_codes not yet initialized
from .test_df_return_codes import we_are_in_docker from .test_df_return_status import we_are_in_docker
if we_are_in_docker(): if we_are_in_docker():
# this test doesn't work in a docker container: # this test doesn't work in a docker container:
...@@ -279,24 +217,23 @@ def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings): ...@@ -279,24 +217,23 @@ def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings):
prod_profile, prod_profile,
] ]
namespace = get_arg_parser().parse_args(args) namespace = get_arg_parser().parse_args(args)
fetcher = DataFetcher(namespace, settings.capo_settings) datafetcher = DataFetcher(namespace, settings.capo_settings)
servers_report = fetcher.servers_report servers_report = datafetcher.servers_report
confirm_retrieve_mode_copy(servers_report) confirm_retrieve_mode_copy(servers_report)
# let's try just one file so we're not sitting here all day # let's try just one file so we're not sitting here all day
for server in servers_report: for server in servers_report:
entry = servers_report[server] entry = servers_report[server]
servers_report = {server: entry} servers_report = {server: entry}
fetcher.servers_report = servers_report datafetcher.servers_report = servers_report
assert fetcher.servers_report[server] is not None assert datafetcher.servers_report[server] is not None
files = fetcher.servers_report[server]["files"] files = datafetcher.servers_report[server]["files"]
fetcher.servers_report[server]["files"] = [files[0]] datafetcher.servers_report[server]["files"] = [files[0]]
break break
with pytest.raises(SystemExit) as exc: with pytest.raises(NGASFetchError):
fetcher.run() datafetcher.run()
assert exc.value.code == ReturnCode.CATASTROPHIC_REQUEST_ERROR.value["code"]
finally: finally:
if props_file.exists(): if props_file.exists():
props_file.unlink() props_file.unlink()
...@@ -314,14 +251,13 @@ def test_dies_with_bad_server_info(make_tempdir, settings): ...@@ -314,14 +251,13 @@ def test_dies_with_bad_server_info(make_tempdir, settings):
"--location-file", "--location-file",
str(location_file), str(location_file),
] ]
try:
with pytest.raises(NGASServiceErrorException):
launch_datafetcher(args, settings.capo_settings) launch_datafetcher(args, settings.capo_settings)
except Exception as exc:
assert exc.value.code == ReturnCode.NGAS_FETCH_ERROR["code"]
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_missing_setting_error_on_bad_destination(settings): def test_missing_setting_exc_on_bad_destination(settings):
args = [ args = [
"--profile", "--profile",
TEST_PROFILE, TEST_PROFILE,
...@@ -330,35 +266,12 @@ def test_missing_setting_error_on_bad_destination(settings): ...@@ -330,35 +266,12 @@ def test_missing_setting_error_on_bad_destination(settings):
"--output-dir", "--output-dir",
"floob", "floob",
] ]
try: with pytest.raises(MissingSettingsException):
launch_datafetcher(args, settings.capo_settings) launch_datafetcher(args, settings.capo_settings)
except Exception as exc:
assert exc.value.code == ReturnCode.MISSING_SETTING["code"]
def write_fake_file(destination: Path, file_info: dict):
filename = file_info["ngas_file_id"]
path = Path(destination, filename)
with open(path, "w") as file:
file.write(f'{str(file_info["size"])}\n')
class MockSuccessfulFetchReturn: @pytest.mark.skip("takes too long; re-enable with WS-179-1")
@staticmethod def test_gets_vlbas_from_report_file(make_tempdir, capo_settings):
def run():
return 0
@pytest.fixture
def mock_successful_fetch_run(monkeypatch):
def mock_run(*args, **kwargs):
return MockSuccessfulFetchReturn().run()
monkeypatch.setattr(DataFetcher, "run", mock_run)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, settings):
location_file = get_locations_file("VLBA_EB") location_file = get_locations_file("VLBA_EB")
args = [ args = [
...@@ -369,12 +282,11 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se ...@@ -369,12 +282,11 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se
"--location-file", "--location-file",
str(location_file), str(location_file),
] ]
fetcher = DataFetcher(get_arg_parser().parse_args(args), settings.capo_settings) datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = fetcher.servers_report servers_report = datafetcher.servers_report
assert len(servers_report) == 1 assert len(servers_report) == 1
return_code = fetcher.run() datafetcher.run()
assert return_code == 0
dest_dir = Path(make_tempdir) dest_dir = Path(make_tempdir)
file_info_dict = dict() file_info_dict = dict()
...@@ -391,8 +303,7 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se ...@@ -391,8 +303,7 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se
file_info_dict[filename["ngas_file_id"]] = filename file_info_dict[filename["ngas_file_id"]] = filename
datafetcher = DataFetcher(get_arg_parser().parse_args(args), settings.capo_settings) datafetcher = DataFetcher(get_arg_parser().parse_args(args), settings.capo_settings)
return_code = datafetcher.run() datafetcher.run()
assert return_code == 0
for filename in file_info_dict: for filename in file_info_dict:
path = Path(dest_dir, filename) path = Path(dest_dir, filename)
...@@ -401,8 +312,8 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se ...@@ -401,8 +312,8 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se
assert int(contents) == file_info_dict[filename]["size"] assert int(contents) == file_info_dict[filename]["size"]
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skip("takes too long; re-enable with WS-179-1")
def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tempdir, settings): def test_gets_large_vla_ebs_from_report_file(make_tempdir, capo_settings):
location_file = get_locations_file("VLA_SMALL_EB") location_file = get_locations_file("VLA_SMALL_EB")
args = [ args = [
"--profile", "--profile",
...@@ -412,14 +323,15 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem ...@@ -412,14 +323,15 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem
"--location-file", "--location-file",
str(location_file), str(location_file),
] ]
fetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings) datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = fetcher.servers_report servers_report = datafetcher.servers_report
assert len(servers_report) == 2 assert len(servers_report) == 2
datafetcher.run()
return_code = fetcher.run() server_file_count = {
assert return_code == 0 "nmngas03.aoc.nrao.edu:7777": 0,
"nmngas04.aoc.nrao.edu:7777": 0,
server_file_count = {"nmngas03.aoc.nrao.edu:7777": 0, "nmngas04.aoc.nrao.edu:7777": 0} }
dest_dir = Path(make_tempdir) dest_dir = Path(make_tempdir)
file_list = list() file_list = list()
for server in servers_report.items(): for server in servers_report.items():
...@@ -440,8 +352,7 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem ...@@ -440,8 +352,7 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem
assert server_file_count["nmngas04.aoc.nrao.edu:7777"] == 41 assert server_file_count["nmngas04.aoc.nrao.edu:7777"] == 41
datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings) datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
return_code = datafetcher.run() datafetcher.run()
assert return_code == 0
found_count = 0 found_count = 0
for file_info in file_list: for file_info in file_list:
...@@ -456,8 +367,8 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem ...@@ -456,8 +367,8 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem
assert found_count == len(file_list) assert found_count == len(file_list)
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skip("takes too long; re-enable with WS-179-1")
def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, settings): def test_gets_images_from_report_file(make_tempdir, capo_settings):
location_file = get_locations_file("IMG") location_file = get_locations_file("IMG")
args = [ args = [
"--profile", "--profile",
...@@ -467,11 +378,14 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s ...@@ -467,11 +378,14 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s
"--location-file", "--location-file",
str(location_file), str(location_file),
] ]
fetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings) datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = fetcher.servers_report servers_report = datafetcher.servers_report
assert len(servers_report) == 2 assert len(servers_report) == 2
server_file_count = {"nmngas01.aoc.nrao.edu:7777": 0, "nmngas02.aoc.nrao.edu:7777": 0} server_file_count = {
"nmngas01.aoc.nrao.edu:7777": 0,
"nmngas02.aoc.nrao.edu:7777": 0,
}
dest_dir = Path(make_tempdir) dest_dir = Path(make_tempdir)
file_list = list() file_list = list()
for server in servers_report.items(): for server in servers_report.items():
...@@ -491,8 +405,7 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s ...@@ -491,8 +405,7 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s
for server_url, count in server_file_count.items(): for server_url, count in server_file_count.items():
assert count == 1 assert count == 1
return_code = fetcher.run() datafetcher.run()
assert return_code == 0
found_count = 0 found_count = 0
for file_info in file_list: for file_info in file_list:
...@@ -508,7 +421,7 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s ...@@ -508,7 +421,7 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempdir, settings): def test_gets_calibration_from_report_file(make_tempdir):
location_file = get_locations_file("CALIBRATION") location_file = get_locations_file("CALIBRATION")
args = [ args = [
"--profile", "--profile",
...@@ -518,8 +431,8 @@ def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempd ...@@ -518,8 +431,8 @@ def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempd
"--location-file", "--location-file",
str(location_file), str(location_file),
] ]
fetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings) datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = fetcher.servers_report servers_report = datafetcher.servers_report
assert len(servers_report) == 1 assert len(servers_report) == 1
fake_file = None fake_file = None
...@@ -538,7 +451,7 @@ def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempd ...@@ -538,7 +451,7 @@ def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempd
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_calibration_from_locator(mock_successful_fetch_run, make_tempdir, settings): def test_gets_calibration_from_locator(make_tempdir, settings):
external_name = LOCATION_REPORTS["CALIBRATION"]["external_name"] external_name = LOCATION_REPORTS["CALIBRATION"]["external_name"]
product_locator = ProductLocatorLookup(settings.db_settings).look_up_locator_for_ext_name( product_locator = ProductLocatorLookup(settings.db_settings).look_up_locator_for_ext_name(
external_name external_name
...@@ -570,6 +483,7 @@ def test_gets_calibration_from_locator(mock_successful_fetch_run, make_tempdir, ...@@ -570,6 +483,7 @@ def test_gets_calibration_from_locator(mock_successful_fetch_run, make_tempdir,
assert int(contents) == file_spec["size"] assert int(contents) == file_spec["size"]
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_gbt_data_from_locator(make_tempdir, settings): def test_gets_gbt_data_from_locator(make_tempdir, settings):
""" Can we cope with GBT data? """ """ Can we cope with GBT data? """
...@@ -601,3 +515,10 @@ def test_gets_gbt_data_from_locator(make_tempdir, settings): ...@@ -601,3 +515,10 @@ def test_gets_gbt_data_from_locator(make_tempdir, settings):
assert fake_file.is_file() assert fake_file.is_file()
contents = fake_file.read_text().strip() contents = fake_file.read_text().strip()
assert int(contents) == file_spec["size"] assert int(contents) == file_spec["size"]
def write_fake_file(destination: Path, file_info: dict):
filename = file_info["ngas_file_id"]
path = Path(destination, filename)
with open(path, "w") as file:
file.write(f'{str(file_info["size"])}\n')
apps/cli/executables/datafetcher/test/test_df_return_codes.py → apps/cli/executables/datafetcher/test/test_df_return_status.py
View file @ 71b97f8f
...@@ -2,19 +2,36 @@ ...@@ -2,19 +2,36 @@
check that arguments passed return expected codes check that arguments passed return expected codes
""" """
# pylint: disable=C0115, C0116, R0801, R0902, R0903, R0914, W0212, W0611 W0613, W0621, W0703, W1203 # pylint: disable=C0115, C0116, E0401, E0402, E1120,R0801, R0902, R0903, R0914,
# pylint: disable=W0212, W0611, W0613, W0621, W0703, W1203
import os import os
import tempfile
from pathlib import Path from pathlib import Path
from typing import List
import pytest import pytest
from datafetcher.errors import (
NoProfileException,
MissingSettingsException,
LocationServiceTimeoutException,
NGASServiceRedirectsException,
NGASFetchError,
NoLocatorException,
SizeMismatchException,
)
from datafetcher.datafetcher import DataFetcher from datafetcher.datafetcher import DataFetcher
from datafetcher.return_codes import ReturnCode from datafetcher.utilities import (
from datafetcher.utilities import get_arg_parser, ProductLocatorLookup get_arg_parser,
ProductLocatorLookup,
path_is_accessible,
)
# N.B. IJ doesn't recognize imported fixtures as being in use. # N.B. IJ doesn't recognize imported fixtures as being in use.
# don't let these imports (make_tempdir, settings) get disappeared. # don't let these imports (make_tempdir, settings) get disappeared.
from .df_pytest_arg_utils import parse_args
from .df_pytest_utils import ( from .df_pytest_utils import (
TEST_PROFILE, TEST_PROFILE,
get_test_capo_settings, get_test_capo_settings,
...@@ -22,16 +39,22 @@ from .df_pytest_utils import ( ...@@ -22,16 +39,22 @@ from .df_pytest_utils import (
capo_settings, capo_settings,
settings, settings,
launch_datafetcher, launch_datafetcher,
MISSING_SETTING,
MISSING_PROFILE,
RUN_ALL, RUN_ALL,
confirm_retrieve_mode_copy, confirm_retrieve_mode_copy,
evaluate_args_and_capo,
) )
from .mock_data_fetcher import MockProdDataFetcher from .mock_data_fetcher import MockProdDataFetcher
def test_launch_df(make_tempdir, settings): def test_launch_df(make_tempdir, settings):
"""
Our "control"; should always pass
:param make_tempdir:
:param settings:
:return:
"""
args = [ args = [
"--product-locator", "--product-locator",
settings.test_data["product_locator"], settings.test_data["product_locator"],
...@@ -41,16 +64,12 @@ def test_launch_df(make_tempdir, settings): ...@@ -41,16 +64,12 @@ def test_launch_df(make_tempdir, settings):
TEST_PROFILE, TEST_PROFILE,
"--dry-run", "--dry-run",
] ]
return_code = launch_datafetcher(args, settings.capo_settings) try_to_launch_df(settings.capo_settings, args)
assert return_code is not None
def test_launch_df_no_args(settings): def test_launch_df_no_args(settings):
try: with pytest.raises(MissingSettingsException):
return_code = launch_datafetcher([], settings.capo_settings) launch_datafetcher([], settings.capo_settings)
assert return_code is not None
except Exception as exc:
pytest.fail(f"{exc}")
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
...@@ -68,8 +87,8 @@ def test_no_args_prints_usage_and_croaks(): ...@@ -68,8 +87,8 @@ def test_no_args_prints_usage_and_croaks():
assert datafetcher.usage == DataFetcher._build_usage_message() assert datafetcher.usage == DataFetcher._build_usage_message()
# @pytest.mark.skipif(not RUN_ALL, reason='debug') @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_omitted_profile_returns_expected_code(make_tempdir, settings): def test_omitted_profile_throws_no_profile_exc(make_tempdir, settings):
""" """
Be sure DF dies with appropriate error message Be sure DF dies with appropriate error message
if called without Capo profile if called without Capo profile
...@@ -93,37 +112,29 @@ def test_omitted_profile_returns_expected_code(make_tempdir, settings): ...@@ -93,37 +112,29 @@ def test_omitted_profile_returns_expected_code(make_tempdir, settings):
str(make_tempdir), str(make_tempdir),
] ]
return_code = launch_datafetcher(args, settings.capo_settings) try:
assert return_code == MISSING_PROFILE with pytest.raises(NoProfileException):
launch_datafetcher(args, settings.capo_settings)
# restore the existing CAPO_PROFILE finally:
os.environ["CAPO_PROFILE"] = existing_capo_profile # restore the existing CAPO_PROFILE
os.environ["CAPO_PROFILE"] = existing_capo_profile
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_omitted_capo_value_returns_expected_code(make_tempdir, settings): def test_omitted_capo_value_throws_missing_setting_exc(capo_settings):
"""
:param make_tempdir: tempdir created on the fly
:param settings: source of Capo settings
:return:
"""
args = [ args = [
"--product-locator", "--product-locator",
"we're not going to get this far", "we're not going to get this far",
"--output-dir", "--output-dir",
"--profile", "--profile",
TEST_PROFILE, "docker",
] ]
result = launch_datafetcher(args, settings.capo_settings) with pytest.raises(MissingSettingsException):
assert result == MISSING_SETTING try_to_launch_df(capo_settings, args)
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_invalid_capo_profile_returns_expected_code(make_tempdir, settings): def test_invalid_capo_profile_raises_no_profile_exc(make_tempdir, settings):
""" """
Be sure DF dies with appropriate error message Be sure DF dies with appropriate error message
if called without Capo profile if called without Capo profile
...@@ -145,8 +156,8 @@ def test_invalid_capo_profile_returns_expected_code(make_tempdir, settings): ...@@ -145,8 +156,8 @@ def test_invalid_capo_profile_returns_expected_code(make_tempdir, settings):
"--output-dir", "--output-dir",
str(make_tempdir), str(make_tempdir),
] ]
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(MissingSettingsException):
assert return_code == MISSING_PROFILE try_to_launch_df(settings.capo_settings, args)
def we_are_in_docker(): def we_are_in_docker():
...@@ -164,59 +175,35 @@ def we_are_in_docker(): ...@@ -164,59 +175,35 @@ def we_are_in_docker():
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_inaccessible_output_dir_returns_expected_code(settings, make_tempdir): def test_inaccessible_output_dir_throws_file_not_found(settings):
umask = os.umask(0o000) tmpdir = Path("go_away")
try: assert not path_is_accessible(tmpdir)
tmpdir = Path(make_tempdir)
tmpdir.chmod(0o666) altered_capo = get_test_capo_settings()
altered_capo["download_dir"] = str(tmpdir)
altered_capo = get_test_capo_settings()
altered_capo["download_dir"] = str(tmpdir) args = [
"--product-locator",
args = [ settings.test_data["product_locator"],
"--product-locator", "--profile",
settings.test_data["product_locator"], TEST_PROFILE,
"--profile", "--output-dir",
TEST_PROFILE, str(tmpdir),
"--output-dir", ]
str(tmpdir),
] with pytest.raises(FileNotFoundError):
try_to_launch_df(settings.capo_settings, args)
namespace = get_arg_parser().parse_args(args)
# N.B. DataFetcher.__init__ throws SystemExit in pytest at command line,
# but in Docker container the failure is interpreted as an Exception
if we_are_in_docker():
try:
DataFetcher(namespace, capo_settings)
except Exception as exc:
assert isinstance(exc, SystemExit)
assert exc.value.code == MISSING_SETTING
else:
with pytest.raises(SystemExit) as exc:
DataFetcher(namespace, settings.capo_settings)
assert exc.value.code == MISSING_SETTING
except Exception as exc:
pytest.fail(f"{exc}")
finally:
try:
os.umask(umask)
except Exception as exc:
pytest.fail(f"{exc}")
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_two_locator_args_returns_expected_code(make_tempdir, settings): def test_locator_plus_file_throws_missing_setting(make_tempdir, capo_settings):
""" """
We should reject invocation with both product locator -and- location We should reject invocation with both product locator -and- location
file. One or the other, people! file. One or the other, people!
:param make_tempdir: tempdir created on the fly :param make_tempdir: tempdir created on the fly
:param settings: source of Capo settings :param capo_settings: source of Capo settings
:return: :return:
""" """
...@@ -225,13 +212,14 @@ def test_two_locator_args_returns_expected_code(make_tempdir, settings): ...@@ -225,13 +212,14 @@ def test_two_locator_args_returns_expected_code(make_tempdir, settings):
"--product-locator", "--product-locator",
"a_locator", "a_locator",
"--location-file", "--location-file",
"location.json" "--output-dir", "location.json",
"--output-dir",
str(make_tempdir), str(make_tempdir),
"--profile", "--profile",
TEST_PROFILE, TEST_PROFILE,
] ]
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(MissingSettingsException):
assert return_code == MISSING_SETTING try_to_launch_df(capo_settings, args)
class MockServiceTimeoutReturn: class MockServiceTimeoutReturn:
...@@ -239,7 +227,7 @@ class MockServiceTimeoutReturn: ...@@ -239,7 +227,7 @@ class MockServiceTimeoutReturn:
@staticmethod @staticmethod
def run(): def run():
return ReturnCode.LOCATOR_SERVICE_TIMEOUT.value["code"] raise LocationServiceTimeoutException()
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
...@@ -256,14 +244,14 @@ def test_locator_service_timeout_returns_expected_code(monkeypatch, settings, ma ...@@ -256,14 +244,14 @@ def test_locator_service_timeout_returns_expected_code(monkeypatch, settings, ma
TEST_PROFILE, TEST_PROFILE,
] ]
monkeypatch.setattr(DataFetcher, "run", mock_run) monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(LocationServiceTimeoutException):
assert return_code == ReturnCode.LOCATOR_SERVICE_TIMEOUT.value["code"] launch_datafetcher(args, settings.capo_settings)
class MockTooManyServiceRedirectsReturn: class MockTooManyServiceRedirectsReturn:
@staticmethod @staticmethod
def run(): def run():
return ReturnCode.TOO_MANY_SERVICE_REDIRECTS.value raise NGASServiceRedirectsException()
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
...@@ -280,14 +268,14 @@ def test_too_many_service_redirects_returns_expected_code(monkeypatch, settings, ...@@ -280,14 +268,14 @@ def test_too_many_service_redirects_returns_expected_code(monkeypatch, settings,
TEST_PROFILE, TEST_PROFILE,
] ]
monkeypatch.setattr(DataFetcher, "run", mock_run) monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(NGASServiceRedirectsException):
assert return_code == ReturnCode.TOO_MANY_SERVICE_REDIRECTS.value launch_datafetcher(args, settings.capo_settings)
class MockCatastrophicServiceErrorReturn: class MockCatastrophicServiceErrorReturn:
@staticmethod @staticmethod
def run(): def run():
return ReturnCode.CATASTROPHIC_REQUEST_ERROR raise NGASFetchError()
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
...@@ -305,11 +293,12 @@ def test_catastrophic_service_error_returns_expected_code(monkeypatch, settings, ...@@ -305,11 +293,12 @@ def test_catastrophic_service_error_returns_expected_code(monkeypatch, settings,
] ]
monkeypatch.setattr(DataFetcher, "run", mock_run) monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(NGASFetchError):
assert return_code == ReturnCode.CATASTROPHIC_REQUEST_ERROR launch_datafetcher(args, settings.capo_settings)
def test_copy_attempt_throws_sys_exit_service_error(monkeypatch, settings, make_tempdir): @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_copy_attempt_raises_ngas_fetch_error(make_tempdir, settings):
args = [ args = [
"--product-locator", "--product-locator",
settings.test_data["product_locator"], settings.test_data["product_locator"],
...@@ -337,9 +326,8 @@ def test_copy_attempt_throws_sys_exit_service_error(monkeypatch, settings, make_ ...@@ -337,9 +326,8 @@ def test_copy_attempt_throws_sys_exit_service_error(monkeypatch, settings, make_
files = fetcher.servers_report[a_server]["files"] files = fetcher.servers_report[a_server]["files"]
fetcher.servers_report[a_server]["files"] = [files[0]] fetcher.servers_report[a_server]["files"] = [files[0]]
with pytest.raises(SystemExit) as exc: with pytest.raises(NGASFetchError):
fetcher.run() fetcher.run()
assert exc.value.code == ReturnCode.CATASTROPHIC_REQUEST_ERROR.value["code"]
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
...@@ -352,12 +340,12 @@ def test_product_locator_not_found_returns_expected_code(make_tempdir, settings) ...@@ -352,12 +340,12 @@ def test_product_locator_not_found_returns_expected_code(make_tempdir, settings)
"--profile", "--profile",
TEST_PROFILE, TEST_PROFILE,
] ]
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(NoLocatorException):
assert return_code == ReturnCode.PRODUCT_LOCATOR_NOT_FOUND.value["code"] launch_datafetcher(args, settings.capo_settings)
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_unable_to_open_location_file_returns_expected_code(make_tempdir, settings): def test_unable_to_open_location_file_throws_file_not_found(make_tempdir, capo_settings):
args = [ args = [
"--location-file", "--location-file",
"location.json", "location.json",
...@@ -366,8 +354,8 @@ def test_unable_to_open_location_file_returns_expected_code(make_tempdir, settin ...@@ -366,8 +354,8 @@ def test_unable_to_open_location_file_returns_expected_code(make_tempdir, settin
"--profile", "--profile",
TEST_PROFILE, TEST_PROFILE,
] ]
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(FileNotFoundError):
assert return_code == ReturnCode.CANNOT_OPEN_LOCATION_FILE.value["code"] try_to_launch_df(capo_settings, args)
class MockNgasFetchError: class MockNgasFetchError:
...@@ -375,7 +363,7 @@ class MockNgasFetchError: ...@@ -375,7 +363,7 @@ class MockNgasFetchError:
@staticmethod @staticmethod
def run(): def run():
return ReturnCode.NGAS_FETCH_ERROR raise NGASFetchError()
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
...@@ -392,8 +380,8 @@ def test_error_fetching_file_from_ngas_returns_expected_code(monkeypatch, settin ...@@ -392,8 +380,8 @@ def test_error_fetching_file_from_ngas_returns_expected_code(monkeypatch, settin
TEST_PROFILE, TEST_PROFILE,
] ]
monkeypatch.setattr(DataFetcher, "run", mock_run) monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(NGASFetchError):
assert return_code == ReturnCode.NGAS_FETCH_ERROR launch_datafetcher(args, settings.capo_settings)
class MockSizeMismatchError: class MockSizeMismatchError:
...@@ -401,7 +389,7 @@ class MockSizeMismatchError: ...@@ -401,7 +389,7 @@ class MockSizeMismatchError:
@staticmethod @staticmethod
def run(): def run():
return ReturnCode.SIZE_MISMATCH raise SizeMismatchException()
@pytest.mark.skipif(not RUN_ALL, reason="debug") @pytest.mark.skipif(not RUN_ALL, reason="debug")
...@@ -418,5 +406,12 @@ def test_unexpected_size_returns_expected_code(monkeypatch, settings, make_tempd ...@@ -418,5 +406,12 @@ def test_unexpected_size_returns_expected_code(monkeypatch, settings, make_tempd
TEST_PROFILE, TEST_PROFILE,
] ]
monkeypatch.setattr(DataFetcher, "run", mock_run) monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings) with pytest.raises(SizeMismatchException):
assert return_code == ReturnCode.SIZE_MISMATCH launch_datafetcher(args, settings.capo_settings)
def try_to_launch_df(capo_settings, args: List[str]):
parse_args(args)
namespace = evaluate_args_and_capo(args, capo_settings)
fetcher = DataFetcher(namespace, capo_settings)
return fetcher.run()