Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ssa/workspaces
1 result
Show changes
Commits on Source (2)
Hide whitespace changes
Inline Side-by-side
Showing
with 644 additions and 790 deletions
apps/cli/executables/datafetcher/datafetcher/datafetcher.py
View file @ 71b97f8f
......@@ -2,24 +2,21 @@
# -*- coding: utf-8 -*-
""" Module for the command line interface to data-fetcher. """
import logging
import sys
from argparse import Namespace
from pathlib import Path
# pylint: disable=C0103, E0402, E0611, R0902, R0903, W0703, W1203
from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.project_fetcher import ParallelFetcher
from datafetcher.return_codes import ReturnCode
from .locations_report import LocationsReport
from .utilities import get_arg_parser, get_capo_settings, FlexLogger, path_is_accessible
# pylint: disable=C0103, W1203, R0902, R0903
from .utilities import get_arg_parser, get_capo_settings, path_is_accessible
_APPLICATION_NAME = "datafetcher"
# pylint: disable=C0103, W0703
logger = logging.getLogger(__name__)
class DataFetcher:
......@@ -48,60 +45,53 @@ class DataFetcher:
def __init__(self, args: Namespace, df_capo_settings: dict):
self.usage = self._build_usage_message()
if args is None or df_capo_settings is None:
self._exit_with_error(ReturnCode.MISSING_SETTING)
raise MissingSettingsException()
self.args = args
self.settings = df_capo_settings
self.verbose = self.args.verbose
try:
self.verbose = self.args.verbose
except AttributeError:
# we don't care; --verbose will be dropped later in WS-179
pass
# required arguments
self.profile = args.profile
if self.profile is None:
raise NoProfileException()
self.output_dir = args.output_dir
if self.output_dir is None:
self._exit_with_error(ReturnCode.MISSING_SETTING)
raise MissingSettingsException("output directory option is missing")
output_dir = Path(self.output_dir)
if not output_dir.is_dir() or not path_is_accessible(output_dir):
logging.error(f"output location {self.output_dir} inaccessible " f"or not found")
self._exit_with_error(ReturnCode.MISSING_SETTING)
raise MissingSettingsException(
f"output location {self.output_dir} inaccessible or not found"
)
self._LOG = FlexLogger(self.__class__.__name__, self.output_dir, self.verbose)
if args.location_file is not None:
if args.product_locator is not None:
self._LOG.error("required: location file OR product locator " "-- not both")
self._exit_with_error(ReturnCode.MISSING_SETTING)
raise MissingSettingsException(
"required: location file OR product locator -- not both"
)
self.location_file = args.location_file
elif args.product_locator is not None:
self.product_locator = args.product_locator
else:
self._LOG.error("you must specify either a location file or a " "product locator")
self._exit_with_error(ReturnCode.MISSING_SETTING)
self.profile = args.profile
if self.profile is None:
self._exit_with_error(ReturnCode.MISSING_PROFILE)
raise MissingSettingsException(
"you must specify either a location file or a product locator"
)
# optional arguments
self.is_dry = args.dry_run
self.force = args.force
self.verbose = args.verbose or False
try:
self.locations_report = self._get_locations()
self.servers_report = self.locations_report.servers_report
except SystemExit as exc:
self._LOG.error(f"{exc}")
if args.location_file:
self._exit_with_error(ReturnCode.CANNOT_OPEN_LOCATION_FILE)
else:
self._exit_with_error(ReturnCode.PRODUCT_LOCATOR_NOT_FOUND)
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected {type(exc)} during init: {exc}")
raise
def _exit_with_error(self, return_code: ReturnCode):
print(self.usage)
sys.exit(return_code.value["code"])
self.verbose = args.verbose or False
except AttributeError:
# verbose is going away soon
self.verbose = False
self.locations_report = self._get_locations()
self.servers_report = self.locations_report.servers_report
@staticmethod
def _build_usage_message() -> str:
......@@ -110,9 +100,6 @@ class DataFetcher:
(--product-locator PRODUCT_LOCATOR | --location-file LOCATION_FILE)
[--dry-run] [--output-dir OUTPUT_DIR] [-v, --verbose]
[--profile PROFILE]\n"""
usage_str += "\n\t\tReturn codes:\n"
for return_code in ReturnCode:
usage_str += f"\n\t\t\t{return_code.value['code']}: " f"{return_code.value['text']}"
return usage_str
def run(self):
......@@ -121,36 +108,12 @@ class DataFetcher:
:return:
"""
try:
fetcher = ParallelFetcher(self.args, self.settings, self._LOG, self.servers_report)
return fetcher.run()
except SystemExit as exc:
self._LOG.error(f"{exc}")
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected exception during run: {exc}")
raise
fetcher = ParallelFetcher(self.args, self.settings, self.servers_report)
fetcher.run()
def _get_locations(self):
try:
capo_settings = get_capo_settings(self.profile)
except MissingSettingsException as exc:
# if a setting couldn't be found, we can be pretty sure that
# the profile's no good
raise NoProfileException from exc
try:
return LocationsReport(self._LOG, self.args, capo_settings)
except Exception as exc:
self._LOG.error(f"{exc}")
if hasattr(self, "location_file"):
self._exit_with_error(ReturnCode.CANNOT_OPEN_LOCATION_FILE)
elif hasattr(self, "product_locator"):
self._exit_with_error(ReturnCode.PRODUCT_LOCATOR_NOT_FOUND)
else:
# should never happen; we've already checked
self._exit_with_error(ReturnCode.MISSING_SETTING)
capo_settings = get_capo_settings(self.profile)
return LocationsReport(self.args, capo_settings)
def main():
......@@ -158,18 +121,12 @@ def main():
from the command line.
"""
parser = get_arg_parser()
try:
args = parser.parse_args()
except Exception as exc:
logging.error(f"{exc}")
return exc.value
except SystemExit as exc:
logging.error(f"{exc}")
return exc.value.code
logging.basicConfig(level=logging.DEBUG)
args = get_arg_parser().parse_args()
settings = get_capo_settings(args.profile)
datafetcher = DataFetcher(args, settings)
return datafetcher.run()
DataFetcher(args, settings).run()
if __name__ == "__main__":
......
apps/cli/executables/datafetcher/datafetcher/errors.py
View file @ 71b97f8f
""" Custom error definitions for data-fetcher """
import logging
import sys
import traceback
from enum import Enum
_LOG = logging.getLogger(__name__)
# pylint: disable=W1202
class Errors(Enum):
"""
Assorted constants and functions involving errors
"""
NO_PROFILE = 1
MISSING_SETTING = 2
LOCATION_SERVICE_TIMEOUT = 3
LOCATION_SERVICE_REDIRECTS = 4
LOCATION_SERVICE_ERROR = 5
NO_LOCATOR = 6
FILE_ERROR = 7
NGAS_SERVICE_TIMEOUT = 8
NGAS_SERVICE_REDIRECTS = 9
NGAS_SERVICE_ERROR = 10
SIZE_MISMATCH = 11
FILE_EXISTS_ERROR = 12
FILE_NOT_FOUND_ERROR = 13
class NoProfileException(Exception):
""" throw this if Capo profile can't be determined"""
class MissingSettingsException(Exception):
""" throw this if a required setting isn't found in command-line args """
class LocationServiceTimeoutException(Exception):
""" throw this if the location service takes too long to return """
class LocationServiceRedirectsException(Exception):
"""throw this if the location service
complains about too many redirects
"""
class LocationServiceErrorException(Exception):
""" throw this if the location service falls over """
class NGASServiceTimeoutException(Exception):
""" throw this if the NGAS service time out """
class NGASServiceRedirectsException(Exception):
""" throw this if the NGAS service complains about too many redirects """
class NGASServiceErrorException(Exception):
""" throw this if the NGAS service falls over """
class NoLocatorException(Exception):
"""throw this if no product locator could be determined from
command-line arguments
"""
class FileErrorException(Exception):
"""throw this if file retriever can't access or create the output
directory
"""
class SizeMismatchException(Exception):
"""throw this if the size of the file retrieved
doesn't match expected size
"""
TERMINAL_ERRORS = {
Errors.NO_PROFILE: "no CAPO profile provided",
Errors.MISSING_SETTING: "missing required setting",
Errors.LOCATION_SERVICE_TIMEOUT: "request to locator service timed out",
Errors.LOCATION_SERVICE_REDIRECTS: "too many redirects on locator service",
Errors.LOCATION_SERVICE_ERROR: "catastrophic error on locator service",
Errors.NO_LOCATOR: "product locator not found",
Errors.FILE_ERROR: "not able to open specified location file",
Errors.FILE_EXISTS_ERROR: "specified location file exists",
Errors.NGAS_SERVICE_TIMEOUT: "request to NGAS timed out",
Errors.NGAS_SERVICE_REDIRECTS: "too many redirects on NGAS service",
Errors.NGAS_SERVICE_ERROR: "catastrophic error on NGAS service",
Errors.SIZE_MISMATCH: "retrieved file not expected size",
Errors.FILE_NOT_FOUND_ERROR: "target directory or file not found",
}
def get_error_descriptions():
""" user-friendly reporting of errors """
result = "Return Codes:\n"
for error in Errors:
result += "\t{}: {}\n".format(error.value, TERMINAL_ERRORS[error])
errors = [
(1, "NoProfileException", "no CAPO profile provided"),
(2, "MissingSettingsException", "missing required setting"),
(
3,
"LocationServiceTimeoutException",
"request to locator service timed out",
),
(
4,
"LocationServiceRedirectsException",
"too many redirects on locator service",
),
(
5,
"LocationServiceErrorException",
"catastrophic error on locator service",
),
(6, "NoLocatorException", "product locator not found"),
(7, "FileErrorException", "not able to open specified location file"),
(8, "NGASServiceTimeoutException", "request to NGAS timed out"),
(
9,
"NGASServiceRedirectsException",
"too many redirects on NGAS service",
),
(10, "NGASServiceErrorException", "catastrophic error on NGAS service"),
(11, "SizeMismatchException", "retrieved file not expected size"),
(12, "FileExistsError", "specified location file exists"),
(13, "FileNotFoundError", "target directory or file not found"),
(14, "NGASFetchError", "trouble retrieving files from NGAS"),
]
class DataFetcherException(Exception):
code: int
message: str
def define_datafetcher_exception(code, name, message):
result = type(name, (DataFetcherException,), {})
result.code, result.message = code, message
return result
def terminal_error(errno):
"""report error, then throw in the towel"""
if errno in TERMINAL_ERRORS:
_LOG.error(TERMINAL_ERRORS[errno])
else:
_LOG.error("unspecified error {}".format(errno))
sys.exit(errno.value)
def exception_to_error(exception):
"""translate an exception to one of our custom errors"""
switcher = {
"NoProfileException": Errors.NO_PROFILE,
"MissingSettingsException": Errors.MISSING_SETTING,
"LocationServiceTimeoutException": Errors.LOCATION_SERVICE_TIMEOUT,
"LocationServiceRedirectsException": Errors.LOCATION_SERVICE_REDIRECTS,
"LocationServiceErrorException": Errors.LOCATION_SERVICE_ERROR,
"NoLocatorException": Errors.NO_LOCATOR,
"FileErrorException": Errors.FILE_ERROR,
"FileExistsError": Errors.FILE_EXISTS_ERROR,
"NGASServiceTimeoutException": Errors.NGAS_SERVICE_TIMEOUT,
"NGASServiceRedirectsException": Errors.NGAS_SERVICE_REDIRECTS,
"NGASServiceErrorException": Errors.NGAS_SERVICE_ERROR,
"SizeMismatchException": Errors.SIZE_MISMATCH,
"FileNotFoundError": Errors.FILE_NOT_FOUND_ERROR,
}
return switcher.get(exception.__class__.__name__)
def terminal_exception(exception):
"""Report this exception, then throw in the towel.
should be used by DataFetcher -ONLY-
"""
errorno = exception_to_error(exception)
_LOG.debug(traceback.format_exc())
_LOG.error(str(exception))
sys.exit(errorno.value)
# Define the exceptions
NoProfileException = define_datafetcher_exception(*errors[0])
MissingSettingsException = define_datafetcher_exception(*errors[1])
LocationServiceTimeoutException = define_datafetcher_exception(*errors[2])
LocationServiceRedirectsException = define_datafetcher_exception(*errors[3])
LocationServiceErrorException = define_datafetcher_exception(*errors[4])
NoLocatorException = define_datafetcher_exception(*errors[5])
FileErrorException = define_datafetcher_exception(*errors[6])
NGASServiceTimeoutException = define_datafetcher_exception(*errors[7])
NGASServiceRedirectsException = define_datafetcher_exception(*errors[8])
NGASServiceErrorException = define_datafetcher_exception(*errors[9])
SizeMismatchException = define_datafetcher_exception(*errors[10])
FileExistsError = define_datafetcher_exception(*errors[11])
FileNotFoundError = define_datafetcher_exception(*errors[12])
NGASFetchError = define_datafetcher_exception(*errors[13])
TERMINAL_ERROR_CODES = "Return Codes:\n"
for error_code, exception_name, message in errors:
TERMINAL_ERROR_CODES += f"\t{error_code}: {message}\n"
apps/cli/executables/datafetcher/datafetcher/file_retrievers.py
View file @ 71b97f8f
......@@ -4,10 +4,13 @@
Implementations of assorted file retrievers.
"""
import http
import logging
import os
from argparse import Namespace
from pathlib import Path
# pylint: disable=C0103, E0401, E0402, R0903, R0914, W1202, W1203
import requests
from bs4 import BeautifulSoup
......@@ -17,21 +20,21 @@ from .errors import (
FileErrorException,
MissingSettingsException,
)
from .utilities import RetrievalMode, Retryer, MAX_TRIES, SLEEP_INTERVAL_SECONDS, FlexLogger
from .utilities import RetrievalMode, Retryer, MAX_TRIES, SLEEP_INTERVAL_SECONDS
_DIRECT_COPY_PLUGIN = "ngamsDirectCopyDppi"
_STREAMING_CHUNK_SIZE = 8192
# pylint: disable=C0103, R0903, R0914
logger = logging.getLogger(__name__)
class NGASFileRetriever:
"""Responsible for getting a file out of NGAS
and saving it to the requested location.
"""
def __init__(self, args: Namespace, logger: FlexLogger):
def __init__(self, args: Namespace):
self.output_dir = args.output_dir
self._LOG = logger
self.logfile = self._LOG.logfile
self.dry_run = args.dry_run
self.force_overwrite = args.force
self.fetch_attempted = False
......@@ -56,7 +59,7 @@ class NGASFileRetriever:
func = (
self._copying_fetch if retrieve_method == RetrievalMode.COPY else self._streaming_fetch
)
retryer = Retryer(func, MAX_TRIES, SLEEP_INTERVAL_SECONDS, self._LOG)
retryer = Retryer(func, MAX_TRIES, SLEEP_INTERVAL_SECONDS)
try:
retryer.retry(download_url, destination, file_spec)
finally:
......@@ -113,7 +116,7 @@ class NGASFileRetriever:
:param file_spec: the file specification of that file
"""
if not self.dry_run:
self._LOG.debug(f"verifying fetch of {destination}")
logger.debug(f"verifying fetch of {destination}")
if not destination.exists():
raise NGASServiceErrorException(f"file not delivered to {destination}")
if file_spec["size"] != os.path.getsize(destination):
......@@ -122,9 +125,9 @@ class NGASFileRetriever:
f"expected {file_spec['size']}, "
f"got {os.path.getsize(destination)}"
)
self._LOG.debug("\tlooks good; sizes match")
logger.debug("\tlooks good; sizes match")
else:
self._LOG.debug("(This was a dry run; no files should have been fetched)")
logger.debug("(This was a dry run; no files should have been fetched)")
def _copying_fetch(self, args: list):
"""Pull a file out of NGAS via the direct copy plugin.
......@@ -138,7 +141,7 @@ class NGASFileRetriever:
"processingPars": "outfile=" + str(destination),
"file_version": file_spec["version"],
}
self._LOG.debug(
logger.debug(
"attempting copying download:\nurl: {}\ndestination: {}".format(
download_url, destination
)
......@@ -148,7 +151,7 @@ class NGASFileRetriever:
response = session.get(download_url, params=params)
if response.status_code != http.HTTPStatus.OK:
self._LOG.error("NGAS does not like this request:\n{}".format(response.url))
logger.error("NGAS does not like this request:\n{}".format(response.url))
soup = BeautifulSoup(response.text, features="lxml")
ngams_status = soup.NgamsStatus.Status
message = ngams_status.get("Message")
......@@ -163,7 +166,7 @@ class NGASFileRetriever:
)
else:
self._LOG.debug(
logger.debug(
f"if this were not a dry run, we would have been copying "
f'{file_spec["relative_path"]}'
)
......@@ -180,7 +183,7 @@ class NGASFileRetriever:
download_url, destination, file_spec = args
params = {"file_id": file_spec["ngas_file_id"], "file_version": file_spec["version"]}
self._LOG.debug(
logger.debug(
"attempting streaming download:\nurl: {}\ndestination: {}".format(
download_url, destination
)
......@@ -191,15 +194,12 @@ class NGASFileRetriever:
try:
response = session.get(download_url, params=params, stream=True)
with open(destination, "wb") as file_to_write:
# a word to the wise: DO NOT try to assign chunk size
# to a variable or to make it a constant. This will
# result in an incomplete download.
for chunk in response.iter_content(chunk_size=_STREAMING_CHUNK_SIZE):
file_to_write.write(chunk)
expected_size = file_spec["size"]
actual_size = os.path.getsize(destination)
if actual_size == 0:
raise FileErrorException(f"{Path(destination).name} " f"was not retrieved")
raise FileErrorException(f"{Path(destination).name} was not retrieved")
if actual_size != expected_size:
raise SizeMismatchException(
f"expected {Path(destination).name} "
......@@ -212,10 +212,10 @@ class NGASFileRetriever:
f"problem connecting with {download_url}"
) from c_err
except AttributeError as a_err:
self._LOG.warning(f"possible problem streaming: {a_err}")
logger.warning(f"possible problem streaming: {a_err}")
if response.status_code != http.HTTPStatus.OK:
self._LOG.error("NGAS does not like this request:\n{}".format(response.url))
logger.error("NGAS does not like this request:\n{}".format(response.url))
soup = BeautifulSoup(response.text, "lxml-xml")
ngams_status = soup.NgamsStatus.Status
message = ngams_status.get("Message")
......@@ -229,10 +229,10 @@ class NGASFileRetriever:
}
)
self._LOG.debug("retrieved {} from {}".format(destination, response.url))
logger.debug(f"retrieved {destination} from {response.url}")
else:
self._LOG.debug(
logger.debug(
f"if this were not a dry run, we would have been streaming "
f'{file_spec["relative_path"]}'
)
......
apps/cli/executables/datafetcher/datafetcher/locations_report.py
View file @ 71b97f8f
......@@ -6,14 +6,18 @@
"""
# pylint: disable=C0103, C0301, E0401, E0402, R0902, R0903, R0914, W0703, W1202, W1203
import copy
import http
import json
import logging
from argparse import Namespace
from typing import Dict
import requests
from . import errors
from .errors import (
LocationServiceTimeoutException,
LocationServiceRedirectsException,
......@@ -21,27 +25,29 @@ from .errors import (
NoLocatorException,
MissingSettingsException,
)
from .utilities import Cluster, RetrievalMode, validate_file_spec, FlexLogger
from .utilities import Cluster, RetrievalMode, validate_file_spec
# pylint: disable=C0103, R0902, R0903, R0914, W0703
logger = logging.getLogger(__name__)
class LocationsReport:
""" Builds a location report """
def __init__(self, logger: FlexLogger, args: Namespace, settings: Dict):
self._LOG = logger
self.logfile = logger.logfile
self._verbose = args and args.verbose
if self._verbose:
logger.set_verbose(True)
def __init__(self, args: Namespace, settings: Dict):
try:
self.verbose = args.verbose or False
except AttributeError:
# doesn't matter; verbose is going away soon
self.verbose = False
self._capture_and_validate_input(args, settings)
self._run()
def _capture_and_validate_input(self, args, settings):
if args is None:
raise MissingSettingsException(
"arguments (locator and/or report file, destination) " "are required"
"arguments (locator and/or report file, destination) are required"
)
self.args = args
if settings is None:
......@@ -139,15 +145,14 @@ class LocationsReport:
to pull in the location report.
"""
self._LOG.debug(f'fetching files from report "{self.location_file}"')
logger.debug(f'fetching files from report "{self.location_file}"')
# try:
try:
with open(self.location_file) as to_read:
result = json.load(to_read)
return result
except FileNotFoundError as err:
self._LOG.error(f"{err}")
raise
raise errors.FileNotFoundError() from err
def _get_location_report_from_service(self):
"""Use 'requests' to fetch the location report from the locator service.
......@@ -156,7 +161,7 @@ class LocationsReport:
"""
url = self.settings["locator_service_url"]
self._LOG.debug("fetching report from {} for {}".format(url, self.product_locator))
logger.debug("fetching report from {} for {}".format(url, self.product_locator))
# this is needed to prevent SSL errors when tests are run
# inside a Docker container
......@@ -172,8 +177,6 @@ class LocationsReport:
raise LocationServiceRedirectsException() from exc_re
except requests.exceptions.RequestException as ex:
raise LocationServiceErrorException(ex) from ex
except Exception as exc:
self._LOG.error(f"{exc}")
if response.status_code == http.HTTPStatus.OK:
return response.json()
......
apps/cli/executables/datafetcher/datafetcher/locations_report_refactor.py
View file @ 71b97f8f
......@@ -30,8 +30,7 @@ from .utilities import Cluster, RetrievalMode, validate_file_spec, get_arg_parse
# pylint: disable=C0103, R0902, R0903, R0914, W0703, W1203
_LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
REQUIRED_SETTINGS = {
"EDU.NRAO.ARCHIVE.DATAFETCHER.DATAFETCHERSETTINGS.LOCATORSERVICEURLPREFIX": "locator_service_url",
......@@ -79,7 +78,7 @@ class LocationsReportRefactor:
return self._add_retrieve_method_field(result)
except JSONDecodeError as js_err:
_LOG.error(f"Unable to parse {self.location_file}")
logger.error(f"Unable to parse {self.location_file}")
raise ReportFileParseException from js_err
def _get_location_report_from_file(self) -> Dict[str, str]:
......@@ -88,19 +87,19 @@ class LocationsReportRefactor:
:return:
"""
_LOG.debug(f'fetching files from report "{self.location_file}"')
logger.debug(f'fetching files from report "{self.location_file}"')
try:
with open(self.location_file) as to_read:
result = json.load(to_read)
return result
except FileNotFoundError as err:
_LOG.error(f"{err}")
logger.error(f"{err}")
raise
except JSONDecodeError as js_err:
_LOG.error(f"Unable to parse {self.location_file}")
logger.error(f"Unable to parse {self.location_file}")
raise ReportFileParseException from js_err
except Exception as exc:
_LOG.error(f"Problem getting location report from f{self.location_file}: {exc}")
logger.error(f"Problem getting location report from f{self.location_file}: {exc}")
raise
def _get_location_report_from_service(self):
......@@ -112,7 +111,7 @@ class LocationsReportRefactor:
"edu.nrao.archive.datafetcher.DataFetcherSettings.locatorServiceUrlPrefix"
)
_LOG.debug(f"fetching report from {url} for {self.product_locator}")
logger.debug(f"fetching report from {url} for {self.product_locator}")
# this is needed to prevent SSL errors when tests are run
# inside a Docker container
......@@ -129,7 +128,7 @@ class LocationsReportRefactor:
except requests.exceptions.RequestException as ex:
raise LocationServiceErrorException(ex) from ex
except Exception as exc:
_LOG.error(f"{exc}")
logger.error(f"{exc}")
if response.status_code == http.HTTPStatus.OK:
return response.json()
......
apps/cli/executables/datafetcher/datafetcher/project_fetcher.py
View file @ 71b97f8f
......@@ -3,50 +3,46 @@
""" Implementations of assorted product fetchers """
import copy
import sys
import logging
from argparse import Namespace
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from pprint import pprint
from typing import Dict
from datafetcher.errors import NGASServiceErrorException
from datafetcher.return_codes import ReturnCode
# pylint: disable=C0103, E0402, R0201, R0902, R0903, W0703, W1202, W1203
from datafetcher.errors import NGASFetchError
from .file_retrievers import NGASFileRetriever
from .utilities import FlexLogger
# pylint: disable=C0103, R0201, R0902, R0903, W0703
logger = logging.getLogger(__name__)
class BaseFetcher:
""" This is a base class for fetchers. """
def __init__(
self, args: Namespace, df_capo_settings: dict, logger: FlexLogger, servers_report: dict
):
def __init__(self, args: Namespace, df_capo_settings: dict, servers_report: dict):
self.args = args
self.output_dir = self.args.output_dir
self._LOG = logger
self.force_overwrite = args.force
self.dry_run = args.dry_run
self.servers_report = servers_report
self.settings = df_capo_settings
self.ngas_retriever = NGASFileRetriever(self.args, self._LOG)
self.ngas_retriever = NGASFileRetriever(self.args)
self.retrieved = []
self.num_files_retrieved = 0
def retrieve_files(self, server, retrieve_method, file_specs):
""" This is the part where we actually fetch the files. """
retriever = NGASFileRetriever(self.args, self._LOG)
retriever = NGASFileRetriever(self.args)
num_files = len(file_specs)
count = 0
for file_spec in file_specs:
count += 1
self._LOG.debug(
logger.debug(
f">>> retrieving {file_spec['relative_path']} "
f"({file_spec['size']} bytes, "
f"no. {count} of {num_files})...."
......@@ -62,17 +58,15 @@ class SerialFetcher(BaseFetcher):
"""
def __init__(
self, args: Namespace, df_capo_settings: Dict, logger: FlexLogger, servers_report: Dict
):
super().__init__(args, df_capo_settings, logger, servers_report)
def __init__(self, args: Namespace, df_capo_settings: Dict, servers_report: Dict):
super().__init__(args, df_capo_settings, servers_report)
def run(self):
""" fetch 'em """
self._LOG.debug("writing to {}".format(self.output_dir))
self._LOG.debug("dry run: {}".format(self.dry_run))
self._LOG.debug(f"force overwrite: {self.force_overwrite}")
logger.debug("writing to {}".format(self.output_dir))
logger.debug("dry run: {}".format(self.dry_run))
logger.debug(f"force overwrite: {self.force_overwrite}")
for server in self.servers_report:
self.retrieve_files(
server,
......@@ -84,10 +78,8 @@ class SerialFetcher(BaseFetcher):
class ParallelFetcher(BaseFetcher):
""" Pull the files out in parallel; try to be clever about it. """
def __init__(
self, args: Namespace, df_capo_settings: dict, logger: FlexLogger, servers_report: dict
):
super().__init__(args, df_capo_settings, logger, servers_report)
def __init__(self, args: Namespace, df_capo_settings: dict, servers_report: dict):
super().__init__(args, df_capo_settings, servers_report)
self.num_files_expected = self._count_files_expected()
self.bucketized_files = self._bucketize_files()
......@@ -134,8 +126,7 @@ class ParallelFetcher(BaseFetcher):
def fetch_bucket(self, bucket):
""" Grab the files in this bucket """
file_sizes = [file["size"] for file in bucket["files"]]
pprint(f"retrieving files {file_sizes} from {bucket['server']}")
self._LOG.debug(
logger.debug(
f"{bucket['retrieve_method']} "
f"{len(bucket['files'])} files from "
f"{bucket['server']}...."
......@@ -149,10 +140,8 @@ class ParallelFetcher(BaseFetcher):
def run(self):
""" Fetch all the files for the product locator """
print(f"running {self.__class__.__name__}")
if self.args.dry_run:
self._LOG.debug("This is a dry run; files will not be fetched")
return 0
logger.debug("This is a dry run; files will not be fetched")
with ThreadPoolExecutor() as executor:
results = executor.map(self.fetch_bucket, self.bucketized_files)
......@@ -162,21 +151,13 @@ class ParallelFetcher(BaseFetcher):
print(f"result has arrived: {result}")
self.num_files_retrieved += result
if self.num_files_retrieved != self.num_files_expected:
self._LOG.error(
logger.error(
f"{self.num_files_expected} files expected, "
f"but only {self.num_files_retrieved} retrieved"
)
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
# successful retrieval
print("returning")
return 0
except (FileExistsError, NGASServiceErrorException) as exc:
self._LOG.error(f"{exc}")
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
except AttributeError:
# (This error sometimes gets thrown after all files
# actually -have- been retrieved. I blame the NGAS API.)
# actually -have- been retrieved. I blame the NGAS API. - JLG)
output_path = Path(self.args.output_dir)
files = [
......@@ -187,17 +168,8 @@ class ParallelFetcher(BaseFetcher):
and not str(file).endswith(".log")
]
if len(files) < self.num_files_expected:
self._LOG.error(
logger.error(
f"{self.num_files_expected} files expected, but only "
f"{self.num_files_retrieved} found"
)
self._exit_with_error(ReturnCode.CATASTROPHIC_REQUEST_ERROR)
return 0
except Exception as exc:
self._LOG.error(f"{exc}")
self._exit_with_error(ReturnCode.NGAS_FETCH_ERROR)
def _exit_with_error(self, return_code: ReturnCode):
sys.exit(return_code.value["code"])
raise NGASFetchError
apps/cli/executables/datafetcher/datafetcher/return_codes.py
View file @ 71b97f8f
""" data-fetcher return codes as specified in README and usage string """
from enum import Enum
class ReturnCode(Enum):
""" Canonical data-fetcher exit code values """
SUCCESS = {"code": 0, "text": "success"}
MISSING_PROFILE = {"code": 1, "text": "no CAPO profile provided"}
MISSING_SETTING = {"code": 2, "text": "missing required setting"}
LOCATOR_SERVICE_TIMEOUT = {"code": 3, "text": "request to locator service timed out"}
TOO_MANY_SERVICE_REDIRECTS = {"code": 4, "text": "too many redirects on locator service"}
CATASTROPHIC_REQUEST_ERROR = {"code": 5, "text": "catastrophic error on request service"}
PRODUCT_LOCATOR_NOT_FOUND = {"code": 6, "text": "product locator not found"}
CANNOT_OPEN_LOCATION_FILE = {"code": 7, "text": "not able to open specified location file"}
NGAS_FETCH_ERROR = {"code": 8, "text": "error fetching file from NGAS server"}
SIZE_MISMATCH = {"code": 9, "text": "retrieved file not expected size"}
apps/cli/executables/datafetcher/datafetcher/utilities.py
View file @ 71b97f8f
......@@ -8,28 +8,26 @@
"""
import argparse
from enum import Enum
import logging
import os
import pathlib
from enum import Enum
from time import time
import time
from typing import Callable
# pylint:disable=C0301, C0303, C0415, E0401, E0402, R0903, W0212, W1202, W0404, W0621, W1203
import psycopg2 as pg
from pycapo import CapoConfig
from .errors import (
get_error_descriptions,
NoProfileException,
MissingSettingsException,
NGASServiceErrorException,
SizeMismatchException,
TERMINAL_ERROR_CODES,
)
# pylint:disable=C0303, C0415, R0903, W0212, W0404, W0621, W1203
LOG_FORMAT = "%(module)s.%(funcName)s, %(lineno)d: %(message)s"
MAX_TRIES = 10
......@@ -50,7 +48,7 @@ SERVER_SPEC_KEYS = ["server", "location", "cluster", "retrieve_method"]
_PROLOGUE = """Retrieve a product (a science product or an ancillary product)
from the NRAO archive, either by specifying the product's locator or by
providing the path to a product locator report."""
_EPILOGUE = get_error_descriptions()
_EPILOGUE = TERMINAL_ERROR_CODES
# This is a dictionary of required CAPO settings
# and the attribute names we'll store them as.
......@@ -61,6 +59,8 @@ REQUIRED_SETTINGS = {
"EDU.NRAO.ARCHIVE.WORKFLOW.CONFIG.REQUESTHANDLERSETTINGS.DOWNLOADDIRECTORY": "download_dir",
}
logger = logging.getLogger(__name__)
def path_is_accessible(path: pathlib.Path) -> bool:
"""
......@@ -88,7 +88,9 @@ def get_arg_parser():
cwd = pathlib.Path().absolute()
parser = argparse.ArgumentParser(
description=_PROLOGUE, epilog=_EPILOGUE, formatter_class=argparse.RawTextHelpFormatter
description=_PROLOGUE,
epilog=_EPILOGUE,
formatter_class=argparse.RawTextHelpFormatter,
)
# Can't find a way of clearing the action groups
# without hitting an internal attribute.
......@@ -121,14 +123,14 @@ def get_arg_parser():
default=cwd,
help="output directory, default current dir",
)
optional_group.add_argument(
"--verbose",
action="store_true",
required=False,
dest="verbose",
default=False,
help="make a lot of noise",
)
# optional_group.add_argument(
# "--verbose",
# action="store_true",
# required=False,
# dest="verbose",
# default=False,
# help="make a lot of noise",
# )
optional_group.add_argument(
"--force",
action="store_true",
......@@ -160,19 +162,19 @@ def get_capo_settings(profile: str):
:param profile: the profile to use
:return: a bunch of settings
"""
result = dict()
if profile is None:
if not profile:
raise NoProfileException("CAPO_PROFILE required; none provided")
capo = CapoConfig(profile=profile)
result = dict()
for setting in REQUIRED_SETTINGS:
setting = setting.upper()
try:
value = capo[setting]
result[REQUIRED_SETTINGS[setting]] = capo[setting]
except KeyError as k_err:
raise MissingSettingsException(
'missing required setting "{}"'.format(setting)
f'missing required setting "{setting}" with profile "{profile}"'
) from k_err
result[REQUIRED_SETTINGS[setting]] = value
return result
......@@ -226,9 +228,7 @@ class ProductLocatorLookup:
password=self.credentials["jdbcPassword"],
) as conn:
cursor = conn.cursor()
sql = (
"SELECT science_product_locator " "FROM science_products " "WHERE external_name=%s"
)
sql = "SELECT science_product_locator FROM science_products WHERE external_name=%s"
cursor.execute(
sql,
(external_name,),
......@@ -237,75 +237,12 @@ class ProductLocatorLookup:
return product_locator[0]
class FlexLogger:
"""
This class wraps a logger, adding the ability to specify
logging level as warning or debug based on the "verbose" flag
"""
def __init__(self, class_name: str, output_dir: pathlib.Path, verbose=False):
"""
Set up logging.
:param class_name: class to be logged
:param output_dir: where log is to be written
:param verbose: if true, additional output
"""
if class_name is None:
raise MissingSettingsException("class name is required")
log_pathname = f"{output_dir}/{class_name}_{str(time())}.log"
try:
self.logfile = pathlib.Path(output_dir, log_pathname)
self.logger = logging.getLogger(str(self.logfile))
self.verbose = verbose
handler = logging.FileHandler(self.logfile)
formatter = logging.Formatter(LOG_FORMAT)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
except (PermissionError, FileNotFoundError) as err:
self.logger.error(f"Problem creating logger for {class_name}: {err}")
raise
level = logging.DEBUG if self.verbose else logging.WARN
self.logger.setLevel(level)
def set_verbose(self, verbose: bool) -> None:
"""
Specify verbose logging:
True == debug and above
False == warnings and above
:param verbose:
:return:
"""
self.verbose = verbose
def debug(self, message: str):
""" log a debug message """
if self.verbose:
self.logger.debug(message)
def warning(self, message: str):
""" log a warning message """
self.logger.warning(message)
def error(self, message: str):
""" log an error """
self.logger.error(message)
class Retryer:
"""
Retry executing a function, or die trying
"""
def __init__(self, func: Callable, max_tries: int, sleep_interval: int, logger: FlexLogger):
def __init__(self, func: Callable, max_tries: int, sleep_interval: int):
self.func = func
self.num_tries = 0
self.max_tries = max_tries
......@@ -320,9 +257,6 @@ class Retryer:
:param args:
:return:
"""
import time
while self.num_tries < self.max_tries and not self.complete:
self.num_tries += 1
......
apps/cli/executables/datafetcher/test/df_pytest_arg_utils.py 0 → 100644
View file @ 71b97f8f
""" Utilities for passing arguments to datafetcher in tests """
import os
from pathlib import Path
from typing import List
from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.utilities import get_arg_parser, path_is_accessible
def parse_args(args: List[str]):
"""
Wrapper around parser.parse_args() so in the event of a foo
an exception is thrown rather than sys.exe
:param args:
:return:
"""
confirm_complete_args(args)
get_arg_parser().parse_args(args)
def confirm_complete_args(args: List[str]):
"""
Let's scrutinize the args -before- calling parse_args()
so we can differentiate among errors.
:param args:
:return:
"""
# we must have a profile
assert "--profile" in args or "CAPO_PROFILE" in os.environ
if "--profile" not in args and "CAPO_PROFILE" in os.environ:
raise NoProfileException("Capo profile is required.")
# we must have an output dir....
assert "--output-dir" in args
# ... and it must exist
args_iter = iter(args)
args_dict = dict(zip(args_iter, args_iter))
output_dir_arg = args_dict["--output-dir"]
output_dir = Path(output_dir_arg)
if not output_dir.exists():
# did they just forget to specify it?
if output_dir_arg.startswith("--"):
raise MissingSettingsException("--output-dir is required")
raise FileNotFoundError(f"output dir '{output_dir_arg}' not found")
if not path_is_accessible(output_dir):
raise FileNotFoundError(f"permission denied on {output_dir}")
# we must have EITHER a product locator OR a locations file....
if "--product-locator" not in args_dict.keys() and "--location-file" not in args_dict.keys():
raise MissingSettingsException("either product locator or locations file required")
if "--product-locator" in args_dict.keys() and "--location-file" in args_dict.keys():
raise MissingSettingsException("product locator OR locations file required -- not both")
if "--product-locator" in args:
product_locator = args_dict["--product-locator"]
assert product_locator
else:
locations_file = args_dict["--location-file"]
if not Path(locations_file).exists():
raise FileNotFoundError(f"{locations_file} not found")
apps/cli/executables/datafetcher/test/df_pytest_utils.py
View file @ 71b97f8f
......@@ -4,45 +4,25 @@
""" Various conveniences for use and re-use in test cases """
import json
import logging
import os
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, str(Path(".").absolute()))
sys.path.insert(0, str(Path("..").absolute()))
# TODO: Some Fine Day: this duplicates same function in package tester.
# CAVEAT PROGRAMMOR: attempts to centralize it have resulted in tears.
def get_project_root() -> Path:
"""
Get the root of this project.
:return:
"""
my_path = Path(__file__)
path = my_path
while not path.name.endswith("workspaces") and not path.name.endswith("packages"):
path = path.parent
return path
from typing import List, Dict
import pytest
from pycapo import CapoConfig
# pylint: disable=C0115, C0116, C0200, R0902, R0903, R0914, R1721, W0212, W0613, W0621, W0703, W1203
sys.path.insert(0, str(get_project_root()))
from shared.workspaces.test.test_data.utilities import (
# pylint: disable=C0115, C0116, C0200, R0902, R0903, R0914, R1721, W0212, W0613,
# pylint: disable=W0621, W0703, W1203
from .df_testdata_utils import (
get_locations_report,
get_test_data_dir,
)
from datafetcher.datafetcher import DataFetcher
from datafetcher.return_codes import ReturnCode
from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.locations_report import LocationsReport
from datafetcher.utilities import (
......@@ -54,8 +34,9 @@ from datafetcher.utilities import (
)
TEST_PROFILE = "docker"
MISSING_SETTING = ReturnCode.MISSING_SETTING.value["code"]
MISSING_PROFILE = ReturnCode.MISSING_PROFILE.value["code"]
# set this to False when debugging one or more tests
# so as not to have to sit thru every test;
# comment out the target test(s)' "@pytest.skip"
RUN_ALL = True
LOCATION_REPORTS = {
......@@ -259,7 +240,7 @@ def make_tempdir() -> Path:
umask = os.umask(0o000)
top_level = tempfile.mkdtemp(prefix="datafetcher_test_", dir="/var/tmp")
os.umask(umask)
yield top_level
return top_level
@pytest.fixture(scope="session")
......@@ -322,41 +303,27 @@ def launch_datafetcher(args: list, df_capo_settings: dict) -> int:
"""
if args is None or len(args) == 0:
return MISSING_SETTING
try:
namespace = evaluate_args_and_capo(args, df_capo_settings)
fetcher = DataFetcher(namespace, df_capo_settings)
return fetcher.run()
except SystemExit as exc:
if hasattr(exc, "value"):
return exc.value.code if hasattr(exc.value, "code") else exc.value
if hasattr(exc, "code"):
return exc.code
raise MissingSettingsException
raise
except (KeyError, NoProfileException) as exc:
logging.error(f"{exc}")
return MISSING_PROFILE
except Exception as exc:
pytest.fail(f"{exc}")
namespace = evaluate_args_and_capo(args, df_capo_settings)
datafetcher = DataFetcher(namespace, df_capo_settings)
datafetcher.run()
def evaluate_args_and_capo(args: list, capo_settings: dict):
def evaluate_args_and_capo(args: List[str], capo_settings: Dict[str, str]):
if args is None or len(args) == 0:
sys.exit(MISSING_SETTING)
raise MissingSettingsException
profile = get_profile_from_args(args)
if profile is None:
profile = capo_settings["profile"]
if profile is None:
sys.exit(MISSING_PROFILE)
raise NoProfileException
else:
args["profile"] = profile
namespace = get_arg_parser().parse_args(args)
return namespace
return get_arg_parser().parse_args(args)
def get_profile_from_args(args: list) -> str:
......
apps/cli/executables/datafetcher/test/df_testdata_utils.py 0 → 100644
View file @ 71b97f8f
""" Helper functions for download product testing """
# NB: this is a copy of the file in shared/workspaces/test/test_data
# Directly accessing that file seems to be troublesome for the test suite. -DKL
import json
import sys
from enum import Enum
from pathlib import Path
from typing import Dict, Tuple
DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
DATE_FORMAT = "%Y-%m-%d"
def get_report_file(basename: str):
"""
Get a locations file from our collection in test_data,
given a basename (.json filename w/o extension)
:param basename:
:return:
"""
test_data_dir = get_test_data_dir()
for file in test_data_dir.glob(basename.upper() + ".json"):
# (there should be only one for this basename)
return file
return None
def get_exec_block_details_from_loc_report(prefix: str, exec_blocks: list):
"""
Fetch and read locations report indicated by basename;
for filegroup IDs of exec blocks, return file info dict
and total size of files
:param prefix:
:param exec_blocks:
:return:
"""
file_info = dict()
total_size = 0
for exec_block in exec_blocks:
basename = prefix + str(exec_block.filegroup_id)
locations_report = None
try:
locations_report = get_locations_report(basename)
except FileNotFoundError:
# special case: GBT product
if basename.startswith("AGBT17B_044"):
locations_report = get_locations_report("AGBT17B_044_02")
total_size += locations_report["aggregate_size"]
for file_spec in locations_report["files"]:
filename = file_spec["ngas_file_id"]
size = file_spec["size"]
file_info[filename] = size
return file_info, total_size
def get_test_data_dir():
""" where's our test data? """
top_level_subdirs = sys.path
shared_ws_src = None
for pathname in top_level_subdirs:
if "shared/workspaces" in pathname:
shared_ws_src = pathname
break
shared_wksp = Path(shared_ws_src).parent
# test data will be a few levels under shared_wksp
for item in shared_wksp.rglob("location_files"):
assert item.is_dir()
return item
return None
def get_file_info_from_json_file(json_filename: str) -> dict:
""" Pluck file information from a .json location file """
to_read = None
for file in Path.cwd().rglob(json_filename):
to_read = file
break
assert to_read is not None
with open(to_read, "r") as content:
file_info = json.loads(content.read())
return file_info
def get_file_info_from_loc_report(locations_info: Dict[str, str]) -> Tuple:
"""
Grab the data we'll need to retrieve the files.
:param locations_info: data from locations report
:return:
"""
file_info = dict()
total_size = 0
total_size += locations_info["aggregate_size"]
for file_spec in locations_info["files"]:
filename = file_spec["ngas_file_id"]
size = file_spec["size"]
file_info[filename] = size
return file_info, total_size
def get_locations_report(basename: str) -> Dict[str, str]:
"""
Get a locations report from a file in test_data.
:param basename: locations report filename w/o exension
:return:
"""
report_path = get_report_file(basename)
if report_path is not None:
with open(report_path, "r") as content:
locations_info = json.loads(content.read())
return locations_info
raise FileNotFoundError(f'{basename.upper() + ".json"} not found')
class Deliverable(Enum):
""" Type of product to be delivered: SDM, BDF, TAR, etc. """
SDM = "SDM"
BDF = "BDF"
MS = "MS"
CMS = "CMS"
IMG = "IMG"
TAR = "TAR"
# VLBA
IDIFITS = "IDIFITS"
@staticmethod
def from_str(key: str):
for dtype in Deliverable:
if dtype.value == key:
return dtype
return None
class DeliverableProduct:
""" Wraps a deliverable and its locations report info. """
def __init__(self, dtype: Deliverable, file_info: dict):
self.type = dtype
self.file_info = file_info
apps/cli/executables/datafetcher/test/fake_archive_service.py
View file @ 71b97f8f
......@@ -7,8 +7,8 @@ from pathlib import Path
from datafetcher.locations_report_refactor import LocationsReportRefactor, UnknownLocatorException
# pylint: disable=C0301, R0903
_LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
PROFILE = "docker"
......
apps/cli/executables/datafetcher/test/mock_data_fetcher.py
View file @ 71b97f8f
""" for testing the attempt to copy rather than stream files """
import sys
import logging
from argparse import Namespace
# pylint: disable=C0103, C0301, E0401, E0402, R0201, R0902, R0903, W0621
from datafetcher.locations_report import LocationsReport
from datafetcher.project_fetcher import ParallelFetcher
from datafetcher.return_codes import ReturnCode
from datafetcher.utilities import get_capo_settings, ExecutionSite, FlexLogger
from datafetcher.utilities import get_capo_settings, ExecutionSite
from datafetcher.errors import MissingSettingsException
from .df_pytest_utils import TEST_PROFILE
# pylint: disable=C0103, R0201, R0902, R0903, W0621
logger = logging.getLogger(__name__)
class MockProdDataFetcher:
......@@ -18,29 +19,15 @@ class MockProdDataFetcher:
def __init__(self, args: Namespace, settings: dict):
if args is None or settings is None:
self._exit_with_error(ReturnCode.MISSING_SETTING)
raise MissingSettingsException()
self.args = args
self.settings = settings
self.verbose = self.args.verbose
self.output_dir = args.output_dir
self.profile = args.profile
self._LOG = FlexLogger(self.__class__.__name__, self.output_dir, self.verbose)
try:
self.locations_report = self._get_locations()
self.servers_report = self.locations_report.servers_report
except SystemExit:
if args.location_file:
self._exit_with_error(ReturnCode.CANNOT_OPEN_LOCATION_FILE)
else:
self._exit_with_error(ReturnCode.PRODUCT_LOCATOR_NOT_FOUND)
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected {type(exc)} during init: {exc}")
raise
self.locations_report = self._get_locations()
self.servers_report = self.locations_report.servers_report
def _get_locations(self):
"""
......@@ -52,7 +39,7 @@ class MockProdDataFetcher:
capo_settings = get_capo_settings(TEST_PROFILE)
capo_settings["execution_site"] = ExecutionSite.DSOC.value
return LocationsReport(self._LOG, self.args, capo_settings)
return LocationsReport(self.args, capo_settings)
def run(self):
"""
......@@ -60,14 +47,4 @@ class MockProdDataFetcher:
:return:
"""
try:
return ParallelFetcher(self.args, self.settings, self._LOG, self.servers_report).run()
except SystemExit as exc:
self._LOG.error(f"{exc}")
raise
except Exception as exc:
self._LOG.error(f">>> throwing unexpected exception during run: {exc}")
raise
def _exit_with_error(self, return_code: ReturnCode):
sys.exit(return_code.value["code"])
return ParallelFetcher(self.args, self.settings, self.servers_report).run()
apps/cli/executables/datafetcher/test/test_df_function.py
View file @ 71b97f8f
......@@ -2,19 +2,19 @@
from pathlib import Path
# pylint: disable=E0401, E0402, W0511
import pytest
import sys
sys.path.insert(0, str(Path(".").absolute()))
from .df_pytest_utils import get_project_root
project_root = get_project_root()
sys.path.insert(0, str(project_root))
from datafetcher.errors import (
NGASFetchError,
MissingSettingsException,
NGASServiceErrorException,
)
# pylint: disable=C0115, C0116, C0200, C0415, R0801, R0902, R0903, R0914, R1721, W0212, W0611, W0613, W0621, W0703, W1203
# pylint: disable=C0115, C0116, C0200, C0415, R0801, R0902, R0903, R0914, R1721,
# pylint: disable=W0212, W0611, W0613, W0621, W0703, W1203
from datafetcher.datafetcher import DataFetcher, ReturnCode
from datafetcher.datafetcher import DataFetcher
from datafetcher.utilities import (
get_arg_parser,
ProductLocatorLookup,
......@@ -23,6 +23,9 @@ from datafetcher.utilities import (
Cluster,
)
from .test_df_return_status import try_to_launch_df
# N.B. these are all in use; SonarLint just doesn't get it
from .df_pytest_utils import (
TEST_PROFILE,
......@@ -35,15 +38,14 @@ from .df_pytest_utils import (
make_tempdir,
RUN_ALL,
confirm_retrieve_mode_copy,
get_mini_exec_block,
)
_LOCATION_FILENAME = "locations.json"
_ASDM_XML = "ASDM.xml"
_EB_EXTERNAL_NAME = "sysstartS.58955.83384832176"
# set this to False when debugging one or more tests
# so as not to have to sit thru every test;
# comment out the target test(s)' @pytest.skip
# TODO: log this in WS-179-3
print(f">>> RUNNING ALL TESTS: {RUN_ALL}")
......@@ -57,16 +59,6 @@ def test_settings_setup(settings):
assert isinstance(settings.test_data, dict)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_usage_statement_makes_sense():
""" Ensure that the datafetcher's "usage" statement is as we expect """
usage = DataFetcher._build_usage_message()
assert usage.startswith("Usage:")
lines = usage.split("\n")
assert len(lines) >= len(ReturnCode) + 1
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_nothing_retrieved_if_dry_locator(make_tempdir, settings):
""" Simulates dry run with product locator """
......@@ -80,8 +72,7 @@ def test_nothing_retrieved_if_dry_locator(make_tempdir, settings):
TEST_PROFILE,
"--dry-run",
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == 0
launch_datafetcher(args, settings.capo_settings)
tempdir_files = Path(make_tempdir).iterdir()
for file in tempdir_files:
if not str(file).endswith(".log") and not str(file).endswith(".json"):
......@@ -111,7 +102,7 @@ def test_nothing_retrieved_if_dry_file(make_tempdir, settings):
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_force_flag_overwrites_existing_file(make_tempdir, settings):
def test_force_flag_overwrites_existing_file(make_tempdir, capo_settings):
top_level = Path(make_tempdir)
location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME)
dest_dir = Path(top_level, _EB_EXTERNAL_NAME)
......@@ -131,19 +122,11 @@ def test_force_flag_overwrites_existing_file(make_tempdir, settings):
str(top_level),
"--force",
]
try:
launch_datafetcher(args, settings.capo_settings)
except SystemExit as ex:
pytest.fail(f"{ex}")
raise
except Exception as exc:
pytest.fail(f"{exc}")
raise
sizes = dict()
launch_datafetcher(args, capo_settings)
# go thru destination directory recursively
# and get everybody's size
sizes = dict()
for file in dest_dir.rglob("*"):
sizes[str(file)] = file.stat().st_size
assert len(sizes) == 37
......@@ -152,16 +135,24 @@ def test_force_flag_overwrites_existing_file(make_tempdir, settings):
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_no_overwrite_without_force(make_tempdir, settings):
def test_no_overwrite_without_force(make_tempdir, capo_settings):
top_level = Path(make_tempdir)
location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME)
locations_dict = get_mini_exec_block()
files = locations_dict["files"]
num_files_expected = 37
assert len(files) == num_files_expected
dest_dir = Path(top_level, _EB_EXTERNAL_NAME)
dest_dir.mkdir(parents=True, exist_ok=True)
# make a fake file to be overwritten
fake_file = dest_dir / _ASDM_XML
if fake_file.exists():
fake_file.unlink()
with open(fake_file, "w") as to_write:
to_write.write("dang! what a kick in the rubber parts!")
assert fake_file.stat().st_size == 38
args = [
"--location-file",
......@@ -172,91 +163,38 @@ def test_no_overwrite_without_force(make_tempdir, settings):
str(top_level),
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.NGAS_FETCH_ERROR.value["code"]
sizes = dict()
for file in dest_dir.rglob("*"):
sizes[str(file)] = file.stat().st_size
assert len(sizes) < 37
fake_size = fake_file.stat().st_size
assert fake_size == 38
with pytest.raises(FileExistsError):
try_to_launch_df(capo_settings, args)
@pytest.mark.skip("verbose mode goes away in WS-179 and isn't in use now")
def test_more_output_when_verbose(make_tempdir, settings):
top_level = Path(make_tempdir)
location_file = get_mini_locations_file(top_level / _LOCATION_FILENAME)
args = [
"--location-file",
str(location_file),
"--profile",
TEST_PROFILE,
"--output-dir",
str(make_tempdir),
"--verbose",
retrieved = [
file for file in top_level.rglob("*") if file.is_file() and not str(file).endswith(".json")
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.SUCCESS.value["code"]
num_files_expected = 37
retrieved = [file for file in top_level.rglob("*") if file.is_file()]
assert num_files_expected == len(retrieved) - 2
verbose_logfile = None
for file in retrieved:
if str(file).endswith(".log"):
verbose_logfile = file
break
assert verbose_logfile is not None
verbose_log_size = verbose_logfile.stat().st_size
assert verbose_log_size > 0
num_not_too_big_files_expected = 28
assert len(retrieved) == num_not_too_big_files_expected
# get rid of all the files we downloaded, plus the log
deleted = [file.unlink() for file in retrieved if not str(file).endswith(".json")]
assert len(deleted) >= num_files_expected
# same download, but without verbose logging
args = [
"--location-file",
str(location_file),
"--profile",
TEST_PROFILE,
"--output-dir",
str(make_tempdir),
deleted = [
file.unlink() for file in retrieved if file.is_file() and not str(file).endswith(".json")
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.SUCCESS.value["code"]
retrieved = [file for file in top_level.rglob("*")]
assert len(retrieved) == num_files_expected + 3
logfile = None
for file in retrieved:
if str(file).endswith(".log"):
logfile = file
break
assert logfile is not None
logsize = logfile.stat().st_size
# successful download, non-verbose logging,
# should result in zero-size log file
assert logsize == 0
assert len(deleted) >= num_not_too_big_files_expected
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings):
"""We set profile to dsoc-prod here so as to force the DF
def test_copy_attempt_throws_fetch_error(make_tempdir, settings):
"""
We set profile to dsoc-prod here so as to force the DF
to try to copy rather than stream
:param make_tempdir: temp directory fixture
:param settings: Capo settings for datafetcher
:return:
"""
# N.B. can't do this import with the rest of the imports up top,
# because test_df_return_codes not yet initialized
from .test_df_return_codes import we_are_in_docker
from .test_df_return_status import we_are_in_docker
if we_are_in_docker():
# this test doesn't work in a docker container:
......@@ -279,24 +217,23 @@ def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings):
prod_profile,
]
namespace = get_arg_parser().parse_args(args)
fetcher = DataFetcher(namespace, settings.capo_settings)
datafetcher = DataFetcher(namespace, settings.capo_settings)
servers_report = fetcher.servers_report
servers_report = datafetcher.servers_report
confirm_retrieve_mode_copy(servers_report)
# let's try just one file so we're not sitting here all day
for server in servers_report:
entry = servers_report[server]
servers_report = {server: entry}
fetcher.servers_report = servers_report
assert fetcher.servers_report[server] is not None
files = fetcher.servers_report[server]["files"]
fetcher.servers_report[server]["files"] = [files[0]]
datafetcher.servers_report = servers_report
assert datafetcher.servers_report[server] is not None
files = datafetcher.servers_report[server]["files"]
datafetcher.servers_report[server]["files"] = [files[0]]
break
with pytest.raises(SystemExit) as exc:
fetcher.run()
assert exc.value.code == ReturnCode.CATASTROPHIC_REQUEST_ERROR.value["code"]
with pytest.raises(NGASFetchError):
datafetcher.run()
finally:
if props_file.exists():
props_file.unlink()
......@@ -314,14 +251,13 @@ def test_dies_with_bad_server_info(make_tempdir, settings):
"--location-file",
str(location_file),
]
try:
with pytest.raises(NGASServiceErrorException):
launch_datafetcher(args, settings.capo_settings)
except Exception as exc:
assert exc.value.code == ReturnCode.NGAS_FETCH_ERROR["code"]
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_missing_setting_error_on_bad_destination(settings):
def test_missing_setting_exc_on_bad_destination(settings):
args = [
"--profile",
TEST_PROFILE,
......@@ -330,35 +266,12 @@ def test_missing_setting_error_on_bad_destination(settings):
"--output-dir",
"floob",
]
try:
with pytest.raises(MissingSettingsException):
launch_datafetcher(args, settings.capo_settings)
except Exception as exc:
assert exc.value.code == ReturnCode.MISSING_SETTING["code"]
def write_fake_file(destination: Path, file_info: dict):
filename = file_info["ngas_file_id"]
path = Path(destination, filename)
with open(path, "w") as file:
file.write(f'{str(file_info["size"])}\n')
class MockSuccessfulFetchReturn:
@staticmethod
def run():
return 0
@pytest.fixture
def mock_successful_fetch_run(monkeypatch):
def mock_run(*args, **kwargs):
return MockSuccessfulFetchReturn().run()
monkeypatch.setattr(DataFetcher, "run", mock_run)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, settings):
@pytest.mark.skip("takes too long; re-enable with WS-179-1")
def test_gets_vlbas_from_report_file(make_tempdir, capo_settings):
location_file = get_locations_file("VLBA_EB")
args = [
......@@ -369,12 +282,11 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se
"--location-file",
str(location_file),
]
fetcher = DataFetcher(get_arg_parser().parse_args(args), settings.capo_settings)
servers_report = fetcher.servers_report
datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = datafetcher.servers_report
assert len(servers_report) == 1
return_code = fetcher.run()
assert return_code == 0
datafetcher.run()
dest_dir = Path(make_tempdir)
file_info_dict = dict()
......@@ -391,8 +303,7 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se
file_info_dict[filename["ngas_file_id"]] = filename
datafetcher = DataFetcher(get_arg_parser().parse_args(args), settings.capo_settings)
return_code = datafetcher.run()
assert return_code == 0
datafetcher.run()
for filename in file_info_dict:
path = Path(dest_dir, filename)
......@@ -401,8 +312,8 @@ def test_gets_vlbas_from_report_file(mock_successful_fetch_run, make_tempdir, se
assert int(contents) == file_info_dict[filename]["size"]
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tempdir, settings):
@pytest.mark.skip("takes too long; re-enable with WS-179-1")
def test_gets_large_vla_ebs_from_report_file(make_tempdir, capo_settings):
location_file = get_locations_file("VLA_SMALL_EB")
args = [
"--profile",
......@@ -412,14 +323,15 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem
"--location-file",
str(location_file),
]
fetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = fetcher.servers_report
datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = datafetcher.servers_report
assert len(servers_report) == 2
datafetcher.run()
return_code = fetcher.run()
assert return_code == 0
server_file_count = {"nmngas03.aoc.nrao.edu:7777": 0, "nmngas04.aoc.nrao.edu:7777": 0}
server_file_count = {
"nmngas03.aoc.nrao.edu:7777": 0,
"nmngas04.aoc.nrao.edu:7777": 0,
}
dest_dir = Path(make_tempdir)
file_list = list()
for server in servers_report.items():
......@@ -440,8 +352,7 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem
assert server_file_count["nmngas04.aoc.nrao.edu:7777"] == 41
datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
return_code = datafetcher.run()
assert return_code == 0
datafetcher.run()
found_count = 0
for file_info in file_list:
......@@ -456,8 +367,8 @@ def test_gets_large_vla_ebs_from_report_file(mock_successful_fetch_run, make_tem
assert found_count == len(file_list)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, settings):
@pytest.mark.skip("takes too long; re-enable with WS-179-1")
def test_gets_images_from_report_file(make_tempdir, capo_settings):
location_file = get_locations_file("IMG")
args = [
"--profile",
......@@ -467,11 +378,14 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s
"--location-file",
str(location_file),
]
fetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = fetcher.servers_report
datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = datafetcher.servers_report
assert len(servers_report) == 2
server_file_count = {"nmngas01.aoc.nrao.edu:7777": 0, "nmngas02.aoc.nrao.edu:7777": 0}
server_file_count = {
"nmngas01.aoc.nrao.edu:7777": 0,
"nmngas02.aoc.nrao.edu:7777": 0,
}
dest_dir = Path(make_tempdir)
file_list = list()
for server in servers_report.items():
......@@ -491,8 +405,7 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s
for server_url, count in server_file_count.items():
assert count == 1
return_code = fetcher.run()
assert return_code == 0
datafetcher.run()
found_count = 0
for file_info in file_list:
......@@ -508,7 +421,7 @@ def test_gets_images_from_report_file(mock_successful_fetch_run, make_tempdir, s
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempdir, settings):
def test_gets_calibration_from_report_file(make_tempdir):
location_file = get_locations_file("CALIBRATION")
args = [
"--profile",
......@@ -518,8 +431,8 @@ def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempd
"--location-file",
str(location_file),
]
fetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = fetcher.servers_report
datafetcher = DataFetcher(get_arg_parser().parse_args(args), capo_settings)
servers_report = datafetcher.servers_report
assert len(servers_report) == 1
fake_file = None
......@@ -538,7 +451,7 @@ def test_gets_calibration_from_report_file(mock_successful_fetch_run, make_tempd
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_calibration_from_locator(mock_successful_fetch_run, make_tempdir, settings):
def test_gets_calibration_from_locator(make_tempdir, settings):
external_name = LOCATION_REPORTS["CALIBRATION"]["external_name"]
product_locator = ProductLocatorLookup(settings.db_settings).look_up_locator_for_ext_name(
external_name
......@@ -570,6 +483,7 @@ def test_gets_calibration_from_locator(mock_successful_fetch_run, make_tempdir,
assert int(contents) == file_spec["size"]
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_gets_gbt_data_from_locator(make_tempdir, settings):
""" Can we cope with GBT data? """
......@@ -601,3 +515,10 @@ def test_gets_gbt_data_from_locator(make_tempdir, settings):
assert fake_file.is_file()
contents = fake_file.read_text().strip()
assert int(contents) == file_spec["size"]
def write_fake_file(destination: Path, file_info: dict):
filename = file_info["ngas_file_id"]
path = Path(destination, filename)
with open(path, "w") as file:
file.write(f'{str(file_info["size"])}\n')
apps/cli/executables/datafetcher/test/test_df_return_codes.py → apps/cli/executables/datafetcher/test/test_df_return_status.py
View file @ 71b97f8f
......@@ -2,19 +2,36 @@
check that arguments passed return expected codes
"""
# pylint: disable=C0115, C0116, R0801, R0902, R0903, R0914, W0212, W0611 W0613, W0621, W0703, W1203
# pylint: disable=C0115, C0116, E0401, E0402, E1120,R0801, R0902, R0903, R0914,
# pylint: disable=W0212, W0611, W0613, W0621, W0703, W1203
import os
import tempfile
from pathlib import Path
from typing import List
import pytest
from datafetcher.errors import (
NoProfileException,
MissingSettingsException,
LocationServiceTimeoutException,
NGASServiceRedirectsException,
NGASFetchError,
NoLocatorException,
SizeMismatchException,
)
from datafetcher.datafetcher import DataFetcher
from datafetcher.return_codes import ReturnCode
from datafetcher.utilities import get_arg_parser, ProductLocatorLookup
from datafetcher.utilities import (
get_arg_parser,
ProductLocatorLookup,
path_is_accessible,
)
# N.B. IJ doesn't recognize imported fixtures as being in use.
# don't let these imports (make_tempdir, settings) get disappeared.
from .df_pytest_arg_utils import parse_args
from .df_pytest_utils import (
TEST_PROFILE,
get_test_capo_settings,
......@@ -22,16 +39,22 @@ from .df_pytest_utils import (
capo_settings,
settings,
launch_datafetcher,
MISSING_SETTING,
MISSING_PROFILE,
RUN_ALL,
confirm_retrieve_mode_copy,
evaluate_args_and_capo,
)
from .mock_data_fetcher import MockProdDataFetcher
def test_launch_df(make_tempdir, settings):
"""
Our "control"; should always pass
:param make_tempdir:
:param settings:
:return:
"""
args = [
"--product-locator",
settings.test_data["product_locator"],
......@@ -41,16 +64,12 @@ def test_launch_df(make_tempdir, settings):
TEST_PROFILE,
"--dry-run",
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code is not None
try_to_launch_df(settings.capo_settings, args)
def test_launch_df_no_args(settings):
try:
return_code = launch_datafetcher([], settings.capo_settings)
assert return_code is not None
except Exception as exc:
pytest.fail(f"{exc}")
with pytest.raises(MissingSettingsException):
launch_datafetcher([], settings.capo_settings)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
......@@ -68,8 +87,8 @@ def test_no_args_prints_usage_and_croaks():
assert datafetcher.usage == DataFetcher._build_usage_message()
# @pytest.mark.skipif(not RUN_ALL, reason='debug')
def test_omitted_profile_returns_expected_code(make_tempdir, settings):
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_omitted_profile_throws_no_profile_exc(make_tempdir, settings):
"""
Be sure DF dies with appropriate error message
if called without Capo profile
......@@ -93,37 +112,29 @@ def test_omitted_profile_returns_expected_code(make_tempdir, settings):
str(make_tempdir),
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == MISSING_PROFILE
# restore the existing CAPO_PROFILE
os.environ["CAPO_PROFILE"] = existing_capo_profile
try:
with pytest.raises(NoProfileException):
launch_datafetcher(args, settings.capo_settings)
finally:
# restore the existing CAPO_PROFILE
os.environ["CAPO_PROFILE"] = existing_capo_profile
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_omitted_capo_value_returns_expected_code(make_tempdir, settings):
"""
:param make_tempdir: tempdir created on the fly
:param settings: source of Capo settings
:return:
"""
def test_omitted_capo_value_throws_missing_setting_exc(capo_settings):
args = [
"--product-locator",
"we're not going to get this far",
"--output-dir",
"--profile",
TEST_PROFILE,
"docker",
]
result = launch_datafetcher(args, settings.capo_settings)
assert result == MISSING_SETTING
with pytest.raises(MissingSettingsException):
try_to_launch_df(capo_settings, args)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_invalid_capo_profile_returns_expected_code(make_tempdir, settings):
def test_invalid_capo_profile_raises_no_profile_exc(make_tempdir, settings):
"""
Be sure DF dies with appropriate error message
if called without Capo profile
......@@ -145,8 +156,8 @@ def test_invalid_capo_profile_returns_expected_code(make_tempdir, settings):
"--output-dir",
str(make_tempdir),
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == MISSING_PROFILE
with pytest.raises(MissingSettingsException):
try_to_launch_df(settings.capo_settings, args)
def we_are_in_docker():
......@@ -164,59 +175,35 @@ def we_are_in_docker():
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_inaccessible_output_dir_returns_expected_code(settings, make_tempdir):
umask = os.umask(0o000)
try:
tmpdir = Path(make_tempdir)
tmpdir.chmod(0o666)
altered_capo = get_test_capo_settings()
altered_capo["download_dir"] = str(tmpdir)
args = [
"--product-locator",
settings.test_data["product_locator"],
"--profile",
TEST_PROFILE,
"--output-dir",
str(tmpdir),
]
namespace = get_arg_parser().parse_args(args)
# N.B. DataFetcher.__init__ throws SystemExit in pytest at command line,
# but in Docker container the failure is interpreted as an Exception
if we_are_in_docker():
try:
DataFetcher(namespace, capo_settings)
except Exception as exc:
assert isinstance(exc, SystemExit)
assert exc.value.code == MISSING_SETTING
else:
with pytest.raises(SystemExit) as exc:
DataFetcher(namespace, settings.capo_settings)
assert exc.value.code == MISSING_SETTING
except Exception as exc:
pytest.fail(f"{exc}")
finally:
try:
os.umask(umask)
except Exception as exc:
pytest.fail(f"{exc}")
def test_inaccessible_output_dir_throws_file_not_found(settings):
tmpdir = Path("go_away")
assert not path_is_accessible(tmpdir)
altered_capo = get_test_capo_settings()
altered_capo["download_dir"] = str(tmpdir)
args = [
"--product-locator",
settings.test_data["product_locator"],
"--profile",
TEST_PROFILE,
"--output-dir",
str(tmpdir),
]
with pytest.raises(FileNotFoundError):
try_to_launch_df(settings.capo_settings, args)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_two_locator_args_returns_expected_code(make_tempdir, settings):
def test_locator_plus_file_throws_missing_setting(make_tempdir, capo_settings):
"""
We should reject invocation with both product locator -and- location
file. One or the other, people!
:param make_tempdir: tempdir created on the fly
:param settings: source of Capo settings
:param capo_settings: source of Capo settings
:return:
"""
......@@ -225,13 +212,14 @@ def test_two_locator_args_returns_expected_code(make_tempdir, settings):
"--product-locator",
"a_locator",
"--location-file",
"location.json" "--output-dir",
"location.json",
"--output-dir",
str(make_tempdir),
"--profile",
TEST_PROFILE,
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == MISSING_SETTING
with pytest.raises(MissingSettingsException):
try_to_launch_df(capo_settings, args)
class MockServiceTimeoutReturn:
......@@ -239,7 +227,7 @@ class MockServiceTimeoutReturn:
@staticmethod
def run():
return ReturnCode.LOCATOR_SERVICE_TIMEOUT.value["code"]
raise LocationServiceTimeoutException()
@pytest.mark.skipif(not RUN_ALL, reason="debug")
......@@ -256,14 +244,14 @@ def test_locator_service_timeout_returns_expected_code(monkeypatch, settings, ma
TEST_PROFILE,
]
monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.LOCATOR_SERVICE_TIMEOUT.value["code"]
with pytest.raises(LocationServiceTimeoutException):
launch_datafetcher(args, settings.capo_settings)
class MockTooManyServiceRedirectsReturn:
@staticmethod
def run():
return ReturnCode.TOO_MANY_SERVICE_REDIRECTS.value
raise NGASServiceRedirectsException()
@pytest.mark.skipif(not RUN_ALL, reason="debug")
......@@ -280,14 +268,14 @@ def test_too_many_service_redirects_returns_expected_code(monkeypatch, settings,
TEST_PROFILE,
]
monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.TOO_MANY_SERVICE_REDIRECTS.value
with pytest.raises(NGASServiceRedirectsException):
launch_datafetcher(args, settings.capo_settings)
class MockCatastrophicServiceErrorReturn:
@staticmethod
def run():
return ReturnCode.CATASTROPHIC_REQUEST_ERROR
raise NGASFetchError()
@pytest.mark.skipif(not RUN_ALL, reason="debug")
......@@ -305,11 +293,12 @@ def test_catastrophic_service_error_returns_expected_code(monkeypatch, settings,
]
monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.CATASTROPHIC_REQUEST_ERROR
with pytest.raises(NGASFetchError):
launch_datafetcher(args, settings.capo_settings)
def test_copy_attempt_throws_sys_exit_service_error(monkeypatch, settings, make_tempdir):
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_copy_attempt_raises_ngas_fetch_error(make_tempdir, settings):
args = [
"--product-locator",
settings.test_data["product_locator"],
......@@ -337,9 +326,8 @@ def test_copy_attempt_throws_sys_exit_service_error(monkeypatch, settings, make_
files = fetcher.servers_report[a_server]["files"]
fetcher.servers_report[a_server]["files"] = [files[0]]
with pytest.raises(SystemExit) as exc:
with pytest.raises(NGASFetchError):
fetcher.run()
assert exc.value.code == ReturnCode.CATASTROPHIC_REQUEST_ERROR.value["code"]
@pytest.mark.skipif(not RUN_ALL, reason="debug")
......@@ -352,12 +340,12 @@ def test_product_locator_not_found_returns_expected_code(make_tempdir, settings)
"--profile",
TEST_PROFILE,
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.PRODUCT_LOCATOR_NOT_FOUND.value["code"]
with pytest.raises(NoLocatorException):
launch_datafetcher(args, settings.capo_settings)
@pytest.mark.skipif(not RUN_ALL, reason="debug")
def test_unable_to_open_location_file_returns_expected_code(make_tempdir, settings):
def test_unable_to_open_location_file_throws_file_not_found(make_tempdir, capo_settings):
args = [
"--location-file",
"location.json",
......@@ -366,8 +354,8 @@ def test_unable_to_open_location_file_returns_expected_code(make_tempdir, settin
"--profile",
TEST_PROFILE,
]
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.CANNOT_OPEN_LOCATION_FILE.value["code"]
with pytest.raises(FileNotFoundError):
try_to_launch_df(capo_settings, args)
class MockNgasFetchError:
......@@ -375,7 +363,7 @@ class MockNgasFetchError:
@staticmethod
def run():
return ReturnCode.NGAS_FETCH_ERROR
raise NGASFetchError()
@pytest.mark.skipif(not RUN_ALL, reason="debug")
......@@ -392,8 +380,8 @@ def test_error_fetching_file_from_ngas_returns_expected_code(monkeypatch, settin
TEST_PROFILE,
]
monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.NGAS_FETCH_ERROR
with pytest.raises(NGASFetchError):
launch_datafetcher(args, settings.capo_settings)
class MockSizeMismatchError:
......@@ -401,7 +389,7 @@ class MockSizeMismatchError:
@staticmethod
def run():
return ReturnCode.SIZE_MISMATCH
raise SizeMismatchException()
@pytest.mark.skipif(not RUN_ALL, reason="debug")
......@@ -418,5 +406,12 @@ def test_unexpected_size_returns_expected_code(monkeypatch, settings, make_tempd
TEST_PROFILE,
]
monkeypatch.setattr(DataFetcher, "run", mock_run)
return_code = launch_datafetcher(args, settings.capo_settings)
assert return_code == ReturnCode.SIZE_MISMATCH
with pytest.raises(SizeMismatchException):
launch_datafetcher(args, settings.capo_settings)
def try_to_launch_df(capo_settings, args: List[str]):
parse_args(args)
namespace = evaluate_args_and_capo(args, capo_settings)
fetcher = DataFetcher(namespace, capo_settings)
return fetcher.run()