diff --git a/_version.py b/_version.py new file mode 100644 index 0000000000000000000000000000000000000000..41dbad0d2dda23b541e627149276c445283d49a6 --- /dev/null +++ b/_version.py @@ -0,0 +1,2 @@ +""" Version information for this package, don't put anything else here. """ +___version___ = '0.1.5' diff --git a/datafetcher/commands.py b/datafetcher/commands.py index 0333f12f8070d8691ecb97ea4156a6c7cf9d77c4..477b9a19a0bb3880a606e4a6f397d007760301ee 100755 --- a/datafetcher/commands.py +++ b/datafetcher/commands.py @@ -7,16 +7,19 @@ import logging import sys import traceback -from errors import NoLocatorException, \ +from .errors import NoLocatorException, \ NGASServiceErrorException, exception_to_error, terminal_exception -from locations_report import LocationsReport -from product_fetchers import ParallelProductFetcher -from utilities import get_arg_parser, get_capo_settings, \ +from .locations_report import LocationsReport +from .product_fetchers import ParallelProductFetcher +from .utilities import get_arg_parser, get_capo_settings, \ FlexLogger class DataFetcher: ''' + TO EXECUTE ALL DF TESTS: + from data/: python -m unittest discover -s datafetcher -p '*_test.py' + example command line that should work with the correct local profile: datafetcher --profile local --output-dir ~/Downloads/ --product-locator \ diff --git a/datafetcher/file_retrievers.py b/datafetcher/file_retrievers.py index 6595d3a5a8b57dbc9c47df8370e992601673c8da..5dd2c58039a3d6b8de141a0e44d949c09ba30096 100644 --- a/datafetcher/file_retrievers.py +++ b/datafetcher/file_retrievers.py @@ -11,10 +11,10 @@ from pathlib import Path import requests from bs4 import BeautifulSoup -from errors import SizeMismatchException, \ +from .errors import SizeMismatchException, \ NGASServiceErrorException, \ FileErrorException, MissingSettingsException -from utilities import RetrievalMode, Retryer, MAX_TRIES, \ +from .utilities import RetrievalMode, Retryer, MAX_TRIES, \ SLEEP_INTERVAL_SECONDS, FlexLogger _DIRECT_COPY_PLUGIN = 'ngamsDirectCopyDppi' diff --git a/datafetcher/locations_report.py b/datafetcher/locations_report.py index 7c1ec201774a3542674464f23817386da394e8b3..7bcdea169edd74c12685d1679a04edf953bdfb2d 100644 --- a/datafetcher/locations_report.py +++ b/datafetcher/locations_report.py @@ -13,11 +13,11 @@ from argparse import Namespace from typing import Dict import requests -from errors import LocationServiceTimeoutException, \ + +from datafetcher.errors import LocationServiceTimeoutException, \ LocationServiceRedirectsException, LocationServiceErrorException, \ NoLocatorException, MissingSettingsException -from utilities import Cluster, RetrievalMode, \ - validate_file_spec, \ +from datafetcher.utilities import Cluster, RetrievalMode, validate_file_spec, \ FlexLogger diff --git a/datafetcher/product_fetchers.py b/datafetcher/product_fetchers.py index 0398d07d1eaa0a95a3abff6140b3ae25795f74aa..0d57b65f72bdf346a13cd3c3047864c8a1821e59 100644 --- a/datafetcher/product_fetchers.py +++ b/datafetcher/product_fetchers.py @@ -8,9 +8,9 @@ from argparse import Namespace from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Dict -from errors import NGASServiceErrorException -from file_retrievers import NGASFileRetriever -from utilities import FlexLogger +from .errors import NGASServiceErrorException +from .file_retrievers import NGASFileRetriever +from .utilities import FlexLogger class BaseProductFetcher: diff --git a/datafetcher/setup.py b/datafetcher/setup.py index c54f4c7d7fe79d865ac1e42a33556f04a9420456..5e3acdec6e11c1914ce4ec754262a1052b585c5a 100644 --- a/datafetcher/setup.py +++ b/datafetcher/setup.py @@ -4,7 +4,7 @@ import os from setuptools import setup, find_packages -NAME = 'datafetcher' +NAME = 'data' HERE = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(HERE, '_version.py')) as f: VERSION = f.readlines()[-1].split()[-1].strip("\"'") @@ -22,7 +22,7 @@ TEST_REQUIRES = [ setup( name=NAME, version=VERSION, - description='grab files from NRAO Archive', + description='NRAO AAT/PPI', long_description=README, author='Stephan Witz', author_email='switz@nrao.edu', diff --git a/datafetcher/test/datafetcher_test.py b/datafetcher/test/datafetcher_test.py index fb5086626c4db5c1166b5642bee3e253916ca5db..8d3447e7109e3dd704cdb09b8036e018ada33489 100644 --- a/datafetcher/test/datafetcher_test.py +++ b/datafetcher/test/datafetcher_test.py @@ -8,17 +8,18 @@ from pathlib import Path from typing import List from unittest.mock import MagicMock -import pytest -from commands import DataFetcher -from errors import Errors -from locations_report import LocationsReport -from utilities import get_capo_settings, get_arg_parser, \ +from datafetcher.commands import DataFetcher +from datafetcher.errors import Errors +from datafetcher.locations_report import LocationsReport +from datafetcher.utilities import get_capo_settings, get_arg_parser, \ ProductLocatorLookup, get_metadata_db_settings, ExecutionSite, \ RetrievalMode, FlexLogger -from testing_utils import get_locations_report, LOCATION_REPORTS, \ - get_mini_locations_file, write_locations_file, DATA_DIR, get_locations_file, \ - find_newest_fetch_log_file +from test.testing_utils import get_locations_report, LOCATION_REPORTS, \ + get_mini_locations_file, write_locations_file, get_locations_file, \ + find_newest_fetch_log_file, get_test_data_dir + +import pytest VLA_SMALL_KEY = 'VLA_SMALL_EB' FETCH_COMMAND = 'datafetcher' @@ -57,6 +58,9 @@ class DataFetcherTestCase(unittest.TestCase): cls.settings = get_capo_settings(cls.profile) cls.db_settings = get_metadata_db_settings(cls.profile) cls.test_data = cls._initialize_test_data(cls) + cls.DATA_DIR = get_test_data_dir() + if cls.DATA_DIR is None: + unittest.fail(f'test data directory not found under {os.getcwd()}') @classmethod def setUp(cls) -> None: @@ -114,7 +118,7 @@ class DataFetcherTestCase(unittest.TestCase): break self.assertTrue(exception_found, 'expecting FileNotFoundError') self.assertEqual(1, term_exc_count, 'terminal_exception should be ' - 'thrown') + 'thrown') def test_nothing_retrieved_if_dry_on_cmd_line(self): location_file = get_mini_locations_file(os.path.join(self.top_level, @@ -354,8 +358,6 @@ class DataFetcherTestCase(unittest.TestCase): fetch = DataFetcher(namespace, self.settings) fetch.run() logfile = fetch.logfile - with open(logfile, 'r') as log: - log_contents = log.readlines() exc_code = exc.value.code expected = Errors.FILE_EXISTS_ERROR.value self.assertEqual(expected, exc_code) @@ -554,7 +556,7 @@ class DataFetcherTestCase(unittest.TestCase): report_spec = LOCATION_REPORTS[VLA_SMALL_KEY] external_name = report_spec['external_name'] - locations_file = os.path.join(DATA_DIR, 'VLA_SMALL_EB_BUSTED.json') + locations_file = os.path.join(self.DATA_DIR, 'VLA_SMALL_EB_BUSTED.json') args = ['--location-file', locations_file, '--output-dir', self.top_level, '--profile', self.profile] namespace = get_arg_parser().parse_args(args) diff --git a/datafetcher/test/locations_report_test.py b/datafetcher/test/locations_report_test.py index 5af87c773424f3506c4b49dfcf114c687b62c510..4eab0b7f422536fcbe8a4b0adb0a61a054a23a4b 100644 --- a/datafetcher/test/locations_report_test.py +++ b/datafetcher/test/locations_report_test.py @@ -6,11 +6,11 @@ from json import JSONDecodeError import pytest -from errors import Errors, NoLocatorException, \ - MissingSettingsException -from locations_report import LocationsReport -from testing_utils import LOCATION_REPORTS, DATA_DIR -from utilities import get_capo_settings, \ +from datafetcher.errors import Errors, MissingSettingsException, \ + NoLocatorException +from datafetcher.locations_report import LocationsReport +from datafetcher.test.testing_utils import LOCATION_REPORTS, get_test_data_dir +from datafetcher.utilities import get_capo_settings, \ get_metadata_db_settings, \ ProductLocatorLookup, get_arg_parser, RetrievalMode, FlexLogger @@ -25,6 +25,9 @@ class LocationsReportTestCase(unittest.TestCase): cls._13b_locator = ProductLocatorLookup(cls.db_settings) \ .look_up_locator_for_ext_name( '13B-014.sb28862036.eb29155786.56782.5720116088') + cls.DATA_DIR = get_test_data_dir() + if cls.DATA_DIR is None: + fail(f'test data directory not found under {os.getcwd()}') @classmethod def setUp(cls) -> None: @@ -141,7 +144,7 @@ class LocationsReportTestCase(unittest.TestCase): def test_gets_expected_images_from_file(self): report_metadata = LOCATION_REPORTS['IMG'] - report_file = os.path.join(DATA_DIR, report_metadata['filename']) + report_file = os.path.join(self.DATA_DIR, report_metadata['filename']) args = ['--location-file', report_file, '--output-dir', None, '--profile', None] @@ -180,7 +183,7 @@ class LocationsReportTestCase(unittest.TestCase): def test_gets_vla_large_from_file_correctly(self): report_metadata = LOCATION_REPORTS['VLA_LARGE_EB'] - report_file = os.path.join(DATA_DIR, report_metadata['filename']) + report_file = os.path.join(self.DATA_DIR, report_metadata['filename']) args = ['--location-file', report_file, '--output-dir', None, '--profile', None] namespace = get_arg_parser().parse_args(args) @@ -208,7 +211,7 @@ class LocationsReportTestCase(unittest.TestCase): def test_gets_vla_small_from_file_correctly(self): report_metadata = LOCATION_REPORTS['VLA_SMALL_EB'] - report_file = os.path.join(DATA_DIR, report_metadata['filename']) + report_file = os.path.join(self.DATA_DIR, report_metadata['filename']) args = ['--location-file', report_file, '--output-dir', None, '--profile', None] namespace = get_arg_parser().parse_args(args) @@ -235,7 +238,7 @@ class LocationsReportTestCase(unittest.TestCase): def test_gets_expected_vlbas_from_file(self): report_metadata = LOCATION_REPORTS['VLBA_EB'] - report_file = os.path.join(DATA_DIR, report_metadata['filename']) + report_file = os.path.join(self.DATA_DIR, report_metadata['filename']) args = ['--location-file', report_file, '--output-dir', None, '--profile', None] @@ -261,7 +264,7 @@ class LocationsReportTestCase(unittest.TestCase): 'these should all be VLBA_VSN0011..UVFITS files') def test_throws_json_error_if_nothing_in_report_file(self): - report_file = os.path.join(DATA_DIR, 'EMPTY.json') + report_file = os.path.join(self.DATA_DIR, 'EMPTY.json') args = ['--location-file', report_file, '--output-dir', None, '--profile', None] namespace = get_arg_parser().parse_args(args) @@ -269,7 +272,7 @@ class LocationsReportTestCase(unittest.TestCase): LocationsReport(self._LOG, namespace, self.settings) def test_throws_json_error_if_report_file_is_not_json(self): - report_file = os.path.join(DATA_DIR, 'NOT_JSON.json') + report_file = os.path.join(self.DATA_DIR, 'NOT_JSON.json') args = ['--location-file', report_file, '--output-dir', None, '--profile', None] namespace = get_arg_parser().parse_args(args) diff --git a/datafetcher/test/logging_test.py b/datafetcher/test/logging_test.py index 7bbf8b0c8dc292f0c64a6e5b6e3c8eede156311f..093232e5c6cc0f63ec13f43d7164e7d5d85cabdc 100644 --- a/datafetcher/test/logging_test.py +++ b/datafetcher/test/logging_test.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from utilities import FlexLogger +from datafetcher.utilities import FlexLogger class FlexLoggerTestCase(unittest.TestCase): diff --git a/datafetcher/test/retriever_test.py b/datafetcher/test/retriever_test.py index 96f6ccd4c45a98d439d456849034f59fe0be15b4..7e213667db4ee9c490e1bd2df2088a079f07a3ad 100644 --- a/datafetcher/test/retriever_test.py +++ b/datafetcher/test/retriever_test.py @@ -10,10 +10,10 @@ from typing import List import pytest -from errors import FileErrorException, MissingSettingsException, \ +from datafetcher.errors import FileErrorException, MissingSettingsException, \ SizeMismatchException, NGASServiceErrorException -from file_retrievers import NGASFileRetriever -from utilities import get_capo_settings, get_metadata_db_settings, \ +from datafetcher.file_retrievers import NGASFileRetriever +from datafetcher.utilities import get_capo_settings, get_metadata_db_settings, \ get_arg_parser, RetrievalMode, path_is_accessible, FlexLogger, MAX_TRIES, \ ProductLocatorLookup, Cluster diff --git a/datafetcher/test/testing_utils.py b/datafetcher/test/testing_utils.py index e2022058edc89f14b3a2c3ebd4942d4cdb4fd989..04835a0dbcb4ae8b7c34f57746a6a7342f193061 100644 --- a/datafetcher/test/testing_utils.py +++ b/datafetcher/test/testing_utils.py @@ -4,8 +4,8 @@ import json import os from pathlib import Path -from locations_report import LocationsReport -from utilities import get_arg_parser +from datafetcher.locations_report import LocationsReport +from datafetcher.utilities import get_arg_parser LOCATION_REPORTS = { 'VLA_SMALL_EB': { @@ -48,14 +48,20 @@ LOCATION_REPORTS = { } -DATA_DIR = './data/' +def get_test_data_dir(): + here = os.path.abspath(os.curdir) + for root, dirnames, filenames in os.walk(here): + if str(root).endswith('test'): + for dirname in dirnames: + if dirname == 'data': + return os.path.join(root, dirname) + return None def get_locations_file(key: str): ''' return the location report file specified by key ''' report_spec = LOCATION_REPORTS[key] filename = report_spec['filename'] - test_data_dir = os.path.join(os.curdir, 'data') - return os.path.join(test_data_dir, filename) + return os.path.join(get_test_data_dir(), filename) def get_locations_report(key: str): ''' return the location report specified by key ''' diff --git a/datafetcher/utilities.py b/datafetcher/utilities.py index e95baadadecafc38efae04430de745b9305cf2cd..0530e22843257732c6870c5ec15b2a9b450add9e 100644 --- a/datafetcher/utilities.py +++ b/datafetcher/utilities.py @@ -16,7 +16,7 @@ from typing import Callable import psycopg2 as pg from pycapo import CapoConfig -from errors import get_error_descriptions, NoProfileException, \ +from datafetcher.errors import get_error_descriptions, NoProfileException, \ MissingSettingsException, NGASServiceErrorException, SizeMismatchException LOG_FORMAT = "%(module)s.%(funcName)s, %(lineno)d: %(message)s" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..cf557ebb4e7e1142b55ab3be7e7824f9b4e17636 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,14 @@ +[metadata] +description-file = README.txt + +[aliases] +test=pytest + +[tool:pytest] +addopts = -s + +[build_sphinx] +source-dir = docs/source +build-dir = docs/build +all_files = 1 +builder = html diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..3b3f4c41cd4c9350e82f16a11218ac55aa2074f4 --- /dev/null +++ b/setup.py @@ -0,0 +1,50 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +from setuptools import setup, find_packages + +NAME = 'data' +HERE = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(HERE, '_version.py')) as f: + VERSION = f.readlines()[-1].split()[-1].strip("\"'") +with open(os.path.join(HERE, 'README.md')) as f: + README = f.read() + +INSTALL_REQUIRES = [ + 'requests==2.23.0', + 'pycapo==0.2.1.post1' +] + +TEST_REQUIRES = [ +] + +setup( + name=NAME, + version=VERSION, + description='NRAO AAT/PPI', + long_description=README, + author='Stephan Witz', + author_email='switz@nrao.edu', + url='TBD', + license="GPL", + install_requires=INSTALL_REQUIRES, + test_requires=TEST_REQUIRES, + # test_suite='datafetcher.test', + keywords=['TBD'], + packages=find_packages(), + classifiers=[ + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 2' + ], + entry_points={ + 'console_scripts': ['datafetcher = datafetcher.commands:main'] + }, +)