diff --git a/Condaless.dockerfile b/Condaless.dockerfile
index 8c4c6de27fe7d4a5ca17a2032a2aee95648734c5..55249d85a9b643b2667a75d7b89fda4d811afbe7 100644
--- a/Condaless.dockerfile
+++ b/Condaless.dockerfile
@@ -1,7 +1,4 @@
-FROM python:3.8-slim
-
-# We happen to expose this port
-EXPOSE 3456:tcp
+FROM python:3.8-slim AS common
# Environment variables
ENV PIP_NO_CACHE_DIR false
@@ -19,6 +16,9 @@ RUN apt-get update \
# Get the code into the image
WORKDIR /code
COPY . .
+
+FROM common AS workflow
+EXPOSE 3456
WORKDIR /code/services/workflow
# Python library installation
@@ -26,3 +26,13 @@ RUN pip install -r requirements.txt \
&& python setup.py develop
ENTRYPOINT pserve --reload development.ini
+
+FROM common AS capability
+EXPOSE 3457
+WORKDIR /code/services/capability
+
+# Python library installation
+RUN pip install -r requirements.txt \
+ && python setup.py develop
+
+ENTRYPOINT pserve --reload development.ini
\ No newline at end of file
diff --git a/Condaless.dockerfile.v2 b/Condaless.dockerfile.v2
new file mode 100644
index 0000000000000000000000000000000000000000..cb7c93c19950f7cf83598ea11a57fc4c88766f04
--- /dev/null
+++ b/Condaless.dockerfile.v2
@@ -0,0 +1,18 @@
+FROM python:3.8-slim
+
+# Get postgres/mysql development stuff in the image
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+ gcc \
+ libmariadb-dev-compat \
+ libpq-dev \
+ && rm -rf /var/lib/apt/lists
+
+# Environment variables
+# - CAPO_PROFILE will be overridden for Dev, Test, and Prod
+ENV PIP_NO_CACHE_DIR false
+ENV CAPO_PROFILE docker
+
+# Set up Capo
+WORKDIR /root/.capo
+COPY docker.properties docker.properties
diff --git a/Makefile b/Makefile
index 0db4ef2ffa2040d7e94286cd4f606783fb2a472a..e67bc85ed0d8294324bf92181bc91018705fefc3 100644
--- a/Makefile
+++ b/Makefile
@@ -49,3 +49,7 @@ docker-images:
pushd services/workflow && docker build -t nrao:workflow . && popd
pushd services/capability && docker build -t nrao:capability . && popd
pushd schema && docker build -t nrao:schema . && popd
+
+.PHONY: docker-base
+docker-base:
+ docker build -t marconi.aoc.nrao.edu/ops/base:workspaces -f Condaless.dockerfile.v2 .
diff --git a/apps/cli/executables/datafetcher/src/datafetcher/datafetcher.py b/apps/cli/executables/datafetcher/src/datafetcher/datafetcher.py
index 5793eb94ed5a697711f6bc1078425d2c9b478823..3968aa664b52d0b596120bba6ca5fb7b97149b50 100755
--- a/apps/cli/executables/datafetcher/src/datafetcher/datafetcher.py
+++ b/apps/cli/executables/datafetcher/src/datafetcher/datafetcher.py
@@ -1,13 +1,14 @@
#!/usr/bin/env/python
# -*- coding: utf-8 -*-
-''' Module for the command line interface to data-fetcher. '''
+""" Module for the command line interface to data-fetcher. """
import logging
import sys
from argparse import Namespace
from pathlib import Path
+from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.project_fetcher import ParallelFetcher
from datafetcher.return_codes import ReturnCode
@@ -15,14 +16,15 @@ from .locations_report import LocationsReport
from .utilities import get_arg_parser, get_capo_settings, FlexLogger, \
path_is_accessible
-# pylint: disable=W1203, R0902, R0903
+# pylint: disable=C0103, W1203, R0902, R0903
_APPLICATION_NAME = 'datafetcher'
-_VALID_PROFILES = ['naasc-dev', 'naasc-test', 'dsoc-test', 'dsoc-dev',
- 'nmprod', 'local']
+
+# pylint: disable=C0103, W0703
+
class DataFetcher:
- '''
+ """
TO EXECUTE ALL DF TESTS: from datafetcher/, just run pytest
example command line that should work with the correct local profile
@@ -41,8 +43,9 @@ class DataFetcher:
to local computer
- execution_site must NOT be DSOC or NAASC
- '''
+ """
+ # TODO Some Fine Day: refactor to reduce cognitive complexity
def __init__(self, args: Namespace, df_capo_settings: dict):
self.usage = self._build_usage_message()
if args is None or df_capo_settings is None:
@@ -79,12 +82,12 @@ class DataFetcher:
self._exit_with_error(ReturnCode.MISSING_SETTING)
self.profile = args.profile
- if self.profile is None or not self.profile in _VALID_PROFILES:
+ if self.profile is None:
self._exit_with_error(ReturnCode.MISSING_PROFILE)
# optional arguments
self.is_dry = args.dry_run
- self.force = args.force
+ self.force = args.force
self.verbose = args.verbose or False
try:
@@ -107,7 +110,6 @@ class DataFetcher:
print(self.usage)
sys.exit(return_code.value['code'])
-
@staticmethod
def _build_usage_message() -> str:
usage_str = 'Usage:'
@@ -143,9 +145,16 @@ class DataFetcher:
def _get_locations(self):
try:
capo_settings = get_capo_settings(self.profile)
+ except MissingSettingsException as exc:
+ # if a setting couldn't be found, we can be pretty sure that
+ # the profile's no good
+ raise NoProfileException from exc
+
+ try:
return LocationsReport(self._LOG,
self.args,
capo_settings)
+
except Exception as exc:
self._LOG.error(f'{exc}')
if hasattr(self, 'location_file'):
@@ -158,9 +167,9 @@ class DataFetcher:
def main():
- ''' this will be executed when fetcher is launched
- from the command line
- '''
+ """ This is executed when fetcher is launched
+ from the command line.
+ """
parser = get_arg_parser()
try:
@@ -169,6 +178,7 @@ def main():
logging.error(f'{exc}')
return exc.value
except SystemExit as exc:
+ logging.error(f'{exc}')
return exc.value.code
settings = get_capo_settings(args.profile)
datafetcher = DataFetcher(args, settings)
diff --git a/apps/cli/executables/datafetcher/src/datafetcher/errors.py b/apps/cli/executables/datafetcher/src/datafetcher/errors.py
index ea71fdf08d00542092ce08521f2ae70d9b0cbbb7..1aca1ec3e07eeac3caedf66e87833b1c41099bdd 100644
--- a/apps/cli/executables/datafetcher/src/datafetcher/errors.py
+++ b/apps/cli/executables/datafetcher/src/datafetcher/errors.py
@@ -1,4 +1,4 @@
-""" Custom error definitions for data fetcher """
+""" Custom error definitions for data-fetcher """
import logging
import sys
@@ -7,6 +7,9 @@ from enum import Enum
_LOG = logging.getLogger(__name__)
+# pylint: disable=W1202
+
+
class Errors(Enum):
"""
Assorted constants and functions involving errors
@@ -29,51 +32,58 @@ class Errors(Enum):
class NoProfileException(Exception):
- pass
+ """ throw this if Capo profile can't be determined"""
class MissingSettingsException(Exception):
- pass
+ """ throw this if a required setting isn't found in command-line args """
class LocationServiceTimeoutException(Exception):
- pass
+ """ throw this if the location service takes too long to return """
class LocationServiceRedirectsException(Exception):
- pass
+ """ throw this if the location service
+ complains about too many redirects
+ """
class LocationServiceErrorException(Exception):
- pass
+ """ throw this if the location service falls over """
class NGASServiceTimeoutException(Exception):
- pass
+ """ throw this if the NGAS service time out """
class NGASServiceRedirectsException(Exception):
- pass
+ """ throw this if the NGAS service complains about too many redirects """
class NGASServiceErrorException(Exception):
- pass
+ """ throw this if the NGAS service falls over """
class NoLocatorException(Exception):
- pass
+ """ throw this if no product locator could be determined from
+ command-line arguments
+
+ """
class FileErrorException(Exception):
- pass
-
+ """ throw this if file retriever can't access or create the output
+ directory
-class FileExistsException(Exception):
- pass
+ """
class SizeMismatchException(Exception):
- pass
+ """ throw this if the size of the file retrieved
+ doesn't match expected size
+
+ """
TERMINAL_ERRORS = {
@@ -92,9 +102,10 @@ TERMINAL_ERRORS = {
Errors.FILE_NOT_FOUND_ERROR: 'target directory or file not found'
}
+
def get_error_descriptions():
- ''' user-friendly display of errors
- '''
+ """ user-friendly reporting of errors """
+
result = 'Return Codes:\n'
for error in Errors:
result += '\t{}: {}\n'.format(error.value, TERMINAL_ERRORS[error])
@@ -102,8 +113,8 @@ def get_error_descriptions():
def terminal_error(errno):
- ''' report error, then throw in the towel
- '''
+ """ report error, then throw in the towel
+ """
if errno in TERMINAL_ERRORS:
_LOG.error(TERMINAL_ERRORS[errno])
else:
@@ -113,8 +124,8 @@ def terminal_error(errno):
def exception_to_error(exception):
- ''' translate an exception to one of our custom errors
- '''
+ """ translate an exception to one of our custom errors
+ """
switcher = {
'NoProfileException': Errors.NO_PROFILE,
'MissingSettingsException': Errors.MISSING_SETTING,
@@ -132,10 +143,11 @@ def exception_to_error(exception):
}
return switcher.get(exception.__class__.__name__)
+
def terminal_exception(exception):
- ''' report exception, then throw in the towel.
+ """ Report this exception, then throw in the towel.
should be used by DataFetcher -ONLY-
- '''
+ """
errorno = exception_to_error(exception)
_LOG.debug(traceback.format_exc())
_LOG.error(str(exception))
diff --git a/apps/cli/executables/datafetcher/src/datafetcher/file_retrievers.py b/apps/cli/executables/datafetcher/src/datafetcher/file_retrievers.py
index 06ba963b7a447f8e54fbc1d58f285defdc1ac6b1..9045ad66834c2845a35ef06cb362d705f9b0a564 100644
--- a/apps/cli/executables/datafetcher/src/datafetcher/file_retrievers.py
+++ b/apps/cli/executables/datafetcher/src/datafetcher/file_retrievers.py
@@ -20,6 +20,7 @@ from .utilities import RetrievalMode, Retryer, MAX_TRIES, \
_DIRECT_COPY_PLUGIN = 'ngamsDirectCopyDppi'
_STREAMING_CHUNK_SIZE = 8192
+# pylint: disable=C0103, R0903, R0914
class NGASFileRetriever:
""" Responsible for getting a file out of NGAS
and saving it to the requested location.
@@ -76,10 +77,10 @@ class NGASFileRetriever:
return output_dir / file_spec['relative_path']
return output_dir / file_spec['subdirectory'] / file_spec['relative_path']
except KeyError as k_err:
- raise MissingSettingsException(k_err)
+ raise MissingSettingsException(k_err) from k_err
except TypeError as t_err:
- raise MissingSettingsException(t_err)
+ raise MissingSettingsException(t_err) from t_err
def _make_basedir(self, destination):
""" Creates the directory (if it doesn't exist) the product will
@@ -97,9 +98,10 @@ class NGASFileRetriever:
umask = os.umask(0o000)
Path(basedir).mkdir(parents=True, exist_ok=True)
os.umask(umask)
- except OSError:
+ except OSError as os_err:
raise FileErrorException(
- f'failure trying to create output directory {basedir}')
+ f'failure trying to create output directory {basedir}') \
+ from os_err
def _check_result(self, destination, file_spec):
""" Confirm that the file was retrieved and its size matches what
@@ -200,9 +202,9 @@ class NGASFileRetriever:
f'was {actual_size} bytes'
)
- except requests.exceptions.ConnectionError:
+ except requests.exceptions.ConnectionError as c_err:
raise NGASServiceErrorException(
- f'problem connecting with {download_url}')
+ f'problem connecting with {download_url}') from c_err
except AttributeError as a_err:
self._LOG.warning(f'possible problem streaming: {a_err}')
diff --git a/apps/cli/executables/datafetcher/src/datafetcher/locations_report.py b/apps/cli/executables/datafetcher/src/datafetcher/locations_report.py
index 39bfacc6e46f3f8d3c1a0c6e48494f2e0b4ccbdd..2c4a3de8570b6cdea5a29ec7d2615ea964522d0c 100644
--- a/apps/cli/executables/datafetcher/src/datafetcher/locations_report.py
+++ b/apps/cli/executables/datafetcher/src/datafetcher/locations_report.py
@@ -20,11 +20,11 @@ from .errors import LocationServiceTimeoutException, \
from .utilities import Cluster, RetrievalMode, validate_file_spec, \
FlexLogger
+# pylint: disable=C0103, R0902, R0903, R0914, W0703
-class LocationsReport:
- ''' the location report class
- '''
+class LocationsReport:
+ """ Builds a location report """
def __init__(self, logger: FlexLogger, args: Namespace, settings: Dict):
self._LOG = logger
@@ -162,23 +162,23 @@ class LocationsReport:
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ':HIGH:!DH:!aNULL'
requests.Session().mount(url, adapter=requests.adapters.HTTPAdapter())
+ response = None
try:
response = requests.get(url,
params={'locator': self.product_locator})
- except requests.exceptions.Timeout:
- raise LocationServiceTimeoutException()
- except requests.exceptions.TooManyRedirects:
- raise LocationServiceRedirectsException()
+ except requests.exceptions.Timeout as exc_to:
+ raise LocationServiceTimeoutException() from exc_to
+ except requests.exceptions.TooManyRedirects as exc_re:
+ raise LocationServiceRedirectsException() from exc_re
except requests.exceptions.RequestException as ex:
- raise LocationServiceErrorException(ex)
+ raise LocationServiceErrorException(ex) from ex
except Exception as exc:
self._LOG.error(f'{exc}')
if response.status_code == http.HTTPStatus.OK:
return response.json()
- elif response.status_code == http.HTTPStatus.NOT_FOUND:
+ if response.status_code == http.HTTPStatus.NOT_FOUND:
raise NoLocatorException(
f'locator "{self.product_locator}" not found')
- else:
- raise LocationServiceErrorException('locator service returned {}'
- .format(response.status_code))
+ raise LocationServiceErrorException('locator service returned {}'
+ .format(response.status_code))
diff --git a/apps/cli/executables/datafetcher/src/datafetcher/project_fetcher.py b/apps/cli/executables/datafetcher/src/datafetcher/project_fetcher.py
index f44cc2988bb0f31fe78ebbc7ec93af01c1a6420c..986822de00173d3a4080907087503d8db9129186 100644
--- a/apps/cli/executables/datafetcher/src/datafetcher/project_fetcher.py
+++ b/apps/cli/executables/datafetcher/src/datafetcher/project_fetcher.py
@@ -15,11 +15,15 @@ from datafetcher.return_codes import ReturnCode
from .file_retrievers import NGASFileRetriever
from .utilities import FlexLogger
+# pylint: disable=C0103, R0201, R0902, R0903, W0703
+
class BaseFetcher:
""" This is a base class for fetchers. """
- def __init__(self, args: Namespace, df_capo_settings: dict, logger: FlexLogger,
+ def __init__(self, args: Namespace,
+ df_capo_settings: dict,
+ logger: FlexLogger,
servers_report: dict):
self.args = args
self.output_dir = self.args.output_dir
@@ -33,7 +37,7 @@ class BaseFetcher:
self.num_files_retrieved = 0
def retrieve_files(self, server, retrieve_method, file_specs):
- ''' this is the part where we actually fetch the files '''
+ """ This is the part where we actually fetch the files. """
retriever = NGASFileRetriever(self.args, self._LOG)
num_files = len(file_specs)
@@ -52,17 +56,18 @@ class BaseFetcher:
class SerialFetcher(BaseFetcher):
- ''' Pull the files out, one right after another;
+ """ Pull the files out, one right after another;
don't try to be clever about it.
- '''
+ """
def __init__(self, args: Namespace, df_capo_settings: Dict, logger: FlexLogger,
servers_report: Dict):
super().__init__(args, df_capo_settings, logger, servers_report)
def run(self):
- ''' fetch 'em '''
+ """ fetch 'em """
+
self._LOG.debug('writing to {}'.format(self.output_dir))
self._LOG.debug('dry run: {}'.format(self.dry_run))
self._LOG.debug(f'force overwrite: {self.force_overwrite}')
@@ -73,23 +78,26 @@ class SerialFetcher(BaseFetcher):
class ParallelFetcher(BaseFetcher):
- ''' Pull the files out in parallel; try to be clever about it. '''
+ """ Pull the files out in parallel; try to be clever about it. """
- def __init__(self, args: Namespace, df_capo_settings: dict, logger: FlexLogger,
+ def __init__(self, args: Namespace,
+ df_capo_settings: dict,
+ logger: FlexLogger,
servers_report: dict):
super().__init__(args, df_capo_settings, logger, servers_report)
self.num_files_expected = self._count_files_expected()
self.bucketized_files = self._bucketize_files()
def _count_files_expected(self):
- ''' determine how many files we expect to retrieve '''
+ """ determine how many files we expect to retrieve """
+
count = 0
for server in self.servers_report:
count += len(self.servers_report[server]['files'])
return count
def _bucketize_files(self):
- ''' Takes the servers report and splits it up into a list of buckets,
+ """ Takes the servers report and splits it up into a list of buckets,
each of which a single thread will handle. There will be X * Y buckets,
where X is the number of servers and Y is the 'threads per host', and
the files for a given server will be distributed among the buckets
@@ -98,7 +106,7 @@ class ParallelFetcher(BaseFetcher):
Basically what we are doing here is splitting up the work among the
threads we'll be creating and creating a list of work
for each thread to do.
- '''
+ """
threads_per_host = int(self.settings['threads_per_host'])
result = list()
@@ -121,8 +129,7 @@ class ParallelFetcher(BaseFetcher):
return result
def fetch_bucket(self, bucket):
- ''' Grab the files in this bucket
- '''
+ """ Grab the files in this bucket """
self._LOG.debug(f"{bucket['retrieve_method']} "
f"{len(bucket['files'])} files from "
f"{bucket['server']}....")
@@ -131,8 +138,8 @@ class ParallelFetcher(BaseFetcher):
bucket['files'])
def run(self):
- ''' Fetch all the files
- '''
+ """ Fetch all the files for the product locator """
+
if self.args.dry_run:
self._LOG.debug('This is a dry run; files will not be fetched')
return 0
diff --git a/apps/cli/executables/datafetcher/src/datafetcher/return_codes.py b/apps/cli/executables/datafetcher/src/datafetcher/return_codes.py
index 4e4629863a5f426b4e0b7e34fa7a9016de79659d..6b1b81c539bac3291a52c589238c9ea7d7a21cf9 100644
--- a/apps/cli/executables/datafetcher/src/datafetcher/return_codes.py
+++ b/apps/cli/executables/datafetcher/src/datafetcher/return_codes.py
@@ -1,17 +1,17 @@
-''' data-fetcher return codes as specified in README and usage string '''
-
+""" data-fetcher return codes as specified in README and usage string """
from enum import Enum
class ReturnCode(Enum):
- SUCCESS = {'code': 0, 'text' : 'success'}
- MISSING_PROFILE = {'code': 1, 'text' : 'no CAPO profile provided'}
- MISSING_SETTING = {'code': 2, 'text' : 'missing required setting'}
- LOCATOR_SERVICE_TIMEOUT = {'code': 3, 'text' : 'request to locator service timed out'}
- TOO_MANY_SERVICE_REDIRECTS = {'code': 4, 'text' : 'too many redirects on locator service'}
- CATASTROPHIC_REQUEST_ERROR = {'code': 5, 'text' : 'catastrophic error on request service'}
- PRODUCT_LOCATOR_NOT_FOUND = {'code': 6, 'text' : 'product locator not found'}
- CANNOT_OPEN_LOCATION_FILE = {'code': 7, 'text' : 'not able to open specified location file'}
- NGAS_FETCH_ERROR = {'code': 8, 'text' : 'error fetching file from NGAS server'}
- SIZE_MISMATCH = {'code': 9, 'text' : 'retrieved file not expected size'}
+ """ Canonical data-fetcher exit code values """
+ SUCCESS = {'code': 0, 'text': 'success'}
+ MISSING_PROFILE = {'code': 1, 'text': 'no CAPO profile provided'}
+ MISSING_SETTING = {'code': 2, 'text': 'missing required setting'}
+ LOCATOR_SERVICE_TIMEOUT = {'code': 3, 'text': 'request to locator service timed out'}
+ TOO_MANY_SERVICE_REDIRECTS = {'code': 4, 'text': 'too many redirects on locator service'}
+ CATASTROPHIC_REQUEST_ERROR = {'code': 5, 'text': 'catastrophic error on request service'}
+ PRODUCT_LOCATOR_NOT_FOUND = {'code': 6, 'text': 'product locator not found'}
+ CANNOT_OPEN_LOCATION_FILE = {'code': 7, 'text': 'not able to open specified location file'}
+ NGAS_FETCH_ERROR = {'code': 8, 'text': 'error fetching file from NGAS server'}
+ SIZE_MISMATCH = {'code': 9, 'text': 'retrieved file not expected size'}
diff --git a/apps/cli/executables/datafetcher/src/datafetcher/utilities.py b/apps/cli/executables/datafetcher/src/datafetcher/utilities.py
index b22c3dfe51a2031079586b7cb74945fe26350be0..576573efccb0067eb4e2d065d382c72e5b9680fc 100644
--- a/apps/cli/executables/datafetcher/src/datafetcher/utilities.py
+++ b/apps/cli/executables/datafetcher/src/datafetcher/utilities.py
@@ -1,8 +1,10 @@
# -*- coding: utf-8 -*-
-"""This is a home for assorted utilities that didn't seem to belong elsewhere,
+"""
+ This is a home for assorted utilities that didn't seem to belong elsewhere,
stuff like the CAPO settings retriever, command line parser generator and
so on.
+
"""
import argparse
@@ -19,6 +21,10 @@ from pycapo import CapoConfig
from .errors import get_error_descriptions, NoProfileException, \
MissingSettingsException, NGASServiceErrorException, SizeMismatchException
+
+# pylint:disable=C0303, C0415, R0903, W0212, W0404, W0621, W1203
+
+
LOG_FORMAT = "%(module)s.%(funcName)s, %(lineno)d: %(message)s"
MAX_TRIES = 10
@@ -28,14 +34,13 @@ FILE_SPEC_KEYS = ['ngas_file_id', 'subdirectory', 'relative_path',
SERVER_SPEC_KEYS = ['server', 'location', 'cluster', 'retrieve_method']
# Prologue and epilogue for the command line parser.
-_PROLOGUE = \
- """Retrieve a product (a science product or an ancillary product) from
- the NRAO archive, either by specifying the product's locator or by
- providing the path to a product locator report.
-"""
+_PROLOGUE = """Retrieve a product (a science product or an ancillary product)
+from the NRAO archive, either by specifying the product's locator or by
+providing the path to a product locator report."""
_EPILOGUE = get_error_descriptions()
-# This is a dictionary of required CAPO settings and the attribute names we'll store them as.
+# This is a dictionary of required CAPO settings
+# and the attribute names we'll store them as.
REQUIRED_SETTINGS = {
'EDU.NRAO.ARCHIVE.DATAFETCHER.DATAFETCHERSETTINGS.LOCATORSERVICEURLPREFIX':
'locator_service_url',
@@ -43,13 +48,18 @@ REQUIRED_SETTINGS = {
'execution_site',
'EDU.NRAO.ARCHIVE.DATAFETCHER.DATAFETCHERSETTINGS.DEFAULTTHREADSPERHOST':
'threads_per_host',
- 'EDU.NRAO.ARCHIVE.WORKFLOW.CONFIG.REQUESTHANDLERSETTINGS.DOWNLOADDIRECTORY' :
+ 'EDU.NRAO.ARCHIVE.WORKFLOW.CONFIG.REQUESTHANDLERSETTINGS.DOWNLOADDIRECTORY':
'download_dir'
}
-def path_is_accessible(path: pathlib.Path):
- ''' Is this path readable, executable, and writable?
- '''
+
+def path_is_accessible(path: pathlib.Path) -> bool:
+ """
+ Is this path readable, executable, and writable?
+ :param path: a file or directory
+ :return:
+ """
+
can_access = os.access(path, os.F_OK)
can_access = can_access and path.is_dir()
can_access = can_access and os.access(path, os.R_OK)
@@ -57,6 +67,7 @@ def path_is_accessible(path: pathlib.Path):
can_access = can_access and os.access(path, os.X_OK)
return can_access
+
def get_arg_parser():
""" Build and return an argument parser with the command line options
for data-fetcher. this is out here and not in a class because Sphinx
@@ -68,8 +79,7 @@ def get_arg_parser():
cwd = pathlib.Path().absolute()
parser = argparse.ArgumentParser(description=_PROLOGUE, epilog=_EPILOGUE,
- formatter_class=
- argparse.RawTextHelpFormatter)
+ formatter_class=argparse.RawTextHelpFormatter)
# Can't find a way of clearing the action groups
# without hitting an internal attribute.
parser._action_groups.pop()
@@ -120,19 +130,21 @@ def get_capo_settings(profile: str):
setting = setting.upper()
try:
value = capo[setting]
- except KeyError:
+ except KeyError as k_err:
raise MissingSettingsException('missing required setting "{}"'
- .format(setting))
+ .format(setting)) from k_err
result[REQUIRED_SETTINGS[setting]] = value
return result
def validate_file_spec(file_spec: dict, retrieve_method_expected: bool):
- '''
+ """
Make sure this nugget of file info contains everything it should.
:param file_spec:
+ :param retrieve_method_expected:
:return:
- '''
+ """
+
for key in FILE_SPEC_KEYS:
if key not in file_spec:
raise MissingSettingsException(f'{key} not found in file_spec')
@@ -141,9 +153,9 @@ def validate_file_spec(file_spec: dict, retrieve_method_expected: bool):
for key in SERVER_SPEC_KEYS:
# if this is before retrieval mode has been set, OK not to have it
- if not key in server.keys() and retrieve_method_expected:
- raise MissingSettingsException(
- f'{key} not found in server spec: {server}')
+ if key not in server.keys() and retrieve_method_expected:
+ raise MissingSettingsException(
+ f'{key} not found in server spec: {server}')
class ProductLocatorLookup:
@@ -158,13 +170,14 @@ class ProductLocatorLookup:
for key, value in self.capo_db_settings.items():
self.credentials[key.replace('metadataDatabase.', '')] = value
- def look_up_locator_for_ext_name(self, external_name):
- '''
- Given a fileset ID or analogous identifier, find its product locator
+ def look_up_locator_for_ext_name(self, external_name) -> str:
+ """
+ Given a fileset ID or analogous identifier, find its product locator.
+
+ :param external_name: e.g., ngas_fileset_id
+ :return: product locator
+ """
- :param external_name:
- :return:
- '''
host, dbname = self.credentials['jdbcUrl'].split(':')[2][2:].split('/')
with pg.connect(dbname=dbname,
@@ -180,16 +193,22 @@ class ProductLocatorLookup:
return product_locator[0]
-class FlexLogger():
- ''' This class wraps a logger, adding the ability to specify
+class FlexLogger:
+ """
+ This class wraps a logger, adding the ability to specify
logging level as warning or debug based on the "verbose" flag
-
- '''
+ """
def __init__(self, class_name: str, output_dir: pathlib.Path,
verbose=False):
- ''' set up logging
- '''
+ """
+ Set up logging.
+ :param class_name: class to be logged
+ :param output_dir: where log is to be written
+ :param verbose: if true, additional output
+
+ """
+
if class_name is None:
raise MissingSettingsException('class name is required')
log_pathname = f'{output_dir}/{class_name}_{str(time())}.log'
@@ -210,25 +229,32 @@ class FlexLogger():
level = logging.DEBUG if self.verbose else logging.WARN
self.logger.setLevel(level)
- def set_verbose(self, verbose: bool):
- ''' specify verbose logging:
+ def set_verbose(self, verbose: bool) -> None:
+ """
+ Specify verbose logging:
True == debug and above
False == warnings and above
- '''
+
+ :param verbose:
+ :return:
+
+ """
+
self.verbose = verbose
def debug(self, message: str):
- ''' log a debug message
- '''
+ """ log a debug message """
+
if self.verbose:
self.logger.debug(message)
def warning(self, message: str):
- ''' log a warning message '''
+ """ log a warning message """
+
self.logger.warning(message)
def error(self, message: str):
- ''' log an error '''
+ """ log an error """
self.logger.error(message)
@@ -313,9 +339,7 @@ class Cluster(Enum):
class ExecutionSite(Enum):
- """
- Where this code is executing
- """
+ """ Where this code is executing """
DSOC = 'DSOC'
NAASC = 'NAASC'
@@ -324,9 +348,7 @@ class ExecutionSite(Enum):
class RetrievalMode(Enum):
- """
- How we're retrieving a file: via streaming or via
- direct copy (plugin)
+ """ How we're retrieving a file: via streaming or via direct copy (plugin)
"""
STREAM = 'stream'
COPY = 'copy'
diff --git a/apps/cli/executables/datafetcher/test/df_pytest_utils.py b/apps/cli/executables/datafetcher/test/df_pytest_utils.py
index 9669baed925b0cff6ecec7f9b8e32f458ec8623e..4a574be36d3350324043d4fdb83e46afbda9ed92 100644
--- a/apps/cli/executables/datafetcher/test/df_pytest_utils.py
+++ b/apps/cli/executables/datafetcher/test/df_pytest_utils.py
@@ -4,116 +4,144 @@
""" Various conveniences for use and re-use in test cases """
import json
+import logging
import os
+import sys
import tempfile
from pathlib import Path
import pytest
+
from pycapo import CapoConfig
+# pylint: disable=C0115, C0116, C0200, R0902, R0903, R0914, R1721, W0212, W0613, W0621, W0703, W1203
+
+from shared.workspaces.test.utilities import get_locations_report, \
+ get_test_data_dir
+
+from datafetcher.datafetcher import DataFetcher
+from datafetcher.return_codes import ReturnCode
from datafetcher.errors import MissingSettingsException, NoProfileException
from datafetcher.locations_report import LocationsReport
from datafetcher.utilities import REQUIRED_SETTINGS, get_arg_parser, \
- ExecutionSite, ProductLocatorLookup
+ ExecutionSite, ProductLocatorLookup, RetrievalMode
TEST_PROFILE = 'local'
+MISSING_SETTING = ReturnCode.MISSING_SETTING.value['code']
+MISSING_PROFILE = ReturnCode.MISSING_PROFILE.value['code']
+RUN_ALL = True
LOCATION_REPORTS = {
'VLA_SMALL_EB': {
- 'filename' : 'VLA_SMALL_EB.json',
- 'external_name' : 'sysstartS.58955.83384832176',
- 'file_count' : 44,
- 'server_count' : 2
+ 'filename': 'VLA_SMALL_EB.json',
+ 'external_name': 'sysstartS.58955.83384832176',
+ 'file_count': 44,
+ 'server_count': 2
},
'VLA_LARGE_EB': {
- 'filename' : 'VLA_LARGE_EB.json',
- 'external_name' : '17B-197.sb34812522.eb35115211.58168.58572621528',
- 'file_count' : 46,
- 'server_count' : 2
+ 'filename': 'VLA_LARGE_EB.json',
+ 'external_name': '17B-197.sb34812522.eb35115211.58168.58572621528',
+ 'file_count': 46,
+ 'server_count': 2
},
'VLA_BAD_SERVER': {
- 'filename' : 'VLA_BAD_SERVER.json',
- 'external_name' : 'TSKY_20min_B2319_18ms_001.58955.86469591435',
- 'file_count' : 1,
- 'server_count' : 1
+ 'filename': 'VLA_BAD_SERVER.json',
+ 'external_name': 'TSKY_20min_B2319_18ms_001.58955.86469591435',
+ 'file_count': 1,
+ 'server_count': 1
},
'IMG': {
- 'filename' : 'IMG.json',
- 'external_name' :
+ 'filename': 'IMG.json',
+ 'external_name':
'VLASS1.1.ql.T01t01.J000232-383000.10.2048.v1',
- 'file_count' : 2,
- 'server_count' : 2
+ 'file_count': 2,
+ 'server_count': 2
},
'VLBA_EB': {
- 'filename' : 'VLBA_EB.json',
- 'external_name' : '',
- 'file_count' : 16,
- 'server_count' : 1
+ 'filename': 'VLBA_EB.json',
+ 'external_name': '',
+ 'file_count': 16,
+ 'server_count': 1
},
'CALIBRATION': {
- 'filename' : 'CALIBRATION.json',
- 'external_name' : '18B-265_2019_12_10_T00_00_59.203.tar',
- 'file_count' : 1,
- 'server_count' : 1
+ 'filename': 'CALIBRATION.json',
+ 'external_name': '18B-265_2019_12_10_T00_00_59.203.tar',
+ 'file_count': 1,
+ 'server_count': 1
},
}
-def get_test_data_dir():
- """ where's our test data? """
- for root, dirnames, _ in os.walk(Path.cwd()):
- if str(root).endswith('test'):
- for dirname in dirnames:
- if dirname == 'data':
- return Path(root, dirname)
- return None
def get_locations_file(key: str):
- ''' return the location report file specified by key '''
+ """
+ Return location report file specified by key
+ :param key: location report name
+ :return:
+
+ """
+
report_spec = LOCATION_REPORTS[key]
filename = report_spec['filename']
return Path(get_test_data_dir(), filename)
-def get_locations_report(key: str):
- ''' return the location report specified by key '''
- report_file = get_locations_file(key)
- with open(report_file, 'r') as content:
- locations_report = json.loads(content.read())
- return locations_report
def write_locations_file(destination: Path, locations_report: LocationsReport):
- ''' write locations report to a file '''
+ """
+
+ :param destination: where locations file is to be written
+ :param locations_report: locations report from which we'll write the file
+
+ :return:
+
+ """
+
with open(destination, 'w') as to_write:
to_dump = {'files': locations_report['files']}
json.dump(to_dump, to_write, indent=4)
return destination
+
def get_mini_exec_block():
- ''' return a location report with large files excised
- '''
+ """
+ Returns a location report with large files excised
+
+ :return: attenuated location report
+
+ """
+
locations_in = get_locations_report('VLA_SMALL_EB')
locations_out = locations_in.copy()
locations_out['files'] = \
[file for file in locations_in['files'] if file['size'] <= 100000]
return locations_out
+
def get_mini_locations_file(destination):
- ''' return a location report file with large files excised
- '''
+ """
+ Returns a location report file with large files excised
+
+ :return: downsized location report file
+
+ """
+
locations_report = get_mini_exec_block()
with open(destination, 'w') as to_write:
to_dump = {'files': locations_report['files']}
json.dump(to_dump, to_write, indent=4)
return destination
+
def get_filenames_for_locator(product_locator: str, settings: dict):
- '''
+ """
For a given product locators, return names of all the files
in its locations report's files report
+
:param product_locator:
:param settings:
:return:
- '''
+ """
+
args = ['--product-locator', product_locator,
'--profile', TEST_PROFILE, '--output-dir', None]
namespace = get_arg_parser().parse_args(args)
@@ -122,10 +150,15 @@ def get_filenames_for_locator(product_locator: str, settings: dict):
return [file['relative_path'] for file in
locations_report.files_report['files']]
+
def find_newest_fetch_log_file(target_dir: Path):
- ''' data-fetcher command line was executed, perhaps more than once;
+ """
+ Data-fetcher command line was executed, perhaps more than once;
find the most recent log
- '''
+
+ :param target_dir: location of log file(s)
+ :return:
+ """
logfiles = list()
for root, _, filenames in os.walk(target_dir):
@@ -139,6 +172,7 @@ def find_newest_fetch_log_file(target_dir: Path):
return None
+
def get_test_capo_settings():
""" get the capo settings we'll need for the tests """
capo = CapoConfig(profile=TEST_PROFILE)
@@ -147,9 +181,9 @@ def get_test_capo_settings():
setting = setting.upper()
try:
result[REQUIRED_SETTINGS[setting]] = capo[setting]
- except KeyError:
+ except KeyError as k_err:
raise MissingSettingsException('missing required setting "{}"'
- .format(setting))
+ .format(setting)) from k_err
if result is None or len(result) == 0:
raise MissingSettingsException('Required Capo settings were not found')
@@ -167,10 +201,11 @@ def get_test_capo_settings():
return result
+
def get_metadata_db_settings(profile):
""" Get Capo settings needed to connect to archive DB
- :param profile:
- :return:
+ :param profile:
+ :return:
"""
result = dict()
if profile is None:
@@ -181,15 +216,19 @@ def get_metadata_db_settings(profile):
for field in qualified_fields:
try:
result[field] = capo.get(field)
- except KeyError:
+ except KeyError as k_err:
raise MissingSettingsException(
- f'missing required setting "{field}"')
+ f'missing required setting "{field}"') from k_err
return result
@pytest.fixture(autouse=True, scope='function')
-def make_tempdir() -> None:
- ''' Creates a new temporary working directory for each test. '''
+def make_tempdir() -> Path:
+ """
+ Creates a new temporary working directory for each test.
+
+ :return:
+ """
umask = os.umask(0o000)
top_level = tempfile.mkdtemp(prefix='datafetcher_test_', dir='/var/tmp')
os.umask(umask)
@@ -198,7 +237,10 @@ def make_tempdir() -> None:
@pytest.fixture(scope='session')
def capo_settings():
- ''' get Capo settings once for whole module '''
+ """
+ Gets Capo settings once for whole module.
+ :return:
+ """
def retrieve_capo_settings() -> CapoConfig:
return get_test_capo_settings()
@@ -209,8 +251,14 @@ def capo_settings():
@pytest.fixture(scope='session')
def settings(capo_settings):
- ''' grabs all the settings we will need for the datafetcher:
+ """
+ Grabs all the settings we will need for the datafetcher:
Capo, database, test data
+
+ :param capo_settings:
+ :return:
+ """
+ ''' g
'''
db_settings = get_metadata_db_settings(TEST_PROFILE)
test_data = _initialize_test_data(db_settings)
@@ -218,7 +266,13 @@ def settings(capo_settings):
def _initialize_test_data(db_settings):
- ''' Set up test data for use in several tests '''
+ """
+ Set up test data for use in several tests
+
+ :param db_settings:
+ :return:
+ """
+
ext_name = '13B-014.sb28862036.eb29155786.56782.5720116088'
product_locator = ProductLocatorLookup(db_settings) \
@@ -228,8 +282,68 @@ def _initialize_test_data(db_settings):
class Settings:
+ """ Encapsulates some settings for use in tests """
def __init__(self, capo_settings, db_settings, test_data):
self.capo_settings = capo_settings
self.db_settings = db_settings
self.test_data = test_data
+
+
+def launch_datafetcher(args: list, df_capo_settings: dict) -> int:
+ """ invoke the DF with these args as in df.main(),
+ launch it with df.run(),
+ and return the appropriate return/error code
+
+ """
+ if args is None or len(args) == 0:
+ return MISSING_SETTING
+
+ try:
+ namespace = evaluate_args_and_capo(args, df_capo_settings)
+ fetcher = DataFetcher(namespace, df_capo_settings)
+ return fetcher.run()
+ except SystemExit as exc:
+ if hasattr(exc, 'value'):
+ return exc.value.code if hasattr(exc.value, 'code') else exc.value
+ if hasattr(exc, 'code'):
+ return exc.code
+
+ raise
+ except (KeyError, NoProfileException) as exc:
+ logging.error(f'{exc}')
+ return MISSING_PROFILE
+ except Exception as exc:
+ pytest.fail(f'{exc}')
+
+
+def evaluate_args_and_capo(args: list, capo_settings: dict):
+
+ if args is None or len(args) == 0:
+ sys.exit(MISSING_SETTING)
+
+ profile = get_profile_from_args(args)
+ if profile is None:
+ profile = capo_settings['profile']
+ if profile is None:
+ sys.exit(MISSING_PROFILE)
+ else:
+ args['profile'] = profile
+
+ namespace = get_arg_parser().parse_args(args)
+ return namespace
+
+
+def get_profile_from_args(args: list) -> str:
+ for i in range(0, len(args)):
+ if args[i] == '--profile' and i < len(args) - 1:
+ profile = args[i + 1]
+ return profile
+
+ return ''
+
+
+def confirm_retrieve_mode_copy(servers_report: dict) -> None:
+ for server in servers_report:
+ entry = servers_report[server]
+ assert entry['retrieve_method'].value == RetrievalMode.COPY.value
diff --git a/apps/cli/executables/datafetcher/test/mock_data_fetcher.py b/apps/cli/executables/datafetcher/test/mock_data_fetcher.py
index 21ed02bab91a114aebfcf5069546e828876b4c43..c1f3473489e8f0aea088f472aa35b3222f18c7ee 100644
--- a/apps/cli/executables/datafetcher/test/mock_data_fetcher.py
+++ b/apps/cli/executables/datafetcher/test/mock_data_fetcher.py
@@ -10,9 +10,12 @@ from datafetcher.utilities import get_capo_settings, ExecutionSite, FlexLogger
from .df_pytest_utils import TEST_PROFILE
+# pylint: disable=C0103, R0201, R0902, R0903, W0621
+
class MockProdDataFetcher:
- ''' Creates and launches a datafetcher using the nmprod profile '''
+ """ Creates and launches a datafetcher using the dsoc-prod profile """
+
def __init__(self, args: Namespace, settings: dict):
if args is None or settings is None:
self._exit_with_error(ReturnCode.MISSING_SETTING)
@@ -42,9 +45,12 @@ class MockProdDataFetcher:
raise
def _get_locations(self):
- ''' create a locations report with DSOC as exec site
+ """
+ Create a locations report with DSOC as exec site
to force copy rather than stream
- '''
+ :return:
+ """
+
capo_settings = get_capo_settings(TEST_PROFILE)
capo_settings['execution_site'] = ExecutionSite.DSOC.value
diff --git a/apps/cli/executables/datafetcher/test/test_df_function.py b/apps/cli/executables/datafetcher/test/test_df_function.py
index 45f6b6f15b0fba1199b57132c820662beec23e99..1c8e83eb98cc21ce2f0cfb2f23cf3fee514caf7c 100644
--- a/apps/cli/executables/datafetcher/test/test_df_function.py
+++ b/apps/cli/executables/datafetcher/test/test_df_function.py
@@ -1,85 +1,30 @@
""" Unit tests for data-fetcher. """
-import sys
from pathlib import Path
import pytest
+
+# pylint: disable=C0115, C0116, C0200, C0415, R0801, R0902, R0903, R0914, R1721, W0212, W0611, W0613, W0621, W0703, W1203
+
from datafetcher.datafetcher import DataFetcher, ReturnCode
from datafetcher.utilities import get_arg_parser, ProductLocatorLookup, \
RetrievalMode, Location, Cluster
-# N.B. IJ is dumb -- NONE of these imports are unused!
+# N.B. these are all in use; SonarLint just doesn't get it
from .df_pytest_utils import TEST_PROFILE, get_mini_locations_file, \
- get_locations_file, LOCATION_REPORTS, make_tempdir, capo_settings, settings
+ get_locations_file, LOCATION_REPORTS, capo_settings, launch_datafetcher, \
+ settings, make_tempdir, RUN_ALL, confirm_retrieve_mode_copy
_LOCATION_FILENAME = 'locations.json'
_ASDM_XML = 'ASDM.xml'
_EB_EXTERNAL_NAME = 'sysstartS.58955.83384832176'
-MISSING_SETTING = ReturnCode.MISSING_SETTING.value['code']
-MISSING_PROFILE = ReturnCode.MISSING_PROFILE.value['code']
-
# set this to False when debugging one or more tests
# so as not to have to sit thru every test;
# comment out the target test(s)' @pytest.skip
-RUN_ALL = True
print(f'>>> RUNNING ALL TESTS: {RUN_ALL}')
-# pylint: disable=C0115, C0116, E0401, E1101, R0902, R0903, R0914, W1203, W0613, W0621
-
-
-def get_profile_from_args(args: list) -> str:
- for i in range(0, len(args)):
- if args[i] == '--profile' and i < len(args) - 1:
- profile = args[i + 1]
- return profile
-
- return None
-
-
-def evaluate_args_and_capo(args: list, capo_settings: dict):
-
- if args is None or len(args) == 0:
- sys.exit(MISSING_SETTING)
-
- profile = get_profile_from_args(args)
- if profile is None:
- profile = capo_settings['profile']
- if profile is None:
- sys.exit(MISSING_PROFILE)
- else:
- args['profile'] = profile
-
- namespace = get_arg_parser().parse_args(args)
- return namespace
-
-
-def launch_datafetcher(args: list, df_capo_settings: dict) -> int:
- """ invoke the DF with these args as in df.main(),
- launch it with df.run(),
- and return the appropriate return/error code
-
- """
- if args is None or len(args) == 0:
- return MISSING_SETTING
-
- try:
- namespace = evaluate_args_and_capo(args, df_capo_settings)
- fetcher = DataFetcher(namespace, df_capo_settings)
- return fetcher.run()
- except SystemExit as exc:
- if hasattr(exc, 'value'):
- return exc.value.code if hasattr(exc.value, 'code') else exc.value
- if hasattr(exc, 'code'):
- return exc.code
-
- raise
- except KeyError:
- sys.exit(MISSING_PROFILE)
- except Exception as exc:
- pytest.fail(f'{exc}')
-
def test_settings_setup(settings):
""" Ensure that the test settings we're using make sense """
@@ -261,7 +206,7 @@ def test_more_output_when_verbose(make_tempdir, settings):
@pytest.mark.skipif(not RUN_ALL, reason='debug')
def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings):
- """ We set profile to nmprod here so as to force the DF
+ """ We set profile to dsoc-prod here so as to force the DF
to try to copy rather than stream
"""
@@ -275,7 +220,7 @@ def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings):
# Instead, we mock this in test_df_return_codes
return
- prod_profile = 'nmprod'
+ prod_profile = 'dsoc-prod'
prod_props_filename = prod_profile + '.properties'
props_file = Path(make_tempdir, prod_props_filename)
@@ -287,9 +232,7 @@ def test_copy_attempt_throws_sys_exit_service_error(make_tempdir, settings):
fetcher = DataFetcher(namespace, settings.capo_settings)
servers_report = fetcher.servers_report
- for server in servers_report:
- entry = servers_report[server]
- assert entry['retrieve_method'].value == RetrievalMode.COPY.value
+ confirm_retrieve_mode_copy(servers_report)
# let's try just one file so we're not sitting here all day
for server in servers_report:
@@ -512,7 +455,7 @@ def test_gets_calibration_from_report_file(mock_successful_fetch_run,
fake_file = None
file_info = None
- # (there will be just one file, therefore iteration)
+ # (there may be more than one file; thus iteration)
for server in servers_report.items():
metadata = server[1]
destination = Path(make_tempdir)
@@ -551,3 +494,28 @@ def test_gets_calibration_from_locator(mock_successful_fetch_run,
assert fake_file.is_file()
contents = fake_file.read_text().strip()
assert int(contents) == file_spec['size']
+
+def test_gets_gbt_data_from_locator(make_tempdir, settings):
+ """ Can we cope with GBT data? """
+
+ external_name = 'AGBT17B_044_553492'
+ product_locator = ProductLocatorLookup(settings.db_settings)\
+ .look_up_locator_for_ext_name(external_name)
+ args = ['--product-locator', product_locator,
+ '--output-dir', str(make_tempdir), '--profile', TEST_PROFILE]
+ namespace = get_arg_parser().parse_args(args)
+ fetch = DataFetcher(namespace, settings.capo_settings)
+ report_files = fetch.locations_report.files_report['files']
+ assert len(report_files) == 1
+
+ file_spec = report_files[0]
+ relative_path = file_spec['relative_path']
+ assert relative_path == 'AGBT17B_044_01.tar'
+ destination = Path(make_tempdir) / relative_path
+ destination.mkdir()
+ write_fake_file(destination, file_spec)
+
+ fake_file = Path(destination, file_spec['ngas_file_id'])
+ assert fake_file.is_file()
+ contents = fake_file.read_text().strip()
+ assert int(contents) == file_spec['size']
\ No newline at end of file
diff --git a/apps/cli/executables/datafetcher/test/test_df_return_codes.py b/apps/cli/executables/datafetcher/test/test_df_return_codes.py
index bdbfc7e916a1820b9c965016d260edc956b976e0..73abbd8575b43e5164ac97f4e6d7b242fc586c9e 100644
--- a/apps/cli/executables/datafetcher/test/test_df_return_codes.py
+++ b/apps/cli/executables/datafetcher/test/test_df_return_codes.py
@@ -1,26 +1,26 @@
-''' Tests of datafetcher return codes:
+""" Tests of datafetcher return codes:
check that arguments passed return expected codes
+"""
-'''
+# pylint: disable=C0115, C0116, R0801, R0902, R0903, R0914, W0212, W0611 W0613, W0621, W0703, W1203
import os
from pathlib import Path
import pytest
+
from datafetcher.datafetcher import DataFetcher
from datafetcher.return_codes import ReturnCode
-from datafetcher.utilities import get_arg_parser, RetrievalMode
+from datafetcher.utilities import get_arg_parser, ProductLocatorLookup
-from .mock_data_fetcher import MockProdDataFetcher
# N.B. IJ doesn't recognize imported fixtures as being in use.
# don't let these imports (make_tempdir, settings) get disappeared.
-from .test_df_function import launch_datafetcher, RUN_ALL, \
- MISSING_PROFILE, MISSING_SETTING
-from .df_pytest_utils import TEST_PROFILE, get_test_capo_settings, make_tempdir, \
- capo_settings, settings
+from .df_pytest_utils import TEST_PROFILE, get_test_capo_settings, \
+ make_tempdir, capo_settings, settings, launch_datafetcher, \
+ MISSING_SETTING, MISSING_PROFILE, RUN_ALL, confirm_retrieve_mode_copy
+from .mock_data_fetcher import MockProdDataFetcher
-# pylint: disable=C0115, C0116, E0401, E1101, R0902, R0903, R0914, W0613, W0621, W1203
def test_launch_df(make_tempdir, settings):
args = ['--product-locator', settings.test_data['product_locator'],
@@ -34,7 +34,7 @@ def test_launch_df(make_tempdir, settings):
def test_launch_df_no_args(settings):
try:
- return_code = launch_datafetcher(None, settings.capo_settings)
+ return_code = launch_datafetcher([], settings.capo_settings)
assert return_code is not None
except Exception as exc:
pytest.fail(f'{exc}')
@@ -42,10 +42,13 @@ def test_launch_df_no_args(settings):
@pytest.mark.skipif(not RUN_ALL, reason='debug')
def test_no_args_prints_usage_and_croaks():
- ''' If the datafetcher is called without any arguments,
+ """
+ If the datafetcher is called without any arguments,
it should print the "usage" statement, then exit
- '''
+ :return:
+
+ """
with pytest.raises(TypeError):
datafetcher = DataFetcher()
@@ -53,38 +56,57 @@ def test_no_args_prints_usage_and_croaks():
# @pytest.mark.skipif(not RUN_ALL, reason='debug')
-def test_omitted_capo_args_returns_expected_code(make_tempdir, settings):
- ''' Be sure DF dies with appropriate error message if called without Capo profile '''
+def test_omitted_profile_returns_expected_code(make_tempdir, settings):
+ """
+ Be sure DF dies with appropriate error message
+ if called without Capo profile
+
+ :param make_tempdir: tempdir created on the fly
+ :param settings: gives us test data
+ :return:
+ """
args = ['--product-locator', settings.test_data['product_locator'],
'--output-dir', str(make_tempdir)]
- with pytest.raises(SystemExit) as exc:
- launch_datafetcher(args, settings.capo_settings)
- assert exc.value.code == 1
+ return_code = launch_datafetcher(args, settings.capo_settings)
+ assert return_code == MISSING_PROFILE
@pytest.mark.skipif(not RUN_ALL, reason='debug')
def test_omitted_capo_value_returns_expected_code(make_tempdir, settings):
- ''' Be sure DF dies with appropriate error message if Capo profile
- argument but no profile value
+ """
+
+ :param make_tempdir: tempdir created on the fly
+ :param settings: source of Capo settings
+
+ :return:
- '''
+ """
args = ['--product-locator', "we're not going to get this far",
- '--output-dir', str(make_tempdir),
- '--profile']
- with pytest.raises(SystemExit) as exc:
- launch_datafetcher(args, settings.capo_settings)
- assert exc.value.code == 1
+ '--output-dir',
+ '--profile', TEST_PROFILE]
+ result = launch_datafetcher(args, settings.capo_settings)
+ assert result == MISSING_SETTING
@pytest.mark.skipif(not RUN_ALL, reason='debug')
def test_invalid_capo_profile_returns_expected_code(make_tempdir, settings):
- ''' Be sure DF dies with appropriate error message if called without Capo profile '''
+ """
+ Be sure DF dies with appropriate error message
+ if called without Capo profile
+
+ :param make_tempdir: tempdir created on the fly
+ :param settings: source of DB settings
+ :return:
+
+ """
+ locator = ProductLocatorLookup(settings.db_settings)\
+ .look_up_locator_for_ext_name('13B-014.sb28862036.eb29155786.56782.5720116088')
args = ['--profile', 'whatevs',
- '--product-locator', "we'll never get this far",
+ '--product-locator', locator,
'--output-dir', str(make_tempdir),
]
return_code = launch_datafetcher(args, settings.capo_settings)
@@ -92,7 +114,11 @@ def test_invalid_capo_profile_returns_expected_code(make_tempdir, settings):
def we_are_in_docker():
- ''' if we are in a Docker container, the SHELL env var will not be set '''
+ """
+ Are we in a Docker container? If so, SHELL env var will not be set
+
+ :return:
+ """
for key, _ in os.environ.items():
if key == 'SHELL':
@@ -103,8 +129,8 @@ def we_are_in_docker():
@pytest.mark.skipif(not RUN_ALL, reason='debug')
def test_inaccessible_output_dir_returns_expected_code(settings, make_tempdir):
+ umask = os.umask(0o000)
try:
- umask = os.umask(0o000)
tmpdir = Path(make_tempdir)
tmpdir.chmod(0o666)
@@ -143,10 +169,16 @@ def test_inaccessible_output_dir_returns_expected_code(settings, make_tempdir):
@pytest.mark.skipif(not RUN_ALL, reason='debug')
def test_two_locator_args_returns_expected_code(make_tempdir, settings):
- ''' We should reject invocation with both product locator -and- location
+ """
+
+ We should reject invocation with both product locator -and- location
file. One or the other, people!
- '''
+ :param make_tempdir: tempdir created on the fly
+ :param settings: source of Capo settings
+
+ :return:
+ """
args = ['--product-locator', 'a_locator',
'--location-file', 'location.json'
@@ -157,7 +189,7 @@ def test_two_locator_args_returns_expected_code(make_tempdir, settings):
class MockServiceTimeoutReturn:
- ''' Simulates locator request service timeout '''
+ """ Simulates locator request service timeout """
@staticmethod
def run():
@@ -235,19 +267,19 @@ def test_copy_attempt_throws_sys_exit_service_error(monkeypatch,
fetcher = MockProdDataFetcher(namespace, settings.capo_settings)
servers_report = fetcher.servers_report
- for server in servers_report:
- entry = servers_report[server]
- assert entry['retrieve_method'].value == RetrievalMode.COPY.value
+ confirm_retrieve_mode_copy(servers_report)
# let's try just one file so we're not sitting here all day
+ a_server = None
for server in servers_report:
entry = servers_report[server]
servers_report = {server: entry}
fetcher.servers_report = servers_report
assert fetcher.servers_report[server] is not None
+ a_server = server
break
- files = fetcher.servers_report[server]['files']
- fetcher.servers_report[server]['files'] = [files[0]]
+ files = fetcher.servers_report[a_server]['files']
+ fetcher.servers_report[a_server]['files'] = [files[0]]
with pytest.raises(SystemExit) as exc:
fetcher.run()
@@ -276,7 +308,7 @@ def test_unable_to_open_location_file_returns_expected_code(make_tempdir,
class MockNgasFetchError:
- ''' Simulates a problem fetching file(s) from NGAS server '''
+ """ Simulates a problem fetching file(s) from NGAS server """
@staticmethod
def run():
@@ -299,7 +331,7 @@ def test_error_fetching_file_from_ngas_returns_expected_code(monkeypatch,
class MockSizeMismatchError:
- ''' Simulates a retrieved file of unexpected size '''
+ """ Simulates a retrieved file of unexpected size """
@staticmethod
def run():
diff --git a/apps/cli/launchers/pymygdala/src/pymygdala/commands.py b/apps/cli/launchers/pymygdala/src/pymygdala/commands.py
index bb322b070823907b7a6de0bbd644f0ffd8337f4d..650b2a9d6f99201e0024441306377a72e1f6cd51 100644
--- a/apps/cli/launchers/pymygdala/src/pymygdala/commands.py
+++ b/apps/cli/launchers/pymygdala/src/pymygdala/commands.py
@@ -9,6 +9,10 @@ import re
from ._version import ___version___ as version
from .models import LogDumper, LogHandler, SendEvent, SendNRAOEvent, MotDBroadcaster
+# stolen from datafetcher; make this universal
+VALID_PROFILES = ['naasc-dev', 'naasc-test', 'dsoc-test', 'dsoc-dev',
+ 'dsoc-prod', 'local']
+
_ERR_MISSING_PROFILE = r"""ERROR: {script} can't deduce the 'profile', give it the --profile argument or set the CAPO_PROFILE environment variable! Geeze!
"""
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b0397d659f9a30254a1f52d9d9641d913baa1925
--- /dev/null
+++ b/docker-compose.local.yml
@@ -0,0 +1,48 @@
+version: '3.8'
+services:
+ db:
+ image: postgres:12.4
+ restart: always
+ environment:
+ POSTGRES_USER: archive
+ POSTGRES_PASSWORD: docker
+ command: postgres -c listen_addresses=*
+
+ amqp:
+ image: rabbitmq:3.8
+ restart: always
+
+ schema:
+ build:
+ context: ./schema/
+ dockerfile: Dockerfile.local
+ depends_on:
+ - db
+ volumes:
+ - ./schema:/code/schema
+
+ workflow:
+ build:
+ context: ./services/workflow/
+ dockerfile: Dockerfile.local
+ ports:
+ - "3456:3456"
+ depends_on:
+ - schema
+ volumes:
+ - ./services/workflow:/code
+ - ./shared:/code/src/shared
+ - ./apps:/code/src/apps
+
+ capability:
+ build:
+ context: ./services/capability/
+ dockerfile: Dockerfile.local
+ ports:
+ - "3457:3457"
+ depends_on:
+ - schema
+ volumes:
+ - ./services/capability:/code
+ - ./shared:/code/src/shared
+ - ./apps:/code/src/apps
diff --git a/schema/Dockerfile.local b/schema/Dockerfile.local
new file mode 100644
index 0000000000000000000000000000000000000000..81c4877c92467de19dfefefdd522285ec76d5dd8
--- /dev/null
+++ b/schema/Dockerfile.local
@@ -0,0 +1,7 @@
+# This is nrao:schema
+# The purpose of this is just to run the alembic migrations against the database in a docker-compose environment
+FROM marconi.aoc.nrao.edu/ops/base:workspaces
+
+WORKDIR /code/schema
+
+ENTRYPOINT ["./bin/run-migrations.sh", "docker"]
diff --git a/schema/bin/run-migrations.sh b/schema/bin/run-migrations.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ee68be168865c1768a3bedb5f68a4c32957c788e
--- /dev/null
+++ b/schema/bin/run-migrations.sh
@@ -0,0 +1,17 @@
+#! /bin/bash
+
+# FOR USE WITH DOCKER DEVELOPMENT ENVIRONMENT ONLY.
+
+# failfast
+set -e
+set -o pipefail
+
+if [[ $# -eq 0 ]] ; then
+ echo 'ERROR: run-migrations requires an argument for CAPO_PROFILE'
+ exit 0
+fi
+
+# Python library installation
+pip install -r requirements.txt
+
+env CAPO_PROFILE=$1 alembic upgrade head
diff --git a/schema/requirements.txt b/schema/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c9431afaba8553b6a03849ff2e3aafaf80bf69c3
--- /dev/null
+++ b/schema/requirements.txt
@@ -0,0 +1,6 @@
+# This file is intended to support the Condaless.dockerfile, not for actual development
+# DO NOT MODIFY THIS FILE, THIS IS FOR DOCKER ONLY!
+
+alembic == 1.4
+pycapo == 0.3.0
+psycopg2 == 2.8
diff --git a/services/capability/Dockerfile.local b/services/capability/Dockerfile.local
new file mode 100644
index 0000000000000000000000000000000000000000..ffdeea4affd66a7b54ded9df7af73356a39a3e10
--- /dev/null
+++ b/services/capability/Dockerfile.local
@@ -0,0 +1,6 @@
+# This is nrao:capability
+FROM marconi.aoc.nrao.edu/ops/base:workspaces
+
+WORKDIR /code/
+
+ENTRYPOINT ["./bin/boot-local.sh"]
diff --git a/services/capability/README.md b/services/capability/README.md
index e0dc6f31b9f24903492a0c38a73e95f8b4d94082..2a6abc47cfc7c56aac4e60c88bb6f16a8926297f 100644
--- a/services/capability/README.md
+++ b/services/capability/README.md
@@ -13,3 +13,5 @@ Then you should be able to run this with:
Once there, try the following URLs:
- http://0.0.0.0:3456/capabilities
+- http://0.0.0.0:3456/capability/request?capability=null
+
diff --git a/services/capability/bin/boot-local.sh b/services/capability/bin/boot-local.sh
new file mode 100755
index 0000000000000000000000000000000000000000..a358ab054fc4945b1e058323a577ebdd6bcb5f5e
--- /dev/null
+++ b/services/capability/bin/boot-local.sh
@@ -0,0 +1,12 @@
+#! /bin/bash
+
+
+# FOR USE WITH DOCKER DEVELOPMENT ENVIRONMENT ONLY.
+# Capability Service
+
+# Python library installation
+pip install -r requirements.txt
+python setup.py develop
+
+# Start development server
+pserve --reload development.ini
diff --git a/services/capability/requirements.txt b/services/capability/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a13b934774651c0032da937d78fb4f604570e56a
--- /dev/null
+++ b/services/capability/requirements.txt
@@ -0,0 +1,18 @@
+# This file is intended to support the Condaless.dockerfile, not for actual development
+# DO NOT MODIFY THIS FILE, THIS IS FOR DOCKER ONLY!
+
+-e ./src/shared/schema
+-e ./src/shared/channels
+-e ./src/shared/workspaces
+-e ./src/apps/cli/utilities/wf_monitor
+
+pycapo == 0.3.0
+pyramid == 1.10
+pyramid_beaker == 0.8
+pyramid_debugtoolbar == 4.5
+pyramid_retry == 2.1.1
+pyramid_tm == 2.2.1
+requests == 2.23
+sqlalchemy == 1.3
+waitress == 1.4
+zope.sqlalchemy == 1.1
diff --git a/services/workflow/Dockerfile.local b/services/workflow/Dockerfile.local
new file mode 100644
index 0000000000000000000000000000000000000000..b6057977d84f408a26a78c7a4470f259c00598ce
--- /dev/null
+++ b/services/workflow/Dockerfile.local
@@ -0,0 +1,6 @@
+# This is nrao:workflow
+FROM marconi.aoc.nrao.edu/ops/base:workspaces
+
+WORKDIR /code/
+
+ENTRYPOINT ["./bin/boot-local.sh"]
\ No newline at end of file
diff --git a/services/workflow/README.md b/services/workflow/README.md
index 14947f8ba6eacbf9563051544a8a1cab7d67324f..b9dff7b35303231500608a1c592f01617aaa6a97 100644
--- a/services/workflow/README.md
+++ b/services/workflow/README.md
@@ -11,7 +11,7 @@ Then you should be able to run this with:
Once there, try the following URLs:
+- http://0.0.0.0:3456/workflows/requests/94
- http://0.0.0.0:3456/workflows
- http://0.0.0.0:3456/workflows/0
-- http://0.0.0.0:3456/workflows/0/files
-- http://0.0.0.0:3456/workflows/0/files/file.txt
+
diff --git a/services/workflow/bin/boot-local.sh b/services/workflow/bin/boot-local.sh
new file mode 100755
index 0000000000000000000000000000000000000000..a7f6a4cc855ca110de73ca6f4f7c221bde3e0182
--- /dev/null
+++ b/services/workflow/bin/boot-local.sh
@@ -0,0 +1,11 @@
+#! /bin/bash
+
+# FOR USE WITH DOCKER DEVELOPMENT ENVIRONMENT ONLY.
+# Workflow Service
+
+# Python library installation
+pip install -r requirements.txt
+python setup.py develop
+
+# Start development server
+pserve --reload development.ini
diff --git a/services/workflow/requirements.txt b/services/workflow/requirements.txt
index 423da295f99c1252686c147b0e22330854201bcf..a13b934774651c0032da937d78fb4f604570e56a 100644
--- a/services/workflow/requirements.txt
+++ b/services/workflow/requirements.txt
@@ -1,10 +1,10 @@
# This file is intended to support the Condaless.dockerfile, not for actual development
# DO NOT MODIFY THIS FILE, THIS IS FOR DOCKER ONLY!
--e ../../shared/schema
--e ../../shared/channels
--e ../../shared/workspaces
--e ../../apps/cli/utilities/wf_monitor
+-e ./src/shared/schema
+-e ./src/shared/channels
+-e ./src/shared/workspaces
+-e ./src/apps/cli/utilities/wf_monitor
pycapo == 0.3.0
pyramid == 1.10
diff --git a/shared/__init__.py b/shared/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/shared/workspaces/__init__.py b/shared/workspaces/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/shared/workspaces/setup.py b/shared/workspaces/setup.py
index a6877866148fca7e694f12af7d2c345dc5178233..5a6907469ba89c78d64ebe7037fbba12bf9e3fe5 100644
--- a/shared/workspaces/setup.py
+++ b/shared/workspaces/setup.py
@@ -8,11 +8,11 @@ VERSION = open("src/workspaces/_version.py").readlines()[-1].split()[-1].strip("
README = Path("README.md").read_text()
requires = [
- "pycapo",
- "marshmallow",
- "ssa-schema",
- "ssa-channels",
- "sqlalchemy",
+ 'pycapo',
+ 'marshmallow',
+ 'ssa-schema',
+ 'sqlalchemy',
+ 'cx_Oracle'
]
tests_require = ["pytest>=5.4,<6.0", "pendulum==2.1.2"]
diff --git a/shared/workspaces/test/test_data/location_files/17A-109_fg_18468.json b/shared/workspaces/test/test_data/location_files/17A-109_fg_18468.json
new file mode 100644
index 0000000000000000000000000000000000000000..32bde904d7761d303360c49c54a7cbf4004d11da
--- /dev/null
+++ b/shared/workspaces/test/test_data/location_files/17A-109_fg_18468.json
@@ -0,0 +1,1111 @@
+{
+ "files": [
+ {
+ "ngas_file_id": "17A-109_2017_02_11_T18_49_09.756.tar",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "17A-109_2017_02_11_T18_49_09.756.tar",
+ "checksum": "655111456",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 308162560,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486843810162.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486843810162",
+ "checksum": "663099363",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 456053712,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486843814178.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486843814178",
+ "checksum": "-1575831526",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 583709759,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844218099.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844218099",
+ "checksum": "49909197",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 12812488,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844226472.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844226472",
+ "checksum": "-179810078",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844235473.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844235473",
+ "checksum": "429529341",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844255472.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844255472",
+ "checksum": "1482990574",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844275472.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844275472",
+ "checksum": "-119129895",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844295481.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844295481",
+ "checksum": "956013048",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844315498.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844315498",
+ "checksum": "-367758597",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844335493.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844335493",
+ "checksum": "-1537359880",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844355471.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844355471",
+ "checksum": "296569084",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844375472.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844375472",
+ "checksum": "1799693437",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844395472.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844395472",
+ "checksum": "1961966131",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342341,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844415472.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844415472",
+ "checksum": "-2098677238",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342341,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844435471.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844435471",
+ "checksum": "158263524",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342341,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844455471.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844455471",
+ "checksum": "350245303",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342341,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844475507.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844475507",
+ "checksum": "751683591",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 34590885,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844500699.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844500699",
+ "checksum": "1609401853",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 276499319,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844523730.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844523730",
+ "checksum": "1790852044",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1198131341,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844557502.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844557502",
+ "checksum": "-1121087048",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 64054248,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844723339.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844723339",
+ "checksum": "315338626",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844724221.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844724221",
+ "checksum": "1674442290",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844744222.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844744222",
+ "checksum": "1287922656",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844764222.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844764222",
+ "checksum": "-972094821",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844784221.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844784221",
+ "checksum": "1454111818",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844804222.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844804222",
+ "checksum": "-1009198755",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844824222.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844824222",
+ "checksum": "1616941529",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844844221.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844844221",
+ "checksum": "1298053941",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844864266.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844864266",
+ "checksum": "801646273",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 24342226,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844884221.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844884221",
+ "checksum": "2041766225",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 26904477,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844910410.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844910410",
+ "checksum": "575271856",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 276499319,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844926183.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844926183",
+ "checksum": "1352508634",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 890920896,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844956129.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844956129",
+ "checksum": "99833159",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5284033620,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486844956129.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486844956129",
+ "checksum": "99833159",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5284033620,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486845045880.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486845045880",
+ "checksum": "28664494",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 983084100,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486845564431.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486845564431",
+ "checksum": "-2045708831",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5284033620,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486845564431.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486845564431",
+ "checksum": "-2045708831",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5284033620,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486845664228.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486845664228",
+ "checksum": "-2053692847",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 983084100,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486846182879.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486846182879",
+ "checksum": "-623334847",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5284033620,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486846182879.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486846182879",
+ "checksum": "-623334847",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5284033620,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486846282749.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486846282749",
+ "checksum": "2010876500",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 983084100,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486846801430.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486846801430",
+ "checksum": "1233750892",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2979953520,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1486846801430.bdf",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1486846801430",
+ "checksum": "1233750892",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2979953520,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806805.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ASDM.xml",
+ "checksum": "1172258150",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 8615,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806806.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Antenna.xml",
+ "checksum": "-677168697",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 10121,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806807.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "CalData.xml",
+ "checksum": "-724222452",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 4038,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806808.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "CalDevice.xml",
+ "checksum": "-830921761",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 582013,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806809.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "CalPointing.xml",
+ "checksum": "1083482267",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 392,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806810.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "CalReduction.xml",
+ "checksum": "1694599615",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 4564,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806811.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ConfigDescription.xml",
+ "checksum": "472738810",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 4090,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806812.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "CorrelatorMode.xml",
+ "checksum": "-1102843093",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 897,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806813.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "DataDescription.xml",
+ "checksum": "-60451139",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 20374,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806814.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "DelayModel.xml",
+ "checksum": "-1617702445",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 389,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806815.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "DelayModelFixedParameters.xml",
+ "checksum": "-1446284715",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1453,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806816.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "DelayModelVariableParameters.xml",
+ "checksum": "-927827839",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 3261,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806817.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Doppler.xml",
+ "checksum": "1273348133",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 380,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806818.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Ephemeris.xml",
+ "checksum": "-959903488",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 386,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806819.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "ExecBlock.xml",
+ "checksum": "-926627879",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2165,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806820.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Feed.xml",
+ "checksum": "1344268705",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 758371,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806821.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Field.xml",
+ "checksum": "-1985034726",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1963,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806822.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Flag.xml",
+ "checksum": "444358800",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 291230,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806823.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Main.xml",
+ "checksum": "1394615011",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 35316,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806824.bin",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Pointing.bin",
+ "checksum": "1411567783",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 122865,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806825.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "PointingModel.xml",
+ "checksum": "728088158",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 10313,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806826.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Polarization.xml",
+ "checksum": "1552196607",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 804,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806827.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Processor.xml",
+ "checksum": "-1327068712",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 612,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806828.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Receiver.xml",
+ "checksum": "849567317",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 21353,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806829.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "SBSummary.xml",
+ "checksum": "-1228933078",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1500,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806830.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Scan.xml",
+ "checksum": "-833365872",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 8631,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806831.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Source.xml",
+ "checksum": "-1155032609",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 92121,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806832.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "SpectralWindow.xml",
+ "checksum": "1255400839",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 40941,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806833.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "State.xml",
+ "checksum": "-685612526",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 551,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806834.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Station.xml",
+ "checksum": "-572802594",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5535,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806835.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Subscan.xml",
+ "checksum": "-2037502563",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 20542,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806836.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "SwitchCycle.xml",
+ "checksum": "1926976049",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 697,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806837.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "SysCal.xml",
+ "checksum": "1719682198",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 377,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806838.bin",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "SysPower.bin",
+ "checksum": "14846238",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 293133635,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1486843806839.sdm",
+ "subdirectory": "17A-109.sb33151327.eb33496982.57795.84034732639",
+ "relative_path": "Weather.xml",
+ "checksum": "-933293551",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 24832,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ }
+ ],
+ "aggregate_size": 45572444823
+}
diff --git a/shared/workspaces/test/test_data/location_files/17A-109_fg_41979.json b/shared/workspaces/test/test_data/location_files/17A-109_fg_41979.json
new file mode 100644
index 0000000000000000000000000000000000000000..0ad9d082bc47ddeb32c83874d2d59803b424eb59
--- /dev/null
+++ b/shared/workspaces/test/test_data/location_files/17A-109_fg_41979.json
@@ -0,0 +1,985 @@
+{
+ "files": [
+ {
+ "ngas_file_id": "17A-109_2017_05_19_T19_25_06.583.tar",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "17A-109_2017_05_19_T19_25_06.583.tar",
+ "checksum": "-1917823728",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 2,
+ "size": 263628800,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495208975419.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495208975419",
+ "checksum": "-144787515",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 714151256,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495209031153.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495209031153",
+ "checksum": "1370765046",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 238896305,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495209554929.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495209554929",
+ "checksum": "-2076123938",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 119451725,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495209564938.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495209564938",
+ "checksum": "-173613419",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 549452213,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495209594847.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495209594847",
+ "checksum": "130192588",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 836119205,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495209714529.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495209714529",
+ "checksum": "-104464587",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 979452701,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495209894045.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495209894045",
+ "checksum": "1016770004",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2699454653,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495210103477.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495210103477",
+ "checksum": "-1844821169",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2556121157,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495210671878.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495210671878",
+ "checksum": "-1341756846",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 406118717,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495211210428.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495211210428",
+ "checksum": "-1083701930",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 812230494,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495211300177.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495211300177",
+ "checksum": "1529871085",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 692785884,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495211479678.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495211479678",
+ "checksum": "1575043793",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2699455332,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495211629276.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495211629276",
+ "checksum": "-1464110576",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2556121800,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495212197730.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495212197730",
+ "checksum": "-449082576",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 406118820,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495212736243.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495212736243",
+ "checksum": "522497003",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2675566410,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495212825976.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495212825976",
+ "checksum": "228005896",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2556121800,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495213394428.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495213394428",
+ "checksum": "-132216914",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 406118820,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495213932977.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495213932977",
+ "checksum": "1382167528",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 836119416,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495214022728.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495214022728",
+ "checksum": "-373617661",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 692785884,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495214202227.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495214202227",
+ "checksum": "-748160633",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2699455332,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495214351826.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495214351826",
+ "checksum": "492163010",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2532232878,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495214920278.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495214920278",
+ "checksum": "-1577909914",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 406118820,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495215458777.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495215458777",
+ "checksum": "-241925708",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2699455332,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495215548541.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495215548541",
+ "checksum": "2125498241",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2556121800,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495216116978.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495216116978",
+ "checksum": "1300175828",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 406118820,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495216655527.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495216655527",
+ "checksum": "-1557323292",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 812230494,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495216745278.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495216745278",
+ "checksum": "-543519127",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 692785884,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495216924777.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495216924777",
+ "checksum": "225409799",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2699455332,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495217074376.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495217074376",
+ "checksum": "-712757804",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2699455332,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495217642829.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495217642829",
+ "checksum": "-1926263831",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 406118820,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495218211293.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495218211293",
+ "checksum": "-1716691546",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2675566410,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495218301030.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495218301030",
+ "checksum": "-1982648223",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2699455332,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495218869478.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495218869478",
+ "checksum": "1338161107",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 406118820,
+ "server": {
+ "server": "nmngas02.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid____evla_bdf_1495219437879.bdf",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDMBinary/uid____evla_bdf_1495219437879",
+ "checksum": "1668836489",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1122786480,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972975.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ASDM.xml",
+ "checksum": "-388730196",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 8613,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972976.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Antenna.xml",
+ "checksum": "-1406746166",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 10914,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972977.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "CalData.xml",
+ "checksum": "2129790671",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 6655,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972978.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "CalDevice.xml",
+ "checksum": "-557576464",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 428455,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972979.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "CalPointing.xml",
+ "checksum": "2040612067",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 392,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972980.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "CalReduction.xml",
+ "checksum": "766431827",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 7548,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972981.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ConfigDescription.xml",
+ "checksum": "403499755",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 3614,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972982.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "CorrelatorMode.xml",
+ "checksum": "-154024918",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 897,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972983.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "DataDescription.xml",
+ "checksum": "-171436082",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 13974,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972984.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "DelayModel.xml",
+ "checksum": "137878731",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 389,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972985.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "DelayModelFixedParameters.xml",
+ "checksum": "-732868035",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1453,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972986.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "DelayModelVariableParameters.xml",
+ "checksum": "-2132151079",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 3258,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972987.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Doppler.xml",
+ "checksum": "-1139741059",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 380,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972988.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Ephemeris.xml",
+ "checksum": "-1496097382",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 386,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972989.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "ExecBlock.xml",
+ "checksum": "1276336790",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 2187,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972990.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Feed.xml",
+ "checksum": "1902005950",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 557905,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972991.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Field.xml",
+ "checksum": "-217023133",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1963,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972992.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Flag.xml",
+ "checksum": "-799820329",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 279897,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972993.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Main.xml",
+ "checksum": "-1996977948",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 31458,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972994.bin",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Pointing.bin",
+ "checksum": "-804680729",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1305,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972995.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "PointingModel.xml",
+ "checksum": "1135443205",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 11107,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972996.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Polarization.xml",
+ "checksum": "1289334348",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 804,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972997.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Processor.xml",
+ "checksum": "-2032915679",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 612,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972998.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Receiver.xml",
+ "checksum": "1476808437",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 14517,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208972999.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "SBSummary.xml",
+ "checksum": "-2039108750",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 1500,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973000.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Scan.xml",
+ "checksum": "539809260",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 16701,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973001.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Source.xml",
+ "checksum": "-1518702232",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 61347,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973002.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "SpectralWindow.xml",
+ "checksum": "-35188444",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 27799,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973003.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "State.xml",
+ "checksum": "1080152948",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 551,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973004.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Station.xml",
+ "checksum": "1166103029",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 5972,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973005.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Subscan.xml",
+ "checksum": "-609938311",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 20039,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973006.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "SwitchCycle.xml",
+ "checksum": "2140647318",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 697,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973007.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "SysCal.xml",
+ "checksum": "1417207942",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 377,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973008.bin",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "SysPower.bin",
+ "checksum": "-1428249263",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 865551713,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ },
+ {
+ "ngas_file_id": "uid___evla_sdm_X1495208973009.sdm",
+ "subdirectory": "17A-109.sb33151331.eb33786546.57892.65940042824",
+ "relative_path": "Weather.xml",
+ "checksum": "628952511",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 77335,
+ "server": {
+ "server": "nmngas01.aoc.nrao.edu:7777",
+ "location": "DSOC",
+ "cluster": "DSOC"
+ }
+ }
+ ],
+ "aggregate_size": 50076899992
+}
diff --git a/shared/workspaces/test/test_data/location_files/A001_X1296_Xa93_RAW.json b/shared/workspaces/test/test_data/location_files/A001_X1296_Xa93_RAW.json
new file mode 100644
index 0000000000000000000000000000000000000000..bf1b505321c0f5d680c6d5b8099c30e0c48f7bf2
--- /dev/null
+++ b/shared/workspaces/test/test_data/location_files/A001_X1296_Xa93_RAW.json
@@ -0,0 +1,265 @@
+{
+ "TAR": {
+ "uid___A002_Xcd8029_Xb0a4.400881588.tar.gz": 457195765
+ },
+ "SDM": {
+ "Annotation.xml": 1505,
+ "Antenna.xml": 4165,
+ "ASDM.xml": 10577,
+ "CalAmpli.xml": 29180,
+ "CalAtmosphere.bin": 21793806,
+ "CalAtmosphere.xml": 1319,
+ "CalData.xml": 11865,
+ "CalDevice.xml": 3478307,
+ "CalPhase.xml": 420017,
+ "CalPointing.xml": 20559,
+ "CalReduction.xml": 12896,
+ "ConfigDescription.xml": 8660,
+ "CorrelatorMode.xml": 1524,
+ "DataDescription.xml": 4873,
+ "ExecBlock.xml": 1938,
+ "Feed.xml": 147490,
+ "Field.xml": 1832,
+ "Flag.xml": 1010289,
+ "Focus.xml": 327030,
+ "FocusModel.xml": 8157,
+ "Main.xml": 152322,
+ "Pointing.bin": 92853427,
+ "Pointing.xml": 963,
+ "PointingModel.xml": 71938,
+ "Polarization.xml": 678,
+ "Processor.xml": 1112,
+ "Receiver.xml": 21041,
+ "SBSummary.xml": 2013,
+ "Scan.xml": 22530,
+ "Source.xml": 39233,
+ "SpectralWindow.xml": 20090,
+ "SquareLawDetector.xml": 1096,
+ "State.xml": 1024,
+ "Station.xml": 3930,
+ "Subscan.xml": 615322,
+ "SwitchCycle.xml": 777,
+ "SysCal.bin": 14468685,
+ "SysCal.xml": 955,
+ "Weather.xml": 141587
+ },
+ "BDF": {
+ "ASDMBinary/uid___A002_Xcd8029_Xb275": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b7": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ee": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb176": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb888": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26f": 380191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb707": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb11b": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52b": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb924": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb829": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b6": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb42e": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6fe": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb282": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2bb": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb621": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6c3": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bf": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f7": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb112": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d7": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb452": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c1": 3966591,
+ "ASDMBinary/uid___A002_Xcd8029_Xb157": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb460": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb417": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb837": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c2": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb780": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0ba": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d0": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb162": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb902": 5973772,
+ "ASDMBinary/uid___A002_Xcd8029_Xb455": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb422": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2be": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb79c": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7c9": 19961996,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52e": 19961996,
+ "ASDMBinary/uid___A002_Xcd8029_Xb42b": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8c6": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb607": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2b5": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7cc": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb777": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8eb": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5e3": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb171": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb35d": 3974767,
+ "ASDMBinary/uid___A002_Xcd8029_Xb391": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb61d": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb278": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0da": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41d": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb175": 19878195,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ef": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb12c": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83c": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b4": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b5": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb11a": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5dd": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26e": 386323,
+ "ASDMBinary/uid___A002_Xcd8029_Xb451": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f4": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb111": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0be": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6c0": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb622": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ba": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60c": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb281": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7c7": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6ff": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb469": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb88a": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb123": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb784": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb610": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c2": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7bd": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ac": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb783": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb159": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c1": 1591240,
+ "ASDMBinary/uid___A002_Xcd8029_Xb480": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb419": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb124": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb381": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb77d": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60d": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb456": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f3": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bb": 1593367,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26b": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b8": 1593367,
+ "ASDMBinary/uid___A002_Xcd8029_Xb477": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52f": 3994012,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5dc": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5cd": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b9": 391888,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41c": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb172": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb12d": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ea": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83d": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb886": 387788,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8c7": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb135": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb42c": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb505": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb922": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb701": 19953796,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7cb": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2b4": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41a": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0dd": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5e2": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb15a": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb884": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb279": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb61e": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb390": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c3": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb88d": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb781": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7ba": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2af": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb423": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb454": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb13e": 73919,
+ "ASDMBinary/uid___A002_Xcd8029_Xb163": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d1": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb383": 19878195,
+ "ASDMBinary/uid___A002_Xcd8029_Xb77f": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb79b": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2bd": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb284": 3966591,
+ "ASDMBinary/uid___A002_Xcd8029_Xb384": 3966591,
+ "ASDMBinary/uid___A002_Xcd8029_Xb620": 19955870,
+ "ASDMBinary/uid___A002_Xcd8029_Xb79e": 3983762,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60a": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb82b": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f6": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d6": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb47b": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c0": 19888415,
+ "ASDMBinary/uid___A002_Xcd8029_Xb416": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb461": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb88c": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0d9": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb156": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb35b": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb274": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb889": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb46a": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0dc": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5e5": 3989912,
+ "ASDMBinary/uid___A002_Xcd8029_Xb177": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ed": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb15f": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb883": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83a": 19953796,
+ "ASDMBinary/uid___A002_Xcd8029_Xb706": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb478": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52c": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb508": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb828": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ff": 250487,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7ca": 3983762,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26c": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb702": 3983780,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5ce": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb921": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b9": 1599517,
+ "ASDMBinary/uid___A002_Xcd8029_Xb506": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb136": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b8": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb270": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb887": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb15b": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41b": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb534": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ae": 386323,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7bb": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c0": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60e": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb77e": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb380": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bc": 1599517,
+ "ASDMBinary/uid___A002_Xcd8029_Xb47f": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d2": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb160": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb13f": 2723899,
+ "ASDMBinary/uid___A002_Xcd8029_Xb900": 8127902,
+ "ASDMBinary/uid___A002_Xcd8029_Xb47a": 19953796,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bd": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7c6": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60b": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb623": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6c1": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb838": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ab": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7be": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c3": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb611": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83b": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb174": 19878195,
+ "ASDMBinary/uid___A002_Xcd8029_Xb533": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb35a": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b5": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8c9": 3983780,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2b0": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb778": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb608": 1649300
+ }
+}
+
diff --git a/shared/workspaces/test/test_data/location_files/AGBT17B_044_02.json b/shared/workspaces/test/test_data/location_files/AGBT17B_044_02.json
new file mode 100644
index 0000000000000000000000000000000000000000..ab781e6f9b7748ecf6d8516cf36cb3219ba0b49a
--- /dev/null
+++ b/shared/workspaces/test/test_data/location_files/AGBT17B_044_02.json
@@ -0,0 +1 @@
+{"files": [{"ngas_file_id": "AGBT17B_044_02__split0000.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0000.tar", "checksum": "1012891531", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 30000000000, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}, {"ngas_file_id": "AGBT17B_044_02__split0001.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0001.tar", "checksum": "950028996", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 30000000000, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}, {"ngas_file_id": "AGBT17B_044_02__split0002.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0002.tar", "checksum": "192919877", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 30000000000, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}, {"ngas_file_id": "AGBT17B_044_02__split0003.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0003.tar", "checksum": "-465285750", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 28571622400, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}], "aggregate_size": 118571622400}
\ No newline at end of file
diff --git a/shared/workspaces/test/test_data/location_files/ALMA_CONT_IMG_4d1b66da.json b/shared/workspaces/test/test_data/location_files/ALMA_CONT_IMG_4d1b66da.json
new file mode 100644
index 0000000000000000000000000000000000000000..18f46b349ad44e32895b3db3d63b6d35495012b5
--- /dev/null
+++ b/shared/workspaces/test/test_data/location_files/ALMA_CONT_IMG_4d1b66da.json
@@ -0,0 +1,19 @@
+{
+ "files": [
+ {
+ "ngas_file_id": "uid____ALMA_continuum_image_4d1b66da-84b2-41fa-8a30-5d3d341246ec.fits",
+ "subdirectory": "uid___A001_X1465_X201e.Per_18-21_sci.spw25_27_29_31.cont.I.tt0.pbcor.fits",
+ "relative_path": "uid___A001_X1465_X201e.Per_18-21_sci.spw25_27_29_31.cont.I.tt0.pbcor.fits",
+ "checksum": "534126887",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 161280,
+ "server": {
+ "server": "cvngas01.cv.nrao.edu:7777",
+ "location": "NAASC",
+ "cluster": "DSOC"
+ }
+ }
+ ],
+ "aggregate_size": 161280
+}
diff --git a/shared/workspaces/test/test_data/location_files/ALMA_CONT_IMG_71595054.json b/shared/workspaces/test/test_data/location_files/ALMA_CONT_IMG_71595054.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7cb6ff3f6a3bc4a017e77ee4c91bfcafd4b911d
--- /dev/null
+++ b/shared/workspaces/test/test_data/location_files/ALMA_CONT_IMG_71595054.json
@@ -0,0 +1,19 @@
+{
+ "files": [
+ {
+ "ngas_file_id": "uid____ALMA_continuum_image_71595054-a06a-4b15-af78-e458557a7597.fits",
+ "subdirectory": "uid___A001_X1465_X201e.Per_18-21_sci.spw25_27_29_31.cont.I.tt1.pbcor.fits",
+ "relative_path": "uid___A001_X1465_X201e.Per_18-21_sci.spw25_27_29_31.cont.I.tt1.pbcor.fits",
+ "checksum": "-840283436",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 161280,
+ "server": {
+ "server": "cvngas01.cv.nrao.edu:7777",
+ "location": "NAASC",
+ "cluster": "DSOC"
+ }
+ }
+ ],
+ "aggregate_size": 161280
+}
diff --git a/shared/workspaces/test/test_data/location_files/ALMA_IMG_CUBE.json b/shared/workspaces/test/test_data/location_files/ALMA_IMG_CUBE.json
new file mode 100644
index 0000000000000000000000000000000000000000..e64e29569c49bfe493849ce7d76f34575e97b360
--- /dev/null
+++ b/shared/workspaces/test/test_data/location_files/ALMA_IMG_CUBE.json
@@ -0,0 +1,19 @@
+{
+ "files": [
+ {
+ "ngas_file_id": "uid____ALMA_image_cube_daf0d76e-dc93-4193-ad29-6dc53692ba41.fits",
+ "subdirectory": "uid___A001_X1465_X201e.Per_18-21_sci.spw25.cube.I.pbcor.fits",
+ "relative_path": "uid___A001_X1465_X201e.Per_18-21_sci.spw25.cube.I.pbcor.fits",
+ "checksum": "-770206062",
+ "checksum_type": "ngamsGenCrc32",
+ "version": 1,
+ "size": 112907520,
+ "server": {
+ "server": "cvngas01.cv.nrao.edu:7777",
+ "location": "NAASC",
+ "cluster": "DSOC"
+ }
+ }
+ ],
+ "aggregate_size": 112907520
+}
diff --git a/apps/cli/executables/datafetcher/test/data/CALIBRATION.json b/shared/workspaces/test/test_data/location_files/CALIBRATION.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/CALIBRATION.json
rename to shared/workspaces/test/test_data/location_files/CALIBRATION.json
diff --git a/apps/cli/executables/datafetcher/test/data/EMPTY.json b/shared/workspaces/test/test_data/location_files/EMPTY.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/EMPTY.json
rename to shared/workspaces/test/test_data/location_files/EMPTY.json
diff --git a/apps/cli/executables/datafetcher/test/data/IMG.json b/shared/workspaces/test/test_data/location_files/IMG.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/IMG.json
rename to shared/workspaces/test/test_data/location_files/IMG.json
diff --git a/apps/cli/executables/datafetcher/test/data/NOT_JSON.json b/shared/workspaces/test/test_data/location_files/NOT_JSON.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/NOT_JSON.json
rename to shared/workspaces/test/test_data/location_files/NOT_JSON.json
diff --git a/apps/cli/executables/datafetcher/test/data/VLA_BAD_SERVER.json b/shared/workspaces/test/test_data/location_files/VLA_BAD_SERVER.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/VLA_BAD_SERVER.json
rename to shared/workspaces/test/test_data/location_files/VLA_BAD_SERVER.json
diff --git a/apps/cli/executables/datafetcher/test/data/VLA_LARGE_EB.json b/shared/workspaces/test/test_data/location_files/VLA_LARGE_EB.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/VLA_LARGE_EB.json
rename to shared/workspaces/test/test_data/location_files/VLA_LARGE_EB.json
diff --git a/apps/cli/executables/datafetcher/test/data/VLA_SMALL_EB.json b/shared/workspaces/test/test_data/location_files/VLA_SMALL_EB.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/VLA_SMALL_EB.json
rename to shared/workspaces/test/test_data/location_files/VLA_SMALL_EB.json
diff --git a/apps/cli/executables/datafetcher/test/data/VLA_SMALL_EB_BUSTED.json b/shared/workspaces/test/test_data/location_files/VLA_SMALL_EB_BUSTED.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/VLA_SMALL_EB_BUSTED.json
rename to shared/workspaces/test/test_data/location_files/VLA_SMALL_EB_BUSTED.json
diff --git a/apps/cli/executables/datafetcher/test/data/VLBA_EB.json b/shared/workspaces/test/test_data/location_files/VLBA_EB.json
similarity index 100%
rename from apps/cli/executables/datafetcher/test/data/VLBA_EB.json
rename to shared/workspaces/test/test_data/location_files/VLBA_EB.json
diff --git a/shared/workspaces/test/test_data/products/__init__.py b/shared/workspaces/test/test_data/products/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/shared/workspaces/test/test_data/products/expected_values_alma.py b/shared/workspaces/test/test_data/products/expected_values_alma.py
new file mode 100644
index 0000000000000000000000000000000000000000..2830a33dbd3d9b216b6fb93f4788a92e93d18de5
--- /dev/null
+++ b/shared/workspaces/test/test_data/products/expected_values_alma.py
@@ -0,0 +1,296 @@
+""" Test data: ALMA download products """
+
+import warnings
+
+from schema import Project, Author, ExecutionBlock, ScienceProduct
+from sqlalchemy import exc as sa_exc
+
+from .expected_values_evla import \
+ EXECUTION_BLOCK, METADATA_INGESTION_DATE, CASA_LOG, PPR_FILENAME
+from ...utilities import get_file_info_from_json_file
+
+EXEC_BLOCK = 'execblock'
+CALIBRATED = 'CALIBRATED'
+WEBLOG = 'weblog.tgz'
+PIPELINE_MANIFEST = 'unknown.download_alma_ms.pipeline_manifest.xml'
+PIPELINE_AQUAREPORT = 'pipeline_aquareport.xml'
+CASA_PIPESCRIPT = 'casa_pipescript.py'
+AUX_PRODUCTS = 'unknown.download_alma_ms.auxproducts.tgz'
+MS_PP_REQUEST = 'unknown.download_alma_ms.pprequest.xml'
+
+# pylint: disable=R0201, R0801, R0902, R0903
+
+
+class AlmaPublicProductSet:
+ """ Test data for ALMA project 2017.1.00297.S """
+
+ def __init__(self):
+ self.project = self.build_project()
+ self.exec_blocks = self.build_exec_blocks()
+ self.science_products = self.build_science_products()
+ self.file_info = get_file_info_from_json_file('A001_X1296_Xa93_RAW.json')
+ self.rawdata_total_size = self._compute_rawdata_size()
+ self.measurement_set, self.basic_ms_size = self.build_basic_ms()
+ self.restored_ms, self.rest_ms_size = self.build_restored_ms()
+
+ def build_project(self) -> Project:
+ """ Returns project metadata for 2017.1.00297.S,
+ including authors' details
+ """
+
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project_code = '2017.1.00297.S'
+ project = Project(
+ project_code=project_code,
+ starttime=56799.3877155556,
+ endtime=56799.4128683333,
+ proprietary_duration=365,
+ total_observation_time=0.0583143749972805,
+ title='Copy of 2012.1.00060.S for testing',
+ abstract="Understanding the physical factors that control the conversion of interstellar gas into stars is of fundamental importance for both developing a predictive physical theory of star formation and understanding the evolution of galaxies from the earliest epochs of cosmic history to the present time. An important aspect of this question is the study of empirical relations that connect the star formation rate in a given region to local properties of the interstellar medium. An important example is the Schmidt-Kennicutt (KS) law for galaxies that relates the surface densities of the star formation rate and the surface densities of interstellar gas in a non-linear fashion. However, it is also known that there is a linear correlation between the total SFR in galaxies and the mass of dense molecular gas as traced by the high excitation HCN molecule. Contrary to the KS relation, this scaling relation suggests that the total SFR depends simply on the total amount of dense molecular gas in a star forming system. Recently, we have begun to test these scaling relations in the Galactic neighborhood where star formation rates can be much better constrained. We found that for local clouds the total SFR scales most directly, and linearly, with the total mass of high extinction (and dense) molecular gas. Furthermore, we found this linear scaling law between SFR and dense gas to extend and extrapolate directly and smoothly to external galaxies. Moreover, our observations also demonstrate that a KS type relation does not exist for molecular clouds in the Galactic neighborhood. This is a direct consequence of a well known scaling law between the mass and size of molecular clouds, Larson's third law. Overall, our results indicate that a linear scaling law, in which the total amount of dense gas controls the SFR, is the fundamental physical relation that connects star formation across the vast scales from individual GMCs to entire galaxies. Critical testing of these ideas require resolved observations of GMCs in external galaxies. Here we propose to use ALMA to evaluate star formation scaling laws in a nearby galaxy where we can obtain resolved observations of individual GMCs. This allows us to obtain observations of a larger sample of GMCs than is accessible in the Galactic neighborhood. An extensive APEX survey of HII regions in the nearby galaxy NGC 300 has provided us with a sample of 36 star-forming regions with CO(2-1) detections and 42 upper limits. We are currently working on obtaining star formation rates for these regions from multi-wavelength ancillary data including our Herschel observations. We propose to use ALMA's unequalled capabilities to obtain snapshot observations of 40 selected regions in CO(2-1) in order to make resolved measurements of cloud structure to obtain sizes and virial masses. As a pilot project, we also propose to observe the brightest subsample in HCN(1-0) as a dense-gas tracer. Our proposed ALMA CO observations will enable us to to test Larson's scaling laws in an external galaxy and to evaluate which formulation of the Schmidt law is the most meaningful and appropriate to apply to spiral galaxies, and in doing so refine Schmidt's original conjecture of a scaling relation between the rate of star formation and gas density."
+ )
+ project.authors = [
+ Author(project_code=project_code,
+ author_id=32550,
+ username='franzbauer',
+ firstname='Franz',
+ lastname='Bauer',
+ pst_person_id=None,
+ is_pi=True),
+
+ Author(project_code=project_code,
+ author_id=32551,
+ username='ezetre',
+ firstname='Ezequiel',
+ lastname='Treister',
+ pst_person_id=None,
+ is_pi=False),
+ Author(project_code=project_code,
+ author_id=32552,
+ username='lho',
+ firstname='Luis',
+ lastname='Ho',
+ pst_person_id=None,
+ is_pi=False),
+ Author(project_code=project_code,
+ author_id=32554,
+ username='jyshangguan',
+ firstname='Jinyi',
+ lastname='Shangguan',
+ pst_person_id=None,
+ is_pi=False),
+ Author(project_code=project_code,
+ author_id=32553,
+ username='rwang',
+ firstname='Ran',
+ lastname='Wang',
+ pst_person_id=None,
+ is_pi=False),
+ ]
+
+ return project
+
+ def build_exec_blocks(self) -> list:
+ """ Returns execution blocks for OUS uid://A001/X1296/Xa93. """
+ alma_ous_id = 'uid://A001/X1296/Xa93'
+ band_code = '05'
+
+ return [
+ ExecutionBlock(
+ project_code=self.project.project_code,
+ execution_block_id=102209,
+ alma_ous_id=alma_ous_id,
+ band_code=band_code,
+ filegroup_id=212043,
+ ngas_fileset_id='uid___A002_Xcd8029_Xa892',
+ starttime=58256.0535855556,
+ endtime=58256.1137983333,
+ calibration_status=CALIBRATED
+ ),
+ ExecutionBlock(
+ project_code=self.project.project_code,
+ execution_block_id=102169,
+ alma_ous_id=alma_ous_id,
+ band_code=band_code,
+ filegroup_id=211958,
+ ngas_fileset_id='uid___A002_Xcd8029_Xfdd',
+ starttime=58255.0338461111,
+ endtime=58255.0919594444,
+ calibration_status=CALIBRATED
+ ),
+ ExecutionBlock(
+ project_code=self.project.project_code,
+ execution_block_id=102211,
+ alma_ous_id=alma_ous_id,
+ band_code=band_code,
+ filegroup_id=212047,
+ ngas_fileset_id='uid___A002_Xcd8029_Xb0a4',
+ starttime=58256.1152683333,
+ endtime=58256.1753838889,
+ calibration_status=CALIBRATED
+ ),
+ ]
+
+ def build_science_products(self) -> list:
+ """ Returns science products assocated with project """
+
+ external_system = 'ALMA Operations'
+ return [
+ ScienceProduct(
+ external_name='uid___A002_Xcd8029_Xfdd',
+ filegroup_id=211958,
+ external_system=external_system,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK
+ ),
+ ScienceProduct(
+ external_name='uid___A002_Xcd8029_Xb0a4',
+ filegroup_id=212047,
+ external_system=external_system,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK
+ ),
+ ScienceProduct(
+ external_name='uid___A002_Xcd8029_Xa892',
+ filegroup_id=212043,
+ external_system=external_system,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK
+ ),
+ ]
+
+ def build_basic_ms(self) -> tuple:
+ """ Returns basic measurement set metadata """
+
+ ms_info = {
+ 'uid___A002_Xcd8029_Xa892': {
+ 'uid___A002_Xcd8029_Xa892.400887655.tar.gz': 504369560,
+ PIPELINE_MANIFEST: 804,
+ CASA_PIPESCRIPT: 680,
+ PPR_FILENAME: 4584,
+ WEBLOG: 1592335,
+ MS_PP_REQUEST: 4584,
+ AUX_PRODUCTS: 839,
+ CASA_LOG: 1729,
+ PIPELINE_AQUAREPORT: 3472,
+ 'uid___A002_Xcd8029_Xa892.ms.tgz': 519234359,
+ },
+ 'uid___A002_Xcd8029_Xb0a4': {
+ 'uid___A002_Xcd8029_Xb0a4.400887655.tar.gz': 503501127,
+ PIPELINE_MANIFEST: 804,
+ CASA_LOG: 1729,
+ WEBLOG: 1576475,
+ CASA_PIPESCRIPT: 680,
+ PIPELINE_AQUAREPORT: 3465,
+ 'uid___A002_Xcd8029_Xb0a4.ms.tgz': 518582184,
+ AUX_PRODUCTS: 844,
+ PPR_FILENAME: 4584,
+ MS_PP_REQUEST: 4584,
+
+ },
+ 'uid___A002_Xcd8029_Xfdd': {
+ 'uid___A002_Xcd8029_Xfdd.400887655.tar.gz': 489883578,
+ PIPELINE_MANIFEST: 802,
+ PIPELINE_AQUAREPORT: 3460,
+ PPR_FILENAME: 4582,
+ MS_PP_REQUEST: 4582,
+ WEBLOG: 1562579,
+ 'uid___A002_Xcd8029_Xfdd.ms.tgz': 504128830,
+ CASA_LOG: 1723,
+ AUX_PRODUCTS: 836,
+ CASA_PIPESCRIPT: 679,
+ }
+ }
+
+ total_size = 0
+ for mous in ms_info.keys():
+ for _, size in ms_info[mous].items():
+ total_size += size
+
+ return ms_info, total_size
+
+ def build_restored_ms(self):
+ """ Returns restored measurement set metadata """
+ ms_info = {
+ 'uid___A002_Xcd8029_Xb0a4': {
+ 'uid___A002_Xcd8029_Xb0a4.ms.tgz': 1325422495,
+ 'uid___A002_Xcd8029_Xa892.ms.tgz': 1331336708,
+ 'uid___A002_Xcd8029_Xfdd.ms.tgz': 1290775660,
+ PPR_FILENAME: 5158,
+
+ },
+ }
+ total_size = 0
+ for _, size in ms_info['uid___A002_Xcd8029_Xb0a4'].items():
+ total_size += size
+ return ms_info, total_size
+
+ def _compute_rawdata_size(self):
+ total_size = 0
+ for file_type in self.file_info.keys():
+ metadata = self.file_info[file_type]
+ for item in metadata.items():
+ total_size += item[1]
+ return total_size
+
+
+class AlmaProprietaryImageProductSet:
+ """ proprietary project 2019.1.00914.S, with image data """
+
+ def __init__(self):
+ self.project_code = '2019.1.00914.S'
+ self.file_info, self.total_size = self._build_file_info()
+
+ def _build_file_info(self) -> tuple:
+ """
+ TODO
+ :return:
+ """
+
+ img_files_metadata = [
+ get_file_info_from_json_file('ALMA_CONT_IMG_4d1b66da.json'),
+ get_file_info_from_json_file('ALMA_CONT_IMG_71595054.json'),
+ get_file_info_from_json_file('ALMA_IMG_CUBE.json'),
+ ]
+
+ file_info = {}
+ total_size = 0
+ for metadata in img_files_metadata:
+ files = metadata['files']
+ for file in files:
+ file_info[file['ngas_file_id']] = file['size']
+ total_size += file['size']
+ return file_info, total_size
+
+
+class AlmaAudiProductSet:
+ """ AUDI products for project 2019.1.00914.S """
+
+ def __init__(self):
+ self.science_products = self.build_science_products()
+ self.file_info = get_file_info_from_json_file(
+ 'A001_X1296_Xa93_RAW.json')
+ self.total_size = self._compute_total_size()
+
+ def _compute_total_size(self):
+ total_size = 0
+ for file_type in self.file_info.keys():
+ metadata = self.file_info[file_type]
+ for item in metadata.items():
+ total_size += item[1]
+ return total_size
+
+ def build_science_products(self) -> list:
+ """ Returns 2019.1.00914.S science product in list """
+
+ return [
+ ScienceProduct(external_name='uid___A002_Xe5aacf_Xdde3',
+ filegroup_id=398280,
+ science_product_type=EXECUTION_BLOCK)
+ ]
diff --git a/shared/workspaces/test/test_data/products/expected_values_evla.py b/shared/workspaces/test/test_data/products/expected_values_evla.py
new file mode 100644
index 0000000000000000000000000000000000000000..181437d44da155c81e6bc1566a14c7dec894d099
--- /dev/null
+++ b/shared/workspaces/test/test_data/products/expected_values_evla.py
@@ -0,0 +1,666 @@
+""" Test data for two VLA EBs, a calibration product, and VLBA and VLASS
+ image products
+"""
+import datetime
+import warnings
+
+from schema import Project, ExecutionBlock, Author, ScienceProduct
+from sqlalchemy import exc as sa_exc
+
+from shared.workspaces.test.utilities import get_locations_report, \
+ DATETIME_FORMAT, DATE_FORMAT, get_exec_block_details_from_loc_report, \
+ get_file_info_from_loc_report
+
+# pylint: disable=C0301, R0201
+
+METADATA_INGESTION_DATE = datetime.datetime.strptime(
+ '2019-12-12 16:04:02.11191', DATETIME_FORMAT)
+EXECUTION_BLOCK = 'Execution Block'
+PPR_FILENAME = 'PPR.xml'
+CASA_LOG = 'casa_commands.log'
+MANIFEST = 'unknown.pipeline_manifest.xml'
+DO_NOT_CALIBRATE = 'Do Not Calibrate'
+VLBA_OPERATIONS = 'VLBA Operations'
+
+
+class VlaProductSet:
+ """ Encapsulates project 17A-109's products """
+
+ def __init__(self):
+ self.project = self.build_project()
+ self.exec_blocks = self.build_exec_blocks()
+ self.science_products = self.build_science_products()
+ self.file_info, self.total_size = get_exec_block_details_from_loc_report(
+ '17a-109_fg_', self.exec_blocks
+ )
+ self.measurement_set = self.build_basic_ms()
+ self.cms = self.build_cms()
+
+ def build_project(self) -> Project:
+ """ Construct VlaProductSet for 17A-109 """
+
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(project_code='17A-109',
+ starttime=57795.8403622685,
+ endtime=57892.7843396991,
+ title="A detailed study of the jet-galaxy interaction in Minkowski's Object",
+ abstract="Minkowski's Object is a dwarf starforming galaxy interacting with an FRI radio jet from a nearby elliptical galaxy. It has been proposed as a prototype for jet-induced star formation, a hypothesis consistent with our recent ALMA observations that map the CO 1-0 emission in the galaxy. In order to better understand the nature of the interaction, we are requesting C- and Ku-band observations at matched resolution to our ALMA and archival VLA L-band data. Faraday synthesis of the C-band polarimetric data will allow us to study the geometry of the interaction, in particular how well the relativistic electron population is mixed with the ionized gas in the star forming region. This will give us important information for our simulations of the interaction and the triggering of star formation in Minkowski's Object. In turn, this will inform on the conditions where jet-induced star formation is likely in higher redshift objects. We will use the combination of the C- and Ku-band data to investigate the variation the spectral index in the interaction region, in particular to search for shock acceleration in the interaction zone.",
+ proprietary_duration=365,
+ total_observation_time=0.0391880786919501,
+ legacy_id='AL988',
+ opt_project_id=33151323,
+ last_addition=datetime.datetime.strptime(
+ '2017-08-28', DATE_FORMAT).date()
+
+ )
+ project.authors = self.build_authors(project.project_code)
+ return project
+
+ def build_authors(self, project_code: str) -> list:
+ """ Returns author details for this project. """
+
+ authors = [Author(project_code=project_code,
+ author_id=780,
+ username='mlacy',
+ firstname='Mark',
+ lastname='Lacy',
+ pst_person_id='885',
+ is_pi=True),
+ Author(project_code=project_code,
+ author_id=781,
+ username='swood',
+ firstname='Sarah',
+ lastname='Wood',
+ pst_person_id='6877',
+ is_pi=False
+ ),
+ Author(project_code=project_code,
+ author_id=782,
+ username='stevecroft',
+ firstname='Steve',
+ lastname='Croft',
+ pst_person_id='2116',
+ is_pi=False
+ ),
+ Author(project_code=project_code,
+ author_id=783,
+ username='KristinaNyland',
+ firstname='Kristina',
+ lastname='Nyland',
+ pst_person_id='2950',
+ is_pi=False
+ ),
+ Author(project_code=project_code,
+ author_id=784,
+ username='fragilep',
+ firstname='Chris',
+ lastname='Fragile',
+ pst_person_id='9238',
+ is_pi=False
+ ), ]
+ return authors
+
+ def build_exec_blocks(self) -> list:
+ """ Returns this project's execution blocks """
+
+ exec_blocks = [
+ ExecutionBlock(execution_block_id=910,
+ filegroup_id=18468,
+ ngas_fileset_id='17A-109.sb33151327.eb33496982.57795.84034732639',
+ scheduling_block_id=33151327,
+ scheduling_block_type='OBSERVER',
+ ost_exec_block_id=33496982,
+ configuration='D',
+ starttime=57795.8403622685,
+ endtime=57795.8795503472,
+ calibration_status='Calibrated',
+ band_code='KU X'),
+ ExecutionBlock(execution_block_id=910,
+ filegroup_id=41979,
+ ngas_fileset_id='17A-109.sb33151331.eb33786546.57892.65940042824',
+ scheduling_block_id=33151331,
+ scheduling_block_type='OBSERVER',
+ ost_exec_block_id=33786546,
+ configuration='C',
+ starttime=57892.6600277778,
+ endtime=57892.7843396991,
+ calibration_status='Calibrated',
+ band_code='C X')
+ ]
+ return exec_blocks
+
+ def build_science_products(self) -> list:
+ """ Returns this project's science products """
+
+ science_products = [
+ ScienceProduct(filegroup_id=18468,
+ external_name='17A-109.sb33151327.eb33496982.57795.84034732639',
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK),
+ ScienceProduct(filegroup_id=41979,
+ external_name='17A-109.sb33151331.eb33786546.57892.65940042824',
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK),
+ ]
+ return science_products
+
+ def build_basic_ms(self) -> dict:
+ """
+ Returns measurement set metadata for a couple of this project's
+ filegroups
+
+ :return:
+ """
+
+ files = {
+ 'fg_41979': [
+ {'filename': 'weblog.tgz', 'size': 1683135},
+ {'filename': 'unknown.pprequest.xml', 'size': 5611},
+ {'filename': 'pipeline_aquareport.xml', 'size': 1351},
+ {'filename': PPR_FILENAME, 'size': 5611},
+ {'filename': 'unknown.auxproducts.tgz', 'size': 615},
+ {'filename': CASA_LOG, 'size': 1995},
+ {'filename': MANIFEST, 'size': 809},
+ {'filename': 'casa_pipescript.py', 'size': 823},
+ # the tar delivered when basic MS is requested
+ {
+ 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.ms.tgz',
+ 'size': 23690977245},
+ ],
+
+ 'fg_18468': [
+ {'filename': 'weblog.tgz', 'size': 1681803},
+ {'filename': 'unknown.pprequest.xml', 'size': 5610},
+ {'filename': 'pipeline_aquareport.xml', 'size': 1351},
+ {'filename': PPR_FILENAME, 'size': 5610},
+ {'filename': 'unknown.auxproducts.tgz', 'size': 615},
+ {'filename': CASA_LOG, 'size': 1995},
+ {'filename': MANIFEST, 'size': 809},
+ {'filename': 'casa_pipescript.py', 'size': 823},
+ # the tar delivered when basic MS is requested
+ {
+ 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.ms.tgz',
+ 'size': 23690977531},
+ ]
+ }
+ return files
+
+ def build_cms(self):
+ """
+ Returns calibrated measurement set metadata
+ for a couple of this project's filegroups
+
+ :return:
+ """
+
+ files = {
+ 'fg_41979': [
+ {'filename':
+ '17A-109.sb33151331.eb33786546.57892.65940042824.ms'
+ '.calapply.txt', 'size': 1058},
+ {'filename': MANIFEST, 'size': 11006},
+ {'filename': 'unknown.session_1.caltables.tgz',
+ 'size': 71653397},
+ {'filename': 'casa_piperestorescript.py', 'size': 205},
+ {'filename': 'flux.csv', 'size': 22},
+ {'filename': PPR_FILENAME, 'size': 1632},
+ {'filename': CASA_LOG, 'size': 133935},
+ # the tar delivered when calibrated MS (aka 'restore') is
+ # requested
+ {
+ 'filename': '17A-109.sb33151331.eb33786546.57892.65940042824.400685418.tar.gz',
+ 'size': 88842463132},
+ ],
+ 'fg_18468': [
+ {'filename': MANIFEST, 'size': 24025},
+ {'filename': 'unknown.session_1.caltables.tgz',
+ 'size': 26463229},
+ {'filename': 'casa_piperestorescript.py', 'size': 1149},
+ {'filename': 'flux.csv', 'size': 22},
+ {'filename': PPR_FILENAME, 'size': 1632},
+ {'filename': CASA_LOG, 'size': 260953},
+ {
+ 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.ms.calapply.txt',
+ 'size': 1058},
+ {
+ 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.400820221.tar.gz',
+ 'size': 47520134173}
+ ]
+ }
+
+ return files
+
+
+class CalibrationProduct:
+ """ Encapsulates calibration test data for 18B-265 """
+
+ def __init__(self):
+ self.project = self.build_project()
+ self.science_product = self.build_science_product()
+ self.file_info, self.total_size = get_file_info_from_loc_report(
+ get_locations_report('calibration'))
+
+ def build_project(self) -> Project:
+ """
+ Returns 18B-265 Project.
+
+ :return:
+
+ """
+
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(
+ project_code='18B-265',
+ legacy_id='AS1535',
+ total_observation_time=0.0316695485962555,
+ proprietary_duration=365,
+ starttime=58745.9972239583,
+ endtime=59043.2400404977,
+ last_addition=datetime.datetime.strptime(
+ '2020-07-13', DATE_FORMAT).date()
+ )
+ project.authors = self.build_authors()
+ return project
+
+ def build_authors(self) -> list:
+ """
+ Returns author details for 18B-265
+ :return:
+
+ """
+
+ return [
+ Author(
+ project_code='18B-265',
+ author_id=59386,
+ username='mansi',
+ firstname='Mansi',
+ lastname='Kasliwal',
+ pst_person_id='8998',
+ is_pi=False
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59387,
+ username='chomiuk',
+ firstname='Laura',
+ lastname='Chomiuk',
+ pst_person_id='701',
+ is_pi=False
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59388,
+ username='kirx',
+ firstname='Kirill',
+ lastname='Sokoloski',
+ pst_person_id='5512',
+ is_pi=False
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59389,
+ username='KojiMukai',
+ firstname='Koji',
+ lastname='Muki',
+ pst_person_id='3674',
+ is_pi=False
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59387,
+ username='JustinLinford',
+ firstname='Justin',
+ lastname='Linford',
+ pst_person_id='3794',
+ is_pi=False
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59390,
+ username='ThomasNelson',
+ firstname='Thomas',
+ lastname='Nelson',
+ pst_person_id='5234',
+ is_pi=False
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59394,
+ username='EliasAydi',
+ firstname='Elias',
+ lastname='Aydi',
+ pst_person_id='10794',
+ is_pi=False
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59392,
+ username='jeno@astro.columbia.edu',
+ firstname='Jennifer',
+ lastname='Sokoloski',
+ pst_person_id='1200',
+ is_pi=True
+ ),
+ Author(
+ project_code='18B-265',
+ author_id=59393,
+ username='amkawash',
+ firstname='Adam',
+ lastname='Kawash',
+ pst_person_id='8687',
+ is_pi=False
+ ),
+
+ ]
+
+ def build_science_product(self):
+ """
+ Returns the science product associated with this calibration.
+
+ :return:
+ """
+
+ return ScienceProduct(filegroup_id=388273,
+ external_name='18B-265_2019_12_10_T00_00_59.203.tar',
+ science_product_type='Calibration',
+ metadata_ingestion_version='1',
+ metadata_ingestion_date=datetime.datetime.strptime(
+ '2019-12-13 23:30:57.730581',
+ DATETIME_FORMAT),
+ external_system='EVLA Processing'
+ )
+
+
+class VlbaProductSet:
+
+ """ Encapsulates a set of products for VLBA project BT142. """
+
+ def __init__(self):
+ self.project = self.build_project()
+ self.exec_blocks = self.build_exec_blocks()
+ self.science_products = self.build_science_products()
+ self.file_info, self.total_size = self.get_vlba_file_info_from_loc_report()
+
+ def build_project(self) -> Project:
+ """
+ Returns BT142 metadata and authors details
+
+ :return:
+ """
+
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(project_code='BT142',
+ starttime=58154.8048148148,
+ endtime=58217.8342361111,
+ title="Jets, outflows, and inclined disks in Seyfert galaxies: the H2O megamasers' view",
+ abstract="Jets, outflows, and inclined disks in Seyfert galaxies: the H2O megamasers' view | Candidate extragalactic water masers associated with accretion disks around AGN are typically identified by a triple-peak profile of the maser spectra. However, a large number of maser sources show single-broad lines or groups of lines clustered in a narrow velocity range. In this proposal, we propose for a study of four galaxies, classified as Seyfert or LINER, hosting bright water masers and showing evidences of ejection/accretion nuclear activity. Our main goals are that of detecting the maser emission at VLBI scales, determining its location and distribution with respect to the main nuclear centers of activity, and derive the nature of the emission. These sources will potentially constitute new case studies for the scarcely-populated class of confirmed jet/outflow masers or very first examples of the exotic, recently-invoked class of 'inclined water maser disks'. Furthermore, the proposed mesurements will set the basis for a number of follow-up (VLBI) studies aim at a better understanding of the physics and disk/jet geometry in Seyferts and LINERs.",
+ proprietary_duration=365,
+ total_observation_time=0.0391880786919501,
+ last_addition=datetime.datetime.strptime(
+ '2018-11-05', DATE_FORMAT).date()
+ )
+ project.authors = self.build_authors()
+
+ return project
+
+ def build_authors(self) -> list:
+ """
+ Returns author details for this project
+ :return:
+ """
+
+ authors = [
+ Author(
+ username='atarchi',
+ firstname='Andrea',
+ lastname='Tarchi',
+ pst_person_id='236',
+ project_code='BT142',
+ author_id=45311,
+ is_pi=True
+ ),
+ Author(
+ username='jbraatz',
+ firstname='James',
+ lastname='Braatz',
+ pst_person_id='26',
+ project_code='BT142',
+ author_id=45310,
+ is_pi=False
+ ),
+ Author(
+ username='pcastangia',
+ firstname='Paola',
+ lastname='Castangia',
+ pst_person_id='541',
+ project_code='BT142',
+ author_id=45312,
+ is_pi=False
+ ),
+ Author(
+ username='gsurcis',
+ firstname='Gabriele',
+ lastname='Surcis',
+ pst_person_id='2512',
+ project_code='BT142',
+ author_id=45313,
+ is_pi=False
+ ),
+ ]
+ return authors
+
+ def build_exec_blocks(self) -> list:
+ """
+ Constructs this project's execution blocks.
+
+ :return:
+ """
+
+ exec_blocks = [
+ ExecutionBlock(
+ execution_block_id=108778,
+ filegroup_id=278938,
+ starttime=58175.0757060185,
+ endtime=58175.325625,
+ calibration_status=DO_NOT_CALIBRATE,
+ band_code='K'
+ ),
+ ExecutionBlock(
+ execution_block_id=108833,
+ filegroup_id=279054,
+ starttime=58160.1166550926,
+ endtime=58160.3043634259,
+ calibration_status=DO_NOT_CALIBRATE,
+ band_code='L'
+ ),
+ ExecutionBlock(
+ execution_block_id=108862,
+ filegroup_id=279119,
+ starttime=58154.8048148148,
+ endtime=58154.9928009259,
+ calibration_status=DO_NOT_CALIBRATE,
+ band_code='L'
+ ),
+ ExecutionBlock(
+ execution_block_id=108630,
+ filegroup_id=278613,
+ starttime=58217.5843171296,
+ endtime=58217.8342361111,
+ calibration_status=DO_NOT_CALIBRATE,
+ band_code='K'
+ ),
+ ]
+ return exec_blocks
+
+ def build_science_products(self):
+ """
+ Returns science products associate with the exec blocks we got
+
+ :return:
+ """
+
+ return [
+ ScienceProduct(
+ filegroup_id=279054,
+ science_product_type=EXECUTION_BLOCK,
+ metadata_ingestion_version='1',
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ external_name='BT142A2',
+ external_system=VLBA_OPERATIONS
+ ),
+ ScienceProduct(
+ filegroup_id=278938,
+ science_product_type=EXECUTION_BLOCK,
+ metadata_ingestion_version='1',
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ external_name='BT142A1',
+ external_system=VLBA_OPERATIONS
+ ),
+ ScienceProduct(
+ filegroup_id=279119,
+ science_product_type=EXECUTION_BLOCK,
+ metadata_ingestion_version='1',
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ external_name='BT142B2',
+ external_system=VLBA_OPERATIONS
+ ),
+ ScienceProduct(
+ filegroup_id=278613,
+ science_product_type=EXECUTION_BLOCK,
+ metadata_ingestion_version='1',
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ external_name='BT142B1',
+ external_system=VLBA_OPERATIONS
+ ),
+ ]
+
+ def get_vlba_file_info_from_loc_report(self) -> tuple:
+ """
+ Pull file metadata from VLBA locations report.
+ We're using this metadata differently from EVLA metadata;
+ hence the bespoke method.
+
+ :return:
+ """
+
+ file_info = dict()
+ locations_report = get_locations_report('vlba_eb')
+
+ for file_spec in locations_report['files']:
+ filename = file_spec['ngas_file_id']
+ size = file_spec['size']
+ file_info[filename] = size
+
+ return file_info, locations_report['aggregate_size']
+
+
+class VlassImageProductSet:
+ """ Encapsulates a VLASS image product """
+
+ def __init__(self):
+ self.project = self.build_project()
+ self.science_products = self.build_science_products()
+ location_report = get_locations_report('img')
+ self.file_info, self.total_size = get_file_info_from_loc_report(location_report)
+
+ def build_project(self) -> Project:
+ """
+ Returns VLASS1.1 project metadata
+
+ :return:
+ """
+
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(project_code='VLASS1.1',
+ title='The Very Large Array Sky Survey',
+ abstract='The Very Large Array Sky Survey (VLASS) is a 5500-hr, community-driven project to survey the whole sky visible to the VLA. It will engage radio astronomy experts, multi-wavelength astronomers and citizen scientists alike. The data will be taken in three passes over the sky to allow the discovery of transient radio sources, and will cover the frequency range 2-4 GHz with an angular resolution of 2.5 arcsec. By utilizing the "on the fly" interferometry mode, the overheads will be much reduced compared to conventional survey techniques. The key science topics to be addressed by the survey are: Imaging Galaxies Through Time and Space; Hidden Explosions; Faraday Tomography of The Magnetic Sky; Peering Though Our Dusty Galaxy; and Missing Physics.',
+ opt_project_id=33997662,
+ starttime=58004.0162552083,
+ endtime=58169.6566603472,
+ total_observation_time=0.158181712955411,
+ last_addition=datetime.datetime.strptime(
+ '2017-09-08', DATE_FORMAT).date(),
+ proprietary_duration=0
+ )
+ project.authors = self.build_authors()
+ return project
+
+ def build_authors(self) -> list:
+ """ Returns VLASS1.1 authors' metadata """
+
+ return [
+ Author(
+ project_code='VLASS1.1',
+ username='vlass',
+ firstname='Vlass',
+ lastname='Scientist',
+ pst_person_id='9707',
+ is_pi=True
+ ),
+ Author(
+ project_code='VLASS1.1',
+ username='cchandle',
+ firstname='Claire',
+ lastname='Chandler',
+ pst_person_id='36',
+ is_pi=False
+ ),
+ Author(
+ project_code='VLASS1.1',
+ username='akimball',
+ firstname='Amy',
+ lastname='Kimball',
+ pst_person_id='1743',
+ is_pi=False
+ ),
+ Author(
+ project_code='VLASS1.1',
+ username='jwrobel',
+ firstname='Joan',
+ lastname='Wrobel',
+ pst_person_id='172',
+ is_pi=False
+ ),
+ Author(
+ project_code='VLASS1.1',
+ username='fschinzel',
+ firstname='Frank',
+ lastname='Schinzel',
+ pst_person_id='2059',
+ is_pi=False
+ ),
+ ]
+
+ def build_science_products(self) -> list:
+ """
+ Returns science product associated with this image file,
+ as a list for consistency with the other ProductSets.
+
+ :return:
+ """
+
+ return [
+ ScienceProduct(
+ filegroup_id=223148,
+ external_name='VLASS1.1.ql.T01t01.J000232-383000.10.2048.v1',
+ metadata_ingestion_version='1',
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ science_product_type='Image',
+ external_system='CASA Pipeline',
+ ),
+ ]
diff --git a/shared/workspaces/test/test_data/products/expected_values_gbt.py b/shared/workspaces/test/test_data/products/expected_values_gbt.py
new file mode 100644
index 0000000000000000000000000000000000000000..b6f57a930da3fdd73eae81ad8b79d2ca9e1a0743
--- /dev/null
+++ b/shared/workspaces/test/test_data/products/expected_values_gbt.py
@@ -0,0 +1,157 @@
+""" Test data: GBT download products """
+
+import datetime
+import warnings
+
+from schema import Project, ExecutionBlock, Author, ScienceProduct
+from sqlalchemy import exc as sa_exc
+
+from shared.workspaces.test.utilities import DATETIME_FORMAT, DATE_FORMAT, \
+ get_exec_block_details_from_loc_report
+
+# pylint: disable=C0301, R0201
+
+class GbtProductSet:
+ """ Encapsulates a GBT project and its products """
+
+ def __init__(self):
+ self.project = self.build_project()
+ self.exec_blocks = self.build_exec_blocks()
+ self.science_products = self.build_science_products()
+ self.file_info, self.total_size = get_exec_block_details_from_loc_report(
+ 'AGBT17B_044_02', self.exec_blocks
+ )
+
+ def build_project(self) -> Project:
+ """ Returns AGBT17B_044 metadata """
+
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(project_code='AGBT17B_044',
+ starttime=58089.1619444443,
+ endtime=58138.2070370372,
+ proprietary_duration=365,
+ total_observation_time=27.055,
+ last_addition=datetime.datetime.strptime(
+ '2020-10-21', DATE_FORMAT).date(),
+ title='Infall and Fragmentation of Cores in the Taurus: I Finding Cores with Infall',
+ abstract=' We propose to conduct a survey of investigating infall candidates and fragmentation toward a complete population of prestellar cores in Taurus in HCN, HCO+, N2H+, and NH2D. The proposed observation will trace infall motions within prestellar cores from 0.05 pc down to 0.0075 pc (1500 AU) spatial scale, which is a typical size scale of individual planetary systems. The scientific goal is to estimate a fraction of infall candidates out of a complete population of prestellar cores and to understand internal velocity structure during final gravitational collapse before forming stars. We shall also search for fragmentation within prestellar cores, and to investigate kinematics involved with fragmentation processes. We propose to conduct pointing observation toward 57 prestellar cores using the ARGUS of the GBT in the 89 GHz transitions of HCN and HCO+, 93 GHz transition of N2H+, and to map 2 prestellar cores, which are found to have infall motions, in N2H+ and NH2D.'
+ )
+ project.authors = self.build_authors()
+ return project
+
+ def build_authors(self) -> list:
+ """ Returns AGBT17B_044 authors' metadata """
+
+ return [
+ Author(
+ username='seo3919',
+ firstname='Youngmin',
+ lastname='Seo',
+ is_pi=True
+ ),
+ Author(
+ username='schurch',
+ firstname='Sarah',
+ lastname='Church',
+ is_pi=False
+ ),
+ Author(
+ username='Paul Goldsmith',
+ firstname='Paul',
+ lastname='Goldsmith',
+ is_pi=False
+ ),
+ Author(
+ username='yancyshirley',
+ firstname='Yancy',
+ lastname='Shirley',
+ is_pi=False
+ ),
+ Author(
+ username='Michael Dunham',
+ firstname='Michael',
+ lastname='Dunham',
+ is_pi=False
+ ),
+ Author(
+ username='DTFrayer',
+ firstname='Dave',
+ lastname='Frayer',
+ is_pi=False
+ ),
+ ]
+
+ def build_exec_blocks(self):
+ """ Returns AGBT17B_044's execution blocks """
+
+ exec_blocks = [
+ ExecutionBlock(execution_block_id=152679,
+ filegroup_id=553493,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ExecutionBlock(execution_block_id=152680,
+ filegroup_id=553494,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ExecutionBlock(execution_block_id=152681,
+ filegroup_id=553495,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ExecutionBlock(execution_block_id=152682,
+ filegroup_id=553496,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ExecutionBlock(execution_block_id=152683,
+ filegroup_id=553497,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ExecutionBlock(execution_block_id=152684,
+ filegroup_id=553498,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ExecutionBlock(execution_block_id=152685,
+ filegroup_id=553499,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ExecutionBlock(execution_block_id=152686,
+ filegroup_id=553500,
+ calibration_level='0',
+ calibration_status='GBT/NOCAL',
+ band_code='N/A'
+ ),
+ ]
+ return exec_blocks
+
+ def build_science_products(self):
+ """ Returns AGBT17B_044's science product
+ (as a list, for consistency with other ProductSets)
+ """
+
+ return [
+ ScienceProduct(
+ filegroup_id=553493,
+ external_system='GBT',
+ science_product_type='dummy1',
+ metadata_ingestion_date=datetime.datetime.strptime(
+ '2020-10-21 15:36:37.876148', DATETIME_FORMAT),
+ metadata_ingestion_version='1',
+ external_name='EXTERNAL NAME 2020-10-21 15:36:37.903180',
+ )
+ ]
diff --git a/shared/workspaces/test/test_test_data_expected_values.py b/shared/workspaces/test/test_test_data_expected_values.py
new file mode 100644
index 0000000000000000000000000000000000000000..59337764854a3c77391aca3f2144ac88e7eb1062
--- /dev/null
+++ b/shared/workspaces/test/test_test_data_expected_values.py
@@ -0,0 +1,320 @@
+""" These tests check test data for various types of products,
+ so we'll know we're getting what we expect when we use the test data in
+ our tests.
+ """
+
+import pytest
+
+from .test_data.products.expected_values_alma import AlmaPublicProductSet, \
+ AlmaProprietaryImageProductSet, AlmaAudiProductSet
+from .test_data.products.expected_values_evla import VlaProductSet, \
+ VlbaProductSet, CalibrationProduct, PPR_FILENAME, VlassImageProductSet
+from .test_data.products.expected_values_gbt import GbtProductSet
+from .utilities import Deliverable, DeliverableProduct
+
+MS_TGZ = '.ms.tgz'
+WEBLOG = 'weblog.tgz'
+
+# pylint: disable=C0301, R0201, R0912
+
+
+def test_gets_evla_eb_products():
+ """
+ Check our execution blocks test data
+ :return:
+ """
+
+ product_set = VlaProductSet()
+ exec_blocks = product_set.exec_blocks
+ assert len(exec_blocks) == 2
+ assert len(product_set.project.authors) == 5
+ assert product_set.total_size == 95649344815
+ assert len(product_set.science_products) == len(exec_blocks)
+
+
+def test_gets_evla_ms():
+ """
+ Check our basic measurement set test data
+ :return:
+ """
+
+ product_set = VlaProductSet()
+ basic_ms = product_set.measurement_set
+ expected_file_count = 9
+ for filegroup in basic_ms:
+ files = basic_ms[filegroup]
+ actual_file_count = 0
+ for item in files:
+ actual_file_count += 1
+ filename = str(item['filename'])
+
+ # spot-check a few files
+ if filename == 'unknown.auxproducts.tgz':
+ assert item['size'] == 615
+ if filename.startswith('17A-109') and filename.endswith(MS_TGZ):
+ if filegroup.endswith('18468'):
+ assert item['size'] == 23690977531
+ elif filename.endswith('41979'):
+ assert item['size'] == 23690977245
+ elif filename == 'unknown.pprequest.xml':
+ if filegroup.endswith('18468'):
+ assert item['size'] == 5610
+ elif filename.endswith('41979'):
+ assert item['size'] == 5611
+ elif filename == WEBLOG:
+ if filegroup.endswith('18468'):
+ assert item['size'] == 1681803
+ elif filename.endswith('41979'):
+ assert item['size'] == 1683135
+
+ assert actual_file_count == expected_file_count
+
+
+def test_gets_evla_cms():
+ """
+ Check our calibrated measurement set ('restore') test data
+ :return:
+ """
+
+ product_set = VlaProductSet()
+ cms = product_set.cms
+ expected_file_count = 8
+ for filegroup in cms:
+ files = cms[filegroup]
+ actual_file_count = 0
+ for item in files:
+ actual_file_count += 1
+ filename = str(item['filename'])
+
+ # spot-check a few files
+ if filename == PPR_FILENAME:
+ assert item['size'] == 1632
+ if filename.startswith('17A-109') and filename.endswith('.tar.tgz'):
+ if filegroup.endswith('18468'):
+ assert item['size'] == 47520134173
+ elif filename.endswith('41979'):
+ assert item['size'] == 88842463132
+ elif filename == 'unknown.pipeline_manifest.xml':
+ if filegroup.endswith('18468'):
+ assert item['size'] == 24025
+ elif filename.endswith('41979'):
+ assert item['size'] == 11006
+ elif filename == 'casa_commands.log':
+ if filegroup.endswith('18468'):
+ assert item['size'] == 260953
+ elif filename.endswith('41979'):
+ assert item['size'] == 133935
+
+ assert actual_file_count == expected_file_count
+
+
+def test_gets_image_product():
+ """ Confirm one test image product with correct size and
+ correct project info
+ """
+
+ product_set = VlassImageProductSet()
+ assert product_set is not None
+ assert not hasattr(product_set, 'exec_blocks')
+ assert product_set.project.project_code == 'VLASS1.1'
+ assert len(product_set.project.authors) == 5
+ assert len(product_set.science_products) == 1
+ assert product_set.total_size == 110851200
+
+
+def test_gets_vlba_products():
+ """ Confirm that we get expected number of VLBA products
+ with correct size and project info
+ """
+
+ product_set = VlbaProductSet()
+ assert product_set is not None
+ exec_blocks = product_set.exec_blocks
+ assert len(exec_blocks) == 4
+ assert len(product_set.project.authors) == 4
+ assert product_set.total_size == 2140560000
+ assert len(product_set.science_products) == len(exec_blocks)
+
+
+def test_gets_calibration():
+ """ Confirm that we get one calibration product with correct size and
+ correct project info
+ """
+
+ product = CalibrationProduct()
+ assert product is not None
+ assert not hasattr(product, 'exec_blocks')
+ assert len(product.project.authors) == 9
+ science_product = product.science_product
+ assert science_product.science_product_type == 'Calibration'
+ assert science_product.external_name == '18B-265_2019_12_10_T00_00_59.203.tar'
+ assert len(product.file_info) == 1
+ assert product.total_size == 27668480
+
+
+def test_gets_alma_raw_data():
+ """ Grab ALMA SDM+BDF test data and confirm we got what we expect """
+
+ product_set = AlmaPublicProductSet()
+
+ file_info = product_set.file_info
+ assert isinstance(file_info, dict)
+ assert len(file_info) == 3
+ assert product_set.rawdata_total_size == 1368988450
+
+ file_sizes = dict()
+ delivered_products = list()
+ for file_type_str in file_info.keys():
+ values = file_info[file_type_str]
+ file_type = Deliverable.from_str(file_type_str)
+ assert file_type is not None
+ file_sizes[file_type] = file_info.values()
+ delivered_products.append(DeliverableProduct(file_type, values))
+
+ # spot-check what we got for each kind of deliverable
+ for delivery in delivered_products:
+ file_count = len(delivery.file_info)
+ if delivery.type == Deliverable.TAR:
+ assert file_count == 1
+ for filename, size in delivery.file_info.items():
+ assert filename.startswith('uid___A002_Xcd8029_Xb0a4') \
+ and filename.endswith('.tar.gz')
+ assert size == 457195765
+ elif delivery.type == Deliverable.SDM:
+ file_count = len(delivery.file_info)
+ assert file_count == 39
+ xml_count = 0
+ bin_count = 0
+ for filename, size in delivery.file_info.items():
+ if filename.endswith('.xml'):
+ xml_count += 1
+ if filename.startswith('Pointing.'):
+ assert size == 963
+ elif filename.endswith('bin'):
+ bin_count += 1
+ if filename.startswith('Pointing.'):
+ assert size == 92853427
+
+ assert xml_count == 36
+ assert bin_count == 3
+ elif delivery.type == Deliverable.BDF:
+ file_count = len(delivery.file_info)
+ assert file_count == 216
+ for file_path, size in delivery.file_info.items():
+ assert file_path.startswith('ASDMBinary')
+ assert size >= 14000
+ else:
+ pytest.fail(f'Unexpected deliverable from ALMA raw data: '
+ f'{delivery.type}')
+
+
+def test_gets_alma_basic_ms():
+ """
+ Analyze our ALMA basic MS test data and ensure the values are as we expect
+
+ """
+
+ product_set = AlmaPublicProductSet()
+ basic_ms = product_set.measurement_set
+ assert isinstance(basic_ms, dict)
+ assert len(basic_ms) == 3
+
+ for name in basic_ms.keys():
+ file_info = basic_ms[name]
+ assert isinstance(file_info, dict)
+ assert len(file_info) == 10
+ weblog_found = False
+ ms_tgz_found = False
+ for filename, size in file_info.items():
+ if filename in (PPR_FILENAME,
+ 'unknown.download_alma_ms.pprequest.xml'):
+ assert size in (4582, 4584)
+ # spot-check some file sizes
+ if name == 'uid___A002_Xcd8029_Xa892':
+ if filename == WEBLOG:
+ assert size == 1592335
+ weblog_found = True
+ elif filename.endswith(MS_TGZ):
+ assert size == 519234359
+ ms_tgz_found = True
+ elif name == 'uid___A002_Xcd8029_Xb0a4':
+ if filename == 'weblog.tgz':
+ assert size == 1576475
+ weblog_found = True
+ elif filename.endswith(MS_TGZ):
+ assert size == 518582184
+ ms_tgz_found = True
+ elif name == 'uid___A002_Xcd8029_Xfdd':
+ if filename == WEBLOG:
+ assert size == 1562579
+ weblog_found = True
+ elif filename.endswith(MS_TGZ):
+ assert size == 504128830
+ ms_tgz_found = True
+ else:
+ pytest.fail(f'Unexpected item found in basic MS: {name}')
+ assert weblog_found and ms_tgz_found
+
+
+def test_gets_alma_restored_ms():
+ """ We're expecting these files inside the tar returned
+ for ALMA restored MS download of 2017.1.00297.S
+ """
+ product_set = AlmaPublicProductSet()
+ restored_ms = product_set.restored_ms
+ assert isinstance(restored_ms, dict)
+ assert len(restored_ms) == 1
+ assert product_set.rest_ms_size == 3947540021
+
+ for name in restored_ms.keys():
+ assert name == 'uid___A002_Xcd8029_Xb0a4'
+ values = restored_ms[name]
+ assert len(values) == 4
+ assert values['uid___A002_Xcd8029_Xb0a4.ms.tgz'] == 1325422495
+ assert values['uid___A002_Xcd8029_Xa892.ms.tgz'] == 1331336708
+ assert values['uid___A002_Xcd8029_Xfdd.ms.tgz'] == 1290775660
+ assert values[PPR_FILENAME] == 5158
+
+
+def test_gets_alma_images():
+ """ Expected download results for a proprietary ALMA project
+ with image data
+ """
+
+ product = AlmaProprietaryImageProductSet()
+ file_info = product.file_info
+ assert len(file_info) == 3
+ total_size = 0
+ for filename, size in file_info.items():
+ total_size += size
+ if '_71595054' in filename or '_4d1b66da' in filename:
+ assert size == 161280
+ else:
+ assert '_image_cube_daf0d76e' in filename
+ assert size == 112907520
+
+ assert total_size == product.total_size
+
+
+def test_gets_audi_product():
+ """
+ Make sure we're getting the product and files we expect for our
+ AlmaAudiProductSet
+
+ :return:
+ """
+
+ product = AlmaAudiProductSet()
+ assert product.total_size == 1368988450
+ file_info = product.file_info
+ assert len(file_info) == 3
+ for file_type in ['SDM', 'BDF', 'TAR']:
+ assert file_type in file_info.keys()
+
+
+def test_gets_gbt_product():
+ """ make sure we're getting the GBT test data values we expect """
+ product_set = GbtProductSet()
+ assert product_set.project.project_code == 'AGBT17B_044'
+ assert product_set.total_size == 948572979200
+ assert len(product_set.file_info) == 4
diff --git a/shared/workspaces/test/utilities.py b/shared/workspaces/test/utilities.py
new file mode 100644
index 0000000000000000000000000000000000000000..8e1a7639907638c30cc7a48116002e6242e84569
--- /dev/null
+++ b/shared/workspaces/test/utilities.py
@@ -0,0 +1,138 @@
+""" Helper functions for download product testing """
+
+import json
+import sys
+from enum import Enum
+from pathlib import Path
+
+
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f'
+DATE_FORMAT = '%Y-%m-%d'
+
+
+def get_report_file(basename: str):
+ """ Get a locations file from our collection in test_data,
+ given a basename (.json filename w/o extension)
+
+ """
+
+ test_data_dir = get_test_data_dir()
+
+ for file in test_data_dir.glob(basename.upper() + '.json'):
+ # (there should be only one for this basename)
+ return file
+
+ return None
+
+
+def get_exec_block_details_from_loc_report(prefix: str, exec_blocks: list):
+ """
+ Fetch and read locations report indicated by basename;
+ for filegroup IDs of exec blocks, return file info dict
+ and total size of files
+
+ :param prefix:
+ :param exec_blocks:
+ :return:
+
+ """
+ file_info = dict()
+ total_size = 0
+ for exec_block in exec_blocks:
+ basename = prefix + str(exec_block.filegroup_id)
+ locations_report = None
+ try:
+ locations_report = get_locations_report(basename)
+ except FileNotFoundError:
+ # special case: GBT product
+ if basename.startswith('AGBT17B_044'):
+ locations_report = get_locations_report('AGBT17B_044_02')
+
+ total_size += locations_report['aggregate_size']
+ for file_spec in locations_report['files']:
+ filename = file_spec['ngas_file_id']
+ size = file_spec['size']
+ file_info[filename] = size
+
+ return file_info, total_size
+
+
+def get_test_data_dir():
+ """ where's our test data? """
+ top_level_subdirs = sys.path
+ shared_ws_src = None
+ for pathname in top_level_subdirs:
+ if 'shared/workspaces' in pathname:
+ shared_ws_src = pathname
+ break
+ shared_wksp = Path(shared_ws_src).parent
+
+ # test data will be a few levels under shared_wksp
+ for item in shared_wksp.rglob('location_files'):
+ assert item.is_dir()
+ return item
+
+ return None
+
+
+def get_file_info_from_json_file(json_filename: str) -> dict:
+ """ Pluck file information from a .json location file """
+
+ to_read = None
+ for file in Path.cwd().rglob(json_filename):
+ to_read = file
+ break
+ assert to_read is not None
+ with open(to_read, 'r') as content:
+ file_info = json.loads(content.read())
+
+ return file_info
+
+
+def get_file_info_from_loc_report(locations_report: dict) -> tuple:
+ file_info = dict()
+ total_size = 0
+ total_size += locations_report['aggregate_size']
+ for file_spec in locations_report['files']:
+ filename = file_spec['ngas_file_id']
+ size = file_spec['size']
+ file_info[filename] = size
+ return file_info, total_size
+
+
+def get_locations_report(basename: str):
+ """ Get a locations report from a file in test_data """
+
+ report_path = get_report_file(basename)
+
+ if report_path is not None:
+ with open(report_path, 'r') as content:
+ locations_report = json.loads(content.read())
+ return locations_report
+
+ raise FileNotFoundError(f'{basename.upper() + ".json"} not found')
+
+
+class Deliverable(Enum):
+ SDM = 'SDM'
+ BDF = 'BDF'
+ MS = 'MS'
+ CMS = 'CMS'
+ IMG = 'IMG'
+ TAR = 'TAR'
+ # VLBA
+ IDIFITS = 'IDIFITS'
+
+ @staticmethod
+ def from_str(key: str):
+ for dtype in Deliverable:
+ if dtype.value == key:
+ return dtype
+ return None
+
+
+class DeliverableProduct:
+
+ def __init__(self, dtype: Deliverable, file_info: dict):
+ self.type = dtype
+ self.file_info = file_info