SSA-6383: added GBT to workspaces test data

shared/workspaces/test/test_data/location_files/AGBT17B_044_02.json

+{"files": [{"ngas_file_id": "AGBT17B_044_02__split0000.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0000.tar", "checksum": "1012891531", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 30000000000, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}, {"ngas_file_id": "AGBT17B_044_02__split0001.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0001.tar", "checksum": "950028996", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 30000000000, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}, {"ngas_file_id": "AGBT17B_044_02__split0002.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0002.tar", "checksum": "192919877", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 30000000000, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}, {"ngas_file_id": "AGBT17B_044_02__split0003.tar", "subdirectory": "EXTERNAL NAME 2020-10-21 15:36:37.903180", "relative_path": "AGBT17B_044_02__split0003.tar", "checksum": "-465285750", "checksum_type": "ngamsGenCrc32", "version": 1, "size": 28571622400, "server": {"server": "vegas-arc-04.cv.nrao.edu:7777", "location": "UVA", "cluster": "DSOC"}}], "aggregate_size": 118571622400}
\ No newline at end of file

shared/workspaces/test/test_data/products/expected_values_gbt.py

+import warnings
+from sqlalchemy import exc as sa_exc
+
+from schema import Project, ExecutionBlock, Author, ScienceProduct
+
+from shared.workspaces.test.utilities import get_locations_report
+
+# TODO: convert date strings to datetime
+class GbtProductSet:
+
+    def __init__(self):
+        self.project = self.build_project()
+        self.exec_blocks = self.build_exec_blocks()
+        self.science_products = self.build_science_products()
+        self.file_info, self.total_size = self.get_file_info()
+
+    def build_project(self):
+        with warnings.catch_warnings():
+                # Suppress SQLAlchemy warnings
+            warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+            project = Project(project_code='AGBT17B_044',
+                              starttime=58089.1619444443,
+                              endtime=58138.2070370372,
+                              proprietary_duration=365,
+                              total_observation_time=27.055,
+                              last_addition='2020-10-21',
+                              title='Infall and Fragmentation of Cores in the Taurus: I Finding Cores with Infall',
+                              abstract=' We propose to conduct a survey of investigating infall candidates and fragmentation toward a complete population of prestellar cores in Taurus in HCN, HCO+, N2H+, and NH2D. The proposed observation will trace infall motions within prestellar cores from 0.05 pc down to 0.0075 pc (1500 AU) spatial scale, which is a typical size scale of individual planetary systems. The scientific goal is to estimate a fraction of infall candidates out of a complete population of prestellar cores and to understand internal velocity structure during final gravitational collapse before forming stars. We shall also search for fragmentation within prestellar cores, and to investigate kinematics involved with fragmentation processes. We propose to conduct pointing observation toward 57 prestellar cores using the ARGUS of the GBT in the 89 GHz transitions of HCN and HCO+, 93 GHz transition of N2H+, and to map 2 prestellar cores, which are found to have infall motions, in N2H+ and NH2D.'
+                        )
+            project.authors = self.build_authors()
+            return project
+
+    def build_authors(self):
+        return [
+            Author(
+                username='seo3919',
+                firstname='Youngmin',
+                lastname='Seo',
+                is_pi=True
+            ),
+            Author(
+                username='schurch',
+                firstname='Sarah',
+                lastname='Church',
+                is_pi=False
+            ),
+            Author(
+                username='Paul Goldsmith',
+                firstname='Paul',
+                lastname='Goldsmith',
+                is_pi=False
+            ),
+            Author(
+                username='yancyshirley',
+                firstname='Yancy',
+                lastname='Shirley',
+                is_pi=False
+            ),
+            Author(
+                username='Michael Dunham',
+                firstname='Michael',
+                lastname='Dunham',
+                is_pi=False
+            ),
+            Author(
+                username='DTFrayer',
+                firstname='Dave',
+                lastname='Frayer',
+                is_pi=False
+            ),
+        ]
+
+    def build_exec_blocks(self):
+        exec_blocks = [
+            ExecutionBlock(execution_block_id=152679,
+                           filegroup_id=553493,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+            ExecutionBlock(execution_block_id=152680,
+                           filegroup_id=553494,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+            ExecutionBlock(execution_block_id=152681,
+                           filegroup_id=553495,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+            ExecutionBlock(execution_block_id=152682,
+                           filegroup_id=553496,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+            ExecutionBlock(execution_block_id=152683,
+                           filegroup_id=553497,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+            ExecutionBlock(execution_block_id=152684,
+                           filegroup_id=553498,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+            ExecutionBlock(execution_block_id=152685,
+                           filegroup_id=553499,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+            ExecutionBlock(execution_block_id=152686,
+                           filegroup_id=553500,
+                           calibration_level=0,
+                           calibration_status='GBT/NOCAL',
+                           band_code='N/A'
+                           ),
+        ]
+        return exec_blocks
+
+    def build_science_products(self):
+        return [
+            ScienceProduct(
+                filegroup_id=553493,
+                external_system='GBT',
+                science_product_type='dummy1',
+                metadata_ingestion_date='2020-10-21 15:36:37.876148',
+                metadata_ingestion_version=1,
+                external_name='EXTERNAL NAME 2020-10-21 15:36:37.903180',
+            )
+        ]
+
+    def get_file_info(self):
+        file_info = dict()
+        total_size = 0
+        for exec_block in self.exec_blocks:
+            basename = 'AGBT17B_044_02' + str(exec_block.filegroup_id)
+            locations_report = get_locations_report(basename)
+            total_size += locations_report['aggregate_size']
+            for file_spec in locations_report['files']:
+                filename = file_spec['ngas_file_id']
+                size = file_spec['size']
+                file_info[filename] = size
+            return file_info, total_size

shared/workspaces/test/test_expected_values.py → shared/workspaces/test/test_test_data_expected_values.py

-""" Tests for EVLA download products"""
+""" These tests check test data for various types of products,
+    so we'll know we're getting what we expect when we use the test data in
+    our tests.
+ """
 
 import pytest
 
-from .test_data.products.expected_values_alma import AlmaPublicProductSet
+from .test_data.products.expected_values_alma import AlmaPublicProductSet, \
+    AlmaImageProduct
 from .test_data.products.expected_values_evla import VlaProductSet, \
     VlassImageProductSet, VlbaProductSet, CalibrationProduct, PPR_FILENAME
 from .utilities import get_locations_report, Deliverable, DeliverableProduct
 
-# FIXME: these location reports are duplicates of the ones in
-#  datafetcher tests. Refactor DF to use these and get rid of the DF copies
-
-""" Example: checking download results"""
-
 MS_TGZ = '.ms.tgz'
 WEBLOG = 'weblog.tgz'
 
 
-def test_gets_vla_products():
-    """ Check test data for our two 17A-109 EBs """
-    product_set = VlaProductSet()
-    assert product_set is not None
+def test_gets_evla_eb_products():
+    """
+    Check our execution blocks test data
+    :return:
+    """
 
-    # check the execution blocks
+    product_set = VlaProductSet()
     exec_blocks = product_set.exec_blocks
     assert len(exec_blocks) == 2
     assert len(product_set.project.authors) == 5
     assert product_set.total_size == 95649344815
     assert len(product_set.science_products) == len(exec_blocks)
 
-    # check the basic measurement set
+def test_gets_evla_ms():
+    """
+    Check our basic measurement set test data
+    :return:
+    """
+    product_set = VlaProductSet()
     basic_ms = product_set.measurement_set
     expected_file_count = 9
     for filegroup in basic_ms:
@@ -59,7 +64,13 @@ def test_gets_vla_products():
 
         assert actual_file_count == expected_file_count
 
-    # check the calibrated measurement set (aka "restore")
+def test_gets_evla_cms():
+    """
+    Check our calibrated measurement set ('restore') test data
+    :return:
+    """
+
+    product_set = VlaProductSet()
     cms = product_set.cms
     expected_file_count = 8
     for filegroup in cms:
@@ -90,7 +101,6 @@ def test_gets_vla_products():
 
         assert actual_file_count == expected_file_count
 
-
 def test_gets_image_product():
     """ Confirm one test image product with correct size and
         correct project info
@@ -191,10 +201,11 @@ def test_gets_alma_raw_data():
 
 
 def test_gets_alma_basic_ms():
-    """ Analyze our ALMA basic MS test data and ensure the values are as we
-        expect 
     """
-    
+    Analyze our ALMA basic MS test data and ensure the values are as we expect
+
+    """
+
     product_set = AlmaPublicProductSet()
     basic_ms = product_set.measurement_set
     assert isinstance(basic_ms, dict)
@@ -256,6 +267,19 @@ def test_gets_alma_restored_ms():
         assert values[PPR_FILENAME] == 5158
 
 
+def test_gets_alma_image():
+    """ Expected download results for a proprietary ALMA project
+        with image data
+    """
+
+    product = AlmaImageProduct()
+    delivered = product.file_info
+    assert len(delivered) == 1
+    file_info = delivered[0]
+    assert 'uid___A001_X1465_X201e' in file_info['filename']
+    assert file_info['filename'].endswith('Per_18-21_sci.spw25.cube.I.pbcor.fits')
+    assert file_info['size'] == 112907520
+
 def test_gets_vla_report_files():
     """ make sure we're getting the VLA test data values we expect
         for our 17B-109 exec blocks of interest
@@ -321,7 +345,6 @@ def test_gets_img_report_file():
     agg_size = locations_dict['aggregate_size']
     assert agg_size == 110851200
 
-
 def test_gets_cal_report_file():
     """ make sure we're getting the calibration test data values we expect """
 
@@ -337,3 +360,18 @@ def test_gets_cal_report_file():
     assert len(files) == 1
     agg_size = locations_dict['aggregate_size']
     assert agg_size == 27668480
+
+def test_gets_gbt_product():
+    """ make sure we're getting the GBT test data values we expect """
+    locations_dict = get_locations_report('AGBT17B_044_02')
+    assert locations_dict is not None
+    assert isinstance(locations_dict, dict)
+
+    assert len(locations_dict) == 2
+    assert 'files' in locations_dict.keys()
+    assert 'aggregate_size' in locations_dict.keys()
+
+    files = locations_dict['files']
+    assert len(files) == 4
+    agg_size = locations_dict['aggregate_size']
+    assert agg_size == 118571622400