From a055432b9f2bf5797f7c6f02d518c52c31b260f4 Mon Sep 17 00:00:00 2001
From: "Janet L. Goldstein" <jgoldste@nrao.edu>
Date: Mon, 28 Sep 2020 08:37:45 -0600
Subject: [PATCH] SSA-6383: ALMA test data and preliminary tests, part I
---
.../alma_A001_X1296_Xa93_rawdata.json | 265 ++++++++++++++++++
.../products/expected_values_alma.py | 241 +++++++++++++++-
.../products/expected_values_evla.py | 190 +++++++------
.../workspaces/test/test_expected_values.py | 168 +++++++++--
shared/workspaces/test/utilities.py | 33 ++-
5 files changed, 778 insertions(+), 119 deletions(-)
create mode 100644 shared/workspaces/test/test_data/products/alma_A001_X1296_Xa93_rawdata.json
diff --git a/shared/workspaces/test/test_data/products/alma_A001_X1296_Xa93_rawdata.json b/shared/workspaces/test/test_data/products/alma_A001_X1296_Xa93_rawdata.json
new file mode 100644
index 000000000..bf1b50532
--- /dev/null
+++ b/shared/workspaces/test/test_data/products/alma_A001_X1296_Xa93_rawdata.json
@@ -0,0 +1,265 @@
+{
+ "TAR": {
+ "uid___A002_Xcd8029_Xb0a4.400881588.tar.gz": 457195765
+ },
+ "SDM": {
+ "Annotation.xml": 1505,
+ "Antenna.xml": 4165,
+ "ASDM.xml": 10577,
+ "CalAmpli.xml": 29180,
+ "CalAtmosphere.bin": 21793806,
+ "CalAtmosphere.xml": 1319,
+ "CalData.xml": 11865,
+ "CalDevice.xml": 3478307,
+ "CalPhase.xml": 420017,
+ "CalPointing.xml": 20559,
+ "CalReduction.xml": 12896,
+ "ConfigDescription.xml": 8660,
+ "CorrelatorMode.xml": 1524,
+ "DataDescription.xml": 4873,
+ "ExecBlock.xml": 1938,
+ "Feed.xml": 147490,
+ "Field.xml": 1832,
+ "Flag.xml": 1010289,
+ "Focus.xml": 327030,
+ "FocusModel.xml": 8157,
+ "Main.xml": 152322,
+ "Pointing.bin": 92853427,
+ "Pointing.xml": 963,
+ "PointingModel.xml": 71938,
+ "Polarization.xml": 678,
+ "Processor.xml": 1112,
+ "Receiver.xml": 21041,
+ "SBSummary.xml": 2013,
+ "Scan.xml": 22530,
+ "Source.xml": 39233,
+ "SpectralWindow.xml": 20090,
+ "SquareLawDetector.xml": 1096,
+ "State.xml": 1024,
+ "Station.xml": 3930,
+ "Subscan.xml": 615322,
+ "SwitchCycle.xml": 777,
+ "SysCal.bin": 14468685,
+ "SysCal.xml": 955,
+ "Weather.xml": 141587
+ },
+ "BDF": {
+ "ASDMBinary/uid___A002_Xcd8029_Xb275": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b7": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ee": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb176": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb888": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26f": 380191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb707": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb11b": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52b": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb924": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb829": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b6": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb42e": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6fe": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb282": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2bb": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb621": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6c3": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bf": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f7": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb112": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d7": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb452": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c1": 3966591,
+ "ASDMBinary/uid___A002_Xcd8029_Xb157": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb460": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb417": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb837": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c2": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb780": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0ba": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d0": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb162": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb902": 5973772,
+ "ASDMBinary/uid___A002_Xcd8029_Xb455": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb422": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2be": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb79c": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7c9": 19961996,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52e": 19961996,
+ "ASDMBinary/uid___A002_Xcd8029_Xb42b": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8c6": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb607": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2b5": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7cc": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb777": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8eb": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5e3": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb171": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb35d": 3974767,
+ "ASDMBinary/uid___A002_Xcd8029_Xb391": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb61d": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb278": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0da": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41d": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb175": 19878195,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ef": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb12c": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83c": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b4": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b5": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb11a": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5dd": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26e": 386323,
+ "ASDMBinary/uid___A002_Xcd8029_Xb451": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f4": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb111": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0be": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6c0": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb622": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ba": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60c": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb281": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7c7": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6ff": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb469": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb88a": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb123": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb784": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb610": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c2": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7bd": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ac": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb783": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb159": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c1": 1591240,
+ "ASDMBinary/uid___A002_Xcd8029_Xb480": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb419": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb124": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb381": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb77d": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60d": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb456": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f3": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bb": 1593367,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26b": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b8": 1593367,
+ "ASDMBinary/uid___A002_Xcd8029_Xb477": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52f": 3994012,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5dc": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5cd": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b9": 391888,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41c": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb172": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb12d": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ea": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83d": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb886": 387788,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8c7": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb135": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb42c": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb505": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb922": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb701": 19953796,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7cb": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2b4": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41a": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0dd": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5e2": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb15a": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb884": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb279": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb61e": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb390": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c3": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb88d": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb781": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7ba": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2af": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb423": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb454": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb13e": 73919,
+ "ASDMBinary/uid___A002_Xcd8029_Xb163": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d1": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb383": 19878195,
+ "ASDMBinary/uid___A002_Xcd8029_Xb77f": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb79b": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2bd": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb284": 3966591,
+ "ASDMBinary/uid___A002_Xcd8029_Xb384": 3966591,
+ "ASDMBinary/uid___A002_Xcd8029_Xb620": 19955870,
+ "ASDMBinary/uid___A002_Xcd8029_Xb79e": 3983762,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60a": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb82b": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8f6": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d6": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb47b": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c0": 19888415,
+ "ASDMBinary/uid___A002_Xcd8029_Xb416": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb461": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb88c": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0d9": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb156": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb35b": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb274": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb889": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb46a": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0dc": 73770,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5e5": 3989912,
+ "ASDMBinary/uid___A002_Xcd8029_Xb177": 27070347,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ed": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb15f": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb883": 14180,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83a": 19953796,
+ "ASDMBinary/uid___A002_Xcd8029_Xb706": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb478": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb52c": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb508": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb828": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8ff": 250487,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7ca": 3983762,
+ "ASDMBinary/uid___A002_Xcd8029_Xb26c": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb702": 3983780,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5ce": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb921": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0b9": 1599517,
+ "ASDMBinary/uid___A002_Xcd8029_Xb506": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb136": 2723822,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b8": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb270": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb887": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb15b": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb41b": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb534": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ae": 386323,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7bb": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0c0": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60e": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb77e": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb380": 811818,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bc": 1599517,
+ "ASDMBinary/uid___A002_Xcd8029_Xb47f": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb5d2": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb160": 1649255,
+ "ASDMBinary/uid___A002_Xcd8029_Xb13f": 2723899,
+ "ASDMBinary/uid___A002_Xcd8029_Xb900": 8127902,
+ "ASDMBinary/uid___A002_Xcd8029_Xb47a": 19953796,
+ "ASDMBinary/uid___A002_Xcd8029_Xb0bd": 1585191,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7c6": 813623,
+ "ASDMBinary/uid___A002_Xcd8029_Xb60b": 379588,
+ "ASDMBinary/uid___A002_Xcd8029_Xb623": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb6c1": 5421830,
+ "ASDMBinary/uid___A002_Xcd8029_Xb838": 27070532,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2ab": 14135,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7be": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2c3": 5421789,
+ "ASDMBinary/uid___A002_Xcd8029_Xb611": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb83b": 3981712,
+ "ASDMBinary/uid___A002_Xcd8029_Xb174": 19878195,
+ "ASDMBinary/uid___A002_Xcd8029_Xb533": 170219,
+ "ASDMBinary/uid___A002_Xcd8029_Xb35a": 169854,
+ "ASDMBinary/uid___A002_Xcd8029_Xb7b5": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb8c9": 3983780,
+ "ASDMBinary/uid___A002_Xcd8029_Xb2b0": 378147,
+ "ASDMBinary/uid___A002_Xcd8029_Xb778": 1649300,
+ "ASDMBinary/uid___A002_Xcd8029_Xb608": 1649300
+ }
+}
+
diff --git a/shared/workspaces/test/test_data/products/expected_values_alma.py b/shared/workspaces/test/test_data/products/expected_values_alma.py
index a0f1b1a57..8e55207be 100644
--- a/shared/workspaces/test/test_data/products/expected_values_alma.py
+++ b/shared/workspaces/test/test_data/products/expected_values_alma.py
@@ -1,10 +1,243 @@
-''' Test data: ALMA download products'''
+""" Test data: ALMA download products """
+import json
+import warnings
+from pathlib import Path
+
+from schema import Project, Author, ExecutionBlock, ScienceProduct
+from sqlalchemy import exc as sa_exc
+
+from .expected_values_evla import \
+ EXECUTION_BLOCK, METADATA_INGESTION_DATE, CASA_LOG, PPR_FILENAME
+
+EXEC_BLOCK = 'execblock'
+CALIBRATED = 'CALIBRATED'
+_WEBLOG = 'weblog.tgz'
class AlmaPublicProductSet:
- # TODO:
+ """ Test data for 2017.1.00297.S """
+
+ def __init__(self):
+ self.project = self.build_project()
+ self.exec_blocks = self.build_exec_blocks()
+ self.science_products = self.build_science_products()
+ self.file_info = self.get_file_info()
+ self.measurement_set = self.build_basic_ms()
+ self.restored_ms = self.build_restored_ms()
+
+ def build_project(self):
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project_code = '2017.1.00297.S'
+ project = Project(
+ project_code=project_code,
+ starttime=56799.3877155556,
+ endtime=56799.4128683333,
+ proprietary_duration=365,
+ total_observation_time=0.0583143749972805,
+ title='Copy of 2012.1.00060.S for testing',
+ abstract = "Understanding the physical factors that control the conversion of interstellar gas into stars is of fundamental importance for both developing a predictive physical theory of star formation and understanding the evolution of galaxies from the earliest epochs of cosmic history to the present time. An important aspect of this question is the study of empirical relations that connect the star formation rate in a given region to local properties of the interstellar medium. An important example is the Schmidt-Kennicutt (KS) law for galaxies that relates the surface densities of the star formation rate and the surface densities of interstellar gas in a non-linear fashion. However, it is also known that there is a linear correlation between the total SFR in galaxies and the mass of dense molecular gas as traced by the high excitation HCN molecule. Contrary to the KS relation, this scaling relation suggests that the total SFR depends simply on the total amount of dense molecular gas in a star forming system. Recently, we have begun to test these scaling relations in the Galactic neighborhood where star formation rates can be much better constrained. We found that for local clouds the total SFR scales most directly, and linearly, with the total mass of high extinction (and dense) molecular gas. Furthermore, we found this linear scaling law between SFR and dense gas to extend and extrapolate directly and smoothly to external galaxies. Moreover, our observations also demonstrate that a KS type relation does not exist for molecular clouds in the Galactic neighborhood. This is a direct consequence of a well known scaling law between the mass and size of molecular clouds, Larson's third law. Overall, our results indicate that a linear scaling law, in which the total amount of dense gas controls the SFR, is the fundamental physical relation that connects star formation across the vast scales from individual GMCs to entire galaxies. Critical testing of these ideas require resolved observations of GMCs in external galaxies. Here we propose to use ALMA to evaluate star formation scaling laws in a nearby galaxy where we can obtain resolved observations of individual GMCs. This allows us to obtain observations of a larger sample of GMCs than is accessible in the Galactic neighborhood. An extensive APEX survey of HII regions in the nearby galaxy NGC 300 has provided us with a sample of 36 star-forming regions with CO(2-1) detections and 42 upper limits. We are currently working on obtaining star formation rates for these regions from multi-wavelength ancillary data including our Herschel observations. We propose to use ALMA's unequalled capabilities to obtain snapshot observations of 40 selected regions in CO(2-1) in order to make resolved measurements of cloud structure to obtain sizes and virial masses. As a pilot project, we also propose to observe the brightest subsample in HCN(1-0) as a dense-gas tracer. Our proposed ALMA CO observations will enable us to to test Larson's scaling laws in an external galaxy and to evaluate which formulation of the Schmidt law is the most meaningful and appropriate to apply to spiral galaxies, and in doing so refine Schmidt's original conjecture of a scaling relation between the rate of star formation and gas density."
+ )
+ project.authors = [
+ Author(project_code=project_code,
+ author_id=32550,
+ username='franzbauer',
+ firstname='Franz',
+ lastname='Bauer',
+ pst_person_id=None,
+ is_pi=True),
+
+ Author(project_code=project_code,
+ author_id=32551,
+ username='ezetre',
+ firstname='Ezequiel',
+ lastname='Treister',
+ pst_person_id=None,
+ is_pi=False),
+ Author(project_code=project_code,
+ author_id=32552,
+ username='lho',
+ firstname='Luis',
+ lastname='Ho',
+ pst_person_id=None,
+ is_pi=False),
+ Author(project_code=project_code,
+ author_id=32554,
+ username='jyshangguan',
+ firstname='Jinyi',
+ lastname='Shangguan',
+ pst_person_id=None,
+ is_pi=False),
+ Author(project_code=project_code,
+ author_id=32553,
+ username='rwang',
+ firstname='Ran',
+ lastname='Wang',
+ pst_person_id=None,
+ is_pi=False),
+ ]
+
+ return project
+
+ def build_exec_blocks(self):
+ alma_ous_id = 'uid://A001/X1296/Xa93'
+ band_code = '05'
+
+ return [
+ ExecutionBlock(
+ project_code=self.project.project_code,
+ execution_block_id=102209,
+ alma_ous_id=alma_ous_id,
+ band_code=band_code,
+ filegroup_id=212043,
+ ngas_fileset_id='uid___A002_Xcd8029_Xa892',
+ starttime=58256.0535855556,
+ endtime=58256.1137983333,
+ calibration_status=CALIBRATED
+ ),
+ ExecutionBlock(
+ project_code=self.project.project_code,
+ execution_block_id=102169,
+ alma_ous_id=alma_ous_id,
+ band_code=band_code,
+ filegroup_id=211958,
+ ngas_fileset_id='uid___A002_Xcd8029_Xfdd',
+ starttime=58255.0338461111,
+ endtime=58255.0919594444,
+ calibration_status=CALIBRATED
+ ),
+ ExecutionBlock(
+ project_code=self.project.project_code,
+ execution_block_id=102211,
+ alma_ous_id=alma_ous_id,
+ band_code=band_code,
+ filegroup_id=212047,
+ ngas_fileset_id='uid___A002_Xcd8029_Xb0a4',
+ starttime=58256.1152683333,
+ endtime=58256.1753838889,
+ calibration_status=CALIBRATED
+ ),
+ ]
+
+ def build_science_products(self):
+ external_system = 'ALMA Operations'
+ return [
+ ScienceProduct(
+ external_name='uid___A002_Xcd8029_Xfdd',
+ filegroup_id=211958,
+ external_system=external_system,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK
+ ),
+ ScienceProduct(
+ external_name='uid___A002_Xcd8029_Xb0a4',
+ filegroup_id=212047,
+ external_system=external_system,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK
+ ),
+ ScienceProduct(
+ external_name='uid___A002_Xcd8029_Xa892',
+ filegroup_id=212043,
+ external_system=external_system,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
+ metadata_ingestion_version='1',
+ science_product_type=EXECUTION_BLOCK
+ ),
+ ]
+
+ def get_file_info(self):
+ """ Since ALMA files are not in NGAS, we can't use
+ location reports to get them; we'll pluck them
+ from a JSON file instead.
+
+ """
+
+ # read in contents from JSON
+ to_read = None
+ for file in Path.cwd().rglob('alma_A001_X1296_Xa93_rawdata.json'):
+ to_read = file
+ break
+ assert to_read is not None
+ with open(to_read, 'r') as content:
+ file_info = json.loads(content.read())
+
+ return file_info
+
+ def build_basic_ms(self):
+ return {
+ 'uid___A002_Xcd8029_Xa892': {
+ 'uid___A002_Xcd8029_Xa892.400887655.tar.gz': 504369560,
+ 'unknown.download_alma_ms.pipeline_manifest.xml': 804,
+ 'casa_pipescript.py': 680,
+ PPR_FILENAME: 4584,
+ _WEBLOG: 1592335,
+ 'unknown.download_alma_ms.pprequest.xml': 4584,
+ 'unknown.download_alma_ms.auxproducts.tgz': 839,
+ CASA_LOG: 1729,
+ 'pipeline_aquareport.xml': 3472,
+ 'uid___A002_Xcd8029_Xa892.ms.tgz': 519234359,
+ },
+ 'uid___A002_Xcd8029_Xb0a4': {
+ 'uid___A002_Xcd8029_Xb0a4.400887655.tar.gz': 503501127,
+ 'unknown.download_alma_ms.pipeline_manifest.xml': 804,
+ CASA_LOG: 1729,
+ _WEBLOG: 1576475,
+ 'casa_pipescript.py': 680,
+ 'pipeline_aquareport.xml': 3465,
+ 'uid___A002_Xcd8029_Xb0a4.ms.tgz': 518582184,
+ 'unknown.download_alma_ms.auxproducts.tgz': 844,
+ PPR_FILENAME: 4584,
+ 'unknown.download_alma_ms.pprequest.xml': 4584,
+
+ },
+ 'uid___A002_Xcd8029_Xfdd': {
+ 'uid___A002_Xcd8029_Xfdd.400887655.tar.gz': 489883578,
+ 'unknown.download_alma_ms.pipeline_manifest.xml': 802,
+ 'pipeline_aquareport.xml': 3460,
+ PPR_FILENAME: 4582,
+ 'unknown.download_alma_ms.pprequest.xml': 4582,
+ _WEBLOG: 1562579,
+ 'uid___A002_Xcd8029_Xfdd.ms.tgz': 504128830,
+ CASA_LOG: 1723,
+ 'unknown.download_alma_ms.auxproducts.tgz': 836,
+ 'casa_pipescript.py': 679,
+ }
+ }
+
+ def build_restored_ms(self):
+ return {
+ 'uid___A002_Xcd8029_Xb0a4': {
+ 'uid___A002_Xcd8029_Xb0a4.ms.tgz': 1325422495,
+ 'uid___A002_Xcd8029_Xa892.ms.tgz': 1331336708,
+ 'uid___A002_Xcd8029_Xfdd.ms.tgz': 1290775660,
+ PPR_FILENAME: 5158,
+
+ },
+ }
+
+
+class AlmaImageProduct:
+ # TODO -- need to find project w/images
pass
+
class AlmaProprietaryProductSet:
- # TODO:
- pass
+ def __init__(self):
+ self.project = self.build_project()
+ self.exec_blocks = self.build_exec_blocks()
+ self.science_products = self.build_science_products()
+ self.file_info, self.total_size = self.get_file_info()
+ self.measurement_set = self.build_basic_ms()
+ self.cms = self.build_cms()
+
+ def build_project(self):
+ project = Project(
+ # TODO:
+ )
+ project.authors = self.build_authors()
+ return project
diff --git a/shared/workspaces/test/test_data/products/expected_values_evla.py b/shared/workspaces/test/test_data/products/expected_values_evla.py
index 497ad474f..c02191d53 100644
--- a/shared/workspaces/test/test_data/products/expected_values_evla.py
+++ b/shared/workspaces/test/test_data/products/expected_values_evla.py
@@ -1,10 +1,15 @@
-from enum import Enum
+""" Test data for two VLA EBs, a calibration product, and VLBA and VLASS
+ image products
+"""
+
+import warnings
+from sqlalchemy import exc as sa_exc
from schema import Project, ExecutionBlock, Author, ScienceProduct
from shared.workspaces.test.utilities import get_locations_report
-EVLA_METADATA_INGEST_DATE = '2019-12-12 16:04:02.11191'
+METADATA_INGESTION_DATE = '2019-12-12 16:04:02.11191'
EXECUTION_BLOCK = 'Execution Block'
PPR_FILENAME = 'PPR.xml'
CASA_LOG = 'casa_commands.log'
@@ -24,18 +29,22 @@ class VlaProductSet:
self.cms = self.build_cms()
def build_project(self):
- project = Project(project_code='17A-109',
- starttime=57795.8403622685,
- endtime=57892.7843396991,
- title="A detailed study of the jet-galaxy interaction in Minkowski's Object",
- abstract="Minkowski's Object is a dwarf starforming galaxy interacting with an FRI radio jet from a nearby elliptical galaxy. It has been proposed as a prototype for jet-induced star formation, a hypothesis consistent with our recent ALMA observations that map the CO 1-0 emission in the galaxy. In order to better understand the nature of the interaction, we are requesting C- and Ku-band observations at matched resolution to our ALMA and archival VLA L-band data. Faraday synthesis of the C-band polarimetric data will allow us to study the geometry of the interaction, in particular how well the relativistic electron population is mixed with the ionized gas in the star forming region. This will give us important information for our simulations of the interaction and the triggering of star formation in Minkowski's Object. In turn, this will inform on the conditions where jet-induced star formation is likely in higher redshift objects. We will use the combination of the C- and Ku-band data to investigate the variation the spectral index in the interaction region, in particular to search for shock acceleration in the interaction zone.",
- proprietary_duration=365,
- total_observation_time=0.0391880786919501,
- legacy_id='AL988',
- opt_project_id=33151323,
- last_addition='2017-08-28'
- )
- project.authors = self.build_authors(project.project_code)
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(project_code='17A-109',
+ starttime=57795.8403622685,
+ endtime=57892.7843396991,
+ title="A detailed study of the jet-galaxy interaction in Minkowski's Object",
+ abstract="Minkowski's Object is a dwarf starforming galaxy interacting with an FRI radio jet from a nearby elliptical galaxy. It has been proposed as a prototype for jet-induced star formation, a hypothesis consistent with our recent ALMA observations that map the CO 1-0 emission in the galaxy. In order to better understand the nature of the interaction, we are requesting C- and Ku-band observations at matched resolution to our ALMA and archival VLA L-band data. Faraday synthesis of the C-band polarimetric data will allow us to study the geometry of the interaction, in particular how well the relativistic electron population is mixed with the ionized gas in the star forming region. This will give us important information for our simulations of the interaction and the triggering of star formation in Minkowski's Object. In turn, this will inform on the conditions where jet-induced star formation is likely in higher redshift objects. We will use the combination of the C- and Ku-band data to investigate the variation the spectral index in the interaction region, in particular to search for shock acceleration in the interaction zone.",
+ proprietary_duration=365,
+ total_observation_time=0.0391880786919501,
+ legacy_id='AL988',
+ opt_project_id=33151323,
+ last_addition='2017-08-28'
+ )
+ project.authors = self.build_authors(project.project_code)
return project
def build_authors(self, project_code: str):
@@ -100,7 +109,7 @@ class VlaProductSet:
scheduling_block_type='OBSERVER',
ost_exec_block_id=33786546,
configuration='C',
- starttime=57892.6600277778 ,
+ starttime=57892.6600277778,
endtime=57892.7843396991,
calibration_status='Calibrated',
band_code='C X')
@@ -112,13 +121,13 @@ class VlaProductSet:
ScienceProduct(filegroup_id=18468,
external_name=
'17A-109.sb33151327.eb33496982.57795.84034732639',
- metadata_ingestion_date=EVLA_METADATA_INGEST_DATE,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
metadata_ingestion_version='1',
science_product_type=EXECUTION_BLOCK),
ScienceProduct(filegroup_id=41979,
external_name=
'17A-109.sb33151331.eb33786546.57892.65940042824',
- metadata_ingestion_date=EVLA_METADATA_INGEST_DATE,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
metadata_ingestion_version='1',
science_product_type=EXECUTION_BLOCK),
]
@@ -127,32 +136,34 @@ class VlaProductSet:
def build_basic_ms(self):
files = {
'fg_41979': [
- { 'filename': 'weblog.tgz', 'size': 1683135 },
- { 'filename': 'unknown.pprequest.xml', 'size': 5611 },
- { 'filename': 'pipeline_aquareport.xml', 'size': 1351 },
- { 'filename': PPR_FILENAME, 'size': 5611 },
- { 'filename': 'unknown.auxproducts.tgz', 'size': 615 },
- { 'filename': CASA_LOG, 'size': 1995 },
- { 'filename': MANIFEST, 'size': 809 },
- { 'filename': 'casa_pipescript.py', 'size': 823 },
+ {'filename': 'weblog.tgz', 'size': 1683135},
+ {'filename': 'unknown.pprequest.xml', 'size': 5611},
+ {'filename': 'pipeline_aquareport.xml', 'size': 1351},
+ {'filename': PPR_FILENAME, 'size': 5611},
+ {'filename': 'unknown.auxproducts.tgz', 'size': 615},
+ {'filename': CASA_LOG, 'size': 1995},
+ {'filename': MANIFEST, 'size': 809},
+ {'filename': 'casa_pipescript.py', 'size': 823},
# the tar delivered when basic MS is requested
- { 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.ms.tgz',
- 'size': 23690977245 },
+ {
+ 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.ms.tgz',
+ 'size': 23690977245},
],
'fg_18468': [
- { 'filename': 'weblog.tgz', 'size': 1681803 },
- { 'filename': 'unknown.pprequest.xml', 'size': 5610 },
- { 'filename': 'pipeline_aquareport.xml', 'size': 1351 },
- { 'filename': PPR_FILENAME, 'size': 5610 },
- { 'filename': 'unknown.auxproducts.tgz', 'size': 615 },
- { 'filename': CASA_LOG, 'size': 1995 },
- { 'filename': MANIFEST, 'size': 809 },
- { 'filename': 'casa_pipescript.py', 'size': 823 },
+ {'filename': 'weblog.tgz', 'size': 1681803},
+ {'filename': 'unknown.pprequest.xml', 'size': 5610},
+ {'filename': 'pipeline_aquareport.xml', 'size': 1351},
+ {'filename': PPR_FILENAME, 'size': 5610},
+ {'filename': 'unknown.auxproducts.tgz', 'size': 615},
+ {'filename': CASA_LOG, 'size': 1995},
+ {'filename': MANIFEST, 'size': 809},
+ {'filename': 'casa_pipescript.py', 'size': 823},
# the tar delivered when basic MS is requested
- { 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.ms.tgz',
- 'size': 23690977531 },
- ]
+ {
+ 'filename': '17A-109.sb33151327.eb33496982.57795.84034732639.ms.tgz',
+ 'size': 23690977531},
+ ]
}
return files
@@ -207,6 +218,7 @@ class VlaProductSet:
file_info[filename] = size
return file_info, total_size
+
class CalibrationProduct:
def __init__(self):
@@ -215,16 +227,20 @@ class CalibrationProduct:
self.file_info, self.total_size = self.get_file_info()
def build_project(self):
- project = Project(
- project_code='18B-265',
- legacy_id='AS1535',
- total_observation_time=0.0316695485962555,
- proprietary_duration=365,
- starttime=58745.9972239583,
- endtime=59043.2400404977,
- last_addition='2020-07-13'
- )
- project.authors = self.build_authors()
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(
+ project_code='18B-265',
+ legacy_id='AS1535',
+ total_observation_time=0.0316695485962555,
+ proprietary_duration=365,
+ starttime=58745.9972239583,
+ endtime=59043.2400404977,
+ last_addition='2020-07-13'
+ )
+ project.authors = self.build_authors()
return project
def build_authors(self):
@@ -345,16 +361,21 @@ class VlbaProductSet:
self.file_info, self.total_size = self.get_file_info()
def build_project(self):
- project = Project(project_code='BT142',
- starttime=58154.8048148148,
- endtime=58217.8342361111,
- title="Jets, outflows, and inclined disks in Seyfert galaxies: the H2O megamasers' view",
- abstract="Jets, outflows, and inclined disks in Seyfert galaxies: the H2O megamasers' view | Candidate extragalactic water masers associated with accretion disks around AGN are typically identified by a triple-peak profile of the maser spectra. However, a large number of maser sources show single-broad lines or groups of lines clustered in a narrow velocity range. In this proposal, we propose for a study of four galaxies, classified as Seyfert or LINER, hosting bright water masers and showing evidences of ejection/accretion nuclear activity. Our main goals are that of detecting the maser emission at VLBI scales, determining its location and distribution with respect to the main nuclear centers of activity, and derive the nature of the emission. These sources will potentially constitute new case studies for the scarcely-populated class of confirmed jet/outflow masers or very first examples of the exotic, recently-invoked class of 'inclined water maser disks'. Furthermore, the proposed mesurements will set the basis for a number of follow-up (VLBI) studies aim at a better understanding of the physics and disk/jet geometry in Seyferts and LINERs.",
- proprietary_duration=365,
- total_observation_time=0.0391880786919501,
- last_addition='2018-11-05'
- )
- project.authors = self.build_authors()
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(project_code='BT142',
+ starttime=58154.8048148148,
+ endtime=58217.8342361111,
+ title="Jets, outflows, and inclined disks in Seyfert galaxies: the H2O megamasers' view",
+ abstract="Jets, outflows, and inclined disks in Seyfert galaxies: the H2O megamasers' view | Candidate extragalactic water masers associated with accretion disks around AGN are typically identified by a triple-peak profile of the maser spectra. However, a large number of maser sources show single-broad lines or groups of lines clustered in a narrow velocity range. In this proposal, we propose for a study of four galaxies, classified as Seyfert or LINER, hosting bright water masers and showing evidences of ejection/accretion nuclear activity. Our main goals are that of detecting the maser emission at VLBI scales, determining its location and distribution with respect to the main nuclear centers of activity, and derive the nature of the emission. These sources will potentially constitute new case studies for the scarcely-populated class of confirmed jet/outflow masers or very first examples of the exotic, recently-invoked class of 'inclined water maser disks'. Furthermore, the proposed mesurements will set the basis for a number of follow-up (VLBI) studies aim at a better understanding of the physics and disk/jet geometry in Seyferts and LINERs.",
+ proprietary_duration=365,
+ total_observation_time=0.0391880786919501,
+ last_addition='2018-11-05'
+ )
+ project.authors = self.build_authors()
+
return project
def build_authors(self):
@@ -441,7 +462,7 @@ class VlbaProductSet:
filegroup_id=279054,
science_product_type=EXECUTION_BLOCK,
metadata_ingestion_version='1',
- metadata_ingestion_date=EVLA_METADATA_INGEST_DATE,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
external_name='BT142A2',
external_system=VLBA_OPERATIONS
),
@@ -449,7 +470,7 @@ class VlbaProductSet:
filegroup_id=278938,
science_product_type=EXECUTION_BLOCK,
metadata_ingestion_version='1',
- metadata_ingestion_date=EVLA_METADATA_INGEST_DATE,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
external_name='BT142A1',
external_system=VLBA_OPERATIONS
),
@@ -457,7 +478,7 @@ class VlbaProductSet:
filegroup_id=279119,
science_product_type=EXECUTION_BLOCK,
metadata_ingestion_version='1',
- metadata_ingestion_date=EVLA_METADATA_INGEST_DATE,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
external_name='BT142B2',
external_system=VLBA_OPERATIONS
),
@@ -465,7 +486,7 @@ class VlbaProductSet:
filegroup_id=278613,
science_product_type=EXECUTION_BLOCK,
metadata_ingestion_version='1',
- metadata_ingestion_date=EVLA_METADATA_INGEST_DATE,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
external_name='BT142B1',
external_system=VLBA_OPERATIONS
),
@@ -491,17 +512,21 @@ class VlassImageProductSet:
self.file_info, self.total_size = self.get_file_info()
def build_project(self):
- project = Project(project_code='VLASS1.1',
- title='The Very Large Array Sky Survey',
- abstract='The Very Large Array Sky Survey (VLASS) is a 5500-hr, community-driven project to survey the whole sky visible to the VLA. It will engage radio astronomy experts, multi-wavelength astronomers and citizen scientists alike. The data will be taken in three passes over the sky to allow the discovery of transient radio sources, and will cover the frequency range 2-4 GHz with an angular resolution of 2.5 arcsec. By utilizing the "on the fly" interferometry mode, the overheads will be much reduced compared to conventional survey techniques. The key science topics to be addressed by the survey are: Imaging Galaxies Through Time and Space; Hidden Explosions; Faraday Tomography of The Magnetic Sky; Peering Though Our Dusty Galaxy; and Missing Physics.',
- opt_project_id=33997662,
- starttime=58004.0162552083,
- endtime=58169.6566603472,
- total_observation_time=0.158181712955411,
- last_addition='2017-09-08',
- proprietary_duration=0
- )
- project.authors = self.build_authors()
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ project = Project(project_code='VLASS1.1',
+ title='The Very Large Array Sky Survey',
+ abstract='The Very Large Array Sky Survey (VLASS) is a 5500-hr, community-driven project to survey the whole sky visible to the VLA. It will engage radio astronomy experts, multi-wavelength astronomers and citizen scientists alike. The data will be taken in three passes over the sky to allow the discovery of transient radio sources, and will cover the frequency range 2-4 GHz with an angular resolution of 2.5 arcsec. By utilizing the "on the fly" interferometry mode, the overheads will be much reduced compared to conventional survey techniques. The key science topics to be addressed by the survey are: Imaging Galaxies Through Time and Space; Hidden Explosions; Faraday Tomography of The Magnetic Sky; Peering Though Our Dusty Galaxy; and Missing Physics.',
+ opt_project_id=33997662,
+ starttime=58004.0162552083,
+ endtime=58169.6566603472,
+ total_observation_time=0.158181712955411,
+ last_addition='2017-09-08',
+ proprietary_duration=0
+ )
+ project.authors = self.build_authors()
return project
def build_authors(self):
@@ -554,7 +579,7 @@ class VlassImageProductSet:
filegroup_id=223148,
external_name='VLASS1.1.ql.T01t01.J000232-383000.10.2048.v1',
metadata_ingestion_version='1',
- metadata_ingestion_date=EVLA_METADATA_INGEST_DATE,
+ metadata_ingestion_date=METADATA_INGESTION_DATE,
science_product_type='Image',
external_system='CASA Pipeline',
),
@@ -570,20 +595,3 @@ class VlassImageProductSet:
size = file_spec['size']
file_info[filename] = size
return file_info, total_size
-
-
-class Deliverable(Enum):
- SDM = 'SDM'
- BDF = 'BDF'
- MS = 'MS'
- CMS = 'CMS'
- IMG = 'IMG'
- # VLBA
- IDIFITS = 'IDIFITS'
-
-
-class DeliverableProduct:
-
- def __init__(self, type: Deliverable, file_info: dict):
- self.type = type
- self.file_info = file_info
diff --git a/shared/workspaces/test/test_expected_values.py b/shared/workspaces/test/test_expected_values.py
index aa51e267f..135ea27a5 100644
--- a/shared/workspaces/test/test_expected_values.py
+++ b/shared/workspaces/test/test_expected_values.py
@@ -1,18 +1,23 @@
-''' Test data: EVLA download products'''
+""" Tests for EVLA download products"""
+import pytest
+from .test_data.products.expected_values_alma import AlmaPublicProductSet
from .test_data.products.expected_values_evla import VlaProductSet, \
- VlassImageProductSet, VlbaProductSet, CalibrationProduct
-from .utilities import get_locations_report
+ VlassImageProductSet, VlbaProductSet, CalibrationProduct, PPR_FILENAME
+from .utilities import get_locations_report, Deliverable, DeliverableProduct
-
-# TODO: these location reports are duplicates of the ones in
+# FIXME: these location reports are duplicates of the ones in
# datafetcher tests. Refactor DF to use these and get rid of the DF copies
-''' Example: checking download results'''
+""" Example: checking download results"""
+
+MS_TGZ = '.ms.tgz'
+WEBLOG = 'weblog.tgz'
+
def test_gets_vla_products():
- ''' check files downloaded for our two 17A-109 EBs '''
+ """ Check test data for our two 17A-109 EBs """
product_set = VlaProductSet()
assert product_set is not None
@@ -36,7 +41,7 @@ def test_gets_vla_products():
# spot-check a few files
if filename == 'unknown.auxproducts.tgz':
assert item['size'] == 615
- if filename.startswith('17A-109') and filename.endswith('.ms.tgz'):
+ if filename.startswith('17A-109') and filename.endswith(MS_TGZ):
if filegroup.endswith('18468'):
assert item['size'] == 23690977531
elif filename.endswith('41979'):
@@ -46,7 +51,7 @@ def test_gets_vla_products():
assert item['size'] == 5610
elif filename.endswith('41979'):
assert item['size'] == 5611
- elif filename == 'weblog.tgz':
+ elif filename == WEBLOG:
if filegroup.endswith('18468'):
assert item['size'] == 1681803
elif filename.endswith('41979'):
@@ -65,7 +70,7 @@ def test_gets_vla_products():
filename = str(item['filename'])
# spot-check a few files
- if filename == 'PPR.xml':
+ if filename == PPR_FILENAME:
assert item['size'] == 1632
if filename.startswith('17A-109') and filename.endswith('.tar.tgz'):
if filegroup.endswith('18468'):
@@ -87,9 +92,9 @@ def test_gets_vla_products():
def test_gets_image_product():
- ''' Confirm that we get one image product with correct size and
+ """ Confirm one test image product with correct size and
correct project info
- '''
+ """
product_set = VlassImageProductSet()
assert product_set is not None
@@ -101,9 +106,9 @@ def test_gets_image_product():
def test_gets_vlba_products():
- ''' Confirm that we get expected number of VLBA products
+ """ Confirm that we get expected number of VLBA products
with correct size and project info
- '''
+ """
product_set = VlbaProductSet()
assert product_set is not None
@@ -115,9 +120,9 @@ def test_gets_vlba_products():
def test_gets_calibration():
- ''' Confirm that we get one calibration product with correct size and
+ """ Confirm that we get one calibration product with correct size and
correct project info
- '''
+ """
product = CalibrationProduct()
assert product is not None
@@ -130,10 +135,131 @@ def test_gets_calibration():
assert product.total_size == 27668480
+def test_gets_alma_raw_data():
+ """ Grab ALMA SDM+BDF test data and confirm we got what we expect """
+
+ product_set = AlmaPublicProductSet()
+
+ file_info = product_set.file_info
+ assert isinstance(file_info, dict)
+ assert len(file_info) == 3
+
+ file_sizes = dict()
+ delivered_products = list()
+ for file_type_str in file_info.keys():
+ values = file_info[file_type_str]
+ file_type = Deliverable.from_str(file_type_str)
+ assert file_type is not None
+ file_sizes[file_type] = file_info.values()
+ delivered_products.append(DeliverableProduct(file_type, values))
+
+ # spot-check what we got for each kind of deliverable
+ for delivery in delivered_products:
+ file_count = len(delivery.file_info)
+ if delivery.type == Deliverable.TAR:
+ assert file_count == 1
+ for filename, size in delivery.file_info.items():
+ assert filename.startswith('uid___A002_Xcd8029_Xb0a4')\
+ and filename.endswith('.tar.gz')
+ assert size == 457195765
+ elif delivery.type == Deliverable.SDM:
+ file_count = len(delivery.file_info)
+ assert file_count == 39
+ xml_count = 0
+ bin_count = 0
+ for filename, size in delivery.file_info.items():
+ if filename.endswith('.xml'):
+ xml_count += 1
+ if filename.startswith('Pointing.'):
+ assert size == 963
+ elif filename.endswith('bin'):
+ bin_count += 1
+ if filename.startswith('Pointing.'):
+ assert size == 92853427
+
+ assert xml_count == 36
+ assert bin_count == 3
+ elif delivery.type == Deliverable.BDF:
+ file_count = len(delivery.file_info)
+ assert file_count == 216
+ for file_path, size in delivery.file_info.items():
+ assert file_path.startswith('ASDMBinary')
+ assert size >= 14000
+ else:
+ pytest.fail(f'Unexpected deliverable from ALMA raw data: '
+ f'{delivery.type}')
+
+
+def test_gets_alma_basic_ms():
+ """ Analyze our ALMA basic MS test data and ensure the values are as we
+ expect
+ """
+
+ product_set = AlmaPublicProductSet()
+ basic_ms = product_set.measurement_set
+ assert isinstance(basic_ms, dict)
+ assert len(basic_ms) == 3
+
+ for name in basic_ms.keys():
+ file_info = basic_ms[name]
+ assert isinstance(file_info, dict)
+ assert len(file_info) == 10
+ weblog_found = False
+ ms_tgz_found = False
+ for filename, size in file_info.items():
+ if filename in (PPR_FILENAME,
+ 'unknown.download_alma_ms.pprequest.xml'):
+ assert size in (4582, 4584)
+ # spot-check some file sizes
+ if name == 'uid___A002_Xcd8029_Xa892':
+ if filename == WEBLOG:
+ assert size == 1592335
+ weblog_found = True
+ elif filename.endswith(MS_TGZ):
+ assert size == 519234359
+ ms_tgz_found = True
+ elif name == 'uid___A002_Xcd8029_Xb0a4':
+ if filename == 'weblog.tgz':
+ assert size == 1576475
+ weblog_found = True
+ elif filename.endswith(MS_TGZ):
+ assert size == 518582184
+ ms_tgz_found = True
+ elif name == 'uid___A002_Xcd8029_Xfdd':
+ if filename == WEBLOG:
+ assert size == 1562579
+ weblog_found = True
+ elif filename.endswith(MS_TGZ):
+ assert size == 504128830
+ ms_tgz_found = True
+ else:
+ pytest.fail(f'Unexpected item found in basic MS: {name}')
+ assert weblog_found and ms_tgz_found
+
+
+def test_gets_alma_restored_ms():
+ """ We're expecting these files inside the tar returned
+ for ALMA restored MS download of 2017.1.00297.S
+ """
+ product_set = AlmaPublicProductSet()
+ restored_ms = product_set.restored_ms
+ assert isinstance(restored_ms, dict)
+ assert len(restored_ms) == 1
+
+ for name in restored_ms.keys():
+ assert name == 'uid___A002_Xcd8029_Xb0a4'
+ values = restored_ms[name]
+ assert len(values) == 4
+ assert values['uid___A002_Xcd8029_Xb0a4.ms.tgz'] == 1325422495
+ assert values['uid___A002_Xcd8029_Xa892.ms.tgz'] == 1331336708
+ assert values['uid___A002_Xcd8029_Xfdd.ms.tgz'] == 1290775660
+ assert values[PPR_FILENAME] == 5158
+
+
def test_gets_vla_report_files():
- ''' make sure we're getting the VLA test data values we expect
+ """ make sure we're getting the VLA test data values we expect
for our 17B-109 exec blocks of interest
- '''
+ """
locations_dict = get_locations_report('17a-109_fg_18468')
assert locations_dict is not None
@@ -163,7 +289,7 @@ def test_gets_vla_report_files():
def test_gets_vlba_report_file():
- ''' make sure we're getting the VLBA test data values we expect '''
+ """ make sure we're getting the VLBA test data values we expect """
locations_dict = get_locations_report('vlba_eb')
assert locations_dict is not None
@@ -180,7 +306,7 @@ def test_gets_vlba_report_file():
def test_gets_img_report_file():
- ''' make sure we're getting the image test data values we expect '''
+ """ make sure we're getting the image test data values we expect """
locations_dict = get_locations_report('img')
assert locations_dict is not None
@@ -197,7 +323,7 @@ def test_gets_img_report_file():
def test_gets_cal_report_file():
- ''' make sure we're getting the calibration test data values we expect '''
+ """ make sure we're getting the calibration test data values we expect """
locations_dict = get_locations_report('calibration')
assert locations_dict is not None
diff --git a/shared/workspaces/test/utilities.py b/shared/workspaces/test/utilities.py
index 41439b7eb..eee30ad8e 100644
--- a/shared/workspaces/test/utilities.py
+++ b/shared/workspaces/test/utilities.py
@@ -1,19 +1,21 @@
-''' Helper functions for download product testing '''
+""" Helper functions for download product testing """
import json
+from enum import Enum
from pathlib import Path
def get_report_file(basename: str):
- ''' Get a locations file from our collection in test_data '''
+ """ Get a locations file from our collection in test_data """
for file in Path.cwd().rglob(basename.upper() + '.json'):
return file
return None
+
def get_locations_report(basename: str):
- ''' Get a locations report from a file in test_data '''
+ """ Get a locations report from a file in test_data """
report_path = get_report_file(basename)
@@ -23,3 +25,28 @@ def get_locations_report(basename: str):
return locations_report
raise FileNotFoundError(f'{basename.upper() + ".json"} not found')
+
+
+class Deliverable(Enum):
+ SDM = 'SDM'
+ BDF = 'BDF'
+ MS = 'MS'
+ CMS = 'CMS'
+ IMG = 'IMG'
+ TAR = 'TAR'
+ # VLBA
+ IDIFITS = 'IDIFITS'
+
+ @staticmethod
+ def from_str(key: str):
+ for dtype in Deliverable:
+ if dtype.value == key:
+ return dtype
+ return None
+
+
+class DeliverableProduct:
+
+ def __init__(self, dtype: Deliverable, file_info: dict):
+ self.type = dtype
+ self.file_info = file_info
--
GitLab