From 668f965843e57a138facfadfb64f331b8c9b1b8e Mon Sep 17 00:00:00 2001
From: Daniel Lyons <dlyons@nrao.edu>
Date: Fri, 30 Jul 2021 15:21:01 +0000
Subject: [PATCH] Adding ingest
---
.../executables/pexable/ingest/LICENSE.txt | 339 +
.../executables/pexable/ingest/MANIFEST.in | 5 +
apps/cli/executables/pexable/ingest/README.md | 3 +
.../cli/executables/pexable/ingest/README.txt | 11 +
.../pexable/ingest/ingest/DBURLUtil.py | 6 +
.../pexable/ingest/ingest/FITSUtil.py | 195 +
.../pexable/ingest/ingest/IDIFITSUtil.py | 251 +
.../ingest/ingest/IngestionManifest.py | 167 +
.../pexable/ingest/ingest/JSONMetadataUtil.py | 100 +
.../pexable/ingest/ingest/NGASUtil.py | 148 +
.../pexable/ingest/ingest/RealFastMetadata.py | 56 +
.../pexable/ingest/ingest/UVFITSUtil.py | 284 +
.../pexable/ingest/ingest/VLBAFITSUtil.py | 294 +
.../pexable/ingest/ingest/__init__.py | 15 +
.../pexable/ingest/ingest/_version.py | 2 +
.../pexable/ingest/ingest/almautils.py | 214 +
.../pexable/ingest/ingest/archive.py | 321 +
.../ingest/ingest/archive_caltables.py | 47 +
.../pexable/ingest/ingest/archiveutils.py | 391 +
.../pexable/ingest/ingest/commands.py | 9 +
.../ingest/ingest/evlasciencedatamodel.py | 108 +
.../pexable/ingest/ingest/files/__init__.py | 6 +
.../pexable/ingest/ingest/files/ingestobs.py | 48 +
.../pexable/ingest/ingest/files/ngasclient.py | 31 +
.../pexable/ingest/ingest/logging-config.ini | 28 +
.../pexable/ingest/ingest/manifestpersist.py | 782 ++
.../ingest/ingest/metadata/__init__.py | 10 +
.../ingest/ingest/metadata/sciencemodel.py | 55 +
.../ingest/ingest/persistVLBAmetadata.py | 317 +
.../pexable/ingest/ingest/persistcaltables.py | 339 +
.../pexable/ingest/ingest/persistimages.py | 268 +
.../pexable/ingest/ingest/persistmetadata.py | 1109 +++
.../pexable/ingest/ingest/projectutils.py | 48 +
.../pexable/ingest/ingest/proposalqueries.py | 20 +
.../pexable/ingest/ingest/proposalutils.py | 44 +
.../ingest/ingest/pymygdala/__init__.py | 9 +
.../ingest/ingest/pymygdala/commands.py | 341 +
.../pexable/ingest/ingest/pymygdala/models.py | 713 ++
.../pexable/ingest/ingest/remotecopy.py | 29 +
.../pexable/ingest/ingest/schema/__init__.py | 85 +
.../pexable/ingest/ingest/schema/almamodel.py | 6686 +++++++++++++++++
.../ingest/ingest/schema/legacy_model.py | 1728 +++++
.../pexable/ingest/ingest/schema/logs.py | 26 +
.../pexable/ingest/ingest/schema/model.py | 1137 +++
.../pexable/ingest/ingest/schema/ngasmodel.py | 50 +
.../pexable/ingest/ingest/schema/optmodel.py | 61 +
.../pexable/ingest/ingest/schema/pstmodel.py | 2135 ++++++
.../ingest/ingest/schema/vlassmodel.py | 482 ++
.../ingest/ingest/weblog_thumbs/__init__.py | 0
.../ingest/weblog_thumbs/thumbnail_finder.py | 130 +
.../ingest/ingest/weblog_thumbs/weblogUtil.py | 153 +
.../ingest/notebooks/find_missing_files.ipynb | 221 +
.../notebooks/find_project_products.ipynb | 274 +
.../ingest/notebooks/missing_ebs.ipynb | 904 +++
.../ingest/notebooks/missing_ebs_VLBA.ipynb | 1078 +++
.../ingest/notebooks/reconciliation.ipynb | 679 ++
.../pexable/ingest/pyat/__init__.py | 104 +
.../pexable/ingest/pyat/_version.py | 2 +
.../pexable/ingest/pyat/events/__init__.py | 1 +
.../pexable/ingest/pyat/events/events.py | 100 +
.../ingest/pyat/legacy/import_by_dir.py | 36 +
.../ingest/pyat/legacy/import_xml_file.py | 265 +
.../ingest/pyat/mark4_import/__init__.py | 0
.../pexable/ingest/pyat/mark4_import/audit.py | 99 +
.../ingest/pyat/mark4_import/commands.py | 148 +
.../ingest/pyat/qa_results/__init__.py | 18 +
.../ingest/pyat/qa_results/commands.py | 267 +
.../ingest/pyat/vlba_grabber/__init__.py | 0
.../pyat/vlba_grabber/ngas_retriever.py | 171 +
.../pexable/ingest/pyat/wf/__init__.py | 3 +
.../pexable/ingest/pyat/wf/commands.py | 497 ++
.../ingest/pyat/wf/ingest_wf_interfaces.py | 596 ++
.../ingest/pyat/wf/ous_wf_interfaces.py | 344 +
.../ingest/pyat/wf/utility_wf_interfaces.py | 64 +
.../pexable/ingest/requirements.txt | 28 +
.../ingest/scripts/alma-data-fetcher.sh | 32 +
apps/cli/executables/pexable/ingest/setup.py | 117 +
.../ingest/tests/functional/__init__.py | 0
.../ingest/tests/functional/test_func_one.py | 10 +
.../pexable/ingest/tests/unit/__init__.py | 0
.../ingest/tests/unit/test_eb_persistence.py | 24 +
.../tests/unit/test_ingestion_manifest.py | 77 +
.../ingest/tests/unit/test_unit_one.py | 10 +
83 files changed, 26005 insertions(+)
create mode 100644 apps/cli/executables/pexable/ingest/LICENSE.txt
create mode 100644 apps/cli/executables/pexable/ingest/MANIFEST.in
create mode 100644 apps/cli/executables/pexable/ingest/README.md
create mode 100644 apps/cli/executables/pexable/ingest/README.txt
create mode 100644 apps/cli/executables/pexable/ingest/ingest/DBURLUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/FITSUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/IDIFITSUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/IngestionManifest.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/JSONMetadataUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/NGASUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/RealFastMetadata.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/UVFITSUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/VLBAFITSUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/_version.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/almautils.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/archive.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/archive_caltables.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/archiveutils.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/commands.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/evlasciencedatamodel.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/files/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/files/ingestobs.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/files/ngasclient.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/logging-config.ini
create mode 100644 apps/cli/executables/pexable/ingest/ingest/manifestpersist.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/metadata/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/metadata/sciencemodel.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/persistVLBAmetadata.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/persistcaltables.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/persistimages.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/persistmetadata.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/projectutils.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/proposalqueries.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/proposalutils.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/pymygdala/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/pymygdala/commands.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/pymygdala/models.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/remotecopy.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/almamodel.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/legacy_model.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/logs.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/model.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/ngasmodel.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/optmodel.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/pstmodel.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/schema/vlassmodel.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/thumbnail_finder.py
create mode 100644 apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/weblogUtil.py
create mode 100644 apps/cli/executables/pexable/ingest/notebooks/find_missing_files.ipynb
create mode 100644 apps/cli/executables/pexable/ingest/notebooks/find_project_products.ipynb
create mode 100644 apps/cli/executables/pexable/ingest/notebooks/missing_ebs.ipynb
create mode 100644 apps/cli/executables/pexable/ingest/notebooks/missing_ebs_VLBA.ipynb
create mode 100644 apps/cli/executables/pexable/ingest/notebooks/reconciliation.ipynb
create mode 100644 apps/cli/executables/pexable/ingest/pyat/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/_version.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/events/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/events/events.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/legacy/import_by_dir.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/legacy/import_xml_file.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/mark4_import/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/mark4_import/audit.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/mark4_import/commands.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/qa_results/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/qa_results/commands.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/vlba_grabber/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/vlba_grabber/ngas_retriever.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/wf/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/wf/commands.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/wf/ingest_wf_interfaces.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/wf/ous_wf_interfaces.py
create mode 100644 apps/cli/executables/pexable/ingest/pyat/wf/utility_wf_interfaces.py
create mode 100644 apps/cli/executables/pexable/ingest/requirements.txt
create mode 100644 apps/cli/executables/pexable/ingest/scripts/alma-data-fetcher.sh
create mode 100644 apps/cli/executables/pexable/ingest/setup.py
create mode 100644 apps/cli/executables/pexable/ingest/tests/functional/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/tests/functional/test_func_one.py
create mode 100644 apps/cli/executables/pexable/ingest/tests/unit/__init__.py
create mode 100644 apps/cli/executables/pexable/ingest/tests/unit/test_eb_persistence.py
create mode 100644 apps/cli/executables/pexable/ingest/tests/unit/test_ingestion_manifest.py
create mode 100644 apps/cli/executables/pexable/ingest/tests/unit/test_unit_one.py
diff --git a/apps/cli/executables/pexable/ingest/LICENSE.txt b/apps/cli/executables/pexable/ingest/LICENSE.txt
new file mode 100644
index 000000000..d159169d1
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/LICENSE.txt
@@ -0,0 +1,339 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/apps/cli/executables/pexable/ingest/MANIFEST.in b/apps/cli/executables/pexable/ingest/MANIFEST.in
new file mode 100644
index 000000000..3a9ac50f8
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/MANIFEST.in
@@ -0,0 +1,5 @@
+include *.txt *.ini *.cfg *.rst *.wsgi *.sh *.spec *.md
+include pyat/ingestion/logging-config.ini
+include MANIFEST.in
+include README.txt
+include scripts/*
diff --git a/apps/cli/executables/pexable/ingest/README.md b/apps/cli/executables/pexable/ingest/README.md
new file mode 100644
index 000000000..e9c1d36aa
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/README.md
@@ -0,0 +1,3 @@
+# Ingest
+
+Ingest is the program that ingests data into the archive.
diff --git a/apps/cli/executables/pexable/ingest/README.txt b/apps/cli/executables/pexable/ingest/README.txt
new file mode 100644
index 000000000..45644e20f
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/README.txt
@@ -0,0 +1,11 @@
+# PyAT: The Python Archive Tools
+
+This is a collection of tools for internal use by the AAT-PPI system, as
+well as libraries and commands for debugging and remote job control. It consists
+of the following pieces:
+
+ - *pymygdala*: archive messaging
+ - *wf*: triggering workflows
+ - *epilogue*: messaging at the conclusion of job executions by qsub
+ - *mrclean*: workflow cleanup
+
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/ingest/DBURLUtil.py b/apps/cli/executables/pexable/ingest/ingest/DBURLUtil.py
new file mode 100644
index 000000000..a33ae5090
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/DBURLUtil.py
@@ -0,0 +1,6 @@
+# jdbc:mysql://my.nrao.edu:3306/nrao_200
+# mysql://pst_ro:XXXXXXXX@my.nrao.edu/nrao_200
+
+
+def getdburlfromjdbc(jdbcurl, username, password):
+ return jdbcurl.replace('jdbc:', '').replace('://', '://' + username + ':' + password + '@')
diff --git a/apps/cli/executables/pexable/ingest/ingest/FITSUtil.py b/apps/cli/executables/pexable/ingest/ingest/FITSUtil.py
new file mode 100644
index 000000000..d0de0a250
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/FITSUtil.py
@@ -0,0 +1,195 @@
+import logging
+import math
+import numpy as np
+from astropy.io import fits
+from astropy.table import Table
+from astropy import units as un
+from . import archiveutils as au
+
+"""
+ Utility module for accessing FITS data
+"""
+
+logger = logging.getLogger('ingestion')
+
+polarizations = {1 : 'I', 2 : 'Q', 3 : 'U', 4 : 'V'}
+
+
+def get_fits_data(filename):
+ """
+ Get FITS header data
+ :param filename: name of the FITS file.
+ :return: target name, ra, dec, image units, min frequency, max frequency
+ """
+
+ hdu = fits.open(filename)
+ header = hdu['PRIMARY'].header
+ image_data = hdu[0].data
+ image_table = Table(image_data)
+ image_array = image_table[0][0]
+
+ source_name = get_source_name(header)
+ ra = get_ra(header)
+ dec = get_dec(header)
+ image_field_of_view = get_image_field_of_view(header)
+
+ ra_element_count = get_ra_element_count(header)
+ dec_element_count = get_dec_element_count(header)
+ spatial_resolution = get_spatial_resolution(header)
+ beam_axis_ratio = get_beam_axis_ratio(header)
+
+ min_frequency = get_min_frequency(header)
+ max_frequency = get_max_frequency(header)
+
+ rest_frequency = get_rest_frequency(header)
+ polarization_id = get_polarization_id(header)
+
+ telescope = get_telescope(header)
+ image_units = get_image_units(header)
+ min_intensity = get_min_intensity(image_array)
+ max_intensity = get_max_intensity(image_array)
+ rms_noise = get_rms_noise(image_array)
+
+ spatial_region = get_spatial_region(header)
+
+ ra_pixel_size = get_ra_pixel_size(header)
+ dec_pixel_size = get_dec_pixel_size(header)
+
+ return source_name, telescope, spatial_resolution, image_field_of_view, min_intensity, max_intensity, ra, dec, min_frequency, max_frequency,\
+ rest_frequency, ra_element_count, dec_element_count, image_units, beam_axis_ratio, polarization_id, rms_noise, spatial_region,\
+ ra_pixel_size, dec_pixel_size
+
+
+def get_source_name(header):
+ return header['OBJECT']
+
+
+def get_ra(header):
+ if header['CTYPE1'] == 'RA---SIN':
+ return header['CRVAL1']
+ else:
+ return 0
+
+
+def get_dec(header):
+ if header['CTYPE2'] == 'DEC--SIN':
+ return header['CRVAL2']
+ else:
+ return 0
+
+
+def get_image_field_of_view(header):
+ return math.sqrt((header['NAXIS1'] * header['CDELT1']) ** 2 + (header['NAXIS2'] * header['CDELT2']) ** 2)
+
+
+def get_ra_element_count(header):
+ if header['CTYPE1'] == 'RA---SIN':
+ return header['NAXIS1']
+ else:
+ return 0
+
+
+def get_dec_element_count(header):
+ if header['CTYPE2'] == 'DEC--SIN':
+ return header['NAXIS2']
+ else:
+ return 0
+
+
+def get_spatial_resolution(header):
+ """
+ :param header:
+ :return: spatial resolution in arcsec
+ """
+ spatial_resolution_degrees = (math.sqrt(header['BMAJ'] * header['BMIN'])) * un.degree
+ return spatial_resolution_degrees.to(un.arcsec).value
+
+
+def get_min_frequency(header):
+ if header['CTYPE3'] == 'FREQ':
+ return header['CRVAL3'] - header['CDELT3'] / 2
+ else:
+ return 0
+
+
+def get_max_frequency(header):
+ if header['CTYPE3'] == 'FREQ':
+ return header['CRVAL3'] + header['CDELT3'] / 2
+ else:
+ return 0
+
+
+def get_rest_frequency(header):
+ return header['RESTFRQ']
+
+
+def get_beam_axis_ratio(header):
+ return header['BMAJ'] / header['BMIN']
+
+
+def get_polarization_id(header):
+ if header['CTYPE4'] == 'STOKES':
+ fits_stokes = round(header['CRVAL4'])
+ polarization_name = polarizations[fits_stokes]
+ return au.getpolarizationcode(polarization_name)
+ else:
+ return 0
+
+
+def get_telescope(header):
+ return header['TELESCOP']
+
+
+def get_image_units(header):
+ return '{} {}'.format(header['BTYPE'], header['BUNIT'])
+
+
+def get_min_intensity(image_array):
+ return np.nanmin(image_array).astype(np.float64)
+
+
+def get_max_intensity(image_array):
+ return np.nanmax(image_array).astype(np.float64)
+
+
+# TODO
+def get_rms_noise(image_array):
+ return 0
+
+
+def get_spatial_region(header):
+ # BOX IRCS {} {} {} {}".format(ra, dec, ra_extent, dec_extent)
+ if header['CTYPE1'] == 'RA---SIN':
+ ra_extent = abs(header['NAXIS1'] * header['CDELT1'])
+ else:
+ ra_extent = 0
+
+ if header['CTYPE2'] == 'DEC--SIN':
+ dec_extent = abs(header['NAXIS2'] * header['CDELT2'])
+ else:
+ dec_extent = 0
+
+ return 'BOX IRCS {} {} {} {}'.format(get_ra(header), get_dec(header), ra_extent, dec_extent)
+
+
+def get_ra_pixel_size(header):
+ if header['CTYPE1'] == 'RA---SIN':
+ return abs(header['CDELT1'])
+ else:
+ return 0
+
+
+def get_dec_pixel_size(header):
+ if header['CTYPE2'] == 'DEC--SIN':
+ return abs(header['CDELT2'])
+ else:
+ return 0
+
+
+
+
+
+
+
+
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/IDIFITSUtil.py b/apps/cli/executables/pexable/ingest/ingest/IDIFITSUtil.py
new file mode 100644
index 000000000..22256ade9
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/IDIFITSUtil.py
@@ -0,0 +1,251 @@
+import os
+import sys
+import re
+import logging
+
+import numpy as np
+from astropy.io import fits
+from astropy.table import Table
+from astropy.time import Time, TimeDelta
+from astropy.coordinates import SkyCoord
+from . import archiveutils as au
+
+"""
+Represents a VLBA IDIFITS file
+"""
+
+LOG = logging.getLogger('ingestion')
+
+class IDIFits:
+ def __init__(self, file):
+ self.idifitsfile = file
+ self.hdu = fits.open(file)
+ self.source_table = Table(self.hdu['SOURCE'].data)
+ self.uv_data = self.hdu['UV_DATA'].data
+ self.frequency_data = self.hdu['FREQUENCY'].data
+ self.frequency_table = Table(self.hdu['FREQUENCY'].data)
+ self.reference_frequency = self.hdu['FREQUENCY'].header['REF_FREQ']
+ self.numbands = self.hdu['FREQUENCY'].header['NO_BAND']
+ # self.spectchannels = self.hdu['FREQUENCY'].header['NO_CHAN']
+ self.scans = self.build_scans(self.uv_data)
+ self.hdu.close()
+
+ def getfitsfile(self):
+ return self.idifitsfile
+
+ def getproject(self):
+ m = re.match('([A-Z]+\d+)', self.hdu['PRIMARY'].header['OBSERVER'])
+ return m.group(0)
+
+ def getexperiment(self):
+ m = re.match('[A-Z]+\d+([A-Z|\d]+)', self.hdu['PRIMARY'].header['OBSERVER'])
+ if m is None:
+ return ""
+ return m.group(1)
+
+ def getcorrelatorpass(self):
+ fflist = os.path.basename(self.getfitsfile()).split('_')
+ return fflist[2]
+
+ def getsourcetable(self):
+ return self.source_table
+
+ def getsource(self, source):
+ """
+ Return source name, RA, and DEC of scan source
+ :param source: the source index of the scan source
+ :return: source name, ra, dec
+ """
+ sourcekey = self.hdu['SOURCE'].data.columns[0].name
+ source_row = self.source_table[self.source_table[sourcekey] == source]
+ # Retrieving a column with 1 element so extract the value
+ # return source_row[0]['SOURCE'], source_row[0]['RAOBS'], source_row[0]['DECOBS']
+ return source_row[0]['SOURCE'], source_row[0]['RAAPP'], source_row[0]['DECAPP']
+
+ def getfrequency(self, frequency):
+ frequency_row = self.frequency_table[self.frequency_table['FREQID'] == frequency]
+ frequencies = frequency_row[0]['BANDFREQ']
+ if np.isscalar(frequencies):
+ # If the frequency is a scalar, put it in a numpy ndarray
+ freq_array = np.ndarray(shape=(1))
+ freq_array[0] = frequency_row[0]['BANDFREQ']
+ return freq_array
+ return frequencies
+
+ def getbandwidth(self, frequency):
+ frequency_row = self.frequency_table[self.frequency_table['FREQID'] == frequency]
+ if np.isscalar(frequency_row[0]['TOTAL_BANDWIDTH']):
+ return frequency_row[0]['TOTAL_BANDWIDTH']
+ else:
+ return frequency_row[0]['TOTAL_BANDWIDTH'][0]
+
+ def getfrequencytable(self):
+ return self.frequency_table
+
+ def get_number_bands(self):
+ return self.numbands
+
+ # def get_num_spectral_channels(self):
+ # return self.spectchannels
+
+ def build_scans(self, uv_data):
+ if len(uv_data) <= 0:
+ LOG.critical(f"UV_DATA table is empty for VLBA file: {self.idifitsfile}")
+ sys.exit(1)
+ sourcekey = self.hdu['SOURCE'].data.columns[0].name
+ scan_list = []
+ polarizations = self.getpolarizations(self.hdu['UV_DATA'].header)
+
+ source = uv_data[0]['SOURCE']
+ scan = 0
+ scan_start_row = 0
+ frequency_ids = [uv_data[scan_start_row]['FREQID']]
+ for row in range(len(uv_data)):
+ if source != uv_data[row]['SOURCE']:
+ scan_end_row = row - 1
+ scan_start_time = Time(uv_data[scan_start_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_start_row]['TIME'], format='jd')
+ scan_end_time = Time(uv_data[scan_end_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_end_row]['TIME'], format='jd')
+ scan_time_on_source = scan_end_time - scan_start_time
+ scan_integration_time = uv_data[scan_start_row]['INTTIM']
+ scan_source = self.source_table[self.source_table[sourcekey] == uv_data[scan_start_row]['SOURCE']]['SOURCE'][0]
+ source_ra = self.source_table[self.source_table[sourcekey] == uv_data[scan_start_row]['SOURCE']]['RAAPP'][0]
+ source_dec = self.source_table[self.source_table[sourcekey] == uv_data[scan_start_row]['SOURCE']]['DECAPP'][0]
+ # c = SkyCoord(source_ra, source_dec, frame='icrs', unit='deg')
+ # source_radec = c.to_string('hmsdms')
+ source_radec = ''
+ frequencies = [(self.get_reference_frequency() + self.getfrequency(f)) for f in frequency_ids]
+ frequency_list = np.concatenate(frequencies, axis=0) / 1000000
+ total_bandwidth = self.getbandwidth(1)
+ current_scan = [scan, scan_source, scan_start_time.mjd, scan_end_time.mjd, scan_time_on_source.sec, scan_integration_time, ' '.join(map(str, frequency_list)), total_bandwidth, source_ra, source_dec, source_radec, polarizations]
+ scan_list.append(current_scan)
+ scan = scan + 1
+ scan_start_row = row
+ frequency_ids.clear()
+
+ frequency_id = uv_data[row]['FREQID']
+ if frequency_id not in frequency_ids:
+ frequency_ids.append(frequency_id)
+
+ source = uv_data[row]['SOURCE']
+
+ # Add in final scan
+ scan_end_row = row
+ scan_start_time = Time(uv_data[scan_start_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_start_row]['TIME'], format='jd')
+ scan_end_time = Time(uv_data[scan_end_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_end_row]['TIME'], format='jd')
+ scan_time_on_source = scan_end_time - scan_start_time
+ scan_integration_time = uv_data[scan_start_row]['INTTIM']
+ scan_source = self.source_table[self.source_table[sourcekey] == uv_data[scan_start_row]['SOURCE']]['SOURCE'][0]
+ source_ra = self.source_table[self.source_table[sourcekey] == uv_data[scan_start_row]['SOURCE']]['RAAPP'][0]
+ source_dec = self.source_table[self.source_table[sourcekey] == uv_data[scan_start_row]['SOURCE']]['DECAPP'][0]
+ # c = SkyCoord(source_ra, source_dec, frame='icrs', unit='deg')
+ # source_radec = c.to_string('hmsdms')
+ source_radec = ''
+ frequencies = [(self.get_reference_frequency() + self.getfrequency(f)) for f in frequency_ids]
+ frequency_list = np.concatenate(frequencies, axis=0) / 1000000
+ total_bandwidth = self.getbandwidth(1)
+ current_scan = [scan, scan_source, scan_start_time.mjd, scan_end_time.mjd, scan_time_on_source.sec, scan_integration_time, ' '.join(map(str, frequency_list)), total_bandwidth, source_ra, source_dec, source_radec, polarizations]
+ scan_list.append(current_scan)
+
+ # Build scan table from list of scans
+ return Table(rows=scan_list, names=('SCAN_ID', 'SOURCE', 'START_TIME', 'END_TIME', 'TIME_ON_SOURCE', 'INTEGRATION_TIME', 'FREQUENCY', 'BANDWIDTH', 'RA', 'DEC', 'RA/DEC', 'POLARIZATION'))
+
+ def get_reference_frequency(self):
+ return self.reference_frequency
+
+ def getpolarizations(self, uv_header):
+ numpolarizations = uv_header['NO_STKD']
+ polarization_code = 0
+ for i in range(numpolarizations):
+ pol_key = f"STK_{i+1}"
+ if pol_key not in uv_header.keys():
+ LOG.warning(f"Stokes key: {pol_key} is not present in UV_DATA table")
+ continue
+ polarization = self.lookup_polarization(uv_header[pol_key])
+ polarization_code = polarization_code + polarization
+ return au.getpolarization(polarization_code)
+
+ def lookup_polarization(self, polarization_code):
+ # Map of polarization codes used by FITS to observation polarization codes
+ polarization_codes = {
+ 1: 'I',
+ 2: 'Q',
+ 3: 'U',
+ 4: 'V',
+ -1: 'RR',
+ -2: 'LL',
+ -3: 'RL',
+ -4: 'LR',
+ -5: 'XX',
+ -6: 'YY',
+ -7: 'XY',
+ -8: 'YX'
+ }
+
+ if polarization_code not in polarization_codes.keys():
+ LOG.error(f"Invalid polarization code: {polarization_code}")
+ sys.exit(1)
+
+ return au.getpolarizationcode(polarization_codes[polarization_code])
+
+ def getuvdatacolumns(self):
+ return self.uv_data.columns
+
+ def gethdulist(self):
+ return self.hdu.info()
+
+ def getheader(self):
+ return self.hdu['PRIMARY'].header
+
+ def getobservingband(self):
+ ref_frequency = self.get_reference_frequency()
+ return self.lookup_observing_band(ref_frequency)
+
+ def getfrequencyoffset(self, freq_id):
+ frequency_row = self.frequency_table[self.frequency_table['FREQID'] == freq_id]
+ return frequency_row['BANDFREQ'][0][1]
+
+ def getbandwidthoffset(self, freq_id):
+ frequency_row = self.frequency_table[self.frequency_table['FREQID'] == freq_id]
+ return frequency_row['CH_WIDTH'][0][1]
+
+ def lookup_observing_band(self, reference_frequency):
+ ref_freq = reference_frequency / 1000000000
+ obs_band = ''
+
+ if 0.058 <= ref_freq <= 0.084:
+ obs_band = '4'
+ elif 0.236 <= ref_freq <= 0.492:
+ obs_band = 'P'
+ elif 1.0 <= ref_freq <= 2.0:
+ obs_band = 'L'
+ elif 2.0 < ref_freq <= 4.0:
+ obs_band = 'S'
+ elif 4.0 < ref_freq <= 8.0:
+ obs_band = 'C'
+ elif 8.0 < ref_freq <= 12.0:
+ obs_band = 'X'
+ elif 12.0 < ref_freq <= 18.0:
+ obs_band = 'Ku'
+ elif 18.0 < ref_freq <= 26.5:
+ obs_band = 'K'
+ elif 26.5 < ref_freq <= 40.0:
+ obs_band = 'Ka'
+ elif 40.0 < ref_freq <= 50.0:
+ obs_band = 'Q'
+ else:
+ LOG.warning(f"Reference frequency {ref_freq} could not be located in a band")
+
+ return obs_band
+
+ def dump_scans(self):
+ self.scans.pprint(max_lines=-1, max_width=-1)
+
+
+def test_1(file):
+ pass
+
+
+if __name__ == '__main__':
+ print("Main")
+ # test_1(os.path.join(vlba_stage, 'VLBA_BS263E_bs263e_BIN0_SRC0_0_180716T144209.idifits'))
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/IngestionManifest.py b/apps/cli/executables/pexable/ingest/ingest/IngestionManifest.py
new file mode 100644
index 000000000..af79feafd
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/IngestionManifest.py
@@ -0,0 +1,167 @@
+import sys
+import logging
+import simplejson as json
+from distutils.util import strtobool
+
+
+"""
+ Utility module for accessing Ingestion Manifest
+"""
+
+LOG = logging.getLogger('ingestion')
+
+INGESTION_MANIFEST_FILE = "ingestion-manifest.json"
+CALIBRATION = "calibration"
+EXECUTION_BLOCK = "execution_block"
+NAASC_CALIBRATION = "naasc_calibration"
+IMAGE = "image"
+CONTINUUM_IMAGE = "continuum_image"
+IMAGE_CUBE = "image_cube"
+
+
+class IngestionManifest:
+
+ def __init__(self, file):
+ self.from_json(file)
+
+ def to_json(self, file):
+ """
+ Save object in JSON file
+ :param file: file to store object
+ :return: None
+ """
+ with open(file, 'w') as manifest_file:
+ return json.dump(self, manifest_file, default=lambda o: o.__dict__,
+ sort_keys=True, indent=4)
+
+ def from_json(self, file):
+ """
+ Load object from JSON file
+ :param file: ingestion manifest file
+ :return: None
+ """
+ with open(file, 'r') as manifest_file:
+ self.__dict__ = json.load(manifest_file)
+
+ def get_parameters(self):
+ return self.parameters
+
+ def get_telescope(self):
+ return self.parameters['telescope']
+
+ def get_operation(self):
+ return self.output_group['science_products'][0]['type']
+
+ def get_path(self):
+ return self.parameters['ingestion_path']
+
+ def get_reingest(self):
+ return self.parameters['reingest']
+
+ def get_ngas_ingest(self):
+ return self.parameters['ngas_ingest']
+
+ def get_additional_metadata_file(self):
+ if 'additional_metadata' in self.__dict__['parameters']:
+ return self.get_parameters()['additional_metadata']
+
+ def get_collection_metadata_file(self):
+ if self.has_collection_metadata_file():
+ return self.get_parameters()['collection_metadata']
+
+ def has_collection_metadata_file(self):
+ return 'collection_metadata' in self.__dict__['parameters']
+
+ # Calibration parameters
+
+ def get_tar_filename(self):
+ return self.output_group['science_products'][0]['filename']
+
+ # Deprecated
+ def get_execution_block_science_product_locator(self):
+ return self.input_group['science_products'][0]['locator']
+
+ def get_execution_block_science_product_locators(self):
+ # Return list of eb science product locators for ebs used in the calibration (same mous)
+ return [input_science_product['locator'] for input_science_product in self.input_group['science_products']]
+
+ def get_input_group(self):
+ if 'input_group' not in self.__dict__:
+ return None
+ return self.input_group
+
+ def get_output_group(self):
+ return self.output_group
+
+ def get_associate_group(self):
+ if 'associate_group' not in self.__dict__:
+ return None
+ return self.associate_group
+
+ def get_group_science_products(self, group):
+ return group['science_products']
+
+ def get_input_group_science_products(self):
+ return self.get_input_group()['science_products']
+
+ def get_output_group_science_products(self):
+ return self.get_output_group()['science_products']
+
+ def get_output_group_ancillary_products(self):
+ return self.get_output_group()['ancillary_products']
+
+ def get_associate_group_science_products(self):
+ return self.get_associate_group()['science_products']
+
+ def get_associate_group_locator(self):
+ return self.get_associate_group_science_products()[0]['locator']
+
+ def get_endtime(self):
+ return self.endtime
+
+ def get_calibration_ingestion_parameters(self):
+ """
+ :return: path to calibration tar file, calibration tar filename,
+ execution_block_science_prduct_locator, reingest, ngas_ingest
+ """
+
+ if self.get_operation() == CALIBRATION:
+ return self.get_path(), self.get_tar_filename(), self.get_execution_block_science_product_locators(), \
+ self.get_reingest(), self.get_ngas_ingest()
+ elif self.get_operation == NAASC_CALIBRATION:
+ return self.get_execution_block_science_product_locators(), self.get_reingest(), self.get_ngas_ingest()
+ else:
+ LOG.critical(f"Method only defined for calibration ingestion")
+ sys.exit(1)
+
+ def get_ingestion_parameters(self):
+ # returning telescope, operation, input product groups, output product groups
+ return self.get_telescope(), self.get_operation(), self.get_input_group(), self.get_output_group(),\
+ self.get_reingest(), self.get_ngas_ingest(), self.get_path()
+
+
+ def get_ingestion_parameters2(self):
+ # returning telescope, operation, reingest, ngas_ingest, path
+ return self.get_telescope(), self.get_operation(), \
+ self.get_reingest(), self.get_ngas_ingest(), self.get_path(), \
+ self.get_additional_metadata_file(), \
+ self.get_collection_metadata_file(),
+
+
+
+def test1():
+ ingestion_manifest = IngestionManifest(INGESTION_MANIFEST_FILE)
+ print(ingestion_manifest.get_input_group())
+ print(ingestion_manifest.get_output_group())
+
+ for science_product in ingestion_manifest.get_input_group_science_products():
+ print(f"Type: {science_product['type']} Locator: {science_product['locator']}")
+
+ for science_product in ingestion_manifest.get_output_group_science_products():
+ print(f"Type: {science_product['type']} File: {science_product['filename']}")
+
+ for ancillary_product in ingestion_manifest.get_output_group_ancillary_products():
+ print(f"Type: {ancillary_product['type']} File: {ancillary_product['filename']}")
+
+if __name__ == '__main__':
+ test1()
diff --git a/apps/cli/executables/pexable/ingest/ingest/JSONMetadataUtil.py b/apps/cli/executables/pexable/ingest/ingest/JSONMetadataUtil.py
new file mode 100644
index 000000000..50046e1f1
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/JSONMetadataUtil.py
@@ -0,0 +1,100 @@
+import logging
+import json
+
+"""
+ Utility module for accessing JSON metadata
+"""
+
+logger = logging.getLogger('ingestion')
+
+
+class JSONMetadata:
+
+ def __init__(self, filename):
+ self._source_file = filename
+ self.from_json(filename)
+
+ def to_json(self, filename):
+ """
+ Save object in JSON file
+ :param filename: file to store object
+ :return: None
+ """
+ with open(filename, 'w') as metadata_file:
+ return json.dump(self, metadata_file, default=lambda o: o.__dict__,
+ sort_keys=True, indent=4)
+
+ def from_json(self, filename):
+ """
+ Load object from JSON file
+ :param filename: file name of JSON file containing object attributes
+ :return: None
+ """
+ try:
+ with open(filename, 'r') as metadata_file:
+ self.__dict__ = json.load(metadata_file)
+ except IOError:
+ logger.critical('Error processing file: {}'.format(filename))
+ exit(-2)
+
+ # Field accessors
+
+ def get_collection_name(self):
+ return self.collection_name
+
+ def get_calibration_level(self):
+ return self.calibration_level
+
+ def get_starttime(self):
+ return self.starttime
+
+ def get_endtime(self):
+ return self.endtime
+
+ def get_exposure_time(self):
+ return self.exposure_time
+
+ def get_image_tags(self):
+ return self.image_tags
+
+ def get_product_tags(self):
+ return self.product_tags
+
+ def get_configurations(self):
+ return str(self.configurations)
+
+ def get_band_code(self):
+ return self.band_code
+
+ def get_project_code(self):
+ return self.project_code
+
+ def get_rms_noise(self):
+ return self.rms_noise
+
+ def isRealFast(self):
+ return self.get_collection_name() == 'realfast_execution_blocks'
+
+
+def test1():
+ filename = 'metadata1.json'
+ o1 = JSONMetadata()
+ o2 = JSONMetadata()
+ o1.collection_name = "vlass"
+ o1.calibration_level = 2
+ o1.starttime = None
+ o1.endtime = None
+ o1.exposure_time = None
+ o1.image_tags = ""
+ o1.product_tags = ""
+ o1.configurations = str(["VLA_B"])
+
+ o1.to_json(filename)
+ o2.from_json(filename)
+ # print(o2.__dict__)
+ assert o1.get_collection_name() == o2.get_collection_name() and o1.get_calibration_level() == o2.get_calibration_level() and \
+ o1.get_starttime() == o2.get_starttime() and o1.get_endtime() == o2.get_endtime() and o1.get_exposure_time() == o2.get_exposure_time() and \
+ o1.get_image_tags() == o2.get_image_tags() and o1.get_product_tags() == o2.get_product_tags() and o1.get_configurations() == o2.get_configurations()
+
+if __name__ == '__main__':
+ test1()
diff --git a/apps/cli/executables/pexable/ingest/ingest/NGASUtil.py b/apps/cli/executables/pexable/ingest/ingest/NGASUtil.py
new file mode 100644
index 000000000..947a835f1
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/NGASUtil.py
@@ -0,0 +1,148 @@
+# NGAS Utility
+
+import os
+import shutil
+import requests
+import jxmlease
+import time
+from pycapo import CapoConfig
+import random
+import uuid
+from urllib.parse import quote_plus
+import logging
+import hashlib
+
+from . import evlasciencedatamodel as sc
+
+capoprofile = os.environ['CAPO_PROFILE']
+config = CapoConfig(profile=capoprofile)
+# Retrieve NGAS host information
+NGASHOSTS = [ x.strip() for x in config.getstring('archive-ingestion.NGASHosts').split(' ') ]
+archivecommand = {'status': 'STATUS', 'archive': 'ARCHIVE', 'retrieve': 'RETRIEVE'}
+LOG = logging.getLogger('ingestion')
+
+
+def random_ngas_host():
+ return random.choice(NGASHOSTS)
+
+
+def ingestevlasdmsintongas(filesetbasepath, bdfpath):
+ filesets = []
+ for fileset in filesets():
+ filesetpath = os.path.join(filesetbasepath, fileset)
+ print('Processing fileset: '.format(fileset))
+
+
+def ingestsdmintongas(sdm, sdmpath, bdfpath, ngas_staging):
+ # Copy SDM table files
+ filelist = []
+ for sdmtable in sdm.sdmtables:
+ table = sdm.sdmtables[sdmtable]
+ if isinstance(sdm.sdmtables[sdmtable], sc.SDMXMLTable):
+ fromextension = '.xml'
+ toextension = '.sdm'
+ elif isinstance(sdm.sdmtables[sdmtable], sc.SDMBinaryTable):
+ fromextension = '.bin'
+ toextension = '.bin'
+ else:
+ LOG.critical('Unrecognizable SDM table type')
+ exit(-1)
+
+ fromfile = os.path.join(sdmpath, table.tablename + fromextension)
+ tofilename = table.entityid.replace(':', '_').replace('/', '_') + toextension
+ tofile = os.path.join(ngas_staging, tofilename)
+ LOG.info(f"SDM ingestion copying from {fromfile} to {tofile}")
+ shutil.copy(fromfile, tofile)
+ filelist.append((tofile, os.stat(tofile).st_size))
+
+ # Link BDF files
+ for bdffile in sdm.bdffiles:
+ extension = '.bdf'
+ filename = bdffile.entityid.replace(':', '_').replace('/', '_')
+ # fromfile = os.path.join(sdmpath, filename) + extension
+ fromfile = os.path.join(bdfpath, filename)
+ tofile = os.path.join(ngas_staging, filename + extension)
+ # TODO don't copy bdfs yet - disk space
+ LOG.info(f"BDF ingestion linking from {fromfile} to {tofile}")
+ os.link(fromfile, tofile)
+ filelist.append((tofile, os.stat(tofile).st_size))
+
+ if len(filelist) > 0:
+ archivefiles(filelist)
+ LOG.info(f"NGAS file list: {filelist}")
+ else:
+ logger.warning('No SDM tables or BDF files to process.')
+
+
+def generate_science_product_locator(telescope, filetype):
+ """
+ Generate unique science product locator for science products
+ :param telescope: the telescope
+ :param filetype: the type of the science product
+ :return: science product locator
+ """
+
+ id = uuid.uuid4()
+ science_product_locator = f"uid://{telescope.lower()}/{filetype}/{id}"
+ return science_product_locator
+
+
+def generate_uuids_for_ngas(telescope, filetype, extension):
+ """
+ Generate unique uuids for ngas storage
+ :param filename: the current name of the file
+ :return: new file name
+ """
+
+ id = uuid.uuid4()
+ file = f"uid____{telescope}_{filetype}_{id}.{extension}"
+ return file
+
+
+def archivefiles(archivefilelist):
+ filecount = 0
+ for archivefile in archivefilelist:
+ # archivefile 0:filename 1: file size on disk
+ file_name = archivefile[0]
+ expected_file_size = archivefile[1]
+ # And away we go:
+ starttime = time.time()
+ NGASHOST = random_ngas_host()
+ filecount += 1
+ LOG.info(f"Ingesting: {file_name}")
+ ngascmd = NGASHOST + archivecommand['archive'] + '?filename=file://' + quote_plus(file_name)
+ LOG.info(ngascmd)
+ response = requests.get(ngascmd)
+ returncode = response.status_code
+ archivemsg = 'Archive request ({}): {} [{}] in {}s'.format(filecount, ngascmd, returncode, time.time() - starttime) # fix units
+ LOG.info(archivemsg)
+ doc = jxmlease.parse(response.text)
+ if returncode == 200:
+ ngas_file_size = int(doc['NgamsStatus']['DiskStatus']['FileStatus'].get_xml_attr('FileSize'))
+ LOG.debug(f"Ingested file size: {ngas_file_size} Expected file size: {expected_file_size}")
+ if ngas_file_size < expected_file_size:
+ LOG.error(f"NGAS ingested file size {ngas_file_size} indicates incomplete ingestion (expected file size {expected_file_size})")
+ exit(-1)
+ elif ngas_file_size > expected_file_size:
+ LOG.info(f"NGAS ingested file size {ngas_file_size} greater than expected {expected_file_size}. Weird, but continuing.")
+ # If they're equal, we're quite happy.
+ else:
+ error_message = doc['NgamsStatus']['Status'].get_xml_attr('Message')
+ LOG.error(f"NGAS ingestion of {archivefile[0]} failed. Return code: {returncode} Message: {error_message}")
+ exit(-1)
+
+
+def sha256(fname):
+ hash_sha256 = hashlib.sha256()
+ with open(fname, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hash_sha256.update(chunk)
+ return hash_sha256.hexdigest()
+
+
+def test_uuid(telescope, filetype, extension):
+ print(generate_uuids_for_ngas(telescope, filetype, extension))
+
+
+if __name__ == '__main__':
+ generate_science_product_locator('EVLA', 'execblock')
diff --git a/apps/cli/executables/pexable/ingest/ingest/RealFastMetadata.py b/apps/cli/executables/pexable/ingest/ingest/RealFastMetadata.py
new file mode 100644
index 000000000..ccd9c6f5b
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/RealFastMetadata.py
@@ -0,0 +1,56 @@
+"""
+Class for holding Realfast Metadata
+"""
+class RealFastMetadata():
+ def __init__(self, json_metadata):
+ self._json_metadata = json_metadata
+
+ def get_transient_dm(self):
+ return self._json_metadata.transient_DM
+
+ def get_portal_candidate_ids(self):
+ return self._json_metadata.portal_candidate_IDs
+
+ def get_transient_ra(self):
+ return self._json_metadata.transient_RA
+
+ def get_transient_ra_error(self):
+ return self._json_metadata.transient_RA_error
+
+ def get_transient_dec(self):
+ return self._json_metadata.transient_Dec
+
+ def get_transient_dec_error(self):
+ return self._json_metadata.transient_Dec_error
+
+ def get_transient_snr(self):
+ return self._json_metadata.transient_SNR
+
+ def get_transient_dm(self):
+ return self._json_metadata.transient_DM
+
+ def get_transient_dm_error(self):
+ return self._json_metadata.transient_DM_error
+
+ def get_preaverage_time(self):
+ return self._json_metadata.preaverage_time
+
+ def get_rfpipe_version(self):
+ return self._json_metadata.rfpipe_version
+
+ def get_prefs_id(self):
+ return self._json_metadata.prefs_Id
+
+ def get_rf_qa_label(self):
+ return self._json_metadata.rf_QA_label
+
+ def get_rf_qa_zero_fraction(self):
+ return self._json_metadata.rf_QA_zero_fraction
+
+ def get_rf_qa_visibility_noise(self):
+ return self._json_metadata.rf_QA_visibility_noise
+
+ def get_rf_qa_image_noise(self):
+ return self._json_metadata.rf_QA_image_noise
+
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/UVFITSUtil.py b/apps/cli/executables/pexable/ingest/ingest/UVFITSUtil.py
new file mode 100644
index 000000000..5b1d9ec13
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/UVFITSUtil.py
@@ -0,0 +1,284 @@
+import os
+import re
+import logging
+from itertools import chain
+
+import numpy as np
+from astropy.io import fits
+from astropy.table import Table, vstack
+from astropy.time import Time, TimeDelta
+from astropy.coordinates import SkyCoord
+from . import archiveutils as au
+
+"""
+Represents a VLBA UVFITS file
+"""
+
+LOG = logging.getLogger('ingestion')
+
+class UVFits:
+ def __init__(self, file):
+ self.fitsfile = file
+
+ with fits.open(self.fitsfile) as fh:
+ self.source_table = Table(fh['SOURCE'].data)
+ self.source_header = fh['SOURCE'].header
+ self.reference_frequency = fh['FREQUENCY'].header['REF_FREQ']
+ self.frequency_table = Table(fh['FREQUENCY'].data)
+ self.uv_header = fh['UV_DATA'].header
+ self.scans = Table()
+
+ i = 1
+ start_scan = 0
+ # Read through trying to find the UV_DATA tables (there may be multiple in older idifits file)
+ # Scan number should continue to increment across UV_DATA tables
+ while True:
+ try:
+ if fh[i].header['EXTNAME'] == 'UV_DATA':
+ uv_data = fh[i].data
+ start_scan, table_scans = self.build_scans(uv_data, start_scan)
+ self.scans = vstack([self.scans, table_scans])
+ i = i + 1
+ except OSError:
+ break
+
+ def getsource(self, source_id):
+ return self.source_table[self.source_table['ID_NO.'] == source_id]['SOURCE']
+
+ def build_scans(self, uv_data, start_scan):
+ scan_list = []
+ polarizations = self.get_polarizations()
+
+ source = uv_data[0]['SOURCE']
+ scan = start_scan
+ scan_start_row = 0
+ frequency_ids = [uv_data[scan_start_row]['FREQID']]
+ for row in range(len(uv_data)):
+ if source != uv_data[row]['SOURCE']:
+ scan_end_row = row - 1
+ scan_start_time = Time(uv_data[scan_start_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_start_row]['TIME'], format='jd')
+ scan_end_time = Time(uv_data[scan_end_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_end_row]['TIME'], format='jd')
+ scan_time_on_source = scan_end_time - scan_start_time
+ scan_integration_time = uv_data[scan_start_row]['INTTIM']
+ scan_source = self.source_table[self.source_table['ID_NO.'] == uv_data[scan_start_row]['SOURCE']]['SOURCE'][0]
+ source_ra = self.source_table[self.source_table['ID_NO.'] == uv_data[scan_start_row]['SOURCE']]['RAAPP'][0]
+ source_dec = self.source_table[self.source_table['ID_NO.'] == uv_data[scan_start_row]['SOURCE']]['DECAPP'][0]
+ c = SkyCoord(source_ra, source_dec, frame='icrs', unit='deg')
+ source_radec = c.to_string('hmsdms')
+ frequencies = [(self.get_reference_frequency() + self.getfrequency(f)) for f in frequency_ids]
+ frequency_list = np.concatenate(frequencies, axis=0) / 1000000
+ total_bandwidth = self.getbandwidth(1)
+ current_scan = [scan, scan_source, scan_start_time.mjd, scan_end_time.mjd, scan_time_on_source.sec, scan_integration_time, ' '.join(map(str, frequency_list)), total_bandwidth, source_ra, source_dec, source_radec, polarizations]
+ scan_list.append(current_scan)
+ scan = scan + 1
+ scan_start_row = row
+ frequency_ids.clear()
+
+ frequency_id = uv_data[row]['FREQID']
+ if frequency_id not in frequency_ids:
+ frequency_ids.append(frequency_id)
+
+ source = uv_data[row]['SOURCE']
+
+ # Add in final scan
+ scan_end_row = row
+ scan_start_time = Time(uv_data[scan_start_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_start_row]['TIME'], format='jd')
+ scan_end_time = Time(uv_data[scan_end_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_end_row]['TIME'], format='jd')
+ scan_time_on_source = scan_end_time - scan_start_time
+ scan_integration_time = uv_data[scan_start_row]['INTTIM']
+ scan_source = self.source_table[self.source_table['ID_NO.'] == uv_data[scan_start_row]['SOURCE']]['SOURCE'][0]
+ source_ra = self.source_table[self.source_table['ID_NO.'] == uv_data[scan_start_row]['SOURCE']]['RAAPP'][0]
+ source_dec = self.source_table[self.source_table['ID_NO.'] == uv_data[scan_start_row]['SOURCE']]['DECAPP'][0]
+ c = SkyCoord(source_ra, source_dec, frame='icrs', unit='deg')
+ source_radec = c.to_string('hmsdms')
+ frequencies = [(self.get_reference_frequency() + self.getfrequency(f)) for f in frequency_ids]
+ frequency_list = np.concatenate(frequencies, axis=0) / 1000000
+ total_bandwidth = self.getbandwidth(1)
+ current_scan = [scan, scan_source, scan_start_time.mjd, scan_end_time.mjd, scan_time_on_source.sec, scan_integration_time, ' '.join(map(str, frequency_list)), total_bandwidth, source_ra, source_dec, source_radec, polarizations]
+ scan_list.append(current_scan)
+
+ # Build scan table from list of scans and increment start scan number in case there is another UV_DATA table
+ return scan + 1, Table(rows=scan_list, names=('SCAN_ID', 'SOURCE', 'START_TIME', 'END_TIME', 'TIME_ON_SOURCE', 'INTEGRATION_TIME', 'FREQUENCY', 'BANDWIDTH', 'RA', 'DEC', 'RA/DEC', 'POLARIZATION'))
+
+ # Observation attributes
+
+ def getfitsfile(self):
+ return self.fitsfile
+
+ def number_of_scans(self):
+ return len(self.scans)
+
+ def get_reference_frequency(self):
+ return self.reference_frequency
+
+ # Scan attributes
+
+ def get_source(self, scan_id):
+ return self.get_column_scalar_data(scan_id, 'SOURCE')
+
+ def get_start_time(self, scan_id):
+ return self.get_column_scalar_data(scan_id, 'START_TIME')
+
+ def get_end_time(self, scan_id):
+ return self.get_column_scalar_data(scan_id, 'END_TIME')
+
+ def get_ra(self, scan_id):
+ return self.get_column_scalar_data(scan_id, 'RA')
+
+ def get_dec(self, scan_id):
+ return self.get_column_scalar_data(scan_id, 'DEC')
+
+ def get_integration_time(self, scan_id):
+ return self.get_column_scalar_data(scan_id, 'INTEGRATION_TIME')
+
+ def get_bandwidth(self, scan_id):
+ return self.get_column_scalar_data(scan_id, 'BANDWIDTH')
+
+ # Helper function
+
+ def get_column_scalar_data(self, scan_id, column_name):
+ return self.scans[self.scans['SCAN_ID'] == scan_id][column_name].data[0]
+
+ # Polarization
+
+ def get_polarizations(self):
+ # The final polarization code is the sum of the individual ones
+ polarization_code = 0
+ number_polarizations = self.uv_header['NO_STKD']
+ for i in range(number_polarizations):
+ pol_key = f"STK_{i+1}"
+ if pol_key not in self.uv_header.keys():
+ LOG.warning(f"Stokes key: {pol_key} is not present in UV_DATA table")
+ continue
+ polarization = self.lookup_polarization(self.uv_header[pol_key])
+ polarization_code = polarization_code + polarization
+ return au.getpolarization(polarization_code)
+ return polarization_code
+
+ def lookup_polarization(self, polarization_code):
+ # Map of polarization codes used by FITS to observation polarization codes
+ polarization_codes = {
+ 1: 'I',
+ 2: 'Q',
+ 3: 'U',
+ 4: 'V',
+ -1: 'RR',
+ -2: 'LL',
+ -3: 'RL',
+ -4: 'LR',
+ -5: 'XX',
+ -6: 'YY',
+ -7: 'XY',
+ -8: 'YX'
+ }
+
+ if polarization_code not in polarization_codes.keys():
+ LOG.error(f"Invalid polarization code: {polarization_code}")
+ exit(-5)
+
+ return au.getpolarizationcode(polarization_codes[polarization_code])
+
+ def getproject(self):
+ return self.source_header['OBSCODE']
+
+ def getexperiment(self):
+ return ''
+
+ def getcorrelatorpass(self):
+ fflist = os.path.basename(self.getfitsfile()).split('_')
+ return fflist[2]
+
+ def getobservingband(self):
+ ref_frequency = self.reference_frequency
+ return self.lookup_observing_band(ref_frequency)
+
+ def getfrequency(self, frequency):
+ frequency_row = self.frequency_table[self.frequency_table['FREQID'] == frequency]
+ frequencies = frequency_row[0]['BANDFREQ']
+ if np.isscalar(frequencies):
+ # If the frequency is a scalar, put it in a numpy ndarray
+ freq_array = np.ndarray(shape=(1))
+ freq_array[0] = frequency_row[0]['BANDFREQ']
+ return freq_array
+ return frequencies
+
+ def getbandwidth(self, frequency):
+ frequency_row = self.frequency_table[self.frequency_table['FREQID'] == frequency]
+ if np.isscalar(frequency_row[0]['TOTAL_BANDWIDTH']):
+ return frequency_row[0]['TOTAL_BANDWIDTH']
+ else:
+ return frequency_row[0]['TOTAL_BANDWIDTH'][0]
+
+ def lookup_observing_band(self, reference_frequency):
+ ref_freq = reference_frequency / 1000000000
+ obs_band = ''
+
+ if 0.058 <= ref_freq <= 0.084:
+ obs_band = '4'
+ elif 0.236 <= ref_freq <= 0.492:
+ obs_band = 'P'
+ elif 1.0 <= ref_freq <= 2.0:
+ obs_band = 'L'
+ elif 2.0 < ref_freq <= 4.0:
+ obs_band = 'S'
+ elif 4.0 < ref_freq <= 8.0:
+ obs_band = 'C'
+ elif 8.0 < ref_freq <= 12.0:
+ obs_band = 'X'
+ elif 12.0 < ref_freq <= 18.0:
+ obs_band = 'Ku'
+ elif 18.0 < ref_freq <= 26.5:
+ obs_band = 'K'
+ elif 26.5 < ref_freq <= 40.0:
+ obs_band = 'Ka'
+ elif 40.0 < ref_freq <= 50.0:
+ obs_band = 'Q'
+ else:
+ LOG.warning(f"Reference frequency {ref_freq} could not be located in a band")
+
+ return obs_band
+
+ def dump_scans(self):
+ self.scans.pprint(max_lines=-1, max_width=-1)
+
+def test_file(file):
+ if os.path.isfile(file):
+ print(f"File {file} exists")
+ assert True
+ else:
+ print(f"File {file} doesn't exist'")
+ assert False
+
+
+def test_headers(file):
+ # with fits.open(file) as fh:
+ # for i in range(1, 17):
+ # print(fh[i].header['EXTNAME'])
+ pass
+
+
+def test_source(uv_fits, source_id, expected):
+ source_name = uv_fits.getsource(source_id)
+ assert source_name == expected
+
+
+def test_scans(uv_fits):
+ columns = uv_fits.scans.columns
+ num_rows = len(uv_fits.scans[:])
+ print(f"Number of rows: {num_rows}")
+ LOG.error(columns)
+
+
+if __name__ == '__main__':
+ path = '/Users/rlively/vlba'
+ file = 'VLBA_VSN005405_file24.uvfits'
+
+ uv_fits_file = os.path.join(path, file)
+ uv_fits = UVFits(uv_fits_file)
+
+ print(f"Fits file: {uv_fits.get_fits_file()}")
+ print(uv_fits.get_polarizations())
+ uv_fits.scans.pprint(max_lines=-1, max_width=-1)
+ print(uv_fits.get_polarizations())
+
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/VLBAFITSUtil.py b/apps/cli/executables/pexable/ingest/ingest/VLBAFITSUtil.py
new file mode 100644
index 000000000..3621c204a
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/VLBAFITSUtil.py
@@ -0,0 +1,294 @@
+import os
+import sys
+import time
+import logging
+import re
+
+import numpy as np
+from astropy.io import fits
+from astropy.table import Table
+from astropy.time import Time, TimeDelta
+from astropy.coordinates import SkyCoord
+
+from . import archiveutils as au
+
+LOG = logging.getLogger('ingestion')
+
+"""
+Represents a VLBA IDIFITS file
+"""
+class VLBAFits:
+ def __init__(self, fits_file):
+ self._idifitsfile = fits_file
+ if not os.path.exists(self.getfitsfile()) or os.path.getsize(self.getfitsfile()) == 0:
+ LOG.error(f"IDIFITS file {self.getfitsfile()} could not be found or is empty.")
+ exit(-1)
+ self._scans = self._build_scans(self.getfitsfile())
+
+ def _build_scans(self, fitsfile):
+ """
+ Build a table of scans from the IDIFITS file. A scan occurs when the source id changes between
+ rows and, in the case of multiple UV_DATA tables, when a new table is encountered (assumption
+ that is consistent with the legacy archive).
+ """
+ scans_start_time = time.time()
+ with fits.open(fitsfile) as hdu:
+ self._project = self._getproject(hdu)
+ self._correlator_pass = self._getcorrelatorpass()
+ self._segment = self._getsegment(hdu)
+ table = 0
+ scan = 0
+ number_rows = 0
+ scan_table_rows = []
+ while True:
+ # Extracting the UV_DATA table(s) in this way is done because older IDIFITS files
+ # were permitted to have multiple UV_DATA tables and the is no way to extract the
+ # list of those tables. hdu['UV_DATA'] only returns the first table
+ try:
+ if 'EXTNAME' in hdu[table].header:
+ if hdu[table].name == 'UV_DATA':
+ # New UV_DATA table triggers a new scan. This is consistent with the legacy
+ # archive, but seems suspicious. Good question for VLBA subsystem scientist.
+ scan = scan + 1 # New table triggers new scan (??)
+ uv_data = hdu[table].data
+ uv_hdr = hdu[table].header
+
+ polarizations = self.getpolarizations(hdu['UV_DATA'].header)
+
+ uv_length = uv_hdr['NAXIS2']
+ if uv_length == 0:
+ LOG.error(f"No UV_DATA table for file {self.getfitsfile()}")
+ exit(-1)
+ scan_start_row = 0
+ for row in range(uv_length):
+ number_rows = number_rows + 1
+
+ if (row < uv_length - 1) and (uv_data[row]['SOURCE'] != uv_data[row+1]['SOURCE']):
+ # Scan break
+ src_mask = self.getsourcetable()[self._source_id_column] == uv_data[row]['SOURCE']
+ source, source_ra, source_dec = self._getsourceinfo(src_mask)
+ frequency_id = uv_data[row]['FREQID']
+ integration_time = uv_data[row]['INTTIM']
+ frequencies = self.getfrequencies(frequency_id)
+ bandwidth = self.getbandwidths(frequency_id)
+ start_time = Time(uv_data[scan_start_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_start_row]['Time'], format='jd')
+ stop_time = Time(uv_data[row]['DATE'], format='jd') + TimeDelta(uv_data[row]['Time'], format='jd')
+
+ scan_table_rows.append((self.getproject(), scan, source, start_time.mjd, stop_time.mjd, round((stop_time - start_time).sec, 2), integration_time, ', '.join(str(x) for x in frequencies), bandwidth, polarizations, source_ra, source_dec))
+
+ scan_start_row = row + 1
+ scan = scan + 1
+ elif (row == uv_length - 1) and (uv_data[row]['SOURCE'] != uv_data[row-1]['SOURCE']):
+ # Last row in uv_data table and different source as next to last row
+ src_mask = self.getsourcetable()[self._source_id_column] == uv_data[row]['SOURCE']
+ source, source_ra, source_dec = self._getsourceinfo(src_mask)
+ frequency_id = uv_data[row]['FREQID']
+ integration_time = uv_data[row]['INTTIM']
+ frequencies = self.getfrequencies(frequency_id)
+ bandwidth = self.getbandwidths(frequency_id)
+ start_time = Time(uv_data[row]['DATE'], format='jd') + TimeDelta(uv_data[row]['Time'], format='jd')
+ stop_time = Time(uv_data[row]['DATE'], format='jd') + TimeDelta(uv_data[row]['Time'], format='jd')
+
+ scan_table_rows.append((self.getproject(), scan, source, start_time.mjd, stop_time.mjd, round((stop_time - start_time).sec, 2), integration_time, ', '.join(str(x) for x in frequencies), bandwidth, polarizations, source_ra, source_dec))
+ elif row == uv_length - 1:
+ # Last row in uv_data table and same source as next to last row
+ src_mask = self.getsourcetable()[self._source_id_column] == uv_data[row]['SOURCE']
+ source, source_ra, source_dec = self._getsourceinfo(src_mask)
+ frequency_id = uv_data[row]['FREQID']
+ integration_time = uv_data[row]['INTTIM']
+ frequencies = self.getfrequencies(frequency_id)
+ bandwidth = self.getbandwidths(frequency_id)
+ start_time = Time(uv_data[scan_start_row]['DATE'], format='jd') + TimeDelta(uv_data[scan_start_row]['Time'], format='jd')
+ stop_time = Time(uv_data[row]['DATE'], format='jd') + TimeDelta(uv_data[row]['Time'], format='jd')
+
+ scan_table_rows.append((self.getproject(), scan, source, start_time.mjd, stop_time.mjd, round((stop_time - start_time).sec, 2), integration_time, ', '.join(str(x) for x in frequencies), bandwidth, polarizations, source_ra, source_dec))
+ elif hdu[table].header['EXTNAME'] == 'SOURCE':
+ # Construct the source table
+ src_table = Table(hdu[table].data)
+ if 'ID_NO.' in src_table.columns:
+ self._source_id_column = 'ID_NO.'
+ elif 'SOURCE_ID' in src_table.columns:
+ self._source_id_column = 'SOURCE_ID'
+
+ self._source_table = src_table
+ elif hdu[table].header['EXTNAME'] == 'FREQUENCY':
+ # Construct the frequency table and get the header
+ frequency_header = hdu[table].header
+ frequency_table = Table(hdu[table].data)
+ self._ref_frequency = frequency_header['REF_FREQ']
+ self._frequency_table = frequency_table
+
+ table = table + 1
+ except OSError:
+ break
+ except IndexError:
+ break
+
+ # The column names are the interface to persistVLBAmetadata. Be careful!
+ # The order in scan_table_rows must correspond to the order of the columns names below. Be careful!
+ scan_table_column_names = ('Project', 'Scan', 'SOURCE', 'START_TIME', 'END_TIME',
+ 'TIME_ON_SOURCE', 'INTEGRATION_TIME', 'FREQUENCY',
+ 'BANDWIDTH', 'POLARIZATION', 'RA', 'DEC')
+ scan_table = Table(rows=scan_table_rows, names=scan_table_column_names)
+ LOG.info(f"File {self.getfitsfile()} has {number_rows} rows and {scan} scans.")
+ LOG.info(f"{scan} scans built in {time.time() - scans_start_time}s")
+ return scan_table # Return a table of scans where each row is built by build_scan
+
+ def getfitsfile(self):
+ return self._idifitsfile
+
+ def getscantable(self):
+ return self._scans
+
+ def _getsourceinfo(self, src_mask):
+ source = self.getsourcetable()[src_mask]['SOURCE'][0]
+ source_ra = self.getsourcetable()[src_mask]['RAAPP'][0]
+ source_dec = self.getsourcetable()[src_mask]['DECAPP'][0]
+
+ c = SkyCoord(source_ra, source_dec, frame='icrs', unit='deg')
+ # For debugging
+ source_radec = c.to_string('hmsdms')
+
+ # Converting to radians
+ source_ra_rad = c.ra.radian
+ source_dec_rad = c.dec.radian
+
+ # Currently returning the ra and dec in degrees, but modifying the return allows
+ # using radians or an 'hmsdms' string
+ return source, source_ra, source_dec
+
+ def _getproject(self, hdu):
+ """
+ Extract the project code from the observer string in the FITS file
+ e.g., BB100 from BB100A1
+ """
+ m = re.match('([A-Z]+\d*)(.*)', hdu['PRIMARY'].header['OBSERVER'])
+ return m.group(1)
+
+ def getproject(self):
+ return self._project
+
+ def _getsegment(self, hdu):
+ """
+ Extract the segment code from the observer string in the FITS file
+ e.g., A1 from BB100A1
+ """
+ m = re.match('([A-Z]+\d*)(.*)', hdu['PRIMARY'].header['OBSERVER'])
+ return m.group(2)
+
+ def getsegment(self):
+ return self._segment
+
+ def _getcorrelatorpass(self):
+ """ Extract the correlator pass from the file name (not present in the IDIFITS file) """
+ fflist = os.path.basename(self.getfitsfile()).split('_')
+ return fflist[2]
+
+ def getcorrelatorpass(self):
+ return self._correlator_pass
+
+ def getsourcetable(self):
+ return self._source_table
+
+ def getreffrequency(self):
+ return self._ref_frequency
+
+ def getfrequencytable(self):
+ return self._frequency_table
+
+ def getfrequencies(self, frequency_id):
+ frequency_row = self.getfrequencytable()[self.getfrequencytable()['FREQID'] == frequency_id]
+ frequencies = frequency_row[0]['BANDFREQ']
+ if np.isscalar(frequencies):
+ # If the frequency is a scalar, put it in a numpy ndarray
+ freq_array = np.ndarray(shape=(1))
+ freq_array[0] = frequency_row[0]['BANDFREQ']
+ return freq_array
+ return (self.getreffrequency() + frequencies) / 1000000
+
+ def getbandwidths(self, frequency_id):
+ frequency_row = self.getfrequencytable()[self.getfrequencytable()['FREQID'] == frequency_id]
+ # If the frequency is a scalar, put it in a numpy ndarray
+ if np.isscalar(frequency_row[0]['TOTAL_BANDWIDTH']):
+ return frequency_row[0]['TOTAL_BANDWIDTH'] / 1000000
+ else:
+ return frequency_row[0]['TOTAL_BANDWIDTH'][0] / 1000000
+
+ def getpolarizations(self, uv_header):
+ numpolarizations = uv_header['NO_STKD']
+ polarization_code = 0
+ for i in range(numpolarizations):
+ pol_key = f"STK_{i+1}"
+ if pol_key not in uv_header.keys():
+ LOG.warning(f"Stokes key: {pol_key} is not present in UV_DATA table")
+ continue
+ polarization = self.lookup_polarization(uv_header[pol_key])
+ polarization_code = polarization_code + polarization
+ return au.getpolarization(polarization_code)
+
+ def lookup_polarization(self, polarization_code):
+ """ Return the polarization code for the numeric polarization in the IDIFITS file """
+ polarization_codes = {
+ 1: 'I',
+ 2: 'Q',
+ 3: 'U',
+ 4: 'V',
+ -1: 'RR',
+ -2: 'LL',
+ -3: 'RL',
+ -4: 'LR',
+ -5: 'XX',
+ -6: 'YY',
+ -7: 'XY',
+ -8: 'YX'
+ }
+
+ if polarization_code not in polarization_codes.keys():
+ LOG.warn(f"Invalid polarization code: {polarization_code}")
+ exit(-1)
+
+ return au.getpolarizationcode(polarization_codes[polarization_code])
+
+ def lookup_observing_band(self):
+ """ Lookup the observing band from the reference frequency """
+ ref_freq = self.getreffrequency() / 1000000000
+ obs_band = ''
+
+ if 0.058 <= ref_freq <= 0.084:
+ obs_band = '4'
+ elif 0.236 <= ref_freq <= 0.492:
+ obs_band = 'P'
+ elif 1.0 <= ref_freq <= 2.0:
+ obs_band = 'L'
+ elif 2.0 < ref_freq <= 4.0:
+ obs_band = 'S'
+ elif 4.0 < ref_freq <= 8.0:
+ obs_band = 'C'
+ elif 8.0 < ref_freq <= 12.0:
+ obs_band = 'X'
+ elif 12.0 < ref_freq <= 18.0:
+ obs_band = 'Ku'
+ elif 18.0 < ref_freq <= 26.5:
+ obs_band = 'K'
+ elif 26.5 < ref_freq <= 40.0:
+ obs_band = 'Ka'
+ elif 40.0 < ref_freq <= 50.0:
+ obs_band = 'Q'
+ else:
+ LOG.warning(f"Reference frequency {ref_freq} could not be located in a band")
+
+ return obs_band
+
+
+if __name__ == '__main__':
+ if (len(sys.argv) != 3):
+ LOG.error(f"Usage: VLBAIDIFits path file")
+ exit(-1)
+
+ path = sys.argv[1]
+ file = sys.argv[2]
+
+ fits = VLBAFits(os.path.join(path, file))
+ fits.getscantable().pprint(max_lines=250, max_width=250)
+
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/__init__.py b/apps/cli/executables/pexable/ingest/ingest/__init__.py
new file mode 100644
index 000000000..a6179fe25
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/__init__.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+
+from .metadata import *
+from .files import *
+
+import numpy
+from psycopg2.extensions import register_adapter, AsIs
+def addapt_numpy_float32(numpy_float32):
+ return AsIs(numpy_float32)
+register_adapter(numpy.float32, addapt_numpy_float32)
+
+__all__ = ['SDM', 'SDMTable', 'SDMBinaryTable', 'buildtable',
+ 'ngasstatus', 'archivefile', 'retrievefile',
+ 'ingest', 'ingestBDF', 'ingestSDM', 'ingestmetadata']
diff --git a/apps/cli/executables/pexable/ingest/ingest/_version.py b/apps/cli/executables/pexable/ingest/ingest/_version.py
new file mode 100644
index 000000000..199527a48
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/_version.py
@@ -0,0 +1,2 @@
+""" Version information for this package, don't put anything else here. """
+___version___ = "4.0.0a1.dev1"
diff --git a/apps/cli/executables/pexable/ingest/ingest/almautils.py b/apps/cli/executables/pexable/ingest/ingest/almautils.py
new file mode 100644
index 000000000..d3b223641
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/almautils.py
@@ -0,0 +1,214 @@
+import logging
+
+from sqlalchemy import select
+from sqlalchemy.orm import aliased
+from sqlalchemy.orm.exc import MultipleResultsFound
+
+from .schema import create_session, Author
+from .schema.almamodel import (
+ BmmvObsproject,
+ BmmvObsproposal,
+ t_mv_obsproposal,
+ t_asp_obs_proposal_author,
+ BmmvSchedblock,
+ ShiftlogEntry,
+ AquaExecblock,
+ AsaProductFile,
+ NgasFiles,
+ AsaScience,
+ AsaEnergy,
+)
+
+LOG = logging.getLogger("ingestion")
+
+session = create_session("ALMA")
+
+
+def getproject(projectid):
+ project = session.query(BmmvObsproject).filter_by(prj_archive_uid=projectid).first()
+ return project
+
+
+def getabstract(projectid):
+ proposal = session.query(BmmvObsproposal).filter_by(obsproject_archive_uid=projectid).first()
+ return proposal.abstract_text
+
+
+def getauthors(projectid):
+ """
+ Return project authors
+ :param projectid: project for which authors are desired
+ :return: list of authors info
+ """
+
+ """
+ select aut.firstname, aut.lastname, aut.account_id, aut.author_index
+ from ALMA.ASP_OBS_PROPOSAL_AUTHOR aut
+ inner join ALMA.MV_OBSPROPOSAL mop
+ on mop.archive_uid = aut.proposal_archive_uid
+ where mop.obsproject_archive_uid = ?
+ ;
+ """
+
+ s = (
+ select(
+ [
+ t_asp_obs_proposal_author.c.firstname,
+ t_asp_obs_proposal_author.c.lastname,
+ t_asp_obs_proposal_author.c.account_id,
+ t_asp_obs_proposal_author.c.author_index,
+ ]
+ )
+ .select_from(
+ t_asp_obs_proposal_author.join(
+ t_mv_obsproposal, t_mv_obsproposal.c.archive_uid == t_asp_obs_proposal_author.c.proposal_archive_uid
+ )
+ )
+ .where(t_mv_obsproposal.c.obsproject_archive_uid == projectid)
+ )
+
+ result = [
+ Author(author_id=None, firstname=r[0], lastname=r[1], username=r[2], project_code=projectid, is_pi=r[3] == 0)
+ for r in session.execute(s).fetchall()
+ ]
+
+ return result
+
+
+def getalmaoushierarchy(execblock):
+ """
+ Build the ALMA OUS hierarchy for this execblock and return the MOUS, GOUS, and SOUS ids
+ :param execblock:
+ :return: MOUS, GOUS, SOUS ids
+ """
+
+ """
+ select SGOUS_STATUS_UID, sbs.GOUS_STATUS_UID, sbs.MOUS_STATUS_UID
+ FROM BMMV_SCHEDBLOCK sbs
+ join ALMA.SHIFTLOG_ENTRIES shifts ON shifts.SE_SB_ID = sbs.ARCHIVE_UID
+ join AQUA_EXECBLOCK ebs ON ebs.EXECBLOCKUID = shifts.SE_EB_UID
+ where ebs.EXECBLOCKUID= ?
+;
+ """
+
+ sbs = aliased(BmmvSchedblock)
+ shifts = aliased(ShiftlogEntry)
+ ebs = aliased(AquaExecblock)
+
+ execblockuid = execblock.replace("___", "://").replace("_", "/")
+ LOG.info(f"Execblock UID: {execblockuid}")
+
+ sb, sh, eb = (
+ session.query(sbs, shifts, ebs)
+ .filter(shifts.se_sb_id == sbs.archive_uid)
+ .filter(ebs.execblockuid == shifts.se_eb_uid)
+ .filter(ebs.execblockuid == execblockuid)
+ .one()
+ )
+
+ LOG.info(
+ f"Execblock: {execblock} MOUS: {sb.mous_status_uid} GOUS: {sb.gous_status_uid} SOUS: {sb.sgous_status_uid}"
+ )
+ return sb.mous_status_uid, sb.gous_status_uid, sb.sgous_status_uid
+
+
+def getalmacalfiles(mous):
+ """
+ :param mous:
+ :return: list of calibration products
+ """
+
+ """
+ select FILE_ID,FILE_SIZE from ngas.ngas_files where file_id in (select ngas_file_id from
+ alma.asa_product_files where ASA_OUS_ID=? and FILE_CLASS in ('calibration','script'));
+ """
+
+ LOG.info(f"Getting calibration products for mous {mous}")
+
+ files = (
+ session.query(AsaProductFile.ngas_file_id, NgasFiles.file_size)
+ .filter(AsaProductFile.asa_ous_id == mous)
+ .filter(AsaProductFile.file_class.in_(["calibration", "script"]))
+ .filter(AsaProductFile.ngas_file_id == NgasFiles.file_id)
+ .all()
+ )
+
+ for file in files:
+ LOG.info(f"File name: {file[0]} size: {file[1]}")
+
+ return [(file[0], file[1]) for file in files]
+
+
+def getschedblockname(mous):
+ """
+ Return the sched block name given the mous id
+
+ :param mous: MOUS uid
+ :return: schedblock name
+ """
+
+ """
+ SELECT SB_NAME FROM BMMV_SCHEDBLOCK WHERE STATUS NOT IN ('Suspended', 'Cancelled') AND STATUS NOT LIKE 'CSV%'
+ AND MOUS_STATUS_UID='uid://A001/X1284/X15db';
+
+ """
+
+ LOG.info(f"Mous id for sched block name: {mous}")
+ sbs = aliased(BmmvSchedblock)
+ try:
+ sb = (
+ session.query(sbs)
+ .filter(sbs.status.notin_(["Suspended", "Cancelled"]))
+ .filter(~(sbs.status.ilike("csv%")))
+ .filter(sbs.mous_status_uid == mous)
+ .one_or_none()
+ )
+ except MultipleResultsFound:
+ LOG.warning("Multiple results found for SB name of {}. Setting to NULL.".format(mous))
+ return None
+
+ if sb is not None:
+ LOG.info(sb.sb_name)
+ return sb.sb_name
+ else:
+ LOG.info("No Appropriate SB Found.")
+ return None
+
+
+def get_source_list(mous):
+ """
+ Get source list from MOUS id
+ :param mous:
+ :return:
+ """
+ # mous = 'uid://A001/X1284/X15db'
+
+ # _SOURCE_DATA_QUERY = """SELECT SOURCE_NAME, RA_SOURCE, DEC_SOURCE, INT_TIME, SPATIAL_SCALE_MIN, SPATIAL_SCALE_MAX, DATASET_ID FROM ALMA.ASA_SCIENCE WHERE ASA_OUS_ID=:mous_uid AND DATASET_ID LIKE :like_this"""
+
+ source_list = (
+ session.query(AsaScience)
+ .filter(AsaScience.asa_ous_id == mous)
+ .filter(AsaScience.dataset_id.ilike(mous + "%"))
+ .all()
+ )
+
+ return source_list
+
+
+def get_spectral_window_data(source):
+ # _SPW_DATA_FOR_SOURCE = """SELECT SPW_NUM, FREQUENCY_MIN, FREQUENCY_MAX, BANDWIDTH, CHANNEL_NUM, RESOLUTION_MIN, RESOLUTION_MAX FROM ALMA.ASA_ENERGY WHERE ASA_DATASET_ID=:dsid"""
+ spectral_window_data = session.query(AsaEnergy).filter(AsaEnergy.asa_dataset_id == source.dataset_id).all()
+
+ return spectral_window_data
+
+
+def update_alma_source_data():
+ pass
+
+
+def update_alma_spectral_window_data():
+ pass
+
+
+def update_alma_spectral_window_source_data():
+ pass
diff --git a/apps/cli/executables/pexable/ingest/ingest/archive.py b/apps/cli/executables/pexable/ingest/ingest/archive.py
new file mode 100644
index 000000000..33abc7657
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/archive.py
@@ -0,0 +1,321 @@
+import argparse
+import glob
+import os
+import time
+import sys
+from pycapo import CapoConfig
+import logging
+from pathlib import Path
+from ._version import ___version___ as _version
+
+from .pymygdala import LogHandler, SendNRAOEvent
+from . import evlasciencedatamodel as sc
+from . import NGASUtil as ng
+from . import persistmetadata as pm
+from . import manifestpersist as mp
+from . import persistcaltables as pct
+from . import persistimages as pimg
+from . import archiveutils as au
+from . import persistVLBAmetadata as vlbamd
+from . import IngestionManifest as im
+from .IngestionManifest import IngestionManifest
+
+capoprofile = os.environ["CAPO_PROFILE"]
+
+LOG = logging.getLogger("ingestion")
+
+LOG.setLevel(logging.DEBUG)
+handler = LogHandler(profile=capoprofile, application="Ingestion")
+handler.setLevel(logging.DEBUG)
+LOG.addHandler(handler)
+
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.DEBUG)
+LOG.addHandler(ch)
+
+_HELP_MESSAGE = """AAT Ingestion Utility, version {}:\n\tCreates metadata structures and places files into NGAS."""
+ingest_parser = argparse.ArgumentParser(
+ description=_HELP_MESSAGE.format(_version), formatter_class=argparse.RawTextHelpFormatter
+)
+
+__send_event = SendNRAOEvent(profile=capoprofile, application="SDM Ingestion")
+
+_INGESTION_MANIFEST_FILENAME = "ingestion_manifest.json"
+_SDM_INGESTION = "meta"
+_BDF_INGESTION = "bdf"
+_CALIBRATION_INGESTION = "cal"
+_IMAGE_INGESTION = "img"
+_VLBA_IDIFITS_INGESTION = "vlba"
+_JSON_MANIFEST_INGESTION = "json"
+_INVALID_INGESTION_OPERATION = "invalid"
+
+
+def main():
+ """
+ Main method for ingesting files into metadata database and NGAS
+ :return:
+ :error returns
+ -5: Invalid operation
+ """
+
+ # Primary flags indicating operation to perform
+ mxg = ingest_parser.add_mutually_exclusive_group(required=True)
+ mxg.add_argument("-b", "--bdf", help="Ingest BDFs for observation", action="store_true")
+ mxg.add_argument("-c", "--cal", help="Ingest calibration products", action="store_true")
+ mxg.add_argument("-i", "--img", help="Ingest imaging products and images", action="store_true")
+ mxg.add_argument("-m", "--meta", help="Ingest science metadata for observation", action="store_true")
+ mxg.add_argument("-v", "--vlba", help="Ingest VLBA science metadata", action="store_true")
+ mxg.add_argument("-j", "--json", help="Ingestion using ingestion manifest", action="store_true")
+ ingest_parser.add_argument("-p", "--path", help="Path to data")
+ ingest_parser.add_argument(
+ "-o", "--file", help="File (observation file set, calibration product tar file, BDF file, image)"
+ )
+ ingest_parser.add_argument("-g", "--filegroup", help="Calibration products parent eb or image project")
+ ingest_parser.add_argument("-r", "--reingest", help="Re-ingest (delete) existing data", action="store_true")
+ ingest_parser.add_argument("-n", "--nocal", help="Do not perform calibration", action="store_true")
+ ingest_parser.add_argument(
+ "-d", "--nongas", help="Only extract metadata, do not ingest into NGAS", action="store_true"
+ )
+
+ args = ingest_parser.parse_args()
+
+ config = CapoConfig(profile=capoprofile)
+
+ operation = getingestionoperation(args)
+ if operation == _INVALID_INGESTION_OPERATION:
+ LOG.critical(f"The operation is unspecified or is invalid.")
+ sys.exit(1)
+ elif operation == _IMAGE_INGESTION:
+ pass
+ # Retrieve paths to SDM tables, BDF files, and the NGAS staging area
+
+ # First the pathy stuff
+ if args.path:
+ sdmpath = args.path
+ bdfpath = args.path
+ calproductpath = args.path
+ else:
+ # TODO This should be an error for the required flags
+ sdmpath = config.getstring("archive-ingestion.SDMPath")
+ bdfpath = config.getstring("archive-ingestion.BDFPath")
+ calproductpath = config.getstring("archive-ingestion.BDFPath")
+
+ if operation == _JSON_MANIFEST_INGESTION:
+ operation = getoperationfromjson()
+
+ if args.meta: # Ingest science metadata for SDM-based observations
+ filegroups = []
+ if args.file: # single file
+ filegroups.append(args.file)
+ else: # all files in path
+ filegroups = [x.name for x in list(Path(sdmpath).glob("*"))]
+ LOG.debug(f"filesets: {filegroups}")
+
+ groupcount = 0
+ starttime = time.time()
+ for filegroup in filegroups:
+ filepath = Path(sdmpath) / filegroup
+ if not filepath.exists():
+ LOG.error(f"Observation {filepath.name} doesn't exist in {filepath.parent}")
+ continue
+ asdm_table = Path(filepath) / "ASDM.xml"
+ if not asdm_table.exists(): # If the directory has ASDM.xml, it is a valid filegroup
+ LOG.error(f"{filepath.name} is missing ASDM.xml. Not a valid observation")
+ continue
+ exec_block = au.getexecutionblockfromname(filegroup)
+ if exec_block is not None: # Already exists
+ if args.reingest:
+ LOG.info(f"Re-ingestion requested")
+ else:
+ LOG.warning(
+ f"Observation {filepath.name} already exists in metadata database and re-ingestion not requested"
+ )
+ continue
+ LOG.info(f"Ingesting SDM metadata for execution block {filegroup}")
+ sdm = sc.buildsdm(filegroup, sdmpath)
+ if len(sdm.bdffiles) > 0: # ingest science meta data
+ project, science_products = pm.persist(sdm, args.nocal)
+ else:
+ LOG.error(f"Execution block: {filepath.name} has no bdfs and will not be ingested")
+ continue
+
+ LOG.info(f"SDM metadata ingestion complete for {filepath.name} in {round(time.time() - starttime, 2)}s")
+ if args.stage:
+ LOG.info(
+ "-t flag set. SDM tables and BDF files will be staged for NGAS ingestion in {}".format(args.stage)
+ )
+ ng.ingestsdmintongas(sdm, os.path.join(sdmpath, filegroup), args.stage)
+ groupcount += 1
+
+ # Send completion even to amygdala
+ send_event(
+ {
+ "logData": {
+ "fileset_id": os.path.basename(filegroup),
+ "ingestion_type": "observation",
+ "project_code": project.project_code,
+ },
+ "message": "ingestion complete",
+ "request": "de nada",
+ }
+ )
+
+ elif args.bdf:
+ # BDF Files
+ # Path (-p) and file set (-o) required
+
+ if args.nongas:
+ LOG.warning("-d flag was set for no ingestion into NGAS. For bdf ingestion, this is a no-op.")
+ sys.exit(0)
+
+ if args.file and args.path:
+ filelist = args.file.split(",")
+ pathfilelist = []
+ for file in filelist:
+ filepath = os.path.join(args.path, file.strip())
+ if not (filepath.endswith(".bdf") or filepath.endswith(".sdm") or filepath.endswith(".bin")):
+ LOG.warning("Invalid extension for ingestion: {}".format(filepath))
+ continue
+ if not os.path.exists(filepath):
+ LOG.error("Path to file doesnt exist: {}".format(filepath))
+ continue
+
+ pathfilelist.append(filepath)
+ if len(pathfilelist) > 0:
+ # Not ingesting bdfs yet
+ pass
+ # ng.archivefiles(pathfilelist)
+ else:
+ LOG.info("No files located for BDF ingestion")
+ else:
+ LOG.error("-b requires arguments -p and -o")
+
+ elif args.cal:
+ # Calibration products
+ # Path (-p) and calibration product tar file (-o) required. Parent execution block (-g) required for calibration
+ # workflow, but not bulk calibration ingestion
+ if args.path and args.file and args.filegroup:
+ calibration_products_file = Path(args.path) / args.file
+ if not calibration_products_file.exists():
+ LOG.error(
+ f"Calibration file {calibration_products_file.name} does not exist in {calibration_products_file.parent}"
+ )
+ sys.exit(1)
+ else:
+ pct.ingestcalmetadata(calibration_products_file, args.filegroup, args.nongas)
+ elif args.path and args.file:
+ calibration_file = Path(args.path) / args.file
+ if not calibration_file.exists():
+ # TODO calfile
+ LOG.error(f"Path to calibration products tar file doesnt exist: {str(calibration_file)}")
+ sys.exit(1)
+ # Ingest file where parent execution block must be extracted from the .tar file
+ pct.ingestcaltable(os.path.join(args.path, args.file))
+ LOG.info(f"Calibration products {os.path.join(args.path, args.file)} ingested")
+ elif args.filegroup:
+ pct.ingestalmacalibration(args.filegroup)
+ else:
+ LOG.error("-c requires arguments -p, -o, and -g for VLA or just -g for ALMA")
+
+ elif args.img:
+ if args.path and args.filegroup:
+ if not os.path.exists(args.path):
+ LOG.error(f"Path to image products doesnt exist: {args.path}")
+ sys.exit(1)
+ LOG.info(f"Ingesting images for project {args.filegroup} in {args.path}")
+
+ filegroup = au.getimagesetfilegroupfromname(os.path.basename(args.path))
+ if filegroup is not None: # Filegroup already exists
+ if args.reingest:
+ LOG.info("Removing image set {} for re-ingestion".format(os.path.basename(args.path)))
+ au.removeimageset(filegroup)
+ else:
+ LOG.warning(
+ "Image set {} already exists in the metadata database. If you wish to re-ingest, use -i -r".format(
+ os.path.basename(args.path)
+ )
+ )
+ sys.exit(1)
+ else:
+ LOG.info("Ingesting image set {}".format(os.path.basename(args.path)))
+
+ pimg.ingestimagemetaData(args.path, args.filegroup, args.nongas)
+ else:
+ LOG.error("-i requires arguments -p and -g")
+ elif args.vlba:
+ if not args.path or not args.file:
+ LOG.error("VLBA ingestion requires a path and file name")
+ sys.exit(1)
+ file_path = Path(args.path)
+ fits_file = file_path / args.file
+ starttime = time.time()
+ existingfile = au.getvlbafile(args.file)
+ if existingfile is not None:
+ if args.reingest:
+ # Re-ingestion - remove the existing file before ingesting
+ au.removefile(existingfile)
+ else:
+ LOG.error(f"File: {fits_file} has already been ingested.")
+ sys.exit(1)
+ vlbamd.persistVLBAMetadata(fits_file)
+ LOG.info(f"IDIFITS file {fits_file.name} ingested in {time.time() - starttime}s")
+ elif args.json:
+ if not args.path:
+ LOG.critical(f"-p flag is required for ingestion via ingestion-manifest.json")
+ sys.exit(1)
+ ingestion_manifest_file = Path(args.path) / im.INGESTION_MANIFEST_FILE
+ if not ingestion_manifest_file.is_file():
+ LOG.critical(
+ f"Ingestion manifest {ingestion_manifest_file.name} does not exist in {ingestion_manifest_file.parent}"
+ )
+ sys.exit(1)
+ ingestion_manifest = IngestionManifest(ingestion_manifest_file)
+ mp.ingest_from_manifest(ingestion_manifest_file.parent, ingestion_manifest)
+ else:
+ LOG.critical("Invalid command-line argument")
+ sys.exit(1)
+
+
+def getingestionoperation(args):
+ if args.meta:
+ return _SDM_INGESTION
+ elif args.bdf:
+ return _BDF_INGESTION
+ elif args.cal:
+ # Path (-p) and calibration product tar file (-o) required. Parent execution block (-g) required for calibration
+ return _CALIBRATION_INGESTION
+ elif args.img:
+ return _IMAGE_INGESTION
+ elif args.vlba:
+ return _VLBA_IDIFITS_INGESTION
+ elif args.json:
+ return _JSON_MANIFEST_INGESTION
+ else:
+ return _INVALID_INGESTION_OPERATION
+
+
+def getoperationfromjson():
+ pass
+
+
+def getargs(operation):
+ if operation == _SDM_INGESTION:
+ pass
+ elif operation == _BDF_INGESTION:
+ pass
+ elif operation == _CALIBRATION_INGESTION:
+ pass
+ elif operation == _IMAGE_INGESTION:
+ pass
+ elif operation == _VLBA_IDIFITS_INGESTION:
+ pass
+ elif operation == _JSON_MANIFEST_INGESTION:
+ pass
+
+
+def send_event(event):
+ __send_event.send(routing_key="ingestion-complete.metadata", event=event)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/apps/cli/executables/pexable/ingest/ingest/archive_caltables.py b/apps/cli/executables/pexable/ingest/ingest/archive_caltables.py
new file mode 100644
index 000000000..6c55bddbc
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/archive_caltables.py
@@ -0,0 +1,47 @@
+import evlasciencedatamodel as sc
+import NGASUtil as ng
+import persistmetadata as pm
+import argparse
+import glob
+import os
+import shutil
+from pycapo import CapoConfig
+from lxml import objectify as obj
+import tarfile
+from persistcaltables import persistcalibrations
+
+# TODO incorporate this in as an option of archive.py ?
+# Capofy the location of the temporary location for tar unpacking
+# No manifest file?
+
+def main():
+ parser = argparse.ArgumentParser(description='Ingest Calibration Tables')
+ parser.add_argument('-f', '--file', help='Calibration File')
+ args = parser.parse_args()
+
+
+ capoprofile = os.environ['CAPO_PROFILE']
+ config = CapoConfig(profile=capoprofile)
+ calibrationtablepath = config.getstring('archive-ingestion.CalibrationTablePath')
+
+ if args.file is None:
+ tarfiles = glob.glob(calibrationtablepath + '/*.tar')
+ else:
+ tarfiles = []
+ tarfiles.append(os.path.join(calibrationtablepath, args.file))
+ print('{} calibration file(s) to be archived'.format(len(tarfiles)))
+ calcount = 0
+ calibrationfiles = []
+ for calfile in tarfiles:
+ print('Tar file to be ingested: {}'.format(tarfile))
+ calcount += 1
+ calibrationfiles.append(calfile)
+
+ ng.archivefiles(calibrationfiles)
+ print('Total of {} calibrations(s) ingested'.format(calcount))
+ persistcalibrations(calibrationfiles)
+
+
+if __name__ == '__main__':
+ # Do the argparse thing
+ main()
diff --git a/apps/cli/executables/pexable/ingest/ingest/archiveutils.py b/apps/cli/executables/pexable/ingest/ingest/archiveutils.py
new file mode 100644
index 000000000..e637f5490
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/archiveutils.py
@@ -0,0 +1,391 @@
+import logging
+
+from sqlalchemy import text
+from sqlalchemy.orm import aliased
+
+from .schema import create_session
+from .schema.model import (
+ Author,
+ Project,
+ ExecutionBlock,
+ Filegroup,
+ File,
+ Polarization,
+ Configuration,
+ DataDescription,
+ AlmaOus,
+ AlmaOusType,
+ Calibration,
+ ScienceProduct,
+ ScienceProductsProductGroup,
+ ProductGroup,
+)
+
+session = create_session("SDM")
+LOG = logging.getLogger("ingestion")
+
+
+def getpolarizationcode(polarization):
+ polarizationid = (
+ session.query(Polarization.polarization_id)
+ .filter(text("name=:polarization"))
+ .params(polarization=polarization)
+ .one_or_none()
+ )
+ if polarizationid is None:
+ return 0
+ session.close()
+ return polarizationid[0]
+
+
+def getproject(projectcode):
+ project = session.query(Project).filter(Project.project_code == projectcode).one_or_none()
+ if project is not None:
+ project.authors
+ session.close()
+ return project
+
+
+def getauthor(projectcode, personid):
+ author = (
+ session.query(Author)
+ .filter(Author.project_code == projectcode)
+ .filter(Author.pst_person_id == str(personid))
+ .first()
+ )
+ session.close()
+ return author
+
+
+def get_filegroup_from_name(external_name):
+ science_product = session.query(ScienceProduct).filter(ScienceProduct.external_name == external_name).one_or_none()
+
+ return science_product.filegroup
+
+
+def get_file_from_group(group):
+ file = session.query(File).filter(File.filegroup1 == group).one_or_none()
+ return file
+
+
+def getexecutionblockfromname(ngas_name):
+ executionblock = (
+ session.query(ExecutionBlock)
+ .filter(ExecutionBlock.ngas_fileset_id.isnot(None))
+ .filter(ExecutionBlock.ngas_fileset_id == ngas_name)
+ .one_or_none()
+ )
+ return executionblock
+
+
+def get_execution_block_from_external_name(external_name):
+ """
+ Get execution block with the given external name
+ :param external_name: external name of the execution block
+ :return: execution block
+ """
+ execution_block = (
+ session.query(ExecutionBlock)
+ .filter(ScienceProduct.external_name == external_name)
+ .filter(ExecutionBlock.science_product_locator == ScienceProduct.science_product_locator)
+ .one_or_none()
+ )
+
+ return execution_block
+
+
+def get_execution_block(science_product_locator):
+ execution_block = (
+ session.query(ExecutionBlock)
+ .filter(ExecutionBlock.ngas_fileset_id.isnot(None))
+ .filter(ExecutionBlock.science_product_locator == science_product_locator)
+ .one_or_none()
+ )
+ session.close()
+ return execution_block
+
+
+def get_science_product_by_spl(science_product_locator):
+ science_product = (
+ session.query(ScienceProduct)
+ .filter(ScienceProduct.science_product_locator == science_product_locator)
+ .one_or_none()
+ )
+ return science_product
+
+
+def get_science_product(filename):
+ """
+ Get the science product associated with this file
+ :param filename:
+ :return:
+ """
+ science_product = (
+ session.query(ScienceProduct)
+ .filter(File.filename == filename)
+ .filter(File.filegroup == Filegroup.filegroup_id)
+ .filter(Calibration.filegroup_id == Filegroup.filegroup_id)
+ .filter(ScienceProduct.science_product_locator == Calibration.science_product_locator)
+ .one_or_none()
+ )
+ return science_product
+
+
+def get_file(filename):
+ file = session.query(File).filter(File.filename == filename).one_or_none()
+ return file
+
+
+def getvlbaexecutionblock(project_code, segment):
+ executionblock = (
+ session.query(ExecutionBlock)
+ .filter(ExecutionBlock.project_code == project_code)
+ .filter(ExecutionBlock.segment == segment)
+ .one_or_none()
+ )
+ session.close()
+ return executionblock
+
+
+def get_vlba_correlations_filegroup(project_code, segment, filegroup_type):
+ filegroup = (
+ session.query(ExecutionBlock, Filegroup)
+ .filter(ExecutionBlock.project_code == project_code)
+ .filter(ExecutionBlock.segment == segment)
+ .filter(ExecutionBlock.filegroup_id == Filegroup.filegroup_id)
+ .filter(Filegroup.type == filegroup_type)
+ .one_or_none()
+ )
+ if filegroup is None:
+ LOG.debug(f"Filegroup type {filegroup_type} not found for segment {project_code}{segment}.")
+ session.close()
+ return filegroup
+
+
+def get_vlba_correlator_pass_filegroup(project_code, segment, filegroup_type):
+ eb, filegroup = (
+ session.query(ExecutionBlock, Filegroup)
+ .filter(ExecutionBlock.project_code == project_code)
+ .filter(ExecutionBlock.segment == segment)
+ .filter(ExecutionBlock.filegroup_id == Filegroup.parent_filegroup_id)
+ .filter(Filegroup.type == filegroup_type)
+ .one_or_none()
+ )
+ if filegroup is None:
+ LOG.debug(f"Filegroup type {filegroup_type} not found for segment {project_code}{segment}.")
+ session.close()
+ return filegroup
+
+
+def getvlbafile(ngas_id):
+ file = session.query(File).filter(File.ngas_id == ngas_id).one_or_none()
+ return file
+
+
+def removeexecutionblock(executionblock):
+ session.delete(executionblock)
+ session.commit()
+
+
+def removefilegroup(filegroup):
+ session.delete(filegroup)
+ session.commit()
+
+
+def removefile(file):
+ session.delete(file)
+ session.commit()
+
+
+def getpolarization(polarizationcode):
+ polarization = session.query(Polarization).filter(Polarization.polarization_id == polarizationcode).one()
+ session.close()
+ return polarization
+
+
+def getdatadescription(bandwidth, frequency, polarization):
+ datadescription = (
+ asession.query(DataDescription)
+ .filter(DataDescription.bandwidth == bandwidth)
+ .filter(DataDescription.frequency == frequency)
+ .filter(DataDescription.polarization_id == polarization)
+ .first()
+ )
+ return datadescription
+
+
+def getconfiguration(execblocid, configurationid, asession):
+ configuration = (
+ asession.query(Configuration)
+ .filter(Configuration.execution_block_id == execblocid)
+ .filter(Configuration.configuration == configurationid)
+ .one_or_none()
+ )
+ return configuration
+
+
+def getimagesetfilegroupfromname(groupname):
+ filegroup = (
+ session.query(Filegroup)
+ .filter(Filegroup.groupname == groupname)
+ .filter(Filegroup.type == "image_set")
+ .one_or_none()
+ )
+ session.close()
+ return filegroup
+
+
+def removeimageset(filegroup):
+ session.delete(filegroup)
+ session.commit()
+
+
+# Check for the existence of OUSes respectively since they will have
+# to be built if they don't exist
+
+
+def ousexists(id, type):
+ """
+
+ :param id: alma_ous_id
+ :param type: MOUS, GOUS, or SOUS
+ :return True if OUS exists in the archive db:
+ """
+
+ cnt = (
+ session.query(AlmaOus, AlmaOusType)
+ .filter(AlmaOus.ous_type == AlmaOusType.ous_type)
+ .filter(AlmaOus.alma_ous_id == id)
+ .filter(AlmaOusType.ous_type == type)
+ .count()
+ )
+ return True if cnt > 0 else False
+
+
+def getprojectcodefrommous(mous):
+ """
+ Get the project code from the MOUS using OUS hierarchy
+ :param mous:
+ :return: project code
+ """
+
+ LOG.info(f"Find project code for MOUS: {mous}")
+
+ almaouss = aliased(AlmaOus)
+ almaousg = aliased(AlmaOus)
+ almaousm = aliased(AlmaOus)
+
+ projectcode = (
+ session.query(almaouss.project_code)
+ .join(almaousg, almaousg.parent_ous_id == almaouss.alma_ous_id)
+ .join(almaousm, almaousm.parent_ous_id == almaousg.alma_ous_id)
+ .filter(almaousm.alma_ous_id == mous)
+ .filter(almaouss.ous_type == "SOUS")
+ .filter(almaousg.ous_type == "GOUS")
+ .filter(almaousm.ous_type == "MOUS")
+ .one_or_none()
+ )
+
+ session.close()
+
+ return projectcode[0]
+
+
+def getmousesfromprojectcode(project_code):
+ """
+ Get MOUSes for a project
+ :param project_code:
+ :return: list of MOUSes
+ """
+
+ mset = set()
+
+ project = session.query(Project).filter(Project.project_code == project_code).one_or_none()
+
+ for sous in project.alma_ouses:
+ gouses = sous.children_ouses
+ for gous in gouses:
+ mouses = gous.children_ouses
+ for mous in mouses:
+ mset.add(mous)
+
+ sessopm / c
+ pse
+ return list(mset)
+
+
+def getcalibrationsfrommous(mousid):
+ """
+ Get calibrations for MOUS
+ :param MOUS id:
+ :return list of calibrations:
+ """
+
+ mous = session.query(AlmaOus).filter(AlmaOus.alma_ous_id == mousid).filter(AlmaOus.ous_type == "MOUS").one_or_none()
+
+ session.close()
+
+ return mous.calibrations
+
+
+def getcalibrationsfromeb(ngas_fileset_id):
+ """
+ Get calibrations for execution block
+ :param MGAS fileset id:
+ :return list of calibrations:
+ """
+
+ eb = session.query(ExecutionBlock).filter(ExecutionBlock.ngas_fileset_id == ngas_fileset_id).one_or_none()
+
+ session.close()
+ return eb.calibrations
+
+
+def getexecutionblocksfrommous(mousid):
+ """
+ Get execution blocks for MOUS
+ :param MOUS id:
+ :return list of execution_blocks:
+ """
+
+ mous = session.query(AlmaOus).filter(AlmaOus.alma_ous_id == mousid).filter(AlmaOus.ous_type == "MOUS").one_or_none()
+
+ session.close()
+ return mous.execution_blocks
+
+
+def get_mous_from_eb_science_product_locator_list(locators):
+ """
+ Get the mous from a list of ebs (should be identical)
+ :param locators:
+ :return:
+ """
+
+ mous_set = set()
+ for locator in locators:
+ eb = session.query(ExecutionBlock).filter(ExecutionBlock.science_product_locator == locator).one_or_none()
+ if eb.alma_ous is not None:
+ mous_set.add(eb.alma_ous)
+ if len(mous_set) > 1:
+ LOG.critical("List of EB science product locators resulted in multiple MOUS")
+
+ LOG.debug(f"MOUS set: {mous_set}")
+ session.close()
+ return mous_set.pop()
+
+
+def get_project_from_science_product_locator(science_product_locator):
+ """
+ Return the project given the science product
+ :param science_product_locator: Science product locator
+ :return: project
+ """
+
+ science_product = (
+ session.query(ScienceProduct)
+ .filter(ScienceProduct.science_product_locator == science_product_locator)
+ .one_or_none()
+ )
+
+ # session.close()
+ return science_product.project
diff --git a/apps/cli/executables/pexable/ingest/ingest/commands.py b/apps/cli/executables/pexable/ingest/ingest/commands.py
new file mode 100644
index 000000000..66bd04ba9
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/commands.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+def ingestion():
+ print('Hello World!')
+
+if __name__ == "__main__":
+ ingestion()
diff --git a/apps/cli/executables/pexable/ingest/ingest/evlasciencedatamodel.py b/apps/cli/executables/pexable/ingest/ingest/evlasciencedatamodel.py
new file mode 100644
index 000000000..c3d454bbb
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/evlasciencedatamodel.py
@@ -0,0 +1,108 @@
+# Construct a Science Data Model from a set of SDM XML tables
+
+# Consider doing validation on the presence of the BDF files
+# General validation of the SDM
+
+import os
+import sys
+from lxml import objectify as obj
+import logging
+
+# Verify the contents of the SDM and cross-verify with BDFs
+from lxml.etree import XMLSyntaxError
+
+logger = logging.getLogger('ingestion')
+
+class SDM:
+ def __init__(self, fileset, sdmbasepath='.', bdfbasepath=None):
+ self.fileset = fileset
+ self.sdmtables = buildsdmtables(self, os.path.join(sdmbasepath, fileset))
+ self.bdffiles = addbdffiles(self)
+
+
+class SDMTable:
+ def __init__(self, tablename, entityid, size):
+ self.tablename = tablename
+ self.entityid = entityid
+ self.size = size
+
+
+class SDMXMLTable(SDMTable):
+ def __init__(self, tablename, entityid, table, size):
+ SDMTable.__init__(self, tablename, entityid, size)
+ self.table = table
+
+
+class SDMBinaryTable(SDMTable):
+ def __init__(self, tablename, entityid, size):
+ SDMTable.__init__(self, tablename, entityid, size)
+
+
+class BDFFile():
+ def __init__(self, entityid, filesize):
+ self.entityid = entityid
+ self.filesize = filesize
+
+
+def buildsdm(fileset='abc23', sdmbasepath='.', bdfbasepath='.'):
+ sdm = SDM(fileset, sdmbasepath, bdfbasepath)
+ return sdm
+
+
+def buildsdmtables(sdm, sdmbasepath):
+ sdmtables = {}
+
+ # Get the SDM Table files
+ sdmtree = obj.parse(os.path.join(sdmbasepath, 'ASDM.xml'))
+ sdmroot = sdmtree.getroot()
+ if sdmroot.find('Table') is None:
+ logger.error('ASDM.xml has no table entries in {}'.format(sdmbasepath))
+ sys.exit(-1)
+ for table in sdmroot.Table:
+ tablepath = os.path.join(sdmbasepath, str(table.Name))
+ if os.path.isfile(tablepath + '.xml'):
+ # Don't parse if it's the old format SysPower.xml due to the size
+ if table.Name == 'SysPower':
+ sdmtabletree = None
+ else:
+ try:
+ sdmtabletree = obj.parse(tablepath + '.xml')
+ sdmxmltable = SDMXMLTable(table.Name.text, table.Entity.get('entityId'), sdmtabletree, os.path.getsize(tablepath + '.xml'))
+ sdmtables[table.Name] = sdmxmltable
+ except XMLSyntaxError:
+ pass
+ elif os.path.isfile(tablepath + '.bin'):
+ sdmbintable = SDMBinaryTable(table.Name.text, table.Entity.get('entityId'), os.path.getsize(tablepath + '.bin'))
+ sdmtables[table.Name] = sdmbintable
+ else:
+ logger.error('SDM Table {} does not exist for {}'.format(table.Name, sdm.fileset))
+ asdmtable = SDMXMLTable('ASDM', sdmroot.Entity.get('entityId'), sdmtree, os.path.getsize(sdmbasepath + '/ASDM.xml'))
+ sdmtables['ASDM'] = asdmtable
+
+ return sdmtables
+
+def addbdffiles(sdm):
+ bdffiles = []
+ maintableroot = sdm.sdmtables['Main'].table.getroot()
+ if len(maintableroot.findall('row')) == 0:
+ logger.error('No BDF files to be loaded for {}'.format(sdm.fileset))
+ return bdffiles
+ for row in maintableroot.row:
+ if row.find('dataUID') is not None:
+ entityid = row.dataUID.EntityRef.get('entityId')
+ elif row.find('dataOid') is not None:
+ entityid = row.dataOid.EntityRef.get('entityId')
+ else:
+ logger.critical('Invalid XML element in Main')
+ if row.find('dataSize') is not None:
+ filesize = row.dataSize
+ else:
+ #TODO for legacy data, find a better way
+ filesize = 0
+ bdffile = BDFFile(entityid, filesize)
+ bdffiles.append(bdffile)
+ return bdffiles
+
+
+if __name__ == '__main__':
+ sdm = buildsdm('15A-017.sb30393593.eb30429744.57067.21779167824', '/Users/rlively/Development/python/testdata', '.')
diff --git a/apps/cli/executables/pexable/ingest/ingest/files/__init__.py b/apps/cli/executables/pexable/ingest/ingest/files/__init__.py
new file mode 100644
index 000000000..36c8f3fc1
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/files/__init__.py
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from .ngasclient import ngasstatus, archivefile, retrievefile
+from .ingestobs import ingest, ingestBDF, ingestSDM, ingestmetadata
+
+__all__ = ["ngasstatus", "archivefile", "retrievefile", "ingest", "ingestBDF", "ingestSDM", "ingestmetadata"]
diff --git a/apps/cli/executables/pexable/ingest/ingest/files/ingestobs.py b/apps/cli/executables/pexable/ingest/ingest/files/ingestobs.py
new file mode 100644
index 000000000..b1665b74f
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/files/ingestobs.py
@@ -0,0 +1,48 @@
+from . import ngasclient as nc
+import os
+import shutil
+
+
+def ingest(sdm,ngasstagingpath):
+ ingestSDM(sdm, ngasstagingpath)
+ ingestBDF(sdm, ngasstagingpath)
+ # ingestmetadata(sdm)
+
+
+def ingestSDM(sdm, ngasstagingpath):
+ for table in sdm.tables:
+ filename = sdm.tables[table].entityid
+ filename = filename.replace('://', '___').replace('/', '_')
+ if (sdm.tables[table].path.endswith('.xml')):
+ extension = '.sdm'
+ else:
+ extension = '.bin'
+ stagingfile = os.path.join(ngasstagingpath, filename) + extension
+ if os.path.exists(sdm.tables[table].path):
+ # For now copy, don't move file
+ # shutil.move(sdm.tables[table].path, stagingfile)
+ shutil.copy(sdm.tables[table].path, stagingfile)
+ if os.path.isfile(stagingfile):
+ nc.archivefile(stagingfile)
+ else:
+ print('File {} does not exist in the ngas staging area'.format(stagingfile))
+
+
+def ingestBDF(sdm, ngasstagingpath):
+ extension = '.bdf'
+ for sourcefile in sdm.bdffiles:
+ destinationfile = os.path.join(ngasstagingpath, os.path.basename(sourcefile)) + extension
+ if os.path.exists(sourcefile):
+ shutil.copy(sourcefile, destinationfile)
+ if os.path.isfile(destinationfile):
+ nc.archivefile(destinationfile)
+ else:
+ print('File {} does not exist in the ngas staging area'.format(destinationfile))
+
+
+def ingestmetadata(sdm):
+ #TODO
+ # Add entries to the metadata db for each SDM Table and BDF ingested into NGAS
+ # Add the science data to the metadata db
+ print('ingesting metadata')
+ # Call the model and pass it the SDM
diff --git a/apps/cli/executables/pexable/ingest/ingest/files/ngasclient.py b/apps/cli/executables/pexable/ingest/ingest/files/ngasclient.py
new file mode 100644
index 000000000..0a6b2d6ba
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/files/ngasclient.py
@@ -0,0 +1,31 @@
+import requests
+import os
+
+#TODO logging
+#TODO capo
+#TODO Error checking
+#TODO Validation
+
+# Example HTTP requests
+# wget -O - http://ngas-alpha.aoc.nrao.edu:7777/STATUS
+# wget -O - http://ngas-alpha.aoc.nrao.edu:7777/ARCHIVE?filename=file:///users/rlively/ngas/Scan.xml
+# wget -O - http://ngas-alpha.aoc.nrao.edu:7777/STATUS?file_id=Scan.xml
+
+NGASHOST = 'http://ngas-alpha.aoc.nrao.edu:7777/'
+archivecommand = {'status': 'STATUS', 'archive': 'ARCHIVE', 'retrieve': 'RETRIEVE'}
+
+def ngasstatus():
+ response = requests.get(NGASHOST + archivecommand['status'])
+ return response
+
+
+def archivefile(path):
+ print('archive request: ' + NGASHOST + archivecommand['archive'] + '?filename=file://' + path)
+ response = requests.get(NGASHOST + archivecommand['archive'] + '?filename=file://' + path)
+ return response
+
+
+def retrievefile(fileid):
+ print(NGASHOST + archivecommand['retrieve'] + '?' + 'file_id=' + fileid)
+ response = requests.get(NGASHOST + archivecommand['retrieve'] + '?' + 'file_id=' + fileid)
+ return response
diff --git a/apps/cli/executables/pexable/ingest/ingest/logging-config.ini b/apps/cli/executables/pexable/ingest/ingest/logging-config.ini
new file mode 100644
index 000000000..6da1860c3
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/logging-config.ini
@@ -0,0 +1,28 @@
+[loggers]
+keys=root,ingestion
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_ingestion]
+level=DEBUG
+handlers=consoleHandler
+qualname=ingestion
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=INFO
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[formatter_simpleFormatter]
+format=%(asctime)s %(module)s[%(funcName)s:%(lineno)s] %(levelname)s: %(message)s
+datefmt=
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/ingest/manifestpersist.py b/apps/cli/executables/pexable/ingest/ingest/manifestpersist.py
new file mode 100644
index 000000000..cb8df01b9
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/manifestpersist.py
@@ -0,0 +1,782 @@
+import logging
+from pathlib import Path
+import datetime
+import hashlib
+import time
+import os
+import sys
+
+from . import IngestionManifest as im
+from .IngestionManifest import IngestionManifest
+
+from pycapo import CapoConfig
+from . import archiveutils as au
+from . import almautils as alu
+from . import remotecopy as rc
+from . import persistmetadata as pm
+from .schema import create_session
+from .schema.model import Filegroup, File, Calibration, ScienceProduct, AncillaryProduct, ProductGroup, ScienceProductsProductGroup, \
+ AlmaSourceDatum, AlmaSpwDatum, Image
+from . import FITSUtil as fi
+from . import NGASUtil as ng
+from .JSONMetadataUtil import JSONMetadata
+
+from . import evlasciencedatamodel as sc
+from .NGASUtil import sha256
+
+config = CapoConfig()
+site_name = config.getstring('archive-ingestion.NGASSite')
+ngas_cluster = config.getstring('archive-ingestion.NGASCluster')
+
+LOG = logging.getLogger('ingestion')
+_SCIENCE_TARGET = 'target'
+
+session = create_session('SDM')
+ngas_files_to_ingest = []
+
+
+def ingest_from_manifest(path, manifest):
+ LOG.info("Ingestion via ingestion manifest")
+ telescope, operation, reingest, ngas_ingest, ingestion_path, additional_metadata_file, collection_metadata_file = manifest.get_ingestion_parameters2()
+ LOG.debug(f"telescope: {telescope}")
+ LOG.debug(f"operation: {operation}")
+ LOG.debug(f"reingest: {reingest}")
+ LOG.debug(f"ngas_ingest: {ngas_ingest}")
+ LOG.debug(f"path: {path}")
+ LOG.debug(f"ingestion_path: {ingestion_path}")
+ LOG.debug(f"collection metadata file: {collection_metadata_file}")
+ input_group = manifest.get_input_group()
+ output_group = manifest.get_output_group()
+ associate_group = manifest.get_associate_group()
+ LOG.debug(f"associaate group: {associate_group}")
+
+ fileset = ''
+
+ if operation == im.NAASC_CALIBRATION:
+ mous = manifest.get_output_group_science_products()[0]['filename']
+ elif operation == im.EXECUTION_BLOCK:
+ fileset = manifest.get_output_group_science_products()[0]['filename']
+ mous = None
+ else:
+ mous = None
+
+ input_science_products, output_science_products = persist_science_metadata(telescope, operation, input_group, output_group, reingest, ngas_ingest, ingestion_path, additional_metadata_file, collection_metadata_file, path)
+ LOG.debug(f"*Input science products-2: {input_science_products}")
+ LOG.debug(f"*Output science products-2: {output_science_products}")
+ LOG.debug(f"*Output group ancillary products: {output_group['ancillary_products']}")
+ # Persist product groups and associated objects (science_product_product_groups, science_product_project)
+ persist_groups(telescope, mous, input_science_products, output_science_products, output_group['ancillary_products'], associate_group, manifest, ingestion_path)
+ if ngas_ingest and len(ngas_files_to_ingest) > 0:
+ ngas_archive(ngas_files_to_ingest, ingestion_path, fileset)
+
+
+def persist_science_metadata(telescope, operation, input_group, output_group, reingest, ngas_ingest, ingestion_path, additional_metadata_file, collection_metadata_file, path):
+ if input_group is not None:
+ # Input group will be empty for SDM ingestion
+ input_group_science_products = input_group['science_products']
+ else:
+ input_group_science_products = []
+ input_science_products = []
+ output_group_science_products = output_group['science_products']
+ output_group_ancillary_products = output_group['ancillary_products']
+
+ LOG.debug(f"Operation: {operation}")
+
+ if operation == im.CALIBRATION:
+ input_science_products, output_science_products = persist_calibration_metadata(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products, reingest, ngas_ingest, ingestion_path)
+ elif operation == im.NAASC_CALIBRATION:
+ input_science_products, output_science_products = persist_naasc_calibration_metadata(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products, reingest, ngas_ingest, ingestion_path)
+ elif operation == im.EXECUTION_BLOCK:
+ output_science_products = persist_sdm(telescope, output_group_science_products, reingest, ingestion_path, additional_metadata_file, collection_metadata_file, path)
+ elif operation == im.CONTINUUM_IMAGE or im.IMAGE_CUBE:
+ input_science_products, output_science_products = persist_images(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products, reingest, ingestion_path, additional_metadata_file, collection_metadata_file, path)
+ else:
+ LOG.critical(f"{operation} is not a valid operation")
+ sys.exit(1)
+
+ return input_science_products, output_science_products
+
+
+def persist_sdm(telescope, output_group_science_products, reingest, ingestion_path, additional_metadata_file, collection_metadata_file, path):
+ LOG.debug(f"SDM OGSP: {output_group_science_products}")
+ input_science_products = []
+ output_science_products = []
+ output_ancillary_products = []
+
+ sdm_identifier = output_group_science_products[0]['filename']
+ if telescope == 'ALMA':
+ # ASDM UIDs are not disk-friendly, so look for a modified name:
+ sdm_identifier = sdm_identifier.replace(':', '_').replace('/', '_')
+
+ exec_block_file = Path(ingestion_path) / sdm_identifier
+ if not exec_block_file.exists():
+ LOG .error(f"File set {exec_block_file.name} not found in {ingestion_path}.")
+ sys.exit(1)
+ # Now make sure the ASDM file exists
+ asdm_file = exec_block_file / 'ASDM.xml'
+ if not asdm_file.exists():
+ LOG.error(f"ASDM.xml not found in {exec_block_file.name}")
+ sys.exit(1)
+
+ execution_block = au.getexecutionblockfromname(exec_block_file.name)
+
+ if execution_block is not None:
+ if reingest:
+ LOG.debug(f"Re-ingestion requested")
+ else:
+ LOG.error(f"Execution block {execution_block.ngas_fileset_id} already exists in archive and re-ingestion not requested.")
+ sys.exit(1)
+
+ else:
+ LOG.debug(f"Initial ingestion of {exec_block_file.name}")
+
+ sdm = sc.buildsdm(exec_block_file.name, ingestion_path)
+ if len(sdm.bdffiles) > 0: # ingest science meta data
+ LOG.debug(f"SDM for {sdm.fileset} has {len(sdm.bdffiles)} BDFs")
+ LOG.debug(f"SDM Tables {sdm.sdmtables.keys()}")
+ starttime = time.time()
+ additional_metadata = None
+ if additional_metadata_file is not None:
+ additional_metadata_file = Path(path) / additional_metadata_file
+ if additional_metadata_file.exists():
+ # Execution block has additional metadata
+ additional_metadata = JSONMetadata(additional_metadata_file)
+ collection_metadata = None
+ if collection_metadata_file is not None:
+ collection_metadata_file = Path(path) / collection_metadata_file
+ if collection_metadata_file.exists():
+ # Execution block has additional metadata
+ collection_metadata = JSONMetadata(collection_metadata_file)
+ project, science_products, ingest_files = pm.persist(sdm, False, additional_metadata, collection_metadata, output_group_science_products, ingestion_path, ngas_cluster)
+ ngas_files_to_ingest.extend(ingest_files)
+ LOG.info(f"SDM metadata ingestion complete for {sdm.fileset} in {round(time.time() - starttime, 2)}s")
+ else:
+ LOG.error(f"Execution block: {sdm.fileset} has no bdfs and will not be ingested")
+
+
+ return science_products
+
+
+def persist_calibration_metadata(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products, reingest, ngas_ingest, ingestion_path):
+ LOG.debug(f"Persisting {telescope} calibration")
+ if not valid_calibration_groups(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products):
+ sys.exit(1)
+
+ LOG.debug(f"IG SP: {input_group_science_products}")
+ LOG.debug(f"OG SP: {output_group_science_products}")
+ LOG.debug(f"OG AP: {output_group_ancillary_products}")
+
+ input_science_products = []
+ output_science_products = []
+
+ for science_product in input_group_science_products:
+ input_science_products.append(au.get_science_product_by_spl(science_product['locator']))
+
+ for science_product in output_group_science_products:
+ calibration_filename = science_product['filename']
+ tar_file = Path(ingestion_path) / calibration_filename
+ if not tar_file.exists():
+ LOG.warning(f"Calibration products file: {tar_file.name} does not exist in {tar_file.parent}.")
+ sys.exit(1)
+ calibration_tar_file = au.get_file(calibration_filename)
+ if calibration_tar_file is not None: # The file already exists
+ if reingest:
+ LOG.info("Re-ingestion")
+ calibration_science_product = au.get_science_product(calibration_filename)
+ calibration_science_product.metadata_ingestion_date = datetime.datetime.utcnow()
+ calibration_science_product.metadata_ingestion_version = int(calibration_science_product.metadata_ingestion_version) + 1
+ output_science_products.append(calibration_science_product)
+ LOG.debug(f"Output science products: {output_science_products}")
+
+ calibration_tar_file.ingestion_time = datetime.datetime.utcnow()
+ calibration_tar_file.checksum = sha256(Path(ingestion_path) / calibration_tar_file.filename)
+ db_session = session.object_session(calibration_science_product)
+ db_session.add(calibration_science_product)
+ db_session.add(calibration_tar_file)
+ ngas_files_to_ingest.append((calibration_tar_file, calibration_tar_file.filesize))
+ else:
+ LOG.error(f"The file {calibration_filename} exists and re-ingestion not requested. Exiting.")
+ sys.exit(1)
+ else:
+ LOG.debug("Initial ingestion")
+
+ calibration_science_product_locator = ng.generate_science_product_locator(input_science_products[0].execution_block.telescope.lower(), science_product['type'])
+
+ # Create calibration Filegroup and File
+ calibration_filegroup = Filegroup(
+ parent_filegroup=input_science_products[0].execution_block.filegroup,
+ type='calibration'
+ )
+
+ calibration_products_file = File(
+ file_path=str(tar_file.parent),
+ ngas_id=ng.generate_uuids_for_ngas(telescope, 'calibration', 'tar'),
+ ngas_location=site_name,
+ ngas_cluster=ngas_cluster,
+ filename=tar_file.name,
+ filegroup1=calibration_filegroup,
+ filesize=tar_file.stat().st_size,
+ format='tar',
+ type='cal',
+ checksum=sha256(tar_file),
+ checksum_type='SHA256',
+ ingestion_time=datetime.datetime.utcnow()
+ )
+
+ ngas_files_to_ingest.append((calibration_products_file, calibration_products_file.filesize))
+
+ calibration = Calibration(
+ science_product_locator=calibration_science_product_locator
+ )
+
+ calibration.execution_block = input_science_products[0].execution_block
+ calibration.project = input_science_products[0].execution_block.project
+ calibration.filegroup = calibration_filegroup
+
+ # Create ScienceProduct
+ output_science_product = ScienceProduct(
+ science_product_locator=calibration_science_product_locator,
+ science_product_type='Calibration',
+ metadata_ingestion_date=datetime.datetime.utcnow(),
+ filegroup=calibration_filegroup,
+ metadata_ingestion_version=1,
+ external_system='EVLA Processing',
+ external_name=tar_file.name
+ )
+
+ output_science_products.append(output_science_product)
+
+ # Update science_products_projects
+ calibration.execution_block.project.science_products.append(output_science_product)
+
+ db_session = session.object_session(calibration.execution_block)
+ db_session.add(calibration_filegroup)
+ db_session.add(calibration_products_file)
+ db_session.add(calibration)
+ db_session.add(output_science_product)
+
+ LOG.debug(f"Input science products: {input_science_products}")
+ LOG.debug(f"Output science products: {output_science_products}")
+
+ db_session.commit()
+
+ # output_auxiliary_products will only contain auxiliary products associated with science products
+ # auxiliary products associated with a group will be added when groups are created
+
+ return input_science_products, output_science_products
+
+
+def persist_naasc_calibration_metadata(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products, reingest, ngas_ingest, ingestion_path):
+ LOG.debug(f"Persisting {telescope} calibration")
+ if not valid_calibration_groups(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products):
+ sys.exit(1)
+
+ LOG.debug(f"IG SP: {input_group_science_products}")
+ LOG.debug(f"OG SP: {output_group_science_products}")
+ LOG.debug(f"OG SP: {output_group_ancillary_products}")
+
+ input_science_products = []
+ output_science_products = []
+ eb_locator_list = []
+
+ for science_product in input_group_science_products:
+ input_science_products.append(au.get_science_product_by_spl(science_product['locator']))
+ eb_locator_list.append(science_product['locator'])
+
+ # Returns AlmaOus object
+ mous = au.get_mous_from_eb_science_product_locator_list(eb_locator_list)
+ session.add(mous)
+
+ LOG.debug(f"Input science products: {input_group_science_products}")
+
+ science_product_locator = ng.generate_science_product_locator(telescope.lower(), 'calibration')
+ session.add(input_science_products[0])
+ project_code = input_science_products[0].filegroup.project_code
+
+ LOG.debug(f"Project code: {project_code}")
+
+ calibration_filegroup = Filegroup(
+ project_code=project_code,
+ type='calibration'
+ )
+
+ LOG.debug(f"Calibration filegroup: project_code={calibration_filegroup.project_code} type={calibration_filegroup.type}")
+
+ session.add(calibration_filegroup)
+
+ calibration_files = alu.getalmacalfiles(mous.alma_ous_id)
+
+ for calibration_file in calibration_files:
+ file = File(
+ ngas_id = calibration_file[0],
+ ngas_location='NAASC',
+ ngas_cluster='NAASC',
+ filename=calibration_file[0],
+ filesize=calibration_file[1],
+ filegroup1=calibration_filegroup,
+ format='cal',
+ type='cal',
+ ingestion_time=datetime.datetime.utcnow()
+ )
+ LOG.debug(f"File ngas_id={file.ngas_id} ngas_location={file.ngas_location} filename={file.filename} filesize={file.filesize} ingestion time: {file.ingestion_time}")
+ session.add(file)
+
+ calibration = Calibration(
+ science_product_locator=science_product_locator,
+ alma_ous=mous,
+ project=input_science_products[0].filegroup.project,
+ filegroup = calibration_filegroup
+ )
+
+ LOG.debug(f"Calibration science product locator={calibration.science_product_locator} alma ous={calibration.alma_ous.alma_ous_id} project code={calibration.project.project_code}")
+
+ session.add(calibration)
+
+ output_science_product = ScienceProduct(
+ science_product_locator=science_product_locator,
+ science_product_type='Calibration',
+ metadata_ingestion_date=datetime.datetime.utcnow(),
+ filegroup=calibration_filegroup,
+ metadata_ingestion_version=1,
+ external_system='ALMA Processing',
+ external_name=mous.alma_ous_id
+
+ )
+
+ output_science_products.append(output_science_product)
+ session.add(output_science_product)
+
+ # Update science_products_projects - this is the problem. No, this is working now
+ calibration.project.science_products.append(output_science_product)
+
+ LOG.debug(f"Science product science product locator={output_science_product.science_product_locator} science product type={output_science_product.science_product_type} metadata ingestion date={output_science_product.metadata_ingestion_date} metadata ingestion version={output_science_product.metadata_ingestion_version}")
+
+ source_list = alu.get_source_list(mous.alma_ous_id)
+
+ for source in source_list:
+ alma_source = AlmaSourceDatum(
+ alma_ous_id=mous.alma_ous_id,
+ field_name=source.source_name,
+ ra=source.ra_source,
+ dec=source.dec_source,
+ integration_time=source.int_time,
+ angular_resolution = source.spatial_scale_min,
+ largest_angular_scale = source.spatial_scale_max,
+ science_field=_SCIENCE_TARGET in source.scan_intent.lower()
+ )
+
+ LOG.debug(f"AlmaSourceDatum alma_ouss_id={alma_source.alma_ous_id} field name={alma_source.field_name} ra={alma_source.ra} dec={alma_source.dec} integration time={alma_source.integration_time}")
+
+ spectral_window_data = alu.get_spectral_window_data(source)
+ for spw in spectral_window_data:
+ spectral_window = AlmaSpwDatum(
+ alma_ous_id = mous.alma_ous_id,
+ spw_name = spw.spectral_window_name,
+ min_frequency = spw.frequency_min,
+ max_frequency = spw.frequency_max,
+ bandwidth = spw.bandwidth,
+ num_channels = spw.channel_num,
+ spectral_resolution = spw.resolution_max
+ )
+ if spw.resolution_min != spw.resolution_max:
+ spectral_window.spectral_resolution = (spw.resolution_max + spw.resolution_min) / 2
+
+ LOG.debug(f"AlmaSpwDatum alma_ous id={spectral_window.alma_ous_id} spw name: {spectral_window.spw_name}")
+
+ session.merge(spectral_window)
+ alma_source.spectral_windows.append(spectral_window)
+ session.merge(alma_source)
+
+ session.commit()
+
+ return input_science_products, output_science_products
+
+
+def persist_images(telescope, input_group_science_products, output_group_science_products, output_group_ancillary_products, reingest, ingestion_path, metadata_file, collection_metadata_file, path):
+ """
+ Persist an image (quicklook, image cube, etc.)
+
+ :param telescope:
+ :param input_group_science_products:
+ :param output_group_science_products:
+ :param output_group_ancillary_products:
+ :param reingest:
+ :param ngas_ingest:
+ :param ingestion_path:
+ :param metadata_file:
+ :param path:
+ :return:
+ """
+
+ LOG.debug(f"IG SP: {input_group_science_products}")
+ LOG.debug(f"OG SP: {output_group_science_products}")
+ LOG.debug(f"OG AP: {output_group_ancillary_products}")
+
+ input_science_products = []
+ output_science_products = []
+
+ # JSON metadata
+ json_metadata_file = Path(path) / metadata_file
+ if not json_metadata_file.exists():
+ LOG.critical(f"Metadata file {json_metadata_file.name} does not exist in {json_metadata_file.parent}")
+ sys.exit(1)
+ json_metadata = JSONMetadata(json_metadata_file)
+ json_metadata.from_json(json_metadata_file)
+ project_code = json_metadata.get_project_code()
+
+ project = au.getproject(project_code)
+
+ LOG.debug(f"Project code: {project.project_code}")
+ if project is None:
+ LOG.error(f"Project {project.project_code} not found")
+ sys.exit(1)
+
+ for science_product in input_group_science_products:
+ input_science_products.append(au.get_science_product_by_spl(science_product['locator']))
+
+ for science_product in output_group_science_products:
+ science_product_type = science_product['type']
+ LOG.debug(f"Science product type: {science_product_type}")
+ image_file = Path(ingestion_path) / science_product['filename']
+ LOG.debug(f"Image path: {image_file.parent} Image file: {image_file.name}")
+ science_product_locator = ng.generate_science_product_locator(telescope.lower(), science_product_type)
+
+ # Create image Filegroup and File
+ filegroup = Filegroup(
+ parent_filegroup=None,
+ # parent_filegroup=input_science_products[0].execution_block.filegroup,
+ type=science_product_type
+ )
+
+ file = File(
+ file_path=str(image_file.parent),
+ ngas_id=ng.generate_uuids_for_ngas(telescope, science_product_type, image_file.suffix[1:]),
+ ngas_location=site_name,
+ ngas_cluster=ngas_cluster,
+ filename=image_file.name,
+ filegroup1=filegroup,
+ filesize=image_file.stat().st_size,
+ format='fits',
+ type=science_product_type,
+ checksum=sha256(image_file),
+ checksum_type='SHA256',
+ ingestion_time=datetime.datetime.utcnow()
+ )
+
+ ngas_files_to_ingest.append((file, file.filesize))
+
+ session.add(filegroup)
+ session.add(file)
+
+ source_name, telescope, spatial_resolution, field_of_view, min_intensity, max_intensity, ra, dec, min_freq, max_freq, rest_frequency, \
+ ra_element_count, dec_element_count, image_units, beam_axis_ratio, polarization_id, rms_noise, spatial_region, ra_pixel_size, \
+ dec_pixel_size = fi.get_fits_data(image_file)
+
+ start_time = json_metadata.get_starttime()
+ if start_time is None:
+ start_time = 0
+ end_time = json_metadata.get_endtime()
+ if end_time is None:
+ end_time = 0
+ if rms_noise is None and json_metadata.get_rms_noise() is None:
+ rms_noise = 0
+
+ image = Image(target_name=source_name,
+ telescope=telescope,
+ spatial_resolution=spatial_resolution,
+ image_field_of_view=field_of_view,
+ max_intensity=max_intensity,
+ min_intensity=min_intensity,
+ rms_noise=rms_noise,
+ polarization_id=polarization_id,
+ ra=ra,
+ dec=dec,
+ min_frequency=min_freq,
+ max_frequency=max_freq,
+ ra_element_count=ra_element_count,
+ dec_element_count=dec_element_count,
+ starttime=start_time,
+ endtime=end_time,
+ exposure_time=end_time,
+ rest_frequency=rest_frequency,
+ image_units=image_units,
+ spatial_region=spatial_region,
+ beam_axis_ratio=beam_axis_ratio,
+ band_code=json_metadata.get_band_code(),
+ ra_pixel_size=ra_pixel_size,
+ dec_pixel_size=dec_pixel_size,
+ tags=json_metadata.get_image_tags(),
+ collection_name=json_metadata.get_collection_name(),
+ calibration_level=json_metadata.get_calibration_level(),
+ science_product_locator=science_product_locator,
+ file=file)
+
+ # Ignore re-ingestion for now, but how would you do this
+
+ external_system = telescope + ' Processing'
+ output_science_product = ScienceProduct(
+ science_product_locator=science_product_locator,
+ science_product_type='Image',
+ metadata_ingestion_date=datetime.datetime.utcnow(),
+ filegroup=filegroup,
+ metadata_ingestion_version=1,
+ external_system=external_system,
+ external_name=image_file.name
+ )
+
+ session.add(image)
+ session.add(output_science_product)
+
+ output_science_product.project = project
+ session.add(output_science_product)
+
+ output_science_products.append(output_science_product)
+
+ # If this science product has ancillary products
+ if 'ancillary_products' in science_product.keys():
+ for anc_product in science_product['ancillary_products']:
+ LOG.debug(f"Ancillary product: {anc_product}")
+ product_file = Path(ingestion_path) / anc_product['filename']
+ ancillary_product_locator = ng.generate_science_product_locator(telescope.lower(), 'ancillary_product')
+ ancillary_product_filegroup = Filegroup (
+ type='ancillary_product',
+ datasize=-1,
+ )
+ if anc_product['type'] == 'thumbnail_image':
+ thumbnail_path = str(Path(ingestion_path) / product_file.name)
+ preview_path = str(Path(calculate_thumbnail_path(thumbnail_path)[0]).parent)
+ else:
+ preview_path = None
+ ancillary_product_file = File (
+ ngas_id=ng.generate_uuids_for_ngas(telescope, 'ancillary_product', product_file.suffix[1:]),
+ ngas_location=site_name,
+ ngas_cluster=ngas_cluster,
+ filename=product_file.name,
+ filesize=product_file.stat().st_size,
+ format=product_file.suffix[1:],
+ type=anc_product['type'],
+ checksum=sha256(product_file),
+ checksum_type='SHA256',
+ ingestion_time=datetime.datetime.utcnow(),
+ filegroup1=ancillary_product_filegroup,
+ preview_storage_path=preview_path
+ )
+
+ ngas_files_to_ingest.append((ancillary_product_file, ancillary_product_file.filesize))
+
+ ancillary_product = AncillaryProduct(
+ ancillary_product_locator=ancillary_product_locator,
+ ancillary_product_type=anc_product['type'],
+ filegroup=ancillary_product_filegroup,
+ science_product_locator=output_science_product.science_product_locator
+ )
+ session.add(ancillary_product_file)
+ session.add(ancillary_product)
+
+ session.commit()
+
+ return input_science_products, output_science_products
+
+
+def persist_groups(telescope, mous, input_science_products, output_science_products, output_group_ancillary_products, associate_group, manifest, ingestion_path):
+ LOG.debug(f"ISPs: {input_science_products}")
+ LOG.debug(f"OSPs: {output_science_products}")
+ LOG.debug(f"OGAPs: {output_group_ancillary_products}")
+ LOG.debug(f"mous: {mous}")
+
+ input_product_group = ProductGroup(
+ product_group_type='pipeline input'
+ )
+
+ db_session = session.object_session(output_science_products[0])
+
+ input_product_group.science_products.extend(input_science_products)
+
+ output_product_group = ProductGroup(
+ product_group_type='pipeline output',
+ parent_product_group=input_product_group,
+ alma_ous_id=mous
+ )
+
+ output_product_group.science_products.extend(output_science_products)
+
+ for anc_product in output_group_ancillary_products:
+ product_file = Path(ingestion_path) / anc_product['filename']
+ ancillary_product_locator = ng.generate_science_product_locator(telescope.lower(), 'ancillary_product')
+ ancillary_product_filegroup = Filegroup (
+ type='ancillary_product',
+ datasize=-1,
+ )
+
+ if anc_product['type'] == 'thumbnail_image':
+ thumbnail_path = str(Path(ingestion_path) / product_file.name)
+ preview_path = str(Path(calculate_thumbnail_path(thumbnail_path)[0]).parent)
+ else:
+ preview_path = None
+
+ ancillary_product_file = File (
+ ngas_id=ng.generate_uuids_for_ngas(telescope, anc_product['type'], product_file.suffix[1:]),
+ ngas_location=site_name,
+ ngas_cluster=ngas_cluster,
+ filename=product_file.name,
+ filesize=product_file.stat().st_size,
+ format=product_file.suffix[1:],
+ type=anc_product['type'],
+ checksum=sha256(product_file),
+ checksum_type='SHA256',
+ ingestion_time=datetime.datetime.utcnow(),
+ filegroup1=ancillary_product_filegroup,
+ preview_storage_path=preview_path
+ )
+
+ ngas_files_to_ingest.append((ancillary_product_file, ancillary_product_file.filesize))
+
+ ancillary_product = AncillaryProduct(
+ ancillary_product_locator=ancillary_product_locator,
+ ancillary_product_type=anc_product['type'],
+ filegroup=ancillary_product_filegroup,
+ product_group=output_product_group
+ )
+ db_session.add(ancillary_product_file)
+ db_session.add(ancillary_product)
+
+ db_session.add(input_product_group)
+ db_session.add(output_product_group)
+
+ # Associate group links ebs
+ if associate_group is not None:
+ donor_science_product = au.get_science_product_by_spl(manifest.get_associate_group_locator())
+ if donor_science_product is None:
+ LOG.critical(f"Science product for donor eb not found: {manifest.get_associate_group_locator()}")
+ else:
+ associate_product_group = ProductGroup(
+ product_group_type='associate',
+ )
+ # Add the SPL for the associated eb
+ associate_product_group.science_products.extend(output_science_products)
+ # Add the SPL for the donor eb
+ local_donor_science_product = db_session.merge(donor_science_product)
+ associate_product_group.science_products.extend([local_donor_science_product])
+
+ db_session.add(associate_product_group)
+
+ db_session.commit()
+
+
+def valid_calibration_groups(telescope, input_science_products, output_science_products, output_ancillary_products):
+ if telescope.lower() == 'evla':
+ if len(input_science_products) != 1:
+ print(f"{len(input_science_products)} science products in input group. Only 1 execution block is permitted")
+ return False
+ if len(output_science_products) != 1:
+ print(f"{len(output_science_products)} science products in output group. Only 1 calibration product is permitted")
+ return False
+ elif telescope.lower() == 'alma':
+ return True
+ if len(output_science_products) != 1:
+ print(f"{len(output_science_products)} science products in output group. Only 1 calibration product is permitted")
+ return False
+ else:
+ print(f"Invalid telescope in manifest: {telescope}")
+ return False
+ return True
+
+
+def ngas_archive(files_to_archive, ingestion_path, fileset=''):
+ LOG.debug(f"Ingesting {len(files_to_archive)} files into ngas....")
+ thumbnails = []
+ files_to_ingest = []
+ for file in files_to_archive:
+ # Some convenience variables:
+ file_type = file[0].type
+ file_format = file[0].format
+ if file_type == 'raw':
+ if file_format == 'sdm':
+ path_to_file = Path(ingestion_path) / fileset / file[0].filename
+ path_to_ngas_rename = Path(ingestion_path) / fileset / file[0].ngas_id
+ elif file_format == 'bdf':
+ path_to_file = Path(ingestion_path) / fileset / 'ASDMBinary' / file[0].filename
+ path_to_ngas_rename = Path(ingestion_path) / fileset / 'ASDMBinary' / file[0].ngas_id
+ else:
+ path_to_file = Path(ingestion_path) / file[0].filename
+ path_to_ngas_rename = Path(ingestion_path) / file[0].ngas_id
+
+ if file_type == 'thumbnail_image':
+ # Thumbnails need to go to the cache as well as NGAS,
+ # track those for the additional step.
+ thumbnails.append(path_to_file)
+
+
+ if path_to_file != path_to_ngas_rename:
+ if os.access(path_to_ngas_rename, os.F_OK):
+ # If the file's NGAS name exists in the staging area already, it means this file was
+ # recently ingested, and we're re-ingesting an updated version. Remove the
+ # old hard link to facilitate the new data.
+ os.unlink(path_to_ngas_rename)
+ # There are cases, in particular with old calibrations, where the
+ # ngas id is the filename. That causes os.link to nuke the file
+ # which is just bad news. So, don't link if we don't have to.
+ os.link(path_to_file, path_to_ngas_rename)
+
+ # add the file with appropriate NGAS naming to the to-do list:
+ # LOG.debug(f"Adding {path_to_ngas_rename} to the ingestion list")
+ files_to_ingest.append([str(path_to_ngas_rename), file[1]])
+
+ # With everything prepped, make the NGAS calls.
+ #LOG.debug(f"Total set of files: {files_to_ingest}")
+ ng.archivefiles(files_to_ingest)
+
+ #Handle the thumbnails:
+ LOG.debug(f"Thumbs: {thumbnails}")
+ # copy_thumbnails(thumbnails, ingestion_path)
+
+
+def calculate_thumbnail_path(thumbnail_image_file):
+ sha1 = hashlib.sha1()
+
+ with open(str(thumbnail_image_file), 'rb') as f:
+ for block in iter(lambda: f.read(1024), b''):
+ sha1.update(block)
+ key = sha1.hexdigest()
+
+ # $1 is the first two characters of the sha1 sum, $2 the second two, and $3 the third two
+ # ea887dea73301fab8b9da760014449b59e1211ae
+ # $ROOT/ea/88/7d/thumbnail
+
+ root_thumbnail_path = config.getstring('archive-ingestion.vlass.ThumbnailPath')
+
+ # Return path without and with ROOT
+ return key[0:2] + '/' + key[2:4] + '/' + key[4:6] + '/' + os.path.basename(thumbnail_image_file), root_thumbnail_path + '/' + key[0:2] + '/' + key[2:4] + '/' + key[4:6] + '/' + os.path.basename(thumbnail_image_file)
+
+
+# def copy_thumbnail(thumbnail_image_file, thumbnail_destination):
+# if not os.path.isdir(os.path.dirname(thumbnail_destination)):
+# os.makedirs(os.path.dirname(thumbnail_destination))
+# shutil.copyfile(thumbnail_image_file, thumbnail_destination)
+
+
+def copy_thumbnails(thumbnails, ingestion_path):
+ for thumbnail in thumbnails:
+ # full path to the png:
+ thumb_source = Path(ingestion_path) / thumbnail
+ # figure out where it goes (based on checksum):
+ doubled_locations = calculate_thumbnail_path(thumb_source)
+ #
+ # The above returns the resultant path with and without the root value from CAPO
+ # we want the second value of the returned tuple
+ #
+ rc.copy_thumbnail(thumb_source, doubled_locations[1])
+
+
+def filesdump(ingest_files):
+ for file in ingest_files:
+ LOG.info(f"File name: {file[0].filename} Size: {file[1]}")
+
+
+def test1():
+ path = Path('.')
+ ingest_from_manifest(path.name, im.INGESTION_MANIFEST_FILE)
+
+
+if __name__ == '__main__':
+ test1()
diff --git a/apps/cli/executables/pexable/ingest/ingest/metadata/__init__.py b/apps/cli/executables/pexable/ingest/ingest/metadata/__init__.py
new file mode 100644
index 000000000..a8a2a5589
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/metadata/__init__.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+from .sciencemodel import SDM, SDMTable, SDMBinaryTable, buildtable
+
+__all__ = [
+ "SDM",
+ "SDMTable",
+ "SDMBinaryTable",
+ "buildtable",
+]
diff --git a/apps/cli/executables/pexable/ingest/ingest/metadata/sciencemodel.py b/apps/cli/executables/pexable/ingest/ingest/metadata/sciencemodel.py
new file mode 100644
index 000000000..46bcc0c34
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/metadata/sciencemodel.py
@@ -0,0 +1,55 @@
+import os
+from lxml import objectify
+
+class SDM:
+ def __init__(self, sdmpath='.', bdfpath='.'):
+ self.sdmpath = os.path.abspath(sdmpath)
+ self.bdfpath = os.path.abspath(bdfpath)
+ self.tables = {} # These are the SDM Tables {name : root of the xml doc}
+ self.bdffiles = []
+ asdmtree = objectify.parse(sdmpath + '/ASDM.xml')
+ self.asdm = asdmtree.getroot()
+
+ # Load the SDM tables
+ for table in self.asdm.Table:
+ entityid = table.Entity.get('entityId')
+ self.tables[table.Name] = buildtable(table.Name, self.sdmpath, entityid)
+ # Load the BDF files
+ bdffiletree = objectify.parse(sdmpath + '/Main.xml')
+ bdfs = bdffiletree.getroot()
+ for rw in bdfs.row:
+ # XML Change
+ if row.dataUID is not None:
+ file = rw.dataUID.EntityRef.get('entityId')
+ elif row.dataOid is not None:
+ file = rw.dataOid.EntityRef.get('entityId')
+ else:
+ raise 'Invalid XML element'
+ # for testing
+ # self.bdffiles.append(os.path.join(bdfpath, file.replace(':///', '____').replace('/', '_') + '.loaded'))
+ self.bdffiles.append(os.path.join(bdfpath, file.replace(':///', '____').replace('/', '_')))
+
+
+def buildtable(name, path, entityid):
+ filename = os.path.join(path, str(name))
+ if os.path.exists(filename + '.xml'):
+ return SDMTable(name, filename + '.xml', entityid)
+ elif os.path.exists(filename + '.bin'):
+ return SDMBinaryTable(name, filename + '.bin', entityid)
+ else:
+ raise 'Not a valid table type'
+
+class SDMTable:
+ def __init__(self, name, path, entityid):
+ self.name = name
+ self.path = path
+ # self._tree = objectify.parse(path)
+ # self._table = self._tree.getroot()
+ self.entityid = entityid
+
+class SDMBinaryTable:
+ def __init__(self, name, path, entityid):
+ self.name = name
+ self.path = path
+ self.entityid = entityid
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/persistVLBAmetadata.py b/apps/cli/executables/pexable/ingest/ingest/persistVLBAmetadata.py
new file mode 100644
index 000000000..bbded0e3d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/persistVLBAmetadata.py
@@ -0,0 +1,317 @@
+"""
+Persist VLBA metadata extracted from .idifits files
+"""
+
+import os
+import sys
+import datetime
+import re
+import logging
+import hashlib
+import time
+
+from pycapo import CapoConfig
+from .VLBAFITSUtil import VLBAFits
+from .schema import create_session
+from . import proposalutils, archiveutils
+from . import NGASUtil as ng
+from .schema.model import Project, Filegroup, File, ExecutionBlock, Scan, Subscan, Author, Intent, Configuration, DataDescription, Polarization, ScienceProduct
+
+session = create_session('VLBA')
+LOG = logging.getLogger('ingestion')
+config = CapoConfig()
+
+def persistVLBAMetadata(fits_file):
+ LOG.info(f"Persisting fits file: {fits_file}")
+ # Basically want to bail out of the whole process if the file name is already in the Files table
+ starttime = time.time()
+ vlba_fits = VLBAFits(fits_file)
+ LOG.info(f"Fits file completed in {time.time() - starttime}s")
+ starttime = time.time()
+ projectcode = vlba_fits.getproject()
+ segment = vlba_fits.getsegment()
+ correlator_pass = vlba_fits.getcorrelatorpass()
+ # vlba_fits.getscantable().pprint(max_lines=250, max_width=250)
+
+ project = persistproject(projectcode)
+ # Persist archive location information
+ correlations_filegroup, correlator_pass_filegroup = persistfilegroups(project, segment, correlator_pass)
+
+ file = persistfile(correlator_pass_filegroup, fits_file)
+
+ # Now the science data
+ executionblock = persistexecblock("VLBA", project, segment, correlations_filegroup, vlba_fits, file)
+ if executionblock is not None:
+ if project.starttime is None or project.starttime > executionblock.starttime:
+ project.starttime = executionblock.starttime
+ if project.endtime is None or project.endtime < executionblock.endtime:
+ project.endtime = executionblock.endtime
+
+ session.add(project)
+
+ session.commit()
+
+ LOG.info(f"Database updates for file {fits_file} completed in {time.time() - starttime}s")
+
+
+def persistproject(projectcode):
+
+ existingproject = archiveutils.getproject(projectcode)
+ if existingproject is None:
+ searchproposalcode = re.sub('([A-Z]+)0', '\g<1>', projectcode)
+ pstproposal = proposalutils.getproposal(searchproposalcode)
+ pstauthors = persistauthors(projectcode)
+ if pstproposal is None:
+ psttitle = ""
+ pstabstract = ""
+ if projectcode.startswith('DQ'):
+ # Data analyst test project
+ proprietaryduration = 0
+ else:
+ proprietaryduration = 365
+ else:
+ if pstproposal.PROPOSAL_TYPE == "Director's Discretionary Time":
+ proprietaryduration = round(365 / 2, 0)
+ else:
+ proprietaryduration = 365
+ psttitle = pstproposal.TITLE
+ pstabstract = pstproposal.ABSTRACT
+
+ project = Project(project_code = projectcode,
+ opt_project_id = 0,
+ legacy_id = projectcode,
+ title = psttitle,
+ abstract = pstabstract,
+ total_observation_time = None,
+ last_addition = datetime.datetime.utcnow(),
+ starttime = None,
+ endtime = None,
+ proprietary_duration = proprietaryduration)
+ project.authors = pstauthors
+ session.add(project)
+ return project
+ return existingproject
+
+
+def persistauthors(project):
+ searchproposalcode = re.sub('([A-Z]+)0', '\g<1>', project)
+ pstauthors = proposalutils.getauthors(searchproposalcode)
+ projauthors = []
+ if pstauthors is None or len(pstauthors) == 0:
+ author = Author(project_code=project,
+ username='operations',
+ firstname='VLBA',
+ lastname='Operations',
+ pst_person_id=10154,
+ is_pi=True)
+ session.add(author)
+ projauthors.append(author)
+ else:
+ for pstauthor in pstauthors:
+ projauthors.append(Author(project_code=project, username=pstauthor.user.personName, firstname=pstauthor.FIRST_NAME, lastname=pstauthor.LAST_NAME,
+ pst_person_id=pstauthor.person_id, is_pi=proposalutils.ispi(pstauthor)))
+
+ return projauthors
+
+
+def persistfilegroups(project, segment, correlator_pass):
+ correlations_filegroup = archiveutils.get_vlba_correlations_filegroup(project.project_code, segment, 'correlations')
+
+ if correlations_filegroup is not None:
+ correlator_pass_filegroup = archiveutils.get_vlba_correlator_pass_filegroup(project.project_code, segment, 'correlator_pass')
+ session.add(correlator_pass_filegroup)
+ correlations_filegroup = correlator_pass_filegroup.parent_filegroup
+ if correlator_pass_filegroup is not None:
+ return correlations_filegroup, correlator_pass_filegroup
+ else:
+ filegroup = Filegroup(parent_filegroup = correlations_filegroup,
+ datasize = 0,
+ type='correlator_pass'
+ )
+ session.add(filegroup)
+ return correlations_filegroup, filegroup
+ else:
+ parent_filegroup = Filegroup(project=project,
+ datasize = 0,
+ type = 'correlations'
+ )
+ filegroup = Filegroup(parent_filegroup = parent_filegroup,
+ datasize = 0,
+ type='correlator_pass'
+ )
+ session.add(parent_filegroup, filegroup)
+ return parent_filegroup, filegroup
+
+
+
+def persistfile(filegroup, fits_file):
+ file_path = os.path.basename(fits_file)
+ ngas_id = os.path.basename(fits_file)
+ existingfile = archiveutils.getvlbafile(ngas_id)
+ site_name = config.getstring('archive-ingestion.NGASSite')
+ cluster_name = config.getstring('archive-ingestion.NGASCluster')
+ if existingfile is None:
+ file = File(file_path=file_path,
+ ngas_id=ngas_id,
+ ngas_location=site_name,
+ ngas_cluster=cluster_name,
+ filegroup1=filegroup,
+ filename=os.path.basename(fits_file),
+ filesize=os.path.getsize(fits_file),
+ format=fits_file.suffix[1:].upper(),
+ type=fits_file.suffix[1:].upper(),
+ checksum=sha256(fits_file),
+ checksum_type='SHA256',
+ ingestion_time=datetime.datetime.utcnow())
+
+ file.filegroup1.datasize = file.filegroup1.datasize + file.filesize
+ session.add(file)
+ return file
+ LOG.critical(f"File {ngas_id} already exists. Exiting.")
+ exit(-1)
+
+
+def persistexecblock(telescope, project, segment, filegroup, vlba_fits, file):
+ execblock = archiveutils.getvlbaexecutionblock(project.project_code, segment)
+ science_product_locator = ng.generate_science_product_locator(telescope, 'execblock')
+ if execblock is None:
+ execblock = ExecutionBlock(
+ ost_exec_block_id=0,
+ filegroup=filegroup,
+ calibration_level=0,
+ telescope='VLBA',
+ configuration=None,
+ scheduling_block_id=0,
+ project=project,
+ segment=segment,
+ ngas_fileset_id=None,
+ starttime=0,
+ endtime=0,
+ calibration_status='Do Not Calibrate',
+ scheduling_block_type=None,
+ band_code=vlba_fits.lookup_observing_band(),
+ science_product_locator=science_product_locator)
+
+ science_product = ScienceProduct(
+ science_product_locator=science_product_locator,
+ science_product_type="Execution Block",
+ metadata_ingestion_date=datetime.datetime.utcnow(),
+ metadata_ingestion_version=1,
+ filegroup=filegroup,
+ external_name=project.project_code+segment,
+ external_system="VLBA Operations",
+ execution_block=execblock
+ )
+
+ execblock.project.science_products.append(science_product)
+
+ session.add(execblock)
+ session.add(science_product)
+
+ starttime, endtime = persistscans(vlba_fits, execblock, filegroup, file)
+ execblock.starttime = starttime
+ execblock.endtime = endtime
+ else:
+ starttime, endtime = persistscans(vlba_fits, execblock, filegroup, file)
+ if starttime < execblock.starttime:
+ execblock.starttime = starttime
+ if endtime > execblock.endtime:
+ execblock.endtime = endtime
+
+ return execblock
+
+
+
+def persistscans(vlba_fits, execution_block, filegroup, file):
+ minstarttime, maxendtime = 1000000000.0, 0.0
+ scans = vlba_fits.getscantable()
+ for scan_number in range(len(scans)):
+ scan = Scan(
+ execution_block=execution_block,
+ filegroup=filegroup,
+ min_bandwidth = 0.0,
+ max_bandwidth = 0.0,
+ min_frequency = 0.0,
+ max_frequency = 0.0,
+ polarization_code = 0
+ )
+ minfrequency, maxfrequency, bandwidth, starttime, endtime, polarization = persistsubscans(execution_block, scan, scans[scan_number], file)
+
+ # MHz --> Hz
+ scan.min_frequency = minfrequency * 1000000
+ scan.max_frequency = maxfrequency * 1000000
+
+ scan.min_bandwidth = bandwidth
+ scan.max_bandwidth = bandwidth
+ # scan.polarization_code = polarization.polarization_id
+ session.add(scan)
+ if starttime < minstarttime:
+ minstarttime = starttime
+ if endtime > maxendtime:
+ maxendtime = endtime
+ return minstarttime, maxendtime
+
+def persistsubscans(execution_block, scan, scandata, file):
+ configuration = persistconfiguration(execution_block, 0)
+ subscan = Subscan(
+ scan=scan,
+ obstype='',
+ starttime=scandata['START_TIME'],
+ endtime=scandata['END_TIME'],
+ sourcename=scandata['SOURCE'].split('\x00', 1)[0], # Fix some source names with terminating nulls
+ sourcetype='STAR',
+ ra=scandata['RA'],
+ dec=scandata['DEC'],
+ exposure_time=scandata['TIME_ON_SOURCE'],
+ integration_time=scandata['INTEGRATION_TIME'],
+ receiver_id=0,
+ backend='DIFX',
+ configuration=configuration,
+ intent='TRACK',
+ file=file
+ )
+
+ bandwidth = scandata['BANDWIDTH']
+ frequencies = [float(freq) for freq in scandata['FREQUENCY'].split(',')]
+ minfrequency = min(frequencies)
+ maxfrequency = max(frequencies)
+ polarization = scandata['POLARIZATION']
+ for frequency in frequencies:
+ persistdatadescription(subscan, configuration, frequency, bandwidth, polarization)
+
+ session.add(subscan)
+ return minfrequency, maxfrequency, bandwidth, subscan.starttime, subscan.endtime, polarization
+
+
+def persistdatadescription(subscan, configuration, frequency, bandwidth, polarization):
+ local_polarization = session.merge(polarization)
+ datadescription = DataDescription(
+ bandwidth=bandwidth,
+ frequency=frequency,
+ polarization=local_polarization,
+ configuration=configuration
+ )
+ session.add(datadescription)
+
+
+def persistconfiguration(execution_block, configuration):
+ configuration = Configuration(
+ execution_block=execution_block,
+ configuration=configuration
+ )
+ session.add(configuration)
+ return configuration
+
+
+def sha256(fname):
+ hash_sha256 = hashlib.sha256()
+ with open(fname, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hash_sha256.update(chunk)
+ return hash_sha256.hexdigest()
+
+
+if __name__ == '__main__':
+ path = sys.argv[1]
+ fits_file = sys.argv[2]
+ persistVLBAMetadata(os.path.join(path, fits_file))
diff --git a/apps/cli/executables/pexable/ingest/ingest/persistcaltables.py b/apps/cli/executables/pexable/ingest/ingest/persistcaltables.py
new file mode 100644
index 000000000..ec7e1d65f
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/persistcaltables.py
@@ -0,0 +1,339 @@
+import datetime
+import logging
+import os
+import sys
+import tarfile
+import hashlib
+from pathlib import Path
+
+from pycapo import CapoConfig
+from . import archiveutils as au
+from . import almautils as alu
+from . import NGASUtil as ng
+from .schema import create_session
+from .schema.model import Filegroup, File, Calibration, ScienceProduct, ProductGroup
+
+LOG = logging.getLogger('ingestion')
+
+session = create_session('SDM')
+
+config = CapoConfig()
+site_name = config.getstring('archive-ingestion.NGASSite')
+ngas_cluster = config.getstring('archive-ingestion.NGASCluster')
+
+
+def ingestcaltable(file):
+ """
+ Ingest a single calibration product tar file. Create the metadata and
+ store the file on NGAS
+ :param file: the tar file to be ingested
+ :return:
+ """
+ # No filegroup is given, so it must be extracted from the tar file
+ found = False
+ tar = tarfile.open(file)
+ for tarinfo in tar:
+ # ms.calapply.txt has the file set id embedded in it
+ if tarinfo.isreg() and tarinfo.name.endswith('.ms.calapply.txt'):
+ found = True
+ if found:
+ fsstart = tarinfo.name.find('products/')
+ fsend = tarinfo.name.find('.ms')
+ fileset = tarinfo.name[fsstart + 9:fsend]
+ ingestcalmetadata(file, fileset)
+ break
+ if not found:
+ LOG.error('Unable to find matching file group for calibration products file {}'.format(file))
+
+
+def ingestcalmetadata(file, external_name, nongas):
+ """
+ Ingest a single calibration product tar file, given the file and associated file group
+ :param file: the tar file to be ingested
+ :param external_name: the external name of the parent execution block
+ :param nongas: flag to indicate if ingestion to NGAS should not occur
+ :return:
+ """
+
+ ngas_id = ng.generate_uuids_for_ngas('evla', 'calibration', 'tar')
+ exec_block = au.get_execution_block_from_external_name(external_name)
+ LOG.debug(f"EB ID: {exec_block.ngas_fileset_id}")
+ if exec_block is None:
+ LOG.error(f"Parent execution block {external_name} not found for calibration products {file}")
+ sys.exit(1)
+ else:
+ if exec_block.filegroup is None:
+ LOG.error(f"Execution block filegroup not found for execution block {external_name}")
+ sys.exit(1)
+
+ calibration_filegroup = au.get_filegroup_from_name(file.name)
+
+ if calibration_filegroup is not None:
+ # Calibration products file exists, replace
+ calfile = au.get_file_from_group(calibration_filegroup)
+ calfile.filesize = file.stat().st_size,
+ calfile.checksum=sha256(file)
+ calfile.checksum_type='SHA256'
+ calfile.ingestion_time=datetime.datetime.utcnow()
+ calfile.ngas_id=ngas_id
+ LOG.info(f"Replacing calibrations products: {file.name} for execution block: {external_name}")
+ else:
+ calibration_filegroup = Filegroup(parent_filegroup=exec_block.filegroup, type='calibration')
+ session.add(calibration_filegroup)
+ filesize = file.stat().st_size
+ site_name = config.getstring('archive-ingestion.NGASSite')
+ ngas_cluster = config.getstring('archive-ingestion.NGASCluster')
+ calfile = File(file_path=groupname,
+ ngas_id=ngas_id,
+ ngas_location=site_name,
+ ngas_cluster=ngas_cluster,
+ filename=os.path.basename(file),
+ filegroup1=calibration_filegroup,
+ filesize=filesize,
+ format='tar',
+ type='cal',
+ checksum=sha256(file),
+ checksum_type='SHA256',
+ ingestion_time=datetime.datetime.utcnow())
+
+ calibration_filegroup.datasize = calfile.filesize
+ session.add(calfile)
+
+ science_product_locator = ng.generate_science_product_locator(execblock.telescope.lower(), 'calibration')
+
+ calibration = Calibration(
+ execution_block=execblock,
+ alma_ous_id=None,
+ filegroup=calibration_filegroup,
+ project_code=exec_block.project_code,
+ science_product_locator=science_product_locator
+ )
+
+ science_product = ScienceProduct(
+ science_product_locator=science_product_locator,
+ science_product_type='Calibration',
+ metadata_ingestion_date=datetime.datetime.utcnow(),
+ metadata_ingestion_version=1, # TBD
+ filegroup=calibration_filegroup
+ )
+
+ # Ordering matters to satisfy FK constraints:
+ session.add(science_product)
+ session.add(calibration)
+
+ # Add to science_products_projects
+ project = execblock.project
+ project.science_products.append(science_product)
+ session.add(project)
+
+ session.commit()
+
+ # Now do the NGAS ingestion, if not disabled
+ if nongas:
+ LOG.info(f"NGAS ingestion is disabled. File {ngas_id} will not be ingested.")
+ else:
+ LOG.info(f"Linking file {file} to {str(Path(file.parent) / ngas_id)}")
+ os.link(file, file.parent / ngas_id)
+ ingestcalfile(file.parent / ngas_id, calfile.filesize)
+
+
+def ingestalmacalibration(mous):
+ LOG.debug(f"MOUS for calibrations: {mous}")
+ # Need the eb filegroup
+ # Get eb from MOUS
+ projectcode = au.getprojectcodefrommous(mous)
+ LOG.debug(f"Project code: {projectcode}")
+ calfilegroup = Filegroup(groupname=mous, project_code=projectcode, type='calibration')
+ session.add(calfilegroup)
+
+ # Now get the files and create File entries
+ # Get the list of files - Alma query
+ calfiles = alu.getalmacalfiles(mous)
+ LOG.info(f"Cal files: {str(calfiles)}")
+ # ALMA calibration files are not stored on our NGAS system, so indicate that:
+ for file in calfiles:
+ calfile = File(
+ ngas_id = file[0],
+ ngas_location='NAASC',
+ ngas_cluster='NAASC',
+ filename=file[0],
+ filesize=file[1],
+ filegroup1=calfilegroup,
+ format='cal',
+ type='cal'
+ )
+ session.add(calfile)
+
+ # Now do the calibrations table
+ calibrations = Calibration(
+ alma_ous_id=mous,
+ filegroup=calfilegroup,
+ project_code=projectcode
+ )
+ session.add(calibrations)
+ # session.commit()
+
+
+def ingest_from_manifest(manifest):
+ telescope = manifest.get_telescope()
+ path, calibration_tar_file, input_product_locators, reingest, ngas_ingest = manifest.get_calibration_ingestion_parameters()
+ ngas_id = ng.generate_uuids_for_ngas('evla', 'calibration', 'tar')
+ # In theory, I should be able to pass a list of execution block science product locators
+ ingest_calibration(telescope, input_product_locators, path, calibration_tar_file, ngas_id, reingest)
+
+ if ngas_ingest:
+ LOG.info(f"TEST Linking file {os.path.join(path, calibration_tar_file)} to {os.path.join(path, ngas_id)}")
+ # LOG.info(f"Linking file {os.path.join(path, calibration_tar_file)} to {os.path.join(path, ngas_id)}")
+ # os.link(os.path.join(path, calibration_tar_file), os.path.join(os.path.dirname(file), ngas_id))
+ # ingestcalfile(os.path.join(path, ngas_id), os.path.getsize(os.path.join(path, ngas_id)))
+ else:
+ LOG.info(f"NGAS ingestion is disabled. File {calibration_tar_file} will not be ingested.")
+
+
+def xingest_from_manifest(manifest):
+ telescope = manifest.get_telescope()
+ # Pass telescope, path to
+ path, calibration_tar_file, input_product_locators, reingest, ngas_ingest = manifest.get_calibration_ingestion_parameters()
+ ngas_id = ng.generate_uuids_for_ngas('evla', 'calibration', 'tar')
+ # In theory, I should be able to pass a list of execution block science product locators
+ ingest_calibration(telescope, input_product_locators, path, calibration_tar_file, ngas_id, reingest)
+
+ if ngas_ingest:
+ LOG.info(f"TEST Linking file {os.path.join(path, calibration_tar_file)} to {os.path.join(path, ngas_id)}")
+ # LOG.info(f"Linking file {os.path.join(path, calibration_tar_file)} to {os.path.join(path, ngas_id)}")
+ # os.link(os.path.join(path, calibration_tar_file), os.path.join(os.path.dirname(file), ngas_id))
+ # ingestcalfile(os.path.join(path, ngas_id), os.path.getsize(os.path.join(path, ngas_id)))
+ else:
+ LOG.info(f"NGAS ingestion is disabled. File {calibration_tar_file} will not be ingested.")
+
+
+def ingest_calibration(telescope, execution_block_science_product_locators, path, calibration_tar_file, ngas_id, reingest):
+ """
+ Ingest calibration products, attach to executiion block, and add to product groups
+ :param telescope: telescope (evla or alma)
+ :param execution_block_science_product_locator: list of science product locators of the execution blocks that were calibrated
+ :param path: Location of calibration products file
+ :param calibration_tar_file: tar file containing the calibration products
+ :param reingest: should existing products be delete and reingested?
+ :return:
+ """
+
+ # Get the parent eb using its science product locator; for now, only do the first
+ # since there should only be one
+ execblock = au.get_execution_block(execution_block_science_product_locators[0])
+ if execblock is None:
+ LOG.error(f"Execution block with science product locator {execution_block_science_product_locator} not found.")
+ sys.exit(1)
+ execution_block_science_product = au.get_science_product_by_spl(execution_block_science_product_locator)
+
+ # Add the calibration products filegroup and file
+ # Look for calibration by filename, in case already ingested
+ calibration_products_file = au.get_file(calibration_tar_file)
+ if calibration_products_file is not None:
+ if reingest:
+ # TODO fix reingestion
+ LOG.debug(f"Reingestion of cal products requested for {calibration_tar_file}")
+ # Delete the existing entry
+ else:
+ LOG.error(f"Calibration products file {calibration_tar_file} already ingested")
+ sys.exit(1)
+ else:
+ LOG.debug(f"Initial ingestion of cal products requested for {os.path.join(path, calibration_tar_file)}")
+
+ calibration_filegroup = Filegroup(
+ groupname=calibration_tar_file,
+ parent_filegroup=execblock.filegroup,
+ type='calibration'
+ )
+ db_session = session.object_session(calibration_filegroup)
+ db_session.add(calibration_filegroup)
+
+ calibration_products_file = File(
+ file_path=os.path.dirname(path),
+ ngas_id=ngas_id,
+ ngas_location=site_name,
+ ngas_cluster=ngas_cluster,
+ filename=calibration_tar_file,
+ filegroup1=calibration_filegroup,
+ filesize=os.path.getsize(os.path.join(path, calibration_tar_file)),
+ format='tar',
+ type='cal',
+ checksum=sha256(os.path.join(path, calibration_tar_file)),
+ checksum_type='SHA256',
+ ingestion_time=datetime.datetime.utcnow()
+ )
+ db_session.add(calibration_products_file)
+
+ science_product_locator = ng.generate_science_product_locator(execblock.telescope.lower(), 'calibration')
+
+ calibration = Calibration(
+ project_code=execblock.project_code,
+ execution_block=execblock,
+ alma_ous_id=None,
+ filegroup=calibration_filegroup,
+ science_product_locator=science_product_locator
+ )
+
+ db_session.add(calibration)
+
+ science_product = ScienceProduct(
+ science_product_locator=science_product_locator,
+ science_product_type='Calibration',
+ metadata_ingestion_date=datetime.datetime.utcnow(),
+ metadata_ingestion_version=1,
+ filegroup=calibration_filegroup
+ )
+
+ db_session.add(science_product)
+
+ # TODO ancillary products for calibrations
+
+ # Add to science_products_projects
+ execblock.project.science_products.append(science_product)
+
+ # Add to product_groups and science_products_product_groups
+ input_product_group = ProductGroup(
+ product_group_type='input_group',
+ parent_product_group=None,
+ ALMA_ous=None
+ )
+ input_product_group.science_products.append(execution_block_science_product)
+ db_session.add(input_product_group)
+
+ output_product_group = ProductGroup(
+ product_group_type='output_group',
+ parent_product_group=input_product_group,
+ ALMA_ous=None
+ )
+ output_product_group.science_products.append(science_product)
+ db_session.add(output_product_group)
+
+ db_session.commit()
+
+
+# def ingest_calibrationx(telescope, execution_block_science_product_locators, path,
+# calibration_tar_file, ngas_id, reingest):
+
+
+def ingestcalfile(calibration_file, file_size):
+ ingestion_list = []
+ ingestion_list.append([calibration_file, file_size])
+ LOG.info(f"Ingesting file: {calibration_file.name}")
+ ng.archivefiles(ingestion_list)
+
+
+def sha256(fname):
+ hash_sha256 = hashlib.sha256()
+ with open(fname, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hash_sha256.update(chunk)
+ return hash_sha256.hexdigest()
+
+
+def test1():
+ LOG.debug("TEST")
+
+
+if __name__ == '__main__':
+ test1()
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/persistimages.py b/apps/cli/executables/pexable/ingest/ingest/persistimages.py
new file mode 100644
index 000000000..6fc63107a
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/persistimages.py
@@ -0,0 +1,268 @@
+import datetime
+import logging
+import os
+import glob
+import tarfile
+import hashlib
+import shutil
+
+from pycapo import CapoConfig
+from . import archiveutils as au
+from . import NGASUtil as ng
+from .schema import create_session
+from .schema.model import Filegroup, File, Image, ImageProduct
+from . import FITSUtil as fi
+from .weblog_thumbs.weblogUtil import Weblog
+from .JSONMetadataUtil import JSONMetadata
+
+LOG = logging.getLogger('ingestion')
+
+session = create_session('SDM')
+
+config = CapoConfig()
+
+
+def ingestimagemetaData(imagepath, projectcode, nongas):
+ """
+ Ingest a image products file and a set of images
+ :param imagepath: the parent directory containing the image products file and the image fits files
+ :param projectcode: the project containing the images
+ :param nongas: flag to indicate if ingestion to NGAS should not occur
+ :return:
+ """
+
+ LOG.info('In image ingestion for project {} located in {}'.format(projectcode, imagepath))
+ project = au.getproject(projectcode)
+ if project is None:
+ LOG.error('Project {} could not be found for images located in {}'.format(projectcode, imagepath))
+ return False
+ else:
+ LOG.info('Loading images for project {} located in {}'.format(project.project_code, imagepath))
+
+ # image set filegroup --> image set
+ groupname = os.path.basename(os.path.normpath(imagepath))
+ imagesetfilegroup = Filegroup(project=project,
+ groupname=groupname,
+ type='image_set')
+ session.add(imagesetfilegroup)
+ session.commit()
+
+ # image product filegroup
+ image_product_filegroup = Filegroup(parent_filegroup_id=imagesetfilegroup.filegroup_id,
+ groupname=groupname,
+ type='image_products')
+ session.add(image_product_filegroup)
+
+ files_to_ingest = []
+
+ # Obtain the NGAS site/cluster info:
+ site_name = config.getstring('archive-ingestion.NGASSite')
+ cluster_name = config.getstring('archive-ingestion.NGASCluster')
+
+ # Image product file
+ image_product_files = glob.glob(imagepath + '/*.tar')
+ image_product_file = image_product_files[0]
+
+ if len(image_product_files) == 0:
+ LOG.error('No image product files found in {}'.format(imagepath))
+ return False
+ elif len(image_product_files) > 1:
+ LOG.error('Multiple image product files found in {}'.format(imagepath))
+ return False
+ else:
+ ngas_id = ng.generate_uuids_for_ngas('evla', 'imageproduct', 'tar')
+ product_file = File(file_path=imagepath,
+ ngas_id=ngas_id,
+ ngas_location=site_name,
+ ngas_cluster=cluster_name,
+ filegroup1=image_product_filegroup,
+ filename=os.path.basename(image_product_file),
+ filesize=os.path.getsize(image_product_file),
+ format='tar',
+ type='image_products',
+ ingestion_time=datetime.datetime.utcnow())
+ session.add(product_file)
+ LOG.info(f"Moving file {image_product_file} to {os.path.join(os.path.dirname(image_product_file), ngas_id)}")
+ os.link(image_product_file, os.path.join(os.path.dirname(image_product_file), ngas_id))
+ files_to_ingest.append([os.path.join(imagepath, ngas_id), product_file.filesize])
+
+ # Image product
+ # Get the json metadata
+ json_metadata_files = glob.glob(imagepath + '/*.json')
+ if len(json_metadata_files) == 0:
+ LOG.error('No json metadata file found in {}'.format(imagepath))
+ elif len(json_metadata_files) > 1:
+ LOG.error('Multiple json metadata files found in {}'.format(imagepath))
+ json_metadata_file = json_metadata_files[0]
+ # File doesn't exist'
+ json_metadata = JSONMetadata(json_metadata_file)
+ json_metadata.from_json(json_metadata_file)
+ image_product = ImageProduct(configurations=json_metadata.get_configurations(),
+ collection_name=json_metadata.get_collection_name(),
+ calibration_level=json_metadata.get_calibration_level(),
+ tags=json_metadata.get_product_tags(),
+ product_file=product_file,
+ project=project,
+ image_set_filegroup=imagesetfilegroup.filegroup_id)
+ session.add(image_product)
+ session.commit()
+
+ # Images file group
+ imagesfilegroup = Filegroup(parent_filegroup=imagesetfilegroup, groupname=groupname, type='images')
+ session.add(imagesfilegroup)
+
+ # Image file(s)
+ imagefiles = glob.glob(imagepath + '/*.fits')
+
+ # Gather the fields for the image table from the weblog - image_product_file is the .tar file
+ # What to extract?
+
+ weblog = get_weblog_from_image_products(image_product_file)
+
+ band_codes = weblog.get_band_codes()
+
+ for imagefile in imagefiles:
+ # imagefile is the full path to a fits image
+ # filename = is the file name itself
+ filename = os.path.basename(imagefile)
+
+ thumbnail_image_file = weblog.find_quicklook_thumbnail_for(filename)
+ if thumbnail_image_file is not None:
+ # Path to the thumbnail without and with the ROOT from CAPO
+ thumbnail_path, thumbnail_destination = calculate_thumbnail_path(thumbnail_image_file)
+ # Copy the thumbnail image to the thumbnails directory
+ copy_thumbnail(thumbnail_image_file, thumbnail_destination)
+ else:
+ LOG.info(f"Skipping thumbnail for {imagefile}.")
+ # Stick a NULL in the field
+ thumbnail_path = None
+
+ if 'rms' in filename:
+ image_type = 'quicklook_image_rms'
+ else:
+ image_type = 'quicklook_image'
+ ngas_id = ng.generate_uuids_for_ngas('evla', 'image', 'fits')
+ imgfile = File(file_path=imagepath,
+ ngas_id=ngas_id,
+ ngas_location=site_name,
+ ngas_cluster=cluster_name,
+ filegroup1=imagesfilegroup,
+ filename=filename,
+ filesize=os.path.getsize(os.path.join(imagepath, imagefile)),
+ format='fits',
+ type=image_type,
+ ingestion_time=datetime.datetime.utcnow())
+ # Gather the fields for the image table from the fits file(s)
+ source_name, telescope, spatial_resolution, field_of_view, min_intensity, max_intensity, ra, dec, min_freq, max_freq, rest_frequency, \
+ ra_element_count, dec_element_count, image_units, beam_axis_ratio, polarization_id, rms_noise, spatial_region, ra_pixel_size,\
+ dec_pixel_size = fi.get_fits_data(imagefile)
+
+ # Add image product id to image
+ image = Image(target_name=source_name,
+ telescope=telescope,
+ thumbnail=thumbnail_path,
+ spatial_resolution=spatial_resolution,
+ image_field_of_view=field_of_view,
+ max_intensity=max_intensity,
+ min_intensity=min_intensity,
+ rms_noise=rms_noise,
+ polarization_id=polarization_id,
+ ra=ra,
+ dec=dec,
+ min_frequency=min_freq,
+ max_frequency=max_freq,
+ ra_element_count=ra_element_count,
+ dec_element_count=dec_element_count,
+ starttime=json_metadata.get_starttime(),
+ endtime=json_metadata.get_endtime(),
+ exposure_time=json_metadata.get_endtime(),
+ rest_frequency=rest_frequency,
+ image_units=image_units,
+ spatial_region=spatial_region,
+ beam_axis_ratio=beam_axis_ratio,
+ band_code=' '.join(band_codes),
+ ra_pixel_size=ra_pixel_size,
+ dec_pixel_size=dec_pixel_size,
+ tags=json_metadata.get_image_tags(),
+ file=imgfile,
+ image_products_id=image_product.image_product_id)
+
+ # Image
+ session.add(imgfile)
+ session.add(image)
+
+ LOG.info(f"Moving file {imagefile} to {os.path.join(os.path.dirname(imagefile), ngas_id)}")
+ os.link(imagefile, os.path.join(os.path.dirname(imagefile), ngas_id))
+ files_to_ingest.append([os.path.join(imagepath, ngas_id), imgfile.filesize])
+
+ session.commit()
+
+ # Now do the NGAS ingestion, if not disabled
+ if nongas:
+ LOG.info(f"NGAS ingestion is disabled. Image products and FITS images will not be ingested.")
+ else:
+ ingestimagefiles(files_to_ingest)
+
+
+def ingestimagefiles(files_to_ingest):
+ """
+ Ingest fits images and image product .tar files into NGAS
+ :param files_to_ingest: files to be ingested
+ :return:
+ """
+ ng.archivefiles(files_to_ingest)
+
+
+
+def calculate_thumbnail_path(thumbnail_image_file):
+ sha1 = hashlib.sha1()
+
+ with open(thumbnail_image_file, 'rb') as f:
+ for block in iter(lambda: f.read(1024), b''):
+ sha1.update(block)
+ key = sha1.hexdigest()
+
+ # $1 is the first two characters of the sha1 sum, $2 the second two, and $3 the third two
+ # ea887dea73301fab8b9da760014449b59e1211ae
+ # $ROOT/ea/88/7d/thumbnail
+
+ root_thumbnail_path = config.getstring('archive-ingestion.vlass.ThumbnailPath')
+
+ # Return path without and with ROOT
+ return key[0:2] + '/' + key[2:4] + '/' + key[4:6] + '/' + os.path.basename(thumbnail_image_file), root_thumbnail_path + '/' + key[0:2] + '/' + key[2:4] + '/' + key[4:6] + '/' + os.path.basename(thumbnail_image_file)
+
+
+def copy_thumbnail(thumbnail_image_file, thumbnail_destination):
+ if not os.path.isdir(os.path.dirname(thumbnail_destination)):
+ os.makedirs(os.path.dirname(thumbnail_destination))
+ shutil.copyfile(thumbnail_image_file, thumbnail_destination)
+
+
+def get_weblog_from_image_products(image_product_file):
+ """
+ Extract data from the weblog
+ :param image_product_file: the image products tar file containing the weblog.tgz
+ :return:
+ """
+
+ products_dir = os.path.dirname(image_product_file)
+ product_tar_file = tarfile.open(image_product_file)
+
+ # The pipeline gives us a uniform weblog.tgz file for VLA stuff... but ALMA
+ # tacks on the sanitzed MOUS UID & the name of the processing procedures.
+ files_list = product_tar_file.getnames()
+ weblog_candidates = [place for place, name in enumerate(files_list) if 'weblog' in name]
+
+ # We expect a single file to match the pattern:
+ if len(weblog_candidates) != 1:
+ # So throw an exception when that's not the case.
+ raise NameError('Cannot isolate the weblog!')
+
+ weblog_file_name = files_list.pop(weblog_candidates[0])
+ product_tar_file.extract(weblog_file_name, path=products_dir)
+ # LOG.info('weblog: {}'.format(os.path.join(products_dir, weblog_file_name)))
+ weblog = Weblog(products_dir,weblog_file_name)
+
+ return weblog
+
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/persistmetadata.py b/apps/cli/executables/pexable/ingest/ingest/persistmetadata.py
new file mode 100644
index 000000000..45498bb38
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/persistmetadata.py
@@ -0,0 +1,1109 @@
+import os
+import math
+import re
+import logging
+import datetime
+from pathlib import Path
+import copy
+
+from . import projectutils, proposalutils, archiveutils, almautils
+from .RealFastMetadata import RealFastMetadata
+from .schema import create_session
+from .schema.model import (
+ Project,
+ Filegroup,
+ File,
+ ExecutionBlock,
+ RealfastExecutionBlock,
+ Scan,
+ Subscan,
+ Author,
+ Intent,
+ Configuration,
+ DataDescription,
+ Polarization,
+ AlmaOus,
+ ScienceProduct,
+ AncillaryProduct,
+)
+from .evlasciencedatamodel import SDMXMLTable, SDMBinaryTable
+from . import NGASUtil as ng
+from pycapo import CapoConfig
+
+LOG = logging.getLogger("ingestion")
+
+session = create_session("SDM")
+
+config = CapoConfig()
+site_name = config.getstring("archive-ingestion.NGASSite")
+cluster_name = config.getstring("archive-ingestion.NGASCluster")
+
+
+def persist(sdm, nocal, additional_metadata, collection_metadata, science_products, ingestion_path, ngas_cluster):
+ """
+ Persist an sdm (Science Data Model) to the metadata db, including the NGAS index to be
+ used by the Data Fetcher via SOLR
+ :param sdm: Science Data Model for an EVLA fileset
+ :param nocal:
+ :param additional_metadata: Addition metadata
+ :param collection_metadata: Collection metadata
+ :param science_products: Science products for output group
+ :param ingestion_path Path of files to be ingested
+ :param ngas_cluster: ngas cluster
+ :return: the project
+ """
+
+ sdmtables = sdm.sdmtables
+ bdffiles = sdm.bdffiles
+ fileset = sdm.fileset
+
+ # The set of XML tables contained in the SDM
+ maintable = sdm.sdmtables["Main"].table.getroot()
+ sbsummarytable = sdm.sdmtables["SBSummary"].table.getroot()
+ execblocktable = sdm.sdmtables["ExecBlock"].table.getroot()
+ scantable = sdm.sdmtables["Scan"].table.getroot()
+ subscantable = sdm.sdmtables["Subscan"].table.getroot()
+ polarizationtable = sdm.sdmtables["Polarization"].table.getroot()
+ spectralwindowtable = sdm.sdmtables["SpectralWindow"].table.getroot()
+ datadescriptiontable = sdm.sdmtables["DataDescription"].table.getroot()
+ configtable = sdm.sdmtables["ConfigDescription"].table.getroot()
+ fieldtable = sdm.sdmtables["Field"].table.getroot()
+ sourcetable = sdm.sdmtables["Source"].table.getroot()
+ receivertable = sdm.sdmtables["Receiver"].table.getroot()
+
+ try:
+ window_receivers = {
+ row.spectralWindowId.text: row.frequencyBand.text.split("_")[-1] for row in receivertable.row
+ }
+ except:
+ LOG.warning("Unable to obtain spectral window -> receiver mapping")
+ window_receivers = {}
+
+ # Persist the NGAS metadata
+ project, parentfilegroup, bdffilemap, sdm_ingest_files = persistngasmetadata(
+ sdmtables, bdffiles, fileset, sbsummarytable, execblocktable, subscantable
+ )
+ session.commit()
+
+ # Persist the science metadata
+ output_products, ingest_files = persistsciencemetadata(
+ fileset,
+ bdffilemap,
+ project,
+ parentfilegroup,
+ sbsummarytable,
+ execblocktable,
+ scantable,
+ subscantable,
+ maintable,
+ configtable,
+ datadescriptiontable,
+ polarizationtable,
+ spectralwindowtable,
+ fieldtable,
+ sourcetable,
+ nocal,
+ window_receivers,
+ additional_metadata,
+ collection_metadata,
+ science_products,
+ ingestion_path,
+ ngas_cluster,
+ )
+ session.commit()
+ return project, output_products, sdm_ingest_files + ingest_files
+
+
+def persistngasmetadata(sdmtables, bdffiles, fileset, sbsummarytable, execblocktable, subscantable):
+ """
+ Persist the NGAS index entries
+ :param sdmtables: the set of sdm tables
+ :param bdffiles: the set of bdf files
+ :param fileset: the file set being persisted
+ :param sbsummarytable: SBSummary table
+ :param execblocktable: ExecBlock table
+ :param subscantable: SubScan table
+ :return: project, filegroup, map of bdffiles
+ """
+
+ # Persist the project
+ ingest_files = []
+ project = persistproject(sbsummarytable, execblocktable, subscantable)
+
+ if project.authors is None or len(project.authors) == 0:
+ # Persist the authors if not an existing project
+ authors = persistauthors(project)
+ for author in authors:
+ session.add(author)
+ project.authors = authors
+
+ # Persist the filegroup
+ parentfilegroup, filegroup = createfilegroup(project)
+
+ # Persist files for sdm tables
+ filegroupsize = 0
+ for sdmtable in sdmtables:
+ file = createsdmtable(sdmtables, filegroup, sdmtable, execblocktable.row.telescopeName)
+ session.add(file)
+ filegroupsize += file.filesize
+ ingest_files.append((file, file.filesize))
+
+ # Persist files for bdf files
+ bdffilemap = {}
+ for bdffile in bdffiles:
+ if execblocktable.row.telescopeName == "EVLA" and "X" in bdffile.entityid:
+ LOG.warning(f"Observation {fileset} has invalid BDF {bdffile.entityid}")
+ continue
+ file = createbdffile(bdffile, filegroup, execblocktable.row.telescopeName)
+ session.add(file)
+ filegroupsize += file.filesize
+ bdffilemap[bdffile.entityid] = file
+ ingest_files.append((file, file.filesize))
+
+ filegroup.datasize = filegroupsize
+ session.add(filegroup)
+ parentfilegroup.datasize = -1
+ session.add(parentfilegroup)
+ session.add(project)
+ return project, parentfilegroup, bdffilemap, ingest_files
+
+
+def persistsciencemetadata(
+ fileset,
+ bdffilemap,
+ project,
+ filegroup,
+ sbsummarytable,
+ execblocktable,
+ scantable,
+ subscantable,
+ maintable,
+ configtable,
+ datadescriptiontable,
+ polarizationtable,
+ spectralwindowtable,
+ fieldtable,
+ sourcetable,
+ nocal,
+ window_receivers,
+ additional_metadata,
+ collection_metadata,
+ output_science_products,
+ ingestion_path,
+ ngas_cluster,
+):
+ """
+ Persist the science meta data from the SDM tables
+ :param bdffilemap:
+ :param project:
+ :param filegroup: Top level filegroup for the observation
+ :param sbsummarytable:
+ :param execblocktable:
+ :param scantable:
+ :param subscantable:
+ :param maintable:
+ :param configtable:
+ :param datadescriptiontable:
+ :param polarizationtable:
+ :param spectralwindowtable:
+ :param fieldtable:
+ :param sourcetable:
+ :param nocal:
+ :param window_receivers:
+ :param additional_metadata: additional metadata
+ :param collection_metadata: collection metadata
+ :param output_science_products: Output group science products
+ :param ingestion_path: Path of files to ingest
+ :param ngas_cluster: ngas cluster
+ :return:
+ """
+ # Persist ExecBlock
+ execblockdict = {}
+ science_products = []
+ ingest_files = []
+ for execblock in execblocktable.row:
+ execblockobject, science_product, files_to_ingest = persistexecblock(
+ fileset,
+ project,
+ filegroup,
+ sbsummarytable,
+ execblocktable,
+ subscantable,
+ nocal,
+ additional_metadata,
+ collection_metadata,
+ output_science_products,
+ ingestion_path,
+ ngas_cluster,
+ )
+ ingest_files = ingest_files + files_to_ingest
+ execblockdict[execblock.execBlockId.text] = execblockobject
+ session.add(science_product, execblockobject)
+ science_products.append(science_product)
+ session.commit()
+
+ for execblock in execblocktable.row:
+ execblockobject = execblockdict[execblock.execBlockId.text]
+ scans = getscans(execblock.execBlockId, scantable)
+ # Persist scans
+ bandset = set()
+ for scan in scans:
+ try:
+ scanobject = persistscan(
+ scan,
+ execblockobject,
+ filegroup,
+ maintable,
+ fieldtable,
+ sourcetable,
+ spectralwindowtable,
+ configtable,
+ datadescriptiontable,
+ polarizationtable,
+ )
+ session.add(scanobject)
+
+ # Persist subscans
+ subscans = getsubscans(scanobject.ost_scan_id, subscantable)
+ for subscan in subscans:
+ subscanobject, bands = persistsubscan(
+ bdffilemap,
+ execblockobject,
+ scanobject,
+ subscan,
+ scan.scanIntent.text,
+ maintable,
+ configtable,
+ datadescriptiontable,
+ polarizationtable,
+ spectralwindowtable,
+ fieldtable,
+ sourcetable,
+ execblocktable,
+ window_receivers,
+ )
+ bandset = bandset.union(bands)
+ session.add(subscanobject)
+ except ValueError:
+ continue
+ execblockbands = " ".join(sorted(bandset))
+ execblockobject.band_code = execblockbands
+ session.add(science_product, execblockobject)
+ return science_products, ingest_files
+
+
+def lookup_external_project(telescope, sbsummarytable):
+ title = ""
+ abstract = ""
+ legacyid = ""
+ schedulingblocktype = sbsummarytable.row.sbType.text
+ authors = []
+
+ # Assign the proprietary duration
+ if schedulingblocktype == "OBSERVER":
+ proprietaryduration = 365
+ else:
+ proprietaryduration = 0
+
+ if telescope == "ALMA":
+ projectid = sbsummarytable.row.projectUID.EntityRef.get("entityId")
+ almaproject = almautils.getproject(projectid)
+ authors.extend(almautils.getauthors(projectid))
+
+ title = almaproject.title
+ abstract = almautils.getabstract(projectid)
+ legacyid = None
+ projectcode = almaproject.code
+ optprojectid = None
+ else:
+ # EVLA stuff
+ projectid = getid(sbsummarytable.row.projectUID.EntityRef)
+
+ if "X" in projectid:
+ # Project id was not available in the sdm table, set to 'Operations'
+ projectcode = "Operations"
+ optprojectid = 0
+ else:
+ optproject = projectutils.getproject(projectid)
+ if optproject is None: # No match in the OPT
+ projectcode = "Not in OPT "
+ optprojectid = 0
+ else:
+ projectcode = optproject.projectcode
+ optprojectid = optproject.id
+
+ # All VLASS project codes should be mapped onto the VLASS proposal
+ if optproject.projectcode.startswith("VLASS"):
+ proposal = proposalutils.getproposal("VLASS")
+ else:
+ proposal = proposalutils.getproposal(optproject.projectcode)
+
+ authors.extend(projectutils.getauthors(projectid))
+
+ if proposal is not None:
+ title = proposal.TITLE
+ abstract = proposal.ABSTRACT
+ legacyid = proposal.LEGACY_ID
+ if proposal.PROPOSAL_TYPE == "Director's Discretionary Time":
+ proprietaryduration = round(365 / 2, 0)
+
+ return ExternalProject(
+ projectcode, title, abstract, proprietaryduration, legacyid=legacyid, optprojectid=optprojectid, authors=authors
+ )
+
+
+def persistproject(sbsummarytable, execblocktable, subscantable):
+ """
+ Persist project
+ :param sbsummarytable:
+ :param execblocktable:
+ :param subscantable:
+ :return:
+ """
+ # If the project is already persisted, just update the last_addition date
+ # and the start time and end time
+ telescope = execblocktable.row.telescopeName
+ extproject = lookup_external_project(telescope, sbsummarytable)
+ existingproject = archiveutils.getproject(extproject.project_code)
+ if existingproject is not None:
+ # Get the min, max for the exec block
+ minstarttime, maxendtime = getminmaxtime(execblocktable.row.execBlockId, subscantable)
+ # Propagate to project if appropriate
+ if minstarttime < existingproject.starttime:
+ existingproject.starttime = minstarttime
+ if maxendtime > existingproject.endtime:
+ existingproject.endtime = maxendtime
+ existingproject.last_addition = datetime.datetime.utcnow()
+ return existingproject
+
+ totaltime = totalobservingtime(execblocktable)
+ # If semester proposal = 365
+ # If test = 0
+ # If semester and DDT = 365 / 2
+ minstarttime, maxendtime = getminmaxtime(execblocktable.row.execBlockId, subscantable)
+ # last_addition - update when another run for this project
+ lastaddition = datetime.datetime.utcnow()
+ project = Project(
+ project_code=extproject.project_code,
+ opt_project_id=extproject.opt_project_id,
+ legacy_id=extproject.legacy_id,
+ title=extproject.title,
+ abstract=extproject.abstract,
+ total_observation_time=totaltime,
+ last_addition=lastaddition,
+ starttime=minstarttime,
+ endtime=maxendtime,
+ proprietary_duration=extproject.proprietary_duration,
+ )
+ if telescope == "ALMA":
+ project.authors = extproject.authors
+
+ return project
+
+
+class ExternalProject:
+ def __init__(self, projectcode, title, abstract, proprietaryduration, **kwargs):
+ self.project_code = projectcode
+ self.opt_project_id = kwargs["optprojectid"]
+ self.legacy_id = kwargs["legacyid"]
+ self.title = title
+ self.abstract = abstract
+ self.proprietary_duration = proprietaryduration
+ self.authors = kwargs["authors"] if "authors" in kwargs else []
+
+
+def persistauthors(project):
+ """
+ Persist authors
+ :param projectid:
+ :return:
+ """
+ saveauthors = []
+ authors = projectutils.getauthors(project.opt_project_id)
+ for author in authors:
+ existingauthor = archiveutils.getauthor(project.project_code, author.globalid)
+ if existingauthor is not None:
+ saveauthors.append(existingauthor)
+ else:
+ # Now get the user information from the PST using the globalid
+ pstuser = proposalutils.getpstuser(author.globalid)
+ saveauthors.append(
+ Author(
+ project_code=project.project_code,
+ username=pstuser.personAuthentication.personName,
+ firstname=pstuser.firstName,
+ lastname=pstuser.lastName,
+ pst_person_id=author.globalid,
+ is_pi=projectutils.ispi(project.project_code, author.globalid),
+ )
+ )
+ return saveauthors
+
+
+def createfilegroup(project):
+ """
+ Create filegroup
+ :param project:
+ :return:
+ """
+ parentfilegroup = Filegroup(project=project, type="execution_block")
+ filegroup = Filegroup(parent_filegroup=parentfilegroup, type="rawdata")
+ return parentfilegroup, filegroup
+
+
+def createsdmtable(sdmtables, filegroup, sdmtable, telescope):
+ """
+ Create file for sdm table
+ :param sdmtables:
+ :param filegroup:
+ :param sdmtable:
+ :param telescope:
+ :return:
+ """
+ file_path = filegroup.type
+ if isinstance(sdmtables[sdmtable], SDMXMLTable):
+ ngas_id = sdmtables[sdmtable].entityid.replace("://", "___").replace("/", "_") + ".sdm"
+ filename = sdmtables[sdmtable].tablename + ".xml"
+ elif isinstance(sdmtables[sdmtable], SDMBinaryTable):
+ ngas_id = sdmtables[sdmtable].entityid.replace("://", "___").replace("/", "_") + ".bin"
+ filename = sdmtables[sdmtable].tablename + ".bin"
+ else:
+ raise Exception("Invalid SDM table type")
+ filesize = sdmtables[sdmtable].size / 1000
+ format = "sdm"
+ type = "raw"
+
+ file = File(
+ file_path=file_path,
+ ngas_id=ngas_id,
+ ngas_location=site_name,
+ ngas_cluster=cluster_name,
+ filegroup1=filegroup,
+ filename=filename,
+ filesize=filesize,
+ format=format,
+ type=type,
+ ingestion_time=datetime.datetime.utcnow(),
+ )
+
+ if telescope == "ALMA":
+ file.ngas_location = "NAASC"
+ file.ngas_cluster = "NAASC"
+
+ return file
+
+
+TELESCOPE_SIZE_CORRECTION_FACTORS = {"EVLA": 1000, "ALMA": 1000}
+
+
+def createbdffile(bdffile, filegroup, telescope):
+ """
+ Create file for bdf
+ :param bdffile:
+ :param filegroup:
+ :return:
+ """
+ file_path = filegroup.type
+ if telescope == "EVLA" and "X" in bdffile.entityid: # dummy uid; I think this is deprecated
+ ngas_id = None
+ filename = bdffile.entityid # The id given in the XML
+ filesize = 0
+ elif ":///" in bdffile.entityid: # EVLA
+ filename = bdffile.entityid.replace(":///", "____").replace("/", "_")
+ ngas_id = filename + ".bdf"
+ # the filesize sometimes appears to be -99, what's that about?
+ filesize = int(bdffile.filesize) / TELESCOPE_SIZE_CORRECTION_FACTORS.get(telescope, 1)
+ if filesize < 0:
+ LOG.warning(f"Negative filesize {filesize} for filename {filename}")
+ elif "://" in bdffile.entityid: # ALMA
+ filename = bdffile.entityid.replace("://", "___").replace("/", "_")
+ ngas_id = filename + ".bdf"
+ # the filesize sometimes appears to be -99, what's that about?
+ filesize = int(bdffile.filesize) / TELESCOPE_SIZE_CORRECTION_FACTORS.get(telescope, 1)
+ if filesize < 0:
+ LOG.warning(f"Negative filesize {filesize} for filename {filename}")
+ format = "bdf"
+ type = "raw"
+ file = File(
+ file_path=file_path,
+ ngas_id=ngas_id,
+ ngas_location=site_name,
+ ngas_cluster=cluster_name,
+ filegroup1=filegroup,
+ filename=filename,
+ filesize=filesize,
+ format=format,
+ type=type,
+ ingestion_time=datetime.datetime.utcnow(),
+ )
+ return file
+
+
+def getid(entityref):
+ entityid = entityref.get("entityId")
+ idloc = entityid.rfind("/")
+ return entityid[idloc + 1 :]
+
+
+def totalobservingtime(execblock):
+ # TODO units?
+ observingtime = 0.0
+ for row in execblock.row:
+ starttime = row.startTime / 86400e09
+ stoptime = row.endTime / 86400e09
+ observingtime += stoptime - starttime
+ return observingtime
+
+
+def persistexecblock(
+ fileset,
+ project,
+ filegroup,
+ sbsummarytable,
+ execblocktable,
+ subscantable,
+ nocal,
+ additional_metadata,
+ collection_metadata,
+ output_science_products,
+ ingestion_path,
+ ngas_cluster,
+):
+
+ files_to_ingest = []
+ ost_exec_block_id = getid(execblocktable.row.execBlockUID.EntityRef)
+ if "X" in ost_exec_block_id:
+ ost_exec_block_id = 0
+ calibration_level = "0"
+ telescope = execblocktable.row.telescopeName.text
+ configuration = None
+ if len(execblocktable.row.findall("configName")) > 0:
+ configuration = execblocktable.row.configName.text
+ scheduling_block_id = getid(sbsummarytable.row.sbSummaryUID.EntityRef)
+ if "X" in scheduling_block_id:
+ scheduling_block_id = 0
+ minstarttime, maxendtime = getminmaxtime(execblocktable.row.execBlockId, subscantable)
+ schedulingblocktype = sbsummarytable.row.sbType.text
+ if nocal:
+ calibration_status = "Do Not Calibrate"
+ else:
+ calibration_status = None
+
+ science_product_locator = ng.generate_science_product_locator(telescope.lower(), "execblock")
+
+ if collection_metadata is not None and collection_metadata.isRealFast():
+ realfast_metadata = RealFastMetadata(collection_metadata)
+ execblock = RealfastExecutionBlock(
+ ost_exec_block_id=ost_exec_block_id,
+ filegroup=filegroup,
+ calibration_level=calibration_level,
+ telescope=telescope,
+ configuration=configuration,
+ scheduling_block_id=scheduling_block_id,
+ project=project,
+ ngas_fileset_id=fileset,
+ starttime=minstarttime,
+ endtime=maxendtime,
+ calibration_status=calibration_status,
+ scheduling_block_type=schedulingblocktype,
+ science_product_locator=science_product_locator,
+ transient_ra=realfast_metadata.get_transient_ra(),
+ transient_ra_error=realfast_metadata.get_transient_ra_error(),
+ transient_dec=realfast_metadata.get_transient_dec(),
+ transient_dec_error=realfast_metadata.get_transient_dec_error(),
+ transient_snr=realfast_metadata.get_transient_snr(),
+ transient_dm=realfast_metadata.get_transient_dm(),
+ transient_dm_error=realfast_metadata.get_transient_dm_error(),
+ preaverage_time=realfast_metadata.get_preaverage_time(),
+ rfpipe_version=realfast_metadata.get_rfpipe_version(),
+ prefs_id=realfast_metadata.get_prefs_id(),
+ rf_qa_label=realfast_metadata.get_rf_qa_label(),
+ rf_qa_zero_fraction=realfast_metadata.get_rf_qa_zero_fraction(),
+ rf_qa_visibility_noise=realfast_metadata.get_rf_qa_visibility_noise(),
+ rf_qa_image_noise=realfast_metadata.get_rf_qa_image_noise(),
+ portal_candidate_ids=realfast_metadata.get_portal_candidate_ids(),
+ )
+ else:
+ execblock = ExecutionBlock(
+ ost_exec_block_id=ost_exec_block_id,
+ filegroup=filegroup,
+ calibration_level=calibration_level,
+ telescope=telescope,
+ configuration=configuration,
+ scheduling_block_id=scheduling_block_id,
+ project=project,
+ ngas_fileset_id=fileset,
+ starttime=minstarttime,
+ endtime=maxendtime,
+ calibration_status=calibration_status,
+ scheduling_block_type=schedulingblocktype,
+ science_product_locator=science_product_locator,
+ )
+ science_product = ScienceProduct(
+ science_product_locator=science_product_locator,
+ science_product_type="Execution Block",
+ metadata_ingestion_date=datetime.datetime.utcnow(),
+ metadata_ingestion_version=1,
+ external_system="VLA Operations",
+ external_name=fileset,
+ filegroup=filegroup,
+ )
+
+ execblock.project.science_products.append(science_product)
+
+ if "ancillary_products" in output_science_products[0].keys():
+ ancillary_products = output_science_products[0]["ancillary_products"]
+ for anc_product in ancillary_products:
+ LOG.debug(f"Ancillary product: {anc_product}")
+ product_file = Path(ingestion_path) / anc_product["filename"]
+ ancillary_product_locator = ng.generate_science_product_locator(telescope.lower(), "ancillary_product")
+ ancillary_product_filegroup = Filegroup(
+ type="ancillary_product",
+ datasize=-1,
+ )
+ if anc_product["type"] == "thumbnail_image":
+ thumbnail_path = str(Path(ingestion_path) / product_file.name)
+ preview_path = str(Path(calculate_thumbnail_path(thumbnail_path)[0]).parent)
+ else:
+ preview_path = None
+ ancillary_product_file = File(
+ ngas_id=ng.generate_uuids_for_ngas(telescope, "ancillary_product", product_file.suffix[1:]),
+ ngas_location=site_name,
+ ngas_cluster=ngas_cluster,
+ filename=product_file.name,
+ filesize=product_file.stat().st_size,
+ format=product_file.suffix[1:],
+ type=anc_product["type"],
+ checksum=ng.sha256(product_file),
+ checksum_type="SHA256",
+ ingestion_time=datetime.datetime.utcnow(),
+ filegroup1=ancillary_product_filegroup,
+ preview_storage_path=preview_path,
+ )
+
+ files_to_ingest.append((ancillary_product_file, ancillary_product_file.filesize))
+
+ ancillary_product = AncillaryProduct(
+ ancillary_product_locator=ancillary_product_locator,
+ ancillary_product_type=anc_product["type"],
+ filegroup=ancillary_product_filegroup,
+ science_product_locator=science_product_locator,
+ )
+ session.add(ancillary_product_filegroup)
+ session.add(ancillary_product_file)
+ session.add(ancillary_product)
+
+ # For everything other than ALMA, just null out the alma_ous_uid
+ if telescope == "ALMA":
+ mous, gous, sous = almautils.getalmaoushierarchy(execblock.ngas_fileset_id)
+ createalmaoushierarchy(project.project_code, mous, gous, sous)
+ execblock.alma_ous_id = mous
+ science_product.external_system = "ALMA Operations"
+ science_product.external_name = fileset
+ else:
+ execblock.alma_ous_id = None
+ science_product.external_system = "VLA Operations"
+ science_product.external_name = fileset
+
+ return execblock, science_product, files_to_ingest
+
+
+def createalmaoushierarchy(project_code, mousid, gousid, sousid):
+ """
+ Create the OUS hierarchy for an ALMA execblock
+ :param project_code:
+ :param mousid:
+ :param gousid:
+ :param sousid:
+ :return:
+ """
+
+ schedblock_name = almautils.getschedblockname(mousid)
+ if archiveutils.ousexists(mousid, "MOUS"):
+ # Nothing needed. Hierarchy already exists, just add eb
+ LOG.info(f"MOUS exists, do nothing")
+ return
+ elif archiveutils.ousexists(gousid, "GOUS"):
+ LOG.info(f"No MOUS, but GOUS exists, add MOUS")
+ mous = AlmaOus(alma_ous_id=mousid, parent_ous_id=gousid, ous_type="MOUS", schedblock_name=schedblock_name)
+ session.add(mous)
+ elif archiveutils.ousexists(sousid, "SOUS"):
+ LOG.info(f"MOUS and GOUS don't exist, but SOUS exists. Add MOUS and GOUS")
+ gous = AlmaOus(alma_ous_id=gousid, parent_ous_id=sousid, ous_type="GOUS")
+ session.add(gous)
+ mous = AlmaOus(alma_ous_id=mousid, parent_ous_id=gousid, ous_type="MOUS", schedblock_name=schedblock_name)
+ session.add(mous)
+ else:
+ LOG.info("SOUS doesn't exist. Create MOUS, GOUS, and SOUS")
+ sous = AlmaOus(alma_ous_id=sousid, project_code=project_code, ous_type="SOUS")
+ session.add(sous)
+ gous = AlmaOus(alma_ous_id=gousid, parent_ous_id=sousid, ous_type="GOUS")
+ session.add(gous)
+ mous = AlmaOus(alma_ous_id=mousid, parent_ous_id=gousid, ous_type="MOUS", schedblock_name=schedblock_name)
+ session.add(mous)
+
+
+def getscans(execblockid, scantable):
+ # Get only scans associated with this
+ scanlist = []
+ for scan in scantable.row:
+ if scan.execBlockId == execblockid:
+ scanlist.append(scan)
+
+ return scanlist
+
+
+def getsubscans(scanid, subscantable):
+ subscanlist = []
+ for subscan in subscantable.row:
+ if subscan.scanNumber.text == scanid:
+ subscanlist.append(subscan)
+ return subscanlist
+
+
+def persistscan(
+ scan,
+ execblock,
+ filegroup,
+ maintable,
+ fieldtable,
+ sourcetable,
+ spectralwindowtable,
+ configtable,
+ datadescriptiontable,
+ polarizationtable,
+):
+ ost_scan_id = scan.scanNumber.text
+ minfrequency, maxfrequency, minbandwidth, maxbandwidth, polarizationcode = getaggregatedatadescriptionfields(
+ ost_scan_id, maintable, configtable, datadescriptiontable, polarizationtable, spectralwindowtable
+ )
+ if "OBSERVE" in scan.scanIntent.text:
+ obstype = "OBS"
+ elif "CALIBRATE" in scan.scanIntent.text:
+ obstype = "CAL"
+ else:
+ obstype = "OTHER"
+ scanobject = Scan(
+ ost_scan_id=ost_scan_id,
+ execution_block=execblock,
+ filegroup=filegroup,
+ max_bandwidth=maxbandwidth,
+ min_bandwidth=minbandwidth,
+ polarization_code=polarizationcode,
+ max_frequency=maxfrequency,
+ min_frequency=minfrequency,
+ )
+ return scanobject
+
+
+def getdatadescriptions(scanid, subscanid, maintable, configtable):
+ maintablerow = getmaintablerow(maintable, scanid, subscanid)
+ configdescriptionrow = getconfigtablerow(configtable, maintablerow.configDescriptionId)
+ dataDescriptions = configdescriptionrow.dataDescriptionId.text.split()[2:]
+ return dataDescriptions
+
+
+def persistdatadescription(
+ scanid, subscanid, subscanobject, datadescriptionid, datadescriptiontable, polarizationtable, spectralwindowtable
+):
+ frequency, bandwidth, polarizationcode, polarizations = getdatadescriptionfields(
+ scanid, subscanid, datadescriptionid, datadescriptiontable, polarizationtable, spectralwindowtable
+ )
+ polarization = archiveutils.getpolarization(polarizationcode)
+ # datadescription = archiveutils.getdatadescription(bandwidth, frequency, polarization.polarization_id, session)
+ # if datadescription is None:
+ datadescription = DataDescription(
+ bandwidth=bandwidth, frequency=frequency, polarization_id=polarization.polarization_id
+ )
+ session.add(datadescription)
+ return datadescription
+
+
+def persistsubscan(
+ bdffilemap,
+ execblock,
+ scan,
+ subscan,
+ scanIntent,
+ maintable,
+ configtable,
+ datadescriptiontable,
+ polarizationtable,
+ spectralwindowtable,
+ fieldtable,
+ sourcetable,
+ execblocktable,
+ window_receivers,
+):
+ backend = "none"
+ intent = " ".join(scanIntent.split(" ")[2:])
+ starttimeMJD = datetoMJD(subscan.startTime)
+ endtimeMJD = datetoMJD(subscan.endTime)
+ subscanid = subscan.subscanNumber.text
+
+ if intent == "CALIBRATE_POINTING":
+ obstype = "POINT"
+ elif intent == "UNSPECIFIED":
+ obstype = "UNKWN"
+ elif "OBSERVE_TARGET" in intent:
+ obstype = "OBS"
+ elif "CALIBRATE" in intent:
+ obstype = "CAL"
+ elif intent == "none":
+ obstype = "TRACK"
+ else:
+ obstype = "OTHER"
+
+ exposure_time = round((endtimeMJD - starttimeMJD) * 86400, 2) # 4 decimal places
+ if subscan.find("numIntegration") is not None and subscan.numIntegration != 0:
+ integration_time = round(exposure_time / subscan.numIntegration, 3)
+ elif subscan.find("numberIntegration") is not None and subscan.numberIntegration != 0:
+ integration_time = round(exposure_time / subscan.numberIntegration, 3)
+ else:
+ integration_time = 0
+ ra, dec = getraanddec(maintable, fieldtable, sourcetable, scan.ost_scan_id, subscan.subscanNumber)
+ # TODO receiver id and build receiver table
+ bands = getbands(
+ scan.ost_scan_id,
+ subscan.subscanNumber,
+ maintable,
+ configtable,
+ datadescriptiontable,
+ spectralwindowtable,
+ execblocktable,
+ window_receivers,
+ )
+ receiver_id = 0
+
+ subscanobject = Subscan(
+ scan=scan,
+ file_id=10,
+ ost_subscan_id=subscanid,
+ obstype=obstype,
+ starttime=starttimeMJD,
+ endtime=endtimeMJD,
+ sourcename="",
+ sourcetype="STAR",
+ ra=ra,
+ dec=dec,
+ exposure_time=exposure_time,
+ integration_time=integration_time,
+ receiver_id=receiver_id,
+ backend=backend,
+ intent=intent,
+ )
+ # Get list of data descriptions for this subscan / config
+ datadescriptionlist = getdatadescriptions(scan.ost_scan_id, subscan.subscanNumber, maintable, configtable)
+ # Build data_descripions rows
+ maintablerow = getmaintablerow(maintable, scan.ost_scan_id, subscan.subscanNumber)
+ configuration = re.sub("\w*_", "", maintablerow.configDescriptionId.text)
+ configobject = archiveutils.getconfiguration(execblock.execution_block_id, int(configuration), session)
+ if configobject is None:
+ # Not found, create a new one
+ configobject = Configuration(configuration=int(configuration), execution_block=execblock)
+ for datadescription in datadescriptionlist:
+ datadescriptionobject = persistdatadescription(
+ scan.ost_scan_id,
+ subscan.subscanNumber,
+ subscanobject,
+ datadescription,
+ datadescriptiontable,
+ polarizationtable,
+ spectralwindowtable,
+ )
+ datadescriptionobject.configuration = configobject
+ session.add(datadescriptionobject)
+ session.add(configobject)
+
+ # XML change
+ bdffileid = None
+ if maintablerow.find("dataUID") is not None:
+ bdffileid = maintablerow.dataUID.EntityRef.get("entityId")
+ elif maintablerow.find("dataOid") is not None:
+ bdffileid = maintablerow.dataOid.EntityRef.get("entityId")
+ else:
+ LOG.critical("Invalid XML element in Main table")
+
+ if execblocktable.row.telescopeName == "EVLA" and "X" in bdffileid:
+ subscanobject.file = None
+ else:
+ subscanobject.file = bdffilemap[bdffileid]
+ fieldtablerow = getfieldrow(fieldtable, maintablerow.fieldId)
+ sourcetablerow = getsourcerow(sourcetable, fieldtablerow.sourceId)
+ subscanobject.sourcename = sourcetablerow.sourceName.text
+ subscanobject.configuration = configobject
+ return subscanobject, bands
+
+
+def getaggregatedatadescriptionfields(
+ scanid, maintable, configtable, datadescriptiontable, polarizationtable, spectralwindowtable
+):
+ minfrequency, maxfrequency, minbandwidth, maxbandwidth, pcode = 0, 0, 0, 0, 240
+ frequencies = []
+ bandwidths = []
+ polarizationset = set()
+ subscans = []
+ datadescriptions = []
+ for row in maintable.row:
+ if row.scanNumber.text == scanid:
+ subscans.append(row.subscanNumber.text)
+ for subscan in subscans:
+ datadescriptions = getdatadescriptions(scanid, subscan, maintable, configtable)
+ for datadescription in datadescriptions:
+ pcode = 0
+ frequency, bandwidth, polarizationcode, polarizations = getdatadescriptionfields(
+ scanid, subscans, datadescription, datadescriptiontable, polarizationtable, spectralwindowtable
+ )
+ frequencies.append(frequency)
+ bandwidths.append(bandwidth)
+ polarizationset = polarizationset.union(polarizations)
+ for polarizationelement in polarizationset:
+ pcode += archiveutils.getpolarizationcode(polarizationelement)
+ minfrequency = min(frequencies)
+ maxfrequency = max(frequencies)
+ minbandwidth = min(bandwidths)
+ maxbandwidth = max(bandwidths)
+
+ return minfrequency, maxfrequency, minbandwidth, maxbandwidth, pcode
+
+
+def getdatadescriptionfields(
+ scanid, subscanid, dataDescriptionId, datadescriptiontable, polarizationtable, spectralwindowtable
+):
+ datadescriptionrow = getdatadescriptionrow(datadescriptiontable, dataDescriptionId)
+ spectralwindowrow = getspectralwindowrow(spectralwindowtable, datadescriptionrow.spectralWindowId)
+ frequency = spectralwindowrow.refFreq.text
+ bandwidth = spectralwindowrow.totBandwidth.text
+ polarizationrow = getpolarizationrow(polarizationtable, datadescriptionrow.polOrHoloId)
+ polarization = polarizationrow.corrType.text
+
+ polarizations = ", ".join(polarization.split()[2:])
+ polarizationlist = polarization.split()[2:]
+ polarizationcode = archiveutils.getpolarizationcode(polarizations)
+
+ return float(frequency), float(bandwidth), int(polarizationcode), polarizationlist
+
+
+def getbands(
+ scanid,
+ subscanid,
+ maintable,
+ configtable,
+ datadescriptiontable,
+ spectralwindowtable,
+ execblocktable,
+ window_receivers,
+):
+ # TODO use this for bands
+
+ bandsset = set()
+ datadescriptionlist = getdatadescriptions(scanid, subscanid, maintable, configtable)
+ # Build data_descripions rows
+ for datadescription in datadescriptionlist:
+ datadescriptionrow = getdatadescriptionrow(datadescriptiontable, datadescription)
+ # NB. The ALMA method here is ostensibly the right way to do this.
+ # However, the CBE in the past guessed what the receiver should be, and
+ # wrote that guess into the SDM in the Receivers table. So, the value
+ # in the Receivers table cannot be trusted in all cases for EVLA, hence,
+ # the inference is made by the spectral window's name, which apparently
+ # is correct in all cases for EVLA.
+ if execblocktable.row.telescopeName == "ALMA":
+ # make sure we have a receiver band
+ if datadescriptionrow.spectralWindowId.text in window_receivers:
+ bandsset.add(window_receivers[datadescriptionrow.spectralWindowId.text])
+ else:
+ LOG.error("Unable to locate any receivers for this spectral window!")
+ else:
+ spectralwindowrow = getspectralwindowrow(spectralwindowtable, datadescriptionrow.spectralWindowId)
+ band = re.sub("\w*_", "", spectralwindowrow.name.text)
+ band = re.sub("#\w*", "", band)
+ bandsset.add(re.sub("#\w*", "", band))
+ return bandsset
+
+
+# Methods to return row matching id
+def getmaintablerow(maintable, scanid, subscanid):
+ for row in maintable.row:
+ if int(row.scanNumber.text) == int(scanid) and int(row.subscanNumber.text) == int(subscanid):
+ return row
+ return None
+
+
+def getconfigtablerow(configtable, configid):
+ for row in configtable.row:
+ if row.configDescriptionId == configid:
+ return row
+ return None
+
+
+def getdatadescriptionrow(datadescriptiontable, datadescriptionid):
+ for row in datadescriptiontable.row:
+ if row.dataDescriptionId == datadescriptionid:
+ return row
+ return None
+
+
+def getpolarizationrow(polarizationtable, polarizationid):
+ for row in polarizationtable.row:
+ if row.polarizationId == polarizationid:
+ return row
+ return None
+
+
+def getspectralwindowrow(spectralwindowtable, spectralwindowid):
+ for row in spectralwindowtable.row:
+ if row.spectralWindowId == spectralwindowid:
+ return row
+ return None
+
+
+def getfieldrow(fieldtable, fieldId):
+ for row in fieldtable.row:
+ if row.fieldId == fieldId:
+ return row
+ return None
+
+
+def getsourcerow(sourcetable, sourceId):
+ for row in sourcetable.row:
+ if row.sourceId == sourceId:
+ return row
+ return None
+
+
+def getraanddec(maintable, fieldtable, sourcetable, scanid, subscanid):
+ mainrow = getmaintablerow(maintable, scanid, subscanid)
+ fieldrow = getfieldrow(fieldtable, mainrow.fieldId)
+ sourcerow = getsourcerow(sourcetable, fieldrow.sourceId)
+ radeclist = sourcerow.direction.text.split(" ")
+ return radeclist[2], radeclist[3]
+
+
+def datetoMJD(subscantime):
+ iat_utc = 0.0
+ asdmMjd = 54500.0
+
+ asdmMjd = (subscantime + iat_utc) / 86400.0e9
+ return asdmMjd
+
+
+def getminmaxtime(execblockid, subscantable):
+ minstarttime, maxendtime = math.inf, -math.inf
+
+ for row in subscantable.row:
+ if row.execBlockId == execblockid:
+ if row.startTime < minstarttime:
+ minstarttime = row.startTime
+ if row.endTime > maxendtime:
+ maxendtime = row.endTime
+ return datetoMJD(minstarttime), datetoMJD(maxendtime)
+
+
+def filesdump(ingest_files):
+ for file in ingest_files:
+ LOG.info(f"File name: {file[0].filename} Size: {file[1]}")
+
+
+def dump_bdf_files(bdffiles):
+ for bdffile in bdffiles:
+ LOG.info(f"BDF file name: {bdffile.entityid} Size: {bdffile.filesize}")
diff --git a/apps/cli/executables/pexable/ingest/ingest/projectutils.py b/apps/cli/executables/pexable/ingest/ingest/projectutils.py
new file mode 100644
index 000000000..ceac2472d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/projectutils.py
@@ -0,0 +1,48 @@
+from sqlalchemy import text
+
+from .schema import create_session
+from .schema.optmodel import Project, Projectauthor, Coauthor
+
+session = create_session("OPT")
+
+
+def getproject(projectid):
+ project = session.query(Project).filter(text("id=:projectid")).params(projectid=projectid).first()
+ session.close()
+ return project
+
+
+# Get the global ids for the PI and coauthors of the project
+def getauthors(projectid):
+ pi = (
+ session.query(Projectauthor)
+ .join(Project, Projectauthor.id == Project.pi)
+ .filter(text("Project.id=:projectid"))
+ .params(projectid=projectid)
+ )
+ coauthors = (
+ session.query(Projectauthor)
+ .join(Coauthor, Projectauthor.id == Coauthor.projectauthor_id)
+ .join(Project, Coauthor.project_id == Project.id)
+ .filter(text("Project.id=:projectid"))
+ .params(projectid=projectid)
+ )
+ authors = pi.union(coauthors).all()
+ session.close()
+ return authors
+
+
+def ispi(projectcode, globalid):
+ pi = (
+ session.query(Projectauthor)
+ .join(Project, Projectauthor.id == Project.pi)
+ .filter(text("Project.projectcode=:projectcode"))
+ .filter(text("Projectauthor.globalid=:globalid"))
+ .params(projectcode=projectcode)
+ .params(globalid=globalid)
+ .one_or_none()
+ )
+ if pi is None:
+ return False
+ else:
+ return True
diff --git a/apps/cli/executables/pexable/ingest/ingest/proposalqueries.py b/apps/cli/executables/pexable/ingest/ingest/proposalqueries.py
new file mode 100644
index 000000000..e9846c1d8
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/proposalqueries.py
@@ -0,0 +1,20 @@
+from sqlalchemy import create_engine
+from sqlalchemy import text
+from sqlalchemy.orm import sessionmaker
+
+from .schema.pstmodel import Proposal
+
+# CAPO !
+pstengine = create_engine('mysql://jerry:garcia@webtest.aoc.nrao.edu/nrao_200', echo=True)
+Session = sessionmaker(bind=pstengine)
+session = Session()
+
+
+def getproposal(proposalcode):
+ proposal = session.query(Proposal).\
+ filter(text("PROP_ID like :proposalcode")).\
+ params(proposalcode='%' + proposalcode + '%').\
+ first()
+ return proposal
+
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/proposalutils.py b/apps/cli/executables/pexable/ingest/ingest/proposalutils.py
new file mode 100644
index 000000000..c558e2cde
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/proposalutils.py
@@ -0,0 +1,44 @@
+from sqlalchemy import text
+
+from .schema import create_session
+from .schema.pstmodel import Proposal, Author, UserAuthentication, Person
+
+session = create_session("PST")
+
+
+def getproposal(proposalcode):
+ proposal = (
+ session.query(Proposal)
+ .filter(text("PROP_ID like :proposalcode or LEGACY_ID like :proposalcode"))
+ .params(proposalcode="%" + proposalcode + "%")
+ .first()
+ )
+ session.close()
+ return proposal
+
+
+def getauthors(proposalcode):
+ authors = (
+ session.query(Author)
+ .join(Proposal, Author.proposal_id == Proposal.proposal_id)
+ .join(UserAuthentication, Author.user_id == UserAuthentication.userAuthentication_id)
+ .filter(text("PROP_ID like :proposalcode or LEGACY_ID like :proposalcode"))
+ .params(proposalcode="%" + proposalcode + "%")
+ .all()
+ )
+
+ session.close()
+ return authors
+
+
+def getpstuser(globalid):
+ user = session.query(Person).filter(text("person_id=:personid")).params(personid=globalid).one()
+
+ return user
+
+
+def ispi(author):
+ if author.proposal.principal_investigator_id == author.author_id:
+ return True
+ else:
+ return False
diff --git a/apps/cli/executables/pexable/ingest/ingest/pymygdala/__init__.py b/apps/cli/executables/pexable/ingest/ingest/pymygdala/__init__.py
new file mode 100644
index 000000000..2d39ea807
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/pymygdala/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+r""" Pymygdala: a CAPO aware Python library built atop pika for interacting with RabbitMQ, plus assorted simple command line applications built on the library.
+
+Using this library you can log messages to RabbitMQ with a standard-isg logging handler and the Python logging package, you can monitor a RabbitMQ server and dump out the logs, and you can send various events to the server.
+Many (most?) of the defaults for things like routing keys, exchange names and CAPO properties are heavily NRAO-centric but you can order-ride them if you want.
+"""
+
+from .models import LogHandler, LogDumper, SendEvent, SendNRAOEvent, RPCEvent
diff --git a/apps/cli/executables/pexable/ingest/ingest/pymygdala/commands.py b/apps/cli/executables/pexable/ingest/ingest/pymygdala/commands.py
new file mode 100644
index 000000000..e61024827
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/pymygdala/commands.py
@@ -0,0 +1,341 @@
+import argparse as ap
+import logging
+import os
+import subprocess
+import sys
+import json
+import re
+
+from pyat import version
+from pyat.pymygdala import (LogDumper, LogHandler, SendEvent, SendNRAOEvent)
+from pyat.pymygdala.models import MotDBroadcaster
+
+_ERR_MISSING_PROFILE = r"""ERROR: {script} can't deduce the 'profile', give it the --profile argument or set the CAPO_PROFILE environment variable! Geeze!
+
+"""
+_ERR_UNKNOWN_LEVEL = r"""ERROR: {script} pass unknown logging level {level}, expected DEBUG, INFO, WARN or ERROR
+"""
+
+_DUMPLOGS_NAME = 'pym-dumplogs'
+_DUMPLOGS_DESCRIPTION = r"""{script}, {version}: connect to a RabbitMQ server and dump the logs out.
+"""
+_DUMPLOGS_EPILOG = r"""Return values:
+ 0: everything worked as expected
+ 1: missing CAPO profile, no --profile argument or environment variable
+ 2: can't connect to specified RabbitMQ server
+"""
+
+
+def get_common_parser(description, epilog):
+ r""" Returns a parser for all these scripts with all the common parameters.
+
+ :param description:
+ :param epilog:
+ :return:
+ """
+ p = ap.ArgumentParser(description=description, epilog=epilog,
+ formatter_class=ap.RawDescriptionHelpFormatter)
+ p.add_argument('--profile', action='store',
+ help='CAPO profile name to use, e.g. test, production')
+ p.add_argument('--exchange', action='store',
+ help='exchange name to use, e.g. archive.logs')
+ p.add_argument('--type', action='store', dest='exchange_type',
+ help='exchange type to use, e.g. fanout')
+ p.add_argument('--hostname', action='store',
+ help='server name of the RabbitMQ server')
+ p.add_argument('--username', action='store',
+ help='username of the RabbitMQ server')
+ p.add_argument('--password', action='store',
+ help='password of the RabbitMQ server')
+ p.add_argument('--port', action='store', type=int,
+ help='port number of the RabbitMQ server')
+ p.add_argument('--key', action='store', dest='routing_key',
+ help='routing key, e.g. archive.logs')
+ return p
+
+
+def ns_to_d(ns):
+ r""" Convert a namespace into a dict, dropping properties that are 'None' along the way.
+
+ :param ns:
+ :return:
+ """
+ return {key: value for key, value in vars(ns).items() if value is not None}
+
+
+def get_settings(parser, script, **kwargs):
+ r""" Get the settings from a given parser, for a given script.
+
+ :param parser:
+ :param script:
+ :return:
+ """
+ d = ns_to_d(parser.parse_args(**kwargs))
+
+ if 'profile' not in d and 'CAPO_PROFILE' not in os.environ:
+ sys.stderr.write(_ERR_MISSING_PROFILE.format(script=script))
+ parser.print_help()
+ sys.exit(1)
+ d['profile'] = os.environ['CAPO_PROFILE'] if 'profile' not in d else d['profile']
+ return d
+
+
+def get_dumplogs_parser():
+ r""" Get a parser for the pym-dumplogs CLI.
+
+ :return:
+ """
+ p = get_common_parser(_DUMPLOGS_DESCRIPTION.format(script=_DUMPLOGS_NAME, version=version),
+ _DUMPLOGS_EPILOG)
+ p.add_argument('--outfile', action='store', default='-',
+ help='write output to file, - for STDOUT')
+ return p
+
+
+def dumplogs():
+ r""" The pym-dumplogs CLI.
+
+ :return:
+ """
+ parser = get_dumplogs_parser()
+ settings = get_settings(parser, _DUMPLOGS_NAME)
+ dumper = LogDumper(**settings)
+ dumper.dump()
+
+
+_SENDLOG_NAME = 'pym-sendlog'
+_SENDLOG_DESCRIPTION = r"""{script}, {version}: a command line tool for logging to RabbitMQ."""
+_SENDLOG_EPILOG = r"""Return values:
+ 0: everything worked as expected
+ 1: missing CAPO profile, no '--profile' argument or environment variable
+ 2: can't connect to specified RabbitMQ server
+ 3: unknown logging level, should be DEBUG, INFO, WARN or ERROR
+"""
+
+
+def get_sendlog_parser():
+ p = get_common_parser(_SENDLOG_DESCRIPTION.format(script=_SENDLOG_NAME, version=version),
+ _SENDLOG_EPILOG)
+ r = p.add_argument_group('Required Arguments')
+ r.add_argument('--level', action='store', required=True,
+ help='logging level, e.g. DEBUG, WARN')
+ r.add_argument('--message', action='store', required=True,
+ help='message to log')
+ r.add_argument('--app', dest='application', action='store', required=True,
+ help='the application name to log as')
+ return p
+
+
+def sendlog():
+ r""" The pym-sendlog CLI.
+
+ :Returns:
+ 0: everything worked as expected
+ 1: can't deduce CAPO_PROFILE
+ 2: error talking to RabbitMQ
+ 3: bad level (not DEBUG, INFO, WARN or ERROR)
+ """
+ parser = get_sendlog_parser()
+ settings = get_settings(parser, _SENDLOG_NAME)
+ log = logging.getLogger(__name__)
+ log.setLevel(logging.DEBUG)
+ handler = LogHandler(**settings)
+ log.addHandler(handler)
+ fn = None
+ if settings['level'] == 'DEBUG':
+ fn = log.debug
+ elif settings['level'] == 'INFO':
+ fn = log.debug
+ elif settings['level'] == 'WARN':
+ fn = log.debug
+ elif settings['level'] == 'ERROR':
+ fn = log.debug
+
+ if fn is not None:
+ try:
+ fn(settings['message'])
+ except:
+ sys.stderr.write(_ERR_MISSING_PROFILE.format(script=_SENDLOG_NAME))
+ parser.print_help()
+ sys.exit(2)
+ else:
+ sys.stderr.write(_ERR_UNKNOWN_LEVEL.format(script=_SENDLOG_NAME, level=settings['level']))
+ parser.print_help()
+ sys.exit(4)
+
+
+def amqp_logging_shell_wrapper():
+ parser = get_common_parser(
+ "logwrapper, {version}: a command line tool for sending console output "
+ "of some other command to RabbitMQ.".format(version=version), "")
+ parser.add_argument("cmd", type=str, nargs="*")
+ settings = get_settings(parser, _SENDLOG_NAME)
+ log = logging.getLogger(__name__)
+ log.setLevel(logging.DEBUG)
+ settings['application'] = settings['cmd'][0]
+ handler = LogHandler(**settings)
+ log.addHandler(handler)
+ with subprocess.Popen(settings['cmd'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ log.info(line.strip())
+ return p.returncode
+
+_SENDEVENT_NAME = 'pym-sendevent'
+_SENDEVENT_DESCRIPTION = r"""{script}, {version}: send an event to a RabbitMQ server."""
+_SENDEVENT_EPILOG = r"""Return values:
+ 0: everything worked as expected
+ 1: missing CAPO profile, no '--profile' argument or environment variable
+ 2: can't connect to specified RabbitMQ server
+"""
+
+
+def get_sendevent_parser():
+ p = get_common_parser(_SENDEVENT_DESCRIPTION.format(script=_SENDEVENT_NAME, version=version),
+ _SENDEVENT_EPILOG)
+ p.add_argument('--event', dest='eventName', action='store',
+ help='eventName to send, e.g. runSdmIngestionWorkflow')
+ p.add_argument('--nrao', dest='nrao_event', action='store_true', default=False,
+ help='add NRAO specific fields to the event')
+ p.add_argument('--message', action='store', default="",
+ help='provide a string for the event message')
+ p.add_argument('--md-name', dest='md_name', action='store', default='logData',
+ help='name of an extra metadata message section')
+ p.add_argument('--metadata', action='append', default=[],
+ help="provide a KEY=VALUE pair in the md-name section of the message; can be used multiple times")
+ r = p.add_argument_group('Required Arguments')
+ r.add_argument('--app', dest='application', action='store', required=True,
+ help='the application name to log as')
+ return p
+
+
+def build_event(settings):
+ result = {'message': settings['message']}
+ if 'md_name' in settings:
+ result[settings['md_name']] = dict(arg.split('=') for arg in settings['metadata'])
+ if 'eventName' in settings:
+ result['eventName'] = settings['eventName']
+ return result
+
+
+def sendevent(**kwargs):
+ r""" The pym-sendevent CLI.
+
+ :Returns:
+ 0: everything worked as expected
+ 1: can't deduce CAPO_PROFILE
+ 2: error talking to RabbitMQ
+ """
+ parser = get_sendevent_parser()
+ settings = get_settings(parser, _SENDEVENT_NAME, **kwargs)
+ event = build_event(settings)
+ try:
+ if settings['nrao_event']:
+ se = SendNRAOEvent(**settings)
+ else:
+ se = SendEvent(**settings)
+ se.send(event)
+ except:
+ sys.stderr.write(_ERR_MISSING_PROFILE.format(script=_SENDEVENT_NAME))
+ parser.print_help()
+ sys.exit(2)
+
+
+def ingestion_complete():
+ parser = get_common_parser("Simulate an ingestion-complete event", "")
+ parser.add_argument("--ingestion_type", type=str, action='store', default='rawdata',
+ help='Type of ingestion which just finished (ex. rawdata, calibration')
+ parser.add_argument("--execblock_id", type=str, action='store',
+ help='Execution block ID which just finished ingesting')
+ parser.add_argument("--skip_screen", type=str, action='store', default='False',
+ help='If set to True launches CIPL independent of current cal status (False by default)')
+ parser.add_argument('--fileset_id', type=str, required=True,
+ help='File set ID of the ingestion that completed')
+ parser.add_argument('--telescope', type=str, action='store', default='EVLA',
+ help='Telescope originating this data')
+ parser.add_argument('--schedblock_id', type=str, action='store',
+ help='Scheduling block ID for this data')
+ parser.add_argument('--schedblock_type', type=str, action='store', default='SCIENCE',
+ help='Scheduling block type (SCIENCE, OPERATIONS, EXPERT, TEST)')
+ parser.add_argument('--project_code', type=str, action='store',
+ help='Project code for this data')
+ parser.add_argument('--vlass', action='store_true', default=False,
+ help='Wait for VLASS Manager reply')
+ parser.add_argument('--file_count', type=str, action='store', default='1',
+ help='Number of files that were just ingested here')
+ args = parser.parse_args()
+
+ settings = get_settings(parser, "ingestion-complete")
+ settings['message'] = 'Ingestion complete'
+ settings['routing_key'] = 'ingestion-complete.' + ('rawdata' if settings['ingestion_type'] == 'observation' else settings['ingestion_type'])
+
+ # let's try parsing the data we need from the fileset ID
+ match = re.search(r'(.*)\.sb(\d+)\.eb(\d+)\..*', args.fileset_id)
+ if match:
+ project_code, schedblock_id, execblock_id = match.groups()
+ else:
+ project_code, schedblock_id, execblock_id = None, None, None
+
+ event = build_event(settings)
+ eb_id = args.execblock_id or execblock_id
+ event['logData'] = {'ingestion_type': args.ingestion_type,
+ 'execblock_id': eb_id,
+ 'override_db_screen': args.skip_screen,
+ 'fileset_id': args.fileset_id,
+ 'telescope': args.telescope,
+ 'schedblock_id': args.schedblock_id or schedblock_id,
+ 'schedblock_type': args.schedblock_type,
+ 'project_code': args.project_code or project_code,
+ 'file_count': args.file_count,
+ 'profile': args.profile}
+
+ try:
+ if 'nrao_event' in settings and settings['nrao_event']:
+ se = SendNRAOEvent(**settings)
+ else:
+ se = SendEvent(**settings)
+
+ # make the queue and get ready for the callback message
+ if args.vlass or 'VLASS' in (args.project_code or project_code):
+ queue = se.channel.queue_declare(queue='', exclusive=True, auto_delete=True, durable=False).method.queue
+ se.channel.queue_bind(queue=queue, exchange="archive.events", routing_key='VlassMngr.' + eb_id)
+
+ se.send(event)
+
+ if args.vlass or 'VLASS' in (args.project_code or project_code):
+ print("message sent, waiting for reply from the manager")
+
+ # we're going to iterate that queue, but we really just want the first message
+ # so we'll exit at the end of the first loop iteration
+ for _, properties, message in se.channel.consume(queue):
+ response = json.loads(message)
+ print('VlassMngr response: ' + response['message'])
+ se.close()
+ sys.exit(0 if response['logData']['launching_jobs'] == 'true' else 1)
+ else:
+ print('message sent')
+ except Exception as e:
+ sys.stdout.write(e)
+ sys.stderr.write(_ERR_MISSING_PROFILE.format(script=_SENDEVENT_NAME))
+ parser.print_help()
+ sys.exit(2)
+
+
+def sendmotd():
+ r""" The pym-sendmotd CLI.
+
+ :Returns:
+ 0: everything worked as expected
+ 1: can't deduce CAPO_PROFILE
+ 2: error talking to RabbitMQ
+ 3: bad level (not DEBUG, INFO, WARN or ERROR)
+ """
+ parser = get_sendlog_parser()
+ settings = get_settings(parser, _SENDLOG_NAME)
+ handler = MotDBroadcaster(**settings)
+ handler.broadcast(settings['message'])
+
+# if __name__ == '__main__':
+# try:
+# sendmotd()
+# except Exception as mex:
+# print('__main__:{0}'.format(mex))
diff --git a/apps/cli/executables/pexable/ingest/ingest/pymygdala/models.py b/apps/cli/executables/pexable/ingest/ingest/pymygdala/models.py
new file mode 100644
index 000000000..ba77a2439
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/pymygdala/models.py
@@ -0,0 +1,713 @@
+# -*- coding: utf-8 -*-
+
+import contextlib
+import datetime
+import getpass
+import logging
+import os
+import socket
+import uuid
+from threading import Lock
+
+import pika
+import simplejson as json
+import sys
+from dateutil.parser import parse
+from pycapo import CapoConfig
+
+# CAPO properties we care about
+CAPO_PROP_HOSTNAME = 'edu.nrao.archive.configuration.AmqpServer.hostname'
+CAPO_PROP_USERNAME = 'edu.nrao.archive.configuration.AmqpServer.username'
+CAPO_PROP_PASSWORD = 'edu.nrao.archive.configuration.AmqpServer.password'
+CAPO_PROP_PORT = 'edu.nrao.archive.configuration.AmqpServer.port'
+
+# Assorted other defaults.
+CONNECTION_ATTEMPTS = 5
+RETRY_DELAY = 500
+SOCKET_TIMEOUT = 5000
+LOG_ROUTING_KEY_FORMAT = '{application}.{level}'
+EVENT_ROUTING_KEY_FORMAT = '{application}.event'
+EVENT_EXCHANGE = 'archive.events'
+EVENT_EXCHANGE_TYPE = 'topic'
+LOG_EXCHANGE = 'archive.logs'
+LOG_EXCHANGE_TYPE = 'fanout'
+MOTD_ROUTING_KEY_FORMAT = '{application}.motd'
+MOTD_EXCHANGE = 'archive.frontend.motd'
+MOTD_EXCHANGE_TYPE = 'fanout'
+MOTD_LEVEL = 'info'
+LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+LOG_LEVEL = logging.NOTSET
+LOGDUMPER_FORMAT = '{hostname} - {date} [{threadName}] {level:<5} {loggerName:<36}#{method}:{line} - {formattedMessage}\n'
+
+
+def _now():
+ r""" Get the current date and time in UTC as a string, round the fractional seconds to the precision that won't break jaxb. """
+ t = datetime.datetime.now()
+ s = t.strftime('%Y-%m-%dT%H:%M:%S.%f')
+ f = round(float(s[-7:]), 3)
+ temp = "%.3f" % f
+ temp = "%s%s" % (s[:-7], temp[1:])
+ return temp + "+00:00"
+
+
+@contextlib.contextmanager
+def _smart_open(filename=None):
+ """If we get a filename and it isn't '-', return a file handle to it for appending, else return a file handle to stdout. Saw on stack overflow, I'm not this clever.
+ http://stackoverflow.com/questions/17602878/how-to-handle-both-with-open-and-sys-stdout-nicely
+ """
+ if filename and filename != '-':
+ fh = open(filename, 'a')
+ else:
+ fh = sys.stdout
+
+ try:
+ yield fh
+ finally:
+ if fh is not sys.stdout:
+ fh.close()
+
+
+class _Pymygdala():
+ r""" Class for sending events to a RabbitMQ server. This gets used as a base class for the other exposed classes in the package.
+
+ :Keyword Arguments:
+ *profile* ("string"): the name of the profile to use, e.g. 'test', 'production'. If missing this defaults to the CAPO_PROFILE environment variable, if that is missing as well it explodes, throwing a ValueError exception. A profile should be a simple word, one without spaces or tabs or evil things in it. This is not checked and no guarantee is provided.
+
+ *application* ("string"): the name of the application sending the event or message, "unknown" by default.
+
+ *hostname* ("string"): name of the AMQP server to use, defaults to CAPO property pymygdala.CAPO_PROP_HOSTNAME.
+
+ *username* ("string"): username to connect to AMQP with, defaults to CAPO property pymygdala.CAPO_PROP_USERNAME.
+
+ *password* ("string"): password to connect to AMQP with, defaults to CAPO property pymygdala.CAPO_PROP_PASSWORD.
+
+ *port* ("int"): port number to connect to AMQP with, defaults to CAPO property pymygdala.CAPO_PROP_PORT.
+
+ *exchange* ("string"): the exchange to use, defaults to pymygdala.EVENT_EXCHANGE.
+
+ *exchange_type* ("string"): type of exchange to use, defaults to pymygdala.EVENT_EXCHANGE_TYPE.
+
+ *routing_key* ("string"): how messages will be routed to queues, format defaults to pymygdala.EVENT_ROUTING_KEY_FORMAT.
+
+ *connection_attempts* (int): maximum number of retry attempts.
+
+ *retry_delay* (int|float): time to wait in seconds, before the next.
+
+ *socket_timeout* (int|float): use for high latency networks.
+
+ :Example:
+ """
+
+ def __init__(self, **kwargs):
+ if 'profile' in kwargs:
+ self.profile = kwargs['profile']
+ elif 'CAPO_PROFILE' in os.environ:
+ self.profile = os.environ['CAPO_PROFILE']
+ else:
+ raise ValueError('Can not deduce CAPO profile to use')
+ self.config = CapoConfig(profile=self.profile)
+ self.application = kwargs.get('application', 'unknown')
+ self.exchange = kwargs.get('exchange', EVENT_EXCHANGE)
+ self.exchange_type = kwargs.get('exchange_type', EVENT_EXCHANGE_TYPE)
+ self.routing_key = kwargs.get('routing_key', EVENT_ROUTING_KEY_FORMAT.format(application=self.application))
+
+ # Connection parameters for talking to RabbitMQ
+ self.connection_parameters = pika.ConnectionParameters(
+ host=kwargs.get('hostname', self.config.getstring(CAPO_PROP_HOSTNAME)),
+ port=kwargs.get('port', self.config.getint(CAPO_PROP_PORT)),
+ connection_attempts=kwargs.get('connection_attempts', CONNECTION_ATTEMPTS),
+ socket_timeout=kwargs.get('socket_timeout', SOCKET_TIMEOUT),
+ retry_delay=kwargs.get('retry_delay', RETRY_DELAY),
+ credentials= \
+ pika.PlainCredentials(username=kwargs.get('username', self.config.getstring(CAPO_PROP_USERNAME)),
+ password=kwargs.get('password', self.config.getstring(CAPO_PROP_PASSWORD))))
+
+ # Parameters we derive or produce and don't let the caller set
+ self.sender_username = getpass.getuser()
+ self.sender_hostname = socket.gethostname()
+ self.connection = None
+ self.channel = None
+ self._lock = Lock()
+ self._open_connection()
+
+ def _close_connection(self):
+ if self.channel:
+ self.channel.close()
+ if self.connection:
+ self.connection.close()
+ self.connection, self.channel = None, None
+
+ def _open_connection(self):
+ self.connection = pika.BlockingConnection(self.connection_parameters)
+ self.channel = self.connection.channel()
+ self.channel.exchange_declare(self.exchange,
+ exchange_type=self.exchange_type,
+ durable=True, auto_delete=False)
+
+ def close(self):
+ self._lock.acquire()
+ try:
+ self._close_connection()
+ finally:
+ self._lock.release()
+
+ def _create_basic_props(self, prop_dict):
+ """Takes a dictionary of values and generates a pika.BasicProperties() call
+
+ From the Spec
+ class pika.spec.BasicProperties(
+ content_type=None,
+ content_encoding=None,
+ headers=None,
+ delivery_mode=None,
+ priority=None,
+ correlation_id=None,
+ reply_to=None,
+ expiration=None,
+ message_id=None,
+ timestamp=None,
+ type=None,
+ user_id=None,
+ app_id=None,
+ cluster_id=None
+ )
+
+
+ """
+ my_content_type = None
+ my_content_encoding = None
+ my_headers = None
+ my_delivery_mode = None
+ my_priority = None
+ my_correlation_id = None
+ my_reply_to = None
+ my_expiration = None
+ my_message_id = None
+ my_timestamp = None
+ my_type = None
+ my_user_id = None
+ my_app_id = None
+ my_cluster_id = None
+
+
+ # TODO: The rest of the properties. I'm sure we'll use them at some point.
+
+ if "reply_to" in prop_dict:
+ my_reply_to = prop_dict["reply_to"]
+
+ if "correlation_id" in prop_dict:
+ my_correlation_id = prop_dict["correlation_id"]
+
+ return pika.BasicProperties(
+ content_type=my_content_type,
+ content_encoding=my_content_encoding,
+ headers=my_headers,
+ delivery_mode=my_delivery_mode,
+ priority=my_priority,
+ correlation_id=my_correlation_id,
+ reply_to=my_reply_to,
+ expiration=my_expiration,
+ message_id=my_message_id,
+ timestamp=my_timestamp,
+ type=my_type,
+ user_id=my_user_id,
+ app_id=my_app_id,
+ cluster_id=my_cluster_id
+ )
+
+ def _send(self, event, routing_key=None, headers=None):
+ """I need a docstring here."""
+ if routing_key is None:
+ routing_key = self.routing_key
+ self._lock.acquire()
+ try:
+ if headers is not None:
+ publication_properties = self._create_basic_props(headers)
+ self.channel.basic_publish(exchange=self.exchange,
+ routing_key=routing_key,
+ properties=publication_properties,
+ body=event)
+ else:
+ self.channel.basic_publish(exchange=self.exchange,
+ routing_key=routing_key,
+ body=event)
+ except Exception:
+ self.channel, self.connection = None, None
+ finally:
+ self._lock.release()
+
+
+class LogHandler(logging.Handler, _Pymygdala):
+ r""" Logging handler for a RabbitMQ server.
+
+ :Keyword Arguments:
+ *profile* ("string"): the name of the profile to use, e.g. 'test', 'production'. If missing this defaults to the CAPO_PROFILE environment variable, if that is missing as well it explodes, throwing a ValueError exception. A profile should be a simple word, one without spaces or tabs or evil things in it. This is not checked and no guarantee is provided.
+
+ *hostname* ("string"): name of the AMQP server to use, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.hostname".
+
+ *username* ("string"): username to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.username".
+
+ *password* ("string"): password to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.password".
+
+ *port* ("int"): port number to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.port".
+
+ *connection_attempts* (int): maximum number of retry attempts.
+
+ *retry_delay* (int|float): time to wait in seconds, before the next.
+
+ *socket_timeout* (int|float): use for high latency networks.
+
+ *exchange* ("string"): the exchange to use, defaults to pymygdala.LOG_EXCHANGE.
+
+ *exchange_type* ("string"): type of exchange to use, defaults to pymygdala.LOG_EXCHANGE_TYPE.
+
+ *application* ("string"): the name of the application sending the event or message, "unknown" by default.
+
+ *level* ("level"): the logging level to use.
+
+ :Example:
+
+ >>> import logging
+ >>> from pyat.pymygdala import LogHandler
+ >>> log = logging.getLogger(__name__)
+ >>> handler = LogHandler(profile='test', application='test-app', level=logging.DEBUG)
+ >>> log.addHandler(handler)
+ >>> log.error("my hovercraft is full of eels")
+
+ """
+
+ def __init__(self, **kwargs):
+ self.level = kwargs.get('level', LOG_LEVEL)
+ self.exchange = kwargs.get('exchange', LOG_EXCHANGE)
+ self.exchange_type = kwargs.get('exchange_type', LOG_EXCHANGE_TYPE)
+ self.application = kwargs.get('application', 'unknown')
+ self.routing_key = LOG_ROUTING_KEY_FORMAT.format(application=self.application, level='WARNING')
+ _Pymygdala.__init__(self, **dict(kwargs,
+ exchange=self.exchange,
+ exchange_type=self.exchange_type,
+ routing_key=self.routing_key))
+ logging.Handler.__init__(self)
+
+ def close(self):
+ _Pymygdala.close(self)
+
+ def record_to_dict(self, record):
+ r""" Turn a Python logging.LogRecord into a simple dictionary, with the structure we expect,
+ that can easily be turned into a JSON string and sent off to RabbitMQ. """
+ d = dict()
+ d['timestamp'] = _now()
+ d['formattedMessage'] = record.msg
+ d['loggerName'] = record.pathname
+ d['level'] = record.levelname
+ d['threadName'] = record.threadName
+ d['referenceMask'] = 1
+ d['filename'] = ''
+ d['class'] = ''
+ d['method'] = record.name
+ d['line'] = record.lineno
+ d['properties'] = dict()
+ d['properties']['user'] = self.sender_username
+ d['properties']['HOSTNAME'] = self.sender_hostname
+ d['properties']['profile'] = self.profile
+ return d
+
+ def emit(self, record):
+ r""" Emit a logging message, build the routing key per emission because it has the logging
+ level as part of it. """
+ self._send(json.dumps(self.record_to_dict(record)),
+ routing_key=LOG_ROUTING_KEY_FORMAT.format(application=self.application,
+ level=record.levelname))
+
+
+class LogDumper(_Pymygdala):
+ r""" Dump the logs from a RabbitMQ server.
+
+ :Keyword Arguments:
+ *profile* ("string"): the name of the profile to use, e.g. 'test', 'production'. If missing this defaults to the CAPO_PROFILE environment variable, if that is missing as well it explodes, throwing a ValueError exception. A profile should be a simple word, one without spaces or tabs or evil things in it. This is not checked and no guarantee is provided.
+
+ *hostname* ("string"): name of the AMQP server to use, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.hostname".
+
+ *username* ("string"): username to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.username".
+
+ *password* ("string"): password to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.password".
+
+ *port* ("int"): port number to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.port".
+
+ *connection_attempts* (int): maximum number of retry attempts.
+
+ *retry_delay* (int|float): time to wait in seconds, before the next.
+
+ *socket_timeout* (int|float): use for high latency networks.
+
+ *exchange* ("string"): the exchange to use, defaults to pymygdala.LOG_EXCHANGE.
+
+ *exchange_type* ("string"): type of exchange to use, defaults to pymygdala.LOG_EXCHANGE_TYPE.
+
+ *outfile* ("string"): the name of the file to write to, default '-' for sys.stdout.
+
+ :Example:
+
+ >>> # Dump the 'test' logs to stdout
+ >>> from pyat.pymygdala import LogDumper
+ >>> dumper = LogDumper(profile='test')
+ >>> dumper.dump()
+
+ """
+
+ def __init__(self, **kwargs):
+ _Pymygdala.__init__(self, **kwargs)
+ self.outfile = kwargs.get('outfile', '-')
+ self.queue = self.channel.queue_declare('', exclusive=True, auto_delete=True,
+ durable=False).method.queue
+ self.channel.queue_bind(self.queue, LOG_EXCHANGE)
+ self.channel.basic_consume(self.queue, on_message_callback=self._callback, auto_ack=True)
+
+ def close(self):
+ _Pymygdala.close(self)
+
+ def _callback(self, ch, method, properties, body):
+ r"""I need more docstrings."""
+ with _smart_open(self.outfile) as fh:
+ try:
+ parsed = json.loads(body)
+ parsed['hostname'] = parsed['properties']['HOSTNAME'] \
+ if 'HOSTNAME' in parsed['properties'] else ''
+
+ if 'formattedMessage' in parsed:
+ parsed['date'] = parse(parsed['timestamp'])
+ fh.write(LOGDUMPER_FORMAT.format(**parsed))
+ else:
+ fh.write('no formatted message: {}\n'.format(body))
+
+ if 'stackTrace' in parsed:
+ for stack in parsed['stackTrace']:
+ for line in stack:
+ fh.write('{}\n'.format(line))
+
+ except Exception as e:
+ fh.write('{}\n'.format(e))
+ fh.write('unparseable: {}\n'.format(body))
+
+ def dump(self):
+ r"""I need more docstrings."""
+ self.channel.start_consuming()
+
+
+def _datetime_to_dict():
+ # Turn 'now' in UTC into the date format events expect to speak.
+ d, t = dict(), dict()
+ now = datetime.datetime.utcnow()
+ d['year'], d['month'], d['day'] = now.year, now.month, now.day
+ t['hour'], t['minute'], t['second'], t['nano'] = now.hour, now.minute, now.second, now.microsecond * 1000
+ return {'date': d, 'time': t}
+
+
+class SendEvent(_Pymygdala):
+ r"""
+
+ :Keyword Arguments:
+ *profile* ("string"): the name of the profile to use, e.g. 'test', 'production'. If missing this defaults to the CAPO_PROFILE environment variable, if that is missing as well it explodes, throwing a ValueError exception. A profile should be a simple word, one without spaces or tabs or evil things in it. This is not checked and no guarantee is provided.
+
+ *application* ("string"): the name of the application sending the event or message, "unknown" by default.
+
+ *hostname* ("string"): name of the AMQP server to use, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.hostname"
+
+ *username* ("string"): username to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.username".
+
+ *password* ("string"): password to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.password".
+
+ *port* ("int"): port number to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.port".
+
+ *connection_attempts* (int): maximum number of retry attempts
+
+ *retry_delay* (int|float): time to wait in seconds, before the next
+
+ *socket_timeout* (int|float): use for high latency networks
+
+ *exchange* ("string"): the exchange to use, defaults to pymygdala.EVENT_EXCHANGE
+
+ *exchange_type* ("string"): type of exchange to use, defaults to pymygdala.EVENT_EXCHANGE_TYPE.
+
+ *routing_key* ("string"): how messages will be routed to queues, format defaults to pymygdala.EVENT_ROUTING_KEY_FORMAT
+
+ :Example:
+
+ >>> # Send the kind of event the solr indexer does when it starts to index
+ >>> from pyat.pymygdala import SendEvent
+ >>> se = SendEvent(profile='test', application='solr-indexer')
+ >>> event = {
+ ... "logData": {
+ ... "dryRun": False,
+ ... "numberOfDocuments": 9999
+ ... },
+ ... "message": "Starting reindexing process",
+ ... "request": "some request"
+ ... }
+ >>> se.send(event)
+
+ """
+
+ def __init__(self, **kwargs):
+ _Pymygdala.__init__(self, **kwargs)
+
+ def close(self):
+ _Pymygdala.close(self)
+
+ def _add_fields(self, event):
+ result = dict()
+ result['user'] = self.sender_username
+ result['hostname'] = self.sender_hostname
+ result['timestamp'] = _datetime_to_dict()
+ result['application'] = self.application
+ result['request'] = 'snek asks nice-like'
+ result['version'] = 1.0
+ result.update(event)
+ return result
+
+ def send(self, event, routing_key=None, headers=None):
+ r""" Send an event to RabbitMQ: an 'event' here is a simple dictionary that can be converted into a JSON string.
+
+ :Required Arguments:
+ *event* ("dict"): a dictionary representing the event to send.
+ :Optional Arguments:
+ *routing_key* ("string"): the routing key of the message, defaults to application.event
+ *headers* ("dict"): a dictionary of headers to send with the event as 'properties'
+
+ """
+ if routing_key is None:
+ routing_key = self.routing_key
+ event = self._add_fields(event)
+ # print(json.dumps(d, sort_keys=True, indent=4 * ' '))
+ self._send(json.dumps(event), routing_key=routing_key, headers=headers)
+
+
+class SendNRAOEvent(SendEvent):
+ r""" This subclasses SendEvent and just add some more fields to the event based on the exchange, so you can use it to trigger AAT/PPI workflows more easily.
+
+ :Keyword Arguments:
+ *profile* ("string"): the name of the profile to use, e.g. 'test', 'production'. If missing this defaults to the CAPO_PROFILE environment variable, if that is missing as well it explodes, throwing a ValueError exception. A profile should be a simple word, one without spaces or tabs or evil things in it. This is not checked and no guarantee is provided.
+
+ *application* ("string"): the name of the application sending the event or message, "unknown" by default.
+
+ *hostname* ("string"): name of the AMQP server to use, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.hostname"
+
+ *username* ("string"): username to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.username".
+
+ *password* ("string"): password to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.password".
+
+ *port* ("int"): port number to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.port".
+
+ *connection_attempts* (int): maximum number of retry attempts
+
+ *retry_delay* (int|float): time to wait in seconds, before the next
+
+ *socket_timeout* (int|float): use for high latency networks
+
+ *exchange* ("string"): the exchange to use, defaults to pymygdala.EVENT_EXCHANGE
+
+ *exchange_type* ("string"): type of exchange to use, defaults to pymygdala.EVENT_EXCHANGE_TYPE.
+
+ *routing_key* ("string"): how messages will be routed to queues, format defaults to pymygdala.EVENT_ROUTING_KEY_FORMAT
+
+ :Example:
+
+ >>> # Send the kind of event the solr indexer does when it starts to index
+ >>> from pyat.pymygdala import SendNRAOEvent
+ >>> se = SendEvent(profile='test', application='solr-indexer')
+ >>> headers = {
+ ... "reply-to": "jim@bob.foo"
+ ... }
+ >>> event = {
+ ... "logData": {
+ ... "dryRun": False,
+ ... "numberOfDocuments": 9999
+ ... },
+ ... "message": "Starting reindexing process",
+ ... "request": "some request"
+ ... }
+ >>> se.send(event, headers=headers)
+
+ """
+
+ def __init__(self, **kwargs):
+ SendEvent.__init__(self, **kwargs)
+
+ def close(self):
+ SendEvent.close(self)
+
+ def _add_fields(self, event):
+ result = SendEvent._add_fields(self, event)
+ if self.exchange == 'archive.workflow-commands':
+ result['type'] = 'edu.nrao.archive.workflow.messaging.commands.StartWorkflow'
+ result['additionalPaths'] = []
+ return result
+
+
+class RPCEvent(SendEvent):
+ """A class to handle the nuances of a dedicated response queue & correlation ID. This is for events which will generate
+ a reply from the system.
+
+ Usage:
+ sender = RPCEvent(...JustLikeTheOthers...)
+ my_id = sender.send()
+ start_workflow(corr_id=my_id)
+ my_answer = sender.get_reply()
+
+
+ TODO: Lay out the total set of variables, highlighting the additions.
+
+ """
+
+ def __init__(self, **kwargs):
+ # Superclass handles most of it:
+ SendEvent.__init__(self, **kwargs)
+
+ # Initializations
+ self.event_corr_id = str(uuid.uuid4())
+ self.answer = None
+ self.reply_channel = None
+
+ # New Channel for the replies:
+ self.reply_channel = self.connection.channel()
+
+ # Start listening to the events exchange for our reply:
+ events_exchange = self.reply_channel.exchange_declare('archive.events', exchange_type='topic', durable=True)
+ specific_queue = "temp-queue-"+self.event_corr_id
+ new_queue = self.reply_channel.queue_declare(specific_queue, durable=False, exclusive=True)
+
+ specific_key = "cli-response."+self.event_corr_id
+ self.reply_channel.queue_bind(specific_queue, 'archive.events', routing_key=specific_key)
+ self.reply_queue = new_queue.method.queue
+
+ # Start Listening
+ self.reply_channel.basic_consume(self.reply_queue, on_message_callback=self._response_handler, auto_ack=True)
+
+ def close(self):
+ # Our superclass handles the basics:
+ SendEvent.close(self)
+
+ # Basic Event handler, only provide the information if the correlation ids match.
+ def _response_handler(self, channel, method, properties, body):
+ # this used to check the properties, but they're harder to customize with an event
+ self.answer = body
+
+ def send(self, event, routing_key=None, headers=None):
+
+ # Configure the passed-in headers with reply information:
+ if headers is not None:
+ headers["correlation_id"] = self.event_corr_id
+ headers["reply_to"] = self.reply_queue
+ else:
+ # No header, populate one with what we need:
+ headers = {
+ "correlation_id": self.event_corr_id,
+ "reply_to": self.reply_queue
+ }
+
+ # Pick up the configurations of our superclass
+ event = super()._add_fields(event)
+
+ # But call back to _Pymgydala for the actual send
+ self._send(json.dumps(event), routing_key=routing_key, headers=headers)
+
+ # return the correlation ID to include in the workflow launch
+ return self.event_corr_id
+
+ def get_reply(self):
+ # If we didn't send an event, we don't expect a reply
+ if self.event_corr_id is None:
+ print("No pending event!")
+ return None
+
+ # Block while we're waiting for our answer
+ # if it has already come, then we'll skip the loop
+ while self.answer is None:
+ # Block, waiting for an answer
+ self.connection.process_data_events()
+
+ decoded_event = json.loads(self.answer)
+
+ self.event_corr_id = None
+ self.answer = None
+
+ # Provide the true payload to caller, instead of the
+ # full event.
+ return decoded_event["logData"]
+
+
+class MotDBroadcaster(_Pymygdala):
+ r""" MotD handler for a RabbitMQ server.
+
+ :Keyword Arguments:
+ *profile* ("string"): the name of the profile to use, e.g. 'test', 'production'. If missing this defaults to the CAPO_PROFILE environment variable, if that is missing as well it explodes, throwing a ValueError exception. A profile should be a simple word, one without spaces or tabs or evil things in it. This is not checked and no guarantee is provided.
+
+ *hostname* ("string"): name of the AMQP server to use, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.hostname".
+
+ *username* ("string"): username to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.username".
+
+ *password* ("string"): password to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.password".
+
+ *port* ("int"): port number to connect to AMQP with, defaults to CAPO property "edu.nrao.archive.configuration.AmqpServer.port".
+
+ *connection_attempts* (int): maximum number of retry attempts.
+
+ *retry_delay* (int|float): time to wait in seconds, before the next.
+
+ *socket_timeout* (int|float): use for high latency networks.
+
+ *exchange* ("string"): the exchange to use, defaults to pymygdala.MOTD_EXCHANGE.
+
+ *exchange_type* ("string"): type of exchange to use, defaults to pymygdala.MOTD_EXCHANGE_TYPE.
+
+ *application* ("string"): the name of the application sending the event or message, "unknown" by default.
+
+ """
+
+ def __init__(self, **kwargs):
+ self.level = kwargs.get('level', MOTD_LEVEL)
+ self.exchange = kwargs.get('exchange', MOTD_EXCHANGE)
+ self.exchange_type = kwargs.get('exchange_type', MOTD_EXCHANGE_TYPE)
+ self.application = kwargs.get('application', 'unknown')
+ self.routing_key = MOTD_ROUTING_KEY_FORMAT.format(application=self.application)
+ _Pymygdala.__init__(self, **dict(kwargs,
+ exchange=self.exchange,
+ exchange_type=self.exchange_type,
+ routing_key=self.routing_key))
+
+ def close(self):
+ _Pymygdala.close(self)
+
+ def broadcast(self, message):
+ r""" Emit a logging message, build the routing key per emission because it has the logging
+ level as part of it. """
+ logging.info('MotD broadcasting: {0}'.format(message))
+ self._send(json.dumps(dict(motd=message, level=self.level)),
+ routing_key=self.routing_key)
+
+
+def test_sendlog():
+ # A quick and dirty test of logging in our environment.
+ LOG = logging.getLogger(__name__)
+ handler = LogHandler(profile='test', application='test-app',
+ level=logging.DEBUG)
+ LOG.addHandler(handler)
+ LOG.error("my hovercraft is full of eels")
+
+
+def test_sendevent():
+ # Send the kind of event the solr indexer does when it starts to index
+ se = SendEvent(profile='test', application='solr-indexer')
+ event = {
+ "logData": {
+ "dryRun": False,
+ "numberOfDocuments": 9999
+ },
+ "message": "Starting reindexing process",
+ "request": "some request"
+ }
+ se.send(event)
+
+
+if __name__ == "__main__":
+ test_sendlog()
+ # test_sendevent()
diff --git a/apps/cli/executables/pexable/ingest/ingest/remotecopy.py b/apps/cli/executables/pexable/ingest/ingest/remotecopy.py
new file mode 100644
index 000000000..06ab60b3c
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/remotecopy.py
@@ -0,0 +1,29 @@
+import logging
+from pathlib import Path
+import pysftp
+from pycapo import CapoConfig
+
+LOG = logging.getLogger('ingestion')
+config = CapoConfig()
+
+KEY_FILE=str(Path.home() / '.ssh' / 'scp_only')
+SSH_USER = 'vlapipe'
+SSH_HOST = 'mcilroy.aoc.nrao.edu'
+OUT_DIR = config.getstring('archive-ingestion.vlass.ThumbnailPath')
+
+# Disable the use of host keys.
+cnopts = pysftp.CnOpts()
+cnopts.hostkeys = None
+
+
+def copy_thumbnail(source, target):
+ """
+ Remote copy of thumbnail images
+ """
+ target_path = Path(target)
+
+ LOG.debug(f"Copying thumbnail from {source} to {target}")
+
+ with pysftp.Connection(SSH_HOST, username=SSH_USER, private_key=KEY_FILE, cnopts=cnopts) as sftp:
+ sftp.makedirs(str(target_path.parent))
+ sftp.put(source, remotepath=str(target_path))
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/__init__.py b/apps/cli/executables/pexable/ingest/ingest/schema/__init__.py
new file mode 100644
index 000000000..5b3ceb827
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/__init__.py
@@ -0,0 +1,85 @@
+# publish our behavior-enhanced table classes
+import sqlalchemy
+from .model import *
+from sqlalchemy.orm import sessionmaker
+from pycapo import CapoConfig
+
+INSTRUMENT_CAPO_PREFIXES = {
+ "SDM": "metadataDatabase.jdbc",
+ "VLBA": "metadataDatabase.jdbc",
+ "ALMA": "almaMetadataDatabase.jdbc",
+ "OPT": "evlaopt.nrao.jdbc.",
+ "PST": "my.nrao.jdbc.",
+ "NGAS": "ngasDatabase.jdbc",
+ "NGAS_NAASC": "almaMetadataDatabase.jdbc",
+ "VLASS": "edu.nrao.vlass.config.VlassMngrSettings.jdbc",
+ "LEGACY": "oracleMetadataDatabase.jdbc",
+}
+
+
+def create_engine(instrument, **kwargs):
+ config = CapoConfig(profile=kwargs["profile"]) if "profile" in kwargs else CapoConfig()
+
+ prefix = INSTRUMENT_CAPO_PREFIXES[instrument]
+
+ user = config[prefix + "Username"]
+ passwd = config[prefix + "Password"]
+ url = config[prefix + "Url"]
+
+ if "oracle" in url:
+ hostport, service_name = url.split("/")[-2:]
+ if instrument == "LEGACY":
+ archiveurl = "oracle://{}:{}@{}/{}".format(user, passwd, hostport, service_name)
+ else:
+ host, port = hostport.split(":")
+ import cx_Oracle
+
+ dsn = cx_Oracle.makedsn(host, port, service_name=service_name)
+ archiveurl = "oracle://{}:{}@{}".format(user, passwd, dsn)
+ else:
+ archiveurl = url.replace("jdbc:", "").replace("://", "://" + user + ":" + passwd + "@")
+
+ return sqlalchemy.create_engine(archiveurl)
+
+
+def create_session(instrument, **kwargs):
+ engine = create_engine(instrument, **kwargs)
+ session_mkr = sessionmaker(engine)
+ session = session_mkr()
+ return session
+
+
+class ArchiveDBSession:
+ """A class to create context manager around an archive connection."""
+
+ def __init__(self, instrument, **kwargs):
+ config = CapoConfig(profile=kwargs["profile"]) if "profile" in kwargs else CapoConfig()
+
+ prefix = INSTRUMENT_CAPO_PREFIXES[instrument]
+
+ user = config[prefix + "Username"]
+ passwd = config[prefix + "Password"]
+ url = config[prefix + "Url"]
+
+ if "oracle" in url:
+ hostport, service_name = url.split("/")[-2:]
+ if instrument == "LEGACY":
+ self.archiveurl = "oracle://{}:{}@{}/{}".format(user, passwd, hostport, service_name)
+ else:
+ host, port = hostport.split(":")
+ import cx_Oracle
+
+ dsn = cx_Oracle.makedsn(host, port, service_name=service_name)
+ self.archiveurl = "oracle://{}:{}@{}".format(user, passwd, dsn)
+ else:
+ self.archiveurl = url.replace("jdbc:", "").replace("://", "://" + user + ":" + passwd + "@")
+ self.session = None
+
+ def __enter__(self):
+ engine = sqlalchemy.create_engine(self.archiveurl)
+ session_mkr = sessionmaker(engine)
+ self.session = session_mkr(bind=engine)
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.session.close()
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/almamodel.py b/apps/cli/executables/pexable/ingest/ingest/schema/almamodel.py
new file mode 100644
index 000000000..62dfa2889
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/almamodel.py
@@ -0,0 +1,6686 @@
+# coding: utf-8
+from sqlalchemy import Column, DateTime, Float, ForeignKey, Index, LargeBinary, Numeric, String, Table, Text, text
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql.sqltypes import NullType
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+t_SLOG_ENTRY_ATTR_bak = Table(
+ 'SLOG_ENTRY_ATTR_bak', metadata,
+ Column('slog_attr_id', Numeric(19, 0, asdecimal=False), nullable=False),
+ Column('slog_attr_type', Numeric(10, 0, asdecimal=False), nullable=False),
+ Column('slog_attr_value', String(256)),
+ Column('slog_se_id', Numeric(19, 0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+class Aaa6(Base):
+ __tablename__ = 'aaa6'
+ __table_args__ = {'schema': 'ALMA'}
+
+ ccc = Column(String(100), primary_key=True)
+
+
+t_aaa7 = Table(
+ 'aaa7', metadata,
+ Column('ddd', String(100)),
+ schema='ALMA'
+)
+
+
+class Account(Base):
+ __tablename__ = 'account'
+ __table_args__ = {'schema': 'ALMA'}
+
+ account_id = Column(String(32), primary_key=True)
+ request_handler_id = Column(Numeric(22, 0, asdecimal=False), unique=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ password_digest = Column(String(256))
+ firstname = Column(String(256), nullable=False)
+ lastname = Column(String(256), nullable=False)
+ initials = Column(String(256))
+ creationtimestamp = Column(String(32))
+ modificationtimestamp = Column(String(32))
+ preferredarc = Column(String(32))
+ email = Column(String(256), unique=True)
+ executive = Column(String(5))
+ aliases = Column(String(256))
+ inst_no = Column(ForeignKey('ALMA.institution.inst_no'), nullable=False)
+ active = Column(String(1), nullable=False, server_default=text("'T' "))
+ account_id_merged_to = Column(ForeignKey('ALMA.account.account_id'))
+ merge_time = Column(DateTime)
+ receiveemails = Column(String(1), nullable=False)
+ passwordtimestamp = Column(DateTime)
+ provides_user_demographics = Column(String(1))
+
+ parent = relationship('Account', remote_side=[account_id])
+ institution = relationship('Institution')
+ nodes = relationship('DrmNode', secondary='ALMA.drm_data_reducer')
+ role = relationship('Role', secondary='ALMA.account_role')
+
+
+class AccountDelegation(Base):
+ __tablename__ = 'account_delegation'
+ __table_args__ = (
+ Index('account_del_unique', 'pi_rh_id', 'project_code', 'delegee_rh_id', 'delegation_type', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ pi_rh_id = Column(ForeignKey('ALMA.account.request_handler_id'), nullable=False)
+ project_code = Column(String(18), nullable=False)
+ delegee_rh_id = Column(ForeignKey('ALMA.account.request_handler_id'), nullable=False)
+ delegation_type = Column(String(10), nullable=False, index=True, server_default=text("'DATA' "))
+
+ delegee_rh = relationship('Account', primaryjoin='AccountDelegation.delegee_rh_id == Account.request_handler_id')
+ pi_rh = relationship('Account', primaryjoin='AccountDelegation.pi_rh_id == Account.request_handler_id')
+
+
+t_account_role = Table(
+ 'account_role', metadata,
+ Column('role_no', ForeignKey('ALMA.role.role_no'), primary_key=True, nullable=False),
+ Column('account_id', ForeignKey('ALMA.account.account_id'), primary_key=True, nullable=False),
+ schema='ALMA'
+)
+
+
+t_alma_alarms = Table(
+ 'alma_alarms', metadata,
+ Column('descriptor', Numeric(10, 0, asdecimal=False)),
+ Column('user_timestamp_secs', Numeric(20, 0, asdecimal=False)),
+ Column('user_timestamp_ms', Numeric(20, 0, asdecimal=False)),
+ Column('activated_by_backup', Numeric(1, 0, asdecimal=False)),
+ Column('terminated_by_backup', Numeric(1, 0, asdecimal=False)),
+ Column('family', String(100)),
+ Column('member', String(100)),
+ Column('code', Numeric(10, 0, asdecimal=False)),
+ Column('user_properties', String(1000)),
+ Column('source_name', String(100)),
+ Column('source_hostname', String(100)),
+ Column('source_timestamp_sec', Numeric(20, 0, asdecimal=False)),
+ Column('source_timestamp_ms', Numeric(20, 0, asdecimal=False)),
+ Column('backup', Numeric(1, 0, asdecimal=False)),
+ Column('version', String(100)),
+ schema='ALMA'
+)
+
+
+class AquaAttachment(Base):
+ __tablename__ = 'aqua_attachment'
+ __table_args__ = (
+ Index('aqua_attachment_uni', 'name', 'ttimestamp', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ attachmentid = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ ttimestamp = Column(DateTime)
+ name = Column(String(256), nullable=False)
+ contents = Column(LargeBinary, nullable=False)
+
+ aqua_session = relationship('AquaSession', secondary='ALMA.aqua_session_attachment')
+ aqua_execblock = relationship('AquaExecblock', secondary='ALMA.aqua_execblock_attachment')
+
+
+class AquaComment(Base):
+ __tablename__ = 'aqua_comment'
+ __table_args__ = (
+ Index('aqua_comment_uni', 'author', 'ttimestamp', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ commentid = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ author = Column(String(32), nullable=False)
+ ttimestamp = Column(DateTime, nullable=False)
+ ccomment = Column(Text)
+
+ aqua_session = relationship('AquaSession', secondary='ALMA.aqua_session_comment')
+ aqua_execblock = relationship('AquaExecblock', secondary='ALMA.aqua_execblock_comment')
+
+
+class AquaExecblock(Base):
+ __tablename__ = 'aqua_execblock'
+ __table_args__ = {'schema': 'ALMA'}
+
+ execblockid = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ execblockuid = Column(String(32), nullable=False, unique=True)
+ sessionid = Column(ForeignKey('ALMA.aqua_session.sessionid'), nullable=False)
+ qa0status = Column(String(32), nullable=False, index=True)
+ starttime = Column(DateTime)
+ endtime = Column(DateTime)
+ finalcommentid = Column(ForeignKey('ALMA.aqua_comment.commentid'), unique=True)
+ schedblockuid = Column(String(32))
+ obsprojectcode = Column(String(25))
+ obsprojectname = Column(String(256))
+ obsprojectuid = Column(String(32))
+ location = Column(String(32))
+ correlatortype = Column(String(4))
+ execfraction = Column(Numeric(5, 3), nullable=False, server_default=text("0 "))
+
+ aqua_comment = relationship('AquaComment')
+ aqua_session = relationship('AquaSession')
+
+
+t_aqua_execblock_attachment = Table(
+ 'aqua_execblock_attachment', metadata,
+ Column('attachmentid', ForeignKey('ALMA.aqua_attachment.attachmentid'), primary_key=True),
+ Column('execblockid', ForeignKey('ALMA.aqua_execblock.execblockid'), nullable=False),
+ schema='ALMA'
+)
+
+
+t_aqua_execblock_comment = Table(
+ 'aqua_execblock_comment', metadata,
+ Column('commentid', ForeignKey('ALMA.aqua_comment.commentid'), primary_key=True),
+ Column('execblockid', ForeignKey('ALMA.aqua_execblock.execblockid')),
+ schema='ALMA'
+)
+
+
+class AquaExecblockEtlDatum(Base):
+ __tablename__ = 'aqua_execblock_etl_data'
+ __table_args__ = (
+ Index('eb_type_unique', 'execblockid', 'etl_data_type', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ etl_data_id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ execblockid = Column(ForeignKey('ALMA.aqua_execblock.execblockid'))
+ etl_data_type = Column(String(16))
+ etl_schema_version = Column(Numeric(3, 0, asdecimal=False))
+ etl_data = Column(Text)
+
+ aqua_execblock = relationship('AquaExecblock')
+
+
+class AquaOu(Base):
+ __tablename__ = 'aqua_ous'
+ __table_args__ = (
+ Index('aqua_ous_uni', 'obsunitsetuid', 'obsunitsetpartid', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ obsunitsetid = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ obsunitsetuid = Column(String(32), nullable=False)
+ qa2status = Column(String(8), nullable=False)
+ finalcommentid = Column(ForeignKey('ALMA.aqua_comment.commentid'), unique=True)
+ obsunitsetpartid = Column(String(32), nullable=False)
+ qa2semipassreason = Column(String(11))
+ qa2statustime = Column(DateTime)
+ achievedsens = Column(Numeric(6, 2))
+ achievedresol = Column(Numeric(6, 2))
+ qa2plcalstatus = Column(String(8))
+ qa2mancalstatus = Column(String(8))
+ plcalfinalcommentid = Column(Numeric(10, 0, asdecimal=False))
+ mcalfinalcommentid = Column(Numeric(10, 0, asdecimal=False))
+ casatickets = Column(String(128))
+ analyst = Column(String(32))
+ execfraction = Column(Numeric(5, 3), nullable=False, server_default=text("0 "))
+ plimgfinalcommentid = Column(Numeric(10, 0, asdecimal=False))
+ mimgfinalcommentid = Column(Numeric(10, 0, asdecimal=False))
+ rmsatbandwidth = Column(NullType)
+ rmscontinuum = Column(NullType)
+ beamsemimajor = Column(NullType)
+ beamsemiminor = Column(NullType)
+ beamposangle = Column(NullType)
+ ous_status_entity_id = Column(String(64))
+
+ aqua_comment = relationship('AquaComment')
+
+
+class AquaOusAttachment(Base):
+ __tablename__ = 'aqua_ous_attachment'
+ __table_args__ = {'schema': 'ALMA'}
+
+ attachmentid = Column(ForeignKey('ALMA.aqua_attachment.attachmentid'), primary_key=True, nullable=False)
+ obsunitsetid = Column(ForeignKey('ALMA.aqua_ous.obsunitsetid'), primary_key=True, nullable=False)
+ attachment_section = Column(String(8))
+
+ aqua_attachment = relationship('AquaAttachment')
+ aqua_ou = relationship('AquaOu')
+
+
+class AquaOusFlag(Base):
+ __tablename__ = 'aqua_ous_flag'
+ __table_args__ = (
+ Index('unique_ous_flag', 'ous_status_entity_id', 'flag_name', unique=True),
+ Index('idx_aqua_ous_flag_val', 'flag_name', 'flag_value'),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(4, 0, asdecimal=False), nullable=False)
+ ous_status_entity_id = Column(ForeignKey('ALMA.obs_unit_set_status.status_entity_id'), nullable=False, index=True)
+ flag_name = Column(String(32), nullable=False)
+ flag_value = Column(String(2000))
+ update_account_id = Column(ForeignKey('ALMA.account.account_id'), nullable=False)
+ update_timestamp = Column(DateTime, nullable=False)
+
+ ous_status_entity = relationship('ObsUnitSetStatu')
+ update_account = relationship('Account')
+
+
+class AquaOusFlagHistory(Base):
+ __tablename__ = 'aqua_ous_flag_history'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ ous_status_entity_id = Column(ForeignKey('ALMA.obs_unit_set_status.status_entity_id'), nullable=False)
+ flag_name = Column(String(32), nullable=False)
+ flag_value = Column(String(2000))
+ update_account_id = Column(ForeignKey('ALMA.account.account_id'), nullable=False)
+ update_timestamp = Column(DateTime, nullable=False)
+
+ ous_status_entity = relationship('ObsUnitSetStatu')
+ update_account = relationship('Account')
+
+
+class AquaPipelineRun(Base):
+ __tablename__ = 'aqua_pipeline_run'
+ __table_args__ = (
+ Index('uk_mous_uid_timestamp', 'mous_uid', 'timestamp', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ gous_uid = Column(String(32))
+ mous_uid = Column(String(32))
+ report = Column(Text)
+ sous_uid = Column(String(32))
+ timestamp = Column(String(32))
+ version = Column(Numeric(10, 0, asdecimal=False))
+ pipeline_run_dir_id = Column(ForeignKey('ALMA.aqua_pipeline_run_directory.id'))
+ weblog_id = Column(ForeignKey('ALMA.aqua_weblog.id'))
+
+ pipeline_run_dir = relationship('AquaPipelineRunDirectory')
+ weblog = relationship('AquaWeblog')
+
+
+class AquaPipelineRunDirectory(Base):
+ __tablename__ = 'aqua_pipeline_run_directory'
+ __table_args__ = (
+ Index('uk_project_code_timestamp', 'project_code', 'timestamp', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ remark = Column(String(2000))
+ last_op_timestamp = Column(String(32))
+ pathname = Column(String(512))
+ project_code = Column(String(32))
+ status = Column(String(32))
+ timestamp = Column(String(32))
+ version = Column(Numeric(10, 0, asdecimal=False))
+ last_op_account = Column(String(32))
+ products_sub_dir = Column(String(512))
+ region = Column(String(32))
+ upload_timestamp = Column(String(32))
+ data_reduction_type = Column(String(32))
+
+
+class AquaQa2EcHistory(Base):
+ __tablename__ = 'aqua_qa2_ec_history'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ ttimestamp = Column(DateTime, nullable=False)
+ entityuid = Column(String(32), nullable=False)
+ author = Column(String(32), nullable=False)
+ ec = Column(NullType)
+
+
+class AquaQa2FailReason(Base):
+ __tablename__ = 'aqua_qa2_fail_reason'
+ __table_args__ = {'schema': 'ALMA'}
+
+ qa2reasonid = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ obsunitsetid = Column(ForeignKey('ALMA.aqua_ous.obsunitsetid'), nullable=False)
+ qa2failreason = Column(String(20), nullable=False)
+
+ aqua_ou = relationship('AquaOu')
+
+
+class AquaQa2ImagingHistory(Base):
+ __tablename__ = 'aqua_qa2_imaging_history'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ ttimestamp = Column(DateTime, nullable=False)
+ entityuid = Column(String(32), nullable=False)
+ author = Column(String(32), nullable=False)
+ rmsatbandwidth = Column(NullType)
+ rmscontinuum = Column(NullType)
+ beamsemimajor = Column(NullType)
+ beamsemiminor = Column(NullType)
+ beamposangle = Column(NullType)
+
+
+class AquaSensitivity(Base):
+ __tablename__ = 'aqua_sensitivity'
+ __table_args__ = {'schema': 'ALMA'}
+
+ asdmuid = Column(String(32), primary_key=True, nullable=False)
+ spwid = Column(Numeric(38, 0, asdecimal=False), primary_key=True, nullable=False)
+ mediantsys = Column(NullType)
+
+
+class AquaSession(Base):
+ __tablename__ = 'aqua_session'
+ __table_args__ = (
+ Index('aqua_session_uni', 'sessionuid', 'sessionpartid', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ sessionid = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ sessionuid = Column(String(32), nullable=False)
+ schedblockuid = Column(String(32), nullable=False)
+ schedblockname = Column(String(128))
+ projectcode = Column(String(64))
+ qlfocussummuid = Column(String(32))
+ qlpointsummuid = Column(String(32))
+ qlphasesummuid = Column(String(32))
+ qlatmsummuid = Column(String(32))
+ qa1status = Column(String(5), nullable=False)
+ starttime = Column(DateTime, nullable=False)
+ endtime = Column(DateTime)
+ finalcommentid = Column(ForeignKey('ALMA.aqua_comment.commentid'), unique=True)
+ sessionpartid = Column(String(32), nullable=False)
+ location = Column(String(32))
+ tmcdbconfigurationname = Column(String(128))
+ correlatortype = Column(String(4))
+
+ aqua_comment = relationship('AquaComment')
+
+
+t_aqua_session_attachment = Table(
+ 'aqua_session_attachment', metadata,
+ Column('attachmentid', ForeignKey('ALMA.aqua_attachment.attachmentid'), primary_key=True),
+ Column('sessionid', ForeignKey('ALMA.aqua_session.sessionid'), nullable=False),
+ schema='ALMA'
+)
+
+
+t_aqua_session_comment = Table(
+ 'aqua_session_comment', metadata,
+ Column('commentid', ForeignKey('ALMA.aqua_comment.commentid'), primary_key=True),
+ Column('sessionid', ForeignKey('ALMA.aqua_session.sessionid')),
+ schema='ALMA'
+)
+
+
+class AquaStatusHistory(Base):
+ __tablename__ = 'aqua_status_history'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ entityuid = Column(String(32), nullable=False)
+ author = Column(String(32), nullable=False)
+ ttimestamp = Column(DateTime, nullable=False)
+ qastatus = Column(String(32), nullable=False)
+ qastatustype = Column(String(32), nullable=False)
+ ccomment = Column(Text)
+
+
+t_aqua_v_execblock = Table(
+ 'aqua_v_execblock', metadata,
+ Column('execblockid', Numeric(10, 0, asdecimal=False), nullable=False),
+ Column('execblockuid', String(32), nullable=False),
+ Column('execfraction', Numeric(5, 3), nullable=False),
+ Column('qa0status', String(32), nullable=False),
+ Column('finalcommentid', Numeric(10, 0, asdecimal=False)),
+ Column('sessionid', Numeric(10, 0, asdecimal=False), nullable=False),
+ Column('sessionuid', String(32)),
+ Column('sessionpartid', String(32)),
+ Column('starttime', DateTime),
+ Column('endtime', DateTime),
+ Column('schedblockuid', String(32)),
+ Column('schedblockname', String(128)),
+ Column('obsprojectcode', String(128)),
+ Column('obsprojectname', String(256)),
+ Column('obsprojectpi', String(20)),
+ Column('obsprojectversion', String(20)),
+ Column('location', String(32)),
+ Column('status', String(32)),
+ Column('archivingstatus', String(32)),
+ Column('bandname', String(10)),
+ Column('representativefrequency', Numeric(15, 10)),
+ Column('almabuild', String(70)),
+ Column('arrayname', String(10)),
+ Column('arraystart', DateTime),
+ Column('arrayend', DateTime),
+ Column('correlatortype', String(4)),
+ Column('arrayfamily', String(11)),
+ Column('arraytype', String(9)),
+ Column('photonicreferencename', String(18)),
+ Column('obsprojectuid', String(32)),
+ schema='ALMA'
+)
+
+
+class AquaWeblog(Base):
+ __tablename__ = 'aqua_weblog'
+ __table_args__ = (
+ Index('uk_ous_uid_timestamp', 'ous_uid', 'timestamp', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ cached = Column(String(1))
+ last_access = Column(String(32))
+ ous_uid = Column(String(32))
+ timestamp = Column(String(32))
+ version = Column(Numeric(10, 0, asdecimal=False))
+ last_op_account = Column(String(32))
+ remark = Column(String(2000))
+ last_op_timestamp = Column(String(32))
+ ngas_file_id = Column(String(128))
+ status = Column(String(32))
+
+
+class AsaBibliography(Base):
+ __tablename__ = 'asa_bibliography'
+ __table_args__ = {'schema': 'ALMA'}
+
+ bibcode = Column(String(30), nullable=False, unique=True)
+ publication_year = Column(Numeric(4, 0, asdecimal=False))
+ journal = Column(String(100), nullable=False)
+ title = Column(String(255), nullable=False, index=True)
+ first_author = Column(String(255), nullable=False)
+ volume = Column(Numeric(8, 0, asdecimal=False))
+ pages = Column(String(100))
+ abstract = Column(String(4000))
+ authors = Column(String(4000))
+ keywords = Column(String(4000))
+ bib_id = Column(Numeric(30, 0, asdecimal=False), primary_key=True)
+
+
+t_asa_bibliography_temp = Table(
+ 'asa_bibliography_temp', metadata,
+ Column('bibcode', String(30), nullable=False),
+ Column('publication_year', Numeric(4, 0, asdecimal=False)),
+ Column('journal', String(100), nullable=False),
+ Column('title', String(255), nullable=False),
+ Column('first_author', String(255), nullable=False),
+ Column('volume', Numeric(8, 0, asdecimal=False)),
+ Column('pages', String(100)),
+ Column('abstract', String(4000)),
+ Column('authors', String(4000)),
+ Column('keywords', String(4000)),
+ Column('bib_id', Numeric(30, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+class AsaColumn(Base):
+ __tablename__ = 'asa_columns'
+ __table_args__ = (
+ Index('asa_columns_unique', 'table_name', 'column_name', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ column_name = Column(String(128), nullable=False)
+ table_name = Column(String(128), nullable=False)
+ description = Column(String(1024))
+ tooltip = Column(String(256))
+ unit = Column(String(32))
+ dimensions = Column(String(32))
+ scale = Column(Float)
+ ucd = Column(String(128))
+ utype = Column(String(512))
+ datatype = Column(String(32))
+ type_size = Column(Numeric(11, 0, asdecimal=False))
+ principal = Column(Numeric(1, 0, asdecimal=False), nullable=False)
+ indexed = Column(Numeric(1, 0, asdecimal=False), nullable=False)
+ std = Column(Numeric(1, 0, asdecimal=False), nullable=False)
+ is_enum = Column(Numeric(1, 0, asdecimal=False), nullable=False)
+ enum_table = Column(String(128))
+ examples = Column(String(1024))
+ is_queriable = Column(String(1), nullable=False)
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+
+
+class AsaDeliveryAsdmOu(Base):
+ __tablename__ = 'asa_delivery_asdm_ous'
+ __table_args__ = (
+ Index('dp_delao_ad_unq', 'asdm_uid', 'deliverable_name', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ asdm_uid = Column(ForeignKey('ALMA.xml_asdm_entities.archive_uid'), index=True)
+ ous_status_id = Column(String(33), index=True)
+ deliverable_name = Column(String(64), nullable=False, index=True)
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ delivery_id = Column(ForeignKey('ALMA.asa_delivery_status.id'), nullable=False)
+
+ xml_asdm_entity = relationship('XmlAsdmEntity')
+ delivery = relationship('AsaDeliveryStatu')
+
+
+class AsaDeliveryStatu(Base):
+ __tablename__ = 'asa_delivery_status'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ delivery_id = Column(String(64), nullable=False, unique=True)
+ pi_account_id = Column(Numeric(22, 0, asdecimal=False), nullable=False)
+ qa2_passed = Column(String(1), nullable=False)
+ notification_date = Column(DateTime)
+ original_release_date = Column(DateTime)
+ release_date = Column(DateTime, server_default=text("to_date('3000-01-01', 'YYYY-MM-DD')"))
+ release_date_comment = Column(String(4000))
+ delivery_name = Column(String(64), nullable=False)
+
+
+class AsaEnergy(Base):
+ __tablename__ = 'asa_energy'
+ __table_args__ = {'schema': 'ALMA'}
+
+ asa_energy_id = Column(String(128), primary_key=True)
+ asa_dataset_id = Column(ForeignKey('ALMA.asa_science.dataset_id'), index=True)
+ frequency_min = Column(NullType, nullable=False)
+ frequency_max = Column(NullType, nullable=False)
+ resolution_min = Column(NullType, nullable=False)
+ resolution_max = Column(NullType, nullable=False)
+ channel_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ channel_width_min = Column(NullType, nullable=False)
+ channel_width_max = Column(NullType, nullable=False)
+ resolving_power_min = Column(NullType, nullable=False)
+ resolving_power_max = Column(NullType, nullable=False)
+ spectral_window_uid = Column(String(64))
+ bandwidth = Column(NullType)
+ pol_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ pol_product = Column(String(64), nullable=False)
+ exptimtp = Column(NullType)
+ exptim7 = Column(NullType)
+ exptim12 = Column(NullType)
+ restfreq = Column(NullType)
+ chanrms = Column(NullType)
+ crval3 = Column(NullType)
+ crpix3 = Column(NullType)
+ crval4 = Column(NullType)
+ crpix4 = Column(NullType)
+ cdelt4 = Column(NullType)
+ parent_asa_energy_ids = Column(String(256))
+ spw_num = Column(Numeric(38, 0, asdecimal=False))
+ spw_id_original = Column(Numeric(38, 0, asdecimal=False))
+ spw_id_orig_list = Column(String(64))
+ spectral_window_name = Column(String(64))
+ asa_energy_product_id = Column(String(128))
+ asdm_spw_id = Column(Numeric(38, 0, asdecimal=False))
+ ms_spw_id = Column(Numeric(38, 0, asdecimal=False))
+ sensitivity_channel = Column(NullType)
+ sensitivity_10kms = Column(NullType)
+ sensitivity_bandwidth = Column(NullType)
+ line_id = Column(String(256))
+ rest_frequency_max = Column(NullType)
+ rest_frequency_min = Column(NullType)
+ rest_frequency_resolution_min = Column(NullType)
+ rest_frequency_resolution_max = Column(NullType)
+ rest_resolving_power_min = Column(NullType)
+ rest_resolving_power_max = Column(NullType)
+ rest_channel_width_min = Column(NullType)
+ rest_channel_width_max = Column(NullType)
+ rest_bandwidth = Column(NullType)
+ parent_energy_id = Column(String(128))
+
+ asa_dataset = relationship('AsaScience')
+
+
+class AsaEnergyTemp(Base):
+ __tablename__ = 'asa_energy_temp'
+ __table_args__ = {'schema': 'ALMA'}
+
+ asa_energy_id = Column(String(128), primary_key=True)
+ asa_dataset_id = Column(String(128), index=True)
+ frequency_min = Column(NullType, nullable=False)
+ frequency_max = Column(NullType, nullable=False)
+ resolution_min = Column(NullType, nullable=False)
+ resolution_max = Column(NullType, nullable=False)
+ channel_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ channel_width_min = Column(NullType, nullable=False)
+ channel_width_max = Column(NullType, nullable=False)
+ resolving_power_min = Column(NullType, nullable=False)
+ resolving_power_max = Column(NullType, nullable=False)
+ spectral_window_uid = Column(String(64))
+ bandwidth = Column(NullType)
+ pol_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ pol_product = Column(String(64), nullable=False)
+ exptimtp = Column(NullType)
+ exptim7 = Column(NullType)
+ exptim12 = Column(NullType)
+ restfreq = Column(NullType)
+ chanrms = Column(NullType)
+ crval3 = Column(NullType)
+ crpix3 = Column(NullType)
+ crval4 = Column(NullType)
+ crpix4 = Column(NullType)
+ cdelt4 = Column(NullType)
+ parent_asa_energy_ids = Column(String(256))
+ spw_num = Column(Numeric(38, 0, asdecimal=False))
+ spw_id_original = Column(Numeric(38, 0, asdecimal=False))
+ spw_id_orig_list = Column(String(64))
+ spectral_window_name = Column(String(64))
+ asa_energy_product_id = Column(String(128), index=True)
+ asdm_spw_id = Column(Numeric(38, 0, asdecimal=False), index=True)
+ ms_spw_id = Column(Numeric(38, 0, asdecimal=False))
+ sensitivity_channel = Column(NullType)
+ sensitivity_10kms = Column(NullType)
+ sensitivity_bandwidth = Column(NullType)
+ line_id = Column(String(256))
+ rest_frequency_max = Column(NullType)
+ rest_frequency_min = Column(NullType)
+ rest_frequency_resolution_min = Column(NullType)
+ rest_frequency_resolution_max = Column(NullType)
+ rest_resolving_power_min = Column(NullType)
+ rest_resolving_power_max = Column(NullType)
+ rest_channel_width_min = Column(NullType)
+ rest_channel_width_max = Column(NullType)
+ rest_bandwidth = Column(NullType)
+ parent_energy_id = Column(String(128))
+
+
+class AsaMailTemplate(Base):
+ __tablename__ = 'asa_mail_templates'
+ __table_args__ = {'schema': 'ALMA'}
+
+ templatename = Column(String(100), primary_key=True, nullable=False)
+ application = Column(String(100), primary_key=True, nullable=False)
+ templatecontent = Column(Text, nullable=False)
+
+
+class AsaOu(Base):
+ __tablename__ = 'asa_ous'
+ __table_args__ = {'schema': 'ALMA'}
+
+ asa_ous_id = Column(String(64), primary_key=True)
+ asa_project_code = Column(String(64))
+ sg_ous_uid = Column(String(64))
+ group_ous_uid = Column(String(64))
+ member_ous_uid = Column(String(64))
+ member_num = Column(Numeric(10, 0, asdecimal=False))
+ completed = Column(Numeric(1, 0, asdecimal=False))
+ xml_scipiperesults_archive_uid = Column(ForeignKey('ALMA.xml_scipiperesults_entities.archive_uid'))
+ individual_files = Column(Numeric(1, 0, asdecimal=False), nullable=False, server_default=text("0 "))
+
+ xml_scipiperesults_entity = relationship('XmlScipiperesultsEntity')
+
+
+class AsaOusRedshift(Base):
+ __tablename__ = 'asa_ous_redshift'
+ __table_args__ = {'schema': 'ALMA'}
+
+ group_ous_uid = Column(String(64), primary_key=True, nullable=False)
+ member_ous_uid = Column(String(64), primary_key=True, nullable=False)
+ source_name = Column(String(256), primary_key=True, nullable=False)
+ best_source_redshift = Column(NullType)
+
+
+class AsaOusTemp(Base):
+ __tablename__ = 'asa_ous_temp'
+ __table_args__ = {'schema': 'ALMA'}
+
+ asa_ous_id = Column(String(64), primary_key=True)
+ asa_project_code = Column(String(64))
+ sg_ous_uid = Column(String(64))
+ group_ous_uid = Column(String(64))
+ member_ous_uid = Column(String(64))
+ member_num = Column(Numeric(10, 0, asdecimal=False))
+ completed = Column(Numeric(1, 0, asdecimal=False))
+ xml_scipiperesults_archive_uid = Column(String(33))
+ individual_files = Column(Numeric(1, 0, asdecimal=False), nullable=False)
+
+
+class AsaProductFile(Base):
+ __tablename__ = 'asa_product_files'
+ __table_args__ = {'schema': 'ALMA'}
+
+ ngas_file_id = Column(String(220), primary_key=True, nullable=False)
+ asa_ous_id = Column(ForeignKey('ALMA.asa_ous.asa_ous_id'), primary_key=True, nullable=False)
+ asa_science_id = Column(ForeignKey('ALMA.asa_science.dataset_id'))
+ asa_energy_id = Column(ForeignKey('ALMA.asa_energy.asa_energy_id'))
+ stored_size = Column(Numeric(20, 0, asdecimal=False), nullable=False)
+ subdirectory = Column(String(128), nullable=False)
+ file_class = Column(String(64))
+ file_type = Column(String(64))
+ pack = Column(Numeric(1, 0, asdecimal=False), nullable=False)
+ vo = Column(Numeric(1, 0, asdecimal=False))
+ creation_date = Column(DateTime, nullable=False)
+ naxis = Column(Numeric(38, 0, asdecimal=False))
+ naxis1 = Column(Numeric(38, 0, asdecimal=False))
+ naxis2 = Column(Numeric(38, 0, asdecimal=False))
+ naxis3 = Column(Numeric(38, 0, asdecimal=False))
+ naxis4 = Column(Numeric(38, 0, asdecimal=False))
+ parent_product_files_id = Column(String(64))
+ main_product_source = Column(Numeric(1, 0, asdecimal=False))
+ main_product_spw = Column(Numeric(1, 0, asdecimal=False))
+ origin = Column(String(32))
+ user_id = Column(String(32))
+ processing_recipe = Column(String(7), nullable=False, server_default=text("'CAL+IMG' "))
+
+ asa_energy = relationship('AsaEnergy')
+ asa_ous = relationship('AsaOu')
+ asa_science = relationship('AsaScience')
+
+
+class AsaProject(Base):
+ __tablename__ = 'asa_project'
+ __table_args__ = {'schema': 'ALMA'}
+
+ code = Column(String(20), primary_key=True)
+ project_uid = Column(String(32), nullable=False, unique=True)
+ proposal_uid = Column(ForeignKey('ALMA.xml_obsproposal_entities.archive_uid'), nullable=False)
+ pi_name = Column(String(256), nullable=False)
+ pi_userid = Column(ForeignKey('ALMA.account.account_id'), nullable=False)
+ coi_name = Column(String(2000))
+ title = Column(String(256))
+ type = Column(String(16), nullable=False)
+ associated_arc = Column(String(8), nullable=False)
+ proposal_cycle = Column(String(6), nullable=False)
+ proposed_start_date = Column(String(128))
+ proposed_end_date = Column(String(128))
+ first_obs_start = Column(String(128))
+ first_obs_end = Column(String(128))
+ last_obs_start = Column(String(128))
+ last_obs_end = Column(String(128))
+ arc_release = Column(String(128))
+ scientific_category = Column(String(200))
+ science_keyword = Column(String(200))
+ proposal_abstract = Column(String(4000))
+
+ account = relationship('Account')
+ xml_obsproposal_entity = relationship('XmlObsproposalEntity')
+
+ def __repr__(self):
+ return '<AsaProject#{code} "{title}" start={first_obs_start} end={last_obs_end}>'.format(**self.__dict__)
+
+
+
+class AsaProjectBibliography(Base):
+ __tablename__ = 'asa_project_bibliography'
+ __table_args__ = (
+ Index('bibprojectcode_pk', 'bibcode', 'project_code', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ bibcode = Column(String(30), nullable=False)
+ project_code = Column(ForeignKey('ALMA.asa_project.code'), primary_key=True, nullable=False)
+ bib_id = Column(ForeignKey('ALMA.asa_bibliography.bib_id'), primary_key=True, nullable=False)
+
+ bib = relationship('AsaBibliography')
+ asa_project = relationship('AsaProject')
+
+
+t_asa_project_bibliography_temp = Table(
+ 'asa_project_bibliography_temp', metadata,
+ Column('bibcode', String(30), nullable=False),
+ Column('project_code', String(20), nullable=False),
+ Column('bib_id', Numeric(30, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+class AsaProjectTemp(Base):
+ __tablename__ = 'asa_project_temp'
+ __table_args__ = {'schema': 'ALMA'}
+
+ code = Column(String(20), primary_key=True)
+ project_uid = Column(String(32), nullable=False)
+ proposal_uid = Column(String(32), nullable=False)
+ pi_name = Column(String(256), nullable=False)
+ pi_userid = Column(String(256), nullable=False)
+ coi_name = Column(String(2000))
+ title = Column(String(256))
+ type = Column(String(16), nullable=False)
+ associated_arc = Column(String(8), nullable=False)
+ proposal_cycle = Column(String(6), nullable=False)
+ proposed_start_date = Column(String(128))
+ proposed_end_date = Column(String(128))
+ first_obs_start = Column(String(128))
+ first_obs_end = Column(String(128))
+ last_obs_start = Column(String(128))
+ last_obs_end = Column(String(128))
+ arc_release = Column(String(128))
+ scientific_category = Column(String(200))
+ science_keyword = Column(String(200))
+ proposal_abstract = Column(String(4000))
+
+
+class AsaScience(Base):
+ __tablename__ = 'asa_science'
+ __table_args__ = {'schema': 'ALMA'}
+
+ dataset_id = Column(String(128), primary_key=True)
+ asdm_uid = Column(String(32), index=True)
+ field_uid = Column(String(32))
+ spectral_window_uid = Column(String(32))
+ project_uid = Column(ForeignKey('ALMA.asa_project.project_uid'))
+ schedblock_uid = Column(String(32))
+ execblock_uid = Column(String(32))
+ ra = Column(NullType, nullable=False)
+ dec = Column(NullType, nullable=False)
+ ra_source = Column(NullType, nullable=False)
+ dec_source = Column(NullType, nullable=False)
+ cx = Column(NullType)
+ cy = Column(NullType)
+ cz = Column(NullType)
+ az_min = Column(NullType, nullable=False)
+ az_max = Column(NullType, nullable=False)
+ elevation_min = Column(NullType, nullable=False)
+ elevation_max = Column(NullType, nullable=False)
+ wcs_cd1_1 = Column(NullType)
+ wcs_cd1_2 = Column(NullType)
+ wcs_cd2_1 = Column(NullType)
+ wcs_cd2_2 = Column(NullType)
+ wcs_crpix1 = Column(NullType)
+ wcs_crpix2 = Column(NullType)
+ wcs_crval1 = Column(NullType)
+ wcs_crval2 = Column(NullType)
+ spatial_scale_min = Column(NullType, nullable=False)
+ spatial_scale_max = Column(NullType, nullable=False)
+ gal_longitude = Column(NullType, nullable=False)
+ gal_latitude = Column(NullType, nullable=False)
+ ecliptic_longitude = Column(NullType, nullable=False)
+ ecliptic_latitude = Column(NullType, nullable=False)
+ footprint = Column(String(2048))
+ fov = Column(NullType, nullable=False)
+ bounding_box = Column(String(2048))
+ source_name = Column(String(256))
+ source_is_in_simbad = Column(String(1), nullable=False, server_default=text("'N' "))
+ source_class = Column(String(128))
+ source_redshift = Column(NullType)
+ source_alma_catalog_id = Column(Numeric(22, 0, asdecimal=False))
+ start_date = Column(DateTime, nullable=False)
+ end_date = Column(DateTime, nullable=False)
+ int_time = Column(Numeric(10, 3), nullable=False)
+ time_resolution = Column(Numeric(8, 4), nullable=False)
+ frequency = Column(NullType)
+ frequency_resolution = Column(NullType)
+ velocity_resolution = Column(NullType)
+ resolving_power = Column(NullType)
+ bandwidth = Column(NullType)
+ rest_frequency = Column(NullType)
+ rest_frequency_min = Column(NullType)
+ rest_frequency_max = Column(NullType)
+ rest_frequency_resolution = Column(NullType)
+ rest_velocity_resolution = Column(NullType, nullable=False)
+ rest_resolving_power = Column(NullType, nullable=False)
+ rest_frequency_support = Column(String(512), nullable=False)
+ channel_num = Column(Numeric(scale=0, asdecimal=False))
+ spectral_window_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ continuum_window_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ obs_unitset_id = Column(String(32), nullable=False)
+ science_goal_ouss_id = Column(ForeignKey('ALMA.obs_unit_set_status.status_entity_id'))
+ group_ouss_id = Column(ForeignKey('ALMA.obs_unit_set_status.status_entity_id'))
+ member_ouss_id = Column(ForeignKey('ALMA.obs_unit_set_status.status_entity_id'))
+ sensitivity = Column(Numeric(22, 0, asdecimal=False))
+ flux_min = Column(NullType)
+ flux_max = Column(NullType)
+ uv_coverage_param1 = Column(NullType)
+ uv_coverage_param2 = Column(NullType)
+ pol_num = Column(Numeric(scale=0, asdecimal=False))
+ pol_products = Column(String(64))
+ airmass = Column(NullType)
+ pwv = Column(NullType)
+ temperature = Column(NullType)
+ windspeed = Column(NullType)
+ winddirection = Column(NullType)
+ antennas = Column(String(660))
+ baseline_max = Column(NullType, nullable=False)
+ baseline_min = Column(NullType, nullable=False)
+ scan_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ project_code = Column(ForeignKey('ALMA.asa_project.code'), nullable=False, index=True)
+ schedblock_name = Column(String(128))
+ observation_category = Column(String(128), nullable=False)
+ scan_intent = Column(String(256))
+ access_format = Column(String(10), nullable=False)
+ processing_level = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ frequency_support = Column(String(4000))
+ spatial_resolution = Column(NullType, nullable=False, server_default=text("-1.0 "))
+ spatres = Column(NullType)
+ mineltp = Column(NullType)
+ minel12 = Column(NullType)
+ minel7 = Column(NullType)
+ max_angscale = Column(NullType)
+ obs_mode = Column(String(80))
+ pipver = Column(String(80))
+ asdm_list = Column(String(200))
+ ant_tp_num = Column(Numeric(3, 0, asdecimal=False), nullable=False, server_default=text("0 "))
+ ant_num = Column(Numeric(3, 0, asdecimal=False), nullable=False, server_default=text("0 "))
+ ant_main_num = Column(Numeric(3, 0, asdecimal=False), nullable=False, server_default=text("0 "))
+ ant_aca_num = Column(Numeric(3, 0, asdecimal=False), nullable=False, server_default=text("0 "))
+ frequency_max = Column(NullType, nullable=False, server_default=text("-1. "))
+ frequency_min = Column(NullType, nullable=False, server_default=text("-1. "))
+ requestedarray = Column(String(8))
+ asa_ous_id = Column(ForeignKey('ALMA.asa_ous.asa_ous_id'))
+ band_list = Column(String(30))
+ antenna_string = Column(String(30))
+ product_type = Column(String(10))
+ dataset_product_id = Column(String(128))
+ rms = Column(NullType)
+ is_mosaic = Column(String(1), nullable=False, server_default=text("'N' "))
+ best_source_redshift = Column(NullType)
+ redshift_provenance = Column(String(1))
+ cont_sensitivity_channel = Column(NullType)
+ cont_sensitivity_10kms = Column(NullType)
+ cont_sensitivity_bandwidth = Column(NullType)
+ parent_dataset_id = Column(String(128))
+
+ asa_ous = relationship('AsaOu')
+ group_ouss = relationship('ObsUnitSetStatu', primaryjoin='AsaScience.group_ouss_id == ObsUnitSetStatu.status_entity_id')
+ member_ouss = relationship('ObsUnitSetStatu', primaryjoin='AsaScience.member_ouss_id == ObsUnitSetStatu.status_entity_id')
+ asa_project = relationship('AsaProject', primaryjoin='AsaScience.project_code == AsaProject.code')
+ asa_project1 = relationship('AsaProject', primaryjoin='AsaScience.project_uid == AsaProject.project_uid')
+ science_goal_ouss = relationship('ObsUnitSetStatu', primaryjoin='AsaScience.science_goal_ouss_id == ObsUnitSetStatu.status_entity_id')
+
+
+class AsaScienceKeyword(Base):
+ __tablename__ = 'asa_science_keyword'
+ __table_args__ = {'schema': 'ALMA'}
+
+ keyword = Column(String(100), nullable=False)
+ category_id = Column(ForeignKey('ALMA.asa_science_keyword_category.id'), nullable=False, index=True)
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+
+ category = relationship('AsaScienceKeywordCategory')
+
+
+class AsaScienceKeywordCategory(Base):
+ __tablename__ = 'asa_science_keyword_category'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(4, 0, asdecimal=False), primary_key=True)
+ catg_name = Column(String(100), nullable=False)
+
+
+class AsaScienceTemp(Base):
+ __tablename__ = 'asa_science_temp'
+ __table_args__ = (
+ Index('asa_tmp_idx3', 'source_name', 'member_ouss_id'),
+ Index('asa_tmp_idx2', 'source_name', 'group_ouss_id'),
+ {'schema': 'ALMA'}
+ )
+
+ dataset_id = Column(String(128), primary_key=True)
+ asdm_uid = Column(String(32))
+ field_uid = Column(String(32))
+ spectral_window_uid = Column(String(32))
+ project_uid = Column(String(32))
+ schedblock_uid = Column(String(32))
+ execblock_uid = Column(String(32))
+ ra = Column(NullType, nullable=False)
+ dec = Column(NullType, nullable=False)
+ ra_source = Column(NullType, nullable=False)
+ dec_source = Column(NullType, nullable=False)
+ cx = Column(NullType)
+ cy = Column(NullType)
+ cz = Column(NullType)
+ az_min = Column(NullType, nullable=False)
+ az_max = Column(NullType, nullable=False)
+ elevation_min = Column(NullType, nullable=False)
+ elevation_max = Column(NullType, nullable=False)
+ wcs_cd1_1 = Column(NullType)
+ wcs_cd1_2 = Column(NullType)
+ wcs_cd2_1 = Column(NullType)
+ wcs_cd2_2 = Column(NullType)
+ wcs_crpix1 = Column(NullType)
+ wcs_crpix2 = Column(NullType)
+ wcs_crval1 = Column(NullType)
+ wcs_crval2 = Column(NullType)
+ spatial_scale_min = Column(NullType, nullable=False)
+ spatial_scale_max = Column(NullType, nullable=False)
+ gal_longitude = Column(NullType, nullable=False)
+ gal_latitude = Column(NullType, nullable=False)
+ ecliptic_longitude = Column(NullType, nullable=False)
+ ecliptic_latitude = Column(NullType, nullable=False)
+ footprint = Column(String(2048))
+ fov = Column(NullType, nullable=False)
+ bounding_box = Column(String(2048))
+ source_name = Column(String(256))
+ source_is_in_simbad = Column(String(1), nullable=False)
+ source_class = Column(String(128))
+ source_redshift = Column(NullType)
+ source_alma_catalog_id = Column(Numeric(22, 0, asdecimal=False))
+ start_date = Column(DateTime, nullable=False)
+ end_date = Column(DateTime, nullable=False)
+ int_time = Column(Numeric(10, 3), nullable=False)
+ time_resolution = Column(Numeric(8, 4), nullable=False)
+ frequency = Column(NullType)
+ band = Column(Numeric(scale=0, asdecimal=False))
+ frequency_resolution = Column(NullType)
+ velocity_resolution = Column(NullType)
+ resolving_power = Column(NullType)
+ bandwidth = Column(NullType)
+ rest_frequency = Column(NullType)
+ rest_frequency_min = Column(NullType)
+ rest_frequency_max = Column(NullType)
+ rest_frequency_resolution = Column(NullType)
+ rest_velocity_resolution = Column(NullType, nullable=False)
+ rest_resolving_power = Column(NullType, nullable=False)
+ rest_frequency_support = Column(String(512), nullable=False)
+ channel_num = Column(Numeric(scale=0, asdecimal=False))
+ spectral_window_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ continuum_window_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ obs_unitset_id = Column(String(32), nullable=False)
+ science_goal_ouss_id = Column(String(32))
+ group_ouss_id = Column(String(32))
+ member_ouss_id = Column(String(32))
+ sensitivity = Column(Numeric(22, 0, asdecimal=False))
+ flux_min = Column(NullType)
+ flux_max = Column(NullType)
+ uv_coverage_param1 = Column(NullType)
+ uv_coverage_param2 = Column(NullType)
+ pol_num = Column(Numeric(scale=0, asdecimal=False))
+ pol_products = Column(String(64))
+ airmass = Column(NullType)
+ pwv = Column(NullType)
+ temperature = Column(NullType)
+ windspeed = Column(NullType)
+ winddirection = Column(NullType)
+ antennas = Column(String(660))
+ baseline_max = Column(NullType, nullable=False)
+ baseline_min = Column(NullType, nullable=False)
+ scan_num = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ project_code = Column(String(20), nullable=False)
+ schedblock_name = Column(String(128))
+ observation_category = Column(String(128), nullable=False)
+ scan_intent = Column(String(256))
+ access_format = Column(String(10), nullable=False)
+ processing_level = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ frequency_support = Column(String(4000))
+ spatial_resolution = Column(NullType, nullable=False)
+ spatres = Column(NullType)
+ mineltp = Column(NullType)
+ minel12 = Column(NullType)
+ minel7 = Column(NullType)
+ max_angscale = Column(NullType)
+ obs_mode = Column(String(80))
+ pipver = Column(String(80))
+ asdm_list = Column(String(200))
+ ant_tp_num = Column(Numeric(3, 0, asdecimal=False), nullable=False)
+ ant_num = Column(Numeric(3, 0, asdecimal=False), nullable=False)
+ ant_main_num = Column(Numeric(3, 0, asdecimal=False), nullable=False)
+ ant_aca_num = Column(Numeric(3, 0, asdecimal=False), nullable=False)
+ frequency_max = Column(NullType, nullable=False)
+ frequency_min = Column(NullType, nullable=False)
+ requestedarray = Column(String(8))
+ asa_ous_id = Column(String(64))
+ band_list = Column(String(30))
+ antenna_string = Column(String(30))
+ product_type = Column(String(10))
+ dataset_product_id = Column(String(128))
+ rms = Column(NullType)
+ is_mosaic = Column(String(1), nullable=False)
+ best_source_redshift = Column(NullType)
+ redshift_provenance = Column(String(1))
+ cont_sensitivity_channel = Column(NullType)
+ cont_sensitivity_10kms = Column(NullType)
+ cont_sensitivity_bandwidth = Column(NullType)
+ parent_dataset_id = Column(String(128))
+
+
+class AsaTable(Base):
+ __tablename__ = 'asa_tables'
+ __table_args__ = (
+ Index('asa_tables_unique', 'schema_name', 'table_name', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ schema_name = Column(String(128), nullable=False)
+ table_name = Column(String(128), nullable=False)
+ table_type = Column(String(5), nullable=False)
+ description = Column(String(1024))
+ utype = Column(String(512))
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+
+
+class AsaTablesKey(Base):
+ __tablename__ = 'asa_tables_keys'
+ __table_args__ = {'schema': 'ALMA'}
+
+ key_id = Column(String(128), primary_key=True)
+ from_table = Column(String(128), nullable=False)
+ target_table = Column(String(128), nullable=False)
+ description = Column(String(1024), nullable=False)
+ utype = Column(String(512))
+
+
+class AsaTablesKeysColumn(Base):
+ __tablename__ = 'asa_tables_keys_columns'
+ __table_args__ = {'schema': 'ALMA'}
+
+ key_id = Column(String(128), primary_key=True)
+ from_column = Column(String(128), nullable=False)
+ target_column = Column(String(128), nullable=False)
+
+
+class AsaTapSchema(Base):
+ __tablename__ = 'asa_tap_schemas'
+ __table_args__ = {'schema': 'ALMA'}
+
+ schema_name = Column(String(128), nullable=False, unique=True)
+ description = Column(String(1024))
+ utype = Column(String(512))
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+
+
+t_asp_accounts = Table(
+ 'asp_accounts', metadata,
+ Column('account_id', String(32), nullable=False),
+ Column('provides_user_demographics', String(1)),
+ Column('last_demographics_page_visit', DateTime),
+ schema='ALMA'
+)
+
+
+t_asp_obs_proposal = Table(
+ 'asp_obs_proposal', metadata,
+ Column('archive_uid', String(33), nullable=False),
+ Column('cycle', String(32)),
+ Column('code', String(64), nullable=False),
+ Column('priority', String(2)),
+ Column('title', String(256)),
+ Column('pi_id', String(32)),
+ Column('affiliation', String(64)),
+ Column('executive', String(32)),
+ Column('qa0passcount', Numeric(asdecimal=False)),
+ Column('final_scientific_category', String(32)),
+ Column('project_state', String(32)),
+ schema='ALMA'
+)
+
+
+t_asp_obs_proposal_author = Table(
+ 'asp_obs_proposal_author', metadata,
+ Column('proposal_archive_uid', String(255), nullable=False),
+ Column('author_index', Numeric(scale=0, asdecimal=False), nullable=False),
+ Column('account_id', String(32), nullable=False),
+ Column('firstname', String(256), nullable=False),
+ Column('initials', String(256)),
+ Column('lastname', String(256), nullable=False),
+ schema='ALMA'
+)
+
+
+t_assignment = Table(
+ 'assignment', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('participant_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('score', Float),
+ Column('proposal_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('state_change_date', DateTime),
+ Column('status', String(32)),
+ Column('assignment_type', String(32)),
+ Column('conflict', String(32)),
+ Column('normalized_score', Float),
+ Column('vote_score', Float),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ Column('rejection_type', String(32)),
+ Column('normalized_vote_score', Numeric(asdecimal=False)),
+ Column('assignment_comment', String(4000)),
+ Column('reason_for_rejection', String(4000)),
+ Column('comment_to_sa', String(4000)),
+ Column('comment_to_phase2', String(4000)),
+ Column('resurrection_justification', String(4000)),
+ schema='ALMA'
+)
+
+
+t_assignment_view = Table(
+ 'assignment_view', metadata,
+ Column('archive_uid', String(33), nullable=False),
+ Column('proposal_code', String(64)),
+ Column('cycle_code', String(32)),
+ Column('title', String(256)),
+ Column('pi_userid', String(32)),
+ Column('abstract_text', Text),
+ Column('scientific_category', String(32)),
+ Column('new_scientific_category', String(16)),
+ Column('overridden_sci_cat', String(32)),
+ Column('proposal_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('feasible', String(8)),
+ Column('arp_avg_score', Float),
+ Column('arp_std_dev_scores', Float),
+ Column('arp_score', Float),
+ Column('arp_prop_strength', String(4000)),
+ Column('arp_prop_weakness', String(4000)),
+ Column('arp_prop_improvement', String(4000)),
+ Column('arp_prop_evaluation', String(4000)),
+ Column('arp_comment', String(4000)),
+ Column('aprc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('aprc_letter_grade', String(1)),
+ Column('aprc_prop_strength', String(4000)),
+ Column('aprc_prop_weakness', String(4000)),
+ Column('aprc_prop_improvement', String(4000)),
+ Column('aprc_prop_evaluation', String(4000)),
+ Column('aprc_comment', String(4000)),
+ Column('dc_letter_grade', String(1)),
+ Column('assignment_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('status', String(32)),
+ Column('conflict', String(32)),
+ Column('assignment_type', String(32)),
+ Column('assignment_comment', String(4000)),
+ Column('comment_to_sa', String(4000)),
+ Column('comment_to_phase2', String(4000)),
+ Column('score', Float),
+ Column('normalized_score', Float),
+ Column('reason_for_rejection', String(4000)),
+ Column('assessor_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('participant_type', String(64), nullable=False),
+ Column('assessor_userid', String(32), nullable=False),
+ Column('assessor_first_name', String(256)),
+ Column('assessor_last_name', String(256)),
+ Column('assessor_email', String(256)),
+ Column('arp_id', Numeric(38, 0, asdecimal=False)),
+ Column('panel_name', String(255)),
+ Column('role_on_panel', String(255)),
+ schema='ALMA'
+)
+
+
+class BmmvObsproject(Base):
+ __tablename__ = 'bmmv_obsproject'
+ __table_args__ = {'schema': 'ALMA'}
+
+ prj_archive_uid = Column(String(33), primary_key=True)
+ deleted = Column(Numeric(1, 0, asdecimal=False), index=True)
+ pi = Column(String(64), index=True)
+ prj_name = Column(String(256))
+ title = Column(String(256))
+ array = Column(String(64))
+ prj_code = Column(String(64), nullable=False)
+ code = Column(String(64), index=True)
+ prj_time_of_creation = Column(String(23))
+ prj_scientific_rank = Column(Numeric(8, 0, asdecimal=False))
+ prj_version = Column(String(32))
+ prj_assigned_priority = Column(Numeric(asdecimal=False))
+ prj_letter_grade = Column(String(2))
+ p2g_attention = Column(String(5), index=True)
+ p2g_attention_reasons = Column(String(4000))
+ manual_mode = Column(String(5))
+
+
+class BmmvObsproposal(Base):
+ __tablename__ = 'bmmv_obsproposal'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ abstract_text = Column(Text)
+ scientific_category = Column(String(32))
+ proposal_type = Column(String(32))
+ pi_userid = Column(String(32))
+ associatedexec = Column(String(32))
+ datereceived = Column(String(32))
+ obsproject_archive_uid = Column(String(32), nullable=False, index=True)
+ projectuid = Column(String(32), nullable=False, index=True)
+ pi_fullname = Column(String(64))
+ organization = Column(String(64))
+ email = Column(String(64))
+ cycle = Column(String(32), index=True)
+ keyword1 = Column(String(128))
+ keyword2 = Column(String(128))
+ keywordcode1 = Column(String(128))
+ keywordcode2 = Column(String(128))
+ studentproject = Column(String(32))
+
+
+class BmmvObsproposalAuthor(Base):
+ __tablename__ = 'bmmv_obsproposal_authors'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), nullable=False, index=True)
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ cycle = Column(String(32), nullable=False)
+ project_archive_uid = Column(String(33), nullable=False)
+ userid = Column(String(32), nullable=False)
+ organisation = Column(String(32))
+ executive = Column(String(32))
+ authtype = Column(String(4), nullable=False)
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ organisation_id = Column(Numeric(38, 0, asdecimal=False))
+
+
+class BmmvObsproposalFeedback(Base):
+ __tablename__ = 'bmmv_obsproposal_feedback'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ nonalmaexectimefraction = Column(String(16))
+ chileexectimefraction = Column(String(16))
+ eaexectimefraction = Column(String(16))
+ euexectimefraction = Column(String(16))
+ naexectimefraction = Column(String(16))
+ integration_time = Column(String(64))
+ integration_time_unit = Column(String(64))
+ receiverband01time = Column(Numeric(asdecimal=False))
+ receiverband02time = Column(Numeric(asdecimal=False))
+ receiverband03time = Column(Numeric(asdecimal=False))
+ receiverband04time = Column(Numeric(asdecimal=False))
+ receiverband05time = Column(Numeric(asdecimal=False))
+ receiverband06time = Column(Numeric(asdecimal=False))
+ receiverband07time = Column(Numeric(asdecimal=False))
+ receiverband08time = Column(Numeric(asdecimal=False))
+ receiverband09time = Column(Numeric(asdecimal=False))
+ receiverband10time = Column(Numeric(asdecimal=False))
+ receiverbanduntime = Column(Numeric(asdecimal=False))
+ array_time_alma = Column(Numeric(asdecimal=False))
+ array_time_unit_alma = Column(String(8))
+ array_time_aca = Column(Numeric(asdecimal=False))
+ array_time_unit_aca = Column(String(8))
+ array_time_twelve_m = Column(Numeric(asdecimal=False))
+ array_time_unit_twelve_m = Column(String(8))
+ array_time_seven_m = Column(Numeric(asdecimal=False))
+ array_time_unit_seven_m = Column(String(8))
+ array_time_tp_array = Column(Numeric(asdecimal=False))
+ array_time_unit_tp_array = Column(String(8))
+ max_data_rate_twelve_m = Column(Numeric(asdecimal=False))
+ max_data_rate_unit_twelve_m = Column(String(8))
+ data_volume_twelve_m = Column(Numeric(asdecimal=False))
+ data_volume_unit_twelve_m = Column(String(8))
+ max_data_rate_seven_m = Column(Numeric(asdecimal=False))
+ max_data_rate_unit_seven_m = Column(String(8))
+ data_volume_seven_m = Column(Numeric(asdecimal=False))
+ data_volume_unit_seven_m = Column(String(8))
+ max_data_rate_tp_array = Column(Numeric(asdecimal=False))
+ max_data_rate_unit_tp_array = Column(String(8))
+ data_volume_tp_array = Column(Numeric(asdecimal=False))
+ data_volume_unit_tp_array = Column(String(8))
+
+
+class BmmvObsproposalTarget(Base):
+ __tablename__ = 'bmmv_obsproposal_target'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ prop_archive_uid = Column(String(33), nullable=False, index=True)
+ science_goal_name = Column(String(1000))
+ source_coord_system = Column(String(10))
+ target_longitude = Column(Numeric(asdecimal=False))
+ target_longitude_unit = Column(String(10))
+ target_latitude = Column(Numeric(asdecimal=False))
+ target_latitude_unit = Column(String(10))
+ desired_sensitivity = Column(Numeric(asdecimal=False))
+ desired_sensitivity_unit = Column(String(10))
+ desired_angular_resolution = Column(Numeric(asdecimal=False))
+ desired_ang_res_unit = Column(String(10))
+ spectral_resolution = Column(Numeric(asdecimal=False))
+ spectral_resolution_unit = Column(String(10))
+ center_frequency = Column(Numeric(asdecimal=False))
+ center_frequency_unit = Column(String(10))
+ band_width = Column(Numeric(asdecimal=False))
+ band_width_unit = Column(String(10))
+ is_sky_frequency = Column(String(5))
+ group_index = Column(Numeric(asdecimal=False))
+ science_goal_index = Column(Numeric(asdecimal=False))
+ start_frequency = Column(Numeric(asdecimal=False))
+ start_frequency_unit = Column(String(10))
+ end_frequency = Column(Numeric(asdecimal=False))
+ end_frequency_unit = Column(String(10))
+
+
+class BmmvObsunitset(Base):
+ __tablename__ = 'bmmv_obsunitset'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ archive_uid = Column(String(32), nullable=False, index=True)
+ partid = Column(String(64))
+ name = Column(String(256))
+ scienceprocessingscript = Column(String(1024))
+ runsciencepipeline = Column(String(32))
+ numberschedblocks = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ ousstatusref = Column(String(32))
+ requestedarray = Column(String(8))
+ is_resubmission = Column(String(5))
+ resubmission_of_name = Column(String(1000))
+ resolution_option = Column(String(32))
+ solar_system_object = Column(String(32))
+ sg_mode = Column(String(16))
+
+
+class BmmvProjTimeConstraint(Base):
+ __tablename__ = 'bmmv_proj_time_constraint'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ prj_archive_uid = Column(String(33), nullable=False, index=True)
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ time_constraint_type = Column(String(32))
+ start_time = Column(String(32))
+ end_time = Column(String(32))
+ allowed_margin = Column(String(32))
+ allowed_margin_unit = Column(String(5))
+ required_delay = Column(String(32))
+ required_delay_unit = Column(String(5))
+ visit_id = Column(String(32))
+ previous_visit_id = Column(String(32))
+ monitoring_length = Column(String(32))
+ monitoring_length_unit = Column(String(5))
+
+
+class BmmvQuicklook(Base):
+ __tablename__ = 'bmmv_quicklook'
+ __table_args__ = (
+ Index('bmmv_quicklook_uni', 'ousstatus_uid', 'ousstatus_part_id', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ archive_uid = Column(String(33), primary_key=True)
+ schedblock_uid = Column(String(64))
+ qlfocussummary_uid = Column(String(64))
+ qlpointingsummary_uid = Column(String(64))
+ qlatmospheresummary_uid = Column(String(64))
+ ousstatus_uid = Column(String(64), nullable=False)
+ ousstatus_part_id = Column(String(64), nullable=False)
+ qlphasesummary_uid = Column(String(64))
+
+
+class BmmvSchedblock(Base):
+ __tablename__ = 'bmmv_schedblock'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ prj_ref = Column(String(32), index=True)
+ receiver_band = Column(String(32))
+ frequency = Column(String(32))
+ ref_ra = Column(String(32))
+ ref_dec = Column(String(32))
+ sb_name = Column(String(32))
+ time_string = Column(String(32))
+ time_units = Column(String(32))
+ receiverband = Column(String(32))
+ refra = Column(String(32))
+ refdec = Column(String(32))
+ sbname = Column(String(32))
+ totalestimatedasstring = Column(String(32))
+ totalestimatedtimeunits = Column(String(32))
+ status = Column(String(32))
+ totalestimatedtime = Column(String(32))
+ execution_count = Column(String(32))
+ requestedarray = Column(String(8))
+ minacceptableangresolution = Column(Float)
+ maxacceptableangresolution = Column(Float)
+ representativefrequency = Column(Float)
+ nominalconfiguration = Column(String(32))
+ gous_status_uid = Column(String(32))
+ mous_status_uid = Column(String(32))
+ sgous_status_uid = Column(String(32))
+
+
+class BmmvSchedblockPolarisation(Base):
+ __tablename__ = 'bmmv_schedblock_polarisation'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ archive_uid = Column(String(33), nullable=False, index=True)
+ products = Column(String(32), nullable=False)
+
+
+class BmmvSchedblockSource(Base):
+ __tablename__ = 'bmmv_schedblock_sources'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ archive_uid = Column(String(33), nullable=False, index=True)
+ field_source_name = Column(String(32), nullable=False)
+
+
+class Country(Base):
+ __tablename__ = 'country'
+ __table_args__ = {'schema': 'ALMA'}
+
+ country_id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ country_name = Column(String(256), nullable=False, unique=True)
+ executive = Column(String(32), nullable=False)
+
+
+t_cycle = Table(
+ 'cycle', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('code', String(255), nullable=False),
+ Column('cycle_step', String(64), nullable=False),
+ Column('total_hours', Numeric(8, 0, asdecimal=False)),
+ Column('ea_percent', Numeric(8, 2)),
+ Column('eu_percent', Numeric(8, 2)),
+ Column('na_percent', Numeric(8, 2)),
+ Column('cl_percent', Numeric(8, 2)),
+ Column('other_percent', Numeric(8, 2)),
+ Column('percent_marker_1', Numeric(8, 0, asdecimal=False)),
+ Column('percent_marker_2', Numeric(8, 0, asdecimal=False)),
+ Column('percent_marker_3', Numeric(8, 0, asdecimal=False)),
+ Column('scientific_categories', String(255)),
+ Column('start_date', DateTime),
+ Column('end_date', DateTime),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ Column('triage_std_dev_factor', Numeric(8, 2)),
+ Column('triage_percent', Numeric(8, 2)),
+ Column('ddt', String(1)),
+ Column('active', String(1)),
+ Column('eu_triage_percent', Numeric(8, 2)),
+ Column('na_triage_percent', Numeric(8, 2)),
+ Column('ea_triage_percent', Numeric(8, 2)),
+ Column('cl_triage_percent', Numeric(8, 2)),
+ Column('duplication_arc_seconds', Numeric(8, 2)),
+ Column('keyword_codes', String(4000)),
+ Column('score_mean', Numeric(8, 6)),
+ Column('score_std_dev', Numeric(8, 6)),
+ schema='ALMA'
+)
+
+
+t_dbmaintain_scripts = Table(
+ 'dbmaintain_scripts', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_dbmaintain_scripts_account = Table(
+ 'dbmaintain_scripts_account', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_dbmaintain_scripts_apdm = Table(
+ 'dbmaintain_scripts_apdm', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_dbmaintain_scripts_asa = Table(
+ 'dbmaintain_scripts_asa', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_dbmaintain_scripts_ph1m = Table(
+ 'dbmaintain_scripts_ph1m', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_dbmaintain_scripts_protrack = Table(
+ 'dbmaintain_scripts_protrack', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_dbmaintain_scripts_version = Table(
+ 'dbmaintain_scripts_version', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+class DpCacheAsdmInfo(Base):
+ __tablename__ = 'dp_cache_asdm_info'
+ __table_args__ = {'schema': 'ALMA'}
+
+ asdm_id = Column(String(64), primary_key=True)
+ tar_size_bytes = Column(Numeric(12, 0, asdecimal=False))
+
+
+class DpCacheXmlInfo(Base):
+ __tablename__ = 'dp_cache_xml_info'
+ __table_args__ = {'schema': 'ALMA'}
+
+ xml_id = Column(String(64), primary_key=True)
+ size_bytes = Column(Numeric(9, 0, asdecimal=False))
+
+
+t_dp_delegation = Table(
+ 'dp_delegation', metadata,
+ Column('id', Numeric(22, 0, asdecimal=False), nullable=False),
+ Column('pi_rh_id', Numeric(22, 0, asdecimal=False), nullable=False),
+ Column('project_code', String(18), nullable=False),
+ Column('delegee_rh_id', Numeric(22, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+t_dp_delegation_backup = Table(
+ 'dp_delegation_backup', metadata,
+ Column('id', Numeric(22, 0, asdecimal=False), nullable=False),
+ Column('pi_rh_id', Numeric(22, 0, asdecimal=False), nullable=False),
+ Column('project_code', String(18), nullable=False),
+ Column('delegee_rh_id', Numeric(22, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+t_dp_delivery_asdm_ous_bkp = Table(
+ 'dp_delivery_asdm_ous_bkp', metadata,
+ Column('asdm_uid', String(33)),
+ Column('ous_status_id', String(33)),
+ Column('deliverable_name', String(64), nullable=False),
+ Column('id', Numeric(22, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+t_drm_data_reducer = Table(
+ 'drm_data_reducer', metadata,
+ Column('dr_user_id', ForeignKey('ALMA.account.account_id'), primary_key=True),
+ Column('node_id', ForeignKey('ALMA.drm_node.id')),
+ schema='ALMA'
+)
+
+
+class DrmDrHistory(Base):
+ __tablename__ = 'drm_dr_history'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ author = Column(String(32))
+ entityuid = Column(String(32), nullable=False)
+ datareducer = Column(String(32))
+ ttimestamp = Column(DateTime, nullable=False)
+ arc = Column(String(8))
+
+
+class DrmDrQualification(Base):
+ __tablename__ = 'drm_dr_qualification'
+ __table_args__ = {'schema': 'ALMA'}
+
+ dr_user_id = Column(ForeignKey('ALMA.account.account_id'), primary_key=True, nullable=False)
+ qualification = Column(String(32), primary_key=True, nullable=False)
+
+ dr_user = relationship('Account')
+
+
+class DrmNode(Base):
+ __tablename__ = 'drm_node'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(3, 0, asdecimal=False), primary_key=True)
+ arc = Column(String(8), nullable=False)
+ node = Column(String(32))
+
+
+t_email_template = Table(
+ 'email_template', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('name', String(255), nullable=False),
+ Column('content_type', String(64)),
+ Column('subject', String(255)),
+ Column('body', Text),
+ Column('query', Text),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ Column('system_email', String(1)),
+ schema='ALMA'
+)
+
+
+class ErrorLog(Base):
+ __tablename__ = 'error_log'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(22, 0, asdecimal=False), primary_key=True)
+ archive_uid = Column(String(32))
+ table_name = Column(String(30))
+ ora_error_code = Column(Numeric(scale=0, asdecimal=False))
+ ora_error_message = Column(String(4000))
+ ora_backtrace = Column(Text)
+ ora_callstack = Column(Text)
+ created_on = Column(DateTime)
+ created_by = Column(String(30))
+
+
+class Eu1(Base):
+ __tablename__ = 'eu1'
+ __table_args__ = {'schema': 'ALMA'}
+
+ data1 = Column(String(100), primary_key=True)
+
+
+class Institution(Base):
+ __tablename__ = 'institution'
+ __table_args__ = (
+ Index('institution_name_unique', 'name1', 'name2', 'name3', 'country_id', 'state', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ inst_no = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ sibling_no = Column(ForeignKey('ALMA.institution.inst_no'))
+ name1 = Column(String(2048), nullable=False)
+ name2 = Column(String(256))
+ name3 = Column(String(256))
+ altnames = Column(String(256))
+ city = Column(String(64))
+ postcode = Column(String(32))
+ executive = Column(String(5), nullable=False)
+ state = Column(String(32))
+ email = Column(String(256))
+ url = Column(String(256))
+ phone = Column(String(128))
+ fax = Column(String(128))
+ recorder = Column(String(256))
+ superseding_no = Column(Numeric(38, 0, asdecimal=False))
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False, server_default=text("0 "))
+ country_id = Column(ForeignKey('ALMA.country.country_id'))
+ user_defined = Column(String(1), server_default=text("'F'"))
+ ignored = Column(String(1), server_default=text("'F'"))
+ address1 = Column(String(512))
+ address2 = Column(String(512))
+ address3 = Column(String(512))
+
+ country = relationship('Country')
+ parent = relationship('Institution', remote_side=[inst_no])
+
+
+t_mv_obsproject = Table(
+ 'mv_obsproject', metadata,
+ Column('prj_archive_uid', String(33), nullable=False),
+ Column('pi', String(32)),
+ Column('prj_name', String(256)),
+ Column('array', String(64)),
+ Column('prj_code', String(64), nullable=False),
+ Column('prj_time_of_creation', String(23)),
+ Column('prj_scientific_rank', Numeric(8, 0, asdecimal=False)),
+ Column('prj_version', String(32)),
+ Column('prj_assigned_priority', Numeric(asdecimal=False)),
+ Column('prj_letter_grade', String(2)),
+ Column('ph1m_priority_grade', String(2)),
+ Column('pi_fullname', String(64)),
+ Column('organization', String(64)),
+ Column('email', String(64)),
+ Column('cycle', String(32)),
+ Column('executive', String(32)),
+ Column('p2g_attention', String(5)),
+ Column('p2g_attention_reasons', String(4000)),
+ schema='ALMA'
+)
+
+
+t_mv_obsproject_derived = Table(
+ 'mv_obsproject_derived', metadata,
+ Column('prj_archive_uid', String(33), nullable=False),
+ Column('total_time', Numeric(asdecimal=False)),
+ Column('used_time', Numeric(asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_mv_obsproposal = Table(
+ 'mv_obsproposal', metadata,
+ Column('archive_uid', String(33), nullable=False),
+ Column('deleted', Numeric(1, 0, asdecimal=False)),
+ Column('title', String(256)),
+ Column('code', String(64)),
+ Column('cycle', String(32)),
+ Column('abstract_text', Text),
+ Column('scientific_category', String(32)),
+ Column('proposal_type', String(32)),
+ Column('pi_userid', String(32)),
+ Column('obsproject_archive_uid', String(32), nullable=False),
+ Column('pi_executive', String(32)),
+ Column('datereceived', String(32)),
+ Column('keyword1', String(128)),
+ Column('keyword2', String(128)),
+ Column('keywordcode1', String(128)),
+ Column('keywordcode2', String(128)),
+ Column('studentproject', String(32)),
+ Column('nonalmaexectimefraction', String(16)),
+ Column('chileexectimefraction', String(16)),
+ Column('eaexectimefraction', String(16)),
+ Column('euexectimefraction', String(16)),
+ Column('naexectimefraction', String(16)),
+ Column('integration_time', Numeric(asdecimal=False)),
+ Column('integration_time_unit', String(8)),
+ Column('receiverband01time', Numeric(asdecimal=False)),
+ Column('receiverband02time', Numeric(asdecimal=False)),
+ Column('receiverband03time', Numeric(asdecimal=False)),
+ Column('receiverband04time', Numeric(asdecimal=False)),
+ Column('receiverband05time', Numeric(asdecimal=False)),
+ Column('receiverband06time', Numeric(asdecimal=False)),
+ Column('receiverband07time', Numeric(asdecimal=False)),
+ Column('receiverband08time', Numeric(asdecimal=False)),
+ Column('receiverband09time', Numeric(asdecimal=False)),
+ Column('receiverband10time', Numeric(asdecimal=False)),
+ Column('receiverbanduntime', Numeric(asdecimal=False)),
+ Column('array_time_alma', Numeric(asdecimal=False)),
+ Column('array_time_unit_alma', String(8)),
+ Column('array_time_twelve_m', Numeric(asdecimal=False)),
+ Column('array_time_unit_twelve_m', String(8)),
+ Column('array_time_aca', Numeric(asdecimal=False)),
+ Column('array_time_unit_aca', String(8)),
+ Column('array_time_seven_m', Numeric(asdecimal=False)),
+ Column('array_time_unit_seven_m', String(8)),
+ Column('array_time_tp_array', Numeric(asdecimal=False)),
+ Column('array_time_unit_tp_array', String(8)),
+ Column('max_data_rate_twelve_m', Numeric(asdecimal=False)),
+ Column('max_data_rate_unit_twelve_m', String(8)),
+ Column('data_volume_twelve_m', Numeric(asdecimal=False)),
+ Column('data_volume_unit_twelve_m', String(8)),
+ Column('max_data_rate_seven_m', Numeric(asdecimal=False)),
+ Column('max_data_rate_unit_seven_m', String(8)),
+ Column('data_volume_seven_m', Numeric(asdecimal=False)),
+ Column('data_volume_unit_seven_m', String(8)),
+ Column('max_data_rate_tp_array', Numeric(asdecimal=False)),
+ Column('max_data_rate_unit_tp_array', String(8)),
+ Column('data_volume_tp_array', Numeric(asdecimal=False)),
+ Column('data_volume_unit_tp_array', String(8)),
+ Column('domain_entity_state', String(32), nullable=False),
+ Column('status_entity_id', String(64), nullable=False),
+ schema='ALMA'
+)
+
+
+t_mv_obsproposal_authors = Table(
+ 'mv_obsproposal_authors', metadata,
+ Column('archive_uid', String(33), nullable=False),
+ Column('prj_code', String(64), nullable=False),
+ Column('deleted', Numeric(1, 0, asdecimal=False)),
+ Column('cycle', String(32), nullable=False),
+ Column('project_archive_uid', String(33), nullable=False),
+ Column('userid', String(32), nullable=False),
+ Column('organisation', String(32)),
+ Column('organisation_id', Numeric(38, 0, asdecimal=False)),
+ Column('executive', String(32)),
+ Column('authtype', String(4), nullable=False),
+ schema='ALMA'
+)
+
+
+t_mv_obsproposal_target = Table(
+ 'mv_obsproposal_target', metadata,
+ Column('id', Numeric(asdecimal=False), nullable=False),
+ Column('prop_archive_uid', String(33), nullable=False),
+ Column('cycle', String(32)),
+ Column('code', String(64)),
+ Column('scientific_category', String(32)),
+ Column('new_scientific_category', String(16)),
+ Column('science_goal_name', String(1000)),
+ Column('source_coord_system', String(10)),
+ Column('target_longitude', Numeric(asdecimal=False)),
+ Column('target_longitude_unit', String(10)),
+ Column('target_latitude', Numeric(asdecimal=False)),
+ Column('target_latitude_unit', String(10)),
+ Column('desired_sensitivity', Numeric(asdecimal=False)),
+ Column('desired_sensitivity_unit', String(10)),
+ Column('desired_angular_resolution', Numeric(asdecimal=False)),
+ Column('desired_ang_res_unit', String(10)),
+ Column('spectral_resolution', Numeric(asdecimal=False)),
+ Column('spectral_resolution_unit', String(10)),
+ Column('center_frequency', Numeric(asdecimal=False)),
+ Column('center_frequency_unit', String(10)),
+ Column('band_width', Numeric(asdecimal=False)),
+ Column('band_width_unit', String(10)),
+ Column('is_sky_frequency', String(4)),
+ Column('group_index', Numeric(asdecimal=False)),
+ Column('aprc_letter_grade', String(1)),
+ Column('dc_letter_grade', String(1)),
+ Column('science_goal_index', Numeric(asdecimal=False)),
+ Column('project_status', String(32), nullable=False),
+ Column('start_frequency', Numeric(asdecimal=False)),
+ Column('start_frequency_unit', String(10)),
+ Column('end_frequency', Numeric(asdecimal=False)),
+ Column('end_frequency_unit', String(10)),
+ schema='ALMA'
+)
+
+
+t_mv_obsunitset = Table(
+ 'mv_obsunitset', metadata,
+ Column('archive_uid', String(32), nullable=False),
+ Column('partid', String(64)),
+ Column('name', String(256)),
+ Column('scienceprocessingscript', String(1024)),
+ Column('runsciencepipeline', String(32)),
+ Column('numberschedblocks', String(32)),
+ Column('requestedarray', String(8)),
+ Column('ousstatusref', String(32)),
+ Column('is_resubmission', String(5)),
+ Column('resubmission_of_name', String(1000)),
+ Column('resolution_option', String(32)),
+ Column('solar_system_object', String(32)),
+ Column('sg_mode', String(16)),
+ schema='ALMA'
+)
+
+
+t_mv_proj_time_constraint = Table(
+ 'mv_proj_time_constraint', metadata,
+ Column('id', Numeric(asdecimal=False), nullable=False),
+ Column('prj_archive_uid', String(33), nullable=False),
+ Column('deleted', Numeric(1, 0, asdecimal=False)),
+ Column('time_constraint_type', String(32)),
+ Column('start_time', String(32)),
+ Column('end_time', String(32)),
+ Column('allowed_margin', String(32)),
+ Column('allowed_margin_unit', String(5)),
+ Column('required_delay', String(32)),
+ Column('required_delay_unit', String(5)),
+ Column('visit_id', String(32)),
+ Column('previous_visit_id', String(32)),
+ Column('monitoring_length', String(32)),
+ Column('monitoring_length_unit', String(5)),
+ schema='ALMA'
+)
+
+
+t_mv_quicklook = Table(
+ 'mv_quicklook', metadata,
+ Column('archive_uid', String(33), nullable=False),
+ Column('schedblock_uid', String(64)),
+ Column('qlfocussummary_uid', String(64)),
+ Column('qlpointingsummary_uid', String(64)),
+ Column('qlatmospheresummary_uid', String(64)),
+ schema='ALMA'
+)
+
+
+t_mv_schedblock = Table(
+ 'mv_schedblock', metadata,
+ Column('sb_archive_uid', String(33), nullable=False),
+ Column('prj_ref', String(32)),
+ Column('receiver_band', String(32)),
+ Column('frequency', String(32)),
+ Column('ref_ra', String(32)),
+ Column('ref_dec', String(32)),
+ Column('sb_name', String(32)),
+ Column('requested_array', String(8)),
+ Column('minacceptableangresolution', Float),
+ Column('maxacceptableangresolution', Float),
+ Column('representativefrequency', Float),
+ Column('status', String(32), nullable=False),
+ Column('used_time', Numeric(asdecimal=False)),
+ Column('time_string', String(32)),
+ Column('time_units', String(32)),
+ Column('execution_count', String(32)),
+ Column('nominalconfiguration', String(32)),
+ Column('time', Numeric(asdecimal=False)),
+ Column('phase1_flag', Numeric(asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_mv_schedblock_sourcename = Table(
+ 'mv_schedblock_sourcename', metadata,
+ Column('id', Numeric(asdecimal=False), nullable=False),
+ Column('archive_uid', String(33), nullable=False),
+ Column('sourcename', String(32), nullable=False),
+ schema='ALMA'
+)
+
+
+class ObsProjectStatu(Base):
+ __tablename__ = 'obs_project_status'
+ __table_args__ = {'schema': 'ALMA'}
+
+ status_entity_id = Column(String(64), primary_key=True)
+ domain_entity_id = Column(String(64), nullable=False, index=True)
+ domain_entity_state = Column(String(32), nullable=False, index=True)
+ obs_project_status_id = Column(String(64), nullable=False)
+ obs_program_status_id = Column(String(64), nullable=False)
+ obs_project_id = Column(ForeignKey('ALMA.xml_obsproject_entities.archive_uid'), nullable=False, unique=True)
+ project_was_timed_out = Column(DateTime)
+ xml = Column(NullType)
+ flags = Column(String(200), index=True)
+
+ obs_project = relationship('XmlObsprojectEntity')
+
+
+class ObsUnitSetStatu(Base):
+ __tablename__ = 'obs_unit_set_status'
+ __table_args__ = (
+ Index('ous_status_project_entity_idx', 'obs_project_id', 'domain_entity_id'),
+ {'schema': 'ALMA'}
+ )
+
+ status_entity_id = Column(String(64), primary_key=True)
+ domain_entity_id = Column(String(64))
+ domain_entity_state = Column(String(32), nullable=False)
+ parent_obs_unit_set_status_id = Column(String(64))
+ obs_project_status_id = Column(String(64), nullable=False)
+ obs_project_id = Column(ForeignKey('ALMA.xml_obsproject_entities.archive_uid'), nullable=False)
+ total_required_time_in_sec = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ total_used_time_in_sec = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ xml = Column(NullType)
+ flags = Column(String(200), index=True)
+
+ obs_project = relationship('XmlObsprojectEntity')
+
+
+class OcdObservingCycle(Base):
+ __tablename__ = 'ocd_observing_cycle'
+ __table_args__ = {'schema': 'ALMA'}
+
+ observingcycleid = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ cycle = Column(String(32), nullable=False, unique=True)
+ description = Column(String(256))
+ clfraction = Column(Numeric(8, 2))
+ eafraction = Column(Numeric(8, 2))
+ eufraction = Column(Numeric(8, 2))
+ nafraction = Column(Numeric(8, 2))
+ otherfraction = Column(Numeric(8, 2))
+
+
+class OcdObservingSession(Base):
+ __tablename__ = 'ocd_observing_session'
+ __table_args__ = (
+ Index('ocd_obssession_uni', 'observingcycleid', 'blocknumber', 'startdate', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ sessionid = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ observingcycleid = Column(ForeignKey('ALMA.ocd_observing_cycle.observingcycleid'), nullable=False)
+ blocknumber = Column(Numeric(8, 0, asdecimal=False), nullable=False)
+ startdate = Column(DateTime, nullable=False)
+ enddate = Column(DateTime, nullable=False)
+ comments = Column(String(256))
+
+ ocd_observing_cycle = relationship('OcdObservingCycle')
+
+
+t_old_dbmaintain_scripts = Table(
+ 'old_dbmaintain_scripts', metadata,
+ Column('file_name', String(150)),
+ Column('file_last_modified_at', Numeric(scale=0, asdecimal=False)),
+ Column('checksum', String(50)),
+ Column('executed_at', String(20)),
+ Column('succeeded', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+class OusOperation(Base):
+ __tablename__ = 'ous_operations'
+ __table_args__ = {'schema': 'ALMA'}
+
+ obs_unit_set_id = Column(String(64), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ data_reducer_account_id = Column(ForeignKey('ALMA.account.account_id'))
+ data_reducer_executive = Column(String(64))
+ qa2recipients = Column(String(512))
+ drm_account_id = Column(ForeignKey('ALMA.account.account_id'))
+
+ data_reducer_account = relationship('Account', primaryjoin='OusOperation.data_reducer_account_id == Account.account_id')
+ drm_account = relationship('Account', primaryjoin='OusOperation.drm_account_id == Account.account_id')
+
+
+t_panel = Table(
+ 'panel', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('cycle_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('panel_name', String(255), nullable=False),
+ Column('minutes', Text),
+ Column('panel_type', String(32), nullable=False),
+ Column('scientific_category', String(32)),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ schema='ALMA'
+)
+
+
+t_panel_member = Table(
+ 'panel_member', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('participant_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('panel_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('role_on_panel', String(255)),
+ Column('package_downloaded', DateTime),
+ schema='ALMA'
+)
+
+
+t_participant = Table(
+ 'participant', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('cycle_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('package_downloaded', DateTime),
+ Column('userid', String(32), nullable=False),
+ Column('participant_type', String(64), nullable=False),
+ Column('alma_location', String(32)),
+ Column('keyword_codes', String(256)),
+ Column('score_mean', Numeric(8, 6)),
+ Column('score_std_dev', Numeric(8, 6)),
+ schema='ALMA'
+)
+
+
+class Ph1mAssignment(Base):
+ __tablename__ = 'ph1m_assignment'
+ __table_args__ = (
+ Index('assignment_unique', 'proposal_id', 'participant_id', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ participant_id = Column(ForeignKey('ALMA.ph1m_participant.id'), nullable=False, index=True)
+ score = Column(Float)
+ proposal_id = Column(ForeignKey('ALMA.ph1m_proposal.id'), nullable=False, index=True)
+ state_change_date = Column(DateTime)
+ status = Column(String(32))
+ assignment_type = Column(String(32))
+ conflict = Column(String(32))
+ normalized_score = Column(Float)
+ vote_score = Column(Float)
+ update_date = Column(DateTime)
+ update_userid = Column(String(32))
+ rejection_type = Column(String(32))
+ normalized_vote_score = Column(Numeric(asdecimal=False))
+ assignment_comment = Column(String(4000))
+ reason_for_rejection = Column(String(4000))
+ comment_to_sa = Column(String(4000))
+ comment_to_phase2 = Column(String(4000))
+ resurrection_justification = Column(String(4000))
+
+ participant = relationship('Ph1mParticipant')
+ proposal = relationship('Ph1mProposal')
+
+
+class Ph1mAutosave(Base):
+ __tablename__ = 'ph1m_autosave'
+ __table_args__ = {'schema': 'ALMA'}
+
+ table_name = Column(String(32), primary_key=True, nullable=False)
+ column_name = Column(String(32), primary_key=True, nullable=False)
+ row_id = Column(Numeric(38, 0, asdecimal=False), primary_key=True, nullable=False)
+ user_id = Column(String(32), primary_key=True, nullable=False)
+ update_date = Column(DateTime, nullable=False)
+ saved_text = Column(Text, nullable=False)
+
+
+class Ph1mCycle(Base):
+ __tablename__ = 'ph1m_cycle'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ code = Column(String(255), nullable=False, unique=True)
+ cycle_step = Column(String(64), nullable=False)
+ total_hours = Column(Numeric(8, 0, asdecimal=False))
+ ea_percent = Column(Numeric(8, 2))
+ eu_percent = Column(Numeric(8, 2))
+ na_percent = Column(Numeric(8, 2))
+ cl_percent = Column(Numeric(8, 2))
+ other_percent = Column(Numeric(8, 2))
+ percent_marker_1 = Column(Numeric(8, 0, asdecimal=False))
+ percent_marker_2 = Column(Numeric(8, 0, asdecimal=False))
+ percent_marker_3 = Column(Numeric(8, 0, asdecimal=False))
+ scientific_categories = Column(String(255))
+ start_date = Column(DateTime)
+ end_date = Column(DateTime)
+ update_date = Column(DateTime)
+ update_userid = Column(String(32))
+ triage_std_dev_factor = Column(Numeric(8, 2))
+ triage_percent = Column(Numeric(8, 2))
+ ddt = Column(String(1), server_default=text("'F'"))
+ active = Column(String(1), server_default=text("'T'"))
+ eu_triage_percent = Column(Numeric(8, 2), server_default=text("70"))
+ na_triage_percent = Column(Numeric(8, 2), server_default=text("70"))
+ ea_triage_percent = Column(Numeric(8, 2), server_default=text("70"))
+ cl_triage_percent = Column(Numeric(8, 2), server_default=text("70"))
+ duplication_arc_seconds = Column(Numeric(8, 2), server_default=text("120"))
+ keyword_codes = Column(String(4000))
+ score_mean = Column(Numeric(8, 6))
+ score_std_dev = Column(Numeric(8, 6))
+ triage_oversub_percent = Column(Numeric(8, 2), server_default=text("""\
+300
+ """))
+
+
+class Ph1mEmailTemplate(Base):
+ __tablename__ = 'ph1m_email_template'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ name = Column(String(255), nullable=False, unique=True)
+ content_type = Column(String(64))
+ subject = Column(String(255))
+ body = Column(Text)
+ query = Column(Text)
+ update_date = Column(DateTime)
+ update_userid = Column(String(32))
+ system_email = Column(String(1), server_default=text("""\
+'F'
+ """))
+
+
+class Ph1mPanel(Base):
+ __tablename__ = 'ph1m_panel'
+ __table_args__ = (
+ Index('unique_panel_name', 'cycle_id', 'panel_name', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ cycle_id = Column(ForeignKey('ALMA.ph1m_cycle.id'), nullable=False)
+ panel_name = Column(String(255), nullable=False)
+ minutes = Column(Text)
+ panel_type = Column(String(32), nullable=False)
+ scientific_category = Column(String(32))
+ update_date = Column(DateTime)
+ update_userid = Column(String(32))
+
+ cycle = relationship('Ph1mCycle')
+ proposals = relationship('Ph1mProposal', secondary='ALMA.ph1m_panel_proposal')
+
+
+class Ph1mPanelMember(Base):
+ __tablename__ = 'ph1m_panel_member'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ participant_id = Column(ForeignKey('ALMA.ph1m_participant.id'), nullable=False)
+ panel_id = Column(ForeignKey('ALMA.ph1m_panel.id'), nullable=False)
+ role_on_panel = Column(String(255))
+ package_downloaded = Column(DateTime)
+
+ panel = relationship('Ph1mPanel')
+ participant = relationship('Ph1mParticipant')
+
+
+t_ph1m_panel_proposal = Table(
+ 'ph1m_panel_proposal', metadata,
+ Column('panel_id', ForeignKey('ALMA.ph1m_panel.id'), primary_key=True, nullable=False),
+ Column('proposal_id', ForeignKey('ALMA.ph1m_proposal.id'), primary_key=True, nullable=False),
+ schema='ALMA'
+)
+
+
+class Ph1mParticipant(Base):
+ __tablename__ = 'ph1m_participant'
+ __table_args__ = (
+ Index('participant_unique', 'userid', 'cycle_id', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ cycle_id = Column(ForeignKey('ALMA.ph1m_cycle.id'), nullable=False)
+ package_downloaded = Column(DateTime)
+ userid = Column(String(32), nullable=False, index=True)
+ participant_type = Column(String(64), nullable=False)
+ alma_location = Column(String(32))
+ keyword_codes = Column(String(256))
+ score_mean = Column(Numeric(8, 6))
+ score_std_dev = Column(Numeric(8, 6))
+ max_workload = Column(Numeric(3, 0, asdecimal=False))
+
+ cycle = relationship('Ph1mCycle')
+
+
+class Ph1mProposal(Base):
+ __tablename__ = 'ph1m_proposal'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ cycle_id = Column(ForeignKey('ALMA.ph1m_cycle.id'), nullable=False)
+ archive_uid = Column(String(255), nullable=False, unique=True)
+ panel_id_back = Column(Numeric(38, 0, asdecimal=False), index=True)
+ cancelled = Column(String(1), server_default=text("'F'"))
+ new_integration_time_hours = Column(Numeric(8, 2), server_default=text("0"))
+ new_scientific_category = Column(String(16))
+ needs_more_time = Column(String(1), server_default=text("'F'"))
+ arp_score = Column(Float)
+ arp_avg_score = Column(Float)
+ arp_std_dev_scores = Column(Float)
+ aprc_letter_grade = Column(String(1))
+ aprc_rank = Column(Numeric(scale=0, asdecimal=False))
+ aprc_original_rank = Column(Numeric(scale=0, asdecimal=False))
+ arp_work_progress = Column(String(10))
+ aprc_work_progress = Column(String(10), server_default=text("'NOT_SEEN'"))
+ feasible = Column(String(8))
+ dc_score = Column(Float)
+ dc_letter_grade = Column(String(1))
+ dc_rank = Column(Numeric(scale=0, asdecimal=False))
+ dc_original_rank = Column(Numeric(scale=0, asdecimal=False))
+ dc_rank_changed = Column(String(1), server_default=text("'F'"))
+ dc_work_progress = Column(String(10), server_default=text("'NOT_SEEN'"))
+ arp_rank = Column(Numeric(scale=0, asdecimal=False))
+ update_date = Column(DateTime)
+ update_userid = Column(String(32))
+ triage_flag = Column(String(1), server_default=text("'A'"))
+ arp_vote_avg = Column(Float)
+ arp_vote_std_dev = Column(Float)
+ arp_nr_votes = Column(Numeric(scale=0, asdecimal=False), server_default=text("0"))
+ arp_original_rank = Column(Numeric(scale=0, asdecimal=False))
+ arp_rank_manually_changed = Column(String(1), server_default=text("'F'"))
+ aprc_rank_manually_changed = Column(String(1), server_default=text("'F'"))
+ arp_score_normalized = Column(Float, server_default=text("999999"))
+ arp_rank_normalized = Column(Float, server_default=text("999999"))
+ duplication_flags = Column(String(250))
+ arp_comment = Column(String(4000))
+ aprc_comment = Column(String(4000))
+ new_time_set_by_sg_descope = Column(String(1))
+ arp_prop_strength = Column(String(4000))
+ arp_prop_weakness = Column(String(4000))
+ arp_prop_improvement = Column(String(4000))
+ arp_prop_evaluation = Column(String(4000))
+ aprc_prop_strength = Column(String(4000))
+ aprc_prop_weakness = Column(String(4000))
+ aprc_prop_improvement = Column(String(4000))
+ aprc_prop_evaluation = Column(String(4000))
+ ta_comment_to_pi = Column(String(4000))
+ arp_grouping_flag = Column(Numeric(2, 0, asdecimal=False))
+ non_standard_twelve_m_hours = Column(Numeric(8, 2))
+ manual_flag = Column(String(255))
+ new_keywordcode1 = Column(String(4))
+ new_keywordcode2 = Column(String(4))
+
+ cycle = relationship('Ph1mCycle')
+
+
+class Ph1mProposalAuthor(Base):
+ __tablename__ = 'ph1m_proposal_author'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ proposal_id = Column(ForeignKey('ALMA.ph1m_proposal.id'), nullable=False, index=True)
+ sequence = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ userid = Column(String(32), nullable=False, index=True)
+ institution_id = Column(Numeric(asdecimal=False))
+ author_type = Column(String(4))
+
+ proposal = relationship('Ph1mProposal')
+
+
+class Ph1mProposalScienceGoal(Base):
+ __tablename__ = 'ph1m_proposal_science_goal'
+ __table_args__ = (
+ Index('ph1m_psg_unique', 'proposal_id', 'sg_name', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(19, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(19, 0, asdecimal=False), nullable=False)
+ proposal_id = Column(ForeignKey('ALMA.ph1m_proposal.id'), nullable=False, index=True)
+ sg_name = Column(String(512), nullable=False)
+ sg_mode = Column(String(10), nullable=False)
+ sg_12m_hours = Column(Numeric(8, 2))
+ sg_aca_hours = Column(Numeric(8, 2))
+ sg_7m_hours = Column(Numeric(8, 2))
+ sg_tp_hours = Column(Numeric(8, 2))
+
+ proposal = relationship('Ph1mProposal')
+
+
+class Ph1mRecommScienceGoal(Base):
+ __tablename__ = 'ph1m_recomm_science_goal'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ recommendation_id = Column(ForeignKey('ALMA.ph1m_recommendation.id'), nullable=False)
+ science_goal_name = Column(String(512))
+ descope = Column(String(1))
+ proposal_science_goal_id = Column(ForeignKey('ALMA.ph1m_proposal_science_goal.id'), index=True)
+
+ proposal_science_goal = relationship('Ph1mProposalScienceGoal')
+ recommendation = relationship('Ph1mRecommendation')
+
+
+class Ph1mRecommendation(Base):
+ __tablename__ = 'ph1m_recommendation'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ recommendation_type = Column(String(32), nullable=False)
+ aprc_flag = Column(String(1), server_default=text("null"))
+ dc_flag = Column(String(1), server_default=text("'F'"))
+ proposal_id = Column(ForeignKey('ALMA.ph1m_proposal.id'))
+ update_date = Column(DateTime)
+ update_userid = Column(String(32))
+ duplicate_proposal_id = Column(ForeignKey('ALMA.ph1m_proposal.id'))
+ arc_seconds = Column(Numeric(8, 2))
+ text = Column(String(4000))
+ text_from_aprc = Column(String(4000))
+ prop_recom_uid = Column(Numeric(19, 0, asdecimal=False))
+
+ duplicate_proposal = relationship('Ph1mProposal', primaryjoin='Ph1mRecommendation.duplicate_proposal_id == Ph1mProposal.id')
+ proposal = relationship('Ph1mProposal', primaryjoin='Ph1mRecommendation.proposal_id == Ph1mProposal.id')
+
+
+t_piv_account = Table(
+ 'piv_account', metadata,
+ Column('account_id', String(32), nullable=False),
+ Column('request_handler_id', Numeric(22, 0, asdecimal=False)),
+ Column('firstname', String(256), nullable=False),
+ Column('lastname', String(256), nullable=False),
+ Column('initials', String(256)),
+ Column('preferredarc', String(32)),
+ Column('email', String(256)),
+ Column('executive', String(5), nullable=False),
+ Column('instno', Numeric(38, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+t_piv_execblock = Table(
+ 'piv_execblock', metadata,
+ Column('execblockuid', String(32), nullable=False),
+ Column('qa0status', String(32), nullable=False),
+ Column('aquastarttime', DateTime),
+ Column('aquaendtime', DateTime),
+ Column('starttime', DateTime),
+ Column('endtime', DateTime),
+ Column('schedblockuid', String(32)),
+ Column('obsprojectpi', String(20)),
+ Column('representativefrequency', Numeric(15, 10)),
+ Column('obsprojectuid', String(32)),
+ schema='ALMA'
+)
+
+
+t_piv_obs_project = Table(
+ 'piv_obs_project', metadata,
+ Column('code', String(64)),
+ Column('pi', String(64)),
+ Column('contact_account_id', String(64)),
+ Column('firstname', String(256)),
+ Column('lastname', String(256)),
+ Column('contact_firstname', String(256)),
+ Column('contact_lastname', String(256)),
+ Column('title', String(256)),
+ Column('prj_version', String(32)),
+ Column('prj_letter_grade', String(2)),
+ Column('prj_grade', String(1)),
+ Column('rank', Numeric(8, 0, asdecimal=False)),
+ Column('obs_project_id', String(64), nullable=False),
+ Column('domain_entity_state', String(32), nullable=False),
+ schema='ALMA'
+)
+
+
+t_piv_obs_proposals = Table(
+ 'piv_obs_proposals', metadata,
+ Column('code', String(64)),
+ Column('title', String(256)),
+ Column('creation_date', String(23)),
+ Column('priority_flag', String(2)),
+ Column('associatedexec', String(32)),
+ Column('scientific_category', String(32)),
+ Column('scientific_category_string', String(4000)),
+ Column('abstract_text', Text),
+ Column('cycle', String(32)),
+ Column('proposal_type', String(32)),
+ Column('projectuid', String(32), nullable=False),
+ Column('archiveuid', String(33), nullable=False),
+ Column('consensus_report', String(4000)),
+ Column('consensus_report1', String(4000)),
+ Column('project_state', String(32), nullable=False),
+ schema='ALMA'
+)
+
+
+t_piv_obs_unit_set = Table(
+ 'piv_obs_unit_set', metadata,
+ Column('obs_project_id', String(64), nullable=False),
+ Column('status_entity_id', String(64), nullable=False),
+ Column('parent_obs_unit_set_status_id', String(64)),
+ Column('domain_entity_state', String(32), nullable=False),
+ Column('partid', String(64)),
+ Column('name', String(256)),
+ Column('requestedarray', String(8)),
+ Column('qa2status', String(8)),
+ schema='ALMA'
+)
+
+
+t_piv_proposal_authors = Table(
+ 'piv_proposal_authors', metadata,
+ Column('account_id', String(32), nullable=False),
+ Column('firstname', String(256), nullable=False),
+ Column('lastname', String(256), nullable=False),
+ Column('initials', String(256)),
+ Column('sequence', Numeric(22, 0, asdecimal=False), nullable=False),
+ Column('author_type', String(4), nullable=False),
+ Column('project_uid', String(32), nullable=False),
+ schema='ALMA'
+)
+
+
+t_piv_sched_block = Table(
+ 'piv_sched_block', metadata,
+ Column('obs_project_id', String(64), nullable=False),
+ Column('status_entity_id', String(64), nullable=False),
+ Column('parent_obs_unit_set_status_id', String(64), nullable=False),
+ Column('domain_entity_state', String(32), nullable=False),
+ Column('flags', String(200)),
+ Column('sched_block_uid', String(64), nullable=False),
+ Column('sb_name', String(32)),
+ Column('totalestimatedtime', String(32)),
+ Column('totalestimatedtimeunits', String(32)),
+ Column('ref_ra', String(32)),
+ Column('ref_dec', String(32)),
+ Column('execution_count', String(32)),
+ Column('qa2status', String(8)),
+ Column('nominalconfiguration', String(32)),
+ schema='ALMA'
+)
+
+
+t_piv_science_goal = Table(
+ 'piv_science_goal', metadata,
+ Column('project_uid', String(33), nullable=False),
+ Column('project_code', String(64)),
+ Column('science_goal_name', String(256)),
+ Column('ous_part_id', String(256)),
+ Column('ous_status_uid', String(64)),
+ Column('sb_uid', String(64)),
+ schema='ALMA'
+)
+
+
+t_piv_science_goal_ous_subview = Table(
+ 'piv_science_goal_ous_subview', metadata,
+ Column('ous_status_uid', String(64), nullable=False),
+ Column('project_uid', String(64), nullable=False),
+ Column('ous_part_id', String(64)),
+ Column('sb_uid', String(64), nullable=False),
+ schema='ALMA'
+)
+
+
+t_piv_science_goal_subview = Table(
+ 'piv_science_goal_subview', metadata,
+ Column('project_uid', String(33), nullable=False),
+ Column('project_code', String(64)),
+ Column('science_goal_name', String(256)),
+ Column('ous_part_id', String(256)),
+ schema='ALMA'
+)
+
+
+t_piv_state_changes = Table(
+ 'piv_state_changes', metadata,
+ Column('state_changes_id', Numeric(32, 0, asdecimal=False), nullable=False),
+ Column('status_entity_id', String(64), nullable=False),
+ Column('domain_entity_id', String(64), nullable=False),
+ Column('domain_entity_state', String(32), nullable=False),
+ Column('timestamp', DateTime, nullable=False),
+ Column('entity_type', String(3), nullable=False),
+ Column('domain_part_id', String(64)),
+ Column('child_sb_uid', String(64)),
+ schema='ALMA'
+)
+
+
+class PrjAttachment(Base):
+ __tablename__ = 'prj_attachment'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ obs_project_archive_uid = Column(String(64), nullable=False)
+ auth_account_id = Column(String(64), nullable=False)
+ created = Column(DateTime, nullable=False)
+ filename = Column(String(256), nullable=False)
+ contents = Column(LargeBinary, nullable=False)
+
+
+class PrjComment(Base):
+ __tablename__ = 'prj_comment'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ obs_project_archive_uid = Column(String(64), nullable=False)
+ auth_account_id = Column(String(64), nullable=False)
+ created = Column(DateTime, nullable=False)
+ updated = Column(DateTime)
+ comment_text = Column(String(3500), nullable=False)
+
+
+class PrjOperation(Base):
+ __tablename__ = 'prj_operations'
+ __table_args__ = {'schema': 'ALMA'}
+
+ obs_project_archive_uid = Column(String(64), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ p2g_account_id = Column(String(64))
+ contact_account_id = Column(String(64))
+ scops_ticket = Column(String(8))
+ recipients = Column(String(512))
+ helpdesk_ticket = Column(String(8))
+
+
+class ProjectCode(Base):
+ __tablename__ = 'project_codes'
+ __table_args__ = {'schema': 'ALMA'}
+
+ year = Column(String(4), primary_key=True, nullable=False)
+ period = Column(String(64), primary_key=True, nullable=False)
+ sequence = Column(Numeric(asdecimal=False), nullable=False)
+ type_code = Column(String(3), primary_key=True, nullable=False)
+
+
+t_project_codes_bkp = Table(
+ 'project_codes_bkp', metadata,
+ Column('year', String(4), nullable=False),
+ Column('period', String(64), nullable=False),
+ Column('sequence', Numeric(asdecimal=False), nullable=False),
+ Column('type_code', String(3), nullable=False),
+ schema='ALMA'
+)
+
+
+t_proposal = Table(
+ 'proposal', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('cycle_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('archive_uid', String(255), nullable=False),
+ Column('panel_id_back', Numeric(38, 0, asdecimal=False)),
+ Column('cancelled', String(1)),
+ Column('new_integration_time_hours', Numeric(8, 2)),
+ Column('new_scientific_category', String(16)),
+ Column('needs_more_time', String(1)),
+ Column('arp_score', Float),
+ Column('arp_avg_score', Float),
+ Column('arp_std_dev_scores', Float),
+ Column('aprc_letter_grade', String(1)),
+ Column('aprc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('aprc_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('arp_work_progress', String(10)),
+ Column('aprc_work_progress', String(10)),
+ Column('feasible', String(8)),
+ Column('dc_score', Float),
+ Column('dc_letter_grade', String(1)),
+ Column('dc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('dc_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('dc_rank_changed', String(1)),
+ Column('dc_work_progress', String(10)),
+ Column('arp_rank', Numeric(scale=0, asdecimal=False)),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ Column('triage_flag', String(1)),
+ Column('arp_vote_avg', Float),
+ Column('arp_vote_std_dev', Float),
+ Column('arp_nr_votes', Numeric(scale=0, asdecimal=False)),
+ Column('arp_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('arp_rank_manually_changed', String(1)),
+ Column('aprc_rank_manually_changed', String(1)),
+ Column('arp_score_normalized', Float),
+ Column('arp_rank_normalized', Float),
+ Column('duplication_flags', String(250)),
+ Column('arp_comment', String(4000)),
+ Column('aprc_comment', String(4000)),
+ Column('new_time_set_by_sg_descope', String(1)),
+ Column('arp_prop_strength', String(4000)),
+ Column('arp_prop_weakness', String(4000)),
+ Column('arp_prop_improvement', String(4000)),
+ Column('arp_prop_evaluation', String(4000)),
+ Column('aprc_prop_strength', String(4000)),
+ Column('aprc_prop_weakness', String(4000)),
+ Column('aprc_prop_improvement', String(4000)),
+ Column('aprc_prop_evaluation', String(4000)),
+ Column('ta_comment_to_pi', String(4000)),
+ Column('arp_grouping_flag', Numeric(2, 0, asdecimal=False)),
+ Column('non_standard_twelve_m_hours', Numeric(8, 2)),
+ schema='ALMA'
+)
+
+
+t_proposal12_back = Table(
+ 'proposal12_back', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('cycle_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('archive_uid', String(255), nullable=False),
+ Column('alma_review_panel_id', Numeric(38, 0, asdecimal=False)),
+ Column('cancelled', String(1)),
+ Column('new_integration_time_hours', Numeric(8, 2)),
+ Column('new_scientific_category', String(16)),
+ Column('needs_more_time', String(1)),
+ Column('arp_score', Float),
+ Column('arp_comment', Text),
+ Column('arp_avg_score', Float),
+ Column('arp_std_dev_scores', Float),
+ Column('aprc_comment', Text),
+ Column('aprc_letter_grade', String(1)),
+ Column('aprc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('aprc_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('arp_work_progress', String(10)),
+ Column('aprc_work_progress', String(10)),
+ Column('feasible', String(8)),
+ Column('dc_score', Float),
+ Column('dc_letter_grade', String(1)),
+ Column('dc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('dc_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('dc_rank_changed', String(1)),
+ Column('dc_work_progress', String(10)),
+ Column('arp_rank', Numeric(scale=0, asdecimal=False)),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ Column('triage_flag', String(1)),
+ Column('arp_vote_avg', Float),
+ Column('arp_vote_std_dev', Float),
+ Column('arp_nr_votes', Numeric(scale=0, asdecimal=False)),
+ Column('arp_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('arp_rank_manually_changed', String(1)),
+ Column('aprc_rank_manually_changed', String(1)),
+ Column('arp_score_normalized', Float),
+ Column('arp_rank_normalized', Float),
+ Column('duplication_flags', String(250)),
+ schema='ALMA'
+)
+
+
+t_proposal_author = Table(
+ 'proposal_author', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('proposal_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('sequence', Numeric(scale=0, asdecimal=False), nullable=False),
+ Column('userid', String(32), nullable=False),
+ Column('institution_id', Numeric(asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_proposal_cycle2_backup = Table(
+ 'proposal_cycle2_backup', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('cycle_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('archive_uid', String(255), nullable=False),
+ Column('alma_review_panel_id', Numeric(38, 0, asdecimal=False)),
+ Column('cancelled', String(1)),
+ Column('new_integration_time_hours', Numeric(8, 2)),
+ Column('new_scientific_category', String(16)),
+ Column('needs_more_time', String(1)),
+ Column('arp_score', Float),
+ Column('arp_comment', Text),
+ Column('arp_avg_score', Float),
+ Column('arp_std_dev_scores', Float),
+ Column('aprc_comment', Text),
+ Column('aprc_letter_grade', String(1)),
+ Column('aprc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('aprc_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('arp_work_progress', String(10)),
+ Column('aprc_work_progress', String(10)),
+ Column('feasible', String(8)),
+ Column('dc_score', Float),
+ Column('dc_letter_grade', String(1)),
+ Column('dc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('dc_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('dc_rank_changed', String(1)),
+ Column('dc_work_progress', String(10)),
+ Column('arp_rank', Numeric(scale=0, asdecimal=False)),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ Column('triage_flag', String(1)),
+ Column('arp_vote_avg', Float),
+ Column('arp_vote_std_dev', Float),
+ Column('arp_nr_votes', Numeric(scale=0, asdecimal=False)),
+ Column('arp_original_rank', Numeric(scale=0, asdecimal=False)),
+ Column('arp_rank_manually_changed', String(1)),
+ Column('aprc_rank_manually_changed', String(1)),
+ Column('arp_score_normalized', Float),
+ Column('arp_rank_normalized', Float),
+ Column('duplication_flags', String(250)),
+ schema='ALMA'
+)
+
+
+t_recommendation = Table(
+ 'recommendation', metadata,
+ Column('id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('version', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('recommendation_type', String(32), nullable=False),
+ Column('aprc_flag', String(1)),
+ Column('dc_flag', String(1)),
+ Column('proposal_id', Numeric(38, 0, asdecimal=False)),
+ Column('update_date', DateTime),
+ Column('update_userid', String(32)),
+ Column('duplicate_proposal_id', Numeric(38, 0, asdecimal=False)),
+ Column('arc_seconds', Numeric(8, 2)),
+ Column('text', String(4000)),
+ Column('text_from_aprc', String(4000)),
+ schema='ALMA'
+)
+
+
+class Role(Base):
+ __tablename__ = 'role'
+ __table_args__ = (
+ Index('role_name_unique', 'application', 'name', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ role_no = Column(Numeric(38, 0, asdecimal=False), primary_key=True)
+ version = Column(Numeric(38, 0, asdecimal=False), nullable=False)
+ application = Column(String(32), nullable=False)
+ name = Column(String(32), nullable=False)
+ parent_role = Column(ForeignKey('ALMA.role.role_no'))
+
+ parent = relationship('Role', remote_side=[role_no])
+
+
+class SchedBlockStatu(Base):
+ __tablename__ = 'sched_block_status'
+ __table_args__ = (
+ Index('sched_block_status_idx', 'obs_project_id', 'domain_entity_id'),
+ {'schema': 'ALMA'}
+ )
+
+ status_entity_id = Column(String(64), primary_key=True)
+ domain_entity_id = Column(ForeignKey('ALMA.xml_schedblock_entities.archive_uid'), nullable=False, index=True)
+ domain_entity_state = Column(String(32), nullable=False)
+ parent_obs_unit_set_status_id = Column(String(64), nullable=False, index=True)
+ obs_project_status_id = Column(String(64), nullable=False)
+ obs_project_id = Column(ForeignKey('ALMA.xml_obsproject_entities.archive_uid'), nullable=False)
+ total_required_time_in_sec = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ total_used_time_in_sec = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ xml = Column(NullType)
+ flags = Column(String(200), index=True)
+
+ domain_entity = relationship('XmlSchedblockEntity')
+ obs_project = relationship('XmlObsprojectEntity')
+
+
+class SchemaVersion(Base):
+ __tablename__ = 'schema_version'
+ __table_args__ = {'schema': 'ALMA'}
+
+ name = Column(String(32), primary_key=True)
+ value = Column(String(32), nullable=False)
+
+
+class ShiftlogEntry(Base):
+ __tablename__ = 'shiftlog_entries'
+ __table_args__ = {'schema': 'ALMA'}
+
+ se_id = Column(Numeric(19, 0, asdecimal=False), primary_key=True)
+ se_type = Column(Numeric(10, 0, asdecimal=False), nullable=False, index=True)
+ se_subject = Column(String(256))
+ se_timestamp = Column(DateTime, nullable=False, unique=True)
+ se_author = Column(String(32))
+ se_start = Column(DateTime, index=True)
+ se_project_code = Column(String(128), index=True)
+ se_sb_code = Column(String(128))
+ se_sb_id = Column(String(32), index=True)
+ se_eb_uid = Column(String(32))
+ se_location = Column(String(32), index=True)
+ se_status = Column(String(32))
+ se_calibration = Column(String(1))
+ se_qa0flag = Column(String(32))
+ se_archiving_status = Column(String(32))
+ se_test_activity = Column(String(64))
+ se_ispowercut = Column(String(1))
+ se_pcrecoveryend = Column(DateTime)
+ se_pcrecoverystart = Column(DateTime)
+ se_wrecoveryend12m = Column(DateTime)
+ se_wrecoveryend7m = Column(DateTime)
+ se_wrecoveryendtp = Column(DateTime)
+ se_wrecoverystart12m = Column(DateTime)
+ se_wrecoverystart7m = Column(DateTime)
+ se_wrecoverystarttp = Column(DateTime)
+ se_arrayentry_id = Column(ForeignKey('ALMA.shiftlog_entries.se_id'))
+ se_arrayname = Column(String(10))
+ se_arraytype = Column(String(9))
+ se_arrayfamily = Column(String(11))
+ se_correlatortype = Column(String(4))
+ se_photonicreferencename = Column(String(18))
+ se_almabuild = Column(String(70))
+ se_downtimetype = Column(String(9))
+ se_mainactivity = Column(String(100))
+ se_mainantennaname = Column(String(50))
+ se_mainresponsible = Column(String(50))
+ se_bandname = Column(String(10))
+ se_executive = Column(String(23))
+ se_obsprojectname = Column(String(256))
+ se_obsprojectpi = Column(String(20))
+ se_obsprojectversion = Column(String(20))
+ se_acsversion = Column(String(12))
+ se_shiftactivity = Column(String(11))
+ se_dashboardantennasavailable = Column(String(2))
+ se_dashboardantennasdelivered = Column(String(2))
+ se_dashboardantennaspresent = Column(String(2))
+ se_pwv = Column(Numeric(18, 10))
+ se_reprfrequency = Column(Numeric(15, 10))
+ se_test = Column(String(1))
+
+ se_arrayentry = relationship('ShiftlogEntry', remote_side=[se_id])
+
+
+t_shiftlog_entries_bak = Table(
+ 'shiftlog_entries_bak', metadata,
+ Column('se_id', Numeric(19, 0, asdecimal=False)),
+ Column('se_type', Numeric(10, 0, asdecimal=False), nullable=False),
+ Column('se_subject', String(256)),
+ Column('se_timestamp', DateTime, nullable=False),
+ Column('se_author', String(32)),
+ Column('se_start', DateTime),
+ Column('se_project_code', String(128)),
+ Column('se_sb_code', String(128)),
+ Column('se_sb_id', String(32)),
+ Column('se_eb_uid', String(32)),
+ Column('se_location', String(32)),
+ Column('se_status', String(32)),
+ Column('se_calibration', String(1)),
+ Column('se_qa0flag', String(10)),
+ Column('se_archiving_status', String(32)),
+ Column('se_test_activity', String(64)),
+ Column('se_ispowercut', String(1)),
+ Column('se_pcrecoveryend', DateTime),
+ Column('se_pcrecoverystart', DateTime),
+ Column('se_wrecoveryend12m', DateTime),
+ Column('se_wrecoveryend7m', DateTime),
+ Column('se_wrecoveryendtp', DateTime),
+ Column('se_wrecoverystart12m', DateTime),
+ Column('se_wrecoverystart7m', DateTime),
+ Column('se_wrecoverystarttp', DateTime),
+ Column('se_arrayentry_id', Numeric(10, 0, asdecimal=False)),
+ Column('se_arrayname', String(10)),
+ Column('se_arraytype', String(9)),
+ Column('se_arrayfamily', String(11)),
+ Column('se_correlatortype', String(4)),
+ Column('se_photonicreferencename', String(18)),
+ Column('se_almabuild', String(70)),
+ Column('se_downtimetype', String(9)),
+ Column('se_mainactivity', String(100)),
+ Column('se_mainantennaname', String(50)),
+ Column('se_mainresponsible', String(50)),
+ Column('se_bandname', String(10)),
+ Column('se_executive', String(20)),
+ Column('se_obsprojectname', String(256)),
+ Column('se_obsprojectpi', String(20)),
+ Column('se_obsprojectversion', String(20)),
+ Column('se_acsversion', String(12)),
+ Column('se_shiftactivity', String(11)),
+ Column('se_dashboardantennasavailable', String(2)),
+ Column('se_dashboardantennasdelivered', String(2)),
+ Column('se_dashboardantennaspresent', String(2)),
+ Column('se_pwv', Numeric(18, 10)),
+ Column('se_reprfrequency', Numeric(15, 10)),
+ schema='ALMA'
+)
+
+
+class ShiftlogReply(Base):
+ __tablename__ = 'shiftlog_reply'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(19, 0, asdecimal=False), primary_key=True)
+ reply_comment = Column(Text)
+ timestamp = Column(DateTime)
+ author = Column(String(32))
+ entry_id = Column(ForeignKey('ALMA.shiftlog_entries.se_id'), nullable=False, index=True)
+
+ entry = relationship('ShiftlogEntry')
+
+
+t_shiftlog_reply_bak = Table(
+ 'shiftlog_reply_bak', metadata,
+ Column('id', Numeric(19, 0, asdecimal=False)),
+ Column('reply_comment', Text),
+ Column('timestamp', DateTime),
+ Column('author', String(32)),
+ Column('entry_id', Numeric(19, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+class ShiftlogSbUsedTime(Base):
+ __tablename__ = 'shiftlog_sb_used_time'
+ __table_args__ = {'schema': 'ALMA'}
+
+ sb_uid = Column(String(33), primary_key=True)
+ used_time = Column(Numeric(asdecimal=False), nullable=False)
+
+
+t_shiftlog_sb_used_time_bak = Table(
+ 'shiftlog_sb_used_time_bak', metadata,
+ Column('sb_uid', String(33), nullable=False),
+ Column('used_time', Numeric(asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+class SlogAllocatedTimeInterval(Base):
+ __tablename__ = 'slog_allocated_time_interval'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(10, 0, asdecimal=False), primary_key=True)
+ entry_id = Column(ForeignKey('ALMA.shiftlog_entries.se_id'), nullable=False)
+ arrayfamily = Column(String(11), nullable=False)
+ starttime = Column(DateTime, nullable=False)
+ endtime = Column(DateTime)
+
+ entry = relationship('ShiftlogEntry')
+
+
+class SlogEntryAttach(Base):
+ __tablename__ = 'slog_entry_attach'
+ __table_args__ = {'schema': 'ALMA'}
+
+ slog_attach_id = Column(Numeric(19, 0, asdecimal=False), primary_key=True)
+ slog_attach_name = Column(String(256))
+ slog_attach_contents = Column(LargeBinary)
+ slog_se_id = Column(ForeignKey('ALMA.shiftlog_entries.se_id'), nullable=False, index=True)
+
+ slog_se = relationship('ShiftlogEntry')
+
+
+t_slog_entry_attach_bak = Table(
+ 'slog_entry_attach_bak', metadata,
+ Column('slog_attach_id', Numeric(19, 0, asdecimal=False)),
+ Column('slog_attach_name', String(256)),
+ Column('slog_attach_contents', LargeBinary),
+ Column('slog_se_id', Numeric(19, 0, asdecimal=False), nullable=False),
+ schema='ALMA'
+)
+
+
+class SlogEntryAttr(Base):
+ __tablename__ = 'slog_entry_attr'
+ __table_args__ = {'schema': 'ALMA'}
+
+ slog_attr_id = Column(Numeric(19, 0, asdecimal=False), primary_key=True)
+ slog_attr_type = Column(Numeric(10, 0, asdecimal=False), nullable=False)
+ slog_attr_value = Column(String(256))
+ slog_se_id = Column(ForeignKey('ALMA.shiftlog_entries.se_id'), index=True)
+
+ slog_se = relationship('ShiftlogEntry')
+
+
+t_slog_entry_attr_bak = Table(
+ 'slog_entry_attr_bak', metadata,
+ Column('slog_attr_id', Numeric(19, 0, asdecimal=False), nullable=False),
+ Column('slog_attr_type', Numeric(10, 0, asdecimal=False), nullable=False),
+ Column('slog_attr_value', String(256)),
+ Column('slog_se_id', Numeric(19, 0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+class StateChange(Base):
+ __tablename__ = 'state_changes'
+ __table_args__ = (
+ Index('id_perf_schs', 'status_entity_id', 'domain_entity_state'),
+ {'schema': 'ALMA'}
+ )
+
+ state_changes_id = Column(Numeric(32, 0, asdecimal=False), primary_key=True)
+ status_entity_id = Column(String(64), nullable=False, index=True)
+ domain_entity_id = Column(String(64), nullable=False, index=True)
+ domain_entity_state = Column(String(32), nullable=False)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ location = Column(String(3), nullable=False)
+ user_id = Column(String(64), nullable=False)
+ subsystem = Column(String(32), nullable=False)
+ info = Column(String(2000))
+ entity_type = Column(String(3), nullable=False)
+ domain_part_id = Column(String(64))
+ flags = Column(String(200), index=True)
+
+
+t_submission_service_account = Table(
+ 'submission_service_account', metadata,
+ Column('account_id', String(32), nullable=False),
+ Column('password_digest', String(256)),
+ schema='ALMA'
+)
+
+
+t_submission_service_roles = Table(
+ 'submission_service_roles', metadata,
+ Column('account_id', String(32)),
+ Column('name', String(66)),
+ schema='ALMA'
+)
+
+
+t_submission_service_roles_orig = Table(
+ 'submission_service_roles_orig', metadata,
+ Column('account_id', String(32), nullable=False),
+ Column('name', String(66)),
+ schema='ALMA'
+)
+
+
+t_test_23 = Table(
+ 'test_23', metadata,
+ Column('a', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_test_24 = Table(
+ 'test_24', metadata,
+ Column('a', Numeric(scale=0, asdecimal=False)),
+ schema='ALMA'
+)
+
+
+class TrustedAccount(Base):
+ __tablename__ = 'trusted_accounts'
+ __table_args__ = (
+ Index('trustedacc_unique', 'alma_account_id', 'trusted_account_id', 'trusted_institution', unique=True),
+ {'schema': 'ALMA'}
+ )
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ alma_account_id = Column(ForeignKey('ALMA.account.account_id'), nullable=False)
+ trusted_account_id = Column(String(32), nullable=False)
+ trusted_institution = Column(String(10), nullable=False)
+
+ alma_account = relationship('Account')
+
+
+t_uid_lookup = Table(
+ 'uid_lookup', metadata,
+ Column('archive_uid', String(33)),
+ Column('schemaname', String(30)),
+ schema='ALMA'
+)
+
+
+class UserDemographic(Base):
+ __tablename__ = 'user_demographics'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ version = Column(Numeric(asdecimal=False))
+ account_id = Column(ForeignKey('ALMA.account.account_id'), nullable=False)
+ observing_mm_submm = Column(String(32), nullable=False)
+ observing_ir = Column(String(32), nullable=False)
+ observing_radio = Column(String(32), nullable=False)
+ observing_theory_modeling = Column(String(32), nullable=False)
+ observing_xray = Column(String(32), nullable=False)
+ professional_status = Column(String(32), nullable=False)
+ techniques_interferometry = Column(String(32), nullable=False)
+ techniques_total_power = Column(String(32), nullable=False)
+ year_of_phd = Column(Numeric(scale=0, asdecimal=False), nullable=False)
+ last_update = Column(DateTime, nullable=False)
+ observing_uv_optical = Column(String(32), nullable=False)
+
+ account = relationship('Account')
+
+
+t_v_region_props = Table(
+ 'v_region_props', metadata,
+ Column('v_proposal_id', Numeric(38, 0, asdecimal=False), nullable=False),
+ Column('v_pi_executive', String(32)),
+ Column('v_aprc_letter_grade', String(1)),
+ Column('v_aprc_rank', Numeric(scale=0, asdecimal=False)),
+ Column('v_arp_score', Float),
+ Column('regional_rank', Numeric(asdecimal=False)),
+ Column('nr_props_region', Numeric(asdecimal=False)),
+ schema='ALMA'
+)
+
+
+t_v_schedblock_polarisation = Table(
+ 'v_schedblock_polarisation', metadata,
+ Column('archive_uid', String(33), nullable=False),
+ Column('products', String(32), nullable=False),
+ schema='ALMA'
+)
+
+
+t_watchdog = Table(
+ 'watchdog', metadata,
+ Column('field', DateTime),
+ Column('site', String(5)),
+ schema='ALMA'
+)
+
+
+class XmlAcapolarizationEntity(Base):
+ __tablename__ = 'xml_acapolarization_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAccummodeEntity(Base):
+ __tablename__ = 'xml_accummode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAcsalarmmessageEntity(Base):
+ __tablename__ = 'xml_acsalarmmessage_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAcscommandcenterpEntity(Base):
+ __tablename__ = 'xml_acscommandcenterp_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAcscommandcentertEntity(Base):
+ __tablename__ = 'xml_acscommandcentert_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAcserrorEntity(Base):
+ __tablename__ = 'xml_acserror_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAcslogtsEntity(Base):
+ __tablename__ = 'xml_acslogts_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAddressEntity(Base):
+ __tablename__ = 'xml_address_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAlarmsystemstatsEntity(Base):
+ __tablename__ = 'xml_alarmsystemstats_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAlmaradiometertabEntity(Base):
+ __tablename__ = 'xml_almaradiometertab_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAnnotationtableEntity(Base):
+ __tablename__ = 'xml_annotationtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAntennamakeEntity(Base):
+ __tablename__ = 'xml_antennamake_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAntennamotionpattEntity(Base):
+ __tablename__ = 'xml_antennamotionpatt_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAntennatableEntity(Base):
+ __tablename__ = 'xml_antennatable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAntennatypeEntity(Base):
+ __tablename__ = 'xml_antennatype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAsdmEntity(Base):
+ __tablename__ = 'xml_asdm_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAsdmbinarytableEntity(Base):
+ __tablename__ = 'xml_asdmbinarytable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAsiconfigurationEntity(Base):
+ __tablename__ = 'xml_asiconfiguration_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAsimessageEntity(Base):
+ __tablename__ = 'xml_asimessage_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAssociatedcalnatuEntity(Base):
+ __tablename__ = 'xml_associatedcalnatu_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAssociatedfieldnaEntity(Base):
+ __tablename__ = 'xml_associatedfieldna_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAtmphasecorrectioEntity(Base):
+ __tablename__ = 'xml_atmphasecorrectio_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlAxisnameEntity(Base):
+ __tablename__ = 'xml_axisname_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlBasebandnameEntity(Base):
+ __tablename__ = 'xml_basebandname_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlBaselinereferenceEntity(Base):
+ __tablename__ = 'xml_baselinereference_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlBeamtableEntity(Base):
+ __tablename__ = 'xml_beamtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlBinarydataflagsEntity(Base):
+ __tablename__ = 'xml_binarydataflags_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlBulktestEntity(Base):
+ __tablename__ = 'xml_bulktest_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalamplitableEntity(Base):
+ __tablename__ = 'xml_calamplitable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalappphasetableEntity(Base):
+ __tablename__ = 'xml_calappphasetable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalatmospheretablEntity(Base):
+ __tablename__ = 'xml_calatmospheretabl_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalbandpasstableEntity(Base):
+ __tablename__ = 'xml_calbandpasstable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalcurvetableEntity(Base):
+ __tablename__ = 'xml_calcurvetable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalcurvetypeEntity(Base):
+ __tablename__ = 'xml_calcurvetype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCaldataoriginEntity(Base):
+ __tablename__ = 'xml_caldataorigin_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCaldatatableEntity(Base):
+ __tablename__ = 'xml_caldatatable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCaldelaytableEntity(Base):
+ __tablename__ = 'xml_caldelaytable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCaldevicetableEntity(Base):
+ __tablename__ = 'xml_caldevicetable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalfluxtableEntity(Base):
+ __tablename__ = 'xml_calfluxtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalfocusmodeltablEntity(Base):
+ __tablename__ = 'xml_calfocusmodeltabl_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalfocustableEntity(Base):
+ __tablename__ = 'xml_calfocustable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalgaintableEntity(Base):
+ __tablename__ = 'xml_calgaintable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalholographytablEntity(Base):
+ __tablename__ = 'xml_calholographytabl_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalibrationconfigEntity(Base):
+ __tablename__ = 'xml_calibrationconfig_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalibrationdeviceEntity(Base):
+ __tablename__ = 'xml_calibrationdevice_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalibrationfunctiEntity(Base):
+ __tablename__ = 'xml_calibrationfuncti_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalibrationmodeEntity(Base):
+ __tablename__ = 'xml_calibrationmode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalibrationsetEntity(Base):
+ __tablename__ = 'xml_calibrationset_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalphasetableEntity(Base):
+ __tablename__ = 'xml_calphasetable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalpointingmodeltEntity(Base):
+ __tablename__ = 'xml_calpointingmodelt_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalpointingtableEntity(Base):
+ __tablename__ = 'xml_calpointingtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalpositiontableEntity(Base):
+ __tablename__ = 'xml_calpositiontable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalprimarybeamtabEntity(Base):
+ __tablename__ = 'xml_calprimarybeamtab_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalqueryparameterEntity(Base):
+ __tablename__ = 'xml_calqueryparameter_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalreductiontableEntity(Base):
+ __tablename__ = 'xml_calreductiontable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalseeingtableEntity(Base):
+ __tablename__ = 'xml_calseeingtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCaltypeEntity(Base):
+ __tablename__ = 'xml_caltype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCalwvrtableEntity(Base):
+ __tablename__ = 'xml_calwvrtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCommonentityEntity(Base):
+ __tablename__ = 'xml_commonentity_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCommontypesEntity(Base):
+ __tablename__ = 'xml_commontypes_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlConfigdescriptionEntity(Base):
+ __tablename__ = 'xml_configdescription_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCorrelationbitEntity(Base):
+ __tablename__ = 'xml_correlationbit_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCorrelationmodeEntity(Base):
+ __tablename__ = 'xml_correlationmode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCorrelatorcalibraEntity(Base):
+ __tablename__ = 'xml_correlatorcalibra_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCorrelatormodetabEntity(Base):
+ __tablename__ = 'xml_correlatormodetab_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCorrelatornameEntity(Base):
+ __tablename__ = 'xml_correlatorname_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlCorrelatortypeEntity(Base):
+ __tablename__ = 'xml_correlatortype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDatacontentEntity(Base):
+ __tablename__ = 'xml_datacontent_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDatadescriptiontaEntity(Base):
+ __tablename__ = 'xml_datadescriptionta_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDatascaleEntity(Base):
+ __tablename__ = 'xml_datascale_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDelaymodelfixedpaEntity(Base):
+ __tablename__ = 'xml_delaymodelfixedpa_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDelaymodeltableEntity(Base):
+ __tablename__ = 'xml_delaymodeltable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDelaymodelvariablEntity(Base):
+ __tablename__ = 'xml_delaymodelvariabl_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDetectorbandtypeEntity(Base):
+ __tablename__ = 'xml_detectorbandtype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDifferencetypeEntity(Base):
+ __tablename__ = 'xml_differencetype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDirectionreferencEntity(Base):
+ __tablename__ = 'xml_directionreferenc_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDopplerreferencecEntity(Base):
+ __tablename__ = 'xml_dopplerreferencec_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDopplertableEntity(Base):
+ __tablename__ = 'xml_dopplertable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlDopplertrackingmoEntity(Base):
+ __tablename__ = 'xml_dopplertrackingmo_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlEphemeristableEntity(Base):
+ __tablename__ = 'xml_ephemeristable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlExecblocktableEntity(Base):
+ __tablename__ = 'xml_execblocktable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlExecconfigEntity(Base):
+ __tablename__ = 'xml_execconfig_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFeedtableEntity(Base):
+ __tablename__ = 'xml_feedtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFieldcodeEntity(Base):
+ __tablename__ = 'xml_fieldcode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFieldtableEntity(Base):
+ __tablename__ = 'xml_fieldtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFiltermodeEntity(Base):
+ __tablename__ = 'xml_filtermode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFlagcmdtableEntity(Base):
+ __tablename__ = 'xml_flagcmdtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFlagtableEntity(Base):
+ __tablename__ = 'xml_flagtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFluxcalibrationmeEntity(Base):
+ __tablename__ = 'xml_fluxcalibrationme_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFocusmethodEntity(Base):
+ __tablename__ = 'xml_focusmethod_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFocusmodeltableEntity(Base):
+ __tablename__ = 'xml_focusmodeltable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFocustableEntity(Base):
+ __tablename__ = 'xml_focustable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFreqoffsettableEntity(Base):
+ __tablename__ = 'xml_freqoffsettable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlFrequencyreferencEntity(Base):
+ __tablename__ = 'xml_frequencyreferenc_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlGaintrackingtableEntity(Base):
+ __tablename__ = 'xml_gaintrackingtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlHistorytableEntity(Base):
+ __tablename__ = 'xml_historytable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlHolographychannelEntity(Base):
+ __tablename__ = 'xml_holographychannel_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlHolographytableEntity(Base):
+ __tablename__ = 'xml_holographytable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlIdentifierrangeEntity(Base):
+ __tablename__ = 'xml_identifierrange_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlInvalidatingcondiEntity(Base):
+ __tablename__ = 'xml_invalidatingcondi_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlLoggingmiEntity(Base):
+ __tablename__ = 'xml_loggingmi_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlMaintableEntity(Base):
+ __tablename__ = 'xml_maintable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlMetaHistory(Base):
+ __tablename__ = 'xml_meta_history'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True, nullable=False)
+ timestamp = Column(DateTime, primary_key=True, nullable=False)
+ metacolumn = Column(Numeric(16, 0, asdecimal=False), primary_key=True, nullable=False)
+ newvalue = Column(String(64))
+
+
+t_xml_metainfo = Table(
+ 'xml_metainfo', metadata,
+ Column('name', String(32)),
+ Column('value', String(128)),
+ schema='ALMA'
+)
+
+
+t_xml_metainfo_backup = Table(
+ 'xml_metainfo_backup', metadata,
+ Column('name', String(32)),
+ Column('value', String(128)),
+ schema='ALMA'
+)
+
+
+class XmlNamespace(Base):
+ __tablename__ = 'xml_namespaces'
+ __table_args__ = {'schema': 'ALMA'}
+
+ prefix = Column(String(16), primary_key=True)
+ namespace = Column(String(128))
+
+
+class XmlNetsidebandEntity(Base):
+ __tablename__ = 'xml_netsideband_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObsattachmentEntity(Base):
+ __tablename__ = 'xml_obsattachment_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObservationtableEntity(Base):
+ __tablename__ = 'xml_observationtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObservingcontrolsEntity(Base):
+ __tablename__ = 'xml_observingcontrols_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObservingmodeEntity(Base):
+ __tablename__ = 'xml_observingmode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObsprojectEntity(Base):
+ __tablename__ = 'xml_obsproject_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObsproposalEntity(Base):
+ __tablename__ = 'xml_obsproposal_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObsreviewEntity(Base):
+ __tablename__ = 'xml_obsreview_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlObstooluserprefsEntity(Base):
+ __tablename__ = 'xml_obstooluserprefs_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlOtconfigurationEntity(Base):
+ __tablename__ = 'xml_otconfiguration_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlOusstatusEntity(Base):
+ __tablename__ = 'xml_ousstatus_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPointingmethodEntity(Base):
+ __tablename__ = 'xml_pointingmethod_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPointingmodelmodeEntity(Base):
+ __tablename__ = 'xml_pointingmodelmode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPointingmodeltablEntity(Base):
+ __tablename__ = 'xml_pointingmodeltabl_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPointingtableEntity(Base):
+ __tablename__ = 'xml_pointingtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPolarizationtableEntity(Base):
+ __tablename__ = 'xml_polarizationtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPolarizationtypeEntity(Base):
+ __tablename__ = 'xml_polarizationtype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPositionmethodEntity(Base):
+ __tablename__ = 'xml_positionmethod_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPositionreferenceEntity(Base):
+ __tablename__ = 'xml_positionreference_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPreferencesEntity(Base):
+ __tablename__ = 'xml_preferences_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPrimarybeamdescriEntity(Base):
+ __tablename__ = 'xml_primarybeamdescri_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPrimitivedatatypeEntity(Base):
+ __tablename__ = 'xml_primitivedatatype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlProcessorsubtypeEntity(Base):
+ __tablename__ = 'xml_processorsubtype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlProcessortableEntity(Base):
+ __tablename__ = 'xml_processortable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlProcessortypeEntity(Base):
+ __tablename__ = 'xml_processortype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlProjectstatusEntity(Base):
+ __tablename__ = 'xml_projectstatus_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPsetEntity(Base):
+ __tablename__ = 'xml_pset_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlPsetdefEntity(Base):
+ __tablename__ = 'xml_psetdef_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQlalarmsEntity(Base):
+ __tablename__ = 'xml_qlalarms_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQlatmospheresummaEntity(Base):
+ __tablename__ = 'xml_qlatmospheresumma_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQlfocussummaryEntity(Base):
+ __tablename__ = 'xml_qlfocussummary_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQlphasesummaryEntity(Base):
+ __tablename__ = 'xml_qlphasesummary_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQlpointingsummaryEntity(Base):
+ __tablename__ = 'xml_qlpointingsummary_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQuicklookdisplayEntity(Base):
+ __tablename__ = 'xml_quicklookdisplay_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQuicklookdisplayxEntity(Base):
+ __tablename__ = 'xml_quicklookdisplayx_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQuicklookresultEntity(Base):
+ __tablename__ = 'xml_quicklookresult_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlQuicklooksummaryEntity(Base):
+ __tablename__ = 'xml_quicklooksummary_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlRadialvelocityrefEntity(Base):
+ __tablename__ = 'xml_radialvelocityref_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlReceiverbandEntity(Base):
+ __tablename__ = 'xml_receiverband_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlReceiversidebandEntity(Base):
+ __tablename__ = 'xml_receiversideband_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlReceivertableEntity(Base):
+ __tablename__ = 'xml_receivertable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlRole(Base):
+ __tablename__ = 'xml_roles'
+ __table_args__ = {'schema': 'ALMA'}
+
+ rolename = Column(String(64), primary_key=True)
+
+
+class XmlSbstatusEntity(Base):
+ __tablename__ = 'xml_sbstatus_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSbsummarytableEntity(Base):
+ __tablename__ = 'xml_sbsummarytable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSbtypeEntity(Base):
+ __tablename__ = 'xml_sbtype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlScaletableEntity(Base):
+ __tablename__ = 'xml_scaletable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlScanintentEntity(Base):
+ __tablename__ = 'xml_scanintent_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlScantableEntity(Base):
+ __tablename__ = 'xml_scantable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSchedblockEntity(Base):
+ __tablename__ = 'xml_schedblock_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False), index=True)
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSchedulermodeEntity(Base):
+ __tablename__ = 'xml_schedulermode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSchedulingpolicyEntity(Base):
+ __tablename__ = 'xml_schedulingpolicy_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSchemaEntity(Base):
+ __tablename__ = 'xml_schema_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ schemaname = Column(String(32), nullable=False)
+ version = Column(Numeric(16, 0, asdecimal=False), nullable=False)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(Text)
+ schemauid = Column(String(33))
+ owner = Column(String(128))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(128))
+ writepermissions = Column(String(128))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSchemanamespace(Base):
+ __tablename__ = 'xml_schemanamespaces'
+ __table_args__ = {'schema': 'ALMA'}
+
+ schemauid = Column(String(33), primary_key=True, nullable=False)
+ prefix = Column(String(16), primary_key=True, nullable=False)
+
+
+class XmlScipiperequestEntity(Base):
+ __tablename__ = 'xml_scipiperequest_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlScipiperesultsEntity(Base):
+ __tablename__ = 'xml_scipiperesults_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSdmdataheaderEntity(Base):
+ __tablename__ = 'xml_sdmdataheader_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSeeingtableEntity(Base):
+ __tablename__ = 'xml_seeingtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSidebandprocessinEntity(Base):
+ __tablename__ = 'xml_sidebandprocessin_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSourcemodelEntity(Base):
+ __tablename__ = 'xml_sourcemodel_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSourcetableEntity(Base):
+ __tablename__ = 'xml_sourcetable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSpecialsbEntity(Base):
+ __tablename__ = 'xml_specialsb_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSpectralresolutioEntity(Base):
+ __tablename__ = 'xml_spectralresolutio_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSpectralwindowtabEntity(Base):
+ __tablename__ = 'xml_spectralwindowtab_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False, index=True)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSquarelawdetectorEntity(Base):
+ __tablename__ = 'xml_squarelawdetector_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlStatetableEntity(Base):
+ __tablename__ = 'xml_statetable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlStationtableEntity(Base):
+ __tablename__ = 'xml_stationtable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlStationtypeEntity(Base):
+ __tablename__ = 'xml_stationtype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlStokesparameterEntity(Base):
+ __tablename__ = 'xml_stokesparameter_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlStylesheet(Base):
+ __tablename__ = 'xml_stylesheets'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemaname = Column(String(32), nullable=False)
+ schemauid = Column(String(33), nullable=False)
+ version_from = Column(Numeric(16, 0, asdecimal=False))
+ version_to = Column(Numeric(16, 0, asdecimal=False))
+ alma_release = Column(String(32), nullable=False)
+
+
+class XmlSubscanfieldsourcEntity(Base):
+ __tablename__ = 'xml_subscanfieldsourc_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSubscanintentEntity(Base):
+ __tablename__ = 'xml_subscanintent_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSubscanspectralspEntity(Base):
+ __tablename__ = 'xml_subscanspectralsp_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSubscantableEntity(Base):
+ __tablename__ = 'xml_subscantable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSwitchcycletableEntity(Base):
+ __tablename__ = 'xml_switchcycletable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSwitchingmodeEntity(Base):
+ __tablename__ = 'xml_switchingmode_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSynthprofEntity(Base):
+ __tablename__ = 'xml_synthprof_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSyscalmethodEntity(Base):
+ __tablename__ = 'xml_syscalmethod_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSyscaltableEntity(Base):
+ __tablename__ = 'xml_syscaltable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlSyspowertableEntity(Base):
+ __tablename__ = 'xml_syspowertable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTestobsprojectEntity(Base):
+ __tablename__ = 'xml_testobsproject_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTestobsproposalEntity(Base):
+ __tablename__ = 'xml_testobsproposal_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTestprojectEntity(Base):
+ __tablename__ = 'xml_testproject_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTestschedblockEntity(Base):
+ __tablename__ = 'xml_testschedblock_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTestvaluetypesEntity(Base):
+ __tablename__ = 'xml_testvaluetypes_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTimesamplingEntity(Base):
+ __tablename__ = 'xml_timesampling_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTimescaleEntity(Base):
+ __tablename__ = 'xml_timescale_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlTotalpowertableEntity(Base):
+ __tablename__ = 'xml_totalpowertable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlUpdate(Base):
+ __tablename__ = 'xml_updates'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(asdecimal=False), primary_key=True)
+ asdm_uid = Column(ForeignKey('ALMA.xml_asdm_entities.archive_uid'), nullable=False)
+ timestamp = Column(DateTime, nullable=False)
+ modified_by = Column(String(32), nullable=False)
+ hostname = Column(String(32), nullable=False)
+ elaboration = Column(String(4000), nullable=False)
+ delta_version = Column(Numeric(5, 0, asdecimal=False), nullable=False)
+
+ xml_asdm_entity = relationship('XmlAsdmEntity')
+
+
+class XmlUserEntity(Base):
+ __tablename__ = 'xml_user_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlUserrole(Base):
+ __tablename__ = 'xml_userroles'
+ __table_args__ = {'schema': 'ALMA'}
+
+ username = Column(String(64), primary_key=True, nullable=False)
+ rolename = Column(String(64), primary_key=True, nullable=False)
+
+
+class XmlUser(Base):
+ __tablename__ = 'xml_users'
+ __table_args__ = {'schema': 'ALMA'}
+
+ username = Column(String(64), primary_key=True)
+
+
+class XmlValuetypesEntity(Base):
+ __tablename__ = 'xml_valuetypes_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlWeathertableEntity(Base):
+ __tablename__ = 'xml_weathertable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlWeighttypeEntity(Base):
+ __tablename__ = 'xml_weighttype_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlWindowfunctionEntity(Base):
+ __tablename__ = 'xml_windowfunction_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlWvmcaltableEntity(Base):
+ __tablename__ = 'xml_wvmcaltable_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class XmlWvrmethodEntity(Base):
+ __tablename__ = 'xml_wvrmethod_entities'
+ __table_args__ = {'schema': 'ALMA'}
+
+ archive_uid = Column(String(33), primary_key=True)
+ timestamp = Column(DateTime, nullable=False)
+ xml = Column(NullType)
+ schemauid = Column(String(33), nullable=False)
+ owner = Column(String(32))
+ deleted = Column(Numeric(1, 0, asdecimal=False))
+ readpermissions = Column(String(8))
+ writepermissions = Column(String(8))
+ hidden = Column(Numeric(1, 0, asdecimal=False))
+ dirty = Column(Numeric(1, 0, asdecimal=False))
+ virtual = Column(Numeric(1, 0, asdecimal=False))
+
+
+class Xslt(Base):
+ __tablename__ = 'xslt'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(Numeric(3, 0, asdecimal=False), primary_key=True)
+ t_xsl = Column(NullType)
+
+
+class AquaOusComment(AquaComment):
+ __tablename__ = 'aqua_ous_comment'
+ __table_args__ = {'schema': 'ALMA'}
+
+ commentid = Column(ForeignKey('ALMA.aqua_comment.commentid'), primary_key=True)
+ obsunitsetid = Column(ForeignKey('ALMA.aqua_ous.obsunitsetid'))
+ comment_section = Column(String(8))
+
+ aqua_ou = relationship('AquaOu')
+
+
+class AquaQa2ReasonHistory(AquaStatusHistory):
+ __tablename__ = 'aqua_qa2_reason_history'
+ __table_args__ = {'schema': 'ALMA'}
+
+ id = Column(ForeignKey('ALMA.aqua_status_history.id'), primary_key=True)
+ qa2reason = Column(String(500))
+
+
+class NgasFiles(Base):
+ __tablename__ = 'NGAS_FILES'
+ __table_args__ = {'schema': 'NGAS'}
+
+ file_id = Column(String(220), primary_key=True)
+ file_size = Column(Numeric(11, 0, asdecimal=True))
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/legacy_model.py b/apps/cli/executables/pexable/ingest/ingest/schema/legacy_model.py
new file mode 100644
index 000000000..7ba1264c9
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/legacy_model.py
@@ -0,0 +1,1728 @@
+# -*- coding: utf-8 -*-
+"""
+An incomplete declarative base model of the legacy archive
+Author: Richard Falardeau <rfalarde@nrao.edu>
+"""
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import CHAR, Column, DateTime, Float, Index, Integer, Numeric, String, Table, VARCHAR, text
+from sqlalchemy.dialects.oracle import NUMBER
+from sqlalchemy.sql.sqltypes import NullType
+
+# pylint: disable=invalid-name
+Base = declarative_base()
+metadata = Base.metadata
+
+
+# pylint: disable=too-few-public-methods
+class LegacyProject(Base):
+ r"""The model of the project table in the legacy db"""
+ __tablename__ = 'project'
+
+ project_code = Column(String(16), primary_key=True)
+ observer = Column(String(48))
+ observer_id = Column(Numeric)
+ starttime = Column(Numeric, primary_key=True)
+ stoptime = Column(Numeric, primary_key=True)
+ proprietary = Column(Numeric)
+ telescope = Column(String(12))
+ telescope_config = Column(String(36))
+ obs_bands = Column(String(24))
+ total_obs_time = Column(Numeric)
+ num_segments = Column(Numeric)
+ arch_files = Column(Numeric)
+ row_date = Column(Numeric)
+ raw_project_code = Column(String(16))
+ project_lock = Column(String(8))
+ unlock_expire = Column(Numeric)
+ grant_access = Column(String(12))
+ proprietary_duration = Column(Numeric)
+ warned = Column(Numeric)
+ project_id = Column(Numeric)
+
+ def __repr__(self):
+ r""" Return the string representation of this class.
+
+ :return: The string representation of self.
+ """
+ repr_str_list = ['{c}(']
+ temp = ['{0}={{s.{0}}}'.format(key) for key in vars(self)
+ if not key.startswith('_')]
+ self_list = ', '.join(temp)
+ repr_str_list.append(self_list)
+ repr_str_list.append(')')
+
+ return ''.join(repr_str_list).format(c=self.__class__.__name__, s=self)
+
+
+t_afl_safe = Table(
+ 'afl_safe', metadata,
+ Column('site_code', VARCHAR(8), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('root_directory', VARCHAR(128), nullable=False),
+ Column('arch_file', VARCHAR(128), nullable=False),
+ Column('mimetype', VARCHAR(80)),
+ Column('common_file_id', VARCHAR(64)),
+ Column('ngas_host', VARCHAR(64)),
+ schema='E2EMGR'
+)
+
+
+class Alia(Base):
+ __tablename__ = 'alias'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ source_id = Column(VARCHAR(12), server_default=text("' '"))
+ jsource_id = Column(VARCHAR(12), nullable=False)
+ prog_code = Column(VARCHAR(8), server_default=text("' '"))
+ pk = Column(Integer, primary_key=True)
+
+
+t_alltables_columns = Table(
+ 'alltables_columns', metadata,
+ Column('column_name', VARCHAR(32), nullable=False, index=True),
+ Column('table_name', VARCHAR(256), nullable=False),
+ Column('description', VARCHAR(128), nullable=False),
+ Column('unit', VARCHAR(16), nullable=False),
+ Column('ucd', VARCHAR(64), nullable=False),
+ Column('utype', VARCHAR(64), nullable=False),
+ Column('datatype', VARCHAR(32), nullable=False),
+ Column('varsize', NUMBER(asdecimal=False), nullable=False),
+ Column('primary', VARCHAR(16), nullable=False),
+ Column('indexed', VARCHAR(16), nullable=False),
+ Column('std', VARCHAR(16), nullable=False),
+ Column('catagory', VARCHAR(32), nullable=False),
+ Column('example', VARCHAR(128), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_antenna = Table(
+ 'antenna', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('antenna_id', NUMBER(asdecimal=False), nullable=False),
+ Column('name', VARCHAR(12), nullable=False),
+ Column('station', VARCHAR(18), nullable=False),
+ Column('mount', VARCHAR(8), nullable=False),
+ Column('dish_diameter', NUMBER(asdecimal=False), nullable=False),
+ Column('antenna_type', VARCHAR(14), nullable=False),
+ Column('axis_off', NUMBER(asdecimal=False), nullable=False),
+ Column('frame', VARCHAR(12), nullable=False),
+ Index('antenna_idx', 'project_code', 'arch_file_id', 'antenna_id'),
+ schema='E2EMGR'
+)
+
+
+t_archfileloc_ngas_migration = Table(
+ 'archfileloc_ngas_migration', metadata,
+ Column('site_code', VARCHAR(8), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('root_directory', VARCHAR(128), nullable=False),
+ Column('arch_file', VARCHAR(128), nullable=False),
+ Column('mimetype', VARCHAR(80)),
+ Column('common_file_id', VARCHAR(64)),
+ Column('ngas_host', VARCHAR(64)),
+ schema='E2EMGR'
+)
+
+
+t_archfilelocation = Table(
+ 'archfilelocation', metadata,
+ Column('site_code', VARCHAR(8), nullable=False, index=True),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('root_directory', VARCHAR(128), nullable=False),
+ Column('arch_file', VARCHAR(128), nullable=False, index=True),
+ Column('mimetype', VARCHAR(80)),
+ Column('common_file_id', VARCHAR(64)),
+ Column('ngas_host', VARCHAR(64)),
+ schema='E2EMGR'
+)
+
+
+t_archfilelocationgbt = Table(
+ 'archfilelocationgbt', metadata,
+ Column('site_code', VARCHAR(8), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('root_directory', VARCHAR(128), nullable=False),
+ Column('arch_file', VARCHAR(128), nullable=False),
+ Column('mimetype', VARCHAR(80)),
+ Column('common_file_id', VARCHAR(64)),
+ Column('ngas_host', VARCHAR(64)),
+ schema='E2EMGR'
+)
+
+
+t_archive = Table(
+ 'archive', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(8), nullable=False),
+ Column('arch_format', VARCHAR(8), nullable=False),
+ Column('data_type', VARCHAR(12), nullable=False),
+ Column('subarray_id', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False, index=True),
+ Column('arch_file', VARCHAR(128), nullable=False, index=True),
+ Column('arch_file_date', NUMBER(asdecimal=False), nullable=False),
+ Column('catalog_date', NUMBER(asdecimal=False), nullable=False),
+ Column('file_size', NUMBER(asdecimal=False), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False, index=True),
+ Column('annex1_id', NUMBER(asdecimal=False), nullable=False),
+ Column('obs_bands', VARCHAR(24), nullable=False),
+ Column('project_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('sb_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('eb_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('datamsg', VARCHAR(128), server_default=text("'none'")),
+ Column('datalevel', NUMBER(8, 0, False), server_default=text("0")),
+ Column('sb_type', VARCHAR(32), server_default=text("'none'")),
+ Index('archive3_idx', 'starttime', 'stoptime'),
+ schema='E2EMGR'
+)
+
+
+t_archivegbt = Table(
+ 'archivegbt', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(8), nullable=False),
+ Column('arch_format', VARCHAR(8), nullable=False),
+ Column('data_type', VARCHAR(12), nullable=False),
+ Column('subarray_id', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file', VARCHAR(128), nullable=False),
+ Column('arch_file_date', NUMBER(asdecimal=False), nullable=False),
+ Column('catalog_date', NUMBER(asdecimal=False), nullable=False),
+ Column('file_size', NUMBER(asdecimal=False), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False),
+ Column('annex1_id', NUMBER(asdecimal=False), nullable=False),
+ Column('obs_bands', VARCHAR(16), nullable=False),
+ Column('project_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('sb_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('eb_id', NUMBER(19, 0, False), server_default=text("""\
+0
+""")),
+ Column('datamsg', VARCHAR(128), server_default=text("'none'")),
+ Column('datalevel', NUMBER(8, 0, False), server_default=text("0")),
+ Column('sb_type', VARCHAR(32), nullable=False, server_default=text("'none' ")),
+ schema='E2EMGR'
+)
+
+
+t_archivegbtold = Table(
+ 'archivegbtold', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(8), nullable=False),
+ Column('arch_format', VARCHAR(8), nullable=False),
+ Column('data_type', VARCHAR(12), nullable=False),
+ Column('subarray_id', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file', VARCHAR(128), nullable=False),
+ Column('arch_file_date', NUMBER(asdecimal=False), nullable=False),
+ Column('catalog_date', NUMBER(asdecimal=False), nullable=False),
+ Column('file_size', NUMBER(asdecimal=False), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False),
+ Column('annex1_id', NUMBER(asdecimal=False), nullable=False),
+ Column('obs_bands', VARCHAR(16), nullable=False),
+ Column('project_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('sb_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('eb_id', NUMBER(19, 0, False), server_default=text("0")),
+ Column('datamsg', VARCHAR(128), server_default=text("'none'")),
+ Column('datalevel', NUMBER(8, 0, False), server_default=text("0")),
+ Column('sb_type', VARCHAR(32), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+t_archivesites = Table(
+ 'archivesites', metadata,
+ Column('site_code', VARCHAR(8), nullable=False, index=True),
+ Column('host', VARCHAR(24), nullable=False),
+ Column('root_directory', VARCHAR(64), nullable=False),
+ Column('ftp_directory', VARCHAR(64), nullable=False),
+ Column('site_name', VARCHAR(64), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_archvla = Table(
+ 'archvla', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(8), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_asa_columns = Table(
+ 'asa_columns', metadata,
+ Column('column_name', VARCHAR(32), index=True, server_default=text("'none'")),
+ Column('table_name', VARCHAR(32), server_default=text("'none'")),
+ Column('description', VARCHAR(64), server_default=text("'none'")),
+ Column('unit', VARCHAR(16), server_default=text("'none'")),
+ Column('ucd', VARCHAR(64), server_default=text("'none'")),
+ Column('utype', VARCHAR(64), server_default=text("'none'")),
+ Column('datatype', VARCHAR(32), server_default=text("'none'")),
+ Column('varsize', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('primary', VARCHAR(16), server_default=text("'none'")),
+ Column('indexed', VARCHAR(16), server_default=text("'none'")),
+ Column('std', VARCHAR(16), server_default=text("'none'")),
+ Column('tooltip', VARCHAR(128), server_default=text("'none'")),
+ Column('examples', VARCHAR(128), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+t_asa_project = Table(
+ 'asa_project', metadata,
+ Column('project_code', VARCHAR(64), server_default=text("'none'")),
+ Column('project_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('proposal_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('pi_name', VARCHAR(64), server_default=text("'none'")),
+ Column('pi_id', VARCHAR(64), server_default=text("'none'")),
+ Column('coi_name', VARCHAR(64), server_default=text("'none'")),
+ Column('title', VARCHAR(64), server_default=text("'none'")),
+ Column('type', VARCHAR(64), server_default=text("'none'")),
+ Column('associated_arc', VARCHAR(64), server_default=text("'none'")),
+ Column('proposal_cycle', VARCHAR(64), server_default=text("'none'")),
+ Column('proposal_abstract', VARCHAR(64), server_default=text("'none'")),
+ Column('proposed_start_date', VARCHAR(64), server_default=text("'none'")),
+ Column('proposed_end_date', VARCHAR(64), server_default=text("'none'")),
+ Column('first_obs_start', VARCHAR(64), server_default=text("'none'")),
+ Column('first_obs_end', VARCHAR(64), server_default=text("'none'")),
+ Column('last_obs_start', VARCHAR(64), server_default=text("'none'")),
+ Column('last_obs_end', VARCHAR(64), server_default=text("'none'")),
+ Column('arc_release', VARCHAR(64), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+t_asa_projects = Table(
+ 'asa_projects', metadata,
+ Column('project_code', VARCHAR(64), server_default=text("'none'")),
+ Column('project_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('proposal_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('pi_name', VARCHAR(64), server_default=text("'none'")),
+ Column('pi_id', VARCHAR(64), server_default=text("'none'")),
+ Column('coi_name', VARCHAR(64), server_default=text("'none'")),
+ Column('title', VARCHAR(64), server_default=text("'none'")),
+ Column('type', VARCHAR(64), server_default=text("'none'")),
+ Column('associated_arc', VARCHAR(64), server_default=text("'none'")),
+ Column('proposal_cycle', VARCHAR(64), server_default=text("'none'")),
+ Column('proposal_abstract', VARCHAR(64), server_default=text("'none'")),
+ Column('proposed_start_date', VARCHAR(64), server_default=text("'none'")),
+ Column('proposed_end_date', VARCHAR(64), server_default=text("'none'")),
+ Column('first_obs_start', VARCHAR(64), server_default=text("'none'")),
+ Column('first_obs_end', VARCHAR(64), server_default=text("'none'")),
+ Column('last_obs_start', VARCHAR(64), server_default=text("'none'")),
+ Column('last_obs_end', VARCHAR(64), server_default=text("'none'")),
+ Column('arc_release', VARCHAR(64), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+t_asa_science = Table(
+ 'asa_science', metadata,
+ Column('dataset_id', VARCHAR(64), server_default=text("'none'")),
+ Column('asdm_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('field_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('spectral_window_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('project_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('schedblock_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('execblock_uid', VARCHAR(64), server_default=text("'none'")),
+ Column('ra', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('dec', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('ra_source', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('dec_source', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('cx', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('cy', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('cz', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('az_min', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('az_max', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('elevation_min', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('elevation_max', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_cd1_1', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_cd1_2', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_cd2_1', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_cd2_2', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_crpix1', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_crpix2', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_crval1', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('wcs_crval2', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('spatial_scale_min', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('spatial_scale_max', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('gal_longitude', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('gal_latitude', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('ecliptic_longitude', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('ecliptic_latitude', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('footprint', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('fov', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('bounding_box', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('source_name', VARCHAR(64), server_default=text("'none'")),
+ Column('source_is_in_simbad', VARCHAR(64), server_default=text("'none'")),
+ Column('source_class', VARCHAR(64), server_default=text("'none'")),
+ Column('source_redshift', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('source_alma_catalog_id', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('start_date', VARCHAR(64), server_default=text("'none'")),
+ Column('end_date', VARCHAR(64), server_default=text("'none'")),
+ Column('int_time', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('time_resolution', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('frequency', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('frequency_max', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('frequency_min', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('frequency_resolution', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('bandwidth', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('rest_frequency', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('rest_frequency_min', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('rest_frequency_max', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('rest_frequency_resolution', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('rest_bandwidth', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('band', VARCHAR(64), server_default=text("'none'")),
+ Column('frequency_support', VARCHAR(128), server_default=text("'none'")),
+ Column('rest_frequency_support', VARCHAR(128), server_default=text("'none'")),
+ Column('velocity_resolution', NUMBER(16, 10, True), server_default=text("0.0")),
+ Column('resolving_power', NUMBER(16, 10, True), server_default=text("0.0")),
+ Column('rest_velocity_resolution', NUMBER(16, 10, True), server_default=text("0.0")),
+ Column('rest_resolving_power', NUMBER(16, 10, True), server_default=text("0.0")),
+ Column('channel_num', VARCHAR(32), server_default=text("'none'")),
+ Column('spectral_window_num', VARCHAR(32), server_default=text("'none'")),
+ Column('continuum_window_num', VARCHAR(32), server_default=text("'none'")),
+ Column('obs_unitset_id', VARCHAR(128), server_default=text("'none'")),
+ Column('science_goal_ouss_id', VARCHAR(128), server_default=text("'none'")),
+ Column('group_ouss_id', VARCHAR(128), server_default=text("'none'")),
+ Column('member_ouss_id', VARCHAR(128), server_default=text("'none'")),
+ Column('sensitivity', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('flux_min', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('flux_max', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('uv_coverage_param1', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('uv_coverage_param2', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('pol_num', NUMBER(16, 0, False), server_default=text("0")),
+ Column('pol_products', VARCHAR(128), server_default=text("'none'")),
+ Column('airmass', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('pwv', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('temperature', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('windspeed', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('winddirection', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('ant_num', NUMBER(16, 0, False), server_default=text("0")),
+ Column('ant_aca_num', NUMBER(16, 0, False), server_default=text("0")),
+ Column('ant_main_num', NUMBER(16, 0, False), server_default=text("0")),
+ Column('antennas', VARCHAR(128), server_default=text("'none'")),
+ Column('baseline_max', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('baseline_min', NUMBER(16, 0, False), server_default=text("0.0")),
+ Column('scan_num', NUMBER(6, 0, False), server_default=text("0")),
+ Column('project_code', VARCHAR(64), server_default=text("'none'")),
+ Column('schedblock_name', VARCHAR(64), server_default=text("'none'")),
+ Column('observation_category', VARCHAR(64), server_default=text("'none'")),
+ Column('scan_intent', VARCHAR(64), server_default=text("'none'")),
+ Column('access_format', VARCHAR(64), server_default=text("'none'")),
+ Column('processing_level', NUMBER(6, 0, False), server_default=text("""\
+0
+""")),
+ schema='E2EMGR'
+)
+
+
+t_calib_parm_pos = Table(
+ 'calib_parm_pos', metadata,
+ Column('jsource_id', VARCHAR(10), nullable=False),
+ Column('epoch', DateTime),
+ Column('rarad', NUMBER(12, 8, True), server_default=text("0")),
+ Column('decrad', NUMBER(12, 8, True), server_default=text("0")),
+ Column('raerr', NUMBER(10, 8, True), server_default=text("0")),
+ Column('decerr', NUMBER(10, 6, True), server_default=text("0")),
+ Column('telescope', VARCHAR(5), nullable=False),
+ Column('config', VARCHAR(5), server_default=text("' '")),
+ Column('obs_freq', Float, server_default=text("0")),
+ Column('freq_min', Float, server_default=text("0")),
+ Column('freq_max', Float, server_default=text("0")),
+ Column('uv_min', NUMBER(12, 6, True), server_default=text("0")),
+ Column('uv_max', NUMBER(12, 6, True), server_default=text("0")),
+ Column('flux', NUMBER(8, 4, True), server_default=text("0")),
+ Column('resolution', NUMBER(6, 4, True), server_default=text("0")),
+ Column('variability', NUMBER(6, 4, True), server_default=text("0")),
+ Column('cdvlba', CHAR(2), server_default=text("' '")),
+ Column('cd_vla', CHAR(2), server_default=text("' '")),
+ Column('entry_date', DateTime),
+ Column('parm_remarks', VARCHAR(64), server_default=text("' '")),
+ schema='E2EMGR'
+)
+
+
+class CalibSrcPo(Base):
+ __tablename__ = 'calib_src_pos'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ remarks = Column(VARCHAR(200), nullable=False)
+ ssize = Column(NUMBER(10, 0, False), server_default=text("0"))
+ flux = Column(NUMBER(10, 6, True), server_default=text("0"))
+ mfreq = Column(NUMBER(asdecimal=False), server_default=text("0"))
+ ufreq = Column(NUMBER(asdecimal=False), server_default=text("0"))
+ lfreq = Column(NUMBER(asdecimal=False), server_default=text("0"))
+ distance = Column(NUMBER(asdecimal=False), server_default=text("0"))
+ ddec = Column(NUMBER(10, 6, True), server_default=text("0"))
+ dra = Column(NUMBER(10, 6, True), server_default=text("0"))
+ repoch = Column(DateTime)
+ parallax = Column(NUMBER(8, 6, True), server_default=text("0"))
+ decerr = Column(NUMBER(12, 8, True), server_default=text("0"))
+ raerr = Column(NUMBER(12, 8, True), server_default=text("0"))
+ decrad = Column(NUMBER(12, 8, True), server_default=text("0"))
+ rarad = Column(NUMBER(12, 8, True), server_default=text("0"))
+ epoch = Column(DateTime)
+ calcode = Column(CHAR(6), server_default=text("' '"))
+ prog_code = Column(VARCHAR(6), server_default=text("' '"))
+ dqualifier = Column(Integer, server_default=text("0"))
+ cqualifier = Column(Integer, server_default=text("0"))
+ uqualifier = Column(Integer, server_default=text("0"))
+ entrydate = Column(DateTime)
+ jsource_id = Column(VARCHAR(12), nullable=False)
+ pk = Column(Integer, primary_key=True)
+
+
+t_config = Table(
+ 'config', metadata,
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(8), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_datadesc = Table(
+ 'datadesc', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('data_desc_id', NUMBER(asdecimal=False), nullable=False, index=True),
+ Column('sub_desc_id', NUMBER(asdecimal=False), nullable=False, index=True),
+ Column('if_band', VARCHAR(4), nullable=False),
+ Column('if_ref_freq', NUMBER(asdecimal=False), nullable=False),
+ Column('pol1', NUMBER(asdecimal=False), nullable=False),
+ Column('pol2', NUMBER(asdecimal=False), nullable=False),
+ Column('pol3', NUMBER(asdecimal=False), nullable=False),
+ Column('pol4', NUMBER(asdecimal=False), nullable=False),
+ Column('if_conv_chain', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_chan_id', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_ref_freq', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_bandw', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol1', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol2', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol3', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol4', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_net_sideband', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_num_chans', NUMBER(asdecimal=False), nullable=False),
+ Column('corr_mode', VARCHAR(24)),
+ Column('if_chan', VARCHAR(8)),
+ Column('backend_id', VARCHAR(24), server_default=text("""\
+'none'
+ """)),
+ Column('receiver_id', VARCHAR(24), nullable=False, server_default=text("'none' ")),
+ Column('veldef', VARCHAR(24), server_default=text("'none'")),
+ Column('version', VARCHAR(16), server_default=text("'none'")),
+ Column('velocity', NUMBER(asdecimal=False), server_default=text("0.0")),
+ schema='E2EMGR'
+)
+
+
+t_datadescgbt = Table(
+ 'datadescgbt', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('data_desc_id', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_desc_id', NUMBER(asdecimal=False), nullable=False),
+ Column('if_band', VARCHAR(4), nullable=False),
+ Column('if_ref_freq', NUMBER(asdecimal=False), nullable=False),
+ Column('pol1', NUMBER(asdecimal=False), nullable=False),
+ Column('pol2', NUMBER(asdecimal=False), nullable=False),
+ Column('pol3', NUMBER(asdecimal=False), nullable=False),
+ Column('pol4', NUMBER(asdecimal=False), nullable=False),
+ Column('if_conv_chain', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_chan_id', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_ref_freq', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_bandw', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol1', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol2', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol3', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol4', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_net_sideband', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_num_chans', NUMBER(asdecimal=False), nullable=False),
+ Column('corr_mode', VARCHAR(24), nullable=False),
+ Column('if_chan', VARCHAR(8), nullable=False),
+ Column('receiver_id', VARCHAR(24), server_default=text("'none'")),
+ Column('backend_id', VARCHAR(24), server_default=text("'none'")),
+ Column('velocity', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('veldef', VARCHAR(24), server_default=text("'none'")),
+ Column('version', VARCHAR(16), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+t_dataproblems = Table(
+ 'dataproblems', metadata,
+ Column('error_code', VARCHAR(16), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('if_bands', VARCHAR(24), nullable=False),
+ Column('error_status', VARCHAR(12), nullable=False),
+ Column('description', VARCHAR(64), nullable=False),
+ Column('doc_link', VARCHAR(64), nullable=False),
+ Column('severity', NUMBER(asdecimal=False), nullable=False, server_default=text("0 ")),
+ Index('dataproblems1_idx', 'starttime', 'stoptime'),
+ schema='E2EMGR'
+)
+
+
+t_dataproblink = Table(
+ 'dataproblink', metadata,
+ Column('error_code', VARCHAR(16), nullable=False, index=True),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_downloadlog = Table(
+ 'downloadlog', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('download_date', DateTime, nullable=False, index=True),
+ Column('email_addr', VARCHAR(48), nullable=False, index=True),
+ Column('ip_addr', VARCHAR(24), nullable=False),
+ Column('status', VARCHAR(16), nullable=False),
+ Column('sources', VARCHAR(64), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file', VARCHAR(128), nullable=False, index=True),
+ Column('out_file', VARCHAR(128), nullable=False),
+ Column('result', VARCHAR(32), nullable=False),
+ Column('file_size', NUMBER(asdecimal=False)),
+ Column('code', VARCHAR(12)),
+ schema='E2EMGR'
+)
+
+
+t_emailaddr = Table(
+ 'emailaddr', metadata,
+ Column('userlastname', VARCHAR(48), nullable=False),
+ Column('userinitials', VARCHAR(24)),
+ Column('emailaddr', VARCHAR(64)),
+ Index('emailaddr1_idx', 'userlastname', 'userinitials'),
+ schema='E2EMGR'
+)
+
+
+t_file_set_properties = Table(
+ 'file_set_properties', metadata,
+ Column('file_set_id', VARCHAR(128), server_default=text("'none'")),
+ Column('file_set_alias', VARCHAR(128), server_default=text("'none'")),
+ Column('file_set_category', VARCHAR(32), server_default=text("'none'")),
+ Column('category_key', VARCHAR(48), server_default=text("'none'")),
+ Column('starttime', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('stoptime', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('telescope', VARCHAR(32), server_default=text("'none'")),
+ Column('telescope_config', VARCHAR(48), server_default=text("'none'")),
+ Column('obs_bands', VARCHAR(48), server_default=text("'none'")),
+ Column('processing_history', VARCHAR(128), server_default=text("'none'")),
+ Column('collection', VARCHAR(48), server_default=text("'none'")),
+ Column('calib_level', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('n_file_subsets', NUMBER(asdecimal=False), server_default=text("""\
+0
+ """)),
+ Index('set_properties_indx', 'file_set_id', 'category_key'),
+ schema='E2EMGR'
+)
+
+
+class FirstImage(Base):
+ __tablename__ = 'first_image'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ id = Column(VARCHAR(32), primary_key=True)
+ obs_id = Column(VARCHAR(32), server_default=text("null"))
+ archive_id = Column(VARCHAR(128), nullable=False)
+ htm_id = Column(NUMBER(14, 0, False), server_default=text("0"))
+ preview_id = Column(VARCHAR(128), nullable=False)
+ access_format = Column(VARCHAR(32), nullable=False)
+ access_estsize = Column(NUMBER(14, 0, False), server_default=text("0"))
+ dataproduct_type = Column(VARCHAR(16), server_default=text("null"))
+ dataproduct_subtype = Column(VARCHAR(32), server_default=text("null"))
+ calib_level = Column(NUMBER(6, 0, False), server_default=text("0"))
+ dataset_length = Column(NUMBER(14, 0, False), server_default=text("0"))
+ im_nsubarrays = Column(NUMBER(8, 0, False), server_default=text("1"))
+ im_naxes = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis1 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis2 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis3 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis4 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_pixtype = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes1 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes2 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes3 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes4 = Column(VARCHAR(16), server_default=text("null"))
+ im_ra1 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec1 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra2 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec2 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra3 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec3 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra4 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec4 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_scale = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ obs_title = Column(VARCHAR(128), server_default=text("null"))
+ obs_creator_name = Column(VARCHAR(32), server_default=text("null"))
+ obs_collection = Column(VARCHAR(32), server_default=text("null"))
+ obs_creator_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_publisher_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_dataset_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_release_date = Column(VARCHAR(20), server_default=text("null"))
+ obs_creation_date = Column(VARCHAR(20), server_default=text("null"))
+ obs_creation_type = Column(VARCHAR(16), server_default=text("null"))
+ facility_name = Column(VARCHAR(20), server_default=text("null"))
+ instrument_name = Column(VARCHAR(20), server_default=text("null"))
+ obs_bandpass = Column(VARCHAR(20), server_default=text("null"))
+ obs_datasource = Column(VARCHAR(20), server_default=text("null"))
+ proposal_id = Column(VARCHAR(20), server_default=text("null"))
+ target_name = Column(VARCHAR(20), server_default=text("null"))
+ target_class = Column(VARCHAR(20), server_default=text("null"))
+ s_ra = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ s_dec = Column(NUMBER(asdecimal=False), index=True, server_default=text("0.0"))
+ s_fov = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ s_region = Column(VARCHAR(128), server_default=text("null"))
+ s_calib_status = Column(VARCHAR(32), server_default=text("'none'"))
+ s_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_min = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_max = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_respower = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_min = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_max = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_exptime = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ o_ucd = Column(VARCHAR(20), server_default=text("null"))
+ o_unit = Column(VARCHAR(32), server_default=text("'none'"))
+ pol_states = Column(VARCHAR(20), server_default=text("null"))
+ project_code = Column(VARCHAR(24), server_default=text("null"))
+ obs_session = Column(VARCHAR(24), server_default=text("null"))
+ file_id = Column(VARCHAR(64), server_default=text("null"))
+ file_set_id = Column(VARCHAR(128), server_default=text("null"))
+ file_subset_id = Column(VARCHAR(128), server_default=text("null"))
+ dataset = Column(VARCHAR(128), server_default=text("null"))
+ image_file = Column(VARCHAR(128), server_default=text("null"))
+ obs_file_id = Column(VARCHAR(64), server_default=text("null"))
+ cal_file_id = Column(VARCHAR(64), server_default=text("null"))
+ root_dir = Column(VARCHAR(128), server_default=text("null"))
+ file_dir = Column(VARCHAR(128), server_default=text("""\
+null
+"""))
+
+
+t_firstcatalog = Table(
+ 'firstcatalog', metadata,
+ Column('rarad', NUMBER(asdecimal=False), nullable=False),
+ Column('decrad', NUMBER(asdecimal=False), nullable=False),
+ Column('warn', VARCHAR(2), nullable=False),
+ Column('pflux', NUMBER(asdecimal=False), nullable=False),
+ Column('flux', NUMBER(asdecimal=False), nullable=False),
+ Column('rms', NUMBER(asdecimal=False), nullable=False),
+ Column('majorax', NUMBER(asdecimal=False), nullable=False),
+ Column('minorax', NUMBER(asdecimal=False), nullable=False),
+ Column('posangle', NUMBER(asdecimal=False), nullable=False),
+ Column('fmajorax', NUMBER(asdecimal=False), nullable=False),
+ Column('fminorax', NUMBER(asdecimal=False), nullable=False),
+ Column('fposangle', NUMBER(asdecimal=False), nullable=False),
+ Column('field', VARCHAR(14), nullable=False),
+ Column('name', VARCHAR(16)),
+ schema='E2EMGR'
+)
+
+
+t_image = Table(
+ 'image', metadata,
+ Column('identity', VARCHAR(16), nullable=False, index=True),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('teleconfig', VARCHAR(8), nullable=False),
+ Column('instrument', VARCHAR(24), nullable=False),
+ Column('observer', VARCHAR(48), nullable=False),
+ Column('projectcode', VARCHAR(16), nullable=False),
+ Column('fieldname', VARCHAR(24), nullable=False),
+ Column('naxes', NUMBER(asdecimal=False), nullable=False),
+ Column('imagemax', NUMBER(asdecimal=False), nullable=False),
+ Column('imagemin', NUMBER(asdecimal=False), nullable=False),
+ Column('imagesens', NUMBER(asdecimal=False), nullable=False),
+ Column('imagescale', NUMBER(asdecimal=False), nullable=False),
+ Column('imagezero', NUMBER(asdecimal=False), nullable=False),
+ Column('imageunits', VARCHAR(16), nullable=False),
+ Column('restoremaj', NUMBER(asdecimal=False), nullable=False),
+ Column('restoremin', NUMBER(asdecimal=False), nullable=False),
+ Column('restorepa', NUMBER(asdecimal=False), nullable=False),
+ Column('firsttime', NUMBER(asdecimal=False), nullable=False),
+ Column('midobstime', NUMBER(asdecimal=False), nullable=False),
+ Column('exposure', NUMBER(asdecimal=False), nullable=False),
+ Column('imagedate', NUMBER(asdecimal=False), nullable=False),
+ Column('coordframe', VARCHAR(16), nullable=False),
+ Column('projection', VARCHAR(16), nullable=False),
+ Column('equinox', NUMBER(asdecimal=False), nullable=False),
+ Column('region', VARCHAR(16), nullable=False),
+ Column('racenter', NUMBER(asdecimal=False), nullable=False),
+ Column('deccenter', NUMBER(asdecimal=False), nullable=False),
+ Column('ramin', NUMBER(asdecimal=False), nullable=False),
+ Column('ramax', NUMBER(asdecimal=False), nullable=False),
+ Column('decmin', NUMBER(asdecimal=False), nullable=False),
+ Column('decmax', NUMBER(asdecimal=False), nullable=False),
+ Column('rarefpix', NUMBER(asdecimal=False), nullable=False),
+ Column('raref', NUMBER(asdecimal=False), nullable=False),
+ Column('radelt', NUMBER(asdecimal=False), nullable=False),
+ Column('rapixels', NUMBER(asdecimal=False), nullable=False),
+ Column('decrefpix', NUMBER(asdecimal=False), nullable=False),
+ Column('decref', NUMBER(asdecimal=False), nullable=False),
+ Column('decdelt', NUMBER(asdecimal=False), nullable=False),
+ Column('decpixels', NUMBER(asdecimal=False), nullable=False),
+ Column('rotangle', NUMBER(asdecimal=False), nullable=False),
+ Column('obsband', VARCHAR(8), nullable=False),
+ Column('frequency', NUMBER(asdecimal=False), nullable=False),
+ Column('bandwd', NUMBER(asdecimal=False), nullable=False),
+ Column('nspect', NUMBER(asdecimal=False), nullable=False),
+ Column('reffreqpix', NUMBER(asdecimal=False), nullable=False),
+ Column('reffreq', NUMBER(asdecimal=False), nullable=False),
+ Column('freqincre', NUMBER(asdecimal=False), nullable=False),
+ Column('npolar', NUMBER(asdecimal=False), nullable=False),
+ Column('polpixels', VARCHAR(12), nullable=False),
+ Column('imagequal', VARCHAR(6), nullable=False),
+ Column('photoerr', NUMBER(asdecimal=False), nullable=False),
+ Column('raerr', NUMBER(asdecimal=False), nullable=False),
+ Column('decerr', NUMBER(asdecimal=False), nullable=False),
+ Column('specterr', NUMBER(asdecimal=False), nullable=False),
+ Column('imagefile', VARCHAR(128), nullable=False),
+ Column('imageformat', VARCHAR(24), nullable=False),
+ Column('rootdirectory', VARCHAR(128), nullable=False),
+ Column('refurl', VARCHAR(128), nullable=False),
+ Column('collection', VARCHAR(24)),
+ Column('filesize', NUMBER(asdecimal=False)),
+ Column('arch_file_id', NUMBER(asdecimal=False)),
+ Column('common_file_id', VARCHAR(80)),
+ Index('image1_idx', 'racenter', 'deccenter'),
+ schema='E2EMGR'
+)
+
+
+t_ingest_stats = Table(
+ 'ingest_stats', metadata,
+ Column('ingestion_date', VARCHAR(24), server_default=text("'none'")),
+ Column('ingestion_total', NUMBER(20, 0, False), server_default=text("0")),
+ Column('telescope', VARCHAR(12), server_default=text("'none'")),
+ Column('data_type', VARCHAR(24), server_default=text("'none'")),
+ Column('root_dir', VARCHAR(128), server_default=text("'none'")),
+ Column('ngas_host', VARCHAR(64), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+t_logfiles = Table(
+ 'logfiles', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('raw_project_code', VARCHAR(16), nullable=False),
+ Column('dynamic_tag', VARCHAR(16), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('observer', VARCHAR(48), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('sched_type', VARCHAR(16), nullable=False),
+ Column('site_code', VARCHAR(8), nullable=False),
+ Column('root_directory', VARCHAR(64), nullable=False),
+ Column('file_name', VARCHAR(64), nullable=False),
+ Column('mimetype', VARCHAR(80), nullable=False),
+ Column('obsfiles', VARCHAR(128), nullable=False),
+ schema='E2EMGR'
+)
+
+
+class NgasCache(Base):
+ __tablename__ = 'ngas_cache'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ disk_id = Column(VARCHAR(128), primary_key=True, nullable=False)
+ file_id = Column(VARCHAR(64), primary_key=True, nullable=False)
+ file_version = Column(Integer, primary_key=True, nullable=False)
+ cache_time = Column(NUMBER(16, 6, True), nullable=False)
+ cache_delete = Column(Integer, nullable=False)
+
+
+t_ngas_file_sets = Table(
+ 'ngas_file_sets', metadata,
+ Column('file_id', VARCHAR(64), nullable=False),
+ Column('file_version', NUMBER(12, 0, False), server_default=text("1")),
+ Column('file_set_id', VARCHAR(64), nullable=False),
+ Column('entity_type_name', VARCHAR(48), nullable=False),
+ Column('entity_id', VARCHAR(48), nullable=False),
+ Column('format', VARCHAR(48), server_default=text("'none'")),
+ Column('file_id_alias', VARCHAR(128), server_default=text("'none'")),
+ Column('native_ext', VARCHAR(24), server_default=text("'none'")),
+ Column('file_subset_id', VARCHAR(48), server_default=text("'none'")),
+ Index('ngas_file_sets_indx', 'file_id', 'file_version', 'file_set_id'),
+ schema='E2EMGR'
+)
+
+
+class NgasMirroringBookkeeping(Base):
+ __tablename__ = 'ngas_mirroring_bookkeeping'
+ __table_args__ = (
+ Index('nmb_thost_status_shost_idx', 'target_cluster', 'target_host', 'status', 'source_host'),
+ Index('nmb_iter_status_idx', 'iteration', 'status'),
+ {'schema': 'E2EMGR'}
+ )
+
+ file_id = Column(VARCHAR(64), primary_key=True, nullable=False)
+ file_version = Column(NUMBER(22, 0, False), primary_key=True, nullable=False)
+ file_size = Column(NUMBER(20, 0, False))
+ disk_id = Column(VARCHAR(128))
+ host_id = Column(VARCHAR(32))
+ format = Column(VARCHAR(32))
+ status = Column(CHAR(8), nullable=False)
+ target_cluster = Column(VARCHAR(64))
+ target_host = Column(VARCHAR(64))
+ archive_command = Column(VARCHAR(118))
+ source_host = Column(VARCHAR(64), nullable=False)
+ retrieve_command = Column(VARCHAR(118))
+ ingestion_date = Column(VARCHAR(23))
+ ingestion_time = Column(Float)
+ iteration = Column(NUMBER(22, 0, False), primary_key=True, nullable=False)
+ checksum = Column(VARCHAR(64), nullable=False)
+ staging_file = Column(VARCHAR(256))
+ attempt = Column(NUMBER(4, 0, False))
+ downloaded_bytes = Column(NUMBER(22, 0, False))
+
+
+class NgasMirroringHist(Base):
+ __tablename__ = 'ngas_mirroring_hist'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ instance_id = Column(VARCHAR(32), nullable=False)
+ file_id = Column(VARCHAR(64), primary_key=True, nullable=False)
+ file_version = Column(Integer, primary_key=True, nullable=False)
+ ingestion_date = Column(VARCHAR(23), nullable=False)
+ srv_list_id = Column(Integer, nullable=False)
+ xml_file_info = Column(VARCHAR(2000), nullable=False)
+ status = Column(Integer, nullable=False)
+ message = Column(VARCHAR(2000))
+ last_activity_time = Column(VARCHAR(23), nullable=False)
+ scheduling_time = Column(VARCHAR(23), nullable=False)
+
+
+class NgasMirroringQueue(Base):
+ __tablename__ = 'ngas_mirroring_queue'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ instance_id = Column(VARCHAR(32), nullable=False)
+ file_id = Column(VARCHAR(64), primary_key=True, nullable=False)
+ file_version = Column(Integer, primary_key=True, nullable=False)
+ ingestion_date = Column(VARCHAR(23), nullable=False)
+ srv_list_id = Column(Integer, nullable=False)
+ xml_file_info = Column(VARCHAR(2000), nullable=False)
+ status = Column(Integer, nullable=False)
+ message = Column(VARCHAR(2000))
+ last_activity_time = Column(VARCHAR(23), nullable=False)
+ scheduling_time = Column(VARCHAR(23), nullable=False)
+
+
+class NgasSrvList(Base):
+ __tablename__ = 'ngas_srv_list'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ srv_list_id = Column(Integer, primary_key=True)
+ srv_list = Column(VARCHAR(255), nullable=False)
+ creation_date = Column(VARCHAR(23), nullable=False)
+
+
+class Nvsscatalog(Base):
+ __tablename__ = 'nvsscatalog'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ rarad = Column(NUMBER(asdecimal=False), primary_key=True, nullable=False)
+ decrad = Column(NUMBER(asdecimal=False), primary_key=True, nullable=False)
+ raerror = Column(NUMBER(asdecimal=False), nullable=False)
+ decerror = Column(NUMBER(asdecimal=False), nullable=False)
+ flux = Column(NUMBER(asdecimal=False), nullable=False)
+ fluxerror = Column(NUMBER(asdecimal=False), nullable=False)
+ majorsym = Column(VARCHAR(4), nullable=False)
+ majorax = Column(NUMBER(asdecimal=False), nullable=False)
+ majorerr = Column(NUMBER(asdecimal=False), nullable=False)
+ minorsym = Column(VARCHAR(4), nullable=False)
+ minorax = Column(NUMBER(asdecimal=False), nullable=False)
+ minorerr = Column(NUMBER(asdecimal=False), nullable=False)
+ posangle = Column(NUMBER(asdecimal=False), nullable=False)
+ paerror = Column(NUMBER(asdecimal=False), nullable=False)
+ rescode = Column(VARCHAR(4), nullable=False)
+ reserror = Column(NUMBER(asdecimal=False), nullable=False)
+ pflux = Column(NUMBER(asdecimal=False), nullable=False)
+ pfluxerror = Column(NUMBER(asdecimal=False), nullable=False)
+ pangle = Column(NUMBER(asdecimal=False), nullable=False)
+ pangleerror = Column(NUMBER(asdecimal=False), nullable=False)
+ field = Column(VARCHAR(10), nullable=False)
+ centerx = Column(NUMBER(asdecimal=False), nullable=False)
+ centery = Column(NUMBER(asdecimal=False), nullable=False)
+ name = Column(VARCHAR(10))
+
+
+t_observation = Table(
+ 'observation', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('obs_type', VARCHAR(24), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('source_id', VARCHAR(24), nullable=False, index=True),
+ Column('source_type', VARCHAR(8), nullable=False),
+ Column('calib_type', VARCHAR(8), nullable=False),
+ Column('corr_mode', VARCHAR(16), nullable=False),
+ Column('ra2000', NUMBER(asdecimal=False), nullable=False),
+ Column('dec2000', NUMBER(asdecimal=False), nullable=False),
+ Column('frame', VARCHAR(20), nullable=False),
+ Column('subarray_id', NUMBER(asdecimal=False), nullable=False),
+ Column('data_desc_id', NUMBER(asdecimal=False), nullable=False, index=True),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False, index=True),
+ Column('exposure', NUMBER(asdecimal=False), nullable=False),
+ Column('interval', NUMBER(asdecimal=False), nullable=False),
+ Column('uv_min', NUMBER(asdecimal=False), nullable=False),
+ Column('uv_max', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_desc_id', NullType),
+ Column('elev_min', NUMBER(asdecimal=False), nullable=False),
+ Column('elev_max', NUMBER(asdecimal=False), nullable=False),
+ Column('n_ants', NUMBER(asdecimal=False)),
+ Column('ra_epoch', NUMBER(asdecimal=False)),
+ Column('dec_epoch', NUMBER(asdecimal=False)),
+ Column('bdf_file', VARCHAR(64)),
+ Column('config_desc_id', VARCHAR(48), server_default=text("'none'")),
+ Column('scan', NUMBER(19, 0, False), server_default=text("0")),
+ Column('subscan', NUMBER(19, 0, False), server_default=text("0")),
+ Column('scan_intent', VARCHAR(256), server_default=text("'none'")),
+ Column('procname', VARCHAR(24), server_default=text("'none'")),
+ Column('proctype', VARCHAR(24), server_default=text("'none'")),
+ Column('gbtobstype', VARCHAR(24), server_default=text("'none'")),
+ Column('msgflag', VARCHAR(128), server_default=text("'none'")),
+ Column('msglevel', NUMBER(8, 0, False), server_default=text("0")),
+ Index('observation4_idx', 'starttime', 'stoptime'),
+ Index('observation5_idx', 'ra2000', 'dec2000'),
+ schema='E2EMGR'
+)
+
+
+t_observationgbt = Table(
+ 'observationgbt', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('obs_type', VARCHAR(24), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('source_id', VARCHAR(16), nullable=False),
+ Column('source_type', VARCHAR(8), nullable=False),
+ Column('calib_type', VARCHAR(8), nullable=False),
+ Column('corr_mode', VARCHAR(16), nullable=False),
+ Column('ra2000', NUMBER(asdecimal=False), nullable=False),
+ Column('dec2000', NUMBER(asdecimal=False), nullable=False),
+ Column('frame', VARCHAR(20), nullable=False),
+ Column('subarray_id', NUMBER(asdecimal=False), nullable=False),
+ Column('data_desc_id', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('exposure', NUMBER(asdecimal=False), nullable=False),
+ Column('interval', NUMBER(asdecimal=False), nullable=False),
+ Column('uv_min', NUMBER(asdecimal=False), nullable=False),
+ Column('uv_max', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_desc_id', NullType),
+ Column('elev_min', NUMBER(asdecimal=False), nullable=False),
+ Column('elev_max', NUMBER(asdecimal=False), nullable=False),
+ Column('n_ants', NUMBER(asdecimal=False), nullable=False),
+ Column('ra_epoch', NUMBER(asdecimal=False), nullable=False),
+ Column('dec_epoch', NUMBER(asdecimal=False), nullable=False),
+ Column('bdf_file', VARCHAR(64), server_default=text("'none'")),
+ Column('config_desc_id', VARCHAR(48), server_default=text("'none'")),
+ Column('scan', NUMBER(19, 0, False), server_default=text("0")),
+ Column('subscan', NUMBER(19, 0, False), server_default=text("0")),
+ Column('scan_intent', VARCHAR(256), server_default=text("'none'")),
+ Column('procname', VARCHAR(24), server_default=text("'none'")),
+ Column('proctype', VARCHAR(24), server_default=text("'none'")),
+ Column('gbtobstype', VARCHAR(24), server_default=text("'none'")),
+ Column('msgflag', VARCHAR(128), server_default=text("""\
+'none'
+""")),
+ Column('msglevel', NUMBER(8, 0, False), server_default=text("0")),
+ schema='E2EMGR'
+)
+
+
+t_obsscripts = Table(
+ 'obsscripts', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('raw_project_code', VARCHAR(16), nullable=False),
+ Column('dynamic_tag', VARCHAR(16), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('observer', VARCHAR(48), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('sched_type', VARCHAR(16), nullable=False),
+ Column('site_code', VARCHAR(8), nullable=False),
+ Column('root_directory', VARCHAR(64), nullable=False),
+ Column('file_name', VARCHAR(64), nullable=False),
+ Column('mimetype', VARCHAR(80), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_obsset = Table(
+ 'obsset', metadata,
+ Column('dataproduct_type', VARCHAR(24), nullable=False),
+ Column('dataproduct_subtype', VARCHAR(24), nullable=False),
+ Column('calib_level', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('target_name', VARCHAR(48), nullable=False),
+ Column('target_class', VARCHAR(24), nullable=False),
+ Column('obs_id', VARCHAR(64), nullable=False),
+ Column('obs_title', VARCHAR(24), nullable=False),
+ Column('obs_collection', VARCHAR(48), nullable=False),
+ Column('obs_creation_date', VARCHAR(24), nullable=False),
+ Column('obs_creator_name', VARCHAR(48), nullable=False),
+ Column('obs_creator_did', VARCHAR(48), nullable=False),
+ Column('obs_release_date', VARCHAR(24), nullable=False),
+ Column('obs_publisher_did', VARCHAR(64), nullable=False),
+ Column('publisher_id', VARCHAR(24), nullable=False),
+ Column('bib_reference', VARCHAR(24), nullable=False),
+ Column('data_rights', VARCHAR(24), nullable=False),
+ Column('access_url', VARCHAR(128), nullable=False),
+ Column('access_format', VARCHAR(24), nullable=False),
+ Column('access_estsize', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_ra', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_dec', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_fov', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_region', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_resolution', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_ucd', VARCHAR(48), nullable=False),
+ Column('s_unit', VARCHAR(24), nullable=False),
+ Column('s_resolution_min', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_resolution_max', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('s_calib_status', VARCHAR(48), nullable=False),
+ Column('s_stat_error', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('t_min', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('t_max', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('t_exptime', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('t_resolution', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('t_calib_status', VARCHAR(48), nullable=False),
+ Column('t_stat_error', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('em_ucd', VARCHAR(48), nullable=False),
+ Column('em_unit', VARCHAR(24), nullable=False),
+ Column('em_calib_status', VARCHAR(48), nullable=False),
+ Column('em_min', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('em_max', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('em_res_power', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('em_res_power_min', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('em_res_power_max', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('em_resolution', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('em_stat_error', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('o_ucd', VARCHAR(128), nullable=False),
+ Column('o_unit', VARCHAR(24), nullable=False),
+ Column('o_calib_status', VARCHAR(24), nullable=False),
+ Column('o_stat_error', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('pol_states', VARCHAR(24), nullable=False),
+ Column('facility_name', VARCHAR(48), nullable=False),
+ Column('instrument_name', VARCHAR(48), nullable=False),
+ Column('proposal_id', VARCHAR(48), nullable=False),
+ Column('project_code', VARCHAR(24), nullable=False),
+ Column('obs_session', VARCHAR(24), nullable=False),
+ Column('file_id', VARCHAR(64), nullable=False),
+ Column('file_set_id', VARCHAR(128), nullable=False),
+ Column('file_subset_id', VARCHAR(128), nullable=False),
+ Column('dataset', VARCHAR(128), nullable=False),
+ Column('image_file', VARCHAR(128), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), server_default=text("""\
+0
+""")),
+ Index('obsset_indx', 'file_set_id', 'file_id', 'arch_file_id'),
+ schema='E2EMGR'
+)
+
+
+t_obssummary = Table(
+ 'obssummary', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('source_id', VARCHAR(24), nullable=False, index=True),
+ Column('if_band', VARCHAR(4), nullable=False),
+ Column('sub_ref_freq', NUMBER(asdecimal=False), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('exposure', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_bandw', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(8), nullable=False),
+ Column('sub_num_chans', NUMBER(asdecimal=False), nullable=False),
+ Column('ra2000', NUMBER(asdecimal=False), nullable=False),
+ Column('dec2000', NUMBER(asdecimal=False), nullable=False),
+ Column('project_lock', VARCHAR(8), nullable=False),
+ Column('sub_pol1', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol2', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol3', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_pol4', NUMBER(asdecimal=False), nullable=False),
+ Column('subarray_id', NUMBER(asdecimal=False), nullable=False),
+ Column('corr_mode', VARCHAR(8), nullable=False),
+ Column('if_chan', VARCHAR(8), nullable=False),
+ Column('n_ants', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_file_id', NUMBER(asdecimal=False), nullable=False),
+ Column('sub_chan_id', NUMBER(asdecimal=False), nullable=False),
+ Column('n_images', NUMBER(asdecimal=False), nullable=False),
+ Column('obssummary_uid', VARCHAR(48), nullable=False, index=True),
+ Column('file_set_id', VARCHAR(64), nullable=False),
+ Column('radeg', NUMBER(asdecimal=False)),
+ Column('decdeg', NUMBER(asdecimal=False)),
+ Column('distance', NUMBER(asdecimal=False)),
+ Column('rms', NUMBER(asdecimal=False)),
+ Column('resol', NUMBER(asdecimal=False)),
+ Column('fov', NUMBER(asdecimal=False)),
+ Column('polar', VARCHAR(16)),
+ Index('obssummary2_idx', 'ra2000', 'dec2000'),
+ Index('obssummary4_idx', 'starttime', 'stoptime'),
+ Index('obssummary6_idx', 'radeg', 'decdeg'),
+ schema='E2EMGR'
+)
+
+
+t_obssummary_columns = Table(
+ 'obssummary_columns', metadata,
+ Column('column_name', VARCHAR(32), nullable=False, index=True),
+ Column('table_name', VARCHAR(32), nullable=False),
+ Column('description', VARCHAR(64), nullable=False),
+ Column('unit', VARCHAR(16), nullable=False),
+ Column('ucd', VARCHAR(64), nullable=False),
+ Column('utype', VARCHAR(64), nullable=False),
+ Column('datatype', VARCHAR(32), nullable=False),
+ Column('varsize', NUMBER(asdecimal=False), nullable=False),
+ Column('primary', VARCHAR(16), nullable=False),
+ Column('indexed', VARCHAR(16), nullable=False),
+ Column('std', VARCHAR(16), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_omsstnsched = Table(
+ 'omsstnsched', metadata,
+ Column('proposal', VARCHAR(5), nullable=False, index=True),
+ Column('segment', VARCHAR(2), nullable=False),
+ Column('staid', VARCHAR(5), nullable=False),
+ Column('start_time', DateTime, nullable=False),
+ Column('stop_time', DateTime, nullable=False),
+ Column('mjdstart', NUMBER(16, 8, True)),
+ Column('mjdstop', NUMBER(16, 8, True)),
+ Index('omsstnsched2_idx', 'mjdstart', 'mjdstop'),
+ schema='E2EMGR'
+)
+
+
+t_pidum = Table(
+ 'pidum', metadata,
+ Column('proposal', VARCHAR(16), nullable=False),
+ Column('pi_lastname', VARCHAR(48), nullable=False),
+ Column('pi_firstname', VARCHAR(48), nullable=False),
+ Column('pi_name', VARCHAR(64), nullable=False),
+ Column('peoplekey', NUMBER(asdecimal=False), nullable=False),
+ schema='E2EMGR'
+)
+
+
+t_pipequeue = Table(
+ 'pipequeue', metadata,
+ Column('project_code', VARCHAR(16), server_default=text("'none'")),
+ Column('arch_file_id', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('arch_file', VARCHAR(128), server_default=text("'none'")),
+ Column('file_set_type', VARCHAR(16), server_default=text("'RAW'")),
+ Column('sb_type', VARCHAR(32), server_default=text("'none'")),
+ Column('queuedate', VARCHAR(23), server_default=text("'none'")),
+ Column('status', VARCHAR(16), server_default=text("""\
+'none'
+""")),
+ Column('submitdate', VARCHAR(23), server_default=text("'none'")),
+ schema='E2EMGR'
+)
+
+
+t_pipequeuetemp = Table(
+ 'pipequeuetemp', metadata,
+ Column('project_code', VARCHAR(16), server_default=text("'none'")),
+ Column('arch_file_id', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('arch_file', VARCHAR(128), server_default=text("'none'")),
+ Column('file_set_type', VARCHAR(16), server_default=text("'RAW'")),
+ Column('sb_type', VARCHAR(32), server_default=text("'none'")),
+ Column('queuedate', DateTime, server_default=text("sysdate")),
+ Column('status', VARCHAR(16), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+t_project = Table(
+ 'project', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('observer', VARCHAR(48), nullable=False, index=True),
+ Column('observer_id', NUMBER(asdecimal=False), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('proprietary', NUMBER(asdecimal=False), nullable=False, index=True),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(36), nullable=False),
+ Column('obs_bands', VARCHAR(24), nullable=False),
+ Column('total_obs_time', NUMBER(asdecimal=False), nullable=False),
+ Column('num_segments', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_files', NUMBER(asdecimal=False), nullable=False),
+ Column('row_date', NUMBER(asdecimal=False), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False, index=True),
+ Column('project_lock', VARCHAR(8), nullable=False, index=True),
+ Column('unlock_expire', NUMBER(asdecimal=False), nullable=False),
+ Column('grant_access', VARCHAR(12), nullable=False),
+ Column('proprietary_duration', NUMBER(asdecimal=False), nullable=False),
+ Column('warned', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('project_id', NUMBER(asdecimal=False), server_default=text("0")),
+ Index('project2_idx', 'starttime', 'stoptime'),
+ schema='E2EMGR'
+)
+
+
+t_projectgbt = Table(
+ 'projectgbt', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('observer', VARCHAR(48), nullable=False),
+ Column('observer_id', NUMBER(asdecimal=False), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('proprietary', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(24), nullable=False),
+ Column('obs_bands', VARCHAR(24), nullable=False),
+ Column('total_obs_time', NUMBER(asdecimal=False), nullable=False),
+ Column('num_segments', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_files', NUMBER(asdecimal=False), nullable=False),
+ Column('row_date', NUMBER(asdecimal=False), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False),
+ Column('project_lock', VARCHAR(8), nullable=False),
+ Column('unlock_expire', NUMBER(asdecimal=False), nullable=False),
+ Column('grant_access', VARCHAR(12), nullable=False),
+ Column('proprietary_duration', NUMBER(asdecimal=False), nullable=False),
+ Column('warned', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('project_id', NUMBER(asdecimal=False), server_default=text("""\
+0
+ """)),
+ schema='E2EMGR'
+)
+
+
+t_projectgbtold = Table(
+ 'projectgbtold', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('observer', VARCHAR(48), nullable=False),
+ Column('observer_id', NUMBER(asdecimal=False), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('proprietary', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(24), nullable=False),
+ Column('obs_bands', VARCHAR(24), nullable=False),
+ Column('total_obs_time', NUMBER(asdecimal=False), nullable=False),
+ Column('num_segments', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_files', NUMBER(asdecimal=False), nullable=False),
+ Column('row_date', NUMBER(asdecimal=False), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False),
+ Column('project_lock', VARCHAR(8), nullable=False),
+ Column('unlock_expire', NUMBER(asdecimal=False), nullable=False),
+ Column('grant_access', VARCHAR(12), nullable=False),
+ Column('proprietary_duration', NUMBER(asdecimal=False), nullable=False),
+ Column('warned', NUMBER(asdecimal=False), server_default=text("0.0")),
+ Column('project_id', NUMBER(asdecimal=False), server_default=text("""\
+0
+ """)),
+ schema='E2EMGR'
+)
+
+
+t_proposal = Table(
+ 'proposal', metadata,
+ Column('project_code', VARCHAR(16), nullable=False, index=True),
+ Column('pi_name', VARCHAR(48), nullable=False, index=True),
+ Column('pi_email', VARCHAR(64), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False, index=True),
+ schema='E2EMGR'
+)
+
+
+t_proprietary = Table(
+ 'proprietary', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('observer', VARCHAR(24), nullable=False),
+ Column('proprietary', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False),
+ Column('keyword', VARCHAR(12), nullable=False),
+ Column('date_expires', NUMBER(asdecimal=False), nullable=False),
+ Column('entry_date', NUMBER(asdecimal=False), nullable=False),
+ Column('email_addr', VARCHAR(64), nullable=False),
+ Index('proprietary_idx', 'project_code', 'observer'),
+ schema='E2EMGR'
+)
+
+
+t_segment = Table(
+ 'segment', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('segment', VARCHAR(4), nullable=False),
+ Column('observer', VARCHAR(36), nullable=False, index=True),
+ Column('observer_id', NUMBER(asdecimal=False), nullable=False),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('proprietary', NUMBER(asdecimal=False), nullable=False),
+ Column('telescope', VARCHAR(12), nullable=False),
+ Column('telescope_config', VARCHAR(24), nullable=False),
+ Column('obs_bands', VARCHAR(16), nullable=False),
+ Column('total_obs_time', NUMBER(asdecimal=False), nullable=False),
+ Column('num_scans', NUMBER(asdecimal=False), nullable=False),
+ Column('arch_files', NUMBER(asdecimal=False), nullable=False),
+ Column('row_date', NUMBER(asdecimal=False), nullable=False),
+ Column('raw_project_code', VARCHAR(16), nullable=False, index=True),
+ Column('project_lock', VARCHAR(8), nullable=False),
+ Column('unlock_expire', NUMBER(asdecimal=False), nullable=False),
+ Column('grant_access', VARCHAR(12), nullable=False),
+ Column('proprietary_duration', NUMBER(asdecimal=False), nullable=False),
+ Index('segment1_idx', 'project_code', 'segment'),
+ Index('segment2_idx', 'starttime', 'stoptime'),
+ schema='E2EMGR'
+)
+
+
+t_shiporders = Table(
+ 'shiporders', metadata,
+ Column('file_set_id', VARCHAR(96), nullable=False),
+ Column('order_id', NUMBER(asdecimal=False), nullable=False),
+ Column('user_id', NUMBER(asdecimal=False), nullable=False),
+ Column('emailaddr', VARCHAR(48), server_default=text("'none'")),
+ Column('submission_date', VARCHAR(48), server_default=text("'none'")),
+ Column('status', VARCHAR(24), server_default=text("""\
+'none'
+""")),
+ Column('useros', VARCHAR(24), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+class Siav2image(Base):
+ __tablename__ = 'siav2image'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ id = Column(VARCHAR(32), primary_key=True)
+ obs_id = Column(VARCHAR(32), server_default=text("null"))
+ archive_id = Column(VARCHAR(128), nullable=False)
+ htm_id = Column(NUMBER(14, 0, False), server_default=text("0"))
+ preview_id = Column(VARCHAR(128), nullable=False)
+ access_format = Column(VARCHAR(32), nullable=False)
+ access_estsize = Column(NUMBER(14, 0, False), server_default=text("0"))
+ dataproduct_type = Column(VARCHAR(16), server_default=text("null"))
+ dataproduct_subtype = Column(VARCHAR(32), server_default=text("null"))
+ calib_level = Column(NUMBER(6, 0, False), server_default=text("0"))
+ dataset_length = Column(NUMBER(14, 0, False), server_default=text("0"))
+ im_nsubarrays = Column(NUMBER(8, 0, False), server_default=text("1"))
+ im_naxes = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis1 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis2 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis3 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis4 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_pixtype = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes1 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes2 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes3 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes4 = Column(VARCHAR(16), server_default=text("null"))
+ im_ra1 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec1 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra2 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec2 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra3 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec3 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra4 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec4 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_scale = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ obs_title = Column(VARCHAR(128), server_default=text("null"))
+ obs_creator_name = Column(VARCHAR(32), server_default=text("null"))
+ obs_collection = Column(VARCHAR(32), server_default=text("null"))
+ obs_creator_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_publisher_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_dataset_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_release_date = Column(VARCHAR(20), server_default=text("null"))
+ obs_creation_date = Column(VARCHAR(20), server_default=text("null"))
+ obs_creation_type = Column(VARCHAR(16), server_default=text("null"))
+ facility_name = Column(VARCHAR(20), server_default=text("null"))
+ instrument_name = Column(VARCHAR(20), server_default=text("null"))
+ obs_bandpass = Column(VARCHAR(20), server_default=text("null"))
+ obs_datasource = Column(VARCHAR(20), server_default=text("null"))
+ proposal_id = Column(VARCHAR(20), server_default=text("null"))
+ target_name = Column(VARCHAR(20), server_default=text("null"))
+ target_class = Column(VARCHAR(20), server_default=text("null"))
+ s_ra = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ s_dec = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ s_fov = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ s_region = Column(VARCHAR(128), server_default=text("null"))
+ s_calib_status = Column(VARCHAR(32), server_default=text("'none'"))
+ s_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_min = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_max = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_respower = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_min = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_max = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_exptime = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ o_ucd = Column(VARCHAR(20), server_default=text("null"))
+ o_unit = Column(VARCHAR(32), server_default=text("'none'"))
+ pol_states = Column(VARCHAR(20), server_default=text("null"))
+ project_code = Column(VARCHAR(24), server_default=text("null"))
+ obs_session = Column(VARCHAR(24), server_default=text("null"))
+ file_id = Column(VARCHAR(64), server_default=text("null"))
+ file_set_id = Column(VARCHAR(128), server_default=text("null"))
+ file_subset_id = Column(VARCHAR(128), server_default=text("null"))
+ dataset = Column(VARCHAR(128), server_default=text("null"))
+ image_file = Column(VARCHAR(128), server_default=text("null"))
+ obs_file_id = Column(NUMBER(asdecimal=False), server_default=text("0"))
+ cal_file_id = Column(NUMBER(asdecimal=False), server_default=text("0"))
+ root_dir = Column(VARCHAR(128), server_default=text("null"))
+ file_dir = Column(VARCHAR(128), server_default=text("""\
+null
+"""))
+
+
+t_subarray = Table(
+ 'subarray', metadata,
+ Column('project_code', VARCHAR(16), nullable=False),
+ Column('segment', VARCHAR(4)),
+ Column('starttime', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptime', NUMBER(asdecimal=False), nullable=False),
+ Column('subarray_id', NUMBER(asdecimal=False), nullable=False),
+ Column('antenna_id', NUMBER(asdecimal=False), nullable=False),
+ Index('subarray_idx', 'project_code', 'starttime', 'stoptime', 'subarray_id'),
+ schema='E2EMGR'
+)
+
+
+t_sumlist = Table(
+ 'sumlist', metadata,
+ Column('programid', VARCHAR(8), nullable=False, index=True),
+ Column('sourcename', VARCHAR(16), nullable=False, index=True),
+ Column('qualifier', VARCHAR(2), nullable=False),
+ Column('observerid', NUMBER(asdecimal=False), nullable=False),
+ Column('observername', VARCHAR(20), nullable=False),
+ Column('centerofbw1', NUMBER(asdecimal=False), nullable=False),
+ Column('centerofbw2', NUMBER(asdecimal=False), nullable=False),
+ Column('centerofbw3', NUMBER(asdecimal=False), nullable=False),
+ Column('centerofbw4', NUMBER(asdecimal=False), nullable=False),
+ Column('bandwidth1', NUMBER(asdecimal=False), nullable=False),
+ Column('bandwidth2', NUMBER(asdecimal=False), nullable=False),
+ Column('bandwidth3', NUMBER(asdecimal=False), nullable=False),
+ Column('bandwidth4', NUMBER(asdecimal=False), nullable=False),
+ Column('netsideband1', VARCHAR(2), nullable=False),
+ Column('netsideband2', VARCHAR(2), nullable=False),
+ Column('netsideband3', VARCHAR(2), nullable=False),
+ Column('netsideband4', VARCHAR(2), nullable=False),
+ Column('obsmode', VARCHAR(3), nullable=False),
+ Column('calcode', VARCHAR(3), nullable=False),
+ Column('correlator', VARCHAR(4), nullable=False),
+ Column('planetary', VARCHAR(2), nullable=False),
+ Column('subarrayid', VARCHAR(2), nullable=False),
+ Column('epoch', VARCHAR(5), nullable=False),
+ Column('ra1950', NUMBER(asdecimal=False), nullable=False),
+ Column('dec1950', NUMBER(asdecimal=False), nullable=False),
+ Column('ra2000', NUMBER(asdecimal=False), nullable=False),
+ Column('dec2000', NUMBER(asdecimal=False), nullable=False),
+ Column('mjad', DateTime, nullable=False, index=True),
+ Column('antennae', NUMBER(asdecimal=False), nullable=False),
+ Column('starttimelst', NUMBER(asdecimal=False), nullable=False),
+ Column('starttimeiat', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptimelst', NUMBER(asdecimal=False), nullable=False),
+ Column('stoptimeiat', NUMBER(asdecimal=False), nullable=False),
+ Column('timeonsource', NUMBER(asdecimal=False), nullable=False),
+ Column('arrayconfig', VARCHAR(6), nullable=False),
+ Column('numchan1', NUMBER(asdecimal=False), nullable=False),
+ Column('numchan2', NUMBER(asdecimal=False), nullable=False),
+ Column('numchan3', NUMBER(asdecimal=False), nullable=False),
+ Column('numchan4', NUMBER(asdecimal=False), nullable=False),
+ Column('inttime', NUMBER(asdecimal=False), nullable=False),
+ Column('velocity1', NUMBER(asdecimal=False), nullable=False),
+ Column('velocity2', NUMBER(asdecimal=False), nullable=False),
+ Column('velocity3', NUMBER(asdecimal=False), nullable=False),
+ Column('velocity4', NUMBER(asdecimal=False), nullable=False),
+ Column('restfreq1', NUMBER(asdecimal=False), nullable=False),
+ Column('restfreq2', NUMBER(asdecimal=False), nullable=False),
+ Column('restfreq3', NUMBER(asdecimal=False), nullable=False),
+ Column('restfreq4', NUMBER(asdecimal=False), nullable=False),
+ Column('velsyscode', VARCHAR(10), nullable=False),
+ Column('apoptions', VARCHAR(2), nullable=False),
+ Column('dataselect1', NUMBER(asdecimal=False), nullable=False),
+ Column('dataselect2', NUMBER(asdecimal=False), nullable=False),
+ Column('dataselect3', NUMBER(asdecimal=False), nullable=False),
+ Column('dataselect4', NUMBER(asdecimal=False), nullable=False),
+ Column('chansep1', NUMBER(asdecimal=False), nullable=False),
+ Column('chansep2', NUMBER(asdecimal=False), nullable=False),
+ Column('chansep3', NUMBER(asdecimal=False), nullable=False),
+ Column('chansep4', NUMBER(asdecimal=False), nullable=False),
+ Column('oldtapenum', VARCHAR(8), nullable=False),
+ Column('newtapenum', VARCHAR(8), nullable=False),
+ Column('filenumber', NUMBER(asdecimal=False), nullable=False),
+ Index('sumlist4_idx', 'starttimeiat', 'stoptimeiat'),
+ Index('sumlist5_idx', 'newtapenum', 'filenumber'),
+ Index('sumlist3_idx', 'ra2000', 'dec2000'),
+ schema='E2EMGR'
+)
+
+
+t_validity_chk = Table(
+ 'validity_chk', metadata,
+ Column('arch_file_id', NUMBER(asdecimal=False), server_default=text("0")),
+ Column('file_type', VARCHAR(16), server_default=text("'none'")),
+ Column('last_checked', DateTime, server_default=text("sysdate")),
+ Column('status', VARCHAR(16), server_default=text("""\
+'none'
+""")),
+ schema='E2EMGR'
+)
+
+
+class VlapipeImage(Base):
+ __tablename__ = 'vlapipe_image'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ id = Column(VARCHAR(32), primary_key=True)
+ obs_id = Column(VARCHAR(32), server_default=text("null"))
+ archive_id = Column(VARCHAR(128), nullable=False)
+ htm_id = Column(NUMBER(14, 0, False), server_default=text("0"))
+ preview_id = Column(VARCHAR(128), nullable=False)
+ access_format = Column(VARCHAR(32), nullable=False)
+ access_estsize = Column(NUMBER(14, 0, False), server_default=text("0"))
+ dataproduct_type = Column(VARCHAR(16), server_default=text("null"))
+ dataproduct_subtype = Column(VARCHAR(32), server_default=text("null"))
+ calib_level = Column(NUMBER(6, 0, False), server_default=text("0"))
+ dataset_length = Column(NUMBER(14, 0, False), server_default=text("0"))
+ im_nsubarrays = Column(NUMBER(8, 0, False), server_default=text("1"))
+ im_naxes = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis1 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis2 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis3 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_naxis4 = Column(NUMBER(8, 0, False), server_default=text("0"))
+ im_pixtype = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes1 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes2 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes3 = Column(VARCHAR(16), server_default=text("null"))
+ im_wcsaxes4 = Column(VARCHAR(16), server_default=text("null"))
+ im_ra1 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec1 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra2 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec2 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra3 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec3 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_ra4 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_dec4 = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ im_scale = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ obs_title = Column(VARCHAR(128), server_default=text("null"))
+ obs_creator_name = Column(VARCHAR(32), server_default=text("null"))
+ obs_collection = Column(VARCHAR(32), server_default=text("null"))
+ obs_creator_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_publisher_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_dataset_did = Column(VARCHAR(160), server_default=text("null"))
+ obs_release_date = Column(VARCHAR(20), server_default=text("null"))
+ obs_creation_date = Column(VARCHAR(20), server_default=text("null"))
+ obs_creation_type = Column(VARCHAR(16), server_default=text("null"))
+ facility_name = Column(VARCHAR(20), server_default=text("null"))
+ instrument_name = Column(VARCHAR(20), server_default=text("null"))
+ obs_bandpass = Column(VARCHAR(20), server_default=text("null"))
+ obs_datasource = Column(VARCHAR(20), server_default=text("null"))
+ proposal_id = Column(VARCHAR(20), server_default=text("null"))
+ target_name = Column(VARCHAR(20), server_default=text("null"))
+ target_class = Column(VARCHAR(20), server_default=text("null"))
+ s_ra = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ s_dec = Column(NUMBER(asdecimal=False), index=True, server_default=text("0.0"))
+ s_fov = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ s_region = Column(VARCHAR(128), server_default=text("null"))
+ s_calib_status = Column(VARCHAR(32), server_default=text("'none'"))
+ s_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_min = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_max = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ em_respower = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_min = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_max = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_exptime = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ t_resolution = Column(NUMBER(asdecimal=False), server_default=text("0.0"))
+ o_ucd = Column(VARCHAR(20), server_default=text("null"))
+ o_unit = Column(VARCHAR(32), server_default=text("'none'"))
+ pol_states = Column(VARCHAR(20), server_default=text("null"))
+ project_code = Column(VARCHAR(24), server_default=text("null"))
+ obs_session = Column(VARCHAR(24), server_default=text("null"))
+ file_id = Column(VARCHAR(64), server_default=text("null"))
+ file_set_id = Column(VARCHAR(128), server_default=text("null"))
+ file_subset_id = Column(VARCHAR(128), server_default=text("null"))
+ dataset = Column(VARCHAR(128), server_default=text("null"))
+ image_file = Column(VARCHAR(128), server_default=text("null"))
+ obs_file_id = Column(VARCHAR(64), server_default=text("null"))
+ cal_file_id = Column(VARCHAR(64), server_default=text("null"))
+ root_dir = Column(VARCHAR(128), server_default=text("null"))
+ file_dir = Column(VARCHAR(128), server_default=text("""\
+null
+"""))
+
+
+t_weather = Table(
+ 'weather', metadata,
+ Column('telescope', VARCHAR(8), nullable=False, index=True),
+ Column('timestamp', NUMBER(12, 6, True), nullable=False, index=True, comment='MJD (decimal)'),
+ Column('pressure', NUMBER(7, 2, True), nullable=False, comment='millibars'),
+ Column('temp', NUMBER(7, 2, True), nullable=False, comment='degrees C'),
+ Column('dewtemp', NUMBER(7, 2, True), nullable=False, comment='degrees C'),
+ Column('windspeed', NUMBER(6, 2, True), nullable=False, comment='m/sec'),
+ Column('winddir', NUMBER(6, 1, True), nullable=False, comment='degrees from North'),
+ schema='E2EMGR'
+)
+
+
+class Wensscatalog(Base):
+ __tablename__ = 'wensscatalog'
+ __table_args__ = {'schema': 'E2EMGR'}
+
+ wensscode = Column(VARCHAR(3), nullable=False)
+ name = Column(VARCHAR(13), primary_key=True, nullable=False)
+ raradb = Column(NUMBER(asdecimal=False), nullable=False)
+ decradb = Column(NUMBER(asdecimal=False), nullable=False)
+ raradj = Column(NUMBER(asdecimal=False), primary_key=True, nullable=False)
+ decradj = Column(NUMBER(asdecimal=False), primary_key=True, nullable=False)
+ srctype = Column(VARCHAR(1), nullable=False)
+ warn = Column(VARCHAR(1), nullable=False)
+ pflux = Column(NUMBER(asdecimal=False), nullable=False)
+ flux = Column(NUMBER(asdecimal=False), nullable=False)
+ majorax = Column(NUMBER(asdecimal=False), nullable=False)
+ minorax = Column(NUMBER(asdecimal=False), nullable=False)
+ posangle = Column(NUMBER(asdecimal=False), nullable=False)
+ rms = Column(NUMBER(asdecimal=False), nullable=False)
+ field = Column(VARCHAR(9), nullable=False)
+ survey = Column(VARCHAR(10), nullable=False)
+ rarad = Column(NUMBER(asdecimal=False))
+ decrad = Column(NUMBER(asdecimal=False))
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/logs.py b/apps/cli/executables/pexable/ingest/ingest/schema/logs.py
new file mode 100644
index 000000000..3ec7fbf08
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/logs.py
@@ -0,0 +1,26 @@
+import pendulum
+
+from . import create_session
+from .model import t_logs as logs
+
+session = create_session("SDM")
+
+
+def since(time):
+ query = logs.select().where(logs.c.timestamp > time).order_by("timestamp")
+ return [dict(r) for r in session.execute(query)]
+
+
+def for_request(request_id):
+ query = (
+ logs.select().where(logs.c.properties["requestId"].astext == str(request_id)).order_by("timestamp").limit(100)
+ )
+ return list(dict(r) for r in session.execute(query))
+
+
+if __name__ == "__main__":
+ print("since:")
+ print(since(pendulum.now().subtract(hours=1)))
+ print()
+ print("for 13099859:")
+ print(for_request(13099859))
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/model.py b/apps/cli/executables/pexable/ingest/ingest/schema/model.py
new file mode 100644
index 000000000..5095fec75
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/model.py
@@ -0,0 +1,1137 @@
+# coding: utf-8
+from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, Date, DateTime, Float, ForeignKey, Integer, JSON, Numeric, String, Table, Text, UniqueConstraint, text, ForeignKeyConstraint
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+class Account(Base):
+ __tablename__ = 'account'
+ __table_args__ = (
+ UniqueConstraint('provider', 'provider_principal'),
+ )
+
+ account_id = Column(String, primary_key=True)
+ request_handler_id = Column(Numeric)
+ version = Column(Numeric, nullable=False, server_default=text("1"))
+ password_digest = Column(String)
+ firstname = Column(String, nullable=False)
+ lastname = Column(String, nullable=False)
+ initials = Column(String)
+ creationtimestamp = Column(String)
+ modificationtimestamp = Column(String)
+ preferredarc = Column(String)
+ email = Column(String)
+ executive = Column(String, nullable=False, server_default=text("'na'::character varying"))
+ aliases = Column(String)
+ inst_no = Column(Numeric, nullable=False, server_default=text("1"))
+ provider = Column(String)
+ provider_principal = Column(String)
+
+ role = relationship('Role', secondary='account_role')
+
+
+t_account_role = Table(
+ 'account_role', metadata,
+ Column('role_no', ForeignKey('role.role_no'), nullable=False),
+ Column('account_id', ForeignKey('account.account_id'), nullable=False)
+)
+
+
+class AlmaOusType(Base):
+ __tablename__ = 'alma_ous_types'
+
+ ous_type = Column(String, primary_key=True)
+ description = Column(String)
+
+
+t_alma_ous_view = Table(
+ 'alma_ous_view', metadata,
+ Column('project_code', String),
+ Column('sous', String),
+ Column('gous', String),
+ Column('mous', String),
+ Column('schedblock_name', String)
+)
+
+
+class AlmaOus(Base):
+ __tablename__ = 'alma_ouses'
+
+ alma_ous_id = Column(String, primary_key=True)
+ parent_ous_id = Column(ForeignKey('alma_ouses.alma_ous_id'))
+ project_code = Column(ForeignKey('projects.project_code'))
+ ous_type = Column(ForeignKey('alma_ous_types.ous_type'), nullable=False)
+ schedblock_name = Column(String)
+
+ alma_ous_type = relationship('AlmaOusType')
+ parent_ous = relationship('AlmaOus', remote_side=[alma_ous_id], backref=backref('children_ouses', cascade="all, delete-orphan"))
+ calibrations = relationship('Calibration')
+ execution_blocks = relationship('ExecutionBlock')
+ project = relationship('Project')
+
+
+class AlmaSourceDatum(Base):
+ __tablename__ = 'alma_source_data'
+
+ alma_ous_id = Column(String, primary_key=True, nullable=False)
+ field_name = Column(String, primary_key=True, nullable=False)
+ ra = Column(Float(53))
+ dec = Column(Float(53))
+ integration_time = Column(Float(53))
+ angular_resolution = Column(Float(53))
+ largest_angular_scale = Column(Float(53))
+ science_field = Column(Boolean, server_default=text("false"))
+
+ spectral_windows = relationship('AlmaSpwDatum', secondary='alma_spw_sources', back_populates='sources')
+
+ def __repr__(self):
+ return "<AlmaSourceDatum#{alma_ous_id} {field_name} {ra} {dec} {integration_time} {angular_resolution} {largest_angular_scale}>" \
+ .format(**self.__dict__)
+
+
+
+class AlmaSpwDatum(Base):
+ __tablename__ = 'alma_spw_data'
+
+ alma_ous_id = Column(String, primary_key=True, nullable=False)
+ spw_name = Column(String, primary_key=True, nullable=False)
+ min_frequency = Column(Float(53))
+ max_frequency = Column(Float(53))
+ bandwidth = Column(Float(53))
+ num_channels = Column(Integer)
+ spectral_resolution = Column(Float(53))
+
+ sources = relationship('AlmaSourceDatum', secondary='alma_spw_sources', back_populates='spectral_windows')
+
+ def __repr__(self):
+ return "<AlmaSpwDatum#{alma_ous_id} {spw_name} {min_frequency} {max_frequency} {bandwidth} {num_channels} {spectral_resolution}>" \
+ .format(**self.__dict__)
+
+
+t_alma_spw_sources = Table(
+ 'alma_spw_sources', metadata,
+ Column('alma_ous_id', String, primary_key=True, nullable=False),
+ Column('spw_name', String, primary_key=True, nullable=False),
+ Column('field_name', String, primary_key=True, nullable=False),
+ ForeignKeyConstraint(['alma_ous_id', 'field_name'], ['alma_source_data.alma_ous_id', 'alma_source_data.field_name']),
+ ForeignKeyConstraint(['alma_ous_id', 'spw_name'], ['alma_spw_data.alma_ous_id', 'alma_spw_data.spw_name'])
+)
+
+
+t_alma_project_codes = Table(
+ 'alma_project_codes', metadata,
+ Column('code', String),
+ Column('uid', String)
+)
+
+
+class AlmaReingestionQueue(Base):
+ __tablename__ = 'alma_reingestion_queue'
+
+ asdm_uid = Column(String, primary_key=True)
+ observation_finished = Column(DateTime(True), nullable=False)
+ last_updated = Column(DateTime(True), nullable=False, server_default=text("now()"))
+ state = Column(String, nullable=False, server_default=text("'WAITING'::character varying"))
+
+
+# class AlmaSourceDatum(Base):
+# __tablename__ = 'alma_source_data'
+#
+# alma_ous_id = Column(String, primary_key=True, nullable=False)
+# field_name = Column(String, primary_key=True, nullable=False)
+# ra = Column(Float(53))
+# dec = Column(Float(53))
+# integration_time = Column(Float(53))
+# angular_resolution = Column(Float(53))
+# largest_angular_scale = Column(Float(53))
+#
+# alma_ouss = relationship('AlmaSpwDatum', secondary='alma_spw_sources')
+
+
+# class AlmaSpwDatum(Base):
+# __tablename__ = 'alma_spw_data'
+#
+# alma_ous_id = Column(String, primary_key=True, nullable=False)
+# spw_number = Column(Integer, primary_key=True, nullable=False)
+# min_frequency = Column(Float(53))
+# max_frequency = Column(Float(53))
+# bandwidth = Column(Float(53))
+# num_channels = Column(Integer)
+# spectral_resolution = Column(Float(53))
+
+
+# t_alma_spw_sources = Table(
+# 'alma_spw_sources', metadata,
+# Column('alma_ous_id', String, primary_key=True, nullable=False),
+# Column('spw_number', Integer, primary_key=True, nullable=False),
+# Column('field_name', String, primary_key=True, nullable=False),
+# ForeignKeyConstraint(['alma_ous_id', 'field_name'], ['alma_source_data.alma_ous_id', 'alma_source_data.field_name']),
+# ForeignKeyConstraint(['alma_ous_id', 'spw_number'], ['alma_spw_data.alma_ous_id', 'alma_spw_data.spw_number'])
+# )
+
+
+class AncillaryProduct(Base):
+ __tablename__ = 'ancillary_products'
+
+ ancillary_product_locator = Column(String, primary_key=True)
+ ancillary_product_type = Column(String, nullable=False)
+ filegroup_id = Column(ForeignKey('filegroups.filegroup_id'), nullable=False)
+ product_group_id = Column(ForeignKey('product_groups.product_group_id'))
+ science_product_locator = Column(ForeignKey('science_products.science_product_locator'))
+
+ filegroup = relationship('Filegroup')
+ product_group = relationship('ProductGroup')
+ science_product = relationship('ScienceProduct', backref=backref('ancillary_products'))
+
+ @property
+ def locator(self):
+ return self.ancillary_product_locator
+
+ @property
+ def subdirectory(self):
+ return ''
+
+
+class Archive(Base):
+ __tablename__ = 'archive'
+
+ id = Column(Numeric, primary_key=True)
+ name = Column(String(10), nullable=False)
+
+
+class ArchiveLog(Base):
+ __tablename__ = 'archive_log'
+ __table_args__ = (
+ UniqueConstraint('request_id', 'dataset_name'),
+ )
+
+ log_id = Column(Numeric, primary_key=True, server_default=text("nextval('log_sequence'::regclass)"))
+ ip = Column(String)
+ host_name = Column(String)
+ account_id = Column(Numeric)
+ country = Column(String)
+ local_host_name = Column(String)
+ user_agent = Column(String)
+ start_date = Column(DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP"))
+ program = Column(String, nullable=False)
+ version = Column(String)
+ resource_name = Column(String)
+ query = Column(String)
+ size_lines = Column(Numeric)
+ size_bytes_wire = Column(Numeric)
+ size_bytes_uncompressed = Column(Numeric)
+ duration = Column(Numeric)
+ medium = Column(String)
+ user_category = Column(Numeric)
+ pi_request = Column(String(1))
+ member_state = Column(String(1))
+ eso_user = Column(String(1))
+ error_code = Column(Numeric)
+ error_string = Column(String)
+ file_id = Column(String)
+ archive_class = Column(String)
+ service_type = Column(String)
+ dataset_name = Column(String)
+ request_id = Column(Numeric)
+ archive_id = Column(ForeignKey('archive.id'))
+
+ archive = relationship('Archive')
+
+
+class Author(Base):
+ __tablename__ = 'authors'
+
+ author_id = Column(Integer, primary_key=True, server_default=text("nextval('authors_author_id_seq'::regclass)"))
+ project_code = Column(ForeignKey('projects.project_code', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+ username = Column(String, nullable=False)
+ firstname = Column(String, nullable=False)
+ lastname = Column(String, nullable=False)
+ pst_person_id = Column(String)
+ is_pi = Column(Boolean, nullable=False)
+
+ project = relationship('Project')
+
+ def __repr__(self):
+ return "<Author#{author_id} {username} '{firstname} {lastname}'>" \
+ .format(**self.__dict__)
+
+
+class CalibrationStatusValue(Base):
+ __tablename__ = 'calibration_status_values'
+
+ status = Column(String, primary_key=True)
+ description = Column(String, nullable=False)
+
+
+class Calibration(Base):
+ __tablename__ = 'calibrations'
+
+ calibration_id = Column(Integer, primary_key=True, server_default=text("nextval('calibrations_calibration_id_seq'::regclass)"))
+ alma_ous_id = Column(ForeignKey('alma_ouses.alma_ous_id'))
+ project_code = Column(ForeignKey('projects.project_code'), nullable=False)
+ execution_block_id = Column(ForeignKey('execution_blocks.execution_block_id'))
+ filegroup_id = Column(ForeignKey('filegroups.filegroup_id'), nullable=False)
+ science_product_locator = Column(ForeignKey('science_products.science_product_locator'), unique=True)
+
+ alma_ous = relationship('AlmaOus')
+ execution_block = relationship('ExecutionBlock')
+ filegroup = relationship('Filegroup')
+ project = relationship('Project')
+ science_product = relationship('ScienceProduct', uselist=False)
+
+ def __repr__(self):
+ return "<Calibration#{calibration_id} {execution_block_id} '{alma_ous_id} {filegroup_id} {project_code}'>" \
+ .format(**self.__dict__)
+
+class VLASSCalibration(Calibration):
+ __tablename__ = 'VLASS_calibrations'
+
+ calibration_id = Column(ForeignKey('calibrations.calibration_id'), primary_key=True)
+
+
+class Catalog(Base):
+ __tablename__ = 'catalogs'
+
+ catalog_id = Column(Integer, primary_key=True, server_default=text("nextval('catalogs_catalog_id_seq'::regclass)"))
+ science_product_locator = Column(ForeignKey('science_products.science_product_locator'), nullable=False)
+
+ science_product = relationship('ScienceProduct')
+
+
+class VlassCatalog(Catalog):
+ __tablename__ = 'vlass_catalogs'
+
+ catalog_id = Column(ForeignKey('catalogs.catalog_id'), primary_key=True)
+ tile = Column(String)
+ type = Column(String)
+
+
+class Configuration(Base):
+ __tablename__ = 'configurations'
+
+ configuration_id = Column(Integer, primary_key=True, server_default=text("nextval('configurations_configuration_id_seq'::regclass)"))
+ configuration = Column(Integer, nullable=False)
+ execution_block_id = Column(ForeignKey('execution_blocks.execution_block_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False, index=True)
+
+ execution_block = relationship('ExecutionBlock')
+
+ def __repr__(self):
+ return "<Configuration#{configuration_id}>".format(**self.__dict__)
+
+
+
+class DataDescription(Base):
+ __tablename__ = 'data_descriptions'
+
+ data_description_id = Column(Integer, primary_key=True, server_default=text("nextval('data_descriptions_data_description_id_seq'::regclass)"))
+ bandwidth = Column(Float(53), nullable=False)
+ frequency = Column(Float(53), nullable=False)
+ polarization_id = Column(ForeignKey('polarizations.polarization_id'), nullable=False)
+ configuration_id = Column(ForeignKey('configurations.configuration_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False, index=True)
+
+ configuration = relationship('Configuration')
+ polarization = relationship('Polarization')
+ subscans = relationship('Subscan', secondary='subscan_data_descriptions')
+
+ def __repr__(self):
+ return "<DataDescription#{data_description_id} Freq={frequency} BW={bandwidth}>" \
+ .format(**self.__dict__)
+
+class Databasechangelog(Base):
+ __tablename__ = 'databasechangelog'
+
+ id = Column(String(63), primary_key=True, nullable=False)
+ author = Column(String(63), primary_key=True, nullable=False)
+ filename = Column(String(200), primary_key=True, nullable=False)
+ dateexecuted = Column(DateTime(True), nullable=False)
+ orderexecuted = Column(Integer, nullable=False)
+ exectype = Column(String(10), nullable=False)
+ md5sum = Column(String(35))
+ description = Column(String(255))
+ comments = Column(String(255))
+ tag = Column(String(255))
+ liquibase = Column(String(20))
+ contexts = Column(String(255))
+ labels = Column(String(255))
+ deployment_id = Column(String(10))
+
+
+class Databasechangeloglock(Base):
+ __tablename__ = 'databasechangeloglock'
+
+ id = Column(Integer, primary_key=True)
+ locked = Column(Boolean, nullable=False)
+ lockgranted = Column(DateTime(True))
+ lockedby = Column(String(255))
+
+
+class Event(Base):
+ __tablename__ = 'events'
+
+ id = Column(Integer, primary_key=True, server_default=text("nextval('events_id_seq'::regclass)"))
+ application = Column(String, nullable=False)
+ user = Column(String)
+ request = Column(String)
+ message = Column(String, nullable=False)
+ log_data = Column(JSON)
+ version = Column(Float(53), nullable=False)
+ timestamp = Column(DateTime(True), nullable=False)
+
+ def __repr__(self):
+ return "<Event#{id} app={application} user={user} request={request} timestamp={timestamp}>" \
+ .format(**self.__dict__)
+
+
+t_execblock_start_stop = Table(
+ 'execblock_start_stop', metadata,
+ Column('execution_block_id', Integer),
+ Column('starttime', Float(53)),
+ Column('endtime', Float(53))
+)
+
+
+class ExecutionBlock(Base):
+ __tablename__ = 'execution_blocks'
+ __table_args__ = (
+ CheckConstraint("(((telescope)::text = 'EVLA'::text) AND (ngas_fileset_id IS NOT NULL)) OR ((telescope)::text <> 'EVLA'::text)"),
+ )
+
+ execution_block_id = Column(Integer, primary_key=True, server_default=text("nextval('execution_blocks_execution_block_id_seq'::regclass)"))
+ ost_exec_block_id = Column(Integer)
+ filegroup_id = Column(ForeignKey('filegroups.filegroup_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+ calibration_level = Column(String, nullable=False)
+ telescope = Column(String, nullable=False)
+ configuration = Column(String)
+ scheduling_block_id = Column(Integer)
+ ngas_fileset_id = Column(String)
+ project_code = Column(ForeignKey('projects.project_code', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+ starttime = Column(Float(53))
+ endtime = Column(Float(53))
+ calibration_status = Column(ForeignKey('calibration_status_values.status'), nullable=False, server_default=text("'Unknown'::character varying"))
+ scheduling_block_type = Column(String)
+ band_code = Column(String)
+ ingestion_complete = Column(Boolean, nullable=False, server_default=text("false"))
+ segment = Column(String(2))
+ alma_ous_id = Column(ForeignKey('alma_ouses.alma_ous_id'))
+ science_product_locator = Column(ForeignKey('science_products.science_product_locator'), unique=True)
+
+ alma_ous = relationship('AlmaOus')
+ calibration_status_value = relationship('CalibrationStatusValue')
+ filegroup = relationship('Filegroup')
+ project = relationship('Project')
+ science_product = relationship('ScienceProduct', uselist=False)
+ scans = relationship('Scan', cascade="all, delete-orphan")
+ calibrations = relationship('Calibration')
+ type = Column(String)
+
+ __mapper_args__ = {
+ 'polymorphic_identity': 'execblock',
+ 'polymorphic_on': type
+ }
+
+ def __repr__(self):
+ return "<ExecutionBlock#{execution_block_id} project={project_code} start={starttime} end={endtime}>".format(**self.__dict__)
+
+
+class RealfastExecutionBlock(ExecutionBlock):
+ __tablename__ = 'realfast_execution_blocks'
+
+ execution_block_id = Column(ForeignKey('execution_blocks.execution_block_id'), primary_key=True)
+ transient_ra = Column(String)
+ transient_ra_error = Column(Float(53))
+ transient_dec = Column(String)
+ transient_dec_error = Column(Float(53))
+ transient_snr = Column(Float(53))
+ transient_dm = Column(Float(53))
+ transient_dm_error = Column(Float(53))
+ preaverage_time = Column(Float(53))
+ rfpipe_version = Column(String)
+ prefs_id = Column(String)
+ rf_qa_label = Column(String)
+ rf_qa_zero_fraction = Column(Float(53))
+ rf_qa_visibility_noise = Column(Float(53))
+ rf_qa_image_noise = Column(Float(53))
+ portal_candidate_ids = Column(String)
+
+ __mapper_args__ = dict(
+ polymorphic_identity='realfast',
+ inherit_condition=(execution_block_id == ExecutionBlock.execution_block_id)
+ )
+
+ def __repr__(self):
+ return "<RealfastExecutionBlock#{execution_block_id} project={project_code} start={starttime} end={endtime}>".format(**self.__dict__)
+
+
+class ExternalProductSystem(Base):
+ __tablename__ = 'external_product_systems'
+
+ name = Column(String, primary_key=True)
+
+ def __repr__(self):
+ return "<ExternalProductSystem#{name}>".format(**self.__dict__)
+
+
+class FailedIngestion(Base):
+ __tablename__ = 'failed_ingestions'
+
+ filename = Column(String(), primary_key=True)
+ telescope = Column(String())
+ last_attempted = Column(DateTime(True))
+ category = Column(String())
+ failure_reason = Column(String())
+ resolution = Column(String())
+ state = Column(String())
+
+
+class Filegroup(Base):
+ __tablename__ = 'filegroups'
+ __table_args__ = (
+ CheckConstraint('(CASE WHEN (project_code IS NOT NULL) THEN 1 ELSE 0 END + CASE WHEN (parent_filegroup_id IS NOT NULL) THEN 1 ELSE 0 END) = 1'),
+ )
+
+ filegroup_id = Column(Integer, primary_key=True, server_default=text("nextval('filegroups_filegroup_id_seq'::regclass)"))
+ project_code = Column(ForeignKey('projects.project_code', ondelete='CASCADE', onupdate='CASCADE'))
+ parent_filegroup_id = Column(ForeignKey('filegroups.filegroup_id', ondelete='CASCADE', onupdate='CASCADE'))
+ datasize = Column(BigInteger)
+ type = Column(String)
+
+ parent_filegroup = relationship('Filegroup', remote_side=[filegroup_id], backref=backref('children_filegroups', cascade="all, delete-orphan"))
+ project = relationship('Project')
+
+ execution_blocks = relationship('ExecutionBlock', cascade="all, delete-orphan")
+ files = relationship('File', cascade="all, delete-orphan")
+ scans = relationship('Scan')
+
+ @property
+ def all_files(self):
+ """
+ :return: all the files under this filegroup, recursively.
+
+ Beware performance issues here!
+ """
+ # this is a gross and probably expensive albeit beautiful way to write this query
+ return self.files + [file for subgroup in self.children_filegroups for file in subgroup.all_files]
+
+ def __repr__(self):
+ return "<Filegroup#{filegroup_id} project={project_code}>".format(**self.__dict__)
+
+
+class File(Base):
+ __tablename__ = 'files'
+
+ file_id = Column(Integer, primary_key=True, server_default=text("nextval('files_file_id_seq'::regclass)"))
+ file_path = Column(String)
+ ngas_id = Column(String)
+ filegroup = Column(ForeignKey('filegroups.filegroup_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False, index=True)
+ filename = Column(String, nullable=False)
+ filesize = Column(BigInteger, nullable=False)
+ format = Column(String, nullable=False)
+ type = Column(String, nullable=False)
+ checksum = Column(String)
+ checksum_type = Column(String)
+ ingestion_time = Column(DateTime(True))
+ ngas_cluster = Column(String)
+ ngas_location = Column(String)
+ preview_storage_path = Column(String)
+
+ filegroup1 = relationship('Filegroup')
+ images = relationship('Image', cascade="all, delete-orphan")
+ subscans = relationship('Subscan')
+
+ def __repr__(self):
+ return "<File#{file_id} {file_path}/{filename} ngas_id={ngas_id}>".format(**self.__dict__)
+
+class ImageProduct(Base):
+ __tablename__ = 'image_products'
+
+ image_product_id = Column(Integer, primary_key=True, server_default=text("nextval('image_products_image_product_id_seq'::regclass)"))
+ project_code = Column(ForeignKey('projects.project_code'), nullable=False)
+ configurations = Column(String(255))
+ collection_name = Column(String(255))
+ calibration_level = Column(Integer, server_default=text("2"))
+ product_file_id = Column(ForeignKey('files.file_id', ondelete='CASCADE', onupdate='CASCADE'), ForeignKey('files.file_id'), nullable=False)
+ tags = Column(String(255))
+ image_set_filegroup = Column(ForeignKey('filegroups.filegroup_id', ondelete='CASCADE', onupdate='CASCADE'))
+
+ filegroup = relationship('Filegroup')
+ product_file = relationship('File', primaryjoin='ImageProduct.product_file_id == File.file_id')
+ product_file1 = relationship('File', primaryjoin='ImageProduct.product_file_id == File.file_id')
+ project = relationship('Project')
+
+
+class Image(Base):
+ __tablename__ = 'images'
+
+ image_id = Column(Integer, primary_key=True, server_default=text("nextval('images_image_id_seq'::regclass)"))
+ file_id = Column(ForeignKey('files.file_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+ target_name = Column(String, nullable=False)
+ telescope = Column(String, nullable=False)
+ thumbnail = Column(String)
+ spatial_resolution = Column(Float(53), nullable=False)
+ image_field_of_view = Column(Float(53), nullable=False)
+ max_intensity = Column(Float(53), nullable=False)
+ min_intensity = Column(Float(53), nullable=False)
+ rms_noise = Column(Float(53), nullable=False)
+ polarization_id = Column(ForeignKey('polarizations.polarization_id'), nullable=False)
+ ra = Column(String(255), nullable=False)
+ dec = Column(String(255), nullable=False)
+ min_frequency = Column(Float(53), nullable=False)
+ max_frequency = Column(Float(53), nullable=False)
+ ra_element_count = Column(Integer, nullable=False)
+ dec_element_count = Column(Integer, nullable=False)
+ starttime = Column(Float(53))
+ endtime = Column(Float(53))
+ exposure_time = Column(Float(53))
+ rest_frequency = Column(Float(53))
+ image_units = Column(String(255))
+ spatial_region = Column(Text)
+ beam_axis_ratio = Column(Float(53))
+ band_code = Column(String)
+ ra_pixel_size = Column(Float(53), nullable=False)
+ dec_pixel_size = Column(Float(53), nullable=False)
+ tags = Column(String)
+ science_product_locator = Column(ForeignKey('science_products.science_product_locator'), unique=True)
+ collection_name = Column(String)
+ configurations = Column(String)
+ calibration_level = Column(Integer)
+
+ file = relationship('File')
+ polarization = relationship('Polarization')
+ science_product = relationship('ScienceProduct', uselist=False)
+
+ def __repr__(self):
+ return "<Image#{image_id} target={target_name} telescope={telescope}>".format(**self.__dict__)
+
+class VlassImage(Image):
+ __tablename__ = 'vlass_images'
+
+ image_id = Column(ForeignKey('images.image_id'), primary_key=True)
+ epoch = Column(ForeignKey('vlass_epochs.epoch'))
+ tile = Column(String)
+ type = Column(ForeignKey('vlass_image_types.type'))
+
+ vlass_epoch = relationship('VlassEpoch')
+ vlass_image_type = relationship('VlassImageType')
+
+
+class Intent(Base):
+ __tablename__ = 'intents'
+
+ intent_name = Column(String, primary_key=True)
+
+ subscans = relationship('Subscan', secondary='subscan_intents')
+
+ def __repr__(self):
+ return "<Intent {0}>".format(self.intent_name)
+
+
+t_logs = Table(
+ 'logs', metadata,
+ Column('filename', String),
+ Column('class', String),
+ Column('method', String),
+ Column('line', Integer),
+ Column('arguments', String),
+ Column('timestamp', DateTime(True)),
+ Column('formatted_message', String),
+ Column('logger_name', String),
+ Column('level', String),
+ Column('thread_name', String),
+ Column('reference_mask', Integer),
+ Column('properties', JSON),
+ Column('stacktrace', Text),
+ Column('request_id', Integer)
+)
+
+
+class Polarization(Base):
+ __tablename__ = 'polarizations'
+
+ polarization_id = Column(Integer, primary_key=True)
+ name = Column(String, nullable=False)
+ description = Column(String, nullable=False)
+
+ def __repr__(self):
+ return "<Polarization {0}>".format(self.name)
+
+
+class ProductGroup(Base):
+ __tablename__ = 'product_groups'
+
+ product_group_id = Column(Integer, primary_key=True, server_default=text("nextval('product_groups_product_group_id_seq'::regclass)"))
+ product_group_type = Column(String, nullable=False)
+ alma_ous_id = Column(ForeignKey('alma_ouses.alma_ous_id'))
+ program_id = Column(ForeignKey('programs.program_id'))
+ parent_product_group_id = Column(ForeignKey('product_groups.product_group_id'))
+
+ ALMA_ous = relationship('AlmaOus')
+ parent_product_group = relationship('ProductGroup', remote_side=[product_group_id])
+ program = relationship('Program')
+ science_products = relationship('ScienceProduct', secondary='science_products_product_groups')
+
+
+class Program(Base):
+ __tablename__ = 'programs'
+
+ program_id = Column(String, primary_key=True)
+ project_code = Column(ForeignKey('projects.project_code'), nullable=False)
+ program_type = Column(String, nullable=False)
+ parent_program_id = Column(ForeignKey('programs.program_id'))
+
+ parent_program = relationship('Program', remote_side=[program_id])
+ project = relationship('Project')
+
+
+class Project(Base):
+ __tablename__ = 'projects'
+
+ project_code = Column(String, primary_key=True)
+ legacy_id = Column(String)
+ total_observation_time = Column(Float(53))
+ opt_project_id = Column(Integer)
+ title = Column(String)
+ abstract = Column(Text)
+ proprietary_expiration = Column(DateTime)
+ last_addition = Column(Date)
+ starttime = Column(Float(53))
+ endtime = Column(Float(53))
+ proprietary_duration = Column(Float(53))
+
+ science_products = relationship('ScienceProduct', secondary='science_products_projects')
+ authors = relationship('Author')
+ execution_blocks = relationship('ExecutionBlock')
+ file_groups = relationship('Filegroup')
+ alma_ouses = relationship('AlmaOus', backref='projects')
+
+ def __repr__(self):
+ return '<Project#{project_code} "{title}" start={starttime} end={endtime}>'.format(**self.__dict__)
+
+
+class Receiver(Base):
+ __tablename__ = 'receivers'
+
+ receiver_id = Column(Integer, primary_key=True, server_default=text("nextval('receivers_receiver_id_seq'::regclass)"))
+ description = Column(String, nullable=False)
+
+ def __repr__(self):
+ return '<Receiver {0}>'.format(self.receiver_id)
+
+
+class ReingestionState(Base):
+ __tablename__ = 'reingestion_states'
+
+ state = Column(String, primary_key=True)
+
+
+class RhCommand(Base):
+ __tablename__ = 'rh_commands'
+
+ command_id = Column(Numeric, primary_key=True)
+ name = Column(String, nullable=False, unique=True)
+ arguments = Column(String)
+ description = Column(String)
+ host = Column(String)
+ script = Column(String, nullable=False)
+ staging_area = Column(String)
+ execution_level = Column(String, nullable=False)
+
+
+class RhDataEntity(Base):
+ __tablename__ = 'rh_data_entities'
+ __table_args__ = (
+ UniqueConstraint('data_entity_name', 'request_id'),
+ )
+
+ data_entity_id = Column(Numeric, primary_key=True)
+ data_entity_name = Column(String, nullable=False)
+ request_id = Column(ForeignKey('rh_requests.request_id'), nullable=False, index=True)
+ state = Column(String, nullable=False)
+ meta_data_uri = Column(String)
+
+ request = relationship('RhRequest')
+
+
+class RhFile(Base):
+ __tablename__ = 'rh_files'
+ __table_args__ = (
+ CheckConstraint('((request_id IS NOT NULL) AND (data_entity_id IS NULL)) OR ((request_id IS NULL) AND (data_entity_id IS NOT NULL))'),
+ )
+
+ file_id = Column(Numeric, primary_key=True)
+ request_id = Column(ForeignKey('rh_requests.request_id'), index=True)
+ data_entity_id = Column(ForeignKey('rh_data_entities.data_entity_id'), index=True)
+ file_key = Column(String, nullable=False, index=True)
+ namespace = Column(String, nullable=False)
+ file_name = Column(String, nullable=False)
+ file_type = Column(String)
+ file_uri = Column(String, nullable=False)
+ file_size = Column(Numeric, nullable=False)
+ rank = Column(Numeric)
+ permission = Column(String)
+ medium_id = Column(Numeric)
+
+ data_entity = relationship('RhDataEntity')
+ request = relationship('RhRequest')
+
+
+class RhOperatorRequest(Base):
+ __tablename__ = 'rh_operator_requests'
+
+ request_id = Column(Numeric, primary_key=True)
+ creation_date = Column(DateTime, nullable=False)
+ priority = Column(String, nullable=False)
+ resolved_flag = Column(String(1), nullable=False)
+ action_id = Column(Numeric)
+ request_description = Column(String)
+
+
+class RhProcess(Base):
+ __tablename__ = 'rh_processes'
+ __table_args__ = (
+ CheckConstraint('((request_id IS NOT NULL) AND (data_entity_id IS NULL)) OR ((request_id IS NULL) AND (data_entity_id IS NOT NULL))'),
+ )
+
+ process_id = Column(Numeric, primary_key=True)
+ state = Column(String, nullable=False)
+ command_id = Column(ForeignKey('rh_commands.command_id'), nullable=False)
+ job_id = Column(String)
+ delivery_method = Column(String)
+ arguments = Column(String)
+ file_id = Column(ForeignKey('rh_files.file_id'))
+ request_id = Column(ForeignKey('rh_requests.request_id'))
+ data_entity_id = Column(ForeignKey('rh_data_entities.data_entity_id'))
+ process_type = Column(String, nullable=False)
+
+ command = relationship('RhCommand')
+ data_entity = relationship('RhDataEntity')
+ file = relationship('RhFile')
+ request = relationship('RhRequest')
+
+
+class RhRequestParameter(Base):
+ __tablename__ = 'rh_request_parameters'
+
+ request_id = Column(ForeignKey('rh_requests.request_id'), primary_key=True, nullable=False)
+ pname = Column(String, primary_key=True, nullable=False)
+ pvalue = Column(String)
+
+ request = relationship('RhRequest')
+
+
+class RhRequest(Base):
+ __tablename__ = 'rh_requests'
+
+ request_id = Column(Numeric, primary_key=True)
+ account_id = Column(Numeric, nullable=False)
+ request_state = Column(String, nullable=False)
+ creation_date = Column(DateTime, nullable=False)
+ request_description = Column(String)
+ delivery_method_type = Column(String)
+ delivery_media_type = Column(String)
+ delivery_media_address = Column(String)
+ completion_date = Column(DateTime)
+ priority_code = Column(Numeric, nullable=False)
+ ds_file_size = Column(Numeric)
+ aux_file_size = Column(Numeric)
+ total_files_number = Column(Numeric)
+ total_processes_number = Column(Numeric)
+ notification_email = Column(String)
+ archive_id = Column(ForeignKey('archive.id'))
+
+ archive = relationship('Archive')
+
+
+t_rh_requests_mediums = Table(
+ 'rh_requests_mediums', metadata,
+ Column('request_id', ForeignKey('rh_requests.request_id'), primary_key=True, nullable=False),
+ Column('medium_id', ForeignKey('rh_site_delivery_medium.medium_id'), primary_key=True, nullable=False)
+)
+
+
+class RhSiteDeliveryMedium(Base):
+ __tablename__ = 'rh_site_delivery_medium'
+
+ medium_id = Column(Numeric, primary_key=True)
+ active = Column(String(1), nullable=False)
+ media_type = Column(String, nullable=False)
+ host_name = Column(String, nullable=False)
+ device = Column(String, nullable=False)
+ capacity = Column(Float, nullable=False)
+ request_id = Column(Numeric)
+ state = Column(String, nullable=False)
+ last_mod = Column(DateTime, server_default=text("'2018-04-18 15:21:15.783814'::timestamp without time zone"))
+ status_message = Column(String, nullable=False)
+
+ requests = relationship('RhRequest', secondary='rh_requests_mediums')
+
+
+class Role(Base):
+ __tablename__ = 'role'
+ __table_args__ = (
+ UniqueConstraint('application', 'name'),
+ )
+
+ role_no = Column(Numeric, primary_key=True)
+ version = Column(Numeric, nullable=False)
+ application = Column(String, nullable=False)
+ name = Column(String, nullable=False)
+ parent_role = Column(ForeignKey('role.role_no'))
+
+ parent = relationship('Role', remote_side=[role_no])
+
+
+t_rsl_eb_cal_map = Table(
+ 'rsl_eb_cal_map', metadata,
+ Column('eb_locator', String),
+ Column('cal_locator', String)
+)
+
+
+class Scan(Base):
+ __tablename__ = 'scans'
+
+ scan_id = Column(Integer, primary_key=True, server_default=text("nextval('scans_scan_id_seq'::regclass)"))
+ ost_scan_id = Column(Integer)
+ execution_block_id = Column(ForeignKey('execution_blocks.execution_block_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+ filegroup_id = Column(ForeignKey('filegroups.filegroup_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+ max_bandwidth = Column(Float(53), nullable=False)
+ min_bandwidth = Column(Float(53), nullable=False)
+ polarization_code = Column(Integer, nullable=False)
+ max_frequency = Column(Float(53), nullable=False)
+ min_frequency = Column(Float(53), nullable=False)
+ filename = Column(String)
+
+ execution_block = relationship('ExecutionBlock')
+ filegroup = relationship('Filegroup')
+ subscans = relationship('Subscan', cascade="all, delete-orphan")
+
+ def __repr__(self):
+ return '<Scan#{0}>'.format(self.scan_id)
+
+
+class ScienceProductGroupType(Base):
+ __tablename__ = 'science_product_group_types'
+
+ science_product_group_type = Column(String, primary_key=True)
+ description = Column(String)
+
+
+class ScienceProductType(Base):
+ __tablename__ = 'science_product_types'
+
+ science_product_type = Column(String, primary_key=True)
+ description = Column(String)
+
+
+class ScienceProduct(Base):
+ __tablename__ = 'science_products'
+
+ science_product_locator = Column(String, primary_key=True)
+ science_product_type = Column(ForeignKey('science_product_types.science_product_type'), nullable=False)
+ metadata_ingestion_date = Column(DateTime, nullable=False)
+ metadata_ingestion_version = Column(String, nullable=False)
+ filegroup_id = Column(ForeignKey('filegroups.filegroup_id'), nullable=False)
+ external_name = Column(String)
+ external_system = Column(ForeignKey('external_product_systems.name'))
+
+ external_product_system = relationship('ExternalProductSystem')
+ filegroup = relationship('Filegroup')
+ science_product_type1 = relationship('ScienceProductType')
+
+ execution_block = relationship('ExecutionBlock', uselist=False, back_populates='science_product')
+ project = relationship('Project', uselist=False, secondary='science_products_projects')
+
+ @property
+ def locator(self):
+ return self.science_product_locator
+
+ @property
+ def subdirectory(self):
+ if self.science_product_type == 'Execution Block':
+ return self.execution_block.ngas_fileset_id
+ else:
+ return self.external_name
+
+
+ def __repr__(self):
+ return f"<ScienceProduct#{self.science_product_locator} type={self.science_product_type}>"
+
+class ScienceProductsProductGroup(Base):
+ __tablename__ = 'science_products_product_groups'
+
+ science_product_locator = Column(ForeignKey('science_products.science_product_locator'), primary_key=True, nullable=False)
+ product_group_id = Column(ForeignKey('product_groups.product_group_id'), primary_key=True, nullable=False)
+
+ product_group = relationship('ProductGroup')
+ science_product = relationship('ScienceProduct')
+
+
+t_science_products_projects = Table(
+ 'science_products_projects', metadata,
+ Column('science_product_locator', ForeignKey('science_products.science_product_locator'), primary_key=True, nullable=False),
+ Column('project_code', ForeignKey('projects.project_code'), primary_key=True, nullable=False)
+)
+
+
+t_source_3c48 = Table(
+ 'source_3c48', metadata,
+ Column('project_code', String),
+ Column('execution_block_id', Integer),
+ Column('ost_scan_id', Integer),
+ Column('starttime', DateTime(True)),
+ Column('ngas_id', String),
+ Column('source', Text),
+ Column('receiver_band', Text)
+)
+
+
+t_subscan_data_descriptions = Table(
+ 'subscan_data_descriptions', metadata,
+ Column('subscan_id', ForeignKey('subscans.subscan_id'), primary_key=True, nullable=False),
+ Column('data_description_id', ForeignKey('data_descriptions.data_description_id'), primary_key=True, nullable=False, index=True)
+)
+
+
+t_subscan_intents = Table(
+ 'subscan_intents', metadata,
+ Column('intent_name', ForeignKey('intents.intent_name'), primary_key=True, nullable=False),
+ Column('subscan_id', ForeignKey('subscans.subscan_id', ondelete='CASCADE', onupdate='CASCADE'), primary_key=True, nullable=False)
+)
+
+
+class Subscan(Base):
+ __tablename__ = 'subscans'
+
+ subscan_id = Column(Integer, primary_key=True, server_default=text("nextval('subscans_subscan_id_seq'::regclass)"))
+ scan_id = Column(ForeignKey('scans.scan_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False)
+ file_id = Column(ForeignKey('files.file_id', ondelete='CASCADE', onupdate='CASCADE'), index=True)
+ ost_subscan_id = Column(Integer)
+ obstype = Column(String, nullable=False)
+ starttime = Column(Float(53), nullable=False)
+ endtime = Column(Float(53), nullable=False)
+ sourcename = Column(String, nullable=False)
+ sourcetype = Column(String, nullable=False)
+ ra = Column(Float(53), nullable=False)
+ dec = Column(Float(53), nullable=False)
+ exposure_time = Column(Float(53), nullable=False)
+ integration_time = Column(Float(53), nullable=False)
+ receiver_id = Column(ForeignKey('receivers.receiver_id'), nullable=False)
+ backend = Column(String, nullable=False)
+ intent = Column(String)
+ configuration_id = Column(ForeignKey('configurations.configuration_id', ondelete='CASCADE', onupdate='CASCADE'), nullable=False, index=True)
+
+ configuration = relationship('Configuration')
+ file = relationship('File')
+ receiver = relationship('Receiver')
+ scan = relationship('Scan')
+ data_descriptions = relationship('DataDescription', secondary='subscan_data_descriptions')
+ intents = relationship('Intent', secondary='subscan_intents')
+
+ def __repr__(self):
+ return '<Subscan#{subscan_id} {obstype} start={starttime} end={endtime} ra={ra} dec={dec} backend={backend} intent={intent}>'.format(**self.__dict__)
+
+
+t_total_file_sizes = Table(
+ 'total_file_sizes', metadata,
+ Column('filegroup_id', Integer),
+ Column('parent_filegroup_id', Integer),
+ Column('filesize', Numeric)
+)
+
+
+class VlassCalibrationType(Base):
+ __tablename__ = 'vlass_calibration_types'
+
+ type = Column(String, primary_key=True)
+ description = Column(String)
+
+
+t_vlass_calibrations = Table(
+ 'vlass_calibrations', metadata,
+ Column('calibration_id', ForeignKey('calibrations.calibration_id'), primary_key=True),
+ Column('type', ForeignKey('vlass_calibration_types.type'))
+)
+
+
+class VlassEpoch(Base):
+ __tablename__ = 'vlass_epochs'
+
+ epoch = Column(String, primary_key=True)
+ description = Column(String)
+
+ execution_blocks = relationship('ExecutionBlock', secondary='vlass_execution_blocks')
+
+
+class VlassExecutionBlockTile(Base):
+ __tablename__ = 'vlass_execution_block_tiles'
+
+ execution_block_id = Column(ForeignKey('vlass_execution_blocks.execution_block_id'), primary_key=True, nullable=False)
+ tile = Column(String, primary_key=True, nullable=False)
+
+ execution_block = relationship('VLASSExecutionBlock')
+
+
+class VLASSExecutionBlock(ExecutionBlock):
+ __tablename__ = 'vlass_execution_blocks'
+
+ __mapper_args__ = {
+ 'polymorphic_identity': 'vlass'
+ }
+
+ execution_block_id = Column(ForeignKey('execution_blocks.execution_block_id'), primary_key=True)
+ epoch = Column(ForeignKey('vlass_epochs.epoch'))
+
+ execution_block = relationship('ExecutionBlock')
+ vlass_epoch = relationship('VlassEpoch')
+
+
+class VlassImageType(Base):
+ __tablename__ = 'vlass_image_types'
+
+ type = Column(String, primary_key=True)
+ description = Column(String)
+
+
+class VlbaReingestionQueue(Base):
+ __tablename__ = 'vlba_reingestion_queue'
+
+ filename = Column(String, primary_key=True)
+ observation_finished = Column(DateTime(True), nullable=False)
+ last_updated = Column(DateTime(True), nullable=False, server_default=text("now()"))
+ state = Column(ForeignKey('reingestion_states.state'), nullable=False, server_default=text("'WAITING'::character varying"))
+
+ reingestion_state = relationship('ReingestionState')
+
+
+class ProblemSeverityList(Base):
+ __tablename__ = 'problem_severity_list'
+
+ severity = Column(Integer, primary_key=True)
+ description = Column(String)
+
+
+class ProblemType(Base):
+ __tablename__ = 'problem_types'
+
+ problem_type_id = Column(Integer, primary_key=True, server_default=text("nextval('problem_types_problem_type_id_seq'::regclass)"))
+ description = Column(String)
+
+
+class ScienceProductComment(Base):
+ __tablename__ = 'science_product_comments'
+
+ comment_id = Column(Integer, primary_key=True, server_default=text("nextval('science_product_comments_comment_id_seq'::regclass)"))
+ problem_type_id = Column(ForeignKey('problem_types.problem_type_id'), nullable=False)
+ science_product_locator = Column(ForeignKey('science_products.science_product_locator'), nullable=False)
+ severity = Column(ForeignKey('problem_severity_list.severity'), nullable=False)
+ comment = Column(String)
+ doc_link = Column(String)
+
+ problem_type = relationship('ProblemType')
+ science_product = relationship('ScienceProduct')
+ problem_severity_list = relationship('ProblemSeverityList')
+
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/ngasmodel.py b/apps/cli/executables/pexable/ingest/ingest/schema/ngasmodel.py
new file mode 100644
index 000000000..7f484aeb4
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/ngasmodel.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from sqlalchemy import Column, ForeignKey, Integer, String
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+class NGASHost(Base):
+ __tablename__ = 'ngas_hosts'
+ host_id = Column('host_id', String, primary_key=True)
+ domain = Column('domain', String)
+ port = Column('srv_port', Integer)
+ cluster_name = Column('cluster_name', String)
+ disks = relationship('NGASDisk', backref='host')
+
+ @property
+ def server(self):
+ return f'{self.host_id}.{self.domain}'
+
+ def __repr__(self):
+ return f'<NGASHost {self.host_id}.{self.domain}:{self.port}>'
+
+
+class NGASDisk(Base):
+ __tablename__ = 'ngas_disks'
+ disk_id = Column('disk_id', String, primary_key=True)
+ mounted = Column('mounted', Integer)
+ host_id = Column('host_id', ForeignKey('ngas_hosts.host_id'))
+ files = relationship('NGASFile', backref='disk')
+
+ def __repr__(self):
+ return f'<NGASDisk#{self.disk_id} mounted={self.mounted}>'
+
+
+class NGASFile(Base):
+ __tablename__ = 'ngas_files'
+ file_id = Column('file_id', String, primary_key=True)
+ file_name = Column('file_name', String)
+ ingestion_date = Column('ingestion_date', String)
+ file_size = Column('file_size', Integer)
+ version = Column('file_version', Integer, primary_key=True)
+ disk_id = Column('disk_id', ForeignKey('ngas_disks.disk_id'))
+ checksum = Column('checksum', String)
+ checksum_plugin = Column('checksum_plugin', String)
+ format = Column('format', String)
+
+ def __repr__(self):
+ return f'<NGASFile#{self.file_id} name={self.file_name} size={self.file_size} version={self.version}>'
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/optmodel.py b/apps/cli/executables/pexable/ingest/ingest/schema/optmodel.py
new file mode 100644
index 000000000..2a8c845a5
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/optmodel.py
@@ -0,0 +1,61 @@
+# coding: utf-8
+from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, String, Text, text
+from sqlalchemy.orm import relationship
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+class Coauthor(Base):
+ __tablename__ = 'coauthors'
+ __table_args__ = (
+ Index('coauthors_idx_project_id_idx', 'idx', 'project_id'),
+ )
+
+ project_id = Column(ForeignKey('project.id'), primary_key=True, nullable=False, index=True)
+ projectauthor_id = Column(ForeignKey('projectauthor.id'), nullable=False, unique=True)
+ idx = Column(Integer, primary_key=True, nullable=False)
+
+ project = relationship('Project')
+ projectauthor = relationship('Projectauthor', uselist=False)
+
+
+class Project(Base):
+ __tablename__ = 'project'
+
+ id = Column(Integer, primary_key=True, server_default=text("nextval('project_id_seq'::regclass)"))
+ rowversion = Column(Integer, nullable=False)
+ projectcode = Column(String, nullable=False, unique=True)
+ title = Column(String)
+ proposalcode = Column(String, nullable=False)
+ createdon = Column(DateTime)
+ createdby = Column(Integer)
+ lastupdatedon = Column(DateTime)
+ lastupdatedby = Column(Integer)
+ comments = Column(Text)
+ projecttype = Column(String)
+ test = Column(Boolean, server_default=text("false"))
+ telescope = Column(String)
+ refereemedian = Column(Float(53))
+ has_submitted_sb = Column(Boolean, nullable=False, server_default=text("false"))
+ is_complete = Column(Boolean, nullable=False, server_default=text("false"))
+ hoursallocated = Column(Float(53), nullable=False)
+ hoursusedsuccess = Column(Float(53), nullable=False)
+ hoursusedfailure = Column(Float(53), nullable=False)
+ pi = Column(ForeignKey('projectauthor.id'), index=True)
+ contactauthor = Column(ForeignKey('projectauthor.id'), index=True)
+
+ projectauthor = relationship('Projectauthor', primaryjoin='Project.contactauthor == Projectauthor.id')
+ projectauthor1 = relationship('Projectauthor', primaryjoin='Project.pi == Projectauthor.id')
+
+
+class Projectauthor(Base):
+ __tablename__ = 'projectauthor'
+
+ id = Column(Integer, primary_key=True, server_default=text("nextval('projectauthor_id_seq'::regclass)"))
+ rowversion = Column(Integer, nullable=False)
+ globalid = Column(Integer, nullable=False)
+ presentonproposal = Column(Boolean, nullable=False, server_default=text("false"))
+ receivesemail = Column(Boolean, nullable=False, server_default=text("true"))
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/pstmodel.py b/apps/cli/executables/pexable/ingest/ingest/schema/pstmodel.py
new file mode 100644
index 000000000..7f202283d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/pstmodel.py
@@ -0,0 +1,2135 @@
+# coding: utf-8
+from sqlalchemy import BigInteger, Column, Date, DateTime, Float, ForeignKey, Index, Integer, SmallInteger, String, Table, Text, text
+from sqlalchemy.dialects.mysql.types import BIT, MEDIUMBLOB
+from sqlalchemy.orm import relationship
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+class AbstractBug(Base):
+ __tablename__ = 'AbstractBug'
+
+ bug_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ id = Column(Integer, nullable=False)
+ status = Column(String(255), nullable=False)
+ statusNumber = Column(Integer, nullable=False, index=True)
+ type = Column(String(255), nullable=False)
+ typeNumber = Column(Integer, nullable=False, index=True)
+ issueDate = Column(Date, index=True)
+ completeDate = Column(Date, index=True)
+ author = Column(String(255))
+ location = Column(String(255))
+ description = Column(String)
+ resolution = Column(String)
+ versionNumber = Column(String(255))
+ osName = Column(String(255))
+ email = Column(String)
+
+
+class GBTVEGASSpectrometer(Base):
+ __tablename__ = 'GBT_VEGAS_Spectrometer'
+
+ id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ mode = Column(Integer, nullable=False)
+ bandwidth = Column(Float(asdecimal=True), nullable=False)
+ restFrequencies = Column(String(255), nullable=False)
+ spectralResolution = Column(Float(asdecimal=True), nullable=False)
+ integrationTime = Column(Float(asdecimal=True), nullable=False)
+ dataRate = Column(Float(asdecimal=True), nullable=False)
+ vegas_id = Column(ForeignKey('GBT_VEGAS_14A.Id', ondelete='CASCADE'), index=True)
+
+ vegas = relationship('GBTVEGAS14A')
+
+
+class GBTVegasPulsar(Base):
+ __tablename__ = 'GBT_Vegas_Pulsar'
+
+ Id = Column(BigInteger, primary_key=True)
+ COMMENTS = Column(String(512))
+ BAND_WIDTH = Column(String(128))
+ BAND_WIDTH_UNIT = Column(String(255))
+ OBSERVING_MODE = Column(String(128))
+ POLAR_CROSS_PROD = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ Dedispersion = Column(String(255))
+ BITS = Column(String(255))
+
+
+class NormalizedScienceType(Base):
+ __tablename__ = 'NormalizedScienceTypes'
+
+ id = Column(Integer, primary_key=True)
+ cycle_id = Column(Integer, nullable=False)
+ science_type = Column(Text, nullable=False)
+ normalized = Column(Integer, nullable=False)
+
+
+class RESOURCE(Base):
+ __tablename__ = 'RESOURCES'
+
+ resource_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ RESOURCE_GROUP = Column(Text)
+ RESOURCE_NAME = Column(Text)
+ PROPOSAL_ID = Column(ForeignKey('proposal.proposal_id'), nullable=False, index=True)
+ TELESCOPE = Column(String(255), nullable=False)
+ DISPLAY_POSITION = Column(Float(asdecimal=True), nullable=False)
+
+ proposal = relationship('Proposal')
+
+
+class GMVARESOURCE(RESOURCE):
+ __tablename__ = 'GMVA_RESOURCE'
+
+ Id = Column(ForeignKey('RESOURCES.resource_id', ondelete='CASCADE'), primary_key=True, index=True)
+ FRONT_END = Column(String(255))
+ BACK_END = Column(String(255))
+ VLBA = Column(BIT(1), nullable=False)
+ EUROPEAN = Column(BIT(1), nullable=False)
+ BREWSTER = Column(BIT(1), nullable=False)
+ FORT_DAVIS = Column(BIT(1), nullable=False)
+ KITT_PEAK = Column(BIT(1), nullable=False)
+ LOS_ALAMOS = Column(BIT(1), nullable=False)
+ MAUNA_KEA = Column(BIT(1), nullable=False)
+ NORTH_LIBERTY = Column(BIT(1), nullable=False)
+ OWENS_VALLEY = Column(BIT(1), nullable=False)
+ PIE_TOWN = Column(BIT(1), nullable=False)
+ EFFELSBERG = Column(BIT(1), nullable=False)
+ PLATEAU_DE_BURE = Column(BIT(1), nullable=False)
+ PICO_VELETA = Column(BIT(1), nullable=False)
+ ONSALA = Column(BIT(1), nullable=False)
+ METSAHOVI = Column(BIT(1), nullable=False)
+ YEBES = Column(BIT(1), nullable=False)
+ OTHER_STATIONS = Column(String(255))
+ FULL_POLAR = Column(BIT(1), nullable=False)
+ AVERAGING_TIME = Column(String(255))
+ SPECTRAL_POINTS = Column(String(255))
+ MULTI_CORR_PASS = Column(String(255))
+ MARK4_CONVERSION = Column(BIT(1), nullable=False)
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ SAMPLING_LEVEL = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ BITS = Column(String(255), nullable=False)
+ POLAR_CROSS_PROD = Column(String(255))
+ AGG_BIT_RATE = Column(String(255))
+ NO_OF_FIELDS = Column(Integer)
+ Wideband_Observing_System = Column(BIT(1), nullable=False)
+ Observing_System = Column(String(255), nullable=False)
+ GREEN_BANK = Column(BIT(1), nullable=False)
+
+
+class GBTRESOURCE(RESOURCE):
+ __tablename__ = 'GBT_RESOURCE'
+
+ Id = Column(ForeignKey('RESOURCES.resource_id', ondelete='CASCADE'), primary_key=True, index=True)
+ FRONT_END = Column(String(255))
+ BACK_END = Column(String(255))
+ RECEIVER_OTHER = Column(String(255))
+
+
+class GBTGuppi(GBTRESOURCE):
+ __tablename__ = 'GBT_Guppi'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ COMMENTS = Column(Text)
+ BAND_WIDTH = Column(String(128))
+ BAND_WIDTH_UNIT = Column(String(128))
+ OBSERVING_MODE = Column(String(128))
+ POLAR_CROSS_PROD = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ Dedispersion = Column(String(255))
+ Bits = Column(String(255), server_default=text("'1'"))
+
+
+class GBTBACKENDBREAKTHROUGHLISTEN(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_BREAKTHROUGH_LISTEN'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ COMMENTS = Column(String(512))
+
+
+class GBTBACKENDPROCESSORPULSAR(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_PROCESSOR_PULSAR'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ BITS = Column(String(255), nullable=False)
+ SPECTRAL_RESOLUTION = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ POLAR_CROSS_PROD = Column(String(255))
+ PHASE_BINS = Column(String(255))
+
+
+class GBTZpectrometer(GBTRESOURCE):
+ __tablename__ = 'GBT_Zpectrometer'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ COMMENTS = Column(Text)
+ PI_PRE_APPROVAL = Column(BIT(1))
+
+
+class GBTVEGAS14A(GBTRESOURCE):
+ __tablename__ = 'GBT_VEGAS_14A'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ observing_type = Column(String(255))
+ number_beams = Column(Integer)
+
+
+class GBTBACKENDRADAR(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_RADAR'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ PI_PRE_APPROVAL = Column(BIT(1))
+ SPECTRAL_RESOLUTION = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ Bits = Column(String(255), server_default=text("'1'"))
+
+
+class GBTSPECTROSPIGOT(GBTRESOURCE):
+ __tablename__ = 'GBT_SPECTRO_SPIGOT'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ POLAR_CROSS_PRODUCT = Column(String(255))
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ BITS = Column(String(255), nullable=False)
+ SPECTRAL_RESOLUTION = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+
+
+class GBTBACKENDCGSR2(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_CGSR2'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ PI_PRE_APPROVAL = Column(BIT(1))
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+
+
+class GBTBACKENDPROCESSORLINE(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_PROCESSOR_LINE'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ BITS = Column(String(255), nullable=False)
+ SPECTRAL_RESOLUTION = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ POLAR_CROSS_PROD = Column(String(255))
+
+
+class GBTBACKENDGASP(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_GASP'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ PI_PRE_APPROVAL = Column(BIT(1))
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+
+
+class GBTBACKENDBCPM(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_BCPM'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+
+
+class GBTSPECTOMETERLINE(GBTRESOURCE):
+ __tablename__ = 'GBT_SPECTOMETER_LINE'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ BITS = Column(String(255), nullable=False)
+ SPECTRAL_RESOLUTION = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ SAMPLING_LEVEL = Column(String(255))
+ BEAMS = Column(String(255))
+ POLAR_CROSS_PRODUCT = Column(String(255))
+
+
+class GBTVEGA(GBTRESOURCE):
+ __tablename__ = 'GBT_VEGAS'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ BITS = Column(String(255), nullable=False)
+ SPECTRAL_RESOLUTION = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ CROSS_POLARIZATION = Column(BIT(1))
+ BEAMS = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ Resolution = Column(String(255))
+ MinimumInterval = Column(String(255))
+
+
+class GBTBACKENDJPLRADAR(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_JPL_RADAR'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ PI_PRE_APPROVAL = Column(BIT(1))
+ SPECTRAL_RESOLUTION = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ BITS = Column(String(255))
+
+
+class GBTBACKENDDIGITAL(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_DIGITAL'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ Bits = Column(String(255), server_default=text("'1'"))
+
+
+class GBTMUSTANG(GBTRESOURCE):
+ __tablename__ = 'GBT_MUSTANG'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ COMMENTS = Column(Text)
+ Array = Column(String(255))
+
+
+class GBTBACKENDOTHER(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_OTHER'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SPECIAL_BACKEND = Column(String(255))
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ BITS = Column(String(255), nullable=False)
+ COMMENTS = Column(Text)
+
+
+class GBTBACKENDCALTECHCONTINUUM(GBTRESOURCE):
+ __tablename__ = 'GBT_BACKEND_CALTECH_CONTINUUM'
+
+ Id = Column(ForeignKey('GBT_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String(255))
+ SKY_FREQUENCY_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(String(255))
+ TIME_RESOLUTION_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+
+
+class VLBARESOURCE(RESOURCE):
+ __tablename__ = 'VLBA_RESOURCE'
+
+ Id = Column(ForeignKey('RESOURCES.resource_id', ondelete='CASCADE'), primary_key=True, index=True)
+ FRONT_END = Column(String(255))
+ BACK_END = Column(String(255))
+ VLBA = Column(BIT(1), nullable=False)
+ HSA = Column(BIT(1), nullable=False)
+ BREWSTER = Column(BIT(1), nullable=False)
+ FORT_DAVIS = Column(BIT(1), nullable=False)
+ HANCOCK = Column(BIT(1), nullable=False)
+ KITT_PEAK = Column(BIT(1), nullable=False)
+ LOS_ALAMOS = Column(BIT(1), nullable=False)
+ MAUNA_KEA = Column(BIT(1), nullable=False)
+ NORTH_LIBERTY = Column(BIT(1), nullable=False)
+ OWENS_VALLEY = Column(BIT(1), nullable=False)
+ PIE_TOWN = Column(BIT(1), nullable=False)
+ ST_CROIX = Column(BIT(1), nullable=False)
+ GBT = Column(BIT(1), nullable=False, index=True)
+ ARECIBO = Column(BIT(1), nullable=False)
+ EFFELSBERG = Column(BIT(1), nullable=False)
+ VLA_Y1 = Column(BIT(1), nullable=False, index=True)
+ VLA_Y27 = Column(BIT(1), nullable=False, index=True)
+ GEODETIC = Column(String(255))
+ FULL_POLAR = Column(BIT(1), nullable=False)
+ PULSAR_GATE = Column(BIT(1), nullable=False)
+ MULTIPLE_FIELDS = Column(String(255))
+ AVERAGING_TIME = Column(String(255))
+ SPECTRAL_POINTS = Column(String(255))
+ MULTI_CORR_PASS = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ SAMPLING_LEVEL = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ BITS = Column(String(255), nullable=False)
+ POLAR_CROSS_PROD = Column(String(255))
+ AGG_BIT_RATE = Column(String(255))
+ NO_OF_FIELDS = Column(Integer, server_default=text("'0'"))
+ Wideband_Observing_System = Column(BIT(1), nullable=False)
+ Observing_Mode = Column(String(255), server_default=text("'Standard'"))
+ RSROComments = Column(String)
+ Observing_System = Column(String(255), server_default=text("'Legacy System'"))
+ MARK4_CONVERSION = Column(BIT(1))
+ shared_risk = Column(BIT(1), nullable=False)
+
+
+class VLARESOURCE(RESOURCE):
+ __tablename__ = 'VLA_RESOURCE'
+
+ Id = Column(ForeignKey('RESOURCES.resource_id', ondelete='CASCADE'), primary_key=True, index=True)
+ FRONT_END = Column(String(255))
+ BACK_END = Column(String(255))
+ CONFIGURATION = Column(String(255))
+
+
+class VLABACKENDGENERAL13A(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_GENERAL13A'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ OBSERVING_CHOICE = Column(String(255), nullable=False)
+ FILE_CONTENTS = Column(MEDIUMBLOB)
+ REST_FREQUENCIES = Column(String)
+
+
+class VLABACKENDSRO13A(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_SRO13A'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SRO_CATALOG_ID = Column(String(255), nullable=False)
+ SRO_RESOURCE_NAME = Column(String(255), nullable=False)
+ SRO_RESOURCE_ID = Column(String(255), nullable=False)
+
+
+class VLABACKENDOSRODUAL(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_OSRODUAL'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String)
+ REST_FREQUENCY_TYPE = Column(String(255))
+ SKY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ HANNING_SMOOTH = Column(String(255))
+ SPECTRAL_RESOL = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ IF_MODE = Column(SmallInteger)
+ Polarization = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ CHANNEL_WIDTH = Column(String(255))
+ CHANNEL_WIDTH_UNIT = Column(String(255))
+
+
+class VLABACKENDSINGLE(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_SINGLE'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ SKY_FREQUENCY = Column(Text)
+ SKY_UNIT = Column(String(255))
+ REST_FREQUENCY_TYPE = Column(String(255))
+
+
+class VLABACKENDECSO(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_ECSO'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ COMMENTS = Column(Text)
+
+
+class VLABACKENDOSRO2(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_OSRO2'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(Text)
+ REST_FREQUENCY_TYPE = Column(String(255))
+ SKY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ HANNING_SMOOTH = Column(String(255))
+ SPECTRAL_RESOL = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ IF_MODE = Column(SmallInteger)
+ Polarization = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ CHANNEL_WIDTH = Column(String(255))
+ CHANNEL_WIDTH_UNIT = Column(String(255))
+
+
+class VLABACKENDOSROFULL(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_OSROFULL'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String)
+ REST_FREQUENCY_TYPE = Column(String(255))
+ SKY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ HANNING_SMOOTH = Column(String(255))
+ SPECTRAL_RESOL = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ IF_MODE = Column(SmallInteger)
+ Polarization = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ CHANNEL_WIDTH = Column(String(255))
+ CHANNEL_WIDTH_UNIT = Column(String(255))
+
+
+class VLABACKENDHIGH(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_HIGH'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SPECTRAL_RESOL = Column(Text)
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ TIME_RESOLUTION = Column(Text)
+ TIME_RESOLUTION_UNIT = Column(String(255))
+
+
+class VLABACKENDOSROSINGLE(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_OSROSINGLE'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(String)
+ REST_FREQUENCY_TYPE = Column(String(255))
+ SKY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ HANNING_SMOOTH = Column(String(255))
+ SPECTRAL_RESOL = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ IF_MODE = Column(SmallInteger)
+ Polarization = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ CHANNEL_WIDTH = Column(String(255))
+ CHANNEL_WIDTH_UNIT = Column(String(255))
+
+
+class VLABACKENDOSRO1(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_OSRO1'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(Text)
+ REST_FREQUENCY_TYPE = Column(String(255))
+ SKY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ HANNING_SMOOTH = Column(String(255))
+ SPECTRAL_RESOL = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ IF_MODE = Column(SmallInteger)
+ Polarization = Column(String(255))
+ CHANNEL_COUNT = Column(Integer)
+ CHANNEL_WIDTH = Column(String(255))
+ CHANNEL_WIDTH_UNIT = Column(String(255))
+
+
+class VLABACKENDRSRO13A(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_RSRO13A'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ DESCRIPTION = Column(String(255), nullable=False)
+
+
+class VLABACKENDMULTI(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_MULTI'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SKY_FREQUENCY = Column(Text)
+ REST_FREQUENCY_TYPE = Column(String(255))
+ SKY_UNIT = Column(String(255))
+ BAND_WIDTH = Column(String(255))
+ BAND_WIDTH_UNIT = Column(String(255))
+ HANNING_SMOOTH = Column(String(255))
+ SPECTRAL_RESOL = Column(String(255))
+ SPECTRAL_RESOL_UNIT = Column(String(255))
+ IF_MODE = Column(SmallInteger)
+ CHANNEL_COUNT = Column(Integer)
+
+
+class VLABACKENDRSRO(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_RSRO'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ COMMENTS = Column(Text)
+
+
+class VLABACKENDSTAFF13A(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_STAFF13A'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ DESCRIPTION = Column(String(255), nullable=False)
+
+
+class VLABACKENDOTHER(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_OTHER'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ SPECIAL_BACK_END = Column(String(255))
+
+
+class VLABACKENDWIDEBAND(VLARESOURCE):
+ __tablename__ = 'VLA_BACKEND_WIDEBAND'
+
+ Id = Column(ForeignKey('VLA_RESOURCE.Id', ondelete='CASCADE'), primary_key=True, index=True)
+ BASEBAND_CENTERS = Column(String(255), nullable=False)
+ POLARIZATION_PRODUCTS = Column(String(255), nullable=False)
+ DUMP_TIME = Column(String(255), nullable=False)
+ DATA_RATE = Column(String(255), nullable=False)
+ BASEBANDS = Column(String(255), nullable=False)
+ SHARED_RISK = Column(BIT(1), nullable=False)
+ DATA_RATE_JUSTIFICATION = Column(String(255))
+
+
+class TechJustification(Base):
+ __tablename__ = 'TechJustification'
+
+ id = Column(Integer, primary_key=True)
+ proposal_id = Column(Integer, nullable=False, unique=True)
+ arrayconfig = Column(Text)
+ daynight = Column(Text)
+ receivers = Column(Text)
+ correlator = Column(Text)
+ sensitivity = Column(Text)
+ integrationTime = Column(Text)
+ dumpTime = Column(Text)
+ imaging = Column(Text)
+ rfi = Column(Text)
+ stations = Column(Text)
+ weather = Column(Text)
+ dates = Column(Text)
+ minlength = Column(Text)
+ phaseref = Column(Text)
+ polarization = Column(Text)
+ hsa = Column(Text)
+ resource = Column(Text)
+ mapping = Column(Text)
+ overhead = Column(Text)
+ novel = Column(Text)
+ pulsar = Column(Text)
+ other = Column(Text)
+ mosaic = Column(Text)
+ bitrate = Column(Text)
+ joint = Column(Text)
+ total_correlator_output_data_size = Column(Text)
+
+
+class TechJustificationFile(Base):
+ __tablename__ = 'TechJustificationFile'
+
+ id = Column(Integer, primary_key=True)
+ techjust_id = Column(Integer, nullable=False)
+ filename = Column(Text, nullable=False)
+ filetype = Column(Text, nullable=False)
+ filesize = Column(Integer, nullable=False)
+ file = Column(MEDIUMBLOB)
+
+
+class ValidConfiguration(Base):
+ __tablename__ = 'ValidConfigurations'
+
+ id = Column(Integer, primary_key=True)
+ cycle_id = Column(Integer, nullable=False)
+ configuration = Column(Text, nullable=False)
+
+
+class Addres(Base):
+ __tablename__ = 'address'
+
+ address_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ defaultAddress = Column(BIT(1))
+ addressDescription = Column(String(255))
+ street1 = Column(String(255))
+ street2 = Column(String(255))
+ street3 = Column(String(255))
+ street4 = Column(String(255))
+ city = Column(String(255))
+ postalCode = Column(String(255))
+ region = Column(String(255))
+ addressState_id = Column(ForeignKey('addressState.addressState_id'), index=True)
+ addressCountry_id = Column(ForeignKey('addressCountry.addressCountry_id'), index=True)
+ person_id = Column(ForeignKey('person.person_id'), index=True)
+ organization_id = Column(ForeignKey('organization.organization_id'), index=True)
+ updatedOn = Column(DateTime)
+ updatedBy = Column(Integer)
+
+ addressCountry = relationship('AddressCountry')
+ addressState = relationship('AddressState')
+ organization = relationship('Organization')
+ person = relationship('Person')
+
+
+class AddressCountry(Base):
+ __tablename__ = 'addressCountry'
+
+ addressCountry_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ addressCountry = Column(String(255), nullable=False)
+ alpha_2 = Column(String(2))
+ alpha_3 = Column(String(3))
+ idd_isd = Column(String(25))
+ status = Column(String(25))
+
+
+t_addressCountry_bck = Table(
+ 'addressCountry_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class AddressState(Base):
+ __tablename__ = 'addressState'
+
+ addressState_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ stateName = Column(String(255), nullable=False)
+ state = Column(String(255))
+
+
+t_addressState_bck = Table(
+ 'addressState_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class AddressType(Base):
+ __tablename__ = 'addressType'
+
+ addressType_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ addressType = Column(String(255), nullable=False)
+
+ addresss = relationship('Addres', secondary='address_addressType')
+
+
+t_addressType_bck = Table(
+ 'addressType_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_address_addressType = Table(
+ 'address_addressType', metadata,
+ Column('address_id', ForeignKey('address.address_id'), primary_key=True, nullable=False, index=True),
+ Column('addressType_id', ForeignKey('addressType.addressType_id'), primary_key=True, nullable=False, index=True)
+)
+
+
+t_address_bck = Table(
+ 'address_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class Author(Base):
+ __tablename__ = 'author'
+
+ author_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ AFFILIATION = Column(String(255))
+ DOMESTIC = Column(BIT(1))
+ NEW_USER = Column(BIT(1))
+ EMAIL = Column(String(255), nullable=False)
+ ADDRESS = Column(String(255))
+ FIRST_NAME = Column(String(255))
+ LAST_NAME = Column(String(255))
+ PROFESSIONAL_STATUS = Column(String(255))
+ THESIS_OBSERVING = Column(BIT(1), nullable=False)
+ GRADUATION_YEAR = Column(String(255))
+ TELEPHONE = Column(String(255))
+ OLDAUTHOR_ID = Column(String(255), nullable=False)
+ DISPLAY_POSITION = Column(Integer, nullable=False)
+ STORAGE_ORDER = Column(Integer, nullable=False)
+ OTHER_AWARDS = Column(Text)
+ SUPPORT_REQUESTER = Column(BIT(1), nullable=False)
+ SUPPORTED = Column(BIT(1), nullable=False)
+ BUDGET = Column(Float(asdecimal=True))
+ ASSIGNMENT = Column(Text)
+ proposal_id = Column(ForeignKey('proposal.proposal_id'), nullable=False, index=True)
+ user_id = Column(ForeignKey('userAuthentication.userAuthentication_id'), index=True)
+ dissertationPlan_id = Column(ForeignKey('dissertationPlan.dissertationPlan_id'), index=True)
+ person_id = Column(BigInteger, nullable=False)
+ country_id = Column(ForeignKey('addressCountry.addressCountry_id'), index=True)
+ state_id = Column(ForeignKey('addressState.addressState_id'), index=True)
+ organization_id = Column(ForeignKey('organization.organization_id'), index=True)
+
+ country = relationship('AddressCountry')
+ dissertationPlan = relationship('DissertationPlan')
+ organization = relationship('Organization')
+ proposal = relationship('Proposal', primaryjoin='Author.proposal_id == Proposal.proposal_id', lazy='subquery')
+ state = relationship('AddressState')
+ user = relationship('UserAuthentication', lazy='subquery')
+
+
+t_author_bck = Table(
+ 'author_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class Bug(Base):
+ __tablename__ = 'bug'
+
+ bug_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ id = Column(Integer, nullable=False)
+ status = Column(String(255), nullable=False)
+ statusNumber = Column(Integer, nullable=False, index=True)
+ type = Column(String(255), nullable=False)
+ typeNumber = Column(Integer, nullable=False, index=True)
+ issueDate = Column(Date, index=True)
+ completeDate = Column(Date, index=True)
+ author = Column(String(255))
+ location = Column(String(255))
+ description = Column(Text)
+ resolution = Column(Text)
+ versionNumber = Column(String(255))
+ osName = Column(String(255))
+ email = Column(String(255))
+
+
+class CasaRegionSetup(Base):
+ __tablename__ = 'casaRegionSetup'
+
+ regionSetup_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ region = Column(String(128), nullable=False, index=True)
+ userName = Column(String(128), nullable=False, index=True)
+ firstName = Column(String(128), nullable=False)
+ lastName = Column(String(128), nullable=False)
+ email = Column(Text, nullable=False)
+ user_id = Column(ForeignKey('userAuthentication.userAuthentication_id'), index=True)
+
+ user = relationship('UserAuthentication')
+
+
+class CasaUserSetup(Base):
+ __tablename__ = 'casaUserSetup'
+
+ userSetup_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ role_id = Column(ForeignKey('role.role_id'), nullable=False, index=True)
+ user_id = Column(ForeignKey('userAuthentication.userAuthentication_id'), nullable=False, index=True)
+ region_id = Column(ForeignKey('casaRegionSetup.regionSetup_id'), nullable=False, index=True)
+ region = Column(String(128), nullable=False, index=True)
+ userName = Column(String(128), nullable=False, index=True)
+ firstName = Column(String(128), nullable=False)
+ lastName = Column(String(128), nullable=False)
+ email = Column(Text, nullable=False)
+
+ region1 = relationship('CasaRegionSetup')
+ role = relationship('Role')
+ user = relationship('UserAuthentication')
+
+
+class CasaBug(Base):
+ __tablename__ = 'casa_bug'
+
+ bug_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ id = Column(Integer, nullable=False)
+ status = Column(String(255), nullable=False)
+ statusNumber = Column(Integer, nullable=False, index=True)
+ type = Column(String(255), nullable=False)
+ typeNumber = Column(Integer, nullable=False, index=True)
+ issueDate = Column(Date, index=True)
+ completeDate = Column(Date, index=True)
+ author = Column(String(255))
+ location = Column(String(255))
+ description = Column(Text)
+ resolution = Column(Text)
+ versionNumber = Column(String(255))
+ osName = Column(String(255))
+ email = Column(String(255))
+ region_id = Column(ForeignKey('casaRegionSetup.regionSetup_id'), index=True)
+ regionName = Column(String(255))
+ uss_id = Column(ForeignKey('casaUserSetup.userSetup_id'), index=True)
+ username = Column(String(255))
+
+ region = relationship('CasaRegionSetup')
+ uss = relationship('CasaUserSetup')
+
+
+class Casalog(Base):
+ __tablename__ = 'casalog'
+
+ casalog_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ date = Column(DateTime, nullable=False, index=True)
+ pagename = Column(String(50), nullable=False)
+ module = Column(String(20), nullable=False)
+ action = Column(String(24))
+ ipaddress = Column(String(128))
+ username = Column(String(255), nullable=False, index=True)
+
+
+class DissertationPlan(Base):
+ __tablename__ = 'dissertationPlan'
+
+ dissertationPlan_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ FILE_NAME = Column(String(255), nullable=False)
+ FILE_ID = Column(String(255), nullable=False)
+ BYTE_SIZE = Column(Integer)
+ PAGE_COUNT = Column(Integer)
+ LAST_DATE = Column(DateTime)
+ CONTENT_TYPE = Column(String(255), nullable=False)
+ FILE_TYPE = Column(String(255))
+ FILE_CONTENTS = Column(MEDIUMBLOB)
+ PERSON_ID = Column(ForeignKey('person.person_id'), unique=True)
+ USER_ID = Column(ForeignKey('userAuthentication.userAuthentication_id'), unique=True)
+
+ person = relationship('Person')
+ userAuthentication = relationship('UserAuthentication')
+
+
+class Email(Base):
+ __tablename__ = 'email'
+
+ email_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ emailType = Column(String(255))
+ email = Column(String(255), nullable=False)
+ defaultEmail = Column(BIT(1), nullable=False)
+ person_id = Column(ForeignKey('person.person_id'), index=True)
+ organization_id = Column(ForeignKey('organization.organization_id'), index=True)
+
+ organization = relationship('Organization')
+ person = relationship('Person')
+
+
+t_email_bck = Table(
+ 'email_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class EntryStatu(Base):
+ __tablename__ = 'entryStatus'
+
+ entryStatus_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ entryStatus = Column(String(255), nullable=False)
+
+
+t_entryStatus_bck = Table(
+ 'entryStatus_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_fix_15A_codes = Table(
+ 'fix_15A_codes', metadata,
+ Column('proposal_id', BigInteger, nullable=False, server_default=text("'0'")),
+ Column('prop_id', String(255), nullable=False),
+ Column('new_id', String(320), nullable=False, server_default=text("''")),
+ Column('pi_name', String(511)),
+ Column('email', String(255), nullable=False),
+ Column('deadline_date', DateTime)
+)
+
+
+class JustificationFile(Base):
+ __tablename__ = 'justificationFile'
+
+ justificationFile_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ JUSTIFICATION_FILE_NAME = Column(String(255), nullable=False)
+ JUSTIFICATION_FILE_ID = Column(String(255), nullable=False)
+ JUSTIFICATION_BYTE_SIZE = Column(Integer)
+ JUSTIFICATION_PAGE_COUNT = Column(Integer)
+ JUST_STORAGE_DATE = Column(DateTime)
+ CONTENT_TYPE = Column(String(255), nullable=False)
+ JUSTIFICATION_FILE_TYPE = Column(String(255))
+ FILE_CONTENTS = Column(MEDIUMBLOB)
+
+
+t_justification_bck = Table(
+ 'justification_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class ObservingType(Base):
+ __tablename__ = 'observingType'
+
+ observingType_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ proposal_id = Column(ForeignKey('proposal.proposal_id'), index=True)
+ display_position = Column(Integer)
+ observingType = Column(String(255), nullable=False)
+
+ proposal = relationship('Proposal')
+
+
+t_observingType_bck = Table(
+ 'observingType_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class Organization(Base):
+ __tablename__ = 'organization'
+
+ organization_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ formalName = Column(String(255), nullable=False)
+ shortName = Column(String(255))
+ mailingName = Column(String(255))
+ departmentName = Column(String(255))
+ url = Column(String(255))
+ contactPerson1 = Column(String(255))
+ contactPerson2 = Column(String(255))
+ parentId = Column(Integer)
+ updatedOn = Column(DateTime)
+ updatedBy = Column(Integer)
+ entryStatus = Column(ForeignKey('entryStatus.entryStatus_id'), index=True)
+
+ entryStatu = relationship('EntryStatu')
+ persons = relationship('Person', secondary='person_organization')
+
+
+class OrganizationType(Base):
+ __tablename__ = 'organizationType'
+
+ organizationType_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ organizationType = Column(String(255), nullable=False)
+
+ organizations = relationship('Organization', secondary='organization_organizationType')
+
+
+t_organizationType_bck = Table(
+ 'organizationType_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_organization_bck = Table(
+ 'organization_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_organization_organizationType = Table(
+ 'organization_organizationType', metadata,
+ Column('organization_id', ForeignKey('organization.organization_id'), primary_key=True, nullable=False, index=True),
+ Column('organizationType_id', ForeignKey('organizationType.organizationType_id'), primary_key=True, nullable=False, index=True)
+)
+
+
+class Pagelog(Base):
+ __tablename__ = 'pagelog'
+
+ pagelog_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ date = Column(DateTime, nullable=False, index=True)
+ pagename = Column(String(50), nullable=False)
+ module = Column(String(20), nullable=False)
+ action = Column(String(255))
+ recordable = Column(BIT(1))
+ username = Column(String(255), nullable=False, index=True)
+ sessionid = Column(Text)
+ proposalid = Column(String(128))
+ ipAddress = Column(String(255))
+
+
+class Permissionset(Base):
+ __tablename__ = 'permissionset'
+
+ permissionset_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ home_view = Column(BIT(1), nullable=False)
+ home_full = Column(BIT(1), nullable=False)
+ proposalModule_view = Column(BIT(1), nullable=False)
+ proposalModule_full = Column(BIT(1), nullable=False)
+ myproposals_view = Column(BIT(1), nullable=False)
+ myproposals_full = Column(BIT(1), nullable=False)
+ proposalList_view = Column(BIT(1), nullable=False)
+ proposalList_full = Column(BIT(1), nullable=False)
+ settingsModule_view = Column(BIT(1), nullable=False)
+ settingsModule_full = Column(BIT(1), nullable=False)
+ myprofile_view = Column(BIT(1), nullable=False)
+ myprofile_full = Column(BIT(1), nullable=False)
+ roleList_view = Column(BIT(1), nullable=False)
+ roleList_full = Column(BIT(1), nullable=False)
+ profileList_view = Column(BIT(1), nullable=False)
+ profileList_full = Column(BIT(1), nullable=False)
+ organizationList_view = Column(BIT(1), nullable=False)
+ organizationList_full = Column(BIT(1), nullable=False)
+ bug_view = Column(BIT(1), nullable=False)
+ bug_full = Column(BIT(1), nullable=False)
+ analyticsModule_view = Column(BIT(1), nullable=False)
+ analyticsModule_full = Column(BIT(1), nullable=False)
+ a_proposal_view = Column(BIT(1), nullable=False)
+ a_proposal_full = Column(BIT(1), nullable=False)
+ a_casa_view = Column(BIT(1), nullable=False)
+ a_casa_full = Column(BIT(1), nullable=False)
+ casaModule_view = Column(BIT(1), nullable=False)
+ casaModule_full = Column(BIT(1), nullable=False)
+ casaHome_view = Column(BIT(1), nullable=False)
+ casaHome_full = Column(BIT(1), nullable=False)
+ casa_bugs_view = Column(BIT(1), nullable=False)
+ casa_bugs_full = Column(BIT(1), nullable=False)
+ casa_setup_view = Column(BIT(1), nullable=False)
+ casa_setup_full = Column(BIT(1), nullable=False)
+ pscAccess = Column(BIT(1), nullable=False)
+ reviewCategory_view = Column(BIT(1), nullable=False)
+ reviewCategory_full = Column(BIT(1), nullable=False)
+ available_authors_view = Column(BIT(1), nullable=False)
+ available_authors_full = Column(BIT(1), nullable=False)
+ available_organizations_view = Column(BIT(1), nullable=False)
+ available_organizations_full = Column(BIT(1), nullable=False)
+ profilesModule_view = Column(BIT(1), nullable=False)
+ profilesModule_full = Column(BIT(1), nullable=False)
+ adminModule_view = Column(BIT(1), nullable=False)
+ adminModule_full = Column(BIT(1), nullable=False)
+ groupList_view = Column(BIT(1), nullable=False)
+ groupList_full = Column(BIT(1), nullable=False)
+ reviewsModule_view = Column(BIT(1), nullable=False)
+ reviewsModule_full = Column(BIT(1), nullable=False)
+ myreviews_view = Column(BIT(1), nullable=False)
+ myreviews_full = Column(BIT(1), nullable=False)
+ refereeSetup_view = Column(BIT(1), nullable=False)
+ refereeSetup_full = Column(BIT(1), nullable=False)
+ mergeAccounts = Column(BIT(1), nullable=False)
+ editRefereeComments = Column(BIT(1), nullable=False)
+ observationsModule_view = Column(BIT(1), nullable=False)
+ observationsModule_full = Column(BIT(1), nullable=False)
+ helpdeskModule_view = Column(BIT(1), nullable=False)
+ helpdeskModule_full = Column(BIT(1), nullable=False)
+ reviewCycles_view = Column(BIT(1), nullable=False)
+ reviewCycles_full = Column(BIT(1), nullable=False)
+ tech_review_panel_view = Column(BIT(1), nullable=False)
+ tech_review_panel_full = Column(BIT(1), nullable=False)
+ scientific_review_panel_view = Column(BIT(1), nullable=False)
+ scientific_review_panel_full = Column(BIT(1), nullable=False)
+ reviewsSetup_view = Column(BIT(1), nullable=False)
+ reviewsSetup_full = Column(BIT(1), nullable=False)
+ technical_review_categorySetup_view = Column(BIT(1), nullable=False)
+ technical_review_categorySetup_full = Column(BIT(1), nullable=False)
+ scientific_review_categorySetup_view = Column(BIT(1), nullable=False)
+ scientific_review_categorySetup_full = Column(BIT(1), nullable=False)
+ technical_refereeSetup_view = Column(BIT(1), nullable=False)
+ technical_refereeSetup_full = Column(BIT(1), nullable=False)
+ scientific_refereeSetup_view = Column(BIT(1), nullable=False)
+ scientific_refereeSetup_full = Column(BIT(1), nullable=False)
+ reviews_summary_view = Column(BIT(1), nullable=False)
+ reviews_summary_full = Column(BIT(1), nullable=False)
+ panel_reviews_view = Column(BIT(1), nullable=False)
+ panel_reviews_full = Column(BIT(1), nullable=False)
+ tac_reviews_view = Column(BIT(1), nullable=False)
+ tac_reviews_full = Column(BIT(1), nullable=False)
+ tac_members_view = Column(BIT(1), nullable=False)
+ tac_members_full = Column(BIT(1), nullable=False)
+ tac_panel_view = Column(BIT(1), nullable=False)
+ tac_panel_full = Column(BIT(1), nullable=False)
+ dataProcessingModule_view = Column(BIT(1), nullable=False)
+ dataProcessingModule_full = Column(BIT(1), nullable=False)
+
+
+t_permissionset_bck = Table(
+ 'permissionset_bck', metadata,
+ Column('permissionset_id', BigInteger, nullable=False, server_default=text("'0'")),
+ Column('objectversion', Integer, nullable=False, server_default=text("'0'")),
+ Column('home_view', BIT(1), nullable=False),
+ Column('home_full', BIT(1), nullable=False),
+ Column('proposalModule_view', BIT(1), nullable=False),
+ Column('proposalModule_full', BIT(1), nullable=False),
+ Column('myproposals_view', BIT(1), nullable=False),
+ Column('myproposals_full', BIT(1), nullable=False),
+ Column('proposalList_view', BIT(1), nullable=False),
+ Column('proposalList_full', BIT(1), nullable=False),
+ Column('settingsModule_view', BIT(1), nullable=False),
+ Column('settingsModule_full', BIT(1), nullable=False),
+ Column('myprofile_view', BIT(1), nullable=False),
+ Column('myprofile_full', BIT(1), nullable=False),
+ Column('roleList_view', BIT(1), nullable=False),
+ Column('roleList_full', BIT(1), nullable=False),
+ Column('profileList_view', BIT(1), nullable=False),
+ Column('profileList_full', BIT(1), nullable=False),
+ Column('organizationList_view', BIT(1), nullable=False),
+ Column('organizationList_full', BIT(1), nullable=False),
+ Column('bug_view', BIT(1), nullable=False),
+ Column('bug_full', BIT(1), nullable=False),
+ Column('analyticsModule_view', BIT(1), nullable=False),
+ Column('analyticsModule_full', BIT(1), nullable=False),
+ Column('a_summary_view', BIT(1), nullable=False),
+ Column('a_summary_full', BIT(1), nullable=False)
+)
+
+
+class Person(Base):
+ __tablename__ = 'person'
+ __table_args__ = (
+ Index('person_i2', 'firstName', 'lastName'),
+ )
+
+ person_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ prefix = Column(ForeignKey('personPrefix.personPrefix_id'), index=True)
+ firstName = Column(String(255))
+ middleName = Column(String(255))
+ lastName = Column(String(255))
+ preferredName = Column(String(255))
+ suffix = Column(ForeignKey('personSuffix.personSuffix_id'), index=True)
+ aipsNumber = Column(SmallInteger)
+ gender = Column(String(255))
+ url = Column(String(255))
+ unknownAffiliationName = Column(String(255))
+ unknownAffiliationCountry = Column(ForeignKey('addressCountry.addressCountry_id'), index=True)
+ personContactLevel_id = Column(ForeignKey('personContactLevel.personContactLevel_id'), index=True)
+ entryStatus = Column(ForeignKey('entryStatus.entryStatus_id'), index=True)
+ defaultOrganization_id = Column(ForeignKey('organization.organization_id'), index=True)
+ personAuthentication_id = Column(ForeignKey('userAuthentication.userAuthentication_id'), index=True)
+ personSpouse_id = Column(ForeignKey('personSpouse.personSpouse_id'), index=True)
+ outOfDateReporterId = Column(Integer)
+ updatedOn = Column(DateTime)
+ updatedBy = Column(Integer)
+ graduationYear = Column(SmallInteger)
+ timezone = Column(String(255))
+ lastLogin = Column(DateTime)
+ lastObservation = Column(DateTime)
+ incomplete = Column(BIT(1))
+ affiliation = Column(Text)
+ enabled = Column(BIT(1), nullable=False, index=True)
+ gender_for_metrics = Column(String(255))
+
+ defaultOrganization = relationship('Organization')
+ entryStatu = relationship('EntryStatu')
+ personAuthentication = relationship('UserAuthentication')
+ personContactLevel = relationship('PersonContactLevel')
+ personSpouse = relationship('PersonSpouse')
+ personPrefix = relationship('PersonPrefix')
+ personSuffix = relationship('PersonSuffix')
+ addressCountry = relationship('AddressCountry')
+ reviewCategorys = relationship('ReviewCategory', secondary='person_reviewCategory')
+
+
+class PersonContactLevel(Base):
+ __tablename__ = 'personContactLevel'
+
+ personContactLevel_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ personContactLevel = Column(String(255), nullable=False)
+
+
+t_personContactLevel_bck = Table(
+ 'personContactLevel_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class PersonGroup(Base):
+ __tablename__ = 'personGroup'
+
+ personGroup_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ personGroupType = Column(SmallInteger, nullable=False)
+ personGroupName = Column(String(255), nullable=False)
+ personGroupDescription = Column(String(255), nullable=False)
+
+ persons = relationship('Person', secondary='person_personGroup')
+
+
+t_personGroup_bck = Table(
+ 'personGroup_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class PersonNote(Base):
+ __tablename__ = 'personNote'
+
+ userNote_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ personNoteText = Column(String(255), nullable=False)
+ updatedOn = Column(DateTime)
+ updatedBy = Column(Integer)
+ person_id = Column(ForeignKey('person.person_id'), index=True)
+
+ person = relationship('Person')
+
+
+t_personNote_bck = Table(
+ 'personNote_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class PersonPrefix(Base):
+ __tablename__ = 'personPrefix'
+
+ personPrefix_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ personPrefix = Column(String(255), nullable=False, unique=True)
+
+
+t_personPrefix_bck = Table(
+ 'personPrefix_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class PersonSpouse(Base):
+ __tablename__ = 'personSpouse'
+
+ personSpouse_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ prefix = Column(ForeignKey('personPrefix.personPrefix_id'), index=True)
+ firstName = Column(String(255))
+ middleName = Column(String(255))
+ lastName = Column(String(255))
+ preferredName = Column(String(255))
+ suffix = Column(ForeignKey('personSuffix.personSuffix_id'), index=True)
+ gender = Column(String(1))
+
+ personPrefix = relationship('PersonPrefix')
+ personSuffix = relationship('PersonSuffix')
+
+
+class PersonSuffix(Base):
+ __tablename__ = 'personSuffix'
+
+ personSuffix_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ personSuffix = Column(String(255), nullable=False, unique=True)
+
+
+t_personSuffix_bck = Table(
+ 'personSuffix_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class PersonType(Base):
+ __tablename__ = 'personType'
+
+ personType_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ personType = Column(String(255), nullable=False)
+ educationType = Column(BIT(1), nullable=False)
+
+ persons = relationship('Person', secondary='person_personType')
+
+
+t_personType_bck = Table(
+ 'personType_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_person_bck = Table(
+ 'person_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_person_organization = Table(
+ 'person_organization', metadata,
+ Column('organization_id', ForeignKey('organization.organization_id'), primary_key=True, nullable=False, index=True),
+ Column('person_id', ForeignKey('person.person_id'), primary_key=True, nullable=False, index=True)
+)
+
+
+t_person_personGroup = Table(
+ 'person_personGroup', metadata,
+ Column('person_id', ForeignKey('person.person_id'), primary_key=True, nullable=False, index=True),
+ Column('personGroup_id', ForeignKey('personGroup.personGroup_id', ondelete='CASCADE'), primary_key=True, nullable=False, index=True)
+)
+
+
+t_person_personType = Table(
+ 'person_personType', metadata,
+ Column('person_id', ForeignKey('person.person_id'), primary_key=True, nullable=False, index=True),
+ Column('personType_id', ForeignKey('personType.personType_id'), primary_key=True, nullable=False, index=True)
+)
+
+
+t_person_reviewCategory = Table(
+ 'person_reviewCategory', metadata,
+ Column('person_id', ForeignKey('person.person_id'), primary_key=True, nullable=False, index=True),
+ Column('reviewCategory_id', ForeignKey('reviewCategory.reviewCategory_id'), primary_key=True, nullable=False, index=True)
+)
+
+
+class Phone(Base):
+ __tablename__ = 'phone'
+
+ phone_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ phoneType = Column(String(255))
+ phone = Column(String(255), nullable=False)
+ defaultPhone = Column(BIT(1), nullable=False)
+ fax = Column(BIT(1), nullable=False)
+ person_id = Column(ForeignKey('person.person_id'), index=True)
+ organization_id = Column(ForeignKey('organization.organization_id'), index=True)
+
+ organization = relationship('Organization')
+ person = relationship('Person')
+
+
+t_phone_bck = Table(
+ 'phone_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class Proposal(Base):
+ __tablename__ = 'proposal'
+
+ proposal_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ ABSTRACT = Column(Text)
+ CREATED_DATE = Column(DateTime, nullable=False)
+ PROP_ID = Column(String(255), nullable=False, unique=True)
+ LEGACY_ID = Column(String(255, 'latin1_general_cs'))
+ PRESENT = Column(String(255))
+ PROPOSAL_TYPE = Column(String(255))
+ JOINT_PROPOSAL_TYPE = Column(String(255))
+ RAPID_RESPONSE_TYPE = Column(String(255))
+ STAFF_SUPPORT = Column(String(255))
+ STATUS = Column(String(255))
+ PLAN_SUBMITTED = Column(String(255))
+ MODIFIED_DATE = Column(DateTime, nullable=False)
+ SUBMITTED_DATE = Column(DateTime)
+ DEADLINE_DATE = Column(DateTime)
+ TITLE = Column(Text)
+ PROCESSED = Column(BIT(1), nullable=False)
+ RELATED_PROPOSALS = Column(Text)
+ PREV_PROP_IDS = Column(Text)
+ LOCK_USER_ID = Column(String(255))
+ LOCK_MILLIS = Column(BigInteger)
+ LOCK_USER_INFO = Column(String(255))
+ OBSERVING_TYPE_OTHER = Column(String(255))
+ OLD_CONTACT = Column(String(255))
+ OLD_PI = Column(String(255))
+ OLD_EDITOR = Column(String(255))
+ justificationFile_id = Column(ForeignKey('justificationFile.justificationFile_id'), index=True)
+ editor_id = Column(ForeignKey('author.author_id'), index=True)
+ contact_id = Column(ForeignKey('author.author_id'), index=True)
+ principal_investigator_id = Column(ForeignKey('author.author_id'), index=True)
+ comments = Column(Text)
+ category1_id = Column(ForeignKey('reviewCategory.reviewCategory_id'), index=True)
+ category2_id = Column(ForeignKey('reviewCategory.reviewCategory_id'), index=True)
+ TELESCOPE = Column(String(255), nullable=False)
+ category3_id = Column(ForeignKey('reviewCategory.reviewCategory_id'), index=True)
+ displayTechnicalReviews = Column(BIT(1))
+ reviewersConflict = Column(BIT(1))
+ technicalCategory_id = Column(ForeignKey('technical_review_category.tech_reviewCategory_id'), index=True)
+ scienceCategory = Column(String(255))
+ public = Column(BIT(1), nullable=False)
+ allocated_hours = Column(Float(asdecimal=True), server_default=text("'0'"))
+ disposition_letter = Column(MEDIUMBLOB)
+ trigger_criteria = Column(String)
+ techJustification_id = Column(Integer)
+ uses_GBT = Column(BIT(1), nullable=False)
+ uses_VLA = Column(BIT(1), nullable=False)
+ allocated_hours_a = Column(Float(asdecimal=True), server_default=text("'0'"))
+ allocated_hours_b = Column(Float(asdecimal=True), server_default=text("'0'"))
+ allocated_hours_c = Column(Float(asdecimal=True), server_default=text("'0'"))
+ sponsor = Column(String(255))
+ hst_orbits_requested = Column(String(255))
+ external = Column(BIT(1), nullable=False)
+ total_time = Column(Float(asdecimal=True), server_default=text("'0'"))
+ external_proposal_id = Column(String(255))
+ swift_ksecs = Column(String(255))
+ chandra_ksecs = Column(String(255))
+
+ category1 = relationship('ReviewCategory', primaryjoin='Proposal.category1_id == ReviewCategory.reviewCategory_id')
+ category2 = relationship('ReviewCategory', primaryjoin='Proposal.category2_id == ReviewCategory.reviewCategory_id')
+ category3 = relationship('ReviewCategory', primaryjoin='Proposal.category3_id == ReviewCategory.reviewCategory_id')
+ contact = relationship('Author', primaryjoin='Proposal.contact_id == Author.author_id')
+ editor = relationship('Author', primaryjoin='Proposal.editor_id == Author.author_id')
+ justificationFile = relationship('JustificationFile')
+ principal_investigator = relationship('Author', primaryjoin='Proposal.principal_investigator_id == Author.author_id')
+ technicalCategory = relationship('TechnicalReviewCategory')
+
+
+t_proposal_15A_bck = Table(
+ 'proposal_15A_bck', metadata,
+ Column('proposal_id', BigInteger, nullable=False, server_default=text("'0'")),
+ Column('objectversion', Integer, nullable=False),
+ Column('ABSTRACT', Text),
+ Column('CREATED_DATE', DateTime, nullable=False),
+ Column('PROP_ID', String(255), nullable=False),
+ Column('LEGACY_ID', String(255, 'latin1_general_cs')),
+ Column('PRESENT', String(255)),
+ Column('PROPOSAL_TYPE', String(255)),
+ Column('JOINT_PROPOSAL_TYPE', String(255)),
+ Column('RAPID_RESPONSE_TYPE', String(255)),
+ Column('STAFF_SUPPORT', String(255)),
+ Column('STATUS', String(255)),
+ Column('PLAN_SUBMITTED', String(255)),
+ Column('MODIFIED_DATE', DateTime, nullable=False),
+ Column('SUBMITTED_DATE', DateTime),
+ Column('DEADLINE_DATE', DateTime),
+ Column('TITLE', Text),
+ Column('PROCESSED', BIT(1), nullable=False),
+ Column('RELATED_PROPOSALS', Text),
+ Column('PREV_PROP_IDS', Text),
+ Column('LOCK_USER_ID', String(255)),
+ Column('LOCK_MILLIS', BigInteger),
+ Column('LOCK_USER_INFO', String(255)),
+ Column('OBSERVING_TYPE_OTHER', String(255)),
+ Column('OLD_CONTACT', String(255)),
+ Column('OLD_PI', String(255)),
+ Column('OLD_EDITOR', String(255)),
+ Column('justificationFile_id', BigInteger),
+ Column('editor_id', BigInteger),
+ Column('contact_id', BigInteger),
+ Column('principal_investigator_id', BigInteger),
+ Column('comments', Text),
+ Column('category1_id', BigInteger),
+ Column('category2_id', BigInteger),
+ Column('TELESCOPE', String(255), nullable=False),
+ Column('category3_id', BigInteger),
+ Column('displayTechnicalReviews', BIT(1)),
+ Column('reviewersConflict', BIT(1)),
+ Column('technicalCategory_id', BigInteger),
+ Column('scienceCategory', String(255)),
+ Column('public', BIT(1), nullable=False),
+ Column('allocated_hours', Float(asdecimal=True), server_default=text("'0'")),
+ Column('disposition_letter', MEDIUMBLOB),
+ Column('trigger_criteria', String),
+ Column('techJustification_id', Integer),
+ Column('uses_GBT', BIT(1), nullable=False),
+ Column('uses_VLA', BIT(1), nullable=False),
+ Column('allocated_hours_a', Float(asdecimal=True), server_default=text("'0'")),
+ Column('allocated_hours_b', Float(asdecimal=True), server_default=text("'0'")),
+ Column('allocated_hours_c', Float(asdecimal=True), server_default=text("'0'")),
+ Column('sponsor', String(255)),
+ Column('hst_orbits_requested', String(255))
+)
+
+
+t_proposal_bck = Table(
+ 'proposal_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class ProposalNormalizedScore(Base):
+ __tablename__ = 'proposal_normalized_scores'
+
+ normalized_scores_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ review_cycle_id = Column(ForeignKey('review_cycles.review_cycle_id'), nullable=False, index=True)
+ proposal_id = Column(ForeignKey('proposal.proposal_id'), nullable=False, index=True)
+ scientificCategory = Column(String(255), nullable=False)
+ avgNormalizedScore = Column(Float(asdecimal=True), nullable=False)
+ normalizedScoreStdev = Column(Float(asdecimal=True), nullable=False)
+ srpScores = Column(Float(asdecimal=True), nullable=False)
+ complete = Column(BIT(1), nullable=False)
+ finalized = Column(BIT(1), nullable=False)
+
+ proposal = relationship('Proposal')
+ review_cycle = relationship('ReviewCycle')
+
+
+t_proposal_normalized_scores_backup = Table(
+ 'proposal_normalized_scores_backup', metadata,
+ Column('normalized_scores_id', BigInteger, nullable=False, server_default=text("'0'")),
+ Column('objectversion', Integer, nullable=False),
+ Column('review_cycle_id', BigInteger, nullable=False),
+ Column('proposal_id', BigInteger, nullable=False),
+ Column('scientificCategory', String(255), nullable=False),
+ Column('avgNormalizedScore', Float(asdecimal=True), nullable=False),
+ Column('normalizedScoreStdev', Float(asdecimal=True), nullable=False),
+ Column('srpScores', Float(asdecimal=True), nullable=False),
+ Column('complete', BIT(1), nullable=False),
+ Column('finalized', BIT(1), nullable=False)
+)
+
+
+t_proposal_public_bit_allocated_hours_backup = Table(
+ 'proposal_public_bit_allocated_hours_backup', metadata,
+ Column('proposal_id', BigInteger, nullable=False, server_default=text("'0'")),
+ Column('public', BIT(1), nullable=False),
+ Column('allocated_hours', Float(asdecimal=True), server_default=text("'0'"))
+)
+
+
+class ProposalReview(Base):
+ __tablename__ = 'proposal_reviews'
+
+ review_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ complete = Column(BIT(1), nullable=False)
+ chair = Column(BIT(1), nullable=False)
+ conflict = Column(BIT(1), nullable=False)
+ reasons = Column(String)
+ review_cycle_id = Column(ForeignKey('review_cycles.review_cycle_id'), nullable=False, index=True)
+ referee_id = Column(ForeignKey('referees.referee_id'), nullable=False, index=True)
+ proposal_id = Column(ForeignKey('proposal.proposal_id'), nullable=False, index=True)
+ saved = Column(BIT(1), nullable=False)
+ close = Column(BIT(1), nullable=False)
+ reviewsComplete = Column(BIT(1), nullable=False)
+
+ proposal = relationship('Proposal')
+ referee = relationship('Referee')
+ review_cycle = relationship('ReviewCycle')
+
+
+class ScientificReview(ProposalReview):
+ __tablename__ = 'scientific_reviews'
+
+ review_id = Column(ForeignKey('proposal_reviews.review_id', ondelete='CASCADE'), primary_key=True, index=True)
+ scientificCategory = Column(String(255), nullable=False)
+ pctRecommendTime = Column(Float(asdecimal=True))
+ score = Column(Float(asdecimal=True), server_default=text("'0'"))
+ comments = Column(String)
+ normalizedScore = Column(Float(asdecimal=True), nullable=False, server_default=text("'0'"))
+
+
+class TechnicalReview(ProposalReview):
+ __tablename__ = 'technical_reviews'
+
+ review_id = Column(ForeignKey('proposal_reviews.review_id', ondelete='CASCADE'), primary_key=True, index=True)
+ commentsForAuthors = Column(String)
+ commentsForTAC = Column(String)
+
+
+class ProposalTacReview(Base):
+ __tablename__ = 'proposal_tac_reviews'
+
+ tac_review_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ review_cycle_id = Column(ForeignKey('review_cycles.review_cycle_id'), nullable=False, index=True)
+ proposal_id = Column(ForeignKey('proposal.proposal_id'), nullable=False, index=True)
+ totalTimeRequested = Column(Float(asdecimal=True), nullable=False)
+ totalTimeAllocated = Column(Float(asdecimal=True), nullable=False)
+ schedulingPriority = Column(String(255), nullable=False)
+ complete = Column(BIT(1), nullable=False)
+ finalized = Column(BIT(1), nullable=False)
+ normalizedSRPScore = Column(Float(asdecimal=True), nullable=False, server_default=text("'0'"))
+ reviewPostponed = Column(BIT(1), nullable=False)
+
+ proposal = relationship('Proposal')
+ review_cycle = relationship('ReviewCycle')
+
+
+# class RefereeComment(Base):
+# __tablename__ = 'refereeComments'
+#
+# refereeComments_id = Column(BigInteger, primary_key=True)
+# objectversion = Column(Integer, nullable=False)
+# person_id = Column(ForeignKey('person.person_id'), index=True)
+# comments = Column(Text)
+# pctRecommendTime = Column(String(255))
+# rating = Column(String(255))
+# commentsToPSC = Column(Text)
+# complete = Column(BIT(1), nullable=False)
+#
+# person = relationship('Person')
+#
+#
+# class RefereeComment(Base):
+# __tablename__ = 'referee_comment'
+#
+# proposal_id = Column(ForeignKey('proposal.proposal_id'), primary_key=True, nullable=False, index=True)
+# commentid = Column(ForeignKey('refereeComments.refereeComments_id'), nullable=False, index=True)
+# userid = Column(ForeignKey('userAuthentication.userAuthentication_id'), primary_key=True, nullable=False, index=True)
+#
+# parent = relationship('RefereeComment', remote_side=[proposal_id, userid])
+# proposal = relationship('Proposal')
+# userAuthentication = relationship('UserAuthentication')
+
+
+class Referee(Base):
+ __tablename__ = 'referees'
+
+ referee_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ person_id = Column(ForeignKey('person.person_id'), nullable=False, index=True)
+ user_id = Column(ForeignKey('userAuthentication.userAuthentication_id'), nullable=False, index=True)
+ active = Column(BIT(1), nullable=False)
+ acceptReviews = Column(BIT(1), nullable=False)
+ type = Column(Integer, nullable=False)
+
+ person = relationship('Person')
+ user = relationship('UserAuthentication')
+
+
+t_resource_bck = Table(
+ 'resource_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False),
+ Column('class', String(255), nullable=False)
+)
+
+
+class ReviewCategory(Base):
+ __tablename__ = 'reviewCategory'
+
+ reviewCategory_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ telescope = Column(String(255), nullable=False)
+ code = Column(String(128), nullable=False, index=True)
+ description = Column(Text)
+
+
+class ReviewCycle(Base):
+ __tablename__ = 'review_cycles'
+
+ review_cycle_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ title = Column(String(255), nullable=False)
+ proposal_deadline = Column(Date, nullable=False)
+ review_deadline = Column(Date, nullable=False)
+ status = Column(String(255), nullable=False)
+ code = Column(String(3), nullable=False, server_default=text("'xxx'"))
+
+
+class Role(Base):
+ __tablename__ = 'role'
+
+ role_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ name = Column(String(128), nullable=False, index=True)
+ description = Column(Text)
+ permissionset_id = Column(ForeignKey('permissionset.permissionset_id'), nullable=False, index=True)
+
+ permissionset = relationship('Permissionset')
+
+
+class ScientificCategory(Base):
+ __tablename__ = 'scientificCategory'
+
+ scientificCategory_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ proposal_id = Column(ForeignKey('proposal.proposal_id'), index=True)
+ display_position = Column(Integer)
+ scientificCategory = Column(String(255), nullable=False)
+
+ proposal = relationship('Proposal')
+
+
+class ScientificReviewPanel(Base):
+ __tablename__ = 'scientific_review_panel'
+
+ sr_panel_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ review_cycle_id = Column(ForeignKey('review_cycles.review_cycle_id'), nullable=False, index=True)
+ referee_id = Column(ForeignKey('referees.referee_id'), nullable=False, index=True)
+ chair = Column(BIT(1), nullable=False)
+ scientificCategory = Column(String(255), nullable=False)
+
+ referee = relationship('Referee')
+ review_cycle = relationship('ReviewCycle')
+
+
+t_scientific_reviews_backup = Table(
+ 'scientific_reviews_backup', metadata,
+ Column('review_id', BigInteger, nullable=False),
+ Column('scientificCategory', String(255), nullable=False),
+ Column('pctRecommendTime', Float(asdecimal=True)),
+ Column('score', Float(asdecimal=True), server_default=text("'0'")),
+ Column('comments', String),
+ Column('normalizedScore', Float(asdecimal=True), nullable=False, server_default=text("'0'"))
+)
+
+
+t_scientific_type = Table(
+ 'scientific_type', metadata,
+ Column('code', String(16), index=True),
+ Column('name', String(50), index=True),
+ Column('abbreviation', String(3)),
+ Column('description', String(255)),
+ Column('legacy', Integer)
+)
+
+
+class SearchPreference(Base):
+ __tablename__ = 'searchPreferences'
+
+ preference_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ username = Column(String(128), nullable=False)
+ pagename = Column(Text, nullable=False)
+ listname = Column(String(255))
+ value = Column(String(255))
+
+
+class Session(Base):
+ __tablename__ = 'session'
+
+ session_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ SESSION_NAME = Column(Text)
+ SESSION_TIME = Column(Float(asdecimal=True))
+ REPEATS = Column(Integer)
+ SEPARATION = Column(String(255))
+ INTERVAL_TIME = Column(Integer)
+ CONSTRAINT_FIELD = Column(Text)
+ COMMENTS = Column(Text)
+ MINIMUM_LST = Column(Text)
+ MAXIMUM_LST = Column(Text)
+ ELEVATION_MINIMUM = Column(Text)
+ SESSION_TIME_CALCULATED = Column(BIT(1), nullable=False)
+ DISPLAY_POSITION = Column(Integer, nullable=False)
+ PROPOSAL_ID = Column(ForeignKey('proposal.proposal_id'), nullable=False, index=True)
+ MODIFIED_SESSION = Column(BIT(1), nullable=False)
+
+ proposal = relationship('Proposal')
+
+
+class ModifiedSession(Session):
+ __tablename__ = 'ModifiedSession'
+
+ session_id = Column(ForeignKey('session.session_id'), primary_key=True, index=True)
+ CREATED_DATE = Column(DateTime, nullable=False)
+ REQUESTED_SESSION_TIME = Column(Float(asdecimal=True))
+ REQUESTED_REPEATS = Column(Integer)
+ schedulingPriority = Column(String(255), server_default=text("''"))
+ ALLOCATED_SESSION_TIME = Column(Float(asdecimal=True), server_default=text("'0'"))
+ ALLOCATED_REPEATS = Column(Integer, server_default=text("'0'"))
+ START_LST = Column(String)
+ STOP_LST = Column(String)
+
+
+class SessionPair(Base):
+ __tablename__ = 'sessionPair'
+
+ sessionPair_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ PAIR_IDENTIFIER = Column(String(255))
+ FIGURE_OF_MERIT = Column(Text)
+ FIGURE_UNITS = Column(String(255))
+ PAIR_TIME = Column(Float(asdecimal=True))
+ PAIR_TIME_UNIT = Column(String(255))
+ CONSTRAINT_FIELD = Column(Text)
+ DISPLAY_POSITION = Column(Integer, nullable=False)
+ RESOURCE_GROUP = Column(String(255))
+ SOURCE_GROUP = Column(String(255))
+ SESSION_ID = Column(ForeignKey('session.session_id', ondelete='CASCADE'), nullable=False, index=True)
+ SUB_ARRAY = Column(Integer, nullable=False)
+ SUB_ARRAY_COUNT = Column(Integer, nullable=False)
+ SUB_ARRAY_GROUP = Column(String(255), nullable=False)
+
+ session = relationship('Session')
+
+
+t_sessionPair_bck = Table(
+ 'sessionPair_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class SessionResource(Base):
+ __tablename__ = 'sessionResource'
+
+ sessionResource_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ sessionPair_id = Column(ForeignKey('sessionPair.sessionPair_id', ondelete='CASCADE'), nullable=False, index=True)
+ resource_id = Column(ForeignKey('RESOURCES.resource_id', ondelete='CASCADE'), nullable=False, index=True)
+ display_position = Column(Integer, nullable=False)
+ SESSION_ID = Column(ForeignKey('session.session_id', ondelete='CASCADE'), nullable=False, index=True)
+
+ session = relationship('Session')
+ resource = relationship('RESOURCE')
+ sessionPair = relationship('SessionPair')
+
+
+t_sessionResource_bck = Table(
+ 'sessionResource_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class SessionSource(Base):
+ __tablename__ = 'sessionSource'
+
+ sessionSource_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ sessionPair_id = Column(ForeignKey('sessionPair.sessionPair_id', ondelete='CASCADE'), nullable=False, index=True)
+ source_id = Column(ForeignKey('source.source_id', ondelete='CASCADE'), nullable=False, index=True)
+ display_position = Column(Integer, nullable=False)
+ SESSION_ID = Column(ForeignKey('session.session_id', ondelete='CASCADE'), nullable=False, index=True)
+
+ session = relationship('Session')
+ sessionPair = relationship('SessionPair')
+ source = relationship('Source')
+
+
+t_sessionSource_bck = Table(
+ 'sessionSource_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_session_bck = Table(
+ 'session_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class Source(Base):
+ __tablename__ = 'source'
+
+ source_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ proposal_id = Column(ForeignKey('proposal.proposal_id'), nullable=False, index=True)
+ display_position = Column(Float(asdecimal=True), nullable=False, index=True)
+ source_group = Column(Text)
+ target_name = Column(Text)
+ coordinate = Column(String(255))
+ right_ascension = Column(Text)
+ declination = Column(Text)
+ right_ascension_range = Column(Text)
+ declination_range = Column(Text)
+ velocity_type = Column(String(255), nullable=False)
+ velocity_redshift = Column(Text, nullable=False)
+ convention = Column(String(255), nullable=False, server_default=text("'Radio'"))
+ coordinate_system = Column(String(255), nullable=False, server_default=text("'Equatorial'"))
+ referenceFrame = Column(String(255), nullable=False, server_default=text("'LSRK'"))
+ calibrator = Column(BIT(1), nullable=False)
+
+ proposal = relationship('Proposal')
+
+
+t_source_bck = Table(
+ 'source_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class SrpComment(Base):
+ __tablename__ = 'srp_comments'
+
+ srp_comments_id = Column(BigInteger, primary_key=True)
+ person_id = Column(ForeignKey('person.person_id'), nullable=False, index=True)
+ referee_id = Column(ForeignKey('referees.referee_id'), index=True)
+ commentsForAuthors = Column(String)
+ commentsForTAC = Column(String)
+ byPSTAdmin = Column(BIT(1), nullable=False)
+ lastUpdated = Column(DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP"))
+
+ person = relationship('Person')
+ referee = relationship('Referee')
+
+
+class SrpCommentsMap(Base):
+ __tablename__ = 'srp_comments_map'
+
+ normalized_scores_id = Column(ForeignKey('proposal_normalized_scores.normalized_scores_id'), primary_key=True, nullable=False, index=True)
+ srp_comments_id = Column(ForeignKey('srp_comments.srp_comments_id'), nullable=False, index=True)
+ person_id = Column(ForeignKey('person.person_id'), primary_key=True, nullable=False, index=True)
+
+ normalized_scores = relationship('ProposalNormalizedScore')
+ person = relationship('Person')
+ srp_comments = relationship('SrpComment')
+
+
+t_srp_comments_tmp = Table(
+ 'srp_comments_tmp', metadata,
+ Column('srp_comments_id', BigInteger, nullable=False, server_default=text("'0'")),
+ Column('person_id', BigInteger, nullable=False),
+ Column('referee_id', BigInteger),
+ Column('commentsForAuthors', String),
+ Column('commentsForTAC', String),
+ Column('byPSTAdmin', BIT(1), nullable=False),
+ Column('lastUpdated', DateTime)
+)
+
+
+class StudentSupport(Base):
+ __tablename__ = 'studentSupport'
+
+ studentSupport_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ travel = Column(Text)
+ hardware = Column(Text)
+ miscellaneous_budget = Column(Float(asdecimal=True))
+ applied = Column(BIT(1), nullable=False)
+ propId = Column(String(255), nullable=False)
+ proposal_id = Column(ForeignKey('proposal.proposal_id', ondelete='CASCADE'), unique=True)
+ sos_pi_id = Column(ForeignKey('author.author_id'), index=True)
+
+ proposal = relationship('Proposal')
+ sos_pi = relationship('Author')
+
+
+t_studentSupport_bck = Table(
+ 'studentSupport_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class TacComment(Base):
+ __tablename__ = 'tac_comments'
+
+ tac_comments_id = Column(BigInteger, primary_key=True)
+ person_id = Column(ForeignKey('person.person_id'), nullable=False, index=True)
+ referee_id = Column(ForeignKey('referees.referee_id'), index=True)
+ commentsForAuthors = Column(String)
+ commentsForTAC = Column(String)
+ byPSTAdmin = Column(BIT(1), nullable=False)
+
+ person = relationship('Person')
+ referee = relationship('Referee')
+
+
+class TacCommentsMap(Base):
+ __tablename__ = 'tac_comments_map'
+
+ tac_review_id = Column(ForeignKey('proposal_tac_reviews.tac_review_id'), primary_key=True, nullable=False, index=True)
+ tac_comments_id = Column(ForeignKey('tac_comments.tac_comments_id'), nullable=False, index=True)
+ person_id = Column(ForeignKey('person.person_id'), primary_key=True, nullable=False, index=True)
+
+ person = relationship('Person')
+ tac_comments = relationship('TacComment')
+ tac_review = relationship('ProposalTacReview')
+
+
+class TacPanel(Base):
+ __tablename__ = 'tac_panel'
+
+ tac_panel_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ review_cycle_id = Column(ForeignKey('review_cycles.review_cycle_id'), nullable=False, index=True)
+ referee_id = Column(ForeignKey('referees.referee_id'), nullable=False, index=True)
+ chair = Column(BIT(1), nullable=False)
+
+ referee = relationship('Referee')
+ review_cycle = relationship('ReviewCycle')
+
+
+class TechnicalReviewCategory(Base):
+ __tablename__ = 'technical_review_category'
+
+ tech_reviewCategory_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ name = Column(String, nullable=False)
+ code = Column(String(128), nullable=False, index=True)
+ description = Column(String)
+ telescope = Column(String(255), nullable=False)
+
+
+class TechnicalReviewPanel(Base):
+ __tablename__ = 'technical_review_panel'
+
+ tr_panel_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ review_cycle_id = Column(ForeignKey('review_cycles.review_cycle_id'), nullable=False, index=True)
+ referee_id = Column(ForeignKey('referees.referee_id'), nullable=False, index=True)
+ technical_category_id = Column(ForeignKey('technical_review_category.tech_reviewCategory_id'), nullable=False, index=True)
+ telescope = Column(String(255), nullable=False)
+
+ referee = relationship('Referee')
+ review_cycle = relationship('ReviewCycle')
+ technical_category = relationship('TechnicalReviewCategory')
+
+
+class Uisetting(Base):
+ __tablename__ = 'uisetting'
+
+ uisetting_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ FLYOVER_HELP = Column(BIT(1), nullable=False)
+ TELESCOPE = Column(String(255))
+ EXPIRATION_RECOVERY = Column(String(255))
+ EDITOR_ROLE = Column(String(255))
+ PROPOSALS_PER_PAGE = Column(Integer)
+ CLIENT_SIDE_CHANGE_INDICATOR = Column(String(255))
+
+
+t_uisettings_bck = Table(
+ 'uisettings_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+t_url_bck = Table(
+ 'url_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
+
+
+class UserAuthentication(Base):
+ __tablename__ = 'userAuthentication'
+
+ userAuthentication_id = Column(BigInteger, primary_key=True)
+ objectversion = Column(Integer, nullable=False)
+ role_id = Column(ForeignKey('role.role_id'), nullable=False, index=True)
+ permissionset_id = Column(ForeignKey('permissionset.permissionset_id'), nullable=False, index=True)
+ uisetting_id = Column(ForeignKey('uisetting.uisetting_id'), index=True)
+ personName = Column(String(128), nullable=False, unique=True)
+ password = Column(String(255), nullable=False)
+ displayName = Column(String(255), nullable=False)
+ authenticationToken = Column(String(255))
+ changePassword = Column(BIT(1), nullable=False)
+ rememberLogin = Column(BIT(1))
+ defaultAccount_id = Column(ForeignKey('userAuthentication.userAuthentication_id'), index=True)
+
+ defaultAccount = relationship('UserAuthentication', remote_side=[userAuthentication_id])
+ permissionset = relationship('Permissionset')
+ role = relationship('Role')
+ uisetting = relationship('Uisetting')
+
+
+t_userAuthentication_bck = Table(
+ 'userAuthentication_bck', metadata,
+ Column('newid', BigInteger, nullable=False),
+ Column('oldid', BigInteger, nullable=False)
+)
diff --git a/apps/cli/executables/pexable/ingest/ingest/schema/vlassmodel.py b/apps/cli/executables/pexable/ingest/ingest/schema/vlassmodel.py
new file mode 100644
index 000000000..4b29b8085
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/schema/vlassmodel.py
@@ -0,0 +1,482 @@
+# coding: utf-8
+from sqlalchemy import BigInteger, Boolean, Column, DateTime, Float, ForeignKey, Integer, SmallInteger, String, Table, Text, text
+from sqlalchemy.orm import relationship
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+t_calibration_product_minitile = Table(
+ 'calibration_product_minitile', metadata,
+ Column('calibration_product_id', ForeignKey('calibration_product.calibration_product_id'), primary_key=True, nullable=False),
+ Column('minitile_id', ForeignKey('minitile.id'), primary_key=True, nullable=False)
+)
+
+
+class Calibrator(Base):
+ __tablename__ = 'calibrator'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ sct_id = Column(BigInteger)
+ sct_catalog_id = Column(BigInteger)
+ type = Column(Integer)
+
+ schedblocks = relationship('Schedblock', secondary='fluxcal_schedblock')
+ minitiles = relationship('Minitile', secondary='minitile_calibrator')
+ schedblocks1 = relationship('Schedblock', secondary='phasecal_schedblock')
+ schedblocks2 = relationship('Schedblock', secondary='polcal_schedblock')
+
+
+class CalibratorFlux(Base):
+ __tablename__ = 'calibrator_flux'
+
+ id = Column(Integer, primary_key=True)
+ frequency = Column(Float(53))
+ obs_date = Column(DateTime)
+ flux = Column(Float(53))
+ sp_idx = Column(Float(53))
+ calibrator_id = Column(ForeignKey('calibrator.id'))
+ uv_min = Column(Float(53))
+ uv_max = Column(Float(53))
+
+ calibrator = relationship('Calibrator')
+
+
+class CalibratorPol(Base):
+ __tablename__ = 'calibrator_pol'
+
+ id = Column(Integer, primary_key=True)
+ fraction = Column(Float(53))
+ pos_angle = Column(Float(53))
+ rm = Column(Float(53))
+ ref_freq = Column(Float(53))
+ calibrator_id = Column(ForeignKey('calibrator.id'))
+
+ calibrator = relationship('Calibrator')
+
+
+class ConfigurationFile(Base):
+ __tablename__ = 'configuration_file'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ content_type = Column(String(255))
+ contents = Column(Text)
+ jobspec_id = Column(ForeignKey('jobspec.id'), index=True)
+
+ jobspec = relationship('Jobspec')
+
+
+class ConfigurationFileTemplate(Base):
+ __tablename__ = 'configuration_file_template'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ content_type = Column(String(255))
+ contents = Column(Text)
+ doc = Column(Text)
+ workflow_id = Column(ForeignKey('workflow.id'))
+
+ workflow = relationship('Workflow')
+
+
+t_databasechangelog = Table(
+ 'databasechangelog', metadata,
+ Column('id', String(255), nullable=False),
+ Column('author', String(255), nullable=False),
+ Column('filename', String(255), nullable=False),
+ Column('dateexecuted', DateTime, nullable=False),
+ Column('orderexecuted', Integer, nullable=False),
+ Column('exectype', String(10), nullable=False),
+ Column('md5sum', String(35)),
+ Column('description', String(255)),
+ Column('comments', String(255)),
+ Column('tag', String(255)),
+ Column('liquibase', String(20)),
+ Column('contexts', String(255)),
+ Column('labels', String(255)),
+ Column('deployment_id', String(10))
+)
+
+
+class Databasechangeloglock(Base):
+ __tablename__ = 'databasechangeloglock'
+
+ id = Column(Integer, primary_key=True)
+ locked = Column(Boolean, nullable=False)
+ lockgranted = Column(DateTime)
+ lockedby = Column(String(255))
+
+
+t_fluxcal_schedblock = Table(
+ 'fluxcal_schedblock', metadata,
+ Column('schedblock_id', ForeignKey('schedblock.schedblock_id'), primary_key=True, nullable=False),
+ Column('calibrator_id', ForeignKey('calibrator.id'), primary_key=True, nullable=False)
+)
+
+
+
+
+class InputProductVersion(Base):
+ __tablename__ = 'input_product_versions'
+
+ id = Column(Integer, primary_key=True)
+ product_version_id = Column(ForeignKey('product_version.id'))
+ jobspec_id = Column(ForeignKey('jobspec.id'))
+
+ jobspec = relationship('Jobspec')
+ product_version = relationship('ProductVersion')
+ product_versions = relationship('ProductVersion', secondary='inputs_to_product_versions')
+
+
+t_inputs_to_product_versions = Table(
+ 'inputs_to_product_versions', metadata,
+ Column('intputs_id', ForeignKey('input_product_versions.id'), primary_key=True, nullable=False),
+ Column('product_version_id', ForeignKey('product_version.id'), primary_key=True, nullable=False, index=True)
+)
+
+
+class Job(Base):
+ __tablename__ = 'job'
+
+ id = Column(Integer, primary_key=True)
+ start_timestamp = Column(DateTime)
+ end_timestamp = Column(DateTime)
+ status = Column(Integer)
+ json_results = Column(Text)
+ qa_analyst = Column(String(255))
+ notes = Column(Text)
+ jobspec_id = Column(ForeignKey('jobspec.id'), nullable=False, index=True)
+ name = Column(String(255))
+ archival_job_id = Column(ForeignKey('job.id'), index=True)
+ arch_status = Column(Integer)
+
+ archival_job = relationship('Job', remote_side=[id])
+ jobspec = relationship('Jobspec')
+
+
+class JobTaskTemplate(Base):
+ __tablename__ = 'job_task_template'
+
+ id = Column(Integer, primary_key=True)
+ order_num = Column(SmallInteger)
+ name = Column(String(255))
+ command_template = Column(String(255))
+ doc = Column(Text)
+ workflow_id = Column(ForeignKey('workflow.id'))
+
+ workflow = relationship('Workflow')
+
+
+class Jobspec(Base):
+ __tablename__ = 'jobspec'
+
+ id = Column(Integer, primary_key=True)
+ json = Column(Text)
+ creation_date = Column(DateTime)
+ next_id = Column(ForeignKey('jobspec.id'), index=True)
+ queue_id = Column(ForeignKey('queue.id'), index=True)
+ name = Column(String(255))
+ enqueued = Column(Boolean)
+ sdm_id = Column(String(255))
+
+ next = relationship('Jobspec', remote_side=[id])
+ queue = relationship('Queue', primaryjoin='Jobspec.queue_id == Queue.id')
+ product_versions = relationship('ProductVersion', secondary='jobspec_to_product_version')
+
+
+t_jobspec_to_product_version = Table(
+ 'jobspec_to_product_version', metadata,
+ Column('product_version_id', ForeignKey('product_version.id'), primary_key=True, nullable=False, index=True),
+ Column('jobspec_id', ForeignKey('jobspec.id'), primary_key=True, nullable=False)
+)
+
+
+class Jobtask(Base):
+ __tablename__ = 'jobtask'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ start_timestamp = Column(DateTime)
+ end_timestamp = Column(DateTime)
+ status = Column(Integer)
+ json_results = Column(Text)
+ is_final = Column(Boolean)
+ job_id = Column(ForeignKey('job.id'), index=True)
+ pbs_job_id = Column(String(255))
+
+ job = relationship('Job')
+
+
+class JsonConfiguration(Base):
+ __tablename__ = 'json_configuration'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ json = Column(Text)
+ product_type_id = Column(ForeignKey('product_type.id'))
+ product_version_id = Column(ForeignKey('product_version.id'))
+ product_id = Column(ForeignKey('product.id'))
+
+ product = relationship('Product')
+ product_type = relationship('ProductType')
+ product_version = relationship('ProductVersion')
+
+
+class Minitile(Base):
+ __tablename__ = 'minitile'
+
+ id = Column(Integer, primary_key=True, server_default=text("nextval('minitile_id_seq'::regclass)"))
+ name = Column(String(255))
+ epoch = Column(Integer)
+ tier = Column(Integer)
+ ra_min = Column(Float(53))
+ dec_min = Column(Float(53))
+ ra_max = Column(Float(53))
+ dec_max = Column(Float(53))
+ area = Column(Float(53))
+ factor = Column(Float(53))
+ first_epoch_half = Column(Boolean)
+ custom_1 = Column(String(255))
+ custom_2 = Column(String(255))
+ custom_3 = Column(String(255))
+ custom_4 = Column(String(255))
+ custom_5 = Column(String(255))
+ definitions = Column(Text)
+
+ schedblocks = relationship('Schedblock', secondary='schedblock_minitile')
+
+
+t_minitile_calibrator = Table(
+ 'minitile_calibrator', metadata,
+ Column('minitile_id', ForeignKey('minitile.id'), primary_key=True, nullable=False),
+ Column('calibrator_id', ForeignKey('calibrator.id'), primary_key=True, nullable=False)
+)
+
+
+t_phasecal_schedblock = Table(
+ 'phasecal_schedblock', metadata,
+ Column('schedblock_id', ForeignKey('schedblock.schedblock_id'), primary_key=True, nullable=False),
+ Column('calibrator_id', ForeignKey('calibrator.id'), primary_key=True, nullable=False)
+)
+
+
+t_polcal_schedblock = Table(
+ 'polcal_schedblock', metadata,
+ Column('schedblock_id', ForeignKey('schedblock.schedblock_id'), primary_key=True, nullable=False),
+ Column('calibrator_id', ForeignKey('calibrator.id'), primary_key=True, nullable=False)
+)
+
+
+class Prerequisite(Base):
+ __tablename__ = 'prerequisite'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ ready = Column(Boolean)
+ product_id = Column(ForeignKey('product.id'), nullable=False, index=True)
+ req_product_id = Column(ForeignKey('product.id'))
+
+ product = relationship('Product', primaryjoin='Prerequisite.product_id == Product.id')
+ req_product = relationship('Product', primaryjoin='Prerequisite.req_product_id == Product.id')
+
+
+class PrerequisiteArchiveid(Base):
+ __tablename__ = 'prerequisite_archiveids'
+
+ prerequisite_id = Column(ForeignKey('prerequisite.id'), primary_key=True, nullable=False)
+ archive_id = Column(String(255), nullable=False)
+ name = Column(String(255), primary_key=True, nullable=False)
+
+ prerequisite = relationship('Prerequisite')
+
+
+class Product(Base):
+ __tablename__ = 'product'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ category = Column(Integer)
+ epoch = Column(Integer)
+ notes = Column(Text)
+ product_type_id = Column(ForeignKey('product_type.id'), nullable=False)
+ completed = Column(Boolean)
+ status = Column(Integer, nullable=False, server_default=text("0"))
+
+ product_type = relationship('ProductType')
+ tags = relationship('Tag', secondary='product_tag')
+
+
+class CalibrationProduct(Product):
+ __tablename__ = 'calibration_product'
+
+ calibration_product_id = Column(ForeignKey('product.id'), primary_key=True)
+
+ minitiles = relationship('Minitile', secondary='calibration_product_minitile')
+
+
+#
+# NOTE:
+# slqacodegen created a much more complete model of the database, but that
+# model included somewhat-circular references among the foreign keys for
+# image_product and it's sub-tables. This configuration was causing
+# issues using the model within SqlAlchemy for metadata lookups in the
+# service layer (ingestion_metadata.py).
+#
+# To postpone this particular part of refactoring the VLASS system, I have
+# deleted image_product and all associated tables, as they weren't required
+# for the current use-case.
+#
+# - JLS 08/29/2019
+#
+
+
+class Schedblock(Product):
+ __tablename__ = 'schedblock'
+
+ schedblock_id = Column(ForeignKey('product.id'), primary_key=True)
+ name = Column(String(255))
+ opt_id = Column(BigInteger)
+ flux_position = Column(Integer)
+ lst_start = Column(Float)
+ lst_end = Column(Float)
+ submitted = Column(Boolean)
+
+
+t_product_tag = Table(
+ 'product_tag', metadata,
+ Column('product_id', ForeignKey('product.id'), primary_key=True, nullable=False),
+ Column('tag_id', ForeignKey('tag.id'), primary_key=True, nullable=False)
+)
+
+
+class ProductType(Base):
+ __tablename__ = 'product_type'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ event_type = Column(String(255))
+
+
+class ProductVersion(Base):
+ __tablename__ = 'product_version'
+
+ id = Column(Integer, primary_key=True)
+ number = Column(Integer)
+ notes = Column(Text)
+ product_id = Column(ForeignKey('product.id'))
+ status = Column(Integer)
+
+ product = relationship('Product')
+
+
+class ProductVersionArchiveid(Base):
+ __tablename__ = 'product_version_archiveids'
+
+ product_version_id = Column(ForeignKey('product_version.id'), primary_key=True, nullable=False)
+ archive_id = Column(String(255), nullable=False)
+ name = Column(String(255), primary_key=True, nullable=False)
+
+ product_version = relationship('ProductVersion')
+
+
+class Queue(Base):
+ __tablename__ = 'queue'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ workflow_id = Column(ForeignKey('workflow.id'))
+ head_jobspec_id = Column(ForeignKey('jobspec.id'))
+ tail_jobspec_id = Column(ForeignKey('jobspec.id'))
+ submit = Column(Boolean)
+ project_code = Column(String(255))
+ create_job = Column(Boolean)
+ auto_qa = Column(Boolean, server_default=text("false"))
+
+ head_jobspec = relationship('Jobspec', primaryjoin='Queue.head_jobspec_id == Jobspec.id')
+ tail_jobspec = relationship('Jobspec', primaryjoin='Queue.tail_jobspec_id == Jobspec.id')
+ workflow = relationship('Workflow')
+
+
+t_rawdata = Table(
+ 'rawdata', metadata,
+ Column('opt_id', BigInteger),
+ Column('sdm_id', String(255)),
+ Column('schedblock_id', ForeignKey('schedblock.schedblock_id')),
+ Column('rawdata_id', ForeignKey('product.id'), nullable=False)
+)
+
+
+class Scan(Base):
+ __tablename__ = 'scan'
+
+ id = Column(Integer, primary_key=True)
+ start_source_id = Column(ForeignKey('source.id'))
+ end_source_id = Column(ForeignKey('source.id'))
+ num_phase_centers = Column(Integer)
+ minitile_id = Column(ForeignKey('minitile.id'))
+
+ end_source = relationship('Source', primaryjoin='Scan.end_source_id == Source.id')
+ minitile = relationship('Minitile')
+ start_source = relationship('Source', primaryjoin='Scan.start_source_id == Source.id')
+
+
+t_schedblock_minitile = Table(
+ 'schedblock_minitile', metadata,
+ Column('schedblock_id', ForeignKey('schedblock.schedblock_id'), primary_key=True, nullable=False),
+ Column('minitile_id', ForeignKey('minitile.id'), primary_key=True, nullable=False)
+)
+
+
+class Setting(Base):
+ __tablename__ = 'settings'
+
+ id = Column(String(1), primary_key=True, server_default=text("'X'::bpchar"))
+ allow_job_deletion = Column(Boolean, server_default=text("false"))
+ allow_product_deletion = Column(Boolean, server_default=text("false"))
+
+
+class Source(Base):
+ __tablename__ = 'source'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ sct_catalog_id = Column(BigInteger)
+ sct_id = Column(BigInteger)
+ ra = Column(Float(53))
+ decl = Column(Float(53))
+
+
+class Tag(Base):
+ __tablename__ = 'tag'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+
+
+class Workflow(Base):
+ __tablename__ = 'workflow'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(255))
+ host = Column(String(255))
+ port = Column(Integer)
+ json = Column(Text)
+ enqueue = Column(Boolean)
+ product_type_id = Column(ForeignKey('product_type.id'))
+ arch_workflow_id = Column(ForeignKey('workflow.id'))
+
+ arch_workflow = relationship('Workflow', remote_side=[id])
+ product_type = relationship('ProductType')
+
+
+class WorkflowProjectCode(Base):
+ __tablename__ = 'workflow_project_codes'
+
+ project_code_id = Column(ForeignKey('workflow.id'), primary_key=True, nullable=False)
+ project_code = Column(String(255), primary_key=True, nullable=False)
+
+ project_code1 = relationship('Workflow')
diff --git a/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/__init__.py b/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/thumbnail_finder.py b/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/thumbnail_finder.py
new file mode 100644
index 000000000..486936b36
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/thumbnail_finder.py
@@ -0,0 +1,130 @@
+import os
+import sys
+import logging
+import logging.handlers
+import argparse as ap
+from .. import version
+from ..pymygdala import LogHandler
+from .weblogUtil import Weblog
+
+# Global logger access:
+logger = logging.getLogger("thumbnail_cheater")
+logger.setLevel(logging.WARN)
+
+#
+# using the old quicklook weblog raiding, expand it to do AUDI continuum & cubes too.
+#
+# The thumbnails don't exactly match their filename, so we might need to modify the name somewhat
+# (undo the +/- -> underscore transformation, remove the .sky portion) to mesh with the manifest
+# building utility
+#
+#
+#
+
+_DESCRIPTION = """Cheating Thumbnail Utility, version {}:\n\tFinds, unpacks, and searches a weblog for thumbnail \n\t
+images. Searchings in stage7 (quicklook mode) or stages 10 and 12 (audi mode) \n\t
+for images approximating that of the provided file."""
+
+
+def _make_parser():
+ r""" Build a command line and/or argument parser """
+
+ result = ap.ArgumentParser(description=_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. test, production')
+ result.add_argument('-v', '--verbose', action='store_true',
+ help='Provide more log feedback.')
+ result.add_argument('-q', '--quicklook', action='store_true',
+ help='Search using quicklook assumptions (stage7)')
+ result.add_argument('-a', '--audi', action='store_true',
+ help='Search using audi assumptions (stages 10 and 12)')
+ result.add_argument('filename', type=str, action='store',
+ help='name of the science product of interest')
+ return result
+
+
+def weblog_raid():
+ parser = _make_parser()
+ arguments = parser.parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == arguments.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != arguments.profile:
+ os.environ['CAPO_PROFILE'] = arguments.profile
+
+ # integrate the logs with pymygdala's logging system
+ handler = LogHandler(profile=os.environ['CAPO_PROFILE'], application='thumbnail_cheater')
+
+ if arguments.verbose:
+ handler.setLevel(logging.DEBUG)
+ logger.setLevel(logging.DEBUG)
+ logger.debug("Enabling more complete feedback.")
+ else:
+ handler.setLevel(logging.WARN)
+
+ logger.addHandler(handler)
+
+ # A last check: pick a mode
+ if (not arguments.audi) and (not arguments.quicklook):
+ logger.error("Only two choices: audi or quicklook. Pick one.")
+ exit(-5)
+
+ # Get the contents of our working directory (ingestion's staging area)
+ contents = os.listdir(path=os.getcwd())
+ logger.debug(f"Searching for a weblog within: {contents}")
+
+ # and find the weblog file among the list (stolen from persistimages): provides the index in contents
+ weblog_candidates = [place for place, name in enumerate(contents) if 'weblog' in name]
+
+ # We expect a single file to match the pattern:
+ if len(weblog_candidates) != 1:
+ # So throw an exception when that's not the case.
+ logger.error("Cannot isolate the weblog, exiting.")
+ exit(-3)
+
+ weblog_path = os.getcwd()
+ weblog_filename = contents[weblog_candidates[0]]
+ weblog = Weblog(weblog_path, weblog_filename)
+
+ logger.debug(f"hunting in the weblog ({weblog_filename}) for a companion to: {arguments.filename}")
+
+ if arguments.quicklook:
+ thumbnail = weblog.find_quicklook_thumbnail_for(arguments.filename)
+ else:
+ if 'cube' in arguments.filename:
+ thumbnail = weblog.find_cube_thumbnail_for(arguments.filename)
+ else:
+ thumbnail = weblog.find_continuum_thumbnail_for(arguments.filename)
+
+ if thumbnail is None:
+ if 'tt1' in arguments.filename:
+ logger.debug(f"Didn't find a thumbnail for {arguments.filename}, as expected")
+ exit(0)
+ else:
+ logger.error(f"Failed to find expected thumbnail for {arguments.filename}")
+ exit(-13)
+
+ logger.debug(f"Discovered {thumbnail} for {arguments.filename}.")
+
+ #
+ # So, now we have the thumbnail, but we need a variant on it's name copied out into
+ # the working directory. (dump the s10_0 sky iter1.image etc).
+ #
+ # the thumbnail matching algorithm will need to handle the +/- -> _ that's less easy
+ # from this direction.
+ #
+ desired_thumbnail = os.path.basename(thumbnail)
+ desired_thumbnail = desired_thumbnail.replace('.s12_0', '')
+ desired_thumbnail = desired_thumbnail.replace('.s10_0', '')
+ desired_thumbnail = desired_thumbnail.replace('.sky', '')
+ # AUDI images drop the .iter1.image bit, but vlass qls don't.
+ if not arguments.quicklook:
+ desired_thumbnail = desired_thumbnail.replace('.iter1.image', '')
+
+ logger.debug(f"Copying the thumbnail to {desired_thumbnail}")
+ os.link(thumbnail, desired_thumbnail)
diff --git a/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/weblogUtil.py b/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/weblogUtil.py
new file mode 100644
index 000000000..d0500c5e5
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/ingest/weblog_thumbs/weblogUtil.py
@@ -0,0 +1,153 @@
+
+import os
+import glob
+import tarfile
+import logging
+
+LOG = logging.getLogger('thumbnail_cheater')
+
+
+class Weblog:
+ """A class to encapsulate the extraction of data from the initial page
+ of a CASA Pipeline weblog. Uses BeautifulSoup for the processing.
+
+ Given the path to a file & it's name: extract & scrape the basic information
+ from the index.html file.
+
+ Initially: required the construction of a full path to the index file file as a string:
+ mylog = new Weblog("/export/home/earth/image_scratch/pipeline-20180222T184022/html/index.html")
+
+ Then use the methods to extract the relevant information from the file.
+ """
+
+ # -------------------------------------------------------------------------
+ # Internal Use Methods
+ # -------------------------------------------------------------------------
+
+ def __init__(self, path_to_file, filename):
+ """Open the weblog's index.html file and perform parse the document."""
+ self.indexFilePath = self.find_index_file(path_to_file, filename)
+ #self.thumbnailImages = self.collect_potential_quicklook_thumbnails(self.indexFilePath)
+
+
+ def find_index_file(self, products_dir, weblog_file_name):
+ weblog_tar_file = tarfile.open(os.path.join(products_dir, weblog_file_name))
+ weblog_tar_file.extractall(path=products_dir)
+ weblog_index_html = glob.glob(products_dir + '/pipeline-*/html/index.html')
+ return weblog_index_html[0]
+
+
+ # -------------------------------------------------------------------
+ # Mixed, probably mostly internal.
+ # -------------------------------------------------------------------
+
+
+ def collect_potential_quicklook_thumbnails(self, imageFilePath):
+ """
+ Retrieve the list of thumnbnail images from the appropriate stage for quicklook imaging:
+ :param imageFilePath: path to the weblog
+ :return: list of thumbnail images
+ """
+ return glob.glob(os.path.dirname(imageFilePath) + '/stage7/thumbs/*.sky.png', recursive=True)
+
+ def collect_potential_continuum_thumbnails(self, imageFilePath):
+ """
+ Retrieve the list of thumnbnail images in the weblog stage for continuum imaging:
+ :param imageFilePath: path to the weblog
+ :return: list of thumbnail images
+ """
+ return glob.glob(os.path.dirname(imageFilePath) + '/stage10/thumbs/*.sky.png', recursive=True)
+
+ def collect_potential_cube_thumbnails(self, imageFilePath):
+ """
+ Retrieve the list of thumnbnail images in the weblog's cube imaging stage
+ :param imageFilePath: path to the weblog
+ :return: list of thumbnail images
+ """
+ return glob.glob(os.path.dirname(imageFilePath) + '/stage12/*.sky.png', recursive=True)
+
+ # --------------------------------------------------------------------
+ # External Methods
+ # --------------------------------------------------------------------
+
+
+
+
+ def find_quicklook_thumbnail_for(self, filename):
+ """
+ Get the thumbnail image from the weblog for the specified filename
+ :param filename: FITS image to search for thumbnail
+ :return: thumbnail image path in the weblog
+ """
+
+ thumbnailImages = self.collect_potential_quicklook_thumbnails(self.indexFilePath)
+
+ for thumbnailImage in thumbnailImages:
+ # logger.info('Checking if {} is a thumbnail for {}'.format(os.path.basename(thumbnailImage), filename.replace('-', '_').replace('+', '_').replace('fits', 'sky.png')))
+ if os.path.basename(thumbnailImage) == filename.replace('-', '_').replace('+', '_').replace('fits', 'sky.png'):
+ return thumbnailImage
+
+ LOG.error('No thumbnail found for image: {}'.format(filename))
+ return None
+
+ def find_continuum_thumbnail_for(self, filename):
+ """
+ Get the thumbnail image from the weblog for the specified filename
+ :param filename: FITS image to search for thumbnail
+ :return: thumbnail image path in the weblog
+ """
+
+ thumbnailImages = self.collect_potential_continuum_thumbnails(self.indexFilePath)
+
+ # Three generic replacements:
+ expected_thumb = filename.replace('-', '_').replace('+', '_').replace('fits', 'sky.png')
+
+ # And one based upon the filename (because there are two variants in what comes out of the
+ # continuum imaging step:
+ if 'tt' in expected_thumb:
+ expected_thumb = expected_thumb.replace('I.tt', 'I.iter1.image.tt')
+ else:
+ expected_thumb = expected_thumb.replace('I.pbcor', 'I.iter1.image.pbcor')
+
+ LOG.debug(f"Searching for a file named: {expected_thumb}")
+
+ for thumbnailImage in thumbnailImages:
+
+ currentThumb = os.path.basename(thumbnailImage)
+ # there's an extra portion stuffed in the beginning of the name, it's best handled here:
+ comparisonThumb = currentThumb.replace('.s10_0', '')
+ LOG.debug(f"\tcomparing to: {currentThumb}({comparisonThumb})")
+
+ if comparisonThumb == expected_thumb:
+ return thumbnailImage
+
+ LOG.error('No thumbnail found for image: {}'.format(filename))
+ return None
+
+ def find_cube_thumbnail_for(self, filename):
+ """
+ Get the thumbnail image from the weblog for the specified filename. There are several
+ transformations we need to make in order to match the relevant file.
+
+ :param filename: FITS image to search for thumbnail
+ :return: full path to the (unaltered) thumbnail image for copying.
+ """
+
+ thumbnailImages = self.collect_potential_cube_thumbnails(self.indexFilePath)
+ # The thumbnail images in the weblog don't exactly match the resultant files.
+ # So, let's meet them halfway:
+ expected_thumb = filename.replace('-', '_').replace('+', '_').replace('fits', 'sky.png').replace('I.pbcor', 'I.iter1.image.pbcor')
+ LOG.debug(f"Searching for a file named: {expected_thumb}")
+
+ for thumbnailImage in thumbnailImages:
+ currentThumb = os.path.basename(thumbnailImage)
+ # there's an extra portion stuffed in the beginning of the name, it's best handled here:
+ comparisonThumb = currentThumb.replace('.s12_0', '')
+ LOG.debug(f"\tcomparing to: {currentThumb}({comparisonThumb})")
+
+ if comparisonThumb == expected_thumb:
+ return thumbnailImage
+
+ LOG.error('No thumbnail found for image: {}'.format(filename))
+ return None
+
diff --git a/apps/cli/executables/pexable/ingest/notebooks/find_missing_files.ipynb b/apps/cli/executables/pexable/ingest/notebooks/find_missing_files.ipynb
new file mode 100644
index 000000000..037d01186
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/notebooks/find_missing_files.ipynb
@@ -0,0 +1,221 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Figuring out Joins with pandas and sqlalchemy\n",
+ "\n",
+ "A significant performance issue we have is that, while there are ~8000 files in NGAS we need to check, checking them individually takes forever. So we need to pull out all the files first, and then do a big query against NGAS, and match things up. Doing this is going to require a massive join, which made me think I should try pandas, and this is me trying it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import pandas\n",
+ "import pyat.schema\n",
+ "from pyat.schema.model import ScienceProduct, AncillaryProduct, Filegroup, File\n",
+ "from pyat.schema.ngasmodel import NGASFile\n",
+ "import tqdm\n",
+ "\n",
+ "def groups_of(df, n):\n",
+ " \"\"\"Return sub-dataframes for every N rows in the data frame.\"\"\"\n",
+ " next_chunk, remainder = df.head(n), df.tail(-n)\n",
+ " while not next_chunk.empty:\n",
+ " yield next_chunk\n",
+ " next_chunk, remainder = remainder.head(n), remainder.tail(-n)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "session = pyat.schema.create_session('SDM')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The next query is fairly intense, because I need to recursively find all the file groups under a product in order to get all their files. We'll need to do something similar to take care of ancillary products in a bit."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [
+ {
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mMemoryError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m<ipython-input-3-76997686e89d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[0mjoin\u001b[0m \u001b[0mrgroups\u001b[0m \u001b[0mon\u001b[0m \u001b[0msp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfilegroup_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrgroups\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtop_fg\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 12\u001b[0m join files f on f.filegroup = rgroups.child_fg\"\"\"\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mspl_files\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpandas\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_sql\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 14\u001b[0m \u001b[0mspl_files\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32mc:\\users\\jplank\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pandas\\io\\sql.py\u001b[0m in \u001b[0;36mread_sql\u001b[1;34m(sql, con, index_col, coerce_float, params, parse_dates, columns, chunksize)\u001b[0m\n\u001b[0;32m 436\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoerce_float\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 437\u001b[0m \u001b[0mparse_dates\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparse_dates\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 438\u001b[1;33m \u001b[0mchunksize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 439\u001b[0m )\n\u001b[0;32m 440\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32mc:\\users\\jplank\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pandas\\io\\sql.py\u001b[0m in \u001b[0;36mread_query\u001b[1;34m(self, sql, index_col, coerce_float, parse_dates, params, chunksize)\u001b[0m\n\u001b[0;32m 1235\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1236\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoerce_float\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1237\u001b[1;33m \u001b[0mparse_dates\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparse_dates\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1238\u001b[0m )\n\u001b[0;32m 1239\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mframe\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32mc:\\users\\jplank\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pandas\\io\\sql.py\u001b[0m in \u001b[0;36m_wrap_result\u001b[1;34m(data, columns, index_col, coerce_float, parse_dates)\u001b[0m\n\u001b[0;32m 122\u001b[0m \u001b[1;34m\"\"\"Wrap result set of query in a DataFrame.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 123\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 124\u001b[1;33m \u001b[0mframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_records\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoerce_float\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 125\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 126\u001b[0m \u001b[0mframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_parse_date_columns\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparse_dates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32mc:\\users\\jplank\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mfrom_records\u001b[1;34m(cls, data, index, exclude, columns, coerce_float, nrows)\u001b[0m\n\u001b[0;32m 1634\u001b[0m \u001b[0marr_columns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1635\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1636\u001b[1;33m \u001b[0marrays\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marr_columns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_arrays\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoerce_float\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1637\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1638\u001b[0m \u001b[0marr_columns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marr_columns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32mc:\\users\\jplank\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pandas\\core\\internals\\construction.py\u001b[0m in \u001b[0;36mto_arrays\u001b[1;34m(data, columns, coerce_float, dtype)\u001b[0m\n\u001b[0;32m 482\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 483\u001b[0m \u001b[1;31m# last ditch effort\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 484\u001b[1;33m \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 485\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0m_list_to_arrays\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoerce_float\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 486\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32mc:\\users\\jplank\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pandas\\core\\internals\\construction.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m 482\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 483\u001b[0m \u001b[1;31m# last ditch effort\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 484\u001b[1;33m \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 485\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0m_list_to_arrays\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoerce_float\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 486\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mMemoryError\u001b[0m: "
+ ],
+ "ename": "MemoryError",
+ "evalue": "",
+ "output_type": "error"
+ }
+ ],
+ "source": [
+ "query = \"\"\"with recursive\n",
+ " rgroups as (\n",
+ " select sp.filegroup_id as top_fg, sp.filegroup_id as child_fg\n",
+ " from science_products sp\n",
+ " -- where sp.science_product_locator = 'uid://evla/execblock/5a2e125e-8043-4354-b7a1-0dcc9b018019'\n",
+ " union all\n",
+ " select rgroups.top_fg as top_fg, fg.filegroup_id\n",
+ " from rgroups\n",
+ " join filegroups fg on rgroups.child_fg = fg.parent_filegroup_id)\n",
+ "select sp.science_product_locator, f.ngas_id, f.ngas_cluster, f.ngas_location from science_products sp\n",
+ "join rgroups on sp.filegroup_id = rgroups.top_fg\n",
+ "join files f on f.filegroup = rgroups.child_fg\"\"\"\n",
+ "spl_files = pandas.read_sql(query, session.bind)\n",
+ "spl_files"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now we have all the SPLs and their NGAS IDs in a Pandas table, so we can join that against the table we get out from NGAS. But now we need to get that table out from NGAS. First, we need to select only the files that have `ngas_cluster` of `DSOC`, and then we need to turn that column into an argument to the SQL, so that we only get things that might match."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dsoc_files = spl_files.query('ngas_cluster==\"DSOC\"')\n",
+ "dsoc_files"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now we can select all the NGAS IDs and start to build the query."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "ngas = pyat.schema.create_session('NGAS')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "#ngas_ids = pandas.read_sql(ngas.query(NGASFile.file_id).filter(NGASFile.file_id.in_(dsoc_files.ngas_id.values)).statement, ngas.bind)\n",
+ "ngas_ids = pandas.read_sql(ngas.query(NGASFile.file_id).statement, ngas.connection())\n",
+ "ngas_ids['present'] = True\n",
+ "ngas_ids"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "joined = ngas_ids.set_index('file_id').join(dsoc_files.set_index('ngas_id'), how='right')\n",
+ "joined"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "missing = joined[joined.present.isna()]\n",
+ "missing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "for spl, missing in missing.groupby('science_product_locator'):\n",
+ " print(f'{spl} missing {len(missing)} files')"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.8"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/notebooks/find_project_products.ipynb b/apps/cli/executables/pexable/ingest/notebooks/find_project_products.ipynb
new file mode 100644
index 000000000..7b421c2bf
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/notebooks/find_project_products.ipynb
@@ -0,0 +1,274 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas\n",
+ "from tqdm.auto import tqdm\n",
+ "import pyat.schema\n",
+ "from pyat.schema.model import Project, ScienceProduct, AncillaryProduct, File\n",
+ "\n",
+ "tqdm.pandas()\n",
+ "session = pyat.schema.create_session('SDM')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<Project#14A-346 \"A Deep, High-Resolution JVLA Study of the Sgr A Complex\" start=56794.23685474537 end=56803.49623726852>"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "project = session.query(Project).filter(Project.project_code == '14A-346').one()\n",
+ "project"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>locator</th>\n",
+ " <th>ngas_id</th>\n",
+ " <th>ngas_cluster</th>\n",
+ " <th>ngas_location</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>uid://evla/execblock/27a8542c-8293-4c68-a0d0-4...</td>\n",
+ " <td>uid___evla_sdm_X1400304811516.sdm</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>uid://evla/execblock/27a8542c-8293-4c68-a0d0-4...</td>\n",
+ " <td>uid___evla_sdm_X1400304811503.sdm</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>uid://evla/execblock/27a8542c-8293-4c68-a0d0-4...</td>\n",
+ " <td>uid___evla_sdm_X1400304811504.sdm</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>uid://evla/execblock/27a8542c-8293-4c68-a0d0-4...</td>\n",
+ " <td>uid___evla_sdm_X1400304811505.sdm</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>uid://evla/execblock/27a8542c-8293-4c68-a0d0-4...</td>\n",
+ " <td>uid___evla_sdm_X1400304811506.sdm</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>141</th>\n",
+ " <td>uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a...</td>\n",
+ " <td>uid____evla_bdf_1401103281456.bdf</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>142</th>\n",
+ " <td>uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a...</td>\n",
+ " <td>uid____evla_bdf_1401103730258.bdf</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>143</th>\n",
+ " <td>uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a...</td>\n",
+ " <td>uid____evla_bdf_1401104179008.bdf</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>144</th>\n",
+ " <td>uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a...</td>\n",
+ " <td>uid____evla_bdf_1401104318605.bdf</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>145</th>\n",
+ " <td>uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a...</td>\n",
+ " <td>uid____evla_bdf_1401104736504.bdf</td>\n",
+ " <td>DSOC</td>\n",
+ " <td>DSOC</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>146 rows × 4 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " locator \\\n",
+ "0 uid://evla/execblock/27a8542c-8293-4c68-a0d0-4... \n",
+ "1 uid://evla/execblock/27a8542c-8293-4c68-a0d0-4... \n",
+ "2 uid://evla/execblock/27a8542c-8293-4c68-a0d0-4... \n",
+ "3 uid://evla/execblock/27a8542c-8293-4c68-a0d0-4... \n",
+ "4 uid://evla/execblock/27a8542c-8293-4c68-a0d0-4... \n",
+ ".. ... \n",
+ "141 uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a... \n",
+ "142 uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a... \n",
+ "143 uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a... \n",
+ "144 uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a... \n",
+ "145 uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a... \n",
+ "\n",
+ " ngas_id ngas_cluster ngas_location \n",
+ "0 uid___evla_sdm_X1400304811516.sdm DSOC DSOC \n",
+ "1 uid___evla_sdm_X1400304811503.sdm DSOC DSOC \n",
+ "2 uid___evla_sdm_X1400304811504.sdm DSOC DSOC \n",
+ "3 uid___evla_sdm_X1400304811505.sdm DSOC DSOC \n",
+ "4 uid___evla_sdm_X1400304811506.sdm DSOC DSOC \n",
+ ".. ... ... ... \n",
+ "141 uid____evla_bdf_1401103281456.bdf DSOC DSOC \n",
+ "142 uid____evla_bdf_1401103730258.bdf DSOC DSOC \n",
+ "143 uid____evla_bdf_1401104179008.bdf DSOC DSOC \n",
+ "144 uid____evla_bdf_1401104318605.bdf DSOC DSOC \n",
+ "145 uid____evla_bdf_1401104736504.bdf DSOC DSOC \n",
+ "\n",
+ "[146 rows x 4 columns]"
+ ]
+ },
+ "execution_count": 42,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "files = []\n",
+ "for sp in project.science_products:\n",
+ " files.extend((sp.locator, f.ngas_id, f.ngas_cluster, f.ngas_location) for f in sp.filegroup.all_files)\n",
+ " for ap in sp.ancillary_products:\n",
+ " files.extend((ap.locator, f.ngas_id, f.ngas_cluster, f.ngas_location) for f in ap.filegroup.all_files)\n",
+ "files = pandas.DataFrame(files, columns=['locator', 'ngas_id', 'ngas_cluster', 'ngas_location'])\n",
+ "files"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'uid://evla/execblock/27a8542c-8293-4c68-a0d0-41416521de86': Int64Index([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,\n",
+ " 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,\n",
+ " 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,\n",
+ " 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67,\n",
+ " 68, 69, 70, 71, 72],\n",
+ " dtype='int64'),\n",
+ " 'uid://evla/execblock/f6f897eb-c716-4acf-a8f1-a19471c06ce4': Int64Index([ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85,\n",
+ " 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98,\n",
+ " 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111,\n",
+ " 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124,\n",
+ " 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137,\n",
+ " 138, 139, 140, 141, 142, 143, 144, 145],\n",
+ " dtype='int64')}"
+ ]
+ },
+ "execution_count": 47,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "files.groupby('locator').groups"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['bar']"
+ ]
+ },
+ "execution_count": 54,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "t = pandas.DataFrame([('foo', True),('bar',False),('baz',True)], columns=['name','present'])\n",
+ "list(t[t.present==False].name)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.8"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/apps/cli/executables/pexable/ingest/notebooks/missing_ebs.ipynb b/apps/cli/executables/pexable/ingest/notebooks/missing_ebs.ipynb
new file mode 100644
index 000000000..97b3ebf88
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/notebooks/missing_ebs.ipynb
@@ -0,0 +1,904 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import pandas\n",
+ "import numpy\n",
+ "import pyat.schema\n",
+ "from pyat.schema.model import ScienceProduct, AncillaryProduct, Filegroup, File\n",
+ "from pyat.schema.ngasmodel import NGASFile\n",
+ "import tqdm\n",
+ "import cx_Oracle\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "#First, I need to get the EVLA ebs that are in the legacy archive.\n",
+ "legacy = pyat.schema.create_session('LEGACY')\n",
+ "query =\"\"\"select ARCHFILELOCATION.ARCH_FILE, ARCHIVE.ARCH_FILE AS ngas_id, PROJECT_CODE \n",
+ "from ARCHIVE inner join ARCHFILELOCATION on ARCHIVE.ARCH_FILE = ARCHFILELOCATION.ARCH_FILE\n",
+ "where TELESCOPE='EVLA'\"\"\"\n",
+ "legacy_result = pandas.read_sql(query, legacy.bind)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>arch_file</th>\n",
+ " <th>ngas_id</th>\n",
+ " <th>project_code</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>09A-106.sb3347506.eb3569223.55619.69491549768</td>\n",
+ " <td>09A-106.sb3347506.eb3569223.55619.69491549768</td>\n",
+ " <td>09A-106</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>09A-106.sb3352622.eb3591699.55625.67939193287</td>\n",
+ " <td>09A-106.sb3352622.eb3591699.55625.67939193287</td>\n",
+ " <td>09A-106</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>09A-106.sb3362167.eb3592502.55632.66018666667</td>\n",
+ " <td>09A-106.sb3362167.eb3592502.55632.66018666667</td>\n",
+ " <td>09A-106</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>09A-106.sb3684616.eb3709112.55639.641143518515</td>\n",
+ " <td>09A-106.sb3684616.eb3709112.55639.641143518515</td>\n",
+ " <td>09A-106</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>09A-106.sb3861088.eb3867871.55653.602983530094</td>\n",
+ " <td>09A-106.sb3861088.eb3867871.55653.602983530094</td>\n",
+ " <td>09A-106</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>97665</th>\n",
+ " <td>sysstartK_001.56870.000027395836</td>\n",
+ " <td>sysstartK_001.56870.000027395836</td>\n",
+ " <td>Operations</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>97666</th>\n",
+ " <td>sysstartK_001.56874.75462980324</td>\n",
+ " <td>sysstartK_001.56874.75462980324</td>\n",
+ " <td>Operations</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>97667</th>\n",
+ " <td>sysstartK_001.56881.80496854166</td>\n",
+ " <td>sysstartK_001.56881.80496854166</td>\n",
+ " <td>Operations</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>97668</th>\n",
+ " <td>sysstartK_001.56890.0215975</td>\n",
+ " <td>sysstartK_001.56890.0215975</td>\n",
+ " <td>Operations</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>97669</th>\n",
+ " <td>sysstartK_001.56896.97181207176</td>\n",
+ " <td>sysstartK_001.56896.97181207176</td>\n",
+ " <td>Operations</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>97670 rows × 3 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " arch_file \\\n",
+ "0 09A-106.sb3347506.eb3569223.55619.69491549768 \n",
+ "1 09A-106.sb3352622.eb3591699.55625.67939193287 \n",
+ "2 09A-106.sb3362167.eb3592502.55632.66018666667 \n",
+ "3 09A-106.sb3684616.eb3709112.55639.641143518515 \n",
+ "4 09A-106.sb3861088.eb3867871.55653.602983530094 \n",
+ "... ... \n",
+ "97665 sysstartK_001.56870.000027395836 \n",
+ "97666 sysstartK_001.56874.75462980324 \n",
+ "97667 sysstartK_001.56881.80496854166 \n",
+ "97668 sysstartK_001.56890.0215975 \n",
+ "97669 sysstartK_001.56896.97181207176 \n",
+ "\n",
+ " ngas_id project_code \n",
+ "0 09A-106.sb3347506.eb3569223.55619.69491549768 09A-106 \n",
+ "1 09A-106.sb3352622.eb3591699.55625.67939193287 09A-106 \n",
+ "2 09A-106.sb3362167.eb3592502.55632.66018666667 09A-106 \n",
+ "3 09A-106.sb3684616.eb3709112.55639.641143518515 09A-106 \n",
+ "4 09A-106.sb3861088.eb3867871.55653.602983530094 09A-106 \n",
+ "... ... ... \n",
+ "97665 sysstartK_001.56870.000027395836 Operations \n",
+ "97666 sysstartK_001.56874.75462980324 Operations \n",
+ "97667 sysstartK_001.56881.80496854166 Operations \n",
+ "97668 sysstartK_001.56890.0215975 Operations \n",
+ "97669 sysstartK_001.56896.97181207176 Operations \n",
+ "\n",
+ "[97670 rows x 3 columns]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "legacy_result"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#next, I need the ebs in the new archive\n",
+ "session = pyat.schema.create_session('SDM', profile='nmprod')\n",
+ "query = \"\"\"select execution_blocks.execution_block_id, execution_blocks.project_code, execution_blocks.ngas_fileset_id \n",
+ "from execution_blocks\n",
+ " where telescope='EVLA' \"\"\"\n",
+ "spl_files = pandas.read_sql(query, session.bind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ },
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>execution_block_id</th>\n",
+ " <th>project_code</th>\n",
+ " <th>ngas_fileset_id</th>\n",
+ " <th>present</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>132133</td>\n",
+ " <td>Operations</td>\n",
+ " <td>sysstartKa.58626.88931716435</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>132164</td>\n",
+ " <td>Operations</td>\n",
+ " <td>X_osro_001.58626.89927145833</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>140754</td>\n",
+ " <td>Operations</td>\n",
+ " <td>delay_NX_000.58779.85314850694</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>140779</td>\n",
+ " <td>Operations</td>\n",
+ " <td>sysstartQ.58779.96363196759</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>140789</td>\n",
+ " <td>Operations</td>\n",
+ " <td>sysstartX_000.58781.0298977662</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>87791</th>\n",
+ " <td>140612</td>\n",
+ " <td>TPUL0001</td>\n",
+ " <td>phasing_X_4GHz.q34.58778.90662894676</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>87792</th>\n",
+ " <td>132012</td>\n",
+ " <td>Operations</td>\n",
+ " <td>19A-393_TEST_B2319.58626.80279469908</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>87793</th>\n",
+ " <td>140635</td>\n",
+ " <td>Operations</td>\n",
+ " <td>sysstartL.58778.95344793981</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>87794</th>\n",
+ " <td>140639</td>\n",
+ " <td>Operations</td>\n",
+ " <td>sysstartS.58778.9574991088</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>87795</th>\n",
+ " <td>140650</td>\n",
+ " <td>Operations</td>\n",
+ " <td>sysstartKa.58778.972608125</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>87796 rows × 4 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " execution_block_id project_code ngas_fileset_id \\\n",
+ "0 132133 Operations sysstartKa.58626.88931716435 \n",
+ "1 132164 Operations X_osro_001.58626.89927145833 \n",
+ "2 140754 Operations delay_NX_000.58779.85314850694 \n",
+ "3 140779 Operations sysstartQ.58779.96363196759 \n",
+ "4 140789 Operations sysstartX_000.58781.0298977662 \n",
+ "... ... ... ... \n",
+ "87791 140612 TPUL0001 phasing_X_4GHz.q34.58778.90662894676 \n",
+ "87792 132012 Operations 19A-393_TEST_B2319.58626.80279469908 \n",
+ "87793 140635 Operations sysstartL.58778.95344793981 \n",
+ "87794 140639 Operations sysstartS.58778.9574991088 \n",
+ "87795 140650 Operations sysstartKa.58778.972608125 \n",
+ "\n",
+ " present \n",
+ "0 True \n",
+ "1 True \n",
+ "2 True \n",
+ "3 True \n",
+ "4 True \n",
+ "... ... \n",
+ "87791 True \n",
+ "87792 True \n",
+ "87793 True \n",
+ "87794 True \n",
+ "87795 True \n",
+ "\n",
+ "[87796 rows x 4 columns]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "spl_files['present'] = True\n",
+ "spl_files"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>arch_file</th>\n",
+ " <th>project_code_legacy</th>\n",
+ " <th>execution_block_id</th>\n",
+ " <th>project_code</th>\n",
+ " <th>present</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngas_id</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>09A-106.sb3347506.eb3569223.55619.69491549768</th>\n",
+ " <td>09A-106.sb3347506.eb3569223.55619.69491549768</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>58594.0</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>09A-106.sb3352622.eb3591699.55625.67939193287</th>\n",
+ " <td>09A-106.sb3352622.eb3591699.55625.67939193287</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>57805.0</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>09A-106.sb3362167.eb3592502.55632.66018666667</th>\n",
+ " <td>09A-106.sb3362167.eb3592502.55632.66018666667</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>57806.0</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>09A-106.sb3684616.eb3709112.55639.641143518515</th>\n",
+ " <td>09A-106.sb3684616.eb3709112.55639.641143518515</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>57807.0</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>09A-106.sb3861088.eb3867871.55653.602983530094</th>\n",
+ " <td>09A-106.sb3861088.eb3867871.55653.602983530094</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>57145.0</td>\n",
+ " <td>09A-106</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>sysstartK_001.56870.000027395836</th>\n",
+ " <td>sysstartK_001.56870.000027395836</td>\n",
+ " <td>Operations</td>\n",
+ " <td>27244.0</td>\n",
+ " <td>Operations</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>sysstartK_001.56874.75462980324</th>\n",
+ " <td>sysstartK_001.56874.75462980324</td>\n",
+ " <td>Operations</td>\n",
+ " <td>27245.0</td>\n",
+ " <td>Operations</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>sysstartK_001.56881.80496854166</th>\n",
+ " <td>sysstartK_001.56881.80496854166</td>\n",
+ " <td>Operations</td>\n",
+ " <td>27246.0</td>\n",
+ " <td>Operations</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>sysstartK_001.56890.0215975</th>\n",
+ " <td>sysstartK_001.56890.0215975</td>\n",
+ " <td>Operations</td>\n",
+ " <td>27247.0</td>\n",
+ " <td>Operations</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>sysstartK_001.56896.97181207176</th>\n",
+ " <td>sysstartK_001.56896.97181207176</td>\n",
+ " <td>Operations</td>\n",
+ " <td>27248.0</td>\n",
+ " <td>Operations</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>97670 rows × 5 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " arch_file \\\n",
+ "ngas_id \n",
+ "09A-106.sb3347506.eb3569223.55619.69491549768 09A-106.sb3347506.eb3569223.55619.69491549768 \n",
+ "09A-106.sb3352622.eb3591699.55625.67939193287 09A-106.sb3352622.eb3591699.55625.67939193287 \n",
+ "09A-106.sb3362167.eb3592502.55632.66018666667 09A-106.sb3362167.eb3592502.55632.66018666667 \n",
+ "09A-106.sb3684616.eb3709112.55639.641143518515 09A-106.sb3684616.eb3709112.55639.641143518515 \n",
+ "09A-106.sb3861088.eb3867871.55653.602983530094 09A-106.sb3861088.eb3867871.55653.602983530094 \n",
+ "... ... \n",
+ "sysstartK_001.56870.000027395836 sysstartK_001.56870.000027395836 \n",
+ "sysstartK_001.56874.75462980324 sysstartK_001.56874.75462980324 \n",
+ "sysstartK_001.56881.80496854166 sysstartK_001.56881.80496854166 \n",
+ "sysstartK_001.56890.0215975 sysstartK_001.56890.0215975 \n",
+ "sysstartK_001.56896.97181207176 sysstartK_001.56896.97181207176 \n",
+ "\n",
+ " project_code_legacy \\\n",
+ "ngas_id \n",
+ "09A-106.sb3347506.eb3569223.55619.69491549768 09A-106 \n",
+ "09A-106.sb3352622.eb3591699.55625.67939193287 09A-106 \n",
+ "09A-106.sb3362167.eb3592502.55632.66018666667 09A-106 \n",
+ "09A-106.sb3684616.eb3709112.55639.641143518515 09A-106 \n",
+ "09A-106.sb3861088.eb3867871.55653.602983530094 09A-106 \n",
+ "... ... \n",
+ "sysstartK_001.56870.000027395836 Operations \n",
+ "sysstartK_001.56874.75462980324 Operations \n",
+ "sysstartK_001.56881.80496854166 Operations \n",
+ "sysstartK_001.56890.0215975 Operations \n",
+ "sysstartK_001.56896.97181207176 Operations \n",
+ "\n",
+ " execution_block_id \\\n",
+ "ngas_id \n",
+ "09A-106.sb3347506.eb3569223.55619.69491549768 58594.0 \n",
+ "09A-106.sb3352622.eb3591699.55625.67939193287 57805.0 \n",
+ "09A-106.sb3362167.eb3592502.55632.66018666667 57806.0 \n",
+ "09A-106.sb3684616.eb3709112.55639.641143518515 57807.0 \n",
+ "09A-106.sb3861088.eb3867871.55653.602983530094 57145.0 \n",
+ "... ... \n",
+ "sysstartK_001.56870.000027395836 27244.0 \n",
+ "sysstartK_001.56874.75462980324 27245.0 \n",
+ "sysstartK_001.56881.80496854166 27246.0 \n",
+ "sysstartK_001.56890.0215975 27247.0 \n",
+ "sysstartK_001.56896.97181207176 27248.0 \n",
+ "\n",
+ " project_code present \n",
+ "ngas_id \n",
+ "09A-106.sb3347506.eb3569223.55619.69491549768 09A-106 True \n",
+ "09A-106.sb3352622.eb3591699.55625.67939193287 09A-106 True \n",
+ "09A-106.sb3362167.eb3592502.55632.66018666667 09A-106 True \n",
+ "09A-106.sb3684616.eb3709112.55639.641143518515 09A-106 True \n",
+ "09A-106.sb3861088.eb3867871.55653.602983530094 09A-106 True \n",
+ "... ... ... \n",
+ "sysstartK_001.56870.000027395836 Operations True \n",
+ "sysstartK_001.56874.75462980324 Operations True \n",
+ "sysstartK_001.56881.80496854166 Operations True \n",
+ "sysstartK_001.56890.0215975 Operations True \n",
+ "sysstartK_001.56896.97181207176 Operations True \n",
+ "\n",
+ "[97670 rows x 5 columns]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#now, compare the two:\n",
+ "combined = legacy_result.set_index('ngas_id').join(spl_files.set_index('ngas_fileset_id'), on='ngas_id', how='left', lsuffix='_legacy')\n",
+ "combined"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array(['BM479.sb35568863.eb35574175.58348.80606886574',\n",
+ " 'C_baseband.55131.65250361111', 'C_baseband.55132.839772604166',\n",
+ " ..., '14A-007_20140315_1394888122236',\n",
+ " '14A-007_20140422_1398171325552', '14A-007_20140430_1398855322758'],\n",
+ " dtype=object)"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "missing = combined[combined.present!=True]\n",
+ "missing[['arch_file','project_code', 'project_code_legacy']]\n",
+ "missing[\"arch_file\"].unique()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>project_code</th>\n",
+ " <th>project_code_legacy</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngas_id</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>BM479.sb35568863.eb35574175.58348.80606886574</th>\n",
+ " <td>NaN</td>\n",
+ " <td>BM479</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>C_baseband.55131.65250361111</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C_baseband</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>C_quad1_002.57994.82833423611</th>\n",
+ " <td>NaN</td>\n",
+ " <td>Operations</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>CrsroXrsro_3C84_001.55692.92237219907</th>\n",
+ " <td>NaN</td>\n",
+ " <td>TRSR0001</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>L_realfast_B.58101.78498135417</th>\n",
+ " <td>NaN</td>\n",
+ " <td>158_2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>22_1.sb30475501.eb30476883.57688.66450266204</th>\n",
+ " <td>NaN</td>\n",
+ " <td>22_1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>14A-000_20140228_1393862124523</th>\n",
+ " <td>NaN</td>\n",
+ " <td>14A-000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>14A-002_20140222_1393068920209</th>\n",
+ " <td>NaN</td>\n",
+ " <td>14A-002</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>14A-003_20140625_1404241354555</th>\n",
+ " <td>NaN</td>\n",
+ " <td>14A-003</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>14A-007_20140315_1394888122236</th>\n",
+ " <td>NaN</td>\n",
+ " <td>14A-007</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>808 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " project_code project_code_legacy\n",
+ "ngas_id \n",
+ "BM479.sb35568863.eb35574175.58348.80606886574 NaN BM479\n",
+ "C_baseband.55131.65250361111 NaN C_baseband\n",
+ "C_quad1_002.57994.82833423611 NaN Operations\n",
+ "CrsroXrsro_3C84_001.55692.92237219907 NaN TRSR0001\n",
+ "L_realfast_B.58101.78498135417 NaN 158_2\n",
+ "... ... ...\n",
+ "22_1.sb30475501.eb30476883.57688.66450266204 NaN 22_1\n",
+ "14A-000_20140228_1393862124523 NaN 14A-000\n",
+ "14A-002_20140222_1393068920209 NaN 14A-002\n",
+ "14A-003_20140625_1404241354555 NaN 14A-003\n",
+ "14A-007_20140315_1394888122236 NaN 14A-007\n",
+ "\n",
+ "[808 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#missing.columns = ['project_code','is_present']\n",
+ "#this does not create exactly the right file; the header row will need modified.\n",
+ "missing[['project_code', 'project_code_legacy']].drop_duplicates().to_csv('missing_ngas_filesets_EVLA.csv')\n",
+ "missing[['project_code', 'project_code_legacy']].drop_duplicates()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "c:\\users\\jplank\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
+ "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+ "Try using .loc[row_indexer,col_indexer] = value instead\n",
+ "\n",
+ "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+ " \"\"\"Entry point for launching an IPython kernel.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>filecount</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>project_code_legacy</th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>10C-133</th>\n",
+ " <td>4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>11A-226</th>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>11B-050</th>\n",
+ " <td>7</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>12A-176</th>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>12B-068</th>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>not_in_PDS</th>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>saturntest</th>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>saturntest2</th>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>saturntest_000</th>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>widarbase04-08</th>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>808 rows × 1 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " filecount\n",
+ "project_code_legacy \n",
+ "10C-133 4\n",
+ "11A-226 2\n",
+ "11B-050 7\n",
+ "12A-176 2\n",
+ "12B-068 1\n",
+ "... ...\n",
+ "not_in_PDS 5\n",
+ "saturntest 1\n",
+ "saturntest2 1\n",
+ "saturntest_000 1\n",
+ "widarbase04-08 1\n",
+ "\n",
+ "[808 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "missing['filecount'] = 1\n",
+ "missing[['project_code_legacy', 'filecount']].groupby('project_code_legacy').count()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "12.75110064502918"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "legacy_count = legacy_result.size\n",
+ "missing_count = missing.size\n",
+ "missing_percent = (missing_count/legacy_count) * 100\n",
+ "missing_percent"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.7"
+ },
+ "pycharm": {
+ "stem_cell": {
+ "cell_type": "raw",
+ "metadata": {
+ "collapsed": false
+ },
+ "source": []
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/apps/cli/executables/pexable/ingest/notebooks/missing_ebs_VLBA.ipynb b/apps/cli/executables/pexable/ingest/notebooks/missing_ebs_VLBA.ipynb
new file mode 100644
index 000000000..920b39a02
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/notebooks/missing_ebs_VLBA.ipynb
@@ -0,0 +1,1078 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import pandas\n",
+ "import numpy\n",
+ "import pyat.schema\n",
+ "from pyat.schema.model import ScienceProduct, AncillaryProduct, Filegroup, File\n",
+ "from pyat.schema.ngasmodel import NGASFile\n",
+ "import tqdm\n",
+ "import cx_Oracle\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "First, I need to get the EVLA ebs that are in the legacy archive.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "legacy = pyat.schema.create_session('LEGACY')\n",
+ "query =\"\"\"select ARCHFILELOCATION.ARCH_FILE, ARCHIVE.ARCH_FILE AS ngas_id, PROJECT_CODE \n",
+ "from ARCHIVE inner join ARCHFILELOCATION on ARCHIVE.ARCH_FILE = ARCHFILELOCATION.ARCH_FILE\n",
+ "where TELESCOPE='VLBA'\"\"\"\n",
+ "legacy_result = pandas.read_sql(query, legacy.bind)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ },
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>arch_file</th>\n",
+ " <th>ngas_id</th>\n",
+ " <th>project_code</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255...</td>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC21_0_120831T16255...</td>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC21_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC22_0_120831T16255...</td>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC22_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC23_0_120831T16255...</td>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC23_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC24_0_120831T16255...</td>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC24_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>395565</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T15104...</td>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>395566</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T15104...</td>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>395567</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T15104...</td>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>395568</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T15104...</td>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>395569</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T15104...</td>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>395570 rows × 3 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " arch_file \\\n",
+ "0 VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255... \n",
+ "1 VLBA_BD161EV_bd161ev_BIN0_SRC21_0_120831T16255... \n",
+ "2 VLBA_BD161EV_bd161ev_BIN0_SRC22_0_120831T16255... \n",
+ "3 VLBA_BD161EV_bd161ev_BIN0_SRC23_0_120831T16255... \n",
+ "4 VLBA_BD161EV_bd161ev_BIN0_SRC24_0_120831T16255... \n",
+ "... ... \n",
+ "395565 VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T15104... \n",
+ "395566 VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T15104... \n",
+ "395567 VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T15104... \n",
+ "395568 VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T15104... \n",
+ "395569 VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T15104... \n",
+ "\n",
+ " ngas_id project_code \n",
+ "0 VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255... BD0161 \n",
+ "1 VLBA_BD161EV_bd161ev_BIN0_SRC21_0_120831T16255... BD0161 \n",
+ "2 VLBA_BD161EV_bd161ev_BIN0_SRC22_0_120831T16255... BD0161 \n",
+ "3 VLBA_BD161EV_bd161ev_BIN0_SRC23_0_120831T16255... BD0161 \n",
+ "4 VLBA_BD161EV_bd161ev_BIN0_SRC24_0_120831T16255... BD0161 \n",
+ "... ... ... \n",
+ "395565 VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T15104... DQ2008 \n",
+ "395566 VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T15104... DQ2008 \n",
+ "395567 VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T15104... DQ2008 \n",
+ "395568 VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T15104... DQ2008 \n",
+ "395569 VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T15104... DQ2008 \n",
+ "\n",
+ "[395570 rows x 3 columns]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "legacy_result"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "#next, I need the ebs in the new archive\n",
+ "session = pyat.schema.create_session('SDM', profile='nmprod')\n",
+ "query = \"\"\"select execution_blocks.execution_block_id, execution_blocks.project_code, files.ngas_id from execution_blocks\n",
+ "inner join scans on scans.execution_block_id = execution_blocks.execution_block_id\n",
+ "inner join subscans on subscans.scan_id= scans.scan_id\n",
+ "inner join files on files.file_id = subscans.file_id\n",
+ " where telescope='VLBA' \"\"\"\n",
+ "spl_files = pandas.read_sql(query, session.bind)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ },
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>execution_block_id</th>\n",
+ " <th>project_code</th>\n",
+ " <th>ngas_id</th>\n",
+ " <th>present</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>111500</td>\n",
+ " <td>BD215</td>\n",
+ " <td>VLBA_BD215F4_bd215f4_BIN0_SRC00_1_181126T15503...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>108013</td>\n",
+ " <td>RV131</td>\n",
+ " <td>VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>108013</td>\n",
+ " <td>RV131</td>\n",
+ " <td>VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>108013</td>\n",
+ " <td>RV131</td>\n",
+ " <td>VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>108013</td>\n",
+ " <td>RV131</td>\n",
+ " <td>VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>8710119</th>\n",
+ " <td>148437</td>\n",
+ " <td>BD215</td>\n",
+ " <td>VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>8710120</th>\n",
+ " <td>148437</td>\n",
+ " <td>BD215</td>\n",
+ " <td>VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>8710121</th>\n",
+ " <td>148437</td>\n",
+ " <td>BD215</td>\n",
+ " <td>VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>8710122</th>\n",
+ " <td>148437</td>\n",
+ " <td>BD215</td>\n",
+ " <td>VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>8710123</th>\n",
+ " <td>148437</td>\n",
+ " <td>BD215</td>\n",
+ " <td>VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1...</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>8710124 rows × 4 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " execution_block_id project_code \\\n",
+ "0 111500 BD215 \n",
+ "1 108013 RV131 \n",
+ "2 108013 RV131 \n",
+ "3 108013 RV131 \n",
+ "4 108013 RV131 \n",
+ "... ... ... \n",
+ "8710119 148437 BD215 \n",
+ "8710120 148437 BD215 \n",
+ "8710121 148437 BD215 \n",
+ "8710122 148437 BD215 \n",
+ "8710123 148437 BD215 \n",
+ "\n",
+ " ngas_id present \n",
+ "0 VLBA_BD215F4_bd215f4_BIN0_SRC00_1_181126T15503... True \n",
+ "1 VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi... True \n",
+ "2 VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi... True \n",
+ "3 VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi... True \n",
+ "4 VLBA_RV131_rv131_BIN0_SRC0_0_181019T143850.idi... True \n",
+ "... ... ... \n",
+ "8710119 VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1... True \n",
+ "8710120 VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1... True \n",
+ "8710121 VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1... True \n",
+ "8710122 VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1... True \n",
+ "8710123 VLBA_BD215D4_bd215d4recor_BIN0_SRC0_1_200429T1... True \n",
+ "\n",
+ "[8710124 rows x 4 columns]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "spl_files['present'] = True\n",
+ "spl_files"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>arch_file</th>\n",
+ " <th>project_code_legacy</th>\n",
+ " <th>execution_block_id</th>\n",
+ " <th>project_code</th>\n",
+ " <th>present</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngas_id</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558.idifits</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " <td>126942.0</td>\n",
+ " <td>BD161</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558.idifits</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " <td>126942.0</td>\n",
+ " <td>BD161</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558.idifits</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " <td>126942.0</td>\n",
+ " <td>BD161</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558.idifits</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " <td>126942.0</td>\n",
+ " <td>BD161</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558.idifits</th>\n",
+ " <td>VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255...</td>\n",
+ " <td>BD0161</td>\n",
+ " <td>126942.0</td>\n",
+ " <td>BD161</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T151040.idifits</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>147110.0</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T151040.idifits</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>147110.0</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T151040.idifits</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>147110.0</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T151040.idifits</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>147110.0</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T151040.idifits</th>\n",
+ " <td>VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T15104...</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>147110.0</td>\n",
+ " <td>DQ2008</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>12243992 rows × 5 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " arch_file \\\n",
+ "ngas_id \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255... \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255... \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255... \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255... \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T16255... \n",
+ "... ... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T151040... VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T15104... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T151040... VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T15104... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T151040... VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T15104... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T151040... VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T15104... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T151040... VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T15104... \n",
+ "\n",
+ " project_code_legacy \\\n",
+ "ngas_id \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD0161 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD0161 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD0161 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD0161 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD0161 \n",
+ "... ... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T151040... DQ2008 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T151040... DQ2008 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T151040... DQ2008 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T151040... DQ2008 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T151040... DQ2008 \n",
+ "\n",
+ " execution_block_id \\\n",
+ "ngas_id \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... 126942.0 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... 126942.0 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... 126942.0 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... 126942.0 \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... 126942.0 \n",
+ "... ... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T151040... 147110.0 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T151040... 147110.0 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T151040... 147110.0 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T151040... 147110.0 \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T151040... 147110.0 \n",
+ "\n",
+ " project_code present \n",
+ "ngas_id \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD161 True \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD161 True \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD161 True \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD161 True \n",
+ "VLBA_BD161EV_bd161ev_BIN0_SRC20_0_120831T162558... BD161 True \n",
+ "... ... ... \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_06_200304T151040... DQ2008 True \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_07_200304T151040... DQ2008 True \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_08_200304T151040... DQ2008 True \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_09_200304T151040... DQ2008 True \n",
+ "VLBA_DQ2008B_dq2008b_BIN0_SRC0_10_200304T151040... DQ2008 True \n",
+ "\n",
+ "[12243992 rows x 5 columns]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#now, compare the two:\n",
+ "combined = legacy_result.set_index('ngas_id').join(spl_files.set_index('ngas_id'), on='ngas_id', how='left', lsuffix='_legacy')\n",
+ "combined"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array(['GMVA_c041aPart1.uvfits', 'GMVA_c041aPart2.uvfits',\n",
+ " 'GMVA_c041aPart3.uvfits', ...,\n",
+ " 'VLBA_UG003W_ug003wrecor_BIN0_SRC0_0_200430T144230.mark4.tar.gz',\n",
+ " 'VLBA_UH003A_uh003a_BIN0_SRC0_0_180507T195629.mark4.tar.gz',\n",
+ " 'VLBA_UH003B_uh003b_BIN0_SRC0_0_180515T142206.mark4.tar.gz'],\n",
+ " dtype=object)"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "missing = combined[combined.present!=True]\n",
+ "missing[['arch_file','project_code', 'project_code_legacy']]\n",
+ "missing[\"arch_file\"].unique()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>project_code</th>\n",
+ " <th>project_code_legacy</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngas_id</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>GMVA_c041aPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C041A</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c041bPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C041B</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c042aPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C042A</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c052aPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C052A</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c052bPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C052B</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BP245P_bp245p_BIN0_SRC0_0_200302T232704.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>BP0245</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BP246A_bp246arecor_BIN0_SRC0_0_200226T221424.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>BP0246</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BR095_FPOL2IF0.5MHZ128CH_4.6601GHZ03.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>BR0095</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2020_dq2020_BIN0_SRC0_01_200602T151219.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>DQ2020</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2022_dq2022_BIN0_SRC0_00_200615T153638.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>DQ2022</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>2955 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " project_code \\\n",
+ "ngas_id \n",
+ "GMVA_c041aPart1.uvfits NaN \n",
+ "GMVA_c041bPart1.uvfits NaN \n",
+ "GMVA_c042aPart1.uvfits NaN \n",
+ "GMVA_c052aPart1.uvfits NaN \n",
+ "GMVA_c052bPart1.uvfits NaN \n",
+ "... ... \n",
+ "VLBA_BP245P_bp245p_BIN0_SRC0_0_200302T232704.id... NaN \n",
+ "VLBA_BP246A_bp246arecor_BIN0_SRC0_0_200226T2214... NaN \n",
+ "VLBA_BR095_FPOL2IF0.5MHZ128CH_4.6601GHZ03.uvfits NaN \n",
+ "VLBA_DQ2020_dq2020_BIN0_SRC0_01_200602T151219.i... NaN \n",
+ "VLBA_DQ2022_dq2022_BIN0_SRC0_00_200615T153638.i... NaN \n",
+ "\n",
+ " project_code_legacy \n",
+ "ngas_id \n",
+ "GMVA_c041aPart1.uvfits C041A \n",
+ "GMVA_c041bPart1.uvfits C041B \n",
+ "GMVA_c042aPart1.uvfits C042A \n",
+ "GMVA_c052aPart1.uvfits C052A \n",
+ "GMVA_c052bPart1.uvfits C052B \n",
+ "... ... \n",
+ "VLBA_BP245P_bp245p_BIN0_SRC0_0_200302T232704.id... BP0245 \n",
+ "VLBA_BP246A_bp246arecor_BIN0_SRC0_0_200226T2214... BP0246 \n",
+ "VLBA_BR095_FPOL2IF0.5MHZ128CH_4.6601GHZ03.uvfits BR0095 \n",
+ "VLBA_DQ2020_dq2020_BIN0_SRC0_01_200602T151219.i... DQ2020 \n",
+ "VLBA_DQ2022_dq2022_BIN0_SRC0_00_200615T153638.i... DQ2022 \n",
+ "\n",
+ "[2955 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "drop_mk_4 = missing[['project_code', 'project_code_legacy']].drop_duplicates()\n",
+ "mk_4 = drop_mk_4.filter(like='.mark4.tar.gz', axis=0)\n",
+ "drop_mk_4 = drop_mk_4[~drop_mk_4.index.isin(mk_4.index)]\n",
+ "drop_mk_4"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>project_code</th>\n",
+ " <th>project_code_legacy</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngas_id</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>GMVA_c041aPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C041A</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c041bPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C041B</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c042aPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C042A</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c052aPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C052A</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>GMVA_c052bPart1.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>C052B</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BP245P_bp245p_BIN0_SRC0_0_200302T232704.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>BP0245</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BP246A_bp246arecor_BIN0_SRC0_0_200226T221424.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>BP0246</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_BR095_FPOL2IF0.5MHZ128CH_4.6601GHZ03.uvfits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>BR0095</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2020_dq2020_BIN0_SRC0_01_200602T151219.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>DQ2020</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>VLBA_DQ2022_dq2022_BIN0_SRC0_00_200615T153638.idifits</th>\n",
+ " <td>NaN</td>\n",
+ " <td>DQ2022</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>2955 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " project_code \\\n",
+ "ngas_id \n",
+ "GMVA_c041aPart1.uvfits NaN \n",
+ "GMVA_c041bPart1.uvfits NaN \n",
+ "GMVA_c042aPart1.uvfits NaN \n",
+ "GMVA_c052aPart1.uvfits NaN \n",
+ "GMVA_c052bPart1.uvfits NaN \n",
+ "... ... \n",
+ "VLBA_BP245P_bp245p_BIN0_SRC0_0_200302T232704.id... NaN \n",
+ "VLBA_BP246A_bp246arecor_BIN0_SRC0_0_200226T2214... NaN \n",
+ "VLBA_BR095_FPOL2IF0.5MHZ128CH_4.6601GHZ03.uvfits NaN \n",
+ "VLBA_DQ2020_dq2020_BIN0_SRC0_01_200602T151219.i... NaN \n",
+ "VLBA_DQ2022_dq2022_BIN0_SRC0_00_200615T153638.i... NaN \n",
+ "\n",
+ " project_code_legacy \n",
+ "ngas_id \n",
+ "GMVA_c041aPart1.uvfits C041A \n",
+ "GMVA_c041bPart1.uvfits C041B \n",
+ "GMVA_c042aPart1.uvfits C042A \n",
+ "GMVA_c052aPart1.uvfits C052A \n",
+ "GMVA_c052bPart1.uvfits C052B \n",
+ "... ... \n",
+ "VLBA_BP245P_bp245p_BIN0_SRC0_0_200302T232704.id... BP0245 \n",
+ "VLBA_BP246A_bp246arecor_BIN0_SRC0_0_200226T2214... BP0246 \n",
+ "VLBA_BR095_FPOL2IF0.5MHZ128CH_4.6601GHZ03.uvfits BR0095 \n",
+ "VLBA_DQ2020_dq2020_BIN0_SRC0_01_200602T151219.i... DQ2020 \n",
+ "VLBA_DQ2022_dq2022_BIN0_SRC0_00_200615T153638.i... DQ2022 \n",
+ "\n",
+ "[2955 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#missing.columns = ['project_code','is_present']\n",
+ "drop_mk_4.to_csv('missing_ngas_files_VLBA.csv')\n",
+ "drop_mk_4"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>filecount</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>project_code_legacy</th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>AB0108</th>\n",
+ " <td>49</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>AD0592</th>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>AP0296</th>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>BA0007</th>\n",
+ " <td>20</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>BA0008</th>\n",
+ " <td>3261</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ZT0004</th>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ZU0001</th>\n",
+ " <td>4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ZU0002</th>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>gl035a</th>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>no_good</th>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>3882 rows × 1 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " filecount\n",
+ "project_code_legacy \n",
+ "AB0108 49\n",
+ "AD0592 3\n",
+ "AP0296 2\n",
+ "BA0007 20\n",
+ "BA0008 3261\n",
+ "... ...\n",
+ "ZT0004 1\n",
+ "ZU0001 4\n",
+ "ZU0002 1\n",
+ "gl035a 2\n",
+ "no_good 5\n",
+ "\n",
+ "[3882 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#missing['filecount'] = 1\n",
+ "missing[['project_code_legacy', 'filecount']].groupby('project_code_legacy').count()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.9872135249778754"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "legacy_count = legacy_result.drop_duplicates().size\n",
+ "missing_count = missing[['project_code', 'project_code_legacy']].drop_duplicates().size\n",
+ "missing_percent = (missing_count/legacy_count) * 100\n",
+ "missing_percent"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.7"
+ },
+ "pycharm": {
+ "stem_cell": {
+ "cell_type": "raw",
+ "metadata": {
+ "collapsed": false
+ },
+ "source": []
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/apps/cli/executables/pexable/ingest/notebooks/reconciliation.ipynb b/apps/cli/executables/pexable/ingest/notebooks/reconciliation.ipynb
new file mode 100644
index 000000000..c4e3ebf44
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/notebooks/reconciliation.ipynb
@@ -0,0 +1,679 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas\n",
+ "from pandas import DataFrame\n",
+ "from tqdm.auto import tqdm\n",
+ "import pyat.schema\n",
+ "from pyat.schema.model import Project, ScienceProduct, AncillaryProduct, File\n",
+ "from pyat.schema.legacy_model import *\n",
+ "from pyat.schema.ngasmodel import NGASFile\n",
+ "from sqlalchemy.sql import *"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tqdm.pandas()\n",
+ "session = pyat.schema.create_session('SDM', profile='local')\n",
+ "ngas = pyat.schema.create_session('NGAS', profile='local')\n",
+ "legacy = pyat.schema.create_session('LEGACY', profile='local')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>ngas_id</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>ngastest_Source_13_1.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>ngastest_Source_14_2.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>ngastest_Source_15_3.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>ngastest_Source_16_4.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>ngastest_Source_1.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2963462</th>\n",
+ " <td>uid___X1_X2_X1224623858320.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2963463</th>\n",
+ " <td>uid___X1_X2_X1224623858321.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2963464</th>\n",
+ " <td>uid___X1_X2_X1224623858322.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2963465</th>\n",
+ " <td>uid___X1_X2_X1224623858323.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2963466</th>\n",
+ " <td>uid___X1_X2_X1224623858324.sdm</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>2963467 rows × 1 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " ngas_id\n",
+ "0 ngastest_Source_13_1.sdm\n",
+ "1 ngastest_Source_14_2.sdm\n",
+ "2 ngastest_Source_15_3.sdm\n",
+ "3 ngastest_Source_16_4.sdm\n",
+ "4 ngastest_Source_1.sdm\n",
+ "... ...\n",
+ "2963462 uid___X1_X2_X1224623858320.sdm\n",
+ "2963463 uid___X1_X2_X1224623858321.sdm\n",
+ "2963464 uid___X1_X2_X1224623858322.sdm\n",
+ "2963465 uid___X1_X2_X1224623858323.sdm\n",
+ "2963466 uid___X1_X2_X1224623858324.sdm\n",
+ "\n",
+ "[2963467 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ngas_sdms = pandas.read_sql(\"select distinct file_id as ngas_id from ngas_files where format = 'application/x-sdm' AND file_id = 'uid___X1_X2_X1224623858320.sdm' order by file_id\", ngas.connection())\n",
+ "query = select([NGASFile.file_id.label('ngas_id')]).where(NGASFile.format == 'application/x-sdm').order_by(NGASFile.file_id)\n",
+ "result = pandas.read_sql(query, ngas.connection())\n",
+ "result"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>ngas_id</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>uid___A002_X2b8998_X192.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>uid___A002_X2b8998_X2bf.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>uid___A002_X2b8998_X65.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>uid___A002_X2b8bb6_X1.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>uid___A002_X2b8bb7_X1.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4749058</th>\n",
+ " <td>uid___evla_sdm_X1583372543642.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4749059</th>\n",
+ " <td>uid___evla_sdm_X1583372543643.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4749060</th>\n",
+ " <td>uid___evla_sdm_X1583372543644.sdm</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4749061</th>\n",
+ " <td>uid___evla_sdm_X1583372543645.bin</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4749062</th>\n",
+ " <td>uid___evla_sdm_X1583372543646.sdm</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>4749063 rows × 1 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " ngas_id\n",
+ "0 uid___A002_X2b8998_X192.sdm\n",
+ "1 uid___A002_X2b8998_X2bf.sdm\n",
+ "2 uid___A002_X2b8998_X65.sdm\n",
+ "3 uid___A002_X2b8bb6_X1.sdm\n",
+ "4 uid___A002_X2b8bb7_X1.sdm\n",
+ "... ...\n",
+ "4749058 uid___evla_sdm_X1583372543642.sdm\n",
+ "4749059 uid___evla_sdm_X1583372543643.sdm\n",
+ "4749060 uid___evla_sdm_X1583372543644.sdm\n",
+ "4749061 uid___evla_sdm_X1583372543645.bin\n",
+ "4749062 uid___evla_sdm_X1583372543646.sdm\n",
+ "\n",
+ "[4749063 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "archive_sdms = pandas.read_sql(\"select ngas_id from files where format = 'sdm' order by ngas_id\", session.connection())\n",
+ "archive_sdms"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>present</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngas_id</th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_1.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_13_1.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_14_2.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_15_3.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_16_4.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_aocngas-11sdmtest.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_aocngas-12sdmtest.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_aocngas-9sdmtest.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_test091002SW.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid__evla_sdm_X1260824882273.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>2745728 rows × 1 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " present\n",
+ "ngas_id \n",
+ "ngastest_Source_1.sdm NaN\n",
+ "ngastest_Source_13_1.sdm NaN\n",
+ "ngastest_Source_14_2.sdm NaN\n",
+ "ngastest_Source_15_3.sdm NaN\n",
+ "ngastest_Source_16_4.sdm NaN\n",
+ "... ...\n",
+ "uid___evla_sdm_aocngas-11sdmtest.sdm NaN\n",
+ "uid___evla_sdm_aocngas-12sdmtest.sdm NaN\n",
+ "uid___evla_sdm_aocngas-9sdmtest.sdm NaN\n",
+ "uid___evla_sdm_test091002SW.sdm NaN\n",
+ "uid__evla_sdm_X1260824882273.sdm NaN\n",
+ "\n",
+ "[2745728 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "archive_sdms['present'] = True\n",
+ "result = ngas_sdms.set_index('ngas_id').join(archive_sdms.set_index('ngas_id'))\n",
+ "result"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>present</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngas_id</th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_1.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_13_1.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_14_2.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_15_3.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>ngastest_Source_16_4.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_aocngas-11sdmtest.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_aocngas-12sdmtest.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_aocngas-9sdmtest.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid___evla_sdm_test091002SW.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>uid__evla_sdm_X1260824882273.sdm</th>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>53178 rows × 1 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " present\n",
+ "ngas_id \n",
+ "ngastest_Source_1.sdm NaN\n",
+ "ngastest_Source_13_1.sdm NaN\n",
+ "ngastest_Source_14_2.sdm NaN\n",
+ "ngastest_Source_15_3.sdm NaN\n",
+ "ngastest_Source_16_4.sdm NaN\n",
+ "... ...\n",
+ "uid___evla_sdm_aocngas-11sdmtest.sdm NaN\n",
+ "uid___evla_sdm_aocngas-12sdmtest.sdm NaN\n",
+ "uid___evla_sdm_aocngas-9sdmtest.sdm NaN\n",
+ "uid___evla_sdm_test091002SW.sdm NaN\n",
+ "uid__evla_sdm_X1260824882273.sdm NaN\n",
+ "\n",
+ "[53178 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "missing = result[result.present.isna()]\n",
+ "missing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sqlalchemy.sql import *\n",
+ "from pyat.schema.legacy_model import *\n",
+ "from pyat.datafinder.util import groups_of"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "matching missing SDMs with legacy archive metadata\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "3072fc7b64d1416d9c7e9258c8a4b5f0",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>file_id</th>\n",
+ " <th>project_code</th>\n",
+ " <th>sb_id</th>\n",
+ " <th>eb_id</th>\n",
+ " <th>filename</th>\n",
+ " <th>file_set_id</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>uid___X1_X2_X1224284503144.sdm</td>\n",
+ " <td>Csdmtest_019</td>\n",
+ " <td>0</td>\n",
+ " <td>0</td>\n",
+ " <td>ASDM.xml</td>\n",
+ " <td>Csdmtest_019.54756.95949090278</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " file_id project_code sb_id eb_id filename \\\n",
+ "0 uid___X1_X2_X1224284503144.sdm Csdmtest_019 0 0 ASDM.xml \n",
+ "\n",
+ " file_set_id \n",
+ "0 Csdmtest_019.54756.95949090278 "
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import math\n",
+ "count = 1000\n",
+ "legacy = pyat.schema.create_session('LEGACY', profile='local')\n",
+ "matching = DataFrame([], columns=['file_id', 'project_code', 'sb_id', 'eb_id', 'filename'])\n",
+ "print('matching missing SDMs with legacy archive metadata')\n",
+ "for group in tqdm(groups_of(missing, count), total=math.ceil(len(missing)/count)):\n",
+ " matching = matching.append(pandas.read_sql(\n",
+ " select([t_ngas_file_sets.c.file_id.distinct(), \n",
+ " t_ngas_file_sets.c.file_set_id,\n",
+ " t_archive.c.project_code, \n",
+ " t_archive.c.sb_id, \n",
+ " t_archive.c.eb_id,\n",
+ " case([(t_ngas_file_sets.c.file_id.like('%.sdm'), func.replace(t_ngas_file_sets.c.entity_type_name, 'Table', '') + '.xml'),\n",
+ " (t_ngas_file_sets.c.file_id.like('%.bin'), func.replace(t_ngas_file_sets.c.entity_type_name, 'Table', '') + '.bin'),\n",
+ " (t_ngas_file_sets.c.file_id.like('%.bdf'), func.replace(t_ngas_file_sets.c.file_id, '.bdf', ''))]).label('filename')])\n",
+ " .where(and_(t_archive.c.arch_file == t_ngas_file_sets.c.file_set_id,\n",
+ " t_ngas_file_sets.c.file_id.in_(group.index)))\n",
+ " .order_by(t_ngas_file_sets.c.file_id),\n",
+ " legacy.connection()))\n",
+ "matching"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>file_id</th>\n",
+ " <th>project_code</th>\n",
+ " <th>sb_id</th>\n",
+ " <th>eb_id</th>\n",
+ " <th>filename</th>\n",
+ " <th>file_set_id</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>uid___X1_X2_X1224284503144.sdm</td>\n",
+ " <td>Csdmtest_019</td>\n",
+ " <td>0</td>\n",
+ " <td>0</td>\n",
+ " <td>ASDM.xml</td>\n",
+ " <td>Csdmtest_019.54756.95949090278</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " file_id project_code sb_id eb_id filename \\\n",
+ "0 uid___X1_X2_X1224284503144.sdm Csdmtest_019 0 0 ASDM.xml \n",
+ "\n",
+ " file_set_id \n",
+ "0 Csdmtest_019.54756.95949090278 "
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "res = list(matching.groupby('file_id'))[0]\n",
+ "res[1]\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.8"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/apps/cli/executables/pexable/ingest/pyat/__init__.py b/apps/cli/executables/pexable/ingest/pyat/__init__.py
new file mode 100644
index 000000000..6736eb206
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/__init__.py
@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+"""
+This is the pyat package, one of the core modules for the NRAO archive workflow system
+
+Authors: Daniel K Lyons <dlyons@nrao.edu>
+ Rick Lively <rlively@nrao.edu>
+ James Sheckar <jsheckar@nrao.edu>
+ Richard Falardeau <rfalarde@nrao.edu>
+"""
+import os
+from os import path
+from pathlib import Path
+import sys
+import logging
+import pycapo
+from pyat._version import ___version___ as version
+
+LOG_MESSAGE_FORMATTER = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+_CAPO_PROPERTIES_ROOT = '/home/casa/capo'
+
+
+def get_console_logger(application_name, level=logging.INFO):
+ r"""
+ A function to build a console logger for the caller
+ :param application_name: the callers name
+ :param level: the log level to set, defaulting to INFO if not set
+ :return: a console logger
+ """
+ console_log = logging.getLogger(application_name)
+ console_log.setLevel(level)
+ stdout_handler = logging.StreamHandler(sys.stdout)
+ stdout_handler.setLevel(level)
+ stdout_handler.setFormatter(LOG_MESSAGE_FORMATTER)
+ console_log.addHandler(stdout_handler)
+ return console_log
+
+
+_LOG = get_console_logger(__name__, logging.DEBUG)
+
+
+def profile_file_in_path(profile, properties_path):
+ """
+ Check if the profile name provided is a valid name for a capo property file, by checking
+ to see if a <profile>.properties file can be found in the path provided
+ :param profile: the profile name we want to look for
+ :param properties_path: a path to a directory where we expect to find capo property files
+ :return: true if the profile given has a corresponding <profile>.properties file in the
+ path provided.
+ """
+ valid_profile = False
+ profile_names = [prop.split('.properties')[0] for prop in os.listdir(properties_path)
+ if prop.endswith('.properties')]
+ if profile in profile_names:
+ valid_profile = True
+ return valid_profile
+
+
+def get_my_capo_config(**kwargs):
+ r"""
+ A function to return a CapoConfig for the profile either passed in, found in the environment
+ or derived from the location we've installed pyat. We do a rudamentary test of the profile
+ against a series of known capo properties directories (CAPO_PATH, /home/casa/capo, or
+ $HOME/.capo) and print a warning if the profile is not found as a valid properties file in any
+ of those location. This is solely to augment the cryptic exception messages that might be
+ thrown when trying to access a capo key from a file that doesn't exist.
+ :param kwargs: may or may not contain a 'profile'
+ :return: a CapoConfig object for the profile derived from the kwargs, env, or deployment
+ location
+ """
+ if 'profile' in kwargs and kwargs['profile']:
+ profile = kwargs['profile']
+ elif 'CAPO_PROFILE' in os.environ:
+ profile = os.environ['CAPO_PROFILE']
+ else:
+ # try to synthesize a profile from our installation root
+ # if you don't set the profile and end up here, but you're running locally... this might
+ # result in a profile of "<your virtualenv>", it's best to actually set it.
+ path_breakdown = os.path.abspath(sys.argv[0]).split(os.path.sep)
+ # CV sticks an extra subdirectory into the installation
+ if path_breakdown[-3] == 'current':
+ profile = path_breakdown[-4]
+ else:
+ profile = path_breakdown[-3]
+
+ # We should have a profile at this point, but it might not be valid (if the user didn't provide
+ # one), so now we'll try checking to see if that profile matchies a <profile>.properties file
+ # in one of our known properties directories. We'll check for a CAPO_PATH, then,
+ # /home/casa/capo, then $HOME/.capo, and, if all that fails, we'll print a warning to the
+ # console to augment any exception that might be thrown by trying to access a property from a
+ # properties files that doesn't exist.
+ profile_exists = False
+ if 'CAPO_PATH' in os.environ:
+ profile_exists = profile_file_in_path(profile, os.environ['CAPO_PATH'])
+ elif path.exists(_CAPO_PROPERTIES_ROOT):
+ profile_exists = profile_file_in_path(profile, _CAPO_PROPERTIES_ROOT)
+ elif path.exists(str(Path.home()) + '/.capo'):
+ profile_exists = profile_file_in_path(profile, str(Path.home()) + '/.capo')
+
+ if not profile_exists:
+ _LOG.error(f'The capo profile "{profile}" does not appear to match any known profile '
+ f'names. This might result in unexpected behavior if the application attempts '
+ f'to access a property from a file that does not exist.')
+
+ return pycapo.CapoConfig(profile=profile)
diff --git a/apps/cli/executables/pexable/ingest/pyat/_version.py b/apps/cli/executables/pexable/ingest/pyat/_version.py
new file mode 100644
index 000000000..7d348773d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/_version.py
@@ -0,0 +1,2 @@
+""" Version information for this package, don't put anything else here. """
+___version___ = '3.9.4'
diff --git a/apps/cli/executables/pexable/ingest/pyat/events/__init__.py b/apps/cli/executables/pexable/ingest/pyat/events/__init__.py
new file mode 100644
index 000000000..bbdc7b44d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/events/__init__.py
@@ -0,0 +1 @@
+from pyat.events.events import *
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/pyat/events/events.py b/apps/cli/executables/pexable/ingest/pyat/events/events.py
new file mode 100644
index 000000000..68b0e5dec
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/events/events.py
@@ -0,0 +1,100 @@
+import json
+import os
+
+import datetime
+import pika
+from pycapo import CapoConfig
+
+EVENTS_EXCHANGE = 'archive.events'
+
+""""
+A simple API for sending an archive event from Python.
+"""
+
+# Local variables
+_connection, _channel = None, None
+
+def broadcast(message, **kwargs):
+ """
+ Broadcasts a message to the archive event system.
+
+ Keyword arguments for the message:
+
+ :param: application: the application sending the message (defaults to 'python')
+ :param: user: the user sending the event (defaults to the currently logged in user)
+ :param: request: the request this event is in reference to, if applicable
+ :param: message: the message itself (will override the formal parameter)
+ :param: logData: a dictionary of additional fields to include in the event
+ :param: timestamp: the current timestamp
+ :param: version: version of the event system to use ('1.0' is the only value at the time of writing)
+
+ Keyword arguments for the AMQP connection. These do not need to be specified:
+
+ :param: hostname: the hostname to connect to
+ :param: port: the port to connect to
+ :param: connection_attempts: the number of connection attempts to try
+ :param: socket_timeout: the time to wait for a socket to connect
+ :param: retry_delay: the time to wait between retrying the connection
+ :param: username: the username to connect to as
+ :param: password: the password to use when connecting
+
+ :return: None
+ """
+ # first, connect
+ _ensure_connected(**kwargs)
+
+ # second, send the event
+ _send(_format_message(message, **kwargs), **kwargs)
+
+
+def _ensure_connected(**kwargs):
+ global _connection, _channel
+ if not _connection:
+ config = CapoConfig(profile=kwargs.get('profile', None)).settings('edu.nrao.archive.configuration.AmqpServer')
+ connection_parameters = pika.ConnectionParameters(
+ host=kwargs.get('hostname', config.hostname),
+ port=int(kwargs.get('port', config.port)),
+ connection_attempts=kwargs.get('connection_attempts', 5),
+ socket_timeout=kwargs.get('socket_timeout', 5000),
+ retry_delay=kwargs.get('retry_delay', 500),
+ credentials=
+ pika.PlainCredentials(
+ username=kwargs.get('username', config.username),
+ password=kwargs.get('password', config.password)))
+ _connection = pika.BlockingConnection(connection_parameters)
+ _channel = _connection.channel()
+ _channel.exchange_declare('archive.events',
+ exchange_type='topic',
+ durable=True, auto_delete=False)
+
+
+def _format_message(message, **kwargs):
+ default_message = {
+ 'application': 'python',
+ 'user': 'nobody',
+ 'request': None,
+ 'message': message,
+ 'logData': {},
+ 'timestamp': _datetime_to_dict(),
+ 'version': '1.0'
+ }
+
+ fields = ['application', 'user', 'request', 'message', 'logData',
+ 'timestamp', 'version']
+
+ return {k: kwargs.get(k, default_message[k]) for k in fields}
+
+
+def _send(message, **kwargs):
+ _channel.basic_publish(exchange=EVENTS_EXCHANGE,
+ routing_key=kwargs.get('routing_key', '{}.event'.format(message['application'])),
+ body=json.dumps(message))
+
+
+def _datetime_to_dict():
+ # Turn 'now' in UTC into the date format events expect to speak.
+ d, t = dict(), dict()
+ now = datetime.datetime.utcnow()
+ d['year'], d['month'], d['day'] = now.year, now.month, now.day
+ t['hour'], t['minute'], t['second'], t['nano'] = now.hour, now.minute, now.second, now.microsecond * 1000
+ return {'date': d, 'time': t}
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/pyat/legacy/import_by_dir.py b/apps/cli/executables/pexable/ingest/pyat/legacy/import_by_dir.py
new file mode 100644
index 000000000..3847d5288
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/legacy/import_by_dir.py
@@ -0,0 +1,36 @@
+from os import listdir
+from os.path import isfile, join, exists
+import sys
+import import_xml_file
+
+#get a list of xml files in the given directory:
+directory = sys.argv[1]
+print (directory)
+files = []
+preexisting_count = 0
+
+for filename in listdir(directory):
+ if not filename.endswith('.xml'): continue
+ if exists(filename+'.log'):
+ preexisting_count = preexisting_count +1
+ continue
+ fullname = join(directory, filename)
+ files.append(filename)
+
+print ('files found: ', len(files))
+print ('preexisting: ', preexisting_count)
+
+#process each file:
+current = 1
+problems = []
+for f in files:
+ print('proccesing {} of {}: {}'.format(current, len(files), f))
+
+ result = import_xml_file.parseFile(directory+'/'+f, f+".log")
+ if result != 0:
+ problems.append(f)
+
+ current = current + 1
+
+print('{} problems found:'.format(len(problems)))
+print(problems)
diff --git a/apps/cli/executables/pexable/ingest/pyat/legacy/import_xml_file.py b/apps/cli/executables/pexable/ingest/pyat/legacy/import_xml_file.py
new file mode 100644
index 000000000..51b61f48c
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/legacy/import_xml_file.py
@@ -0,0 +1,265 @@
+import sys
+import psycopg2
+import datetime
+from lxml import etree
+import os
+from pycapo import CapoConfig
+
+profile = os.environ['CAPO_PROFILE']
+
+def getElement(root, element):
+ return root.find(".//{http://www.w3schools.com/VLAarchive}"+element)
+
+def getElementList(root, element):
+ return root.findall(".//{http://www.w3schools.com/VLAarchive}"+element)
+
+def connect(profile):
+ """ Connect to the PostgreSQL database server """
+ conn = None
+ try:
+
+ config = CapoConfig(profile=profile)
+ user = config['metadataDatabase.jdbcUsername']
+ password = config['metadataDatabase.jdbcPassword']
+ url = config['metadataDatabase.jdbcUrl']
+ archiveurl = url.replace('jdbc:', '').replace('://', '://' + user + ':' + password + '@')
+
+ # connect to the PostgreSQL server
+ print('Connecting to '+archiveurl)
+ conn = psycopg2.connect(archiveurl)
+
+
+ return conn
+
+ except (Exception, psycopg2.DatabaseError) as error:
+ print(error)
+
+
+def parseFile(file_name, log_file):
+ #set up the log file:
+ log = open(log_file, "w")
+
+ #open and parse the file:
+ try:
+ input_file = open(file_name, 'r')
+ tree = etree.parse(input_file)
+ except:
+ print("Could not parse file {}!".format(file_name))
+ log.write("could not parse file {}!".format(file_name))
+ return(1)
+
+ # connect to the DB:
+ try:
+ conn = connect(profile)
+ cur = conn.cursor()
+ cur.execute('SELECT version()')
+ db_version = cur.fetchone()
+ except:
+ print("Could not connect to the database!")
+ log.write("could not connect to the database!")
+ return(1)
+
+ #collect the values we need from the xml:
+ root = tree.getroot()
+
+ project = {}
+ project['project_code'] = getElement(root, "observing_program_Id").text
+ project['configuration'] = getElement(root, "configuration").text
+ project['starttime'] = getElement(root, "start_time").text
+ project['endtime'] = getElement(root, "end_time").text
+ project['prop_expiration'] = 0
+ project['prop_duration'] = 365
+
+ log.write(str(project))
+
+ if project['project_code'] is None:
+ print("No project code found: exiting.")
+ log.write("no project code found.")
+ return(1)
+
+ #create the project, if necessary
+ cur.execute('SELECT starttime, endtime from projects where project_code=%s', [project['project_code']])
+ existing_project = cur.fetchone()
+ if existing_project is not None:
+ #update the project, if necessary
+ project['starttime'] = existing_project[0]
+ project['endtime'] = existing_project[1]
+ log.write("Project {} exists.".format(project['project_code']))
+ else:
+ #create the project
+ log.write("Creating Project:")
+ log.write("project code: %s, legacy_id: %s, title: %s, abstract: %s, start_time: %s, end_time: %s"
+ %(project['project_code'], project['project_code'], "null", "null", project['starttime'], project['endtime']))
+ cur.execute("INSERT INTO projects (project_code, legacy_id, starttime, endtime) VALUES (%s, %s, %s, %s);"
+ , [project['project_code'], project['project_code'], project['starttime'], project['endtime']])
+ author = {}
+ author['name'] = getElement(root, 'observer_name').text
+ name = author['name'].split(" ", 1)
+
+ #create the author, if necessary
+ cur.execute('SELECT firstname, lastname from authors where project_code=%s', [project['project_code']])
+ existing_authors = cur.fetchone()
+ if existing_authors is not None:
+ #authors already exist
+ log.write("Project %s has author %s %s." %(project['project_code'], existing_authors[0], existing_authors[1]))
+ else:
+ #create the author
+ log.write("Creating Author %s:" %(author['name']))
+ cur.execute("INSERT INTO authors (project_code, username, firstname, lastname, is_pi) VALUES (%s, %s, %s, %s, %s);",
+ [project['project_code'], 'unknown', name[0], name[1], True])
+
+ eb = {}
+ eb['configuration'] = getElement(root, "configuration").text
+ band_list = getElementList(root, "band")
+ bands = '';
+ for e in band_list:
+ if e.text is not None:
+ bands = bands+e.text+' '
+
+ eb['band_code'] = bands
+
+ filesize = os.stat(file_name[:-4]+".exp").st_size
+
+ #assume for now that all entries in a file belong to the same new EB:
+ cur.execute("insert into filegroups (project_code, type, datasize) values (%s, %s, %s) returning filegroup_id;"
+ , (project['project_code'], "execution_block", filesize))
+ filegroup_id = cur.fetchone()[0]
+
+ #there is one .exp file per .xml file:
+ filename = file_name.split('/')[-1][:-4]+".exp"
+ cur.execute("insert into files (filegroup, filename, filesize, format, type, ngas_location, ngas_cluster, ngas_id)"
+ " values(%s, %s, %s, %s, %s, %s, %s, %s)"
+ , (filegroup_id, filename, filesize, "exp", "raw", "DSOC", "DSOC", filename))
+
+ cur.execute("insert into science_products (science_product_locator, science_product_type, metadata_ingestion_date, metadata_ingestion_version, filegroup_id, external_name) "
+ "values ('uid://VLA/execblock/'||gen_random_uuid(), %s, %s, %s, %s, %s) returning science_product_locator;"
+ , ("Execution Block", datetime.datetime.now(), 1, filegroup_id, filename))
+ spl = cur.fetchone()[0]
+
+ log.write("creating science_products_projects entry:")
+ cur.execute("insert into science_products_projects (science_product_locator, project_code) values(%s, %s)", (spl, project['project_code']))
+
+ log.write("Creating EB: {}, {}, {}, {}, {}, {}, {}, {}"
+ .format( filegroup_id, 0, 'VLA', project['project_code'], 'DO Not Calibrate', eb['band_code'], eb['configuration'], True))
+ cur.execute("insert into execution_blocks (science_product_locator, filegroup_id, calibration_level, telescope, project_code"
+ ", calibration_status, band_code, configuration, ingestion_complete) "
+ "values (%s, %s, 0, 'VLA', %s, 'Do Not Calibrate', %s, %s, True) returning execution_block_id;"
+ , (spl, filegroup_id, project['project_code'], eb['band_code'], eb['configuration']))
+ eb_id = cur.fetchone()[0]
+
+ scan = {}
+
+ subscans = getElementList(root, "scan")
+ log.write("found %s scans".format(len(subscans)))
+ for e in subscans:
+ #create one scan entry and a subscan entry for each record:
+ cur.execute("insert into scans (execution_block_id, filegroup_id, max_bandwidth, min_bandwidth, polarization_code, max_frequency, min_frequency) "
+ "values (%s, %s, %s, %s, %s, %s, %s) returning scan_id;"
+ , (eb_id, filegroup_id, 0, 0, 0, 0, 0))
+ scan_id = cur.fetchone()[0]
+
+ scan_max_freq = 0.0
+ scan_min_freq = sys.float_info.max
+ scan_max_bw = 0.0
+ scan_min_bw = sys.float_info.max
+ scan_pol_code = 0
+ eb_start = sys.float_info.max
+ eb_end = 0.0
+
+ subscan = {}
+ subscan['starttime'] = getElement(e, 'start_time').text
+ subscan['endtime'] = getElement(e, 'end_time').text
+ subscan['ra'] = getElement(e, 'right_ascension_J2000').text
+ subscan['dec'] = getElement(e, 'declination_J2000').text
+ subscan['sourcename'] = getElement(e, "source_name").text
+ subscan['sourcetype'] = "NO IDEA"
+ subscan['obstype'] = ""
+ subscan['exposure_time'] = 0.0
+ subscan['integration_time'] = getElement(e, "integration_time").text
+ subscan['receiver_id'] = 0
+ subscan['backend'] = "none"
+
+ if subscan['sourcename'] is None or len(subscan['sourcename']) <= 0:
+ subscan['sourcename'] = 'Unknown'
+
+ #polarizations are annoying and fiddly.
+ #create a configuration for the subscan, and create a data_description linked to the configuration for each polarization
+ # in the XML
+ log.write("creating configuration")
+ cur.execute("insert into configurations (configuration, execution_block_id) values (%s, %s) returning configuration_id",
+ (0, eb_id))
+ config_id = cur.fetchone()
+
+ spectral_windows = getElementList(e, "spectral_window")
+ for sw in spectral_windows:
+ window = {}
+ window['frequency'] = getElement(sw, 'center_frequency').text
+ window['bandwidth'] = getElement(sw, 'bandwidth').text
+ window['channels'] = getElement(sw, 'number_of_channels').text
+ window['polarization'] = getElement(sw, 'polarizations').text
+ polarization_id = 0
+ if 'LL' in window['polarization']:
+ polarization_id += 16
+ if 'LR' in window['polarization']:
+ polarization_id += 32
+ if 'RL' in window['polarization']:
+ polarization_id += 64
+ if 'RR' in window['polarization']:
+ polarization_id += 128
+
+ #set some scan aggregate values:
+ scan_max_freq = max(scan_max_freq, float(window['frequency']))
+ scan_min_freq = min(scan_min_freq, float(window['frequency']))
+ scan_max_bw = max(scan_max_bw, float(window['bandwidth']))
+ scan_min_bw = min(scan_min_bw, float(window['bandwidth']))
+ scan_pol_code = scan_pol_code | polarization_id #bitwise or
+
+ log.write(str(window))
+ cur.execute("insert into data_descriptions (bandwidth, frequency, polarization_id, configuration_id) values (%s, %s, %s, %s)"
+ , (window['bandwidth'], window['frequency'], polarization_id, config_id))
+ #end for spectral windows
+
+ log.write(str(subscan))
+ cur.execute("insert into subscans (scan_id, obstype, starttime, endtime, sourcename, sourcetype, ra, dec, exposure_time"
+ ", integration_time, receiver_id, backend, configuration_id)"
+ "values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
+ , (scan_id, subscan['obstype'], subscan['starttime'], subscan['endtime'], subscan['sourcename'], subscan['sourcetype']
+ , subscan['ra'], subscan['dec'], subscan['exposure_time'], subscan['integration_time'], subscan['receiver_id']
+ , subscan['backend'], config_id))
+
+ #set some project aggregate values:
+ project['starttime'] = min(float(project['starttime']), float(subscan['starttime']))
+ project['endtime'] = max(float(project['endtime']), float(subscan['endtime']))
+ eb_start = min(eb_start,float( subscan['starttime']))
+ eb_end = max(eb_end, float(subscan['endtime']))
+
+ if scan_min_bw == sys.float_info.max:
+ scan_min_bw = 0;
+ if scan_min_freq == sys.float_info.max:
+ scan_min_freq = 0;
+
+ cur.execute("update scans set max_bandwidth=%s, min_bandwidth=%s, max_frequency=%s, min_frequency=%s, polarization_code=%s where scan_id=%s"
+ ,(scan_max_bw, scan_min_bw, scan_max_freq, scan_min_freq, scan_pol_code, scan_id))
+
+ #end for subscans
+ if eb_start == sys.float_info.max:
+ eb_start = 0;
+
+ #update the eb with aggregated values:
+ cur.execute("update execution_blocks set starttime=%s, endtime=%s where execution_block_id=%s"
+ , (eb_start, eb_end, eb_id))
+
+ #update the project with aggregate values:
+ cur.execute("update projects set starttime=%s, endtime=%s, proprietary_expiration=mjd_to_timestamp(%s), last_addition = mjd_to_timestamp(%s), proprietary_duration=%s where project_code = %s"
+ , (project['starttime'], project['endtime'], float(project['endtime'])+365.0, project['endtime'], 365, project['project_code']))
+
+ cur.execute("commit;");
+ return 0;
+
+
+
+
+
+
+
+
diff --git a/apps/cli/executables/pexable/ingest/pyat/mark4_import/__init__.py b/apps/cli/executables/pexable/ingest/pyat/mark4_import/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/cli/executables/pexable/ingest/pyat/mark4_import/audit.py b/apps/cli/executables/pexable/ingest/pyat/mark4_import/audit.py
new file mode 100644
index 000000000..89068aeaa
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/mark4_import/audit.py
@@ -0,0 +1,99 @@
+import datetime
+import pandas as pd
+import logging
+import sys
+from pycapo import CapoConfig
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker # an Engine, which the Session will use for connection
+from pyat.schema.model import Filegroup
+import argparse as ap
+from pyat import version
+from pyat.schema import create_engine, create_session
+
+_NOW = datetime.datetime.utcnow()
+_LOCATION = 'DSOC'
+_FILE_TYPE = 'MARK4'
+_MAX_ROWS_TO_IMPORT = 2
+
+_LOG = logging.getLogger("audit")
+_LOG.setLevel(logging.INFO)
+ch = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+ch.setLevel(logging.DEBUG)
+_LOG.addHandler(ch)
+_DESCRIPTION = """VLBA Audit version {}."""
+
+
+def audit_vlba():
+ """
+ Find matching idifits / mark4 pairs in the legacy archive. Create a mark4 file entry in the
+ new archive and attach it to the same filegroup as the idifits file.
+ :return:
+ """
+
+ args = _make_parser().parse_args()
+
+ ngas_session = create_session('NGAS', profile=args.profile)
+ ngas_engine = create_engine('NGAS', profile=args.profile)
+
+ ngas_vlba_files = pd.read_sql("""
+ select f.file_id filename
+ from ngas_files f
+ where f.file_id like 'VLBA%%'
+ and f.file_id not like '%%difx.log'
+ and f.file_id not like '%%mark4%%'
+ """,
+ ngas_engine)
+
+ ngas_session.close()
+ _LOG.info(f"{len(ngas_vlba_files)} VLBA files in NGAS")
+
+ # legacy_engine = create_engine('LEGACY', profile=args.profile)
+ # legacy_session = create_session('LEGACY', profile=args.profile)
+ # legacy_vlba_files = pd.read_sql("""
+ # select arch_file filename
+ # from archive
+ # where arch_file like 'VLBA%%'
+ # """,
+ # legacy_engine)
+ #
+ # legacy_session.close()
+ #
+ # _LOG.info(f"{len(legacy_vlba_files)} VLBA files in legacy archive")
+
+ archive_engine = create_engine('SDM', profile=args.profile)
+ archive_session = create_session('SDM', profile=args.profile)
+
+ archive_vlba_files = pd.read_sql("""
+ select filename
+ from files
+ where filename like 'VLBA%%'
+ """,
+ archive_engine)
+
+ archive_session.close()
+
+ _LOG.info(f"{len(archive_vlba_files)} VLBA files in new archive")
+
+ # file_merge = legacy_vlba_files.merge(ngas_vlba_files, indicator=True, how='outer')
+ # file_merge = legacy_vlba_files.merge(ngas_vlba_files, indicator=True, how='outer')
+ file_merge = ngas_vlba_files.merge(archive_vlba_files, indicator=True, how='outer')
+ diff = file_merge[file_merge['_merge'] == 'left_only']
+
+ _LOG.info(f"{len(diff)} unmatching entries")
+ _LOG.info(diff['filename'].head(25))
+
+
+def _make_parser():
+ r""" Build a command line parser for this app: this is external to MrBooks and/or CapoSettings because both need it,
+ and Sphinx will want a look at ot to build docs. """
+ result = ap.ArgumentParser(description=_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store',
+ help='profile name to use, e.g. test, production')
+ return result
+
+
+if __name__ == "__main__":
+ audit_vlba()
diff --git a/apps/cli/executables/pexable/ingest/pyat/mark4_import/commands.py b/apps/cli/executables/pexable/ingest/pyat/mark4_import/commands.py
new file mode 100644
index 000000000..e51c6d495
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/mark4_import/commands.py
@@ -0,0 +1,148 @@
+import datetime
+import pandas as pd
+import logging
+import sys
+from pycapo import CapoConfig
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker # an Engine, which the Session will use for connection
+from pyat.schema.model import Filegroup
+import argparse as ap
+from pyat import version
+
+
+_NOW = datetime.datetime.utcnow()
+_LOCATION = 'DSOC'
+_FILE_TYPE = 'MARK4'
+_MAX_ROWS_TO_IMPORT = 2
+
+_LOG = logging.getLogger("mark4_import")
+_LOG.setLevel(logging.INFO)
+ch = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+ch.setLevel(logging.DEBUG)
+_LOG.addHandler(ch)
+_DESCRIPTION = """Mark4 import version {}."""
+
+
+def mark4_import():
+ """
+ Find matching idifits / mark4 pairs in the legacy archive. Create a mark4 file entry in the
+ new archive and attach it to the same filegroup as the idifits file.
+ :return:
+ """
+ args = _make_parser().parse_args()
+ nconf = CapoConfig(profile=args.profile).settings('ngasDatabase')
+ aconf = CapoConfig(profile=args.profile).settings('metadataDatabase')
+
+ ngas_engine = create_engine(nconf.jdbcUrl.lstrip('jdbc:')
+ .replace('://', f'://{nconf.jdbcUsername}:{nconf.jdbcPassword}@'))
+ archive_engine = create_engine(aconf.jdbcUrl.lstrip('jdbc:')
+ .replace('://', f'://{aconf.jdbcUsername}:{aconf.jdbcPassword}@'))
+
+ ngasSession = sessionmaker(bind=ngas_engine)# create a Session
+ archiveSession = sessionmaker(bind=archive_engine)# create a Session
+
+ session = ngasSession()
+
+ # Construct list of matching mark4 and idifits files in NGAS database
+ mark4_matches = pd.read_sql("""
+ select nf1.file_id file_path,
+ nf1.file_id ngas_id,
+ nf1.file_id filename,
+ nf1.file_size filesize,
+ nf2.file_id idifits
+ from ngas_files nf1
+ inner join ngas_files nf2
+ on reverse(regexp_replace(reverse(nf1.file_name), '.*?_(.*)', '')) =
+ reverse(regexp_replace(reverse(nf2.file_name), '.*?_(.*)', ''))
+ where nf1.file_name like '%%mark4.tar.gz'
+ and nf2.file_name like '%%.idifits'""",
+ ngas_engine)
+
+ session.close()
+ _LOG.info(f"{len(mark4_matches)} MARK4 files in NGAS with matching IDIFITS file")
+
+ session = archiveSession()
+
+ # Construct list of IDIFITS files in archive database with no matching MARK4 file
+ idifits_matches = pd.read_sql("""
+ select f1.filename idifits,
+ f1.filegroup filegroup
+ from files f1
+ where f1.format = 'IDIFITS'
+ and not exists (
+ select 1
+ from files f2
+ where f2.format = 'MARK4'
+ and reverse(regexp_replace(reverse(f2.filename), '.*?_(.*)', '')) =
+ reverse(regexp_replace(reverse(f1.filename), '.*?_(.*)', ''))
+ )
+ """,
+ archive_engine)
+
+ _LOG.info(f"{len(idifits_matches)} unmatched IDIFITS files in archive")
+
+ file_merge = mark4_matches.merge(idifits_matches)
+ _LOG.info(f"{len(file_merge)} merged entries")
+
+ if len(file_merge) == 0:
+ _LOG.info(f"No mark4 files found to be imported")
+ session.close()
+ exit(0)
+
+ # Add in constant columns not in database
+ file_merge['type'] = _FILE_TYPE
+ file_merge['format'] = _FILE_TYPE
+ file_merge['checksum'] = None
+ file_merge['checksum_type'] = None
+ file_merge['ingestion_time'] = _NOW
+ file_merge['preview_storage_path'] = None
+ file_merge['ngas_location'] = _LOCATION
+ file_merge['ngas_cluster'] = _LOCATION
+
+ # Prepare data for insert into database
+ mark4_data = file_merge[['file_path', 'ngas_id', 'filegroup', 'filename', 'filesize', 'format', 'type', 'checksum', 'checksum_type', 'ingestion_time', 'preview_storage_path', 'ngas_location', 'ngas_cluster']]
+ # Limit on number of rows to insert - for debugging
+ # mark4_data = mark4_data.head(_MAX_ROWS_TO_IMPORT)
+
+ # Append rows to file table in archive database
+ mark4_data.to_sql('files', con=archive_engine, index=False, if_exists='append')
+
+ # Update filesize in filegroup containing new mark4 files
+ update_file_sizes(mark4_data, session)
+ session.close()
+
+
+def update_file_sizes(mark4_data, session):
+ """
+ Update file size for parent filegroup
+ :param mark4_data: dataframe containing the merged data
+ :param session:
+ :return:
+ """
+ for row in range(len(mark4_data)):
+ mark4_row = mark4_data.iloc[row]
+ filegroup = session.query(Filegroup). \
+ filter(Filegroup.filegroup_id==int(mark4_row['filegroup'])). \
+ one_or_none()
+ if filegroup is not None:
+ filegroup.datasize = filegroup.datasize + int(mark4_row['filesize'])
+ session.add(filegroup)
+ else:
+ _LOG.warning(f"Filegroup not found for id {mark4_row['filegroup']}")
+ session.commit()
+
+
+def _make_parser():
+ r""" Build a command line parser for this app: this is external to MrBooks and/or CapoSettings because both need it,
+ and Sphinx will want a look at ot to build docs. """
+ result = ap.ArgumentParser(description=_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store',
+ help='profile name to use, e.g. test, production')
+ return result
+
+
+if __name__ == "__main__":
+ mark4_import()
diff --git a/apps/cli/executables/pexable/ingest/pyat/qa_results/__init__.py b/apps/cli/executables/pexable/ingest/pyat/qa_results/__init__.py
new file mode 100644
index 000000000..a594901df
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/qa_results/__init__.py
@@ -0,0 +1,18 @@
+r"""
+ A nice, clean command-line interface for the Data Analysts to
+ notify the newArchive of the PASS/FAIL status of CIPL run
+ results.
+
+ On a PASS: send an ArchiveEvent signifying that we
+ should run the calibration ingestion workflow
+
+ On a FAIL: send an ArchiveEvent signifying that the
+ data is no good, and should be marked as
+ DoNotCalibrate
+
+
+ In either case, the script needs to identify a fileSetId from the
+ subdirectory name in the QA2 directory (this is used by Amygdala to
+ uniquely identify the observation).
+
+"""
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/pyat/qa_results/commands.py b/apps/cli/executables/pexable/ingest/pyat/qa_results/commands.py
new file mode 100644
index 000000000..cd421d833
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/qa_results/commands.py
@@ -0,0 +1,267 @@
+
+from __future__ import print_function
+
+import argparse as ap
+import getpass
+import os
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import sys
+from pyat import version
+from pyat.pymygdala import RPCEvent
+from pycapo import CapoConfig
+
+# Help Messages
+_PASS_DESCRIPTION = """AAT/PPI QA Pass {}: The calibration jobs listed are considered acceptable. This tool initiates
+ the ingestion of calibration products for the indicated directories and updates the list of calibrations for the
+ associated execution blocks."""
+
+_FAIL_DESCRIPTION = """AAT/PPI QA Fail {}: This tool marks the execution blocks related to the indicated directories
+as not appropriate for automated calibration, and initiates a clean up procedure to remove the directories from the QA
+area. """
+
+# Pipeline Processing Request
+PPR = "PPR"
+PPR_FILENAME = PPR + ".xml"
+
+
+def _make_parser(status):
+ r""" Build a command line parser for this app: this is external to MrClean and/or CapoSettings because both need
+ it, and Sphinx will want a look at ot to build docs. """
+
+ if status == "PASS":
+ _DESCRIPTION = _PASS_DESCRIPTION
+ else:
+ _DESCRIPTION = _FAIL_DESCRIPTION
+
+ result = ap.ArgumentParser(description=_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. test, production')
+ result.add_argument('-f', '--fileSetId', action='store', default='', help='provide a fileSetId for the directory instead of searching for it')
+ # For capturing DA comments, eventually.
+ # result.add_argument('-c', '--comment', type=str, action='store', default='No Comment Provided', help='Comment on the EB.')
+ result.add_argument('-e', '--email', type=str, action='store', default='', help='Email for completion notification')
+ result.add_argument('directoryName', type = str, action='store', nargs='+', help='name of the qa2 subdirectory to which the status applies')
+ return result
+
+
+def rawdata_check(path):
+ #TODO: Integrate this with both searches
+ # If we arrived here, then we need another route:
+ #
+ # the rawdata is stored in a directory named for
+ # fileSetId
+ raw_path = Path(path)
+ fileList = list(raw_path.glob('**/*'))
+ if(1 < len(fileList)):
+ # This lists: [fsid, ASDMBinary] if successful
+ fileSetId = fileList[0].name.replace("/", "") # make sure no trailing slash
+ return fileSetId
+
+
+def qa_search(path, subdir):
+ r"""Seach the /qa2/ subdirectory for the listed directory and extract the fileSetId from the
+ name of the flagversions file. return an empty string if this search fails"""
+ fileSetId = ''
+
+
+ # Where we're looking
+ qaPath = Path(path + '/' + subdir + '/products/')
+
+ # There's quite a bit of junk in the directory, and we only need the one tgz file
+ # so screen out the rest of them
+ #
+ # Loosened the glob to allow non NN[ABCD]-NNN.sb... identifiers through
+ fileList = list(qaPath.glob('**/*.ms.flagversions.tgz'))
+ if(0 < len(fileList)):
+ flagTableFile = fileList[0].name
+ fileSetId = flagTableFile.split(".ms")[0]
+ return fileSetId
+
+ #
+ # If we arrived here, then we need another route:
+ #
+ # the rawdata is stored in a directory named for
+ # fileSetId
+ raw_path = Path(path + '/' + subdir + '/rawdata/')
+ fileList = list(raw_path.glob('**/'))
+ if(1 < len(fileList)):
+ # This lists: [rawdata, fsid, ASDMBinary] if successful
+ fileSetId = fileList[1].name.replace("/", "") # make sure no trailing slash
+ return fileSetId
+
+ #
+ # Ok, now things are getting weird.
+ #
+ return fileSetId
+
+
+def spool_search(path, subdir):
+ r"""Search the /spool/ for the desired directory, check for
+ a rawdata subdirectory and parse the PPR to discover the fileSetId.
+ return an empty string if you can't acquire the fileSetId"""
+ fileSetId = ''
+
+ # If we're here, it's because there's nothing useful in the qa2
+ # directory, which has most of the data in it....
+ #
+
+ # Try the rawdata directory again
+ #
+ #
+ # the rawdata is stored in a directory named for
+ # fileSetId
+ raw_path = Path(path + '/' + subdir + '/rawdata/')
+ fileList = list(raw_path.glob('**/'))
+ if(1 < len(fileList)):
+ # This lists: [rawdata, fsid, ASDMBinary] if successful
+ fileSetId = fileList[1].name.replace("/", "") # make sure no trailing slash
+ return fileSetId
+
+ # No luck with the raw data directory here in spool...
+ #
+ # So that leaves us with one last thing to try: PPR.xml
+
+ pprFile = path+'/'+subdir+'/working/' + PPR_FILENAME
+ try:
+ fileTree = ET.parse(pprFile)
+ #
+ # Now walk the structure of the PPR file down to the piece we need
+ #
+ procRequests = fileTree.find('ProcessingRequests')
+ procReq = procRequests.find('ProcessingRequest')
+ dataSet = procReq.find('DataSet')
+ sdmElement = dataSet.find('SdmIdentifier').text
+ if sdmElement is not None:
+ fileSetId = sdmElement
+ return fileSetId
+
+ except ET.ParseError as pe:
+ print("Error parsing PPR file.")
+ except StopIteration as si:
+ print('No SdmIdentifier in the PPR file.')
+ except FileNotFoundError as fnf:
+ print('No PPR file found.')
+
+ # We haven't managed to find the fileSetId: indicate that
+ return ''
+
+
+def qa_list(status):
+ r"""From the given directory(s) find the fileSetId(s) required for either workflow or database update
+ purposes (either from some filenames in the directory, or potentially parsing the PPR if that is deemed
+ necessary). """
+ # Get the command line arguments
+ args = _make_parser(status).parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ capo = CapoConfig()
+
+ qaDirectory = capo.getstring("edu.nrao.archive.workflow.config.CiplWorkflowSettings.qaDirectory")
+ spoolDirectory = capo.getstring("edu.nrao.archive.workflow.config.CiplWorkflowSettings.spoolDirectory")
+
+
+ #
+ # So, the fileSetId is both in the name of a secondary tar file in the QA directory,
+ # and in the casa PPR file (which lives in spoolDirectory/$subDirName/working/)
+ # The former is by far the easier to deal with.
+ #
+ # A QA products directory has the following tgz files
+ #
+ # 13B-326.sb29065738.eb29113188.56764.86226100694.ms.flagversions.tgz
+ # unknown.session_1.caltables.tgz
+ # weblog.tgz
+ #
+ # The first of which is the fileSetId, so for each directory we were given,
+ # parse the fileSetId and send an event for each.
+ #
+ for resultsDirectory in args.directoryName:
+ # consistency: we'll place the /s
+ resultsDirectory = resultsDirectory.replace("/", "")
+
+ # If we haven't been provided with the fileSetId, go find it
+ if('' == args.fileSetId):
+ fileSetId = qa_search(qaDirectory, resultsDirectory)
+ if('' == fileSetId):
+ fileSetId = spool_search(spoolDirectory, resultsDirectory)
+ if ('' == fileSetId):
+ print("Failed to obtain a fileSetId for directory " + resultsDirectory)
+ return
+ else:
+ fileSetId = args.fileSetId
+
+ # We've got the information we need, go ahead and send the event
+ print("Sending "+status+" for "+resultsDirectory+" ("+fileSetId+")")
+ qa_single(fileSetId, resultsDirectory, status, args.email)
+
+
+def qa_single(fileset_id, results_directory, status, notify_email):
+ r"""Takes two strings: fileSetId (parsed from a file in the directory we're interested in)
+ and status (Pass or Fail), and builds string to execute to create the appropriate event
+ for updating the database and/or launching a workflow."""
+
+ # A little parsing work: The project code is the first part of the fileSetId
+ project_code = fileset_id.split('.')[0]
+
+ # We'll want to know who sent the message:
+ analyst_name = getpass.getuser()
+
+ # now prep the event, use an RPC event to get data back from the system:
+ broadcaster = RPCEvent(profile=os.environ['CAPO_PROFILE'], exchange='archive.events', application='qa-script')
+
+ event_data = {
+ "user": analyst_name,
+ "message": "{}={}".format(fileset_id, status),
+ "logData": {
+ "projectCode": project_code,
+ "status": status,
+ "type": "calibration",
+ "fileset_id": fileset_id,
+ "markedDirectory": results_directory
+ }
+ }
+
+ if '' != notify_email:
+ event_data["logData"]["email"] = notify_email
+
+ broadcaster.send(event_data)
+
+ # Now get the reply: (uncomment when that part is done)
+ ingestion_data = broadcaster.get_reply()
+
+
+ directory = None
+ log_id = None
+
+ if "working_directory" in ingestion_data:
+ directory = ingestion_data["working_directory"]
+
+ if "logging_id" in ingestion_data:
+ log_id = ingestion_data["logging_id"]
+
+ if "PASS" == status:
+ workflow_name = 'Calibration Ingestion'
+ else:
+ workflow_name = 'Qa Cleanup'
+
+ print("{} ({}) is running in {}".format(workflow_name, log_id, directory))
+
+ return
+
+
+def qa_pass():
+ qa_list("PASS")
+
+
+def qa_fail():
+ qa_list("FAIL")
diff --git a/apps/cli/executables/pexable/ingest/pyat/vlba_grabber/__init__.py b/apps/cli/executables/pexable/ingest/pyat/vlba_grabber/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/cli/executables/pexable/ingest/pyat/vlba_grabber/ngas_retriever.py b/apps/cli/executables/pexable/ingest/pyat/vlba_grabber/ngas_retriever.py
new file mode 100644
index 000000000..8f15c23f1
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/vlba_grabber/ngas_retriever.py
@@ -0,0 +1,171 @@
+"""
+ A simple, low level, ngas retriever with verification.
+
+ Given a file, and a location, interact with NGAS to find the server, and initiate a direct copy from NGAS
+ and verify that the transfer was completed appropriately.
+
+"""
+import os
+import sys
+import requests
+import argparse as ap
+import logging
+import logging.handlers
+
+from pyat import version
+from pyat.pymygdala import LogHandler
+from pycapo import CapoConfig
+from bs4 import BeautifulSoup
+
+# Constants: Might want to become CAPO properties.
+_NGAS_MASTER_NODE = 'aocngas-master'
+_NGAS_PORT = 7777
+
+_DESCRIPTION = """AAT/PPI uitility for low-level NGAS interactions. Version {}
+ This tool interacts with the NGAS HTTP interface to facilitate the transfer of a file to disk."""
+
+# set up logging:
+logger = logging.getLogger("ngas_retriever")
+logger.setLevel(logging.DEBUG)
+
+
+def _make_parser():
+ r""" Build a command line and/or argument parser """
+
+ result = ap.ArgumentParser(description=_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. test, production')
+ result.add_argument('-d', '--destination', type=str, action='store', default='',
+ help='Location to place the file (default is current working directory)')
+ # result.add_argument('-e', '--email', type=str, action='store', default='', help='Email for completion notification')
+ result.add_argument('filename', type=str, action='store',
+ help='NGAS file id to retrieve.')
+ return result
+
+# A typical response we need to extract information from:
+#
+# <?xml version="1.0" encoding="UTF-8"?>
+# <!DOCTYPE NgamsStatus SYSTEM "http://aocngas-master.aoc.nrao.edu:7777/RETRIEVE?internal=ngamsStatus.dtd">
+# <NgamsStatus>
+# <Status CompletionTime="2018-10-24T11:27:24.637" Date="2018-10-24T11:27:24.629" HostId="aocngas-master" Message="Successfully handled command STATUS" RequestId="4150" RequestTime="2018-10-24T11:27:24.623" State="ONLINE" Status="SUCCESS" SubState="IDLE" Version="v4.2.2-ALMA/2013-12-101T16:00:00"/>
+# <DiskStatus Archive="NRAO-ARCHIVE" AvailableMb="83" BytesStored="1091949384027" Checksum="" Completed="1" CompletionDate="2010-08-02T09:17:17.368" DiskId="3c790665f4f353cb2ae19faaa85ea754" HostId="aocngas-4" InstallationDate="2009-12-04T13:23:21.957" LastCheck="" LogicalName="NRAO-ARCH-M-000184" Manufacturer="Western Digital" MountPoint="/NGAS/volumes/Volume01" Mounted="1" NumberOfFiles="33445" SlotId="Volume01" TotalDiskWriteTime="19938" Type="raid1_part2">
+# <FileStatus AccessDate="" Checksum="711574960" ChecksumPlugIn="ngamsGenCrc32" Compression="" CreationDate="2010-02-11T10:35:02.000" FileId="uid___evla_sdm_X1265909221067.sdm" FileName="data/2010-02-11/1/uid___evla_sdm_X1265909221067.sdm" FileSize="395" FileStatus="00000000" FileVersion="1" Format="application/x-sdm" Group="" Ignore="0" IngestionDate="2010-02-11T10:35:02.032" ModificationDate="" Owner="" Permissions="" Tag="" UncompressedFileSize="395"/>
+# </DiskStatus>
+# </NgamsStatus>
+
+def perform_preliminary_lookup(filename):
+
+ ## Eventually, we'll want the master node & ports as CAPO properties: config = CapoConfig()
+
+ status_url = "http://{}.aoc.nrao.edu:{}/STATUS?file_id={}".format(_NGAS_MASTER_NODE, _NGAS_PORT, filename)
+
+ status_info = requests.get(status_url)
+
+ if requests.codes.ok == status_info.status_code:
+
+ # read and parse out the HostId, FileSize (maybe FileStatus?)
+ xml_content = status_info.text
+ response_handle = BeautifulSoup(xml_content, "lxml-xml")
+ # print(xml_content)
+
+ disk_handle = response_handle.NgamsStatus.DiskStatus
+ hostname = disk_handle.get("HostId")
+
+
+ file_handle = response_handle.find("FileStatus")
+
+ file_status = file_handle.get('FileStatus')
+ file_size = file_handle.get('FileSize')
+
+ return hostname, file_size
+
+ logger.error(f"Got code {status_info.status_code} from NGAS, cannot proceed.")
+
+ # If something went wrong, indicate that by the return value
+ return None, None
+
+
+def retrieve_file(filename, server, destination, file_size):
+ #
+ # This is a pure 'yank and put', no renaming or anything fancy.
+ #
+
+ retrieve_url = "http://{}.aoc.nrao.edu:{}/RETRIEVE".format(server, _NGAS_PORT)
+
+ # The details of the retrieval request
+ parameters = {
+ 'file_id':filename,
+ 'processing':'ngamsDirectCopyDppi',
+ 'processingPars':'outfile='+destination+'/'+filename
+ }
+
+ copy_response = requests.get(retrieve_url, params=parameters)
+
+ if requests.codes.ok == copy_response.status_code:
+
+ # xml_content = copy_response.text
+ # response_handle = BeautifulSoup(xml_content, "lxml-xml")
+
+ logger.info("Success!")
+
+ # Unfortunately, the return from the direct copy call doesn't provide us much:
+ #
+ # {'Server': 'NGAMS/v4.2.2-ALMA/2013-12-101T16:00:00', 'Date': 'Wed, 24 Oct 2018 21:44:43 GMT', 'Expires': 'Wed, 24 Oct 2018 21:44:43 GMT', 'Content-type': 'application/x-bdf', 'Content-length': '74', 'Content-disposition': 'attachment; filename=""', 'Accept-Ranges': 'bytes'}
+ # /NGAS/volumes/Volume32/data/2018-10-23/1/uid____evla_bdf_1540297250925.bdf
+ else:
+ logger.error(f"Direct Copy Call failed with code: {copy_response.status_code}")
+
+ xml_content = copy_response.text
+ response_handle = BeautifulSoup(xml_content, "lxml-xml")
+
+ status_handle = response_handle.NgamsStatus.Status
+ error_message = status_handle.get("Message")
+ logger.error(f"Error details: {error_message}")
+
+ logger.info(f"{filename} retrieval complete.")
+
+
+def ngas_retriever():
+ """The heart of the retrieval process """
+
+ parser = _make_parser()
+
+ arguments = parser.parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == arguments.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != arguments.profile:
+ os.environ['CAPO_PROFILE'] = arguments.profile
+
+ # integrate the logs with pymygdala's logging system
+ handler = LogHandler(profile=arguments.profile, application='aoc_ngas_retriever')
+ handler.setLevel(logging.DEBUG)
+ logger.addHandler(handler)
+
+
+ if '' == arguments.destination:
+ delivery_location = os.getcwd()
+ else:
+ delivery_location = arguments.destination
+
+ if delivery_location.endswith('/'):
+ # remove any trailing / for consistency:
+ delivery_location = delivery_location[:-1]
+
+ server, file_size = perform_preliminary_lookup(arguments.filename)
+
+ # If we don't have the info we need, exit with an error:
+ if (None == server) or (None == file_size):
+ exit(-10)
+
+ logger.info(f"We want to get {arguments.filename} from {server}, and we expect {file_size} bytes")
+
+ retrieve_file(arguments.filename, server, delivery_location, file_size)
+
+if __name__ == "__main__":
+ ngas_retriever()
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/pyat/wf/__init__.py b/apps/cli/executables/pexable/ingest/pyat/wf/__init__.py
new file mode 100644
index 000000000..b94ad2c78
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/wf/__init__.py
@@ -0,0 +1,3 @@
+r""" wf: utility for running workflows
+"""
+from pyat.wf.commands import wf, WorkflowTracker
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/pyat/wf/commands.py b/apps/cli/executables/pexable/ingest/pyat/wf/commands.py
new file mode 100644
index 000000000..f857a2778
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/wf/commands.py
@@ -0,0 +1,497 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import argparse as ap
+import json
+import logging
+import os
+
+import pika
+import sys
+import time
+
+from pyat import version
+from pyat.pymygdala import LogHandler, RPCEvent
+from pycapo import CapoConfig
+
+_DESCRIPTION = """AAT/PPI workflow launcher, version {}. Launch a workflow from
+the command line."""
+
+
+def _make_parser():
+ r""" Build a command line parser for this app: this is external to MrClean and/or CapoSettings because both need
+ it, and Sphinx will want a look at ot to build docs. """
+ result = ap.ArgumentParser(description=_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store',
+ help='profile name to use, e.g. test, production')
+ result.add_argument('-e', '--exchange', action='store', default='archive.workflow-commands',
+ help='exchange to send to (you probably want the default)')
+ result.add_argument('-s', '--processingsite', action='store', default='DSOC', choices=['NAASC', 'DSOC'],
+ help='site where processing should occur (either DSOC or NAASC)')
+ result.add_argument('-p', '--property', action='append', default=[],
+ help="provide a KEY=VALUE pair to be passed to the workflow; can be used multiple times")
+ result.add_argument('workflow', action='store', help='name of the workflow to initiate; e.g. runDownloadWorkflow, runSdmIngestionWorkflow')
+ return result
+
+def wf(**kwargs):
+ # parse out the command line args here and decide what to do
+ args = _make_parser().parse_args(**kwargs)
+
+ connection = connect_to_amqp(args.profile)
+ channel = connection.channel()
+
+ # make the exchange we need
+ channel.exchange_declare(args.exchange, 'topic', durable=True)
+
+ #process the properties passed in
+ properties = dict(arg.split('=') for arg in args.property)
+
+ # if it's in there, fileSetIds needs some special care.
+ if 'fileSetIds' in properties:
+ #The script needs to pass an array in the json. This drop spare square brackets
+ #and split according to commas to create a list which is then decoded properly
+ middleman = properties['fileSetIds']
+ #replace any square braces in the input
+ idList = middleman.replace('[','').replace(']','').split(',')
+ #make fileSetIds a list of strings, not a string with a list
+ properties['fileSetIds'] = idList
+
+ # if it's in there, the bdf list needs some special care.
+ if 'bdfs' in properties:
+ #The script needs to pass an array in the json. This drop spare square brackets
+ #and split according to commas to create a list which is then decoded properly
+ middleman = properties['bdfs']
+ #replace any square braces in the input
+ idList = middleman.replace('[','').replace(']','').split(',')
+ #make fileSetIds a list of strings, not a string with a list
+ properties['bdfs'] = idList
+
+ # The sets of imaging parameters need to be a JsonArray, so we have work to do:
+ if 'optimizedImagingParameters' in properties:
+ cube_list = []
+ json_parameters = json.loads(properties['optimizedImagingParameters'])
+
+ # create a list of cubes, which will get interpreted correctly:
+ for cube in json_parameters:
+ cube_list.append(cube)
+
+ # replace the old string we were passed with the new value
+ properties['optimizedImagingParameters'] = cube_list
+
+ # build the message
+ message = {'eventName': args.workflow,
+ 'type': 'edu.nrao.archive.workflow.messaging.commands.StartWorkflow',
+ 'additionalPaths': [],
+ 'metadata': properties}
+
+ # dispatch it
+ channel.basic_publish(exchange=args.exchange,
+ routing_key=args.processingsite.lower() + '.start',
+ body=json.dumps(message))
+
+
+def connect_to_amqp(profile):
+ # load up Capo
+ capo = CapoConfig(profile=profile)
+
+ # get AMQP settings
+ hostname = capo.getstring(
+ 'edu.nrao.archive.configuration.AmqpServer.hostname')
+ username = capo.getstring(
+ 'edu.nrao.archive.configuration.AmqpServer.username')
+ password = capo.getstring(
+ 'edu.nrao.archive.configuration.AmqpServer.password')
+
+ # connect
+ connection = pika.BlockingConnection(
+ pika.ConnectionParameters(hostname,
+ credentials=pika.PlainCredentials(username,
+ password)))
+ return connection
+
+
+#
+# Helper class for tracking a workflow:
+#
+class WorkflowTracker:
+ """An encapsulation of using an RPCEvent object to generate an ID
+ and receive the working directory and loggingId for a workflow
+ launched via wf.
+
+ Usage:
+ my_tracker = new WorkflowTracker(current_profile)
+ uid_value = my_tracker.prepareWorkflowTracking("runMyWorkflow", <optional email address for completion>)
+
+ # create a wf call as usual, but include another key in the metadata:
+ wf_arguments += " -p cliCorrId=" + uid_value
+ # send the wf command
+
+
+ my_tracker.waitForReply()
+
+ wd_name = my_tracker.getWorkingDirectory()
+ lid = my_tracker.getLoggingId()
+
+ print("Yadda, yadda, {} yadda {}".format(wd_name,lid))
+
+
+ """
+
+ def __init__(self, profile):
+ self._rpc = RPCEvent(profile=profile, exchange='archive.events', application='commandline')
+
+ self._work_dir_name = None
+ self._log_id = None
+
+ def close(self):
+ self._rpc.close()
+
+ def prepareWorkflowTracking(self, workflow_name, email=None):
+
+ # Create the event: Versions with and without an email.
+ if email is None:
+ event = {
+ "message": "Tracking Request for" + workflow_name,
+ "request": "monitor",
+ "logData": {
+ "workflowId": workflow_name
+ }
+ }
+ else:
+ event = {
+ "message": "Tracking Request for " + workflow_name,
+ "request": "monitor",
+ "logData": {
+ "workflowId": workflow_name,
+ "email": email
+ }
+ }
+
+ tracking_key = self._rpc.send(event)
+ return tracking_key
+
+ def waitForReply(self):
+ # Now, wait for the initial processing and populate our variables
+ # This will block until the direct reply arrives.
+ answer = self._rpc.get_reply()
+
+ if "working_directory" in answer:
+ self._work_dir_name = answer["working_directory"]
+
+ if "logging_id" in answer:
+ self._log_id = answer["logging_id"]
+
+ def getWorkingDirectory(self):
+ return self._work_dir_name
+
+ def getLoggingId(self):
+ return self._log_id
+
+#
+# Specific WF usages
+#
+
+
+# CIPL CLI
+_RUN_CIPL_DESCRIPTION = """AAT/PPI Calibration Launcher, version {}: This tool initiates the standard CASA Calibration
+Pipeline on the indicated execution blocks. Note: This tool ignores the ciplRunState Capo property. It will always
+ start the requested calibrations."""
+
+
+def _make_cipl_parser():
+ r"""Command-line CIPL Launch Parser"""
+ result = ap.ArgumentParser(description=_RUN_CIPL_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. nmtest, mnprod')
+ result.add_argument('-C', '--casa', action='store', default='',
+ help='Path to Casa package to use')
+ result.add_argument('fileSetIds', type = str, action='store', nargs='+', help='File Set Identifiers to be run through CIPL')
+ return result
+
+
+def launch_calibrations():
+ r"""Take given fileSetId(s) and launch the CIPL workflow for each"""
+
+ #parse our arguments, get profile and list of Ids
+ args = _make_cipl_parser().parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ #
+ # Command line we want is:
+ #
+ # wf -P [test|nmprod] -p workflowName=CiplCalibration -p fileSetIds=arg runCiplCalibrationWorkflow
+ # or
+ # wf -P [nmtest|nmprod] -p workflowName=CiplCalibration -p casaHome=arg -p fileSetIds=arg runCiplCalibrationWorkflow
+ #
+ # The pieces of the string
+ clBase = "-p workflowName=CiplCalibration -p processingSite=DSOC -P " #profile here
+ clFSIDPiece = " -p fileSetIds=" # fileSetId here
+ clTrackPiece = " -p cliCorrId=" # tracking UID here
+ if '' != args.casa:
+ clCasaPiece = " -p casaHome=" + args.casa
+ # insert the casa parameter here, since it is fixed for all fileSetIds
+ clFSIDPiece = clCasaPiece + clFSIDPiece
+
+ clFinish = " runCiplCalibrationWorkflow"
+
+ # create a tracking object for these workflows:
+ ranger = WorkflowTracker(os.environ['CAPO_PROFILE'])
+
+ #
+ # Loop over the provided fileSetIds and send a command for each,
+ # waiting for the previous one to report back information before
+ # starting the next.
+ #
+ for id in args.fileSetIds:
+
+ # CIPL itself send emails at start & finish, so it'd be redundant here
+ this_uid = ranger.prepareWorkflowTracking('runCiplCalibrationWorkflow')
+
+ wfArgs = clBase + os.environ['CAPO_PROFILE'] + clFSIDPiece + id + clTrackPiece + this_uid + clFinish
+ wf(args=wfArgs.split(' '))
+
+ # Wait for information from this workflow before we proceed:
+ ranger.waitForReply()
+
+ print("CIPL ({}) for {} running in: {}".format(ranger.getLoggingId(), id, ranger.getWorkingDirectory()))
+
+
+# CIPL Reprocess CLI
+_RUN_CIPL_REPRO_DESCRIPTION = """AAT/PPI Calibration Reprocessing, version {}: This tool initiates a re-run of the CASA
+calibration pipeline on the desired directories. The data are placed in the Spool for the processing and returned to the
+QA area upon successful completion."""
+
+
+def _make_cipl_reprocess_parser():
+ r"""Parser"""
+ result = ap.ArgumentParser(description=_RUN_CIPL_REPRO_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. test, production')
+ result.add_argument('-C', '--casa', action='store', default='',
+ help='Path to Casa package to use')
+ result.add_argument('-m', '--main', action='store_true', help='Work in the main spool directory, not the cipl subdirectory')
+ result.add_argument('directoriesOfInterest', type = str, action='store', nargs='+', help='Name of the directories to be re-run through CIPL')
+ return result
+
+
+def launch_recalibrations():
+ """From a (list of) directory(s) launch the CIPL Reprocessing workflow for each"""
+
+ #parse our arguments, get profile and list of Ids
+ args = _make_cipl_reprocess_parser().parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+
+ #
+ # Command line we want is:
+ # (processingSite for message routing)
+ #
+ # wf -P [nmtest|nmprod] -p workflowName=CiplReprocess -p processingSite=DSOC -p targetDirectory=arg runCiplReprocessingWorkflow
+ #
+ # The pieces of the string
+ clBase = "-p workflowName=CiplReprocess -p processingSite=DSOC -P " #profile here
+ clArgPiece = " -p targetDirectory=" #fileSetId here
+ if '' != args.casa:
+ clCasaPiece = " -p casaHome=" + args.casa
+ # insert the casa parameter here, since it is fixed for all fileSetIds
+ clArgPiece = clCasaPiece + clArgPiece
+ clArg2Piece = " -p useMainDirectory=true"
+ clFinish = " runCiplReprocessingWorkflow"
+ #clFinish = " runTestWorkflow"
+ #
+ # Loop over the provided directories and send a command for each:
+ #
+ for id in args.directoriesOfInterest:
+ #add the extra -p call if we're using the super-directory
+ if(args.main):
+ totalCommand = clBase+ os.environ['CAPO_PROFILE'] +clArgPiece+ id +clArg2Piece+clFinish
+ else:
+ totalCommand = clBase+ os.environ['CAPO_PROFILE'] +clArgPiece+ id +clFinish
+ wf(args=totalCommand.split(' '))
+ time.sleep(0.25) # 1/4 second deplay between workflow launches for saftey
+
+
+# SDM reingestion CLI
+_RUN_SDM_REINGEST_DESCRIPTION = """AAT/PPI SDM Re-ingestion, version {}: This tool initiates a download of the requested
+Science Data Models, following which the files are re-parsed for an update of the Archive metadata database."""
+
+
+def _make_sdm_reingest_parser():
+ result = ap.ArgumentParser(description=_RUN_SDM_REINGEST_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store',
+ help='profile name to use, e.g. test, production')
+ result.add_argument('fileSetId', type = str, action='store', nargs='+',
+ help='Name of the fileset whose SDM is to be reingested')
+ return result
+
+
+def launch_reingest():
+ """Given a fileset ID, pull the (presumably altered by QA) SDM and re-run ingestion """
+
+ # TODO once this is working:
+ # - accept a list of fileset IDs
+ # - accept a telescope name
+
+ args = _make_sdm_reingest_parser().parse_args()
+
+ # command line:
+ # Test cases:
+ # 1. no SDM source directory supplied
+ # -- should ultimately fail if data not present under archive-ingestion.SDMPath
+ # wf -P [test|nmprod] -p workflowName=Reingestion -p telescope=EVLA \
+ # -p dataSetId=<FSID> runSdmReingestWorkflow
+ # example: wf -P test -p workflowName=Reingestion -p telescope=EVLA \
+ # -p dataSetId=3_4.sb31082669.eb31333837.57311.64081266204 \
+ # -p fileSetIds=[3_4.sb31082669.eb31333837.57311.64081266204] \
+ # runSdmReingestWorkflow
+ # 2. user provides SDM source directory
+ # wf -P [test|nmprod] -p workflowName=Reingestion -p telescope=EVLA -p dataSetId=<FSID> \
+ # -p SDMPath=</path/to/SDM> runSdmIngestionWorkflow
+ # example: wf -P test -p workflowName=Reingestion -p telescope=EVLA \
+ # -p dataSetId=3_4.sb31082669.eb31333837.57311.64081266204 \
+ # -p fileSetIds=[3_4.sb31082669.eb31333837.57311.64081266204] \
+ # -p SDMPath=/lustre/aoc/cluster/pipeline/test/spool/3_4_2015_10_16_T21_58_09.207/rawdata \
+ # runSdmReingestWorkflow
+
+ reArg0 = "-p dataSetId="
+ reArg1 = "-p fileSetIds=["
+ reArg2 = "] -p workflowName=Reingestion "
+ reArg3 = "-p telescope=EVLA"
+ reLast = " runSdmReingestWorkflow"
+ command_line = "-P " + args.profile + reArg0 + args.fileSetId + reArg2
+ if len(args) < 5:
+ # the usual ingestion case: grab SDM from standard archive location
+ command_line += reArg3
+ else:
+ # manual or auto re-ingest: grab SDM from user-supplied location0
+ # (use case: SDM was corrupt and fixed manually, now needs to be reingested)
+ reArg4 = " -p SDMPath=" + args.sdmSubdir
+ command_line = "-P " + args.profile + reArg0 + args.fileSetId + reArg1 + args.fileSetId + reArg2 + reArg3 + reArg4
+ command_line += reLast
+
+ LOG = logging.getLogger(__name__)
+ handler = LogHandler(profile='test', application='test-app',
+ level=logging.DEBUG)
+ LOG.addHandler(handler)
+ #OG.debug("Using this command line: {}" %command_line)
+ # in case that syntax is wrong:
+ LOG.debug("Using command line '%s'" %command_line)
+
+ wf(args=command_line.split(' '))
+
+
+# RestoreToCache Workflow CLI
+_RUN_RESTORE_DESCRIPTION = """AAT/PPI Restore to Cache, version {}: This tool takes a FileSetId, and the name of a
+calibration products tar file. It will then initiate a CASA Restore Workflow. The resultant Calibrated Measurement Set
+is placed into the Cache as defined by the Capo settings provided."""
+
+
+def _make_r2c_parser():
+ r"""Command-line CIPL Launch Parser"""
+ result = ap.ArgumentParser(description=_RUN_RESTORE_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. nmtest, mnprod')
+ result.add_argument('-C', '--casa', action='store', default='',
+ help='Path to Casa package to use')
+ result.add_argument('-E', '--email', action='store', default=None,
+ help='Optional email address for a completion message')
+ # Placeholder, workflow can't do this yet.
+ # result.add_argument('-D', '--destination', action='store', default='', help='Override location for CMS delivery')
+ result.add_argument('fileSetId', type = str, action='store', help='File Set Identifier to be Restored')
+ result.add_argument('calFile', type=str, action='store', help='Calibration File to use in the Restore')
+ return result
+
+
+def launch_restore_to_cache():
+ r"""Take given fileSetId(s) and launch the CIPL workflow for each"""
+
+ # parse our arguments, get profile and list of Ids
+ args = _make_r2c_parser().parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ # Obtain the Project Code from the fileSetId value (careful of VLASSA.B format)
+ fsid = args.fileSetId
+ if fsid.startswith("VLASS"):
+ components = fsid.split('.')
+ projectCode = components[0] + '.' + components[1] # Grab the first two substrings
+ else:
+ projectCode = fsid.split('.')[0] # Only up to the first .
+
+ # Send an event to initiate workflow monitoring:
+ my_tracker = WorkflowTracker(args.profile)
+ monitor_key = my_tracker.prepareWorkflowTracking("runRestoreWorkflow", email=args.email)
+
+ # Launch the workflow
+ launch_restore(args, projectCode, monitor_key)
+
+ # Now wait for information:
+ my_tracker.waitForReply()
+
+ print("Restore ({}) running in: {}".format(my_tracker.getLoggingId(), my_tracker.getWorkingDirectory()))
+
+
+def launch_restore(args, project_code, tracking_key):
+ #
+ # An Example Command Line from workflow testing:
+ #
+ # wf -p processingSite=DSOC -p workflowName=RestoreWorkflow -p telescope=EVLA
+ # -p calFile=16A-439_2016_10_09_T14_48_37.675.tar
+ # -p fileSetIds=16A-439.sb32267588.eb32942323.57670.592283148144
+ # -p projectCodeOrDataType=16A-439
+ # -p deliveryFormat=CMS runRestoreWorkflow
+ #
+ # Build the wf command line, starting with all the fixed pieces:
+ arguments = "-P " + os.environ['CAPO_PROFILE']
+ # get the exchange from our requested profile (since we might be sending this to the non-default manager)
+ configuration = CapoConfig(profile=os.environ['CAPO_PROFILE'])
+ arguments += " -e " + configuration.get('edu.nrao.archive.workflow.config.WorkflowManagerSettings.workflowCommandsExchangeName')
+ # Required Parameters
+ arguments += " -p workflowName=RestoreWorkflow -p processingSite=DSOC -p deliveryFormat=CMS -p telescope=EVLA"
+ arguments += " -p fileSetIds=" + args.fileSetId
+ arguments += " -p calFile=" + args.calFile
+ arguments += " -p projectCodeOrDataType=" + project_code
+ arguments += " -p cliCorrId=" + tracking_key
+ # The Optional Parameters:
+ if '' != args.casa:
+ arguments += " -p casaHome=" + args.casa
+ # Option not available yet, so don't display or use it
+ # if '' != args.destination:
+ # arguments += " -p deliveryDirectory=" + args.destination
+ # The Finale
+ arguments += " runRestoreWorkflow"
+ # Build our command line and execute:
+ wf(args=arguments.split(' '))
+ # Talk to the user, these CLIs are bad on feedback
+ print("Restoring a CMS of {} to the cache using calibration {}.\n".format(args.fileSetId, args.calFile))
+
+
diff --git a/apps/cli/executables/pexable/ingest/pyat/wf/ingest_wf_interfaces.py b/apps/cli/executables/pexable/ingest/pyat/wf/ingest_wf_interfaces.py
new file mode 100644
index 000000000..6c51577ce
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/wf/ingest_wf_interfaces.py
@@ -0,0 +1,596 @@
+#
+# With the expansion of the number of command line tools built within the wf/commands.py file, it's getting a bit
+# difficult to find what you need. This file contains the various CLI interfaces to the ingestion-focused workflows.
+#
+import argparse as ap
+import os
+import sys
+import time
+import logging
+
+
+from pyat.wf import wf, WorkflowTracker
+from pathlib import Path
+from pyat import version
+from pycapo import CapoConfig
+from pyat.wf.ous_wf_interfaces import lookup_single_locator, lookup_project_code, lookup_eb_uid
+
+# Ingestion Workflow CLIs
+def _make_ingestion_parser(ingestion_description, major_arg):
+ r"""Shared Command-line Ingestion Launch Parser"""
+ result = ap.ArgumentParser(description=ingestion_description.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. nmtest, mnprod')
+ result.add_argument('-p', '--path', action='store', default='',
+ help='Path to data (overrides CAPO setting)')
+ result.add_argument('fileInfo', type = str, action='store', nargs='+', help=major_arg)
+ return result
+
+
+def launch_sdm_ingestion():
+ r"""CLI for SDM Ingestion Workflow"""
+ sdm_description = """Manual SDM Ingestion, version {}: Initiates an SDM Ingestion workflow for each listed execution
+ block."""
+ sdm_argument = 'FileSet Identifiers(s) to ingest'
+ parser = _make_ingestion_parser(sdm_description, sdm_argument)
+ args = parser.parse_args()
+
+ # Automatic Profile Determination:
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ #
+ # Command line we want is:
+ #
+ # wf -P [test|nmprod] -p workflowName=SdmIngestion -p dataSetId=arg runSdmIngestionWorkflow
+ # or
+ # wf -P [nmtest|nmprod] -p workflowName=SdmIngestion -p SDMPath=arg -p datasetId=arg runSdmIngestionWorkflow
+ #
+ # The pieces of the string
+ clBase = "-p workflowName=SdmIngestion -p processingSite=DSOC -P " #profile here
+ clArgPiece = " -p datasetId=" #fileSetId here
+ if '' != args.path:
+ clPathPiece = " -p SDMPath=" + args.path
+ # insert the casa parameter here, since it is fixed for all fileSetIds
+ clArgPiece = clPathPiece + clArgPiece
+
+ clFinish = " runSdmIngestionWorkflow"
+
+
+ #
+ # Loop over the provided fileSetIds and send a command for each:
+ #
+ for id in args.fileInfo:
+ wfArgs = clBase + os.environ['CAPO_PROFILE'] + clArgPiece + id + clFinish
+ #print("wf "+wfArgs)
+ wf(args=wfArgs.split(' '))
+ time.sleep(0.25) # 1/4 second delay between workflow launches for saftey
+
+
+def launch_bdf_ingestion():
+ r"""CLI for BDF Ingestion Workflow"""
+ bdf_description = """Manual BDF Ingestion, version {}: Launches a BDF ingestion workflow for the listed files."""
+ bdf_argument = 'BDF File(s) to Ingest'
+ parser = _make_ingestion_parser(bdf_description, bdf_argument)
+ args = parser.parse_args()
+
+ # Automatic Profile Determination:
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ #
+ # Command line we want is:
+ #
+ # wf -P [test|nmprod] -p workflowName=SdmIngestion -p bdfs=[list from arg] runBdfIngestionWorkflow
+ # or
+ # wf -P [nmtest|nmprod] -p workflowName=SdmIngestion -p SDMPath=arg -p bdfs=[list from arg] runBdfIngestionWorkflow
+ #
+ # The pieces of the string
+ clBase = "-p workflowName=BdfIngestion -p processingSite=DSOC -P " #profile here
+ clArgPiece = " -p bdfs=" #fileSetId here
+ if '' != args.path:
+ clPathPiece = " -p BDFPath=" + args.path
+ # insert the casa parameter here, since it is fixed for all fileSetIds
+ clArgPiece = clPathPiece + clArgPiece
+
+ clFinish = " runBdfIngestionWorkflow"
+
+ # We need to massage the list of filenames into a string without spaces
+ temp_string = str(args.fileInfo)
+ bdf_string = temp_string.replace(' ', '')
+
+ # Put it all together (all files in one command:
+ wf_arguments = clBase + os.environ['CAPO_PROFILE'] + clArgPiece + bdf_string + clFinish
+ #print("wf "+wf_arguments)
+ wf(args=wf_arguments.split(' '))
+
+
+def launch_idifits_ingestion():
+ r"""CLI for VLBA IDI Fits file ingestion"""
+ idi_description = """Manual VLBA IDI FITS Ingestion, version {}: Initiates an ingestion workflow for each idi fits
+ file in the named directory."""
+ idi_argument = 'Folder named for the project & segment to process (all contained IDIFITS files will be ingested)'
+ parser = _make_ingestion_parser(idi_description, idi_argument)
+ args = parser.parse_args()
+
+ # Automatic Profile Determination:
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ # We need our full path, so check for the -p override, or extract it from CAPO
+ settings = CapoConfig(profile=os.environ['CAPO_PROFILE'])
+ path_base_string = settings.getstring('archive-ingestion.VLBAPath')
+ if '' != args.path:
+ path_base_string = args.path
+
+ base_ingestion = Path(path_base_string)
+
+ # A couple checks:
+ if not base_ingestion.exists():
+ logging.error(f"Base ingestion path {path_base_string} doesn't exist!")
+ exit(-1)
+ if not base_ingestion.is_dir() or not base_ingestion.is_absolute():
+ logging.error(f"Base ingestion path is inappropriate: {path_base_string}")
+ exit(-1)
+
+
+ #
+ # So, for each directory we're given, we need to:
+ #
+ # 1) extract the legacy ID to provide the workflow
+ # 2) obtain a listing of the fits files, and launch a workflow to ingest each one.
+ #
+ for directory_name in args.fileInfo:
+ # The directories are named in a pattern, which makes my life easier:
+ # (ProjectCode)(Segment).(CorrelatorPass)
+ # The project code (what we call LegacyId) follows AA111
+ # The segment follows A[0-9]*
+ #directory_components = directory_name.split(".")
+ # legacy id is the first 5 characters of the first piece, which is guaranteed to exist from split()
+ #legacy_id = directory_components[0][:5]
+ # print(f"The Legacy id for {directory_name} is {legacy_id}.")
+
+ #
+ # Now get the filenames:
+ #
+ ingestion_path = base_ingestion / directory_name
+
+ files = ingestion_path.glob("*.idifits")
+ for file in files:
+ print(f"\tLaunching workflow to ingest: {file.name} from {str(ingestion_path)}")
+
+ #
+ # The files have names in the format:
+ # VLBA_DQ840_dq840_BIN0_SRC0_02_181016T191428.idifits*
+ #
+ # VLBA_[PC][Segment]_.......
+ # We can use this to extract the project code on a per-file basis.
+ # This is safer for a bulk-ingestion of legacy data.
+ name_components = file.name.split("_")
+
+ if 3 >= len(name_components):
+ # if we don't have the requisite number of pieces, then skip it!
+ print(f"Error! File {file.name} has only {len(name_components)} subsections.")
+ continue
+
+ legacy_id = name_components[1][:5]
+ #print(f"Legacy id for {file.name} is {legacy_id}.")
+
+ #
+ # Command line we want to emulate:
+ #
+ # wf -P nmtest
+ # -p ingestionPath=/home/e2earchive/hold/difx/BD215G3.bd215g3.cata
+ # -p subDirOrFileName=VLBA_BD215G3_bd215g3_BIN0_SRC1_0_181001T142116.idifitsVLBA_BD215G3_bd215g3_BIN0_SRC1_0_181001T142116.idifits
+ # -p projectCode=BD215 -p ingestionType=vlba -p workflowName=VlbaIngestion runVlbaIngestionWorkflow
+ #
+ # The pieces of the string
+ cl_base = "-p workflowName=VlbaIngestion -p ingestionType=vlba -s DSOC -P " #profile here
+ cl_arg_piece = " -p subDirOrFileName=" + file.name
+ cl_path_piece = " -p ingestionPath=" + str(ingestion_path)
+ cl_code_piece = " -p projectCode=" + legacy_id
+ cl_finish = " runVlbaIngestionWorkflow"
+
+ total_command = cl_base + os.environ['CAPO_PROFILE'] + cl_arg_piece + cl_path_piece + cl_code_piece + cl_finish
+ # print(f"\tFor WF: {total_command}")
+
+ wf(args=total_command.split(' '))
+ time.sleep(1.25) # Wait a bit between workflow launches
+
+ print(f"Finished with {directory_name}.\n")
+
+
+#
+# Image ingestion is a slightly different beast: You point to a directory, and we collect the FITS files
+# and bundle the ancillary files into a tar bundle for ingestion.
+#
+# For image reingestion, you assume the directory already has just the FITS files and the tar bundle.
+#
+def _make_image_ingestion_parser(ingestion_description):
+ r"""Shared Command-line Ingestion Launch Parser"""
+ result = ap.ArgumentParser(description=ingestion_description.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. nmtest, mnprod')
+ result.add_argument('projectCode', action='store', type=str, help='Project for these Images')
+ result.add_argument('imageSetLoc', action='store', type=str, nargs='+',
+ help='Directory from which to draw FITS images & ancillary file(s)')
+ return result
+
+
+def launch_image_ingestion():
+ r"""CLI for the Image Ingestion Workflow"""
+ image_description = """Manual Image Ingestion, version {}: Initiates an image ingestion workflow
+ for each listed location (note, all sets ingested with a single command will
+ share the same project)"""
+ parser = _make_image_ingestion_parser(image_description)
+ args = parser.parse_args()
+
+ # Automatic Profile Determination:
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ #
+ # Command line we want is:
+ #
+ # wf -P [test|nmprod] -p workflowName=SdmIngestion -p dataSetId=arg runSdmIngestionWorkflow
+ # or
+ # wf -P [nmtest|nmprod] -p workflowName=SdmIngestion -p SDMPath=arg -p datasetId=arg runSdmIngestionWorkflow
+ #
+ # The pieces of the string
+ clBase = "-p workflowName=QuicklookImageIngestion -p processingSite=DSOC -p ingestionType=vlass_quicklook -P " #profile here
+ clProjectPiece = " -p projectCode="+args.projectCode
+ clArgPath = " -p ingestionPath="
+ clArgPiece = " -p subDirOrFileName="
+ clFinish = " runQuicklookIngestionWorkflow"
+
+
+ #
+ # Loop over the provided directories and send a command for each:
+ #
+ for id in args.imageSetLoc:
+ id_as_path = Path(id)
+ subdirectory = str(id_as_path.name)
+ ingest_path = str(id_as_path.parent)
+ wfArgs = clBase + os.environ['CAPO_PROFILE'] + clProjectPiece + clArgPath + ingest_path + clArgPiece + subdirectory + clFinish
+ # print("wf "+wfArgs)
+ wf(args=wfArgs.split(' '))
+ time.sleep(0.25) # 1/4 second delay between workflow launches for saftey
+
+
+def launch_image_reingestion():
+ r"""CLI for the Image Ingestion Workflow"""
+ image_description = """Manual Image Metadat Reingestion, version {}: provided the location of an image set
+ (Fits files + tar) this workflow deletes the existing metadata & rebuilds the entries. A full cycle
+ reingestion workflow is possible, but not yet necessary. """
+ parser = _make_image_ingestion_parser(image_description)
+ args = parser.parse_args()
+
+ # Automatic Profile Determination:
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ #
+ # Command line we want is:
+ #
+ # wf -P [test|nmprod] -p workflowName=SdmIngestion -p dataSetId=arg runSdmIngestionWorkflow
+ # or
+ # wf -P [nmtest|nmprod] -p workflowName=SdmIngestion -p SDMPath=arg -p datasetId=arg runSdmIngestionWorkflow
+ #
+ # The pieces of the string
+ clBase = "-p workflowName=ImageReingestion -p processingSite=DSOC -P " #profile here
+ clProjectPiece = " -p projectCode="+args.projectCode
+ clArgPiece = " -p sourceDirectory="
+ clFinish = " runImageReingestionWorkflow"
+
+
+ #
+ # Loop over the provided directories and send a command for each:
+ #
+ for id in args.imageSetLoc:
+ wfArgs = clBase + os.environ['CAPO_PROFILE'] + clProjectPiece + clArgPiece + id + clFinish
+ # print("wf "+wfArgs)
+ wf(args=wfArgs.split(' '))
+ time.sleep(0.25) # 1/4 second delay between workflow launches for saftey
+
+
+_WORKFLOW_NAME = 'AoiIngestionWorkflow'
+
+_RUN_AOI_DESCRIPTION = """AAT/PPI ALMA Optimized Image Ingestion, version {}: This tool initiates AOI for the supplied
+ fileSetId(s). Upon successful processing, the request state is changed to PENDINGQA, and the user is notified
+ that the request awaits QA. When the data have passed QA, the data are delivered, request state is changed to
+ COMPLETE and the user sent a download link. """
+
+
+def find_mous_id(path_to_data):
+ r""" Somewhere in the path will be the external name,
+ either just below the top-level directory (if no request ID)
+ or underneath top-level-dir/request_id/.
+ """
+
+ external_name = None
+ filename = None
+ rawdata_dir = None
+ for base, dirs, files in os.walk(path_to_data):
+ for dir in dirs:
+ # A directory name corresponding to the mousId will have underscores rather than
+ # colon & slashes
+ if str(dir).endswith('rawdata'):
+ rawdata_dir = dir
+ break
+
+ if rawdata_dir is not None:
+ for base, dirs, files in os.walk(path_to_data):
+ # the ASA_OUS_ID will be hidden in the name of a data file;
+ # it -won't- be a directory name
+ for file in files:
+ if str(file).startswith("member."):
+ filename = str(file)
+ break
+
+ if filename is not None:
+ parts = filename.split('.')
+ external_name = parts[1]
+ break
+
+ if external_name is not None:
+ break
+
+ if external_name is not None:
+ # translate to OUS ID format
+ mousId = external_name.replace("___", "://").replace("_", "/")
+ return mousId
+
+ return None
+
+
+def find_working_subdirectory(request_path):
+ """
+ A utility for handling external requests. The tool is
+ provided a directory named for the request id, and we
+ need to provide the name of the actual working directory
+ underneath.
+
+ Note: for an externally-initiated AUDI there should be
+ a single subdirectory, if we find more than one, we've
+ got an issue.
+ :return: name of subdirectory (or None if an error occurs)
+ """
+
+ request_subdir = None
+
+ for component in request_path.iterdir():
+ if component.is_dir():
+ # we really only ought to have one...
+ if request_subdir is None:
+ request_subdir = component.name
+ else:
+ # if we've already filled in an answer
+ # and found another subdirectory....
+ # something's wrong.
+ return None
+
+ return request_subdir
+
+
+def launch_alma_image_ingest_qa():
+ r"""Launch runAoiIngestionWorkflow for data in given image-qa subdirectory
+ Based on ingest_wf_interfaces.launch_image_ingestion() -- thanks, Jim
+ """
+
+ # example command lines:
+ #
+ # audiPass uid___A002_Xd3607d_X2b0d_2018_10_16_T14_19_15.962 -E jgoldste@nrao.edu
+ # audiPass 299837780
+ #
+ # subdirectory is assumed to exist under AlmaImagingWorkflowSettings.qaDirectory
+ #
+
+ # parse our arguments; get profile and other info we need
+ args = _make_audi_pass_parser().parse_args()
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ capo_profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = capo_profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(capo_profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ capo_profile = os.environ['CAPO_PROFILE']
+ configuration = CapoConfig(profile=capo_profile)
+ top_level_dir = configuration.get('edu.nrao.archive.workflow.config.AlmaImagingWorkflowSettings.qaDirectory')
+
+ # stray '/' cause problems:
+ provided_qa_dir = args.sourceDirectory.replace('/', '')
+ path_to_data = os.path.join(top_level_dir, provided_qa_dir)
+
+ if not os.path.isdir(path_to_data):
+ raise NotADirectoryError(f'{path_to_data} not found')
+
+ # Check if we were given a request Id
+ try:
+ request_id = int(provided_qa_dir)
+ # with a request id, our subdirectory of interest is one level down,
+ # go find it
+ subdirectory = find_working_subdirectory(Path(top_level_dir, provided_qa_dir))
+
+ if subdirectory is None:
+ print(f'Error: Could not identify the working directory under {provided_qa_dir}.')
+
+ exit(-1)
+
+ except ValueError:
+ # If this fails, then we have an internal run
+ # no request id, just directly given the working directory
+ request_id = None
+ subdirectory = provided_qa_dir
+
+ mous_id = find_mous_id(path_to_data)
+
+ if mous_id is not None:
+ try:
+ project_code = lookup_project_code(mous_id)
+ except TypeError:
+ print(f'>>> No project code found for MOUS ID {mous_id}')
+ exit(-1)
+ else:
+ print(f'Error: MOUS ID not found in {provided_qa_dir}.')
+ exit(-1)
+
+ if project_code is None:
+ print(f'Error: No project code found for MOUS ID {mous_id}.')
+ exit(-1)
+
+ my_tracker = WorkflowTracker(capo_profile)
+ monitor_key = my_tracker.prepareWorkflowTracking(_WORKFLOW_NAME, email=args.email)
+
+ launch_audi_pass(mous_id, project_code, subdirectory, request_id, monitor_key)
+
+ # Now wait for information:
+ my_tracker.waitForReply()
+
+ print(f'Image ingestion QA ({my_tracker.getLoggingId()}) running in: {my_tracker.getWorkingDirectory()}')
+
+
+def _make_audi_pass_parser():
+
+ r"""Command-line AOI Parser"""
+
+ result = ap.ArgumentParser(description=_RUN_AOI_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. nmtest, mnprod')
+
+ result.add_argument('sourceDirectory', action='store', type=str,
+ help='image-qa subdirectory containing data')
+
+ result.add_argument('-E', '--email', action='store', default=None,
+ help='Optional email address for a completion message')
+
+ # Placeholder; workflow can't do this yet.
+ # result.add_argument('-D', '--destination', action='store', default='', help='Override location for AUDI pass')
+
+ return result
+
+
+def launch_audi_pass(data_set_id, project_code, subdir, request_id, tracking_key):
+ # build pieces of command line
+
+ capo_profile = os.environ['CAPO_PROFILE']
+ arguments = '-P ' + capo_profile
+ arguments += ' -s NAASC'
+ arguments += ' -p telescope=ALMA'
+ arguments += ' -p projectCode=' + project_code
+ arguments += ' -p workflowName=AoiIngestionWorkflow'
+ arguments += ' -p subDirOrFileName=' +subdir
+ arguments += ' -p dataSetId=' + data_set_id
+ if request_id is not None:
+ arguments += ' -p requestId=' + str(request_id)
+ arguments += ' -p ingestionType=alma_aoi'
+ arguments += ' -p cliCorrId=' + tracking_key
+ arguments += ' run'+ _WORKFLOW_NAME
+
+ wf(args=arguments.split(' '))
+
+#
+#
+# Collection-specific Variants
+#
+#
+
+def _make_realfast_parser(ingestion_description, major_arg):
+ r"""Shared Command-line Ingestion Launch Parser"""
+ result = ap.ArgumentParser(description=ingestion_description.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. nmtest, mnprod')
+ result.add_argument('-s', '--sdm_path', action='store', default='',
+ help='Path to the RealFast SDM (overrides CAPO setting)')
+ result.add_argument('-p', '--png_path', action='store', default='',
+ help='Path to the candidate PNG files (overrides CAPO setting)')
+ result.add_argument('sdmName', type = str, action='store', nargs='+', help=major_arg)
+ return result
+
+
+def launch_realfast_sdm_ingestion():
+ r"""CLI for RealFast SDM Ingestion Workflow"""
+ rf_description = """RealFast SDM Ingestion, version {}: Initiates an ingestion workflow for the SDM and ancillary
+ files for each execution block listed."""
+ rf_argument = 'FileSet Identifiers(s) to ingest'
+ parser = _make_realfast_parser(rf_description, rf_argument)
+ args = parser.parse_args()
+
+ # Automatic Profile Determination:
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ #
+ # Very similar to regular SDM ingestion, the command line we want is:
+ #
+ # wf -P [test|nmprod] -p workflowName=RfSdmIngestion -p dataSetId=arg runRealFastSdmIngestionWorkflow
+ # or
+ # wf -P [nmtest|nmprod] -p workflowName=RfSdmIngestion -p SDMPath=arg -p pngPath=arg -p datasetId=arg runRealFastSdmIngestionWorkflow
+ #
+ # The pieces of the string
+ clBase = "-p workflowName=RfSdmIngestion -p processingSite=DSOC -P" #profile here
+ clArgPiece = " -p ingestionType=realfast_sdm -p datasetId=" #fileSetId here
+ if '' != args.sdm_path:
+ clPathPiece = " -p SDMPath=" + args.sdm_path
+ # insert the casa parameter here, since it is fixed for all fileSetIds
+ clArgPiece = clPathPiece + clArgPiece
+
+ if '' != args.png_path:
+ clPathPiece = " -p pngPath=" + args.png_path
+ # insert the casa parameter here, since it is fixed for all fileSetIds
+ clArgPiece = clPathPiece + clArgPiece
+
+
+ clFinish = " runRealFastIngestionWorkflow"
+
+
+ #
+ # Loop over the provided fileSetIds and send a command for each:
+ #
+ for id in args.sdmName:
+ wfArgs = clBase + os.environ['CAPO_PROFILE'] + clArgPiece + id + clFinish
+ print("wf "+wfArgs)
+ wf(args=wfArgs.split(' '))
+ time.sleep(0.25) # 1/4 second delay between workflow launches for safety
diff --git a/apps/cli/executables/pexable/ingest/pyat/wf/ous_wf_interfaces.py b/apps/cli/executables/pexable/ingest/pyat/wf/ous_wf_interfaces.py
new file mode 100644
index 000000000..b7448ee3d
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/wf/ous_wf_interfaces.py
@@ -0,0 +1,344 @@
+
+import re
+import os
+import sys
+import json
+import psycopg2
+import pyat.schema
+import argparse as ap
+import cx_Oracle as almadb
+
+from pyat import version
+from datetime import datetime
+from pycapo import CapoConfig
+from pathlib import Path
+from pyat.wf import wf, WorkflowTracker
+
+# Alma Restores CLI:
+_RUN_ALMA_RESTORE_DESCRIPTION = """AAT/PPI ALMA Data Restore, version {}: This tool takes a Member OUS Status UID,
+ and from there it launches an ALMA specific version of the Restore workflow using the current version of the
+ calibration information from the NAASC archives. """
+
+
+
+def _make_alma_restore_parser():
+ r"""Command-line ALMA Restore Launch Parser"""
+ result = ap.ArgumentParser(description=_RUN_ALMA_RESTORE_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. nmtest, mnprod')
+ result.add_argument('-C', '--casa', action='store', default='',
+ help='Path to Casa package to use')
+ result.add_argument('-E', '--email', action='store', default=None,
+ help='Optional email address for a completion message')
+ # Placeholder, workflow can't do this yet.
+ # result.add_argument('-D', '--destination', action='store', default='', help='Override location for CMS delivery')
+ result.add_argument('mousUid', type = str, action='store', help='Member ObsUnitSet Identifier to be Restored')
+
+ return result
+
+
+def connect_alma(config):
+ return pyat.schema.create_session('ALMA').connection().engine.raw_connection()
+
+def connect_metadata(config):
+ return pyat.schema.create_session('SDM').connection().engine.raw_connection()
+
+
+def lookup_project_code(mous_uid):
+ r"""We want the project code to simplify the job of the workflow (and, as a smoke test for whether
+ the alma archive will behave itself for this process)"""
+
+ configuration = CapoConfig()
+
+ alma_handle = connect_alma(configuration).cursor()
+
+ _ALMA_PC_QUERY="select ASA_PROJECT_CODE from ASA_OUS where ASA_OUS_ID= :mous_id"
+
+ alma_handle.execute(_ALMA_PC_QUERY, mous_id=mous_uid)
+
+ result = alma_handle.fetchone()[0]
+
+ return result
+
+
+def lookup_single_locator(mous_uid):
+ configuration = CapoConfig()
+
+ _AAT_PL_QUERY = """SELECT science_product_locator FROM execution_blocks WHERE alma_ous_id=%s"""
+
+ aat_handle = connect_metadata(configuration).cursor()
+ aat_handle.execute(_AAT_PL_QUERY, [mous_uid])
+
+ return aat_handle.fetchone()[0]
+
+
+def lookup_eb_uid(mous_uid):
+ r"""Another query, we need a fileSetId related to our MOUS"""
+ configuration = CapoConfig()
+
+ alma_handle = connect_alma(configuration).cursor()
+
+ _ALMA_EB_QUERY="select ASDM_UID from ASA_SCIENCE where MEMBER_OUSS_ID= :mous_id"
+
+ alma_handle.execute(_ALMA_EB_QUERY, mous_id=mous_uid)
+
+ # We'll get a bunch of results, but we only need one ASDM to proceed
+ result = alma_handle.fetchone()[0]
+
+ return result
+
+
+def send_alma_ous_workflow_start(args, project_code, locator, file_set_id, tracking_key, cube_params=None):
+ r"""Build the arguments for the WF call, and off we go!"""
+ # An Example Command Line from workflow testing:
+ #
+ # wf -P nmtest -s DSOC
+ # -p ousStatusId=uid://A001/X1284/X265f
+ # -p fileSetIds=uid://A002/Xc7111c/X3979
+ # -p telescope=ALMA
+ # -p projectCodeOrDataType=2017.1.00886.L
+ # runAlmaRestoreTest
+ #
+ # Build the wf command line, starting with all the fixed pieces:
+ arguments = "-s NAASC -p processingSite=NAASC -p telescope=ALMA"
+ # Required Parameters
+ arguments += " -P " + os.environ['CAPO_PROFILE']
+ arguments += " -p productLocator=" + locator
+ arguments += " -p fileSetIds=" + file_set_id
+ arguments += " -p ousStatusId=" + args.mousUid
+ arguments += " -p projectCodeOrDataType=" + project_code
+ arguments += " -p cliCorrId=" + tracking_key
+
+ # # Initial attempt to get it working again: Give it an explict working directory:
+ # santized_mous = args.mousUid.replace(':', '_').replace('/', '_')
+ # timestamp = datetime.now().isoformat().replace('-','_').replace(':','_')
+ # arguments += " -p relativePath="+santized_mous+'_'+timestamp
+
+ # Control whether we just restore, or image as well:
+ if cube_params is None:
+ arguments += " -p workflowName=AlmaOusRestoreWorkflow"
+ arguments += " -p downloadDataFormat=CMS"
+ arguments += " -p deliveryFormat=CMS"
+ else:
+ arguments += " -p workflowName=AlmaOusImagingWorkflow"
+ arguments += " -p downloadDataFormat=IMG"
+ arguments += " -p deliveryFormat=IMG"
+ arguments += " -p optimizedImagingParameters=" + cube_params
+ # The Optional Parameters:
+ if '' != args.casa:
+ arguments += " -p casaHome=" + args.casa
+ else:
+ config = CapoConfig()
+ arguments += " -p casaHome=" + config['edu.nrao.archive.workflow.config.CasaVersions.homeForAlmaRestore']
+ # Option not available yet, so don't display or use it
+ # if '' != args.destination:
+ # arguments += " -p deliveryDirectory=" + args.destination
+ # The Finale
+ if cube_params is None:
+ arguments += " runAlmaBasicRestoreWorkflow"
+ else:
+ arguments += " runAlmaOptimizedImagingWorkflow"
+ # Build our command line and execute:
+ wf(args=arguments.split(' '))
+
+ # Talk to the user, these CLIs are bad on feedback
+ print("Initiating processing of {} to spool.\n".format(args.mousUid))
+
+
+def launch_alma_restore():
+ r"""Take given mous_uid and launch a restore for it. One/command, these are expensive."""
+
+ # parse our arguments, get profile and list of Ids
+ args = _make_alma_restore_parser().parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+
+ # In order to use the direct reply functionality, we need to
+ # send these messages via the NM system.
+ if "va" in args.profile:
+ profile_to_use = args.profile.replace('va', 'nm')
+ else:
+ profile_to_use = args.profile
+
+ os.environ['CAPO_PROFILE'] = profile_to_use
+
+ # Obtain the Project Code & an ASDM UID: Require queries of the ALMA database:
+ project_code = lookup_project_code(args.mousUid)
+ asdm_uid = lookup_eb_uid(args.mousUid)
+ asdm_locator = lookup_single_locator(args.mousUid)
+
+ # Send an event to initiate workflow monitoring:
+
+ my_tracker = WorkflowTracker(args.profile)
+ monitor_key = my_tracker.prepareWorkflowTracking("runAlmaRestoreWorkflow", email=args.email)
+
+ # Launch the workflow
+ send_alma_ous_workflow_start(args, project_code, asdm_locator, asdm_uid, monitor_key)
+
+ # Now wait for information:
+ my_tracker.waitForReply()
+
+ # and tell the user where things are happening:
+ print("Alma Restore ({}) running in: {}".format(my_tracker.getLoggingId(), my_tracker.getWorkingDirectory()))
+
+
+# Optimized Imaging
+_RUN_ALMA_IMAGING_DESCRIPTION = """AAT/PPI ALMA Optimized Imaging, version {}: This tool takes a Member OUS Status UID
+ and json file of parameters. It then proceeds to initate the Optimized Imaging variant of the ALMA OUS processing
+ workflow. It makes use of the template_hifa_cubeimage.xml recipe provided by the pipeline. """
+
+
+def _make_alma_cube_imaging_parser():
+ r"""Command-line ALMA Restore Launch Parser"""
+ result = ap.ArgumentParser(description=_RUN_ALMA_RESTORE_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. vatest, vaprod')
+ result.add_argument('-C', '--casa', action='store', default='',
+ help='Path to Casa package to use')
+ result.add_argument('-E', '--email', action='store', default=None,
+ help='Optional email address for a completion message')
+ result.add_argument('file', action='store', type=str, help='Name of the parameters file to use for imaging.')
+ result.add_argument('mousUid', type = str, action='store', help='Member ObsUnitSet Identifier to be Imaged')
+
+ return result
+
+
+def launch_alma_cube_imaging():
+ r"""
+ A wrapper to set up the metadata for and tracking of an Alma Optimized
+ Imaging workflow (note: this is a variant on the restore above. Except
+ for the imaging parameters & the downloadDataFormat value, things should
+ be pretty much the same.
+ """
+
+ args = _make_alma_cube_imaging_parser().parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+
+ # In order to use the direct reply functionality, we need to
+ # send these messages via the NM system.
+ if "va" in args.profile:
+ profile_to_use = args.profile.replace('va', 'nm')
+ else:
+ profile_to_use = args.profile
+
+ os.environ['CAPO_PROFILE'] = profile_to_use
+
+ parameters_file = Path(args.file)
+ if not parameters_file.exists():
+ print(f"Can't find file {parameters_file}! Not executing workflow.")
+ exit(5)
+
+ # if we got here, then the file exists, read the contents for passing on to the workflow:
+ file_handle = open(parameters_file, 'r')
+ raw_parameters = file_handle.read()
+ # remove all tab/newline/spaces to allow easier writing of the json by peoples:
+ parameters_json = re.sub(r"[\n\t\s]*", "", raw_parameters)
+
+ try:
+ structured_json = json.loads(parameters_json)
+ print(f"Parameters loaded in: {structured_json}")
+ # Note: the json parsing here is just for validation purposes.
+ except ValueError:
+ print(f"{parameters_file} is not properly formatted Json. Attempted to parse: {parameters_json}")
+ exit(3)
+
+ # Obtain the Project Code & an ASDM UID: Require queries of the ALMA database:
+ project_code = lookup_project_code(args.mousUid)
+ asdm_uid = lookup_eb_uid(args.mousUid)
+ asdm_locator = lookup_single_locator(args.mousUid)
+
+ # Send an event to initiate workflow monitoring:
+ my_tracker = WorkflowTracker(args.profile)
+ monitor_key = my_tracker.prepareWorkflowTracking("runAlmaOptimizedCubeWorkflow", email=args.email)
+
+ # Launch the workflow
+ send_alma_ous_workflow_start(args, project_code, asdm_locator, asdm_uid, monitor_key, cube_params=parameters_json)
+
+ # Now wait for information:
+ my_tracker.waitForReply()
+
+ # and tell the user where things are happening:
+ print("Alma Cube Imaging ({}) running in: {}".format(my_tracker.getLoggingId(), my_tracker.getWorkingDirectory()))
+
+# Optimized Imaging Reprocess
+
+_RUN_AOI_REPRO_DESCRIPTION = """AAT/PPI Alma Imaging Reprocessing, version {}: This tool initiates a re-run of the CASA imaging pipeline on the desired directory. The data are placed in the Spool for the processing and returned to the QA area upon successful completion."""
+
+
+def _make_aoi_reprocess_parser():
+ r"""Parser"""
+ result = ap.ArgumentParser(description=_RUN_AOI_REPRO_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. test, production')
+ result.add_argument('-I', '--image_casa', action='store', default='',
+ help='Specify a CASA for imaging purposes (overrides what was chosen in the UI)')
+ result.add_argument('-K', '--restore_casa', action='store', default='',
+ help='The C5 Kludge(tm): Separates restore processing, and sets the CASA version for restore')
+ result.add_argument('-r', '--request', action='store', default='', type=str,
+ help='The external RequestId for this run (if applicable)')
+ result.add_argument('directoryOfInterest', type = str, action='store',
+ help='Name of the directories to be re-run through AOI')
+ return result
+
+
+def launch_reimaging_run():
+ """From a (list of) directory(s) launch the AOI Reprocessing workflow for each"""
+
+ #parse our arguments, get profile and list of Ids
+ args = _make_aoi_reprocess_parser().parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+
+ #
+ # Command line we want is similar to that for CIPL reprocessing:
+ # (processingSite for message routing)
+ #
+ # wf -P [nmtest|nmprod] -p workflowName=CiplReprocess -p processingSite=DSOC -p targetDirectory=arg runCiplReprocessingWorkflow
+ #
+ # Build up the arguments:
+ arguments = "-p workflowName=AoiReprocess -p processingSite=NAASC -p telescope=ALMA -s NAASC"
+ arguments += " -P " +os.environ['CAPO_PROFILE']
+ arguments += " -p targetDirectory=" + args.directoryOfInterest
+
+ if '' != args.image_casa:
+ arguments += " -p casaHome=" + args.image_casa
+
+ # Setting this variable activates The C5 Kludge(tm)
+ if '' != args.restore_casa:
+ arguments += " -p restoreCasa=" + args.restore_casa
+
+ if '' != args.request:
+ arguments += " -p requestId=" + args.request
+
+ arguments += " runAoiReprocessingWorkflow"
+ #clFinish = " runTestWorkflow"
+
+ wf(args=arguments.split(' '))
+ print(f"Re-running CASA for {args.directoryOfInterest} in spool. You'll recieve an email shortly.")
+
diff --git a/apps/cli/executables/pexable/ingest/pyat/wf/utility_wf_interfaces.py b/apps/cli/executables/pexable/ingest/pyat/wf/utility_wf_interfaces.py
new file mode 100644
index 000000000..645babe24
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/pyat/wf/utility_wf_interfaces.py
@@ -0,0 +1,64 @@
+import argparse as ap
+import os
+import sys
+
+
+from pyat.wf import wf
+from pyat import version
+
+
+_RUN_QACLEAN_DESCRIPTION = """"""
+
+
+def _make_qaclean_parser():
+ r"""Command-line CIPL Launch Parser"""
+ result = ap.ArgumentParser(description=_RUN_QACLEAN_DESCRIPTION.format(version),
+ formatter_class=ap.RawTextHelpFormatter)
+ result.add_argument('-P', '--profile', action='store', default='',
+ help='profile name to use, e.g. dsoc-dev, dsoc-prod')
+ result.add_argument('workingDirectory', type = str, action='store', nargs='+',
+ help='Working Directories to remove from the qa area.')
+ return result
+
+
+def launch_deletions():
+ r"""Direct launch of the QaCleanupWorkflow (because SRDP)"""
+
+ # parse our arguments, get profile and list of Ids
+ args = _make_qaclean_parser().parse_args()
+
+ # Shamelessly stolen from epilogue with a twist: allow for explict profile setting via the CL
+ if 'CAPO_PROFILE' not in os.environ and '' == args.profile:
+ # try to synthesize a profile from our installation root
+ profile = os.path.abspath(sys.argv[0]).split(os.path.sep)[-3]
+ os.environ['CAPO_PROFILE'] = profile
+ print('No CAPO_PROFILE explicitly set, synthesizing {0} by path'.format(str(profile)))
+ elif '' != args.profile:
+ os.environ['CAPO_PROFILE'] = args.profile
+
+ #
+ # Command line we want is:
+ #
+ # wf -s [] -P [] -p workflowName=QaCleanup -p =arg runQaCleanupWorkflow
+ #
+ # The pieces of the string
+ profile_name = os.environ['CAPO_PROFILE']
+ if (profile_name.find('dsoc') == -1) and (profile_name.find('nm') == -1):
+ # Running in CV:
+ cl_basics = "-s NAASC -P "
+ else:
+ # Running in NM:
+ cl_basics = "-s DSOC -P " # profile here
+ cl_wf_args = " -p workflowName=QaCleanup -p subDirOrFileName=" #
+ cl_finish = " runQaCleanupWorkflow"
+
+ #
+ # Loop over the provided fileSetIds and send a command for each,
+ # waiting for the previous one to report back information before
+ # starting the next.
+ #
+ for id in args.workingDirectory:
+
+ wfArgs =cl_basics + profile_name + cl_wf_args + id + cl_finish
+ wf(args=wfArgs.split(' '))
+ print(f"Launched workflow to delete {id} from the QA areas.")
diff --git a/apps/cli/executables/pexable/ingest/requirements.txt b/apps/cli/executables/pexable/ingest/requirements.txt
new file mode 100644
index 000000000..b34c7f92a
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/requirements.txt
@@ -0,0 +1,28 @@
+astropy==4.3.post1
+bcrypt==3.2.0
+certifi==2021.5.30
+cffi==1.14.6
+charset-normalizer==2.0.3
+cryptography==3.4.7
+cx-Oracle==8.2.1
+greenlet==1.1.0
+idna==3.2
+jxmlease==1.0.3
+lxml==4.6.3
+mysqlclient==2.0.3
+numpy==1.21.1
+paramiko==2.7.2
+pex==2.1.43
+pika==1.2.0
+psycopg2-binary==2.9.1
+pycapo==0.3.1
+pycparser==2.20
+pyerfa==2.0.0
+PyNaCl==1.4.0
+pysftp==0.2.9
+python-dateutil==2.8.2
+requests==2.26.0
+simplejson==3.17.3
+six==1.16.0
+SQLAlchemy==1.4.22
+urllib3==1.26.6
diff --git a/apps/cli/executables/pexable/ingest/scripts/alma-data-fetcher.sh b/apps/cli/executables/pexable/ingest/scripts/alma-data-fetcher.sh
new file mode 100644
index 000000000..f3f2932d0
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/scripts/alma-data-fetcher.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# -o nounset does not work if you source the ACS bash profile thing :)
+set -o errexit -o xtrace
+
+JAVA_HOME=/opt/local/java/java8
+PATH=$JAVA_HOME/bin:$PATH
+ACS_RETAIN=true
+ALMASW_ROOTDIR=$(capo -q OodtSettings.almaswRootdir)
+ALMASW_RELEASE=$(capo -q OodtSettings.almaswRelease)
+
+source ${ALMASW_ROOTDIR}/${ALMASW_RELEASE}/ACSSW/config/.acs/.bash_profile.acs
+
+# Let's make sure to fail if we try a data fetch inside the DSOC data center
+if ! hostname -f | grep -q .cv.; then
+ echo "ERROR! Cannot fetch ALMA data outside the NAASC data center!"
+ exit 1
+fi
+
+# argument 1 is the directory
+# argument 2 is optionally "-m" for metadata-only
+# argument 3 is (or two) is the ASDM ID
+
+DIRECTORY=$1; shift
+DATA_TYPE=$1; shift
+DATASET=$1; shift
+
+if [ "$DATA_TYPE" = "SDMonly" ]; then
+ asdmExportLight -d "$DIRECTORY/rawdata" -m "$DATASET"
+else
+ asdmExportLight -d "$DIRECTORY/rawdata" "$DATASET"
+fi
diff --git a/apps/cli/executables/pexable/ingest/setup.py b/apps/cli/executables/pexable/ingest/setup.py
new file mode 100644
index 000000000..63e9aad4f
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/setup.py
@@ -0,0 +1,117 @@
+"""A setuptools based setup module.
+
+See:
+https://packaging.python.org/en/latest/distributing.html
+https://github.com/pypa/sampleproject
+"""
+
+# To use a consistent encoding
+from codecs import open
+from os import path
+
+# Always prefer setuptools over distutils
+from setuptools import setup, find_packages
+
+# For matching the version string.
+import re
+
+this_module = "ingest"
+here = path.abspath(path.dirname(__file__))
+
+# Get the long description from the README file
+with open(path.join(here, "README.txt"), encoding="utf-8") as f:
+ long_description = f.read()
+
+
+def read(*parts):
+ with open(path.join(here, *parts), "r") as fp:
+ return fp.read()
+
+
+def find_version(*file_paths):
+ version_file = read(*file_paths)
+ version_match = re.search(r"^___version___ = ['\"]([^'\"]*)['\"]", version_file, re.M)
+ if version_match:
+ return version_match.group(1)
+ raise RuntimeError("Unable to find version string.")
+
+
+with open("requirements.txt") as f:
+ requires = f.read().splitlines()
+
+setup(
+ name=this_module,
+ # Versions should comply with PEP440. For a discussion on single-sourcing
+ # the version across setup.py and the project code, see
+ # https://packaging.python.org/en/latest/single_source_version.html
+ version=find_version(this_module, "_version.py"),
+ description="NRAO Archive Ingest",
+ long_description=long_description,
+ # The project's main homepage.
+ # url='https://github.com/pypa/sampleproject',
+ # Author details
+ author="SSA Team",
+ author_email="dms-ssa@nrao.edu",
+ # Choose your license
+ license="GPL",
+ # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
+ classifiers=[
+ # How mature is this project? Common values are
+ # 3 - Alpha
+ # 4 - Beta
+ # 5 - Production/Stable
+ "Development Status :: 4 - Beta",
+ # Indicate who your project is intended for
+ "Intended Audience :: Developers",
+ "Topic :: Software Development :: Build Tools",
+ # Pick your license as you wish (should match "license" above)
+ "License :: OSI Approved :: GPL License",
+ # Specify the Python versions you support here. In particular, ensure
+ # that you indicate whether you support Python 2, Python 3 or both.
+ "Programming Language :: Python :: 3.6",
+ ],
+ # What does your project relate to?
+ keywords="nrao archive",
+ # You can just specify the packages manually here if your project is
+ # simple. Or you can use find_packages().
+ packages=find_packages(),
+ include_package_data=True,
+ # Alternatively, if you want to distribute just a my_module.py, uncomment
+ # this:
+ # py_modules=["my_module"],
+ # List run-time dependencies here. These will be installed by pip when
+ # your project is installed. For an analysis of "install_requires" vs pip's
+ # requirements files see:
+ # https://packaging.python.org/en/latest/requirements.html
+ install_requires=requires,
+ # tests
+ test_suite="pyat.tests",
+ tests_require=["pytest-runner==2.9"],
+ # List additional groups of dependencies here (e.g. development
+ # dependencies). You can install these using the following syntax,
+ # for example:
+ # $ pip install -e .[dev,test]
+ # extras_require={
+ # 'dev': ['check-manifest'],
+ # 'test': ['coverage'],
+ # },
+ # If there are data files included in your packages that need to be
+ # installed, specify them here. If using Python 2.6 or less, then these
+ # have to be included in MANIFEST.in as well.
+ # package_data={
+ # 'sample': ['package_data.dat'],
+ # },
+ # Although 'package_data' is the preferred approach, in some case you may
+ # need to place data files outside of your packages. See:
+ # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
+ # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
+ # data_files=[('my_data', ['data/data_file'])],
+ # To provide executable scripts, use entry points in preference to the
+ # "scripts" keyword. Entry points provide cross-platform support and allow
+ # pip to create the appropriate form of executable for the target platform.
+ entry_points={
+ "console_scripts": [
+ "ingest = ingest.archive:main",
+ ],
+ },
+)
diff --git a/apps/cli/executables/pexable/ingest/tests/functional/__init__.py b/apps/cli/executables/pexable/ingest/tests/functional/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/cli/executables/pexable/ingest/tests/functional/test_func_one.py b/apps/cli/executables/pexable/ingest/tests/functional/test_func_one.py
new file mode 100644
index 000000000..ea532bae5
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/tests/functional/test_func_one.py
@@ -0,0 +1,10 @@
+import pytest
+
+
+@pytest.mark.skip(reason='Example skipped test')
+def test_one():
+ assert 1 == 0
+
+
+def test_two():
+ assert 1 == 1
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/tests/unit/__init__.py b/apps/cli/executables/pexable/ingest/tests/unit/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/cli/executables/pexable/ingest/tests/unit/test_eb_persistence.py b/apps/cli/executables/pexable/ingest/tests/unit/test_eb_persistence.py
new file mode 100644
index 000000000..f476b5516
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/tests/unit/test_eb_persistence.py
@@ -0,0 +1,24 @@
+import pyat
+from pyat.schema import RealfastExecutionBlock, Filegroup, Project, ExecutionBlock
+
+
+def test_realfast_execblock_persistence():
+ p = Project()
+ p.project_code = 'foo'
+ fg = Filegroup()
+ eb = RealfastExecutionBlock(filegroup=fg, calibration_level=1, telescope='VLA', project_code='foo')
+
+ session = pyat.schema.create_session('SDM', profile='local')
+ try:
+ session.add(p)
+ session.add(eb)
+ session.commit()
+ eb2 = session.query(RealfastExecutionBlock).all()[-1]
+ assert eb2 == eb
+ eb3 = session.query(ExecutionBlock).get(eb2.execution_block_id)
+ assert eb3 == eb
+ finally:
+ session.delete(eb)
+ session.delete(fg)
+ session.delete(p)
+ session.commit()
\ No newline at end of file
diff --git a/apps/cli/executables/pexable/ingest/tests/unit/test_ingestion_manifest.py b/apps/cli/executables/pexable/ingest/tests/unit/test_ingestion_manifest.py
new file mode 100644
index 000000000..7c219bbc5
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/tests/unit/test_ingestion_manifest.py
@@ -0,0 +1,77 @@
+
+import pytest
+
+from pathlib import Path
+
+from pyat.ingestion.IngestionManifest import IngestionManifest
+from pyat.ingestion.IngestionManifest import INGESTION_MANIFEST_FILE
+import pyat.ingestion.manifestpersist as mp
+from pyat.ingestion.JSONMetadataUtil import JSONMetadata
+from pyat.ingestion.RealFastMetadata import RealFastMetadata
+from pyat.schema import RealfastExecutionBlock
+
+
+@pytest.fixture
+def ingestion_manifest():
+ ingestion_manifest = IngestionManifest(INGESTION_MANIFEST_FILE)
+ return ingestion_manifest
+
+@pytest.fixture
+def realfast_metadata():
+ manifest = IngestionManifest(INGESTION_MANIFEST_FILE)
+ if manifest.has_collection():
+ collection_metadata_file = Path(".") / manifest.get_additional_metadata_file()
+ json_metadata = JSONMetadata(collection_metadata_file)
+ if json_metadata.isRealFast():
+ realfast_metadata = RealFastMetadata(json_metadata)
+ return realfast_metadata
+ else:
+ return None
+ else:
+ return None
+
+
+@pytest.mark.skip(reason='Ignoring for now')
+def test_associate_group(ingestion_manifest):
+ assert ingestion_manifest.get_associate_group_locator() == "uid://evla/execblock/abcd-efgh-ijkl-mnop"
+
+
+@pytest.mark.skip(reason='Ignoring for now')
+def test_type(realfast_metadata):
+ assert type(realfast_metadata) == RealFastMetadata
+
+
+#TODO
+# NGAS files
+@pytest.mark.skip(reason='Ignoring for now')
+def test_realfast_attributes(realfast_metadata):
+ assert realfast_metadata.get_transient_dm() == 576.9
+ assert realfast_metadata.get_transient_ra() == "01:55:50.2715"
+ assert realfast_metadata.get_transient_ra_error() == 0.05664506927973141
+ assert realfast_metadata.get_transient_dec() == "+31:27:54.8557"
+ assert realfast_metadata.get_transient_dec_error() == 0.8496760391959711
+ assert realfast_metadata.get_transient_snr() == 7.483555793762207
+ assert realfast_metadata.get_transient_dm() == 576.9
+ assert realfast_metadata.get_transient_dm_error() == 576.9
+ assert realfast_metadata.get_preaverage_time() == 0.019999999999999997
+ assert realfast_metadata.get_rfpipe_version() == "1.5.7"
+ assert realfast_metadata.get_prefs_id() == "f245c8a5fad6005dbe85fbee85428b4d"
+ assert realfast_metadata.get_rf_qa_label() == "Questionable"
+ assert realfast_metadata.get_rf_qa_zero_fraction() == 0.2119006529116276
+ assert realfast_metadata.get_rf_qa_visibility_noise() == 0.017208602279424667
+ assert realfast_metadata.get_rf_qa_image_noise() == 847.9179077148438
+
+
+@pytest.mark.skip(reason='Ignoring for now')
+def test_realfast_execution_block(realfast_metadata):
+ execution_block = RealfastExecutionBlock()
+ assert type(execution_block) == RealfastExecutionBlock
+ execution_block.transient_ra = realfast_metadata.get_transient_ra()
+ assert execution_block.transient_ra == realfast_metadata.get_transient_ra()
+
+
+def test_manifest():
+ path = Path("./tests")
+ manifest = IngestionManifest(path / INGESTION_MANIFEST_FILE)
+ assert manifest.has_collection_metadata_file()
+
diff --git a/apps/cli/executables/pexable/ingest/tests/unit/test_unit_one.py b/apps/cli/executables/pexable/ingest/tests/unit/test_unit_one.py
new file mode 100644
index 000000000..ea532bae5
--- /dev/null
+++ b/apps/cli/executables/pexable/ingest/tests/unit/test_unit_one.py
@@ -0,0 +1,10 @@
+import pytest
+
+
+@pytest.mark.skip(reason='Example skipped test')
+def test_one():
+ assert 1 == 0
+
+
+def test_two():
+ assert 1 == 1
\ No newline at end of file
--
GitLab