Merge pull request #12 in SSA/data from SSA-6315-skeletal-build to main

* commit '340b5921': (48 commits)
  Moved recipes, tools, pkgs, and metadata dirs from data/ to data/build/ for a cleaner root dir. Adjusted code to correct for this change.
  Added some error checking and comments.
  Completed recipe for building packages and transferring them to builder.
  gitignore update
  gitignore update
  Updated build rule of Makefile to use buildout recipe.
  Bump to PyCapo 0.3.1
  Add a basic Makefile that builds all the conda packages for us
  Fix the imports to no longer rely on pyat and instead the broken out data packages
  Added simple recipe for building packages.
  Refactor and remove numpy dependencies
  Adding zc.buildout to the environment.yml
  Add zc.buildout to the conda support packages
  Corrected another dependency typo.
  Corrected dependency typo.
  gitignore addition
  Changed a common import error I kept running into in several files.
  Adding meta.yaml files I created for a couple external packages.
  Updated gitignore
  Another updated init file from main branch.
  ...

.gitignore

@@ -46,3 +46,16 @@ docs/_build
 /workflow-all/workflow-jobs/src/main/resources/edu/nrao/archive/workflow/scripts/open-permissions.sh
 /archiveIface/archiveIface/webpack/stats.json
 /git-info.txt
+develop-eggs
+build/metadata
+#**/build/**
+apps/**/build/**
+services/**/build/**
+shared/**/build/**
+projects_checklist.txt
+.installed.cfg
+.ipynb_checkpoints
+deps.png
+build/pkgs
+eggs
+parts
\ No newline at end of file

Makefile

+all: ${HOME}/miniconda3 ${HOME}/miniconda3/envs/data metadata build
+
+${HOME}/miniconda3:
+	@echo Please install miniconda to ${HOME}/miniconda3
+	@exit 1
+
+${HOME}/miniconda3/envs/data: environment.yml
+	conda env update
+	touch $@
+
+.PHONY: metadata
+metadata:
+	buildout parts=gen_metadata
+
+.PHONY: build
+build:
+	buildout parts=build_pkgs name=all

apps/cli/executables/alma_product_fetch/src/alma_product_fetch/__init__.py

-r"""
-    An import of the Joint Alma Observatories listfiles.py into the AAT/PPI system.
-
-    Given an MOUS Status UID (the most common identifier for a processing unit),
-    perform a database query to obtain the list of files required for a restore
-    and proceed to obtain them from the NAASC NGAS machines.
-
-    Initially written to use wget, this is rewritten to use requests and to
-    attempt to validate the data retrieved.
-
-"""

apps/cli/executables/alma_product_fetch/src/alma_product_fetch/commands.py

@@ -10,7 +10,7 @@ import cx_Oracle as almadb
 import requests
 import sys
 from ._version import ___version___ as version
-from pymygdala import LogHandler
+from pymygdala.models import LogHandler
 from pycapo import CapoConfig
 
 """

apps/cli/executables/alma_reingester/setup.py

@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['cx-Oracle', 'pycapo', 'psycopg2', 'events', 'schema'],
+    install_requires=['cx_Oracle', 'pycapo', 'psycopg2', 'events', 'schema'],
     keywords=[],
     packages=['alma_reingester'],
     package_dir={'':'src'},

apps/cli/executables/datafetcher/conda/meta.yaml

+{% set name = "datafetcher" %}
+{% set version = "4.0.0a1.dev1" %}
+
+package:
+  name: "{{ name|lower }}"
+  version: "{{ version }}"
+
+build:
+  entry_points:
+    - datafetcher = work.datafetcher.commands:main
+  script: {{ PYTHON }} setup.py develop
+
+source:
+  path: ../../apps/cli/executables/datafetcher
+#  - path: ../../apps/cli/executables/datafetcher/src/datafetcher
+
+requirements:
+  build:
+    - python==3.8
+    - pika>=1.1,<2
+    - pycapo>=0.3.0,<1.0
+    - beautifulsoup4>=4.9.1,<5.0
+    - lxml>=4.3.2,<5.0
+    - psycopg2>=2.8.5,<3.0
+    - pyopenssl>=19.1.0,<20.0
+    - requests>=2.23,<3.0
+  run:
+    - python==3.8
+#    - pika>=1.1,<2
+#    - pycapo>=0.3.0,<1.0
+#    - beautifulsoup4>=4.9.1,<5.0
+#    - lxml>=4.3.2,<5.0
+#    - psycopg2>=2.8.5,<3.0
+#    - pyopenssl>=19.1.0,<20.0
+#    - requests>=2.23,<3.0
+
+#test:
+#  source_files:
+#    - test/
+#  requires:
+#    - pytest>=5.4,<6.0
+#  commands:
+#    - pytest -vv --log-level=DEBUG --showlocals test/datafetcher_test.py
+
+about:
+  license: "GPL"
+  license_family: "GPL"
+  summary: "NRAO Archive Data Fetcher Script"
\ No newline at end of file

apps/cli/executables/datafetcher/setup.py

@@ -9,29 +9,17 @@ VERSION = open('_version.py').readlines()[-1].split()[-1].strip("\"'")
 README = Path('README.md').read_text()
 
 requires = [
-    'pika==1.1.0',
-    'pycapo==0.2.1post1',
-    'bs4==0.0.1',
-    'beautifulsoup4==4.9.1',
-    'lxml==4.3.2',
-    'psycopg2==2.8.5',
-    'pyinstaller==3.2',
-    'pyopenssl=19.1.0',
-    'tox==3.1.3',
-    'tox-pyenv==1.1.0'
+    'pika>=1.1,<2',
+    'pycapo>=0.3.0,<1.0',
+    'beautifulsoup4>=4.9.1,<5.0',
+    'lxml>=4.3.2,<5.0',
+    'psycopg2>=2.8.5,<3.0',
+    'pyopenssl>=19.1.0,<20.0',
+    'requests>=2.23,<3.0'
 ]
 
 tests_require = [
-    'bs4==0.0.1',
-    'beautifulsoup4==4.9.1',
-    'psycopg2==2.8.5',
-    'pyopenssl=19.1.0',
-    'pyinstaller==3.2',
-    'pytest==5.4',
-    'pytest-runner==4.2',
-    'tox==3.1.3',
-    'tox-pyenv==1.1.0',
-
+    'pytest>=5.4,<6.0'
 ]
 setup(
     name=Path().absolute().name,
@@ -42,7 +30,8 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['requests', 'pycapo'],
+    install_requires=requires,
+    tests_require=tests_require,
     keywords=[],
     packages=['datafetcher'],
     package_dir={'':'src'},

apps/cli/executables/datafetcher/test/testing_utils.py

@@ -9,11 +9,11 @@ from pathlib import Path
 
 from pycapo import CapoConfig
 
-from src.datafetcher.errors import \
+from datafetcher.errors import \
     MissingSettingsException, NoProfileException
-from src.datafetcher.locations_report import \
+from datafetcher.locations_report import \
     LocationsReport
-from src.datafetcher.utilities import \
+from datafetcher.utilities import \
     REQUIRED_SETTINGS, get_arg_parser, \
     ExecutionSite
 

apps/cli/executables/epilogue/setup.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from setuptools import setup, find_packages
 
-VERSION = open('src/_version.py').readlines()[-1].split()[-1].strip("\"'")
+VERSION = open('src/epilogue/_version.py').readlines()[-1].split()[-1].strip("\"'")
 README = Path('README.md').read_text()
 
 setup(

apps/cli/executables/ingestion/src/ingestion/__init__.py

-# -*- coding: utf-8 -*-
-
-
-from .metadata import *
-from .files import *
-
-import numpy
-from psycopg2.extensions import register_adapter, AsIs
-def addapt_numpy_float32(numpy_float32):
-    return AsIs(numpy_float32)
-register_adapter(numpy.float32, addapt_numpy_float32)
-
-__all__ = ['SDM', 'SDMTable', 'SDMBinaryTable', 'buildtable',
-           'ngasstatus', 'archivefile', 'retrievefile',
-           'ingest', 'ingestBDF', 'ingestSDM', 'ingestmetadata']