Merge pull request #15 in SSA/data from SSA-6319-null-executable to main

* commit '24dcfec0':
  Changed test naming convention. Added custom fixture for initializing Null object.
  Updated README with correct information.
  Added username/password collection to ensure a connection to builder.
  Changed import to work with conda build testing.
  Added dependency for transfer_to_builder
  Created unit tests.
  Fixed typo that was causing print_error to not run correctly.
  Removed pymygdala integration; made logging self-contained; added two more functions
  Fixed crash that would occur when a package has no requirements.
  Fixed bug in build_packages that would report package not valid when building it for the first time.
  Updated logging to use pymygdala's LogHandler class.
  First prototype of null executable.

apps/cli/executables/null/README.md

+This is the null executable, a baseline test of the functionality of the Workspaces system.
+
+It can:
+- Print a message to stderr
+- Print a message to stdout
+- Exit with status code -1
+- Exit with random status code in [-50, 50]
+- Sleep for 5 seconds
+- Abort and dump the core
+ 
+```
+usage: null [-h] [-v] [-pe | -g | -ef | -er | -n | -d]
+
+Workspaces null executable, a status capture test of the system. Version 4.0.0a1.dev1
+
+optional arguments:
+  -h, --help          show this help message and exit
+  -pe, --print-error  print out aggressive message to stderr
+  -g, --greeting      print out a friendly greeting to stdout
+  -ef, --exit-fail    print error message and exit with status code -1
+  -er, --exit-random  print error message and exit with random status code within [-50, 50]
+  -n, --nap           take a short nap
+  -d, --dump          abort program and dump core
+
+options:
+  settings for altering program behavior
+
+  -v, --verbose       allow the program the gift of speech
+```

apps/cli/executables/null/setup.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from pathlib import Path
+from setuptools import setup
+
+VERSION = open('src/null/_version.py').readlines()[-1].split()[-1].strip("\"'")
+README = Path('README.md').read_text()
+
+# requires = [
+# ]
+
+tests_require = [
+    'pytest>=5.4,<6.0'
+]
+setup(
+    name=Path().absolute().name,
+    version=VERSION,
+    description='Workspaces null executable.',
+    long_description=README,
+    author='NRAO SSA Team',
+    author_email='dms-ssa@nrao.edu',
+    url='TBD',
+    license="GPL",
+    # install_requires=requires,
+    tests_require=tests_require,
+    keywords=[],
+    packages=['null'],
+    package_dir={'':'src'},
+    classifiers=[
+        'Programming Language :: Python :: 3.8'
+    ],
+    entry_points={
+        'console_scripts': ['null = null.null:main']
+    },
+)

apps/cli/executables/null/src/null/_version.py

+""" Version information for this package, don't put anything else here. """
+___version___ = '4.0.0a1.dev1'

apps/cli/executables/null/src/null/null.py

+""" Module for the null executable. Performs some very basic actions
+    and utilizes pymygdala's LogHandler for logging. """
+
+import os
+import sys
+import time
+import random
+import logging
+import argparse
+
+from ._version import ___version___ as version
+
+_DESCRIPTION = """Workspaces null executable, a status capture test of the system. Version {}"""
+
+logger = logging.getLogger("null")
+logger.setLevel(logging.INFO)
+handler = logging.StreamHandler(stream=sys.stdout)
+
+class Null:
+    def __init__(self, args, verbose):
+        self.args = args
+        if verbose:
+            logger.setLevel(logging.DEBUG)
+        self.args_to_funcs = {
+            'print-error': self.print_error,
+            'greeting': self.print_greeting,
+            'exit-fail': self.exit_with_failure,
+            'exit-random': self.exit_randomly,
+            'nap': self.take_nap,
+            'dump': self.dump_core
+        }
+
+    def print_error(self):
+        logger.removeHandler(handler)
+        err_handler = logging.StreamHandler(stream=sys.stderr)
+        logger.addHandler(err_handler)
+        logger.error("ERROR: This is an error.")
+
+    def print_greeting(self):
+        logger.info("Hello, world!")
+        logger.debug("And goodbye, world...")
+
+    def exit_with_failure(self):
+        logger.error("Error purposefully induced. Exiting with status code -1...")
+        sys.exit(-1)
+
+    def exit_randomly(self):
+        status_code = random.randint(-50, 50)
+        logger.debug("Exiting with status code {}".format(status_code))
+        sys.exit(status_code)
+
+    def take_nap(self):
+        logger.debug("Going to sleep...")
+        time.sleep(5)
+        logger.debug("Waking up.")
+
+    def dump_core(self):
+        logger.debug("Aborting and dumping core...", stack_info=True)
+        os.abort()
+
+    def execute(self):
+        """
+        Executes command specified by CL arguments.
+        """
+        for arg, val in vars(self.args).items():
+            if val and arg in self.args_to_funcs:
+                self.args_to_funcs[arg]()
+
+def make_arg_parser():
+    """
+    Creates an argparse arguments parser with appropriate options
+    :return: Said argument parser
+    """
+    parser = argparse.ArgumentParser(description=_DESCRIPTION.format(version),
+                                     formatter_class=argparse.RawTextHelpFormatter)
+    options = parser.add_argument_group('options', 'settings for altering program behavior')
+    options.add_argument('-v', '--verbose', action='store_true',
+                         required=False, dest='verbose', default=False,
+                         help='allow the program the gift of speech')
+
+    functions = parser.add_mutually_exclusive_group(required=False)
+    functions.add_argument('-pe', '--print-error', action='store_true',
+                           required=False, dest='print-error', default=False,
+                           help='print out aggressive message to stderr')
+    functions.add_argument('-g', '--greeting', action='store_true',
+                           required=False, dest='greeting', default=False,
+                           help='print out a friendly greeting to stdout')
+    functions.add_argument('-ef', '--exit-fail', action='store_true',
+                           required=False, dest='exit-fail', default=False,
+                           help='print error message and exit with status code -1')
+    functions.add_argument('-er', '--exit-random', action='store_true',
+                           required=False, dest='exit-random', default=False,
+                           help='print error message and exit with random status code within [-50, 50]')
+    functions.add_argument('-n', '--nap', action='store_true',
+                           required=False, dest='nap', default=False,
+                           help='take a short nap')
+    functions.add_argument('-d', '--dump', action='store_true',
+                           required=False, dest='dump', default=False,
+                           help='abort program and dump core')
+    return parser
+
+def main():
+    arg_parser = make_arg_parser()
+    args = arg_parser.parse_args()
+    logger.addHandler(handler)
+
+    executable = Null(args, args.verbose)
+    executable.execute()
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

apps/cli/executables/null/test/test_null.py

+import pytest
+import argparse
+
+# from null.null import Null
+from ..src.null.null import Null
+
+@pytest.fixture()
+def null():
+    null = Null(argparse.Namespace(), True)
+    return null
+
+def test_print_error(null, caplog):
+    null.print_error()
+    assert 'ERROR: This is an error.' in caplog.text
+
+def test_print_greeting(null, caplog):
+    null.print_greeting()
+    assert 'Hello, world!' in caplog.text
+    assert 'And goodbye, world...' in caplog.text
+
+def test_exit_with_failure(null, caplog):
+    with pytest.raises(SystemExit) as e:
+        null.exit_with_failure()
+        assert 'Error purposefully induced. Exiting with status code -1...' in caplog.text
+        assert e.value.code == -1
+
+def test_exit_randomly(null, caplog):
+    with pytest.raises(SystemExit) as e:
+        null.exit_randomly()
+        assert 'Exiting with status code' in caplog.text
+        assert -50 <= e.value.code <= 50
+
+def test_take_nap(null, caplog):
+    null.take_nap()
+    assert 'Going to sleep...' in caplog.text
+    assert 'Waking up.' in caplog.text
\ No newline at end of file

build/recipes/build_pkgs/build_pkgs.py

 import subprocess
 
-def get_pkg_list():
+def get_dirs():
     """
-    Run a couple shell commands to parse the metadata directory for its packages.
-    :return: List of packages in metadata directory
+    Finds all subdirectories containing setup.py files.
+    :return: List of directories as strings.
     """
-    find_proc = subprocess.run(["find", "build/metadata",
-                                  "-name", "meta.yaml"],
-                                 stdout=subprocess.PIPE)
-    paths = find_proc.stdout.decode('utf-8')
-    fmt_paths = paths.replace("build/metadata/", "").replace("/meta.yaml", "")
-    return fmt_paths.split('\n')
+    find = subprocess.run([
+        'find', '.', '-name', 'setup.py', '-not', '-path', './build/recipes/*'
+    ], stdout=subprocess.PIPE)
+    dirs = find.stdout.decode('utf-8').split('\n')
+    dirs_cpy = dirs
+
+    for i, d in enumerate(dirs_cpy):
+        dirs[i] = d.replace('/setup.py', '')
+
+    return dirs
+
+def get_names(dirs):
+    """
+    Generate list of subproject names based on the rule that the name of the
+    subproject directory will be the name of the subproject.
+    :return: List of names as strings.
+    """
+    names = []
+    for d in dirs:
+        if d != '':
+            name = d.split('/')[-1]
+
+            if name == "archive":
+                # Case with ./services/archive having special dir structure
+                name = "services"
+            names.append(name)
+
+    return names
 
 class Recipe:
     def __init__(self, buildout, name, options):
@@ -23,7 +45,7 @@ class Recipe:
         """
         self.name = name
         self.options = options
-        self.pkg_list = get_pkg_list()
+        self.pkg_list = get_names(get_dirs())
 
     def install(self):
         """

build/recipes/setup_to_meta/setup_to_meta.py

@@ -46,18 +46,20 @@ class MetadataGenerator:
         """
         reqs_string = ''
         reqs_list = ''
+        reqs_string += 'requirements:\n'
+        build_reqs = '  build:\n'
+        run_reqs = '  run:\n'
+        host_reqs = '  host:\n'
+        reqs_list += '    - python={}\n'.format(PYTHON_VERSION)
+
         if 'install_requires' in self.setup.keys():
-            reqs_string += 'requirements:\n'
-            build_reqs = '  build:\n'
-            run_reqs = '  run:\n'
-            host_reqs = '  host:\n'
-            reqs_list += '    - python={}\n'.format(PYTHON_VERSION)
             for req in self.setup['install_requires']:
                 reqs_list += '    - {}\n'.format(req)
-            reqs_string += build_reqs + reqs_list + \
-                           run_reqs + reqs_list + \
-                           host_reqs + reqs_list + \
-                           '\n'
+
+        reqs_string += build_reqs + reqs_list + \
+                       run_reqs + reqs_list + \
+                       host_reqs + reqs_list + \
+                       '\n'
         return reqs_string
 
     def fmt_test(self):
@@ -88,8 +90,11 @@ class MetadataGenerator:
 
     def generate(self):
         # Filter numpy etc. out of the requirements
-        self.setup['install_requires'] = [req for req in self.setup['install_requires'] if req != 'numpy']
-    
+        try:
+            self.setup['install_requires'] = [req for req in self.setup['install_requires'] if req != 'numpy']
+        except KeyError:
+            pass
+
         name = self.setup['name']
         version = self.setup['version']
         entry_points = self.fmt_ep()

build/tools/transfer_to_builder.py

-import subprocess, paramiko, fnmatch, os
+import subprocess
+import paramiko
+import fnmatch
+import os
+import getpass
+
 from scp import SCPClient
 
 def get_build_pkg_names():
@@ -23,7 +28,11 @@ def create_ssh_client(server):
     client = paramiko.SSHClient()
     client.load_system_host_keys()
     client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    client.connect(server)
+
+    username = input("Enter NRAO username: ")
+    password = getpass.getpass(prompt="Enter NRAO password: ")
+
+    client.connect(server, username=username, password=password)
     return client
 
 def transfer_packages(pkg_names):
@@ -34,9 +43,9 @@ def transfer_packages(pkg_names):
     if len(pkg_names):
         builder_addr = "builder.aoc.nrao.edu"
         builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch"
-        ssh = create_ssh_client(builder_addr)
-        with SCPClient(ssh.get_transport()) as scp:
-            [scp.put(pkg, builder_path) for pkg in pkg_names]
+        with create_ssh_client(builder_addr) as ssh:
+            with SCPClient(ssh.get_transport()) as scp:
+                [scp.put(pkg, builder_path) for pkg in pkg_names]
         cmd_cd = "cd {}".format(builder_path)
         cmd_index = "conda index .."
         cmd_chmod = "chmod -f 664 *"

environment.yml

@@ -13,6 +13,7 @@ dependencies:
   - jxmlease=1.0
   - lxml=4.5
   - mysqlclient=1.4
+  - paramiko
   - pandas=1.0
   - pendulum=2.1
   - pid=2.2