diff --git a/.gitignore b/.gitignore
index 37bec31ad4e16a0912958f4fede2c304b5005649..0e5ed67a5153d0321ecac4f9517f0f0e896e6223 100644
--- a/.gitignore
+++ b/.gitignore
@@ -57,5 +57,5 @@ projects_checklist.txt
.ipynb_checkpoints
deps.png
build/pkgs
-eggs
-parts
\ No newline at end of file
+**/eggs
+**/parts
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 79d404658d21dd65b7f8ba54abcc1c23d91e20be..9aa967a1c4b5a7478695dd3eb770f4101b33897d 100644
--- a/Makefile
+++ b/Makefile
@@ -15,3 +15,7 @@ metadata:
.PHONY: build
build:
buildout parts=build_pkgs name=all
+
+.PHONY: test
+test:
+ buildout parts=test name=all
\ No newline at end of file
diff --git a/apps/cli/executables/null/setup.py b/apps/cli/executables/null/setup.py
index 91b6b575c50c3c08f10dbc16446d91301d71d40c..c7a5dcaf94e82b73b2284a472f99e3d09c767d2c 100644
--- a/apps/cli/executables/null/setup.py
+++ b/apps/cli/executables/null/setup.py
@@ -7,9 +7,6 @@ from setuptools import setup
VERSION = open('src/null/_version.py').readlines()[-1].split()[-1].strip("\"'")
README = Path('README.md').read_text()
-# requires = [
-# ]
-
tests_require = [
'pytest>=5.4,<6.0'
]
@@ -22,11 +19,10 @@ setup(
author_email='dms-ssa@nrao.edu',
url='TBD',
license="GPL",
- # install_requires=requires,
tests_require=tests_require,
keywords=[],
packages=['null'],
- package_dir={'':'src'},
+ package_dir={'': 'src'},
classifiers=[
'Programming Language :: Python :: 3.8'
],
diff --git a/apps/cli/executables/null/src/null/null.py b/apps/cli/executables/null/src/null/null.py
index 6f806ae60910206b36c8a97d3fb9eca700595529..1cdacad80b9c8e028ce8d0f4ddd6f31bb4dae0e6 100644
--- a/apps/cli/executables/null/src/null/null.py
+++ b/apps/cli/executables/null/src/null/null.py
@@ -16,47 +16,75 @@ logger = logging.getLogger("null")
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(stream=sys.stdout)
-class Null:
- def __init__(self, args, verbose):
- self.args = args
- if verbose:
- logger.setLevel(logging.DEBUG)
- self.args_to_funcs = {
- 'print-error': self.print_error,
- 'greeting': self.print_greeting,
- 'exit-fail': self.exit_with_failure,
- 'exit-random': self.exit_randomly,
- 'nap': self.take_nap,
- 'dump': self.dump_core
- }
- def print_error(self):
- logger.removeHandler(handler)
- err_handler = logging.StreamHandler(stream=sys.stderr)
- logger.addHandler(err_handler)
- logger.error("ERROR: This is an error.")
+def print_error():
+ """
+ Logs an error message to stderr.
+ """
+ logger.removeHandler(handler)
+ err_handler = logging.StreamHandler(stream=sys.stderr)
+ logger.addHandler(err_handler)
+ logger.error("ERROR: This is an error.")
+
+
+def print_greeting():
+ """
+ Prints a friendly greeting to stdout
+ """
+ logger.info("Hello, world!")
+ logger.debug("And goodbye, world...")
+
+
+def exit_with_failure():
+ """
+ Exits with status code -1
+ """
+ logger.error("Error purposefully induced. Exiting with status code -1...")
+ sys.exit(-1)
+
+
+def exit_randomly():
+ """
+ Exits with a random status code between -50 and 50
+ """
+ status_code = random.randint(-50, 50)
+ logger.debug("Exiting with status code {}".format(status_code))
+ sys.exit(status_code)
- def print_greeting(self):
- logger.info("Hello, world!")
- logger.debug("And goodbye, world...")
- def exit_with_failure(self):
- logger.error("Error purposefully induced. Exiting with status code -1...")
- sys.exit(-1)
+def take_nap():
+ """
+ Sleeps for 5 seconds
+ """
+ logger.debug("Going to sleep...")
+ time.sleep(5)
+ logger.debug("Waking up.")
- def exit_randomly(self):
- status_code = random.randint(-50, 50)
- logger.debug("Exiting with status code {}".format(status_code))
- sys.exit(status_code)
- def take_nap(self):
- logger.debug("Going to sleep...")
- time.sleep(5)
- logger.debug("Waking up.")
+def dump_core():
+ """
+ Makes a call to os.abort() which dumps the core
+ """
+ logger.debug("Aborting and dumping core...", stack_info=True)
+ os.abort()
+
- def dump_core(self):
- logger.debug("Aborting and dumping core...", stack_info=True)
- os.abort()
+class Null:
+ """
+ Null executable that executes null functionality based on arguments given
+ """
+ def __init__(self, args: argparse.Namespace, verbose: bool):
+ self.args = args
+ if verbose:
+ logger.setLevel(logging.DEBUG)
+ self.args_to_funcs = {
+ 'print-error': print_error,
+ 'greeting': print_greeting,
+ 'exit-fail': exit_with_failure,
+ 'exit-random': exit_randomly,
+ 'nap': take_nap,
+ 'dump': dump_core
+ }
def execute(self):
"""
@@ -66,7 +94,8 @@ class Null:
if val and arg in self.args_to_funcs:
self.args_to_funcs[arg]()
-def make_arg_parser():
+
+def make_arg_parser() -> argparse.ArgumentParser:
"""
Creates an argparse arguments parser with appropriate options
:return: Said argument parser
@@ -99,6 +128,7 @@ def make_arg_parser():
help='abort program and dump core')
return parser
+
def main():
arg_parser = make_arg_parser()
args = arg_parser.parse_args()
@@ -107,5 +137,6 @@ def main():
executable = Null(args, args.verbose)
executable.execute()
+
if __name__ == '__main__':
- main()
\ No newline at end of file
+ main()
diff --git a/apps/cli/executables/null/test/test_null.py b/apps/cli/executables/null/test/test_null.py
index 7dd49717c246daa13db90f959016aa4ce59af8dc..83f6b5fc4564742e1c7a881ec33707bf58d6f088 100644
--- a/apps/cli/executables/null/test/test_null.py
+++ b/apps/cli/executables/null/test/test_null.py
@@ -1,36 +1,41 @@
import pytest
import argparse
-# from null.null import Null
-from ..src.null.null import Null
+from null.null import Null
+
@pytest.fixture()
def null():
null = Null(argparse.Namespace(), True)
return null
+
def test_print_error(null, caplog):
null.print_error()
assert 'ERROR: This is an error.' in caplog.text
+
def test_print_greeting(null, caplog):
null.print_greeting()
assert 'Hello, world!' in caplog.text
assert 'And goodbye, world...' in caplog.text
+
def test_exit_with_failure(null, caplog):
with pytest.raises(SystemExit) as e:
null.exit_with_failure()
assert 'Error purposefully induced. Exiting with status code -1...' in caplog.text
assert e.value.code == -1
+
def test_exit_randomly(null, caplog):
with pytest.raises(SystemExit) as e:
null.exit_randomly()
assert 'Exiting with status code' in caplog.text
assert -50 <= e.value.code <= 50
+
def test_take_nap(null, caplog):
null.take_nap()
assert 'Going to sleep...' in caplog.text
- assert 'Waking up.' in caplog.text
\ No newline at end of file
+ assert 'Waking up.' in caplog.text
diff --git a/apps/cli/executables/null/__init__.py b/build/recipes/build_pkgs/__init__.py
similarity index 100%
rename from apps/cli/executables/null/__init__.py
rename to build/recipes/build_pkgs/__init__.py
diff --git a/build/recipes/build_pkgs/build_pkgs.py b/build/recipes/build_pkgs/build_pkgs.py
index f542b7b3b63d112b31c109be6e3f02f4552439f3..7b3324ad8d465a489f0302974870a1b2fd0d8015 100644
--- a/build/recipes/build_pkgs/build_pkgs.py
+++ b/build/recipes/build_pkgs/build_pkgs.py
@@ -1,10 +1,18 @@
+import logging
import subprocess
+from typing import List, Dict, Any, Callable, Optional
-def get_dirs():
+from zc.buildout.buildout import Buildout, Options
+
+logger = logging.getLogger("buildout/build_pkgs")
+
+
+def get_dirs() -> List[str]:
"""
Finds all subdirectories containing setup.py files.
:return: List of directories as strings.
"""
+ logger.debug("Getting list of directories containing setup.py files...")
find = subprocess.run([
'find', '.', '-name', 'setup.py', '-not', '-path', './build/recipes/*'
], stdout=subprocess.PIPE)
@@ -14,14 +22,17 @@ def get_dirs():
for i, d in enumerate(dirs_cpy):
dirs[i] = d.replace('/setup.py', '')
+ logger.debug("Done getting directories.")
return dirs
-def get_names(dirs):
+
+def get_names(dirs: List[str]) -> List[str]:
"""
Generate list of subproject names based on the rule that the name of the
subproject directory will be the name of the subproject.
:return: List of names as strings.
"""
+ logger.debug("Generating list of subproject names...")
names = []
for d in dirs:
if d != '':
@@ -32,10 +43,12 @@ def get_names(dirs):
name = "services"
names.append(name)
+ logger.debug("Done generating.")
return names
+
class Recipe:
- def __init__(self, buildout, name, options):
+ def __init__(self, buildout: Optional[Buildout], name: str, options: Options):
"""
Initializes fields needed for recipe.
:param buildout: (Boilerplate) Dictionary of options from buildout section
@@ -47,19 +60,22 @@ class Recipe:
self.options = options
self.pkg_list = get_names(get_dirs())
- def install(self):
+ def install(self) -> Any:
"""
Install method that runs when recipe has components it needs to install.
:return: Paths to files, as strings, created by the recipe.
"""
if self.options['name'] == "all":
+ logger.warning("WARNING: You've requested all packages to be built. This will take a long time.")
+ logger.warning("If only one or a few packages have changed, consider specifying them "
+ "in a comma-separated list.")
pkgs = self.pkg_list
else:
pkgs = self.options['name'].split(',')
for p in pkgs:
if p not in self.pkg_list or p == '':
- print("Package {} not valid. Skipping.".format(p))
+ logger.error(f"Package {p} not valid. Skipping.")
continue
subprocess.run(["conda", "build", "build/metadata/{}".format(p), "--output-folder", "build/pkgs/"],
stdout=subprocess.PIPE)
@@ -69,4 +85,4 @@ class Recipe:
return self.options.created()
- update = install
\ No newline at end of file
+ update = install
diff --git a/build/recipes/build_pkgs/setup.py b/build/recipes/build_pkgs/setup.py
index db69c236a8a32ec6f0a16d2be9ca896b83df5875..04e432da6c878e5a4f3243cdf83b10081671854a 100644
--- a/build/recipes/build_pkgs/setup.py
+++ b/build/recipes/build_pkgs/setup.py
@@ -3,6 +3,6 @@ from setuptools import setup
setup(
name='build_pkgs',
version='0.1',
- py_modules = ['build_pkgs'],
- entry_points = {"zc.buildout": ["default=build_pkgs:Recipe"]},
-)
\ No newline at end of file
+ py_modules=['build_pkgs'],
+ entry_points={"zc.buildout": ["default=build_pkgs:Recipe"]},
+)
diff --git a/build/recipes/build_pkgs/test/__init__.py b/build/recipes/build_pkgs/test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/build/recipes/build_pkgs/test/conftest.py b/build/recipes/build_pkgs/test/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..46126efdfc2d3f2a05a8cf0e83d51c1acc8bdc3f
--- /dev/null
+++ b/build/recipes/build_pkgs/test/conftest.py
@@ -0,0 +1,18 @@
+import pytest
+import zc.buildout.testing
+from .. import build_pkgs
+
+
+@pytest.fixture(scope='module')
+def recipe() -> build_pkgs.Recipe:
+ """
+ pytest fixture that initializes zc.buildout objects for use in testing.
+ Initializes Buildout, Options, and Recipe objects.
+
+ :return: Initialized recipe object for build_pkgs
+ """
+ from .. import build_pkgs
+ buildout = zc.buildout.testing.Buildout()
+ options = buildout.Options(buildout, 'build_pkgs', {'recipe': 'build_pkgs', 'name': 'null'})
+ recipe = build_pkgs.Recipe(buildout=buildout, name=None, options=options)
+ return recipe
diff --git a/build/recipes/build_pkgs/test/test_build_pkgs.py b/build/recipes/build_pkgs/test/test_build_pkgs.py
new file mode 100644
index 0000000000000000000000000000000000000000..188c5e670ca796fad5dc6d275b60b6b9670b71c2
--- /dev/null
+++ b/build/recipes/build_pkgs/test/test_build_pkgs.py
@@ -0,0 +1,32 @@
+import os
+from typing import List
+
+from .. import build_pkgs
+
+
+class TestBuildPkgs:
+ def test_get_names(self):
+ """
+ Test that build_pkgs correctly gets the package name from
+ :return:
+ """
+ d = 'apps/cli/executables/null'
+ assert build_pkgs.get_names([d]) == ['null']
+
+ def test_get_dirs(self):
+ """
+ Test that build_pkgs correctly finds and stores directory paths
+ of directories containing setup.py files.
+ """
+ assert './apps/cli/executables/null' in build_pkgs.get_dirs()
+
+ def test_output(self, recipe: build_pkgs.Recipe):
+ """
+ Test that the package specified in the recipe has been built correctly.
+ """
+ created = recipe.install()
+
+ for path in created:
+ if len(path) > 0:
+ assert path is not None, "conda build failed to build package"
+ assert os.path.exists(path)
diff --git a/build/recipes/setup_to_meta/__init__.py b/build/recipes/setup_to_meta/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/build/recipes/setup_to_meta/setup.py b/build/recipes/setup_to_meta/setup.py
index 24ade227647ff5b327d2f7a892a04a6a98ea004e..8a2ebea1329a7390c9016fa06838e78cb2843ecf 100644
--- a/build/recipes/setup_to_meta/setup.py
+++ b/build/recipes/setup_to_meta/setup.py
@@ -3,6 +3,6 @@ from setuptools import setup
setup(
name='setup_to_meta',
version='0.1',
- py_modules = ['setup_to_meta'],
- entry_points = {"zc.buildout": ["default=setup_to_meta:Recipe"]},
-)
\ No newline at end of file
+ py_modules=['setup_to_meta'],
+ entry_points={"zc.buildout": ["default=setup_to_meta:Recipe"]},
+)
diff --git a/build/recipes/setup_to_meta/setup_to_meta.py b/build/recipes/setup_to_meta/setup_to_meta.py
index e65c1ba2ae9e104a35dfb543cc9985c0224ab107..783afd3766ca01ad586f53953463bde491396c94 100644
--- a/build/recipes/setup_to_meta/setup_to_meta.py
+++ b/build/recipes/setup_to_meta/setup_to_meta.py
@@ -1,13 +1,22 @@
+import os
import json
+import logging
+import subprocess
+from typing import List, Any, Dict, Callable, Optional
-import setuptools, importlib, subprocess, os, re
+from zc.buildout.buildout import Buildout, Options
PYTHON_VERSION = '3.8'
+logger = logging.getLogger("buildout/setup_to_meta")
-def write_metafile(metadata, filepath):
+
+def write_metafile(metadata: str, filepath: str):
"""
Writes given metadata to file with given path.
+ :param metadata: String containing conda recipe metadata to be written
+ :param filepath: String containing the path to conda recipe file (meta.yaml)
"""
+ logger.debug(f"Writing meta.yaml file at {filepath}...")
try:
os.makedirs(filepath[:-10])
except FileExistsError:
@@ -15,18 +24,18 @@ def write_metafile(metadata, filepath):
with open(filepath, 'w') as f:
f.write(metadata)
+ logger.debug("Done writing.")
class MetadataGenerator:
"""
Uses given info extracted from setup.py file to fill out metadata template.
"""
-
- def __init__(self, setup, path):
+ def __init__(self, setup: Dict[str, str], path: str):
self.setup = setup
self.path = path
- def fmt_ep(self):
+ def fmt_ep(self) -> str:
"""
Format entry points section of metadata.
:return: Formatted string if entry points exists; else empty string.
@@ -39,7 +48,7 @@ class MetadataGenerator:
ep_string += ' '
return ep_string
- def fmt_reqs(self):
+ def fmt_reqs(self) -> str:
"""
Format requirements section of metadata.
:return: Formatted string if requirements exists; else empty string.
@@ -62,7 +71,7 @@ class MetadataGenerator:
'\n'
return reqs_string
- def fmt_test(self):
+ def fmt_test(self) -> str:
"""
Format test section of metadata.
NOTE: May need further tweaking to be smarter based on individual project
@@ -88,7 +97,8 @@ class MetadataGenerator:
)
return test_string
- def generate(self):
+ def generate(self) -> str:
+ logger.debug(f"Generating meta.yaml file from {self.path}...")
# Filter numpy etc. out of the requirements
try:
self.setup['install_requires'] = [req for req in self.setup['install_requires'] if req != 'numpy']
@@ -106,7 +116,8 @@ class MetadataGenerator:
with open('build/tools/metafile_template.txt', 'r') as f:
metadata = f.read()
-
+
+ logger.debug("Done generating.")
return metadata.format(
name = name,
version = version,
@@ -119,22 +130,25 @@ class MetadataGenerator:
)
-def parse_setup(d):
+def parse_setup(d: str) -> Dict[str, str]:
"""
Function for running parse_setup.py on each directory with a setup.py file.
NOTE: Contains a hack for getting parse_setup.py to run in each directory.
:param d: Directory with a setup.py file.
:return: Data collected from parse_setup.py.
"""
+ logger.debug(f"Parsing setup.py at {d}...")
subprocess.run(['cp', 'build/tools/parse_setup.py', d])
os.chdir(d)
proc = subprocess.run(['python3', 'parse_setup.py'], stdout=subprocess.PIPE)
os.chdir(root)
subprocess.run(['rm', '{}/parse_setup.py'.format(d)])
+ logger.debug("Done parsing.")
return json.loads(proc.stdout)
-def get_outputs(names):
+
+def get_outputs(names: List[str]) -> List[str]:
"""
Generate list of metadata files that will be created.
:param dirs: List of dirs of all subprojects with a setup.py file.
@@ -146,11 +160,12 @@ def get_outputs(names):
return outputs
-def get_dirs():
+def get_dirs() -> List[str]:
"""
Finds all subdirectories containing setup.py files.
:return: List of directories as strings.
"""
+ logger.debug("Finding list of directories containing setup.py files...")
find = subprocess.run([
'find', '.', '-name', 'setup.py', '-not', '-path', './build/recipes/*'
], stdout=subprocess.PIPE)
@@ -160,14 +175,17 @@ def get_dirs():
for i, d in enumerate(dirs_cpy):
dirs[i] = d.replace('/setup.py', '')
+ logger.debug("Done finding directories.")
return dirs
-def get_names(dirs):
+
+def get_names(dirs: List[str]) -> List[str]:
"""
Generate list of subproject names based on the rule that the name of the
subproject directory will be the name of the subproject.
:return: List of names as strings.
"""
+ logger.debug("Getting list of names...")
names = []
for d in dirs:
if d != '':
@@ -178,9 +196,11 @@ def get_names(dirs):
name = "services"
names.append(name)
+ logger.debug("Done getting list of names.")
return names
-def del_substrings(s, substrings):
+
+def del_substrings(s: str, substrings: List[str]):
"""
Function for deleting multiple substrings from a string.
:param s: String to be modified.
@@ -192,15 +212,17 @@ def del_substrings(s, substrings):
return s
+
root = os.getcwd()
+
class Recipe:
"""
Buildout Recipe class.
For more detailed information, see the link.
http://www.buildout.org/en/latest/topics/writing-recipes.html
"""
- def __init__(self, buildout, name, options):
+ def __init__(self, buildout: Optional[Buildout], name: Optional[str], options: Options):
"""
Initializes fields needed for recipe.
:param buildout: (Boilerplate) Dictionary of options from buildout section
@@ -213,8 +235,7 @@ class Recipe:
self.outputs = get_outputs(self.names)
self.options = options
- # TODO: Keep track of path in setup_dict
- def install(self):
+ def install(self) -> Any:
"""
Install method that runs when recipe has components it needs to install.
:return: Paths to files, as strings, created by the recipe.
@@ -224,10 +245,10 @@ class Recipe:
setup_data = parse_setup(d)
metadata = MetadataGenerator(setup_data, d).generate()
write_metafile(metadata, self.outputs[i])
- # Pass created file into options.created()
+ # Buildout-specific operation: pass created file into options.created()
self.options.created(self.outputs[i])
return self.options.created()
# No special procedure for updating vs. installing
- update = install
\ No newline at end of file
+ update = install
diff --git a/build/recipes/setup_to_meta/test/__init__.py b/build/recipes/setup_to_meta/test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/build/recipes/setup_to_meta/test/conftest.py b/build/recipes/setup_to_meta/test/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..877092d84ed217e63b96ae18f0b49205f9d70abd
--- /dev/null
+++ b/build/recipes/setup_to_meta/test/conftest.py
@@ -0,0 +1,27 @@
+import pytest
+import zc.buildout.testing
+
+from .. import setup_to_meta
+
+
+@pytest.fixture(scope='module')
+def recipe() -> setup_to_meta.Recipe:
+ """
+ pytest fixture that initializes zc.buildout objects for use in testing.
+ Initializes Buildout, Options, and Recipe objects.
+
+ :return: Initialized recipe object for setup_to_meta
+ """
+ from .. import setup_to_meta
+ buildout = zc.buildout.testing.Buildout()
+ options = buildout.Options(
+ buildout,
+ 'gen_metadata',
+ {'recipe': 'setup_to_meta'}
+ )
+ recipe = setup_to_meta.Recipe(
+ buildout=buildout,
+ name=None,
+ options=options
+ )
+ return recipe
diff --git a/build/recipes/setup_to_meta/test/test_setup_to_meta.py b/build/recipes/setup_to_meta/test/test_setup_to_meta.py
new file mode 100644
index 0000000000000000000000000000000000000000..8cb514d110f903e577b61fdaa503fe139a9bad03
--- /dev/null
+++ b/build/recipes/setup_to_meta/test/test_setup_to_meta.py
@@ -0,0 +1,56 @@
+from typing import Dict, List
+
+from .. import setup_to_meta
+
+
+class TestSetupToMeta:
+ def test_del_substrings(self):
+ """
+ Tests that del_substrings function properly deletes substrings from a given string
+ """
+ replaced = setup_to_meta.del_substrings('heallob, woarlcd', ['a', 'b', 'c'])
+ assert replaced == 'hello, world'
+
+ def test_get_names(self):
+ """
+ Tests that setup_to_meta correctly gets the package name from the package path
+ """
+ d = 'apps/cli/executables/null'
+ assert setup_to_meta.get_names([d]) == ['null']
+
+ def test_get_dirs(self):
+ """
+ Tests that setup_to_meta correctly finds and stores directory paths
+ of packages containing setup.py files
+ """
+ assert './apps/cli/executables/null' in setup_to_meta.get_dirs()
+
+ def test_get_outputs(self):
+ """
+ Tests that setup_to_meta correctly generates a list of output paths given
+ a list of package names.
+ """
+ assert setup_to_meta.get_outputs(['null']) == ['build/metadata/null/meta.yaml']
+
+ def test_parse_setup(self):
+ """
+ Tests that parse_setup correctly parses a setup.py file into a dictionary.
+ """
+ setup_data = setup_to_meta.parse_setup('apps/cli/executables/null')
+ keys = ['name', 'version', 'description', 'license']
+ for key in keys:
+ assert key in setup_data
+
+ def test_output(self, recipe: setup_to_meta.Recipe):
+ """
+ Test that metadata was successfully created and contains data.
+
+ Checking for 'package' is an arbitrary check for content that will always
+ occur in a correct recipe.
+ :param recipe: Fixture that initializes recipe class in setup_to_meta.py
+ """
+ created = recipe.install()
+
+ for path in created:
+ with open(path, 'r') as f:
+ assert 'package' in f.read()
diff --git a/build/recipes/test_recipes/__init__.py b/build/recipes/test_recipes/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/build/recipes/test_recipes/setup.py b/build/recipes/test_recipes/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..c97dae42aaa38bdbdab402d89a8a0d437d0f7071
--- /dev/null
+++ b/build/recipes/test_recipes/setup.py
@@ -0,0 +1,8 @@
+from setuptools import setup
+
+setup(
+ name='test_recipes',
+ version='0.1',
+ py_modules = ['test_recipes'],
+ entry_points = {"zc.buildout": ["default=test_recipes:Recipe"]},
+)
\ No newline at end of file
diff --git a/build/recipes/test_recipes/test/__init__.py b/build/recipes/test_recipes/test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/build/recipes/test_recipes/test/test_test_recipes.py b/build/recipes/test_recipes/test/test_test_recipes.py
new file mode 100644
index 0000000000000000000000000000000000000000..912bc274ba93235ccdc85c82e1949993855cafd5
--- /dev/null
+++ b/build/recipes/test_recipes/test/test_test_recipes.py
@@ -0,0 +1,13 @@
+from .. import test_recipes
+
+
+class TestRecipes:
+ def test_get_recipes(self):
+ """
+ Test that test_recipes is able to successfully retrieve names and paths
+ for all buildout recipes.
+ """
+ recipe_names = ['setup_to_meta', 'build_pkgs', 'test_recipes']
+ recipes = test_recipes.get_recipes()
+ for recipe in recipe_names:
+ assert recipe in recipes
diff --git a/build/recipes/test_recipes/test_recipes.py b/build/recipes/test_recipes/test_recipes.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd09a2c49ee94434e2a4e6823c0be5029e3af9da
--- /dev/null
+++ b/build/recipes/test_recipes/test_recipes.py
@@ -0,0 +1,76 @@
+import subprocess
+import logging
+from typing import Dict, Optional
+
+from zc.buildout.buildout import Buildout, Options
+
+logger = logging.getLogger("buildout/test_recipes")
+
+
+def get_recipes() -> Dict[str, str]:
+ """
+ Get all currently installed buildout recipes (including this one!)
+ :return: Dictionary with format {recipe_name: recipe_path_from_root,}
+ """
+ logger.debug("Getting list of recipes...")
+ recipes = {}
+ find = subprocess.run(['find', './build/recipes', '-name', 'setup.py'],
+ stdout=subprocess.PIPE)
+ paths = find.stdout.decode('utf-8').split('\n')
+ dirs = []
+ names = []
+ for p in paths:
+ if len(p) > 1:
+ # Exclude empty result from find
+ dirs.append(p.replace('/setup.py', ''))
+ names.append(p.split('/')[-2])
+ for d, n in zip(dirs, names):
+ recipes[n] = d
+ logger.debug("Done getting recipes.")
+
+ return recipes
+
+
+class Recipe:
+ """
+ Buildout Recipe class.
+ For more detailed information, see the link.
+ http://www.buildout.org/en/latest/topics/writing-recipes.html
+ """
+
+ def run_test(self, recipe: str):
+ """
+ Run test for given recipe.
+ :param recipe: Name of recipe to be run.
+ """
+ logger.debug(f"Running tests for recipe {recipe}...")
+ subprocess.run(['pytest', '-vv', '--log-level=DEBUG', '--showlocals',
+ self.recipes[recipe]])
+
+ def __init__(self, buildout: Optional[Buildout], name: str, options: Options):
+ """
+ Initializes fields needed for recipe.
+ :param buildout: (Boilerplate) Dictionary of options from buildout section
+ of buildout.cfg
+ :param name: (Boilerplate) Name of section that uses this recipe.
+ :param options: (Boilerplate) Options of section that uses this recipe.
+ """
+ self.name = name
+ self.options = options
+ self.recipes = get_recipes()
+ self.choice = options['name']
+
+ def install(self):
+ """
+ Install method that runs when recipe has components it needs to install.
+
+ In this case, nothing is "installed" per se.
+ :return: Paths to files, as strings, created by the recipe.
+ """
+ if self.choice == 'all':
+ # Run tests for all recipes
+ for recipe in self.recipes:
+ self.run_test(recipe)
+ else:
+ if self.choice in self.recipes:
+ self.run_test(self.choice)
diff --git a/build/tools/parse_setup.py b/build/tools/parse_setup.py
index 8acd807b95980c86bcedb4f1a2b220ea5300da79..6d1c8ab8fd78bd0096d40322fc3ca926dacf5aea 100644
--- a/build/tools/parse_setup.py
+++ b/build/tools/parse_setup.py
@@ -3,6 +3,8 @@ import setuptools
import json
data = {}
+
+
def my_setup(*args, **kwargs):
"""
A replacement for setuptools.setup().
@@ -20,6 +22,7 @@ def my_setup(*args, **kwargs):
for field in fields:
data[field] = kwargs.get(field)
+
def main():
# Author of these shenanigans: Daniel Lyons (but you already knew that)
@@ -36,5 +39,6 @@ def main():
# Instead of exiting, we now have populated our global variable, without doing any parsing
json.dump(data, sys.stdout)
+
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/build/tools/transfer_to_builder.py b/build/tools/transfer_to_builder.py
index 64d7e4fc7ff61c4fdc8b32d4f8c7fd627f7981d8..4f9b67094afb4ecaafb325c1b8b7635dffa36dca 100644
--- a/build/tools/transfer_to_builder.py
+++ b/build/tools/transfer_to_builder.py
@@ -1,25 +1,38 @@
-import subprocess
-import paramiko
-import fnmatch
import os
+import sys
+import fnmatch
import getpass
+import logging
+from typing import List
+
+import paramiko
+import subprocess
from scp import SCPClient
-def get_build_pkg_names():
+logger = logging.getLogger("buildtools/transfer_to_builder")
+logger.setLevel(logging.INFO)
+hander = logging.StreamHandler(stream=sys.stdout)
+
+
+def get_build_pkg_names() -> List[str]:
"""
Search through pkgs directory for built .tar.bz2 packages
:return: List of package archive file names
"""
pkg_names = []
d = "build/pkgs/noarch/"
- for file in os.listdir(d):
- if fnmatch.fnmatch(file, "*.tar.bz2"):
- pkg_names.append(d + file)
+ try:
+ for file in os.listdir(d):
+ if fnmatch.fnmatch(file, "*.tar.bz2"):
+ pkg_names.append(d + file)
+ except FileNotFoundError as e:
+ logger.error(e)
return pkg_names
-def create_ssh_client(server):
+
+def create_ssh_client(server: str) -> paramiko.SSHClient:
"""
Use paramiko to load SSH keys if they exist and set up an SSH connection to a server.
:param server: The server to connect to
@@ -29,17 +42,28 @@ def create_ssh_client(server):
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
- username = input("Enter NRAO username: ")
- password = getpass.getpass(prompt="Enter NRAO password: ")
+ while True:
+ username = input("Enter NRAO username: ")
+ password = getpass.getpass(prompt="Enter NRAO password: ")
+
+ try:
+ client.connect(server, username=username, password=password)
+ except paramiko.AuthenticationException as e:
+ logger.error(e)
+ logger.error("Invalid credentials. Try again.")
+ continue
+ break
- client.connect(server, username=username, password=password)
return client
-def transfer_packages(pkg_names):
+
+def transfer_packages(pkg_names: List[str]):
"""
Use shell commands to transfer build archives to builder and update its conda package index.
:param pkg_names: Names of the .tar.bz2 files for the built packages.
"""
+ logger.addHandler(hander)
+
if len(pkg_names):
builder_addr = "builder.aoc.nrao.edu"
builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch"
@@ -54,7 +78,9 @@ def transfer_packages(pkg_names):
cmd_index + " && " +
cmd_chmod])
else:
- print("No packages found in build/pkgs/noarch. Did conda build successfully build the package(s)?")
+ logger.error("No packages found in build/pkgs/noarch. "
+ "Did conda build successfully build the package(s)?")
+
if __name__ == "__main__":
- transfer_packages(get_build_pkg_names())
\ No newline at end of file
+ transfer_packages(get_build_pkg_names())
diff --git a/buildout.cfg b/buildout.cfg
index 5ae4f1cc3931524a3c061e563efccd0d0a489b60..58fcedbeb7a329c3e1ca06c0e77023c95c2ff0b6 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -3,13 +3,22 @@
# e.g. buildout parts=build_pkgs will specify build_pkgs
# as a part that should be installed
[buildout]
-develop = build/recipes/setup_to_meta build/recipes/build_pkgs
+develop = build/recipes/setup_to_meta build/recipes/build_pkgs build/recipes/test_recipes
+
+# Section for testing buildout recipes
+# Depends on `name` in [buildout] to specify which recipe to test
+# Specify name via command line using `buildout parts=test name={name}`
+# Specify 'all' to test all recipes
+[test]
+recipe = test_recipes
+name = ${buildout:name}
# Section for building internal tools using conda
# Depends on gen_metadata
# Depends on `name` in [buildout] to specify which package to install
# Specify name via command line using
# `buildout parts=build_pkgs name={name}`
+# Specify 'all' to build all packages (this takes a LONG time)
[build_pkgs]
=> gen_metadata
recipe = build_pkgs