diff --git a/.gitignore b/.gitignore index 37bec31ad4e16a0912958f4fede2c304b5005649..0e5ed67a5153d0321ecac4f9517f0f0e896e6223 100644 --- a/.gitignore +++ b/.gitignore @@ -57,5 +57,5 @@ projects_checklist.txt .ipynb_checkpoints deps.png build/pkgs -eggs -parts \ No newline at end of file +**/eggs +**/parts \ No newline at end of file diff --git a/Makefile b/Makefile index 79d404658d21dd65b7f8ba54abcc1c23d91e20be..9aa967a1c4b5a7478695dd3eb770f4101b33897d 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,7 @@ metadata: .PHONY: build build: buildout parts=build_pkgs name=all + +.PHONY: test +test: + buildout parts=test name=all \ No newline at end of file diff --git a/apps/cli/executables/null/test/test_null.py b/apps/cli/executables/null/test/test_null.py index 7dd49717c246daa13db90f959016aa4ce59af8dc..ae8acdf8fec8828de51b8569fcb0c7be99293d8a 100644 --- a/apps/cli/executables/null/test/test_null.py +++ b/apps/cli/executables/null/test/test_null.py @@ -1,8 +1,7 @@ import pytest import argparse -# from null.null import Null -from ..src.null.null import Null +from null.null import Null @pytest.fixture() def null(): diff --git a/apps/cli/executables/null/__init__.py b/build/recipes/build_pkgs/__init__.py similarity index 100% rename from apps/cli/executables/null/__init__.py rename to build/recipes/build_pkgs/__init__.py diff --git a/build/recipes/build_pkgs/build_pkgs.py b/build/recipes/build_pkgs/build_pkgs.py index f542b7b3b63d112b31c109be6e3f02f4552439f3..304d2bad22677d40b0e410e899aafd56ed66dd12 100644 --- a/build/recipes/build_pkgs/build_pkgs.py +++ b/build/recipes/build_pkgs/build_pkgs.py @@ -1,10 +1,14 @@ import subprocess +import logging + +logger = logging.getLogger("buildout/build_pkgs") def get_dirs(): """ Finds all subdirectories containing setup.py files. :return: List of directories as strings. """ + logger.debug("Getting list of directories containing setup.py files...") find = subprocess.run([ 'find', '.', '-name', 'setup.py', '-not', '-path', './build/recipes/*' ], stdout=subprocess.PIPE) @@ -14,6 +18,7 @@ def get_dirs(): for i, d in enumerate(dirs_cpy): dirs[i] = d.replace('/setup.py', '') + logger.debug("Done getting directories.") return dirs def get_names(dirs): @@ -22,6 +27,7 @@ def get_names(dirs): subproject directory will be the name of the subproject. :return: List of names as strings. """ + logger.debug("Generating list of subproject names...") names = [] for d in dirs: if d != '': @@ -32,6 +38,7 @@ def get_names(dirs): name = "services" names.append(name) + logger.debug("Done generating.") return names class Recipe: @@ -53,13 +60,16 @@ class Recipe: :return: Paths to files, as strings, created by the recipe. """ if self.options['name'] == "all": + logger.warning("WARNING: You've requested all packages to be built. This will take a long time.") + logger.warning("If only one or a few packages have changed, consider specifying them " + "in a comma-separated list.") pkgs = self.pkg_list else: pkgs = self.options['name'].split(',') for p in pkgs: if p not in self.pkg_list or p == '': - print("Package {} not valid. Skipping.".format(p)) + logger.error(f"Package {p} not valid. Skipping.") continue subprocess.run(["conda", "build", "build/metadata/{}".format(p), "--output-folder", "build/pkgs/"], stdout=subprocess.PIPE) diff --git a/build/recipes/build_pkgs/test/__init__.py b/build/recipes/build_pkgs/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/build/recipes/build_pkgs/test/conftest.py b/build/recipes/build_pkgs/test/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..e98638c7ae9666bdb27979cbb8e20c1f899271b8 --- /dev/null +++ b/build/recipes/build_pkgs/test/conftest.py @@ -0,0 +1,16 @@ +import pytest +import zc.buildout.testing + +@pytest.fixture(scope='module') +def recipe(): + """ + pytest fixture that initializes zc.buildout objects for use in testing. + Initializes Buildout, Options, and Recipe objects. + + :return: Initialized recipe object for build_pkgs + """ + from .. import build_pkgs + buildout = zc.buildout.testing.Buildout() + options = buildout.Options(buildout, 'build_pkgs', {'recipe': 'build_pkgs', 'name': 'null'}) + recipe = build_pkgs.Recipe(buildout=buildout, name=None, options=options) + return recipe \ No newline at end of file diff --git a/build/recipes/build_pkgs/test/test_build_pkgs.py b/build/recipes/build_pkgs/test/test_build_pkgs.py new file mode 100644 index 0000000000000000000000000000000000000000..c0dfae66cbebc4a4540506aff006532ed8a15497 --- /dev/null +++ b/build/recipes/build_pkgs/test/test_build_pkgs.py @@ -0,0 +1,30 @@ +import os +from .. import build_pkgs + +class TestBuildPkgs: + def test_get_names(self): + """ + Test that build_pkgs correctly gets the package name from + :return: + """ + d = 'apps/cli/executables/null' + assert build_pkgs.get_names([d]) == ['null'] + + def test_get_dirs(self): + """ + Test that build_pkgs correctly finds and stores directory paths + of directories containing setup.py files. + """ + assert './apps/cli/executables/null' in build_pkgs.get_dirs() + + def test_output(self, recipe): + """ + Test that the package specified in the recipe has been built correctly. + """ + created = recipe.install() + + for path in created: + print(path) + if len(path) > 0: + assert path is not None, "conda build failed to build package" + assert os.path.exists(path) diff --git a/build/recipes/setup_to_meta/__init__.py b/build/recipes/setup_to_meta/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/build/recipes/setup_to_meta/setup_to_meta.py b/build/recipes/setup_to_meta/setup_to_meta.py index e65c1ba2ae9e104a35dfb543cc9985c0224ab107..f7e32134d73d9f2eaef14a99f01a8e1d6e840099 100644 --- a/build/recipes/setup_to_meta/setup_to_meta.py +++ b/build/recipes/setup_to_meta/setup_to_meta.py @@ -1,13 +1,17 @@ -import json -import setuptools, importlib, subprocess, os, re +import subprocess +import logging +import json +import os PYTHON_VERSION = '3.8' +logger = logging.getLogger("buildout/setup_to_meta") def write_metafile(metadata, filepath): """ Writes given metadata to file with given path. """ + logger.debug(f"Writing meta.yaml file at {filepath}...") try: os.makedirs(filepath[:-10]) except FileExistsError: @@ -15,6 +19,7 @@ def write_metafile(metadata, filepath): with open(filepath, 'w') as f: f.write(metadata) + logger.debug("Done writing.") class MetadataGenerator: @@ -89,6 +94,7 @@ class MetadataGenerator: return test_string def generate(self): + logger.debug(f"Generating meta.yaml file from {self.path}...") # Filter numpy etc. out of the requirements try: self.setup['install_requires'] = [req for req in self.setup['install_requires'] if req != 'numpy'] @@ -106,7 +112,8 @@ class MetadataGenerator: with open('build/tools/metafile_template.txt', 'r') as f: metadata = f.read() - + + logger.debug("Done generating.") return metadata.format( name = name, version = version, @@ -126,12 +133,14 @@ def parse_setup(d): :param d: Directory with a setup.py file. :return: Data collected from parse_setup.py. """ + logger.debug(f"Parsing setup.py at {d}...") subprocess.run(['cp', 'build/tools/parse_setup.py', d]) os.chdir(d) proc = subprocess.run(['python3', 'parse_setup.py'], stdout=subprocess.PIPE) os.chdir(root) subprocess.run(['rm', '{}/parse_setup.py'.format(d)]) + logger.debug("Done parsing.") return json.loads(proc.stdout) def get_outputs(names): @@ -151,6 +160,7 @@ def get_dirs(): Finds all subdirectories containing setup.py files. :return: List of directories as strings. """ + logger.debug("Finding list of directories containing setup.py files...") find = subprocess.run([ 'find', '.', '-name', 'setup.py', '-not', '-path', './build/recipes/*' ], stdout=subprocess.PIPE) @@ -160,6 +170,7 @@ def get_dirs(): for i, d in enumerate(dirs_cpy): dirs[i] = d.replace('/setup.py', '') + logger.debug("Done finding directories.") return dirs def get_names(dirs): @@ -168,6 +179,7 @@ def get_names(dirs): subproject directory will be the name of the subproject. :return: List of names as strings. """ + logger.debug("Getting list of names...") names = [] for d in dirs: if d != '': @@ -178,6 +190,7 @@ def get_names(dirs): name = "services" names.append(name) + logger.debug("Done getting list of names.") return names def del_substrings(s, substrings): @@ -213,7 +226,6 @@ class Recipe: self.outputs = get_outputs(self.names) self.options = options - # TODO: Keep track of path in setup_dict def install(self): """ Install method that runs when recipe has components it needs to install. diff --git a/build/recipes/setup_to_meta/test/__init__.py b/build/recipes/setup_to_meta/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/build/recipes/setup_to_meta/test/conftest.py b/build/recipes/setup_to_meta/test/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..2f6dfb21d3d7ca162dc8b378bf4be2ce707f77ab --- /dev/null +++ b/build/recipes/setup_to_meta/test/conftest.py @@ -0,0 +1,16 @@ +import pytest +import zc.buildout.testing + +@pytest.fixture(scope='module') +def recipe(): + """ + pytest fixture that initializes zc.buildout objects for use in testing. + Initializes Buildout, Options, and Recipe objects. + + :return: Initialized recipe object for setup_to_meta + """ + from .. import setup_to_meta + buildout = zc.buildout.testing.Buildout() + options = buildout.Options(buildout, 'gen_metadata', {'recipe': 'setup_to_meta'}) + recipe = setup_to_meta.Recipe(buildout=buildout, name=None, options=options) + return recipe \ No newline at end of file diff --git a/build/recipes/setup_to_meta/test/test_setup_to_meta.py b/build/recipes/setup_to_meta/test/test_setup_to_meta.py new file mode 100644 index 0000000000000000000000000000000000000000..7b0162ae61965a5d88941af339f95761072d795e --- /dev/null +++ b/build/recipes/setup_to_meta/test/test_setup_to_meta.py @@ -0,0 +1,53 @@ +from .. import setup_to_meta + +class TestSetupToMeta: + def test_del_substrings(self): + """ + Tests that del_substrings function properly deletes substrings from a given string + """ + replaced = setup_to_meta.del_substrings('heallob, woarlcd', ['a', 'b', 'c']) + assert replaced == 'hello, world' + + def test_get_names(self): + """ + Tests that setup_to_meta correctly gets the package name from the package path + """ + d = 'apps/cli/executables/null' + assert setup_to_meta.get_names([d]) == ['null'] + + def test_get_dirs(self): + """ + Tests that setup_to_meta correctly finds and stores directory paths + of packages containing setup.py files + """ + assert './apps/cli/executables/null' in setup_to_meta.get_dirs() + + def test_get_outputs(self): + """ + Tests that setup_to_meta correctly generates a list of output paths given + a list of package names. + """ + assert setup_to_meta.get_outputs(['null']) == ['build/metadata/null/meta.yaml'] + + def test_parse_setup(self): + """ + Tests that parse_setup correctly parses a setup.py file into a dictionary. + """ + setup_data = setup_to_meta.parse_setup('apps/cli/executables/null') + keys = ['name', 'version', 'description', 'license'] + for key in keys: + assert key in setup_data + + def test_output(self, recipe): + """ + Test that metadata was successfully created and contains data. + + Checking for 'package' is an arbitrary check for content that will always + occur in a correct recipe. + :param recipe: Fixture that initializes recipe class in setup_to_meta.py + """ + created = recipe.install() + + for path in created: + with open(path, 'r') as f: + assert 'package' in f.read() diff --git a/build/recipes/test_recipes/__init__.py b/build/recipes/test_recipes/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/build/recipes/test_recipes/setup.py b/build/recipes/test_recipes/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..c97dae42aaa38bdbdab402d89a8a0d437d0f7071 --- /dev/null +++ b/build/recipes/test_recipes/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup + +setup( + name='test_recipes', + version='0.1', + py_modules = ['test_recipes'], + entry_points = {"zc.buildout": ["default=test_recipes:Recipe"]}, +) \ No newline at end of file diff --git a/build/recipes/test_recipes/test/__init__.py b/build/recipes/test_recipes/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/build/recipes/test_recipes/test/test_test_recipes.py b/build/recipes/test_recipes/test/test_test_recipes.py new file mode 100644 index 0000000000000000000000000000000000000000..57034eefdff357e8e0b09ddb02c556b57b0f2fff --- /dev/null +++ b/build/recipes/test_recipes/test/test_test_recipes.py @@ -0,0 +1,12 @@ +from .. import test_recipes + +class TestRecipes: + def test_get_recipes(self): + """ + Test that test_recipes is able to successfully retrieve names and paths + for all buildout recipes. + """ + recipe_names = ['setup_to_meta', 'build_pkgs', 'test_recipes'] + recipes = test_recipes.get_recipes() + for recipe in recipe_names: + assert recipe in recipes diff --git a/build/recipes/test_recipes/test_recipes.py b/build/recipes/test_recipes/test_recipes.py new file mode 100644 index 0000000000000000000000000000000000000000..479cc05141ca024ae4db3a58f36ebf6f570100d2 --- /dev/null +++ b/build/recipes/test_recipes/test_recipes.py @@ -0,0 +1,71 @@ +import subprocess +import logging + +logger = logging.getLogger("buildout/test_recipes") + +def get_recipes(): + """ + Get all currently installed buildout recipes (including this one!) + :return: Dictionary with format {recipe_name: recipe_path_from_root,} + """ + logger.debug("Getting list of recipes...") + recipes = {} + find = subprocess.run(['find', './build/recipes', '-name', 'setup.py'], + stdout=subprocess.PIPE) + paths = find.stdout.decode('utf-8').split('\n') + dirs = [] + names = [] + for p in paths: + if len(p) > 1: + # Exclude empty result from find + dirs.append(p.replace('/setup.py', '')) + names.append(p.split('/')[-2]) + for d, n in zip(dirs, names): + recipes[n] = d + logger.debug("Done getting recipes.") + + return recipes + +class Recipe: + """ + Buildout Recipe class. + For more detailed information, see the link. + http://www.buildout.org/en/latest/topics/writing-recipes.html + """ + + def run_test(self, recipe): + """ + Run test for given recipe. + :param recipe: Name of recipe to be run. + """ + logger.debug(f"Running tests for recipe {recipe}...") + subprocess.run(['pytest', '-vv', '--log-level=DEBUG', '--showlocals', + self.recipes[recipe]]) + + def __init__(self, buildout, name, options): + """ + Initializes fields needed for recipe. + :param buildout: (Boilerplate) Dictionary of options from buildout section + of buildout.cfg + :param name: (Boilerplate) Name of section that uses this recipe. + :param options: (Boilerplate) Options of section that uses this recipe. + """ + self.name = name + self.options = options + self.recipes = get_recipes() + self.choice = options['name'] + + def install(self): + """ + Install method that runs when recipe has components it needs to install. + + In this case, nothing is "installed" per se. + :return: Paths to files, as strings, created by the recipe. + """ + if self.choice == 'all': + # Run tests for all recipes + for recipe in self.recipes: + self.run_test(recipe) + else: + if self.choice in self.recipes: + self.run_test(self.choice) \ No newline at end of file diff --git a/build/tools/transfer_to_builder.py b/build/tools/transfer_to_builder.py index 64d7e4fc7ff61c4fdc8b32d4f8c7fd627f7981d8..fa43851a3f4ccf5312ba40249f082155b2e1e1b1 100644 --- a/build/tools/transfer_to_builder.py +++ b/build/tools/transfer_to_builder.py @@ -1,11 +1,17 @@ import subprocess import paramiko +import logging import fnmatch -import os import getpass +import sys +import os from scp import SCPClient +logger = logging.getLogger("buildtools/transfer_to_builder") +logger.setLevel(logging.INFO) +hander = logging.StreamHandler(stream=sys.stdout) + def get_build_pkg_names(): """ Search through pkgs directory for built .tar.bz2 packages @@ -13,9 +19,12 @@ def get_build_pkg_names(): """ pkg_names = [] d = "build/pkgs/noarch/" - for file in os.listdir(d): - if fnmatch.fnmatch(file, "*.tar.bz2"): - pkg_names.append(d + file) + try: + for file in os.listdir(d): + if fnmatch.fnmatch(file, "*.tar.bz2"): + pkg_names.append(d + file) + except FileNotFoundError as e: + logger.error(e) return pkg_names @@ -29,10 +38,18 @@ def create_ssh_client(server): client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - username = input("Enter NRAO username: ") - password = getpass.getpass(prompt="Enter NRAO password: ") + while True: + username = input("Enter NRAO username: ") + password = getpass.getpass(prompt="Enter NRAO password: ") + + try: + client.connect(server, username=username, password=password) + except paramiko.AuthenticationException as e: + logger.error(e) + logger.error("Invalid credentials. Try again.") + continue + break - client.connect(server, username=username, password=password) return client def transfer_packages(pkg_names): @@ -40,6 +57,8 @@ def transfer_packages(pkg_names): Use shell commands to transfer build archives to builder and update its conda package index. :param pkg_names: Names of the .tar.bz2 files for the built packages. """ + logger.addHandler(hander) + if len(pkg_names): builder_addr = "builder.aoc.nrao.edu" builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch" @@ -54,7 +73,8 @@ def transfer_packages(pkg_names): cmd_index + " && " + cmd_chmod]) else: - print("No packages found in build/pkgs/noarch. Did conda build successfully build the package(s)?") + logger.error("No packages found in build/pkgs/noarch. " + "Did conda build successfully build the package(s)?") if __name__ == "__main__": transfer_packages(get_build_pkg_names()) \ No newline at end of file diff --git a/buildout.cfg b/buildout.cfg index 5ae4f1cc3931524a3c061e563efccd0d0a489b60..58fcedbeb7a329c3e1ca06c0e77023c95c2ff0b6 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -3,13 +3,22 @@ # e.g. buildout parts=build_pkgs will specify build_pkgs # as a part that should be installed [buildout] -develop = build/recipes/setup_to_meta build/recipes/build_pkgs +develop = build/recipes/setup_to_meta build/recipes/build_pkgs build/recipes/test_recipes + +# Section for testing buildout recipes +# Depends on `name` in [buildout] to specify which recipe to test +# Specify name via command line using `buildout parts=test name={name}` +# Specify 'all' to test all recipes +[test] +recipe = test_recipes +name = ${buildout:name} # Section for building internal tools using conda # Depends on gen_metadata # Depends on `name` in [buildout] to specify which package to install # Specify name via command line using # `buildout parts=build_pkgs name={name}` +# Specify 'all' to build all packages (this takes a LONG time) [build_pkgs] => gen_metadata recipe = build_pkgs