From bfa1793dee350e112483605651b88148ae095ca8 Mon Sep 17 00:00:00 2001
From: nhertz <nhertz@nrao.edu>
Date: Wed, 2 Sep 2020 17:08:59 -0600
Subject: [PATCH] Added type hinting and PEP8-ified
---
build/recipes/build_pkgs/build_pkgs.py | 40 +++---
build/recipes/build_pkgs/setup.py | 6 +-
build/recipes/build_pkgs/test/conftest.py | 12 +-
.../build_pkgs/test/test_build_pkgs.py | 8 +-
build/recipes/setup_to_meta/setup.py | 6 +-
build/recipes/setup_to_meta/setup_to_meta.py | 117 ++++++++++--------
build/recipes/setup_to_meta/test/conftest.py | 21 +++-
.../setup_to_meta/test/test_setup_to_meta.py | 13 +-
8 files changed, 128 insertions(+), 95 deletions(-)
diff --git a/build/recipes/build_pkgs/build_pkgs.py b/build/recipes/build_pkgs/build_pkgs.py
index 304d2bad2..0cafdde39 100644
--- a/build/recipes/build_pkgs/build_pkgs.py
+++ b/build/recipes/build_pkgs/build_pkgs.py
@@ -1,19 +1,23 @@
-import subprocess
import logging
+import subprocess
+from typing import List, Dict, Any, Callable, Optional
+
+from zc.buildout.buildout import Buildout, Options
-logger = logging.getLogger("buildout/build_pkgs")
+logger: logging.Logger = logging.getLogger("buildout/build_pkgs")
-def get_dirs():
+
+def get_dirs() -> List[str]:
"""
Finds all subdirectories containing setup.py files.
:return: List of directories as strings.
"""
logger.debug("Getting list of directories containing setup.py files...")
- find = subprocess.run([
+ find: subprocess.CompletedProcess = subprocess.run([
'find', '.', '-name', 'setup.py', '-not', '-path', './build/recipes/*'
], stdout=subprocess.PIPE)
- dirs = find.stdout.decode('utf-8').split('\n')
- dirs_cpy = dirs
+ dirs: List[str] = find.stdout.decode('utf-8').split('\n')
+ dirs_cpy: List[str] = dirs
for i, d in enumerate(dirs_cpy):
dirs[i] = d.replace('/setup.py', '')
@@ -21,17 +25,18 @@ def get_dirs():
logger.debug("Done getting directories.")
return dirs
-def get_names(dirs):
+
+def get_names(dirs: List[str]) -> List[str]:
"""
Generate list of subproject names based on the rule that the name of the
subproject directory will be the name of the subproject.
:return: List of names as strings.
"""
logger.debug("Generating list of subproject names...")
- names = []
+ names: List[str] = []
for d in dirs:
if d != '':
- name = d.split('/')[-1]
+ name: str = d.split('/')[-1]
if name == "archive":
# Case with ./services/archive having special dir structure
@@ -41,8 +46,9 @@ def get_names(dirs):
logger.debug("Done generating.")
return names
+
class Recipe:
- def __init__(self, buildout, name, options):
+ def __init__(self, buildout: Optional[Buildout], name: str, options: Options):
"""
Initializes fields needed for recipe.
:param buildout: (Boilerplate) Dictionary of options from buildout section
@@ -50,11 +56,11 @@ class Recipe:
:param name: (Boilerplate) Name of section that uses this recipe.
:param options: (Boilerplate) Options of section that uses this recipe.
"""
- self.name = name
- self.options = options
- self.pkg_list = get_names(get_dirs())
+ self.name: str = name
+ self.options: Options = options
+ self.pkg_list: List[str] = get_names(get_dirs())
- def install(self):
+ def install(self) -> Any:
"""
Install method that runs when recipe has components it needs to install.
:return: Paths to files, as strings, created by the recipe.
@@ -63,9 +69,9 @@ class Recipe:
logger.warning("WARNING: You've requested all packages to be built. This will take a long time.")
logger.warning("If only one or a few packages have changed, consider specifying them "
"in a comma-separated list.")
- pkgs = self.pkg_list
+ pkgs: List[str] = self.pkg_list
else:
- pkgs = self.options['name'].split(',')
+ pkgs: List[str] = self.options['name'].split(',')
for p in pkgs:
if p not in self.pkg_list or p == '':
@@ -79,4 +85,4 @@ class Recipe:
return self.options.created()
- update = install
\ No newline at end of file
+ update: Callable = install
diff --git a/build/recipes/build_pkgs/setup.py b/build/recipes/build_pkgs/setup.py
index db69c236a..04e432da6 100644
--- a/build/recipes/build_pkgs/setup.py
+++ b/build/recipes/build_pkgs/setup.py
@@ -3,6 +3,6 @@ from setuptools import setup
setup(
name='build_pkgs',
version='0.1',
- py_modules = ['build_pkgs'],
- entry_points = {"zc.buildout": ["default=build_pkgs:Recipe"]},
-)
\ No newline at end of file
+ py_modules=['build_pkgs'],
+ entry_points={"zc.buildout": ["default=build_pkgs:Recipe"]},
+)
diff --git a/build/recipes/build_pkgs/test/conftest.py b/build/recipes/build_pkgs/test/conftest.py
index e98638c7a..5d7d7756f 100644
--- a/build/recipes/build_pkgs/test/conftest.py
+++ b/build/recipes/build_pkgs/test/conftest.py
@@ -1,8 +1,10 @@
import pytest
import zc.buildout.testing
+from .. import build_pkgs
+
@pytest.fixture(scope='module')
-def recipe():
+def recipe() -> build_pkgs.Recipe:
"""
pytest fixture that initializes zc.buildout objects for use in testing.
Initializes Buildout, Options, and Recipe objects.
@@ -10,7 +12,7 @@ def recipe():
:return: Initialized recipe object for build_pkgs
"""
from .. import build_pkgs
- buildout = zc.buildout.testing.Buildout()
- options = buildout.Options(buildout, 'build_pkgs', {'recipe': 'build_pkgs', 'name': 'null'})
- recipe = build_pkgs.Recipe(buildout=buildout, name=None, options=options)
- return recipe
\ No newline at end of file
+ buildout: zc.buildout.testing.Buildout = zc.buildout.testing.Buildout()
+ options: buildout.Options = buildout.Options(buildout, 'build_pkgs', {'recipe': 'build_pkgs', 'name': 'null'})
+ recipe: build_pkgs.Recipe = build_pkgs.Recipe(buildout=buildout, name=None, options=options)
+ return recipe
diff --git a/build/recipes/build_pkgs/test/test_build_pkgs.py b/build/recipes/build_pkgs/test/test_build_pkgs.py
index c0dfae66c..10758442d 100644
--- a/build/recipes/build_pkgs/test/test_build_pkgs.py
+++ b/build/recipes/build_pkgs/test/test_build_pkgs.py
@@ -1,13 +1,16 @@
import os
+from typing import List
+
from .. import build_pkgs
+
class TestBuildPkgs:
def test_get_names(self):
"""
Test that build_pkgs correctly gets the package name from
:return:
"""
- d = 'apps/cli/executables/null'
+ d: str = 'apps/cli/executables/null'
assert build_pkgs.get_names([d]) == ['null']
def test_get_dirs(self):
@@ -21,10 +24,9 @@ class TestBuildPkgs:
"""
Test that the package specified in the recipe has been built correctly.
"""
- created = recipe.install()
+ created: List[str] = recipe.install()
for path in created:
- print(path)
if len(path) > 0:
assert path is not None, "conda build failed to build package"
assert os.path.exists(path)
diff --git a/build/recipes/setup_to_meta/setup.py b/build/recipes/setup_to_meta/setup.py
index 24ade2276..8a2ebea13 100644
--- a/build/recipes/setup_to_meta/setup.py
+++ b/build/recipes/setup_to_meta/setup.py
@@ -3,6 +3,6 @@ from setuptools import setup
setup(
name='setup_to_meta',
version='0.1',
- py_modules = ['setup_to_meta'],
- entry_points = {"zc.buildout": ["default=setup_to_meta:Recipe"]},
-)
\ No newline at end of file
+ py_modules=['setup_to_meta'],
+ entry_points={"zc.buildout": ["default=setup_to_meta:Recipe"]},
+)
diff --git a/build/recipes/setup_to_meta/setup_to_meta.py b/build/recipes/setup_to_meta/setup_to_meta.py
index f7e32134d..b8ed06744 100644
--- a/build/recipes/setup_to_meta/setup_to_meta.py
+++ b/build/recipes/setup_to_meta/setup_to_meta.py
@@ -1,15 +1,20 @@
-
-import subprocess
-import logging
-import json
import os
+import json
+import logging
+import subprocess
+from typing import List, Any, Dict, Callable, Optional
+
+from zc.buildout.buildout import Buildout, Options
-PYTHON_VERSION = '3.8'
-logger = logging.getLogger("buildout/setup_to_meta")
+PYTHON_VERSION: str = '3.8'
+logger: logging.Logger = logging.getLogger("buildout/setup_to_meta")
-def write_metafile(metadata, filepath):
+
+def write_metafile(metadata: str, filepath: str):
"""
Writes given metadata to file with given path.
+ :param metadata: String containing conda recipe metadata to be written
+ :param filepath: String containing the path to conda recipe file (meta.yaml)
"""
logger.debug(f"Writing meta.yaml file at {filepath}...")
try:
@@ -26,17 +31,16 @@ class MetadataGenerator:
"""
Uses given info extracted from setup.py file to fill out metadata template.
"""
-
- def __init__(self, setup, path):
- self.setup = setup
- self.path = path
+ def __init__(self, setup: Dict[str, str], path: str):
+ self.setup: Dict[str, str] = setup
+ self.path: str = path
- def fmt_ep(self):
+ def fmt_ep(self) -> str:
"""
Format entry points section of metadata.
:return: Formatted string if entry points exists; else empty string.
"""
- ep_string = ''
+ ep_string: str = ''
if 'entry_points' in self.setup.keys() and 'console_scripts' in self.setup['entry_points']:
ep_string += 'entry_points:\n'
for ep in self.setup['entry_points']['console_scripts']:
@@ -44,17 +48,17 @@ class MetadataGenerator:
ep_string += ' '
return ep_string
- def fmt_reqs(self):
+ def fmt_reqs(self) -> str:
"""
Format requirements section of metadata.
:return: Formatted string if requirements exists; else empty string.
"""
- reqs_string = ''
- reqs_list = ''
+ reqs_string: str = ''
+ reqs_list: str = ''
reqs_string += 'requirements:\n'
- build_reqs = ' build:\n'
- run_reqs = ' run:\n'
- host_reqs = ' host:\n'
+ build_reqs: str = ' build:\n'
+ run_reqs: str = ' run:\n'
+ host_reqs: str = ' host:\n'
reqs_list += ' - python={}\n'.format(PYTHON_VERSION)
if 'install_requires' in self.setup.keys():
@@ -67,14 +71,14 @@ class MetadataGenerator:
'\n'
return reqs_string
- def fmt_test(self):
+ def fmt_test(self) -> str:
"""
Format test section of metadata.
NOTE: May need further tweaking to be smarter based on individual project
needs. For now, it's pretty dumb.
:return: Formatted string if tests_require exists; else empty string.
"""
- test_string = ''
+ test_string: str = ''
if 'tests_require' in self.setup.keys():
test_string += (
'test:\n'
@@ -93,7 +97,7 @@ class MetadataGenerator:
)
return test_string
- def generate(self):
+ def generate(self) -> str:
logger.debug(f"Generating meta.yaml file from {self.path}...")
# Filter numpy etc. out of the requirements
try:
@@ -101,17 +105,17 @@ class MetadataGenerator:
except KeyError:
pass
- name = self.setup['name']
- version = self.setup['version']
- entry_points = self.fmt_ep()
- pth = self.path.replace("./", "")
- requirements = self.fmt_reqs()
- test = self.fmt_test()
- lic = self.setup['license']
- summary = self.setup['description']
+ name: str = self.setup['name']
+ version: str = self.setup['version']
+ entry_points: str = self.fmt_ep()
+ pth: str = self.path.replace("./", "")
+ requirements: str = self.fmt_reqs()
+ test: str = self.fmt_test()
+ lic: str = self.setup['license']
+ summary: str = self.setup['description']
with open('build/tools/metafile_template.txt', 'r') as f:
- metadata = f.read()
+ metadata: str = f.read()
logger.debug("Done generating.")
return metadata.format(
@@ -126,7 +130,7 @@ class MetadataGenerator:
)
-def parse_setup(d):
+def parse_setup(d: str) -> Dict[str, str]:
"""
Function for running parse_setup.py on each directory with a setup.py file.
NOTE: Contains a hack for getting parse_setup.py to run in each directory.
@@ -136,36 +140,37 @@ def parse_setup(d):
logger.debug(f"Parsing setup.py at {d}...")
subprocess.run(['cp', 'build/tools/parse_setup.py', d])
os.chdir(d)
- proc = subprocess.run(['python3', 'parse_setup.py'], stdout=subprocess.PIPE)
+ proc: subprocess.CompletedProcess = subprocess.run(['python3', 'parse_setup.py'], stdout=subprocess.PIPE)
os.chdir(root)
subprocess.run(['rm', '{}/parse_setup.py'.format(d)])
logger.debug("Done parsing.")
return json.loads(proc.stdout)
-def get_outputs(names):
+
+def get_outputs(names: List[str]) -> List[str]:
"""
Generate list of metadata files that will be created.
:param dirs: List of dirs of all subprojects with a setup.py file.
:return: List of paths to output files as strings.
"""
- outputs = []
+ outputs: List[str] = []
for name in names:
outputs.append("build/metadata/{}/meta.yaml".format(name))
return outputs
-def get_dirs():
+def get_dirs() -> List[str]:
"""
Finds all subdirectories containing setup.py files.
:return: List of directories as strings.
"""
logger.debug("Finding list of directories containing setup.py files...")
- find = subprocess.run([
+ find: subprocess.CompletedProcess = subprocess.run([
'find', '.', '-name', 'setup.py', '-not', '-path', './build/recipes/*'
], stdout=subprocess.PIPE)
- dirs = find.stdout.decode('utf-8').split('\n')
- dirs_cpy = dirs
+ dirs: List[str] = find.stdout.decode('utf-8').split('\n')
+ dirs_cpy: List[str] = dirs
for i, d in enumerate(dirs_cpy):
dirs[i] = d.replace('/setup.py', '')
@@ -173,14 +178,15 @@ def get_dirs():
logger.debug("Done finding directories.")
return dirs
-def get_names(dirs):
+
+def get_names(dirs: List[str]) -> List[str]:
"""
Generate list of subproject names based on the rule that the name of the
subproject directory will be the name of the subproject.
:return: List of names as strings.
"""
logger.debug("Getting list of names...")
- names = []
+ names: List[str] = []
for d in dirs:
if d != '':
name = d.split('/')[-1]
@@ -193,7 +199,8 @@ def get_names(dirs):
logger.debug("Done getting list of names.")
return names
-def del_substrings(s, substrings):
+
+def del_substrings(s: str, substrings: List[str]):
"""
Function for deleting multiple substrings from a string.
:param s: String to be modified.
@@ -201,11 +208,13 @@ def del_substrings(s, substrings):
:return: Modified string.
"""
for replace in substrings:
- s = s.replace(replace, '')
+ s: str = s.replace(replace, '')
return s
-root = os.getcwd()
+
+root: str = os.getcwd()
+
class Recipe:
"""
@@ -213,7 +222,7 @@ class Recipe:
For more detailed information, see the link.
http://www.buildout.org/en/latest/topics/writing-recipes.html
"""
- def __init__(self, buildout, name, options):
+ def __init__(self, buildout: Optional[Buildout], name: Optional[str], options: Options):
"""
Initializes fields needed for recipe.
:param buildout: (Boilerplate) Dictionary of options from buildout section
@@ -221,25 +230,25 @@ class Recipe:
:param name: (Boilerplate) Name of section that uses this recipe.
:param options: (Boilerplate) Options of section that uses this recipe.
"""
- self.dirs = get_dirs()
- self.names = get_names(self.dirs)
- self.outputs = get_outputs(self.names)
- self.options = options
+ self.dirs: List[str] = get_dirs()
+ self.names: List[str] = get_names(self.dirs)
+ self.outputs: List[str] = get_outputs(self.names)
+ self.options: Options = options
- def install(self):
+ def install(self) -> Any:
"""
Install method that runs when recipe has components it needs to install.
:return: Paths to files, as strings, created by the recipe.
"""
for i, d in enumerate(self.dirs):
if d != '':
- setup_data = parse_setup(d)
- metadata = MetadataGenerator(setup_data, d).generate()
+ setup_data: Dict[str, str] = parse_setup(d)
+ metadata: str = MetadataGenerator(setup_data, d).generate()
write_metafile(metadata, self.outputs[i])
- # Pass created file into options.created()
+ # Buildout-specific operation: pass created file into options.created()
self.options.created(self.outputs[i])
return self.options.created()
# No special procedure for updating vs. installing
- update = install
\ No newline at end of file
+ update: Callable = install
diff --git a/build/recipes/setup_to_meta/test/conftest.py b/build/recipes/setup_to_meta/test/conftest.py
index 2f6dfb21d..c9f4b3a91 100644
--- a/build/recipes/setup_to_meta/test/conftest.py
+++ b/build/recipes/setup_to_meta/test/conftest.py
@@ -1,8 +1,11 @@
import pytest
import zc.buildout.testing
+from .. import setup_to_meta
+
+
@pytest.fixture(scope='module')
-def recipe():
+def recipe() -> setup_to_meta.Recipe:
"""
pytest fixture that initializes zc.buildout objects for use in testing.
Initializes Buildout, Options, and Recipe objects.
@@ -10,7 +13,15 @@ def recipe():
:return: Initialized recipe object for setup_to_meta
"""
from .. import setup_to_meta
- buildout = zc.buildout.testing.Buildout()
- options = buildout.Options(buildout, 'gen_metadata', {'recipe': 'setup_to_meta'})
- recipe = setup_to_meta.Recipe(buildout=buildout, name=None, options=options)
- return recipe
\ No newline at end of file
+ buildout: zc.buildout.testing.Buildout = zc.buildout.testing.Buildout()
+ options: buildout.Options = buildout.Options(
+ buildout,
+ 'gen_metadata',
+ {'recipe': 'setup_to_meta'}
+ )
+ recipe: setup_to_meta.Recipe = setup_to_meta.Recipe(
+ buildout=buildout,
+ name=None,
+ options=options
+ )
+ return recipe
diff --git a/build/recipes/setup_to_meta/test/test_setup_to_meta.py b/build/recipes/setup_to_meta/test/test_setup_to_meta.py
index 7b0162ae6..4b17197ca 100644
--- a/build/recipes/setup_to_meta/test/test_setup_to_meta.py
+++ b/build/recipes/setup_to_meta/test/test_setup_to_meta.py
@@ -1,18 +1,21 @@
+from typing import Dict, List
+
from .. import setup_to_meta
+
class TestSetupToMeta:
def test_del_substrings(self):
"""
Tests that del_substrings function properly deletes substrings from a given string
"""
- replaced = setup_to_meta.del_substrings('heallob, woarlcd', ['a', 'b', 'c'])
+ replaced: str = setup_to_meta.del_substrings('heallob, woarlcd', ['a', 'b', 'c'])
assert replaced == 'hello, world'
def test_get_names(self):
"""
Tests that setup_to_meta correctly gets the package name from the package path
"""
- d = 'apps/cli/executables/null'
+ d: str = 'apps/cli/executables/null'
assert setup_to_meta.get_names([d]) == ['null']
def test_get_dirs(self):
@@ -33,8 +36,8 @@ class TestSetupToMeta:
"""
Tests that parse_setup correctly parses a setup.py file into a dictionary.
"""
- setup_data = setup_to_meta.parse_setup('apps/cli/executables/null')
- keys = ['name', 'version', 'description', 'license']
+ setup_data: Dict[str, str] = setup_to_meta.parse_setup('apps/cli/executables/null')
+ keys: List[str] = ['name', 'version', 'description', 'license']
for key in keys:
assert key in setup_data
@@ -46,7 +49,7 @@ class TestSetupToMeta:
occur in a correct recipe.
:param recipe: Fixture that initializes recipe class in setup_to_meta.py
"""
- created = recipe.install()
+ created: List[str] = recipe.install()
for path in created:
with open(path, 'r') as f:
--
GitLab