import setuptools, importlib, subprocess, os, re PYTHON_VERSION = '3.8' def write_metafile(metadata, filepath): """ Writes given metadata to file with given path. """ try: os.makedirs(filepath[:-10]) except FileExistsError: pass with open(filepath, 'w') as f: f.write(metadata) def generate_metadata(setup_dict, path): """ Uses given info extracted from setup.py file to fill out metadata template. :param setup_dict: Dictionary of info extracted from setup.py. :param path: Path to root directory of the subproject. :return: String of metadata, ready to be written to a file. """ def fmt_ep(): """ Format entry points section of metadata. :return: Formatted string if entry points exists; else empty string. """ ep_string = '' if 'entry_points' in setup_dict.keys(): ep_string += 'entry_points:\n' for ep in setup_dict['entry_points']: ep_string += ' - {}\n'.format(ep) ep_string += ' ' return ep_string def fmt_reqs(): """ Format requirements section of metadata. :return: Formatted string if requirements exists; else empty string. """ reqs_string = '' reqs_list = '' if 'install_requires' in setup_dict.keys(): reqs_string += 'requirements:\n' build_reqs = ' build:\n' run_reqs = ' run:\n' host_reqs = ' host:\n' reqs_list += ' - python={}\n'.format(PYTHON_VERSION) for req in setup_dict['install_requires'].split(', '): reqs_list += ' - {}\n'.format(req) reqs_string += build_reqs + reqs_list + \ run_reqs + reqs_list + \ host_reqs + reqs_list + \ '\n' return reqs_string def fmt_test(): """ Format test section of metadata. NOTE: May need further tweaking to be smarter based on individual project needs. For now, it's pretty dumb. :return: Formatted string if tests_require exists; else empty string. """ test_string = '' if 'tests_require' in setup_dict.keys(): test_string += ( 'test:\n' ' source_files:\n' ' - test/\n' ' requires:\n' ) for req in setup_dict['tests_require'].split(', '): test_string += ' - {}\n'.format(req) test_string += ( ' commands:\n' ' - pytest -vv --log-level=DEBUG --showlocals\n' '\n' ) return test_string name = setup_dict['name'] version = setup_dict['version'] entry_points = fmt_ep() pth = path.replace("./", "") requirements = fmt_reqs() test = fmt_test() lic = setup_dict['license'] summary = setup_dict['description'] with open('tools/metafile_template.txt', 'r') as f: metadata = f.read() return metadata.format( name = name, version = version, entry_points = entry_points, path = "../../" + pth, requirements = requirements, test = test, license = lic, summary = summary ) def data_to_dict(setup_data): """ Translates parsed setup.py data from unformatted list to formatted dictionary. :param setup_data: Data extracted from setup.py using parse_setup.py :return: Dictionary of formatted data. """ data_dict = {} r_entry = r"([a-z-_]+): " r_id = r"[a-zA-Z0-9-_]" r_ep = r"{{(?:[a-zA-Z0-9-_.]+):\s*(?P<values>(?:{0}+\s*=\s*[a-zA-Z0-9-_.]+:{0}+(?:,\s*)?)*)}}".format(r_id) for d in setup_data: result = re.match(r_entry, d) if result: substrings = [result.group(0), '[', ']', '\'', '"'] field = result.group(1) value = del_substrings(d, substrings) if value == "None": continue if re.match(r_ep, value): # Parse entry points value = re.match(r_ep, value).group('values') value = re.split(',\s*', value) data_dict[field] = value return data_dict def parse_setup(d): """ Function for running parse_setup.py on each directory with a setup.py file. NOTE: Contains a hack for getting parse_setup.py to run in each directory. :param d: Directory with a setup.py file. :return: Data collected from parse_setup.py. """ subprocess.run(['cp', 'tools/parse_setup.py', d]) os.chdir(d) proc = subprocess.run(['python3', 'parse_setup.py'], stdout=subprocess.PIPE) setup_data = (proc.stdout.decode('utf-8')).split('\n') os.chdir(root) subprocess.run(['rm', '{}/parse_setup.py'.format(d)]) return setup_data def get_outputs(names): """ Generate list of metadata files that will be created. :param dirs: List of dirs of all subprojects with a setup.py file. :return: List of paths to output files as strings. """ outputs = [] for name in names: outputs.append("metadata/{}/meta.yaml".format(name)) return outputs def get_dirs(): """ Finds all subdirectories containing setup.py files. :return: List of directories as strings. """ find = subprocess.run([ 'find', '.', '-name', 'setup.py', '-not', '-path', './src/*' ], stdout=subprocess.PIPE) dirs = find.stdout.decode('utf-8').split('\n') dirs_cpy = dirs for i, d in enumerate(dirs_cpy): dirs[i] = d.replace('/setup.py', '') return dirs def get_names(dirs): """ Generate list of subproject names based on the rule that the name of the subproject directory will be the name of the subproject. :return: List of names as strings. """ names = [] for d in dirs: if d != '': name = d.split('/')[-1] if name == "archive": # Case with ./services/archive having special dir structure name = "services" names.append(name) return names def del_substrings(s, substrings): """ Function for deleting multiple substrings from a string. :param s: String to be modified. :param substrings: List of substrings to be targeted for deletion. :return: Modified string. """ for replace in substrings: s = s.replace(replace, '') return s root = os.getcwd() class Recipe: """ Buildout Recipe class. For more detailed information, see the link. http://www.buildout.org/en/latest/topics/writing-recipes.html """ def __init__(self, buildout, name, options): """ Initializes fields needed for recipe. :param buildout: (Boilerplate) Dictionary of options from buildout section of buildout.cfg :param name: (Boilerplate) Name of section that uses this recipe. :param options: (Boilerplate) Options of section that uses this recipe. """ self.dirs = get_dirs() self.names = get_names(self.dirs) self.outputs = get_outputs(self.names) self.options = options # TODO: Keep track of path in setup_dict def install(self): """ Install method that runs when recipe has components it needs to install. :return: Paths to files, as strings, created by the recipe. """ for i, d in enumerate(self.dirs): if d != '': setup_data = parse_setup(d) data_dict = data_to_dict(setup_data) metadata = generate_metadata(data_dict, d) write_metafile(metadata, self.outputs[i]) # Pass created file into options.created() self.options.created(self.outputs[i]) return self.options.created() # No special procedure for updating vs. installing update = install