setup_to_meta.py

import setuptools, importlib, subprocess, os, re

PYTHON_VERSION = '3.8'

def write_metafile(metadata, filepath):
    """
    Writes given metadata to file with given path.
    """
    try:
        os.makedirs(filepath[:-10])
    except FileExistsError:
        pass

    with open(filepath, 'w') as f:
        f.write(metadata)

def generate_metadata(setup_dict, path):
    """
    Uses given info extracted from setup.py file to fill out metadata template.
    :param setup_dict: Dictionary of info extracted from setup.py.
    :param path: Path to root directory of the subproject.
    :return: String of metadata, ready to be written to a file.
    """
    def fmt_ep():
        """
        Format entry points section of metadata.
        :return: Formatted string if entry points exists; else empty string.
        """
        ep_string = ''
        if 'entry_points' in setup_dict.keys():
            ep_string += 'entry_points:\n'
            for ep in setup_dict['entry_points']:
                ep_string += '    - {}\n'.format(ep)
        return ep_string
    def fmt_reqs():
        """
        Format requirements section of metadata.
        :return: Formatted string if requirements exists; else empty string.
        """
        reqs_string = ''
        reqs_list = ''
        if 'install_requires' in setup_dict.keys():
            reqs_string += 'requirements:\n'
            build_reqs = '  build:\n'
            run_reqs = '  run:\n'
            host_reqs = '  host:\n'
            reqs_list += '    - python={}\n'.format(PYTHON_VERSION)
            for req in setup_dict['install_requires'].split(', '):
                reqs_list += '    - {}\n'.format(req)
            reqs_string += build_reqs + reqs_list + \
                           run_reqs + reqs_list + \
                           host_reqs + reqs_list + \
                           '\n'
        return reqs_string
    def fmt_test():
        """
        Format test section of metadata.
        NOTE: May need further tweaking to be smarter based on individual project
        needs. For now, it's pretty dumb.
        :return: Formatted string if tests_require exists; else empty string.
        """
        test_string = ''
        if 'tests_require' in setup_dict.keys():
            test_string += (
                'test:\n'
                '  source_files:\n'
                '    - test/\n'
                '  requires:\n'
            )

            for req in setup_dict['tests_require'].split(', '):
                test_string += '    - {}\n'.format(req)

            test_string += (
                '  commands:\n'
                '    - pytest -vv --log-level=DEBUG --showlocals\n'
                '\n'
            )
        return test_string

    name = setup_dict['name']
    version = setup_dict['version']
    entry_points = fmt_ep()
    pth = path.replace("./", "")
    requirements = fmt_reqs()
    test = fmt_test()
    lic = setup_dict['license']
    summary = setup_dict['description']

    with open('tools/metafile_template.txt', 'r') as f:
        metadata = f.read()

    return metadata.format(
        name = name,
        version = version,
        entry_points = entry_points,
        path = "../../" + pth,
        requirements = requirements,
        test = test,
        license = lic,
        summary = summary
    )

def data_to_dict(setup_data):
    """
    Translates parsed setup.py data from unformatted list to formatted dictionary.
    :param setup_data: Data extracted from setup.py using parse_setup.py
    :return: Dictionary of formatted data.
    """
    data_dict = {}
    r_entry = r"([a-z-_]+): "
    r_id = r"[a-zA-Z0-9-_]"
    r_ep = r"{{(?:[a-zA-Z0-9-_.]+):\s*(?P<values>(?:{0}+\s*=\s*[a-zA-Z0-9-_.]+:{0}+(?:,\s*)?)*)}}".format(r_id)

    for d in setup_data:
        result = re.match(r_entry, d)
        if result:
            substrings = [result.group(0), '[', ']', '\'', '"']
            field = result.group(1)
            value = del_substrings(d, substrings)
            if value == "None":
                continue
            if re.match(r_ep, value):
                # Parse entry points
                value = re.match(r_ep, value).group('values')
                value = re.split(',\s*', value)
            data_dict[field] = value

    return data_dict

def parse_setup(d):
    """
    Function for running parse_setup.py on each directory with a setup.py file.
    NOTE: Contains a hack for getting parse_setup.py to run in each directory.
    :param d: Directory with a setup.py file.
    :return: Data collected from parse_setup.py.
    """
    subprocess.run(['cp', 'tools/parse_setup.py', d])
    os.chdir(d)
    proc = subprocess.run(['python3', 'parse_setup.py'], stdout=subprocess.PIPE)
    setup_data = (proc.stdout.decode('utf-8')).split('\n')
    os.chdir(root)
    subprocess.run(['rm', '{}/parse_setup.py'.format(d)])

    return setup_data

def get_outputs(names):
    """
    Generate list of metadata files that will be created.
    :param dirs: List of dirs of all subprojects with a setup.py file.
    :return: List of paths to output files as strings.
    """
    outputs = []
    for name in names:
        outputs.append("metadata/{}/meta.yaml".format(name))

    return outputs

def get_dirs():
    """
    Finds all subdirectories containing setup.py files.
    :return: List of directories as strings.
    """
    find = subprocess.run([
        'find', '.', '-name', 'setup.py', '-not', '-path', './src/*'
    ], stdout=subprocess.PIPE)
    dirs = find.stdout.decode('utf-8').split('\n')
    dirs_cpy = dirs

    for i, d in enumerate(dirs_cpy):
        dirs[i] = d.replace('/setup.py', '')

    return dirs

def get_names(dirs):
    """
    Generate list of subproject names based on the rule that the name of the
    subproject directory will be the name of the subproject.
    :return: List of names as strings.
    """
    names = []
    for d in dirs:
        if d != '':
            name = d.split('/')[-1]

            if name == "archive":
                # Case with ./services/archive having special dir structure
                name = "services"
            names.append(name)

    return names

def del_substrings(s, substrings):
    """
    Function for deleting multiple substrings from a string.
    :param s: String to be modified.
    :param substrings: List of substrings to be targeted for deletion.
    :return: Modified string.
    """
    for replace in substrings:
        s = s.replace(replace, '')

    return s

root = os.getcwd()

class Recipe:
    """
    Buildout Recipe class.
    For more detailed information, see the link.
    http://www.buildout.org/en/latest/topics/writing-recipes.html
    """
    def __init__(self, buildout, name, options):
        """
        Initializes fields needed for recipe.
        :param buildout: (Boilerplate) Dictionary of options from buildout section
        of buildout.cfg
        :param name: (Boilerplate) Name of section that uses this recipe.
        :param options: (Boilerplate) Options of section that uses this recipe.
        """
        self.dirs = get_dirs()
        self.names = get_names(self.dirs)
        self.outputs = get_outputs(self.names)
        self.options = options

    # TODO: Keep track of path in setup_dict
    def install(self):
        """
        Install method that runs when recipe has components it needs to install.
        :return: Paths to files, as strings, created by the recipe.
        """
        for i, d in enumerate(self.dirs):
            if d != '':
                setup_data = parse_setup(d)
                data_dict = data_to_dict(setup_data)
                metadata = generate_metadata(data_dict, d)
                write_metafile(metadata, self.outputs[i])
                # Pass created file into options.created()
                self.options.created(self.outputs[i])

        return self.options.created()

    # No special procedure for updating vs. installing
    update = install