Skip to content
Snippets Groups Projects
Commit d3d5f587 authored by Nathan Hertz's avatar Nathan Hertz
Browse files

Added SConstruct file for testing scons; added @dlyons' Jupyter notebook file... 

Added SConstruct file for testing scons; added @dlyons' Jupyter notebook file for generating Makefiles
parent 6c44801c
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Python Meta-make.ipynb 0 → 100644
+ 283
− 0
View file @ d3d5f587
%% Cell type:code id: tags:
``` python
import os
from pathlib import Path
import platform
import sys
from typing import List, Optional, TextIO
```
%% Cell type:markdown id: tags:
A directory contains a project if there is a `setup.py` file in it.
%% Cell type:code id: tags:
``` python
projects = list(proj for (proj, subdirs, files) in os.walk(Path()) if 'setup.py' in files)
projects
```
%% Output
['./shared/schema',
'./shared/support',
'./shared/messaging/events',
'./apps/cli/executables/ingestion',
'./apps/cli/executables/weblog_thumbs',
'./apps/cli/executables/alma_reingester',
'./apps/cli/executables/alma_product_fetch',
'./apps/cli/executables/epilogue',
'./apps/cli/executables/datafetcher',
'./apps/cli/executables/vlba_grabber',
'./apps/cli/utilities/s_code_project_updater',
'./apps/cli/utilities/proprietary_setter',
'./apps/cli/utilities/mr_clean',
'./apps/cli/utilities/faultchecker',
'./apps/cli/utilities/mr_books',
'./apps/cli/utilities/dumplogs',
'./apps/cli/utilities/datafinder',
'./apps/cli/utilities/qa_results',
'./apps/cli/launchers/pymygdala',
'./apps/cli/launchers/wf',
'./services/archive']
%% Cell type:markdown id: tags:
Here are some things we know about projects:
- Every project has a name, which is the same as the name of the directory that contains it.
- Every project has a version, which happens to be in a file in `src/$PROJECT/_version.py` in the same format
- Every project has certain dependencies, which are enumerated in the setup.py as `install_requires`
Let's build up this abstraction.
%% Cell type:code id: tags:
``` python
class Project:
def __init__(self, path: Path):
self.path = path
self.name = path.name
@property
def version(self) -> str:
"""Compute and return the version in the _version.py file under the project."""
# to compute the version, we must locate and parse the _version.py file
version_file = self.path / 'src' / self.name / '_version.py'
# this line is basically cribbed from the standard setup.py file
return version_file.open().readlines()[-1].split()[-1].strip("\"'")
@property
def dependencies(self) -> List[str]:
"""Returns the list of dependencies under this project"""
# to compute the dependencies, we have to do something a bit more gross
# we have to open the setup.py file and look for a line with "install_requires"
# once we have that line, we have to parse out the Python list
# we can go ahead and cheat here with eval() for today.
#
# Unfortunately, this doesn't work if they span multiple lines, which is a
# frequent occurrence
setup_file = self.path / 'setup.py'
for line in setup_file.open().readlines():
if 'install_requires' in line:
return eval(line.split('=')[1].strip().strip(','))
# if we made it here, we never found that line, so this project
# has no dependencies; let's make life easy on ourselves and
# keep the API simple
return []
def __repr__(self) -> str:
return f'<Project name={self.name} version={self.version}>'
p1 = Project(Path('apps/cli/executables/datafetcher'))
p1
```
%% Output
<Project name=datafetcher version=4.0.0a1.dev1>
%% Cell type:code id: tags:
``` python
p1.dependencies
```
%% Output
['requests', 'pycapo']
%% Cell type:markdown id: tags:
The Makefile we want to build is going to have a certain structure. Namely, it should:
- Have a top-level entry point, probably a `.PHONY` that is easy for a parent Makefile to target
- The top-level entry point is going to depend on all the projects we have located
- Each project will be a target, a dependency of the top-level entry point
Based on the project structure we also know this:
- A project rule will depend on that project's dependencies
This should cause the build order to fall out rather naturally.
%% Cell type:code id: tags:
``` python
alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz
```
%% Output
'x86_64'
%% Cell type:code id: tags:
``` python
class Makefile:
def __init__(self, projects: List[Project]):
self.projects = projects
def build(self):
output = sys.stdout
self.build_entry_point(output)
self.build_projects(output)
def build_entry_point(self, output: TextIO):
"""This builds the toplevel entry point, ready for inclusion in another Makefile"""
toplevels = ' '.join(self.project_target(project) for project in self.projects)
output.write('.PHONY: all-python-projects\n')
output.write(f'all-python-projects: {toplevels}\n\n')
def project_target(self, project: Project) -> str:
"""This converts a project to a distfile name, so that Make can know if it is up to date"""
return f'{project.path}/dist/{project.name}-{project.version}.macosx-10.9-x86_64.tar.gz'
def build_projects(self, output: TextIO):
"""This generates all of the targets for each project we know about."""
for project in self.projects:
self.build_project(project, output)
def build_project(self, project: Project, output: TextIO):
dependencies = [self.lookup_dependency(dep) for dep in project.dependencies]
target_dependencies = ' '.join(self.project_target(dep) for dep in dependencies if dep)
output.write(f'{self.project_target(project)}: {target_dependencies}\n')
output.write(f'\tcd {project.path} && python setup.py bdist\n\n')
def lookup_dependency(self, project_name: str) -> Optional[Project]:
return next((project for project in self.projects if project.name == project_name), None)
def load_projects() -> List[Project]:
# the next line is commented out due to the fragility of the setup.py parsing logic
# return [Project(Path(proj)) for (proj, subdirs, files) in os.walk(Path()) if 'setup.py' in files]
return [Project(Path('shared/schema')), Project(Path('apps/cli/executables/alma_reingester')), Project(Path('shared/messaging/events'))]
m = Makefile(load_projects())
m.build()
```
%% Output
.PHONY: all-python-projects
all-python-projects: shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz apps/cli/executables/alma_reingester/dist/alma_reingester-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz
shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz:
cd shared/schema && python setup.py bdist
apps/cli/executables/alma_reingester/dist/alma_reingester-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz
cd apps/cli/executables/alma_reingester && python setup.py bdist
shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz:
cd shared/messaging/events && python setup.py bdist
SConstruct 0 → 100644
+ 19
− 0
View file @ d3d5f587
import os
from pathlib import Path
def setup_project(target, source, env):
dir = str(source[0])[:-9]
home = os.getcwd()
env.Execute("cd {} && python3 setup.py bdist && cd {}".format(dir, home))
builder = Builder(action = setup_project)
env = Environment(BUILDERS = {'Setup' : builder})
projects = list(proj for (proj, subdirs, files) in os.walk(Path()) if "setup.py" in files)
output = []
for path in projects:
name = path.split('/')[-1]
output.append("{}/dist/{}".format(path.replace('.', os.getcwd()), name + "-4.0.0a1.dev1.macosx-10.15-x86_64.tar.gz"))
for src, out in zip(projects, output):
env.Setup(source=src + "/setup.py", target=out)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment