merging SSA-6315-skeletal-build

Makefile

+include conda.mk
+include setuptools.mk
+include alma_fetch.mk
+
+SHELL := /bin/bash
+
+build-base:
+	@echo "Building..."
+	make miniconda-installed
+	make build-python
+	@echo "Finished build."
+
+build-almafetch:
+	@echo "Building ALMA fetcher."
+	make build-alma
+

Python Meta-make.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from pathlib import Path\n",
+    "import platform\n",
+    "import sys\n",
+    "from typing import List, Optional, TextIO"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "A directory contains a project if there is a `setup.py` file in it."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['./shared/schema',\n",
+       " './shared/support',\n",
+       " './shared/messaging/events',\n",
+       " './apps/cli/executables/ingestion',\n",
+       " './apps/cli/executables/weblog_thumbs',\n",
+       " './apps/cli/executables/alma_reingester',\n",
+       " './apps/cli/executables/alma_product_fetch',\n",
+       " './apps/cli/executables/epilogue',\n",
+       " './apps/cli/executables/datafetcher',\n",
+       " './apps/cli/executables/vlba_grabber',\n",
+       " './apps/cli/utilities/s_code_project_updater',\n",
+       " './apps/cli/utilities/proprietary_setter',\n",
+       " './apps/cli/utilities/mr_clean',\n",
+       " './apps/cli/utilities/faultchecker',\n",
+       " './apps/cli/utilities/mr_books',\n",
+       " './apps/cli/utilities/dumplogs',\n",
+       " './apps/cli/utilities/datafinder',\n",
+       " './apps/cli/utilities/qa_results',\n",
+       " './apps/cli/launchers/pymygdala',\n",
+       " './apps/cli/launchers/wf',\n",
+       " './services/archive']"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "projects = list(proj for (proj, subdirs, files) in os.walk(Path()) if 'setup.py' in files)\n",
+    "projects"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Here are some things we know about projects:\n",
+    "\n",
+    "- Every project has a name, which is the same as the name of the directory that contains it.\n",
+    "- Every project has a version, which happens to be in a file in `src/$PROJECT/_version.py` in the same format\n",
+    "- Every project has certain dependencies, which are enumerated in the setup.py as `install_requires`\n",
+    "\n",
+    "Let's build up this abstraction."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Project name=datafetcher version=4.0.0a1.dev1>"
+      ]
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "class Project:\n",
+    "    def __init__(self, path: Path):\n",
+    "        self.path = path\n",
+    "        self.name = path.name\n",
+    "    \n",
+    "    @property\n",
+    "    def version(self) -> str:\n",
+    "        \"\"\"Compute and return the version in the _version.py file under the project.\"\"\"\n",
+    "        # to compute the version, we must locate and parse the _version.py file\n",
+    "        version_file = self.path / 'src' / self.name / '_version.py'\n",
+    "        \n",
+    "        # this line is basically cribbed from the standard setup.py file\n",
+    "        return version_file.open().readlines()[-1].split()[-1].strip(\"\\\"'\")\n",
+    "    \n",
+    "    @property\n",
+    "    def dependencies(self) -> List[str]:\n",
+    "        \"\"\"Returns the list of dependencies under this project\"\"\"\n",
+    "        \n",
+    "        # to compute the dependencies, we have to do something a bit more gross\n",
+    "        # we have to open the setup.py file and look for a line with \"install_requires\"\n",
+    "        # once we have that line, we have to parse out the Python list\n",
+    "        # we can go ahead and cheat here with eval() for today.\n",
+    "        #\n",
+    "        # Unfortunately, this doesn't work if they span multiple lines, which is a\n",
+    "        # frequent occurrence\n",
+    "        setup_file = self.path / 'setup.py'\n",
+    "        \n",
+    "        for line in setup_file.open().readlines():\n",
+    "            if 'install_requires' in line:\n",
+    "                return eval(line.split('=')[1].strip().strip(','))\n",
+    "        \n",
+    "        # if we made it here, we never found that line, so this project \n",
+    "        # has no dependencies; let's make life easy on ourselves and \n",
+    "        # keep the API simple\n",
+    "        return []\n",
+    "    \n",
+    "    def __repr__(self) -> str:\n",
+    "        return f'<Project name={self.name} version={self.version}>'\n",
+    "    \n",
+    "p1 = Project(Path('apps/cli/executables/datafetcher'))\n",
+    "p1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['requests', 'pycapo']"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p1.dependencies"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The Makefile we want to build is going to have a certain structure. Namely, it should:\n",
+    "\n",
+    "- Have a top-level entry point, probably a `.PHONY` that is easy for a parent Makefile to target\n",
+    "- The top-level entry point is going to depend on all the projects we have located\n",
+    "- Each project will be a target, a dependency of the top-level entry point\n",
+    "\n",
+    "Based on the project structure we also know this:\n",
+    "\n",
+    "- A project rule will depend on that project's dependencies\n",
+    "\n",
+    "This should cause the build order to fall out rather naturally."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'x86_64'"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      ".PHONY: all-python-projects\n",
+      "all-python-projects: shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz apps/cli/executables/alma_reingester/dist/alma_reingester-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz\n",
+      "\n",
+      "shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: \n",
+      "\tcd shared/schema && python setup.py bdist\n",
+      "\n",
+      "apps/cli/executables/alma_reingester/dist/alma_reingester-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz\n",
+      "\tcd apps/cli/executables/alma_reingester && python setup.py bdist\n",
+      "\n",
+      "shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: \n",
+      "\tcd shared/messaging/events && python setup.py bdist\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "class Makefile:\n",
+    "    def __init__(self, projects: List[Project]):\n",
+    "        self.projects = projects\n",
+    "    \n",
+    "    def build(self):\n",
+    "        output = sys.stdout\n",
+    "        self.build_entry_point(output)\n",
+    "        self.build_projects(output)\n",
+    "    \n",
+    "    def build_entry_point(self, output: TextIO):\n",
+    "        \"\"\"This builds the toplevel entry point, ready for inclusion in another Makefile\"\"\"\n",
+    "        toplevels = ' '.join(self.project_target(project) for project in self.projects)\n",
+    "        output.write('.PHONY: all-python-projects\\n')\n",
+    "        output.write(f'all-python-projects: {toplevels}\\n\\n')\n",
+    "    \n",
+    "    def project_target(self, project: Project) -> str:\n",
+    "        \"\"\"This converts a project to a distfile name, so that Make can know if it is up to date\"\"\"\n",
+    "        return f'{project.path}/dist/{project.name}-{project.version}.macosx-10.9-x86_64.tar.gz'\n",
+    "    \n",
+    "    def build_projects(self, output: TextIO):\n",
+    "        \"\"\"This generates all of the targets for each project we know about.\"\"\"\n",
+    "        for project in self.projects:\n",
+    "            self.build_project(project, output)\n",
+    "            \n",
+    "    def build_project(self, project: Project, output: TextIO):\n",
+    "        dependencies = [self.lookup_dependency(dep) for dep in project.dependencies]\n",
+    "        target_dependencies = ' '.join(self.project_target(dep) for dep in dependencies if dep)\n",
+    "        output.write(f'{self.project_target(project)}: {target_dependencies}\\n')\n",
+    "        output.write(f'\\tcd {project.path} && python setup.py bdist\\n\\n')\n",
+    "    \n",
+    "    def lookup_dependency(self, project_name: str) -> Optional[Project]:\n",
+    "        return next((project for project in self.projects if project.name == project_name), None)\n",
+    "\n",
+    "def load_projects() -> List[Project]:\n",
+    "    # the next line is commented out due to the fragility of the setup.py parsing logic\n",
+    "    # return [Project(Path(proj)) for (proj, subdirs, files) in os.walk(Path()) if 'setup.py' in files]\n",
+    "    return [Project(Path('shared/schema')), Project(Path('apps/cli/executables/alma_reingester')), Project(Path('shared/messaging/events'))]\n",
+    "\n",
+    "m = Makefile(load_projects())\n",
+    "m.build()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

SConstruct

+import os
+from pathlib import Path
+
+def setup_project(target, source, env):
+    dir = str(source[0])[:-9]
+    home = os.getcwd()
+    env.Execute("cd {} && python3 setup.py bdist && cd {}".format(dir, home))
+
+builder = Builder(action = setup_project)
+env = Environment(BUILDERS = {'Setup' : builder})
+
+projects = list(proj for (proj, subdirs, files) in os.walk(Path()) if "setup.py" in files)
+output = []
+for path in projects:
+    name = path.split('/')[-1]
+    output.append("{}/dist/{}".format(path.replace('.', os.getcwd()), name + "-4.0.0a1.dev1.macosx-10.15-x86_64.tar.gz"))
+
+for src, out in zip(projects, output):
+    env.Setup(source=src + "/setup.py", target=out)

alma_fetch.mk

+SHELL := /bin/bash
+alma_product_fetch_sources := $(shell find apps/cli/executables/alma_product_fetch -type f)
+
+build-alma: apps/cli/executables/alma_product_fetch/dist/alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz
+	@echo "built"
+
+apps/cli/executables/alma_product_fetch/dist/alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: $(alma_product_fetch_sources)
+	@echo building alma product fetch
+	cd apps/cli/executables/alma_product_fetch && \
+	python setup.py bdist
+
+clean-alma:
+	rm -f apps/cli/executables/alma_product_fetch/dist/alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz
\ No newline at end of file

apps/cli/executables/datafetcher/setup.py

@@ -2,37 +2,25 @@
 # -*- coding: utf-8 -*-
 
 from pathlib import Path
-
 from setuptools import setup
 
 VERSION = open('_version.py').readlines()[-1].split()[-1].strip("\"'")
 README = Path('README.md').read_text()
 
 requires = [
-    'pika==1.1.0',
-    'pycapo==0.3.0',
-    'bs4==0.0.1',
-    'beautifulsoup4==4.9.1',
-    'lxml==4.3.2',
-    'psycopg2==2.8.5',
-    'pyinstaller==3.2',
-    'pyopenssl=19.1.0',
-    'tox==3.1.3',
-    'tox-pyenv==1.1.0'
+    'pika>=1.1,<2',
+    'pycapo>=0.3.0,<1.0',
+    'beautifulsoup4>=4.9.1,<5.0',
+    'lxml>=4.3.2,<5.0',
+    'psycopg2>=2.8.5,<3.0',
+    'pyopenssl>=19.1.0,<20.0',
+    'requests>=2.23,<3.0'
 ]
 
 tests_require = [
-    'bs4==0.0.1',
-    'beautifulsoup4==4.9.1',
-    'psycopg2==2.8.5',
-    'pyopenssl=19.1.0',
-    'pyinstaller==3.2',
-    'pytest==5.4',
-    'pytest-runner==4.2',
-    'tox==3.1.3',
-    'tox-pyenv==1.1.0',
-
+    'pytest>=5.4,<6.0'
 ]
+
 setup(
     name=Path().absolute().name,
     version=VERSION,
@@ -42,10 +30,11 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['requests', 'pycapo'],
+    install_requires=requires,
+    tests_require=tests_require,
     keywords=[],
     packages=['datafetcher'],
-    package_dir={'':'src'},
+    package_dir={'': 'src'},
     classifiers=[
         'Programming Language :: Python :: 3.8'
     ],

apps/cli/executables/datafetcher/test/testing_utils.py

@@ -9,11 +9,11 @@ from pathlib import Path
 
 from pycapo import CapoConfig
 
-from src.datafetcher.errors import \
+from datafetcher.errors import \
     MissingSettingsException, NoProfileException
-from src.datafetcher.locations_report import \
+from datafetcher.locations_report import \
     LocationsReport
-from src.datafetcher.utilities import \
+from datafetcher.utilities import \
     REQUIRED_SETTINGS, get_arg_parser, \
     ExecutionSite
 

buildout.cfg

+[buildout]
+develop = src
+parts = alma-product-fetch
+
+[alma-product-fetch]
+recipe = setuprecipe
+path = apps/cli/executables/alma_product_fetch
\ No newline at end of file

conda.mk

+SHELL := /bin/bash
+os := $(shell uname -s)
+conda-install := $(shell which conda)
+
+miniconda-installed conda-install:
+	# Check if data env is activated
+	@([ ${CONDA_DEFAULT_ENV} == data ] && conda env update) || \
+	conda activate data
+
+conda-install:
+	if [ -z conda-install ]; \
+	then \
+		@url="https://repo.anaconda.com/miniconda/" \
+		if [ "$(os)" == "Darwin" ]; \
+		then \
+			file="Miniconda3-latest-MacOSX-x86_64.sh"; \
+		elif [ "$(os)" == "Linux" ]; \
+		then \
+			file="Miniconda3-latest-Linux-x86_64.sh"; \
+		fi \
+		wget $(addsuffix $(file),$(url)) -P ${HOME} \
+		chmod +x ${HOME}/$$file \
+		$$HOME/$$file \
+	fi
+
+clean-conda:
+	conda deactivate
\ No newline at end of file

setuptools.mk

+SHELL := /bin/bash
+py := python3
+home_dir := $(shell pwd)
+setup_dirs := $(shell find . -name setup.py | tr '\n' ' ' | sed "s/setup.py//g")
+
+build-python $(setup_dirs):
+	@for dir in $(setup_dirs) ; do \
+  		cd $$dir ; \
+  		$(py) setup.py develop ; \
+  		cd $(home_dir) ; \
+	done
+
+clean-python:
+	rm -Rf *.pyc *.pyo *~ *.egg-info
\ No newline at end of file

src/bdistrecipe/bdistrecipe.py

+import os
+
+class Recipe:
+    def __init__(self, buildout, name, options):
+        options['created'] = "{}/dist/{}".format(options['path'], name + "-4.0.0a1.dev1.tar.bz2")
+        self.options = options
+
+    def install(self):
+        self.options.created(self.options['created'])
+        home = os.getcwd()
+
+        # cd into correct directory
+        os.chdir(self.options['path'])
+        # Run setup.py
+        os.system("conda run -n data python3 setup.py bdist")
+        os.chdir(home)
+        # Deploy conda package to builder
+        builder_path = "/home/builder.aoc.nrao.edu/content/conda/"
+        platform = "osx64"
+        os.system("scp {}/{} builder:{}".format(home, self.options['created'], builder_path + platform))
+        os.system("ssh builder 'cd {} && conda index'".format(builder_path))
+
+        return self.options.created()
+
+    update = install
\ No newline at end of file

src/bdistrecipe/setup.py

+from setuptools import setup
+
+setup(
+    name='bdistrecipe',
+    version='0.1',
+    py_modules = ['bdistrecipe'],
+    entry_points = {"zc.buildout": ["default=bdistrecipe:Recipe"]}
+)
\ No newline at end of file

src/condarecipe/condarecipe.py

+import os
+
+class Recipe:
+    def __init__(self, buildout, name, options):
+        options['created'] = "{}/envs/{}".format(options['path'], name + options['env'])
+        self.options = options
+
+    def install(self):
+        self.options.created(self.options['created'])
+        os.system("source /opt/miniconda3/etc/profile.d/conda.sh && conda activate data")
+
+        return self.options.created()
+
+    update = install
\ No newline at end of file

src/condarecipe/setup.py

+from setuptools import setup
+
+setup(
+    name='condarecipe',
+    version='0.1',
+    py_modules = ['condarecipe'],
+    entry_points = {"zc.buildout": ["default=condarecipe:Recipe"]},
+)
\ No newline at end of file

src/setup.py

+from setuptools import setup
+
+setup(
+    name='setuprecipe',
+    version='0.1',
+    py_modules = ['setuprecipe'],
+    entry_points = {"zc.buildout": ["default=setuprecipe:Recipe"]},
+)
\ No newline at end of file

src/setuprecipe.py

+import os
+
+class Recipe:
+    def __init__(self, buildout, name, options):
+        options['created'] = "{}/dist/{}".format(options['path'], name + "-4.0.0a1.dev1.macosx-10.15-x86_64.tar.gz")
+        self.options = options
+
+    def install(self):
+        self.options.created(self.options['created'])
+        # home = os.getcwd()
+
+        # cd into correct directory
+        os.chdir(self.options['path'])
+        os.system("ls -la")
+        # Run setup.py
+        os.system("python3 setup.py bdist")
+
+        return self.options.created()
+
+    update = install
\ No newline at end of file

support/conda/datafetcher/meta.yaml

+{% set name = "datafetcher" %}
+{% set version = "4.0.0a1.dev1" %}
+
+package:
+  name: "{{ name|lower }}"
+  version: "{{ version }}"
+
+build:
+  entry_points:
+    - datafetcher = datafetcher.commands:main
+  script: {{ PYTHON }} setup.py install
+
+source:
+  path: ../../../apps/cli/executables/datafetcher
+
+requirements:
+  build:
+    - python==3.8
+    - pika>=1.1,<2
+    - pycapo>=0.3.0,<1.0
+    - beautifulsoup4>=4.9.1,<5.0
+    - lxml>=4.3.2,<5.0
+    - psycopg2>=2.8.5,<3.0
+    - pyopenssl>=19.1.0,<20.0
+    - requests>=2.23,<3.0
+  run:
+    - python==3.8
+    - pika>=1.1,<2
+    - pycapo>=0.3.0,<1.0
+    - beautifulsoup4>=4.9.1,<5.0
+    - lxml>=4.3.2,<5.0
+    - psycopg2>=2.8.5,<3.0
+    - pyopenssl>=19.1.0,<20.0
+    - requests>=2.23,<3.0
+
+test:
+  source_files:
+    - test/
+  requires:
+    - pytest>=5.4,<6.0
+  commands:
+    - pytest -vv --log-level=DEBUG --showlocals test/datafetcher_test.py
+
+about:
+  license: "GPL"
+  license_family: "GPL"
+  summary: "NRAO Archive Data Fetcher Script"

support/conda/events/meta.yaml

+{% set name = "events" %}
+{% set version = "4.0.0a1.dev1" %}
+
+package:
+  name: "{{ name|lower }}"
+  version: "{{ version }}"
+
+source:
+  path: /Users/nhertz/data/shared/messaging/events
+
+build:
+  number: 0
+  script:
+    - python setup.py install
+
+requirements:
+  run:
+    - python=3.8
+    - pika
+    - pycapo
+
+about:
+  license: "GPL"
+  license_family: "GPL"
+  summary: "NRAO Archive Data Fetcher Script"

support/conda/pycapo/meta.yaml

 {% set name = "pycapo" %}
-{% set version = "0.2.1.post1" %}
+{% set version = "0.3.0" %}
 
 package:
   name: "{{ name|lower }}"
@@ -7,7 +7,7 @@ package:
 
 source:
   url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
-  sha256: 703002d913d4e38a4bcce7748674ef58d4fd63c9fd099316b4643a89955ff361
+  sha256: 507d47278a66f2a67dc817e2dae533374ad0c8312aed7ab6e23e1b322baef5c6
 
 build:
   number: 0