Merge branch 'main' into SSA-6324-pytest-branch2-scode-update

.gitignore

@@ -46,3 +46,16 @@ docs/_build
 /workflow-all/workflow-jobs/src/main/resources/edu/nrao/archive/workflow/scripts/open-permissions.sh
 /archiveIface/archiveIface/webpack/stats.json
 /git-info.txt
+develop-eggs
+build/metadata
+#**/build/**
+apps/**/build/**
+services/**/build/**
+shared/**/build/**
+projects_checklist.txt
+.installed.cfg
+.ipynb_checkpoints
+deps.png
+build/pkgs
+eggs
+parts
\ No newline at end of file

Makefile

-include conda.mk
-include setuptools.mk
-include alma_fetch.mk
+all: ${HOME}/miniconda3 ${HOME}/miniconda3/envs/data metadata build
 
-SHELL := /bin/bash
+${HOME}/miniconda3:
+	@echo Please install miniconda to ${HOME}/miniconda3
+	@exit 1
 
-build-base:
-	@echo "Building..."
-	make miniconda-installed
-	make build-python
-	@echo "Finished build."
+${HOME}/miniconda3/envs/data: environment.yml
+	conda env update
+	touch $@
 
-build-almafetch:
-	@echo "Building ALMA fetcher."
-	make build-alma
+.PHONY: metadata
+metadata:
+	buildout parts=gen_metadata
 
+.PHONY: build
+build:
+	buildout parts=build_pkgs name=all

Python Meta-make.ipynb

-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "from pathlib import Path\n",
-    "import platform\n",
-    "import sys\n",
-    "from typing import List, Optional, TextIO"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "A directory contains a project if there is a `setup.py` file in it."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['./shared/schema',\n",
-       " './shared/support',\n",
-       " './shared/messaging/events',\n",
-       " './apps/cli/executables/ingestion',\n",
-       " './apps/cli/executables/weblog_thumbs',\n",
-       " './apps/cli/executables/alma_reingester',\n",
-       " './apps/cli/executables/alma_product_fetch',\n",
-       " './apps/cli/executables/epilogue',\n",
-       " './apps/cli/executables/datafetcher',\n",
-       " './apps/cli/executables/vlba_grabber',\n",
-       " './apps/cli/utilities/s_code_project_updater',\n",
-       " './apps/cli/utilities/proprietary_setter',\n",
-       " './apps/cli/utilities/mr_clean',\n",
-       " './apps/cli/utilities/faultchecker',\n",
-       " './apps/cli/utilities/mr_books',\n",
-       " './apps/cli/utilities/dumplogs',\n",
-       " './apps/cli/utilities/datafinder',\n",
-       " './apps/cli/utilities/qa_results',\n",
-       " './apps/cli/launchers/pymygdala',\n",
-       " './apps/cli/launchers/wf',\n",
-       " './services/archive']"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "projects = list(proj for (proj, subdirs, files) in os.walk(Path()) if 'setup.py' in files)\n",
-    "projects"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here are some things we know about projects:\n",
-    "\n",
-    "- Every project has a name, which is the same as the name of the directory that contains it.\n",
-    "- Every project has a version, which happens to be in a file in `src/$PROJECT/_version.py` in the same format\n",
-    "- Every project has certain dependencies, which are enumerated in the setup.py as `install_requires`\n",
-    "\n",
-    "Let's build up this abstraction."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<Project name=datafetcher version=4.0.0a1.dev1>"
-      ]
-     },
-     "execution_count": 44,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "class Project:\n",
-    "    def __init__(self, path: Path):\n",
-    "        self.path = path\n",
-    "        self.name = path.name\n",
-    "    \n",
-    "    @property\n",
-    "    def version(self) -> str:\n",
-    "        \"\"\"Compute and return the version in the _version.py file under the project.\"\"\"\n",
-    "        # to compute the version, we must locate and parse the _version.py file\n",
-    "        version_file = self.path / 'src' / self.name / '_version.py'\n",
-    "        \n",
-    "        # this line is basically cribbed from the standard setup.py file\n",
-    "        return version_file.open().readlines()[-1].split()[-1].strip(\"\\\"'\")\n",
-    "    \n",
-    "    @property\n",
-    "    def dependencies(self) -> List[str]:\n",
-    "        \"\"\"Returns the list of dependencies under this project\"\"\"\n",
-    "        \n",
-    "        # to compute the dependencies, we have to do something a bit more gross\n",
-    "        # we have to open the setup.py file and look for a line with \"install_requires\"\n",
-    "        # once we have that line, we have to parse out the Python list\n",
-    "        # we can go ahead and cheat here with eval() for today.\n",
-    "        #\n",
-    "        # Unfortunately, this doesn't work if they span multiple lines, which is a\n",
-    "        # frequent occurrence\n",
-    "        setup_file = self.path / 'setup.py'\n",
-    "        \n",
-    "        for line in setup_file.open().readlines():\n",
-    "            if 'install_requires' in line:\n",
-    "                return eval(line.split('=')[1].strip().strip(','))\n",
-    "        \n",
-    "        # if we made it here, we never found that line, so this project \n",
-    "        # has no dependencies; let's make life easy on ourselves and \n",
-    "        # keep the API simple\n",
-    "        return []\n",
-    "    \n",
-    "    def __repr__(self) -> str:\n",
-    "        return f'<Project name={self.name} version={self.version}>'\n",
-    "    \n",
-    "p1 = Project(Path('apps/cli/executables/datafetcher'))\n",
-    "p1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['requests', 'pycapo']"
-      ]
-     },
-     "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "p1.dependencies"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The Makefile we want to build is going to have a certain structure. Namely, it should:\n",
-    "\n",
-    "- Have a top-level entry point, probably a `.PHONY` that is easy for a parent Makefile to target\n",
-    "- The top-level entry point is going to depend on all the projects we have located\n",
-    "- Each project will be a target, a dependency of the top-level entry point\n",
-    "\n",
-    "Based on the project structure we also know this:\n",
-    "\n",
-    "- A project rule will depend on that project's dependencies\n",
-    "\n",
-    "This should cause the build order to fall out rather naturally."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'x86_64'"
-      ]
-     },
-     "execution_count": 23,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 46,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      ".PHONY: all-python-projects\n",
-      "all-python-projects: shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz apps/cli/executables/alma_reingester/dist/alma_reingester-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz\n",
-      "\n",
-      "shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: \n",
-      "\tcd shared/schema && python setup.py bdist\n",
-      "\n",
-      "apps/cli/executables/alma_reingester/dist/alma_reingester-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz shared/schema/dist/schema-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz\n",
-      "\tcd apps/cli/executables/alma_reingester && python setup.py bdist\n",
-      "\n",
-      "shared/messaging/events/dist/events-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: \n",
-      "\tcd shared/messaging/events && python setup.py bdist\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "class Makefile:\n",
-    "    def __init__(self, projects: List[Project]):\n",
-    "        self.projects = projects\n",
-    "    \n",
-    "    def build(self):\n",
-    "        output = sys.stdout\n",
-    "        self.build_entry_point(output)\n",
-    "        self.build_projects(output)\n",
-    "    \n",
-    "    def build_entry_point(self, output: TextIO):\n",
-    "        \"\"\"This builds the toplevel entry point, ready for inclusion in another Makefile\"\"\"\n",
-    "        toplevels = ' '.join(self.project_target(project) for project in self.projects)\n",
-    "        output.write('.PHONY: all-python-projects\\n')\n",
-    "        output.write(f'all-python-projects: {toplevels}\\n\\n')\n",
-    "    \n",
-    "    def project_target(self, project: Project) -> str:\n",
-    "        \"\"\"This converts a project to a distfile name, so that Make can know if it is up to date\"\"\"\n",
-    "        return f'{project.path}/dist/{project.name}-{project.version}.macosx-10.9-x86_64.tar.gz'\n",
-    "    \n",
-    "    def build_projects(self, output: TextIO):\n",
-    "        \"\"\"This generates all of the targets for each project we know about.\"\"\"\n",
-    "        for project in self.projects:\n",
-    "            self.build_project(project, output)\n",
-    "            \n",
-    "    def build_project(self, project: Project, output: TextIO):\n",
-    "        dependencies = [self.lookup_dependency(dep) for dep in project.dependencies]\n",
-    "        target_dependencies = ' '.join(self.project_target(dep) for dep in dependencies if dep)\n",
-    "        output.write(f'{self.project_target(project)}: {target_dependencies}\\n')\n",
-    "        output.write(f'\\tcd {project.path} && python setup.py bdist\\n\\n')\n",
-    "    \n",
-    "    def lookup_dependency(self, project_name: str) -> Optional[Project]:\n",
-    "        return next((project for project in self.projects if project.name == project_name), None)\n",
-    "\n",
-    "def load_projects() -> List[Project]:\n",
-    "    # the next line is commented out due to the fragility of the setup.py parsing logic\n",
-    "    # return [Project(Path(proj)) for (proj, subdirs, files) in os.walk(Path()) if 'setup.py' in files]\n",
-    "    return [Project(Path('shared/schema')), Project(Path('apps/cli/executables/alma_reingester')), Project(Path('shared/messaging/events'))]\n",
-    "\n",
-    "m = Makefile(load_projects())\n",
-    "m.build()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

SConstruct

-import os
-from pathlib import Path
-
-def setup_project(target, source, env):
-    dir = str(source[0])[:-9]
-    home = os.getcwd()
-    env.Execute("cd {} && python3 setup.py bdist && cd {}".format(dir, home))
-
-builder = Builder(action = setup_project)
-env = Environment(BUILDERS = {'Setup' : builder})
-
-projects = list(proj for (proj, subdirs, files) in os.walk(Path()) if "setup.py" in files)
-output = []
-for path in projects:
-    name = path.split('/')[-1]
-    output.append("{}/dist/{}".format(path.replace('.', os.getcwd()), name + "-4.0.0a1.dev1.macosx-10.15-x86_64.tar.gz"))
-
-for src, out in zip(projects, output):
-    env.Setup(source=src + "/setup.py", target=out)

alma_fetch.mk

-SHELL := /bin/bash
-alma_product_fetch_sources := $(shell find apps/cli/executables/alma_product_fetch -type f)
-
-build-alma: apps/cli/executables/alma_product_fetch/dist/alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz
-	@echo "built"
-
-apps/cli/executables/alma_product_fetch/dist/alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz: $(alma_product_fetch_sources)
-	@echo building alma product fetch
-	cd apps/cli/executables/alma_product_fetch && \
-	python setup.py bdist
-
-clean-alma:
-	rm -f apps/cli/executables/alma_product_fetch/dist/alma_product_fetch-4.0.0a1.dev1.macosx-10.9-x86_64.tar.gz
\ No newline at end of file

apps/cli/executables/alma_product_fetch/src/alma_product_fetch/__init__.py

-r"""
-    An import of the Joint Alma Observatories listfiles.py into the AAT/PPI system.
-
-    Given an MOUS Status UID (the most common identifier for a processing unit),
-    perform a database query to obtain the list of files required for a restore
-    and proceed to obtain them from the NAASC NGAS machines.
-
-    Initially written to use wget, this is rewritten to use requests and to
-    attempt to validate the data retrieved.
-
-"""

apps/cli/executables/alma_product_fetch/src/alma_product_fetch/commands.py

@@ -10,7 +10,7 @@ import cx_Oracle as almadb
 import requests
 import sys
 from ._version import ___version___ as version
-from pymygdala import LogHandler
+from pymygdala.models import LogHandler
 from pycapo import CapoConfig
 
 """

apps/cli/executables/alma_reingester/setup.py

@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['cx-Oracle', 'pycapo', 'psycopg2', 'events', 'schema'],
+    install_requires=['cx_Oracle', 'pycapo', 'psycopg2', 'events', 'schema'],
     keywords=[],
     packages=['alma_reingester'],
     package_dir={'':'src'},

support/conda/datafetcher/meta.yaml → apps/cli/executables/datafetcher/conda/meta.yaml

@@ -7,11 +7,12 @@ package:
 
 build:
   entry_points:
-    - datafetcher = datafetcher.commands:main
-  script: {{ PYTHON }} setup.py install
+    - datafetcher = work.datafetcher.commands:main
+  script: {{ PYTHON }} setup.py develop
 
 source:
-  path: ../../../apps/cli/executables/datafetcher
+  path: ../../apps/cli/executables/datafetcher
+#  - path: ../../apps/cli/executables/datafetcher/src/datafetcher
 
 requirements:
   build:
@@ -25,23 +26,23 @@ requirements:
     - requests>=2.23,<3.0
   run:
     - python==3.8
-    - pika>=1.1,<2
-    - pycapo>=0.3.0,<1.0
-    - beautifulsoup4>=4.9.1,<5.0
-    - lxml>=4.3.2,<5.0
-    - psycopg2>=2.8.5,<3.0
-    - pyopenssl>=19.1.0,<20.0
-    - requests>=2.23,<3.0
+#    - pika>=1.1,<2
+#    - pycapo>=0.3.0,<1.0
+#    - beautifulsoup4>=4.9.1,<5.0
+#    - lxml>=4.3.2,<5.0
+#    - psycopg2>=2.8.5,<3.0
+#    - pyopenssl>=19.1.0,<20.0
+#    - requests>=2.23,<3.0
 
-test:
-  source_files:
-    - test/
-  requires:
-    - pytest>=5.4,<6.0
-  commands:
-    - pytest -vv --log-level=DEBUG --showlocals test/datafetcher_test.py
+#test:
+#  source_files:
+#    - test/
+#  requires:
+#    - pytest>=5.4,<6.0
+#  commands:
+#    - pytest -vv --log-level=DEBUG --showlocals test/datafetcher_test.py
 
 about:
   license: "GPL"
   license_family: "GPL"
-  summary: "NRAO Archive Data Fetcher Script"
+  summary: "NRAO Archive Data Fetcher Script"
\ No newline at end of file

apps/cli/executables/epilogue/setup.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from setuptools import setup, find_packages
 
-VERSION = open('src/_version.py').readlines()[-1].split()[-1].strip("\"'")
+VERSION = open('src/epilogue/_version.py').readlines()[-1].split()[-1].strip("\"'")
 README = Path('README.md').read_text()
 
 setup(