Merge branch 'SSA-6320-workflow-service' into release/WS-0.1

apps/cli/executables/alma_product_fetch/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/alma_product_fetch/_version.py').readlines()[-1].split()[-1]
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive ALMA Product Fetcher',
     long_description=README,

apps/cli/executables/alma_reingester/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/alma_reingester/_version.py').readlines()[-1].split()[-1].st
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive ALMA Reingester',
     long_description=README,
@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['cx_Oracle', 'pycapo', 'psycopg2', 'events', 'schema'],
+    install_requires=['cx_Oracle', 'pycapo', 'psycopg2', 'ssa-events', 'ssa-schema'],
     keywords=[],
     packages=['alma_reingester'],
     package_dir={'':'src'},

apps/cli/executables/datafetcher/setup.py

@@ -22,7 +22,7 @@ tests_require = [
 ]
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Data Fetcher Script',
     long_description=README,

apps/cli/executables/epilogue/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/epilogue/_version.py').readlines()[-1].split()[-1].strip("\"
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Job Epilogue Script',
     long_description=README,

apps/cli/executables/ingestion/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/ingestion/_version.py').readlines()[-1].split()[-1].strip("\
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Ingestion',
     long_description=README,
@@ -22,11 +22,11 @@ setup(
                       'numpy',
                       'psycopg2',
                       'pycapo',
-                      'pymygdala',
                       'requests',
-                      'schema',
-                      'sqlalchemy',
-                      'weblog_thumbs'],
+                      'ssa-pymygdala',
+                      'ssa-schema',
+                      'ssa-weblog_thumbs',
+                      'sqlalchemy'],
     keywords=[],
     packages=['ingestion'],
     package_dir={'':'src'},

apps/cli/executables/null/setup.py

@@ -11,7 +11,7 @@ tests_require = [
     'pytest>=5.4,<6.0'
 ]
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='Workspaces null executable.',
     long_description=README,

apps/cli/executables/vlba_grabber/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/vlba_grabber/_version.py').readlines()[-1].split()[-1].strip
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive VLBA Grabber',
     long_description=README,
@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['requests', 'pymygdala', 'pycapo', 'beautifulsoup4'],
+    install_requires=['requests', 'ssa-pymygdala', 'pycapo', 'beautifulsoup4'],
     keywords=[],
     packages=['vlba_grabber'],
     package_dir={'':'src'},

apps/cli/executables/vulture/README.md

+# Vulture
+
+Vulture is a fake HTCondor. It doesn't submit jobs to a cluster, instead it runs them locally. 
+
+This tool is intended to bypass HTCondor for integration testing purposes.

apps/cli/executables/vulture/setup.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from pathlib import Path
+from setuptools import setup
+
+VERSION = open('src/vulture/_version.py').readlines()[-1].split()[-1].strip("\"'")
+README = Path('README.md').read_text()
+
+tests_require = [
+    'pytest>=5.4,<6.0'
+]
+setup(
+    name='ssa-' + Path().absolute().name,
+    version=VERSION,
+    description='HTCondor fake executor.',
+    long_description=README,
+    author='NRAO SSA Team',
+    author_email='dms-ssa@nrao.edu',
+    url='TBD',
+    license="GPL",
+    # install_requires=requires,
+    tests_require=tests_require,
+    keywords=[],
+    packages=['vulture'],
+    package_dir={'':'src'},
+    classifiers=[
+        'Programming Language :: Python :: 3.8'
+    ],
+    entry_points={
+        'console_scripts': ['vulture = vulture.execute:main']
+    },
+)

apps/cli/executables/vulture/src/vulture/_version.py

+""" Version information for this package, don't put anything else here. """
+___version___ = '4.0.0a1.dev1'

apps/cli/executables/vulture/src/vulture/condorlite.py

+class Job:
+    @staticmethod
+    def parse(file):
+        job = Job()
+        # do something here to iterate the file contents and parse it
+        raise NotImplementedError
+
+    def execute(self):
+        raise NotImplementedError
+
+
+class Dag:
+    @staticmethod
+    def parse(file):
+        dag = Dag()
+        # do something here to iterate the file contents and parse it
+        raise NotImplementedError
+
+
+    def execute(self):
+        raise NotImplementedError
+

apps/cli/executables/vulture/src/vulture/execute.py

+"""
+CLI to our fake dagman executor and fake job executor
+"""
+
+import argparse as ap
+from pathlib import Path
+
+
+def execute_dag(file: Path):
+    """
+    Execute the DAG in the given file.
+    :param file:
+    :return:
+    """
+    dag = Dag.parse(file)
+    dag.execute()
+
+
+def execute_job(file: Path):
+    """
+    Execute the job in the given file.
+    :param file:
+    :return:
+    """
+    job = Job.parse(file)
+    job.execute()
+
+
+def main():
+    parser = ap.ArgumentParser('vulture', description='Tool to run HTCondor job specs and DAG workflows locally')
+    subparsers = parser.add_subparsers()
+    dag = subparsers.add_parser('dag', help='DAG operations')
+    sub2 = dag.add_subparsers()
+    execute = sub2.add_parser('execute', description='Execute an HTCondor DAGMAN DAG')
+    execute.set_defaults(func=execute_dag)
+    execute.add_argument('file', nargs=1, type=Path, help='DAGMAN file to execute')
+
+    job = subparsers.add_parser('job', help='Job operations')
+    sub2 = job.add_subparsers()
+    execute = sub2.add_parser('execute', description='Execute an HTCondor job specification')
+    execute.set_defaults(func=execute_job)
+    execute.add_argument('file', nargs=1, type=Path, help='Job file to execute')
+
+    args = parser.parse_args()
+
+    if 'func' in args and 'file' in args:
+        args.func(args.file)
+    else:
+        parser.print_usage()

apps/cli/executables/weblog_thumbs/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/weblog_thumbs/_version.py').readlines()[-1].split()[-1].stri
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Weblog Thumbs',
     long_description=README,
@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['pymygdala'],
+    install_requires=['ssa-pymygdala'],
     keywords=[],
     packages=['weblog_thumbs'],
     package_dir={'':'src'},

apps/cli/launchers/pymygdala/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/pymygdala/_version.py').readlines()[-1].split()[-1].strip("\
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Pymygdala General Message Sender',
     long_description=README,

apps/cli/launchers/wf/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/wf/_version.py').readlines()[-1].split()[-1].strip("\"'")
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Workflow Commands',
     long_description=README,
@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['pika', 'psycopg2', 'cx_Oracle', 'pymygdala', 'pycapo', 'schema'],
+    install_requires=['pika', 'psycopg2', 'cx_Oracle', 'ssa-pymygdala', 'pycapo', 'ssa-schema'],
     keywords=[],
     packages=['wf'],
     package_dir={'':'src'},

apps/cli/utilities/datafinder/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/datafinder/_version.py').readlines()[-1].split()[-1].strip("
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Data Finding Tools',
     long_description=README,
@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['pandas', 'requests', 'schema', 'sqlalchemy', 'tqdm'],
+    install_requires=['pandas', 'requests', 'ssa-schema', 'sqlalchemy', 'tqdm'],
     keywords=[],
     packages=['datafinder'],
     package_dir={'':'src'},

apps/cli/utilities/dumplogs/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/dumplogs/_version.py').readlines()[-1].split()[-1].strip("\"
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Dumplogs',
     long_description=README,

apps/cli/utilities/faultchecker/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/faultchecker/_version.py').readlines()[-1].split()[-1].strip
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive Fault Checker',
     long_description=README,

apps/cli/utilities/mr_books/setup.py

@@ -8,7 +8,7 @@ VERSION = open('src/mr_books/_version.py').readlines()[-1].split()[-1].strip("\"
 README = Path('README.md').read_text()
 
 setup(
-    name=Path().absolute().name,
+    name='ssa-' + Path().absolute().name,
     version=VERSION,
     description='NRAO Archive mr_books',
     long_description=README,
@@ -16,7 +16,7 @@ setup(
     author_email='dms-ssa@nrao.edu',
     url='TBD',
     license="GPL",
-    install_requires=['pendulum', 'pid', 'pycapo', 'pymygdala', 'schema', 'sqlalchemy'],
+    install_requires=['pendulum', 'pid', 'pycapo', 'ssa-pymygdala', 'ssa-schema', 'sqlalchemy'],
     keywords=[],
     packages=['mr_books'],
     package_dir={'':'src'},