From 523fbca10ed5cf72fdedf7805b4ef47dad722133 Mon Sep 17 00:00:00 2001
From: Daniel K Lyons <dlyons@nrao.edu>
Date: Wed, 17 Jun 2020 13:20:36 -0600
Subject: [PATCH] wf
---
apps/cli/executables/ingestion/setup.py | 14 +++-
apps/cli/launchers/stdwf/stdwf.py | 77 -------------------
apps/cli/launchers/wf/README.md | 0
apps/cli/launchers/wf/__init__.py | 3 -
apps/cli/launchers/wf/setup.py | 45 +++++++++++
apps/cli/launchers/wf/src/wf/__init__.py | 3 +
apps/cli/launchers/wf/src/wf/_version.py | 2 +
.../cli/launchers/wf/{ => src/wf}/commands.py | 4 +-
.../wf/{ => src/wf}/ingest_wf_interfaces.py | 6 +-
.../wf/{ => src/wf}/ous_wf_interfaces.py | 6 +-
.../wf/{ => src/wf}/utility_wf_interfaces.py | 4 +-
11 files changed, 72 insertions(+), 92 deletions(-)
delete mode 100644 apps/cli/launchers/stdwf/stdwf.py
create mode 100644 apps/cli/launchers/wf/README.md
delete mode 100644 apps/cli/launchers/wf/__init__.py
create mode 100644 apps/cli/launchers/wf/setup.py
create mode 100644 apps/cli/launchers/wf/src/wf/__init__.py
create mode 100644 apps/cli/launchers/wf/src/wf/_version.py
rename apps/cli/launchers/wf/{ => src/wf}/commands.py (99%)
rename apps/cli/launchers/wf/{ => src/wf}/ingest_wf_interfaces.py (99%)
rename apps/cli/launchers/wf/{ => src/wf}/ous_wf_interfaces.py (99%)
rename apps/cli/launchers/wf/{ => src/wf}/utility_wf_interfaces.py (97%)
diff --git a/apps/cli/executables/ingestion/setup.py b/apps/cli/executables/ingestion/setup.py
index 0d1e51d63..a689b4e90 100644
--- a/apps/cli/executables/ingestion/setup.py
+++ b/apps/cli/executables/ingestion/setup.py
@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
from pathlib import Path
-from setuptools import setup, find_packages
+from setuptools import setup
VERSION = open('src/ingestion/_version.py').readlines()[-1].split()[-1].strip("\"'")
README = Path('README.md').read_text()
@@ -16,7 +16,17 @@ setup(
author_email='dms-ssa@nrao.edu',
url='TBD',
license="GPL",
- install_requires=['numpy', 'psycopg2', 'pycapo', 'pymygdala', 'lxml', 'sqlalchemy', 'schema', 'astropy', 'requests', 'jxmlease', 'weblog_thumbs'],
+ install_requires=['astropy',
+ 'jxmlease',
+ 'lxml',
+ 'numpy',
+ 'psycopg2',
+ 'pycapo',
+ 'pymygdala',
+ 'requests',
+ 'schema',
+ 'sqlalchemy',
+ 'weblog_thumbs'],
keywords=[],
packages=['ingestion'],
package_dir={'':'src'},
diff --git a/apps/cli/launchers/stdwf/stdwf.py b/apps/cli/launchers/stdwf/stdwf.py
deleted file mode 100644
index 1ea732786..000000000
--- a/apps/cli/launchers/stdwf/stdwf.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-The Standard Workflow
-
-The standard workflow reads a PPR and executes the following steps:
-
-1. Fetch data
-2. Run CASA
-3. Deliver results
-4. Ingest
-
-The standard workflow is actually structured as a dependency graph. Raw
-data is a dependency for running CASA, and the dependency is satisfied by
-running the data fetcher. Everything runs in a recovery mode, as if the task
-crashed some earlier time and is being rerun.
-"""
-
-class Node:
- """
- A node in the dependency graph.
- """
-
- def __init__(self, action=lambda: True):
- self.action = action
- self.dependencies = []
-
- def depends_on(self, other_node):
- """Add a dependency to this action."""
- self.dependencies.append(other_node)
-
- def execute(self):
- """
- Executes this node, if the dependencies all return true.
- Returns True if the execution is successful.
- """
- for dependency in self.dependencies:
- if not dependency.execute():
- return False
-
- # if we made it here, execute this action
- return self.action()
-
-
-class PPR:
- @classmethod
- def parse(cls, ppr_filename):
- "Parse the PPR XML file"
- pass
-
- def build_fetch_step(self): pass
- def build_casa_step(self): pass
- def build_delivery_step(self): pass
- def build_ingest_step(self): pass
-
-class Plan:
- def __init__(self):
- self.ppr = PPR.parse("PPR.xml")
-
- def generate_plan(self):
- "Read the XML for the PPR and generate the plan from it"
- fetch_step = Node(self.ppr.build_fetch_step())
- processing_step = Node(self.ppr.build_casa_step())
- delivery_step = Node(self.ppr.build_delivery_step())
- ingest_step = Node(self.ppr.build_ingest_step())
-
- processing_step.depends_on(fetch_step)
- delivery_step .depends_on(processing_step)
- ingest_step .depends_on(processing_step)
-
- goal = Node()
- goal.depends_on(delivery_step)
- goal.depends_on(ingest_step)
-
- return goal
-
-if __name__ == '__main__':
- plan = Plan()
- plan.generate_plan().execute()
diff --git a/apps/cli/launchers/wf/README.md b/apps/cli/launchers/wf/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/cli/launchers/wf/__init__.py b/apps/cli/launchers/wf/__init__.py
deleted file mode 100644
index b94ad2c78..000000000
--- a/apps/cli/launchers/wf/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-r""" wf: utility for running workflows
-"""
-from pyat.wf.commands import wf, WorkflowTracker
\ No newline at end of file
diff --git a/apps/cli/launchers/wf/setup.py b/apps/cli/launchers/wf/setup.py
new file mode 100644
index 000000000..c496a495b
--- /dev/null
+++ b/apps/cli/launchers/wf/setup.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from pathlib import Path
+from setuptools import setup
+
+VERSION = open('src/wf/_version.py').readlines()[-1].split()[-1].strip("\"'")
+README = Path('README.md').read_text()
+
+setup(
+ name=Path().absolute().name,
+ version=VERSION,
+ description='NRAO Archive Workflow Commands',
+ long_description=README,
+ author='NRAO SSA Team',
+ author_email='dms-ssa@nrao.edu',
+ url='TBD',
+ license="GPL",
+ install_requires=['pika', 'psycopg2', 'cx-Oracle', 'pymygdala', 'pycapo', 'schema'],
+ keywords=[],
+ packages=['wf'],
+ package_dir={'':'src'},
+ classifiers=[
+ 'Programming Language :: Python :: 3.8'
+ ],
+ entry_points={
+ 'console_scripts': [
+ 'wf = wf.commands:wf',
+ 'calFileSets = wf.commands:launch_calibrations',
+ 'recalDirectories = wf.commands:launch_recalibrations',
+ 'reingest = wf.commands:launch_reingest',
+ 'sdmIngest = wf.ingest_wf_interfaces:launch_sdm_ingestion',
+ 'bdfIngest = wf.ingest_wf_interfaces:launch_bdf_ingestion',
+ 'restoreToCache = wf.commands:launch_restore_to_cache',
+ 'almaRestore = wf.ous_wf_interfaces:launch_alma_restore',
+ 'almaImageCube = wf.ous_wf_interfaces:launch_alma_cube_imaging',
+ 'almaReimageCube = wf.ous_wf_interfaces:launch_reimaging_run',
+ 'audiPass = wf.ingest_wf_interfaces:launch_alma_image_ingest_qa',
+ 'vlbaFitsIngest = wf.ingest_wf_interfaces:launch_idifits_ingestion',
+ 'vlassQlIngest = wf.ingest_wf_interfaces:launch_image_ingestion',
+ 'vlassQlReingest = wf.ingest_wf_interfaces:launch_image_reingestion',
+ 'qaClean = wf.utility_wf_interfaces:launch_deletions',
+ ]
+ },
+)
diff --git a/apps/cli/launchers/wf/src/wf/__init__.py b/apps/cli/launchers/wf/src/wf/__init__.py
new file mode 100644
index 000000000..d8682ebfd
--- /dev/null
+++ b/apps/cli/launchers/wf/src/wf/__init__.py
@@ -0,0 +1,3 @@
+r""" wf: utility for running workflows
+"""
+from .commands import wf, WorkflowTracker
\ No newline at end of file
diff --git a/apps/cli/launchers/wf/src/wf/_version.py b/apps/cli/launchers/wf/src/wf/_version.py
new file mode 100644
index 000000000..e63ec3a14
--- /dev/null
+++ b/apps/cli/launchers/wf/src/wf/_version.py
@@ -0,0 +1,2 @@
+""" Version information for this package, don't put anything else here. """
+___version___ = '4.0a1.dev1'
diff --git a/apps/cli/launchers/wf/commands.py b/apps/cli/launchers/wf/src/wf/commands.py
similarity index 99%
rename from apps/cli/launchers/wf/commands.py
rename to apps/cli/launchers/wf/src/wf/commands.py
index f857a2778..a78c70155 100644
--- a/apps/cli/launchers/wf/commands.py
+++ b/apps/cli/launchers/wf/src/wf/commands.py
@@ -11,8 +11,8 @@ import pika
import sys
import time
-from pyat import version
-from pyat.pymygdala import LogHandler, RPCEvent
+from ._version import ___version___ as version
+from pymygdala import LogHandler, RPCEvent
from pycapo import CapoConfig
_DESCRIPTION = """AAT/PPI workflow launcher, version {}. Launch a workflow from
diff --git a/apps/cli/launchers/wf/ingest_wf_interfaces.py b/apps/cli/launchers/wf/src/wf/ingest_wf_interfaces.py
similarity index 99%
rename from apps/cli/launchers/wf/ingest_wf_interfaces.py
rename to apps/cli/launchers/wf/src/wf/ingest_wf_interfaces.py
index fb92352b2..917c70646 100644
--- a/apps/cli/launchers/wf/ingest_wf_interfaces.py
+++ b/apps/cli/launchers/wf/src/wf/ingest_wf_interfaces.py
@@ -9,11 +9,11 @@ import time
import logging
-from pyat.wf import wf, WorkflowTracker
+from . import wf, WorkflowTracker
from pathlib import Path
-from pyat import version
+from _version import ___version___ as version
from pycapo import CapoConfig
-from pyat.wf.ous_wf_interfaces import lookup_single_locator, lookup_project_code, lookup_eb_uid
+from .ous_wf_interfaces import lookup_single_locator, lookup_project_code, lookup_eb_uid
# Ingestion Workflow CLIs
def _make_ingestion_parser(ingestion_description, major_arg):
diff --git a/apps/cli/launchers/wf/ous_wf_interfaces.py b/apps/cli/launchers/wf/src/wf/ous_wf_interfaces.py
similarity index 99%
rename from apps/cli/launchers/wf/ous_wf_interfaces.py
rename to apps/cli/launchers/wf/src/wf/ous_wf_interfaces.py
index f5831068d..c0aeb9d64 100644
--- a/apps/cli/launchers/wf/ous_wf_interfaces.py
+++ b/apps/cli/launchers/wf/src/wf/ous_wf_interfaces.py
@@ -4,15 +4,15 @@ import os
import sys
import json
import psycopg2
-import pyat.schema
+import schema
import argparse as ap
import cx_Oracle as almadb
-from pyat import version
+from _version import ___version___ as version
from datetime import datetime
from pycapo import CapoConfig
from pathlib import Path
-from pyat.wf import wf, WorkflowTracker
+from . import wf, WorkflowTracker
# Alma Restores CLI:
_RUN_ALMA_RESTORE_DESCRIPTION = """AAT/PPI ALMA Data Restore, version {}: This tool takes a Member OUS Status UID,
diff --git a/apps/cli/launchers/wf/utility_wf_interfaces.py b/apps/cli/launchers/wf/src/wf/utility_wf_interfaces.py
similarity index 97%
rename from apps/cli/launchers/wf/utility_wf_interfaces.py
rename to apps/cli/launchers/wf/src/wf/utility_wf_interfaces.py
index 645babe24..39fb57c92 100644
--- a/apps/cli/launchers/wf/utility_wf_interfaces.py
+++ b/apps/cli/launchers/wf/src/wf/utility_wf_interfaces.py
@@ -3,8 +3,8 @@ import os
import sys
-from pyat.wf import wf
-from pyat import version
+from . import wf
+from _version import ___version___ as version
_RUN_QACLEAN_DESCRIPTION = """"""
--
GitLab