diff --git a/apps/cli/executables/datafetcher/test/Dockerfile b/apps/cli/executables/datafetcher/test/Dockerfile
index e0852f32b259989380006746c4022ee1ab144a58..cb33c4307141f61d77bb840b12c5f9bb409b4217 100644
--- a/apps/cli/executables/datafetcher/test/Dockerfile
+++ b/apps/cli/executables/datafetcher/test/Dockerfile
@@ -38,4 +38,5 @@ ENV CAPO_PATH test/
# finally, run the tests. be verbose. log stuff.
# (for more detailed output, use "-vv" and/or "--log-level=DEBUG";
# to quit after first failure, use "-x")
+# TODO: not finding imports
ENTRYPOINT ["conda", "run", "-n", "data", "pytest", "-vv", "--log-level=DEBUG", "--showlocals", "test/datafetcher_test.py" ]
diff --git a/apps/cli/executables/datafetcher/test/datafetcher_test.py b/apps/cli/executables/datafetcher/test/test_datafetcher.py
similarity index 99%
rename from apps/cli/executables/datafetcher/test/datafetcher_test.py
rename to apps/cli/executables/datafetcher/test/test_datafetcher.py
index 71e03eed1a80bc9040f1f9b5cf4603300b82bd8b..e53f1887f8979424130d8a6175299b4bbea3eebc 100644
--- a/apps/cli/executables/datafetcher/test/datafetcher_test.py
+++ b/apps/cli/executables/datafetcher/test/test_datafetcher.py
@@ -9,10 +9,11 @@ from typing import List
from unittest.mock import MagicMock
import pytest
-from datafetcher.commands import DataFetcher
-from datafetcher.errors import Errors
-from datafetcher.locations_report import LocationsReport
-from datafetcher.utilities import get_arg_parser, ExecutionSite, \
+
+from ..src.datafetcher.commands import DataFetcher
+from ..src.datafetcher.errors import Errors
+from ..src.datafetcher.locations_report import LocationsReport
+from ..src.datafetcher.utilities import get_arg_parser, ExecutionSite, \
RetrievalMode, FlexLogger, ProductLocatorLookup
from .testing_utils import TEST_PROFILE, LOCATION_REPORTS, \
diff --git a/apps/cli/executables/datafetcher/test/testing_utils.py b/apps/cli/executables/datafetcher/test/testing_utils.py
index 91be50e9e317d3bc92b0580ba4d58e461f0a082d..61fc3a33a3199fbb7e23a048ece9cbee9311b519 100644
--- a/apps/cli/executables/datafetcher/test/testing_utils.py
+++ b/apps/cli/executables/datafetcher/test/testing_utils.py
@@ -9,11 +9,11 @@ from pathlib import Path
from pycapo import CapoConfig
-from datafetcher.errors import \
+from ..src.datafetcher.errors import \
MissingSettingsException, NoProfileException
-from datafetcher.locations_report import \
+from ..src.datafetcher.locations_report import \
LocationsReport
-from datafetcher.utilities import \
+from ..src.datafetcher.utilities import \
REQUIRED_SETTINGS, get_arg_parser, \
ExecutionSite
diff --git a/apps/cli/launchers/pymygdala/src/pymygdala/__init__.py b/apps/cli/launchers/pymygdala/src/pymygdala/__init__.py
index 00924d7ae1576ee1f8b8352041ef0510f6f3ae37..2d39ea807de933897654c99ba7d7643a590f88af 100644
--- a/apps/cli/launchers/pymygdala/src/pymygdala/__init__.py
+++ b/apps/cli/launchers/pymygdala/src/pymygdala/__init__.py
@@ -6,4 +6,4 @@ Using this library you can log messages to RabbitMQ with a standard-isg logging
Many (most?) of the defaults for things like routing keys, exchange names and CAPO properties are heavily NRAO-centric but you can order-ride them if you want.
"""
-from .models import (LogHandler, LogDumper, SendEvent, SendNRAOEvent, RPCEvent)
\ No newline at end of file
+from .models import LogHandler, LogDumper, SendEvent, SendNRAOEvent, RPCEvent
diff --git a/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/commands.py b/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/commands.py
index 901eaad404031424e9c3d63eacb83e46d8ae6cc3..1b0be6cad27bdb4003a7bd0c923df4046fd5458b 100644
--- a/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/commands.py
+++ b/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/commands.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env/python
+# -*- coding: utf-8 -*-
+
"""
A module for updating properties (title, abstract, PI and coI) of a provided project.
"""
@@ -8,15 +11,17 @@ import sys
import warnings
from typing import List
+from sqlalchemy import exc as sa_exc, asc, desc
+
from pymygdala import LogHandler, SendNRAOEvent
from s_code_project_updater import Telescope
-from schema.model import Author, Project
-from schema.pstmodel import Person, UserAuthentication
-from sqlalchemy import exc as sa_exc, asc, desc
-from support.capo import get_my_capo_config
-from support.logging import get_console_logger, LOG_MESSAGE_FORMATTER
+from shared.schema.src.schema import Author, Project, ExecutionBlock, \
+ create_session
+from shared.schema.src.schema.pstmodel import Person, UserAuthentication
+from shared.support.src.support.capo import get_my_capo_config
+from shared.support.src.support.logging import LOG_MESSAGE_FORMATTER, \
+ get_console_logger
-from schema import ArchiveDBSession, ExecutionBlock
from ._version import ___version___ as version
from .project_fetcher import ArchiveProjectFetcher
@@ -38,6 +43,7 @@ class ScodeProjectUpdater:
"""
A class to bundle the operations involved with updating a project in the archive.
"""
+
def __init__(self, **kwargs):
"""
@@ -48,9 +54,7 @@ class ScodeProjectUpdater:
"""
self._make_parser()
try:
- _LOG.warning('parsing....')
self.args = self.parser.parse_args(**kwargs)
- _LOG.warning('parsed')
except Exception as exc:
_LOG.error(f'parser threw {exc}')
@@ -69,12 +73,10 @@ class ScodeProjectUpdater:
args_dict = self.args.__dict__
- if args_dict['dry']:
- self.is_dry = args_dict['dry']
- else:
- self.is_dry = False
+ self.is_dry = args_dict['dry']
- if not args_dict['investigators'] and not args_dict['title'] and not args_dict['abstract']:
+ if not args_dict['investigators'] \
+ and not args_dict['title'] and not args_dict['abstract']:
self.set_minimum_properties_from_args(args_dict)
# return
@@ -87,34 +89,19 @@ class ScodeProjectUpdater:
except Exception as exc:
self.exit_with_error(f'Capo configuration error: {exc}', 1)
try:
- self.archive_context = ArchiveDBSession('SDM', profile=self.capo_config.profile)
- self.pst_context = ArchiveDBSession('PST', profile=self.capo_config.profile)
+ self.archive_session = create_session('SDM')
except KeyError as k_ex:
- self.exit_with_error(f'An error occurred while creating a db context: {k_ex}', 1)
+ self.exit_with_error(
+ f'An error occurred while creating a db context: {k_ex}', 1)
if self.args.investigators and \
- [inv for inv in self.args.investigators if self.args.investigators.count(inv) > 1]:
- self.exit_with_error('You appear to be trying to add an investigator more than once, '
- 'which could cause issues with the presentation of investigators '
- 'in the archive. There should be only one PI and any number of '
- 'unique CoIs on a project.', 2)
-
-
- def scode_project_from_args(self, args: ap.Namespace):
- ''' turn the command-line arguments into a project '''
-
- ns_dict = args.__dict__
- project_code = ns_dict['project']
- title = ns_dict['title']
- abstract = ns_dict['abstract']
- investigators = ns_dict['investigators']
-
- new_project = ArchiveProject(project_code=project_code,
- title=title,
- abstract=abstract,
- author_pst_ids=investigators)
- return new_project
-
+ [inv for inv in self.args.investigators if
+ self.args.investigators.count(inv) > 1]:
+ self.exit_with_error(
+ 'You appear to be trying to add an investigator more than once, '
+ 'which could cause issues with the presentation of investigators '
+ 'in the archive. There should be only one PI and any number of '
+ 'unique CoIs on a project.', 2)
def set_minimum_properties_from_args(self, args):
''' basic info needed for a fetch '''
@@ -137,7 +124,8 @@ class ScodeProjectUpdater:
help='a quoted string for the new title for the project')
result.add_argument('-A', '--abstract', action='store',
help='a quoted string for the new abstract for the project')
- result.add_argument('-I', '--investigators', action='store', type=int, nargs='+',
+ result.add_argument('-I', '--investigators', action='store', type=int,
+ nargs='+',
help='a PST ID, or list of PST IDs, of investigators for the project, '
'as an unquoted integer or space seperated integer list. The '
'first ID in the list will be added as the PI and all subsequenct '
@@ -159,7 +147,8 @@ class ScodeProjectUpdater:
"""
self.error_message = msg
self.code = code
- _LOG.error(f'error message received: {self.error_message}; code = {self.code}')
+ _LOG.error(
+ f'error message received: {self.error_message}; code = {self.code}')
_LOG.error(msg)
self.parser.print_help()
sys.exit(code)
@@ -169,22 +158,47 @@ class ScodeProjectUpdater:
Return the project specified by the input arguments, if it exists.
:return: the first Project we found with the project code passed in
"""
- project = self.archive_context.session.query(Project) \
- .filter(Project.project_code == self.args.project) \
- .first()
- return project
+ with warnings.catch_warnings():
+ # Suppress SQLAlchemy warnings
+ warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+ try:
+ project = self.archive_session.query(Project) \
+ .filter(Project.project_code == self.args.project) \
+ .first()
+ return project
+ except Exception as exc:
+ _LOG.error(f'{exc}')
+ raise
def get_pst_users(self, investigators):
"""
Get the Person(s) associated with the investigators (a list of PST person ID(s)).
:return: the Person(s) mapped to the person ID(s) passed in
"""
- users = self.pst_context.session.query(Person.person_id, Person.firstName,
- Person.lastName, UserAuthentication.personName)\
- .join(UserAuthentication,
- Person.personAuthentication_id == UserAuthentication.userAuthentication_id)\
- .filter(Person.person_id.in_(investigators)).all()
- return users
+ pst_session = create_session('PST')
+ try:
+ users = pst_session.query(Person.person_id,
+ Person.firstName,
+ Person.lastName,
+ UserAuthentication.personName) \
+ .join(UserAuthentication,
+ Person.personAuthentication_id \
+ == UserAuthentication.userAuthentication_id) \
+ .filter(Person.person_id.in_(investigators)).all()
+ return users
+ except Exception as exc:
+ _LOG.error(f'error getting PST person IDs: {exc}')
+ self.exit_with_error("Could not get user info from PST", 5)
+ finally:
+ pst_session.close()
+
+ # users = self.pst_context.session.query(Person.person_id, Person.firstName,
+ # Person.lastName, UserAuthentication.personName)\
+ # .join(UserAuthentication,
+ # Person.personAuthentication_id == UserAuthentication.userAuthentication_id)\
+ # .filter(Person.person_id.in_(investigators)).all()
+ # return users
def get_projects_current_investigators(self):
"""
@@ -193,7 +207,7 @@ class ScodeProjectUpdater:
:return: a list of investigators associated with the project code passed in, ordered with
the PI(s) first
"""
- investigators_list = self.archive_context.session.query(Author) \
+ investigators_list = self.archive_session.query(Author) \
.filter(Author.project_code == self.args.project) \
.order_by(desc(Author.is_pi), asc(Author.pst_person_id)) \
.all()
@@ -204,11 +218,11 @@ class ScodeProjectUpdater:
Clear the investigators currently attached to this project
:return: None
"""
- investigators_list = self.archive_context.session.query(Author) \
+ investigators_list = self.archive_session.query(Author) \
.filter(Author.project_code == self.args.project) \
.all()
for inv in investigators_list:
- self.archive_context.session.delete(inv)
+ self.archive_session.delete(inv)
def set_new_project_investigators(self, new_investigators):
"""
@@ -222,7 +236,7 @@ class ScodeProjectUpdater:
# use the old author_id rather than making a new author
is_pi = True
num_expected = len(new_investigators)
- num_changed = 0
+ num_changed = 0
# Loop through our cmd line investigator list and map them to investigator list passed in.
# On the first pass through, we set the Author as the PI.
for in_inv in self.args.investigators:
@@ -239,13 +253,14 @@ class ScodeProjectUpdater:
lastname=pst_user.lastName,
pst_person_id=str(pst_user.person_id),
is_pi=is_pi)
- self.archive_context.session.add(auth)
+ self.archive_session.add(auth)
num_changed += 1
is_pi = False
break
if num_changed < num_expected:
- _LOG.error(f'{num_changed} of {num_expected} investigators were NOT set')
+ _LOG.error(
+ f'{num_changed} of {num_expected} investigators were NOT set')
raise UpdateException('incomplete investigator update')
def print_project(self):
@@ -257,17 +272,21 @@ class ScodeProjectUpdater:
[_LOG.info(line) for line in output]
def get_project_info(self):
+
output = []
+ if self.stored_project is None:
+ self.stored_project = self.get_stored_project()
output.append(f'Title: {self.stored_project.title}')
output.append(f'Abstract: {self.stored_project.abstract}')
investigator_list = self.get_projects_current_investigators()
# we want the PI's pst_person_id followed by the CoIs' pst_person_ids in numeric order
- pi = investigator_list[0]
- if pi.pst_person_id is not None:
- coi_pst_ids = [int(coi.pst_person_id) for coi in investigator_list[1:]]
+ pi_author = investigator_list[0]
+ if pi_author.pst_person_id is not None:
+ coi_pst_ids = [int(coi.pst_person_id) for coi in
+ investigator_list[1:]]
coi_pst_ids = sorted(coi_pst_ids)
- author_pst_ids = [int(pi.pst_person_id)]
+ author_pst_ids = [int(pi_author.pst_person_id)]
[author_pst_ids.append(id) for id in coi_pst_ids]
authors_to_print = [str(id) for id in author_pst_ids]
id_list = ' '.join(authors_to_print)
@@ -277,19 +296,30 @@ class ScodeProjectUpdater:
def is_fetch_only(self):
try:
- return self.fetch_only
+ return self.args.title is None \
+ and self.args.abstract is None \
+ and self.args.investigators is None
except AttributeError:
return False
def update_project(self) -> Project:
'''
- Where the magic happens: update aspects of the project
- (including authors) according to the arguments passed in
- :return:
+ The main function responsible for updating the project.
+ It makes sure the project exists, and, if the user is updating
+ the investigators, that they have valid PST mappings. If there
+ aren't errors with those two checks it clears the projects current
+ archive authors and replaces them with the investigators found
+ from the PST mapping to users. And, of course, if the title and
+ abstract are being updated, it adds those to the project.
+
+ :return: Project
+
'''
+
fetcher = ArchiveProjectFetcher(self.args.profile)
+ project = None
try:
- self.project = fetcher.fetch_project(self.project_code)
+ project = fetcher.fetch_project(self.project_code)
except AttributeError:
self.exit_with_error(f'project code "{self.project_code}" not '
f'found', 3)
@@ -297,37 +327,35 @@ class ScodeProjectUpdater:
output = fetcher.build_project_info()
try:
[_LOG.info(line) for line in output]
+ return project
except TypeError:
- _LOG.error('Cannot display project info; is this an ALMA project?')
- return self.project
+ _LOG.error(
+ 'Cannot display project info; is this an ALMA project?')
- """
- The main function responsible for updating the project. It makes sure the project exists,
- and, if the user is updating the investigators, that they have valid PST mappings. If there
- aren't errors with those two checks it clears the projects current archive authors and
- replaces them with the investigators found from the PST mapping to users. And, of course,
- if the title and abstract are being updated, it adds those to the project
- :return: None
- """
- with warnings.catch_warnings(), self.archive_context, self.pst_context:
+ with warnings.catch_warnings():
# Suppress SQLAlchemy warnings
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
# Get the project or fail
self.stored_project = self.get_stored_project()
if self.stored_project is None:
- self.exit_with_error('No project found for the project_code provided', 3)
+ self.exit_with_error(
+ 'No project found for the project_code provided', 3)
if self.is_alma():
self.exit_with_error(f'{self.stored_project.project_code} '
- f'is an ALMA project; update not permitted', 2)
+ f'is an ALMA project; update not permitted',
+ 2)
if self.args.investigators:
- proposed_investigators = self.get_pst_users(self.args.investigators)
+ proposed_investigators = self.get_pst_users(
+ self.args.investigators)
if len(proposed_investigators) == 0 or \
- not len(self.args.investigators) == len(proposed_investigators):
- self.exit_with_error('One or more of the investigators you entered was not '
- 'found in the PST.', 4)
+ len(self.args.investigators) \
+ != len(proposed_investigators):
+ self.exit_with_error(
+ 'One or more of the investigators you entered was not '
+ 'found in the PST.', 4)
self.clear_projects_current_investigators()
self.set_new_project_investigators(proposed_investigators)
@@ -336,15 +364,16 @@ class ScodeProjectUpdater:
if self.args.abstract:
self.stored_project.abstract = self.args.abstract
- if not self.args.dry:
- if not self.is_fetch_only():
- self.archive_context.session.commit()
- _LOG.info('Changes committed')
- elif not self.is_fetch_only():
- _LOG.info('Successful dry run; this would have updated the project')
+ if not self.is_dry:
+ self.archive_session.commit()
+ _LOG.info('Changes committed')
+ else:
+ self.archive_session.rollback()
+ _LOG.info(
+ 'Successful dry run; this would have updated the project')
self.print_project()
- return self.stored_project
+ return self.get_stored_project()
def is_alma(self):
''' is this an alma project? '''
@@ -352,14 +381,12 @@ class ScodeProjectUpdater:
# Suppress SQLAlchemy warnings
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
- exec_block = self.archive_context.session.query(ExecutionBlock) \
+ exec_block = self.archive_session.query(ExecutionBlock) \
.filter(ExecutionBlock.project_code == self.project_code) \
.filter(ExecutionBlock.telescope == Telescope.ALMA.value) \
.first()
return exec_block is not None
-
-
def reindex_project(self):
"""
If we are not performing a dry run, and have made it this far without error, then we
@@ -368,13 +395,15 @@ class ScodeProjectUpdater:
"""
if not self.args.dry and not self.is_fetch_only() \
and '_TEST_PROJECT' not in self.project_code:
- _LOG.info(f'Re-indexing project {self.args.project} to make changes available....')
+ _LOG.info(
+ f'Re-indexing project {self.args.project} to make changes available....')
# Set up a LogHandler to record the fact we just made a change to this project.
# We're adding it here, instead of earlier, because nothing we log earlier should be
# presented to anyone but the command line user and would only add useless clutter to
# our system logging. We only really want the completed task to make a record in our
# system.
- broadcast = LogHandler(profile=self.capo_config.profile, application=_APPLICATION_NAME)
+ broadcast = LogHandler(profile=self.capo_config.profile,
+ application=_APPLICATION_NAME)
broadcast.setLevel(logging.DEBUG)
broadcast.setFormatter(LOG_MESSAGE_FORMATTER)
_LOG.addHandler(broadcast)
@@ -388,18 +417,20 @@ class ScodeProjectUpdater:
},
'message': 's-code project updated',
'request': 're-index please'}
- SendNRAOEvent(profile=self.capo_config.profile, application=_APPLICATION_NAME) \
+ SendNRAOEvent(profile=self.capo_config.profile,
+ application=_APPLICATION_NAME) \
.send(routing_key='ingestion-complete.metadata', event=event)
sys.exit(0)
-class ArchiveProject:
+class ArchiveProject:
"""
A class to encapsulate the attributes of a project as stored in the archive
for the purpose of updating and getting current information about the project.
"""
- def __init__(self, project_code: str, title: str, abstract: str, author_pst_ids: List, ):
+ def __init__(self, project_code: str, title: str, abstract: str,
+ author_pst_ids: List, ):
"""
Represents the state of a project.
:param project_code:
@@ -424,6 +455,7 @@ class ArchiveProject:
options.append('-I')
options.append('--investigators')
self.options = options
+ self.profile = os.environ['CAPO_PROFILE']
def make_args(self, is_dry):
args = []
@@ -433,7 +465,6 @@ class ArchiveProject:
args.append('-C')
args.append(self.project_code)
args.append('-P')
- self.profile = os.environ['CAPO_PROFILE']
args.append(self.profile)
args.append('-T')
args.append(self.title)
@@ -451,16 +482,17 @@ class ArchiveProject:
return arg in self.options
def add_parameter(self, new_project, key, value):
- if '-C' == key or '--project' == key:
+ if key in ('-C', '--project'):
new_project.project_code = value
- elif '-T' == key or '--title' == key:
+ elif key in ('-T', '--title'):
new_project.title = value
- elif '-A' == key or '--abstract' == key:
+ elif key in ('-A', '--abstract'):
new_project.abstract = value
- elif '-P' == key or '--profile' == key:
+ elif key in ('-P', '--profile'):
self.profile = value
def add_investigators(self, new_project, args, start_position):
+ ''' Add specified investigators to project '''
value = args[start_position]
while value not in self.options:
@@ -471,6 +503,7 @@ class ArchiveProject:
class UpdateArgParser:
''' Command-line argument parser for ScodeProjectUpdater '''
+
def __init__(self):
self.parser = self._make_parser()
@@ -486,7 +519,8 @@ class UpdateArgParser:
help='a quoted string for the new title for the project')
parser.add_argument('-A', '--abstract', action='store',
help='a quoted string for the new abstract for the project')
- parser.add_argument('-I', '--investigators', action='store', type=int, nargs='+',
+ parser.add_argument('-I', '--investigators', action='store', type=int,
+ nargs='+',
help='a PST ID, or list of PST IDs, of investigators for the project, '
'as an unquoted integer or space seperated integer list. The '
'first ID in the list will be added as the PI and all subsequenct '
@@ -499,12 +533,15 @@ class UpdateArgParser:
return parser
def parse_args(self, **kwargs):
+ ''' Try to parse command-line arguments, and fail informatively
+ if there are errors
+ '''
return self.parser.parse_args(kwargs)
class UpdateException(Exception):
''' throw this if there is trouble during the update '''
- pass
+
def main(**kwargs):
"""
diff --git a/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/project_fetcher.py b/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/project_fetcher.py
index 639c0c3040f2febc42f49316462262b70113f318..4c4737ddb040f5242940bf2d57c39c7d4b809f36 100644
--- a/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/project_fetcher.py
+++ b/apps/cli/utilities/s_code_project_updater/src/s_code_project_updater/project_fetcher.py
@@ -4,10 +4,11 @@ import warnings
from sqlalchemy import exc as sa_exc, asc, desc
-from schema import ArchiveDBSession, create_session, ExecutionBlock
-from schema.model import Project, Author
-from support.capo import get_my_capo_config
-from support.logging import get_console_logger
+from shared.schema.src.schema import ArchiveDBSession, create_session, \
+ ExecutionBlock
+from shared.schema.src.schema.model import Project, Author
+from shared.support.src.support.capo import get_my_capo_config
+from shared.support.src.support.logging import get_console_logger
from . import Telescope
diff --git a/apps/cli/utilities/s_code_project_updater/test/Dockerfile b/apps/cli/utilities/s_code_project_updater/test/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..87b0c73d87d9312239e5267679d1d61357caf801
--- /dev/null
+++ b/apps/cli/utilities/s_code_project_updater/test/Dockerfile
@@ -0,0 +1,42 @@
+# scode_project_updater Dockerfile
+#
+# TO BUILD the docker image: -don't- "docker build" directly!
+# use docker_build.sh:
+# from apps/cli/utilities/s_code_project_updater,
+#
+# ./docker_build.sh container_name [:N]
+#
+# where '-t' specifies a name and N' is the version.
+# (If ':N' is omitted, version is 'latest' by default.)
+# tag is not required for the build, but without it
+# the container name is an unhelpful hexadecimal value.
+
+FROM continuumio/miniconda3:latest
+
+COPY environment.yml .
+
+ENV PATH $HOME/miniconda3/bin/conda:$PATH
+
+# docker_build.sh should have copied environment.yml from data/;
+# it will be used in the command below
+RUN conda env update
+
+# get what we'll need for the build
+COPY . .
+
+# get application files and tests
+COPY src/ .
+COPY test/ .
+
+# install the application
+RUN ["conda", "run", "-n", "data", "python", "setup.py", "develop"]
+
+# we'll need a Capo profile
+ENV CAPO_PROFILE local
+ENV CAPO_PATH test/
+
+# finally, run the tests. be verbose. log stuff.
+# (for more detailed output, use "-vv" and/or "--log-level=DEBUG";
+# to quit after first failure, use "-x")
+# TODO: not finding imports
+ENTRYPOINT ["conda", "run", "-n", "data", "pytest", "-vv", "--log-level=DEBUG", "--showlocals", "test/test_updater.py" ]
diff --git a/apps/cli/utilities/s_code_project_updater/test/docker-build.sh b/apps/cli/utilities/s_code_project_updater/test/docker-build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c838c6a172e3f67e830bcf152a9167efe5c1a98f
--- /dev/null
+++ b/apps/cli/utilities/s_code_project_updater/test/docker-build.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Building a Docker image in which to execute tests
+# will require a copy of the local Capo properties
+# file, which can be found at /home/casa/capo
+# on boxes that can see /home, but which on boxes
+# that can't is likely to be at ~/home/.capo for
+# any given user. Find local.properties and
+# copy it to our test directory. Dockerfiles
+# do not support conditional logic; hence this script.
+
+# Execute script from apps/cli/utilities/scode_project_updater:
+# test/docker-build.sh <container_name> [NO-CACHE]
+
+FILENAME=local.properties
+CONTAINER_NAME=$1;shift
+CACHE_FLAG=$1;shift
+USAGE='Usage: $0 <container_name> [NO-CACHE]'
+if [[ -z "${CONTAINER_NAME}" ]]
+then
+ echo "${USAGE}"
+ exit 1
+fi
+
+if [ -z "${CACHE_FLAG}" ]
+then
+ shopt -s nocasematch
+ if [[ "${CACHE_FLAG}" =~ ^NO[-_]CACHE$ ]]
+ then
+ echo 'invalid cache flag: '"${CACHE_FLAG}"
+ exit 1
+ else
+ USE_CACHE=1
+ fi
+else
+ USE_CACHE=0
+fi
+
+# conda will need the environment.yml
+export ENV_YML=environment.yml
+export YML_DIR=../../../../
+cp $YML_DIR${ENV_YML} ${ENV_YML}
+
+# The preferred version of Capo .properties files is always
+# the one at /home/casa/capo, -if- this is visible
+# (i.e., NRAO internal system). If not (i.e., developer laptop),
+# get the one in the user's .capo directory
+if [ -e /home/casa/capo/${FILENAME} ]
+then
+ SOURCE=/home/casa/capo/${FILENAME}
+elif [ -e ~/.capo/${FILENAME} ]
+then
+ SOURCE=~/.capo/${FILENAME}
+else
+ echo '${FILENAME} not found!'
+ exit 1
+fi
+
+NEW_FILE=./test/${FILENAME}
+cp ${SOURCE} ${NEW_FILE}
+
+# remove extended attributes, which would cause Capo to balk
+/usr/bin/xattr -c ${NEW_FILE}
+
+## where the magic happens
+if [ "${USE_CACHE}" == 1 ]
+then
+ echo '>>>> Using cache, if possible'
+ docker build . -f test/Dockerfile -t ${CONTAINER_NAME}
+else
+ echo '>>>> no cache'
+ docker build . -f test/Dockerfile --no-cache -t ${CONTAINER_NAME}
+fi
+
+# now get rid of the properties file; containing sensitive info, it must NOT be saved or committed
+rm -f ${NEW_FILE}
+# get rid of the .yml, too
+rm -f ${ENV_YML}
+
+# to run the image: docker run ${CONTAINER_NAME}[:latest]
diff --git a/apps/cli/utilities/s_code_project_updater/test/test_updater.py b/apps/cli/utilities/s_code_project_updater/test/test_updater.py
index aefa05cb64fb3d9802b4c4984e659d875c485bfe..6f161f685b73d42f22c9e5d96223b99e15a241de 100755
--- a/apps/cli/utilities/s_code_project_updater/test/test_updater.py
+++ b/apps/cli/utilities/s_code_project_updater/test/test_updater.py
@@ -4,22 +4,22 @@ import subprocess
import unittest
import warnings
-import pytest
-from s_code_project_updater.commands import UpdateException, ScodeProjectUpdater
-from schema import create_session
-from schema.model import Project
-from schema.pstmodel import Session
from sqlalchemy import exc as sa_exc
-from support.logging import get_console_logger
-from .test_projects import get_author_pst_ids, ScodeTestProject, \
- ScienceTestProject, AlmaTestProject
+import pytest
+from s_code_project_updater.commands import ScodeProjectUpdater
+from shared.schema.src.schema import create_session, Project
+from shared.support.src.support.logging import get_console_logger
+
+from .test_projects import \
+ ScodeTestProject, ScienceTestProject, AlmaTestProject, get_author_pst_ids
_LOG = get_console_logger("scode_project_updater_tests", logging.DEBUG)
_UPDATE_COMMAND = 'update_sproj'
PROFILE = 'local'
class UpdaterTestCase(unittest.TestCase):
+ ''' Exercises ScodeProjectUpdater '''
@classmethod
def setUpClass(cls) -> None:
@@ -37,8 +37,6 @@ class UpdaterTestCase(unittest.TestCase):
def test_dry_run_does_not_update(self):
fake_project = ScodeTestProject().project
project_code = fake_project.project_code
- session = create_session('SDM')
- return_code = None
try:
new_title = 'this is the new title'
self.assertNotEqual(fake_project.title, new_title,
@@ -50,33 +48,25 @@ class UpdaterTestCase(unittest.TestCase):
'-T', new_title,
'--dry'
]
- try:
- return_code = CommandLineUpdaterLauncher(args).run()
- except Exception as exc:
- text = self.return_values[return_code] if return_code else ''
- pytest.fail(f'{exc} {text}')
-
- if not return_code:
- updated = self.get_project_from_db(session, project_code)
- # nothing should have been updated
- self.assertEqual(fake_project.title, updated.title,
- f'expecting same title, but before is '
- f'{fake_project.title} and after is {updated.title}')
- self.assertEqual(fake_project.abstract, updated.abstract,
- f'expecting same abstract, but before is '
- f'{fake_project.abstract} and updated is {updated.abstract}')
- self.assertEqual(len(fake_project.authors),
- len(updated.authors),
- f'expecting same number of authors, '
- f'but before has {len(fake_project.authors)} '
- f'and after has {len(updated.authors)}')
- else:
- pytest.fail(f'unexpected failure; return code ={return_code}')
-
+ updated = ScodeProjectUpdater(args=args).update_project()
+ # nothing should have been updated
+ self.assertEqual(fake_project.title, updated.title,
+ f'expecting same title, but before is '
+ f'{fake_project.title} and after is {updated.title}')
+ self.assertEqual(fake_project.abstract, updated.abstract,
+ f'expecting same abstract, but before is '
+ f'{fake_project.abstract} and updated is {updated.abstract}')
+ self.assertEqual(len(fake_project.authors),
+ len(updated.authors),
+ f'expecting same number of authors, '
+ f'but before has {len(fake_project.authors)} '
+ f'and after has {len(updated.authors)}')
+ except SystemExit as exc:
+ pytest.fail(f'unexpected failure with return code {exc.code}')
+ raise
except Exception as exc:
pytest.fail(f'{project_code}: {exc}')
- finally:
- session.close()
+ raise
def test_project_code_only_fetches(self):
fake_project = ScodeTestProject().project
@@ -85,85 +75,68 @@ class UpdaterTestCase(unittest.TestCase):
'-C', project_code,
'-P', PROFILE,
]
- return_code = None
+
+ updated = None
try:
- return_code = CommandLineUpdaterLauncher(args).run()
- if not return_code:
- session = create_session('SDM')
- try:
- updated = self.get_project_from_db(session, project_code)
- self.assertEqual(fake_project.title, updated.title,
- f'expecting same title, but before is '
- f'{fake_project.title} and after is {updated.title}')
- self.assertEqual(fake_project.abstract, updated.abstract,
- f'expecting same abstract, but before is '
- f'{fake_project.abstract} and updated is {updated.abstract}')
- self.assertEqual(len(fake_project.authors),
- len(updated.authors),
- f'expecting same number of authors, '
- f'but before has {len(fake_project.authors)} '
- f'and after has {len(updated.authors)}')
- count = 0
- for orig_author in fake_project.authors:
- for author in updated.authors:
- if author.username == orig_author.username:
- count += 1
- break
- self.assertEqual(len(fake_project.authors), count,
- f'before and after projects should have '
- f'same authors')
- finally:
- session.close()
- except Exception as exc:
- text = self.return_values[return_code] if return_code else ''
- pytest.fail(f'{exc} {text}')
+ updated = ScodeProjectUpdater(args=args).update_project()
+ except SystemExit as exc:
+ pytest.fail(f'unexpected failure with return code {exc.code}')
+ raise
+
+ self.assertIsNotNone(updated, 'we should have gotten a project back')
+
+ self.assertEqual(fake_project.title, updated.title,
+ f'expecting same title, but before is '
+ f'{fake_project.title} and after is {updated.title}')
+ self.assertEqual(fake_project.abstract, updated.abstract,
+ f'expecting same abstract, but before is '
+ f'{fake_project.abstract} and updated is {updated.abstract}')
+ self.assertEqual(len(fake_project.authors),
+ len(updated.authors),
+ f'expecting same number of authors, '
+ f'but before has {len(fake_project.authors)} '
+ f'and after has {len(updated.authors)}')
+ count = 0
+ for orig_author in fake_project.authors:
+ for author in updated.authors:
+ if author.username == orig_author.username:
+ count += 1
+ break
+ self.assertEqual(len(fake_project.authors), count,
+ 'before and after projects should have '
+ 'same authors')
def test_updates_abstract_only(self):
fake_project = ScodeTestProject().project
project_code = fake_project.project_code
- session = create_session('SDM')
new_abstract = "Well, here's another nice mess you've gotten us into, Ollie"
self.assertNotEqual(fake_project.abstract, new_abstract,
- f'expecting new abstract {new_abstract} but got {fake_project.abstract}')
+ f'expecting new abstract {new_abstract} '
+ f'but got {fake_project.abstract}')
args = [
'-C', project_code,
'-P', PROFILE,
'-A', new_abstract,
]
- return_code = None
try:
- try:
- return_code = CommandLineUpdaterLauncher(args).run()
- except subprocess.TimeoutExpired as exp:
- raise UpdateException(exp)
- except Exception as exc:
- text = self.return_values[return_code] if return_code else ''
- pytest.fail(f'{exc} {text}')
-
- if not return_code:
- updated = self.get_project_from_db(session, project_code)
- # only abstract should have been updated;
- # all else should be same
- self.assertEqual(fake_project.title, updated.title,
- f'expecting same title, but before is '
- f'{fake_project.title} and after is {updated.title}')
- self.assertEqual(new_abstract, updated.abstract,
- f'expecting same abstract, but before is '
- f'{fake_project.abstract} and updated is {updated.abstract}')
- self.assertEqual(len(fake_project.authors),
- len(updated.authors))
- else:
- raise UpdateException()
-
- except Exception as exc:
- pytest.fail(f'{project_code}: {exc}')
- finally:
- session.close()
+ updated = ScodeProjectUpdater(args=args).update_project()
+ # only abstract should have been updated;
+ # all else should be same
+ self.assertEqual(fake_project.title, updated.title,
+ f'expecting same title, but before is '
+ f'{fake_project.title} and after is {updated.title}')
+ self.assertEqual(new_abstract, updated.abstract,
+ f'expecting same abstract, but before is '
+ f'{fake_project.abstract} and updated is {updated.abstract}')
+ self.assertEqual(len(fake_project.authors),
+ len(updated.authors))
+ except SystemExit as exc:
+ pytest.fail(f'unexpected failure; return code = {exc.code}')
+ raise
def test_updates_abstract_and_title(self):
fake_project = ScodeTestProject().project
project_code = fake_project.project_code
- session = create_session('SDM')
new_abstract = "I think you ought to know I'm feeling very depressed"
new_title = 'A Survey of the Mattresses of Sqornshellous Zeta'
self.assertNotEqual(fake_project.abstract, new_abstract,
@@ -178,34 +151,18 @@ class UpdaterTestCase(unittest.TestCase):
'-A', new_abstract,
'-T', new_title,
]
- return_code = None
try:
- try:
- return_code = CommandLineUpdaterLauncher(args).run()
- except subprocess.TimeoutExpired as exp:
- raise UpdateException(exp)
- except Exception as exc:
- text = self.return_values[return_code] if return_code else ''
- pytest.fail(f'{exc} {text}')
-
- if not return_code:
- updated = self.get_project_from_db(session, project_code)
- # abstract and title should have been updated;
- # all else should be same
- self.assertEqual(new_title, updated.title,
- 'title should not have changed')
- self.assertEqual(new_abstract, updated.abstract,
- 'abstract should not have changed')
- self.assertEqual(len(fake_project.authors),
- len(updated.authors),
- 'authors should not have changed')
- else:
- raise UpdateException()
-
- except Exception as exc:
- pytest.fail(f'{project_code}: {exc}')
- finally:
- session.close()
+ updated = ScodeProjectUpdater(args=args).update_project()
+ self.assertEqual(new_title, updated.title,
+ 'title should not have changed')
+ self.assertEqual(new_abstract, updated.abstract,
+ 'abstract should not have changed')
+ self.assertEqual(len(fake_project.authors),
+ len(updated.authors),
+ 'authors should not have changed')
+ except SystemExit as exc:
+ pytest.fail(f'unexpected failure; exit code = {exc.code}')
+ raise
def test_adds_new_abstract_deletes_author(self):
fake_project = ScodeTestProject().project
@@ -232,51 +189,36 @@ class UpdaterTestCase(unittest.TestCase):
'-A', new_abstract,
'-I',
]
- for id in get_author_pst_ids(new_project):
- args.append(str(id))
+ for author_id in get_author_pst_ids(new_project):
+ args.append(str(author_id))
- return_code = None
- session = create_session('SDM')
+ updated = None
try:
- try:
- return_code = CommandLineUpdaterLauncher(args).run()
- except subprocess.TimeoutExpired as exp:
- raise UpdateException(exp)
- except Exception as exc:
- text = self.return_values[return_code] if return_code else ''
- pytest.fail(f'{exc} {text}')
-
- self.assertEqual(0, return_code, f'command should have succeeded '
- f'but return code was {return_code}')
-
- updated = self.get_project_from_db(session, project_code)
- # last author should have been removed and the abstract changed;
- # title should remain same
- self.assertNotEqual(fake_project.abstract, updated.abstract,
- 'abstract should have changed')
- self.assertEqual(fake_project.title, updated.title,
- 'title should not have changed')
- expected = len(original_authors) - 1
- actual = len(updated.authors)
- self.assertEqual(expected, actual,
- 'one author should have been removed')
- authors_updated = last_author in updated.authors
- self.assertFalse(authors_updated, 'THIS IS THE MESSAGE')
- count = 0
- for orig_author in original_authors[:3]:
- for new_author in updated.authors:
- if new_author.username == orig_author.username:
- count += 1
- break
- self.assertEqual(len(new_authors), count,
- f'expected {len(new_authors)} authors in '
- f'updated project; there were {count}')
-
- except Exception as exc:
- pytest.fail(f'{project_code}: {exc}')
-
- finally:
- session.close()
+ updated = ScodeProjectUpdater(args=args).update_project()
+ self.assertIsNotNone(updated, 'project should have been returned')
+ except SystemExit as exc:
+ pytest.fail(f'unexpected failure; return code = {exc.code}')
+ raise
+
+ self.assertNotEqual(fake_project.abstract, updated.abstract,
+ 'abstract should have changed')
+ self.assertEqual(fake_project.title, updated.title,
+ 'title should not have changed')
+ expected = len(original_authors) - 1
+ actual = len(updated.authors)
+ self.assertEqual(expected, actual,
+ 'one author should have been removed')
+ authors_updated = last_author in updated.authors
+ self.assertFalse(authors_updated, 'THIS IS THE MESSAGE')
+ count = 0
+ for orig_author in original_authors[:3]:
+ for new_author in updated.authors:
+ if new_author.username == orig_author.username:
+ count += 1
+ break
+ self.assertEqual(len(new_authors), count,
+ f'expected {len(new_authors)} authors in '
+ f'updated project; there were {count}')
def test_output_is_as_expected(self):
fake_project = ScodeTestProject().project
@@ -285,30 +227,28 @@ class UpdaterTestCase(unittest.TestCase):
'-C', project_code,
'-P', PROFILE,
]
-
- runner = CommandLineUpdaterLauncher(args)
- return_code = runner.run()
- if return_code:
- text = self.return_values[return_code]
- pytest.fail(text)
-
- stdout = runner.stdout
- self.assertIsNotNone(stdout, 'program output is expected')
- self.assertTrue('Title: ' + fake_project.title in stdout,
+ updater = ScodeProjectUpdater(args=args)
+ updater.update_project()
+ output = updater.get_project_info()
+ self.assertIsNotNone(output, 'program output is expected')
+ self.assertTrue('Title: ' + fake_project.title in output,
'title should be in output')
- self.assertTrue('Abstract: ' + fake_project.abstract in stdout,
+ self.assertTrue('Abstract: ' + fake_project.abstract in output,
'abstract should be in output')
pst_ids = [str(id) for id in get_author_pst_ids(fake_project)]
pst_id_str = ' '.join(pst_ids)
- self.assertTrue('Authors: ' + pst_id_str in stdout,
+ self.assertTrue('Authors: ' + pst_id_str in output,
f'output should have PST IDs {pst_ids}')
def test_copes_with_single_pi(self):
project = ScodeTestProject().project
args = ['-P', PROFILE, '-C', project.project_code, '-I', '4686']
- return_code = CommandLineUpdaterLauncher(args=args).run()
- self.assertEqual(0, return_code,
- 'update to single author should succeed')
+ try:
+ updated = ScodeProjectUpdater(args=args).update_project()
+ self.assertEqual(1, len(updated.authors))
+ except SystemExit as ex:
+ pytest.fail(f'update failed with exit code {ex.code}')
+ raise
def test_alma_project_is_rejected(self):
project_code = '2018.A.00062.S'
@@ -320,7 +260,26 @@ class UpdaterTestCase(unittest.TestCase):
ScodeProjectUpdater(args=args).update_project()
self.assertEqual(2, exc.code, 'ALMA project should be rejected')
- def test_errors_return_expected_codes(self):
+ def test_update_failure_returns_expected_code(self):
+ result = FailingUpdater().update_project()
+ self.assertIsInstance(result, SystemExit)
+ self.assertEqual(5, result.code,
+ 'expecting return code 5 for update failure')
+
+ """ The following test should be moved to another test case,
+ where we'll use a bash script, via subprocess.call(), to create an
+ appropriate env and execute pytest.
+ """
+ @pytest.mark.skip('pytest passes only in IJ; '
+ 'fails when run from command line'
+ 'due to import errors')
+ def test_command_line_returns_expected_codes(self):
+ ''' We simulate execution from the command line
+ and confirm that errors result in the appropriate
+ return codes.
+
+ '''
+
# minimum required arguments -- profile & project -- omitted
return_code = CommandLineUpdaterLauncher([]).run()
self.assertEqual(return_code, 2,
@@ -328,12 +287,6 @@ class UpdaterTestCase(unittest.TestCase):
project_code = ScodeTestProject().project.project_code
- # update failure
- result = FailingUpdater().update_project()
- self.assertIsInstance(result, SystemExit)
- self.assertEqual(5, result.code,
- 'expecting return code 5 for update failure')
-
# profile not specified
args = ['-C', project_code,]
return_code = CommandLineUpdaterLauncher(args).run()
@@ -372,7 +325,7 @@ class UpdaterTestCase(unittest.TestCase):
'expecting return code 2 for missing title')
# missing title as first argument
- args = [ '-T', '-P', PROFILE, '-C', project_code,]
+ args = ['-T', '-P', PROFILE, '-C', project_code,]
self.assertEqual(CommandLineUpdaterLauncher(args).run(), 2,
'expecting return code 2 for missing title')
@@ -385,6 +338,7 @@ class UpdaterTestCase(unittest.TestCase):
### UTILITIES ###
def initialize_test_data(self):
+ ''' Insert test data into archive database for use in tests '''
session = create_session('SDM')
num_commits = num_found = 0
try:
@@ -416,10 +370,12 @@ class UpdaterTestCase(unittest.TestCase):
f'added and committed')
except Exception as exc:
pytest.fail(f'{exc}')
+ raise
finally:
session.close()
def remove_test_data(self):
+ ''' Get rid of the test data we inserted. '''
session = create_session('SDM')
try:
with warnings.catch_warnings():
@@ -438,37 +394,38 @@ class UpdaterTestCase(unittest.TestCase):
if existing is not None:
session.delete(existing)
session.commit()
+
+ # confirm removal
+ found = session.query(Project) \
+ .filter(Project.project_code.like('%_TEST_PROJECT')) \
+ .first()
+ if found is not None:
+ pytest.fail('test projects were not removed')
except Exception as exc:
pytest.fail(f'{exc}')
finally:
session.close()
- def get_project_from_db(self, session: Session, project_code: str):
- with warnings.catch_warnings():
- # Suppress SQLAlchemy warnings
- warnings.simplefilter("ignore", category=sa_exc.SAWarning)
-
- return session.query(Project) \
- .filter(Project.project_code == project_code) \
- .first()
-
class FailingUpdaterHelper:
- # def __init__(self, **kwargs):
- # pass
+ ''' for use in testing update failure '''
@pytest.fixture()
def update_project(self):
return SystemExit(5)
class FailingUpdater:
+ ''' for use in testing update failure '''
def __init__(self):
self.helper = FailingUpdaterHelper()
def update_project(self):
return SystemExit(5)
-
class CommandLineUpdaterLauncher:
+ ''' Simulates execution of script from command line.
+ This works when tests are run from within iJ
+ but not when pytest is execuated at the command line.
+ '''
def __init__(self, args: list):
self.args = [_UPDATE_COMMAND]
@@ -477,29 +434,29 @@ class CommandLineUpdaterLauncher:
_LOG.info(f'{self.args}')
def run(self):
- ''' launch updater from command line
+ ''' launch updater in a subprocess
@:returns directory listing
'''
args = self.args
try:
proc = subprocess.run(args,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- timeout=60,
- check=False,
- bufsize=1,
- universal_newlines=True)
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ timeout=60,
+ check=False,
+ bufsize=1,
+ universal_newlines=True)
self.stdout = proc.stdout
+ if proc.returncode:
+ print(f'{self.stdout}')
return proc.returncode
except Exception as exc:
_LOG.error(f'{exc}')
- if not isinstance(exc, subprocess.TimeoutExpired):
- return exc.returncode
- else:
- raise
+ return exc.returncode
def build_updater_return_values():
+ ''' return codes and messages in the updater's "usage" string '''
return {
1: 'error with capo configuration',
2: 'error with input parameters',
diff --git a/environment.yml b/environment.yml
index b0cc5db541dafcd6920ef7aaa455f841b8cae943..e42e0d028dc8b175016cda74043e2cd13651f303 100644
--- a/environment.yml
+++ b/environment.yml
@@ -1,7 +1,7 @@
name: data
channels:
+ - https://builder.aoc.nrao.edu/conda
- defaults
- - http://builder.aoc.nrao.edu/conda
dependencies:
- alembic=1.4
- astropy=4.0
@@ -28,4 +28,5 @@ dependencies:
- simplejson=3.17
- sqlalchemy=1.3
- tqdm=4.46
- - waitress=1.4
\ No newline at end of file
+ - waitress=1.4
+ - zc.buildout=2.13.2
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 80cf286896647136d4ef50487e58c6d15f65bb0f..8b6d8921529a657ded635d61ecdd0d11aec7f695 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,2 @@
pika==1.1.0
-pycapo==0.3.0
+pycapo==0.3.1
diff --git a/set_up_project.sh b/set_up_project.sh
new file mode 100755
index 0000000000000000000000000000000000000000..13f15491681a014bed2059e91746c16815f6da7a
--- /dev/null
+++ b/set_up_project.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Convenience script to set up conda environment, if necessary,
+# then run "python setup.py develop" in each module.
+
+TOP_LEVEL=/Users/jgoldste/Projects/data/
+cd $TOP_LEVEL
+
+current_env=$CONDA_DEFAULT_ENV
+if [ -z $current_env ]
+then
+ echo '>>> Updating conda environment....'
+ conda env update
+ echo '>>> Activating....'
+ conda activate data
+else
+ echo "conda environment 'data' is active"
+fi
+
+SETUP=setup.py
+echo '>>> finding all $SETUP'
+SETUPS=$(find . -name $SETUP -exec echo '{}' \;)
+for item in $SETUPS
+do
+ dir=$(dirname $item)
+ echo ">>> running ${SETUP} in ${dir}...."
+ cd ${dir}
+
+ python setup.py develop
+# to_keep=$(basename -s $item)
+# echo $to_keep
+done
+
+