Resolved merge conflict

apps/cli/executables/datafetcher/test/testing_utils.py

@@ -9,13 +9,9 @@ from pathlib import Path
 
 from pycapo import CapoConfig
 
-from datafetcher.errors import \
-    MissingSettingsException, NoProfileException
-from datafetcher.locations_report import \
-    LocationsReport
-from datafetcher.utilities import \
-    REQUIRED_SETTINGS, get_arg_parser, \
-    ExecutionSite
+from datafetcher.errors import MissingSettingsException, NoProfileException
+from datafetcher.locations_report import LocationsReport
+from datafetcher.utilities import REQUIRED_SETTINGS, get_arg_parser, ExecutionSite
 
 TEST_PROFILE = 'local'
 
@@ -155,8 +151,6 @@ def get_test_capo_settings():
     if result is None or len(result) == 0:
         raise MissingSettingsException('Required Capo settings were not found')
 
-    for setting in result:
-        print(f'{setting} = {result[setting]}')
     # be sure execution site is not DSOC nor NAASC
     exec_site = result['execution_site']
     if ExecutionSite.DSOC.value in exec_site or ExecutionSite.NAASC.value in \

apps/cli/utilities/proprietary_setter/setup.py

@@ -24,6 +24,6 @@ setup(
         'Programming Language :: Python :: 3.8'
     ],
     entry_points={
-        'console_scripts': ['proj_prop_period = proprietary_setter.commands:main']
+        'console_scripts': ['proj_prop_period = proprietary_setter.prop_setter:main']
     },
 )

apps/cli/utilities/proprietary_setter/src/proprietary_setter/commands.py

-# -*- coding: utf-8 -*-
-"""
-A module for updating the proprietary period of a project, whether that be to set it as
-proprietary or to make it public.
-
-Author: Richard Falardeau <rfalarde@nrao.edu>
-"""
-# pylint: disable=logging-format-interpolation
-
-import argparse as ap
-import sys
-import warnings
-import logging
-from astropy.time import Time
-from sqlalchemy import exc as sa_exc
-
-from ._version import ___version___ as version
-from support.logging import get_console_logger, LOG_MESSAGE_FORMATTER
-from support.capo import get_my_capo_config
-from schema import create_session
-from schema.model import Project
-from schema.legacy_model import LegacyProject
-from pymygdala import LogHandler, SendNRAOEvent
-
-
-_APPLICATION_NAME = 'proprietary_setter'
-_LOG = get_console_logger(_APPLICATION_NAME)
-_MISSING_PROFILE = """unable to derive the capo profile from the environment, """ +\
-                   """provide the capo profile through the -P argument or set the """ +\
-                   """CAPO_PROFILE environment variable."""
-_DISALLOWED_DURATION = """invalid proprietary duration, only integer values """ +\
-                       """between 0 and 730 are allowed."""
-_DESCRIPTION = """A tool for setting the proprietary duration of a project"""
-_EPILOG = """Return values:
-0: everything worked,
-1: can't deduce which profile to use
-2: invalid proprietary duration
-3: DB update failed"""
-
-
-class UpdateException(Exception):
-    r"""A wrapper for exceptions caught during our attempt to update the proprietary period.
-    Since we aren't passing the argparse object to helper functions, we want to catch some
-    obviously library exceptions in sqlalchemy, but also use it for some error trapping and
-    'handling'"""
-
-
-def _make_parser():
-    r"""
-    Build a command line parser to take the parameters for setting a projects
-    proprietary period.
-
-    :return result: an argparse object with the profile, project and
-    duration in its namespace"""
-    result = ap.ArgumentParser(description=_DESCRIPTION.format(version),
-                               formatter_class=ap.RawTextHelpFormatter,
-                               epilog=_EPILOG)
-    result.add_argument('-P', '--profile', action='store',
-                        help='profile name to use, e.g. nmtest, dsoc-test, or nmprod')
-    result.add_argument('project', action='store',
-                        help='project_code to update proprietary duration')
-    result.add_argument('duration', action='store',
-                        type=int,
-                        help='an integer duration to apply to the project; '
-                             '0 (immediately public) to 730 (private for two years from today)')
-    return result
-
-
-def set_project_proprietary_state(capo_config, project_code, proprietary_duration):
-    r"""
-    Set the proprietary period on the project code to the proprietary period provide on both the
-    archive and the legacy archive.
-        Note:   Since the capo_config will only really set the db parameters for the archive,
-                all updates to the legacy archive are actually live on production.
-    :param capo_config: the capo_config we're running under, which determines which db we update
-    :param project_code: the project code to update
-    :param proprietary_duration: an integer value for the new proprietary period (in days)
-    """
-    try:
-        with warnings.catch_warnings():
-            # This is solely to suppress the SQLAlchemy warning messages
-            warnings.simplefilter("ignore", category=sa_exc.SAWarning)
-            a_session = create_session('SDM', profile=capo_config.profile)
-            legacy_session = create_session('LEGACY', profile=capo_config.profile)
-            new_mjd_endtime = Time.now().mjd
-
-            # In the archive, the project code is a PK, we should only ever get one
-            project = a_session.query(Project) \
-                .filter(Project.project_code == project_code) \
-                .first()
-            # And if we don't, throw an exception before we commit anything
-            if project is not None:
-                project.proprietary_duration = proprietary_duration
-                if proprietary_duration != 0:
-                    project.endtime = new_mjd_endtime
-            else:
-                raise UpdateException(f'Project {project_code} was not found in the archive')
-
-            # We're not so lucky in the legacy archive, so we need to get all instances
-            leg_project = legacy_session.query(LegacyProject) \
-                .filter(LegacyProject.project_code == project_code) \
-                .all()
-            # Loop over each to set the period, or throw an exception if not found
-            if len(leg_project) > 0:
-                for project_instance in leg_project:
-                    project_instance.proprietary_duration = proprietary_duration
-                    project_instance.unlock_expire = proprietary_duration
-                    if proprietary_duration != 0:
-                        project_instance.proprietary = new_mjd_endtime
-                        project_instance.project_lock = 'LTIME'
-                    else:
-                        project_instance.project_lock = 'PUBLIC'
-            else:
-                raise UpdateException(f'Project {project_code} was not found in the legacy archive')
-
-            a_session.commit()
-            a_session.close()
-            legacy_session.commit()
-            legacy_session.close()
-
-    except Exception as update_exception:
-        raise UpdateException(f'DB update failed for the following reason: {update_exception}')
-
-
-def main(**kwargs):
-    r"""
-    The main entry point for this script.  Builds the parser, checks params, gets a profile and then
-    attempts to update the db.  If that succeeds, we kick off an amygdala even to tell the system
-    to re-index the project so the archive will reflect the new status.
-    :param kwargs: command line arguments to be passed to our parser builder
-    :return: nothing, if we complete the system will exit normally, if not we'll set a system
-    exit code.
-    """
-    parser = _make_parser()
-    args = parser.parse_args(**kwargs)
-
-    capo_config = get_my_capo_config(profile=args.profile)
-
-    if args.duration not in range(0, 730):
-        _LOG.error(_DISALLOWED_DURATION)
-        parser.print_help()
-        sys.exit(2)
-
-    try:
-        set_project_proprietary_state(capo_config, args.project, args.duration)
-    except UpdateException as update_exception:
-        _LOG.error(update_exception)
-        parser.print_help()
-        sys.exit(3)
-
-    # Set up a LogHandler to record the fact we just made a change to this project.
-    # We're adding it here, instead of earlier, because nothing we log earlier should be presented
-    # to anyone but the command line user and would only add useless clutter to our system logging.
-    # We only really want the completed task to make a record in our system.
-    broadcast = LogHandler(profile=capo_config.profile, application=_APPLICATION_NAME)
-    broadcast.setLevel(logging.DEBUG)
-    broadcast.setFormatter(LOG_MESSAGE_FORMATTER)
-    _LOG.addHandler(broadcast)
-    _LOG.info(f'Attempting to update proprietary period for {args.project}.')
-    if args.duration != 0:
-        _LOG.info(f'Locking for {args.duration} days from today')
-    else:
-        _LOG.info('Unlocking')
-
-    event = {'logData': {'project_code': args.project,
-                         'proprietary_duration': args.duration,
-                         'ingestion_type': 'evla_sdm'
-                         },
-             'message': 'proprietary period updated',
-             'request': 're-index please'}
-    SendNRAOEvent(profile=capo_config.profile, application=_APPLICATION_NAME) \
-        .send(routing_key='ingestion-complete.metadata', event=event)
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

apps/cli/utilities/proprietary_setter/src/proprietary_setter/project_finders.py

+# -*- coding: utf-8 -*-
+
+''' Project finders pull project metadata
+    from the "new" and legacy archive DBs.
+'''
+
+import logging
+import re
+import warnings
+from pprint import pprint
+from sqlalchemy import exc as sa_exc
+
+from schema import create_session, Project
+from schema.legacy_model import LegacyProject
+from support.logging import get_console_logger
+
+# pylint: disable=logging-format-interpolation
+
+class NewArchiveProjectFinder:
+    ''' Pulls specified project from archive DB '''
+
+    def __init__(self, profile: str):
+        self.profile = profile
+        self._LOG = get_console_logger(self.__class__.__name__)
+
+    def find_project(self, project_code: str):
+        ''' get specified project'''
+        session = create_session('SDM', profile=self.profile)
+        try:
+            with warnings.catch_warnings():
+                # suppress SQLAlchemy warnings
+                warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+                # In the archive, the project code is a PK;
+                # we should only  ever get one
+                project = session.query(Project) \
+                    .filter(Project.project_code == project_code) \
+                    .first()
+
+            return project
+
+        except Exception as ex:
+            self._LOG.error(f'{ex}')
+            raise NewArchiveProjectException(ex)
+
+        finally:
+            session.close()
+
+
+LEGACY_QUERY = """select PROJECT_CODE,
+OBSERVER,
+STARTTIME,
+STOPTIME,
+PROPRIETARY_DURATION,
+STOPTIME + PROPRIETARY_DURATION,
+proprietary,
+UNLOCK_EXPIRE,
+PROJECT_LOCK
+from PROJECT
+where PROJECT_CODE= :project_code 
+order by PROPRIETARY desc"""
+
+
+class LegacyProjectFinder:
+    ''' Pulls specified project from legacy DB '''
+
+    def __init__(self, profile: str):
+        self.profile = profile
+        self._LOG = get_console_logger(self.__class__.__name__)
+
+    def find_projects(self, project_code: str) -> list:
+        ''' project codes are not unique in the legacy archive '''
+
+        legacy_project_code = make_legacy_project_code(project_code)
+        session = create_session('LEGACY',
+                                 profile=self.profile)
+        try:
+
+            with warnings.catch_warnings():
+                # suppress SQLAlchemy warnings
+                warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+                try:
+                    projects = session.query(LegacyProject) \
+                        .filter(LegacyProject.project_code
+                                .in_((project_code, legacy_project_code))) \
+                        .all()
+                finally:
+                    session.close()
+            return projects
+
+        except Exception as ex:
+            self._LOG.error(f'{ex}')
+            raise LegacyProjectException(ex)
+
+        finally:
+            session.close()
+
+    def find_proprietary_metadata(self, project_code: str):
+        ''' Get proprietary, lock, prop duration from legacy archive '''
+        session = create_session('LEGACY',
+                                 profile=self.profile)
+        try:
+
+            with warnings.catch_warnings():
+                # suppress SQLAlchemy warnings
+                warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+                cursor = session.connection().engine.raw_connection().cursor()
+                cursor.execute(LEGACY_QUERY, project_code=project_code)
+
+            results = list()
+            for row in cursor.fetchall():
+                (proj_code, operator, starttime, stoptime, prop_dur, stop_dur,
+                 proprietary, unlock_expire, proj_lock) = row
+                proj_metadata = {'project_code': proj_code,
+                                 'operator': operator,
+                                 'starttime': starttime,
+                                 'stoptime': stoptime,
+                                 'prop_duration': prop_dur,
+                                 'stop_dur': stop_dur,
+                                 'proprietary': proprietary,
+                                 'unlock_expire': unlock_expire,
+                                 'project_lock': proj_lock,
+                                 }
+                self._LOG.info(pprint(proj_metadata))
+                results.append(proj_metadata)
+
+            return results
+
+        except Exception as ex:
+            self._LOG.error(f'{ex}')
+            raise LegacyProjectException(ex)
+
+        finally:
+            session.close()
+
+
+def make_legacy_project_code(project_code: str):
+    ''' Some project codes in the legacy DB have an extra 0 in them.
+        Create such a project code from a given project code
+        for use in searches.
+
+     '''
+    legacy_project_code = project_code.replace('0', '00', 1)
+    if legacy_project_code != project_code:
+        return legacy_project_code
+
+    # project code didn't have a zero in it
+    match = re.match(r"(([A-Za-z]+)([-0-9]+))", project_code)
+    if match is None:
+        return project_code
+    parts = match.groups()
+    if len(parts) == 3:
+        return parts[1] + '0' + parts[2]
+
+    logging.error(f"don't know what to do with {project_code}")
+    return project_code
+
+
+class NewArchiveProjectException(Exception):
+    ''' raise this when something goes wrong in querying or updating
+        the archive DB
+    '''
+
+
+class LegacyProjectException(Exception):
+    ''' raise this when something goes wrong in querying or updating
+        the legacy DB
+    '''