diff --git a/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py b/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py index 90639e3ea971a3758ce1b574d4d0cefeaf68529d..98e898e9f91dbb9a05b07c5b8f9e34d6e084f06b 100644 --- a/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py +++ b/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py @@ -1,20 +1,24 @@ """ LocationsReport conveniences """ import http -from abc import abstractmethod, ABC +import logging +from abc import ABC, abstractmethod from enum import Enum from pathlib import Path from typing import NamedTuple, Optional - # pylint: disable=E0239, E0401, E0402, E1136, R0201, R0902, R0903, R0913, W0613 import requests from marshmallow import Schema, fields, post_load +from marshmallow.decorators import validates_schema from pycapo import CapoConfig -from .exceptions import LocatorServiceException, FetchError +from .exceptions import FetchError, LocatorServiceException from .interfaces import LocatedFile, LocationReport, Locator from .ngas import NgasConnection +logger = logging.getLogger("productfetcher.locations") +logger.setLevel(logging.INFO) + class Location(Enum): """ @@ -248,7 +252,7 @@ class NgasFileSchema(Schema): """ One of the items in a location report's "files" list """ ngas_file_id = fields.Str() - subdirectory = fields.Str() + subdirectory = fields.Str(allow_none=True) relative_path = fields.Str() checksum = fields.Integer() checksum_type = fields.Str() @@ -258,6 +262,12 @@ class NgasFileSchema(Schema): table = fields.Str() archive_uid = fields.Str() + @validates_schema + def validate_subdirectory(self, data, **kwargs): + # Fill in a subdirectory value that is None with the default + if data["subdirectory"] is None: + data["subdirectory"] = "retrieved-products" + @post_load def make_filespec(self, data, **kwargs): if "table" in data: diff --git a/apps/cli/executables/pexable/productfetcher/tests/test_locations.py b/apps/cli/executables/pexable/productfetcher/tests/test_locations.py index 3c0cc50c43c5cbf9ba1b62b8dd79f17f639a0c16..7a4d3f56bcd4250039207c7339c8dc0ea86af30c 100644 --- a/apps/cli/executables/pexable/productfetcher/tests/test_locations.py +++ b/apps/cli/executables/pexable/productfetcher/tests/test_locations.py @@ -4,21 +4,21 @@ from pathlib import Path from typing import NamedTuple +from unittest.mock import MagicMock, patch -from unittest.mock import patch, MagicMock +import marshmallow.exceptions import pytest - - import requests_mock - from productfetcher.interfaces import LocationReport from productfetcher.locations import ( + Cluster, FileLocator, - OracleXml, + Location, NgasFile, + NgasFileSchema, NgasServer, - Location, - Cluster, + NgasServerSchema, + OracleXml, ServiceLocator, ) @@ -151,3 +151,41 @@ def test_direct_copy_detection(): for location in Location: for cluster in Cluster: assert not NgasServer("", location, cluster).can_direct_copy_to(home) + + +class TestNgasFileSchema: + def test_null_value_in_subdirectory_field(self): + """ + Tests that when the subdirectory value is provided to NgasFileSchema it will accept it and fill it in with + a default value + """ + fake_oracle_xml_data = { + "archive_uid": "1341983", + "table": "cloth", + "subdirectory": None, + "relative_path": "relpath", + "size": 1000000, + } + try: + fake_file = NgasFileSchema().load(data=fake_oracle_xml_data) + assert type(fake_file) is OracleXml + assert fake_file.subdirectory == "retrieved-products" + except marshmallow.exceptions.ValidationError: + pytest.fail("Subdirectory field should be nullable.") + + fake_ngas_file_data = { + "ngas_file_id": "12345", + "subdirectory": None, + "relative_path": "relpath", + "checksum": 12345, + "checksum_type": "foo", + "version": 1, + "size": 1000000, + "server": {"server": "nmngas07", "location": "DSOC", "cluster": "DSOC"}, + } + try: + fake_file = NgasFileSchema().load(data=fake_ngas_file_data) + assert type(fake_file) is NgasFile + assert fake_file.subdirectory == "retrieved-products" + except marshmallow.exceptions.ValidationError: + pytest.fail("Subdirectory field should be nullable.")