Fixed failing on null `subdirectory` value, and included a default value

validator that will fill in `subdirectory` if it's `None`; also includes
test

apps/cli/executables/pexable/productfetcher/productfetcher/locations.py

 """ LocationsReport conveniences """
 import http
-from abc import abstractmethod, ABC
+import logging
+from abc import ABC, abstractmethod
 from enum import Enum
 from pathlib import Path
 from typing import NamedTuple, Optional
 
-
 # pylint: disable=E0239, E0401, E0402, E1136, R0201, R0902, R0903, R0913, W0613
 import requests
 from marshmallow import Schema, fields, post_load
+from marshmallow.decorators import validates_schema
 from pycapo import CapoConfig
 
-from .exceptions import LocatorServiceException, FetchError
+from .exceptions import FetchError, LocatorServiceException
 from .interfaces import LocatedFile, LocationReport, Locator
 from .ngas import NgasConnection
 
+logger = logging.getLogger("productfetcher.locations")
+logger.setLevel(logging.INFO)
+
 
 class Location(Enum):
     """
@@ -248,7 +252,7 @@ class NgasFileSchema(Schema):
     """ One of the items in a location report's "files" list """
 
     ngas_file_id = fields.Str()
-    subdirectory = fields.Str()
+    subdirectory = fields.Str(allow_none=True)
     relative_path = fields.Str()
     checksum = fields.Integer()
     checksum_type = fields.Str()
@@ -258,6 +262,12 @@ class NgasFileSchema(Schema):
     table = fields.Str()
     archive_uid = fields.Str()
 
+    @validates_schema
+    def validate_subdirectory(self, data, **kwargs):
+        # Fill in a subdirectory value that is None with the default
+        if data["subdirectory"] is None:
+            data["subdirectory"] = "retrieved-products"
+
     @post_load
     def make_filespec(self, data, **kwargs):
         if "table" in data:

apps/cli/executables/pexable/productfetcher/tests/test_locations.py

@@ -4,21 +4,21 @@
 
 from pathlib import Path
 from typing import NamedTuple
+from unittest.mock import MagicMock, patch
 
-from unittest.mock import patch, MagicMock
+import marshmallow.exceptions
 import pytest
-
-
 import requests_mock
-
 from productfetcher.interfaces import LocationReport
 from productfetcher.locations import (
+    Cluster,
     FileLocator,
-    OracleXml,
+    Location,
     NgasFile,
+    NgasFileSchema,
     NgasServer,
-    Location,
-    Cluster,
+    NgasServerSchema,
+    OracleXml,
     ServiceLocator,
 )
 
@@ -151,3 +151,41 @@ def test_direct_copy_detection():
     for location in Location:
         for cluster in Cluster:
             assert not NgasServer("", location, cluster).can_direct_copy_to(home)
+
+
+class TestNgasFileSchema:
+    def test_null_value_in_subdirectory_field(self):
+        """
+        Tests that when the subdirectory value is provided to NgasFileSchema it will accept it and fill it in with
+        a default value
+        """
+        fake_oracle_xml_data = {
+            "archive_uid": "1341983",
+            "table": "cloth",
+            "subdirectory": None,
+            "relative_path": "relpath",
+            "size": 1000000,
+        }
+        try:
+            fake_file = NgasFileSchema().load(data=fake_oracle_xml_data)
+            assert type(fake_file) is OracleXml
+            assert fake_file.subdirectory == "retrieved-products"
+        except marshmallow.exceptions.ValidationError:
+            pytest.fail("Subdirectory field should be nullable.")
+
+        fake_ngas_file_data = {
+            "ngas_file_id": "12345",
+            "subdirectory": None,
+            "relative_path": "relpath",
+            "checksum": 12345,
+            "checksum_type": "foo",
+            "version": 1,
+            "size": 1000000,
+            "server": {"server": "nmngas07", "location": "DSOC", "cluster": "DSOC"},
+        }
+        try:
+            fake_file = NgasFileSchema().load(data=fake_ngas_file_data)
+            assert type(fake_file) is NgasFile
+            assert fake_file.subdirectory == "retrieved-products"
+        except marshmallow.exceptions.ValidationError:
+            pytest.fail("Subdirectory field should be nullable.")