make productfectcher use default threads capo property

apps/cli/executables/pexable/productfetcher/productfetcher/product_fetcher.py

@@ -24,6 +24,8 @@ from pathlib import Path
 from typing import List, Type
 
 # pylint: disable=E0401, E0402, R0913
+from pycapo import CapoConfig
+
 from .exceptions import FetchError
 from .fetcher_factory import ConfiguredFetcherFactory
 from .fetchers import ForceMode
@@ -263,6 +265,8 @@ class FetchContext:
         :return: an argparse 'parser' with command line options for productfetcher.
         """
 
+        default_threads = CapoConfig().settings("edu.nrao.workspaces.ProductFetcherSettings").defaultThreadsPerHost
+
         parser = argparse.ArgumentParser(
             description=_PROLOGUE,
             formatter_class=argparse.RawTextHelpFormatter,
@@ -272,9 +276,9 @@ class FetchContext:
             CLIParam.CONCURRENCY.value,
             action="store",
             dest="concurrency",
-            help="maxmimum threads per plan (set to 1 for no threading)",
+            help="maximum threads per plan (set to 1 for no threading)",
             type=int,
-            default=16,
+            default=default_threads,
         )
 
         parser.add_argument(

apps/cli/executables/pexable/productfetcher/tests/test_product_fetcher.py

@@ -21,6 +21,7 @@
 
 from pathlib import Path
 from typing import List, Union
+from unittest.mock import patch
 
 import pytest
 from productfetcher.fetchers import DryRunFakeFileFetcher, ForceMode
@@ -87,27 +88,29 @@ def test_multiple_locator_fetching(capsys, resource_path_root):
     :param capsys:
     :return:
     """
-    img = resource_path_root / "location_files" / "IMG.json"
-    cal = resource_path_root / "location_files" / "CALIBRATION.json"
+    with patch("productfetcher.product_fetcher.CapoConfig") as mocked_capo:
+        mocked_capo.return_value.settings.return_value.defaultThreadsPerHost = 1
+        img = resource_path_root / "location_files" / "IMG.json"
+        cal = resource_path_root / "location_files" / "CALIBRATION.json"
 
-    # parse the command line with these two
-    fc = FetchContext.parse_commandline([CLIParam.FILE.value, str(img), CLIParam.FILE.value, str(cal)])
-    assert len(fc.locators) == 2
+        # parse the command line with these two
+        fc = FetchContext.parse_commandline([CLIParam.FILE.value, str(img), CLIParam.FILE.value, str(cal)])
+        assert len(fc.locators) == 2
 
-    # let's make the plan and ensure we have all the stuff we expect from both
-    plan = fc.generate_plan()
+        # let's make the plan and ensure we have all the stuff we expect from both
+        plan = fc.generate_plan()
 
-    # we'll need to open these files ourselves to figure out what fetchers we expect
-    for locator_file in [img, cal]:
-        for file in FileLocator(locator_file).locate().files:
-            seen = False
+        # we'll need to open these files ourselves to figure out what fetchers we expect
+        for locator_file in [img, cal]:
+            for file in FileLocator(locator_file).locate().files:
+                seen = False
 
-            # there may be a more "test friendly" way of doing this, such as by asking the plan
-            # if it is fetching a certain file, but it seems like a lot of refactoring for this
-            # one test, so I'm going to leave it alone for now
-            for hostgroup_fetcher in plan.fetchers:
-                seen = seen or file in [fetcher.file for fetcher in hostgroup_fetcher.fetchers]
-            assert seen
+                # there may be a more "test friendly" way of doing this, such as by asking the plan
+                # if it is fetching a certain file, but it seems like a lot of refactoring for this
+                # one test, so I'm going to leave it alone for now
+                for hostgroup_fetcher in plan.fetchers:
+                    seen = seen or file in [fetcher.file for fetcher in hostgroup_fetcher.fetchers]
+                assert seen
 
 
 def test_argument_parsing(capsys):
@@ -117,32 +120,34 @@ def test_argument_parsing(capsys):
     :param capsys:
     :return:
     """
-    # ensure that various combinations do not work
-    with pytest.raises(SystemExit):
-        # must have an SPL or a file
-        FetchContext.parse_commandline([])
-
-    # check the dry run value
-    fc = FetchContext.parse_commandline([CLIParam.DRY.value, CLIParam.FILE.value, "foo"])
-    assert fc.dry_run
-    fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo"])
-    assert not fc.dry_run
-
-    # check the force value
-    fc = FetchContext.parse_commandline([CLIParam.FORCE.value, CLIParam.FILE.value, "foo"])
-    assert fc.force == ForceMode.FORCE
-    fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo"])
-    assert fc.force == ForceMode.NORMAL
-
-    fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo"])
-    assert isinstance(fc.locators[0], FileLocator)
-    assert fc.locators[0].file == Path("foo")
-
-    fc = FetchContext.parse_commandline([CLIParam.SPL.value, "uid://this/is/a/fakesy"])
-    assert isinstance(fc.locators[0], ServiceLocator)
-    assert fc.locators[0].spl == "uid://this/is/a/fakesy"
-
-    fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo", CLIParam.CONCURRENCY.value, "732"])
-    assert fc.concurrency == 732
-
-    capsys.readouterr()
+    with patch("productfetcher.product_fetcher.CapoConfig") as mocked_capo:
+        mocked_capo.return_value.settings.return_value.defaultThreadsPerHost = 1
+        # ensure that various combinations do not work
+        with pytest.raises(SystemExit):
+            # must have an SPL or a file
+            FetchContext.parse_commandline([])
+
+        # check the dry run value
+        fc = FetchContext.parse_commandline([CLIParam.DRY.value, CLIParam.FILE.value, "foo"])
+        assert fc.dry_run
+        fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo"])
+        assert not fc.dry_run
+
+        # check the force value
+        fc = FetchContext.parse_commandline([CLIParam.FORCE.value, CLIParam.FILE.value, "foo"])
+        assert fc.force == ForceMode.FORCE
+        fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo"])
+        assert fc.force == ForceMode.NORMAL
+
+        fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo"])
+        assert isinstance(fc.locators[0], FileLocator)
+        assert fc.locators[0].file == Path("foo")
+
+        fc = FetchContext.parse_commandline([CLIParam.SPL.value, "uid://this/is/a/fakesy"])
+        assert isinstance(fc.locators[0], ServiceLocator)
+        assert fc.locators[0].spl == "uid://this/is/a/fakesy"
+
+        fc = FetchContext.parse_commandline([CLIParam.FILE.value, "foo", CLIParam.CONCURRENCY.value, "732"])
+        assert fc.concurrency == 732
+
+        capsys.readouterr()

docker.properties

@@ -25,7 +25,7 @@ edu.nrao.workspaces.CapabilitySettings.externalServiceUrl = http://capability:34
 edu.nrao.workspaces.ProcessingSettings.useCasa = false
 edu.nrao.workspaces.ProcessingSettings.rootDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/spool
 edu.nrao.workspaces.ProcessingSettings.scriptLocation = /lustre/aoc/cluster/pipeline/docker/workspaces/sbin
-edu.nrao.workspaces.ProcessingSettings.ramInGb = 0.23G
+edu.nrao.workspaces.ProcessingSettings.ramInGb = 0.2G
 edu.nrao.workspaces.ProcessingSettings.autoGenerateStandardCals = False
 edu.nrao.workspaces.ProcessingSettings.CasaVersion.vlassSeci = /home/casa/packages/pipeline/casa-6.1.3-3-pipeline-2021.1.1.32