WS-397: update capo properties

apps/cli/executables/pexable/casa_envoy/casa_envoy/palaver.py

@@ -21,7 +21,7 @@ logger.addHandler(logging.StreamHandler(sys.stdout))
 
 
 def _get_settings(cwd: pathlib.Path, args: list):
-    use_casa = CapoConfig().getboolean("edu.nrao.archive.workspaces.ProcessingSettings.useCasa")
+    use_casa = CapoConfig().getboolean("edu.nrao.workspaces.ProcessingSettings.useCasa")
     casa_home = (
         CapoConfig().settings("edu.nrao.archive.workflow.config.CasaVersions").homeForReprocessing
     )

apps/cli/executables/pexable/conveyor/conveyor/conveyor.py

@@ -49,18 +49,13 @@ def arg_parser() -> argparse.ArgumentParser:
 
 
 def _get_settings(filename: str) -> Dict[str, str]:
-    delivery_area = (
-        CapoConfig().settings("edu.nrao.archive.workspaces.DeliverySettings").ciplDelivery
-    )
-    weblog_cache_area = (
-        CapoConfig().settings("edu.nrao.archive.workspaces.DeliverySettings").cacheWeblogDirectory
-    )
-    image_qa_area = (
-        CapoConfig().settings("edu.nrao.archive.workspaces.DeliverySettings").standardImageDelivery
-    )
-    workspaces_lustre_root_dir = (
-        CapoConfig().settings("edu.nrao.archive.workspaces.ProcessingSettings").rootDirectory
-    )
+    delivery_properties = CapoConfig().settings("edu.nrao.workspaces.DeliverySettings")
+    processing_properties = CapoConfig().settings("edu.nrao.workspaces.ProcessingSettings")
+
+    delivery_area = delivery_properties.standardCalibrationDelivery
+    weblog_cache_area = delivery_properties.cacheWeblogDirectory
+    image_qa_area = delivery_properties.standardImageDelivery
+    workspaces_lustre_root_dir = processing_properties.rootDirectory
 
     current_root_directory = str(pathlib.Path.cwd().parent)
     current_subdirectory = str(pathlib.Path.cwd().stem)

apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingest.py

@@ -16,7 +16,7 @@ logger.addHandler(logging.StreamHandler(sys.stdout))
 
 
 def _get_settings(filename: str) -> dict:
-    ingestion_settings = CapoConfig().settings("edu.nrao.archive.workspaces.IngestionSettings")
+    ingestion_settings = CapoConfig().settings("edu.nrao.workspaces.IngestionSettings")
     staging_root_dir = ingestion_settings.stagingDirectory
     storage_root_dir = ingestion_settings.storageDirectory
 

apps/cli/executables/pexable/productfetcher/productfetcher/locations.py

@@ -223,12 +223,14 @@ class ServiceLocator(Locator):
         self.spl = science_product_locator
 
     def locate(self) -> LocationReport:
-        settings = CapoConfig().settings("edu.nrao.archive.datafetcher.DataFetcherSettings")
+        settings = CapoConfig().settings("edu.nrao.workspaces.ProductFetcherSettings")
 
         # this is needed to prevent SSL errors when tests are run
         # inside a Docker container
         requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ":HIGH:!DH:!aNULL"
-        requests.Session().mount(settings.locatorServiceUrlPrefix, adapter=requests.adapters.HTTPAdapter())
+        requests.Session().mount(
+            settings.locatorServiceUrlPrefix, adapter=requests.adapters.HTTPAdapter()
+        )
 
         try:
             response = requests.get(settings.locatorServiceUrlPrefix, params={"locator": self.spl})

apps/cli/executables/wf_framework/sh/calibration-table-collector.sh

@@ -32,9 +32,9 @@ SDM_ID=$1;shift
 CAL_PROCESSING_DATETIME=$1;shift
 
 # Get the spool, staging and storage paths from CAPO
-SPOOL_DIR=$(./pycapo -P "${CAPO_PROFILE}" -q edu.nrao.archive.workspaces.ProcessingSettings.rootDirectory)
-STAGING_DIR=$(./pycapo -P "${CAPO_PROFILE}" -q edu.nrao.archive.workspaces.IngestionSettings.stagingDirectory)
-STORAGE_DIR=$(./pycapo -P "${CAPO_PROFILE}" -q edu.nrao.archive.workspaces.IngestionSettings.storageDirectory)
+SPOOL_DIR=$(./pycapo -P "${CAPO_PROFILE}" -q edu.nrao.workspaces.ProcessingSettings.rootDirectory)
+STAGING_DIR=$(./pycapo -P "${CAPO_PROFILE}" -q edu.nrao.workspaces.IngestionSettings.stagingDirectory)
+STORAGE_DIR=$(./pycapo -P "${CAPO_PROFILE}" -q edu.nrao.workspaces.IngestionSettings.storageDirectory)
 
 # File name for output tar of calibration products
 #

apps/cli/executables/wf_framework/sh/ingest-request.sh

@@ -24,7 +24,7 @@ case $option in
   ;;
 esac
 
-WORKFLOW_SERVICE=$(capo -q edu.nrao.archive.workspaces.WorkflowSettings.serviceUrl)
+WORKFLOW_SERVICE=$(capo -q edu.nrao.workspaces.WorkflowSettings.serviceUrl)
 
 if [ "$action" = "ingest_cal" ]; then
     curl -X POST $WORKFLOW_SERVICE/workflows/std_calibration/requests/$2/ingest

apps/web/src/environments/environment.dev.ts

 export const environment = {
     production: true,
     // archive search services
-    apiUrl: 'https://webtest.aoc.nrao.edu/archive-services/',
+    apiUrl: 'https://webtest.aoc.nrao.edu/archive-service/',
     // workspace services
     workspacesUrl: 'https://ws-dev.nrao.edu/'
 };

apps/web/src/environments/environment.prod.ts

 export const environment = {
   production: true,
   // archive search services
-  apiUrl: 'https://webtest.aoc.nrao.edu/archive-services/',
+  apiUrl: 'https://webtest.aoc.nrao.edu/archive-service/',
   // workspace services
   workspacesUrl: 'http://localhost:3457/'
 };

apps/web/src/environments/environment.test.ts

 export const environment = {
   production: true,
   // archive search services
-  apiUrl: 'https://webtest.aoc.nrao.edu/archive-services/',
+  apiUrl: 'https://webtest.aoc.nrao.edu/archive-service/',
   // workspace services
   workspacesUrl: 'http://shipman.aoc.nrao.edu:3457/'
 };

apps/web/src/environments/environment.ts

@@ -5,7 +5,7 @@
 export const environment = {
   production: false,
   // archive search services
-  apiUrl: 'https://webtest.aoc.nrao.edu/archive-services/',
+  apiUrl: 'https://webtest.aoc.nrao.edu/archive-service/',
   // workspace services
   workspacesUrl: 'http://localhost:3457/'
 };

docker.properties

@@ -12,19 +12,19 @@ edu.nrao.archive.configuration.AmqpServer.port     = 5672
 #
 # Workspaces settings
 #
-edu.nrao.archive.workspaces.CapabilitySettings.serviceUrl = http://capability:3457
-edu.nrao.archive.workspaces.WorkflowSettings.serviceUrl = http://workflow:3456
-edu.nrao.archive.workspaces.NotificationSettings.serviceUrl = http://notification:3458
-edu.nrao.archive.workspaces.UISettings.serviceUrl = http://localhost:4444/workspaces
-
+edu.nrao.workspaces.CapabilitySettings.serviceUrl = http://capability:3457
+edu.nrao.workspaces.WorkflowSettings.serviceUrl = http://workflow:3456
+edu.nrao.workspaces.NotificationSettings.serviceUrl = http://notification:3458
+edu.nrao.workspaces.UISettings.serviceUrl = http://localhost:4444/workspaces
 
 #
 # Processing Settings
 #
-edu.nrao.archive.workspaces.ProcessingSettings.useCasa = false
-edu.nrao.archive.workspaces.ProcessingSettings.rootDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/spool
-edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation = /packages/apps/cli/executables/wf_framework/sh
-edu.nrao.archive.workspaces.ProcessingSettings.ramInGb = 1G
+edu.nrao.workspaces.ProcessingSettings.useCasa = false
+edu.nrao.workspaces.ProcessingSettings.rootDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/spool
+edu.nrao.workspaces.ProcessingSettings.scriptLocation = /lustre/aoc/cluster/pipeline/docker/workspaces/sbin
+edu.nrao.workspaces.ProcessingSettings.ramInGb = 1G
+edu.nrao.workspaces.ProcessingSettings.autoGenerateStandardCals = False
 
 edu.nrao.archive.workflow.config.CasaVersions.homeForReprocessing = /home/casa/packages/pipeline/current
 
@@ -34,37 +34,31 @@ edu.nrao.archive.workflow.config.CasaVersions.homeForReprocessing = /home/casa/p
 edu.nrao.archive.workflow.config.DeliverySettings.nraoDownloadDirectory = /tmp/delivery_root
 edu.nrao.archive.workflow.config.DeliverySettings.nraoDownloadUrl = http://localhost:4444/dl
 
-
 #
 # Ingestion Settings
 #
-edu.nrao.archive.workspaces.IngestionSettings.stagingDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/staging
-edu.nrao.archive.workspaces.IngestionSettings.storageDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/storage
+edu.nrao.workspaces.IngestionSettings.stagingDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/staging
+edu.nrao.workspaces.IngestionSettings.storageDirectory = /lustre/aoc/cluster/pipeline/docker/workspaces/storage
 
 #
 # Standard Calibration Settings
 #
-edu.nrao.archive.workspaces.DeliverySettings.ciplDelivery =  /lustre/aoc/cluster/pipeline/docker/workspaces/qa2
-edu.nrao.archive.workspaces.DeliverySettings.cacheWeblogDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/cache/weblog
+edu.nrao.workspaces.DeliverySettings.standardCalibrationDelivery =  /lustre/aoc/cluster/pipeline/docker/workspaces/qa2
+edu.nrao.workspaces.DeliverySettings.cacheWeblogDirectory =  /lustre/aoc/cluster/pipeline/docker/workspaces/cache/weblog
 
 #
 # Standard Imaging Settings
 #
-edu.nrao.archive.workspaces.DeliverySettings.standardImageDelivery = /lustre/aoc/cluster/pipeline/docker/workspaces/image-qa
-
+edu.nrao.workspaces.DeliverySettings.standardImageDelivery = /lustre/aoc/cluster/pipeline/docker/workspaces/image-qa
 
 #
-# Data fetcher settings
+# Product Fetcher settings
 #
-edu.nrao.archive.workflow.config.DataFetcherSettings.ramInGb = 16
-edu.nrao.archive.workflow.config.DataFetcherSettings.clusterTimeout = 01:00:00:00
-edu.nrao.archive.datafetcher.DataFetcherSettings.locatorServiceUrlPrefix = https://webtest.aoc.nrao.edu/archive-services/location?locator
-edu.nrao.archive.datafetcher.DataFetcherSettings.defaultThreadsPerHost = 4
-edu.nrao.archive.datafetcher.DataFetcherSettings.executionSite = local_test
-edu.nrao.archive.datafetcher.DataFetcherSettings.downloadDirectory=/tmp/
-edu.nrao.archive.workflow.config.RequestHandlerSettings.downloadDirectory=/tmp/
+edu.nrao.workspaces.ProductFetcherSettings.locatorServiceUrlPrefix = https://webtest.aoc.nrao.edu/archive-service/location?locator
+edu.nrao.workspaces.ProductFetcherSettings.defaultThreadsPerHost = 4
+edu.nrao.workspaces.ProductFetcherSettings.executionSite = local_test
 
 #
 # Notification Settings
 #
-edu.nrao.archive.workspaces.NotificationSettings.analystEmail = workspaces-analysts@listmgr.nrao.edu
+edu.nrao.workspaces.NotificationSettings.analystEmail = workspaces-analysts-test@listmgr.nrao.edu

schema/versions/291edd508557_add_std_calibration_script.py

@@ -11,21 +11,24 @@ from pycapo import CapoConfig
 
 
 # revision identifiers, used by Alembic.
-revision = '291edd508557'
-down_revision = '16b40772e51a'
+revision = "291edd508557"
+down_revision = "16b40772e51a"
 branch_labels = None
 depends_on = None
 
 
 def get_script_location() -> str:
-    return CapoConfig().settings("edu.nrao.archive.workspaces.ProcessingSettings").scriptLocation
+    return CapoConfig().settings("edu.nrao.workspaces.ProcessingSettings").scriptLocation
 
 
 def upgrade():
     path = get_script_location()
-    script_body = """#!/bin/sh
+    script_body = (
+        """#!/bin/sh
 
-    export PATH=$PATH:""" + path + """
+    export PATH=$PATH:"""
+        + path
+        + """
 
     framework.sh -d .
     cd rawdata/
@@ -34,11 +37,14 @@ def upgrade():
     casa_envoy --standard-cal $3 $4
     deliver -r ./products --prefix $2
     """
+    )
 
     op.execute(
         """
         INSERT INTO workflow_templates (filename, content, workflow_name)
-        VALUES ('std_calibration.sh', E'""" + script_body + """' , 'std_calibration')
+        VALUES ('std_calibration.sh', E'"""
+        + script_body
+        + """' , 'std_calibration')
         """
     )
 

schema/versions/2b6e18877c8e_update_std_calibration_workflow_.py

@@ -36,7 +36,7 @@ queue
     )
 
     script_contents = """#!/bin/sh
-SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
    
 export PATH=$PATH:${SCRIPTS_DIR}
 
@@ -58,7 +58,7 @@ def downgrade():
         UPDATE workflow_templates
         SET content = 
         E'#!/bin/sh
-        SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+        SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
            
         export PATH=$PATH:${SCRIPTS_DIR}
         

schema/versions/7d53b022c5b3_std_calibration_add_chmod_call_before_.py

@@ -56,7 +56,7 @@ queue
 
 def downgrade():
     script_contents = """#!/bin/sh
-SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
    
 export PATH=$PATH:${SCRIPTS_DIR}
 

schema/versions/e4452ab33088_correcting_calibration_script.py

@@ -10,8 +10,8 @@ import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = 'e4452ab33088'
-down_revision = '291edd508557'
+revision = "e4452ab33088"
+down_revision = "291edd508557"
 branch_labels = None
 depends_on = None
 
@@ -22,7 +22,7 @@ def upgrade():
         UPDATE workflow_templates
         SET content = 
         E'#!/bin/sh
-        SCRIPTS_DIR=$(pycapo -q edu.nrao.archive.workspaces.ProcessingSettings.scriptLocation)
+        SCRIPTS_DIR=$(pycapo -q edu.nrao.workspaces.ProcessingSettings.scriptLocation)
            
         export PATH=$PATH:${SCRIPTS_DIR}
         
@@ -33,7 +33,8 @@ def upgrade():
         casa_envoy --standard-cal $3 $4
         deliver-cipl.sh ${PWD##*/}'
         WHERE filename = 'std_calibration.sh'
-        """)
+        """
+    )
     pass
 
 

services/workflow/bin/boot-condor-and-workflow.sh

@@ -23,8 +23,8 @@ SPOOL_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/spool
 QA_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/qa2
 WEBLOG_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/cache
 IMAGE_QA_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/image-qa
-STAGING_DIR=lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/staging
-STORAGE_DIR=lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/storage
+STAGING_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/staging
+STORAGE_DIR=/lustre/aoc/cluster/pipeline/"$CAPO_PROFILE"/workspaces/storage
 mkdir -p "$SPOOL_DIR"
 mkdir -p "$QA_DIR"
 mkdir -p "$WEBLOG_DIR"

shared/workspaces/test/test_archive_service.py

@@ -74,7 +74,7 @@ class TestArchiveService:
         params = {
             "parameters": {
                 "product_locator": spl,
-                "user_email": "workspaces-analysts@listmgr.nrao.edu",
+                "user_email": "workspaces-analysts-test@listmgr.nrao.edu",
                 "metadata": fake_metadata,
             }
         }

shared/workspaces/workspaces/notification/services/notification_service.py

@@ -15,10 +15,8 @@ logger = logging.getLogger(__name__)
 
 class NotificationServiceRESTClient(NotificationServiceIF):
     def __init__(self):
-        self.url = (
-            CapoConfig().settings("edu.nrao.archive.workspaces.NotificationSettings").serviceUrl
-        )
-        self.linkUrl = CapoConfig().settings("edu.nrao.archive.workspaces.UISettings").serviceUrl
+        self.url = CapoConfig().settings("edu.nrao.workspaces.NotificationSettings").serviceUrl
+        self.linkUrl = CapoConfig().settings("edu.nrao.workspaces.UISettings").serviceUrl
 
     def notify_submitted(self, request: CapabilityRequestIF):
         # bail out early if we have no user email
@@ -76,7 +74,7 @@ class NotificationServiceRESTClient(NotificationServiceIF):
         """
         logger.info("SENDING QA-READY NOTIFICATION!!!")
         analyst_email = (
-            CapoConfig().settings("edu.nrao.archive.workspaces.NotificationSettings").analystEmail
+            CapoConfig().settings("edu.nrao.workspaces.NotificationSettings").analystEmail
         )
         requests.post(
             f"{self.url}/notify/ready-for-QA_email/send",

shared/workspaces/workspaces/products/services/archive_service.py

@@ -70,11 +70,9 @@ class ArchiveService(ArchiveServiceIF):
             - obs_end_time: End time of observation
         :return: HTTP Response for attempt to create std_calibration request
         """
-        capability_url = (
-            CapoConfig().settings("edu.nrao.archive.workspaces.CapabilitySettings").serviceUrl
-        )
+        capability_url = CapoConfig().settings("edu.nrao.workspaces.CapabilitySettings").serviceUrl
         analyst_email = (
-            CapoConfig().settings("edu.nrao.archive.workspaces.NotificationSettings").analystEmail
+            CapoConfig().settings("edu.nrao.workspaces.NotificationSettings").analystEmail
         )
         parameters = {
             "parameters": {

shared/workspaces/workspaces/workflow/services/workflow_service.py

@@ -31,7 +31,7 @@ class WorkflowServiceRESTClient(WorkflowServiceIF):
     """This is the workflow service REST client."""
 
     def __init__(self):
-        self.url = CapoConfig().settings("edu.nrao.archive.workspaces.WorkflowSettings").serviceUrl
+        self.url = CapoConfig().settings("edu.nrao.workspaces.WorkflowSettings").serviceUrl
 
     def execute(self, request: WorkflowRequestIF):
         """
@@ -141,9 +141,7 @@ class WorkflowService(WorkflowServiceIF):
         self.message_router = Router("workflow")
         self.message_router.register(self)
         self.info = info
-        self.processing_settings = CapoConfig().settings(
-            "edu.nrao.archive.workspaces.ProcessingSettings"
-        )
+        self.processing_settings = CapoConfig().settings("edu.nrao.workspaces.ProcessingSettings")
 
     def attach_file_to_request(self, request: WorkflowRequestIF, filename: str, content: bytes):
         """