diff --git a/apps/cli/executables/pexable/delivery/delivery/finder.py b/apps/cli/executables/pexable/delivery/delivery/finder.py index ed49291030d9b126415c835099a46c5db2a5febe..71029040418ee924153f40b38d1d696a960249b4 100644 --- a/apps/cli/executables/pexable/delivery/delivery/finder.py +++ b/apps/cli/executables/pexable/delivery/delivery/finder.py @@ -8,7 +8,7 @@ import os import pathlib from typing import Iterator -from .products import ExecutionBlock, SpooledProduct +from .products import ExecutionBlock, SpooledProduct, CalibratedMS class ProductFinder(abc.ABC): @@ -46,5 +46,7 @@ class HeuristicProductFinder(ProductFinder): :return: List of products """ for subdir in self.path.glob("*"): - if subdir.is_dir() and not subdir.name.startswith("tmp"): + if subdir.is_dir() and subdir.name.endswith(".ms"): + yield CalibratedMS(subdir) + elif subdir.is_dir() and not subdir.name.startswith("tmp"): yield ExecutionBlock(subdir) diff --git a/apps/cli/executables/pexable/delivery/delivery/products.py b/apps/cli/executables/pexable/delivery/delivery/products.py index 82f4f72be4520f5706188c8015f4396b8c1d0e05..c2b6c533ebb0cdfc8e4213ab5a9b981e45f7329b 100644 --- a/apps/cli/executables/pexable/delivery/delivery/products.py +++ b/apps/cli/executables/pexable/delivery/delivery/products.py @@ -1,4 +1,5 @@ import abc +import glob import pathlib import re from typing import Dict, Tuple @@ -59,6 +60,23 @@ class ExecutionBlock(SpooledProduct): return f"execution block {self.eb_name}" +class CalibratedMS(SpooledProduct): + """ + A Calibrated Measurement Set is the end result of a restore capability request and is used as an input to imaging. + Deliver all files in the request's products directory. + """ + + @property + def cms_name(self): + return self.path.absolute().name + + def deliver_to(self, destination: Destination): + destination.add_directory(self.path, self.cms_name[:-3]) + + def __str__(self): + return f"calibrated MS {self.cms_name}" + + # Future types of product that might be needed: # class Calibration(SpooledProduct): diff --git a/apps/cli/executables/pexable/vela/test/test_forger.py b/apps/cli/executables/pexable/vela/test/test_forger.py index 8745406a2cf7bef67aa431ac55dc29cfc6371521..a7b78b7de80a5b82ad75b3a07b2fefa65faa59c8 100644 --- a/apps/cli/executables/pexable/vela/test/test_forger.py +++ b/apps/cli/executables/pexable/vela/test/test_forger.py @@ -64,9 +64,9 @@ class TestVelaProduct: def test_forge_measurement_set(self, mock_chdir, mock_mkdir): with patch("builtins.open", mock_open()) as o: VelaProduct("test.json").forge_measurement_set(sdm_id="brain_001.58099.678886747686") - assert o.call_count == 5 + assert o.call_count == 10 handle = o() - assert handle.write.call_count == 5 + assert handle.write.call_count == 10 assert mock_mkdir.call_count == 6 assert mock_chdir.call_count == 2 diff --git a/apps/cli/executables/pexable/vela/vela/forger.py b/apps/cli/executables/pexable/vela/vela/forger.py index c08acfb2a7bee01aea248b99a2222b2b584d8755..9192bab3929991e96ec4b14a5b29f947b54d924d 100644 --- a/apps/cli/executables/pexable/vela/vela/forger.py +++ b/apps/cli/executables/pexable/vela/vela/forger.py @@ -79,7 +79,7 @@ class VelaProduct: sdm_id = ( parameters["fileSetIds"][0] - if isinstance(parameters["workflowName"], list) + if isinstance(parameters["fileSetIds"], list) else parameters["fileSetIds"] ) filename = sdm_id + ".ms.calapply.txt" @@ -120,6 +120,9 @@ class VelaProduct: for i in range(0, 5): dirname = f"FORGED_MS_DIR_{i}" os.mkdir(dirname) + dir_file = open(f"./{dirname}/table.f{i}", "x") + dir_file.write("I am a file.") + dir_file.close() file = open(f"table.f{i}", "x") file.write("I am a table.") file.close() diff --git a/apps/web/src/app/workspaces/components/capability-request/components/request-operations/request-operations.component.html b/apps/web/src/app/workspaces/components/capability-request/components/request-operations/request-operations.component.html index 744327aceb8dbac2d6ae1c23a3497eb68484643c..ca54d5034a2008b92f50baafebaff5322726be19 100644 --- a/apps/web/src/app/workspaces/components/capability-request/components/request-operations/request-operations.component.html +++ b/apps/web/src/app/workspaces/components/capability-request/components/request-operations/request-operations.component.html @@ -8,7 +8,7 @@ ></app-create-new-version-form> </div> <div class="col-auto d-flex" *ngIf="capabilityRequest.state === 'Complete' && - capabilityRequest.capability_name === 'std_calibration' || capabilityRequest.capability_name === 'restore_cms'"> + (capabilityRequest.capability_name === 'std_calibration' || capabilityRequest.capability_name === 'restore_cms')"> <button id="create-image-request" type="button" diff --git a/schema/versions/fd68d858d354_restore_corrections.py b/schema/versions/fd68d858d354_restore_corrections.py new file mode 100644 index 0000000000000000000000000000000000000000..d7da0446e667bc76f243859b263b3d21f49cb9f6 --- /dev/null +++ b/schema/versions/fd68d858d354_restore_corrections.py @@ -0,0 +1,162 @@ +"""restore corrections + +Revision ID: fd68d858d354 +Revises: 87dae5acfd34 +Create Date: 2021-07-20 10:45:50.997748 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "fd68d858d354" +down_revision = "87dae5acfd34" +branch_labels = None +depends_on = None + + +restore_condor_content = """executable = restore_cms.sh +arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml + +output = restore.out +error = restore.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json +transfer_output_files = working, rawdata, products, delivery.json + +request_memory = {{ramInGb}} +getenv = True +environment = "CAPO_PATH=/home/casa/capo" + +queue + + +""" + +restore_img_content = """executable = std_restore_imaging.sh +arguments = {{product_locator}} {{cal_product_locator}} {{request_id}} metadata.json PPR.xml + +output = restore.out +error = restore.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json +transfer_output_files = working, rawdata, products, delivery.json + +request_memory = {{ramInGb}} +getenv = True +environment = "CAPO_PATH=/home/casa/capo" + +queue + + +""" + + +old_restore_img = """executable = std_restore_imaging.sh +arguments = {{product_locator}} {{cal_product_locator}} {{request_id}} metadata.json PPR.xml + +output = restore.out +error = restore.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json +transfer_output_files = working, rawdata, products + +request_memory = {{ramInGb}} +getenv = True +environment = "CAPO_PATH=/home/casa/capo" + +queue + + +""" + +old_restore_condor = """executable = restore_cms.sh +arguments = {{product_locator}} {{cal_locator}} {{request_id}} metadata.json PPR.xml + +output = restore.out +error = restore.err +log = condor.log + +SBIN_PATH = /lustre/aoc/cluster/pipeline/$ENV(CAPO_PROFILE)/workspaces/sbin +should_transfer_files = yes +transfer_input_files = $(SBIN_PATH)/.matplotlib, $(SBIN_PATH)/pycapo, $(SBIN_PATH)/framework.sh, $(SBIN_PATH)/productfetcher, $(SBIN_PATH)/casa_envoy, $(SBIN_PATH)/vela, $(SBIN_PATH)/deliver, ./PPR.xml, ./metadata.json +transfer_output_files = working, rawdata, products + +request_memory = {{ramInGb}} +getenv = True +environment = "CAPO_PATH=/home/casa/capo" + +queue + + +""" + + +def upgrade(): + op.execute( + """ + UPDATE capabilities + SET capability_steps='prepare-and-run-workflow restore_cms\nawait-workflow' + WHERE capability_name='restore_cms' + """ + ) + op.execute( + """ + UPDATE capabilities + SET capability_steps='prepare-and-run-workflow std_restore_imaging\nawait-workflow' + WHERE capability_name='std_restore_imaging' + """ + ) + + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{restore_condor_content}' WHERE filename='restore_cms.condor' + """ + ) + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{restore_img_content}' WHERE filename='std_restore_imaging.condor' + """ + ) + + +def downgrade(): + op.execute( + """ + UPDATE capabilities + SET capability_steps='prepare-and-run-workflow restore_cms\nawait-workflow\nawait-qa' + WHERE capability_name='restore_cms' + """ + ) + op.execute( + """ + UPDATE capabilities + SET capability_steps='prepare-and-run-workflow std_restore_imaging\nawait-workflow\nawait-qa' + WHERE capability_name='std_restore_imaging' + """ + ) + + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{old_restore_condor}' WHERE filename='restore_cms.condor' + """ + ) + op.execute( + f""" + UPDATE workflow_templates + SET content=E'{old_restore_img}' WHERE filename='std_restore_imaging.condor' + """ + ) diff --git a/shared/workspaces/workspaces/workflow/services/workflow_service.py b/shared/workspaces/workspaces/workflow/services/workflow_service.py index 8cb4febbad184d4bc655a8a785708f93d5e9669e..9945cf411da0c04d122b260c33ee1ad121da5be2 100644 --- a/shared/workspaces/workspaces/workflow/services/workflow_service.py +++ b/shared/workspaces/workspaces/workflow/services/workflow_service.py @@ -483,6 +483,8 @@ class WorkflowService(WorkflowServiceIF): status = WorkflowRequestState.Complete.name elif message["type"] == "workflow-failed": status = WorkflowRequestState.Failed.name + elif message["type"] == "delivery": + status = WorkflowRequestState.Complete.name else: status = "Unknown"