From bca5350bdab56d933c6d9c431b1d44444e2e1efa Mon Sep 17 00:00:00 2001
From: chausman <chausman@nrao.edu>
Date: Wed, 14 Jul 2021 09:43:42 -0600
Subject: [PATCH] refactor test structure
---
.../casa_envoy/test/{ => input_files}/PPR.xml | 0
.../test/{ => input_files}/cmsimage-PPR.xml | 0
.../{ => input_files}/image-metadata.json | 0
.../test/{ => input_files}/restore.json | 0
.../test/{ => input_files}/test.json | 0
.../pexable/casa_envoy/test/test_auditor.py | 22 +++++++++-------
.../casa_envoy/test/test_casa_envoy.py | 13 ++++++----
.../casa_envoy/test/test_foundation.py | 7 +++---
.../pexable/casa_envoy/test/test_launchers.py | 10 ++++----
.../7200d0d19938_add_restore_capability.py | 25 +++++++++++++++++++
10 files changed, 55 insertions(+), 22 deletions(-)
rename apps/cli/executables/pexable/casa_envoy/test/{ => input_files}/PPR.xml (100%)
rename apps/cli/executables/pexable/casa_envoy/test/{ => input_files}/cmsimage-PPR.xml (100%)
rename apps/cli/executables/pexable/casa_envoy/test/{ => input_files}/image-metadata.json (100%)
rename apps/cli/executables/pexable/casa_envoy/test/{ => input_files}/restore.json (100%)
rename apps/cli/executables/pexable/casa_envoy/test/{ => input_files}/test.json (100%)
diff --git a/apps/cli/executables/pexable/casa_envoy/test/PPR.xml b/apps/cli/executables/pexable/casa_envoy/test/input_files/PPR.xml
similarity index 100%
rename from apps/cli/executables/pexable/casa_envoy/test/PPR.xml
rename to apps/cli/executables/pexable/casa_envoy/test/input_files/PPR.xml
diff --git a/apps/cli/executables/pexable/casa_envoy/test/cmsimage-PPR.xml b/apps/cli/executables/pexable/casa_envoy/test/input_files/cmsimage-PPR.xml
similarity index 100%
rename from apps/cli/executables/pexable/casa_envoy/test/cmsimage-PPR.xml
rename to apps/cli/executables/pexable/casa_envoy/test/input_files/cmsimage-PPR.xml
diff --git a/apps/cli/executables/pexable/casa_envoy/test/image-metadata.json b/apps/cli/executables/pexable/casa_envoy/test/input_files/image-metadata.json
similarity index 100%
rename from apps/cli/executables/pexable/casa_envoy/test/image-metadata.json
rename to apps/cli/executables/pexable/casa_envoy/test/input_files/image-metadata.json
diff --git a/apps/cli/executables/pexable/casa_envoy/test/restore.json b/apps/cli/executables/pexable/casa_envoy/test/input_files/restore.json
similarity index 100%
rename from apps/cli/executables/pexable/casa_envoy/test/restore.json
rename to apps/cli/executables/pexable/casa_envoy/test/input_files/restore.json
diff --git a/apps/cli/executables/pexable/casa_envoy/test/test.json b/apps/cli/executables/pexable/casa_envoy/test/input_files/test.json
similarity index 100%
rename from apps/cli/executables/pexable/casa_envoy/test/test.json
rename to apps/cli/executables/pexable/casa_envoy/test/input_files/test.json
diff --git a/apps/cli/executables/pexable/casa_envoy/test/test_auditor.py b/apps/cli/executables/pexable/casa_envoy/test/test_auditor.py
index cc896a22b..f5ed148b1 100644
--- a/apps/cli/executables/pexable/casa_envoy/test/test_auditor.py
+++ b/apps/cli/executables/pexable/casa_envoy/test/test_auditor.py
@@ -18,13 +18,15 @@ cal_settings = {
"homeForReprocessing": "/home/casa/packages/pipeline/current",
"rootDirectory": "/tmp/workspaces_tmp/",
"processingDirectory": "tmpiox5trbp",
- "metadata": "test/test.json",
- "ppr": "test/PPR.xml",
+ "metadata": "test/input_files/test.json",
+ "ppr": "test/input_files/PPR.xml",
"product_type": "standard-cal",
}
-test_ppr = AbstractTextFile(filename="test/PPR.xml", content=Path("test/PPR.xml").read_text())
+test_ppr = AbstractTextFile(
+ filename="test/input_files/PPR.xml", content=Path("test/input_files/PPR.xml").read_text()
+)
test_cal_metadata = AbstractTextFile(
- filename="test/test.json", content=Path("test/test.json").read_text()
+ filename="test/input_files/test.json", content=Path("test/input_files/test.json").read_text()
)
img_settings = {
@@ -32,15 +34,17 @@ img_settings = {
"homeForReprocessing": "/home/casa/packages/pipeline/current",
"rootDirectory": "/tmp/workspaces_tmp/",
"processingDirectory": "tmpiox5trbp",
- "metadata": "test/image-metadata.json",
- "ppr": "test/cmsimage-PPR.xml",
+ "metadata": "test/input_files/image-metadata.json",
+ "ppr": "test/input_files/cmsimage-PPR.xml",
"product_type": "standard-img",
}
test_img_ppr = AbstractTextFile(
- filename="test/cmsimage-PPR.xml", content=Path("test/cmsimage-PPR.xml").read_text()
+ filename="test/input_files/cmsimage-PPR.xml",
+ content=Path("test/input_files/cmsimage-PPR.xml").read_text(),
)
test_img_metadata = AbstractTextFile(
- filename="test/image-metadata.json", content=Path("test/image-metadata.json").read_text()
+ filename="test/input_files/image-metadata.json",
+ content=Path("test/input_files/image-metadata.json").read_text(),
)
@@ -138,7 +142,7 @@ class TestAuditFiles:
ppr = AuditFiles([test_cal_metadata, test_ppr], cal_settings).correct_for_condor(
ppr=test_ppr
)
- assert ppr.filename == "test/PPR.xml"
+ assert ppr.filename == "test/input_files/PPR.xml"
def test_audit(self):
result = AuditFiles([test_cal_metadata, test_ppr], cal_settings).audit()
diff --git a/apps/cli/executables/pexable/casa_envoy/test/test_casa_envoy.py b/apps/cli/executables/pexable/casa_envoy/test/test_casa_envoy.py
index e28c7391e..8b2baef9b 100644
--- a/apps/cli/executables/pexable/casa_envoy/test/test_casa_envoy.py
+++ b/apps/cli/executables/pexable/casa_envoy/test/test_casa_envoy.py
@@ -12,8 +12,8 @@ expected_settings = {
"rootDirectory": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool",
"processingDirectory": "tmpo1ca1pp_",
"parent_path": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool/tmpo1ca1pp_",
- "metadata": "test/test.json",
- "ppr": "test/PPR.xml",
+ "metadata": "test/input_files/test.json",
+ "ppr": "test/input_files/PPR.xml",
"product_type": "standard-cal",
}
args = argparse.Namespace()
@@ -21,7 +21,7 @@ args = argparse.Namespace()
class TestPalaver:
def test_get_settings(self):
- args.standard_cal = ["test/test.json", "test/PPR.xml"]
+ args.standard_cal = ["test/input_files/test.json", "test/input_files/PPR.xml"]
args.restore = False
with patch(
@@ -41,7 +41,7 @@ class TestPalaver:
@patch("os.chdir")
@patch("os.getcwd")
def test_main_cal(self, mock_cwd, mock_chdir):
- args.standard_cal = ["test/test.json", "test/PPR.xml"]
+ args.standard_cal = ["test/input_files/test.json", "test/input_files/PPR.xml"]
args.integrated = False
with patch("argparse.ArgumentParser.parse_args", MagicMock(return_value=args)) as mock_args:
@@ -54,7 +54,10 @@ class TestPalaver:
@patch("os.chdir")
@patch("os.getcwd")
def test_main_img(self, mock_cwd, mock_chdir):
- args.standard_img = ["test/image-metadata.json", "test/cmsimage-PPR.xml"]
+ args.standard_img = [
+ "test/input_files/image-metadata.json",
+ "test/input_files/cmsimage-PPR.xml",
+ ]
with patch("argparse.ArgumentParser.parse_args", MagicMock(return_value=args)) as mock_args:
with patch("casa_envoy.launchers.ImagingLauncher.launch_casa") as img_launcher:
diff --git a/apps/cli/executables/pexable/casa_envoy/test/test_foundation.py b/apps/cli/executables/pexable/casa_envoy/test/test_foundation.py
index 2482d2276..13e3ed77d 100644
--- a/apps/cli/executables/pexable/casa_envoy/test/test_foundation.py
+++ b/apps/cli/executables/pexable/casa_envoy/test/test_foundation.py
@@ -15,13 +15,14 @@ parameters = expected_settings = {
"rootDirectory": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool",
"processingDirectory": "tmpo1ca1pp_",
"parent_path": "/lustre/aoc/cluster/pipeline/docker/workspaces/spool/tmpo1ca1pp_",
- "metadata": "test/restore.json",
- "ppr": "test/PPR.xml",
+ "metadata": "test/input_files/restore.json",
+ "ppr": "test/input_files/PPR.xml",
"product_type": "restore",
}
test_restore_metadata = AbstractTextFile(
- filename="test/restore.json", content=Path("test/restore.json").read_text()
+ filename="test/input_files/restore.json",
+ content=Path("test/input_files/restore.json").read_text(),
)
foundation = RestoreFoundation(parameters=parameters, metadata=test_restore_metadata)
diff --git a/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py b/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py
index 38b235c42..cb5dcc81f 100644
--- a/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py
+++ b/apps/cli/executables/pexable/casa_envoy/test/test_launchers.py
@@ -8,8 +8,8 @@ cal_parameters = {
"homeForReprocessing": "/home/casa/packages/pipeline/current",
"rootDirectory": "/tmp/workspaces_tmp/",
"processingDirectory": "tmpiox5trbp",
- "metadata": "test/test.json",
- "ppr": "test/PPR.xml",
+ "metadata": "test/input_files/test.json",
+ "ppr": "test/input_files/PPR.xml",
"product_type": "standard-cal",
}
img_parameters = {
@@ -17,8 +17,8 @@ img_parameters = {
"homeForReprocessing": "/home/casa/packages/pipeline/current",
"rootDirectory": "/tmp/workspaces_tmp/",
"processingDirectory": "tmpiox5trbp",
- "metadata": "test/image-metadata.json",
- "ppr": "test/cmsimage-PPR.xml",
+ "metadata": "test/input_files/image-metadata.json",
+ "ppr": "test/input_files/cmsimage-PPR.xml",
"product_type": "standard-img",
}
@@ -28,7 +28,7 @@ class TestCasaLauncher:
CasaLauncher(parameters=cal_parameters).setup_environment()
assert os.environ.get("SCIPIPE_ROOTDIR") == cal_parameters["rootDirectory"]
assert os.environ.get("CASA_HOME") == cal_parameters["homeForReprocessing"]
- assert os.environ.get("PPR_FILENAME") == "test/PPR.xml"
+ assert os.environ.get("PPR_FILENAME") == "test/input_files/PPR.xml"
@patch("subprocess.run")
@patch("os.chdir")
diff --git a/schema/versions/7200d0d19938_add_restore_capability.py b/schema/versions/7200d0d19938_add_restore_capability.py
index 3de56bd48..5441624c5 100644
--- a/schema/versions/7200d0d19938_add_restore_capability.py
+++ b/schema/versions/7200d0d19938_add_restore_capability.py
@@ -52,6 +52,25 @@ cp ./working/{{sdmId}}.ms ./products/
"""
+metadata_content = """{
+ "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"]
+ "workflowName": "std_calibration",
+ "systemId": "{{request_id}}",
+ "creationTime": "{{created_at}}",
+ "productLocator": "{{product_locator}}",
+ "calProductLocator": "{{cal_locator}}",
+ "projectMetadata": {
+ "projectCode": "{{projectCode}}",
+ "title": "{{title}}",
+ "startTime": "{{startTime}}",
+ "observer": "{{observer}}"
+ },
+ "destinationDirectory": "{{root_directory}}/{{relative_path}}"
+}
+
+"""
+
+
def upgrade():
restore_steps = """prepare-and-run-workflow restore_cms
await-workflow
@@ -83,6 +102,12 @@ await-qa
VALUES ('restore_cms.sh', E'{restore_script_content}', 'restore_cms')
"""
)
+ op.execute(
+ f"""
+ INSERT INTO workflow_templates (filename, content, workflow_name)
+ VALUES ('metadata.json', E'{metadata_content}', 'restore_cms')
+ """
+ )
def downgrade():
--
GitLab