Removed unnecessary clean-up from tests

apps/cli/executables/pexable/ingest_envoy/test/conftest.py

@@ -43,10 +43,10 @@ UNWANTED = ["ignore_me.fits", "just_a_lotta_nothing", "uninteresting_metadata.xm
 @pytest.fixture(scope="function")
 def ingest_path(tmp_path: Path) -> Path:
     """
-    Make an "ingestion path" for tests
+    Make a directory to use as the ingestion staging dir, or curation source
 
-    :param tmp_path: built-in pytest fixture; temporary home for ingestion location
-    :return:
+    :param tmp_path: built-in pytest fixture, Pytest cleans them up periodically
+    :return: Path to new directory
     """
 
     # cast is necessary because otherwise we get a LocalPath, which doesn't work
@@ -57,6 +57,12 @@ def ingest_path(tmp_path: Path) -> Path:
 
 @pytest.fixture
 def alternate_manifest_destination(tmp_path: Path) -> Path:
+    """
+    Make an alternative directory to ingest_path for tests to put their manifests in
+
+    :param tmp_path: built-in pytest fixture, Pytest cleans them up periodically
+    :return: Path to new directory
+    """
     alternate_manifest_destination = tmp_path / "manifest_destination"
     alternate_manifest_destination.mkdir()
     return alternate_manifest_destination

apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py

@@ -19,7 +19,6 @@
 
 import json
 import logging
-import shutil
 import sys
 import tarfile
 from pathlib import Path
@@ -109,8 +108,6 @@ def test_filters_cal_input_files(ingest_path: Path):
     assert sp_out.filename in EVLA_CAL_INPUT_FILENAMES
     assert sp_out.filename not in UNWANTED
 
-    shutil.rmtree(ingest_path)
-
 
 def test_writes_expected_evla_cal_output_files(ingest_path: Path):
     """
@@ -138,8 +135,6 @@ def test_writes_expected_evla_cal_output_files(ingest_path: Path):
     tars = [file for file in ingestion_files if file.name.endswith(TARFILE_EXT)]
     assert len(tars) >= 2
 
-    shutil.rmtree(ingest_path)
-
 
 @pytest.mark.skip("TODO")
 def test_writes_final_evla_cal_output(ingest_path: Path):
@@ -269,8 +264,6 @@ def test_ingestion_artifacts_tar_filename_built_just_once(ingest_path: Path):
         ).build()
         mock.assert_called_once()
 
-    shutil.rmtree(ingest_path)
-
 
 def test_ingestion_artifacts_tar_correct(ingest_path: Path):
     """
@@ -301,8 +294,6 @@ def test_ingestion_artifacts_tar_correct(ingest_path: Path):
     mf_path = Path(member.name)
     assert mf_path.name == MANIFEST_FILENAME
 
-    shutil.rmtree(ingest_path)
-
 
 def test_evla_cal_manifest_matches_example(ingest_path: Path):
     """
@@ -355,8 +346,6 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
         logger.info(f"{ap.filename}: {ap.type}, group with {ap.group_with}")
     assert len(actual_aps) == len(expected_aps)
 
-    shutil.rmtree(ingest_path)
-
 
 def test_evla_cal_final_manifest_finds_init_weblog(ingest_path: Path):
     """
@@ -468,5 +457,3 @@ def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
         if ap.filename == WEBLOG_FILENAME or ap.filename == INIT_WEBLOG_FILENAME:
             found_count += 1
     assert found_count == 2
-
-    shutil.rmtree(ingest_path)

apps/cli/executables/pexable/ingest_envoy/test/test_img_manifest_example.py

@@ -19,7 +19,6 @@
 /lustre/aoc/cluster/pipeline/dsoc-dev/workspaces/staging/tmpx_ratuqh"""
 
 import json
-import shutil
 import tarfile
 from pathlib import Path
 
@@ -52,95 +51,92 @@ def test_image_manifest_creation(ingest_path: Path):
     """Test manifest creation for an image"""
 
     populate_fake_tmpx_ratuqh_ingest_path(ingest_path)
-    try:
-        manifest, manifest_file = build_tmpx_ratuqh_image_manifest(ingest_path)
-
-        # Check that top-level groups match expectations
-        mf_json = manifest.to_dict()
-        keys = ["parameters", "input_group", "output_group"]
-        assert len(mf_json) == len(keys)
-        for key in keys:
-            assert key in mf_json.keys()
-
-        """
-        The ancillary_products section of the manifest we build
-        should match the one in the example.
-        """
-        output_group = manifest.output_group
-        aps = output_group.ancillary_products
-        assert len(aps) == len(ANCILLARY_PRODUCTS)
-
-        # Check contents of ancillary_products
-        weblog_candidates = [ap for ap in manifest.output_group.ancillary_products if ap.filename == WEBLOG_FILENAME]
-        assert len(weblog_candidates) == 1
-        assert weblog_candidates[0].type == AncillaryProductType.PIPELINE_WEBLOG
-
-        ingest_artifacts_tar = find_ingestion_artifacts_tar(ingest_path)
-        ingest_artifacts_candidates = [
-            ap for ap in manifest.output_group.ancillary_products if ap.filename == ingest_artifacts_tar.name
-        ]
-        assert len(ingest_artifacts_candidates) == 1
-        assert ingest_artifacts_candidates[0].type == AncillaryProductType.INGESTION_ARTIFACTS
-
-        maybe_pips = [file for file in ingest_path.glob(AncillaryProductType.PIPELINE_ARTIFACTS.value + "*.tar")]
-        assert len(maybe_pips) == 1
-        pip_artie = maybe_pips[0]
-        pipeline_artifacts_candidates = [
-            ap for ap in manifest.output_group.ancillary_products if ap.filename == pip_artie.name
-        ]
-        assert len(pipeline_artifacts_candidates) == 1
-        assert pipeline_artifacts_candidates[0].type == AncillaryProductType.PIPELINE_ARTIFACTS
-
-        # Inspect the manifest's JSON dict
-        """
-        The output_group section of the manifest we build
-        should match the one in the example:
-        * a "science_products" section containing two science products comprising "type", "filename",
-        and six ancillary products on each science product
-        * an "ancillary products" section comprising three ancillary products
-        """
-        mf_json = manifest.to_dict()
-        og_json = mf_json[IngestionManifestKey.OUTPUT_GROUP.value]
-        assert len(og_json[IngestionManifestKey.SCIENCE_PRODUCTS.value]) == 2
-
-        for sp_json in og_json[IngestionManifestKey.SCIENCE_PRODUCTS.value]:
-            assert len(sp_json) == 3
-            assert set(sp_json.keys()) == {IngestionManifestKey.ANCILLARY_PRODUCTS.value, "type", "filename"}
-            # and ancillary products belonging to the first science product...
-            sp_ap_jsons = sp_json[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
-            assert len(sp_ap_jsons) == 6
-
-        # ... and ancillary products twisting in the wind all by themselves
-        ap_jsons = og_json[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
-        assert len(ap_jsons) == 3
-
-        # there should be a weblog, the ingestion artifacts tar, and the pipeline artifacts tar
-        filenames_found = []
-        for ap_json in ap_jsons:
-            filename = ap_json["filename"]
-            if filename.endswith(TARFILE_EXT):
-                if filename.startswith(AncillaryProductType.INGESTION_ARTIFACTS.value) or filename.startswith(
-                    AncillaryProductType.PIPELINE_ARTIFACTS.value
-                ):
-                    filenames_found.append(filename)
-            elif filename == WEBLOG_FILENAME:
+    manifest, manifest_file = build_tmpx_ratuqh_image_manifest(ingest_path)
+
+    # Check that top-level groups match expectations
+    mf_json = manifest.to_dict()
+    keys = ["parameters", "input_group", "output_group"]
+    assert len(mf_json) == len(keys)
+    for key in keys:
+        assert key in mf_json.keys()
+
+    """
+    The ancillary_products section of the manifest we build
+    should match the one in the example.
+    """
+    output_group = manifest.output_group
+    aps = output_group.ancillary_products
+    assert len(aps) == len(ANCILLARY_PRODUCTS)
+
+    # Check contents of ancillary_products
+    weblog_candidates = [ap for ap in manifest.output_group.ancillary_products if ap.filename == WEBLOG_FILENAME]
+    assert len(weblog_candidates) == 1
+    assert weblog_candidates[0].type == AncillaryProductType.PIPELINE_WEBLOG
+
+    ingest_artifacts_tar = find_ingestion_artifacts_tar(ingest_path)
+    ingest_artifacts_candidates = [
+        ap for ap in manifest.output_group.ancillary_products if ap.filename == ingest_artifacts_tar.name
+    ]
+    assert len(ingest_artifacts_candidates) == 1
+    assert ingest_artifacts_candidates[0].type == AncillaryProductType.INGESTION_ARTIFACTS
+
+    maybe_pips = [file for file in ingest_path.glob(AncillaryProductType.PIPELINE_ARTIFACTS.value + "*.tar")]
+    assert len(maybe_pips) == 1
+    pip_artie = maybe_pips[0]
+    pipeline_artifacts_candidates = [
+        ap for ap in manifest.output_group.ancillary_products if ap.filename == pip_artie.name
+    ]
+    assert len(pipeline_artifacts_candidates) == 1
+    assert pipeline_artifacts_candidates[0].type == AncillaryProductType.PIPELINE_ARTIFACTS
+
+    # Inspect the manifest's JSON dict
+    """
+    The output_group section of the manifest we build
+    should match the one in the example:
+    * a "science_products" section containing two science products comprising "type", "filename",
+    and six ancillary products on each science product
+    * an "ancillary products" section comprising three ancillary products
+    """
+    mf_json = manifest.to_dict()
+    og_json = mf_json[IngestionManifestKey.OUTPUT_GROUP.value]
+    assert len(og_json[IngestionManifestKey.SCIENCE_PRODUCTS.value]) == 2
+
+    for sp_json in og_json[IngestionManifestKey.SCIENCE_PRODUCTS.value]:
+        assert len(sp_json) == 3
+        assert set(sp_json.keys()) == {IngestionManifestKey.ANCILLARY_PRODUCTS.value, "type", "filename"}
+        # and ancillary products belonging to the first science product...
+        sp_ap_jsons = sp_json[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
+        assert len(sp_ap_jsons) == 6
+
+    # ... and ancillary products twisting in the wind all by themselves
+    ap_jsons = og_json[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
+    assert len(ap_jsons) == 3
+
+    # there should be a weblog, the ingestion artifacts tar, and the pipeline artifacts tar
+    filenames_found = []
+    for ap_json in ap_jsons:
+        filename = ap_json["filename"]
+        if filename.endswith(TARFILE_EXT):
+            if filename.startswith(AncillaryProductType.INGESTION_ARTIFACTS.value) or filename.startswith(
+                AncillaryProductType.PIPELINE_ARTIFACTS.value
+            ):
                 filenames_found.append(filename)
+        elif filename == WEBLOG_FILENAME:
+            filenames_found.append(filename)
+
+    assert len(filenames_found) == 3
 
-        assert len(filenames_found) == 3
-
-        """
-        The ingestion_artifacts tar should contain ONLY
-        the ingestion manifest and the additional metadata file
-        """
-        addl_md = manifest.parameters.additional_metadata
-        ing_arties_tar = find_ingestion_artifacts_tar(ingest_path)
-        with tarfile.open(ing_arties_tar, "r") as tar:
-            members = tar.getmembers()
-            assert len(members) == 2
-            for member in members:
-                assert member.name.endswith(MANIFEST_FILENAME) or member.name.endswith(addl_md.filename)
-    finally:
-        shutil.rmtree(ingest_path)
+    """
+    The ingestion_artifacts tar should contain ONLY
+    the ingestion manifest and the additional metadata file
+    """
+    addl_md = manifest.parameters.additional_metadata
+    ing_arties_tar = find_ingestion_artifacts_tar(ingest_path)
+    with tarfile.open(ing_arties_tar, "r") as tar:
+        members = tar.getmembers()
+        assert len(members) == 2
+        for member in members:
+            assert member.name.endswith(MANIFEST_FILENAME) or member.name.endswith(addl_md.filename)
 
 
 def test_catches_invalid_sp_type():

apps/cli/executables/pexable/ingest_envoy/test/test_manifest_builder_entry_points.py

@@ -20,18 +20,9 @@
 import logging
 
 # pylint: disable=E0401, E0402, R1721, W0611, W0621
-import shutil
 import sys
 from pathlib import Path
 
-from ingest_envoy.ingestion_manifest import (
-    IngestionManifest,
-    IngestionManifestBuilder,
-    find_manifest,
-)
-from ingest_envoy.manifest_components import INGESTION_ARTIFACTS_NAME, TARFILE_EXT
-from ingest_envoy.utilities import ScienceProductType, Telescope
-
 from conftest import (
     ANCILLARY_PRODUCTS,
     OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_K,
@@ -40,6 +31,13 @@ from conftest import (
     populate_fake_evla_cal_ingest_path,
     populate_fake_tmpx_ratuqh_ingest_path,
 )
+from ingest_envoy.ingestion_manifest import (
+    IngestionManifest,
+    IngestionManifestBuilder,
+    find_manifest,
+)
+from ingest_envoy.manifest_components import INGESTION_ARTIFACTS_NAME, TARFILE_EXT
+from ingest_envoy.utilities import ScienceProductType, Telescope
 
 logger = logging.getLogger(IngestionManifest.__name__)
 logger.setLevel(logging.INFO)
@@ -96,8 +94,12 @@ def test_entry_point_for_image(ingest_path: Path):
     # we should be starting out with various image manifest input files
     # and CASA byproducts, a random file, and -not- the image ingestion
     # manifest yet to be created
-    expected_file_count_before = len(ANCILLARY_PRODUCTS) + len(OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_K) +\
-                                 len(OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_X) + 2
+    expected_file_count_before = (
+        len(ANCILLARY_PRODUCTS)
+        + len(OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_K)
+        + len(OUTPUT_GROUP_SCIENCE_PRODUCT_ANCILLARIES_X)
+        + 2
+    )
     ingestion_files_before = [file for file in ingest_path.iterdir()]
     assert len(ingestion_files_before) == expected_file_count_before
 
@@ -123,4 +125,3 @@ def test_entry_point_for_image(ingest_path: Path):
                 print(f">>> {file.name} present after manifest build")
 
     assert len(ingestion_files_after) == expected_file_count_before + 2
-    shutil.rmtree(ingest_path)