test_cli.py

#
# Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA.
#
# This file is part of NRAO Workspaces.
#
# Workspaces is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Workspaces is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Workspaces.  If not, see <https://www.gnu.org/licenses/>.
# Testing the CLI
import datetime
import filecmp
import json
import pathlib
import shutil
import tarfile
from test import (
    RESTORE_AUXPRODUCTS_NAME,
    TEST_RESTORE_METADATA,
    get_expected_files_and_dirs_for_restore,
)
from test.conftest import assert_restore_delivered_only_expected_files
from unittest.mock import patch

import pytest
from delivery.context import DeliveryContext
from delivery.delivery import Delivery, main


def verify_extracted_directory(
    subdirectory: str,
    delivered_subdirectory: str,
    tar_path: pathlib.Path,
    extraction_target: pathlib.Path,
    original_data_path: str,
):
    """
    Verify that an extracted directory has the same contents as the supplied temporary directory.
    Useful for testing tar-related functionality

    :param subdirectory:       subdirectory to look for inside extraction area
    :param tar_path:           path to the tarfile to examine
    :param extraction_target:  location to extract to
    :param original_data_path: location of the original files to compare to
    :return:
    """

    # is it actually a tar?
    assert tarfile.is_tarfile(tar_path)

    # let's unpack it
    shutil.unpack_archive(tar_path, extraction_target / "extracted")

    # did it output what we expect?
    assert (extraction_target / "extracted" / delivered_subdirectory).exists()

    # compare the extracted results with the source
    assert_directories_are_same(
        extraction_target / "extracted" / delivered_subdirectory, (original_data_path + "/" + subdirectory)
    )


def assert_directories_are_same(left, right):
    """
    Check that the contents of two directories are the same as far as we care
    :param left:
    :param right:
    :return:
    """
    compare_dirs = filecmp.dircmp(left, right)

    # did the comparison report they are the same
    assert len(compare_dirs.left_only) == 0
    assert len(compare_dirs.right_only) == 0
    assert len(compare_dirs.funny_files) == 0


def test_local_rawdata_no_tar(resource_path_root, tmpdir_factory, capsys):
    """
    Test that local delivery works without tar (the simplest case)
    """
    temp_directory = str(tmpdir_factory.mktemp("test_basic_rawdata_no_tar"))
    test_data_path = resource_path_root / "724126739"
    eb_name = "17A-109.sb33151331.eb33786546.57892.65940042824"
    main(["-r", "-l", temp_directory, str(test_data_path)])

    # compare the source and destination
    assert_directories_are_same(
        temp_directory + "/17A-109/observation.57892.6594042824/" + eb_name,
        test_data_path / "b2f536ca-bdce-4073-b9ee-96d8266109e7" / eb_name,
    )

    # ensure that we actually got a delivery file with the proper contents
    with open("delivery.json", "r") as delivery_results_file:
        results = json.load(delivery_results_file)
        assert len(results.keys()) == 3
        assert results["delivered_to"] == temp_directory
    capsys.readouterr()


def test_local_restore_no_tar(restore_directory: pathlib.Path, tmpdir_factory, capsys):
    """
    Test that local delivery works without tar (the simplest case)
    """
    temp_directory = str(tmpdir_factory.mktemp("test_basic_restore_no_tar"))
    main(["--restore", "-l", temp_directory, str(restore_directory)])

    deliver_rel_path_root = (
        f"{temp_directory}/{TEST_RESTORE_METADATA.project_code}/{TEST_RESTORE_METADATA.pipeline_spec}"
    )
    expected_files, expected_dirs_to_file_counts = get_expected_files_and_dirs_for_restore(deliver_rel_path_root)
    expected_files.add(pathlib.Path(temp_directory + "/SHA1SUMS"))
    expected_files = {pathlib.Path(file).resolve() for file in expected_files}
    actual_files = list(p.resolve() for p in pathlib.Path(temp_directory).rglob("*"))
    assert_restore_delivered_only_expected_files(actual_files, expected_files, expected_dirs_to_file_counts)

    # ensure that we actually got a delivery file with the proper contents
    with open("delivery.json", "r") as delivery_results_file:
        results = json.load(delivery_results_file)
        assert len(results.keys()) == 3
        assert results["delivered_to"] == temp_directory
    capsys.readouterr()


def test_local_restore_no_tar_with_flagtemplate(
    restore_directory: pathlib.Path, tmp_path_factory: pytest.TempPathFactory, capsys
):
    """
    Test that local delivery works without tar (the simplest case)
    """
    auxproducts_path = restore_directory.parent / RESTORE_AUXPRODUCTS_NAME
    assert auxproducts_path.is_file()
    # To avoid possibly polluting the repo,
    #     use a tmp dir for the restore since the auxproducts tar needs to be copied into it
    src_dir = str(tmp_path_factory.mktemp("test_basic_restore_no_tar_flagtemplate_src"))
    shutil.copytree(restore_directory, src_dir, dirs_exist_ok=True)
    shutil.copy2(auxproducts_path, src_dir + "/products")

    dest_dir = str(tmp_path_factory.mktemp("test_basic_restore_no_tar_flagtemplate_dest"))
    main(["--restore", "-l", dest_dir, src_dir])

    deliver_rel_path_root = f"{dest_dir}/{TEST_RESTORE_METADATA.project_code}/{TEST_RESTORE_METADATA.pipeline_spec}"
    expected_files, expected_dirs_to_file_counts = get_expected_files_and_dirs_for_restore(
        deliver_rel_path_root, do_include_flagtemplate=True
    )
    expected_files.add(pathlib.Path(dest_dir + "/SHA1SUMS"))
    expected_files = {pathlib.Path(file).resolve() for file in expected_files}
    actual_files = list(p.resolve() for p in pathlib.Path(dest_dir).rglob("*"))
    assert_restore_delivered_only_expected_files(actual_files, expected_files, expected_dirs_to_file_counts)

    # ensure that we actually got a delivery file with the proper contents
    with open("delivery.json", "r") as delivery_results_file:
        results = json.load(delivery_results_file)
        assert len(results.keys()) == 3
        assert results["delivered_to"] == dest_dir
    capsys.readouterr()


def test_local_rawdata_with_tar(resource_path_root, tmpdir_factory, capsys):
    """
    Test that local delivery works with tar
    """
    temp_directory = pathlib.Path(tmpdir_factory.mktemp("test_basic_rawdata_with_tar"))
    test_data_path = resource_path_root / "724126739"
    with patch("delivery.destinations.tar.datetime", wraps=datetime.datetime) as dt:
        dt.now.return_value = datetime.datetime(2022, 1, 3, 8, 14, 56)
        main(["-r", "-t", "-l", str(temp_directory), str(test_data_path)])
    eb_name = "17A-109.sb33151331.eb33786546.57892.65940042824"
    tar_path = temp_directory / "NRAO_archive_17A-109_20220103-081456.tar"

    # does a tar exist where we think
    assert tar_path.exists()

    # do we only have it and the SHA1SUMS
    assert len(list(temp_directory.iterdir())) == 2

    verify_extracted_directory(
        "b2f536ca-bdce-4073-b9ee-96d8266109e7/" + eb_name,
        "17A-109/observation.57892.6594042824/" + eb_name,
        tar_path,
        temp_directory,
        str(test_data_path) + "/",
    )

    with open("delivery.json", "r") as delivery_results_file:
        results = json.load(delivery_results_file)
        assert len(results.keys()) == 3
        assert results["delivered_to"] == str(temp_directory)
    capsys.readouterr()


def test_local_restore_with_tar(restore_directory: pathlib.Path, tmpdir_factory, capsys):
    """
    Test that local restore delivery works with tar
    """
    temp_directory = pathlib.Path(tmpdir_factory.mktemp("test_basic_restore_with_tar"))
    with patch("delivery.destinations.tar.datetime", wraps=datetime.datetime) as dt:
        dt.now.return_value = datetime.datetime(2022, 1, 3, 8, 14, 56)
        main(["--restore", "-t", "-l", str(temp_directory), str(restore_directory)])
    tar_path = temp_directory / f"NRAO_archive_{TEST_RESTORE_METADATA.project_code}_20220103-081456.tar"

    # does a tar exist where we think
    assert len(list(temp_directory.iterdir())) == 2
    assert set(temp_directory.iterdir()) == {tar_path, temp_directory / "SHA1SUMS"}

    # Extract the tar and make sure it looks like an untarred, delivered restore
    assert tarfile.is_tarfile(tar_path)
    extraction_dir = temp_directory / "extracted"
    shutil.unpack_archive(tar_path, extraction_dir)
    actual_files = list(extraction_dir.rglob("*"))
    expected_files, expected_dirs_to_filecounts = get_expected_files_and_dirs_for_restore(
        str(extraction_dir / TEST_RESTORE_METADATA.project_code / TEST_RESTORE_METADATA.pipeline_spec)
    )
    assert_restore_delivered_only_expected_files(actual_files, expected_files, expected_dirs_to_filecounts)

    with open("delivery.json", "r") as delivery_results_file:
        results = json.load(delivery_results_file)
        assert len(results.keys()) == 3
        assert results["delivered_to"] == str(temp_directory)
    capsys.readouterr()


def test_web_rawdata_no_tar(resource_path_root, tmpdir_factory, capsys):
    """
    Test that delivery works to a web destination without tar
    """
    temp_directory = pathlib.Path(tmpdir_factory.mktemp("test_web_rawdata_no_tar"))
    test_data_path = resource_path_root / "724126739"
    eb_name = "17A-109.sb33151331.eb33786546.57892.65940042824"
    test_context = DeliveryContext.parse_commandline(["-r", str(test_data_path)])
    with patch("delivery.destinations.sharedweb.CapoConfig") as mocked_capo_config:
        mocked_capo_config.return_value.settings.return_value.downloadDirectory = str(temp_directory)
        mocked_capo_config.return_value.settings.return_value.downloadUrl = "http://testing"
        assert str(temp_directory) == mocked_capo_config().settings().downloadDirectory
        results = Delivery().deliver(test_context)

    # check the relationship between the delivery root and the URL
    assert str(temp_directory / results["url"].lstrip("http://testing")) == results["delivered_to"]
    actual_delivery_dir = pathlib.Path(results["delivered_to"])

    # compare the source and destination
    assert_directories_are_same(
        actual_delivery_dir / "17A-109/observation.57892.6594042824" / eb_name,
        test_data_path / "b2f536ca-bdce-4073-b9ee-96d8266109e7" / eb_name,
    )
    capsys.readouterr()


def test_web_rawdata_no_tmp(resource_path_root, tmpdir_factory, capsys):
    """
    Test that delivery works to a web destination and doesn't deliver tmp directories
    """
    temp_directory = pathlib.Path(tmpdir_factory.mktemp("test_web_rawdata_no_tmp"))
    test_data_path = resource_path_root / "724126739_with_tmp"
    test_context = DeliveryContext.parse_commandline(["-r", str(test_data_path)])
    with patch("delivery.destinations.sharedweb.CapoConfig") as mocked_capo_config:
        mocked_capo_config.return_value.settings.return_value.downloadDirectory = str(temp_directory)
        mocked_capo_config.return_value.settings.return_value.downloadUrl = "http://testing"
        assert str(temp_directory) == mocked_capo_config().settings().downloadDirectory
        results = Delivery().deliver(test_context)

    # check the relationship between the delivery root and the URL
    assert str(temp_directory / results["url"].lstrip("http://testing")) == results["delivered_to"]
    actual_delivery_dir = pathlib.Path(results["delivered_to"])

    # compare the source and destination
    delivered_contents = [d for d in actual_delivery_dir.iterdir()]
    test_data_contents = [d for d in test_data_path.iterdir()]
    assert test_data_path / "tmp123" in test_data_contents
    assert not actual_delivery_dir / "tmp123" in delivered_contents

    capsys.readouterr()


def test_web_rawdata_no_tar_with_prefix(resource_path_root, tmpdir_factory, capsys):
    """
    Test that delivery works to a web destination without tar
    """
    temp_directory = pathlib.Path(tmpdir_factory.mktemp("test_web_rawdata_no_tar"))
    test_data_path = resource_path_root / "724126739"
    eb_name = "17A-109.sb33151331.eb33786546.57892.65940042824"
    prefix = "1234"
    test_context = DeliveryContext.parse_commandline(["-r", str(test_data_path), "--prefix", prefix])
    with patch("delivery.destinations.sharedweb.CapoConfig") as mocked_capo_config:
        mocked_capo_config.return_value.settings.return_value.downloadDirectory = str(temp_directory)
        mocked_capo_config.return_value.settings.return_value.downloadUrl = "http://testing"
        assert str(temp_directory) == mocked_capo_config().settings().downloadDirectory
        results = Delivery().deliver(test_context)

    # check the relationship between the delivery root and the URL
    assert str(temp_directory / results["url"].lstrip("http://testing")) == results["delivered_to"]
    actual_delivery_dir = pathlib.Path(results["delivered_to"])

    # compare the source and destination
    assert_directories_are_same(
        actual_delivery_dir / "17A-109/observation.57892.6594042824" / eb_name,
        test_data_path / "b2f536ca-bdce-4073-b9ee-96d8266109e7" / eb_name,
    )

    # make sure that 1234 wound up in the URL and in the delivery directory
    assert f"/{prefix}/" in results["url"]
    assert f"/{prefix}/" in str(actual_delivery_dir)
    capsys.readouterr()


def test_web_rawdata_with_tar(resource_path_root, tmpdir_factory, capsys):
    """
    Test that delivery works to a web destination with tar
    """
    temp_directory = pathlib.Path(tmpdir_factory.mktemp("test_web_rawdata_with_tar"))
    test_data_path = resource_path_root / "724126739"
    test_context = DeliveryContext.parse_commandline(["-r", "-t", str(test_data_path)])

    with patch("delivery.destinations.sharedweb.CapoConfig") as mocked_capo_config:
        mocked_capo_config.return_value.settings.return_value.downloadDirectory = str(temp_directory)
        mocked_capo_config.return_value.settings.return_value.downloadUrl = "http://testing"
        assert str(temp_directory) == mocked_capo_config().settings().downloadDirectory

        with patch("delivery.destinations.tar.datetime", wraps=datetime.datetime) as dt:
            dt.now.return_value = datetime.datetime(2022, 1, 3, 8, 14, 56)
            results = Delivery().deliver(test_context)

    eb_name = "17A-109.sb33151331.eb33786546.57892.65940042824"
    tar_name = "NRAO_archive_17A-109_20220103-081456.tar"

    # check the relationship between the delivery root and the URL
    assert str(temp_directory / results["url"].lstrip("http://testing")) == results["delivered_to"]
    actual_delivery_dir = pathlib.Path(results["delivered_to"])

    # does a tar exist where we think
    tar_path = actual_delivery_dir / tar_name
    assert tar_path.exists()

    # is it the only thing there (did cleanup work)
    assert len(list(actual_delivery_dir.iterdir())) == 3

    verify_extracted_directory(
        "b2f536ca-bdce-4073-b9ee-96d8266109e7/" + eb_name,
        "17A-109/observation.57892.6594042824/" + eb_name,
        tar_path,
        temp_directory,
        str(test_data_path),
    )
    capsys.readouterr()


@pytest.mark.skip()
def test_web_calibration(tmpdir_factory, capsys, resource_path_root):
    """
    Tests that delivery can correctly parse a calibration's products and deliver them correctly according to the
    new delivery standards for calibrations, found here:
    https://open-confluence.nrao.edu/pages/viewpage.action?spaceKey=SPR&title=Delivery+Directory+Improvements
    """
    test_context = DeliveryContext.parse_commandline(["-t", str(temp_directory)])

    with patch("delivery.destinations.sharedweb.CapoConfig") as mocked_capo_config:
        mocked_capo_config.return_value.settings.return_value.downloadDirectory = str(temp_directory)
        mocked_capo_config.return_value.settings.return_value.downloadUrl = "http://testing"
        assert str(temp_directory) == mocked_capo_config().settings().downloadDirectory
        results = Delivery().deliver(test_context)
        # TODO: Check that results match the file structure specified in the Confluence page:
        #   <project-code>
        #   └── calibration_pipeline.<pipeline-id-1>.<pipeline-id-2>
        #       |-- <project-code><timestamp>.tar
        #       └── weblog.tgz
        print(results)
    capsys.readouterr()