diff --git a/apps/cli/executables/delivery/delivery/deliverer.py b/apps/cli/executables/delivery/delivery/deliverer.py
index 73cb3af363f4279dc6251b08f3a90507c0d34785..06ee1101780e017b348824a25340ecb3bc40f321 100644
--- a/apps/cli/executables/delivery/delivery/deliverer.py
+++ b/apps/cli/executables/delivery/delivery/deliverer.py
@@ -5,7 +5,7 @@
# -------------------------------------------------------------------------
import pathlib
-from .destinations.interfaces import *
+from .destinations.interfaces import DestinationDecorator, DeliveryContextIF, Destination
from .destinations.tar import TarArchiver
from .destinations.local import LocalDestination
from .destinations.checksum import ChecksumDecorator
diff --git a/apps/cli/executables/delivery/delivery/delivery.py b/apps/cli/executables/delivery/delivery/delivery.py
index 19fff561235d3aa9a32ce87add823d75f474c145..526b52ec38b271e607f0fcbd7a7e553a840a46ba 100644
--- a/apps/cli/executables/delivery/delivery/delivery.py
+++ b/apps/cli/executables/delivery/delivery/delivery.py
@@ -1,4 +1,6 @@
+import json
import pathlib
+from typing import Dict
from .context import DeliveryContext
from .finder import HeuristicProductFinder, ProductFinder
@@ -23,7 +25,7 @@ class Delivery:
else:
return HeuristicProductFinder(source)
- def deliver(self, context: DeliveryContext):
+ def deliver(self, context: DeliveryContext) -> Dict:
"""
Primary entrypoint to delivery process
@@ -36,17 +38,26 @@ class Delivery:
# loop over the products and deliver them
with context.create_destination() as destination:
for product in finder.find_products():
+ print(f"Delivering {product} to {destination}")
product.deliver_to(destination)
# the last thing we should do is return the URL to the delivered stuff.
# for a local delivery, this will be a file URL; otherwise it will be an http URL.
- return destination.result_url()
+ return destination.results()
def main(args=None):
"""CLI entry point"""
+ # parse the arguments
context = DeliveryContext.parse_commandline(args)
- print(Delivery().deliver(context))
+
+ # perform the delivery
+ delivery = Delivery().deliver(context)
+
+ # write the results to a file
+ with open("delivery.json", "w") as result_file:
+ # indent=2 causes the file to be pretty-printed for humans
+ json.dump(delivery, result_file, indent=2)
if __name__ == "__main__":
diff --git a/apps/cli/executables/delivery/delivery/destinations/checksum.py b/apps/cli/executables/delivery/delivery/destinations/checksum.py
index ab85b8ae42e3c90f9a094b0a67e861aca6a57801..bd909f15de56e0b0190bc2ac29db0f4c5d51da99 100644
--- a/apps/cli/executables/delivery/delivery/destinations/checksum.py
+++ b/apps/cli/executables/delivery/delivery/destinations/checksum.py
@@ -57,3 +57,6 @@ class ChecksumDecorator(DestinationDecorator):
# now proceed
super().close()
+
+ def __str__(self):
+ return str(self.underlying) + " with SHA1SUM"
diff --git a/apps/cli/executables/delivery/delivery/destinations/fetchfile.py b/apps/cli/executables/delivery/delivery/destinations/fetchfile.py
index 089103fa1b8e409e374313781e6998b596877eb6..ebd5bafb73c96b8984f35b75f816964d36816aaf 100644
--- a/apps/cli/executables/delivery/delivery/destinations/fetchfile.py
+++ b/apps/cli/executables/delivery/delivery/destinations/fetchfile.py
@@ -22,9 +22,8 @@ class FetchFileGenerator(DestinationDecorator):
super().add_file(file, relative_path)
# add a line to the fetch script for this file
- self.fetch_script.file().writelines(
- [f"wget {self.underlying.result_url()}/{relative_path}\n".encode("utf8")]
- )
+ url = self.underlying.results()["url"]
+ self.fetch_script.file().writelines([f"wget {url}/{relative_path}\n".encode("utf8")])
def close(self):
# first close the script
@@ -32,3 +31,6 @@ class FetchFileGenerator(DestinationDecorator):
# proceed
super().close()
+
+ def __str__(self):
+ return str(self.underlying) + " with fetch-all.sh"
diff --git a/apps/cli/executables/delivery/delivery/destinations/interfaces.py b/apps/cli/executables/delivery/delivery/destinations/interfaces.py
index 4a3a2ea0dddc007d512d6803126fd28544ce70ac..2df84387158c75218bbe8841edbdecc49da3b4c7 100644
--- a/apps/cli/executables/delivery/delivery/destinations/interfaces.py
+++ b/apps/cli/executables/delivery/delivery/destinations/interfaces.py
@@ -1,6 +1,6 @@
import pathlib
from abc import ABC, abstractmethod
-from typing import BinaryIO
+from typing import BinaryIO, Dict
class DeliveryContextIF(ABC):
@@ -105,10 +105,16 @@ class Destination(ABC):
pass
@abstractmethod
- def result_url(self) -> str:
+ def results(self) -> Dict:
"""
- Returns a URL to the results.
- :return: URL pointing to the results
+ Returns some result information, to be returned to the caller. Expected keys include:
+
+ ``delivered_to``
+ A filesystem location where the delivery placed files
+ ``url``
+ A URL to the delivery location, if it can be accessed via a browser
+
+ :return: result information
"""
pass
@@ -171,9 +177,12 @@ class DestinationDecorator(Destination):
"""
self.underlying.close()
- def result_url(self) -> str:
+ def results(self) -> Dict:
"""
In most cases you should leave this alone unless you want to modify the URL that gets
returned to the console at the end of delivery.
"""
- return self.underlying.result_url()
+ return self.underlying.results()
+
+ def __str__(self):
+ return str(self.underlying)
diff --git a/apps/cli/executables/delivery/delivery/destinations/local.py b/apps/cli/executables/delivery/delivery/destinations/local.py
index 7302e2acc08a083f88993755533232b2afa8cd3b..7e99ba73ed04b104ef0de19cfcf4836720078f87 100644
--- a/apps/cli/executables/delivery/delivery/destinations/local.py
+++ b/apps/cli/executables/delivery/delivery/destinations/local.py
@@ -1,7 +1,7 @@
import os
import shutil
import tempfile
-from typing import BinaryIO
+from typing import BinaryIO, Dict
from .interfaces import Destination, DeliveryContextIF, DestinationTempFile
import pathlib
@@ -50,11 +50,17 @@ class LocalDestination(Destination):
# hand off to the LocalDestinationTempFile, which handles the rest
return LocalDestinationTempFile(self, relative_path, rawfile)
- def result_url(self) -> str:
+ def results(self) -> Dict:
"""
Return a file:/// URL for this location
"""
- return pathlib.Path(self.path.absolute()).as_uri()
+
+ # we could also supply a file:/// URL, which would be constructed like so:
+ # "url": pathlib.Path(self.path.absolute()).as_uri()
+ return {"delivered_to": str(self.path)}
+
+ def __str__(self):
+ return f"local destination {self.path}"
class LocalDestinationTempFile(DestinationTempFile):
diff --git a/apps/cli/executables/delivery/delivery/destinations/sharedweb.py b/apps/cli/executables/delivery/delivery/destinations/sharedweb.py
index bd5b9ee34ad2d1035019a5ab5f17f1fe51764bb2..bb33831aa6ab3e037818daaacaa3af504855ceed 100644
--- a/apps/cli/executables/delivery/delivery/destinations/sharedweb.py
+++ b/apps/cli/executables/delivery/delivery/destinations/sharedweb.py
@@ -1,4 +1,5 @@
import pathlib
+from typing import Dict
from .interfaces import DestinationDecorator, DeliveryContextIF
from .local import LocalDestination
@@ -22,16 +23,19 @@ class SharedWebDestination(DestinationDecorator):
username = "anonymous"
# determine the destination directory
- destination_dir = pathlib.Path(capo.nraoDownloadDirectory) / username / context.token()
+ self.destination_dir = pathlib.Path(capo.nraoDownloadDirectory) / username / context.token()
# LocalDestination won't create a directory, so we'll do that here
- destination_dir.mkdir(parents=True)
+ self.destination_dir.mkdir(parents=True)
# we're ready to build the local destination and thread it through our super constructor
- super().__init__(LocalDestination(context, destination_dir))
+ super().__init__(LocalDestination(context, self.destination_dir))
# generate the download URL for later usage
self.download_url = f"{capo.nraoDownloadUrl}/{username}/{context.token()}"
- def result_url(self) -> str:
- return self.download_url
+ def results(self) -> Dict:
+ return {"delivered_to": str(self.destination_dir), "url": self.download_url}
+
+ def __str__(self):
+ return f"shared web root at {self.destination_dir}"
diff --git a/apps/cli/executables/delivery/delivery/destinations/tar.py b/apps/cli/executables/delivery/delivery/destinations/tar.py
index 141edd7a8f10ff03164d2ad155790522cf1216ee..672d79b9cfdd01b6b8877030c8e2846b1fe6f949 100644
--- a/apps/cli/executables/delivery/delivery/destinations/tar.py
+++ b/apps/cli/executables/delivery/delivery/destinations/tar.py
@@ -55,3 +55,6 @@ class TarArchiver(DestinationDecorator):
# now we can proceed
self.underlying.close()
+
+ def __str__(self):
+ return f"tar archive in " + str(self.underlying)
diff --git a/apps/cli/executables/delivery/delivery/products.py b/apps/cli/executables/delivery/delivery/products.py
index 4a4e46526a1329050bdde54e318248c6f1170004..7f1c5b66f39108d81c8046c91bc7d2b3df13f75d 100644
--- a/apps/cli/executables/delivery/delivery/products.py
+++ b/apps/cli/executables/delivery/delivery/products.py
@@ -45,10 +45,16 @@ class ExecutionBlock(SpooledProduct):
eventually.
"""
+ @property
+ def eb_name(self):
+ return self.path.absolute().name
+
def deliver_to(self, destination: Destination):
- eb_name = self.path.absolute().name
# let's use our directory name as the relative path
- destination.add_directory(self.path, eb_name)
+ destination.add_directory(self.path, self.eb_name)
+
+ def __str__(self):
+ return f"execution block {self.eb_name}"
# Future types of product that might be needed:
diff --git a/apps/cli/executables/delivery/test/test_api.py b/apps/cli/executables/delivery/test/test_api.py
index e23c4c56e0a39cf7335db453d9c00f1efcda5ac7..f275c8d0f26817752247e2217febf7604ed53d2a 100644
--- a/apps/cli/executables/delivery/test/test_api.py
+++ b/apps/cli/executables/delivery/test/test_api.py
@@ -60,6 +60,11 @@ def test_local_delivery_add_file(tmpdir, file_to_deliver: pathlib.Path, dest_dir
# see if the content of the file is intact
assert (dest_dir / file_to_deliver.name).read_text(encoding=None) == "content"
+ # see if the results are alright
+ results = local_dest.results()
+ assert len(results.keys()) == 1
+ assert results["delivered_to"] == str(dest_dir)
+
def test_local_delivery_add_directory(
tmpdir: pathlib.Path, file_to_deliver: pathlib.Path, dest_dir: pathlib.Path
@@ -71,9 +76,13 @@ def test_local_delivery_add_directory(
local_dest.add_directory(file_to_deliver.parent, file_to_deliver.parent.name)
compare_dirs = filecmp.dircmp(file_to_deliver.parent, dest_dir / "prior")
+
# see if the destination got all the files from source
- assert (
- len(compare_dirs.left_only) == 0
- and len(compare_dirs.right_only) == 0
- and len(compare_dirs.funny_files) == 0
- )
+ assert len(compare_dirs.left_only) == 0
+ assert len(compare_dirs.right_only) == 0
+ assert len(compare_dirs.funny_files) == 0
+
+ # see if the results are alright
+ results = local_dest.results()
+ assert len(results.keys()) == 1
+ assert results["delivered_to"] == str(dest_dir)
diff --git a/apps/cli/executables/delivery/test/test_cli.py b/apps/cli/executables/delivery/test/test_cli.py
index c6ab65578fcc103470ca5263c69d63f16ee0951f..98e555a99d3eafc6a668cbe349949948497723f5 100644
--- a/apps/cli/executables/delivery/test/test_cli.py
+++ b/apps/cli/executables/delivery/test/test_cli.py
@@ -1,5 +1,6 @@
# Testing the CLI
import filecmp
+import json
import pathlib
import shutil
import tarfile
@@ -64,9 +65,16 @@ def test_local_rawdata_no_tar(tmpdir_factory):
test_data_path = "../../../../shared/workspaces/test/test_data/spool/724126739/"
eb_name = "17A-109.sb33151331.eb33786546.57892.65940042824"
main(["-r", "-l", temp_directory, test_data_path])
+
# compare the source and destination
assert_directories_are_same(temp_directory + "/" + eb_name, (test_data_path + eb_name))
+ # ensure that we actually got a delivery file with the proper contents
+ with open("delivery.json", "r") as delivery_results_file:
+ results = json.load(delivery_results_file)
+ assert len(results.keys()) == 1
+ assert results["delivered_to"] == temp_directory
+
def test_local_rawdata_with_tar(tmpdir_factory):
"""
@@ -86,6 +94,11 @@ def test_local_rawdata_with_tar(tmpdir_factory):
verify_extracted_directory(eb_name, tar_path, temp_directory, test_data_path)
+ with open("delivery.json", "r") as delivery_results_file:
+ results = json.load(delivery_results_file)
+ assert len(results.keys()) == 1
+ assert results["delivered_to"] == str(temp_directory)
+
# @pytest.mark.skip(reason="Test needs more dev time")
def test_web_rawdata_no_tar(tmpdir_factory):
@@ -100,10 +113,11 @@ def test_web_rawdata_no_tar(tmpdir_factory):
mocked_capo_settings.return_value.nraoDownloadDirectory = str(temp_directory)
mocked_capo_settings.return_value.nraoDownloadUrl = "http://testing"
assert str(temp_directory) == mocked_capo_settings().nraoDownloadDirectory
- destination_url = Delivery().deliver(test_context)
+ results = Delivery().deliver(test_context)
- # determine the destination by looking at the URL
- actual_delivery_dir = temp_directory / destination_url.lstrip("http://testing")
+ # check the relationship between the delivery root and the URL
+ assert str(temp_directory / results["url"].lstrip("http://testing")) == results["delivered_to"]
+ actual_delivery_dir = pathlib.Path(results["delivered_to"])
# compare the source and destination
assert_directories_are_same(actual_delivery_dir / eb_name, f"{test_data_path}{eb_name}")
@@ -121,11 +135,12 @@ def test_web_rawdata_with_tar(tmpdir_factory):
mocked_capo_settings.return_value.nraoDownloadDirectory = temp_directory
mocked_capo_settings.return_value.nraoDownloadUrl = "http://testing"
assert temp_directory == mocked_capo_settings().nraoDownloadDirectory
- destination_url = Delivery().deliver(test_context)
+ results = Delivery().deliver(test_context)
eb_name = "17A-109.sb33151331.eb33786546.57892.65940042824"
- # determine the destination by looking at the URL
- actual_delivery_dir = temp_directory / destination_url.lstrip("http://testing")
+ # check the relationship between the delivery root and the URL
+ assert str(temp_directory / results["url"].lstrip("http://testing")) == results["delivered_to"]
+ actual_delivery_dir = pathlib.Path(results["delivered_to"])
# does a tar exist where we think
tar_path = actual_delivery_dir / (eb_name + ".tar")