Delivery rework

Clean up destination code and add some functionality

• Split the destinations into separate files
• New DestinationTempFile for writing tempfiles into the destination which eventually get added
• All of the close() methods are now streaming
• Implemented ChecksumDecorator, so we get a SHA1SUMS file
• Implemented FetchFile decorator, so we get a rudimentary fetch-all.sh script
• Configured Docker and delivery for local development to have a separate volume mount for serving files

apps/cli/executables/delivery/delivery/destinations/checksum.py

+import hashlib
+import pathlib
+
+from .interfaces import DestinationDecorator, Destination
+
+
+class ChecksumDecorator(DestinationDecorator):
+    """
+    Generate a SHA1SUMS file in the content root with an entry for every file that got delivered.
+    """
+
+    def __init__(self, underlying: Destination):
+        super().__init__(underlying)
+        self.sumsfile = underlying.create_file("SHA1SUMS")
+
+    def add_file(self, file: pathlib.Path, relative_path: str):
+        # add the file to the archive the normal way
+        super().add_file(file, relative_path)
+
+        # generate the hash and keep it
+        self.sumsfile.file().writelines([self.hash_file_line(file, relative_path).encode("utf8")])
+
+    def hash_file_line(self, file: pathlib.Path, relative_path: str):
+        """
+        Generate a line for the checksum file.
+
+        :param file:            file to hash
+        :param relative_path:   relative path to show in checksum file
+        :return:                a line like "<hash>  <relative_path>"
+        """
+        return f"{self.hash_file(file)}  {relative_path}\n"
+
+    @staticmethod
+    def hash_file(file: pathlib.Path):
+        """
+        Hash the supplied file
+        :param file: file to hash
+        :return: string in SHA1SUMS format (hexdigest)
+        """
+        # You would expect performance to be worse than calling a C program here, but in my own testing I found
+        # that the ensuing block is somewhat *faster* than calling "shasum" directly. I was able to hash a 1 GB
+        # random file in 1.2 seconds with the following code, versus 1.8 seconds with shasum. This is across about 10
+        # trials of each, with a warm cache in each case. So I would not refactor this code to shell out without
+        # doing more performance tests
+        #
+        # code courtesy of Stack Overflow: https://stackoverflow.com/a/59056796/812818
+        with open(file, "rb") as f:
+            file_hash = hashlib.sha1()
+            while chunk := f.read(8192):
+                file_hash.update(chunk)
+        return file_hash.hexdigest()  # to get a printable str instead of bytes
+
+    def close(self):
+        # first close the hash file
+        self.sumsfile.close()
+
+        # now proceed
+        super().close()

[Janet Goldstein]

should this be pydoc?