# # Copyright (C) 2021 Associated Universities, Inc. Washington DC, USA. # # This file is part of NRAO Workspaces. # # Workspaces is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Workspaces is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Workspaces. If not, see <https://www.gnu.org/licenses/>. import pathlib from abc import ABC, abstractmethod from typing import BinaryIO, Dict, Sequence class DeliveryContextIF(ABC): """ The DeliveryContext is something that is available to destinations during processing for shared utility functions. """ @abstractmethod def token(self) -> str: """ Returns a unique token for this delivery. Guaranteed to be the same across multiple calls. :return: a string token """ pass class DestinationTempFile(ABC): """ A DestinationFile is a file that you can create in the destination. Initially a temporary file, it will be added to the destination when you are finished with it. This is a way to create files you can write to during delivery, which are still routed through the destination mechanism at the end. """ @abstractmethod def close(self): """ Close the file (and add it to the destination at your desired path) """ pass @abstractmethod def file(self) -> BinaryIO: """ Access the raw file for writing :return: a file for writing """ pass @abstractmethod def filename(self) -> str: """ Access the temporary path of this file during construction :return: the path to the temporary file """ pass @abstractmethod def chmod(self, mode): """ Modify the file's permissions during construction Makes use of pathlib.chmod & os.chmod, so the mode parameter should follow their conventions (0o777, for instance) :param mode: parameter passed through to libraries """ pass class Destination(ABC): """ Destinations are locations that files can be copied into. They might not always be on a local disk; FTP or Globus could also be destinations. The destination API is very simply, consisting just of adding files. """ def __init__(self, context: DeliveryContextIF): self.context = context @abstractmethod def add_file(self, file: pathlib.Path, relative_path: str): """ Add a file to the destination at the given relative path. :param file: the file (whose contents we are delivering) :param relative_path: the relative path to that file (in the delivery root) """ pass @abstractmethod def create_file(self, relative_path: str) -> DestinationTempFile: """ Create a file in the destination. When the file is closed, it will be added to the destination via the add_file method at the specified relative path. :param relative_path: the relative path where the file should eventually be placed """ pass def add_path_entry(self, path: pathlib.Path, relative_path: str): """ Add a path entry, without regard for whether it is a file or a directory. :param path: path to add to the destination :param relative_path: relative path to place the path at """ if path.is_file(): self.add_file(path, relative_path) else: self.add_directory(path, relative_path) def add_directory(self, directory: pathlib.Path, relative_path: str): """ Add a directory and its contents recursively to the destination at the given path. Do not override this method! add_file must be called for every file that gets delivered. :param directory: the directory (whose files we will deliver) :param relative_path: the relative path to this directory (in the delivery root) """ for entry in directory.iterdir(): if entry.is_file(): self.add_file(entry, relative_path + "/" + entry.name) else: self.add_directory(entry, relative_path + "/" + entry.name) def close(self): """ Close the destination, signalling to this and possibly destinations in the stack that we are finished adding new files to the destination. """ pass @abstractmethod def results(self) -> Dict: """ Returns some result information, to be returned to the caller. Expected keys include: ``delivered_to`` A filesystem location where the delivery placed files ``url`` A URL to the delivery location, if it can be accessed via a browser :return: result information """ pass def __truediv__(self, path: str) -> "SubdirectoryDecorator": return SubdirectoryDecorator(self, path) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): # ensure that if we are used as a context manager ('with' statement) # that the destinations do get properly closed self.close() def excluding(self, *extensions: str): return ExcludingDecorator(self, extensions) class DestinationDecorator(Destination): """ This is a useful base class for destinations that augment the functionality of other destinations. In general, if you add functionality to a destination, you should do it through the decorator facility and then make corresponding changes to the builder to support it. """ def __init__(self, underlying: Destination): """ Create this destination wrapping an underlying destination. :param underlying: the underlying destination that does real-er work """ self.underlying = underlying super().__init__(underlying.context) def add_file(self, file: pathlib.Path, relative_path: str): """ Add a file to the destination. In most cases, your decorator should intercept this call, do something useful with the file and then propagate it to the underlying destination. :param file: file to add to the destination :param relative_path: path to the file in the result """ self.underlying.add_file(file, relative_path) def create_file(self, relative_path: str) -> DestinationTempFile: """ Create a temporary file with the underlying destination. In most cases your decorator should leave this call alone; if you override, be sure to read the comments in the method and copy the hack. :param relative_path: path to the eventual home of this temporary file :return: a DestinationTempFile that can be written to """ # This is a bit of a hack, but there's only one thing that makes temporary files properly now # and it's necessary for the rest of the machinery to work. Better ideas for solving this are welcome. # if we don't have the following lines of code, adding the tempfile at the end of processing # goes directly to the LocalDestination rather than passing through all the layers of the layer cake temporary_file = self.underlying.create_file(relative_path) temporary_file.destination = self return temporary_file def close(self): """ Close the underlying destination. In most cases, your decorator will want to intercept this call to do any finalization work that might be needed, such as closing DestinationTempFiles. It's essential that you remember to propagate the close call to the underlying destination, or the rest of delivery will fail. """ self.underlying.close() def results(self) -> Dict: """ In most cases you should leave this alone unless you want to modify the URL that gets returned to the console at the end of delivery. """ return self.underlying.results() def __str__(self): return str(self.underlying) class ExcludingDecorator(DestinationDecorator): """ A wrapper for making it easy to omit files with certain extensions during delivery. """ def __init__(self, underlying: Destination, exclude_extensions: Sequence[str]): super().__init__(underlying) self.exclude_extensions = exclude_extensions def add_file(self, file: pathlib.Path, relative_path: str): if file.suffix not in self.exclude_extensions: self.underlying.add_file(file, relative_path) class SubdirectoryDecorator(DestinationDecorator): """ A wrapper for making it easy to descend to subdirectories during delivery. """ def __init__(self, underlying: Destination, subdirectory: str): super().__init__(underlying) self.subdirectory = subdirectory def add_file(self, file: pathlib.Path, relative_path: str): self.underlying.add_file(file, self.subdirectory + "/" + relative_path) def __str__(self): return str(self.underlying) + "/" + self.subdirectory