toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/lib/io.py

@@ -11,6 +11,8 @@ from contextlib import contextmanager
 from io import BytesIO
 from typing import IO, Any, Callable, Optional, Protocol, Union
 
+from toil.lib.directory import get_directory_item, TOIL_DIR_URI_SCHEME
+from toil.lib.url import URLAccess
 from toil.lib.memoize import memoize
 from toil.lib.misc import StrPath
 
@@ -39,9 +41,8 @@ def get_toil_home() -> str:
 
 TOIL_URI_SCHEME = "toilfile:"
 
-
 STANDARD_SCHEMES = ["http:", "https:", "s3:", "gs:", "ftp:"]
-REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME]
+REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME, TOIL_DIR_URI_SCHEME]
 ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
 
 def is_standard_url(filename: str) -> bool:
@@ -75,11 +76,25 @@ def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
     return False
 
 def is_toil_url(filename: str) -> bool:
+    """
+    Return True if a URL is a toilfile: or toildir: URL.
+    """
+    return is_url_with_scheme(filename, [TOIL_URI_SCHEME, TOIL_DIR_URI_SCHEME])
+
+def is_toil_file_url(filename: str) -> bool:
     """
     Return True if a URL is a toilfile: URL.
     """
     return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
 
+def is_toil_dir_url(filename: str) -> bool:
+    """
+    Return True if a URL is a toildir: URL.
+
+    Note that this may point to either a direcotry or a leaf file.
+    """
+    return is_url_with_scheme(filename, [TOIL_DIR_URI_SCHEME])
+
 def is_file_url(filename: str) -> bool:
     """
     Return True if a URL is a file: URL.
@@ -88,6 +103,22 @@ def is_file_url(filename: str) -> bool:
     """
     return is_url_with_scheme(filename, ["file:"])
 
+def is_directory_url(filename: str) -> bool:
+    """
+    Return True if a URL points to a directory.
+
+    Covers toildir: URLs and deterrmines if they point to directories or leaf
+    files. Also covers other supported remote URL schemes.
+    """
+
+    if is_toil_file_url(filename):
+        # Direct file URLs aren't directories.
+        return False
+    if is_toil_dir_url(filename):
+        # This is a toildir: URL but might be a file or a root or subdirectory.
+        return not isinstance(get_directory_item(filename), str)
+    return URLAccess.get_is_directory(filename)
+
 def mkdtemp(
     suffix: Optional[str] = None,
     prefix: Optional[str] = None,

toil/lib/memoize.py

@@ -60,17 +60,22 @@ def sync_memoize(f: Callable[[MAT], MRT]) -> Callable[[MAT], MRT]:
 
 def parse_iso_utc(s: str) -> datetime.datetime:
     """
-    Parses an ISO time with a hard-coded Z for zulu-time (UTC) at the end. Other timezones are
-    not supported. Returns a timezone-naive datetime object.
+    Parses an RFC 3339 ISO 8601 time in the UTC timezone.
+
+    Other timezones are not supported. Returns a timezone-aware UTC datetime
+    object.
 
     :param s: The ISO-formatted time
 
-    :return: A timezone-naive datetime object
+    :return: A timezone-aware UTC datetime object
+
+    :raises ValueError: if the string is not in the correct format or is not in
+        the UTC timezone.
 
     >>> parse_iso_utc('2016-04-27T00:28:04.000Z')
-    datetime.datetime(2016, 4, 27, 0, 28, 4)
+    datetime.datetime(2016, 4, 27, 0, 28, 4, tzinfo=datetime.timezone.utc)
     >>> parse_iso_utc('2016-04-27T00:28:04Z')
-    datetime.datetime(2016, 4, 27, 0, 28, 4)
+    datetime.datetime(2016, 4, 27, 0, 28, 4, tzinfo=datetime.timezone.utc)
     >>> parse_iso_utc('2016-04-27T00:28:04X')
     Traceback (most recent call last):
     ...
@@ -83,8 +88,17 @@ def parse_iso_utc(s: str) -> datetime.datetime:
     if not m:
         raise ValueError(f"Not a valid ISO datetime in UTC: {s}")
     else:
-        fmt = "%Y-%m-%dT%H:%M:%S" + (".%f" if m.group(7) else "") + "Z"
-        return datetime.datetime.strptime(s, fmt)
+        if m.group(8) != "Z" and not m.group(8).endswith("00:00"):
+            raise ValueError(f"Not in the UTC time zone: {s}")
+        if m.group(8) == "Z":
+            # Convert to an offset for parsing
+            s = s[:-1] + "-00:00"
+
+        fmt = "%Y-%m-%dT%H:%M:%S" + (".%f" if m.group(7) else "") + "%z"
+        parsed = datetime.datetime.strptime(s, fmt)
+        # We should have guaranteed that this is in UTC
+        assert parsed.tzinfo is not None
+        return parsed
 
 
 def strict_bool(s: str) -> bool:

toil/lib/misc.py

@@ -27,7 +27,7 @@ def get_public_ip() -> str:
     try:
         # Try to get the internet-facing IP by attempting a connection
         # to a non-existent server and reading what IP was used.
-        ip = "127.0.0.1"
+        ip: str = "127.0.0.1"
         with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:
             # 203.0.113.0/24 is reserved as TEST-NET-3 by RFC 5737, so
             # there is guaranteed to be no one listening on the other

toil/lib/pipes.py

@@ -0,0 +1,385 @@
+import errno
+import logging
+import os
+import hashlib
+import threading
+
+from abc import ABC, abstractmethod
+from typing import Optional, TextIO, BinaryIO, IO, Any
+
+from toil.lib.checksum import ChecksumError
+from toil.lib.threading import ExceptionalThread
+
+log = logging.getLogger(__name__)
+
+
+class WritablePipe(ABC):
+    """
+    An object-oriented wrapper for os.pipe. Clients should subclass it, implement
+    :meth:`.readFrom` to consume the readable end of the pipe, then instantiate the class as a
+    context manager to get the writable end. See the example below.
+
+    >>> import sys, shutil, codecs
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         shutil.copyfileobj(codecs.getreader('utf-8')(readable), sys.stdout)
+    >>> with MyPipe() as writable:
+    ...     _ = writable.write('Hello, world!\\n'.encode('utf-8'))
+    Hello, world!
+
+    Each instance of this class creates a thread and invokes the readFrom method in that thread.
+    The thread will be join()ed upon normal exit from the context manager, i.e. the body of the
+    `with` statement. If an exception occurs, the thread will not be joined but a well-behaved
+    :meth:`.readFrom` implementation will terminate shortly thereafter due to the pipe having
+    been closed.
+
+    Now, exceptions in the reader thread will be reraised in the main thread:
+
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as writable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+
+    More complicated, less illustrative tests:
+
+    Same as above, but proving that handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as writable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+
+    Exceptions in the body of the with statement aren't masked, and handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         pass
+    >>> with MyPipe() as writable:
+    ...     raise RuntimeError('Hello, world!')
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+    """
+
+    def __init__(self, encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+        """
+        The specified encoding and errors apply to the writable end of the pipe.
+
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super().__init__()
+        self.encoding: Optional[str] = encoding
+        self.errors: Optional[str] = errors
+        self.readable_fh: Optional[int] = None
+        self.writable: Optional[IO[Any]] = None
+        self.thread: Optional[ExceptionalThread] = None
+        self.reader_done: bool = False
+
+    def __enter__(self) -> IO[Any]:
+        self.readable_fh, writable_fh = os.pipe()
+        self.writable = os.fdopen(
+            writable_fh,
+            "wb" if self.encoding == None else "wt",
+            encoding=self.encoding,
+            errors=self.errors,
+        )
+        self.thread = ExceptionalThread(target=self._reader)
+        self.thread.start()
+        return self.writable
+
+    def __exit__(self, exc_type: Optional[str], exc_val: Optional[str], exc_tb: Optional[str]) -> None:
+        # Closing the writable end will send EOF to the readable and cause the reader thread
+        # to finish.
+        # TODO: Can close() fail? If so, would we try and clean up after the reader?
+        assert self.writable is not None
+        self.writable.close()
+        try:
+            if self.thread is not None:
+                # reraises any exception that was raised in the thread
+                self.thread.join()
+        except Exception as e:
+            if exc_type is None:
+                # Only raise the child exception if there wasn't
+                # already an exception in the main thread
+                raise
+            else:
+                log.error(
+                    "Swallowing additional exception in reader thread: %s", str(e)
+                )
+        finally:
+            # The responsibility for closing the readable end is generally that of the reader
+            # thread. To cover the small window before the reader takes over we also close it here.
+            # TODO: Does that make any sense?
+            if self.readable_fh is not None:
+                # Close the file handle. The reader thread must be dead now.
+                try:
+                    os.close(self.readable_fh)
+                except OSError as e:
+                    # OSError: [Errno 9] Bad file descriptor implies this file handle is already closed
+                    if not e.errno == errno.EBADF:
+                        raise e
+
+    @abstractmethod
+    def readFrom(self, readable: IO[Any]) -> None:
+        """
+        Implement this method to read data from the pipe. This method should support both
+        binary and text mode output.
+
+        :param file readable: the file object representing the readable end of the pipe. Do not
+            explicitly invoke the close() method of the object; that will be done automatically.
+        """
+        raise NotImplementedError()
+
+    def _reader(self) -> None:
+        assert self.readable_fh is not None
+        with os.fdopen(self.readable_fh, "rb") as readable:
+            # TODO: If the reader somehow crashes here, both threads might try
+            # to close readable_fh.  Fortunately we don't do anything that
+            # should be able to fail here.
+            # TODO: Use a real mutex; this None-flagging logic doesn't seem race-free.
+            self.readable_fh = None  # signal to parent thread that we've taken over
+            self.readFrom(readable)
+            self.reader_done = True
+
+
+
+class ReadablePipe(ABC):
+    """
+    An object-oriented wrapper for os.pipe. Clients should subclass it, implement
+    :meth:`.writeTo` to place data into the writable end of the pipe, then instantiate the class
+    as a context manager to get the writable end. See the example below.
+
+    >>> import sys, shutil, codecs
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable: IO[Any]) -> None:
+    ...         writable.write('Hello, world!\\n'.encode('utf-8'))
+    >>> with MyPipe() as readable:
+    ...     shutil.copyfileobj(codecs.getreader('utf-8')(readable), sys.stdout)
+    Hello, world!
+
+    Each instance of this class creates a thread and invokes the :meth:`.writeTo` method in that
+    thread. The thread will be join()ed upon normal exit from the context manager, i.e. the body
+    of the `with` statement. If an exception occurs, the thread will not be joined but a
+    well-behaved :meth:`.writeTo` implementation will terminate shortly thereafter due to the
+    pipe having been closed.
+
+    Now, exceptions in the reader thread will be reraised in the main thread:
+
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable):
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as readable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+
+    More complicated, less illustrative tests:
+
+    Same as above, but proving that handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable: IO[Any]) -> None:
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as readable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+
+    Exceptions in the body of the with statement aren't masked, and handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable):
+    ...         pass
+    >>> with MyPipe() as readable:
+    ...     raise RuntimeError('Hello, world!')
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+    """
+
+    @abstractmethod
+    def writeTo(self, writable: IO[Any]) -> None:
+        """
+        Implement this method to write data from the pipe. This method should support both
+        binary and text mode input.
+
+        :param file writable: the file object representing the writable end of the pipe. Do not
+            explicitly invoke the close() method of the object, that will be done automatically.
+        """
+        raise NotImplementedError()
+
+    def _writer(self) -> None:
+        assert self.writable_fh is not None
+        try:
+            with os.fdopen(self.writable_fh, "wb") as writable:
+                self.writeTo(writable)
+        except OSError as e:
+            # The other side of the pipe may have been closed by the
+            # reading thread, which is OK.
+            if e.errno != errno.EPIPE:
+                raise
+
+    def __init__(self, encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+        """
+        The specified encoding and errors apply to the readable end of the pipe.
+
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super().__init__()
+        self.encoding: Optional[str] = encoding
+        self.errors: Optional[str] = errors
+        self.writable_fh: Optional[int] = None
+        self.readable: Optional[IO[Any]] = None
+        self.thread: Optional[ExceptionalThread] = None
+
+    def __enter__(self) -> IO[Any]:
+        readable_fh, self.writable_fh = os.pipe()
+        self.readable = os.fdopen(
+            readable_fh,
+            "rb" if self.encoding == None else "rt",
+            encoding=self.encoding,
+            errors=self.errors,
+        )
+        self.thread = ExceptionalThread(target=self._writer)
+        self.thread.start()
+        return self.readable
+
+    def __exit__(self, exc_type: Optional[str], exc_val: Optional[str], exc_tb: Optional[str]) -> None:
+        # Close the read end of the pipe. The writing thread may
+        # still be writing to the other end, but this will wake it up
+        # if that's the case.
+        assert self.readable is not None
+        self.readable.close()
+        try:
+            if self.thread is not None:
+                # reraises any exception that was raised in the thread
+                self.thread.join()
+        except:
+            if exc_type is None:
+                # Only raise the child exception if there wasn't
+                # already an exception in the main thread
+                raise
+
+
+class ReadableTransformingPipe(ReadablePipe):
+    """
+    A pipe which is constructed around a readable stream, and which provides a
+    context manager that gives a readable stream.
+
+    Useful as a base class for pipes which have to transform or otherwise visit
+    bytes that flow through them, instead of just consuming or producing data.
+
+    Clients should subclass it and implement :meth:`.transform`, like so:
+
+    >>> import sys, shutil, codecs
+    >>> class MyPipe(ReadableTransformingPipe):
+    ...     def transform(self, readable, writable):
+    ...         writable.write(readable.read().decode('utf-8').upper().encode('utf-8'))
+    >>> class SourcePipe(ReadablePipe):
+    ...     def writeTo(self, writable):
+    ...         writable.write('Hello, world!\\n'.encode('utf-8'))
+    >>> with SourcePipe() as source:
+    ...     with MyPipe(source) as transformed:
+    ...         shutil.copyfileobj(codecs.getreader('utf-8')(transformed), sys.stdout)
+    HELLO, WORLD!
+
+    The :meth:`.transform` method runs in its own thread, and should move data
+    chunk by chunk instead of all at once. It should finish normally if it
+    encounters either an EOF on the readable, or a :class:`BrokenPipeError` on
+    the writable. This means that it should make sure to actually catch a
+    :class:`BrokenPipeError` when writing.
+
+    See also: :class:`toil.lib.misc.WriteWatchingStream`.
+
+    """
+
+    def __init__(self, source: IO[Any], encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+        """
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super().__init__(encoding=encoding, errors=errors)
+        self.source = source
+
+    @abstractmethod
+    def transform(self, readable: IO[Any], writable: IO[Any]) -> None:
+        """
+        Implement this method to ship data through the pipe.
+
+        :param file readable: the input stream file object to transform.
+
+        :param file writable: the file object representing the writable end of the pipe. Do not
+            explicitly invoke the close() method of the object, that will be done automatically.
+        """
+        raise NotImplementedError()
+
+    def writeTo(self, writable: IO[Any]) -> None:
+        self.transform(self.source, writable)
+
+
+class HashingPipe(ReadableTransformingPipe):
+    """
+    Class which checksums all the data read through it. If it
+    reaches EOF and the checksum isn't correct, raises ChecksumError.
+
+    Assumes info actually has a checksum.
+    """
+    def __init__(self, source: IO[Any], encoding: Optional[str] = None, errors: Optional[str] = None, checksum_to_verify: Optional[str] = None) -> None:
+        """
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super(HashingPipe, self).__init__(source=source, encoding=encoding, errors=errors)
+        self.checksum_to_verify = checksum_to_verify
+
+    def transform(self, readable: IO[Any], writable: IO[Any]) -> None:
+        hash_object = hashlib.sha1()
+        contents = readable.read(1024 * 1024)
+        while contents != b'':
+            hash_object.update(contents)
+            try:
+                writable.write(contents)
+            except BrokenPipeError:
+                # Read was stopped early by user code.
+                # Can't check the checksum.
+                return
+            contents = readable.read(1024 * 1024)
+        final_computed_checksum = f'sha1${hash_object.hexdigest()}'
+        if not self.checksum_to_verify == final_computed_checksum:
+            raise ChecksumError(f'Checksum mismatch. Expected: {self.checksum_to_verify} Actual: {final_computed_checksum}')

toil/lib/plugins.py

@@ -0,0 +1,106 @@
+# Copyright (C) 2015-2025 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generic plugin system for Toil plugins.
+
+Plugins come in Python packages named::
+
+    toil_{PLUGIN_TYPE}_{WHATEVER}
+
+When looking for plugins, Toil will list all the Python packages with the right
+name prefix for the given type of plugin, and load them. The plugin modules
+then have an opportunity to import :meth:`register_plugin` and register
+themselves.
+"""
+
+import importlib
+from typing import Any, Literal, Union
+import pkgutil
+from toil.lib.memoize import memoize
+
+
+PluginType = Union[Literal["batch_system"], Literal["url_access"]]
+plugin_types: list[PluginType] = ["batch_system", "url_access"]
+
+_registry: dict[str, dict[str, Any]] = {k: {} for k in plugin_types}
+
+def register_plugin(
+    plugin_type: PluginType, plugin_name: str, plugin_being_registered: Any
+) -> None:
+    """
+    Adds a plugin to the registry for the given type of plugin.
+
+    :param plugin_name: For batch systems, this is the string the user will use
+        to select the batch system on the command line with ``--batchSystem``.
+        For URL access plugins, this is the URL scheme that the plugin
+        implements.
+    :param plugin_being_registered: This is a function that, when called,
+        imports and returns a plugin-provided class type. For batch systems,
+        the resulting type must extend
+        :class:`toil.batchSystems.abstractBatchSystem.AbstractBatchSystem`. For
+        URL access plugins, it must extend :class:`toil.lib.url.URLAccess`.
+        Note that the function used here should return the class itslef; it
+        should not construct an instance of the class.
+    """
+    _registry[plugin_type][plugin_name] = plugin_being_registered
+
+def remove_plugin(
+    plugin_type: PluginType, plugin_name: str) -> None:
+    """
+    Removes a plugin from the registry for the given type of plugin.
+    """
+    try:
+        del _registry[plugin_type][plugin_name]
+    except KeyError:
+        # If the plugin does not exist, it can be ignored
+        pass
+
+def get_plugin_names(plugin_type:PluginType) -> list[str]:
+    """
+    Get the names of all the available plugins of the given type.
+    """
+    _load_all_plugins(plugin_type)
+    return list(_registry[plugin_type].keys())
+
+def get_plugin(plugin_type: PluginType, plugin_name: str) -> Any:
+    """
+    Get a plugin class factory function by name.
+
+    :raises: KeyError if plugin_name is not the name of a plugin of the given
+        type.
+    """
+    _load_all_plugins(plugin_type)
+    return _registry[plugin_type][plugin_name]
+
+
+def _plugin_name_prefix(plugin_type: PluginType) -> str:
+    """
+    Get prefix for plugin type. 
+    
+    Any packages with prefix will count as toil plugins of that type.
+    """
+    return f"toil_{plugin_type}_"
+
+@memoize
+def _load_all_plugins(plugin_type: PluginType) -> None:
+    """
+    Load all the plugins of the given type that are installed.
+    """
+    prefix = _plugin_name_prefix(plugin_type)
+    for finder, name, is_pkg in pkgutil.iter_modules():
+        # For all installed packages
+        if name.startswith(prefix):
+            # If it is a Toil batch system plugin, import it
+            importlib.import_module(name)

toil/lib/retry.py

@@ -172,7 +172,7 @@ class ErrorCondition:
 
     def __init__(
         self,
-        error: Optional[Any] = None,
+        error: Optional[type[BaseException]] = None,
         error_codes: list[int] = None,
         boto_error_codes: list[str] = None,
         error_message_must_include: str = None,

toil/lib/threading.py

@@ -226,7 +226,7 @@ class ExceptionalThread(threading.Thread):
         if not self.is_alive() and self.exc_info is not None:
             exc_type, exc_value, traceback = self.exc_info
             self.exc_info = None
-            raise_(exc_type, exc_value, traceback)
+            raise_(exc_type, exc_value, traceback) 
 
 
 def cpu_count() -> int: