toil 9.0.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/lib/io.py

@@ -11,6 +11,8 @@ from contextlib import contextmanager
 from io import BytesIO
 from typing import IO, Any, Callable, Optional, Protocol, Union
 
+from toil.lib.directory import get_directory_item, TOIL_DIR_URI_SCHEME
+from toil.lib.url import URLAccess
 from toil.lib.memoize import memoize
 from toil.lib.misc import StrPath
 
@@ -39,9 +41,8 @@ def get_toil_home() -> str:
 
 TOIL_URI_SCHEME = "toilfile:"
 
-
 STANDARD_SCHEMES = ["http:", "https:", "s3:", "gs:", "ftp:"]
-REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME]
+REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME, TOIL_DIR_URI_SCHEME]
 ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
 
 def is_standard_url(filename: str) -> bool:
@@ -75,11 +76,25 @@ def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
     return False
 
 def is_toil_url(filename: str) -> bool:
+    """
+    Return True if a URL is a toilfile: or toildir: URL.
+    """
+    return is_url_with_scheme(filename, [TOIL_URI_SCHEME, TOIL_DIR_URI_SCHEME])
+
+def is_toil_file_url(filename: str) -> bool:
     """
     Return True if a URL is a toilfile: URL.
     """
     return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
 
+def is_toil_dir_url(filename: str) -> bool:
+    """
+    Return True if a URL is a toildir: URL.
+
+    Note that this may point to either a direcotry or a leaf file.
+    """
+    return is_url_with_scheme(filename, [TOIL_DIR_URI_SCHEME])
+
 def is_file_url(filename: str) -> bool:
     """
     Return True if a URL is a file: URL.
@@ -88,6 +103,22 @@ def is_file_url(filename: str) -> bool:
     """
     return is_url_with_scheme(filename, ["file:"])
 
+def is_directory_url(filename: str) -> bool:
+    """
+    Return True if a URL points to a directory.
+
+    Covers toildir: URLs and deterrmines if they point to directories or leaf
+    files. Also covers other supported remote URL schemes.
+    """
+
+    if is_toil_file_url(filename):
+        # Direct file URLs aren't directories.
+        return False
+    if is_toil_dir_url(filename):
+        # This is a toildir: URL but might be a file or a root or subdirectory.
+        return not isinstance(get_directory_item(filename), str)
+    return URLAccess.get_is_directory(filename)
+
 def mkdtemp(
     suffix: Optional[str] = None,
     prefix: Optional[str] = None,

toil/lib/memoize.py

@@ -60,17 +60,22 @@ def sync_memoize(f: Callable[[MAT], MRT]) -> Callable[[MAT], MRT]:
 
 def parse_iso_utc(s: str) -> datetime.datetime:
     """
-    Parses an ISO time with a hard-coded Z for zulu-time (UTC) at the end. Other timezones are
-    not supported. Returns a timezone-naive datetime object.
+    Parses an RFC 3339 ISO 8601 time in the UTC timezone.
+
+    Other timezones are not supported. Returns a timezone-aware UTC datetime
+    object.
 
     :param s: The ISO-formatted time
 
-    :return: A timezone-naive datetime object
+    :return: A timezone-aware UTC datetime object
+
+    :raises ValueError: if the string is not in the correct format or is not in
+        the UTC timezone.
 
     >>> parse_iso_utc('2016-04-27T00:28:04.000Z')
-    datetime.datetime(2016, 4, 27, 0, 28, 4)
+    datetime.datetime(2016, 4, 27, 0, 28, 4, tzinfo=datetime.timezone.utc)
     >>> parse_iso_utc('2016-04-27T00:28:04Z')
-    datetime.datetime(2016, 4, 27, 0, 28, 4)
+    datetime.datetime(2016, 4, 27, 0, 28, 4, tzinfo=datetime.timezone.utc)
     >>> parse_iso_utc('2016-04-27T00:28:04X')
     Traceback (most recent call last):
     ...
@@ -83,8 +88,17 @@ def parse_iso_utc(s: str) -> datetime.datetime:
     if not m:
         raise ValueError(f"Not a valid ISO datetime in UTC: {s}")
     else:
-        fmt = "%Y-%m-%dT%H:%M:%S" + (".%f" if m.group(7) else "") + "Z"
-        return datetime.datetime.strptime(s, fmt)
+        if m.group(8) != "Z" and not m.group(8).endswith("00:00"):
+            raise ValueError(f"Not in the UTC time zone: {s}")
+        if m.group(8) == "Z":
+            # Convert to an offset for parsing
+            s = s[:-1] + "-00:00"
+
+        fmt = "%Y-%m-%dT%H:%M:%S" + (".%f" if m.group(7) else "") + "%z"
+        parsed = datetime.datetime.strptime(s, fmt)
+        # We should have guaranteed that this is in UTC
+        assert parsed.tzinfo is not None
+        return parsed
 
 
 def strict_bool(s: str) -> bool:

toil/lib/pipes.py

@@ -0,0 +1,385 @@
+import errno
+import logging
+import os
+import hashlib
+import threading
+
+from abc import ABC, abstractmethod
+from typing import Optional, TextIO, BinaryIO, IO, Any
+
+from toil.lib.checksum import ChecksumError
+from toil.lib.threading import ExceptionalThread
+
+log = logging.getLogger(__name__)
+
+
+class WritablePipe(ABC):
+    """
+    An object-oriented wrapper for os.pipe. Clients should subclass it, implement
+    :meth:`.readFrom` to consume the readable end of the pipe, then instantiate the class as a
+    context manager to get the writable end. See the example below.
+
+    >>> import sys, shutil, codecs
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         shutil.copyfileobj(codecs.getreader('utf-8')(readable), sys.stdout)
+    >>> with MyPipe() as writable:
+    ...     _ = writable.write('Hello, world!\\n'.encode('utf-8'))
+    Hello, world!
+
+    Each instance of this class creates a thread and invokes the readFrom method in that thread.
+    The thread will be join()ed upon normal exit from the context manager, i.e. the body of the
+    `with` statement. If an exception occurs, the thread will not be joined but a well-behaved
+    :meth:`.readFrom` implementation will terminate shortly thereafter due to the pipe having
+    been closed.
+
+    Now, exceptions in the reader thread will be reraised in the main thread:
+
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as writable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+
+    More complicated, less illustrative tests:
+
+    Same as above, but proving that handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as writable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+
+    Exceptions in the body of the with statement aren't masked, and handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(WritablePipe):
+    ...     def readFrom(self, readable):
+    ...         pass
+    >>> with MyPipe() as writable:
+    ...     raise RuntimeError('Hello, world!')
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+    """
+
+    def __init__(self, encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+        """
+        The specified encoding and errors apply to the writable end of the pipe.
+
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super().__init__()
+        self.encoding: Optional[str] = encoding
+        self.errors: Optional[str] = errors
+        self.readable_fh: Optional[int] = None
+        self.writable: Optional[IO[Any]] = None
+        self.thread: Optional[ExceptionalThread] = None
+        self.reader_done: bool = False
+
+    def __enter__(self) -> IO[Any]:
+        self.readable_fh, writable_fh = os.pipe()
+        self.writable = os.fdopen(
+            writable_fh,
+            "wb" if self.encoding == None else "wt",
+            encoding=self.encoding,
+            errors=self.errors,
+        )
+        self.thread = ExceptionalThread(target=self._reader)
+        self.thread.start()
+        return self.writable
+
+    def __exit__(self, exc_type: Optional[str], exc_val: Optional[str], exc_tb: Optional[str]) -> None:
+        # Closing the writable end will send EOF to the readable and cause the reader thread
+        # to finish.
+        # TODO: Can close() fail? If so, would we try and clean up after the reader?
+        assert self.writable is not None
+        self.writable.close()
+        try:
+            if self.thread is not None:
+                # reraises any exception that was raised in the thread
+                self.thread.join()
+        except Exception as e:
+            if exc_type is None:
+                # Only raise the child exception if there wasn't
+                # already an exception in the main thread
+                raise
+            else:
+                log.error(
+                    "Swallowing additional exception in reader thread: %s", str(e)
+                )
+        finally:
+            # The responsibility for closing the readable end is generally that of the reader
+            # thread. To cover the small window before the reader takes over we also close it here.
+            # TODO: Does that make any sense?
+            if self.readable_fh is not None:
+                # Close the file handle. The reader thread must be dead now.
+                try:
+                    os.close(self.readable_fh)
+                except OSError as e:
+                    # OSError: [Errno 9] Bad file descriptor implies this file handle is already closed
+                    if not e.errno == errno.EBADF:
+                        raise e
+
+    @abstractmethod
+    def readFrom(self, readable: IO[Any]) -> None:
+        """
+        Implement this method to read data from the pipe. This method should support both
+        binary and text mode output.
+
+        :param file readable: the file object representing the readable end of the pipe. Do not
+            explicitly invoke the close() method of the object; that will be done automatically.
+        """
+        raise NotImplementedError()
+
+    def _reader(self) -> None:
+        assert self.readable_fh is not None
+        with os.fdopen(self.readable_fh, "rb") as readable:
+            # TODO: If the reader somehow crashes here, both threads might try
+            # to close readable_fh.  Fortunately we don't do anything that
+            # should be able to fail here.
+            # TODO: Use a real mutex; this None-flagging logic doesn't seem race-free.
+            self.readable_fh = None  # signal to parent thread that we've taken over
+            self.readFrom(readable)
+            self.reader_done = True
+
+
+
+class ReadablePipe(ABC):
+    """
+    An object-oriented wrapper for os.pipe. Clients should subclass it, implement
+    :meth:`.writeTo` to place data into the writable end of the pipe, then instantiate the class
+    as a context manager to get the writable end. See the example below.
+
+    >>> import sys, shutil, codecs
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable: IO[Any]) -> None:
+    ...         writable.write('Hello, world!\\n'.encode('utf-8'))
+    >>> with MyPipe() as readable:
+    ...     shutil.copyfileobj(codecs.getreader('utf-8')(readable), sys.stdout)
+    Hello, world!
+
+    Each instance of this class creates a thread and invokes the :meth:`.writeTo` method in that
+    thread. The thread will be join()ed upon normal exit from the context manager, i.e. the body
+    of the `with` statement. If an exception occurs, the thread will not be joined but a
+    well-behaved :meth:`.writeTo` implementation will terminate shortly thereafter due to the
+    pipe having been closed.
+
+    Now, exceptions in the reader thread will be reraised in the main thread:
+
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable):
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as readable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+
+    More complicated, less illustrative tests:
+
+    Same as above, but proving that handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable: IO[Any]) -> None:
+    ...         raise RuntimeError('Hello, world!')
+    >>> with MyPipe() as readable:
+    ...     pass
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+
+    Exceptions in the body of the with statement aren't masked, and handles are closed:
+
+    >>> x = os.dup(0); os.close(x)
+    >>> class MyPipe(ReadablePipe):
+    ...     def writeTo(self, writable):
+    ...         pass
+    >>> with MyPipe() as readable:
+    ...     raise RuntimeError('Hello, world!')
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Hello, world!
+    >>> y = os.dup(0); os.close(y); x == y
+    True
+    """
+
+    @abstractmethod
+    def writeTo(self, writable: IO[Any]) -> None:
+        """
+        Implement this method to write data from the pipe. This method should support both
+        binary and text mode input.
+
+        :param file writable: the file object representing the writable end of the pipe. Do not
+            explicitly invoke the close() method of the object, that will be done automatically.
+        """
+        raise NotImplementedError()
+
+    def _writer(self) -> None:
+        assert self.writable_fh is not None
+        try:
+            with os.fdopen(self.writable_fh, "wb") as writable:
+                self.writeTo(writable)
+        except OSError as e:
+            # The other side of the pipe may have been closed by the
+            # reading thread, which is OK.
+            if e.errno != errno.EPIPE:
+                raise
+
+    def __init__(self, encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+        """
+        The specified encoding and errors apply to the readable end of the pipe.
+
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super().__init__()
+        self.encoding: Optional[str] = encoding
+        self.errors: Optional[str] = errors
+        self.writable_fh: Optional[int] = None
+        self.readable: Optional[IO[Any]] = None
+        self.thread: Optional[ExceptionalThread] = None
+
+    def __enter__(self) -> IO[Any]:
+        readable_fh, self.writable_fh = os.pipe()
+        self.readable = os.fdopen(
+            readable_fh,
+            "rb" if self.encoding == None else "rt",
+            encoding=self.encoding,
+            errors=self.errors,
+        )
+        self.thread = ExceptionalThread(target=self._writer)
+        self.thread.start()
+        return self.readable
+
+    def __exit__(self, exc_type: Optional[str], exc_val: Optional[str], exc_tb: Optional[str]) -> None:
+        # Close the read end of the pipe. The writing thread may
+        # still be writing to the other end, but this will wake it up
+        # if that's the case.
+        assert self.readable is not None
+        self.readable.close()
+        try:
+            if self.thread is not None:
+                # reraises any exception that was raised in the thread
+                self.thread.join()
+        except:
+            if exc_type is None:
+                # Only raise the child exception if there wasn't
+                # already an exception in the main thread
+                raise
+
+
+class ReadableTransformingPipe(ReadablePipe):
+    """
+    A pipe which is constructed around a readable stream, and which provides a
+    context manager that gives a readable stream.
+
+    Useful as a base class for pipes which have to transform or otherwise visit
+    bytes that flow through them, instead of just consuming or producing data.
+
+    Clients should subclass it and implement :meth:`.transform`, like so:
+
+    >>> import sys, shutil, codecs
+    >>> class MyPipe(ReadableTransformingPipe):
+    ...     def transform(self, readable, writable):
+    ...         writable.write(readable.read().decode('utf-8').upper().encode('utf-8'))
+    >>> class SourcePipe(ReadablePipe):
+    ...     def writeTo(self, writable):
+    ...         writable.write('Hello, world!\\n'.encode('utf-8'))
+    >>> with SourcePipe() as source:
+    ...     with MyPipe(source) as transformed:
+    ...         shutil.copyfileobj(codecs.getreader('utf-8')(transformed), sys.stdout)
+    HELLO, WORLD!
+
+    The :meth:`.transform` method runs in its own thread, and should move data
+    chunk by chunk instead of all at once. It should finish normally if it
+    encounters either an EOF on the readable, or a :class:`BrokenPipeError` on
+    the writable. This means that it should make sure to actually catch a
+    :class:`BrokenPipeError` when writing.
+
+    See also: :class:`toil.lib.misc.WriteWatchingStream`.
+
+    """
+
+    def __init__(self, source: IO[Any], encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+        """
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super().__init__(encoding=encoding, errors=errors)
+        self.source = source
+
+    @abstractmethod
+    def transform(self, readable: IO[Any], writable: IO[Any]) -> None:
+        """
+        Implement this method to ship data through the pipe.
+
+        :param file readable: the input stream file object to transform.
+
+        :param file writable: the file object representing the writable end of the pipe. Do not
+            explicitly invoke the close() method of the object, that will be done automatically.
+        """
+        raise NotImplementedError()
+
+    def writeTo(self, writable: IO[Any]) -> None:
+        self.transform(self.source, writable)
+
+
+class HashingPipe(ReadableTransformingPipe):
+    """
+    Class which checksums all the data read through it. If it
+    reaches EOF and the checksum isn't correct, raises ChecksumError.
+
+    Assumes info actually has a checksum.
+    """
+    def __init__(self, source: IO[Any], encoding: Optional[str] = None, errors: Optional[str] = None, checksum_to_verify: Optional[str] = None) -> None:
+        """
+        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
+                as for encode(). Defaults to None which represents binary mode.
+
+        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
+                are the same as for open(). Defaults to 'strict' when an encoding is specified.
+        """
+        super(HashingPipe, self).__init__(source=source, encoding=encoding, errors=errors)
+        self.checksum_to_verify = checksum_to_verify
+
+    def transform(self, readable: IO[Any], writable: IO[Any]) -> None:
+        hash_object = hashlib.sha1()
+        contents = readable.read(1024 * 1024)
+        while contents != b'':
+            hash_object.update(contents)
+            try:
+                writable.write(contents)
+            except BrokenPipeError:
+                # Read was stopped early by user code.
+                # Can't check the checksum.
+                return
+            contents = readable.read(1024 * 1024)
+        final_computed_checksum = f'sha1${hash_object.hexdigest()}'
+        if not self.checksum_to_verify == final_computed_checksum:
+            raise ChecksumError(f'Checksum mismatch. Expected: {self.checksum_to_verify} Actual: {final_computed_checksum}')

toil/lib/retry.py

@@ -172,7 +172,7 @@ class ErrorCondition:
 
     def __init__(
         self,
-        error: Optional[Any] = None,
+        error: Optional[type[BaseException]] = None,
         error_codes: list[int] = None,
         boto_error_codes: list[str] = None,
         error_message_must_include: str = None,

toil/lib/threading.py

@@ -226,7 +226,7 @@ class ExceptionalThread(threading.Thread):
         if not self.is_alive() and self.exc_info is not None:
             exc_type, exc_value, traceback = self.exc_info
             self.exc_info = None
-            raise_(exc_type, exc_value, traceback)
+            raise_(exc_type, exc_value, traceback) 
 
 
 def cpu_count() -> int:

toil/lib/web.py

@@ -17,13 +17,12 @@ Contains functions for making web requests with Toil.
 
 All web requests should go through this module, to make sure they use the right
 user agent.
-
+>>> httpserver = getfixture("httpserver")
+>>> handler = httpserver.expect_request("/path").respond_with_json({})
 >>> from toil.lib.web import web_session
->>> web_session.get("https://example.com")
-
+>>> web_session.get(httpserver.url_for("/path"))
+<Response [200]>
 """
-
-import logging
 import requests
 import sys
 

toil/provisioners/__init__.py

@@ -127,8 +127,11 @@ def parse_node_types(
 
     Inputs should look something like this:
 
-    >>> parse_node_types('c5.4xlarge/c5a.4xlarge:0.42,t2.large')
-    [({'c5.4xlarge', 'c5a.4xlarge'}, 0.42), ({'t2.large'}, None)]
+    >>> types = parse_node_types('c5.4xlarge/c5a.4xlarge:0.42,t2.large')
+    >>> sorted(types[0][0]), types[0][1]
+    (['c5.4xlarge', 'c5a.4xlarge'], 0.42)
+    >>> sorted(types[1][0]), types[1][1]
+    (['t2.large'], None)
 
     :param node_type_specs: A string defining node types