toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

toil/lib/io.py

@@ -6,15 +6,15 @@ import stat
 import sys
 import tempfile
 import uuid
-from collections.abc import Iterator, Iterable
+from collections.abc import Callable, Iterable, Iterator
 from contextlib import contextmanager
 from io import BytesIO
-from typing import IO, Any, Callable, Optional, Protocol, Union
+from typing import IO, Any, Protocol
 
-from toil.lib.directory import get_directory_item, TOIL_DIR_URI_SCHEME
-from toil.lib.url import URLAccess
+from toil.lib.directory import TOIL_DIR_URI_SCHEME, get_directory_item
 from toil.lib.memoize import memoize
 from toil.lib.misc import StrPath
+from toil.lib.url import URLAccess
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +27,7 @@ def get_toil_home() -> str:
     Raises an error if it does not exist and cannot be created. Safe to run
     simultaneously in multiple processes.
     """
-   
+
     # TODO: should this use an XDG config directory or ~/.config to not clutter the
     # base home directory?
     toil_home_dir = os.path.join(os.path.expanduser("~"), ".toil")
@@ -39,24 +39,28 @@ def get_toil_home() -> str:
         )
     return dir_path
 
+
 TOIL_URI_SCHEME = "toilfile:"
 
 STANDARD_SCHEMES = ["http:", "https:", "s3:", "gs:", "ftp:"]
 REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME, TOIL_DIR_URI_SCHEME]
 ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
 
+
 def is_standard_url(filename: str) -> bool:
     """
     Return True if the given URL is a non-Toil, non-file: URL.
     """
     return is_url_with_scheme(filename, STANDARD_SCHEMES)
 
+
 def is_remote_url(filename: str) -> bool:
     """
     Decide if a filename is a known, non-file kind of URL
     """
     return is_url_with_scheme(filename, REMOTE_SCHEMES)
 
+
 def is_any_url(filename: str) -> bool:
     """
     Decide if a string is a URI like http:// or file://.
@@ -65,6 +69,7 @@ def is_any_url(filename: str) -> bool:
     """
     return is_url_with_scheme(filename, ALL_SCHEMES)
 
+
 def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
     """
     Return True if filename is a URL with any of the given schemes and False otherwise.
@@ -75,26 +80,30 @@ def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
             return True
     return False
 
+
 def is_toil_url(filename: str) -> bool:
     """
     Return True if a URL is a toilfile: or toildir: URL.
     """
     return is_url_with_scheme(filename, [TOIL_URI_SCHEME, TOIL_DIR_URI_SCHEME])
 
+
 def is_toil_file_url(filename: str) -> bool:
     """
     Return True if a URL is a toilfile: URL.
     """
     return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
 
+
 def is_toil_dir_url(filename: str) -> bool:
     """
     Return True if a URL is a toildir: URL.
 
-    Note that this may point to either a direcotry or a leaf file.
+    Note that this may point to either a directory or a leaf file.
     """
     return is_url_with_scheme(filename, [TOIL_DIR_URI_SCHEME])
 
+
 def is_file_url(filename: str) -> bool:
     """
     Return True if a URL is a file: URL.
@@ -103,6 +112,7 @@ def is_file_url(filename: str) -> bool:
     """
     return is_url_with_scheme(filename, ["file:"])
 
+
 def is_directory_url(filename: str) -> bool:
     """
     Return True if a URL points to a directory.
@@ -119,10 +129,11 @@ def is_directory_url(filename: str) -> bool:
         return not isinstance(get_directory_item(filename), str)
     return URLAccess.get_is_directory(filename)
 
+
 def mkdtemp(
-    suffix: Optional[str] = None,
-    prefix: Optional[str] = None,
-    dir: Optional[StrPath] = None,
+    suffix: str | None = None,
+    prefix: str | None = None,
+    dir: StrPath | None = None,
 ) -> str:
     """
     Make a temporary directory like tempfile.mkdtemp, but with relaxed permissions.
@@ -147,7 +158,7 @@ def mkdtemp(
     return result
 
 
-def robust_rmtree(path: Union[str, bytes]) -> None:
+def robust_rmtree(path: str | bytes) -> None:
     """
     Robustly tries to delete paths.
 
@@ -257,9 +268,7 @@ def AtomicFileCreate(final_path: StrPath, keep: bool = False) -> Iterator[str]:
         raise
 
 
-def atomic_copy(
-    src_path: str, dest_path: str, executable: Optional[bool] = None
-) -> None:
+def atomic_copy(src_path: str, dest_path: str, executable: bool | None = None) -> None:
     """Copy a file using posix atomic creations semantics."""
     if executable is None:
         executable = os.stat(src_path).st_mode & stat.S_IXUSR != 0
@@ -280,7 +289,7 @@ def atomic_copyobj(
             os.chmod(dest_path_tmp, os.stat(dest_path_tmp).st_mode | stat.S_IXUSR)
 
 
-def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) -> str:
+def make_public_dir(in_directory: str, suggested_name: str | None = None) -> str:
     """
     Make a publicly-accessible directory in the given directory.
 
@@ -318,7 +327,7 @@ def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) ->
     return this_should_never_happen
 
 
-def try_path(path: str, min_size: int = 100 * 1024 * 1024) -> Optional[str]:
+def try_path(path: str, min_size: int = 100 * 1024 * 1024) -> str | None:
     """
     Try to use the given path. Return it if it exists or can be made,
     and we can make things within it, or None otherwise.
@@ -416,6 +425,7 @@ class WriteWatchingStream:
 
         self.backingStream.close()
 
+
 class ReadableFileObj(Protocol):
     """
     Protocol that is more specific than what file_digest takes as an argument.
@@ -423,10 +433,12 @@ class ReadableFileObj(Protocol):
     Would extend the protocol from Typeshed for hashlib but those are only
     declared for 3.11+.
     """
+
     def readinto(self, buf: bytearray, /) -> int: ...
     def readable(self) -> bool: ...
     def read(self, number: int) -> bytes: ...
 
+
 # hashlib._Hash seems to not appear at runtime
 def file_digest(f: ReadableFileObj, alg_name: str) -> "hashlib._Hash":
     """
@@ -441,5 +453,3 @@ def file_digest(f: ReadableFileObj, alg_name: str) -> "hashlib._Hash":
         hasher.update(buffer)
         buffer = f.read(BUFFER_SIZE)
     return hasher
-
-

toil/lib/memoize.py

@@ -15,9 +15,10 @@
 # 5.14.2018: copied into Toil from https://github.com/BD2KGenomics/bd2k-python-lib
 import datetime
 import re
+from collections.abc import Callable
 from functools import lru_cache, wraps
 from threading import Lock
-from typing import Any, Callable, TypeVar
+from typing import Any, TypeVar
 
 memoize = lru_cache(maxsize=None)
 """

toil/lib/misc.py

@@ -9,17 +9,14 @@ import sys
 import time
 from collections.abc import Iterator
 from contextlib import closing
-from typing import Optional, Union
-if sys.version_info >= (3, 10):
-    from typing import TypeAlias
-else:
-    from typing_extensions import TypeAlias
+from typing import TypeAlias, Union
 
 logger = logging.getLogger(__name__)
 
 StrPath: TypeAlias = Union[str, os.PathLike[str]]
 FileDescriptorOrPath: TypeAlias = Union[int, bytes, os.PathLike[bytes], StrPath]
 
+
 def get_public_ip() -> str:
     """Get the IP that this machine uses to contact the internet.
 
@@ -69,17 +66,24 @@ def unix_now_ms() -> float:
     """Return the current time in milliseconds since the Unix epoch."""
     return time.time() * 1000
 
+
 def unix_seconds_to_timestamp(timestamp: float) -> str:
     """
     Convert a time in seconds since the Unix epoch to an ISO 8601 string.
     """
-    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
+    return datetime.datetime.fromtimestamp(
+        timestamp, tz=datetime.timezone.utc
+    ).isoformat()
+
 
 def unix_seconds_to_local_time(timestamp: float) -> datetime.datetime:
     """
     Returns a local time corresponding to the given number of seconds since the Unix epoch.
     """
-    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).astimezone()
+    return datetime.datetime.fromtimestamp(
+        timestamp, tz=datetime.timezone.utc
+    ).astimezone()
+
 
 def seconds_to_duration(time_difference: float) -> str:
     """
@@ -157,11 +161,11 @@ class CalledProcessErrorStderr(subprocess.CalledProcessError):
 def call_command(
     cmd: list[str],
     *args: str,
-    input: Optional[str] = None,
-    timeout: Optional[float] = None,
+    input: str | None = None,
+    timeout: float | None = None,
     useCLocale: bool = True,
-    env: Optional[dict[str, str]] = None,
-    quiet: Optional[bool] = False
+    env: dict[str, str] | None = None,
+    quiet: bool | None = False,
 ) -> str:
     """
     Simplified calling of external commands.

toil/lib/pipes.py

@@ -1,11 +1,9 @@
 import errno
+import hashlib
 import logging
 import os
-import hashlib
-import threading
-
 from abc import ABC, abstractmethod
-from typing import Optional, TextIO, BinaryIO, IO, Any
+from typing import IO, Any
 
 from toil.lib.checksum import ChecksumError
 from toil.lib.threading import ExceptionalThread
@@ -75,7 +73,7 @@ class WritablePipe(ABC):
     True
     """
 
-    def __init__(self, encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+    def __init__(self, encoding: str | None = None, errors: str | None = None) -> None:
         """
         The specified encoding and errors apply to the writable end of the pipe.
 
@@ -86,11 +84,11 @@ class WritablePipe(ABC):
                 are the same as for open(). Defaults to 'strict' when an encoding is specified.
         """
         super().__init__()
-        self.encoding: Optional[str] = encoding
-        self.errors: Optional[str] = errors
-        self.readable_fh: Optional[int] = None
-        self.writable: Optional[IO[Any]] = None
-        self.thread: Optional[ExceptionalThread] = None
+        self.encoding: str | None = encoding
+        self.errors: str | None = errors
+        self.readable_fh: int | None = None
+        self.writable: IO[Any] | None = None
+        self.thread: ExceptionalThread | None = None
         self.reader_done: bool = False
 
     def __enter__(self) -> IO[Any]:
@@ -105,7 +103,9 @@ class WritablePipe(ABC):
         self.thread.start()
         return self.writable
 
-    def __exit__(self, exc_type: Optional[str], exc_val: Optional[str], exc_tb: Optional[str]) -> None:
+    def __exit__(
+        self, exc_type: str | None, exc_val: str | None, exc_tb: str | None
+    ) -> None:
         # Closing the writable end will send EOF to the readable and cause the reader thread
         # to finish.
         # TODO: Can close() fail? If so, would we try and clean up after the reader?
@@ -160,7 +160,6 @@ class WritablePipe(ABC):
             self.reader_done = True
 
 
-
 class ReadablePipe(ABC):
     """
     An object-oriented wrapper for os.pipe. Clients should subclass it, implement
@@ -245,7 +244,7 @@ class ReadablePipe(ABC):
             if e.errno != errno.EPIPE:
                 raise
 
-    def __init__(self, encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+    def __init__(self, encoding: str | None = None, errors: str | None = None) -> None:
         """
         The specified encoding and errors apply to the readable end of the pipe.
 
@@ -256,11 +255,11 @@ class ReadablePipe(ABC):
                 are the same as for open(). Defaults to 'strict' when an encoding is specified.
         """
         super().__init__()
-        self.encoding: Optional[str] = encoding
-        self.errors: Optional[str] = errors
-        self.writable_fh: Optional[int] = None
-        self.readable: Optional[IO[Any]] = None
-        self.thread: Optional[ExceptionalThread] = None
+        self.encoding: str | None = encoding
+        self.errors: str | None = errors
+        self.writable_fh: int | None = None
+        self.readable: IO[Any] | None = None
+        self.thread: ExceptionalThread | None = None
 
     def __enter__(self) -> IO[Any]:
         readable_fh, self.writable_fh = os.pipe()
@@ -274,7 +273,9 @@ class ReadablePipe(ABC):
         self.thread.start()
         return self.readable
 
-    def __exit__(self, exc_type: Optional[str], exc_val: Optional[str], exc_tb: Optional[str]) -> None:
+    def __exit__(
+        self, exc_type: str | None, exc_val: str | None, exc_tb: str | None
+    ) -> None:
         # Close the read end of the pipe. The writing thread may
         # still be writing to the other end, but this will wake it up
         # if that's the case.
@@ -323,7 +324,12 @@ class ReadableTransformingPipe(ReadablePipe):
 
     """
 
-    def __init__(self, source: IO[Any], encoding: Optional[str] = None, errors: Optional[str] = None) -> None:
+    def __init__(
+        self,
+        source: IO[Any],
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> None:
         """
         :param str encoding: the name of the encoding used to encode the file. Encodings are the same
                 as for encode(). Defaults to None which represents binary mode.
@@ -357,7 +363,14 @@ class HashingPipe(ReadableTransformingPipe):
 
     Assumes info actually has a checksum.
     """
-    def __init__(self, source: IO[Any], encoding: Optional[str] = None, errors: Optional[str] = None, checksum_to_verify: Optional[str] = None) -> None:
+
+    def __init__(
+        self,
+        source: IO[Any],
+        encoding: str | None = None,
+        errors: str | None = None,
+        checksum_to_verify: str | None = None,
+    ) -> None:
         """
         :param str encoding: the name of the encoding used to encode the file. Encodings are the same
                 as for encode(). Defaults to None which represents binary mode.
@@ -365,13 +378,13 @@ class HashingPipe(ReadableTransformingPipe):
         :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
                 are the same as for open(). Defaults to 'strict' when an encoding is specified.
         """
-        super(HashingPipe, self).__init__(source=source, encoding=encoding, errors=errors)
+        super().__init__(source=source, encoding=encoding, errors=errors)
         self.checksum_to_verify = checksum_to_verify
 
     def transform(self, readable: IO[Any], writable: IO[Any]) -> None:
         hash_object = hashlib.sha1()
         contents = readable.read(1024 * 1024)
-        while contents != b'':
+        while contents != b"":
             hash_object.update(contents)
             try:
                 writable.write(contents)
@@ -380,6 +393,8 @@ class HashingPipe(ReadableTransformingPipe):
                 # Can't check the checksum.
                 return
             contents = readable.read(1024 * 1024)
-        final_computed_checksum = f'sha1${hash_object.hexdigest()}'
+        final_computed_checksum = f"sha1${hash_object.hexdigest()}"
         if not self.checksum_to_verify == final_computed_checksum:
-            raise ChecksumError(f'Checksum mismatch. Expected: {self.checksum_to_verify} Actual: {final_computed_checksum}')
+            raise ChecksumError(
+                f"Checksum mismatch. Expected: {self.checksum_to_verify} Actual: {final_computed_checksum}"
+            )

toil/lib/plugins.py

@@ -26,16 +26,17 @@ themselves.
 """
 
 import importlib
-from typing import Any, Literal, Union
 import pkgutil
-from toil.lib.memoize import memoize
+from typing import Any, Literal, Union
 
+from toil.lib.memoize import memoize
 
 PluginType = Union[Literal["batch_system"], Literal["url_access"]]
 plugin_types: list[PluginType] = ["batch_system", "url_access"]
 
 _registry: dict[str, dict[str, Any]] = {k: {} for k in plugin_types}
 
+
 def register_plugin(
     plugin_type: PluginType, plugin_name: str, plugin_being_registered: Any
 ) -> None:
@@ -51,13 +52,13 @@ def register_plugin(
         the resulting type must extend
         :class:`toil.batchSystems.abstractBatchSystem.AbstractBatchSystem`. For
         URL access plugins, it must extend :class:`toil.lib.url.URLAccess`.
-        Note that the function used here should return the class itslef; it
+        Note that the function used here should return the class itself; it
         should not construct an instance of the class.
     """
     _registry[plugin_type][plugin_name] = plugin_being_registered
 
-def remove_plugin(
-    plugin_type: PluginType, plugin_name: str) -> None:
+
+def remove_plugin(plugin_type: PluginType, plugin_name: str) -> None:
     """
     Removes a plugin from the registry for the given type of plugin.
     """
@@ -67,13 +68,15 @@ def remove_plugin(
         # If the plugin does not exist, it can be ignored
         pass
 
-def get_plugin_names(plugin_type:PluginType) -> list[str]:
+
+def get_plugin_names(plugin_type: PluginType) -> list[str]:
     """
     Get the names of all the available plugins of the given type.
     """
     _load_all_plugins(plugin_type)
     return list(_registry[plugin_type].keys())
 
+
 def get_plugin(plugin_type: PluginType, plugin_name: str) -> Any:
     """
     Get a plugin class factory function by name.
@@ -87,12 +90,13 @@ def get_plugin(plugin_type: PluginType, plugin_name: str) -> Any:
 
 def _plugin_name_prefix(plugin_type: PluginType) -> str:
     """
-    Get prefix for plugin type. 
-    
+    Get prefix for plugin type.
+
     Any packages with prefix will count as toil plugins of that type.
     """
     return f"toil_{plugin_type}_"
 
+
 @memoize
 def _load_all_plugins(plugin_type: PluginType) -> None:
     """

toil/lib/resources.py

@@ -19,6 +19,7 @@ import sys
 
 from toil.lib.misc import StrPath
 
+
 class ResourceMonitor:
     """
     Global resource monitoring widget.

toil/lib/retry.py

@@ -131,9 +131,9 @@ import sqlite3
 import time
 import traceback
 import urllib.error
-from collections.abc import Generator, Iterable, Sequence
+from collections.abc import Callable, Generator, Iterable, Sequence
 from contextlib import contextmanager
-from typing import Any, Callable, ContextManager, Optional, TypeVar, Union
+from typing import ContextManager, TypeVar
 
 import requests.exceptions
 import urllib3.exceptions
@@ -172,7 +172,7 @@ class ErrorCondition:
 
     def __init__(
         self,
-        error: Optional[type[BaseException]] = None,
+        error: type[BaseException] | None = None,
         error_codes: list[int] = None,
         boto_error_codes: list[str] = None,
         error_message_must_include: str = None,
@@ -227,11 +227,11 @@ RT = TypeVar("RT")
 
 
 def retry(
-    intervals: Optional[list] = None,
+    intervals: list | None = None,
     infinite_retries: bool = False,
-    errors: Optional[Sequence[Union[ErrorCondition, type[Exception]]]] = None,
-    log_message: Optional[tuple[Callable, str]] = None,
-    prepare: Optional[list[Callable]] = None,
+    errors: Sequence[ErrorCondition | type[Exception]] | None = None,
+    log_message: tuple[Callable, str] | None = None,
+    prepare: list[Callable] | None = None,
 ) -> Callable[[Callable[..., RT]], Callable[..., RT]]:
     """
     Retry a function if it fails with any Exception defined in "errors".
@@ -332,17 +332,14 @@ def return_status_code(e):
     if botocore:
         if isinstance(e, botocore.exceptions.ClientError):
             return e.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
-
-    if isinstance(e, requests.exceptions.HTTPError):
-        return e.response.status_code
-    elif isinstance(e, http.client.HTTPException) or isinstance(
-        e, urllib3.exceptions.HTTPError
-    ):
-        return e.status
-    elif isinstance(e, urllib.error.HTTPError):
-        return e.code
-    else:
-        raise ValueError(f"Unsupported error type; cannot grok status code: {e}.")
+    match e:
+        case requests.exceptions.HTTPError():
+            return e.response.status_code
+        case http.client.HTTPException() | urllib3.exceptions.HTTPError():
+            return e.status
+        case urllib.error.HTTPError():
+            return e.code
+    raise ValueError(f"Unsupported error type; cannot grok status code: {e}.")
 
 
 def get_error_code(e: Exception) -> str:
@@ -441,7 +438,7 @@ def get_error_body(e: Exception) -> str:
     return f"{get_error_code(e)}: {get_error_message(e)}"
 
 
-def meets_error_message_condition(e: Exception, error_message: Optional[str]):
+def meets_error_message_condition(e: Exception, error_message: str | None):
     if error_message:
         if kubernetes:
             if isinstance(e, kubernetes.client.rest.ApiException):
@@ -450,26 +447,25 @@ def meets_error_message_condition(e: Exception, error_message: Optional[str]):
         if botocore:
             if isinstance(e, botocore.exceptions.ClientError):
                 return error_message in str(e)
-
-        if isinstance(e, http.client.HTTPException) or isinstance(
-            e, urllib3.exceptions.HTTPError
-        ):
-            return error_message in e.reason
-        elif isinstance(e, sqlite3.OperationalError):
-            return error_message in str(e)
-        elif isinstance(e, urllib.error.HTTPError):
-            return error_message in e.msg
-        elif isinstance(e, requests.exceptions.HTTPError):
-            return error_message in e.raw
-        elif hasattr(e, "msg"):
-            return error_message in e.msg
-        else:
-            return error_message in traceback.format_exc()
+        match e:
+            case (
+                http.client.HTTPException() | urllib3.exceptions.HTTPError()
+            ) as c_exc:
+                return error_message in c_exc.reason
+            case sqlite3.OperationalError():
+                return error_message in str(e)
+            case urllib.error.HTTPError() as u_err:
+                return error_message in u_err.msg
+            case requests.exceptions.HTTPError() as r_err:
+                return error_message in r_err.raw
+            case Exception() as exc if hasattr(exc, "msg"):
+                return error_message in exc.msg
+        return error_message in traceback.format_exc()
     else:  # there is no error message, so the user is not expecting it to be present
         return True
 
 
-def meets_error_code_condition(e: Exception, error_codes: Optional[list[int]]):
+def meets_error_code_condition(e: Exception, error_codes: list[int] | None):
     """These are expected to be normal HTTP error codes, like 404 or 500."""
     if error_codes:
         status_code = get_error_status(e)
@@ -478,9 +474,7 @@ def meets_error_code_condition(e: Exception, error_codes: Optional[list[int]]):
         return True
 
 
-def meets_boto_error_code_condition(
-    e: Exception, boto_error_codes: Optional[list[str]]
-):
+def meets_boto_error_code_condition(e: Exception, boto_error_codes: list[str] | None):
     """These are expected to be AWS's custom error aliases, like 'BucketNotFound' or 'AccessDenied'."""
     if boto_error_codes:
         status_code = get_error_code(e)