toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

toil/lib/conversions.py

@@ -2,28 +2,55 @@
 Conversion utilities for mapping memory, disk, core declarations from strings to numbers and vice versa.
 Also contains general conversion functions
 """
+
 import math
 import urllib.parse
-
-from typing import Optional, SupportsInt, Union, List
+from typing import SupportsInt
 
 KIB = 1024
-MIB = 1024 ** 2
-GIB = 1024 ** 3
-TIB = 1024 ** 4
-PIB = 1024 ** 5
-EIB = 1024 ** 6
+MIB = 1024**2
+GIB = 1024**3
+TIB = 1024**4
+PIB = 1024**5
+EIB = 1024**6
 
 KB = 1000
-MB = 1000 ** 2
-GB = 1000 ** 3
-TB = 1000 ** 4
-PB = 1000 ** 5
-EB = 1000 ** 6
+MB = 1000**2
+GB = 1000**3
+TB = 1000**4
+PB = 1000**5
+EB = 1000**6
 
 # See https://en.wikipedia.org/wiki/Binary_prefix
-BINARY_PREFIXES = ['ki', 'mi', 'gi', 'ti', 'pi', 'ei', 'kib', 'mib', 'gib', 'tib', 'pib', 'eib']
-DECIMAL_PREFIXES = ['b', 'k', 'm', 'g', 't', 'p', 'e', 'kb', 'mb', 'gb', 'tb', 'pb', 'eb']
+BINARY_PREFIXES = [
+    "ki",
+    "mi",
+    "gi",
+    "ti",
+    "pi",
+    "ei",
+    "kib",
+    "mib",
+    "gib",
+    "tib",
+    "pib",
+    "eib",
+]
+DECIMAL_PREFIXES = [
+    "b",
+    "k",
+    "m",
+    "g",
+    "t",
+    "p",
+    "e",
+    "kb",
+    "mb",
+    "gb",
+    "tb",
+    "pb",
+    "eb",
+]
 VALID_PREFIXES = BINARY_PREFIXES + DECIMAL_PREFIXES
 
 
@@ -113,14 +140,14 @@ def bytes2human(n: SupportsInt) -> str:
     return f"{value:.1f} {unit}"
 
 
-def b_to_mib(n: Union[int, float]) -> float:
+def b_to_mib(n: int | float) -> float:
     """
     Convert a number from bytes to mibibytes.
     """
     return convert_units(n, "b", "mib")
 
 
-def mib_to_b(n: Union[int, float]) -> float:
+def mib_to_b(n: int | float) -> float:
     """
     Convert a number from mibibytes to bytes.
     """
@@ -170,12 +197,12 @@ def strtobool(val: str) -> bool:
     raise ValueError(f'Cannot convert "{val}" to a bool')
 
 
-def opt_strtobool(b: Optional[str]) -> Optional[bool]:
+def opt_strtobool(b: str | None) -> bool | None:
     """Convert an optional string representation of bool to None or bool"""
     return b if b is None else strtobool(b)
 
 
-def modify_url(url: str, remove: List[str]) -> str:
+def modify_url(url: str, remove: list[str]) -> str:
     """
     Given a valid URL string, split out the params, remove any offending
     params in 'remove', and return the cleaned URL.

toil/lib/directory.py

@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import base64
-
+import json
+from collections.abc import Iterator
+from typing import Union
 from urllib.parse import quote, unquote
 
-from typing import Iterator, Optional, Union
-
 TOIL_DIR_URI_SCHEME = "toildir:"
 
 
@@ -44,7 +43,7 @@ def check_directory_dict_invariants(contents: DirectoryContents) -> None:
 
 def decode_directory(
     dir_path: str,
-) -> tuple[DirectoryContents, Optional[str], str, Optional[str], Optional[str]]:
+) -> tuple[DirectoryContents, str | None, str, str | None, str | None]:
     """
     Decode a directory from a "toildir:" path to a directory (or a file in it).
 
@@ -69,9 +68,9 @@ def decode_directory(
     # Before the first slash is the encoded data describing the directory contents
     encoded_name, encoded_source, dir_data = parts[0].split(":")
     # Decode the name and source, replacing empty string with None again.
-    name: Optional[str] = unquote(encoded_name) or None
-    source: Optional[str] = unquote(encoded_source) or None
-    
+    name: str | None = unquote(encoded_name) or None
+    source: str | None = unquote(encoded_source) or None
+
     # We need the unique key identifying this directory, which is where it came
     # from if stored, or the encoded data itself otherwise.
     # TODO: Is this too complicated?
@@ -91,7 +90,12 @@ def decode_directory(
         # We have a path below this
         return contents, parts[1], directory_identifier, name, source
 
-def encode_directory(contents: DirectoryContents, name: Optional[str] = None, source: Optional[str] = None) -> str:
+
+def encode_directory(
+    contents: DirectoryContents,
+    name: str | None = None,
+    source: str | None = None,
+) -> str:
     """
     Encode a directory from a "toildir:" path to a directory (or a file in it).
 
@@ -109,9 +113,7 @@ def encode_directory(contents: DirectoryContents, name: Optional[str] = None, so
         TOIL_DIR_URI_SCHEME[:-1],
         quote(name or "", safe=""),
         quote(source or "", safe=""),
-        base64.urlsafe_b64encode(
-            json.dumps(contents).encode("utf-8")
-        ).decode("utf-8"),
+        base64.urlsafe_b64encode(json.dumps(contents).encode("utf-8")).decode("utf-8"),
     ]
 
     return ":".join(parts)
@@ -131,15 +133,18 @@ def directory_item_exists(dir_path: str) -> bool:
         return False
     return True
 
-def get_directory_contents_item(contents: DirectoryContents, remaining_path: Optional[str]) -> Union[DirectoryContents, str]:
+
+def get_directory_contents_item(
+    contents: DirectoryContents, remaining_path: str | None
+) -> DirectoryContents | str:
     """
     Get a subdirectory or file from a decoded directory and remaining path.
     """
 
     if remaining_path is None:
         return contents
-    
-    here: Union[str, DirectoryContents] = contents
+
+    here: str | DirectoryContents = contents
     for part in remaining_path.split("/"):
         if not isinstance(here, dict):
             # We're trying to go inside a file
@@ -151,20 +156,24 @@ def get_directory_contents_item(contents: DirectoryContents, remaining_path: Opt
     # If we get here we successfully looked up the thing in the structure
     return here
 
-def get_directory_item(dir_path: str) -> Union[DirectoryContents, str]:
+
+def get_directory_item(dir_path: str) -> DirectoryContents | str:
     """
     Get a subdirectory or file from a URL pointing to or into a toildir: directory.
     """
 
     contents, remaining_path, _, _, _ = decode_directory(dir_path)
-    
+
     try:
         return get_directory_contents_item(contents, remaining_path)
     except FileNotFoundError:
         # Rewrite file not found to be about the full thing we went to look up.
         raise FileNotFoundError(dir_path)
 
-def directory_contents_items(contents: DirectoryContents) -> Iterator[tuple[str, Union[str, None]]]:
+
+def directory_contents_items(
+    contents: DirectoryContents,
+) -> Iterator[tuple[str, str | None]]:
     """
     Yield each file or directory under the given contents, including itself.
 
@@ -188,7 +197,8 @@ def directory_contents_items(contents: DirectoryContents) -> Iterator[tuple[str,
             for child_path, child_value in directory_contents_items(v):
                 yield (f"{k}/{child_path}", child_value)
 
-def directory_items(dir_path: str) -> Iterator[tuple[str, Union[str, None]]]:
+
+def directory_items(dir_path: str) -> Iterator[tuple[str, str | None]]:
     """
     Yield each file or directory under the given path, including itself.
 
@@ -208,10 +218,3 @@ def directory_items(dir_path: str) -> Iterator[tuple[str, Union[str, None]]]:
     else:
         # It's a directory in there.
         yield from directory_contents_items(item)
-
-        
-
-
-
-
-

toil/lib/docker.py

@@ -11,23 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from collections.abc import Generator
 import base64
 import logging
 import os
 import re
 import struct
+from collections.abc import Generator
 from shlex import quote
-from typing import (
-    cast,
-    overload,
-    Any,
-    Literal,
-    NoReturn,
-    Optional,
-    Union,
-    TYPE_CHECKING,
-)
+from typing import TYPE_CHECKING, Any, Literal, NoReturn, Union, cast, overload
 
 import requests
 
@@ -38,7 +29,10 @@ from docker.errors import (  # type: ignore[import-not-found]
     NotFound,
     create_api_error_from_http_exception,
 )
-from docker.utils.socket import consume_socket_output, demux_adaptor  # type: ignore[import-not-found]
+from docker.utils.socket import (  # type: ignore[import-not-found]
+    consume_socket_output,
+    demux_adaptor,
+)
 from toil.lib.accelerators import get_host_accelerator_numbers
 
 if TYPE_CHECKING:
@@ -76,24 +70,24 @@ def subprocessDockerCall(*args: Any, **kwargs: Any) -> NoReturn:
 def apiDockerCall(
     job: "Job",
     image: str,
-    parameters: Optional[Union[list[str], list[list[str]]]] = None,
-    deferParam: Optional[int] = None,
-    volumes: Optional[dict[str, dict[str, str]]] = None,
-    working_dir: Optional[str] = None,
-    containerName: Optional[str] = None,
-    entrypoint: Optional[Union[str, list[str]]] = None,
+    parameters: list[str] | list[list[str]] | None = None,
+    deferParam: int | None = None,
+    volumes: dict[str, dict[str, str]] | None = None,
+    working_dir: str | None = None,
+    containerName: str | None = None,
+    entrypoint: str | list[str] | None = None,
     detach: Literal[False] = False,
-    log_config: Optional[dict[str, str]] = None,
-    auto_remove: Optional[bool] = None,
-    remove: Optional[bool] = False,
-    user: Optional[str] = None,
-    environment: Optional[dict[str, str]] = None,
-    stdout: Optional[bool] = None,
+    log_config: dict[str, str] | None = None,
+    auto_remove: bool | None = None,
+    remove: bool | None = False,
+    user: str | None = None,
+    environment: dict[str, str] | None = None,
+    stdout: bool | None = None,
     stderr: bool = False,
     stream: bool = False,
     demux: Literal[False] = False,
-    streamfile: Optional[str] = None,
-    accelerators: Optional[list[int]] = None,
+    streamfile: str | None = None,
+    accelerators: list[int] | None = None,
     timeout: int = 365 * 24 * 60 * 60,
     **kwargs: Any,
 ) -> str:
@@ -104,24 +98,24 @@ def apiDockerCall(
 def apiDockerCall(
     job: "Job",
     image: str,
-    parameters: Optional[Union[list[str], list[list[str]]]] = None,
-    deferParam: Optional[int] = None,
-    volumes: Optional[dict[str, dict[str, str]]] = None,
-    working_dir: Optional[str] = None,
-    containerName: Optional[str] = None,
-    entrypoint: Optional[Union[str, list[str]]] = None,
+    parameters: list[str] | list[list[str]] | None = None,
+    deferParam: int | None = None,
+    volumes: dict[str, dict[str, str]] | None = None,
+    working_dir: str | None = None,
+    containerName: str | None = None,
+    entrypoint: str | list[str] | None = None,
     detach: Literal[False] = False,
-    log_config: Optional[dict[str, str]] = None,
-    auto_remove: Optional[bool] = None,
-    remove: Optional[bool] = False,
-    user: Optional[str] = None,
-    environment: Optional[dict[str, str]] = None,
-    stdout: Optional[bool] = None,
+    log_config: dict[str, str] | None = None,
+    auto_remove: bool | None = None,
+    remove: bool | None = False,
+    user: str | None = None,
+    environment: dict[str, str] | None = None,
+    stdout: bool | None = None,
     stderr: bool = False,
     stream: Literal[True] = True,
     demux: Literal[True] = True,
-    streamfile: Optional[str] = None,
-    accelerators: Optional[list[int]] = None,
+    streamfile: str | None = None,
+    accelerators: list[int] | None = None,
     timeout: int = 365 * 24 * 60 * 60,
     **kwargs: Any,
 ) -> Generator[tuple[str, str]]:
@@ -132,24 +126,24 @@ def apiDockerCall(
 def apiDockerCall(
     job: "Job",
     image: str,
-    parameters: Optional[Union[list[str], list[list[str]]]] = None,
-    deferParam: Optional[int] = None,
-    volumes: Optional[dict[str, dict[str, str]]] = None,
-    working_dir: Optional[str] = None,
-    containerName: Optional[str] = None,
-    entrypoint: Optional[Union[str, list[str]]] = None,
+    parameters: list[str] | list[list[str]] | None = None,
+    deferParam: int | None = None,
+    volumes: dict[str, dict[str, str]] | None = None,
+    working_dir: str | None = None,
+    containerName: str | None = None,
+    entrypoint: str | list[str] | None = None,
     detach: Literal[False] = False,
-    log_config: Optional[dict[str, str]] = None,
-    auto_remove: Optional[bool] = None,
-    remove: Optional[bool] = False,
-    user: Optional[str] = None,
-    environment: Optional[dict[str, str]] = None,
-    stdout: Optional[bool] = None,
+    log_config: dict[str, str] | None = None,
+    auto_remove: bool | None = None,
+    remove: bool | None = False,
+    user: str | None = None,
+    environment: dict[str, str] | None = None,
+    stdout: bool | None = None,
     stderr: bool = False,
     stream: Literal[False] = False,
     demux: Literal[True] = True,
-    streamfile: Optional[str] = None,
-    accelerators: Optional[list[int]] = None,
+    streamfile: str | None = None,
+    accelerators: list[int] | None = None,
     timeout: int = 365 * 24 * 60 * 60,
     **kwargs: Any,
 ) -> tuple[str, str]:
@@ -160,24 +154,24 @@ def apiDockerCall(
 def apiDockerCall(
     job: "Job",
     image: str,
-    parameters: Optional[Union[list[str], list[list[str]]]] = None,
-    deferParam: Optional[int] = None,
-    volumes: Optional[dict[str, dict[str, str]]] = None,
-    working_dir: Optional[str] = None,
-    containerName: Optional[str] = None,
-    entrypoint: Optional[Union[str, list[str]]] = None,
+    parameters: list[str] | list[list[str]] | None = None,
+    deferParam: int | None = None,
+    volumes: dict[str, dict[str, str]] | None = None,
+    working_dir: str | None = None,
+    containerName: str | None = None,
+    entrypoint: str | list[str] | None = None,
     detach: Literal[True] = True,
-    log_config: Optional[dict[str, str]] = None,
-    auto_remove: Optional[bool] = None,
-    remove: Optional[bool] = False,
-    user: Optional[str] = None,
-    environment: Optional[dict[str, str]] = None,
-    stdout: Optional[bool] = None,
+    log_config: dict[str, str] | None = None,
+    auto_remove: bool | None = None,
+    remove: bool | None = False,
+    user: str | None = None,
+    environment: dict[str, str] | None = None,
+    stdout: bool | None = None,
     stderr: bool = False,
     stream: bool = False,
     demux: bool = False,
-    streamfile: Optional[str] = None,
-    accelerators: Optional[list[int]] = None,
+    streamfile: str | None = None,
+    accelerators: list[int] | None = None,
     timeout: int = 365 * 24 * 60 * 60,
     **kwargs: Any,
 ) -> "Container":
@@ -188,24 +182,24 @@ def apiDockerCall(
 def apiDockerCall(
     job: "Job",
     image: str,
-    parameters: Optional[Union[list[str], list[list[str]]]] = None,
-    deferParam: Optional[int] = None,
-    volumes: Optional[dict[str, dict[str, str]]] = None,
-    working_dir: Optional[str] = None,
-    containerName: Optional[str] = None,
-    entrypoint: Optional[Union[str, list[str]]] = None,
+    parameters: list[str] | list[list[str]] | None = None,
+    deferParam: int | None = None,
+    volumes: dict[str, dict[str, str]] | None = None,
+    working_dir: str | None = None,
+    containerName: str | None = None,
+    entrypoint: str | list[str] | None = None,
     detach: bool = False,
-    log_config: Optional[dict[str, str]] = None,
-    auto_remove: Optional[bool] = None,
-    remove: Optional[bool] = False,
-    user: Optional[str] = None,
-    environment: Optional[dict[str, str]] = None,
-    stdout: Optional[bool] = None,
+    log_config: dict[str, str] | None = None,
+    auto_remove: bool | None = None,
+    remove: bool | None = False,
+    user: str | None = None,
+    environment: dict[str, str] | None = None,
+    stdout: bool | None = None,
     stderr: bool = False,
     stream: bool = False,
     demux: bool = False,
-    streamfile: Optional[str] = None,
-    accelerators: Optional[list[int]] = None,
+    streamfile: str | None = None,
+    accelerators: list[int] | None = None,
     timeout: int = 365 * 24 * 60 * 60,
     **kwargs: Any,
 ) -> Union[
@@ -216,24 +210,24 @@ def apiDockerCall(
 def apiDockerCall(
     job: "Job",
     image: str,
-    parameters: Optional[Union[list[str], list[list[str]]]] = None,
-    deferParam: Optional[int] = None,
-    volumes: Optional[dict[str, dict[str, str]]] = None,
-    working_dir: Optional[str] = None,
-    containerName: Optional[str] = None,
-    entrypoint: Optional[Union[str, list[str]]] = None,
+    parameters: list[str] | list[list[str]] | None = None,
+    deferParam: int | None = None,
+    volumes: dict[str, dict[str, str]] | None = None,
+    working_dir: str | None = None,
+    containerName: str | None = None,
+    entrypoint: str | list[str] | None = None,
     detach: bool = False,
-    log_config: Optional[dict[str, str]] = None,
-    auto_remove: Optional[bool] = None,
-    remove: Optional[bool] = False,
-    user: Optional[str] = None,
-    environment: Optional[dict[str, str]] = None,
-    stdout: Optional[bool] = None,
+    log_config: dict[str, str] | None = None,
+    auto_remove: bool | None = None,
+    remove: bool | None = False,
+    user: str | None = None,
+    environment: dict[str, str] | None = None,
+    stdout: bool | None = None,
     stderr: bool = False,
     stream: bool = False,
     demux: bool = False,
-    streamfile: Optional[str] = None,
-    accelerators: Optional[list[int]] = None,
+    streamfile: str | None = None,
+    accelerators: list[int] | None = None,
     timeout: int = 365 * 24 * 60 * 60,
     **kwargs: Any,
 ) -> Union[
@@ -369,7 +363,7 @@ def apiDockerCall(
             " ".join(quote(arg) for arg in command) for command in parameters
         ]
         pipe_prefix = "set -eo pipefail && "
-        command: Union[None, str, list[str]] = [pipe_prefix + " | ".join(chain_params)]
+        command: None | str | list[str] = [pipe_prefix + " | ".join(chain_params)]
         logger.debug("Calling docker with: " + repr(command))
 
     # If 'parameters' is a normal list, join all elements into a single string
@@ -592,7 +586,7 @@ def dockerStop(container_name: str, remove: bool = False) -> None:
 
 def containerIsRunning(
     container_name: str, timeout: int = 365 * 24 * 60 * 60
-) -> Optional[bool]:
+) -> bool | None:
     """
     Checks whether the container is running or not.
 
@@ -635,7 +629,7 @@ def getContainerName(job: "Job") -> str:
 
 def _multiplexed_response_stream_helper(
     response: requests.Response,
-) -> Generator[tuple[Any, Union[bytes, Any]]]:
+) -> Generator[tuple[Any, bytes | Any]]:
     """
     A generator of multiplexed data blocks coming from a response stream modified from:
     https://github.com/docker/docker-py/blob/4.3.1-release/docker/api/client.py#L370