toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

toil/lib/aws/session.py

@@ -15,7 +15,7 @@ import collections
 import logging
 import os
 import threading
-from typing import TYPE_CHECKING, Literal, Optional, cast, overload
+from typing import TYPE_CHECKING, Literal, cast, overload
 
 import boto3
 import boto3.resources.base
@@ -55,7 +55,7 @@ ANONYMOUS_CONFIG = Config(signature_version=botocore.UNSIGNED)
 _init_lock = threading.RLock()
 
 
-def _new_boto3_session(region_name: Optional[str] = None) -> Session:
+def _new_boto3_session(region_name: str | None = None) -> Session:
     """
     This is the One True Place where new Boto3 sessions should be made, and
     prepares them with the necessary credential caching. Does *not* cache
@@ -115,26 +115,26 @@ class AWSConnectionManager:
         """
         # This stores Boto3 sessions in .item of a thread-local storage, by
         # region.
-        self.sessions_by_region: dict[Optional[str], threading.local] = (
+        self.sessions_by_region: dict[str | None, threading.local] = (
             collections.defaultdict(threading.local)
         )
         # This stores Boto3 resources in .item of a thread-local storage, by
         # (region, service name, endpoint URL) tuples
         self.resource_cache: dict[
-            tuple[Optional[str], str, Optional[str]], threading.local
+            tuple[str | None, str, str | None], threading.local
         ] = collections.defaultdict(threading.local)
         # This stores Boto3 clients in .item of a thread-local storage, by
         # (region, service name, endpoint URL) tuples
-        self.client_cache: dict[
-            tuple[Optional[str], str, Optional[str]], threading.local
-        ] = collections.defaultdict(threading.local)
+        self.client_cache: dict[tuple[str | None, str, str | None], threading.local] = (
+            collections.defaultdict(threading.local)
+        )
         # This stores Boto 2 connections in .item of a thread-local storage, by
         # (region, service name) tuples.
-        self.boto2_cache: dict[tuple[Optional[str], str], threading.local] = (
+        self.boto2_cache: dict[tuple[str | None, str], threading.local] = (
             collections.defaultdict(threading.local)
         )
 
-    def session(self, region: Optional[str]) -> boto3.session.Session:
+    def session(self, region: str | None) -> boto3.session.Session:
         """
         Get the Boto3 Session to use for the given region.
         """
@@ -148,34 +148,34 @@ class AWSConnectionManager:
     @overload
     def resource(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["s3"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "S3ServiceResource": ...
     @overload
     def resource(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["iam"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "IAMServiceResource": ...
     @overload
     def resource(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["ec2"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "EC2ServiceResource": ...
 
     def resource(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: str,
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> boto3.resources.base.ServiceResource:
         """
         Get the Boto3 Resource to use with the given service (like 'ec2') in the given region.
@@ -205,58 +205,58 @@ class AWSConnectionManager:
     @overload
     def client(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["ec2"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "EC2Client": ...
     @overload
     def client(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["iam"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "IAMClient": ...
     @overload
     def client(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["s3"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "S3Client": ...
     @overload
     def client(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["sts"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "STSClient": ...
     @overload
     def client(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["sdb"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "SimpleDBClient": ...
     @overload
     def client(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["autoscaling"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> "AutoScalingClient": ...
 
     def client(
         self,
-        region: Optional[str],
+        region: str | None,
         service_name: Literal["ec2", "iam", "s3", "sts", "sdb", "autoscaling"],
-        endpoint_url: Optional[str] = None,
-        config: Optional[Config] = None,
+        endpoint_url: str | None = None,
+        config: Config | None = None,
     ) -> botocore.client.BaseClient:
         """
         Get the Boto3 Client to use with the given service (like 'ec2') in the given region.
@@ -298,7 +298,7 @@ class AWSConnectionManager:
 _global_manager = AWSConnectionManager()
 
 
-def establish_boto3_session(region_name: Optional[str] = None) -> Session:
+def establish_boto3_session(region_name: str | None = None) -> Session:
     """
     Get a Boto 3 session usable by the current thread.
 
@@ -312,52 +312,52 @@ def establish_boto3_session(region_name: Optional[str] = None) -> Session:
 @overload
 def client(
     service_name: Literal["ec2"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "EC2Client": ...
 @overload
 def client(
     service_name: Literal["iam"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "IAMClient": ...
 @overload
 def client(
     service_name: Literal["s3"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "S3Client": ...
 @overload
 def client(
     service_name: Literal["sts"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "STSClient": ...
 @overload
 def client(
     service_name: Literal["sdb"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "SimpleDBClient": ...
 @overload
 def client(
     service_name: Literal["autoscaling"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "AutoScalingClient": ...
 
 
 def client(
     service_name: Literal["ec2", "iam", "s3", "sts", "sdb", "autoscaling"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> botocore.client.BaseClient:
     """
     Get a Boto 3 client for a particular AWS service, usable by the current thread.
@@ -374,31 +374,31 @@ def client(
 @overload
 def resource(
     service_name: Literal["s3"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "S3ServiceResource": ...
 @overload
 def resource(
     service_name: Literal["iam"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "IAMServiceResource": ...
 @overload
 def resource(
     service_name: Literal["ec2"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> "EC2ServiceResource": ...
 
 
 def resource(
     service_name: Literal["s3", "iam", "ec2"],
-    region_name: Optional[str] = None,
-    endpoint_url: Optional[str] = None,
-    config: Optional[Config] = None,
+    region_name: str | None = None,
+    endpoint_url: str | None = None,
+    config: Config | None = None,
 ) -> boto3.resources.base.ServiceResource:
     """
     Get a Boto 3 resource for a particular AWS service, usable by the current thread.

toil/lib/aws/utils.py

@@ -15,20 +15,11 @@ import errno
 import logging
 import os
 import socket
-from collections.abc import Iterable, Iterator
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    ContextManager,
-    Literal,
-    Optional,
-    Union,
-    cast,
-)
-from urllib.parse import ParseResult, urlparse
+from collections.abc import Callable, Iterable, Iterator
+from typing import TYPE_CHECKING, Any, ContextManager, Literal, cast
+from urllib.parse import ParseResult
 
-# To import toil.lib.aws.session, the AWS libraries must be installed 
+# To import toil.lib.aws.session, the AWS libraries must be installed
 from toil.lib.aws import AWSRegionName, AWSServerErrors, session
 from toil.lib.conversions import strtobool
 from toil.lib.memoize import memoize
@@ -73,7 +64,7 @@ THROTTLED_ERROR_CODES = [
 
 @retry(errors=[AWSServerErrors])
 def delete_sdb_domain(
-    sdb_domain_name: str, region: Optional[str] = None, quiet: bool = True
+    sdb_domain_name: str, region: str | None = None, quiet: bool = True
 ) -> None:
     sdb_client = session.client("sdb", region_name=region)
     sdb_client.delete_domain(DomainName=sdb_domain_name)
@@ -168,7 +159,9 @@ def delete_s3_bucket(
         s3_resource.Bucket(bucket).delete()
         # S3 bucket deletion is only eventually-consistent. See
         # <https://docs.aws.amazon.com/AmazonS3/latest/userguide/delete-bucket.html>
-        printq(f"\n * S3 bucket successfully scheduled for deletion: {bucket}\n\n", quiet)
+        printq(
+            f"\n * S3 bucket successfully scheduled for deletion: {bucket}\n\n", quiet
+        )
     except s3_resource.meta.client.exceptions.NoSuchBucket:
         printq(f"\n * S3 bucket no longer exists: {bucket}\n\n", quiet)
 
@@ -242,9 +235,9 @@ class NoBucketLocationError(Exception):
 
 def get_bucket_region(
     bucket_name: str,
-    endpoint_url: Optional[str] = None,
-    only_strategies: Optional[set[int]] = None,
-    anonymous: Optional[bool] = None
+    endpoint_url: str | None = None,
+    only_strategies: set[int] | None = None,
+    anonymous: bool | None = None,
 ) -> str:
     """
     Get the AWS region name associated with the given S3 bucket, or raise NoBucketLocationError.
@@ -262,7 +255,7 @@ def get_bucket_region(
     config = session.ANONYMOUS_CONFIG if anonymous else None
     s3_client = session.client("s3", endpoint_url=endpoint_url, config=config)
 
-    def attempt_get_bucket_location() -> Optional[str]:
+    def attempt_get_bucket_location() -> str | None:
         """
         Try and get the bucket location from the normal API call.
         """
@@ -270,7 +263,7 @@ def get_bucket_region(
             "LocationConstraint", None
         )
 
-    def attempt_get_bucket_location_from_us_east_1() -> Optional[str]:
+    def attempt_get_bucket_location_from_us_east_1() -> str | None:
         """
         Try and get the bucket location from the normal API call, but against us-east-1
         """
@@ -289,7 +282,7 @@ def get_bucket_region(
             "LocationConstraint", None
         )
 
-    def attempt_head_bucket() -> Optional[str]:
+    def attempt_head_bucket() -> str | None:
         """
         Try and get the bucket location from calling HeadBucket and inspecting
         the headers.
@@ -303,7 +296,7 @@ def get_bucket_region(
 
     # Compose a list of strategies we want to try in order, which may work.
     # None is an acceptable return type that actually means something.
-    strategies: list[Callable[[], Optional[str]]] = []
+    strategies: list[Callable[[], str | None]] = []
     strategies.append(attempt_get_bucket_location)
     if not endpoint_url:
         # We should only try to talk to us-east-1 if we don't have a custom
@@ -338,25 +331,30 @@ def get_bucket_region(
                         raise
                 except KeyError as e:
                     # If we get a weird head response we will have a KeyError
-                    logger.debug("Strategy %d to get bucket location did not work: %s", i + 1, e)
+                    logger.debug(
+                        "Strategy %d to get bucket location did not work: %s", i + 1, e
+                    )
                     error_logs.append((i + 1, str(e)))
                     last_error = e
 
     error_messages = []
     for rank, message in error_logs:
-        error_messages.append(f"Strategy {rank} failed to get bucket location because: {message}")
+        error_messages.append(
+            f"Strategy {rank} failed to get bucket location because: {message}"
+        )
     # If we get here we ran out of attempts.
     raise NoBucketLocationError(
         "Could not get bucket location: " + "\n".join(error_messages)
     ) from last_error
 
+
 @memoize
 def get_bucket_region_if_available(
     bucket_name: str,
-    endpoint_url: Optional[str] = None,
-    only_strategies: Optional[set[int]] = None,
-    anonymous: Optional[bool] = None
-) -> Optional[str]:
+    endpoint_url: str | None = None,
+    only_strategies: set[int] | None = None,
+    anonymous: bool | None = None,
+) -> str | None:
     """
     Get the AWS region name associated with the given S3 bucket, or return None.
 
@@ -369,21 +367,26 @@ def get_bucket_region_if_available(
     try:
         return get_bucket_region(bucket_name, endpoint_url, only_strategies, anonymous)
     except Exception as e:
-        if isinstance(e, NoBucketLocationError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+        if isinstance(e, NoBucketLocationError) or (
+            isinstance(e, ClientError) and get_error_status(e) == 403
+        ):
             # We can't know
             return None
         else:
             raise
 
+
 def region_to_bucket_location(region: str) -> str:
     return "" if region == "us-east-1" else region
 
 
-def bucket_location_to_region(location: Optional[str]) -> str:
+def bucket_location_to_region(location: str | None) -> str:
     return "us-east-1" if location == "" or location is None else location
 
 
-def get_object_for_url(url: ParseResult, existing: Optional[bool] = None, anonymous: Optional[bool] = None) -> "S3Object":
+def get_object_for_url(
+    url: ParseResult, existing: bool | None = None, anonymous: bool | None = None
+) -> "S3Object":
     """
     Extracts a key (object) from a given parsed s3:// URL.
 
@@ -400,11 +403,11 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None, anonym
     bucket_name = url.netloc
 
     # Decide if we need to override Boto's built-in URL here.
-    endpoint_url: Optional[str] = None
+    endpoint_url: str | None = None
     host = os.environ.get("TOIL_S3_HOST", None)
     port = os.environ.get("TOIL_S3_PORT", None)
     protocol = "https"
-    if strtobool(os.environ.get("TOIL_S3_USE_SSL", 'True')) is False:
+    if strtobool(os.environ.get("TOIL_S3_USE_SSL", "True")) is False:
         protocol = "http"
     if host:
         endpoint_url = f"{protocol}://{host}" + f":{port}" if port else ""
@@ -412,13 +415,17 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None, anonym
     # TODO: OrdinaryCallingFormat equivalent in boto3?
     # if botoargs:
     #     botoargs['calling_format'] = boto.s3.connection.OrdinaryCallingFormat()
-    
+
     config = session.ANONYMOUS_CONFIG if anonymous else None
     # Get the bucket's region to avoid a redirect per request.
     # Cache the result
-    region = get_bucket_region_if_available(bucket_name, endpoint_url=endpoint_url, anonymous=anonymous)
+    region = get_bucket_region_if_available(
+        bucket_name, endpoint_url=endpoint_url, anonymous=anonymous
+    )
     if region is not None:
-        s3 = session.resource("s3", region_name=region, endpoint_url=endpoint_url, config=config)
+        s3 = session.resource(
+            "s3", region_name=region, endpoint_url=endpoint_url, config=config
+        )
     else:
         # We can't get the bucket location, perhaps because we don't have
         # permission to do that.
@@ -461,7 +468,7 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None, anonym
 
 
 @retry(errors=[AWSServerErrors])
-def list_objects_for_url(url: ParseResult, anonymous: Optional[bool] = None) -> list[str]:
+def list_objects_for_url(url: ParseResult, anonymous: bool | None = None) -> list[str]:
     """
     Extracts a key (object) from a given parsed s3:// URL. The URL will be
     supplemented with a trailing slash if it is missing.
@@ -478,20 +485,20 @@ def list_objects_for_url(url: ParseResult, anonymous: Optional[bool] = None) ->
 
     # Decide if we need to override Boto's built-in URL here.
     # TODO: Deduplicate with get_object_for_url, or push down into session module
-    endpoint_url: Optional[str] = None
+    endpoint_url: str | None = None
     host = os.environ.get("TOIL_S3_HOST", None)
     port = os.environ.get("TOIL_S3_PORT", None)
     protocol = "https"
-    if strtobool(os.environ.get("TOIL_S3_USE_SSL", 'True')) is False:
+    if strtobool(os.environ.get("TOIL_S3_USE_SSL", "True")) is False:
         protocol = "http"
     if host:
         endpoint_url = f"{protocol}://{host}" + f":{port}" if port else ""
-    
+
     config = session.ANONYMOUS_CONFIG if anonymous else None
     client = session.client("s3", endpoint_url=endpoint_url, config=config)
 
     listing = []
-    
+
     try:
         paginator = client.get_paginator("list_objects_v2")
         result = paginator.paginate(Bucket=bucket_name, Prefix=key_name, Delimiter="/")
@@ -513,12 +520,13 @@ def list_objects_for_url(url: ParseResult, anonymous: Optional[bool] = None) ->
         else:
             raise
 
-
     logger.debug("Found in %s items: %s", url, listing)
     return listing
 
 
-def flatten_tags(tags: dict[str, str]) -> list[dict[Union[Literal["Key"], Literal["Value"]], str]]:
+def flatten_tags(
+    tags: dict[str, str],
+) -> list[dict[Literal["Key"] | Literal["Value"], str]]:
     """
     Convert tags from a key to value dict into a list of 'Key': xxx, 'Value': xxx dicts.
     """

toil/lib/checksum.py

@@ -11,11 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import logging
 import hashlib
-
+import logging
 from io import BytesIO
-from typing import BinaryIO, Union, List, TYPE_CHECKING
+from typing import TYPE_CHECKING, BinaryIO
 
 from toil.lib.aws.config import S3_PART_SIZE
 
@@ -33,16 +32,17 @@ class ChecksumError(Exception):
 
 class Etag:
     """A hasher for s3 etags."""
+
     def __init__(self, chunk_size: int) -> None:
         self.etag_bytes: int = 0
-        self.etag_parts: List[bytes] = []
+        self.etag_parts: list[bytes] = []
         self.etag_hasher: "_Hash" = hashlib.md5()
         self.chunk_size: int = chunk_size
 
     def update(self, chunk: bytes) -> None:
         if self.etag_bytes + len(chunk) > self.chunk_size:
-            chunk_head = chunk[:self.chunk_size - self.etag_bytes]
-            chunk_tail = chunk[self.chunk_size - self.etag_bytes:]
+            chunk_head = chunk[: self.chunk_size - self.etag_bytes]
+            chunk_tail = chunk[self.chunk_size - self.etag_bytes :]
             self.etag_hasher.update(chunk_head)
             self.etag_parts.append(self.etag_hasher.digest())
             self.etag_hasher = hashlib.md5()
@@ -58,31 +58,35 @@ class Etag:
             self.etag_bytes = 0
         if len(self.etag_parts) > 1:
             etag = hashlib.md5(b"".join(self.etag_parts)).hexdigest()
-            return f'{etag}-{len(self.etag_parts)}'
+            return f"{etag}-{len(self.etag_parts)}"
         else:
             return self.etag_hasher.hexdigest()
 
 
-hashers = {'sha1':    hashlib.sha1(),
-           'sha256':  hashlib.sha256(),
-           'etag': Etag(chunk_size=S3_PART_SIZE)}
+hashers = {
+    "sha1": hashlib.sha1(),
+    "sha256": hashlib.sha256(),
+    "etag": Etag(chunk_size=S3_PART_SIZE),
+}
 
 
-def compute_checksum_for_file(local_file_path: str, algorithm: str = 'sha1') -> str:
-    with open(local_file_path, 'rb') as fh:
+def compute_checksum_for_file(local_file_path: str, algorithm: str = "sha1") -> str:
+    with open(local_file_path, "rb") as fh:
         checksum_result = compute_checksum_for_content(fh, algorithm=algorithm)
     return checksum_result
 
 
-def compute_checksum_for_content(fh: Union[BinaryIO, BytesIO], algorithm: str = 'sha1') -> str:
+def compute_checksum_for_content(
+    fh: BinaryIO | BytesIO, algorithm: str = "sha1"
+) -> str:
     """
     Note: Chunk size matters for s3 etags, and must be the same to get the same hash from the same object.
     Therefore this buffer is not modifiable throughout Toil.
     """
     hasher: "_Hash" = hashers[algorithm]  # type: ignore
     contents = fh.read(S3_PART_SIZE)
-    while contents != b'':
+    while contents != b"":
         hasher.update(contents)
         contents = fh.read(S3_PART_SIZE)
 
-    return f'{algorithm}${hasher.hexdigest()}'
+    return f"{algorithm}${hasher.hexdigest()}"

toil/lib/compatibility.py

@@ -1,6 +1,7 @@
 import functools
 import warnings
-from typing import Any, Callable, Union
+from collections.abc import Callable
+from typing import Any
 
 
 def deprecated(new_function_name: str) -> Callable[..., Any]:
@@ -18,7 +19,7 @@ def deprecated(new_function_name: str) -> Callable[..., Any]:
     return decorate
 
 
-def compat_bytes(s: Union[bytes, str]) -> str:
+def compat_bytes(s: bytes | str) -> str:
     return s.decode("utf-8") if isinstance(s, bytes) else s