toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/jobStores/aws/jobStore.py

@@ -21,46 +21,53 @@ import reprlib
 import stat
 import time
 import uuid
+from collections.abc import Generator
 from contextlib import contextmanager
 from io import BytesIO
-from typing import List, Optional, IO, Dict, Union, Generator, Tuple, cast, TYPE_CHECKING
+from typing import IO, TYPE_CHECKING, Optional, Union, cast
 from urllib.parse import ParseResult, parse_qs, urlencode, urlsplit, urlunsplit
 
 from botocore.exceptions import ClientError
-from mypy_boto3_sdb import SimpleDBClient
-from mypy_boto3_sdb.type_defs import ReplaceableItemTypeDef, ReplaceableAttributeTypeDef, SelectResultTypeDef, ItemTypeDef, AttributeTypeDef, DeletableItemTypeDef, UpdateConditionTypeDef
 
 import toil.lib.encryption as encryption
 from toil.fileStores import FileID
 from toil.job import Job, JobDescription
-from toil.jobStores.abstractJobStore import (AbstractJobStore,
-                                             ConcurrentFileModificationException,
-                                             JobStoreExistsException,
-                                             NoSuchFileException,
-                                             NoSuchJobException,
-                                             NoSuchJobStoreException, LocatorException)
-from toil.jobStores.aws.utils import (SDBHelper,
-                                      ServerSideCopyProhibitedError,
-                                      copyKeyMultipart,
-                                      fileSizeAndTime,
-                                      no_such_sdb_domain,
-                                      retry_sdb,
-                                      sdb_unavailable,
-                                      uploadFile,
-                                      uploadFromPath)
-from toil.jobStores.utils import (ReadablePipe,
-                                  ReadableTransformingPipe,
-                                  WritablePipe)
+from toil.jobStores.abstractJobStore import (
+    AbstractJobStore,
+    ConcurrentFileModificationException,
+    JobStoreExistsException,
+    LocatorException,
+    NoSuchFileException,
+    NoSuchJobException,
+    NoSuchJobStoreException,
+)
+from toil.jobStores.aws.utils import (
+    SDBHelper,
+    ServerSideCopyProhibitedError,
+    copyKeyMultipart,
+    fileSizeAndTime,
+    no_such_sdb_domain,
+    retry_sdb,
+    sdb_unavailable,
+    uploadFile,
+    uploadFromPath,
+)
+from toil.jobStores.utils import ReadablePipe, ReadableTransformingPipe, WritablePipe
 from toil.lib.aws import build_tag_dict_from_env
 from toil.lib.aws.session import establish_boto3_session
-from toil.lib.aws.utils import (create_s3_bucket,
-                                enable_public_objects,
-                                flatten_tags,
-                                get_bucket_region,
-                                get_object_for_url,
-                                list_objects_for_url,
-                                retry_s3,
-                                retryable_s3_errors, boto3_pager, get_item_from_attributes)
+from toil.lib.aws.utils import (
+    NoBucketLocationError,
+    boto3_pager,
+    create_s3_bucket,
+    enable_public_objects,
+    flatten_tags,
+    get_bucket_region,
+    get_item_from_attributes,
+    get_object_for_url,
+    list_objects_for_url,
+    retry_s3,
+    retryable_s3_errors,
+)
 from toil.lib.compatibility import compat_bytes
 from toil.lib.ec2nodes import EC2Regions
 from toil.lib.exceptions import panic
@@ -70,11 +77,20 @@ from toil.lib.objects import InnerClass
 from toil.lib.retry import get_error_code, get_error_status, retry
 
 if TYPE_CHECKING:
+    from mypy_boto3_sdb.type_defs import (
+        AttributeTypeDef,
+        DeletableItemTypeDef,
+        ItemTypeDef,
+        ReplaceableAttributeTypeDef,
+        ReplaceableItemTypeDef,
+        UpdateConditionTypeDef,
+    )
+
     from toil import Config
 
 boto3_session = establish_boto3_session()
-s3_boto3_resource = boto3_session.resource('s3')
-s3_boto3_client = boto3_session.client('s3')
+s3_boto3_resource = boto3_session.resource("s3")
+s3_boto3_client = boto3_session.client("s3")
 logger = logging.getLogger(__name__)
 
 # Sometimes we have to wait for multipart uploads to become real. How long
@@ -89,6 +105,7 @@ class ChecksumError(Exception):
 
 class DomainDoesNotExist(Exception):
     """Raised when a domain that is expected to exist does not exist."""
+
     def __init__(self, domain_name):
         super().__init__(f"Expected domain {domain_name} to exist!")
 
@@ -106,14 +123,14 @@ class AWSJobStore(AbstractJobStore):
     # URLs where the may interfere with the certificate common name. We use a double
     # underscore as a separator instead.
     #
-    bucketNameRe = re.compile(r'^[a-z0-9][a-z0-9-]+[a-z0-9]$')
+    bucketNameRe = re.compile(r"^[a-z0-9][a-z0-9-]+[a-z0-9]$")
 
     # See http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
     #
     minBucketNameLen = 3
     maxBucketNameLen = 63
     maxNameLen = 10
-    nameSeparator = '--'
+    nameSeparator = "--"
 
     def __init__(self, locator: str, partSize: int = 50 << 20) -> None:
         """
@@ -124,23 +141,35 @@ class AWSJobStore(AbstractJobStore):
                whole file
         """
         super().__init__(locator)
-        region, namePrefix = locator.split(':')
+        region, namePrefix = locator.split(":")
         regions = EC2Regions.keys()
         if region not in regions:
             raise ValueError(f'Region "{region}" is not one of: {regions}')
         if not self.bucketNameRe.match(namePrefix):
-            raise ValueError("Invalid name prefix '%s'. Name prefixes must contain only digits, "
-                             "hyphens or lower-case letters and must not start or end in a "
-                             "hyphen." % namePrefix)
+            raise ValueError(
+                "Invalid name prefix '%s'. Name prefixes must contain only digits, "
+                "hyphens or lower-case letters and must not start or end in a "
+                "hyphen." % namePrefix
+            )
         # Reserve 13 for separator and suffix
-        if len(namePrefix) > self.maxBucketNameLen - self.maxNameLen - len(self.nameSeparator):
-            raise ValueError("Invalid name prefix '%s'. Name prefixes may not be longer than 50 "
-                             "characters." % namePrefix)
-        if '--' in namePrefix:
-            raise ValueError("Invalid name prefix '%s'. Name prefixes may not contain "
-                             "%s." % (namePrefix, self.nameSeparator))
-        logger.debug("Instantiating %s for region %s and name prefix '%s'",
-                     self.__class__, region, namePrefix)
+        if len(namePrefix) > self.maxBucketNameLen - self.maxNameLen - len(
+            self.nameSeparator
+        ):
+            raise ValueError(
+                "Invalid name prefix '%s'. Name prefixes may not be longer than 50 "
+                "characters." % namePrefix
+            )
+        if "--" in namePrefix:
+            raise ValueError(
+                "Invalid name prefix '%s'. Name prefixes may not contain "
+                "%s." % (namePrefix, self.nameSeparator)
+            )
+        logger.debug(
+            "Instantiating %s for region %s and name prefix '%s'",
+            self.__class__,
+            region,
+            namePrefix,
+        )
         self.region = region
         self.name_prefix = namePrefix
         self.part_size = partSize
@@ -149,7 +178,7 @@ class AWSJobStore(AbstractJobStore):
         self.files_bucket = None
         self.db = boto3_session.client(service_name="sdb", region_name=region)
 
-        self.s3_resource = boto3_session.resource('s3', region_name=self.region)
+        self.s3_resource = boto3_session.resource("s3", region_name=self.region)
         self.s3_client = self.s3_resource.meta.client
 
     def initialize(self, config: "Config") -> None:
@@ -176,7 +205,12 @@ class AWSJobStore(AbstractJobStore):
         self._bind(create=False)
         super().resume()
 
-    def _bind(self, create: bool = False, block: bool = True, check_versioning_consistency: bool = True) -> None:
+    def _bind(
+        self,
+        create: bool = False,
+        block: bool = True,
+        check_versioning_consistency: bool = True,
+    ) -> None:
         def qualify(name):
             assert len(name) <= self.maxNameLen
             return self.name_prefix + self.nameSeparator + name
@@ -191,11 +225,13 @@ class AWSJobStore(AbstractJobStore):
             self.files_domain_name = qualify("files")
             self._bindDomain(self.files_domain_name, create=create, block=block)
         if self.files_bucket is None:
-            self.files_bucket = self._bindBucket(qualify('files'),
-                                                 create=create,
-                                                 block=block,
-                                                 versioning=True,
-                                                 check_versioning_consistency=check_versioning_consistency)
+            self.files_bucket = self._bindBucket(
+                qualify("files"),
+                create=create,
+                block=block,
+                versioning=True,
+                check_versioning_consistency=check_versioning_consistency,
+            )
 
     @property
     def _registered(self) -> Optional[bool]:
@@ -217,25 +253,31 @@ class AWSJobStore(AbstractJobStore):
         # can't handle job stores that were partially created by 3.3.0, though.
         registry_domain_name = "toil-registry"
         try:
-            self._bindDomain(domain_name=registry_domain_name,
-                             create=False,
-                             block=False)
+            self._bindDomain(
+                domain_name=registry_domain_name, create=False, block=False
+            )
         except DomainDoesNotExist:
             return False
 
         for attempt in retry_sdb():
             with attempt:
-                get_result = self.db.get_attributes(DomainName=registry_domain_name,
-                                                    ItemName=self.name_prefix,
-                                                    AttributeNames=['exists'],
-                                                    ConsistentRead=True)
-                attributes: List[AttributeTypeDef] = get_result.get("Attributes", [])  # the documentation says 'Attributes' should always exist, but this is not true
-                exists: Optional[str] = get_item_from_attributes(attributes=attributes, name="exists")
+                get_result = self.db.get_attributes(
+                    DomainName=registry_domain_name,
+                    ItemName=self.name_prefix,
+                    AttributeNames=["exists"],
+                    ConsistentRead=True,
+                )
+                attributes: list["AttributeTypeDef"] = get_result.get(
+                    "Attributes", []
+                )  # the documentation says 'Attributes' should always exist, but this is not true
+                exists: Optional[str] = get_item_from_attributes(
+                    attributes=attributes, name="exists"
+                )
                 if exists is None:
                     return False
-                elif exists == 'True':
+                elif exists == "True":
                     return True
-                elif exists == 'False':
+                elif exists == "False":
                     return None
                 else:
                     assert False
@@ -244,31 +286,40 @@ class AWSJobStore(AbstractJobStore):
     def _registered(self, value: bool) -> None:
         registry_domain_name = "toil-registry"
         try:
-            self._bindDomain(domain_name=registry_domain_name,
-                             # Only create registry domain when registering or
-                             # transitioning a store
-                             create=value is not False,
-                             block=False)
+            self._bindDomain(
+                domain_name=registry_domain_name,
+                # Only create registry domain when registering or
+                # transitioning a store
+                create=value is not False,
+                block=False,
+            )
         except DomainDoesNotExist:
             pass
         else:
             for attempt in retry_sdb():
                 with attempt:
                     if value is False:
-                        self.db.delete_attributes(DomainName=registry_domain_name,
-                                                  ItemName=self.name_prefix)
+                        self.db.delete_attributes(
+                            DomainName=registry_domain_name, ItemName=self.name_prefix
+                        )
                     else:
                         if value is True:
-                            attributes: List[ReplaceableAttributeTypeDef] = [{"Name": "exists", "Value": "True", "Replace": True}]
+                            attributes: list["ReplaceableAttributeTypeDef"] = [
+                                {"Name": "exists", "Value": "True", "Replace": True}
+                            ]
                         elif value is None:
-                            attributes = [{"Name": "exists", "Value": "False", "Replace": True}]
+                            attributes = [
+                                {"Name": "exists", "Value": "False", "Replace": True}
+                            ]
                         else:
                             assert False
-                        self.db.put_attributes(DomainName=registry_domain_name,
-                                               ItemName=self.name_prefix,
-                                               Attributes=attributes)
+                        self.db.put_attributes(
+                            DomainName=registry_domain_name,
+                            ItemName=self.name_prefix,
+                            Attributes=attributes,
+                        )
 
-    def _checkItem(self, item: ItemTypeDef, enforce: bool = True) -> None:
+    def _checkItem(self, item: "ItemTypeDef", enforce: bool = True) -> None:
         """
         Make sure that the given SimpleDB item actually has the attributes we think it should.
 
@@ -278,22 +329,31 @@ class AWSJobStore(AbstractJobStore):
         """
         self._checkAttributes(item["Attributes"], enforce)
 
-    def _checkAttributes(self, attributes: List[AttributeTypeDef], enforce: bool = True) -> None:
+    def _checkAttributes(
+        self, attributes: list["AttributeTypeDef"], enforce: bool = True
+    ) -> None:
         if get_item_from_attributes(attributes=attributes, name="overlargeID") is None:
-            logger.error("overlargeID attribute isn't present: either SimpleDB entry is "
-                         "corrupt or jobstore is from an extremely old Toil: %s", attributes)
+            logger.error(
+                "overlargeID attribute isn't present: either SimpleDB entry is "
+                "corrupt or jobstore is from an extremely old Toil: %s",
+                attributes,
+            )
             if enforce:
-                raise RuntimeError("encountered SimpleDB entry missing required attribute "
-                                   "'overlargeID'; is your job store ancient?")
+                raise RuntimeError(
+                    "encountered SimpleDB entry missing required attribute "
+                    "'overlargeID'; is your job store ancient?"
+                )
 
-    def _awsJobFromAttributes(self, attributes: List[AttributeTypeDef]) -> Job:
+    def _awsJobFromAttributes(self, attributes: list["AttributeTypeDef"]) -> Job:
         """
         Get a Toil Job object from attributes that are defined in an item from the DB
         :param attributes: List of attributes
         :return: Toil job
         """
         self._checkAttributes(attributes)
-        overlarge_id_value = get_item_from_attributes(attributes=attributes, name="overlargeID")
+        overlarge_id_value = get_item_from_attributes(
+            attributes=attributes, name="overlargeID"
+        )
         if overlarge_id_value:
             assert self.file_exists(overlarge_id_value)
             # This is an overlarge job, download the actual attributes
@@ -309,15 +369,14 @@ class AWSJobStore(AbstractJobStore):
             job.assignConfig(self.config)
         return job
 
-    def _awsJobFromItem(self, item: ItemTypeDef) -> Job:
+    def _awsJobFromItem(self, item: "ItemTypeDef") -> Job:
         """
         Get a Toil Job object from an item from the DB
-        :param item: ItemTypeDef
         :return: Toil Job
         """
         return self._awsJobFromAttributes(item["Attributes"])
 
-    def _awsJobToAttributes(self, job: JobDescription) -> List[AttributeTypeDef]:
+    def _awsJobToAttributes(self, job: JobDescription) -> list["AttributeTypeDef"]:
         binary = pickle.dumps(job, protocol=pickle.HIGHEST_PROTOCOL)
         if len(binary) > SDBHelper.maxBinarySize(extraReservedChunks=1):
             # Store as an overlarge job in S3
@@ -330,7 +389,7 @@ class AWSJobStore(AbstractJobStore):
             item["overlargeID"] = ""
         return SDBHelper.attributeDictToList(item)
 
-    def _awsJobToItem(self, job: JobDescription, name: str) -> ItemTypeDef:
+    def _awsJobToItem(self, job: JobDescription, name: str) -> "ItemTypeDef":
         return {"Name": name, "Attributes": self._awsJobToAttributes(job)}
 
     jobsPerBatchInsert = 25
@@ -339,27 +398,34 @@ class AWSJobStore(AbstractJobStore):
     def batch(self) -> None:
         self._batchedUpdates = []
         yield
-        batches = [self._batchedUpdates[i:i + self.jobsPerBatchInsert] for i in
-                   range(0, len(self._batchedUpdates), self.jobsPerBatchInsert)]
+        batches = [
+            self._batchedUpdates[i : i + self.jobsPerBatchInsert]
+            for i in range(0, len(self._batchedUpdates), self.jobsPerBatchInsert)
+        ]
 
         for batch in batches:
-            items: List[ReplaceableItemTypeDef] = []
+            items: list["ReplaceableItemTypeDef"] = []
             for jobDescription in batch:
-                item_attributes: List[ReplaceableAttributeTypeDef] = []
+                item_attributes: list["ReplaceableAttributeTypeDef"] = []
                 jobDescription.pre_update_hook()
                 item_name = compat_bytes(jobDescription.jobStoreID)
-                got_job_attributes: List[AttributeTypeDef] = self._awsJobToAttributes(jobDescription)
+                got_job_attributes: list["AttributeTypeDef"] = self._awsJobToAttributes(
+                    jobDescription
+                )
                 for each_attribute in got_job_attributes:
-                    new_attribute: ReplaceableAttributeTypeDef = {"Name": each_attribute["Name"],
-                                                                  "Value": each_attribute["Value"],
-                                                                  "Replace": True}
+                    new_attribute: "ReplaceableAttributeTypeDef" = {
+                        "Name": each_attribute["Name"],
+                        "Value": each_attribute["Value"],
+                        "Replace": True,
+                    }
                     item_attributes.append(new_attribute)
-                items.append({"Name": item_name,
-                              "Attributes": item_attributes})
+                items.append({"Name": item_name, "Attributes": item_attributes})
 
             for attempt in retry_sdb():
                 with attempt:
-                    self.db.batch_put_attributes(DomainName=self.jobs_domain_name, Items=items)
+                    self.db.batch_put_attributes(
+                        DomainName=self.jobs_domain_name, Items=items
+                    )
         self._batchedUpdates = None
 
     def assign_job_id(self, job_description: JobDescription) -> None:
@@ -377,19 +443,28 @@ class AWSJobStore(AbstractJobStore):
     def job_exists(self, job_id: Union[bytes, str]) -> bool:
         for attempt in retry_sdb():
             with attempt:
-                return len(self.db.get_attributes(DomainName=self.jobs_domain_name,
-                                                  ItemName=compat_bytes(job_id),
-                                                  AttributeNames=[SDBHelper.presenceIndicator()],
-                                                  ConsistentRead=True).get("Attributes", [])) > 0
+                return (
+                    len(
+                        self.db.get_attributes(
+                            DomainName=self.jobs_domain_name,
+                            ItemName=compat_bytes(job_id),
+                            AttributeNames=[SDBHelper.presenceIndicator()],
+                            ConsistentRead=True,
+                        ).get("Attributes", [])
+                    )
+                    > 0
+                )
 
     def jobs(self) -> Generator[Job, None, None]:
-        job_items: Optional[List[ItemTypeDef]] = None
+        job_items: Optional[list["ItemTypeDef"]] = None
         for attempt in retry_sdb():
             with attempt:
-                job_items = boto3_pager(self.db.select,
-                                        "Items",
-                                        ConsistentRead=True,
-                                        SelectExpression="select * from `%s`" % self.jobs_domain_name)
+                job_items = boto3_pager(
+                    self.db.select,
+                    "Items",
+                    ConsistentRead=True,
+                    SelectExpression="select * from `%s`" % self.jobs_domain_name,
+                )
         assert job_items is not None
         for jobItem in job_items:
             yield self._awsJobFromItem(jobItem)
@@ -398,9 +473,11 @@ class AWSJobStore(AbstractJobStore):
         item_attributes = None
         for attempt in retry_sdb():
             with attempt:
-                item_attributes = self.db.get_attributes(DomainName=self.jobs_domain_name,
-                                                         ItemName=compat_bytes(job_id),
-                                                         ConsistentRead=True).get("Attributes", [])
+                item_attributes = self.db.get_attributes(
+                    DomainName=self.jobs_domain_name,
+                    ItemName=compat_bytes(job_id),
+                    ConsistentRead=True,
+                ).get("Attributes", [])
         if not item_attributes:
             raise NoSuchJobException(job_id)
         job = self._awsJobFromAttributes(item_attributes)
@@ -413,11 +490,17 @@ class AWSJobStore(AbstractJobStore):
         logger.debug("Updating job %s", job_description.jobStoreID)
         job_description.pre_update_hook()
         job_attributes = self._awsJobToAttributes(job_description)
-        update_attributes: List[ReplaceableAttributeTypeDef] = [{"Name": attribute["Name"], "Value": attribute["Value"], "Replace": True}
-                                                                for attribute in job_attributes]
+        update_attributes: list["ReplaceableAttributeTypeDef"] = [
+            {"Name": attribute["Name"], "Value": attribute["Value"], "Replace": True}
+            for attribute in job_attributes
+        ]
         for attempt in retry_sdb():
             with attempt:
-                self.db.put_attributes(DomainName=self.jobs_domain_name, ItemName=compat_bytes(job_description.jobStoreID), Attributes=update_attributes)
+                self.db.put_attributes(
+                    DomainName=self.jobs_domain_name,
+                    ItemName=compat_bytes(job_description.jobStoreID),
+                    Attributes=update_attributes,
+                )
 
     itemsPerBatchDelete = 25
 
@@ -428,51 +511,75 @@ class AWSJobStore(AbstractJobStore):
         # If the job is overlarge, delete its file from the filestore
         for attempt in retry_sdb():
             with attempt:
-                attributes = self.db.get_attributes(DomainName=self.jobs_domain_name,
-                                                    ItemName=compat_bytes(job_id),
-                                                    ConsistentRead=True).get("Attributes", [])
+                attributes = self.db.get_attributes(
+                    DomainName=self.jobs_domain_name,
+                    ItemName=compat_bytes(job_id),
+                    ConsistentRead=True,
+                ).get("Attributes", [])
         # If the overlargeID has fallen off, maybe we partially deleted the
         # attributes of the item? Or raced on it? Or hit SimpleDB being merely
         # eventually consistent? We should still be able to get rid of it.
         self._checkAttributes(attributes, enforce=False)
-        overlarge_id_value = get_item_from_attributes(attributes=attributes, name="overlargeID")
+        overlarge_id_value = get_item_from_attributes(
+            attributes=attributes, name="overlargeID"
+        )
         if overlarge_id_value:
             logger.debug("Deleting job from filestore")
             self.delete_file(overlarge_id_value)
         for attempt in retry_sdb():
             with attempt:
-                self.db.delete_attributes(DomainName=self.jobs_domain_name, ItemName=compat_bytes(job_id))
-        items: Optional[List[ItemTypeDef]] = None
+                self.db.delete_attributes(
+                    DomainName=self.jobs_domain_name, ItemName=compat_bytes(job_id)
+                )
+        items: Optional[list["ItemTypeDef"]] = None
         for attempt in retry_sdb():
             with attempt:
-                items = list(boto3_pager(self.db.select,
-                                         "Items",
-                                         ConsistentRead=True,
-                                         SelectExpression=f"select version from `{self.files_domain_name}` where ownerID='{job_id}'"))
+                items = list(
+                    boto3_pager(
+                        self.db.select,
+                        "Items",
+                        ConsistentRead=True,
+                        SelectExpression=f"select version from `{self.files_domain_name}` where ownerID='{job_id}'",
+                    )
+                )
         assert items is not None
         if items:
-            logger.debug("Deleting %d file(s) associated with job %s", len(items), job_id)
+            logger.debug(
+                "Deleting %d file(s) associated with job %s", len(items), job_id
+            )
             n = self.itemsPerBatchDelete
-            batches = [items[i:i + n] for i in range(0, len(items), n)]
+            batches = [items[i : i + n] for i in range(0, len(items), n)]
             for batch in batches:
-                delete_items: List[DeletableItemTypeDef] = [{"Name": item["Name"]} for item in batch]
+                delete_items: list["DeletableItemTypeDef"] = [
+                    {"Name": item["Name"]} for item in batch
+                ]
                 for attempt in retry_sdb():
                     with attempt:
-                        self.db.batch_delete_attributes(DomainName=self.files_domain_name, Items=delete_items)
+                        self.db.batch_delete_attributes(
+                            DomainName=self.files_domain_name, Items=delete_items
+                        )
             for item in items:
-                item: ItemTypeDef
-                version = get_item_from_attributes(attributes=item["Attributes"], name="version")
+                item: "ItemTypeDef"
+                version = get_item_from_attributes(
+                    attributes=item["Attributes"], name="version"
+                )
                 for attempt in retry_s3():
                     with attempt:
                         if version:
-                            self.s3_client.delete_object(Bucket=self.files_bucket.name,
-                                                         Key=compat_bytes(item["Name"]),
-                                                         VersionId=version)
+                            self.s3_client.delete_object(
+                                Bucket=self.files_bucket.name,
+                                Key=compat_bytes(item["Name"]),
+                                VersionId=version,
+                            )
                         else:
-                            self.s3_client.delete_object(Bucket=self.files_bucket.name,
-                                                         Key=compat_bytes(item["Name"]))
+                            self.s3_client.delete_object(
+                                Bucket=self.files_bucket.name,
+                                Key=compat_bytes(item["Name"]),
+                            )
 
-    def get_empty_file_store_id(self, jobStoreID=None, cleanup=False, basename=None) -> FileID:
+    def get_empty_file_store_id(
+        self, jobStoreID=None, cleanup=False, basename=None
+    ) -> FileID:
         info = self.FileInfo.create(jobStoreID if cleanup else None)
         with info.uploadStream() as _:
             # Empty
@@ -481,8 +588,14 @@ class AWSJobStore(AbstractJobStore):
         logger.debug("Created %r.", info)
         return info.fileID
 
-    def _import_file(self, otherCls, uri: ParseResult, shared_file_name: Optional[str] = None,
-                     hardlink: bool = False, symlink: bool = True) -> Optional[FileID]:
+    def _import_file(
+        self,
+        otherCls,
+        uri: ParseResult,
+        shared_file_name: Optional[str] = None,
+        hardlink: bool = False,
+        symlink: bool = True,
+    ) -> Optional[FileID]:
         try:
             if issubclass(otherCls, AWSJobStore):
                 srcObj = get_object_for_url(uri, existing=True)
@@ -492,15 +605,19 @@ class AWSJobStore(AbstractJobStore):
                 else:
                     self._requireValidSharedFileName(shared_file_name)
                     jobStoreFileID = self._shared_file_id(shared_file_name)
-                    info = self.FileInfo.loadOrCreate(jobStoreFileID=jobStoreFileID,
-                                                      ownerID=str(self.sharedFileOwnerID),
-                                                      encrypted=None)
+                    info = self.FileInfo.loadOrCreate(
+                        jobStoreFileID=jobStoreFileID,
+                        ownerID=str(self.sharedFileOwnerID),
+                        encrypted=None,
+                    )
                 info.copyFrom(srcObj)
                 info.save()
                 return FileID(info.fileID, size) if shared_file_name is None else None
-        except ServerSideCopyProhibitedError:
-            # AWS refuses to do this copy for us
-            logger.warning("Falling back to copying via the local machine. This could get expensive!")
+        except (NoBucketLocationError, ServerSideCopyProhibitedError):
+            # AWS refuses to tell us where the bucket is or do this copy for us
+            logger.warning(
+                "Falling back to copying via the local machine. This could get expensive!"
+            )
 
         # copy if exception
         return super()._import_file(otherCls, uri, shared_file_name=shared_file_name)
@@ -512,71 +629,112 @@ class AWSJobStore(AbstractJobStore):
                 info = self.FileInfo.loadOrFail(file_id)
                 info.copyTo(dstObj)
                 return
-        except ServerSideCopyProhibitedError:
-            # AWS refuses to do this copy for us
-            logger.warning("Falling back to copying via the local machine. This could get expensive!")
+        except (NoBucketLocationError, ServerSideCopyProhibitedError):
+            # AWS refuses to tell us where the bucket is or do this copy for us
+            logger.warning(
+                "Falling back to copying via the local machine. This could get expensive!"
+            )
         else:
             super()._default_export_file(otherCls, file_id, uri)
 
+    ###
+    # URL access implementation
+    ###
+
+    # URL access methods aren't used by the rest of the job store methods.
+
     @classmethod
     def _url_exists(cls, url: ParseResult) -> bool:
         try:
-            get_object_for_url(url, existing=True)
+            try:
+                get_object_for_url(url, existing=True, anonymous=True)
+            except PermissionError:
+                # If we can't look anonymously, log in
+                get_object_for_url(url, existing=True)
             return True
         except FileNotFoundError:
             # Not a file
-            # Might be a directory.
+            # Might be a directory. Or we might not have access to know.
+            # See if it's a directory.
             return cls._get_is_directory(url)
 
     @classmethod
     def _get_size(cls, url: ParseResult) -> int:
-        return get_object_for_url(url, existing=True).content_length
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+        except PermissionError:
+            src_obj = get_object_for_url(url, existing=True)
+        return src_obj.content_length
 
     @classmethod
     def _read_from_url(cls, url: ParseResult, writable):
-        srcObj = get_object_for_url(url, existing=True)
-        srcObj.download_fileobj(writable)
-        return (
-            srcObj.content_length,
-            False  # executable bit is always False
-        )
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+            src_obj.download_fileobj(writable)
+        except Exception as e:
+            if isinstance(e, PermissionError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+                # The object setup or the download does not have permission. Try again with a login.
+                src_obj = get_object_for_url(url, existing=True)
+                src_obj.download_fileobj(writable)
+            else:
+                raise
+        return (src_obj.content_length, False)  # executable bit is always False
 
     @classmethod
     def _open_url(cls, url: ParseResult) -> IO[bytes]:
-        src_obj = get_object_for_url(url, existing=True)
-        response = src_obj.get()
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+            response = src_obj.get()
+        except Exception as e:
+            if isinstance(e, PermissionError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+                # The object setup or the download does not have permission. Try again with a login.
+                src_obj = get_object_for_url(url, existing=True)
+                response = src_obj.get()
+            else:
+                raise
         # We should get back a response with a stream in 'Body'
-        if 'Body' not in response:
+        if "Body" not in response:
             raise RuntimeError(f"Could not fetch body stream for {url}")
-        return response['Body']
+        return response["Body"]
 
     @classmethod
-    def _write_to_url(cls, readable, url: ParseResult, executable: bool = False) -> None:
+    def _write_to_url(
+        cls, readable, url: ParseResult, executable: bool = False
+    ) -> None:
+        # Don't try to do anonympus writes.
         dstObj = get_object_for_url(url)
 
         logger.debug("Uploading %s", dstObj.key)
         # uploadFile takes care of using multipart upload if the file is larger than partSize (default to 5MB)
-        uploadFile(readable=readable,
-                   resource=s3_boto3_resource,
-                   bucketName=dstObj.bucket_name,
-                   fileID=dstObj.key,
-                   partSize=5 * 1000 * 1000)
+        uploadFile(
+            readable=readable,
+            resource=s3_boto3_resource,
+            bucketName=dstObj.bucket_name,
+            fileID=dstObj.key,
+            partSize=5 * 1000 * 1000,
+        )
 
     @classmethod
-    def _list_url(cls, url: ParseResult) -> List[str]:
-        return list_objects_for_url(url)
+    def _list_url(cls, url: ParseResult) -> list[str]:
+        try:
+            return list_objects_for_url(url, anonymous=True)
+        except PermissionError:
+            return list_objects_for_url(url)
+        
 
     @classmethod
     def _get_is_directory(cls, url: ParseResult) -> bool:
         # We consider it a directory if anything is in it.
         # TODO: Can we just get the first item and not the whole list?
-        return len(list_objects_for_url(url)) > 0
+        return len(cls._list_url(url)) > 0
 
     @classmethod
     def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
-        return url.scheme.lower() == 's3'
+        return url.scheme.lower() == "s3"
 
-    def write_file(self, local_path: FileID, job_id: Optional[FileID] = None, cleanup: bool = False) -> FileID:
+    def write_file(
+        self, local_path: FileID, job_id: Optional[FileID] = None, cleanup: bool = False
+    ) -> FileID:
         info = self.FileInfo.create(job_id if cleanup else None)
         info.upload(local_path, not self.config.disableJobStoreChecksumVerification)
         info.save()
@@ -584,7 +742,14 @@ class AWSJobStore(AbstractJobStore):
         return info.fileID
 
     @contextmanager
-    def write_file_stream(self, job_id: Optional[FileID] = None, cleanup: bool = False, basename=None, encoding=None, errors=None):
+    def write_file_stream(
+        self,
+        job_id: Optional[FileID] = None,
+        cleanup: bool = False,
+        basename=None,
+        encoding=None,
+        errors=None,
+    ):
         info = self.FileInfo.create(job_id if cleanup else None)
         with info.uploadStream(encoding=encoding, errors=errors) as writable:
             yield writable, info.fileID
@@ -592,11 +757,15 @@ class AWSJobStore(AbstractJobStore):
         logger.debug("Wrote %r.", info)
 
     @contextmanager
-    def write_shared_file_stream(self, shared_file_name, encrypted=None, encoding=None, errors=None):
+    def write_shared_file_stream(
+        self, shared_file_name, encrypted=None, encoding=None, errors=None
+    ):
         self._requireValidSharedFileName(shared_file_name)
-        info = self.FileInfo.loadOrCreate(jobStoreFileID=self._shared_file_id(shared_file_name),
-                                          ownerID=str(self.sharedFileOwnerID),
-                                          encrypted=encrypted)
+        info = self.FileInfo.loadOrCreate(
+            jobStoreFileID=self._shared_file_id(shared_file_name),
+            ownerID=str(self.sharedFileOwnerID),
+            encrypted=encrypted,
+        )
         with info.uploadStream(encoding=encoding, errors=errors) as writable:
             yield writable
         info.save()
@@ -629,7 +798,7 @@ class AWSJobStore(AbstractJobStore):
         info = self.FileInfo.loadOrFail(file_id)
         logger.debug("Reading %r into %r.", info, local_path)
         info.download(local_path, not self.config.disableJobStoreChecksumVerification)
-        if getattr(file_id, 'executable', False):
+        if getattr(file_id, "executable", False):
             os.chmod(local_path, os.stat(local_path).st_mode | stat.S_IXUSR)
 
     @contextmanager
@@ -644,7 +813,9 @@ class AWSJobStore(AbstractJobStore):
         self._requireValidSharedFileName(shared_file_name)
         jobStoreFileID = self._shared_file_id(shared_file_name)
         info = self.FileInfo.loadOrFail(jobStoreFileID, customName=shared_file_name)
-        logger.debug("Reading %r for shared file %r into stream.", info, shared_file_name)
+        logger.debug(
+            "Reading %r for shared file %r into stream.", info, shared_file_name
+        )
         with info.downloadStream(encoding=encoding, errors=errors) as readable:
             yield readable
 
@@ -660,7 +831,7 @@ class AWSJobStore(AbstractJobStore):
         with info.uploadStream(multipart=False) as writeable:
             if isinstance(msg, str):
                 # This stream is for binary data, so encode any non-encoded things
-                msg = msg.encode('utf-8', errors='ignore')
+                msg = msg.encode("utf-8", errors="ignore")
             writeable.write(msg)
         info.save()
 
@@ -682,10 +853,12 @@ class AWSJobStore(AbstractJobStore):
         items = None
         for attempt in retry_sdb():
             with attempt:
-                items = boto3_pager(self.db.select,
-                                    "Items",
-                                    ConsistentRead=True,
-                                    SelectExpression="select * from `{}` where ownerID='{}'".format(self.files_domain_name, str(ownerId)))
+                items = boto3_pager(
+                    self.db.select,
+                    "Items",
+                    ConsistentRead=True,
+                    SelectExpression=f"select * from `{self.files_domain_name}` where ownerID='{str(ownerId)}'",
+                )
         assert items is not None
         for item in items:
             info = self.FileInfo.fromItem(item)
@@ -702,13 +875,19 @@ class AWSJobStore(AbstractJobStore):
             with info.uploadStream(allowInlining=False) as f:
                 f.write(info.content)
 
-        self.files_bucket.Object(compat_bytes(jobStoreFileID)).Acl().put(ACL='public-read')
+        self.files_bucket.Object(compat_bytes(jobStoreFileID)).Acl().put(
+            ACL="public-read"
+        )
 
-        url = self.s3_client.generate_presigned_url('get_object',
-                                                    Params={'Bucket': self.files_bucket.name,
-                                                            'Key': compat_bytes(jobStoreFileID),
-                                                            'VersionId': info.version},
-                                                    ExpiresIn=self.publicUrlExpiration.total_seconds())
+        url = self.s3_client.generate_presigned_url(
+            "get_object",
+            Params={
+                "Bucket": self.files_bucket.name,
+                "Key": compat_bytes(jobStoreFileID),
+                "VersionId": info.version,
+            },
+            ExpiresIn=self.publicUrlExpiration.total_seconds(),
+        )
 
         # boto doesn't properly remove the x-amz-security-token parameter when
         # query_auth is False when using an IAM role (see issue #2043). Including the
@@ -716,12 +895,12 @@ class AWSJobStore(AbstractJobStore):
         # even if the resource is public, so we need to remove it.
         scheme, netloc, path, query, fragment = urlsplit(url)
         params = parse_qs(query)
-        if 'x-amz-security-token' in params:
-            del params['x-amz-security-token']
-        if 'AWSAccessKeyId' in params:
-            del params['AWSAccessKeyId']
-        if 'Signature' in params:
-            del params['Signature']
+        if "x-amz-security-token" in params:
+            del params["x-amz-security-token"]
+        if "AWSAccessKeyId" in params:
+            del params["AWSAccessKeyId"]
+        if "Signature" in params:
+            del params["Signature"]
         query = urlencode(params, doseq=True)
         url = urlunsplit((scheme, netloc, path, query, fragment))
         return url
@@ -730,12 +909,14 @@ class AWSJobStore(AbstractJobStore):
         self._requireValidSharedFileName(shared_file_name)
         return self.get_public_url(self._shared_file_id(shared_file_name))
 
-    def _bindBucket(self,
-                    bucket_name: str,
-                    create: bool = False,
-                    block: bool = True,
-                    versioning: bool = False,
-                    check_versioning_consistency: bool = True):
+    def _bindBucket(
+        self,
+        bucket_name: str,
+        create: bool = False,
+        block: bool = True,
+        versioning: bool = False,
+        check_versioning_consistency: bool = True,
+    ):
         """
         Return the Boto Bucket object representing the S3 bucket with the given name. If the
         bucket does not exist and `create` is True, it will be created.
@@ -760,8 +941,7 @@ class AWSJobStore(AbstractJobStore):
             Decide, given an error, whether we should retry binding the bucket.
             """
 
-            if (isinstance(error, ClientError) and
-                    get_error_status(error) in (404, 409)):
+            if isinstance(error, ClientError) and get_error_status(error) in (404, 409):
                 # Handle cases where the bucket creation is in a weird state that might let us proceed.
                 # https://github.com/BD2KGenomics/toil/issues/955
                 # https://github.com/BD2KGenomics/toil/issues/995
@@ -771,7 +951,7 @@ class AWSJobStore(AbstractJobStore):
                 # OperationAborted == 409
                 # NoSuchBucket == 404
                 return True
-            if get_error_code(error) == 'SlowDown':
+            if get_error_code(error) == "SlowDown":
                 # We may get told to SlowDown by AWS when we try to create our
                 # bucket. In that case, we should retry and use the exponential
                 # backoff.
@@ -804,15 +984,17 @@ class AWSJobStore(AbstractJobStore):
                             # NoSuchBucket. We let that kick us back up to the
                             # main retry loop.
                             assert (
-                                    get_bucket_region(bucket_name) == self.region
+                                get_bucket_region(bucket_name) == self.region
                             ), f"bucket_name: {bucket_name}, {get_bucket_region(bucket_name)} != {self.region}"
 
                             tags = build_tag_dict_from_env()
 
                             if tags:
                                 flat_tags = flatten_tags(tags)
-                                bucket_tagging = self.s3_resource.BucketTagging(bucket_name)
-                                bucket_tagging.put(Tagging={'TagSet': flat_tags})
+                                bucket_tagging = self.s3_resource.BucketTagging(
+                                    bucket_name
+                                )
+                                bucket_tagging.put(Tagging={"TagSet": flat_tags})
 
                             # Configure bucket so that we can make objects in
                             # it public, which was the historical default.
@@ -825,7 +1007,9 @@ class AWSJobStore(AbstractJobStore):
                         # This is raised if the user attempts to get a bucket in a region outside
                         # the specified one, if the specified one is not `us-east-1`.  The us-east-1
                         # server allows a user to use buckets from any region.
-                        raise BucketLocationConflictException(get_bucket_region(bucket_name))
+                        raise BucketLocationConflictException(
+                            get_bucket_region(bucket_name)
+                        )
                     else:
                         raise
                 else:
@@ -842,24 +1026,32 @@ class AWSJobStore(AbstractJobStore):
                     # consistent?
                     time.sleep(1)
                     while not self._getBucketVersioning(bucket_name):
-                        logger.warning(f"Waiting for versioning activation on bucket '{bucket_name}'...")
+                        logger.warning(
+                            f"Waiting for versioning activation on bucket '{bucket_name}'..."
+                        )
                         time.sleep(1)
                 elif check_versioning_consistency:
                     # now test for versioning consistency
                     # we should never see any of these errors since 'versioning' should always be true
                     bucket_versioning = self._getBucketVersioning(bucket_name)
                     if bucket_versioning != versioning:
-                        assert False, 'Cannot modify versioning on existing bucket'
+                        assert False, "Cannot modify versioning on existing bucket"
                     elif bucket_versioning is None:
-                        assert False, 'Cannot use a bucket with versioning suspended'
+                        assert False, "Cannot use a bucket with versioning suspended"
                 if bucketExisted:
-                    logger.debug(f"Using pre-existing job store bucket '{bucket_name}'.")
+                    logger.debug(
+                        f"Using pre-existing job store bucket '{bucket_name}'."
+                    )
                 else:
-                    logger.debug(f"Created new job store bucket '{bucket_name}' with versioning state {versioning}.")
+                    logger.debug(
+                        f"Created new job store bucket '{bucket_name}' with versioning state {versioning}."
+                    )
 
                 return bucket
 
-    def _bindDomain(self, domain_name: str, create: bool = False, block: bool = True) -> None:
+    def _bindDomain(
+        self, domain_name: str, create: bool = False, block: bool = True
+    ) -> None:
         """
         Return the Boto3 domain name representing the SDB domain. When create=True, it will
         create the domain if it does not exist.
@@ -878,9 +1070,11 @@ class AWSJobStore(AbstractJobStore):
                 retry timeout expires.
         """
         logger.debug("Binding to job store domain '%s'.", domain_name)
-        retryargs = dict(predicate=lambda e: no_such_sdb_domain(e) or sdb_unavailable(e))
+        retryargs = dict(
+            predicate=lambda e: no_such_sdb_domain(e) or sdb_unavailable(e)
+        )
         if not block:
-            retryargs['timeout'] = 15
+            retryargs["timeout"] = 15
         for attempt in retry_sdb(**retryargs):
             with attempt:
                 try:
@@ -902,13 +1096,13 @@ class AWSJobStore(AbstractJobStore):
         return str(uuid.uuid4())
 
     # A dummy job ID under which all shared files are stored
-    sharedFileOwnerID = uuid.UUID('891f7db6-e4d9-4221-a58e-ab6cc4395f94')
+    sharedFileOwnerID = uuid.UUID("891f7db6-e4d9-4221-a58e-ab6cc4395f94")
 
     # A dummy job ID under which all unread stats files are stored
-    statsFileOwnerID = uuid.UUID('bfcf5286-4bc7-41ef-a85d-9ab415b69d53')
+    statsFileOwnerID = uuid.UUID("bfcf5286-4bc7-41ef-a85d-9ab415b69d53")
 
     # A dummy job ID under which all read stats files are stored
-    readStatsFileOwnerID = uuid.UUID('e77fc3aa-d232-4255-ae04-f64ee8eb0bfa')
+    readStatsFileOwnerID = uuid.UUID("e77fc3aa-d232-4255-ae04-f64ee8eb0bfa")
 
     def _shared_file_id(self, shared_file_name):
         return str(uuid.uuid5(self.sharedFileOwnerID, shared_file_name))
@@ -918,13 +1112,22 @@ class AWSJobStore(AbstractJobStore):
         """
         Represents a file in this job store.
         """
+
         outer = None
         """
         :type: AWSJobStore
         """
 
-        def __init__(self, fileID, ownerID, encrypted,
-                     version=None, content=None, numContentChunks=0, checksum=None):
+        def __init__(
+            self,
+            fileID,
+            ownerID,
+            encrypted,
+            version=None,
+            content=None,
+            numContentChunks=0,
+            checksum=None,
+        ):
             """
             :type fileID: str
             :param fileID: the file's ID
@@ -1003,24 +1206,30 @@ class AWSJobStore(AbstractJobStore):
             assert content is None or isinstance(content, bytes)
             self._content = content
             if content is not None:
-                self.version = ''
+                self.version = ""
 
         @classmethod
         def create(cls, ownerID: str):
-            return cls(str(uuid.uuid4()), ownerID, encrypted=cls.outer.sseKeyPath is not None)
+            return cls(
+                str(uuid.uuid4()), ownerID, encrypted=cls.outer.sseKeyPath is not None
+            )
 
         @classmethod
         def presenceIndicator(cls):
-            return 'encrypted'
+            return "encrypted"
 
         @classmethod
         def exists(cls, jobStoreFileID):
             for attempt in retry_sdb():
                 with attempt:
-                    return bool(cls.outer.db.get_attributes(DomainName=cls.outer.files_domain_name,
-                                                            ItemName=compat_bytes(jobStoreFileID),
-                                                            AttributeNames=[cls.presenceIndicator()],
-                                                            ConsistentRead=True).get("Attributes", []))
+                    return bool(
+                        cls.outer.db.get_attributes(
+                            DomainName=cls.outer.files_domain_name,
+                            ItemName=compat_bytes(jobStoreFileID),
+                            AttributeNames=[cls.presenceIndicator()],
+                            ConsistentRead=True,
+                        ).get("Attributes", [])
+                    )
 
         @classmethod
         def load(cls, jobStoreFileID):
@@ -1029,10 +1238,13 @@ class AWSJobStore(AbstractJobStore):
                     self = cls.fromItem(
                         {
                             "Name": compat_bytes(jobStoreFileID),
-                            "Attributes": cls.outer.db.get_attributes(DomainName=cls.outer.files_domain_name,
-                                                                      ItemName=compat_bytes(jobStoreFileID),
-                                                                      ConsistentRead=True).get("Attributes", [])
-                        })
+                            "Attributes": cls.outer.db.get_attributes(
+                                DomainName=cls.outer.files_domain_name,
+                                ItemName=compat_bytes(jobStoreFileID),
+                                ConsistentRead=True,
+                            ).get("Attributes", []),
+                        }
+                    )
                     return self
 
         @classmethod
@@ -1062,7 +1274,7 @@ class AWSJobStore(AbstractJobStore):
                 return self
 
         @classmethod
-        def fromItem(cls, item: ItemTypeDef):
+        def fromItem(cls, item: "ItemTypeDef"):
             """
             Convert an SDB item to an instance of this class.
 
@@ -1075,7 +1287,9 @@ class AWSJobStore(AbstractJobStore):
                 return s if s is None else str(s)
 
             # ownerID and encrypted are the only mandatory attributes
-            ownerID, encrypted, version, checksum = SDBHelper.get_attributes_from_item(item, ["ownerID", "encrypted", "version", "checksum"])
+            ownerID, encrypted, version, checksum = SDBHelper.get_attributes_from_item(
+                item, ["ownerID", "encrypted", "version", "checksum"]
+            )
             if ownerID is None:
                 assert encrypted is None
                 return None
@@ -1085,14 +1299,23 @@ class AWSJobStore(AbstractJobStore):
                 if encrypted:
                     sseKeyPath = cls.outer.sseKeyPath
                     if sseKeyPath is None:
-                        raise AssertionError('Content is encrypted but no key was provided.')
+                        raise AssertionError(
+                            "Content is encrypted but no key was provided."
+                        )
                     if content is not None:
                         content = encryption.decrypt(content, sseKeyPath)
-                self = cls(fileID=item["Name"], ownerID=ownerID, encrypted=encrypted, version=version,
-                           content=content, numContentChunks=numContentChunks, checksum=checksum)
+                self = cls(
+                    fileID=item["Name"],
+                    ownerID=ownerID,
+                    encrypted=encrypted,
+                    version=version,
+                    content=content,
+                    numContentChunks=numContentChunks,
+                    checksum=checksum,
+                )
                 return self
 
-        def toItem(self) -> Tuple[Dict[str, str], int]:
+        def toItem(self) -> tuple[dict[str, str], int]:
             """
             Convert this instance to a dictionary of attribute names to values
 
@@ -1104,15 +1327,21 @@ class AWSJobStore(AbstractJobStore):
             if self.encrypted and content is not None:
                 sseKeyPath = self.outer.sseKeyPath
                 if sseKeyPath is None:
-                    raise AssertionError('Encryption requested but no key was provided.')
+                    raise AssertionError(
+                        "Encryption requested but no key was provided."
+                    )
                 content = encryption.encrypt(content, sseKeyPath)
             assert content is None or isinstance(content, bytes)
             attributes = self.binaryToAttributes(content)
-            numChunks = int(attributes['numChunks'])
-            attributes.update(dict(ownerID=self.ownerID or '',
-                                   encrypted=str(self.encrypted),
-                                   version=self.version or '',
-                                   checksum=self.checksum or ''))
+            numChunks = int(attributes["numChunks"])
+            attributes.update(
+                dict(
+                    ownerID=self.ownerID or "",
+                    encrypted=str(self.encrypted),
+                    version=self.version or "",
+                    checksum=self.checksum or "",
+                )
+            )
             return attributes, numChunks
 
         @classmethod
@@ -1128,24 +1357,37 @@ class AWSJobStore(AbstractJobStore):
             attributes_boto3 = SDBHelper.attributeDictToList(attributes)
             # False stands for absence
             if self.previousVersion is None:
-                expected: UpdateConditionTypeDef = {"Name": 'version', "Exists": False}
+                expected: "UpdateConditionTypeDef" = {
+                    "Name": "version",
+                    "Exists": False,
+                }
             else:
-                expected = {"Name": 'version', "Value": cast(str, self.previousVersion)}
+                expected = {"Name": "version", "Value": cast(str, self.previousVersion)}
             try:
                 for attempt in retry_sdb():
                     with attempt:
-                        self.outer.db.put_attributes(DomainName=self.outer.files_domain_name,
-                                                     ItemName=compat_bytes(self.fileID),
-                                                     Attributes=[{"Name": attribute["Name"], "Value": attribute["Value"], "Replace": True}
-                                                                 for attribute in attributes_boto3],
-                                                     Expected=expected)
+                        self.outer.db.put_attributes(
+                            DomainName=self.outer.files_domain_name,
+                            ItemName=compat_bytes(self.fileID),
+                            Attributes=[
+                                {
+                                    "Name": attribute["Name"],
+                                    "Value": attribute["Value"],
+                                    "Replace": True,
+                                }
+                                for attribute in attributes_boto3
+                            ],
+                            Expected=expected,
+                        )
                 # clean up the old version of the file if necessary and safe
                 if self.previousVersion and (self.previousVersion != self.version):
                     for attempt in retry_s3():
                         with attempt:
-                            self.outer.s3_client.delete_object(Bucket=self.outer.files_bucket.name,
-                                                               Key=compat_bytes(self.fileID),
-                                                               VersionId=self.previousVersion)
+                            self.outer.s3_client.delete_object(
+                                Bucket=self.outer.files_bucket.name,
+                                Key=compat_bytes(self.fileID),
+                                VersionId=self.previousVersion,
+                            )
                 self._previousVersion = self._version
                 if numNewContentChunks < self._numContentChunks:
                     residualChunks = range(numNewContentChunks, self._numContentChunks)
@@ -1153,19 +1395,26 @@ class AWSJobStore(AbstractJobStore):
                     # boto3 requires providing the value as well as the name in the attribute, and we don't store it locally
                     # the php sdk resolves this issue by not requiring the Value key https://github.com/aws/aws-sdk-php/issues/185
                     # but this doesnt extend to boto3
-                    delete_attributes = self.outer.db.get_attributes(DomainName=self.outer.files_domain_name,
-                                                                     ItemName=compat_bytes(self.fileID),
-                                                                     AttributeNames=[chunk for chunk in residual_chunk_names]).get("Attributes")
+                    delete_attributes = self.outer.db.get_attributes(
+                        DomainName=self.outer.files_domain_name,
+                        ItemName=compat_bytes(self.fileID),
+                        AttributeNames=[chunk for chunk in residual_chunk_names],
+                    ).get("Attributes")
                     for attempt in retry_sdb():
                         with attempt:
-                            self.outer.db.delete_attributes(DomainName=self.outer.files_domain_name,
-                                                            ItemName=compat_bytes(self.fileID),
-                                                            Attributes=delete_attributes)
-                    self.outer.db.get_attributes(DomainName=self.outer.files_domain_name, ItemName=compat_bytes(self.fileID))
+                            self.outer.db.delete_attributes(
+                                DomainName=self.outer.files_domain_name,
+                                ItemName=compat_bytes(self.fileID),
+                                Attributes=delete_attributes,
+                            )
+                    self.outer.db.get_attributes(
+                        DomainName=self.outer.files_domain_name,
+                        ItemName=compat_bytes(self.fileID),
+                    )
 
                 self._numContentChunks = numNewContentChunks
             except ClientError as e:
-                if get_error_code(e) == 'ConditionalCheckFailed':
+                if get_error_code(e) == "ConditionalCheckFailed":
                     raise ConcurrentFileModificationException(self.fileID)
                 else:
                     raise
@@ -1173,24 +1422,30 @@ class AWSJobStore(AbstractJobStore):
         def upload(self, localFilePath, calculateChecksum=True):
             file_size, file_time = fileSizeAndTime(localFilePath)
             if file_size <= self.maxInlinedSize():
-                with open(localFilePath, 'rb') as f:
+                with open(localFilePath, "rb") as f:
                     self.content = f.read()
                 # Clear out any old checksum in case of overwrite
-                self.checksum = ''
+                self.checksum = ""
             else:
                 headerArgs = self._s3EncryptionArgs()
                 # Create a new Resource in case it needs to be on its own thread
-                resource = boto3_session.resource('s3', region_name=self.outer.region)
-
-                self.checksum = self._get_file_checksum(localFilePath) if calculateChecksum else None
-                self.version = uploadFromPath(localFilePath,
-                                              resource=resource,
-                                              bucketName=self.outer.files_bucket.name,
-                                              fileID=compat_bytes(self.fileID),
-                                              headerArgs=headerArgs,
-                                              partSize=self.outer.part_size)
-
-        def _start_checksum(self, to_match=None, algorithm='sha1'):
+                resource = boto3_session.resource("s3", region_name=self.outer.region)
+
+                self.checksum = (
+                    self._get_file_checksum(localFilePath)
+                    if calculateChecksum
+                    else None
+                )
+                self.version = uploadFromPath(
+                    localFilePath,
+                    resource=resource,
+                    bucketName=self.outer.files_bucket.name,
+                    fileID=compat_bytes(self.fileID),
+                    headerArgs=headerArgs,
+                    partSize=self.outer.part_size,
+                )
+
+        def _start_checksum(self, to_match=None, algorithm="sha1"):
             """
             Get a hasher that can be used with _update_checksum and
             _finish_checksum.
@@ -1208,12 +1463,12 @@ class AWSJobStore(AbstractJobStore):
             expected = None
 
             if to_match is not None:
-                parts = to_match.split('$')
+                parts = to_match.split("$")
                 algorithm = parts[0]
                 expected = parts[1]
 
             wrapped = getattr(hashlib, algorithm)()
-            logger.debug(f'Starting {algorithm} checksum to match {expected}')
+            logger.debug(f"Starting {algorithm} checksum to match {expected}")
             return algorithm, wrapped, expected
 
         def _update_checksum(self, checksum_in_progress, data):
@@ -1230,26 +1485,32 @@ class AWSJobStore(AbstractJobStore):
 
             result_hash = checksum_in_progress[1].hexdigest()
 
-            logger.debug(f'Completed checksum with hash {result_hash} vs. expected {checksum_in_progress[2]}')
+            logger.debug(
+                f"Completed checksum with hash {result_hash} vs. expected {checksum_in_progress[2]}"
+            )
             if checksum_in_progress[2] is not None:
                 # We expected a particular hash
                 if result_hash != checksum_in_progress[2]:
-                    raise ChecksumError('Checksum mismatch. Expected: %s Actual: %s' %
-                                        (checksum_in_progress[2], result_hash))
+                    raise ChecksumError(
+                        "Checksum mismatch. Expected: %s Actual: %s"
+                        % (checksum_in_progress[2], result_hash)
+                    )
 
-            return '$'.join([checksum_in_progress[0], result_hash])
+            return "$".join([checksum_in_progress[0], result_hash])
 
         def _get_file_checksum(self, localFilePath, to_match=None):
-            with open(localFilePath, 'rb') as f:
+            with open(localFilePath, "rb") as f:
                 hasher = self._start_checksum(to_match=to_match)
                 contents = f.read(1024 * 1024)
-                while contents != b'':
+                while contents != b"":
                     self._update_checksum(hasher, contents)
                     contents = f.read(1024 * 1024)
                 return self._finish_checksum(hasher)
 
         @contextmanager
-        def uploadStream(self, multipart=True, allowInlining=True, encoding=None, errors=None):
+        def uploadStream(
+            self, multipart=True, allowInlining=True, encoding=None, errors=None
+        ):
             """
             Context manager that gives out a binary or text mode upload stream to upload data.
             """
@@ -1270,14 +1531,14 @@ class AWSJobStore(AbstractJobStore):
                     assert isinstance(buf, bytes)
 
                     if allowInlining and len(buf) <= info.maxInlinedSize():
-                        logger.debug('Inlining content of %d bytes', len(buf))
+                        logger.debug("Inlining content of %d bytes", len(buf))
                         info.content = buf
                         # There will be no checksum
-                        info.checksum = ''
+                        info.checksum = ""
                     else:
                         # We will compute a checksum
                         hasher = info._start_checksum()
-                        logger.debug('Updating checksum with %d bytes', len(buf))
+                        logger.debug("Updating checksum with %d bytes", len(buf))
                         info._update_checksum(hasher, buf)
 
                         client = store.s3_client
@@ -1286,47 +1547,72 @@ class AWSJobStore(AbstractJobStore):
 
                         for attempt in retry_s3():
                             with attempt:
-                                logger.debug('Starting multipart upload')
+                                logger.debug("Starting multipart upload")
                                 # low-level clients are thread safe
-                                upload = client.create_multipart_upload(Bucket=bucket_name,
-                                                                        Key=compat_bytes(info.fileID),
-                                                                        **headerArgs)
-                                uploadId = upload['UploadId']
+                                upload = client.create_multipart_upload(
+                                    Bucket=bucket_name,
+                                    Key=compat_bytes(info.fileID),
+                                    **headerArgs,
+                                )
+                                uploadId = upload["UploadId"]
                                 parts = []
-                                logger.debug('Multipart upload started as %s', uploadId)
+                                logger.debug("Multipart upload started as %s", uploadId)
 
                         for attempt in retry_s3():
                             with attempt:
                                 for i in range(CONSISTENCY_TICKS):
                                     # Sometimes we can create a multipart upload and not see it. Wait around for it.
-                                    response = client.list_multipart_uploads(Bucket=bucket_name,
-                                                                             MaxUploads=1,
-                                                                             Prefix=compat_bytes(info.fileID))
-                                    if ('Uploads' in response and
-                                            len(response['Uploads']) != 0 and
-                                            response['Uploads'][0]['UploadId'] == uploadId):
-
-                                        logger.debug('Multipart upload visible as %s', uploadId)
+                                    response = client.list_multipart_uploads(
+                                        Bucket=bucket_name,
+                                        MaxUploads=1,
+                                        Prefix=compat_bytes(info.fileID),
+                                    )
+                                    if (
+                                        "Uploads" in response
+                                        and len(response["Uploads"]) != 0
+                                        and response["Uploads"][0]["UploadId"]
+                                        == uploadId
+                                    ):
+
+                                        logger.debug(
+                                            "Multipart upload visible as %s", uploadId
+                                        )
                                         break
                                     else:
-                                        logger.debug('Multipart upload %s is not visible; we see %s', uploadId, response.get('Uploads'))
-                                        time.sleep(CONSISTENCY_TIME * 2 ** i)
+                                        logger.debug(
+                                            "Multipart upload %s is not visible; we see %s",
+                                            uploadId,
+                                            response.get("Uploads"),
+                                        )
+                                        time.sleep(CONSISTENCY_TIME * 2**i)
 
                         try:
                             for part_num in itertools.count():
                                 for attempt in retry_s3():
                                     with attempt:
-                                        logger.debug('Uploading part %d of %d bytes to %s', part_num + 1, len(buf), uploadId)
+                                        logger.debug(
+                                            "Uploading part %d of %d bytes to %s",
+                                            part_num + 1,
+                                            len(buf),
+                                            uploadId,
+                                        )
                                         # TODO: include the Content-MD5 header:
                                         #  https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.complete_multipart_upload
-                                        part = client.upload_part(Bucket=bucket_name,
-                                                                  Key=compat_bytes(info.fileID),
-                                                                  PartNumber=part_num + 1,
-                                                                  UploadId=uploadId,
-                                                                  Body=BytesIO(buf),
-                                                                  **headerArgs)
-
-                                        parts.append({"PartNumber": part_num + 1, "ETag": part["ETag"]})
+                                        part = client.upload_part(
+                                            Bucket=bucket_name,
+                                            Key=compat_bytes(info.fileID),
+                                            PartNumber=part_num + 1,
+                                            UploadId=uploadId,
+                                            Body=BytesIO(buf),
+                                            **headerArgs,
+                                        )
+
+                                        parts.append(
+                                            {
+                                                "PartNumber": part_num + 1,
+                                                "ETag": part["ETag"],
+                                            }
+                                        )
 
                                 # Get the next block of data we want to put
                                 buf = readable.read(info.outer.part_size)
@@ -1339,15 +1625,21 @@ class AWSJobStore(AbstractJobStore):
                             with panic(log=logger):
                                 for attempt in retry_s3():
                                     with attempt:
-                                        client.abort_multipart_upload(Bucket=bucket_name,
-                                                                      Key=compat_bytes(info.fileID),
-                                                                      UploadId=uploadId)
+                                        client.abort_multipart_upload(
+                                            Bucket=bucket_name,
+                                            Key=compat_bytes(info.fileID),
+                                            UploadId=uploadId,
+                                        )
 
                         else:
 
-                            while not store._getBucketVersioning(store.files_bucket.name):
-                                logger.warning('Versioning does not appear to be enabled yet. Deferring multipart '
-                                               'upload completion...')
+                            while not store._getBucketVersioning(
+                                store.files_bucket.name
+                            ):
+                                logger.warning(
+                                    "Versioning does not appear to be enabled yet. Deferring multipart "
+                                    "upload completion..."
+                                )
                                 time.sleep(1)
 
                             # Save the checksum
@@ -1359,32 +1651,46 @@ class AWSJobStore(AbstractJobStore):
                                 # in tests
                                 # (https://github.com/DataBiosphere/toil/issues/3894)
                                 with attempt:
-                                    logger.debug('Attempting to complete upload...')
+                                    logger.debug("Attempting to complete upload...")
                                     completed = client.complete_multipart_upload(
                                         Bucket=bucket_name,
                                         Key=compat_bytes(info.fileID),
                                         UploadId=uploadId,
-                                        MultipartUpload={"Parts": parts})
-
-                                    logger.debug('Completed upload object of type %s: %s', str(type(completed)),
-                                                 repr(completed))
-                                    info.version = completed.get('VersionId')
-                                    logger.debug('Completed upload with version %s', str(info.version))
+                                        MultipartUpload={"Parts": parts},
+                                    )
+
+                                    logger.debug(
+                                        "Completed upload object of type %s: %s",
+                                        str(type(completed)),
+                                        repr(completed),
+                                    )
+                                    info.version = completed.get("VersionId")
+                                    logger.debug(
+                                        "Completed upload with version %s",
+                                        str(info.version),
+                                    )
 
                             if info.version is None:
                                 # Somehow we don't know the version. Try and get it.
-                                for attempt in retry_s3(predicate=lambda e: retryable_s3_errors(e) or isinstance(e, AssertionError)):
+                                for attempt in retry_s3(
+                                    predicate=lambda e: retryable_s3_errors(e)
+                                    or isinstance(e, AssertionError)
+                                ):
                                     with attempt:
-                                        version = client.head_object(Bucket=bucket_name,
-                                                                     Key=compat_bytes(info.fileID),
-                                                                     **headerArgs).get('VersionId', None)
-                                        logger.warning('Loaded key for upload with no version and got version %s',
-                                                       str(version))
+                                        version = client.head_object(
+                                            Bucket=bucket_name,
+                                            Key=compat_bytes(info.fileID),
+                                            **headerArgs,
+                                        ).get("VersionId", None)
+                                        logger.warning(
+                                            "Loaded key for upload with no version and got version %s",
+                                            str(version),
+                                        )
                                         info.version = version
                                         assert info.version is not None
 
                     # Make sure we actually wrote something, even if an empty file
-                    assert (bool(info.version) or info.content is not None)
+                    assert bool(info.version) or info.content is not None
 
             class SinglePartPipe(WritablePipe):
                 def readFrom(self, readable):
@@ -1392,10 +1698,10 @@ class AWSJobStore(AbstractJobStore):
                     assert isinstance(buf, bytes)
                     dataLength = len(buf)
                     if allowInlining and dataLength <= info.maxInlinedSize():
-                        logger.debug('Inlining content of %d bytes', len(buf))
+                        logger.debug("Inlining content of %d bytes", len(buf))
                         info.content = buf
                         # There will be no checksum
-                        info.checksum = ''
+                        info.checksum = ""
                     else:
                         # We will compute a checksum
                         hasher = info._start_checksum()
@@ -1409,39 +1715,57 @@ class AWSJobStore(AbstractJobStore):
                         buf = BytesIO(buf)
 
                         while not store._getBucketVersioning(bucket_name):
-                            logger.warning('Versioning does not appear to be enabled yet. Deferring single part '
-                                           'upload...')
+                            logger.warning(
+                                "Versioning does not appear to be enabled yet. Deferring single part "
+                                "upload..."
+                            )
                             time.sleep(1)
 
                         for attempt in retry_s3():
                             with attempt:
-                                logger.debug('Uploading single part of %d bytes', dataLength)
-                                client.upload_fileobj(Bucket=bucket_name,
-                                                      Key=compat_bytes(info.fileID),
-                                                      Fileobj=buf,
-                                                      ExtraArgs=headerArgs)
+                                logger.debug(
+                                    "Uploading single part of %d bytes", dataLength
+                                )
+                                client.upload_fileobj(
+                                    Bucket=bucket_name,
+                                    Key=compat_bytes(info.fileID),
+                                    Fileobj=buf,
+                                    ExtraArgs=headerArgs,
+                                )
 
                                 # use head_object with the SSE headers to access versionId and content_length attributes
-                                headObj = client.head_object(Bucket=bucket_name,
-                                                             Key=compat_bytes(info.fileID),
-                                                             **headerArgs)
-                                assert dataLength == headObj.get('ContentLength', None)
-                                info.version = headObj.get('VersionId', None)
-                                logger.debug('Upload received version %s', str(info.version))
+                                headObj = client.head_object(
+                                    Bucket=bucket_name,
+                                    Key=compat_bytes(info.fileID),
+                                    **headerArgs,
+                                )
+                                assert dataLength == headObj.get("ContentLength", None)
+                                info.version = headObj.get("VersionId", None)
+                                logger.debug(
+                                    "Upload received version %s", str(info.version)
+                                )
 
                         if info.version is None:
                             # Somehow we don't know the version
-                            for attempt in retry_s3(predicate=lambda e: retryable_s3_errors(e) or isinstance(e, AssertionError)):
+                            for attempt in retry_s3(
+                                predicate=lambda e: retryable_s3_errors(e)
+                                or isinstance(e, AssertionError)
+                            ):
                                 with attempt:
-                                    headObj = client.head_object(Bucket=bucket_name,
-                                                                 Key=compat_bytes(info.fileID),
-                                                                 **headerArgs)
-                                    info.version = headObj.get('VersionId', None)
-                                    logger.warning('Reloaded key with no version and got version %s', str(info.version))
+                                    headObj = client.head_object(
+                                        Bucket=bucket_name,
+                                        Key=compat_bytes(info.fileID),
+                                        **headerArgs,
+                                    )
+                                    info.version = headObj.get("VersionId", None)
+                                    logger.warning(
+                                        "Reloaded key with no version and got version %s",
+                                        str(info.version),
+                                    )
                                     assert info.version is not None
 
                     # Make sure we actually wrote something, even if an empty file
-                    assert (bool(info.version) or info.content is not None)
+                    assert bool(info.version) or info.content is not None
 
             if multipart:
                 pipe = MultiPartPipe(encoding=encoding, errors=errors)
@@ -1452,20 +1776,22 @@ class AWSJobStore(AbstractJobStore):
                 yield writable
 
             if not pipe.reader_done:
-                logger.debug(f'Version: {self.version} Content: {self.content}')
-                raise RuntimeError('Escaped context manager without written data being read!')
+                logger.debug(f"Version: {self.version} Content: {self.content}")
+                raise RuntimeError(
+                    "Escaped context manager without written data being read!"
+                )
 
             # We check our work to make sure we have exactly one of embedded
             # content or a real object version.
 
             if self.content is None:
                 if not bool(self.version):
-                    logger.debug(f'Version: {self.version} Content: {self.content}')
-                    raise RuntimeError('No content added and no version created')
+                    logger.debug(f"Version: {self.version} Content: {self.content}")
+                    raise RuntimeError("No content added and no version created")
             else:
                 if bool(self.version):
-                    logger.debug(f'Version: {self.version} Content: {self.content}')
-                    raise RuntimeError('Content added and version created')
+                    logger.debug(f"Version: {self.version} Content: {self.content}")
+                    raise RuntimeError("Content added and version created")
 
         def copyFrom(self, srcObj):
             """
@@ -1475,18 +1801,20 @@ class AWSJobStore(AbstractJobStore):
             """
             assert srcObj.content_length is not None
             if srcObj.content_length <= self.maxInlinedSize():
-                self.content = srcObj.get().get('Body').read()
+                self.content = srcObj.get().get("Body").read()
             else:
                 # Create a new Resource in case it needs to be on its own thread
-                resource = boto3_session.resource('s3', region_name=self.outer.region)
-                self.version = copyKeyMultipart(resource,
-                                                srcBucketName=compat_bytes(srcObj.bucket_name),
-                                                srcKeyName=compat_bytes(srcObj.key),
-                                                srcKeyVersion=compat_bytes(srcObj.version_id),
-                                                dstBucketName=compat_bytes(self.outer.files_bucket.name),
-                                                dstKeyName=compat_bytes(self._fileID),
-                                                sseAlgorithm='AES256',
-                                                sseKey=self._getSSEKey())
+                resource = boto3_session.resource("s3", region_name=self.outer.region)
+                self.version = copyKeyMultipart(
+                    resource,
+                    srcBucketName=compat_bytes(srcObj.bucket_name),
+                    srcKeyName=compat_bytes(srcObj.key),
+                    srcKeyVersion=compat_bytes(srcObj.version_id),
+                    dstBucketName=compat_bytes(self.outer.files_bucket.name),
+                    dstKeyName=compat_bytes(self._fileID),
+                    sseAlgorithm="AES256",
+                    sseKey=self._getSSEKey(),
+                )
 
         def copyTo(self, dstObj):
             """
@@ -1500,35 +1828,43 @@ class AWSJobStore(AbstractJobStore):
                         dstObj.put(Body=self.content)
             elif self.version:
                 # Create a new Resource in case it needs to be on its own thread
-                resource = boto3_session.resource('s3', region_name=self.outer.region)
+                resource = boto3_session.resource("s3", region_name=self.outer.region)
 
                 for attempt in retry_s3():
                     # encrypted = True if self.outer.sseKeyPath else False
                     with attempt:
-                        copyKeyMultipart(resource,
-                                         srcBucketName=compat_bytes(self.outer.files_bucket.name),
-                                         srcKeyName=compat_bytes(self.fileID),
-                                         srcKeyVersion=compat_bytes(self.version),
-                                         dstBucketName=compat_bytes(dstObj.bucket_name),
-                                         dstKeyName=compat_bytes(dstObj.key),
-                                         copySourceSseAlgorithm='AES256',
-                                         copySourceSseKey=self._getSSEKey())
+                        copyKeyMultipart(
+                            resource,
+                            srcBucketName=compat_bytes(self.outer.files_bucket.name),
+                            srcKeyName=compat_bytes(self.fileID),
+                            srcKeyVersion=compat_bytes(self.version),
+                            dstBucketName=compat_bytes(dstObj.bucket_name),
+                            dstKeyName=compat_bytes(dstObj.key),
+                            copySourceSseAlgorithm="AES256",
+                            copySourceSseKey=self._getSSEKey(),
+                        )
             else:
                 assert False
 
         def download(self, localFilePath, verifyChecksum=True):
             if self.content is not None:
                 with AtomicFileCreate(localFilePath) as tmpPath:
-                    with open(tmpPath, 'wb') as f:
+                    with open(tmpPath, "wb") as f:
                         f.write(self.content)
             elif self.version:
                 headerArgs = self._s3EncryptionArgs()
                 obj = self.outer.files_bucket.Object(compat_bytes(self.fileID))
 
-                for attempt in retry_s3(predicate=lambda e: retryable_s3_errors(e) or isinstance(e, ChecksumError)):
+                for attempt in retry_s3(
+                    predicate=lambda e: retryable_s3_errors(e)
+                    or isinstance(e, ChecksumError)
+                ):
                     with attempt:
                         with AtomicFileCreate(localFilePath) as tmpPath:
-                            obj.download_file(Filename=tmpPath, ExtraArgs={'VersionId': self.version, **headerArgs})
+                            obj.download_file(
+                                Filename=tmpPath,
+                                ExtraArgs={"VersionId": self.version, **headerArgs},
+                            )
 
                         if verifyChecksum and self.checksum:
                             try:
@@ -1536,7 +1872,10 @@ class AWSJobStore(AbstractJobStore):
                                 self._get_file_checksum(localFilePath, self.checksum)
                             except ChecksumError as e:
                                 # Annotate checksum mismatches with file name
-                                raise ChecksumError('Checksums do not match for file %s.' % localFilePath) from e
+                                raise ChecksumError(
+                                    "Checksums do not match for file %s."
+                                    % localFilePath
+                                ) from e
                                 # The error will get caught and result in a retry of the download until we run out of retries.
                                 # TODO: handle obviously truncated downloads by resuming instead.
             else:
@@ -1558,7 +1897,10 @@ class AWSJobStore(AbstractJobStore):
                         obj = info.outer.files_bucket.Object(compat_bytes(info.fileID))
                         for attempt in retry_s3():
                             with attempt:
-                                obj.download_fileobj(writable, ExtraArgs={'VersionId': info.version, **headerArgs})
+                                obj.download_fileobj(
+                                    writable,
+                                    ExtraArgs={"VersionId": info.version, **headerArgs},
+                                )
                     else:
                         assert False
 
@@ -1574,7 +1916,7 @@ class AWSJobStore(AbstractJobStore):
                 def transform(self, readable, writable):
                     hasher = info._start_checksum(to_match=info.checksum)
                     contents = readable.read(1024 * 1024)
-                    while contents != b'':
+                    while contents != b"":
                         info._update_checksum(hasher, contents)
                         try:
                             writable.write(contents)
@@ -1591,7 +1933,9 @@ class AWSJobStore(AbstractJobStore):
             if verifyChecksum and self.checksum:
                 with DownloadPipe() as readable:
                     # Interpose a pipe to check the hash
-                    with HashingPipe(readable, encoding=encoding, errors=errors) as verified:
+                    with HashingPipe(
+                        readable, encoding=encoding, errors=errors
+                    ) as verified:
                         yield verified
             else:
                 # Readable end of pipe produces text mode output if encoding specified
@@ -1602,18 +1946,25 @@ class AWSJobStore(AbstractJobStore):
         def delete(self):
             store = self.outer
             if self.previousVersion is not None:
-                expected: UpdateConditionTypeDef = {"Name": 'version', "Value": cast(str, self.previousVersion)}
+                expected: "UpdateConditionTypeDef" = {
+                    "Name": "version",
+                    "Value": cast(str, self.previousVersion),
+                }
                 for attempt in retry_sdb():
                     with attempt:
-                        store.db.delete_attributes(DomainName=store.files_domain_name,
-                                                   ItemName=compat_bytes(self.fileID),
-                                                   Expected=expected)
+                        store.db.delete_attributes(
+                            DomainName=store.files_domain_name,
+                            ItemName=compat_bytes(self.fileID),
+                            Expected=expected,
+                        )
                 if self.previousVersion:
                     for attempt in retry_s3():
                         with attempt:
-                            store.s3_client.delete_object(Bucket=store.files_bucket.name,
-                                                          Key=compat_bytes(self.fileID),
-                                                          VersionId=self.previousVersion)
+                            store.s3_client.delete_object(
+                                Bucket=store.files_bucket.name,
+                                Key=compat_bytes(self.fileID),
+                                VersionId=self.previousVersion,
+                            )
 
         def getSize(self):
             """
@@ -1632,7 +1983,7 @@ class AWSJobStore(AbstractJobStore):
         def _getSSEKey(self) -> Optional[bytes]:
             sseKeyPath = self.outer.sseKeyPath
             if sseKeyPath:
-                with open(sseKeyPath, 'rb') as f:
+                with open(sseKeyPath, "rb") as f:
                     sseKey = f.read()
                 return sseKey
 
@@ -1641,25 +1992,30 @@ class AWSJobStore(AbstractJobStore):
             # parameters and will be used to set the http headers
             if self.encrypted:
                 sseKey = self._getSSEKey()
-                assert sseKey is not None, 'Content is encrypted but no key was provided.'
+                assert (
+                    sseKey is not None
+                ), "Content is encrypted but no key was provided."
                 assert len(sseKey) == 32
                 # boto3 encodes the key and calculates the MD5 for us
-                return {'SSECustomerAlgorithm': 'AES256', 'SSECustomerKey': sseKey}
+                return {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": sseKey}
             else:
                 return {}
 
         def __repr__(self):
             r = custom_repr
-            d = (('fileID', r(self.fileID)),
-                 ('ownerID', r(self.ownerID)),
-                 ('encrypted', r(self.encrypted)),
-                 ('version', r(self.version)),
-                 ('previousVersion', r(self.previousVersion)),
-                 ('content', r(self.content)),
-                 ('checksum', r(self.checksum)),
-                 ('_numContentChunks', r(self._numContentChunks)))
-            return "{}({})".format(type(self).__name__,
-                                   ', '.join(f'{k}={v}' for k, v in d))
+            d = (
+                ("fileID", r(self.fileID)),
+                ("ownerID", r(self.ownerID)),
+                ("encrypted", r(self.encrypted)),
+                ("version", r(self.version)),
+                ("previousVersion", r(self.previousVersion)),
+                ("content", r(self.content)),
+                ("checksum", r(self.checksum)),
+                ("_numContentChunks", r(self._numContentChunks)),
+            )
+            return "{}({})".format(
+                type(self).__name__, ", ".join(f"{k}={v}" for k, v in d)
+            )
 
     versionings = dict(Enabled=True, Disabled=False, Suspended=None)
 
@@ -1696,7 +2052,7 @@ class AWSJobStore(AbstractJobStore):
         if self.files_bucket is not None:
             self._delete_bucket(self.files_bucket)
             self.files_bucket = None
-        for name in 'files_domain_name', 'jobs_domain_name':
+        for name in "files_domain_name", "jobs_domain_name":
             domainName = getattr(self, name)
             if domainName is not None:
                 self._delete_domain(domainName)
@@ -1720,12 +2076,14 @@ class AWSJobStore(AbstractJobStore):
         for attempt in retry_s3():
             with attempt:
                 try:
-                    uploads = s3_boto3_client.list_multipart_uploads(Bucket=bucket.name).get('Uploads')
+                    uploads = s3_boto3_client.list_multipart_uploads(
+                        Bucket=bucket.name
+                    ).get("Uploads")
                     if uploads:
                         for u in uploads:
-                            s3_boto3_client.abort_multipart_upload(Bucket=bucket.name,
-                                                                   Key=u["Key"],
-                                                                   UploadId=u["UploadId"])
+                            s3_boto3_client.abort_multipart_upload(
+                                Bucket=bucket.name, Key=u["Key"], UploadId=u["UploadId"]
+                            )
 
                     bucket.objects.all().delete()
                     bucket.object_versions.delete()
@@ -1745,5 +2103,7 @@ custom_repr = aRepr.repr
 class BucketLocationConflictException(LocatorException):
     def __init__(self, bucketRegion):
         super().__init__(
-            'A bucket with the same name as the jobstore was found in another region (%s). '
-            'Cannot proceed as the unique bucket name is already in use.', locator=bucketRegion)
+            "A bucket with the same name as the jobstore was found in another region (%s). "
+            "Cannot proceed as the unique bucket name is already in use.",
+            locator=bucketRegion,
+        )