toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/job.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import annotations
+
 import collections
 import copy
 import importlib
@@ -27,56 +29,59 @@ from abc import ABCMeta, abstractmethod
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace
 from contextlib import contextmanager
 from io import BytesIO
-from typing import (TYPE_CHECKING,
-                    Any,
-                    Callable,
-                    Dict,
-                    Iterator,
-                    List,
-                    Mapping,
-                    NamedTuple,
-                    Optional,
-                    Sequence,
-                    Set,
-                    Tuple,
-                    TypeVar,
-                    Union,
-                    cast,
-                    overload)
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Mapping,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+    TypedDict,
+    Literal,
+)
+from urllib.error import HTTPError
+from urllib.parse import urlsplit, unquote, urljoin
+
+from toil import memoize
 
+import dill
 from configargparse import ArgParser
 
-from toil.bus import Names
-from toil.lib.compatibility import deprecated
-
-if sys.version_info >= (3, 8):
-    from typing import TypedDict
-else:
-    from typing_extensions import TypedDict
-
-import dill
-# TODO: When this gets into the standard library, get it from there and drop
-# typing-extensions dependency on Pythons that are new enough.
-from typing_extensions import NotRequired
+from toil.lib.io import is_remote_url
 
-if sys.version_info >= (3, 8):
-    from typing import Literal
+if sys.version_info < (3, 11):
+    from typing_extensions import NotRequired
 else:
-    from typing_extensions import Literal
+    from typing import NotRequired
 
+from toil.bus import Names
 from toil.common import Config, Toil, addOptions, safeUnpickleFromStream
 from toil.deferred import DeferredFunction
 from toil.fileStores import FileID
+from toil.lib.compatibility import deprecated
 from toil.lib.conversions import bytes2human, human2bytes
 from toil.lib.expando import Expando
 from toil.lib.resources import ResourceMonitor
 from toil.resource import ModuleDescriptor
 from toil.statsAndLogging import set_logging_from_options
 
+from toil.lib.exceptions import UnimplementedURLException
+
 if TYPE_CHECKING:
     from optparse import OptionParser
 
-    from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
+    from toil.batchSystems.abstractBatchSystem import (
+        BatchJobExitReason
+    )
     from toil.fileStores.abstractFileStore import AbstractFileStore
     from toil.jobStores.abstractJobStore import AbstractJobStore
 
@@ -122,24 +127,28 @@ class ConflictingPredecessorError(Exception):
             f'The given job: "{predecessor.description}" is already a predecessor of job: "{successor.description}".'
         )
 
+
 class DebugStoppingPointReached(BaseException):
     """
     Raised when a job reaches a point at which it has been instructed to stop for debugging.
     """
-    pass
+
 
 class FilesDownloadedStoppingPointReached(DebugStoppingPointReached):
     """
     Raised when a job stops because it was asked to download its files, and the files are downloaded.
     """
 
-    def __init__(self, message, host_and_job_paths: Optional[List[Tuple[str, str]]] = None):
+    def __init__(
+        self, message, host_and_job_paths: Optional[list[tuple[str, str]]] = None
+    ):
         super().__init__(message)
 
         # Save the host and user-code-visible paths of files, in case we're
         # using a container and they are different.
         self.host_and_job_paths = host_and_job_paths
 
+
 class TemporaryID:
     """
     Placeholder for a unregistered job ID used by a JobDescription.
@@ -161,7 +170,7 @@ class TemporaryID:
         return self.__repr__()
 
     def __repr__(self) -> str:
-        return f'TemporaryID({self._value})'
+        return f"TemporaryID({self._value})"
 
     def __hash__(self) -> int:
         return hash(self._value)
@@ -172,6 +181,7 @@ class TemporaryID:
     def __ne__(self, other: Any) -> bool:
         return not isinstance(other, TemporaryID) or self._value != other._value
 
+
 class AcceleratorRequirement(TypedDict):
     """Requirement for one or more computational accelerators, like a GPU or FPGA."""
 
@@ -210,7 +220,10 @@ class AcceleratorRequirement(TypedDict):
 
     # TODO: support requesting any GPU with X amount of vram
 
-def parse_accelerator(spec: Union[int, str, Dict[str, Union[str, int]]]) -> AcceleratorRequirement:
+
+def parse_accelerator(
+    spec: Union[int, str, dict[str, Union[str, int]]]
+) -> AcceleratorRequirement:
     """
     Parse an AcceleratorRequirement specified by user code.
 
@@ -247,16 +260,16 @@ def parse_accelerator(spec: Union[int, str, Dict[str, Union[str, int]]]) -> Acce
     :raises ValueError: if it gets something it can't parse
     :raises TypeError: if it gets something it can't parse because it's the wrong type.
     """
-    KINDS = {'gpu'}
-    BRANDS = {'nvidia', 'amd'}
-    APIS = {'cuda', 'rocm', 'opencl'}
+    KINDS = {"gpu"}
+    BRANDS = {"nvidia", "amd"}
+    APIS = {"cuda", "rocm", "opencl"}
 
-    parsed: AcceleratorRequirement = {'count': 1, 'kind': 'gpu'}
+    parsed: AcceleratorRequirement = {"count": 1, "kind": "gpu"}
 
     if isinstance(spec, int):
-        parsed['count'] = spec
+        parsed["count"] = spec
     elif isinstance(spec, str):
-        parts = spec.split(':')
+        parts = spec.split(":")
 
         if len(parts) > 2:
             raise ValueError("Could not parse AcceleratorRequirement: " + spec)
@@ -265,7 +278,7 @@ def parse_accelerator(spec: Union[int, str, Dict[str, Union[str, int]]]) -> Acce
 
         try:
             # If they have : and then a count, or just a count, handle that.
-            parsed['count'] = int(possible_count)
+            parsed["count"] = int(possible_count)
             if len(parts) > 1:
                 # Then we take whatever was before the colon as text
                 possible_description = parts[0]
@@ -275,73 +288,97 @@ def parse_accelerator(spec: Union[int, str, Dict[str, Union[str, int]]]) -> Acce
             # It doesn't end with a number
             if len(parts) == 2:
                 # We should have a number though.
-                raise ValueError("Could not parse AcceleratorRequirement count in: " + spec)
+                raise ValueError(
+                    "Could not parse AcceleratorRequirement count in: " + spec
+                )
             else:
                 # Must be just the description
                 possible_description = possible_count
 
         # Determine if we have a kind, brand, API, or (by default) model
         if possible_description in KINDS:
-            parsed['kind'] = possible_description
+            parsed["kind"] = possible_description
         elif possible_description in BRANDS:
-            parsed['brand'] = possible_description
+            parsed["brand"] = possible_description
         elif possible_description in APIS:
-            parsed['api'] = possible_description
+            parsed["api"] = possible_description
         else:
             if possible_description is not None:
-                parsed['model'] = possible_description
+                parsed["model"] = possible_description
     elif isinstance(spec, dict):
         # It's a dict, so merge with the defaults.
         parsed.update(spec)
         # TODO: make sure they didn't misspell keys or something
     else:
-        raise TypeError(f"Cannot parse value of type {type(spec)} as an AcceleratorRequirement")
+        raise TypeError(
+            f"Cannot parse value of type {type(spec)} as an AcceleratorRequirement"
+        )
 
-    if parsed['kind'] == 'gpu':
+    if parsed["kind"] == "gpu":
         # Use some smarts about what current GPUs are like to elaborate the
         # description.
 
-        if 'brand' not in parsed and 'model' in parsed:
+        if "brand" not in parsed and "model" in parsed:
             # Try to guess the brand from the model
             for brand in BRANDS:
-                if parsed['model'].startswith(brand):
+                if parsed["model"].startswith(brand):
                     # The model often starts with the brand
-                    parsed['brand'] = brand
+                    parsed["brand"] = brand
                     break
 
-        if 'brand' not in parsed and 'api' in parsed:
+        if "brand" not in parsed and "api" in parsed:
             # Try to guess the brand from the API
-            if parsed['api'] == 'cuda':
+            if parsed["api"] == "cuda":
                 # Only nvidia makes cuda cards
-                parsed['brand'] = 'nvidia'
-            elif parsed['api'] == 'rocm':
+                parsed["brand"] = "nvidia"
+            elif parsed["api"] == "rocm":
                 # Only amd makes rocm cards
-                parsed['brand'] = 'amd'
+                parsed["brand"] = "amd"
 
     return parsed
 
-def accelerator_satisfies(candidate: AcceleratorRequirement, requirement: AcceleratorRequirement, ignore: List[str] = []) -> bool:
+
+def accelerator_satisfies(
+    candidate: AcceleratorRequirement,
+    requirement: AcceleratorRequirement,
+    ignore: list[str] = [],
+) -> bool:
     """
     Test if candidate partially satisfies the given requirement.
 
     :returns: True if the given candidate at least partially satisfies the
               given requirement (i.e. check all fields other than count).
     """
-    for key in ['kind', 'brand', 'api', 'model']:
+    for key in ["kind", "brand", "api", "model"]:
         if key in ignore:
             # Skip this aspect.
             continue
         if key in requirement:
             if key not in candidate:
-                logger.debug('Candidate %s does not satisfy requirement %s because it does not have a %s', candidate, requirement, key)
+                logger.debug(
+                    "Candidate %s does not satisfy requirement %s because it does not have a %s",
+                    candidate,
+                    requirement,
+                    key,
+                )
                 return False
             if candidate[key] != requirement[key]:
-                logger.debug('Candidate %s does not satisfy requirement %s because it does not have the correct %s', candidate, requirement, key)
+                logger.debug(
+                    "Candidate %s does not satisfy requirement %s because it does not have the correct %s",
+                    candidate,
+                    requirement,
+                    key,
+                )
                 return False
     # If all these match or are more specific than required, we match!
     return True
 
-def accelerators_fully_satisfy(candidates: Optional[List[AcceleratorRequirement]], requirement: AcceleratorRequirement, ignore: List[str] = []) -> bool:
+
+def accelerators_fully_satisfy(
+    candidates: Optional[list[AcceleratorRequirement]],
+    requirement: AcceleratorRequirement,
+    ignore: list[str] = [],
+) -> bool:
     """
     Determine if a set of accelerators satisfy a requirement.
 
@@ -352,21 +389,22 @@ def accelerators_fully_satisfy(candidates: Optional[List[AcceleratorRequirement]
                together (i.e. check all fields including count).
     """
 
-    count_remaining = requirement['count']
+    count_remaining = requirement["count"]
 
     if candidates:
         for candidate in candidates:
             if accelerator_satisfies(candidate, requirement, ignore=ignore):
-                if candidate['count'] > count_remaining:
+                if candidate["count"] > count_remaining:
                     # We found all the matching accelerators we need
                     count_remaining = 0
                     break
                 else:
-                    count_remaining -= candidate['count']
+                    count_remaining -= candidate["count"]
 
     # If we have no count left we are fully satisfied
     return count_remaining == 0
 
+
 class RequirementsDict(TypedDict):
     """
     Typed storage for requirements for a job.
@@ -377,22 +415,35 @@ class RequirementsDict(TypedDict):
     cores: NotRequired[Union[int, float]]
     memory: NotRequired[int]
     disk: NotRequired[int]
-    accelerators: NotRequired[List[AcceleratorRequirement]]
+    accelerators: NotRequired[list[AcceleratorRequirement]]
     preemptible: NotRequired[bool]
 
+
 # These must be all the key names in RequirementsDict
 REQUIREMENT_NAMES = ["disk", "memory", "cores", "accelerators", "preemptible"]
 
 # This is the supertype of all value types in RequirementsDict
-ParsedRequirement = Union[int, float, bool, List[AcceleratorRequirement]]
+ParsedRequirement = Union[int, float, bool, list[AcceleratorRequirement]]
 
 # We define some types for things we can parse into different kind of requirements
 ParseableIndivisibleResource = Union[str, int]
 ParseableDivisibleResource = Union[str, int, float]
 ParseableFlag = Union[str, int, bool]
-ParseableAcceleratorRequirement = Union[str, int, Mapping[str, Any], AcceleratorRequirement, Sequence[Union[str, int, Mapping[str, Any], AcceleratorRequirement]]]
+ParseableAcceleratorRequirement = Union[
+    str,
+    int,
+    Mapping[str, Any],
+    AcceleratorRequirement,
+    Sequence[Union[str, int, Mapping[str, Any], AcceleratorRequirement]],
+]
+
+ParseableRequirement = Union[
+    ParseableIndivisibleResource,
+    ParseableDivisibleResource,
+    ParseableFlag,
+    ParseableAcceleratorRequirement,
+]
 
-ParseableRequirement = Union[ParseableIndivisibleResource, ParseableDivisibleResource, ParseableFlag, ParseableAcceleratorRequirement]
 
 class Requirer:
     """
@@ -403,9 +454,7 @@ class Requirer:
 
     _requirementOverrides: RequirementsDict
 
-    def __init__(
-        self, requirements: Mapping[str, ParseableRequirement]
-    ) -> None:
+    def __init__(self, requirements: Mapping[str, ParseableRequirement]) -> None:
         """
         Parse and save the given requirements.
 
@@ -446,12 +495,11 @@ class Requirer:
             raise RuntimeError(f"Config assigned multiple times to {self}")
         self._config = config
 
-
-    def __getstate__(self) -> Dict[str, Any]:
+    def __getstate__(self) -> dict[str, Any]:
         """Return the dict to use as the instance's __dict__ when pickling."""
         # We want to exclude the config from pickling.
         state = self.__dict__.copy()
-        state['_config'] = None
+        state["_config"] = None
         return state
 
     def __copy__(self) -> "Requirer":
@@ -492,37 +540,29 @@ class Requirer:
     @overload
     @staticmethod
     def _parseResource(
-        name: Union[Literal["memory"], Literal["disks"]], value: ParseableIndivisibleResource
-    ) -> int:
-        ...
+        name: Union[Literal["memory"], Literal["disks"]],
+        value: ParseableIndivisibleResource,
+    ) -> int: ...
 
     @overload
     @staticmethod
     def _parseResource(
         name: Literal["cores"], value: ParseableDivisibleResource
-    ) -> Union[int, float]:
-        ...
+    ) -> Union[int, float]: ...
 
     @overload
     @staticmethod
     def _parseResource(
         name: Literal["accelerators"], value: ParseableAcceleratorRequirement
-    ) -> List[AcceleratorRequirement]:
-        ...
+    ) -> list[AcceleratorRequirement]: ...
 
     @overload
     @staticmethod
-    def _parseResource(
-        name: str, value: ParseableRequirement
-    ) -> ParsedRequirement:
-        ...
+    def _parseResource(name: str, value: ParseableRequirement) -> ParsedRequirement: ...
 
     @overload
     @staticmethod
-    def _parseResource(
-        name: str, value: None
-    ) -> None:
-        ...
+    def _parseResource(name: str, value: None) -> None: ...
 
     @staticmethod
     def _parseResource(
@@ -559,43 +599,53 @@ class Requirer:
             # Anything can be None.
             return value
 
-        if name in ('memory', 'disk', 'cores'):
+        if name in ("memory", "disk", "cores"):
             # These should be numbers that accept things like "5G".
             if isinstance(value, (str, bytes)):
                 value = human2bytes(value)
             if isinstance(value, int):
                 return value
-            elif isinstance(value, float) and name == 'cores':
+            elif isinstance(value, float) and name == "cores":
                 # But only cores can be fractional.
                 return value
             else:
-                raise TypeError(f"The '{name}' requirement does not accept values that are of type {type(value)}")
-        elif name == 'preemptible':
+                raise TypeError(
+                    f"The '{name}' requirement does not accept values that are of type {type(value)}"
+                )
+        elif name == "preemptible":
             if isinstance(value, str):
                 if value.lower() == "true":
                     return True
                 elif value.lower() == "false":
                     return False
                 else:
-                    raise ValueError(f"The '{name}' requirement, as a string, must be 'true' or 'false' but is {value}")
+                    raise ValueError(
+                        f"The '{name}' requirement, as a string, must be 'true' or 'false' but is {value}"
+                    )
             elif isinstance(value, int):
                 if value == 1:
                     return True
                 if value == 0:
                     return False
                 else:
-                    raise ValueError(f"The '{name}' requirement, as an int, must be 1 or 0 but is {value}")
+                    raise ValueError(
+                        f"The '{name}' requirement, as an int, must be 1 or 0 but is {value}"
+                    )
             elif isinstance(value, bool):
                 return value
             else:
-                raise TypeError(f"The '{name}' requirement does not accept values that are of type {type(value)}")
-        elif name == 'accelerators':
+                raise TypeError(
+                    f"The '{name}' requirement does not accept values that are of type {type(value)}"
+                )
+        elif name == "accelerators":
             # The type checking for this is delegated to the
             # AcceleratorRequirement class.
             if isinstance(value, list):
-                return [parse_accelerator(v) for v in value] #accelerators={'kind': 'gpu', 'brand': 'nvidia', 'count': 2}
+                return [
+                    parse_accelerator(v) for v in value
+                ]  # accelerators={'kind': 'gpu', 'brand': 'nvidia', 'count': 2}
             else:
-                return [parse_accelerator(value)] #accelerators=1
+                return [parse_accelerator(value)]  # accelerators=1
         else:
             # Anything else we just pass along without opinons
             return cast(ParsedRequirement, value)
@@ -618,7 +668,10 @@ class Requirer:
                 )
             return value
         elif self._config is not None:
-            values = [getattr(self._config, 'default_' + requirement, None), getattr(self._config, 'default' + requirement.capitalize(), None)]
+            values = [
+                getattr(self._config, "default_" + requirement, None),
+                getattr(self._config, "default" + requirement.capitalize(), None),
+            ]
             value = values[0] if values[0] is not None else values[1]
             if value is None:
                 raise AttributeError(
@@ -679,10 +732,13 @@ class Requirer:
         self._requirementOverrides["preemptible"] = Requirer._parseResource(
             "preemptible", val
         )
+
     @property
-    def accelerators(self) -> List[AcceleratorRequirement]:
+    def accelerators(self) -> list[AcceleratorRequirement]:
         """Any accelerators, such as GPUs, that are needed."""
-        return cast(List[AcceleratorRequirement], self._fetchRequirement("accelerators"))
+        return cast(
+            list[AcceleratorRequirement], self._fetchRequirement("accelerators")
+        )
 
     @accelerators.setter
     def accelerators(self, val: ParseableAcceleratorRequirement) -> None:
@@ -705,7 +761,7 @@ class Requirer:
         if isinstance(original_value, (int, float)):
             # This is something we actually can scale up and down
             new_value = original_value * factor
-            if requirement in ('memory', 'disk'):
+            if requirement in ("memory", "disk"):
                 # Must round to an int
                 new_value = math.ceil(new_value)
             setattr(scaled, requirement, new_value)
@@ -723,29 +779,32 @@ class Requirer:
                 if isinstance(v, (int, float)) and v > 1000:
                     # Make large numbers readable
                     v = bytes2human(v)
-                parts.append(f'{k}: {v}')
+                parts.append(f"{k}: {v}")
         if len(parts) == 0:
-            parts = ['no requirements']
-        return ', '.join(parts)
+            parts = ["no requirements"]
+        return ", ".join(parts)
+
 
 class JobBodyReference(NamedTuple):
     """
     Reference from a job description to its body.
     """
+
     file_store_id: str
     """File ID (or special shared file name for the root job) of the job's body."""
-    module_string: str 
+    module_string: str
     """Stringified description of the module needed to load the body."""
 
+
 class JobDescription(Requirer):
     """
     Stores all the information that the Toil Leader ever needs to know about a Job.
-    
+
     This includes:
         * Resource requirements.
         * Which jobs are children or follow-ons or predecessors of this job.
         * A reference to the Job object in the job store.
-    
+
     Can be obtained from an actual (i.e. executable) Job object, and can be
     used to obtain the Job object from the JobStore.
 
@@ -760,8 +819,9 @@ class JobDescription(Requirer):
         requirements: Mapping[str, Union[int, str, bool]],
         jobName: str,
         unitName: Optional[str] = "",
-        displayName: Optional[str] = "",        
-        local: Optional[bool] = None
+        displayName: Optional[str] = "",
+        local: Optional[bool] = None,
+        files: Optional[set[FileID]] = None,
     ) -> None:
         """
         Create a new JobDescription.
@@ -784,6 +844,7 @@ class JobDescription(Requirer):
         :param local: If True, the job is meant to use minimal resources but is
             sensitive to execution latency, and so should be executed by the
             leader.
+        :param files: Set of FileID objects that the job plans to use.
         """
         # Set requirements
         super().__init__(requirements)
@@ -794,10 +855,11 @@ class JobDescription(Requirer):
         # Save names, making sure they are strings and not e.g. bytes or None.
         def makeString(x: Union[str, bytes, None]) -> str:
             if isinstance(x, bytes):
-                return x.decode('utf-8', errors='replace')
+                return x.decode("utf-8", errors="replace")
             if x is None:
                 return ""
             return x
+
         self.jobName = makeString(jobName)
         self.unitName = makeString(unitName)
         self.displayName = makeString(displayName)
@@ -844,7 +906,7 @@ class JobDescription(Requirer):
         # chained-in job with its original ID, and also this job's ID with its
         # original names, or is empty if no chaining has happened.
         # The first job in the chain comes first in the list.
-        self._merged_job_names: List[Names] = []
+        self._merged_job_names: list[Names] = []
 
         # The number of direct predecessors of the job. Needs to be stored at
         # the JobDescription to support dynamically-created jobs with multiple
@@ -867,17 +929,17 @@ class JobDescription(Requirer):
 
         # The IDs of all child jobs of the described job.
         # Children which are done must be removed with filterSuccessors.
-        self.childIDs: Set[str] = set()
+        self.childIDs: set[str] = set()
 
         # The IDs of all follow-on jobs of the described job.
         # Follow-ons which are done must be removed with filterSuccessors.
-        self.followOnIDs: Set[str] = set()
+        self.followOnIDs: set[str] = set()
 
         # We keep our own children and follow-ons in a list of successor
         # phases, along with any successors adopted from jobs we have chained
         # from. When we finish our own children and follow-ons, we may have to
         # go back and finish successors for those jobs.
-        self.successor_phases: List[Set[str]] = [self.followOnIDs, self.childIDs]
+        self.successor_phases: list[set[str]] = [self.followOnIDs, self.childIDs]
 
         # Dict from ServiceHostJob ID to list of child ServiceHostJobs that start after it.
         # All services must have an entry, if only to an empty list.
@@ -893,13 +955,24 @@ class JobDescription(Requirer):
         # And we log who made the version (by PID)
         self._job_version_writer = 0
 
+        # Store FileIDs that the Job will want to use
+        # This currently does not serve much of a purpose except for debugging
+        # In the future, this can be used to improve job scheduling, see https://github.com/DataBiosphere/toil/issues/3071
+        self.files_to_use = files or set()
+
     def get_names(self) -> Names:
         """
         Get the names and ID of this job as a named tuple.
         """
-        return Names(self.jobName, self.unitName, self.displayName, self.displayName, str(self.jobStoreID))
+        return Names(
+            self.jobName,
+            self.unitName,
+            self.displayName,
+            self.displayName,
+            str(self.jobStoreID),
+        )
 
-    def get_chain(self) -> List[Names]:
+    def get_chain(self) -> list[Names]:
         """
         Get all the jobs that executed in this job's chain, in order.
 
@@ -914,7 +987,7 @@ class JobDescription(Requirer):
         else:
             return list(self._merged_job_names)
 
-    def serviceHostIDsInBatches(self) -> Iterator[List[str]]:
+    def serviceHostIDsInBatches(self) -> Iterator[list[str]]:
         """
         Find all batches of service host job IDs that can be started at the same time.
 
@@ -955,14 +1028,13 @@ class JobDescription(Requirer):
         """
 
         for phase in self.successor_phases:
-            for successor in phase:
-                yield successor
+            yield from phase
 
-    def successors_by_phase(self) -> Iterator[Tuple[int, str]]:
+    def successors_by_phase(self) -> Iterator[tuple[int, str]]:
         """
-        Get an iterator over all child/follow-on/chained inherited successor job IDs, along with their phase numbere on the stack.
+        Get an iterator over all child/follow-on/chained inherited successor job IDs, along with their phase number on the stack.
 
-        Phases ececute higher numbers to lower numbers.
+        Phases execute higher numbers to lower numbers.
         """
 
         for i, phase in enumerate(self.successor_phases):
@@ -1003,7 +1075,7 @@ class JobDescription(Requirer):
         """
         self._body = None
 
-    def get_body(self) -> Tuple[str, ModuleDescriptor]:
+    def get_body(self) -> tuple[str, ModuleDescriptor]:
         """
         Get the information needed to load the job body.
 
@@ -1016,9 +1088,11 @@ class JobDescription(Requirer):
         if not self.has_body():
             raise RuntimeError(f"Cannot load the body of a job {self} without one")
 
-        return self._body.file_store_id, ModuleDescriptor.fromCommand(self._body.module_string)
+        return self._body.file_store_id, ModuleDescriptor.fromCommand(
+            self._body.module_string
+        )
 
-    def nextSuccessors(self) -> Optional[Set[str]]:
+    def nextSuccessors(self) -> Optional[set[str]]:
         """
         Return the collection of job IDs for the successors of this job that are ready to run.
 
@@ -1101,7 +1175,9 @@ class JobDescription(Requirer):
         :returns: True if the job appears to be done, and all related child,
                   follow-on, and service jobs appear to be finished and removed.
         """
-        return not self.has_body() and next(self.successorsAndServiceHosts(), None) is None
+        return (
+            not self.has_body() and next(self.successorsAndServiceHosts(), None) is None
+        )
 
     def replace(self, other: "JobDescription") -> None:
         """
@@ -1120,11 +1196,15 @@ class JobDescription(Requirer):
         # TODO: We can't join the job graphs with Job._jobGraphsJoined, is that a problem?
 
         # Take all the successors other than this one
-        old_phases = [{i for i in p if i != self.jobStoreID} for p in other.successor_phases]
+        old_phases = [
+            {i for i in p if i != self.jobStoreID} for p in other.successor_phases
+        ]
         # And drop empty phases
         old_phases = [p for p in old_phases if len(p) > 0]
         # And put in front of our existing phases
-        logger.debug('%s is adopting successor phases from %s of: %s', self, other, old_phases)
+        logger.debug(
+            "%s is adopting successor phases from %s of: %s", self, other, old_phases
+        )
         self.successor_phases = old_phases + self.successor_phases
 
         # When deleting, we need to delete the files for our old ID, and also
@@ -1148,9 +1228,13 @@ class JobDescription(Requirer):
         self.jobStoreID = other.jobStoreID
 
         if len(other.filesToDelete) > 0:
-            raise RuntimeError("Trying to take on the ID of a job that is in the process of being committed!")
+            raise RuntimeError(
+                "Trying to take on the ID of a job that is in the process of being committed!"
+            )
         if len(self.filesToDelete) > 0:
-            raise RuntimeError("Trying to take on the ID of anothe job while in the process of being committed!")
+            raise RuntimeError(
+                "Trying to take on the ID of anothe job while in the process of being committed!"
+            )
 
         self._job_version = other._job_version
         self._job_version_writer = os.getpid()
@@ -1160,7 +1244,9 @@ class JobDescription(Requirer):
         Make sure this JobDescription is not newer than a prospective new version of the JobDescription.
         """
         if other._job_version < self._job_version:
-            raise RuntimeError(f"Cannot replace {self} from PID {self._job_version_writer} with older version {other} from PID {other._job_version_writer}")
+            raise RuntimeError(
+                f"Cannot replace {self} from PID {self._job_version_writer} with older version {other} from PID {other._job_version_writer}"
+            )
 
     def is_updated_by(self, other: "JobDescription") -> bool:
         """
@@ -1177,7 +1263,7 @@ class JobDescription(Requirer):
                 other._job_version_writer,
                 self.jobStoreID,
                 self,
-                self._job_version_writer
+                self._job_version_writer,
             )
             return False
 
@@ -1189,7 +1275,7 @@ class JobDescription(Requirer):
                 other,
                 other._job_version_writer,
                 self,
-                self._job_version_writer
+                self._job_version_writer,
             )
             return False
 
@@ -1229,7 +1315,7 @@ class JobDescription(Requirer):
         """Test if the ServiceHostJob is a service of the described job."""
         return serviceID in self.serviceTree
 
-    def renameReferences(self, renames: Dict[TemporaryID, str]) -> None:
+    def renameReferences(self, renames: dict[TemporaryID, str]) -> None:
         """
         Apply the given dict of ID renames to all references to jobs.
 
@@ -1245,8 +1331,12 @@ class JobDescription(Requirer):
                     # Replace each renamed item one at a time to preserve set identity
                     phase.remove(item)
                     phase.add(renames[item])
-        self.serviceTree = {renames.get(parent, parent): [renames.get(child, child) for child in children]
-                            for parent, children in self.serviceTree.items()}
+        self.serviceTree = {
+            renames.get(parent, parent): [
+                renames.get(child, child) for child in children
+            ]
+            for parent, children in self.serviceTree.items()
+        }
 
     def addPredecessor(self) -> None:
         """Notify the JobDescription that a predecessor has been added to its Job."""
@@ -1264,7 +1354,11 @@ class JobDescription(Requirer):
         :param jobStore: The job store we are being placed into
         """
 
-    def setupJobAfterFailure(self, exit_status: Optional[int] = None, exit_reason: Optional["BatchJobExitReason"] = None) -> None:
+    def setupJobAfterFailure(
+        self,
+        exit_status: Optional[int] = None,
+        exit_reason: Optional["BatchJobExitReason"] = None,
+    ) -> None:
         """
         Configure job after a failure.
 
@@ -1287,30 +1381,49 @@ class JobDescription(Requirer):
         if self._config is None:
             raise RuntimeError("The job's config is not assigned.")
 
-        if self._config.enableUnlimitedPreemptibleRetries and exit_reason == BatchJobExitReason.LOST:
-            logger.info("*Not* reducing try count (%s) of job %s with ID %s",
-                        self.remainingTryCount, self, self.jobStoreID)
+        if (
+            self._config.enableUnlimitedPreemptibleRetries
+            and exit_reason == BatchJobExitReason.LOST
+        ):
+            logger.info(
+                "*Not* reducing try count (%s) of job %s with ID %s",
+                self.remainingTryCount,
+                self,
+                self.jobStoreID,
+            )
         else:
             self.remainingTryCount = max(0, self.remainingTryCount - 1)
-            logger.warning("Due to failure we are reducing the remaining try count of job %s with ID %s to %s",
-                           self, self.jobStoreID, self.remainingTryCount)
+            logger.warning(
+                "Due to failure we are reducing the remaining try count of job %s with ID %s to %s",
+                self,
+                self.jobStoreID,
+                self.remainingTryCount,
+            )
         # Set the default memory to be at least as large as the default, in
         # case this was a malloc failure (we do this because of the combined
         # batch system)
         if exit_reason == BatchJobExitReason.MEMLIMIT and self._config.doubleMem:
             self.memory = self.memory * 2
-            logger.warning("We have doubled the memory of the failed job %s to %s bytes due to doubleMem flag",
-                           self, self.memory)
+            logger.warning(
+                "We have doubled the memory of the failed job %s to %s bytes due to doubleMem flag",
+                self,
+                self.memory,
+            )
         if self.memory < self._config.defaultMemory:
             self.memory = self._config.defaultMemory
-            logger.warning("We have increased the default memory of the failed job %s to %s bytes",
-                           self, self.memory)
+            logger.warning(
+                "We have increased the default memory of the failed job %s to %s bytes",
+                self,
+                self.memory,
+            )
 
         if self.disk < self._config.defaultDisk:
             self.disk = self._config.defaultDisk
-            logger.warning("We have increased the disk of the failed job %s to the default of %s bytes",
-                           self, self.disk)
-
+            logger.warning(
+                "We have increased the disk of the failed job %s to the default of %s bytes",
+                self,
+                self.disk,
+            )
 
     def getLogFileHandle(self, jobStore):
         """
@@ -1360,12 +1473,12 @@ class JobDescription(Requirer):
         """Produce a useful logging string identifying this job."""
         printedName = "'" + self.jobName + "'"
         if self.unitName:
-            printedName += ' ' + self.unitName
+            printedName += " " + self.unitName
 
         if self.jobStoreID is not None:
-            printedName += ' ' + str(self.jobStoreID)
+            printedName += " " + str(self.jobStoreID)
 
-        printedName += ' v' + str(self._job_version)
+        printedName += " v" + str(self._job_version)
 
         return printedName
 
@@ -1374,7 +1487,7 @@ class JobDescription(Requirer):
     # a time, keyed by jobStoreID.
 
     def __repr__(self):
-        return f'{self.__class__.__name__}( **{self.__dict__!r} )'
+        return f"{self.__class__.__name__}( **{self.__dict__!r} )"
 
     def reserve_versions(self, count: int) -> None:
         """
@@ -1394,6 +1507,7 @@ class JobDescription(Requirer):
         self._job_version_writer = os.getpid()
         logger.debug("New job version: %s", self)
 
+
 class ServiceJobDescription(JobDescription):
     """A description of a job that hosts a service."""
 
@@ -1464,7 +1578,7 @@ class CheckpointJobDescription(JobDescription):
             raise RuntimeError(f"Cannot restore an empty checkpoint for a job {self}")
         self._body = self.checkpoint
 
-    def restartCheckpoint(self, jobStore: "AbstractJobStore") -> List[str]:
+    def restartCheckpoint(self, jobStore: "AbstractJobStore") -> list[str]:
         """
         Restart a checkpoint after the total failure of jobs in its subtree.
 
@@ -1475,24 +1589,30 @@ class CheckpointJobDescription(JobDescription):
         Returns a list with the IDs of any successors deleted.
         """
         if self.checkpoint is None:
-            raise RuntimeError("Cannot restart a checkpoint job. The checkpoint was never set.")
+            raise RuntimeError(
+                "Cannot restart a checkpoint job. The checkpoint was never set."
+            )
         successorsDeleted = []
         all_successors = list(self.allSuccessors())
         if len(all_successors) > 0 or self.serviceTree or self.has_body():
             if self.has_body():
                 if self._body != self.checkpoint:
-                    raise RuntimeError("The stored body reference and checkpoint are not the same.")
+                    raise RuntimeError(
+                        "The stored body reference and checkpoint are not the same."
+                    )
                 logger.debug("Checkpoint job already has body set to run")
             else:
                 self.restore_checkpoint()
 
-            jobStore.update_job(self) # Update immediately to ensure that checkpoint
+            jobStore.update_job(self)  # Update immediately to ensure that checkpoint
             # is made before deleting any remaining successors
 
             if len(all_successors) > 0 or self.serviceTree:
                 # If the subtree of successors is not complete restart everything
-                logger.debug("Checkpoint job has unfinished successor jobs, deleting successors: %s, services: %s " %
-                             (all_successors, self.serviceTree.keys()))
+                logger.debug(
+                    "Checkpoint job has unfinished successor jobs, deleting successors: %s, services: %s "
+                    % (all_successors, self.serviceTree.keys())
+                )
 
                 # Delete everything on the stack, as these represent successors to clean
                 # up as we restart the queue
@@ -1505,9 +1625,13 @@ class CheckpointJobDescription(JobDescription):
                             logger.debug("Job %s has already been deleted", otherJobID)
                     if jobDesc.jobStoreID != self.jobStoreID:
                         # Delete everything under us except us.
-                        logger.debug("Checkpoint is deleting old successor job: %s", jobDesc.jobStoreID)
+                        logger.debug(
+                            "Checkpoint is deleting old successor job: %s",
+                            jobDesc.jobStoreID,
+                        )
                         jobStore.delete_job(jobDesc.jobStoreID)
                         successorsDeleted.append(jobDesc.jobStoreID)
+
                 recursiveDelete(self)
 
                 # Cut links to the jobs we deleted.
@@ -1536,6 +1660,7 @@ class Job:
         displayName: Optional[str] = "",
         descriptionClass: Optional[type] = None,
         local: Optional[bool] = None,
+        files: Optional[set[FileID]] = None,
     ) -> None:
         """
         Job initializer.
@@ -1556,6 +1681,7 @@ class Job:
         :param displayName: Human-readable job type display name.
         :param descriptionClass: Override for the JobDescription class used to describe the job.
         :param local: if the job can be run on the leader.
+        :param files: Set of Files that the job will want to use.
 
         :type memory: int or string convertible by toil.lib.conversions.human2bytes to an int
         :type cores: float, int, or string convertible by toil.lib.conversions.human2bytes to an int
@@ -1571,14 +1697,20 @@ class Job:
         jobName = self.__class__.__name__
         displayName = displayName if displayName else jobName
 
-        #Some workflows use preemptable instead of preemptible
+        # Some workflows use preemptable instead of preemptible
         if preemptable and not preemptible:
-            logger.warning("Preemptable as a keyword has been deprecated, please use preemptible.")
+            logger.warning(
+                "Preemptable as a keyword has been deprecated, please use preemptible."
+            )
             preemptible = preemptable
         # Build a requirements dict for the description
-        requirements = {'memory': memory, 'cores': cores, 'disk': disk,
-                        'accelerators': accelerators,
-                        'preemptible': preemptible}
+        requirements = {
+            "memory": memory,
+            "cores": cores,
+            "disk": disk,
+            "accelerators": accelerators,
+            "preemptible": preemptible,
+        }
         if descriptionClass is None:
             if checkpoint:
                 # Actually describe as a checkpoint job
@@ -1594,7 +1726,8 @@ class Job:
             jobName,
             unitName=unitName,
             displayName=displayName,
-            local=local
+            local=local,
+            files=files,
         )
 
         # Private class variables needed to actually execute a job, in the worker.
@@ -1617,7 +1750,9 @@ class Job:
         # Note that self.__module__ is not necessarily this module, i.e. job.py. It is the module
         # defining the class self is an instance of, which may be a subclass of Job that may be
         # defined in a different module.
-        self.userModule: ModuleDescriptor = ModuleDescriptor.forModule(self.__module__).globalize()
+        self.userModule: ModuleDescriptor = ModuleDescriptor.forModule(
+            self.__module__
+        ).globalize()
         # Maps index paths into composite return values to lists of IDs of files containing
         # promised values for those return value items. An index path is a tuple of indices that
         # traverses a nested data structure of lists, dicts, tuples or any other type supporting
@@ -1630,7 +1765,7 @@ class Job:
         self._tempDir = None
 
         # Holds flags set by set_debug_flag()
-        self._debug_flags: Set[str] = set()
+        self._debug_flags: set[str] = set()
 
     def __str__(self):
         """
@@ -1640,7 +1775,7 @@ class Job:
         if self.description is None:
             return repr(self)
         else:
-            return 'Job(' + str(self.description) + ')'
+            return "Job(" + str(self.description) + ")"
 
     def check_initialized(self) -> None:
         """
@@ -1652,8 +1787,10 @@ class Job:
         If __init__() has not been called, raise an error.
         """
         if not hasattr(self, "_description"):
-            raise ValueError(f"Job instance of type {type(self)} has not been initialized. super().__init__() may not "
-                             f"have been called.")
+            raise ValueError(
+                f"Job instance of type {type(self)} has not been initialized. super().__init__() may not "
+                f"have been called."
+            )
 
     @property
     def jobStoreID(self) -> Union[str, TemporaryID]:
@@ -1673,33 +1810,37 @@ class Job:
     def disk(self) -> int:
         """The maximum number of bytes of disk the job will require to run."""
         return self.description.disk
+
     @disk.setter
     def disk(self, val):
-         self.description.disk = val
+        self.description.disk = val
 
     @property
     def memory(self):
         """The maximum number of bytes of memory the job will require to run."""
         return self.description.memory
+
     @memory.setter
     def memory(self, val):
-         self.description.memory = val
+        self.description.memory = val
 
     @property
     def cores(self) -> Union[int, float]:
         """The number of CPU cores required."""
         return self.description.cores
+
     @cores.setter
     def cores(self, val):
-         self.description.cores = val
+        self.description.cores = val
 
     @property
-    def accelerators(self) -> List[AcceleratorRequirement]:
+    def accelerators(self) -> list[AcceleratorRequirement]:
         """Any accelerators, such as GPUs, that are needed."""
         return self.description.accelerators
+
     @accelerators.setter
-    def accelerators(self, val: List[ParseableAcceleratorRequirement]) -> None:
-         self.description.accelerators = val
+    def accelerators(self, val: list[ParseableAcceleratorRequirement]) -> None:
+        self.description.accelerators = val
 
     @property
     def preemptible(self) -> bool:
@@ -1709,15 +1850,30 @@ class Job:
     @deprecated(new_function_name="preemptible")
     def preemptable(self):
         return self.description.preemptible
+
     @preemptible.setter
     def preemptible(self, val):
-         self.description.preemptible = val
+        self.description.preemptible = val
 
     @property
     def checkpoint(self) -> bool:
         """Determine if the job is a checkpoint job or not."""
         return isinstance(self._description, CheckpointJobDescription)
 
+    @property
+    def files_to_use(self) -> set[FileID]:
+        return self.description.files_to_use
+
+    @files_to_use.setter
+    def files_to_use(self, val: set[FileID]):
+        self.description.files_to_use = val
+
+    def add_to_files_to_use(self, val: FileID):
+        self.description.files_to_use.add(val)
+
+    def remove_from_files_to_use(self, val: FileID):
+        self.description.files_to_use.remove(val)
+
     def assignConfig(self, config: Config) -> None:
         """
         Assign the given config object.
@@ -1831,7 +1987,7 @@ class Job:
 
         return followOnJob
 
-    def hasPredecessor(self, job: 'Job') -> bool:
+    def hasPredecessor(self, job: "Job") -> bool:
         """Check if a given job is already a predecessor of this job."""
         return job in self._directPredecessors
 
@@ -1893,7 +2049,9 @@ class Job:
 
     def hasService(self, service: "Job.Service") -> bool:
         """Return True if the given Service is a service of this job, and False otherwise."""
-        return service.hostID is None or self._description.hasServiceHostJob(service.hostID)
+        return service.hostID is None or self._description.hasServiceHostJob(
+            service.hostID
+        )
 
     # Convenience functions for creating jobs
 
@@ -1941,7 +2099,9 @@ class Job:
         :return: The new child job that wraps fn.
         """
         if PromisedRequirement.convertPromises(kwargs):
-            return self.addChild(PromisedRequirementJobFunctionWrappingJob.create(fn, *args, **kwargs))
+            return self.addChild(
+                PromisedRequirementJobFunctionWrappingJob.create(fn, *args, **kwargs)
+            )
         else:
             return self.addChild(JobFunctionWrappingJob(fn, *args, **kwargs))
 
@@ -1957,7 +2117,9 @@ class Job:
         :return: The new follow-on job that wraps fn.
         """
         if PromisedRequirement.convertPromises(kwargs):
-            return self.addFollowOn(PromisedRequirementJobFunctionWrappingJob.create(fn, *args, **kwargs))
+            return self.addFollowOn(
+                PromisedRequirementJobFunctionWrappingJob.create(fn, *args, **kwargs)
+            )
         else:
             return self.addFollowOn(JobFunctionWrappingJob(fn, *args, **kwargs))
 
@@ -2059,8 +2221,12 @@ class Job:
             raise JobPromiseConstraintError(self)
         # TODO: can we guarantee self.jobStoreID is populated and so pass that here?
         with self._promiseJobStore.write_file_stream() as (fileHandle, jobStoreFileID):
-            promise = UnfulfilledPromiseSentinel(str(self.description), jobStoreFileID, False)
-            logger.debug('Issuing promise %s for result of %s', jobStoreFileID, self.description)
+            promise = UnfulfilledPromiseSentinel(
+                str(self.description), jobStoreFileID, False
+            )
+            logger.debug(
+                "Issuing promise %s for result of %s", jobStoreFileID, self.description
+            )
             pickle.dump(promise, fileHandle, pickle.HIGHEST_PROTOCOL)
         self._rvs[path].append(jobStoreFileID)
         return self._promiseJobStore.config.jobStore, jobStoreFileID
@@ -2110,7 +2276,7 @@ class Job:
         self.checkJobGraphAcylic()
         self.checkNewCheckpointsAreLeafVertices()
 
-    def getRootJobs(self) -> Set['Job']:
+    def getRootJobs(self) -> set["Job"]:
         """
         Return the set of root job objects that contain this job.
 
@@ -2142,8 +2308,9 @@ class Job:
         """
         rootJobs = self.getRootJobs()
         if len(rootJobs) != 1:
-            raise JobGraphDeadlockException("Graph does not contain exactly one"
-                                            " root job: %s" % rootJobs)
+            raise JobGraphDeadlockException(
+                "Graph does not contain exactly one" " root job: %s" % rootJobs
+            )
 
     def checkJobGraphAcylic(self):
         """
@@ -2163,15 +2330,15 @@ class Job:
 
         Only deals with jobs created here, rather than loaded from the job store.
         """
-        #Get the root jobs
+        # Get the root jobs
         roots = self.getRootJobs()
         if len(roots) == 0:
             raise JobGraphDeadlockException("Graph contains no root jobs due to cycles")
 
-        #Get implied edges
+        # Get implied edges
         extraEdges = self._getImpliedEdges(roots)
 
-        #Check for directed cycles in the augmented graph
+        # Check for directed cycles in the augmented graph
         visited = set()
         for root in roots:
             root._checkJobGraphAcylicDFS([], visited, extraEdges)
@@ -2181,17 +2348,23 @@ class Job:
         if self not in visited:
             visited.add(self)
             stack.append(self)
-            for successor in [self._registry[jID] for jID in self.description.allSuccessors() if jID in self._registry] + extraEdges[self]:
+            for successor in [
+                self._registry[jID]
+                for jID in self.description.allSuccessors()
+                if jID in self._registry
+            ] + extraEdges[self]:
                 # Grab all the successors in the current registry (i.e. added form this node) and look at them.
                 successor._checkJobGraphAcylicDFS(stack, visited, extraEdges)
             if stack.pop() != self:
                 raise RuntimeError("The stack ordering/elements was changed.")
         if self in stack:
             stack.append(self)
-            raise JobGraphDeadlockException("A cycle of job dependencies has been detected '%s'" % stack)
+            raise JobGraphDeadlockException(
+                "A cycle of job dependencies has been detected '%s'" % stack
+            )
 
     @staticmethod
-    def _getImpliedEdges(roots) -> Dict["Job", List["Job"]]:
+    def _getImpliedEdges(roots) -> dict["Job", list["Job"]]:
         """
         Gets the set of implied edges (between children and follow-ons of a common job).
 
@@ -2201,17 +2374,17 @@ class Job:
 
         :returns: dict from Job object to list of Job objects that must be done before it can start.
         """
-        #Get nodes (Job objects) in job graph
+        # Get nodes (Job objects) in job graph
         nodes = set()
         for root in roots:
             root._collectAllSuccessors(nodes)
 
         ##For each follow-on edge calculate the extra implied edges
-        #Adjacency list of implied edges, i.e. map of jobs to lists of jobs
-        #connected by an implied edge
+        # Adjacency list of implied edges, i.e. map of jobs to lists of jobs
+        # connected by an implied edge
         extraEdges = {n: [] for n in nodes}
         for job in nodes:
-             # Get all the nonempty successor phases
+            # Get all the nonempty successor phases
             phases = [p for p in job.description.successor_phases if len(p) > 0]
             for depth in range(1, len(phases)):
                 # Add edges from all jobs in the earlier/upper subtrees to all
@@ -2231,7 +2404,11 @@ class Job:
                 for inUpper in reacheable:
                     # Add extra edges to the roots of all the lower subtrees
                     # But skip anything in the lower subtree not in the current _registry (i.e. not created hear)
-                    extraEdges[inUpper] += [job._registry[lowerID] for lowerID in lower if lowerID in job._registry]
+                    extraEdges[inUpper] += [
+                        job._registry[lowerID]
+                        for lowerID in lower
+                        if lowerID in job._registry
+                    ]
 
         return extraEdges
 
@@ -2251,17 +2428,21 @@ class Job:
         :raises toil.job.JobGraphDeadlockException: if there exists a job being added to the graph for which \
         checkpoint=True and which is not a leaf.
         """
-        roots = self.getRootJobs() # Roots jobs of component, these are preexisting jobs in the graph
+        roots = (
+            self.getRootJobs()
+        )  # Roots jobs of component, these are preexisting jobs in the graph
 
         # All jobs in the component of the job graph containing self
         jobs = set()
-        list(map(lambda x : x._collectAllSuccessors(jobs), roots))
+        list(map(lambda x: x._collectAllSuccessors(jobs), roots))
 
         # Check for each job for which checkpoint is true that it is a cut vertex or leaf
         for y in [x for x in jobs if x.checkpoint]:
-            if y not in roots: # The roots are the prexisting jobs
+            if y not in roots:  # The roots are the prexisting jobs
                 if not Job._isLeafVertex(y):
-                    raise JobGraphDeadlockException("New checkpoint job %s is not a leaf in the job graph" % y)
+                    raise JobGraphDeadlockException(
+                        "New checkpoint job %s is not a leaf in the job graph" % y
+                    )
 
     ####################################################
     # Deferred function system
@@ -2290,7 +2471,9 @@ class Job:
         :param dict kwargs: The keyword arguments to the function
         """
         if self._defer is None:
-            raise Exception('A deferred function may only be registered with a job while that job is running.')
+            raise Exception(
+                "A deferred function may only be registered with a job while that job is running."
+            )
         self._defer(DeferredFunction.create(function, *args, **kwargs))
 
     ####################################################
@@ -2299,7 +2482,7 @@ class Job:
     # and defining a service (Job.Service)
     ####################################################
 
-    class Runner():
+    class Runner:
         """Used to setup and run Toil workflow."""
 
         @staticmethod
@@ -2315,7 +2498,9 @@ class Job:
             return parser
 
         @staticmethod
-        def getDefaultOptions(jobStore: Optional[str] = None, jobstore_as_flag: bool = False) -> Namespace:
+        def getDefaultOptions(
+            jobStore: Optional[str] = None, jobstore_as_flag: bool = False
+        ) -> Namespace:
             """
             Get default options for a toil workflow.
 
@@ -2326,9 +2511,13 @@ class Job:
             """
             # setting jobstore_as_flag to True allows the user to declare the jobstore in the config file instead
             if not jobstore_as_flag and jobStore is None:
-                raise RuntimeError("The jobstore argument cannot be missing if the jobstore_as_flag argument is set "
-                                   "to False!")
-            parser = Job.Runner.getDefaultArgumentParser(jobstore_as_flag=jobstore_as_flag)
+                raise RuntimeError(
+                    "The jobstore argument cannot be missing if the jobstore_as_flag argument is set "
+                    "to False!"
+                )
+            parser = Job.Runner.getDefaultArgumentParser(
+                jobstore_as_flag=jobstore_as_flag
+            )
             arguments = []
             if jobstore_as_flag and jobStore is not None:
                 arguments = ["--jobstore", jobStore]
@@ -2337,7 +2526,10 @@ class Job:
             return parser.parse_args(args=arguments)
 
         @staticmethod
-        def addToilOptions(parser: Union["OptionParser", ArgumentParser], jobstore_as_flag: bool = False) -> None:
+        def addToilOptions(
+            parser: Union["OptionParser", ArgumentParser],
+            jobstore_as_flag: bool = False,
+        ) -> None:
             """
             Adds the default toil options to an :mod:`optparse` or :mod:`argparse`
             parser object.
@@ -2377,19 +2569,29 @@ class Job:
         Is not executed as a job; runs within a ServiceHostJob.
         """
 
-        def __init__(self, memory=None, cores=None, disk=None, accelerators=None, preemptible=None, unitName=None):
+        def __init__(
+            self,
+            memory=None,
+            cores=None,
+            disk=None,
+            accelerators=None,
+            preemptible=None,
+            unitName=None,
+        ):
             """
             Memory, core and disk requirements are specified identically to as in \
             :func:`toil.job.Job.__init__`.
             """
             # Save the requirements in ourselves so they are visible on `self` to user code.
-            super().__init__({
-                'memory': memory,
-                'cores': cores,
-                'disk': disk,
-                'accelerators': accelerators,
-                'preemptible': preemptible
-            })
+            super().__init__(
+                {
+                    "memory": memory,
+                    "cores": cores,
+                    "disk": disk,
+                    "accelerators": accelerators,
+                    "preemptible": preemptible,
+                }
+            )
 
             # And the unit name
             self.unitName = unitName
@@ -2467,15 +2669,19 @@ class Job:
 
         def filter_main(module_name, class_name):
             try:
-                if module_name == '__main__':
+                if module_name == "__main__":
                     return getattr(userModule, class_name)
                 else:
                     return getattr(importlib.import_module(module_name), class_name)
             except:
-                if module_name == '__main__':
-                    logger.debug('Failed getting %s from module %s.', class_name, userModule)
+                if module_name == "__main__":
+                    logger.debug(
+                        "Failed getting %s from module %s.", class_name, userModule
+                    )
                 else:
-                    logger.debug('Failed getting %s from module %s.', class_name, module_name)
+                    logger.debug(
+                        "Failed getting %s from module %s.", class_name, module_name
+                    )
                 raise
 
         class FilteredUnpickler(pickle.Unpickler):
@@ -2485,7 +2691,9 @@ class Job:
         unpickler = FilteredUnpickler(fileHandle)
 
         runnable = unpickler.load()
-        if requireInstanceOf is not None and not isinstance(runnable, requireInstanceOf):
+        if requireInstanceOf is not None and not isinstance(
+            runnable, requireInstanceOf
+        ):
             raise RuntimeError(f"Did not find a {requireInstanceOf} when expected")
 
         return runnable
@@ -2518,15 +2726,28 @@ class Job:
                 # File may be gone if the job is a service being re-run and the accessing job is
                 # already complete.
                 if jobStore.file_exists(promiseFileStoreID):
-                    logger.debug("Resolve promise %s from %s with a %s", promiseFileStoreID, self, type(promisedValue))
+                    logger.debug(
+                        "Resolve promise %s from %s with a %s",
+                        promiseFileStoreID,
+                        self,
+                        type(promisedValue),
+                    )
                     with jobStore.update_file_stream(promiseFileStoreID) as fileHandle:
                         try:
-                            pickle.dump(promisedValue, fileHandle, pickle.HIGHEST_PROTOCOL)
+                            pickle.dump(
+                                promisedValue, fileHandle, pickle.HIGHEST_PROTOCOL
+                            )
                         except AttributeError:
-                            logger.exception("Could not pickle promise result %s", promisedValue)
+                            logger.exception(
+                                "Could not pickle promise result %s", promisedValue
+                            )
                             raise
                 else:
-                    logger.debug("Do not resolve promise %s from %s because it is no longer needed", promiseFileStoreID, self)
+                    logger.debug(
+                        "Do not resolve promise %s from %s because it is no longer needed",
+                        promiseFileStoreID,
+                        self,
+                    )
 
     # Functions associated with Job.checkJobGraphAcyclic to establish that the job graph does not
     # contain any cycles of dependencies:
@@ -2551,7 +2772,7 @@ class Job:
                         # We added this successor locally
                         todo.append(self._registry[successorID])
 
-    def getTopologicalOrderingOfJobs(self) -> List["Job"]:
+    def getTopologicalOrderingOfJobs(self) -> list["Job"]:
         """
         :returns: a list of jobs such that for all pairs of indices i, j for which i < j, \
         the job at index i can be run before the job at index j.
@@ -2573,8 +2794,8 @@ class Job:
             job = todo[-1]
             todo.pop()
 
-            #Do not add the job to the ordering until all its predecessors have been
-            #added to the ordering
+            # Do not add the job to the ordering until all its predecessors have been
+            # added to the ordering
             outstandingPredecessor = False
             for predJob in job._directPredecessors:
                 if predJob.jobStoreID not in visited:
@@ -2599,7 +2820,7 @@ class Job:
     # Storing Jobs into the JobStore
     ####################################################
 
-    def _register(self, jobStore) -> List[Tuple[TemporaryID, str]]:
+    def _register(self, jobStore) -> list[tuple[TemporaryID, str]]:
         """
         If this job lacks a JobStore-assigned ID, assign this job an ID.
         Must be called for each job before it is saved to the JobStore for the first time.
@@ -2628,7 +2849,7 @@ class Job:
             # We already have an ID. No assignment or reference rewrite necessary.
             return []
 
-    def _renameReferences(self, renames: Dict[TemporaryID, str]) -> None:
+    def _renameReferences(self, renames: dict[TemporaryID, str]) -> None:
         """
         Apply the given dict of ID renames to all references to other jobs.
 
@@ -2664,8 +2885,8 @@ class Job:
 
         # Clear out old Cactus compatibility fields that don't need to be
         # preserved and shouldn't be serialized.
-        if hasattr(self, '_services'):
-            delattr(self, '_services')
+        if hasattr(self, "_services"):
+            delattr(self, "_services")
 
         # Remember fields we will overwrite
         description = self._description
@@ -2683,7 +2904,9 @@ class Job:
                 self._directPredecessors = set()
 
                 # Save the body of the job
-                with jobStore.write_file_stream(description.jobStoreID, cleanup=True) as (fileHandle, fileStoreID):
+                with jobStore.write_file_stream(
+                    description.jobStoreID, cleanup=True
+                ) as (fileHandle, fileStoreID):
                     pickle.dump(self, fileHandle, pickle.HIGHEST_PROTOCOL)
             finally:
                 # Restore important fields (before handling errors)
@@ -2709,7 +2932,12 @@ class Job:
         # Connect the body of the job to the JobDescription
         self._description.attach_body(fileStoreID, userScript)
 
-    def _saveJobGraph(self, jobStore: "AbstractJobStore", saveSelf: bool = False, returnValues: bool = None):
+    def _saveJobGraph(
+        self,
+        jobStore: "AbstractJobStore",
+        saveSelf: bool = False,
+        returnValues: bool = None,
+    ):
         """
         Save job data and new JobDescriptions to the given job store for this
         job and all descending jobs, including services.
@@ -2760,7 +2988,12 @@ class Job:
         # Set up to save last job first, so promises flow the right way
         ordering.reverse()
 
-        logger.debug("Saving graph of %d jobs, %d non-service, %d new", len(allJobs), len(ordering), len(fakeToReal))
+        logger.debug(
+            "Saving graph of %d jobs, %d non-service, %d new",
+            len(allJobs),
+            len(ordering),
+            len(fakeToReal),
+        )
 
         # Make sure we're the root
         if ordering[-1] != self:
@@ -2773,15 +3006,15 @@ class Job:
             if not isinstance(j, ServiceHostJob) and j.jobStoreID not in ordered_ids:
                 raise RuntimeError(f"{j} not found in ordering {ordering}")
 
-
-
         if not saveSelf:
             # Fulfil promises for return values (even if value is None)
             self._fulfillPromises(returnValues, jobStore)
 
         for job in ordering:
             logger.debug("Processing job %s", job.description)
-            for serviceBatch in reversed(list(job.description.serviceHostIDsInBatches())):
+            for serviceBatch in reversed(
+                list(job.description.serviceHostIDsInBatches())
+            ):
                 # For each batch of service host jobs in reverse order they start
                 for serviceID in serviceBatch:
                     logger.debug("Processing service %s", serviceID)
@@ -2819,7 +3052,8 @@ class Job:
         # All other job vertices in the graph are checked by checkNewCheckpointsAreLeafVertices
         if self.checkpoint and not Job._isLeafVertex(self):
             raise JobGraphDeadlockException(
-                'New checkpoint job %s is not a leaf in the job graph' % self)
+                "New checkpoint job %s is not a leaf in the job graph" % self
+            )
 
         # Save the root job and all descendants and services
         self._saveJobGraph(jobStore, saveSelf=True)
@@ -2845,19 +3079,19 @@ class Job:
         :param job_description: the JobDescription of the job to retrieve.
         :returns: The job referenced by the JobDescription.
         """
-        
+
         file_store_id, user_module_descriptor = job_description.get_body()
-        logger.debug('Loading user module %s.', user_module_descriptor)
+        logger.debug("Loading user module %s.", user_module_descriptor)
         user_module = cls._loadUserModule(user_module_descriptor)
 
-        #Loads context manager using file stream
+        # Loads context manager using file stream
         if file_store_id == "firstJob":
             # This one is actually a shared file name and not a file ID.
             manager = job_store.read_shared_file_stream(file_store_id)
         else:
             manager = job_store.read_file_stream(file_store_id)
 
-        #Open and unpickle
+        # Open and unpickle
         with manager as file_handle:
 
             job = cls._unpickle(user_module, file_handle, requireInstanceOf=Job)
@@ -2869,7 +3103,6 @@ class Job:
 
         return job
 
-
     def _run(self, jobGraph=None, fileStore=None, **kwargs):
         """
         Function which worker calls to ultimately invoke
@@ -2934,7 +3167,9 @@ class Job:
             os.chdir(baseDir)
         # Finish up the stats
         if stats is not None:
-            totalCpuTime, totalMemoryUsage = ResourceMonitor.get_total_cpu_time_and_memory_usage()
+            totalCpuTime, totalMemoryUsage = (
+                ResourceMonitor.get_total_cpu_time_and_memory_usage()
+            )
             stats.jobs.append(
                 Expando(
                     time=str(time.time() - startTime),
@@ -2942,7 +3177,7 @@ class Job:
                     class_name=self._jobName(),
                     memory=str(totalMemoryUsage),
                     requested_cores=str(self.cores),
-                    disk=str(fileStore.get_disk_usage())
+                    disk=str(fileStore.get_disk_usage()),
                 )
             )
 
@@ -2987,13 +3222,12 @@ class Job:
         self._defer = None
         self._fileStore = None
 
-
         # Serialize the new Jobs defined by the run method to the jobStore
         self._saveJobGraph(jobStore, saveSelf=False, returnValues=returnValues)
 
         # Clear out the body, because the job is done.
         self.description.detach_body()
-        
+
         # That and the new child/follow-on relationships will need to be
         # recorded later by an update() of the JobDescription.
 
@@ -3016,7 +3250,9 @@ class Job:
 
         return flag in self._debug_flags
 
-    def files_downloaded_hook(self, host_and_job_paths: Optional[List[Tuple[str, str]]] = None) -> None:
+    def files_downloaded_hook(
+        self, host_and_job_paths: Optional[list[tuple[str, str]]] = None
+    ) -> None:
         """
         Function that subclasses can call when they have downloaded their input files.
 
@@ -3031,7 +3267,10 @@ class Job:
             # Stop the worker!
             logger.info("Job has downloaded its files. Stopping.")
             # Send off the path mapping for the debugging wrapper.
-            raise FilesDownloadedStoppingPointReached("Files downloaded", host_and_job_paths=host_and_job_paths)
+            raise FilesDownloadedStoppingPointReached(
+                "Files downloaded", host_and_job_paths=host_and_job_paths
+            )
+
 
 class JobException(Exception):
     """General job exception."""
@@ -3045,6 +3284,7 @@ class JobGraphDeadlockException(JobException):
     An exception raised in the event that a workflow contains an unresolvable \
     dependency, such as a cycle. See :func:`toil.job.Job.checkJobGraphForDeadlocks`.
     """
+
     def __init__(self, string):
         super().__init__(string)
 
@@ -3053,6 +3293,7 @@ class FunctionWrappingJob(Job):
     """
     Job used to wrap a function. In its `run` method the wrapped function is called.
     """
+
     def __init__(self, userFunction, *args, **kwargs):
         """
         :param callable userFunction: The function to wrap. It will be called with ``*args`` and
@@ -3072,7 +3313,9 @@ class FunctionWrappingJob(Job):
         if argSpec.defaults is None:
             argDict = {}
         else:
-            argDict = dict(list(zip(argSpec.args[-len(argSpec.defaults):], argSpec.defaults)))
+            argDict = dict(
+                list(zip(argSpec.args[-len(argSpec.defaults) :], argSpec.defaults))
+            )
 
         def resolve(key, default=None, dehumanize=False):
             try:
@@ -3090,36 +3333,48 @@ class FunctionWrappingJob(Job):
                 value = human2bytes(value)
             return value
 
-        super().__init__(memory=resolve('memory', dehumanize=True),
-                         cores=resolve('cores', dehumanize=True),
-                         disk=resolve('disk', dehumanize=True),
-                         accelerators=resolve('accelerators'),
-                         preemptible=resolve('preemptible'),
-                         checkpoint=resolve('checkpoint', default=False),
-                         unitName=resolve('name', default=None))
+        super().__init__(
+            memory=resolve("memory", dehumanize=True),
+            cores=resolve("cores", dehumanize=True),
+            disk=resolve("disk", dehumanize=True),
+            accelerators=resolve("accelerators"),
+            preemptible=resolve("preemptible"),
+            checkpoint=resolve("checkpoint", default=False),
+            unitName=resolve("name", default=None),
+        )
 
-        self.userFunctionModule = ModuleDescriptor.forModule(userFunction.__module__).globalize()
+        self.userFunctionModule = ModuleDescriptor.forModule(
+            userFunction.__module__
+        ).globalize()
         self.userFunctionName = str(userFunction.__name__)
         self.description.jobName = self.userFunctionName
         self._args = args
         self._kwargs = kwargs
 
     def _getUserFunction(self):
-        logger.debug('Loading user function %s from module %s.',
-                     self.userFunctionName,
-                     self.userFunctionModule)
+        logger.debug(
+            "Loading user function %s from module %s.",
+            self.userFunctionName,
+            self.userFunctionModule,
+        )
         userFunctionModule = self._loadUserModule(self.userFunctionModule)
         return getattr(userFunctionModule, self.userFunctionName)
 
-    def run(self,fileStore):
-        userFunction = self._getUserFunction( )
+    def run(self, fileStore):
+        userFunction = self._getUserFunction()
         return userFunction(*self._args, **self._kwargs)
 
     def getUserScript(self):
         return self.userFunctionModule
 
     def _jobName(self):
-        return ".".join((self.__class__.__name__, self.userFunctionModule.name, self.userFunctionName))
+        return ".".join(
+            (
+                self.__class__.__name__,
+                self.userFunctionModule.name,
+                self.userFunctionName,
+            )
+        )
 
 
 class JobFunctionWrappingJob(FunctionWrappingJob):
@@ -3165,10 +3420,20 @@ class PromisedRequirementFunctionWrappingJob(FunctionWrappingJob):
     Spawns child function using parent function parameters and fulfilled promised
     resource requirements.
     """
+
     def __init__(self, userFunction, *args, **kwargs):
         self._promisedKwargs = kwargs.copy()
         # Replace resource requirements in intermediate job with small values.
-        kwargs.update(dict(disk='1M', memory='32M', cores=0.1, accelerators=[], preemptible=True, preemptable=True))
+        kwargs.update(
+            dict(
+                disk="1M",
+                memory="32M",
+                cores=0.1,
+                accelerators=[],
+                preemptible=True,
+                preemptable=True,
+            )
+        )
         super().__init__(userFunction, *args, **kwargs)
 
     @classmethod
@@ -3193,7 +3458,9 @@ class PromisedRequirementFunctionWrappingJob(FunctionWrappingJob):
         for requirement in REQUIREMENT_NAMES:
             try:
                 if isinstance(self._promisedKwargs[requirement], PromisedRequirement):
-                    self._promisedKwargs[requirement] = self._promisedKwargs[requirement].getValue()
+                    self._promisedKwargs[requirement] = self._promisedKwargs[
+                        requirement
+                    ].getValue()
             except KeyError:
                 pass
 
@@ -3207,7 +3474,9 @@ class PromisedRequirementJobFunctionWrappingJob(PromisedRequirementFunctionWrapp
     def run(self, fileStore):
         self.evaluatePromisedRequirements()
         userFunction = self._getUserFunction()
-        return self.addChildJobFn(userFunction, *self._args, **self._promisedKwargs).rv()
+        return self.addChildJobFn(
+            userFunction, *self._args, **self._promisedKwargs
+        ).rv()
 
 
 class EncapsulatedJob(Job):
@@ -3234,6 +3503,7 @@ class EncapsulatedJob(Job):
     is the return value of the root job, e.g. A().encapsulate().rv() and A().rv() will resolve to
     the same value after A or A.encapsulate() has been run.
     """
+
     def __init__(self, job, unitName=None):
         """
         :param toil.job.Job job: the job to encapsulate.
@@ -3253,7 +3523,12 @@ class EncapsulatedJob(Job):
             Job.addChild(self, job)
             # Use small resource requirements for dummy Job instance.
             # But not too small, or the job won't have enough resources to safely start up Toil.
-            self.encapsulatedFollowOn = Job(disk='100M', memory='512M', cores=0.1, unitName=None if unitName is None else unitName + '-followOn')
+            self.encapsulatedFollowOn = Job(
+                disk="100M",
+                memory="512M",
+                cores=0.1,
+                unitName=None if unitName is None else unitName + "-followOn",
+            )
             Job.addFollowOn(self, self.encapsulatedFollowOn)
         else:
             # Unpickling on the worker, to be run as a no-op.
@@ -3265,17 +3540,25 @@ class EncapsulatedJob(Job):
 
     def addChild(self, childJob):
         if self.encapsulatedFollowOn is None:
-            raise RuntimeError("Children cannot be added to EncapsulatedJob while it is running")
+            raise RuntimeError(
+                "Children cannot be added to EncapsulatedJob while it is running"
+            )
         return Job.addChild(self.encapsulatedFollowOn, childJob)
 
     def addService(self, service, parentService=None):
         if self.encapsulatedFollowOn is None:
-            raise RuntimeError("Services cannot be added to EncapsulatedJob while it is running")
-        return Job.addService(self.encapsulatedFollowOn, service, parentService=parentService)
+            raise RuntimeError(
+                "Services cannot be added to EncapsulatedJob while it is running"
+            )
+        return Job.addService(
+            self.encapsulatedFollowOn, service, parentService=parentService
+        )
 
     def addFollowOn(self, followOnJob):
         if self.encapsulatedFollowOn is None:
-            raise RuntimeError("Follow-ons cannot be added to EncapsulatedJob while it is running")
+            raise RuntimeError(
+                "Follow-ons cannot be added to EncapsulatedJob while it is running"
+            )
         return Job.addFollowOn(self.encapsulatedFollowOn, followOnJob)
 
     def rv(self, *path) -> "Promise":
@@ -3318,6 +3601,7 @@ class ServiceHostJob(Job):
     """
     Job that runs a service. Used internally by Toil. Users should subclass Service instead of using this.
     """
+
     def __init__(self, service):
         """
         This constructor should not be called by a user.
@@ -3328,12 +3612,17 @@ class ServiceHostJob(Job):
 
         # Make sure the service hasn't been given a host already.
         if service.hostID is not None:
-            raise RuntimeError("Cannot set the host. The service has already been given a host.")
+            raise RuntimeError(
+                "Cannot set the host. The service has already been given a host."
+            )
 
         # Make ourselves with name info from the Service and a
         # ServiceJobDescription that has the service control flags.
-        super().__init__(**service.requirements,
-            unitName=service.unitName, descriptionClass=ServiceJobDescription)
+        super().__init__(
+            **service.requirements,
+            unitName=service.unitName,
+            descriptionClass=ServiceJobDescription,
+        )
 
         # Make sure the service knows it has a host now
         service.hostID = self.jobStoreID
@@ -3371,13 +3660,19 @@ class ServiceHostJob(Job):
     # stuff onto us.
 
     def addChild(self, child):
-        raise RuntimeError("Service host jobs cannot have children, follow-ons, or services")
+        raise RuntimeError(
+            "Service host jobs cannot have children, follow-ons, or services"
+        )
 
     def addFollowOn(self, followOn):
-        raise RuntimeError("Service host jobs cannot have children, follow-ons, or services")
+        raise RuntimeError(
+            "Service host jobs cannot have children, follow-ons, or services"
+        )
 
     def addService(self, service, parentService=None):
-        raise RuntimeError("Service host jobs cannot have children, follow-ons, or services")
+        raise RuntimeError(
+            "Service host jobs cannot have children, follow-ons, or services"
+        )
 
     def saveBody(self, jobStore):
         """
@@ -3386,7 +3681,9 @@ class ServiceHostJob(Job):
         # Save unpickled service
         service = self.service
         # Serialize service
-        self.pickledService = pickle.dumps(self.service, protocol=pickle.HIGHEST_PROTOCOL)
+        self.pickledService = pickle.dumps(
+            self.service, protocol=pickle.HIGHEST_PROTOCOL
+        )
         # Clear real service until we have the module to load it back
         self.service = None
         # Save body as normal
@@ -3397,24 +3694,30 @@ class ServiceHostJob(Job):
 
     def run(self, fileStore):
         # Unpickle the service
-        logger.debug('Loading service module %s.', self.serviceModule)
+        logger.debug("Loading service module %s.", self.serviceModule)
         userModule = self._loadUserModule(self.serviceModule)
-        service = self._unpickle(userModule, BytesIO(self.pickledService), requireInstanceOf=Job.Service)
+        service = self._unpickle(
+            userModule, BytesIO(self.pickledService), requireInstanceOf=Job.Service
+        )
         self.pickledService = None
         # Make sure it has the config, since it wasn't load()-ed via the JobStore
         service.assignConfig(fileStore.jobStore.config)
-        #Start the service
+        # Start the service
         startCredentials = service.start(self)
         try:
-            #The start credentials  must be communicated to processes connecting to
-            #the service, to do this while the run method is running we
-            #cheat and set the return value promise within the run method
+            # The start credentials  must be communicated to processes connecting to
+            # the service, to do this while the run method is running we
+            # cheat and set the return value promise within the run method
             self._fulfillPromises(startCredentials, fileStore.jobStore)
-            self._rvs = {}  # Set this to avoid the return values being updated after the
-            #run method has completed!
+            self._rvs = (
+                {}
+            )  # Set this to avoid the return values being updated after the
+            # run method has completed!
 
-            #Now flag that the service is running jobs can connect to it
-            logger.debug("Removing the start jobStoreID to indicate that establishment of the service")
+            # Now flag that the service is running jobs can connect to it
+            logger.debug(
+                "Removing the start jobStoreID to indicate that establishment of the service"
+            )
             if self.description.startJobStoreID is None:
                 raise RuntimeError("No start jobStoreID to remove.")
             if fileStore.jobStore.file_exists(self.description.startJobStoreID):
@@ -3422,23 +3725,33 @@ class ServiceHostJob(Job):
             if fileStore.jobStore.file_exists(self.description.startJobStoreID):
                 raise RuntimeError("The start jobStoreID is not a file.")
 
-            #Now block until we are told to stop, which is indicated by the removal
-            #of a file
+            # Now block until we are told to stop, which is indicated by the removal
+            # of a file
             if self.description.terminateJobStoreID is None:
                 raise RuntimeError("No terminate jobStoreID to use.")
             while True:
                 # Check for the terminate signal
-                if not fileStore.jobStore.file_exists(self.description.terminateJobStoreID):
-                    logger.debug("Detected that the terminate jobStoreID has been removed so exiting")
-                    if not fileStore.jobStore.file_exists(self.description.errorJobStoreID):
-                        raise RuntimeError("Detected the error jobStoreID has been removed so exiting with an error")
+                if not fileStore.jobStore.file_exists(
+                    self.description.terminateJobStoreID
+                ):
+                    logger.debug(
+                        "Detected that the terminate jobStoreID has been removed so exiting"
+                    )
+                    if not fileStore.jobStore.file_exists(
+                        self.description.errorJobStoreID
+                    ):
+                        raise RuntimeError(
+                            "Detected the error jobStoreID has been removed so exiting with an error"
+                        )
                     break
 
                 # Check the service's status and exit if failed or complete
                 try:
                     if not service.check():
-                        logger.debug("The service has finished okay, but we have not been told to terminate. "
-                                     "Waiting for leader to tell us to come back.")
+                        logger.debug(
+                            "The service has finished okay, but we have not been told to terminate. "
+                            "Waiting for leader to tell us to come back."
+                        )
                         # TODO: Adjust leader so that it keys on something
                         # other than the services finishing (assumed to be
                         # after the children) to know when to run follow-on
@@ -3449,7 +3762,9 @@ class ServiceHostJob(Job):
                     logger.debug("Detected abnormal termination of the service")
                     raise
 
-                time.sleep(fileStore.jobStore.config.servicePollingInterval) #Avoid excessive polling
+                time.sleep(
+                    fileStore.jobStore.config.servicePollingInterval
+                )  # Avoid excessive polling
 
             logger.debug("Service is done")
         finally:
@@ -3460,6 +3775,354 @@ class ServiceHostJob(Job):
         return self.serviceModule
 
 
+class FileMetadata(NamedTuple):
+    """
+    Metadata for a file.
+    source is the URL to grab the file from
+    parent_dir is parent directory of the source
+    size is the size of the file. Is none if the filesize cannot be retrieved.
+    """
+
+    source: str
+    parent_dir: str
+    size: Optional[int]
+
+
+def potential_absolute_uris(
+    uri: str,
+    path: list[str],
+    importer: Optional[str] = None,
+    execution_dir: Optional[str] = None,
+) -> Iterator[str]:
+    """
+    Get potential absolute URIs to check for an imported file.
+
+    Given a URI or bare path, yield in turn all the URIs, with schemes, where we
+    should actually try to find it, given that we want to search under/against
+    the given paths or URIs, the current directory, and the given importing WDL
+    document if any.
+    """
+
+    if uri == "":
+        # Empty URIs can't come from anywhere.
+        return
+
+    # We need to brute-force find this URI relative to:
+    #
+    # 1. Itself if a full URI.
+    #
+    # 2. Importer's URL, if importer is a URL and this is a
+    #    host-root-relative URL starting with / or scheme-relative
+    #    starting with //, or just plain relative.
+    #
+    # 3. Current directory, if a relative path.
+    #
+    # 4. All the prefixes in "path".
+    #
+    # If it can't be found anywhere, we ought to (probably) throw
+    # FileNotFoundError like the MiniWDL implementation does, with a
+    # correct errno.
+    #
+    # To do this, we have AbstractFileStore.read_from_url, which can read a
+    # URL into a binary-mode writable, or throw some kind of unspecified
+    # exception if the source doesn't exist or can't be fetched.
+
+    # This holds scheme-applied full URIs for all the places to search.
+    full_path_list = []
+
+    if importer is not None:
+        # Add the place the imported file came form, to search first.
+        full_path_list.append(Toil.normalize_uri(importer))
+
+    # Then the current directory. We need to make sure to include a filename component here or it will treat the current directory with no trailing / as a document and relative paths will look 1 level up.
+    # When importing on a worker, the cwd will be a tmpdir and will result in FileNotFoundError after os.path.abspath, so override with the execution dir
+    full_path_list.append(Toil.normalize_uri(execution_dir or ".") + "/.")
+
+    # Then the specified paths.
+    # TODO:
+    # https://github.com/chanzuckerberg/miniwdl/blob/e3e8ef74e80fbe59f137b0ad40b354957915c345/WDL/Tree.py#L1479-L1482
+    # seems backward actually and might do these first!
+    full_path_list += [Toil.normalize_uri(p) for p in path]
+
+    # This holds all the URIs we tried and failed with.
+    failures: set[str] = set()
+
+    for candidate_base in full_path_list:
+        # Try fetching based off each base URI
+        candidate_uri = urljoin(candidate_base, uri)
+        if candidate_uri in failures:
+            # Already tried this one, maybe we have an absolute uri input.
+            continue
+        logger.debug(
+            "Consider %s which is %s off of %s", candidate_uri, uri, candidate_base
+        )
+
+        # Try it
+        yield candidate_uri
+        # If we come back it didn't work
+        failures.add(candidate_uri)
+
+
+def get_file_sizes(
+    filenames: List[str],
+    file_source: AbstractJobStore,
+    search_paths: Optional[List[str]] = None,
+    include_remote_files: bool = True,
+    execution_dir: Optional[str] = None,
+) -> Dict[str, FileMetadata]:
+    """
+    Resolve relative-URI files in the given environment and turn them into absolute normalized URIs. Returns a dictionary of the *string values* from the WDL file values
+    to a tuple of the normalized URI, parent directory ID, and size of the file. The size of the file may be None, which means unknown size.
+
+    :param filenames: list of filenames to evaluate on
+    :param file_source: Context to search for files with
+    :param task_path: Dotted WDL name of the user-level code doing the
+        importing (probably the workflow name).
+    :param search_paths: If set, try resolving input location relative to the URLs or
+        directories in this list.
+    :param include_remote_files: If set, import files from remote locations. Else leave them as URI references.
+    """
+
+    @memoize
+    def get_filename_size(filename: str) -> FileMetadata:
+        tried = []
+        for candidate_uri in potential_absolute_uris(
+            filename,
+            search_paths if search_paths is not None else [],
+            execution_dir=execution_dir,
+        ):
+            tried.append(candidate_uri)
+            try:
+                if not include_remote_files and is_remote_url(candidate_uri):
+                    # Use remote URIs in place. But we need to find the one that exists.
+                    if not file_source.url_exists(candidate_uri):
+                        # Wasn't found there
+                        continue
+
+                # Now we know this exists, so pass it through
+                # Get filesizes
+                filesize = file_source.get_size(candidate_uri)
+            except UnimplementedURLException as e:
+                # We can't find anything that can even support this URL scheme.
+                # Report to the user, they are probably missing an extra.
+                logger.critical("Error: " + str(e))
+                raise
+            except HTTPError as e:
+                # Something went wrong looking for it there.
+                logger.warning(
+                    "Checked URL %s but got HTTP status %s", candidate_uri, e.code
+                )
+                if e.code == 405:
+                    # 405 Method not allowed, maybe HEAD requests are not supported
+                    filesize = None
+                else:
+                    # Try the next location.
+                    continue
+            except FileNotFoundError:
+                # Wasn't found there
+                continue
+            except Exception:
+                # Something went wrong besides the file not being found. Maybe
+                # we have no auth.
+                logger.error(
+                    "Something went wrong when testing for existence of %s",
+                    candidate_uri,
+                )
+                raise
+
+            # Work out what the basename for the file was
+            file_basename = os.path.basename(urlsplit(candidate_uri).path)
+
+            if file_basename == "":
+                # We can't have files with no basename because we need to
+                # download them at that basename later in WDL.
+                raise RuntimeError(
+                    f"File {candidate_uri} has no basename"
+                )
+
+            # Was actually found
+            if is_remote_url(candidate_uri):
+                # Might be a file URI or other URI.
+                # We need to make sure file URIs and local paths that point to
+                # the same place are treated the same.
+                parsed = urlsplit(candidate_uri)
+                if parsed.scheme == "file:":
+                    # This is a local file URI. Convert to a path for source directory tracking.
+                    parent_dir = os.path.dirname(unquote(parsed.path))
+                else:
+                    # This is some other URL. Get the URL to the parent directory and use that.
+                    parent_dir = urljoin(candidate_uri, ".")
+            else:
+                # Must be a local path
+                parent_dir = os.path.dirname(candidate_uri)
+
+            return cast(FileMetadata, (candidate_uri, parent_dir, filesize))
+        # Not found
+        raise RuntimeError(
+            f"Could not find {filename} at any of: {list(potential_absolute_uris(filename, search_paths if search_paths is not None else []))}"
+        )
+
+    return {k: get_filename_size(k) for k in filenames}
+
+
+class CombineImportsJob(Job):
+    """
+    Combine the outputs of multiple WorkerImportsJob into one promise
+    """
+
+    def __init__(self, d: Sequence[Promised[Dict[str, FileID]]], **kwargs):
+        """
+        :param d: Sequence of dictionaries to merge
+        """
+        self._d = d
+        super().__init__(**kwargs)
+
+    def run(self, file_store: AbstractFileStore) -> Promised[Dict[str, FileID]]:
+        """
+        Merge the dicts
+        """
+        d = unwrap_all(self._d)
+        return {k: v for item in d for k, v in item.items()}
+
+
+class WorkerImportJob(Job):
+    """
+    Job to do file imports on a worker instead of a leader. Assumes all local and cloud files are accessible.
+
+    For the CWL/WDL runners, this class is only used when runImportsOnWorkers is enabled.
+    """
+
+    def __init__(
+        self,
+        filenames: List[str],
+        local: bool = False,
+        **kwargs: Any
+    ):
+        """
+        Setup importing files on a worker.
+        :param filenames: List of file URIs to import
+        :param kwargs: args for the superclass
+        """
+        self.filenames = filenames
+        super().__init__(local=local, **kwargs)
+
+    @staticmethod
+    def import_files(
+        files: List[str], file_source: "AbstractJobStore"
+    ) -> Dict[str, FileID]:
+        """
+        Import a list of files into the jobstore. Returns a mapping of the filename to the associated FileIDs
+
+        When stream is true but the import is not streamable, the worker will run out of
+        disk space and run a new import job with enough disk space instead.
+        :param files: list of files to import
+        :param file_source: AbstractJobStore
+        :return: Dictionary mapping filenames to associated jobstore FileID
+        """
+        # todo: make the import ensure streaming is done instead of relying on running out of disk space
+        path_to_fileid = {}
+
+        @memoize
+        def import_filename(filename: str) -> Optional[FileID]:
+            return file_source.import_file(filename, symlink=True)
+
+        for file in files:
+            imported = import_filename(file)
+            if imported is not None:
+                path_to_fileid[file] = imported
+        return path_to_fileid
+
+    def run(self, file_store: AbstractFileStore) -> Promised[Dict[str, FileID]]:
+        """
+        Import the workflow inputs and then create and run the workflow.
+        :return: Promise of workflow outputs
+        """
+        return self.import_files(self.filenames, file_store.jobStore)
+
+
+class ImportsJob(Job):
+    """
+    Job to organize and delegate files to individual WorkerImportJobs.
+
+    For the CWL/WDL runners, this is only used when runImportsOnWorkers is enabled
+    """
+
+    def __init__(
+        self,
+        file_to_data: Dict[str, FileMetadata],
+        max_batch_size: ParseableIndivisibleResource,
+        import_worker_disk: ParseableIndivisibleResource,
+        **kwargs: Any,
+    ):
+        """
+        Job to take the inputs for a workflow and import them on a worker instead of a leader. Assumes all local and cloud files are accessible.
+
+        This class is only used when runImportsOnWorkers is enabled.
+
+        :param file_to_data: mapping of file source name to file metadata
+        :param max_batch_size: maximum cumulative file size of a batched import
+        """
+        super().__init__(local=True, **kwargs)
+        self._file_to_data = file_to_data
+        self._max_batch_size = max_batch_size
+        self._import_worker_disk = import_worker_disk
+
+    def run(
+        self, file_store: AbstractFileStore
+    ) -> Tuple[Promised[Dict[str, FileID]], Dict[str, FileMetadata]]:
+        """
+        Import the workflow inputs and then create and run the workflow.
+        :return: Tuple of a mapping from the candidate uri to the file id and a mapping of the source filenames to its metadata. The candidate uri is a field in the file metadata
+        """
+        max_batch_size = self._max_batch_size
+        file_to_data = self._file_to_data
+        # Run WDL imports on a worker instead
+
+        filenames = list(file_to_data.keys())
+
+        import_jobs = []
+
+        # This list will hold lists of batched filenames
+        file_batches = []
+
+        # List of filenames for each batch
+        per_batch_files = []
+        per_batch_size = 0
+        while len(filenames) > 0:
+            filename = filenames.pop(0)
+            # See if adding this to the queue will make the batch job too big
+            filesize = file_to_data[filename][2]
+            if per_batch_size + filesize >= max_batch_size:
+                # batch is too big now, store to schedule the batch
+                if len(per_batch_files) == 0:
+                    # schedule the individual file
+                    per_batch_files.append(filename)
+                file_batches.append(per_batch_files)
+                # reset batching calculation
+                per_batch_size = 0
+            else:
+                per_batch_size += filesize
+            per_batch_files.append(filename)
+
+        if per_batch_files:
+            file_batches.append(per_batch_files)
+
+        # Create batch import jobs for each group of files
+        for batch in file_batches:
+            candidate_uris = [file_to_data[filename][0] for filename in batch]
+            import_jobs.append(WorkerImportJob(candidate_uris, disk=self._import_worker_disk))
+
+        for job in import_jobs:
+            self.addChild(job)
+
+        combine_imports_job = CombineImportsJob([job.rv() for job in import_jobs])
+        for job in import_jobs:
+            job.addFollowOn(combine_imports_job)
+        self.addChild(combine_imports_job)
+
+        return combine_imports_job.rv(), file_to_data
+
+
 class Promise:
     """
     References a return value from a method as a *promise* before the method itself is run.
@@ -3520,7 +4183,9 @@ class Promise:
     def __new__(cls, *args) -> "Promise":
         """Instantiate this Promise."""
         if len(args) != 2:
-            raise RuntimeError("Cannot instantiate promise. Invalid number of arguments given (Expected 2).")
+            raise RuntimeError(
+                "Cannot instantiate promise. Invalid number of arguments given (Expected 2)."
+            )
         if isinstance(args[0], Job):
             # Regular instantiation when promise is created, before it is being pickled
             return super().__new__(cls)
@@ -3541,6 +4206,7 @@ class Promise:
             value = safeUnpickleFromStream(fileHandle)
             return value
 
+
 # Machinery for type-safe-ish Toil Python workflows.
 #
 # TODO: Until we make Promise generic on the promised type, and work out how to
@@ -3548,12 +4214,13 @@ class Promise:
 # method returns, this won't actually be type-safe, because any Promise will be
 # a Promised[] for any type.
 
-T = TypeVar('T')
+T = TypeVar("T")
 # We have type shorthand for a promised value.
 # Uses a generic type alias, so you can have a Promised[T]. See <https://github.com/python/mypy/pull/2378>.
 
 Promised = Union[Promise, T]
 
+
 def unwrap(p: Promised[T]) -> T:
     """
     Function for ensuring you actually have a promised value, and not just a promise.
@@ -3562,9 +4229,10 @@ def unwrap(p: Promised[T]) -> T:
     The "unwrap" terminology is borrowed from Rust.
     """
     if isinstance(p, Promise):
-        raise TypeError(f'Attempted to unwrap a value that is still a Promise: {p}')
+        raise TypeError(f"Attempted to unwrap a value that is still a Promise: {p}")
     return p
 
+
 def unwrap_all(p: Sequence[Promised[T]]) -> Sequence[T]:
     """
     Function for ensuring you actually have a collection of promised values,
@@ -3574,9 +4242,12 @@ def unwrap_all(p: Sequence[Promised[T]]) -> Sequence[T]:
     """
     for i, item in enumerate(p):
         if isinstance(item, Promise):
-            raise TypeError(f'Attempted to unwrap a value at index {i} that is still a Promise: {item}')
+            raise TypeError(
+                f"Attempted to unwrap a value at index {i} that is still a Promise: {item}"
+            )
     return p
 
+
 class PromisedRequirement:
     """
     Class for dynamically allocating job function resource requirements.
@@ -3603,13 +4274,15 @@ class PromisedRequirement:
         :param args: variable length argument list
         :type args: int or .Promise
         """
-        if hasattr(valueOrCallable, '__call__'):
+        if hasattr(valueOrCallable, "__call__"):
             if len(args) == 0:
-                raise RuntimeError('Need parameters for PromisedRequirement function.')
+                raise RuntimeError("Need parameters for PromisedRequirement function.")
             func = valueOrCallable
         else:
             if len(args) != 0:
-                raise RuntimeError('Define a PromisedRequirement function to handle multiple arguments.')
+                raise RuntimeError(
+                    "Define a PromisedRequirement function to handle multiple arguments."
+                )
             func = lambda x: x
             args = [valueOrCallable]
 
@@ -3622,7 +4295,7 @@ class PromisedRequirement:
         return func(*self._args)
 
     @staticmethod
-    def convertPromises(kwargs: Dict[str, Any]) -> bool:
+    def convertPromises(kwargs: dict[str, Any]) -> bool:
         """
         Return True if reserved resource keyword is a Promise or PromisedRequirement instance.
 
@@ -3651,15 +4324,15 @@ class UnfulfilledPromiseSentinel:
         self.file_id = file_id
 
     @staticmethod
-    def __setstate__(stateDict: Dict[str, Any]) -> None:
+    def __setstate__(stateDict: dict[str, Any]) -> None:
         """
         Only called when unpickling.
 
         This won't be unpickled unless the promise wasn't resolved, so we throw
         an exception.
         """
-        jobName = stateDict['fulfillingJobName']
-        file_id = stateDict['file_id']
+        jobName = stateDict["fulfillingJobName"]
+        file_id = stateDict["file_id"]
         raise RuntimeError(
             f"This job was passed promise {file_id} that wasn't yet resolved when it "
             f"ran. The job {jobName} that fulfills this promise hasn't yet "