toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/provisioners/clusterScaler.py

@@ -18,27 +18,21 @@ import math
 import os
 import time
 from collections import defaultdict
-from typing import (TYPE_CHECKING,
-                    Any,
-                    Callable,
-                    Dict,
-                    List,
-                    Optional,
-                    Set,
-                    Tuple,
-                    Union)
-
-from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
-                                                   AbstractScalableBatchSystem,
-                                                   NodeInfo)
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
+
+from toil.batchSystems.abstractBatchSystem import (
+    AbstractBatchSystem,
+    AbstractScalableBatchSystem,
+    NodeInfo,
+)
 from toil.bus import ClusterDesiredSizeMessage, ClusterSizeMessage
 from toil.common import Config
-from toil.options.common import defaultTargetTime
 from toil.job import JobDescription, ServiceJobDescription
 from toil.lib.conversions import bytes2human, human2bytes
 from toil.lib.retry import old_retry
 from toil.lib.threading import ExceptionalThread
 from toil.lib.throttle import throttle
+from toil.options.common import defaultTargetTime
 from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
 
 if TYPE_CHECKING:
@@ -48,18 +42,25 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 # Properties of GKE's memory overhead algorithm
-EVICTION_THRESHOLD = human2bytes('100MiB')
-RESERVE_SMALL_LIMIT = human2bytes('1GiB')
-RESERVE_SMALL_AMOUNT = human2bytes('255MiB')
-RESERVE_BREAKPOINTS: List[Union[int, float]] = [human2bytes('4GiB'), human2bytes('8GiB'), human2bytes('16GiB'), human2bytes('128GiB'), math.inf]
+EVICTION_THRESHOLD = human2bytes("100MiB")
+RESERVE_SMALL_LIMIT = human2bytes("1GiB")
+RESERVE_SMALL_AMOUNT = human2bytes("255MiB")
+RESERVE_BREAKPOINTS: list[Union[int, float]] = [
+    human2bytes("4GiB"),
+    human2bytes("8GiB"),
+    human2bytes("16GiB"),
+    human2bytes("128GiB"),
+    math.inf,
+]
 RESERVE_FRACTIONS = [0.25, 0.2, 0.1, 0.06, 0.02]
 
 # Guess of how much disk space on the root volume is used for the OS and essential container images
-OS_SIZE = human2bytes('5G')
+OS_SIZE = human2bytes("5G")
 
 # Define a type for an explanation of why a job can't fit on a node.
 # Consists of a resource name and a constraining value for that resource.
-FailedConstraint = Tuple[str, Union[int, float, bool]]
+FailedConstraint = tuple[str, Union[int, float, bool]]
+
 
 class BinPackedFit:
     """
@@ -80,24 +81,30 @@ class BinPackedFit:
     :returns: The minimum number of minimal node allocations estimated to be required to run all
               the jobs in jobShapes.
     """
-    nodeReservations: Dict[Shape, List['NodeReservation']]
 
-    def __init__(self, nodeShapes: List[Shape], targetTime: float = defaultTargetTime) -> None:
+    nodeReservations: dict[Shape, list["NodeReservation"]]
+
+    def __init__(
+        self, nodeShapes: list[Shape], targetTime: float = defaultTargetTime
+    ) -> None:
         self.nodeShapes = sorted(nodeShapes)
         self.targetTime = targetTime
         self.nodeReservations = {nodeShape: [] for nodeShape in nodeShapes}
 
-    def binPack(self, jobShapes: List[Shape]) -> Dict[Shape, List[FailedConstraint]]:
+    def binPack(self, jobShapes: list[Shape]) -> dict[Shape, list[FailedConstraint]]:
         """
         Pack a list of jobShapes into the fewest nodes reasonable.
-        
+
         Can be run multiple times.
-        
+
         Returns any distinct Shapes that did not fit, mapping to reasons they did not fit.
         """
         # TODO: Check for redundancy with batchsystems.mesos.JobQueue() sorting
-        logger.debug('Running bin packing for node shapes %s and %s job(s).',
-                     self.nodeShapes, len(jobShapes))
+        logger.debug(
+            "Running bin packing for node shapes %s and %s job(s).",
+            self.nodeShapes,
+            len(jobShapes),
+        )
         # Sort in descending order from largest to smallest. The FFD like-strategy will pack the
         # jobs in order from longest to shortest.
         jobShapes.sort()
@@ -111,11 +118,13 @@ class BinPackedFit:
                 could_not_fit[rejection[0]] = rejection[1]
         return could_not_fit
 
-    def addJobShape(self, jobShape: Shape) -> Optional[Tuple[Shape, List[FailedConstraint]]]:
+    def addJobShape(
+        self, jobShape: Shape
+    ) -> Optional[tuple[Shape, list[FailedConstraint]]]:
         """
         Add the job to the first node reservation in which it will fit. (This
         is the bin-packing aspect).
-        
+
         Returns the job shape again, and a list of failed constraints, if it did not fit.
         """
         chosenNodeShape = None
@@ -126,24 +135,33 @@ class BinPackedFit:
                 break
 
         if chosenNodeShape is None:
-            logger.debug("Couldn't fit job with requirements %s into any nodes in the nodeTypes "
-                         "list.", jobShape)
+            logger.debug(
+                "Couldn't fit job with requirements %s into any nodes in the nodeTypes "
+                "list.",
+                jobShape,
+            )
             # Go back and debug why this happened.
-            fewest_constraints: Optional[List[FailedConstraint]] = None
+            fewest_constraints: Optional[list[FailedConstraint]] = None
             for shape in self.nodeShapes:
                 failures = NodeReservation(nodeShape).get_failed_constraints(jobShape)
-                if fewest_constraints is None or len(failures) < len(fewest_constraints):
+                if fewest_constraints is None or len(failures) < len(
+                    fewest_constraints
+                ):
                     # This was closer to fitting.
                     # TODO: Check the actual constraint values so we don't tell
                     # the user to raise the memory on the smallest machine?
                     fewest_constraints = failures
-            
-            return jobShape, fewest_constraints if fewest_constraints is not None else []
+
+            return jobShape, (
+                fewest_constraints if fewest_constraints is not None else []
+            )
 
         # grab current list of job objects appended to this instance type
         nodeReservations = self.nodeReservations[chosenNodeShape]
         for nodeReservation in nodeReservations:
-            if nodeReservation.attemptToAddJob(jobShape, chosenNodeShape, self.targetTime):
+            if nodeReservation.attemptToAddJob(
+                jobShape, chosenNodeShape, self.targetTime
+            ):
                 # We succeeded adding the job to this node reservation. Now we're done.
                 return None
 
@@ -160,7 +178,7 @@ class BinPackedFit:
             reservation = extendThisReservation
         return None
 
-    def getRequiredNodes(self) -> Dict[Shape, int]:
+    def getRequiredNodes(self) -> dict[Shape, int]:
         """Return a dict from node shape to number of nodes required to run the packed jobs."""
         return {
             nodeShape: len(self.nodeReservations[nodeShape])
@@ -184,48 +202,72 @@ class NodeReservation:
         self.nReservation: Optional[NodeReservation] = None
 
     def __str__(self) -> str:
-        return "-------------------\n" \
-               "Current Reservation\n" \
-               "-------------------\n" \
-               "Shape wallTime: %s\n" \
-               "Shape memory: %s\n" \
-               "Shape cores: %s\n" \
-               "Shape disk: %s\n" \
-               "Shape preempt: %s\n" \
-               "\n" \
-               "nReserv wallTime: %s\n" \
-               "nReserv memory: %s\n" \
-               "nReserv cores: %s\n" \
-               "nReserv disk: %s\n" \
-               "nReserv preempt: %s\n" \
-               "\n" \
-               "Time slices: %s\n" \
-               "\n" % \
-               (self.shape.wallTime,
+        return (
+            "-------------------\n"
+            "Current Reservation\n"
+            "-------------------\n"
+            "Shape wallTime: %s\n"
+            "Shape memory: %s\n"
+            "Shape cores: %s\n"
+            "Shape disk: %s\n"
+            "Shape preempt: %s\n"
+            "\n"
+            "nReserv wallTime: %s\n"
+            "nReserv memory: %s\n"
+            "nReserv cores: %s\n"
+            "nReserv disk: %s\n"
+            "nReserv preempt: %s\n"
+            "\n"
+            "Time slices: %s\n"
+            "\n"
+            % (
+                self.shape.wallTime,
                 self.shape.memory,
                 self.shape.cores,
                 self.shape.disk,
                 self.shape.preemptible,
-                self.nReservation.shape.wallTime if self.nReservation is not None else str(None),
-                self.nReservation.shape.memory if self.nReservation is not None else str(None),
-                self.nReservation.shape.cores if self.nReservation is not None else str(None),
-                self.nReservation.shape.disk if self.nReservation is not None else str(None),
-                self.nReservation.shape.preemptible if self.nReservation is not None else str(None),
-                str(len(self.shapes())))
-
-    def get_failed_constraints(self, job_shape: Shape) -> List[FailedConstraint]:
+                (
+                    self.nReservation.shape.wallTime
+                    if self.nReservation is not None
+                    else str(None)
+                ),
+                (
+                    self.nReservation.shape.memory
+                    if self.nReservation is not None
+                    else str(None)
+                ),
+                (
+                    self.nReservation.shape.cores
+                    if self.nReservation is not None
+                    else str(None)
+                ),
+                (
+                    self.nReservation.shape.disk
+                    if self.nReservation is not None
+                    else str(None)
+                ),
+                (
+                    self.nReservation.shape.preemptible
+                    if self.nReservation is not None
+                    else str(None)
+                ),
+                str(len(self.shapes())),
+            )
+        )
+
+    def get_failed_constraints(self, job_shape: Shape) -> list[FailedConstraint]:
         """
         Check if a job shape's resource requirements will fit within this allocation.
-        
+
         If the job does *not* fit, returns the failing constraints: the resources
         that can't be accomodated, and the limits that were hit.
-        
+
         If the job *does* fit, returns an empty list.
-        
+
         Must always agree with fits()! This codepath is slower and used for diagnosis.
         """
-        
-        failures: List[FailedConstraint] = []
+
+        failures: list[FailedConstraint] = []
         if job_shape.memory > self.shape.memory:
             failures.append(("memory", self.shape.memory))
         if job_shape.cores > self.shape.cores:
@@ -235,15 +277,17 @@ class NodeReservation:
         if not job_shape.preemptible and self.shape.preemptible:
             failures.append(("preemptible", self.shape.preemptible))
         return failures
-    
+
     def fits(self, jobShape: Shape) -> bool:
         """Check if a job shape's resource requirements will fit within this allocation."""
-        return jobShape.memory <= self.shape.memory and \
-               jobShape.cores <= self.shape.cores and \
-               jobShape.disk <= self.shape.disk and \
-               (jobShape.preemptible or not self.shape.preemptible)
+        return (
+            jobShape.memory <= self.shape.memory
+            and jobShape.cores <= self.shape.cores
+            and jobShape.disk <= self.shape.disk
+            and (jobShape.preemptible or not self.shape.preemptible)
+        )
 
-    def shapes(self) -> List[Shape]:
+    def shapes(self) -> list[Shape]:
         """Get all time-slice shapes, in order, from this reservation on."""
         shapes = []
         curRes: Optional[NodeReservation] = self
@@ -254,11 +298,13 @@ class NodeReservation:
 
     def subtract(self, jobShape: Shape) -> None:
         """Subtract the resources necessary to run a jobShape from the reservation."""
-        self.shape = Shape(self.shape.wallTime,
-                           self.shape.memory - jobShape.memory,
-                           self.shape.cores - jobShape.cores,
-                           self.shape.disk - jobShape.disk,
-                           self.shape.preemptible)
+        self.shape = Shape(
+            self.shape.wallTime,
+            self.shape.memory - jobShape.memory,
+            self.shape.cores - jobShape.cores,
+            self.shape.disk - jobShape.disk,
+            self.shape.preemptible,
+        )
 
     def attemptToAddJob(
         self, jobShape: Shape, nodeShape: Shape, targetTime: float
@@ -286,27 +332,42 @@ class NodeReservation:
                 # does the job time fit in the reservation's remaining time?
                 if availableTime >= jobShape.wallTime:
                     timeSlice: float = 0
-                    while (startingReservation != endingReservation):
+                    while startingReservation != endingReservation:
                         # removes resources only (NO time) from startingReservation
                         startingReservation.subtract(jobShape)  # type: ignore
                         # set aside the timeSlice
                         timeSlice += startingReservation.shape.wallTime  # type: ignore
                         startingReservation = startingReservation.nReservation  # type: ignore
-                    assert jobShape.wallTime - timeSlice <= startingReservation.shape.wallTime
-                    adjustEndingReservationForJob(endingReservation, jobShape, timeSlice)
+                    assert (
+                        jobShape.wallTime - timeSlice
+                        <= startingReservation.shape.wallTime
+                    )
+                    adjustEndingReservationForJob(
+                        endingReservation, jobShape, timeSlice
+                    )
                     # Packed the job.
                     return True
 
                 # If the job would fit, but is longer than the total node allocation
                 # extend the node allocation
-                elif endingReservation.nReservation == None and startingReservation == self:
+                elif (
+                    endingReservation.nReservation == None
+                    and startingReservation == self
+                ):
                     # Extend the node reservation to accommodate jobShape
                     endingReservation.nReservation = NodeReservation(nodeShape)
             # can't run the job with the current resources
             else:
-                if startingReservationTime + availableTime + endingReservation.shape.wallTime <= targetTime:
+                if (
+                    startingReservationTime
+                    + availableTime
+                    + endingReservation.shape.wallTime
+                    <= targetTime
+                ):
                     startingReservation = endingReservation.nReservation
-                    startingReservationTime += availableTime + endingReservation.shape.wallTime
+                    startingReservationTime += (
+                        availableTime + endingReservation.shape.wallTime
+                    )
                     availableTime = 0
                 else:
                     break
@@ -332,7 +393,9 @@ def adjustEndingReservationForJob(
     """
     if jobShape.wallTime - wallTime < reservation.shape.wallTime:
         # This job only partially fills one of the slices. Create a new slice.
-        reservation.shape, nS = split(reservation.shape, jobShape, jobShape.wallTime - wallTime)
+        reservation.shape, nS = split(
+            reservation.shape, jobShape, jobShape.wallTime - wallTime
+        )
         nS.nReservation = reservation.nReservation
         reservation.nReservation = nS
     else:
@@ -342,30 +405,40 @@ def adjustEndingReservationForJob(
 
 def split(
     nodeShape: Shape, jobShape: Shape, wallTime: float
-) -> Tuple[Shape, NodeReservation]:
+) -> tuple[Shape, NodeReservation]:
     """
     Partition a node allocation into two to fit the job.
 
     Returning the modified shape of the node and a new node reservation for
     the extra time that the job didn't fill.
     """
-    return (Shape(wallTime,
-                  nodeShape.memory - jobShape.memory,
-                  nodeShape.cores - jobShape.cores,
-                  nodeShape.disk - jobShape.disk,
-                  nodeShape.preemptible),
-            NodeReservation(Shape(nodeShape.wallTime - wallTime,
-                                  nodeShape.memory,
-                                  nodeShape.cores,
-                                  nodeShape.disk,
-                                  nodeShape.preemptible)))
-
-
-def binPacking(nodeShapes: List[Shape], jobShapes: List[Shape], goalTime: float) -> Tuple[Dict[Shape, int], Dict[Shape, List[FailedConstraint]]]:
+    return (
+        Shape(
+            wallTime,
+            nodeShape.memory - jobShape.memory,
+            nodeShape.cores - jobShape.cores,
+            nodeShape.disk - jobShape.disk,
+            nodeShape.preemptible,
+        ),
+        NodeReservation(
+            Shape(
+                nodeShape.wallTime - wallTime,
+                nodeShape.memory,
+                nodeShape.cores,
+                nodeShape.disk,
+                nodeShape.preemptible,
+            )
+        ),
+    )
+
+
+def binPacking(
+    nodeShapes: list[Shape], jobShapes: list[Shape], goalTime: float
+) -> tuple[dict[Shape, int], dict[Shape, list[FailedConstraint]]]:
     """
     Using the given node shape bins, pack the given job shapes into nodes to
     get them done in the given amount of time.
-    
+
     Returns a dict saying how many of each node will be needed, a dict from job
     shapes that could not fit to reasons why.
     """
@@ -388,34 +461,37 @@ class ClusterScaler:
         self.provisioner = provisioner
         self.leader = leader
         self.config = config
-        self.static: Dict[bool, Dict[str, "Node"]] = {}
-        
+        self.static: dict[bool, dict[str, "Node"]] = {}
+
         # If we encounter a Shape of job that we don't think we can run, call
         # these callbacks with the Shape that didn't fit and the Shapes that
         # were available.
-        self.on_too_big: List[Callable[[Shape, List[Shape]], Any]] = []
+        self.on_too_big: list[Callable[[Shape, list[Shape]], Any]] = []
 
         # Dictionary of job names to their average runtime, used to estimate wall time of queued
         # jobs for bin-packing
-        self.jobNameToAvgRuntime: Dict[str, float] = {}
-        self.jobNameToNumCompleted: Dict[str, int] = {}
+        self.jobNameToAvgRuntime: dict[str, float] = {}
+        self.jobNameToNumCompleted: dict[str, int] = {}
         self.totalAvgRuntime = 0.0
         self.totalJobsCompleted = 0
 
         self.targetTime: float = config.targetTime
         if self.targetTime <= 0:
-            raise RuntimeError('targetTime (%s) must be a positive integer!' % self.targetTime)
+            raise RuntimeError(
+                "targetTime (%s) must be a positive integer!" % self.targetTime
+            )
         self.betaInertia = config.betaInertia
         if not 0.0 <= self.betaInertia <= 0.9:
-            raise RuntimeError('betaInertia (%f) must be between 0.0 and 0.9!' % self.betaInertia)
-
+            raise RuntimeError(
+                "betaInertia (%f) must be between 0.0 and 0.9!" % self.betaInertia
+            )
 
         # Pull scaling information from the provisioner.
         self.nodeShapeToType = provisioner.getAutoscaledInstanceShapes()
         self.instance_types = list(self.nodeShapeToType.values())
         self.nodeShapes = list(self.nodeShapeToType.keys())
 
-        self.ignoredNodes: Set[str] = set()
+        self.ignoredNodes: set[str] = set()
 
         # A *deficit* exists when we have more jobs that can run on preemptible
         # nodes than we have preemptible nodes. In order to not block these jobs,
@@ -426,13 +502,17 @@ class ClusterScaler:
         # of provisioned preemptible nodes and the number of nodes that were requested.
         # Then, when provisioning non-preemptible nodes of the same type, we attempt to
         # make up the deficit.
-        self.preemptibleNodeDeficit = {instance_type: 0 for instance_type in self.instance_types}
+        self.preemptibleNodeDeficit = {
+            instance_type: 0 for instance_type in self.instance_types
+        }
 
         # Keeps track of the last raw (i.e. float, not limited by
         # max/min nodes) estimates of the number of nodes needed for
         # each node shape. NB: we start with an estimate of 0, so
         # scaling up is smoothed as well.
-        self.previousWeightedEstimate = {nodeShape: 0.0 for nodeShape in self.nodeShapes}
+        self.previousWeightedEstimate = {
+            nodeShape: 0.0 for nodeShape in self.nodeShapes
+        }
 
         assert len(self.nodeShapes) > 0
 
@@ -454,26 +534,38 @@ class ClusterScaler:
         self.nodeShapes.sort()
 
         # Nodes might not actually provide all the resources of their nominal shapes
-        self.node_shapes_after_overhead = self.nodeShapes if config.assume_zero_overhead else [self._reserve_overhead(s) for s in self.nodeShapes]
-        self.without_overhead = {k: v for k, v in zip(self.node_shapes_after_overhead, self.nodeShapes)}
+        self.node_shapes_after_overhead = (
+            self.nodeShapes
+            if config.assume_zero_overhead
+            else [self._reserve_overhead(s) for s in self.nodeShapes]
+        )
+        self.without_overhead = {
+            k: v for k, v in zip(self.node_shapes_after_overhead, self.nodeShapes)
+        }
 
-        #Node shape to number of currently provisioned nodes
-        totalNodes: Dict[Shape, int] = defaultdict(int)
-        if isinstance(leader.batchSystem, AbstractScalableBatchSystem) and leader.provisioner:
+        # Node shape to number of currently provisioned nodes
+        totalNodes: dict[Shape, int] = defaultdict(int)
+        if (
+            isinstance(leader.batchSystem, AbstractScalableBatchSystem)
+            and leader.provisioner
+        ):
             for preemptible in (True, False):
-                nodes: List["Node"] = []
+                nodes: list["Node"] = []
                 for nodeShape, instance_type in self.nodeShapeToType.items():
-                    nodes_thisType = leader.provisioner.getProvisionedWorkers(instance_type=instance_type,
-                                                                              preemptible=preemptible)
+                    nodes_thisType = leader.provisioner.getProvisionedWorkers(
+                        instance_type=instance_type, preemptible=preemptible
+                    )
                     totalNodes[nodeShape] += len(nodes_thisType)
                     nodes.extend(nodes_thisType)
 
                 self.setStaticNodes(nodes, preemptible)
 
-        logger.debug('Starting with the following nodes in the cluster: %s' % totalNodes)
+        logger.debug(
+            "Starting with the following nodes in the cluster: %s" % totalNodes
+        )
 
         if not sum(config.maxNodes) > 0:
-            raise RuntimeError('Not configured to create nodes of any type.')
+            raise RuntimeError("Not configured to create nodes of any type.")
 
     def _round(self, number: float) -> int:
         """
@@ -529,7 +621,7 @@ class ClusterScaler:
         # TODO: Figure out if the disk is an OS disk of a scratch disk
         smaller.disk -= self._disk_overhead(smaller.disk)
 
-        logger.debug('Node shape %s can hold jobs of shape %s', full_node, smaller)
+        logger.debug("Node shape %s can hold jobs of shape %s", full_node, smaller)
 
         return smaller
 
@@ -558,12 +650,21 @@ class ClusterScaler:
             # since the previous breakpoint, like a progressive income tax.
             limit = min(breakpoint, memory_bytes)
             reservation = fraction * (limit - accounted)
-            logger.debug('Reserve %s of memory between %s and %s', bytes2human(reservation), bytes2human(accounted), bytes2human(limit))
+            logger.debug(
+                "Reserve %s of memory between %s and %s",
+                bytes2human(reservation),
+                bytes2human(accounted),
+                bytes2human(limit),
+            )
             reserved += reservation
             accounted = limit
             if accounted >= memory_bytes:
                 break
-        logger.debug('Reserved %s/%s memory for overhead', bytes2human(reserved), bytes2human(memory_bytes))
+        logger.debug(
+            "Reserved %s/%s memory for overhead",
+            bytes2human(reserved),
+            bytes2human(memory_bytes),
+        )
 
         return int(reserved) + EVICTION_THRESHOLD
 
@@ -579,15 +680,20 @@ class ClusterScaler:
 
         if disk_bytes <= disk_needed:
             # We don't think we can actually use any of this disk
-            logger.warning('All %sB of disk on a node type are likely to be needed by the OS! The node probably cannot do any useful work!', bytes2human(disk_bytes))
+            logger.warning(
+                "All %sB of disk on a node type are likely to be needed by the OS! The node probably cannot do any useful work!",
+                bytes2human(disk_bytes),
+            )
             return disk_bytes
 
         if disk_needed * 2 > disk_bytes:
-            logger.warning('A node type has only %sB disk, of which more than half are expected to be used by the OS. Consider using a larger --nodeStorage', bytes2human(disk_bytes))
+            logger.warning(
+                "A node type has only %sB disk, of which more than half are expected to be used by the OS. Consider using a larger --nodeStorage",
+                bytes2human(disk_bytes),
+            )
 
         return disk_needed
 
-
     def getAverageRuntime(self, jobName: str, service: bool = False) -> float:
         if service:
             # We short-circuit service jobs and assume that they will
@@ -599,15 +705,15 @@ class ClusterScaler:
             # be running at once for any actual work to get done.
             return self.targetTime * 24 + 3600
         if jobName in self.jobNameToAvgRuntime:
-            #Have seen jobs of this type before, so estimate
-            #the runtime based on average of previous jobs of this type
+            # Have seen jobs of this type before, so estimate
+            # the runtime based on average of previous jobs of this type
             return self.jobNameToAvgRuntime[jobName]
         elif self.totalAvgRuntime > 0:
-            #Haven't seen this job yet, so estimate its runtime as
-            #the average runtime of all completed jobs
+            # Haven't seen this job yet, so estimate its runtime as
+            # the average runtime of all completed jobs
             return self.totalAvgRuntime
         else:
-            #Have no information whatsoever
+            # Have no information whatsoever
             return 1.0
 
     def addCompletedJob(self, job: JobDescription, wallTime: int) -> None:
@@ -618,21 +724,25 @@ class ClusterScaler:
         :param int wallTime: The wall-time taken to complete the job in seconds.
         """
 
-        #Adjust average runtimes to include this job.
+        # Adjust average runtimes to include this job.
         if job.jobName in self.jobNameToAvgRuntime:
             prevAvg = self.jobNameToAvgRuntime[job.jobName]
             prevNum = self.jobNameToNumCompleted[job.jobName]
-            self.jobNameToAvgRuntime[job.jobName] = float(prevAvg*prevNum + wallTime)/(prevNum + 1)
+            self.jobNameToAvgRuntime[job.jobName] = float(
+                prevAvg * prevNum + wallTime
+            ) / (prevNum + 1)
             self.jobNameToNumCompleted[job.jobName] += 1
         else:
             self.jobNameToAvgRuntime[job.jobName] = wallTime
             self.jobNameToNumCompleted[job.jobName] = 1
 
         self.totalJobsCompleted += 1
-        self.totalAvgRuntime = float(self.totalAvgRuntime * (self.totalJobsCompleted - 1) + \
-                                     wallTime)/self.totalJobsCompleted
+        self.totalAvgRuntime = (
+            float(self.totalAvgRuntime * (self.totalJobsCompleted - 1) + wallTime)
+            / self.totalJobsCompleted
+        )
 
-    def setStaticNodes(self, nodes: List["Node"], preemptible: bool) -> None:
+    def setStaticNodes(self, nodes: list["Node"], preemptible: bool) -> None:
         """
         Used to track statically provisioned nodes. This method must be called
         before any auto-scaled nodes are provisioned.
@@ -642,12 +752,12 @@ class ClusterScaler:
 
         :param nodes: list of Node objects
         """
-        prefix = 'non-' if not preemptible else ''
+        prefix = "non-" if not preemptible else ""
         logger.debug("Adding %s to %spreemptible static nodes", nodes, prefix)
         if nodes is not None:
-            self.static[preemptible] = {node.privateIP : node for node in nodes}
+            self.static[preemptible] = {node.privateIP: node for node in nodes}
 
-    def getStaticNodes(self, preemptible: bool) -> Dict[str, "Node"]:
+    def getStaticNodes(self, preemptible: bool) -> dict[str, "Node"]:
         """
         Returns nodes set in setStaticNodes().
 
@@ -662,14 +772,17 @@ class ClusterScaler:
 
         Returns an integer.
         """
-        weightedEstimate = (1 - self.betaInertia) * estimatedNodeCount + \
-                           self.betaInertia * self.previousWeightedEstimate[nodeShape]
+        weightedEstimate = (
+            1 - self.betaInertia
+        ) * estimatedNodeCount + self.betaInertia * self.previousWeightedEstimate[
+            nodeShape
+        ]
         self.previousWeightedEstimate[nodeShape] = weightedEstimate
         return self._round(weightedEstimate)
 
     def getEstimatedNodeCounts(
-        self, queuedJobShapes: List[Shape], currentNodeCounts: Dict[Shape, int]
-    ) -> Tuple[Dict[Shape, int], Dict[Shape, List[FailedConstraint]]]:
+        self, queuedJobShapes: list[Shape], currentNodeCounts: dict[Shape, int]
+    ) -> tuple[dict[Shape, int], dict[Shape, list[FailedConstraint]]]:
         """
         Given the resource requirements of queued jobs and the current size of the cluster.
 
@@ -682,21 +795,30 @@ class ClusterScaler:
         nodesToRunQueuedJobs, could_not_fit = binPacking(
             jobShapes=queuedJobShapes,
             nodeShapes=self.node_shapes_after_overhead,
-            goalTime=self.targetTime
+            goalTime=self.targetTime,
         )
-        
+
         # Then translate back to get results in terms of full nodes without overhead.
-        nodesToRunQueuedJobs = {self.without_overhead[k]: v for k, v in nodesToRunQueuedJobs.items()}
+        nodesToRunQueuedJobs = {
+            self.without_overhead[k]: v for k, v in nodesToRunQueuedJobs.items()
+        }
 
         estimatedNodeCounts = {}
         for nodeShape in self.nodeShapes:
             instance_type = self.nodeShapeToType[nodeShape]
 
-            logger.debug(f"Nodes of type {instance_type} to run queued jobs: {nodesToRunQueuedJobs[nodeShape]}")
+            logger.debug(
+                f"Nodes of type {instance_type} to run queued jobs: {nodesToRunQueuedJobs[nodeShape]}"
+            )
             # Actual calculation of the estimated number of nodes required
-            estimatedNodeCount = 0 if nodesToRunQueuedJobs[nodeShape] == 0 \
+            estimatedNodeCount = (
+                0
+                if nodesToRunQueuedJobs[nodeShape] == 0
                 else max(1, self._round(nodesToRunQueuedJobs[nodeShape]))
-            logger.debug("Estimating %i nodes of shape %s" % (estimatedNodeCount, nodeShape))
+            )
+            logger.debug(
+                "Estimating %i nodes of shape %s" % (estimatedNodeCount, nodeShape)
+            )
 
             # Use inertia parameter to smooth out fluctuations according to an exponentially
             # weighted moving average.
@@ -710,37 +832,56 @@ class ClusterScaler:
                 # The number of nodes we provision as compensation for missing preemptible
                 # nodes is the product of the deficit (the number of preemptible nodes we did
                 # _not_ allocate) and configuration preference.
-                compensationNodes = self._round(self.preemptibleNodeDeficit[instance_type] * compensation)
+                compensationNodes = self._round(
+                    self.preemptibleNodeDeficit[instance_type] * compensation
+                )
                 if compensationNodes > 0:
-                    logger.debug('Adding %d non-preemptible nodes of type %s to compensate for a '
-                                'deficit of %d preemptible ones.', compensationNodes,
-                                instance_type,
-                                self.preemptibleNodeDeficit[instance_type])
+                    logger.debug(
+                        "Adding %d non-preemptible nodes of type %s to compensate for a "
+                        "deficit of %d preemptible ones.",
+                        compensationNodes,
+                        instance_type,
+                        self.preemptibleNodeDeficit[instance_type],
+                    )
                 estimatedNodeCount += compensationNodes
 
             # Tell everyone how big the cluster is
-            logger.debug("Currently %i nodes of type %s in cluster" % (currentNodeCounts[nodeShape],
-                                                                      instance_type))
-            self.leader.toilState.bus.publish(ClusterSizeMessage(instance_type, currentNodeCounts[nodeShape]))
-            self.leader.toilState.bus.publish(ClusterDesiredSizeMessage(instance_type, estimatedNodeCount))
+            logger.debug(
+                "Currently %i nodes of type %s in cluster"
+                % (currentNodeCounts[nodeShape], instance_type)
+            )
+            self.leader.toilState.bus.publish(
+                ClusterSizeMessage(instance_type, currentNodeCounts[nodeShape])
+            )
+            self.leader.toilState.bus.publish(
+                ClusterDesiredSizeMessage(instance_type, estimatedNodeCount)
+            )
 
             # Bound number using the max and min node parameters
             if estimatedNodeCount > self.maxNodes[nodeShape]:
-                logger.debug('Limiting the estimated number of necessary %s (%s) to the '
-                             'configured maximum (%s).', instance_type,
-                             estimatedNodeCount,
-                             self.maxNodes[nodeShape])
+                logger.debug(
+                    "Limiting the estimated number of necessary %s (%s) to the "
+                    "configured maximum (%s).",
+                    instance_type,
+                    estimatedNodeCount,
+                    self.maxNodes[nodeShape],
+                )
                 estimatedNodeCount = self.maxNodes[nodeShape]
             elif estimatedNodeCount < self.minNodes[nodeShape]:
-                logger.debug('Raising the estimated number of necessary %s (%s) to the '
-                            'configured minimum (%s).', instance_type,
-                            estimatedNodeCount,
-                            self.minNodes[nodeShape])
+                logger.debug(
+                    "Raising the estimated number of necessary %s (%s) to the "
+                    "configured minimum (%s).",
+                    instance_type,
+                    estimatedNodeCount,
+                    self.minNodes[nodeShape],
+                )
                 estimatedNodeCount = self.minNodes[nodeShape]
             estimatedNodeCounts[nodeShape] = estimatedNodeCount
         return estimatedNodeCounts, could_not_fit
 
-    def updateClusterSize(self, estimatedNodeCounts: Dict[Shape, int]) -> Dict[Shape, int]:
+    def updateClusterSize(
+        self, estimatedNodeCounts: dict[Shape, int]
+    ) -> dict[Shape, int]:
         """
         Given the desired and current size of the cluster, attempts to launch/remove instances to get to the desired size.
 
@@ -752,21 +893,26 @@ class ClusterScaler:
         for nodeShape, estimatedNodeCount in estimatedNodeCounts.items():
             instance_type = self.nodeShapeToType[nodeShape]
 
-            newNodeCount = self.setNodeCount(instance_type, estimatedNodeCount, preemptible=nodeShape.preemptible)
+            newNodeCount = self.setNodeCount(
+                instance_type, estimatedNodeCount, preemptible=nodeShape.preemptible
+            )
             # If we were scaling up a preemptible node type and failed to meet
             # our target, we will attempt to compensate for the deficit while scaling
             # non-preemptible nodes of this type.
             if nodeShape.preemptible:
                 if newNodeCount < estimatedNodeCount:
                     deficit = estimatedNodeCount - newNodeCount
-                    logger.debug('Preemptible scaler detected deficit of %d nodes of type %s.' % (deficit, instance_type))
+                    logger.debug(
+                        "Preemptible scaler detected deficit of %d nodes of type %s."
+                        % (deficit, instance_type)
+                    )
                     self.preemptibleNodeDeficit[instance_type] = deficit
                 else:
                     self.preemptibleNodeDeficit[instance_type] = 0
             newNodeCounts[nodeShape] = newNodeCount
 
-        #Attempt to terminate any nodes that we previously designated for
-        #termination, but which still had workers running.
+        # Attempt to terminate any nodes that we previously designated for
+        # termination, but which still had workers running.
         self._terminateIgnoredNodes()
         return newNodeCounts
 
@@ -800,18 +946,29 @@ class ClusterScaler:
                 actual cluster size at the time this method returns.
         """
         if not isinstance(self.leader.batchSystem, AbstractScalableBatchSystem):
-            raise RuntimeError('Non-scalable batch system abusing a scalable-only function.')
+            raise RuntimeError(
+                "Non-scalable batch system abusing a scalable-only function."
+            )
         for attempt in old_retry(predicate=self.provisioner.retryPredicate):
             with attempt:
                 nodes = self.getNodes(preemptible)
                 logger.debug("Cluster contains %i instances" % len(nodes))
 
-                nodes = {node: nodes[node] for node in nodes if node.nodeType == instance_type}
-                ignoredNodes = [node for node in nodes if node.privateIP in self.ignoredNodes]
+                nodes = {
+                    node: nodes[node]
+                    for node in nodes
+                    if node.nodeType == instance_type
+                }
+                ignoredNodes = [
+                    node for node in nodes if node.privateIP in self.ignoredNodes
+                ]
                 numIgnoredNodes = len(ignoredNodes)
                 numCurrentNodes = len(nodes)
-                logger.debug("Cluster contains %i instances of type %s (%i ignored and draining jobs until "
-                             "they can be safely terminated)" % (numCurrentNodes, instance_type, numIgnoredNodes))
+                logger.debug(
+                    "Cluster contains %i instances of type %s (%i ignored and draining jobs until "
+                    "they can be safely terminated)"
+                    % (numCurrentNodes, instance_type, numIgnoredNodes)
+                )
                 if not force:
                     delta = numNodes - (numCurrentNodes - numIgnoredNodes)
                 else:
@@ -819,38 +976,59 @@ class ClusterScaler:
                 if delta > 0 and numIgnoredNodes > 0:
                     # We can un-ignore a few nodes to compensate for the additional nodes we want.
                     numNodesToUnignore = min(delta, numIgnoredNodes)
-                    logger.debug('Unignoring %i nodes because we want to scale back up again.' % numNodesToUnignore)
+                    logger.debug(
+                        "Unignoring %i nodes because we want to scale back up again."
+                        % numNodesToUnignore
+                    )
                     delta -= numNodesToUnignore
 
                     for node in ignoredNodes[:numNodesToUnignore]:
                         self.ignoredNodes.remove(node.privateIP)
                         self.leader.batchSystem.unignoreNode(node.privateIP)
                 if delta > 0:
-                    logger.info('Adding %i %s nodes to get to desired cluster size of %i.',
-                                delta,
-                                'preemptible' if preemptible else 'non-preemptible',
-                                numNodes)
-                    numNodes = numCurrentNodes + self._addNodes(instance_type, numNodes=delta,
-                                                                preemptible=preemptible)
+                    logger.info(
+                        "Adding %i %s nodes to get to desired cluster size of %i.",
+                        delta,
+                        "preemptible" if preemptible else "non-preemptible",
+                        numNodes,
+                    )
+                    numNodes = numCurrentNodes + self._addNodes(
+                        instance_type, numNodes=delta, preemptible=preemptible
+                    )
                 elif delta < 0:
-                    logger.info('Removing %i %s nodes to get to desired cluster size of %i.', -delta, 'preemptible' if preemptible else 'non-preemptible', numNodes)
-                    numNodes = numCurrentNodes - self._removeNodes(nodes,
-                                                                   instance_type=instance_type,
-                                                                   num_nodes=-delta,
-                                                                   preemptible=preemptible,
-                                                                   force=force)
+                    logger.info(
+                        "Removing %i %s nodes to get to desired cluster size of %i.",
+                        -delta,
+                        "preemptible" if preemptible else "non-preemptible",
+                        numNodes,
+                    )
+                    numNodes = numCurrentNodes - self._removeNodes(
+                        nodes,
+                        instance_type=instance_type,
+                        num_nodes=-delta,
+                        preemptible=preemptible,
+                        force=force,
+                    )
                 elif force:
-                    logger.debug('Cluster already at desired size of %i. Nothing to do.', numNodes)
+                    logger.debug(
+                        "Cluster already at desired size of %i. Nothing to do.",
+                        numNodes,
+                    )
                 else:
-                    logger.debug('Cluster (minus ignored nodes) already at desired size of %i. Nothing to do.', numNodes)
+                    logger.debug(
+                        "Cluster (minus ignored nodes) already at desired size of %i. Nothing to do.",
+                        numNodes,
+                    )
         return numNodes
 
     def _addNodes(self, instance_type: str, numNodes: int, preemptible: bool) -> int:
-        return self.provisioner.addNodes(nodeTypes={instance_type}, numNodes=numNodes, preemptible=preemptible)
+        return self.provisioner.addNodes(
+            nodeTypes={instance_type}, numNodes=numNodes, preemptible=preemptible
+        )
 
     def _removeNodes(
         self,
-        nodes: Dict["Node", NodeInfo],
+        nodes: dict["Node", NodeInfo],
         instance_type: str,
         num_nodes: int,
         preemptible: bool = False,
@@ -867,17 +1045,18 @@ class ClusterScaler:
             nodes = self.getNodes(preemptible)
             # Filter down to nodes of the correct node type
 
-            nodes = {node: nodes[node] for node in nodes if
-                     node.nodeType == instance_type}
+            nodes = {
+                node: nodes[node] for node in nodes if node.nodeType == instance_type
+            }
 
             filtered_nodes = self.filter_out_static_nodes(nodes, preemptible)
             filtered_nodes = filtered_nodes[:num_nodes]
 
             # Join nodes and instances on private IP address.
-            logger.debug('Nodes considered to terminate: %s', ' '.join(map(str, nodes)))
+            logger.debug("Nodes considered to terminate: %s", " ".join(map(str, nodes)))
 
             # Tell the batch system to stop sending jobs to these nodes
-            for (node, nodeInfo) in filtered_nodes:
+            for node, nodeInfo in filtered_nodes:
                 self.ignoredNodes.add(node.privateIP)
                 self.leader.batchSystem.ignoreNode(node.privateIP)
 
@@ -886,8 +1065,11 @@ class ClusterScaler:
                 # will be terminated in _removeIgnoredNodes later on
                 # once all jobs have finished, but they will be ignored by
                 # the batch system and cluster scaler from now on
-                filtered_nodes = [(node, nodeInfo) for (node, nodeInfo) in filtered_nodes if
-                                  nodeInfo and nodeInfo.workers < 1]
+                filtered_nodes = [
+                    (node, nodeInfo)
+                    for (node, nodeInfo) in filtered_nodes
+                    if nodeInfo and nodeInfo.workers < 1
+                ]
             nodes_to_terminate = [node for (node, nodeInfo) in filtered_nodes]
             for node in nodes_to_terminate:
                 if node.privateIP in self.ignoredNodes:
@@ -895,10 +1077,12 @@ class ClusterScaler:
                     self.leader.batchSystem.unignoreNode(node.privateIP)
         else:
             # Without load info all we can do is sort instances by time left in billing cycle.
-            nodes_to_terminate = sorted(nodes.keys(), key=lambda x: x.remainingBillingInterval())
+            nodes_to_terminate = sorted(
+                nodes.keys(), key=lambda x: x.remainingBillingInterval()
+            )
             nodes_to_terminate = nodes_to_terminate[:num_nodes]
         number_terminated = len(nodes_to_terminate)
-        logger.debug('Terminating %i instance(s).', number_terminated)
+        logger.debug("Terminating %i instance(s).", number_terminated)
         for node in nodes_to_terminate:
             if node.privateIP in self.ignoredNodes:
                 # TODO: Why are we undoing what was just done above???
@@ -912,7 +1096,9 @@ class ClusterScaler:
         but which still have workers running.
         """
         if not isinstance(self.leader.batchSystem, AbstractScalableBatchSystem):
-            raise RuntimeError('Non-scalable batch system abusing a scalable-only function.')
+            raise RuntimeError(
+                "Non-scalable batch system abusing a scalable-only function."
+            )
 
         # start with a dictionary of all nodes and filter down
         nodes = self.getNodes()
@@ -926,10 +1112,18 @@ class ClusterScaler:
             self.ignoredNodes.remove(ip)
             self.leader.batchSystem.unignoreNode(ip)
 
-        logger.debug("There are %i nodes being ignored by the batch system, "
-                     "checking if they can be terminated" % len(self.ignoredNodes))
-        nodes = {node: info for node, info in nodes.items() if node.privateIP in self.ignoredNodes}
-        nodes = {node: info for node, info in nodes.items() if info and info.workers < 1}
+        logger.debug(
+            "There are %i nodes being ignored by the batch system, "
+            "checking if they can be terminated" % len(self.ignoredNodes)
+        )
+        nodes = {
+            node: info
+            for node, info in nodes.items()
+            if node.privateIP in self.ignoredNodes
+        }
+        nodes = {
+            node: info for node, info in nodes.items() if info and info.workers < 1
+        }
         nodes_to_terminate = list(nodes.keys())
 
         for node in nodes_to_terminate:
@@ -938,25 +1132,32 @@ class ClusterScaler:
         self.provisioner.terminateNodes(nodes_to_terminate)
 
     def filter_out_static_nodes(
-            self,
-            nodes: Dict["Node", NodeInfo],
-            preemptible: bool = False) -> List[Tuple["Node", NodeInfo]]:
+        self, nodes: dict["Node", NodeInfo], preemptible: bool = False
+    ) -> list[tuple["Node", NodeInfo]]:
         filtered_nodes = []
         for node, nodeInfo in nodes.items():
             if node:
-                non = 'non-' if not preemptible else ''
+                non = "non-" if not preemptible else ""
                 if node.privateIP in self.getStaticNodes(preemptible):
                     # we don't want to automatically terminate any statically provisioned nodes
-                    logger.debug(f'Found {node.privateIP} in {non}preemptible static nodes')
+                    logger.debug(
+                        f"Found {node.privateIP} in {non}preemptible static nodes"
+                    )
                 else:
-                    logger.debug(f'Did not find {node.privateIP} in {non}preemptible static nodes')
+                    logger.debug(
+                        f"Did not find {node.privateIP} in {non}preemptible static nodes"
+                    )
                     filtered_nodes.append((node, nodeInfo))
         # Sort nodes by number of workers and time left in billing cycle
-        filtered_nodes.sort(key=lambda node_nodeInfo: (
-            node_nodeInfo[1].workers if node_nodeInfo[1] else 1, node_nodeInfo[0].remainingBillingInterval()))
+        filtered_nodes.sort(
+            key=lambda node_nodeInfo: (
+                node_nodeInfo[1].workers if node_nodeInfo[1] else 1,
+                node_nodeInfo[0].remainingBillingInterval(),
+            )
+        )
         return filtered_nodes
 
-    def getNodes(self, preemptible: Optional[bool] = None) -> Dict["Node", NodeInfo]:
+    def getNodes(self, preemptible: Optional[bool] = None) -> dict["Node", NodeInfo]:
         """
         Returns a dictionary mapping node identifiers of preemptible or non-preemptible nodes to
         NodeInfo objects, one for each node.
@@ -968,25 +1169,31 @@ class ClusterScaler:
                If None, all nodes will be returned.
         """
         if not isinstance(self.leader.batchSystem, AbstractScalableBatchSystem):
-            raise RuntimeError('Non-scalable batch system abusing a scalable-only function.')
+            raise RuntimeError(
+                "Non-scalable batch system abusing a scalable-only function."
+            )
         # nodes seen within the last 600 seconds (10 minutes)
         recent_nodes = self.leader.batchSystem.getNodes(preemptible, timeout=600)
         # all available nodes
         all_nodes = self.leader.batchSystem.getNodes(preemptible)
         # nodes that are supposedly doing something
-        provisioned_nodes = self.provisioner.getProvisionedWorkers(preemptible=preemptible)
+        provisioned_nodes = self.provisioner.getProvisionedWorkers(
+            preemptible=preemptible
+        )
 
         if len(recent_nodes) != len(provisioned_nodes):
             logger.debug("Consolidating state between mesos and provisioner")
 
-        nodeToInfo: Dict["Node", NodeInfo] = {}
+        nodeToInfo: dict["Node", NodeInfo] = {}
         # fixme: what happens if awsFilterImpairedNodes is used?
         # if this assertion is false it means that user-managed nodes are being
         # used that are outside the provisioner's control
         # this would violate many basic assumptions in autoscaling so it currently not allowed
         for node, ip in ((node, node.privateIP) for node in provisioned_nodes):
             if ip not in recent_nodes:
-                logger.debug("Worker node at %s is not reporting executor information", ip)
+                logger.debug(
+                    "Worker node at %s is not reporting executor information", ip
+                )
 
                 # get up-to-date information about the node, if available
                 info = all_nodes.get(ip)
@@ -1009,9 +1216,15 @@ class ClusterScaler:
                     #
                     # In all 3 situations it's safe to fake executor info with 0 workers,
                     # since in all cases there are no workers running.
-                    info = NodeInfo(coresTotal=1, coresUsed=0, requestedCores=0,
-                                    memoryTotal=1, memoryUsed=0, requestedMemory=0,
-                                    workers=0)
+                    info = NodeInfo(
+                        coresTotal=1,
+                        coresUsed=0,
+                        requestedCores=0,
+                        memoryTotal=1,
+                        memoryUsed=0,
+                        requestedMemory=0,
+                        workers=0,
+                    )
             else:
                 # mesos knows about the ip & we have up-to-date information - easy!
                 info = recent_nodes[ip]
@@ -1020,40 +1233,55 @@ class ClusterScaler:
         return nodeToInfo
 
     def shutDown(self) -> None:
-        logger.debug('Forcing provisioner to reduce cluster size to zero.')
+        logger.debug("Forcing provisioner to reduce cluster size to zero.")
         for nodeShape in self.nodeShapes:
             preemptible = nodeShape.preemptible
             instance_type = self.nodeShapeToType[nodeShape]
-            self.setNodeCount(instance_type=instance_type, numNodes=0, preemptible=preemptible, force=True)
+            self.setNodeCount(
+                instance_type=instance_type,
+                numNodes=0,
+                preemptible=preemptible,
+                force=True,
+            )
+
 
 class JobTooBigError(Exception):
     """
     Raised in the scaler thread when a job cannot fit in any available node
     type and is likely to lock up the workflow.
     """
-    
-    def __init__(self, job: Optional[JobDescription] = None, shape: Optional[Shape] = None, constraints: Optional[List[FailedConstraint]] = None):
+
+    def __init__(
+        self,
+        job: Optional[JobDescription] = None,
+        shape: Optional[Shape] = None,
+        constraints: Optional[list[FailedConstraint]] = None,
+    ):
         """
         Make a JobTooBigError.
-        
+
         Can have a job, the job's shape, and the limiting resources and amounts. All are optional.
         """
         self.job = job
         self.shape = shape
         self.constraints = constraints if constraints is not None else []
-        
+
         parts = [
             f"The job {self.job}" if self.job else "A job",
             f" with shape {self.shape}" if self.shape else "",
-            " is too big for any available node type."
+            " is too big for any available node type.",
         ]
-        
+
         if self.constraints:
             parts.append(" It could have fit if it only needed ")
-            parts.append(", ".join([f"{limit} {resource}" for resource, limit in self.constraints]))
-            parts.append(".") 
-        
-        self.msg = ''.join(parts)
+            parts.append(
+                ", ".join(
+                    [f"{limit} {resource}" for resource, limit in self.constraints]
+                )
+            )
+            parts.append(".")
+
+        self.msg = "".join(parts)
         super().__init__()
 
     def __str__(self) -> str:
@@ -1062,6 +1290,7 @@ class JobTooBigError(Exception):
         """
         return self.msg
 
+
 class ScalerThread(ExceptionalThread):
     """
     A thread that automatically scales the number of either preemptible or non-preemptible worker
@@ -1077,10 +1306,17 @@ class ScalerThread(ExceptionalThread):
     is made, else the size of the cluster is adapted. The beta factor is an inertia parameter
     that prevents continual fluctuations in the number of nodes.
     """
-    def __init__(self, provisioner: AbstractProvisioner, leader: "Leader", config: Config, stop_on_exception: bool = False) -> None:
-        super().__init__(name='scaler')
+
+    def __init__(
+        self,
+        provisioner: AbstractProvisioner,
+        leader: "Leader",
+        config: Config,
+        stop_on_exception: bool = False,
+    ) -> None:
+        super().__init__(name="scaler")
         self.scaler = ClusterScaler(provisioner, leader, config)
-        
+
         # Indicates that the scaling thread should shutdown
         self.stop = False
         # Indicates that we should stop the thread if we encounter an error.
@@ -1090,13 +1326,13 @@ class ScalerThread(ExceptionalThread):
         self.stats = None
         if config.clusterStats:
             logger.debug("Starting up cluster statistics...")
-            self.stats = ClusterStats(leader.config.clusterStats,
-                                      leader.batchSystem,
-                                      provisioner.clusterName)
+            self.stats = ClusterStats(
+                leader.config.clusterStats, leader.batchSystem, provisioner.clusterName
+            )
             for preemptible in [True, False]:
                 self.stats.startStats(preemptible=preemptible)
             logger.debug("...Cluster stats started.")
-            
+
     def check(self) -> None:
         """
         Attempt to join any existing scaler threads that may have died or finished.
@@ -1121,20 +1357,27 @@ class ScalerThread(ExceptionalThread):
 
     def tryRun(self) -> None:
         if self.scaler.leader.provisioner is None:
-            raise RuntimeError('No provisioner found for a scaling cluster '
-                               '(cannot access "getProvisionedWorkers").')
+            raise RuntimeError(
+                "No provisioner found for a scaling cluster "
+                '(cannot access "getProvisionedWorkers").'
+            )
         while not self.stop:
             with throttle(self.scaler.config.scaleInterval):
                 try:
                     queuedJobs = self.scaler.leader.getJobs()
                     queuedJobShapes = [
-                        Shape(wallTime=self.scaler.getAverageRuntime(
-                            jobName=job.jobName,
-                            service=isinstance(job, ServiceJobDescription)),
+                        Shape(
+                            wallTime=self.scaler.getAverageRuntime(
+                                jobName=job.jobName,
+                                service=isinstance(job, ServiceJobDescription),
+                            ),
                             memory=job.memory,
                             cores=job.cores,
                             disk=job.disk,
-                            preemptible=job.preemptible) for job in queuedJobs]
+                            preemptible=job.preemptible,
+                        )
+                        for job in queuedJobs
+                    ]
                     currentNodeCounts = {}
                     for nodeShape in self.scaler.nodeShapes:
                         instance_type = self.scaler.nodeShapeToType[nodeShape]
@@ -1144,14 +1387,16 @@ class ScalerThread(ExceptionalThread):
                                 preemptible=nodeShape.preemptible,
                             )
                         )
-                    estimatedNodeCounts, could_not_fit = self.scaler.getEstimatedNodeCounts(
-                        queuedJobShapes, currentNodeCounts
+                    estimatedNodeCounts, could_not_fit = (
+                        self.scaler.getEstimatedNodeCounts(
+                            queuedJobShapes, currentNodeCounts
+                        )
                     )
                     self.scaler.updateClusterSize(estimatedNodeCounts)
                     if self.stats:
                         self.stats.checkStats()
-                    
-                    if len(could_not_fit) != 0: 
+
+                    if len(could_not_fit) != 0:
                         # If we have any jobs left over that we couldn't fit, complain.
                         bad_job: Optional[JobDescription] = None
                         bad_shape: Optional[Shape] = None
@@ -1164,39 +1409,49 @@ class ScalerThread(ExceptionalThread):
                         if bad_shape is None:
                             # If we can't find an offending job, grab an arbitrary offending shape.
                             bad_shape = next(iter(could_not_fit))
-                        
-                        raise JobTooBigError(job=bad_job, shape=bad_shape, constraints=could_not_fit[bad_shape])
-                        
+
+                        raise JobTooBigError(
+                            job=bad_job,
+                            shape=bad_shape,
+                            constraints=could_not_fit[bad_shape],
+                        )
+
                 except:
                     if self.stop_on_exception:
                         logger.critical("Stopping ScalerThread due to an error.")
                         raise
                     else:
-                        logger.exception("Exception encountered in scaler thread. Making a best-effort "
-                                         "attempt to keep going, but things may go wrong from now on.")
+                        logger.exception(
+                            "Exception encountered in scaler thread. Making a best-effort "
+                            "attempt to keep going, but things may go wrong from now on."
+                        )
         self.scaler.shutDown()
 
+
 class ClusterStats:
     def __init__(
         self, path: str, batchSystem: AbstractBatchSystem, clusterName: Optional[str]
     ) -> None:
         logger.debug("Initializing cluster statistics")
-        self.stats: Dict[str, Dict[str, List[Dict[str, Any]]]] = {}
-        self.statsThreads: List[ExceptionalThread] = []
+        self.stats: dict[str, dict[str, list[dict[str, Any]]]] = {}
+        self.statsThreads: list[ExceptionalThread] = []
         self.statsPath = path
         self.stop = False
         self.clusterName = clusterName
         self.batchSystem = batchSystem
-        self.scaleable = isinstance(self.batchSystem, AbstractScalableBatchSystem) \
-            if batchSystem else False
+        self.scaleable = (
+            isinstance(self.batchSystem, AbstractScalableBatchSystem)
+            if batchSystem
+            else False
+        )
 
     def shutDownStats(self) -> None:
         if self.stop:
             return
 
         def getFileName() -> str:
-            extension = '.json'
-            file = '%s-stats' % self.clusterName
+            extension = ".json"
+            file = "%s-stats" % self.clusterName
             counter = 0
             while True:
                 suffix = str(counter).zfill(3) + extension
@@ -1204,12 +1459,13 @@ class ClusterStats:
                 if not os.path.exists(fullName):
                     return fullName
                 counter += 1
+
         if self.statsPath and self.scaleable:
             self.stop = True
             for thread in self.statsThreads:
                 thread.join()
             fileName = getFileName()
-            with open(fileName, 'w') as f:
+            with open(fileName, "w") as f:
                 json.dump(self.stats, f)
 
     def startStats(self, preemptible: bool) -> None:
@@ -1223,22 +1479,26 @@ class ClusterStats:
             thread.join(timeout=0)
 
     def _gatherStats(self, preemptible: bool) -> None:
-        def toDict(nodeInfo: NodeInfo) -> Dict[str, Any]:
+        def toDict(nodeInfo: NodeInfo) -> dict[str, Any]:
             # convert NodeInfo object to dict to improve JSON output
-            return dict(memory=nodeInfo.memoryUsed,
-                        cores=nodeInfo.coresUsed,
-                        memoryTotal=nodeInfo.memoryTotal,
-                        coresTotal=nodeInfo.coresTotal,
-                        requestedCores=nodeInfo.requestedCores,
-                        requestedMemory=nodeInfo.requestedMemory,
-                        workers=nodeInfo.workers,
-                        time=time.time()  # add time stamp
-                        )
+            return dict(
+                memory=nodeInfo.memoryUsed,
+                cores=nodeInfo.coresUsed,
+                memoryTotal=nodeInfo.memoryTotal,
+                coresTotal=nodeInfo.coresTotal,
+                requestedCores=nodeInfo.requestedCores,
+                requestedMemory=nodeInfo.requestedMemory,
+                workers=nodeInfo.workers,
+                time=time.time(),  # add time stamp
+            )
+
         if self.scaleable:
             logger.debug("Starting to gather statistics")
-            stats: Dict[str, List[Dict[str, Any]]] = {}
+            stats: dict[str, list[dict[str, Any]]] = {}
             if not isinstance(self.batchSystem, AbstractScalableBatchSystem):
-                raise RuntimeError('Non-scalable batch system abusing a scalable-only function.')
+                raise RuntimeError(
+                    "Non-scalable batch system abusing a scalable-only function."
+                )
             try:
                 while not self.stop:
                     nodeInfo = self.batchSystem.getNodes(preemptible)
@@ -1255,6 +1515,8 @@ class ClusterStats:
                                 stats[nodeIP] = [nodeStatsDict]
                     time.sleep(60)
             finally:
-                threadName = 'Preemptible' if preemptible else 'Non-preemptible'
-                logger.debug('%s provisioner stats thread shut down successfully.', threadName)
+                threadName = "Preemptible" if preemptible else "Non-preemptible"
+                logger.debug(
+                    "%s provisioner stats thread shut down successfully.", threadName
+                )
                 self.stats[threadName] = stats