toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/provisioners/abstractProvisioner.py

@@ -20,7 +20,7 @@ import tempfile
 import textwrap
 from abc import ABC, abstractmethod
 from functools import total_ordering
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from typing import Any, Optional, Union
 from urllib.parse import quote
 from uuid import uuid4
 
@@ -55,6 +55,7 @@ class Shape:
     The memory and disk attributes store the number of bytes required by a job (or provided by a
     node) in RAM or on disk (SSD or HDD), respectively.
     """
+
     def __init__(
         self,
         wallTime: Union[int, float],
@@ -70,11 +71,13 @@ class Shape:
         self.preemptible = preemptible
 
     def __eq__(self, other: Any) -> bool:
-        return (self.wallTime == other.wallTime and
-                self.memory == other.memory and
-                self.cores == other.cores and
-                self.disk == other.disk and
-                self.preemptible == other.preemptible)
+        return (
+            self.wallTime == other.wallTime
+            and self.memory == other.memory
+            and self.cores == other.cores
+            and self.disk == other.disk
+            and self.preemptible == other.preemptible
+        )
 
     def greater_than(self, other: Any) -> bool:
         if self.preemptible < other.preemptible:
@@ -104,12 +107,13 @@ class Shape:
         return self.greater_than(other)
 
     def __repr__(self) -> str:
-        return "Shape(wallTime=%s, memory=%s, cores=%s, disk=%s, preemptible=%s)" % \
-               (self.wallTime,
-                self.memory,
-                self.cores,
-                self.disk,
-                self.preemptible)
+        return "Shape(wallTime=%s, memory=%s, cores=%s, disk=%s, preemptible=%s)" % (
+            self.wallTime,
+            self.memory,
+            self.cores,
+            self.disk,
+            self.preemptible,
+        )
 
     def __str__(self) -> str:
         return self.__repr__()
@@ -117,17 +121,14 @@ class Shape:
     def __hash__(self) -> int:
         # Since we replaced __eq__ we need to replace __hash__ as well.
         return hash(
-            (self.wallTime,
-             self.memory,
-             self.cores,
-             self.disk,
-             self.preemptible))
+            (self.wallTime, self.memory, self.cores, self.disk, self.preemptible)
+        )
 
 
 class AbstractProvisioner(ABC):
     """Interface for provisioning worker nodes to use in a Toil cluster."""
 
-    LEADER_HOME_DIR = '/root/'  # home directory in the Toil appliance on an instance
+    LEADER_HOME_DIR = "/root/"  # home directory in the Toil appliance on an instance
     cloud: str = None
 
     def __init__(
@@ -136,7 +137,8 @@ class AbstractProvisioner(ABC):
         clusterType: Optional[str] = "mesos",
         zone: Optional[str] = None,
         nodeStorage: int = 50,
-        nodeStorageOverrides: Optional[List[str]] = None,
+        nodeStorageOverrides: Optional[list[str]] = None,
+        enable_fuse: bool = False,
     ) -> None:
         """
         Initialize provisioner.
@@ -160,13 +162,16 @@ class AbstractProvisioner(ABC):
         self._nodeStorage = nodeStorage
         self._nodeStorageOverrides = {}
         for override in nodeStorageOverrides or []:
-            nodeShape, storageOverride = override.split(':')
+            nodeShape, storageOverride = override.split(":")
             self._nodeStorageOverrides[nodeShape] = int(storageOverride)
-        self._leaderPrivateIP = None
+        self._leaderPrivateIP: Optional[str] = None
         # This will hold an SSH public key for Mesos clusters, or the
         # Kubernetes joining information as a dict for Kubernetes clusters.
         self._leaderWorkerAuthentication = None
 
+        # Whether or not to use FUSE on the cluster. If true, the cluster's Toil containers will be launched in privileged mode
+        self.enable_fuse = enable_fuse
+
         if clusterName:
             # Making a new cluster
             self.createClusterSettings()
@@ -175,7 +180,7 @@ class AbstractProvisioner(ABC):
             self.readClusterSettings()
 
     @abstractmethod
-    def supportedClusterTypes(self) -> Set[str]:
+    def supportedClusterTypes(self) -> set[str]:
         """
         Get all the cluster types that this provisioner implementation
         supports.
@@ -241,12 +246,14 @@ class AbstractProvisioner(ABC):
         :param leader: Node to pull credentials from, if not the current machine.
         """
 
-        if self.clusterType == 'mesos':
+        if self.clusterType == "mesos":
             # We're using a Mesos cluster, so set up SSH from leader to workers.
             self._leaderWorkerAuthentication = self._setSSH(leader=leader)
-        elif self.clusterType == 'kubernetes':
+        elif self.clusterType == "kubernetes":
             # We're using a Kubernetes cluster.
-            self._leaderWorkerAuthentication = self._getKubernetesJoiningInfo(leader=leader)
+            self._leaderWorkerAuthentication = self._getKubernetesJoiningInfo(
+                leader=leader
+            )
 
     def _clearLeaderWorkerAuthentication(self):
         """
@@ -273,16 +280,22 @@ class AbstractProvisioner(ABC):
 
         # To work locally or remotely we need to do all our setup work as one
         # big bash -c
-        command = ['bash', '-c', ('set -e; if [ ! -e /root/.sshSuccess ] ; '
-                                  'then ssh-keygen -f /root/.ssh/id_rsa -t rsa -N ""; '
-                                  'touch /root/.sshSuccess; fi; chmod 700 /root/.ssh;')]
+        command = [
+            "bash",
+            "-c",
+            (
+                "set -e; if [ ! -e /root/.sshSuccess ] ; "
+                'then ssh-keygen -f /root/.ssh/id_rsa -t rsa -N ""; '
+                "touch /root/.sshSuccess; fi; chmod 700 /root/.ssh;"
+            ),
+        ]
 
         if leader is None:
             # Run locally
             subprocess.check_call(command)
 
             # Grab from local file
-            with open('/root/.ssh/id_rsa.pub') as f:
+            with open("/root/.ssh/id_rsa.pub") as f:
                 leaderPublicKey = f.read()
         else:
             # Run remotely
@@ -290,20 +303,20 @@ class AbstractProvisioner(ABC):
 
             # Grab from remote file
             with tempfile.TemporaryDirectory() as tmpdir:
-                localFile = os.path.join(tmpdir, 'id_rsa.pub')
-                leader.extractFile('/root/.ssh/id_rsa.pub', localFile, 'toil_leader')
+                localFile = os.path.join(tmpdir, "id_rsa.pub")
+                leader.extractFile("/root/.ssh/id_rsa.pub", localFile, "toil_leader")
 
                 with open(localFile) as f:
                     leaderPublicKey = f.read()
 
         # Drop the key type and keep just the key data
-        leaderPublicKey = leaderPublicKey.split(' ')[1]
+        leaderPublicKey = leaderPublicKey.split(" ")[1]
 
         # confirm it really is an RSA public key
-        assert leaderPublicKey.startswith('AAAAB3NzaC1yc2E'), leaderPublicKey
+        assert leaderPublicKey.startswith("AAAAB3NzaC1yc2E"), leaderPublicKey
         return leaderPublicKey
 
-    def _getKubernetesJoiningInfo(self, leader: Node = None) -> Dict[str, str]:
+    def _getKubernetesJoiningInfo(self, leader: Node = None) -> dict[str, str]:
         """
         Get the Kubernetes joining info created when Kubernetes was set up on
         this node, which is the leader, or on a different specified Node.
@@ -323,22 +336,24 @@ class AbstractProvisioner(ABC):
             # This info is always supposed to be set up before the Toil appliance
             # starts, and mounted in at the same path as on the host. So we just go
             # read it.
-            with open('/etc/kubernetes/worker.ini') as f:
+            with open("/etc/kubernetes/worker.ini") as f:
                 config.read_file(f)
         else:
             # Grab from remote file
             with tempfile.TemporaryDirectory() as tmpdir:
-                localFile = os.path.join(tmpdir, 'worker.ini')
-                leader.extractFile('/etc/kubernetes/worker.ini', localFile, 'toil_leader')
+                localFile = os.path.join(tmpdir, "worker.ini")
+                leader.extractFile(
+                    "/etc/kubernetes/worker.ini", localFile, "toil_leader"
+                )
 
                 with open(localFile) as f:
                     config.read_file(f)
 
         # Grab everything out of the default section where our setup script put
         # it.
-        return dict(config['DEFAULT'])
+        return dict(config["DEFAULT"])
 
-    def setAutoscaledNodeTypes(self, nodeTypes: List[Tuple[Set[str], Optional[float]]]):
+    def setAutoscaledNodeTypes(self, nodeTypes: list[tuple[set[str], Optional[float]]]):
         """
         Set node types, shapes and spot bids for Toil-managed autoscaling.
         :param nodeTypes: A list of node types, as parsed with parse_node_types.
@@ -371,13 +386,13 @@ class AbstractProvisioner(ABC):
         """
         return len(self.getAutoscaledInstanceShapes()) > 0
 
-    def getAutoscaledInstanceShapes(self) -> Dict[Shape, str]:
+    def getAutoscaledInstanceShapes(self) -> dict[Shape, str]:
         """
         Get all the node shapes and their named instance types that the Toil
         autoscaler should manage.
         """
 
-        if hasattr(self, '_shape_to_instance_type'):
+        if hasattr(self, "_shape_to_instance_type"):
             # We have had Toil-managed autoscaling set up
             return dict(self._shape_to_instance_type)
         else:
@@ -414,7 +429,7 @@ class AbstractProvisioner(ABC):
     @abstractmethod
     def addNodes(
         self,
-        nodeTypes: Set[str],
+        nodeTypes: set[str],
         numNodes: int,
         preemptible: bool,
         spotBid: Optional[float] = None,
@@ -429,7 +444,9 @@ class AbstractProvisioner(ABC):
         """
         raise NotImplementedError
 
-    def addManagedNodes(self, nodeTypes: Set[str], minNodes, maxNodes, preemptible, spotBid=None) -> None:
+    def addManagedNodes(
+        self, nodeTypes: set[str], minNodes, maxNodes, preemptible, spotBid=None
+    ) -> None:
         """
         Add a group of managed nodes of the given type, up to the given maximum.
         The nodes will automatically be launched and terminated depending on cluster load.
@@ -444,10 +461,12 @@ class AbstractProvisioner(ABC):
         """
 
         # Not available by default
-        raise ManagedNodesNotSupportedException("Managed nodes not supported by this provisioner")
+        raise ManagedNodesNotSupportedException(
+            "Managed nodes not supported by this provisioner"
+        )
 
     @abstractmethod
-    def terminateNodes(self, nodes: List[Node]) -> None:
+    def terminateNodes(self, nodes: list[Node]) -> None:
         """
         Terminate the nodes represented by given Node objects
 
@@ -463,7 +482,9 @@ class AbstractProvisioner(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def getProvisionedWorkers(self, instance_type: Optional[str] = None, preemptible: Optional[bool] = None) -> List[Node]:
+    def getProvisionedWorkers(
+        self, instance_type: Optional[str] = None, preemptible: Optional[bool] = None
+    ) -> list[Node]:
         """
         Gets all nodes, optionally of the given instance type or
         preemptability, from the provisioner. Includes both static and
@@ -510,7 +531,14 @@ class AbstractProvisioner(ABC):
             # Holds strings like "ssh-rsa actualKeyData" for keys to authorize (independently of cloud provider's system)
             self.sshPublicKeys = []
 
-        def addFile(self, path: str, filesystem: str = 'root', mode: Union[str, int] = '0755', contents: str = '', append: bool = False):
+        def addFile(
+            self,
+            path: str,
+            filesystem: str = "root",
+            mode: Union[str, int] = "0755",
+            contents: str = "",
+            append: bool = False,
+        ):
             """
             Make a file on the instance with the given filesystem, mode, and contents.
 
@@ -522,16 +550,21 @@ class AbstractProvisioner(ABC):
                 mode = int(mode, 8)
             assert isinstance(mode, int)
 
-            contents = 'data:,' + quote(contents.encode('utf-8'))
+            contents = "data:," + quote(contents.encode("utf-8"))
 
-            ignition_file = {'path': path, 'filesystem': filesystem, 'mode': mode, 'contents': {'source': contents}}
+            ignition_file = {
+                "path": path,
+                "filesystem": filesystem,
+                "mode": mode,
+                "contents": {"source": contents},
+            }
 
             if append:
                 ignition_file["append"] = append
 
             self.files.append(ignition_file)
 
-        def addUnit(self, name: str, enabled: bool = True, contents: str = ''):
+        def addUnit(self, name: str, enabled: bool = True, contents: str = ""):
             """
             Make a systemd unit on the instance with the given name (including
             .service), and content. Units will be enabled by default.
@@ -542,7 +575,7 @@ class AbstractProvisioner(ABC):
                 journalctl -xe
             """
 
-            self.units.append({'name': name, 'enabled': enabled, 'contents': contents})
+            self.units.append({"name": name, "enabled": enabled, "contents": contents})
 
         def addSSHRSAKey(self, keyData: str):
             """
@@ -559,30 +592,19 @@ class AbstractProvisioner(ABC):
             # Define the base config.  We're using Flatcar's v2.2.0 fork
             # See: https://github.com/kinvolk/ignition/blob/flatcar-master/doc/configuration-v2_2.md
             config = {
-                'ignition': {
-                    'version': '2.2.0'
-                },
-                'storage': {
-                    'files': self.files
-                },
-                'systemd': {
-                    'units': self.units
-                }
+                "ignition": {"version": "2.2.0"},
+                "storage": {"files": self.files},
+                "systemd": {"units": self.units},
             }
 
             if len(self.sshPublicKeys) > 0:
                 # Add SSH keys if needed
-                config['passwd'] = {
-                    'users': [
-                        {
-                            'name': 'core',
-                            'sshAuthorizedKeys': self.sshPublicKeys
-                        }
-                    ]
+                config["passwd"] = {
+                    "users": [{"name": "core", "sshAuthorizedKeys": self.sshPublicKeys}]
                 }
 
             # Serialize as JSON
-            return json.dumps(config, separators=(',', ':'))
+            return json.dumps(config, separators=(",", ":"))
 
     def getBaseInstanceConfiguration(self) -> InstanceConfiguration:
         """
@@ -592,10 +614,16 @@ class AbstractProvisioner(ABC):
         config = self.InstanceConfiguration()
 
         # We set Flatcar's update reboot strategy to off
-        config.addFile("/etc/coreos/update.conf", mode='0644', contents=textwrap.dedent("""\
+        config.addFile(
+            "/etc/coreos/update.conf",
+            mode="0644",
+            contents=textwrap.dedent(
+                """\
         GROUP=stable
         REBOOT_STRATEGY=off
-        """))
+        """
+            ),
+        )
 
         # Then we have volume mounting. That always happens.
         self.addVolumesService(config)
@@ -617,7 +645,10 @@ class AbstractProvisioner(ABC):
         #
         # TODO: check what kind of instance this is, and what ephemeral volumes
         # *should* be there, and declaratively RAID and mount them.
-        config.addFile("/home/core/volumes.sh", contents=textwrap.dedent("""\
+        config.addFile(
+            "/home/core/volumes.sh",
+            contents=textwrap.dedent(
+                """\
             #!/bin/bash
             set -x
             ephemeral_count=0
@@ -680,9 +711,14 @@ class AbstractProvisioner(ABC):
                 sudo mkdir -p /var/$directory
                 sudo mount --bind /mnt/ephemeral/var/$directory /var/$directory
             done
-            """))
+            """
+            ),
+        )
         # TODO: Make this retry?
-        config.addUnit("volume-mounting.service", contents=textwrap.dedent("""\
+        config.addUnit(
+            "volume-mounting.service",
+            contents=textwrap.dedent(
+                """\
             [Unit]
             Description=mounts ephemeral volumes & bind mounts toil directories
             Before=docker.service
@@ -694,14 +730,19 @@ class AbstractProvisioner(ABC):
 
             [Install]
             WantedBy=multi-user.target
-            """))
+            """
+            ),
+        )
 
     def addNodeExporterService(self, config: InstanceConfiguration):
         """
         Add the node exporter service for Prometheus to an instance configuration.
         """
 
-        config.addUnit("node-exporter.service", contents=textwrap.dedent('''\
+        config.addUnit(
+            "node-exporter.service",
+            contents=textwrap.dedent(
+                """\
             [Unit]
             Description=node-exporter container
             After=docker.service
@@ -724,12 +765,20 @@ class AbstractProvisioner(ABC):
 
             [Install]
             WantedBy=multi-user.target
-            '''))
+            """
+            ),
+        )
 
     def toil_service_env_options(self) -> str:
         return "-e TMPDIR=/var/tmp"
 
-    def add_toil_service(self, config: InstanceConfiguration, role: str, keyPath: str = None, preemptible: bool = False):
+    def add_toil_service(
+        self,
+        config: InstanceConfiguration,
+        role: str,
+        keyPath: str = None,
+        preemptible: bool = False,
+    ):
         """
         Add the Toil leader or worker service to an instance configuration.
 
@@ -746,46 +795,59 @@ class AbstractProvisioner(ABC):
         # transferred. The waitForKey.sh script loops on the new VM until it finds the keyPath file, then it starts the
         # mesos-agent. If there are multiple keys to be transferred, then the last one to be transferred must be
         # set to keyPath.
-        MESOS_LOG_DIR = '--log_dir=/var/lib/mesos '
-        LEADER_DOCKER_ARGS = '--registry=in_memory --cluster={name}'
+        MESOS_LOG_DIR = "--log_dir=/var/lib/mesos "
+        LEADER_DOCKER_ARGS = "--registry=in_memory --cluster={name}"
         # --no-systemd_enable_support is necessary in Ubuntu 16.04 (otherwise,
         # Mesos attempts to contact systemd but can't find its run file)
-        WORKER_DOCKER_ARGS = '--work_dir=/var/lib/mesos --master={ip}:5050 --attributes=preemptible:{preemptible} --no-hostname_lookup --no-systemd_enable_support'
-
-        if self.clusterType == 'mesos':
-            if role == 'leader':
-                entryPoint = 'mesos-master'
-                entryPointArgs = MESOS_LOG_DIR + LEADER_DOCKER_ARGS.format(name=self.clusterName)
-            elif role == 'worker':
-                entryPoint = 'mesos-agent'
-                entryPointArgs = MESOS_LOG_DIR + WORKER_DOCKER_ARGS.format(ip=self._leaderPrivateIP,
-                                                                           preemptible=preemptible)
+        WORKER_DOCKER_ARGS = "--work_dir=/var/lib/mesos --master={ip}:5050 --attributes=preemptible:{preemptible} --no-hostname_lookup --no-systemd_enable_support"
+
+        if self.clusterType == "mesos":
+            if role == "leader":
+                entryPoint = "mesos-master"
+                entryPointArgs = MESOS_LOG_DIR + LEADER_DOCKER_ARGS.format(
+                    name=self.clusterName
+                )
+            elif role == "worker":
+                entryPoint = "mesos-agent"
+                entryPointArgs = MESOS_LOG_DIR + WORKER_DOCKER_ARGS.format(
+                    ip=self._leaderPrivateIP, preemptible=preemptible
+                )
             else:
                 raise RuntimeError("Unknown role %s" % role)
-        elif self.clusterType == 'kubernetes':
-            if role == 'leader':
+        elif self.clusterType == "kubernetes":
+            if role == "leader":
                 # We need *an* entry point or the leader container will finish
                 # and go away, and thus not be available to take user logins.
-                entryPoint = 'sleep'
-                entryPointArgs = 'infinity'
+                entryPoint = "sleep"
+                entryPointArgs = "infinity"
             else:
-                raise RuntimeError('Toil service not needed for %s nodes in a %s cluster',
-                                   role, self.clusterType)
+                raise RuntimeError(
+                    "Toil service not needed for %s nodes in a %s cluster",
+                    role,
+                    self.clusterType,
+                )
         else:
-            raise RuntimeError('Toil service not needed in a %s cluster', self.clusterType)
+            raise RuntimeError(
+                "Toil service not needed in a %s cluster", self.clusterType
+            )
 
         if keyPath:
-            entryPointArgs = keyPath + ' ' + entryPointArgs
+            entryPointArgs = keyPath + " " + entryPointArgs
             entryPoint = "waitForKey.sh"
         customDockerInitCommand = customDockerInitCmd()
         if customDockerInitCommand:
-            entryPointArgs = " ".join(["'" + customDockerInitCommand + "'", entryPoint, entryPointArgs])
+            entryPointArgs = " ".join(
+                ["'" + customDockerInitCommand + "'", entryPoint, entryPointArgs]
+            )
             entryPoint = "customDockerInit.sh"
 
         # Set up the service. Make sure to make it default to using the
         # actually-big temp directory of /var/tmp (see
         # https://systemd.io/TEMPORARY_DIRECTORIES/).
-        config.addUnit(f"toil-{role}.service", contents=textwrap.dedent(f'''\
+        config.addUnit(
+            f"toil-{role}.service",
+            contents=textwrap.dedent(
+                f"""\
             [Unit]
             Description=toil-{role} container
             After=docker.service
@@ -812,15 +874,23 @@ class AbstractProvisioner(ABC):
                 -v /opt:/opt \\
                 -v /etc/kubernetes:/etc/kubernetes \\
                 -v /etc/kubernetes/admin.conf:/root/.kube/config \\
+                {"-e TOIL_KUBERNETES_PRIVILEGED=True --privileged" if self.enable_fuse else 
+                "--security-opt seccomp=unconfined --security-opt systempaths=unconfined"} \\
+                -e TOIL_KUBERNETES_HOST_PATH=/var/lib/toil \\
+                # Pass in a path to use for singularity image caching into the container
+                -e SINGULARITY_CACHEDIR=/var/lib/toil/singularity \\
+                -e MINIWDL__SINGULARITY__IMAGE_CACHE=/var/lib/toil/miniwdl \\
                 --name=toil_{role} \\
                 {applianceSelf()} \\
                 {entryPointArgs}
 
             [Install]
             WantedBy=multi-user.target
-            '''))
+            """
+            ),
+        )
 
-    def getKubernetesValues(self, architecture: str = 'amd64'):
+    def getKubernetesValues(self, architecture: str = "amd64"):
         """
         Returns a dict of Kubernetes component versions and paths for formatting into Kubernetes-related templates.
         """
@@ -847,10 +917,14 @@ class AbstractProvisioner(ABC):
             METRICS_API_VERSION="v0.3.7",
             CLUSTER_NAME=self.clusterName,
             # YAML line that tells the Kubelet to use a cloud provider, if we need one.
-            CLOUD_PROVIDER_SPEC=('cloud-provider: ' + cloud_provider) if cloud_provider else ''
+            CLOUD_PROVIDER_SPEC=(
+                ("cloud-provider: " + cloud_provider) if cloud_provider else ""
+            ),
         )
 
-    def addKubernetesServices(self, config: InstanceConfiguration, architecture: str = 'amd64'):
+    def addKubernetesServices(
+        self, config: InstanceConfiguration, architecture: str = "amd64"
+    ):
         """
         Add installing Kubernetes and Kubeadm and setting up the Kubelet to run when configured to an instance configuration.
         The same process applies to leaders and workers.
@@ -859,7 +933,10 @@ class AbstractProvisioner(ABC):
         values = self.getKubernetesValues(architecture)
 
         # We're going to ship the Kubelet service from Kubernetes' release pipeline via cloud-config
-        config.addUnit("kubelet.service", contents=textwrap.dedent('''\
+        config.addUnit(
+            "kubelet.service",
+            contents=textwrap.dedent(
+                """\
             # This came from https://raw.githubusercontent.com/kubernetes/release/v0.4.0/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service
             # It has been modified to replace /usr/bin with {DOWNLOAD_DIR}
             # License: https://raw.githubusercontent.com/kubernetes/release/v0.4.0/LICENSE
@@ -878,11 +955,16 @@ class AbstractProvisioner(ABC):
 
             [Install]
             WantedBy=multi-user.target
-            ''').format(**values))
+            """
+            ).format(**values),
+        )
 
         # It needs this config file
-        config.addFile("/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", mode='0644',
-                       contents=textwrap.dedent('''\
+        config.addFile(
+            "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf",
+            mode="0644",
+            contents=textwrap.dedent(
+                """\
             # This came from https://raw.githubusercontent.com/kubernetes/release/v0.4.0/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf
             # It has been modified to replace /usr/bin with {DOWNLOAD_DIR}
             # License: https://raw.githubusercontent.com/kubernetes/release/v0.4.0/LICENSE
@@ -898,7 +980,9 @@ class AbstractProvisioner(ABC):
             EnvironmentFile=-/etc/default/kubelet
             ExecStart=
             ExecStart={DOWNLOAD_DIR}/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
-            ''').format(**values))
+            """
+            ).format(**values),
+        )
 
         # Before we let the kubelet try to start, we have to actually download it (and kubeadm)
         # We set up this service so it can restart on failure despite not
@@ -909,7 +993,10 @@ class AbstractProvisioner(ABC):
         # restarts work if the script fails. We also use a condition which
         # treats the service as successful and skips it if it made a file to
         # say it already ran.
-        config.addFile("/home/core/install-kubernetes.sh", contents=textwrap.dedent('''\
+        config.addFile(
+            "/home/core/install-kubernetes.sh",
+            contents=textwrap.dedent(
+                """\
             #!/usr/bin/env bash
             set -e
             FLAG_FILE="{SETUP_STATE_DIR}/install-kubernetes.done"
@@ -928,8 +1015,13 @@ class AbstractProvisioner(ABC):
 
             mkdir -p "{SETUP_STATE_DIR}"
             touch "$FLAG_FILE"
-            ''').format(**values))
-        config.addUnit("install-kubernetes.service", contents=textwrap.dedent('''\
+            """
+            ).format(**values),
+        )
+        config.addUnit(
+            "install-kubernetes.service",
+            contents=textwrap.dedent(
+                """\
             [Unit]
             Description=base Kubernetes installation
             Wants=network-online.target
@@ -947,12 +1039,14 @@ class AbstractProvisioner(ABC):
             [Install]
             WantedBy=multi-user.target
             RequiredBy=kubelet.service
-            ''').format(**values))
+            """
+            ).format(**values),
+        )
 
         # Now we should have the kubeadm command, and the bootlooping kubelet
         # waiting for kubeadm to configure it.
 
-    def getKubernetesAutoscalerSetupCommands(self, values: Dict[str, str]) -> str:
+    def getKubernetesAutoscalerSetupCommands(self, values: dict[str, str]) -> str:
         """
         Return Bash commands that set up the Kubernetes cluster autoscaler for
         provisioning from the environment supported by this provisioner.
@@ -987,7 +1081,11 @@ class AbstractProvisioner(ABC):
 
         # Customize scheduler to pack jobs into as few nodes as possible
         # See: https://kubernetes.io/docs/reference/scheduling/config/#profiles
-        config.addFile("/home/core/scheduler-config.yml", mode='0644', contents=textwrap.dedent('''\
+        config.addFile(
+            "/home/core/scheduler-config.yml",
+            mode="0644",
+            contents=textwrap.dedent(
+                """\
             apiVersion: kubescheduler.config.k8s.io/v1beta1
             kind: KubeSchedulerConfiguration
             clientConnection:
@@ -1001,13 +1099,21 @@ class AbstractProvisioner(ABC):
                     enabled:
                     - name: NodeResourcesMostAllocated
                       weight: 1
-            '''.format(**values)))
+            """.format(
+                    **values
+                )
+            ),
+        )
 
         # Main kubeadm cluster configuration.
         # Make sure to mount the scheduler config where the scheduler can see
         # it, which is undocumented but inferred from
         # https://pkg.go.dev/k8s.io/kubernetes@v1.21.0/cmd/kubeadm/app/apis/kubeadm#ControlPlaneComponent
-        config.addFile("/home/core/kubernetes-leader.yml", mode='0644', contents=textwrap.dedent('''\
+        config.addFile(
+            "/home/core/kubernetes-leader.yml",
+            mode="0644",
+            contents=textwrap.dedent(
+                """\
             apiVersion: kubeadm.k8s.io/v1beta2
             kind: InitConfiguration
             nodeRegistration:
@@ -1039,11 +1145,18 @@ class AbstractProvisioner(ABC):
             serverTLSBootstrap: true
             rotateCertificates: true
             cgroupDriver: systemd
-            '''.format(**values)))
+            """.format(
+                    **values
+                )
+            ),
+        )
 
         # Make a script to apply that and the other cluster components
         # Note that we're escaping {{thing}} as {{{{thing}}}} because we need to match mustaches in a yaml we hack up.
-        config.addFile("/home/core/create-kubernetes-cluster.sh", contents=textwrap.dedent('''\
+        config.addFile(
+            "/home/core/create-kubernetes-cluster.sh",
+            contents=textwrap.dedent(
+                """\
             #!/usr/bin/env bash
             set -e
 
@@ -1076,7 +1189,11 @@ class AbstractProvisioner(ABC):
             kubectl apply -f https://raw.githubusercontent.com/kontena/kubelet-rubber-stamp/release/{RUBBER_STAMP_VERSION}/deploy/role_binding.yaml
             kubectl apply -f https://raw.githubusercontent.com/kontena/kubelet-rubber-stamp/release/{RUBBER_STAMP_VERSION}/deploy/operator.yaml
 
-            ''').format(**values) + self.getKubernetesAutoscalerSetupCommands(values) + textwrap.dedent('''\
+            """
+            ).format(**values)
+            + self.getKubernetesAutoscalerSetupCommands(values)
+            + textwrap.dedent(
+                """\
             # Set up metrics server, which needs serverTLSBootstrap and rubber stamp, and insists on running on a worker
             curl -sSL https://github.com/kubernetes-sigs/metrics-server/releases/download/{METRICS_API_VERSION}/components.yaml | \\
                 sed 's/          - --secure-port=4443/          - --secure-port=4443\\n          - --kubelet-preferred-address-types=Hostname/' | \\
@@ -1090,8 +1207,13 @@ class AbstractProvisioner(ABC):
 
             mkdir -p "{SETUP_STATE_DIR}"
             touch "$FLAG_FILE"
-            ''').format(**values))
-        config.addUnit("create-kubernetes-cluster.service", contents=textwrap.dedent('''\
+            """
+            ).format(**values),
+        )
+        config.addUnit(
+            "create-kubernetes-cluster.service",
+            contents=textwrap.dedent(
+                """\
             [Unit]
             Description=Kubernetes cluster bootstrap
             After=install-kubernetes.service
@@ -1110,10 +1232,15 @@ class AbstractProvisioner(ABC):
             [Install]
             WantedBy=multi-user.target
             RequiredBy=toil-leader.service
-            ''').format(**values))
+            """
+            ).format(**values),
+        )
 
         # We also need a node cleaner service
-        config.addFile("/home/core/cleanup-nodes.sh", contents=textwrap.dedent('''\
+        config.addFile(
+            "/home/core/cleanup-nodes.sh",
+            contents=textwrap.dedent(
+                """\
             #!/usr/bin/env bash
             # cleanup-nodes.sh: constantly clean up NotReady nodes that are tainted as having been deleted
             set -e
@@ -1132,8 +1259,13 @@ class AbstractProvisioner(ABC):
                 done
                 sleep 300
             done
-            ''').format(**values))
-        config.addUnit("cleanup-nodes.service", contents=textwrap.dedent('''\
+            """
+            ).format(**values),
+        )
+        config.addUnit(
+            "cleanup-nodes.service",
+            contents=textwrap.dedent(
+                """\
             [Unit]
             Description=Remove scaled-in nodes
             After=create-kubernetes-cluster.service
@@ -1145,9 +1277,16 @@ class AbstractProvisioner(ABC):
             RestartSec=10
             [Install]
             WantedBy=multi-user.target
-            '''))
+            """
+            ),
+        )
 
-    def addKubernetesWorker(self, config: InstanceConfiguration, authVars: Dict[str, str], preemptible: bool = False):
+    def addKubernetesWorker(
+        self,
+        config: InstanceConfiguration,
+        authVars: dict[str, str],
+        preemptible: bool = False,
+    ):
         """
         Add services to configure as a Kubernetes worker, if Kubernetes is
         already set to be installed.
@@ -1167,10 +1306,16 @@ class AbstractProvisioner(ABC):
         # TODO: We use the same label that EKS uses here, because nothing is standardized.
         # This won't be quite appropriate as we aren't on EKS and we might not
         # even be on AWS, but the batch system should understand it.
-        values['WORKER_LABEL_SPEC'] = 'node-labels: "eks.amazonaws.com/capacityType=SPOT"' if preemptible else ''
+        values["WORKER_LABEL_SPEC"] = (
+            'node-labels: "eks.amazonaws.com/capacityType=SPOT"' if preemptible else ""
+        )
 
         # Kubeadm worker configuration
-        config.addFile("/home/core/kubernetes-worker.yml", mode='0644', contents=textwrap.dedent('''\
+        config.addFile(
+            "/home/core/kubernetes-worker.yml",
+            mode="0644",
+            contents=textwrap.dedent(
+                """\
             apiVersion: kubeadm.k8s.io/v1beta2
             kind: JoinConfiguration
             nodeRegistration:
@@ -1188,10 +1333,17 @@ class AbstractProvisioner(ABC):
             apiVersion: kubelet.config.k8s.io/v1beta1
             kind: KubeletConfiguration
             cgroupDriver: systemd
-            '''.format(**values)))
+            """.format(
+                    **values
+                )
+            ),
+        )
 
         # Make a script to join the cluster using that configuration
-        config.addFile("/home/core/join-kubernetes-cluster.sh", contents=textwrap.dedent('''\
+        config.addFile(
+            "/home/core/join-kubernetes-cluster.sh",
+            contents=textwrap.dedent(
+                """\
             #!/usr/bin/env bash
             set -e
             FLAG_FILE="{SETUP_STATE_DIR}/join-kubernetes-cluster.done"
@@ -1206,9 +1358,14 @@ class AbstractProvisioner(ABC):
 
             mkdir -p "{SETUP_STATE_DIR}"
             touch "$FLAG_FILE"
-            ''').format(**values))
+            """
+            ).format(**values),
+        )
 
-        config.addUnit("join-kubernetes-cluster.service", contents=textwrap.dedent('''\
+        config.addUnit(
+            "join-kubernetes-cluster.service",
+            contents=textwrap.dedent(
+                """\
             [Unit]
             Description=Kubernetes cluster membership
             After=install-kubernetes.service
@@ -1226,9 +1383,17 @@ class AbstractProvisioner(ABC):
 
             [Install]
             WantedBy=multi-user.target
-            ''').format(**values))
+            """
+            ).format(**values),
+        )
 
-    def _getIgnitionUserData(self, role, keyPath=None, preemptible=False, architecture='amd64'):
+    def _getIgnitionUserData(
+        self,
+        role: str,
+        keyPath: Optional[str] = None,
+        preemptible: bool = False,
+        architecture: str = "amd64",
+    ) -> str:
         """
         Return the text (not bytes) user data to pass to a provisioned node.
 
@@ -1242,33 +1407,35 @@ class AbstractProvisioner(ABC):
         # Start with a base config
         config = self.getBaseInstanceConfiguration()
 
-        if self.clusterType == 'kubernetes':
+        if self.clusterType == "kubernetes":
             # Install Kubernetes
             self.addKubernetesServices(config, architecture)
 
-            if role == 'leader':
+            if role == "leader":
                 # Set up the cluster
                 self.addKubernetesLeader(config)
 
             # We can't actually set up a Kubernetes worker without credentials
             # to connect back to the leader.
 
-        if self.clusterType == 'mesos' or role == 'leader':
+        if self.clusterType == "mesos" or role == "leader":
             # Leaders, and all nodes in a Mesos cluster, need a Toil service
             self.add_toil_service(config, role, keyPath, preemptible)
 
-        if role == 'worker' and self._leaderWorkerAuthentication is not None:
+        if role == "worker" and self._leaderWorkerAuthentication is not None:
             # We need to connect the worker to the leader.
-            if self.clusterType == 'mesos':
+            if self.clusterType == "mesos":
                 # This involves an SSH public key form the leader
                 config.addSSHRSAKey(self._leaderWorkerAuthentication)
-            elif self.clusterType == 'kubernetes':
+            elif self.clusterType == "kubernetes":
                 # We can install the Kubernetes worker and make it phone home
                 # to the leader.
                 # TODO: this puts sufficient info to fake a malicious worker
                 # into the worker config, which probably is accessible by
                 # anyone in the cloud account.
-                self.addKubernetesWorker(config, self._leaderWorkerAuthentication, preemptible=preemptible)
+                self.addKubernetesWorker(
+                    config, self._leaderWorkerAuthentication, preemptible=preemptible
+                )
 
         # Make it into a string for Ignition
         user_data = config.toIgnitionConfig()
@@ -1279,21 +1446,29 @@ class AbstractProvisioner(ABC):
         user_data_limit: int = self._get_user_data_limit()
 
         if len(user_data) > user_data_limit:
-            logger.warning(f"Ignition config size exceeds the user data limit ({len(user_data)} > {user_data_limit}).  "
-                           "Writing to cloud storage...")
-
-            src = self._write_file_to_cloud(f'configs/{role}/config-{uuid4()}.ign', contents=user_data.encode('utf-8'))
-
-            return json.dumps({
-                'ignition': {
-                    'version': '2.2.0',
-                    # See: https://github.com/coreos/ignition/blob/spec2x/doc/configuration-v2_2.md
-                    'config': {
-                        'replace': {
-                            'source': src,
-                        }
+            logger.warning(
+                f"Ignition config size exceeds the user data limit ({len(user_data)} > {user_data_limit}).  "
+                "Writing to cloud storage..."
+            )
+
+            src = self._write_file_to_cloud(
+                f"configs/{role}/config-{uuid4()}.ign",
+                contents=user_data.encode("utf-8"),
+            )
+
+            return json.dumps(
+                {
+                    "ignition": {
+                        "version": "2.2.0",
+                        # See: https://github.com/coreos/ignition/blob/spec2x/doc/configuration-v2_2.md
+                        "config": {
+                            "replace": {
+                                "source": src,
+                            }
+                        },
                     }
-                }
-            }, separators=(',', ':'))
+                },
+                separators=(",", ":"),
+            )
 
         return user_data