toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/provisioners/aws/awsProvisioner.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 from __future__ import annotations
 
+import base64
 import json
 import logging
 import os
@@ -25,7 +26,7 @@ import uuid
 from collections.abc import Collection, Iterable
 from functools import wraps
 from shlex import quote
-from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast
 
 # We need these to exist as attributes we can get off of the boto object
 from botocore.exceptions import ClientError
@@ -109,6 +110,11 @@ _INSTANCE_PROFILE_ROLE_NAME = "toil"
 _TAG_KEY_TOIL_NODE_TYPE = "ToilNodeType"
 # The tag that specifies the cluster name on all nodes
 _TAG_KEY_TOIL_CLUSTER_NAME = "clusterName"
+# The tag we use to store the SSH key name.
+# TODO: Get rid of this once
+# <https://github.com/adamchainz/ec2-metadata/pull/562> is merged and we can
+# get the SSH key name from the instance metadata.
+_TAG_KEY_TOIL_SSH_KEY = "sshKeyName"
 # How much storage on the root volume is expected to go to overhead and be
 # unavailable to jobs when the node comes up?
 # TODO: measure
@@ -309,11 +315,28 @@ class AWSProvisioner(AbstractProvisioner):
         for tag in instance["Tags"]:
             if tag.get("Key") == "Name":
                 self.clusterName = tag["Value"]
+            elif tag.get("Key") == _TAG_KEY_TOIL_SSH_KEY:
+                # If we can't get an SSH key from the instance metadata, we
+                # might be able to use this one from the tags.
+                self._keyName = tag["Value"]
         # Determine what subnet we, the leader, are in
         self._leader_subnet = instance["SubnetId"]
         # Determine where to deploy workers.
         self._worker_subnets_by_zone = self._get_good_subnets_like(self._leader_subnet)
 
+        # Find the SSH key name to use to start instances
+        if hasattr(ec2_metadata, 'public_keys') and isinstance(ec2_metadata.public_keys, dict):
+            key_names = list(ec2_metadata.public_keys.keys())
+            if len(key_names) > 0 and isinstance(key_names[0], str):
+                # We have a key name from the EC2 metadata. This should always
+                # be the case once
+                # <https://github.com/adamchainz/ec2-metadata/pull/562> is
+                # merged. Override anything from the tags.
+                self._keyName = key_names[0]
+
+        if not hasattr(self, '_keyName'):
+            raise RuntimeError("Unable to determine the SSH key name the cluster is using")
+
         self._leaderPrivateIP = ec2_metadata.private_ipv4  # this is PRIVATE IP
         self._tags = {
             k: v
@@ -427,7 +450,7 @@ class AWSProvisioner(AbstractProvisioner):
         :return: None
         """
 
-        if "network" in kwargs:
+        if kwargs.get("network") is not None:
             logger.warning(
                 "AWS provisioner does not support a network parameter. Ignoring %s!",
                 kwargs["network"],
@@ -495,6 +518,7 @@ class AWSProvisioner(AbstractProvisioner):
         # Make tags for the leader specifically
         leader_tags = dict(self._tags)
         leader_tags[_TAG_KEY_TOIL_NODE_TYPE] = "leader"
+        leader_tags[_TAG_KEY_TOIL_SSH_KEY] = self._keyName
         logger.debug("Launching leader with tags: %s", leader_tags)
 
         instances: list[Instance] = create_instances(
@@ -995,28 +1019,29 @@ class AWSProvisioner(AbstractProvisioner):
         userData: str = self._getIgnitionUserData(
             "worker", keyPath, preemptible, self._architecture
         )
-        userDataBytes: bytes = b""
-        if isinstance(userData, str):
-            # Spot-market provisioning requires bytes for user data.
-            userDataBytes = userData.encode("utf-8")
-
-        spot_kwargs = {
-            "KeyName": self._keyName,
+        # Boto 3 demands we base64 the user data ourselves *only* for spot
+        # instances, and still wants a str.
+        spot_user_data = base64.b64encode(
+            userData.encode("utf-8")
+        ).decode("utf-8")
+        spot_kwargs: dict[Literal["LaunchSpecification"], dict[str, Any]] = {
             "LaunchSpecification": {
+                "KeyName": self._keyName,
                 "SecurityGroupIds": self._getSecurityGroupIDs(),
                 "InstanceType": type_info.name,
-                "UserData": userDataBytes,
+                "UserData": spot_user_data,
                 "BlockDeviceMappings": bdm,
                 "IamInstanceProfile": {"Arn": self._leaderProfileArn},
                 "Placement": {"AvailabilityZone": zone},
                 "SubnetId": subnet_id,
             },
         }
+
         on_demand_kwargs = {
             "KeyName": self._keyName,
             "SecurityGroupIds": self._getSecurityGroupIDs(),
             "InstanceType": type_info.name,
-            "UserData": userDataBytes,
+            "UserData": userData,
             "BlockDeviceMappings": bdm,
             "IamInstanceProfile": {"Arn": self._leaderProfileArn},
             "Placement": {"AvailabilityZone": zone},
@@ -1032,6 +1057,10 @@ class AWSProvisioner(AbstractProvisioner):
                 # every request in this method
                 if not preemptible:
                     logger.debug("Launching %s non-preemptible nodes", numNodes)
+                    # TODO: Use create_instances() instead, which requires
+                    # refactoring both ondemand and spot sides here to use
+                    # mypy_boto3_ec2.service_resource.Instance objects instead
+                    # of mypy_boto3_ec2.type_defs.InstanceTypeDef
                     instancesLaunched = create_ondemand_instances(
                         boto3_ec2=boto3_ec2,
                         image_id=self._discoverAMI(),
@@ -1059,7 +1088,6 @@ class AWSProvisioner(AbstractProvisioner):
                         reservation
                         for subdict in generatedInstancesLaunched
                         for reservation in subdict["Reservations"]
-                        for key, value in subdict.items()
                     ]
                     # get a flattened list of all requested instances, as before instancesLaunched is a dict of reservations which is a dict of instance requests
                     instancesLaunched = [
@@ -1144,7 +1172,7 @@ class AWSProvisioner(AbstractProvisioner):
             workerInstances = [
                 i
                 for i in workerInstances
-                if preemptible == (i["SpotInstanceRequestId"] is not None)
+                if preemptible == (i.get("SpotInstanceRequestId") is not None)
             ]
             logger.debug(
                 "%spreemptible workers found in cluster: %s",
@@ -1161,7 +1189,7 @@ class AWSProvisioner(AbstractProvisioner):
                 name=i["InstanceId"],
                 launchTime=i["LaunchTime"],
                 nodeType=i["InstanceType"],
-                preemptible=i["SpotInstanceRequestId"] is not None,
+                preemptible=i.get("SpotInstanceRequestId") is not None,
                 tags=collapse_tags(i["Tags"]),
             )
             for i in workerInstances
@@ -1509,11 +1537,15 @@ class AWSProvisioner(AbstractProvisioner):
         tags: list[TagDescriptionTypeDef] = ec2.describe_tags(Filters=[tag_filter])[
             "Tags"
         ]
+        # TODO: Does this reference instance or spot request? Or can it be either?
         idsToCancel = [tag["ResourceId"] for tag in tags]
         return [
             request["SpotInstanceRequestId"]
             for request in requests
-            if request["InstanceId"] in idsToCancel
+            if (
+                request.get("InstanceId") in idsToCancel
+                or request["SpotInstanceRequestId"] in idsToCancel
+            )
         ]
 
     def _createSecurityGroups(self) -> list[str]:

toil/provisioners/node.py

@@ -41,6 +41,21 @@ class Node:
         tags: Optional[dict[str, str]] = None,
         use_private_ip: Optional[bool] = None,
     ) -> None:
+        """
+        Create a new node.
+
+        :param launchTime: Time when the node was launched. If a naive
+            datetime, or a string without timezone information, is assumed to
+            be in UTC.
+
+        :raises ValueError: If launchTime is an improperly formatted string.
+
+        >>> node = Node("127.0.0.1", "127.0.0.1", "localhost",
+        ...             "Decembruary Eleventeenth", None, False)
+        Traceback (most recent call last):
+        ...
+        ValueError: Invalid isoformat string: 'Decembruary Eleventeenth'
+        """
         self.publicIP = publicIP
         self.privateIP = privateIP
         if use_private_ip:
@@ -48,13 +63,25 @@ class Node:
         else:
             self.effectiveIP = self.publicIP or self.privateIP
         self.name = name
+        # Typing should prevent an empty launch time, but just to make sure,
+        # check it at runtime.
+        assert launchTime is not None, (
+            f"Attempted to create a Node {name} without a launch time"
+        )
         if isinstance(launchTime, datetime.datetime):
             self.launchTime = launchTime
         else:
             try:
+                # Parse an RFC 3339 ISO 8601 UTC datetime
                 self.launchTime = parse_iso_utc(launchTime)
             except ValueError:
+                # Parse (almost) any ISO 8601 datetime
                 self.launchTime = datetime.datetime.fromisoformat(launchTime)
+        if self.launchTime.tzinfo is None:
+            # Read naive datatimes as in UTC
+            self.launchTime = self.launchTime.replace(
+                tzinfo=datetime.timezone.utc
+            )
         self.nodeType = nodeType
         self.preemptible = preemptible
         self.tags = tags
@@ -70,22 +97,43 @@ class Node:
 
     def remainingBillingInterval(self) -> float:
         """
-        If the node has a launch time, this function returns a floating point value
-        between 0 and 1.0 representing how far we are into the
-        current billing cycle for the given instance. If the return value is .25, we are one
-        quarter into the billing cycle, with three quarters remaining before we will be charged
-        again for that instance.
+        Returns a floating point value between 0 and 1.0 representing how much
+        time is left in the current billing cycle for the given instance. If
+        the return value is .25, we are three quarters into the billing cycle,
+        with one quarters remaining before we will be charged again for that
+        instance.
 
         Assumes a billing cycle of one hour.
 
-        :return: Float from 0 -> 1.0 representing percentage of pre-paid time left in cycle.
+        :return: Float from 1.0 -> 0.0 representing fraction of pre-paid time
+            remaining in cycle.
+
+        >>> node = Node("127.0.0.1", "127.0.0.1", "localhost",
+        ...             datetime.datetime.utcnow(), None, False)
+        >>> node.remainingBillingInterval() >= 0
+        True
+        >>> node.remainingBillingInterval() <= 1.0
+        True
+        >>> node.remainingBillingInterval() > 0.5
+        True
+        >>> interval1 = node.remainingBillingInterval()
+        >>> time.sleep(1)
+        >>> interval2 = node.remainingBillingInterval()
+        >>> interval2 < interval1
+        True
+
+        >>> node = Node("127.0.0.1", "127.0.0.1", "localhost",
+        ...             datetime.datetime.now(datetime.timezone.utc) - 
+        ...             datetime.timedelta(minutes=5), None, False)
+        >>> node.remainingBillingInterval() < 0.99
+        True
+        >>> node.remainingBillingInterval() > 0.9
+        True
+        
         """
-        if self.launchTime:
-            now = datetime.datetime.utcnow()
-            delta = now - self.launchTime
-            return 1 - delta.total_seconds() / 3600.0 % 1.0
-        else:
-            return 1
+        now = datetime.datetime.now(datetime.timezone.utc)
+        delta = now - self.launchTime
+        return 1 - delta.total_seconds() / 3600.0 % 1.0
 
     def waitForNode(self, role: str, keyName: str = "core") -> None:
         self._waitForSSHPort()

toil/resource.py

@@ -32,7 +32,6 @@ from zipfile import ZipFile
 
 from toil import inVirtualEnv
 from toil.lib.io import mkdtemp
-from toil.lib.iterables import concat
 from toil.lib.memoize import strict_bool
 from toil.lib.retry import ErrorCondition, retry
 from toil.version import exactPython
@@ -619,18 +618,9 @@ class ModuleDescriptor(
             initName = self._initModuleName(self.dirPath)
             if initName:
                 raise ResourceException(
-                    "Toil does not support loading a user script from a package directory. You "
-                    "may want to remove %s from %s or invoke the user script as a module via "
-                    "'PYTHONPATH=\"%s\" %s -m %s.%s'."
-                    % tuple(
-                        concat(
-                            initName,
-                            self.dirPath,
-                            exactPython,
-                            os.path.split(self.dirPath),
-                            self.name,
-                        )
-                    )
+                    f"Toil does not support loading a user script from a package directory. You "
+                    f"may want to remove {initName} from {self.dirPath} or invoke the user script as a module via: "
+                    f"PYTHONPATH='{self.dirPath}' {exactPython} -m {self.dirPath}.{self.name}"
                 )
             return self.dirPath
 

toil/server/app.py

@@ -16,6 +16,7 @@ import logging
 import os
 
 import connexion  # type: ignore
+from connexion.options import SwaggerUIOptions  # type: ignore[import-untyped]
 from configargparse import ArgumentParser
 
 from toil.lib.aws import get_current_aws_region, running_on_ec2, running_on_ecs
@@ -133,8 +134,10 @@ def create_app(args: argparse.Namespace) -> "connexion.FlaskApp":
     """
     Create a "connexion.FlaskApp" instance with Toil server configurations.
     """
+    swagger_ui_options = SwaggerUIOptions(swagger_ui=args.swagger_ui)
+
     flask_app = connexion.FlaskApp(
-        __name__, specification_dir="api_spec/", options={"swagger_ui": args.swagger_ui}
+        __name__, specification_dir="api_spec/", swagger_ui_options=swagger_ui_options
     )
 
     flask_app.app.config["JSON_SORT_KEYS"] = False
@@ -164,16 +167,16 @@ def create_app(args: argparse.Namespace) -> "connexion.FlaskApp":
     if isinstance(backend, ToilBackend):
         # We extend the WES API to allow presenting log data
         base_url = "/toil/wes/v1"
-        flask_app.app.add_url_rule(
+        flask_app.add_url_rule(
             f"{base_url}/logs/<run_id>/stdout", view_func=backend.get_stdout
         )
-        flask_app.app.add_url_rule(
+        flask_app.add_url_rule(
             f"{base_url}/logs/<run_id>/stderr", view_func=backend.get_stderr
         )
         # To be a well-behaved AGC engine we can implement the default status check endpoint
-        flask_app.app.add_url_rule("/engine/v1/status", view_func=backend.get_health)
+        flask_app.add_url_rule("/engine/v1/status", view_func=backend.get_health)
         # And we can provide lost humans some information on what they are looking at
-        flask_app.app.add_url_rule("/", view_func=backend.get_homepage)
+        flask_app.add_url_rule("/", view_func=backend.get_homepage)
 
     return flask_app
 
@@ -201,9 +204,12 @@ def start_server(args: argparse.Namespace) -> None:
     else:
         # start a production WSGI server
         run_app(
-            flask_app.app,
+            flask_app,
             options={
                 "bind": f"{host}:{port}",
                 "workers": args.workers,
+                # The uvicorn worker class must be specified for gunicorn to work on connexion 3
+                # https://github.com/spec-first/connexion/issues/1755#issuecomment-1778522142
+                "worker_class": "uvicorn.workers.UvicornWorker"
             },
         )

toil/server/cli/wes_cwl_runner.py

@@ -14,7 +14,7 @@ import ruamel.yaml
 import schema_salad
 from configargparse import ArgumentParser
 from wes_client.util import WESClient  # type: ignore
-from wes_client.util import wes_reponse as wes_response
+from wes_client.util import wes_response as wes_response
 
 from toil.lib.web import web_session
 from toil.wdl.utils import get_version as get_wdl_version
@@ -144,7 +144,7 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
             return "3.8"
         elif extension == "cwl":
             with open(workflow_file) as f:
-                yaml = ruamel.yaml.YAML(typ='safe', pure=True)
+                yaml = ruamel.yaml.YAML(typ="safe", pure=True)
                 return str(yaml.load(f)["cwlVersion"])
         elif extension == "wdl":
             with open(workflow_file) as f:

toil/server/wes/abstract_backend.py

@@ -224,7 +224,7 @@ class WESBackend:
         )
 
     def collect_attachments(
-        self, run_id: Optional[str], temp_dir: Optional[str]
+        self, args: dict[str, Any], run_id: Optional[str], temp_dir: Optional[str]
     ) -> tuple[str, dict[str, Any]]:
         """
         Collect attachments from the current request by staging uploaded files
@@ -238,48 +238,26 @@ class WESBackend:
             temp_dir = mkdtemp()
         body: dict[str, Any] = {}
         has_attachments = False
-        for key, ls in connexion.request.files.lists():
-            try:
-                for value in ls:
-                    # uploaded files that are required to execute the workflow
-                    if key == "workflow_attachment":
-                        # guard against maliciously constructed filenames
-                        dest = os.path.join(temp_dir, self.secure_path(value.filename))
-                        if not os.path.isdir(os.path.dirname(dest)):
-                            os.makedirs(os.path.dirname(dest))
-                        self.log_for_run(
-                            run_id, f"Staging attachment '{value.filename}' to '{dest}'"
-                        )
-                        value.save(dest)
-                        has_attachments = True
-                        body[key] = (
-                            f"file://{temp_dir}"  # Reference to temp working dir.
-                        )
-
-                    elif key in (
-                        "workflow_params",
-                        "tags",
-                        "workflow_engine_parameters",
-                    ):
-                        content = value.read()
-                        body[key] = json.loads(content.decode("utf-8"))
-                    else:
-                        body[key] = value.read().decode()
-            except Exception as e:
-                raise MalformedRequestException(f"Error reading parameter '{key}': {e}")
-
-        for key, ls in connexion.request.form.lists():
-            try:
-                for value in ls:
-                    if not value:
-                        continue
-                    if key in ("workflow_params", "tags", "workflow_engine_parameters"):
-                        body[key] = json.loads(value)
-                    else:
-                        body[key] = value
-            except Exception as e:
-                raise MalformedRequestException(f"Error reading parameter '{key}': {e}")
-
+        for k, v in args.items():
+            if k == "workflow_attachment":
+                for file in (v or []):
+                    dest = os.path.join(temp_dir, self.secure_path(file.filename))
+                    if not os.path.isdir(os.path.dirname(dest)):
+                        os.makedirs(os.path.dirname(dest))
+                    self.log_for_run(
+                        run_id,
+                        f"Staging attachment '{file.filename}' to '{dest}'",
+                    )
+                    file.save(dest)
+                    has_attachments = True
+                    body["workflow_attachment"] = (
+                            "file://%s" % temp_dir
+                    )  # Reference to temp working dir.
+            elif k in ("workflow_params", "tags", "workflow_engine_parameters"):
+                if v is not None:
+                    body[k] = json.loads(v)
+            else:
+                body[k] = v
         if "workflow_url" in body:
             url, ref = urldefrag(body["workflow_url"])
             if ":" not in url:

toil/server/wes/toil_backend.py

@@ -502,7 +502,7 @@ class ToilBackend(WESBackend):
         }
 
     @handle_errors
-    def run_workflow(self) -> dict[str, str]:
+    def run_workflow(self, **args: Any) -> dict[str, str]:
         """Run a workflow."""
         run_id = self.run_id_prefix + uuid.uuid4().hex
         run = self._get_run(run_id, should_exists=False)
@@ -514,7 +514,7 @@ class ToilBackend(WESBackend):
         # stage the uploaded files to the execution directory, so that we can run the workflow file directly
         temp_dir = run.exec_dir
         try:
-            _, request = self.collect_attachments(run_id, temp_dir=temp_dir)
+            _, request = self.collect_attachments(args, run_id, temp_dir=temp_dir)
         except ValueError:
             run.clean_up()
             raise

toil/test/__init__.py

@@ -49,7 +49,6 @@ from toil.lib.accelerators import (
     have_working_nvidia_smi,
 )
 from toil.lib.io import mkdtemp
-from toil.lib.iterables import concat
 from toil.lib.memoize import memoize
 from toil.lib.threading import ExceptionalThread, cpu_count
 from toil.version import distVersion
@@ -224,7 +223,7 @@ class ToilTest(unittest.TestCase):
 
         :return: The output of the process' stdout if capture=True was passed, None otherwise.
         """
-        argl = list(concat(command, args))
+        argl = [command] + list(args)
         logger.info("Running %r", argl)
         capture = kwargs.pop("capture", False)
         _input = kwargs.pop("input", None)
@@ -395,6 +394,8 @@ def _aws_s3_avail() -> bool:
         boto3_credentials = session.get_credentials()
     except ImportError:
         return False
+    except ProxyConnectionError:
+        return False
     from toil.lib.aws import running_on_ec2
 
     if not (
@@ -440,7 +441,7 @@ def needs_aws_batch(test_item: MT) -> MT:
         test_item
     )
     test_item = needs_env_var(
-        "TOIL_AWS_BATCH_JOB_ROLE_ARN", "an IAM role ARN that grants S3 and SDB access"
+        "TOIL_AWS_BATCH_JOB_ROLE_ARN", "an IAM role ARN that grants S3 access"
     )(test_item)
     try:
         from toil.lib.aws import get_current_aws_region
@@ -467,7 +468,7 @@ def needs_google_storage(test_item: MT) -> MT:
     """
     test_item = _mark_test("google_storage", needs_online(test_item))
     try:
-        from google.cloud import storage  # type: ignore[import-untyped]
+        import google.clould.storage  # type: ignore[import-untyped]
     except ImportError:
         return unittest.skip(
             "Install Toil with the 'google' extra to include this test."
@@ -619,7 +620,7 @@ def needs_htcondor(test_item: MT) -> MT:
     """Use a decorator before test classes or methods to run only if the HTCondor is installed."""
     test_item = _mark_test("htcondor", test_item)
     try:
-        import htcondor  # type: ignore[import-not-found]
+        import htcondor  # type: ignore
 
         htcondor.Collector(os.getenv("TOIL_HTCONDOR_COLLECTOR")).query(
             constraint="False"
@@ -1249,19 +1250,16 @@ class ApplianceTestSupport(ToilTest):
             with self.lock:
                 image = applianceSelf()
                 # Omitting --rm, it's unreliable, see https://github.com/docker/docker/issues/16575
-                args = list(
-                    concat(
-                        "docker",
-                        "run",
-                        "--entrypoint=" + self._entryPoint(),
-                        "--net=host",
-                        "-i",
-                        "--name=" + self.containerName,
-                        ["--volume=%s:%s" % mount for mount in self.mounts.items()],
-                        image,
-                        self._containerCommand(),
-                    )
-                )
+                args = [
+                    "docker",
+                    "run",
+                    f"--entrypoint={self._entryPoint()}",
+                    "--net=host",
+                    "-i",
+                    f"--name={self.containerName}"] + \
+                    ["--volume=%s:%s" % mount for mount in self.mounts.items()] + \
+                    [image] + \
+                    self._containerCommand()
                 logger.info("Running %r", args)
                 self.popen = subprocess.Popen(args)
             self.start()

toil/test/batchSystems/batchSystemTest.py

@@ -42,8 +42,6 @@ from toil.batchSystems.registry import (
     add_batch_system_factory,
     get_batch_system,
     get_batch_systems,
-    restore_batch_system_plugin_state,
-    save_batch_system_plugin_state,
 )
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
 from toil.common import Config, Toil
@@ -69,6 +67,7 @@ from toil.test import (
     pslow,
     pneeds_mesos,
 )
+from toil.lib.plugins import remove_plugin
 
 import pytest
 
@@ -97,15 +96,9 @@ class BatchSystemPluginTest(ToilTest):
     Class for testing batch system plugin functionality.
     """
 
-    def setUp(self) -> None:
-        # Save plugin state so our plugin doesn't stick around after the test
-        # (and create duplicate options)
-        self.__state = save_batch_system_plugin_state()
-        super().setUp()
-
     def tearDown(self) -> None:
         # Restore plugin state
-        restore_batch_system_plugin_state(self.__state)
+        remove_plugin("batch_system", "testBatchSystem")
         super().tearDown()
 
     def test_add_batch_system_factory(self) -> None:

toil/test/batchSystems/batch_system_plugin_test.py

@@ -26,6 +26,7 @@ from toil.batchSystems.registry import add_batch_system_factory
 from toil.common import Toil, addOptions
 from toil.job import JobDescription
 from toil.test import ToilTest
+from toil.lib.plugins import remove_plugin
 
 logger = logging.getLogger(__name__)
 
@@ -68,6 +69,11 @@ class FakeBatchSystem(BatchSystemCleanupSupport):
 
 
 class BatchSystemPluginTest(ToilTest):
+    def tearDown(self) -> None:
+        # Restore plugin state
+        remove_plugin("batch_system", "fake")
+        super().tearDown()
+
     def test_batchsystem_plugin_installable(self):
         """
         Test that installing a batch system plugin works.
@@ -76,6 +82,7 @@ class BatchSystemPluginTest(ToilTest):
 
         def fake_batch_system_factory() -> type[AbstractBatchSystem]:
             return FakeBatchSystem
+        
 
         add_batch_system_factory("fake", fake_batch_system_factory)