toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

toil/statsAndLogging.py

@@ -18,9 +18,10 @@ import logging
 import os
 import time
 from argparse import ArgumentParser, Namespace
+from collections.abc import Callable
 from logging.handlers import RotatingFileHandler
 from threading import Event, Thread
-from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union
+from typing import IO, TYPE_CHECKING, Any, Union
 
 from toil.lib.conversions import strtobool
 from toil.lib.expando import Expando
@@ -60,9 +61,7 @@ class StatsAndLogging:
         self._worker.start()
 
     @classmethod
-    def formatLogStream(
-        cls, stream: Union[IO[str], IO[bytes]], stream_name: str
-    ) -> str:
+    def formatLogStream(cls, stream: IO[str] | IO[bytes], stream_name: str) -> str:
         """
         Given a stream of text or bytes, and the job name, job itself, or some
         other optional stringifyable identity info for the job, return a big
@@ -90,9 +89,9 @@ class StatsAndLogging:
     def logWithFormatting(
         cls,
         stream_name: str,
-        jobLogs: Union[IO[str], IO[bytes]],
+        jobLogs: IO[str] | IO[bytes],
         method: Callable[[str], None] = logger.debug,
-        message: Optional[str] = None,
+        message: str | None = None,
     ) -> None:
         if message is not None:
             method(message)
@@ -174,7 +173,7 @@ class StatsAndLogging:
         startTime = time.time()
         startClock = ResourceMonitor.get_total_cpu_time()
 
-        def callback(fileHandle: Union[IO[bytes], IO[str]]) -> None:
+        def callback(fileHandle: IO[bytes] | IO[str]) -> None:
             statsStr = fileHandle.read()
             if not isinstance(statsStr, str):
                 statsStr = statsStr.decode()
@@ -259,7 +258,7 @@ class StatsAndLogging:
                             cores=float(job.requested_cores),
                             cpu_seconds=float(job.clock),
                             memory_bytes=int(job.memory) * 1024,
-                            disk_bytes=int(job.disk)
+                            disk_bytes=int(job.disk),
                         )
                     except:
                         logger.exception("Could not record job attempt in history!")
@@ -304,7 +303,7 @@ class StatsAndLogging:
         # in addition to cleaning on exceptions, onError should clean if there are any failed jobs
 
 
-def set_log_level(level: str, set_logger: Optional[logging.Logger] = None) -> None:
+def set_log_level(level: str, set_logger: logging.Logger | None = None) -> None:
     """Sets the root logger level to a given string level (like "INFO")."""
     level = "CRITICAL" if level.upper() == "OFF" else level.upper()
     set_logger = set_logger if set_logger else root_logger
@@ -314,7 +313,7 @@ def set_log_level(level: str, set_logger: Optional[logging.Logger] = None) -> No
     suppress_exotic_logging(__name__)
 
 
-def install_log_color(set_logger: Optional[logging.Logger] = None) -> None:
+def install_log_color(set_logger: logging.Logger | None = None) -> None:
     """Make logs colored."""
     # Most of this code is taken from miniwdl
     # delayed import
@@ -349,7 +348,7 @@ def install_log_color(set_logger: Optional[logging.Logger] = None) -> None:
 
 
 def add_logging_options(
-    parser: ArgumentParser, default_level: Optional[int] = None
+    parser: ArgumentParser, default_level: int | None = None
 ) -> None:
     """
     Add logging options to set the global log level.
@@ -424,11 +423,11 @@ def configure_root_logger() -> None:
     root_logger.setLevel(DEFAULT_LOGLEVEL)
 
 
-def log_to_file(log_file: Optional[str], log_rotation: bool) -> None:
+def log_to_file(log_file: str | None, log_rotation: bool) -> None:
     if log_file and log_file not in __loggingFiles:
         logger.debug(f"Logging to file '{log_file}'.")
         __loggingFiles.append(log_file)
-        handler: Union[RotatingFileHandler, logging.FileHandler]
+        handler: RotatingFileHandler | logging.FileHandler
         if log_rotation:
             handler = RotatingFileHandler(log_file, maxBytes=1000000, backupCount=1)
         else:

toil/test/__init__.py

@@ -35,7 +35,8 @@ from pathlib import Path
 from shutil import which
 from tempfile import mkstemp
 from textwrap import dedent
-from typing import Any, Callable, Literal, Optional, TypeVar, Union, cast
+from typing import Any, Literal, Optional, TypeVar, Union, cast
+from collections.abc import Callable
 from unittest.util import strclass
 from urllib.error import HTTPError, URLError
 from urllib.request import urlopen
@@ -60,6 +61,7 @@ except ImportError:
     class ProxyConnectionError(BaseException):  # type: ignore[no-redef]
         ...
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -100,7 +102,7 @@ class ToilTest(unittest.TestCase):
     """
 
     _rootpath: Path
-    _tempBaseDir: Optional[str] = None
+    _tempBaseDir: str | None = None
     _tempDirs: list[str] = []
 
     def setup_method(self, method: Any) -> None:
@@ -179,11 +181,11 @@ class ToilTest(unittest.TestCase):
         assert region
         return region.group(1)
 
-    def _createTempDir(self, purpose: Optional[str] = None) -> str:
+    def _createTempDir(self, purpose: str | None = None) -> str:
         return self._createTempDirEx(self._testMethodName, purpose)
 
     @classmethod
-    def _createTempDirEx(cls, *names: Optional[str]) -> str:
+    def _createTempDirEx(cls, *names: str | None) -> str:
         classname = strclass(cls)
         if classname.startswith("toil.test."):
             classname = classname[len("toil.test.") :]
@@ -207,7 +209,7 @@ class ToilTest(unittest.TestCase):
         return path
 
     @classmethod
-    def _run(cls, command: str, *args: str, **kwargs: Any) -> Optional[str]:
+    def _run(cls, command: str, *args: str, **kwargs: Any) -> str | None:
         """
         Run a command.
 
@@ -265,7 +267,7 @@ else:
         return cast(MT, getattr(pytest_mark, name)(test_item))
 
 
-def get_temp_file(suffix: str = "", rootDir: Optional[str] = None) -> str:
+def get_temp_file(suffix: str = "", rootDir: str | None = None) -> str:
     """Return a string representing a temporary file, that must be manually deleted."""
     if rootDir is None:
         handle, tmp_file = mkstemp(suffix)
@@ -282,7 +284,7 @@ def get_temp_file(suffix: str = "", rootDir: Optional[str] = None) -> str:
         return tmp_file
 
 
-def needs_env_var(var_name: str, comment: Optional[str] = None) -> Callable[[MT], MT]:
+def needs_env_var(var_name: str, comment: str | None = None) -> Callable[[MT], MT]:
     """
     Use as a decorator before test classes or methods to run only if the given
     environment variable is set.
@@ -369,9 +371,7 @@ def needs_aws_s3(test_item: MT) -> MT:
             test_item
         )
     except ProxyConnectionError as e:
-        return unittest.skip(f"Proxy error: {e}, skipping this test.")(
-            test_item
-        )
+        return unittest.skip(f"Proxy error: {e}, skipping this test.")(test_item)
     from toil.lib.aws import running_on_ec2
 
     if not (
@@ -569,6 +569,7 @@ def needs_mesos(test_item: MT) -> MT:
         )(test_item)
     try:
         import psutil  # noqa
+
         # If pymesos is installed, because it isn't typed, mypy sees an
         # import-untyped error here.
         #
@@ -592,6 +593,9 @@ def _mesos_avail() -> bool:
     try:
         import psutil
         import pymesos
+
+        str(psutil)  # to prevent removal of this import
+        str(pymesos)  # to prevent removal of this import
     except ImportError:
         return False
     return True
@@ -793,6 +797,8 @@ def needs_cwl(test_item: MT) -> MT:
 def _cwl_available() -> bool:
     try:
         import cwltool
+
+        str(cwltool)  # to prevent removal of this import
     except ImportError:
         return False
     return True
@@ -1038,7 +1044,7 @@ def timeLimit(seconds: int) -> Generator[None, None, None]:
 
 def make_tests(
     generalMethod: Callable[[Any], Any],
-    targetClass: Optional[Callable[[Any], Any]],
+    targetClass: Callable[[Any], Any] | None,
     **kwargs: Any,
 ) -> None:
     """
@@ -1185,7 +1191,7 @@ class ApplianceTestSupport(ToilTest):
 
     @contextmanager
     def _applianceCluster(
-        self, mounts: dict[str, str], numCores: Optional[int] = None
+        self, mounts: dict[str, str], numCores: int | None = None
     ) -> Generator[
         tuple["ApplianceTestSupport.LeaderThread", "ApplianceTestSupport.WorkerThread"],
         None,
@@ -1244,22 +1250,25 @@ class ApplianceTestSupport(ToilTest):
             self.mounts = mounts
             self.cleanMounts = cleanMounts
             self.containerName = str(uuid.uuid4())
-            self.popen: Optional[subprocess.Popen[bytes]] = None
+            self.popen: subprocess.Popen[bytes] | None = None
 
         def __enter__(self) -> Self:
             with self.lock:
                 image = applianceSelf()
                 # Omitting --rm, it's unreliable, see https://github.com/docker/docker/issues/16575
-                args = [
-                    "docker",
-                    "run",
-                    f"--entrypoint={self._entryPoint()}",
-                    "--net=host",
-                    "-i",
-                    f"--name={self.containerName}"] + \
-                    ["--volume=%s:%s" % mount for mount in self.mounts.items()] + \
-                    [image] + \
-                    self._containerCommand()
+                args = (
+                    [
+                        "docker",
+                        "run",
+                        f"--entrypoint={self._entryPoint()}",
+                        "--net=host",
+                        "-i",
+                        f"--name={self.containerName}",
+                    ]
+                    + ["--volume=%s:%s" % mount for mount in self.mounts.items()]
+                    + [image]
+                    + self._containerCommand()
+                )
                 logger.info("Running %r", args)
                 self.popen = subprocess.Popen(args)
             self.start()
@@ -1347,7 +1356,7 @@ class ApplianceTestSupport(ToilTest):
             self.runOnAppliance("tee", path, input=contents)
 
         def deployScript(
-            self, path: str, packagePath: str, script: Union[str, Callable[..., Any]]
+            self, path: str, packagePath: str, script: str | Callable[..., Any]
         ) -> None:
             """
             Deploy a Python module on the appliance.

toil/test/batchSystems/batchSystemTest.py

@@ -11,22 +11,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from collections.abc import Iterable, Generator
 import argparse
 import fcntl
 import itertools
 import logging
 import os
-from pathlib import Path
 import subprocess
 import sys
 import tempfile
 import textwrap
 import time
 from abc import ABCMeta, abstractmethod
+from collections.abc import Generator
 from fractions import Fraction
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
 from unittest import skipIf
-from typing import Optional, Any, TYPE_CHECKING
+
+import pytest
 
 from toil.batchSystems.abstractBatchSystem import (
     AbstractBatchSystem,
@@ -45,9 +47,9 @@ from toil.batchSystems.registry import (
 )
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
 from toil.common import Config, Toil
-from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import Job, JobDescription, Requirer, ServiceHostJob
 from toil.lib.misc import StrPath
+from toil.lib.plugins import remove_plugin
 from toil.lib.retry import retry_flaky_test
 from toil.lib.threading import cpu_count
 from toil.test import (
@@ -63,13 +65,10 @@ from toil.test import (
     needs_mesos,
     needs_slurm,
     needs_torque,
-    slow,
-    pslow,
     pneeds_mesos,
+    pslow,
+    slow,
 )
-from toil.lib.plugins import remove_plugin
-
-import pytest
 
 if TYPE_CHECKING:
     from toil.batchSystems.mesos.batchSystem import MesosBatchSystem
@@ -119,6 +118,7 @@ class hidden:
 
     http://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class#answer-25695512
     """
+
     class AbstractBatchSystemTest(ToilTest, metaclass=ABCMeta):
         """
         A base test case with generic tests that every batch system should pass.
@@ -159,7 +159,7 @@ class hidden:
             return self.createConfig()
 
         def _mockJobDescription(
-            self, jobStoreID: Optional[str] = None, **kwargs: Any
+            self, jobStoreID: str | None = None, **kwargs: Any
         ) -> JobDescription:
             """
             Create a mock-up JobDescription with the given ID and other parameters.
@@ -818,7 +818,8 @@ class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
             import signal
             import sys
             import time
-            from typing import Any, Iterable
+            from collections.abc import Iterable
+            from typing import Any
 
             def handle_signal(sig: Any, frame: Any) -> None:
                 sys.stderr.write(f"{os.getpid()} ignoring signal {sig}\n")

toil/test/batchSystems/batch_system_plugin_test.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Optional
 
 from configargparse import ArgParser, ArgumentParser
 
@@ -25,8 +24,8 @@ from toil.batchSystems.options import OptionSetter
 from toil.batchSystems.registry import add_batch_system_factory
 from toil.common import Toil, addOptions
 from toil.job import JobDescription
-from toil.test import ToilTest
 from toil.lib.plugins import remove_plugin
+from toil.test import ToilTest
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +39,7 @@ class FakeBatchSystem(BatchSystemCleanupSupport):
         self,
         command: str,
         job_desc: JobDescription,
-        job_environment: Optional[dict[str, str]] = None,
+        job_environment: dict[str, str] | None = None,
     ) -> int:
         pass
 
@@ -53,7 +52,7 @@ class FakeBatchSystem(BatchSystemCleanupSupport):
     def getRunningBatchJobIDs(self) -> dict[int, float]:
         pass
 
-    def getUpdatedBatchJob(self, maxWait: int) -> Optional[UpdatedBatchJobInfo]:
+    def getUpdatedBatchJob(self, maxWait: int) -> UpdatedBatchJobInfo | None:
         pass
 
     def shutdown(self) -> None:
@@ -82,7 +81,6 @@ class BatchSystemPluginTest(ToilTest):
 
         def fake_batch_system_factory() -> type[AbstractBatchSystem]:
             return FakeBatchSystem
-        
 
         add_batch_system_factory("fake", fake_batch_system_factory)
 

toil/test/batchSystems/test_slurm.py

@@ -1,12 +1,11 @@
 import errno
+import logging
+import sys
 import textwrap
+from datetime import datetime, timedelta
 from queue import Queue
 
-import logging
 import pytest
-import sys
-
-from datetime import datetime, timedelta
 
 import toil.batchSystems.slurm
 from toil.batchSystems.abstractBatchSystem import (
@@ -29,6 +28,7 @@ logger = logging.getLogger(__name__)
 # we hope is not days and days away from the time the tests actually run.
 JOB_BASE_TIME = datetime.now().astimezone(None) - timedelta(days=5)
 
+
 def call_either(args, **_) -> str:
     """
     Pretend to call either sacct or scontrol as appropriate.
@@ -40,6 +40,7 @@ def call_either(args, **_) -> str:
     else:
         raise RuntimeError(f"Cannot fake command call: {args}")
 
+
 def call_sacct(args, **_) -> str:
     """
     The arguments passed to `call_command` when executing `sacct` are something like:
@@ -84,26 +85,24 @@ def call_sacct(args, **_) -> str:
 
     # See if they asked for a job list
     try:
-        j_index = args.index('-j')
+        j_index = args.index("-j")
         job_ids = [int(job_id) for job_id in args[j_index + 1].split(",")]
     except ValueError:
         # We're not restricting to a list of jobs.
         job_ids = list(sacct_info.keys())
     # See if they asked for start or end times
     try:
-        flag_index = args.index('-S')
+        flag_index = args.index("-S")
         begin_time = datetime.fromisoformat(args[flag_index + 1]).astimezone(None)
     except ValueError:
         # By default, Slurm uses today at midnight
-        begin_time = datetime.now().astimezone(None).replace(
-            hour=0,
-            minute=0,
-            second=0,
-            microsecond=0,
-            fold=0
+        begin_time = (
+            datetime.now()
+            .astimezone(None)
+            .replace(hour=0, minute=0, second=0, microsecond=0, fold=0)
         )
     try:
-        flag_index = args.index('-E')
+        flag_index = args.index("-E")
         end_time = datetime.fromisoformat(args[flag_index + 1]).astimezone(None)
     except ValueError:
         end_time = None
@@ -251,6 +250,7 @@ def call_sacct_raises(*_):
         1, "sacct: error: Problem talking to the database: " "Connection timed out"
     )
 
+
 def call_sinfo(*_) -> str:
     """
     Simulate asking for partition info from Slurm
@@ -272,6 +272,7 @@ def call_sinfo(*_) -> str:
     )
     return stdout
 
+
 class FakeBatchSystem(BatchSystemSupport):
     """
     Class that implements a minimal Batch System, needed to create a Worker (see below).
@@ -299,13 +300,17 @@ class FakeBatchSystem(BatchSystemSupport):
 
         config.workflowID = str(uuid4())
         config.cleanWorkDir = "always"
-        toil.batchSystems.slurm.SlurmBatchSystem.setOptions(lambda o: setattr(config, o, None))
+        toil.batchSystems.slurm.SlurmBatchSystem.setOptions(
+            lambda o: setattr(config, o, None)
+        )
         return config
 
+
 # Make the mock class not have abstract methods anymore, even though we don't
 # implement them. See <https://stackoverflow.com/a/17345619>.
 FakeBatchSystem.__abstractmethods__ = set()
 
+
 class SlurmTest(ToilTest):
     """
     Class for unit-testing SlurmBatchSystem
@@ -373,7 +378,6 @@ class SlurmTest(ToilTest):
         result = self.worker._getJobDetailsFromSacct(list(expected_result))
         assert result == expected_result, f"{result} != {expected_result}"
 
-
     ####
     #### tests for _getJobDetailsFromScontrol()
     ####
@@ -524,7 +528,7 @@ class SlurmTest(ToilTest):
             (130, BatchJobExitReason.FAILED),
             (0, BatchJobExitReason.FINISHED),
             (0, BatchJobExitReason.FINISHED),
-            None
+            None,
         ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
@@ -625,13 +629,17 @@ class SlurmTest(ToilTest):
         assert "--partition=short" in command
 
         # With a partition override, we should not. But the override will be rewritten.
-        self.worker.boss.config.slurm_args = "--something --partition foo --somethingElse"
+        self.worker.boss.config.slurm_args = (
+            "--something --partition foo --somethingElse"
+        )
         command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
         assert "--partition=short" not in command
         assert "--partition=foo" in command
 
         # All ways of setting partition should work, including =
-        self.worker.boss.config.slurm_args = "--something --partition=foo --somethingElse"
+        self.worker.boss.config.slurm_args = (
+            "--something --partition=foo --somethingElse"
+        )
         command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
         assert "--partition=short" not in command
         assert "--partition=foo" in command
@@ -649,7 +657,9 @@ class SlurmTest(ToilTest):
         assert "--partition=foobar" in command
 
         # But they should be overridden by the argument overrides
-        self.worker.boss.config.slurm_args = "--something --partition=baz --somethingElse"
+        self.worker.boss.config.slurm_args = (
+            "--something --partition=baz --somethingElse"
+        )
         command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
         assert "--partition=baz" in command
 
@@ -669,14 +679,18 @@ class SlurmTest(ToilTest):
 
         # With a time override, we should use it, slightly translated, and it
         # should change the selected partition.
-        self.worker.boss.config.slurm_args = "--something --time 10:00:00 --somethingElse"
+        self.worker.boss.config.slurm_args = (
+            "--something --time 10:00:00 --somethingElse"
+        )
         command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
         logger.debug("Command: %s", command)
         assert "--partition=medium" in command
         assert "--time=0:36000" in command
 
         # All ways of setting time should work, including =
-        self.worker.boss.config.slurm_args = "--something --time=10:00:00 --somethingElse"
+        self.worker.boss.config.slurm_args = (
+            "--something --time=10:00:00 --somethingElse"
+        )
         command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
         logger.debug("Command: %s", command)
         assert "--partition=medium" in command
@@ -724,7 +738,9 @@ class SlurmTest(ToilTest):
         detector = toil.batchSystems.slurm.any_option_detector([])
         self.assertFalse(detector("--anything"))
 
-        detector = toil.batchSystems.slurm.any_option_detector([("foobar", "f"), "many-bothans", ("bazz-only", "B")])
+        detector = toil.batchSystems.slurm.any_option_detector(
+            [("foobar", "f"), "many-bothans", ("bazz-only", "B")]
+        )
 
         self.assertTrue(detector("--foobar"))
         self.assertTrue(detector("-f"))
@@ -733,5 +749,3 @@ class SlurmTest(ToilTest):
         self.assertTrue(detector("-B"))
         self.assertFalse(detector("--no-bazz"))
         self.assertFalse(detector("--foo-bar=--bazz-only"))
-
-

toil/test/cwl/conftest.py

@@ -17,9 +17,10 @@
 import json
 import logging
 from io import StringIO
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any
 
 from cwltest import utils
+
 logger = logging.getLogger(__name__)
 
 collect_ignore = ["spec"]
@@ -30,15 +31,13 @@ collect_ignore = ["spec"]
 # See cwltool's reference implementation:
 # https://github.com/common-workflow-language/cwltool/blob/05af6c1357c327b3146e9f5da40e7c0aa3e6d976/tests/cwl-conformance/cwltool-conftest.py
 def pytest_cwl_execute_test(
-        config: utils.CWLTestConfig,
-        processfile: str,
-        jobfile: Optional[str]
-) -> Tuple[int, Optional[Dict[str, Any]]]:
+    config: utils.CWLTestConfig, processfile: str, jobfile: str | None
+) -> tuple[int, dict[str, Any] | None]:
     """Use Toil to execute CWL tests (equivalent to running toil-cwl-runner)."""
     from toil.cwl.cwltoil import main
 
     stdout = StringIO()
-    argsl: List[str] = [f"--outdir={config.outdir}"]
+    argsl: list[str] = [f"--outdir={config.outdir}"]
     if config.runner_quiet:
         argsl.append("--quiet")
     elif config.verbose: