cluster-tools 2.0.7__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/PKG-INFO +2 -2
  2. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/__init__.py +1 -1
  3. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/call.py +3 -8
  4. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/file_wait_thread.py +6 -11
  5. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/multiprocessing_logging_handler.py +3 -2
  6. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/pickling.py +7 -7
  7. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/reflection.py +1 -1
  8. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/tailf.py +5 -4
  9. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/warning.py +5 -10
  10. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executor_protocol.py +8 -12
  11. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/dask.py +17 -22
  12. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/multiprocessing_.py +16 -21
  13. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/multiprocessing_pickle.py +2 -1
  14. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/sequential.py +5 -5
  15. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/sequential_pickle.py +2 -1
  16. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/remote.py +11 -17
  17. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/cluster_executor.py +36 -46
  18. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/kube.py +15 -17
  19. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/pbs.py +22 -28
  20. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/slurm.py +39 -53
  21. cluster_tools-2.2.0/cluster_tools/version.py +1 -0
  22. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/PKG-INFO +2 -2
  23. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/pyproject.toml +9 -8
  24. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_all.py +20 -32
  25. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_deref_main.py +1 -3
  26. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_kubernetes.py +1 -2
  27. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_multiprocessing.py +12 -12
  28. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_slurm.py +25 -25
  29. cluster_tools-2.0.7/cluster_tools/version.py +0 -1
  30. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/README.md +0 -0
  31. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/string_.py +0 -0
  32. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/__init__.py +0 -0
  33. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/SOURCES.txt +0 -0
  34. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/dependency_links.txt +0 -0
  35. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/requires.txt +0 -0
  36. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/top_level.txt +0 -0
  37. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/setup.cfg +0 -0
  38. {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_dask.py +0 -0
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 2.0.7
3
+ Version: 2.2.0
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
6
  License: MIT
7
7
  Project-URL: Repository, https://github.com/scalableminds/webknossos-libs
8
8
  Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/master/cluster_tools/Changelog.md
9
- Requires-Python: >=3.9
9
+ Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
11
  Requires-Dist: typing-extensions~=4.12.0
12
12
  Provides-Extra: kubernetes
@@ -117,4 +117,4 @@ def get_executor(environment: str, **kwargs: Any) -> "Executor":
117
117
  return SequentialPickleExecutor(**kwargs)
118
118
  elif environment == "multiprocessing_with_pickling":
119
119
  return MultiprocessingPickleExecutor(**kwargs)
120
- raise Exception("Unknown executor: {}".format(environment))
120
+ raise Exception(f"Unknown executor: {environment}")
@@ -1,8 +1,7 @@
1
1
  import subprocess
2
- from typing import Optional, Tuple
3
2
 
4
3
 
5
- def call(command: str, stdin: Optional[str] = None) -> Tuple[str, str, int]:
4
+ def call(command: str, stdin: str | None = None) -> tuple[str, str, int]:
6
5
  """Invokes a shell command as a subprocess, optionally with some
7
6
  data sent to the standard input. Returns the standard output data,
8
7
  the standard error, and the return code.
@@ -31,14 +30,10 @@ class CommandError(Exception):
31
30
  self.stderr = stderr
32
31
 
33
32
  def __str__(self) -> str:
34
- return "%s exited with status %i: %s" % (
35
- repr(self.command),
36
- self.code,
37
- repr(self.stderr),
38
- )
33
+ return f"{self.command!r} exited with status {self.code}: {self.stderr!r}"
39
34
 
40
35
 
41
- def chcall(command: str, stdin: Optional[str] = None) -> Tuple[str, str]:
36
+ def chcall(command: str, stdin: str | None = None) -> tuple[str, str]:
42
37
  """Like ``call`` but raises an exception when the return code is
43
38
  nonzero. Only returns the stdout and stderr data.
44
39
  """
@@ -2,7 +2,8 @@ import logging
2
2
  import os
3
3
  import threading
4
4
  import time
5
- from typing import TYPE_CHECKING, Callable, Dict
5
+ from collections.abc import Callable
6
+ from typing import TYPE_CHECKING
6
7
 
7
8
  if TYPE_CHECKING:
8
9
  from cluster_tools.schedulers.cluster_executor import ClusterExecutor
@@ -28,8 +29,8 @@ class FileWaitThread(threading.Thread):
28
29
  threading.Thread.__init__(self)
29
30
  self.callback = callback
30
31
  self.interval = interval
31
- self.waiting: Dict[str, str] = {}
32
- self.retryMap: Dict[str, int] = {}
32
+ self.waiting: dict[str, str] = {}
33
+ self.retryMap: dict[str, int] = {}
33
34
  self.lock = threading.Lock()
34
35
  self.shutdown = False
35
36
  self.executor = executor
@@ -85,17 +86,11 @@ class FileWaitThread(threading.Thread):
85
86
  if self.retryMap[filename] <= FileWaitThread.MAX_RETRY:
86
87
  # Retry by looping again
87
88
  logging.warning(
88
- "Job state is completed, but {} couldn't be found. Retrying {}/{}".format(
89
- filename,
90
- self.retryMap[filename],
91
- FileWaitThread.MAX_RETRY,
92
- )
89
+ f"Job state is completed, but {filename} couldn't be found. Retrying {self.retryMap[filename]}/{FileWaitThread.MAX_RETRY}"
93
90
  )
94
91
  else:
95
92
  logging.error(
96
- "Job state is completed, but {} couldn't be found.".format(
97
- filename
98
- )
93
+ f"Job state is completed, but {filename} couldn't be found."
99
94
  )
100
95
  handle_completed_job(job_id, filename, True)
101
96
 
@@ -6,11 +6,12 @@ import sys
6
6
  import threading
7
7
  import traceback
8
8
  import warnings
9
+ from collections.abc import Callable, Sequence
9
10
  from logging import getLogger
10
11
  from logging.handlers import QueueHandler
11
12
  from queue import Empty as QueueEmpty
12
13
  from queue import Queue
13
- from typing import Any, Callable, List
14
+ from typing import Any
14
15
 
15
16
  # Inspired by https://stackoverflow.com/a/894284
16
17
 
@@ -106,7 +107,7 @@ class _MultiprocessingLoggingHandler(logging.Handler):
106
107
 
107
108
 
108
109
  def _setup_logging_multiprocessing(
109
- queues: List[Queue], levels: List[int], filters: List[Any]
110
+ queues: list[Queue], levels: list[int], filters: Sequence[Any]
110
111
  ) -> None:
111
112
  """Re-setup logging in a multiprocessing context (only needed if a start_method other than
112
113
  fork is used) by setting up QueueHandler loggers for each queue and level
@@ -1,6 +1,6 @@
1
1
  import pickle
2
2
  import sys
3
- from typing import Any, BinaryIO, Optional
3
+ from typing import Any, BinaryIO
4
4
 
5
5
  from cluster_tools._utils.warning import warn_after
6
6
 
@@ -26,25 +26,25 @@ def dump(*args: Any, **kwargs: Any) -> None:
26
26
 
27
27
  @warn_after("pickle.loads", WARNING_TIMEOUT)
28
28
  def loads(*args: Any, **kwargs: Any) -> Any:
29
- assert (
30
- "custom_main_path" not in kwargs
31
- ), "loads does not implement support for the argument custom_main_path"
29
+ assert "custom_main_path" not in kwargs, (
30
+ "loads does not implement support for the argument custom_main_path"
31
+ )
32
32
  return pickle.loads(*args, **kwargs)
33
33
 
34
34
 
35
35
  class _RenameUnpickler(pickle.Unpickler):
36
- custom_main_path: Optional[str]
36
+ custom_main_path: str | None
37
37
 
38
38
  def find_class(self, module: str, name: str) -> Any:
39
39
  renamed_module = module
40
40
  if module == "__main__" and self.custom_main_path is not None:
41
41
  renamed_module = self.custom_main_path
42
42
 
43
- return super(_RenameUnpickler, self).find_class(renamed_module, name)
43
+ return super().find_class(renamed_module, name)
44
44
 
45
45
 
46
46
  @warn_after("pickle.load", WARNING_TIMEOUT)
47
- def load(f: BinaryIO, custom_main_path: Optional[str] = None) -> Any:
47
+ def load(f: BinaryIO, custom_main_path: str | None = None) -> Any:
48
48
  unpickler = _RenameUnpickler(f)
49
49
  unpickler.custom_main_path = custom_main_path
50
50
  return unpickler.load()
@@ -1,5 +1,5 @@
1
1
  import os
2
- from typing import Callable
2
+ from collections.abc import Callable
3
3
 
4
4
  WARNING_TIMEOUT = 10 * 60 # seconds
5
5
 
@@ -5,7 +5,8 @@
5
5
  import os
6
6
  import sys
7
7
  import time
8
- from typing import Any, Callable
8
+ from collections.abc import Callable
9
+ from typing import Any
9
10
 
10
11
 
11
12
  class Tail:
@@ -58,11 +59,11 @@ class Tail:
58
59
  def check_file_validity(self, file_: str) -> None:
59
60
  """Check whether the a given file exists, readable and is a file"""
60
61
  if not os.access(file_, os.F_OK):
61
- raise TailError("File '%s' does not exist" % (file_))
62
+ raise TailError(f"File '{file_}' does not exist")
62
63
  if not os.access(file_, os.R_OK):
63
- raise TailError("File '%s' not readable" % (file_))
64
+ raise TailError(f"File '{file_}' not readable")
64
65
  if os.path.isdir(file_):
65
- raise TailError("File '%s' is a directory" % (file_))
66
+ raise TailError(f"File '{file_}' is a directory")
66
67
 
67
68
 
68
69
  class TailError(Exception):
@@ -1,8 +1,9 @@
1
1
  import logging
2
2
  import threading
3
3
  import time
4
+ from collections.abc import Callable
4
5
  from concurrent.futures import Future
5
- from typing import Callable, TypeVar
6
+ from typing import TypeVar
6
7
 
7
8
  from typing_extensions import ParamSpec
8
9
 
@@ -24,9 +25,7 @@ def warn_after(
24
25
 
25
26
  def warn_function() -> None:
26
27
  logging.warning(
27
- "Function {} is taking suspiciously long (longer than {} seconds)".format(
28
- job, seconds
29
- )
28
+ f"Function {job} is taking suspiciously long (longer than {seconds} seconds)"
30
29
  )
31
30
  exceeded_timeout[0] = True
32
31
 
@@ -38,9 +37,7 @@ def warn_after(
38
37
  if exceeded_timeout[0]:
39
38
  end_time = time.time()
40
39
  logging.warning(
41
- "Function {} succeeded after all (took {} seconds)".format(
42
- job, int(end_time - start_time)
43
- )
40
+ f"Function {job} succeeded after all (took {int(end_time - start_time)} seconds)"
44
41
  )
45
42
  finally:
46
43
  timer.cancel()
@@ -56,9 +53,7 @@ def enrich_future_with_uncaught_warning(f: Future) -> None:
56
53
  maybe_exception = future.exception()
57
54
  if maybe_exception is not None:
58
55
  logging.error(
59
- "A future crashed with an exception: {}. Future: {}".format(
60
- maybe_exception, future
61
- )
56
+ f"A future crashed with an exception: {maybe_exception}. Future: {future}"
62
57
  )
63
58
 
64
59
  if not hasattr(f, "is_wrapped_by_cluster_tools"):
@@ -1,12 +1,8 @@
1
+ from collections.abc import Callable, Iterable, Iterator
1
2
  from concurrent.futures import Future
3
+ from contextlib import AbstractContextManager
2
4
  from os import PathLike
3
5
  from typing import (
4
- Callable,
5
- ContextManager,
6
- Iterable,
7
- Iterator,
8
- List,
9
- Optional,
10
6
  Protocol,
11
7
  TypeVar,
12
8
  )
@@ -18,9 +14,9 @@ _P = ParamSpec("_P")
18
14
  _S = TypeVar("_S")
19
15
 
20
16
 
21
- class Executor(Protocol, ContextManager["Executor"]):
17
+ class Executor(Protocol, AbstractContextManager["Executor"]):
22
18
  @classmethod
23
- def as_completed(cls, futures: List[Future[_T]]) -> Iterator[Future[_T]]: ...
19
+ def as_completed(cls, futures: list[Future[_T]]) -> Iterator[Future[_T]]: ...
24
20
 
25
21
  def submit(
26
22
  self,
@@ -34,15 +30,15 @@ class Executor(Protocol, ContextManager["Executor"]):
34
30
  self,
35
31
  fn: Callable[[_S], _T],
36
32
  args: Iterable[_S],
37
- output_pickle_path_getter: Optional[Callable[[_S], PathLike]] = None,
38
- ) -> List[Future[_T]]: ...
33
+ output_pickle_path_getter: Callable[[_S], PathLike] | None = None,
34
+ ) -> list[Future[_T]]: ...
39
35
 
40
36
  def map(
41
37
  self,
42
38
  fn: Callable[[_S], _T],
43
39
  iterables: Iterable[_S],
44
- timeout: Optional[float] = None,
45
- chunksize: Optional[int] = None,
40
+ timeout: float | None = None,
41
+ chunksize: int | None = None,
46
42
  ) -> Iterator[_T]: ...
47
43
 
48
44
  def forward_log(self, fut: Future[_T]) -> _T: ...
@@ -2,20 +2,15 @@ import logging
2
2
  import os
3
3
  import re
4
4
  import signal
5
+ from collections.abc import Callable, Iterable, Iterator
5
6
  from concurrent import futures
6
7
  from concurrent.futures import Future
7
8
  from functools import partial
8
9
  from multiprocessing import Queue, get_context
10
+ from pathlib import Path
9
11
  from typing import (
10
12
  TYPE_CHECKING,
11
13
  Any,
12
- Callable,
13
- Dict,
14
- Iterable,
15
- Iterator,
16
- List,
17
- Optional,
18
- Set,
19
14
  TypeVar,
20
15
  cast,
21
16
  )
@@ -41,7 +36,7 @@ def _run_in_nanny(
41
36
  queue: Queue, __fn: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
42
37
  ) -> None:
43
38
  try:
44
- __env = cast(Dict[str, str], kwargs.pop("__env"))
39
+ __env = cast(dict[str, str], kwargs.pop("__env"))
45
40
  for key, value in __env.items():
46
41
  os.environ[key] = value
47
42
 
@@ -82,7 +77,7 @@ def _parse_mem(size: str) -> int:
82
77
  def _handle_kill_through_weakref(
83
78
  executor_ref: "ReferenceType[DaskExecutor]",
84
79
  existing_sigint_handler: Any,
85
- signum: Optional[int],
80
+ signum: int | None,
86
81
  frame: Any,
87
82
  ) -> None:
88
83
  executor = executor_ref()
@@ -107,12 +102,12 @@ class DaskExecutor(futures.Executor):
107
102
  """
108
103
 
109
104
  client: "Client"
110
- pending_futures: Set[Future]
111
- job_resources: Optional[Dict[str, Any]]
105
+ pending_futures: set[Future]
106
+ job_resources: dict[str, Any] | None
112
107
  is_shutting_down = False
113
108
 
114
109
  def __init__(
115
- self, client: "Client", job_resources: Optional[Dict[str, Any]] = None
110
+ self, client: "Client", job_resources: dict[str, Any] | None = None
116
111
  ) -> None:
117
112
  try:
118
113
  import distributed # noqa: F401 unused import
@@ -148,7 +143,7 @@ class DaskExecutor(futures.Executor):
148
143
  @classmethod
149
144
  def from_config(
150
145
  cls,
151
- job_resources: Dict[str, str],
146
+ job_resources: dict[str, str],
152
147
  **_kwargs: Any,
153
148
  ) -> "DaskExecutor":
154
149
  from distributed import Client
@@ -162,7 +157,7 @@ class DaskExecutor(futures.Executor):
162
157
  return cls(client, job_resources=job_resources)
163
158
 
164
159
  @classmethod
165
- def as_completed(cls, futures: List[Future[_T]]) -> Iterator[Future[_T]]:
160
+ def as_completed(cls, futures: list[Future[_T]]) -> Iterator[Future[_T]]:
166
161
  from distributed import as_completed
167
162
 
168
163
  return as_completed(futures)
@@ -183,7 +178,7 @@ class DaskExecutor(futures.Executor):
183
178
  Callable[_P, _T],
184
179
  partial(
185
180
  MultiprocessingExecutor._execute_and_persist_function,
186
- output_pickle_path,
181
+ Path(output_pickle_path),
187
182
  __fn,
188
183
  ),
189
184
  )
@@ -200,8 +195,8 @@ class DaskExecutor(futures.Executor):
200
195
  currently_available_workers = self.client.scheduler_info()["workers"]
201
196
 
202
197
  def check_resources(
203
- job_resources: Optional[Dict[str, float]],
204
- worker_resources: Optional[Dict[str, float]],
198
+ job_resources: dict[str, float] | None,
199
+ worker_resources: dict[str, float] | None,
205
200
  ) -> bool:
206
201
  if job_resources is None or len(job_resources) == 0:
207
202
  return True
@@ -242,8 +237,8 @@ class DaskExecutor(futures.Executor):
242
237
  args: Iterable[
243
238
  _S
244
239
  ], # TODO change: allow more than one arg per call # noqa FIX002 Line contains TODO
245
- output_pickle_path_getter: Optional[Callable[[_S], os.PathLike]] = None,
246
- ) -> List[Future[_T]]:
240
+ output_pickle_path_getter: Callable[[_S], os.PathLike] | None = None,
241
+ ) -> list[Future[_T]]:
247
242
  if output_pickle_path_getter is not None:
248
243
  futs = [
249
244
  self.submit( # type: ignore[call-arg]
@@ -264,8 +259,8 @@ class DaskExecutor(futures.Executor):
264
259
  self,
265
260
  fn: Callable[[_S], _T],
266
261
  iterables: Iterable[_S],
267
- timeout: Optional[float] = None,
268
- chunksize: Optional[int] = None,
262
+ timeout: float | None = None,
263
+ chunksize: int | None = None,
269
264
  ) -> Iterator[_T]:
270
265
  if chunksize is None:
271
266
  chunksize = 1
@@ -277,7 +272,7 @@ class DaskExecutor(futures.Executor):
277
272
  def handle_kill(
278
273
  self,
279
274
  existing_sigint_handler: Any,
280
- signum: Optional[int],
275
+ signum: int | None,
281
276
  frame: Any,
282
277
  ) -> None:
283
278
  if self.is_shutting_down:
@@ -1,6 +1,7 @@
1
1
  import logging
2
2
  import multiprocessing
3
3
  import os
4
+ from collections.abc import Callable, Iterable, Iterator
4
5
  from concurrent import futures
5
6
  from concurrent.futures import Future, ProcessPoolExecutor
6
7
  from functools import partial
@@ -8,12 +9,6 @@ from multiprocessing.context import BaseContext
8
9
  from pathlib import Path
9
10
  from typing import (
10
11
  Any,
11
- Callable,
12
- Iterable,
13
- Iterator,
14
- List,
15
- Optional,
16
- Tuple,
17
12
  TypedDict,
18
13
  TypeVar,
19
14
  cast,
@@ -51,11 +46,11 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
51
46
  def __init__(
52
47
  self,
53
48
  *,
54
- max_workers: Optional[int] = None,
55
- start_method: Optional[str] = None,
56
- mp_context: Optional[BaseContext] = None,
57
- initializer: Optional[Callable] = None,
58
- initargs: Tuple = (),
49
+ max_workers: int | None = None,
50
+ start_method: str | None = None,
51
+ mp_context: BaseContext | None = None,
52
+ initializer: Callable | None = None,
53
+ initargs: tuple = (),
59
54
  **__kwargs: Any,
60
55
  ) -> None:
61
56
  if mp_context is None:
@@ -68,9 +63,9 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
68
63
  else:
69
64
  mp_context = multiprocessing.get_context("spawn")
70
65
  else:
71
- assert (
72
- start_method is None
73
- ), "Cannot use both `start_method` and `mp_context` kwargs."
66
+ assert start_method is None, (
67
+ "Cannot use both `start_method` and `mp_context` kwargs."
68
+ )
74
69
 
75
70
  super().__init__(
76
71
  mp_context=mp_context,
@@ -84,7 +79,7 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
84
79
  self._mp_logging_handler_pool = _MultiprocessingLoggingHandlerPool()
85
80
 
86
81
  @classmethod
87
- def as_completed(cls, futs: List[Future[_T]]) -> Iterator[Future[_T]]:
82
+ def as_completed(cls, futs: list[Future[_T]]) -> Iterator[Future[_T]]:
88
83
  return futures.as_completed(futs)
89
84
 
90
85
  def submit( # type: ignore[override]
@@ -141,8 +136,8 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
141
136
  self,
142
137
  fn: Callable[[_S], _T],
143
138
  iterables: Iterable[_S],
144
- timeout: Optional[float] = None,
145
- chunksize: Optional[int] = None,
139
+ timeout: float | None = None,
140
+ chunksize: int | None = None,
146
141
  ) -> Iterator[_T]:
147
142
  if chunksize is None:
148
143
  chunksize = 1
@@ -151,10 +146,10 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
151
146
  @staticmethod
152
147
  def _setup_logging_and_execute(
153
148
  multiprocessing_logging_setup_fn: Callable[[], None],
154
- fn: Callable[_P, Future[_T]],
149
+ fn: Callable[_P, _T],
155
150
  *args: Any,
156
151
  **kwargs: Any,
157
- ) -> Future[_T]:
152
+ ) -> _T:
158
153
  multiprocessing_logging_setup_fn()
159
154
  return fn(*args, **kwargs)
160
155
 
@@ -186,8 +181,8 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
186
181
  self,
187
182
  fn: Callable[[_S], _T],
188
183
  args: Iterable[_S],
189
- output_pickle_path_getter: Optional[Callable[[_S], os.PathLike]] = None,
190
- ) -> List[Future[_T]]:
184
+ output_pickle_path_getter: Callable[[_S], os.PathLike] | None = None,
185
+ ) -> list[Future[_T]]:
191
186
  if output_pickle_path_getter is not None:
192
187
  futs = [
193
188
  self.submit( # type: ignore[call-arg]
@@ -1,6 +1,7 @@
1
+ from collections.abc import Callable
1
2
  from concurrent.futures import Future
2
3
  from functools import partial
3
- from typing import Callable, TypeVar
4
+ from typing import TypeVar
4
5
 
5
6
  from typing_extensions import ParamSpec
6
7
 
@@ -1,9 +1,9 @@
1
1
  import warnings
2
- from collections.abc import Iterable, Iterator
2
+ from collections.abc import Callable, Iterable, Iterator
3
3
  from concurrent.futures import Executor, Future, as_completed
4
4
  from os import PathLike
5
5
  from pathlib import Path
6
- from typing import Any, Callable, Optional, TypeVar, cast
6
+ from typing import Any, TypeVar, cast
7
7
 
8
8
  from typing_extensions import ParamSpec
9
9
 
@@ -59,7 +59,7 @@ class SequentialExecutor(Executor):
59
59
  self,
60
60
  fn: Callable[[_S], _T],
61
61
  args: Iterable[_S],
62
- output_pickle_path_getter: Optional[Callable[[_S], PathLike]] = None,
62
+ output_pickle_path_getter: Callable[[_S], PathLike] | None = None,
63
63
  ) -> list[Future[_T]]:
64
64
  if output_pickle_path_getter is not None:
65
65
  futs = [
@@ -81,8 +81,8 @@ class SequentialExecutor(Executor):
81
81
  self,
82
82
  fn: Callable[[_S], _T],
83
83
  iterables: Iterable[_S],
84
- timeout: Optional[float] = None,
85
- chunksize: Optional[int] = None,
84
+ timeout: float | None = None,
85
+ chunksize: int | None = None,
86
86
  ) -> Iterator[_T]:
87
87
  if timeout is not None:
88
88
  warnings.warn(
@@ -1,6 +1,7 @@
1
+ from collections.abc import Callable
1
2
  from concurrent.futures import Future
2
3
  from functools import partial
3
- from typing import Callable, TypeVar
4
+ from typing import TypeVar
4
5
 
5
6
  from typing_extensions import ParamSpec
6
7
 
@@ -4,7 +4,7 @@ import logging
4
4
  import os
5
5
  import sys
6
6
  import traceback
7
- from typing import Any, Dict, Optional, Type
7
+ from typing import Any
8
8
 
9
9
  from cluster_tools._utils import pickling
10
10
  from cluster_tools._utils.string_ import with_preliminary_postfix
@@ -14,7 +14,7 @@ from cluster_tools.schedulers.pbs import PBSExecutor
14
14
  from cluster_tools.schedulers.slurm import SlurmExecutor
15
15
 
16
16
 
17
- def get_executor_class(executor_key: str) -> Type[ClusterExecutor]:
17
+ def get_executor_class(executor_key: str) -> type[ClusterExecutor]:
18
18
  return {
19
19
  "slurm": SlurmExecutor,
20
20
  "pbs": PBSExecutor,
@@ -29,21 +29,19 @@ def format_remote_exc() -> str:
29
29
  return "".join(traceback.format_exception(typ, value, tb))
30
30
 
31
31
 
32
- def get_custom_main_path(
33
- workerid: str, executor: Type[ClusterExecutor]
34
- ) -> Optional[str]:
32
+ def get_custom_main_path(workerid: str, executor: type[ClusterExecutor]) -> str | None:
35
33
  custom_main_path = None
36
34
  main_meta_path = executor.get_main_meta_path(cfut_dir, workerid)
37
35
  if os.path.exists(main_meta_path):
38
- with open(main_meta_path, "r", encoding="utf-8") as file:
36
+ with open(main_meta_path, encoding="utf-8") as file:
39
37
  custom_main_path = file.read()
40
38
  return custom_main_path
41
39
 
42
40
 
43
41
  def worker(
44
- executor: Type[ClusterExecutor],
42
+ executor: type[ClusterExecutor],
45
43
  workerid: str,
46
- job_array_index: Optional[int],
44
+ job_array_index: int | None,
47
45
  job_array_index_offset: str,
48
46
  cfut_dir: str,
49
47
  ) -> None:
@@ -75,9 +73,7 @@ def worker(
75
73
  setup_logging(meta_data, executor, cfut_dir)
76
74
 
77
75
  logging.info(
78
- "Job computation started (jobid={}, workerid_with_idx={}).".format(
79
- executor.get_current_job_id(), workerid_with_idx
80
- )
76
+ f"Job computation started (jobid={executor.get_current_job_id()}, workerid_with_idx={workerid_with_idx})."
81
77
  )
82
78
  result = True, fun(*args, **kwargs)
83
79
  logging.info("Job computation completed.")
@@ -98,13 +94,13 @@ def worker(
98
94
  tempfile = str(destfile) + ".tmp"
99
95
  with open(tempfile, "wb") as f:
100
96
  f.write(out)
101
- logging.debug("Pickle file written to {}.".format(tempfile))
97
+ logging.debug(f"Pickle file written to {tempfile}.")
102
98
  os.rename(tempfile, destfile)
103
- logging.debug("Pickle file renamed to {}.".format(destfile))
99
+ logging.debug(f"Pickle file renamed to {destfile}.")
104
100
 
105
101
 
106
102
  def setup_logging(
107
- meta_data: Dict[str, Any], executor: Type[ClusterExecutor], cfut_dir: str
103
+ meta_data: dict[str, Any], executor: type[ClusterExecutor], cfut_dir: str
108
104
  ) -> None:
109
105
  if "logging_setup_fn" in meta_data:
110
106
  logging.debug("Using supplied logging_setup_fn to setup logging.")
@@ -118,9 +114,7 @@ def setup_logging(
118
114
  logging_config = meta_data.get("logging_config", dict())
119
115
 
120
116
  logging.debug(
121
- "Setting up logging.basicConfig (potentially overwriting logging configuration of the main script). Config: {}".format(
122
- logging_config
123
- )
117
+ f"Setting up logging.basicConfig (potentially overwriting logging configuration of the main script). Config: {logging_config}"
124
118
  )
125
119
 
126
120
  # Call basicConfig which is necessary for the logging to work.