cluster-tools 3.5.2__tar.gz → 3.5.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/PKG-INFO +2 -2
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/cluster_executor.py +24 -6
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/slurm.py +28 -1
- cluster_tools-3.5.4/cluster_tools/version.py +1 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/PKG-INFO +2 -2
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/requires.txt +1 -1
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/pyproject.toml +3 -3
- cluster_tools-3.5.2/cluster_tools/version.py +0 -1
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/LICENSE +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/README.md +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/__init__.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/call.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/file_wait_thread.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/multiprocessing_logging_handler.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/pickling.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/reflection.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/string_.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/tailf.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/warning.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executor_protocol.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/batching.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/dask.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/multiprocessing_.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/sequential.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/sequential_pickle.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/remote.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/__init__.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/kube.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/pbs.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/SOURCES.txt +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/dependency_links.txt +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/top_level.txt +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/setup.cfg +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_all.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_batching.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_dask.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_deref_main.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_kubernetes.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_multiprocessing.py +0 -0
- {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_slurm.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cluster_tools
|
|
3
|
-
Version: 3.5.
|
|
3
|
+
Version: 3.5.4
|
|
4
4
|
Summary: Utility library for easily distributing code execution on clusters
|
|
5
5
|
Author-email: scalable minds <hello@scalableminds.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -9,7 +9,7 @@ Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/ma
|
|
|
9
9
|
Requires-Python: >=3.10
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
License-File: LICENSE
|
|
12
|
-
Requires-Dist: typing-extensions~=4.
|
|
12
|
+
Requires-Dist: typing-extensions~=4.15.0
|
|
13
13
|
Provides-Extra: dask
|
|
14
14
|
Requires-Dist: distributed~=2023.9.1; extra == "dask"
|
|
15
15
|
Provides-Extra: kubernetes
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import atexit
|
|
2
2
|
import logging
|
|
3
|
+
import math
|
|
3
4
|
import os
|
|
4
5
|
import signal
|
|
5
6
|
import sys
|
|
@@ -38,6 +39,9 @@ _T = TypeVar("_T")
|
|
|
38
39
|
_P = ParamSpec("_P")
|
|
39
40
|
_S = TypeVar("_S")
|
|
40
41
|
|
|
42
|
+
LOG_FILE_POLLING_INTERVAL_SECONDS = 2
|
|
43
|
+
MAX_LOG_FILE_POLLING_DURATION_SECONDS = 120
|
|
44
|
+
|
|
41
45
|
|
|
42
46
|
def join_messages(strings: list[str]) -> str:
|
|
43
47
|
return " ".join(x.strip() for x in strings if x.strip())
|
|
@@ -776,13 +780,27 @@ class ClusterExecutor(futures.Executor):
|
|
|
776
780
|
tailer = Tail(log_path, log_callback)
|
|
777
781
|
fut.add_done_callback(lambda _: tailer.cancel())
|
|
778
782
|
|
|
779
|
-
#
|
|
780
|
-
|
|
781
|
-
|
|
783
|
+
# Wait until the log file exists or the job is completed/cancelled
|
|
784
|
+
log_file_exists = False
|
|
785
|
+
while not (
|
|
786
|
+
(log_file_exists := os.path.exists(log_path)) or tailer.is_cancelled
|
|
787
|
+
):
|
|
788
|
+
time.sleep(LOG_FILE_POLLING_INTERVAL_SECONDS)
|
|
782
789
|
|
|
783
|
-
#
|
|
784
|
-
#
|
|
785
|
-
|
|
790
|
+
# The job might be done/cancelled but the log file might still not exist (e.g. slow/inconsistent NFS)
|
|
791
|
+
# In this case, we keep polling but with an upper limit to not run into a deadlock.
|
|
792
|
+
retries = 0
|
|
793
|
+
max_retries = math.ceil(
|
|
794
|
+
MAX_LOG_FILE_POLLING_DURATION_SECONDS / LOG_FILE_POLLING_INTERVAL_SECONDS
|
|
795
|
+
)
|
|
796
|
+
while not log_file_exists and retries < max_retries:
|
|
797
|
+
log_file_exists = os.path.exists(log_path)
|
|
798
|
+
retries += 1
|
|
799
|
+
sys.stdout.write(
|
|
800
|
+
f"Job with id {fut.cluster_jobid} is finished but log file couldn't be found at {log_path}. Retrying {retries}/{max_retries}" # type: ignore[attr-defined]
|
|
801
|
+
)
|
|
802
|
+
time.sleep(LOG_FILE_POLLING_INTERVAL_SECONDS)
|
|
803
|
+
tailer.follow(LOG_FILE_POLLING_INTERVAL_SECONDS)
|
|
786
804
|
return fut.result()
|
|
787
805
|
|
|
788
806
|
@abstractmethod
|
|
@@ -54,6 +54,27 @@ SLURM_STATES = {
|
|
|
54
54
|
"Unclear": ["SUSPENDED", "REVOKED", "SIGNALING", "SPECIAL_EXIT", "STAGE_OUT"],
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
# Since Slurm 22.05, some resource flags are not automatically inherited from the
|
|
58
|
+
# enclosing sbatch job by srun calls within the script anymore and must be restated
|
|
59
|
+
# explicitly, or srun computes its own (potentially conflicting) resource request
|
|
60
|
+
# leading to failures like:
|
|
61
|
+
# "srun: fatal: cpus-per-task set by two different environment variables ..."
|
|
62
|
+
# Other job_resources (e.g. --mem, --gres) should be inherited by srun automatically.
|
|
63
|
+
# Therefore, we currently explicitely restate potentially non-inherited resource for the srun call.
|
|
64
|
+
# This is most relevant for cpus-per-task, but we have added some others to be on the safe side.
|
|
65
|
+
# It is worth noting that this change was quite disruptive, particularly for MPI implementations.
|
|
66
|
+
# Because of this, in Slurm 23.11, this requirement was largely walked back.
|
|
67
|
+
# SRUN_CPUS_PER_TASK war reverted back to SLURM_CPUS_PER_TASK and an --external-launcher option was introduced to srun
|
|
68
|
+
# to prevent task launchers from being bound to a single CPU.
|
|
69
|
+
SLURM_SRUN_NON_INHERITED_RESOURCE_KEYS = {
|
|
70
|
+
"cpus-per-task",
|
|
71
|
+
"ntasks",
|
|
72
|
+
"gpus",
|
|
73
|
+
"gpus-per-node",
|
|
74
|
+
"gpus-per-socket",
|
|
75
|
+
"gpus-per-task",
|
|
76
|
+
}
|
|
77
|
+
|
|
57
78
|
SLURM_QUEUE_CHECK_INTERVAL = 1 if "pytest" in sys.modules else 60
|
|
58
79
|
|
|
59
80
|
T = TypeVar("T")
|
|
@@ -303,9 +324,15 @@ class SlurmExecutor(ClusterExecutor):
|
|
|
303
324
|
log_path = self.format_log_file_path(self.cfut_dir, job_id_string)
|
|
304
325
|
|
|
305
326
|
job_resources_lines = []
|
|
327
|
+
srun_resource_args = []
|
|
306
328
|
if self.job_resources is not None:
|
|
307
329
|
for resource, value in self.job_resources.items():
|
|
308
330
|
job_resources_lines += [f"#SBATCH --{resource}={value}"]
|
|
331
|
+
if resource in SLURM_SRUN_NON_INHERITED_RESOURCE_KEYS:
|
|
332
|
+
srun_resource_args += [f"--{resource}={value}"]
|
|
333
|
+
srun_resource_args_str = (
|
|
334
|
+
" " + " ".join(srun_resource_args) if srun_resource_args else ""
|
|
335
|
+
)
|
|
309
336
|
|
|
310
337
|
max_array_size = self.get_max_array_size()
|
|
311
338
|
max_submit_jobs = self.get_max_submit_jobs()
|
|
@@ -340,7 +367,7 @@ class SlurmExecutor(ClusterExecutor):
|
|
|
340
367
|
+ job_resources_lines
|
|
341
368
|
+ [
|
|
342
369
|
*additional_setup_lines,
|
|
343
|
-
f"srun {cmdline} {job_index_start}",
|
|
370
|
+
f"srun{srun_resource_args_str} {cmdline} {job_index_start}",
|
|
344
371
|
]
|
|
345
372
|
)
|
|
346
373
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '3.5.4'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cluster_tools
|
|
3
|
-
Version: 3.5.
|
|
3
|
+
Version: 3.5.4
|
|
4
4
|
Summary: Utility library for easily distributing code execution on clusters
|
|
5
5
|
Author-email: scalable minds <hello@scalableminds.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -9,7 +9,7 @@ Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/ma
|
|
|
9
9
|
Requires-Python: >=3.10
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
License-File: LICENSE
|
|
12
|
-
Requires-Dist: typing-extensions~=4.
|
|
12
|
+
Requires-Dist: typing-extensions~=4.15.0
|
|
13
13
|
Provides-Extra: dask
|
|
14
14
|
Requires-Dist: distributed~=2023.9.1; extra == "dask"
|
|
15
15
|
Provides-Extra: kubernetes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "cluster_tools"
|
|
3
|
-
version = "3.5.
|
|
3
|
+
version = "3.5.4" # filled by dunamai
|
|
4
4
|
description = "Utility library for easily distributing code execution on clusters"
|
|
5
5
|
authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
|
|
6
6
|
readme = "README.md"
|
|
@@ -8,7 +8,7 @@ license-files = ["LICENSE"]
|
|
|
8
8
|
license = "MIT"
|
|
9
9
|
requires-python = ">=3.10"
|
|
10
10
|
dependencies = [
|
|
11
|
-
"typing-extensions ~=4.
|
|
11
|
+
"typing-extensions ~=4.15.0",
|
|
12
12
|
]
|
|
13
13
|
|
|
14
14
|
[project.urls]
|
|
@@ -24,7 +24,7 @@ all=["cluster_tools[kubernetes]", "cluster_tools[dask]"]
|
|
|
24
24
|
[dependency-groups]
|
|
25
25
|
dev = [
|
|
26
26
|
"icecream ~=2.1.1",
|
|
27
|
-
"mypy ~=1.
|
|
27
|
+
"mypy ~=2.1.0",
|
|
28
28
|
"pytest ~=8.3.3",
|
|
29
29
|
"pytest-timeout>=2.4.0",
|
|
30
30
|
"ruff ~=0.11.0",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '3.5.2'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/multiprocessing_logging_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/multiprocessing_pickle.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|