cluster-tools 2.3.8__tar.gz → 2.3.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cluster_tools-2.3.10/LICENSE +21 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/PKG-INFO +4 -2
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/schedulers/slurm.py +48 -33
- cluster_tools-2.3.10/cluster_tools/version.py +1 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools.egg-info/PKG-INFO +4 -2
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools.egg-info/SOURCES.txt +1 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/pyproject.toml +4 -8
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/tests/test_slurm.py +27 -52
- cluster_tools-2.3.8/cluster_tools/version.py +0 -1
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/README.md +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/__init__.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/call.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/file_wait_thread.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/multiprocessing_logging_handler.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/pickling.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/reflection.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/string_.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/tailf.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/warning.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/executor_protocol.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/executors/dask.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/executors/multiprocessing_.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/executors/sequential.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/executors/sequential_pickle.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/remote.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/schedulers/__init__.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/schedulers/cluster_executor.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/schedulers/kube.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/schedulers/pbs.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools.egg-info/dependency_links.txt +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools.egg-info/requires.txt +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools.egg-info/top_level.txt +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/setup.cfg +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/tests/test_all.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/tests/test_dask.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/tests/test_deref_main.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/tests/test_kubernetes.py +0 -0
- {cluster_tools-2.3.8 → cluster_tools-2.3.10}/tests/test_multiprocessing.py +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2019 scalable minds
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cluster_tools
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.10
|
|
4
4
|
Summary: Utility library for easily distributing code execution on clusters
|
|
5
5
|
Author-email: scalable minds <hello@scalableminds.com>
|
|
6
|
-
License: MIT
|
|
6
|
+
License-Expression: MIT
|
|
7
7
|
Project-URL: Repository, https://github.com/scalableminds/webknossos-libs
|
|
8
8
|
Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/master/cluster_tools/Changelog.md
|
|
9
9
|
Requires-Python: >=3.10
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
11
12
|
Requires-Dist: typing-extensions~=4.12.0
|
|
12
13
|
Provides-Extra: kubernetes
|
|
13
14
|
Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
|
|
@@ -16,6 +17,7 @@ Requires-Dist: kubernetes~=27.2.0; extra == "dask"
|
|
|
16
17
|
Provides-Extra: all
|
|
17
18
|
Requires-Dist: cluster_tools[kubernetes]; extra == "all"
|
|
18
19
|
Requires-Dist: cluster_tools[dask]; extra == "all"
|
|
20
|
+
Dynamic: license-file
|
|
19
21
|
|
|
20
22
|
# Cluster Tools
|
|
21
23
|
|
|
@@ -5,6 +5,7 @@ import os
|
|
|
5
5
|
import re
|
|
6
6
|
import sys
|
|
7
7
|
import threading
|
|
8
|
+
import time
|
|
8
9
|
from collections.abc import Callable, Iterable
|
|
9
10
|
from concurrent.futures import Future
|
|
10
11
|
from functools import lru_cache
|
|
@@ -423,8 +424,9 @@ class SlurmExecutor(ClusterExecutor):
|
|
|
423
424
|
|
|
424
425
|
# Call `scontrol show jobid=<job_id>` which should return some output including
|
|
425
426
|
# key=value pairs, such as: "Reason=...", "TimeLimit=...", and "RunTime=..."
|
|
426
|
-
stdout,
|
|
427
|
+
stdout, stderr, exit_code = call(f"scontrol show jobid={job_id_with_index}")
|
|
427
428
|
|
|
429
|
+
properties = None
|
|
428
430
|
if exit_code == 0:
|
|
429
431
|
# Parse stdout into a key-value object
|
|
430
432
|
properties = parse_key_value_pairs(stdout, " ", "=")
|
|
@@ -433,18 +435,28 @@ class SlurmExecutor(ClusterExecutor):
|
|
|
433
435
|
if time_limit_investigation:
|
|
434
436
|
return time_limit_investigation
|
|
435
437
|
|
|
436
|
-
#
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
438
|
+
# Request gathered job information.
|
|
439
|
+
for _ in range(10):
|
|
440
|
+
stdout, _, exit_code = call(
|
|
441
|
+
f"sacct -P --format=JobID,State,MaxRSS,ReqMem --unit K -j {job_id_with_index}"
|
|
442
|
+
)
|
|
443
|
+
stdout_lines = stdout.splitlines()
|
|
441
444
|
|
|
442
|
-
|
|
443
|
-
|
|
445
|
+
if exit_code != 0:
|
|
446
|
+
break
|
|
447
|
+
elif len(stdout_lines) <= 1:
|
|
448
|
+
time.sleep(0.2)
|
|
449
|
+
else:
|
|
450
|
+
# Parse stdout into a key-value object
|
|
451
|
+
memory_limit_investigation = self._investigate_memory_consumption(
|
|
452
|
+
stdout
|
|
453
|
+
)
|
|
454
|
+
if memory_limit_investigation:
|
|
455
|
+
return memory_limit_investigation
|
|
456
|
+
break
|
|
444
457
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
return memory_limit_investigation
|
|
458
|
+
if properties is None:
|
|
459
|
+
return None
|
|
448
460
|
|
|
449
461
|
return self._investigate_exit_code(properties)
|
|
450
462
|
|
|
@@ -466,30 +478,33 @@ class SlurmExecutor(ClusterExecutor):
|
|
|
466
478
|
return (reason, RemoteTimeLimitException)
|
|
467
479
|
|
|
468
480
|
def _investigate_memory_consumption(
|
|
469
|
-
self,
|
|
481
|
+
self, stdout: str
|
|
470
482
|
) -> tuple[str, type[RemoteOutOfMemoryException]] | None:
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
#
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
483
|
+
# Table Format:
|
|
484
|
+
#
|
|
485
|
+
# JobID|State|MaxRSS|ReqMem
|
|
486
|
+
# 91_0|FAILED||30720K
|
|
487
|
+
# 91_0.batch|FAILED|248K|
|
|
488
|
+
# 91_0.0|OUT_OF_MEMORY|164K|
|
|
489
|
+
max_rss = 0
|
|
490
|
+
req_mem = 0
|
|
491
|
+
states = []
|
|
492
|
+
linefilter = re.compile(r"^([^|]*)\|(\w*)\|(\d*)K?\|(\d*)K?$", re.MULTILINE)
|
|
493
|
+
for job_id, state, rss, mem in linefilter.findall(stdout):
|
|
494
|
+
max_rss = max(max_rss, int(rss or 0))
|
|
495
|
+
req_mem = max(req_mem, int(mem or 0))
|
|
496
|
+
states += [state]
|
|
497
|
+
|
|
498
|
+
if "OUT_OF_MEMORY" in states:
|
|
499
|
+
# Check if task plugin killed the job. This is the case if cgroup is used as TaskPlugin and
|
|
500
|
+
# memory limits are enforced.
|
|
501
|
+
reason = f"The job was terminated because it consumed too much memory (Requested: {req_mem / 1000} MB)."
|
|
502
|
+
elif max_rss > req_mem:
|
|
503
|
+
# Check if job accounting canceled the job. This is the case if JobAcctGatherParam=OverMemoryKill
|
|
504
|
+
# is enabled.
|
|
505
|
+
reason = f"The job was probably terminated because it consumed too much memory. Required {max_rss / 1000} MB but requeseted {req_mem / 1000} MB."
|
|
506
|
+
else:
|
|
490
507
|
return None
|
|
491
|
-
|
|
492
|
-
reason = f"The job was probably terminated because it consumed too much memory ({efficiency_note})."
|
|
493
508
|
return (reason, RemoteOutOfMemoryException)
|
|
494
509
|
|
|
495
510
|
def _investigate_exit_code(
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.3.10'
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cluster_tools
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.10
|
|
4
4
|
Summary: Utility library for easily distributing code execution on clusters
|
|
5
5
|
Author-email: scalable minds <hello@scalableminds.com>
|
|
6
|
-
License: MIT
|
|
6
|
+
License-Expression: MIT
|
|
7
7
|
Project-URL: Repository, https://github.com/scalableminds/webknossos-libs
|
|
8
8
|
Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/master/cluster_tools/Changelog.md
|
|
9
9
|
Requires-Python: >=3.10
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
11
12
|
Requires-Dist: typing-extensions~=4.12.0
|
|
12
13
|
Provides-Extra: kubernetes
|
|
13
14
|
Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
|
|
@@ -16,6 +17,7 @@ Requires-Dist: kubernetes~=27.2.0; extra == "dask"
|
|
|
16
17
|
Provides-Extra: all
|
|
17
18
|
Requires-Dist: cluster_tools[kubernetes]; extra == "all"
|
|
18
19
|
Requires-Dist: cluster_tools[dask]; extra == "all"
|
|
20
|
+
Dynamic: license-file
|
|
19
21
|
|
|
20
22
|
# Cluster Tools
|
|
21
23
|
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "cluster_tools"
|
|
3
|
-
version = "2.3.
|
|
3
|
+
version = "2.3.10" # filled by dunamai
|
|
4
4
|
description = "Utility library for easily distributing code execution on clusters"
|
|
5
|
-
authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
|
|
5
|
+
authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
|
|
6
6
|
readme = "README.md"
|
|
7
|
-
license =
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
license = "MIT"
|
|
8
9
|
requires-python = ">=3.10"
|
|
9
10
|
dependencies = [
|
|
10
11
|
"typing-extensions ~=4.12.0",
|
|
@@ -37,11 +38,6 @@ where = ["."]
|
|
|
37
38
|
include = ["cluster_tools*"]
|
|
38
39
|
exclude = ["cluster_tools.tests"]
|
|
39
40
|
|
|
40
|
-
# This is a fix for an issue in setuptools. See: https://github.com/pypa/setuptools/issues/4759
|
|
41
|
-
# This should be removed when the issue is resolved.
|
|
42
|
-
[tool.setuptools]
|
|
43
|
-
license-files = []
|
|
44
|
-
|
|
45
41
|
[tool.ruff]
|
|
46
42
|
# Exclude a variety of commonly ignored directories.
|
|
47
43
|
exclude = [
|
|
@@ -44,12 +44,6 @@ def expect_fork() -> bool:
|
|
|
44
44
|
return True
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
def search_and_replace_in_slurm_config(search_string: str, replace_string: str) -> None:
|
|
48
|
-
chcall(
|
|
49
|
-
f"sed -ci 's/{search_string}/{replace_string}/g' /etc/slurm/slurm.conf && scontrol reconfigure"
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
|
|
53
47
|
def test_map_with_spawn() -> None:
|
|
54
48
|
with cluster_tools.get_executor(
|
|
55
49
|
"slurm", max_workers=5, start_method="spawn"
|
|
@@ -273,33 +267,24 @@ def test_slurm_number_of_submitted_jobs() -> None:
|
|
|
273
267
|
assert executor.get_number_of_submitted_jobs() == 0
|
|
274
268
|
|
|
275
269
|
|
|
270
|
+
@pytest.mark.requires_modified_slurm_config
|
|
276
271
|
def test_slurm_max_array_size() -> None:
|
|
277
|
-
|
|
272
|
+
expected_max_array_size = 2
|
|
278
273
|
|
|
279
274
|
executor = cluster_tools.get_executor("slurm", debug=True)
|
|
280
|
-
original_max_array_size = executor.get_max_array_size()
|
|
281
275
|
|
|
282
|
-
|
|
276
|
+
max_array_size = executor.get_max_array_size()
|
|
277
|
+
assert max_array_size == expected_max_array_size
|
|
283
278
|
|
|
284
|
-
|
|
285
|
-
|
|
279
|
+
with executor:
|
|
280
|
+
futures = executor.map_to_futures(square, range(6))
|
|
281
|
+
concurrent.futures.wait(futures)
|
|
282
|
+
job_ids = [fut.cluster_jobid for fut in futures] # type: ignore[attr-defined]
|
|
286
283
|
|
|
287
|
-
|
|
288
|
-
|
|
284
|
+
# Count how often each job_id occurs which corresponds to the array size of the job
|
|
285
|
+
occurrences = list(Counter(job_ids).values())
|
|
289
286
|
|
|
290
|
-
|
|
291
|
-
futures = executor.map_to_futures(square, range(6))
|
|
292
|
-
concurrent.futures.wait(futures)
|
|
293
|
-
job_ids = [fut.cluster_jobid for fut in futures] # type: ignore[attr-defined]
|
|
294
|
-
|
|
295
|
-
# Count how often each job_id occurs which corresponds to the array size of the job
|
|
296
|
-
occurrences = list(Counter(job_ids).values())
|
|
297
|
-
|
|
298
|
-
assert all(array_size <= max_array_size for array_size in occurrences)
|
|
299
|
-
finally:
|
|
300
|
-
search_and_replace_in_slurm_config(command, "")
|
|
301
|
-
reset_max_array_size = executor.get_max_array_size()
|
|
302
|
-
assert reset_max_array_size == original_max_array_size
|
|
287
|
+
assert all(array_size <= expected_max_array_size for array_size in occurrences)
|
|
303
288
|
|
|
304
289
|
|
|
305
290
|
@pytest.mark.skip(
|
|
@@ -323,38 +308,28 @@ def test_slurm_time_limit() -> None:
|
|
|
323
308
|
)
|
|
324
309
|
|
|
325
310
|
|
|
311
|
+
@pytest.mark.requires_modified_slurm_config
|
|
326
312
|
def test_slurm_memory_limit() -> None:
|
|
327
|
-
# Request
|
|
313
|
+
# Request 30 MB
|
|
328
314
|
executor = cluster_tools.get_executor(
|
|
329
|
-
"slurm",
|
|
315
|
+
"slurm",
|
|
316
|
+
debug=True,
|
|
317
|
+
job_resources={"mem": "30M"}, # 30M is the smallest limit enforced by Cgroups
|
|
330
318
|
)
|
|
331
319
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
original_gather_frequency_config, new_gather_frequency_config
|
|
320
|
+
with executor:
|
|
321
|
+
# Schedule a job that allocates more than 30 MB and let it run for more than 1 second
|
|
322
|
+
# because the frequency of the memory polling is 1 second
|
|
323
|
+
duration = 3
|
|
324
|
+
futures = executor.map_to_futures(
|
|
325
|
+
partial(allocate, duration), [1024 * 1024 * 50]
|
|
339
326
|
)
|
|
327
|
+
concurrent.futures.wait(futures)
|
|
340
328
|
|
|
341
|
-
with
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
futures = executor.map_to_futures(
|
|
346
|
-
partial(allocate, duration), [1024 * 1024 * 2]
|
|
347
|
-
)
|
|
348
|
-
concurrent.futures.wait(futures)
|
|
349
|
-
|
|
350
|
-
# Job should have been killed with a RemoteOutOfMemoryException
|
|
351
|
-
assert all(
|
|
352
|
-
isinstance(fut.exception(), cluster_tools.RemoteOutOfMemoryException)
|
|
353
|
-
for fut in futures
|
|
354
|
-
)
|
|
355
|
-
finally:
|
|
356
|
-
search_and_replace_in_slurm_config(
|
|
357
|
-
new_gather_frequency_config, original_gather_frequency_config
|
|
329
|
+
# Job should have been killed with a RemoteOutOfMemoryException
|
|
330
|
+
assert all(
|
|
331
|
+
isinstance(fut.exception(), cluster_tools.RemoteOutOfMemoryException)
|
|
332
|
+
for fut in futures
|
|
358
333
|
)
|
|
359
334
|
|
|
360
335
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '2.3.8'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/_utils/multiprocessing_logging_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cluster_tools-2.3.8 → cluster_tools-2.3.10}/cluster_tools/executors/multiprocessing_pickle.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|