cluster-tools 2.3.7__tar.gz → 2.3.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. cluster_tools-2.3.9/LICENSE +21 -0
  2. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/PKG-INFO +4 -2
  3. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/schedulers/slurm.py +48 -33
  4. cluster_tools-2.3.9/cluster_tools/version.py +1 -0
  5. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools.egg-info/PKG-INFO +4 -2
  6. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools.egg-info/SOURCES.txt +1 -0
  7. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/pyproject.toml +4 -8
  8. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/tests/test_slurm.py +27 -52
  9. cluster_tools-2.3.7/cluster_tools/version.py +0 -1
  10. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/README.md +0 -0
  11. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/__init__.py +0 -0
  12. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/call.py +0 -0
  13. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/file_wait_thread.py +0 -0
  14. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/multiprocessing_logging_handler.py +0 -0
  15. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/pickling.py +0 -0
  16. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/reflection.py +0 -0
  17. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/string_.py +0 -0
  18. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/tailf.py +0 -0
  19. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/_utils/warning.py +0 -0
  20. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/executor_protocol.py +0 -0
  21. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/executors/dask.py +0 -0
  22. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/executors/multiprocessing_.py +0 -0
  23. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
  24. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/executors/sequential.py +0 -0
  25. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/executors/sequential_pickle.py +0 -0
  26. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/remote.py +0 -0
  27. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/schedulers/__init__.py +0 -0
  28. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/schedulers/cluster_executor.py +0 -0
  29. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/schedulers/kube.py +0 -0
  30. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools/schedulers/pbs.py +0 -0
  31. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools.egg-info/dependency_links.txt +0 -0
  32. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools.egg-info/requires.txt +0 -0
  33. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/cluster_tools.egg-info/top_level.txt +0 -0
  34. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/setup.cfg +0 -0
  35. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/tests/test_all.py +0 -0
  36. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/tests/test_dask.py +0 -0
  37. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/tests/test_deref_main.py +0 -0
  38. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/tests/test_kubernetes.py +0 -0
  39. {cluster_tools-2.3.7 → cluster_tools-2.3.9}/tests/test_multiprocessing.py +0 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 scalable minds
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -1,13 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 2.3.7
3
+ Version: 2.3.9
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
- License: MIT
6
+ License-Expression: MIT
7
7
  Project-URL: Repository, https://github.com/scalableminds/webknossos-libs
8
8
  Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/master/cluster_tools/Changelog.md
9
9
  Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
+ License-File: LICENSE
11
12
  Requires-Dist: typing-extensions~=4.12.0
12
13
  Provides-Extra: kubernetes
13
14
  Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
@@ -16,6 +17,7 @@ Requires-Dist: kubernetes~=27.2.0; extra == "dask"
16
17
  Provides-Extra: all
17
18
  Requires-Dist: cluster_tools[kubernetes]; extra == "all"
18
19
  Requires-Dist: cluster_tools[dask]; extra == "all"
20
+ Dynamic: license-file
19
21
 
20
22
  # Cluster Tools
21
23
 
@@ -5,6 +5,7 @@ import os
5
5
  import re
6
6
  import sys
7
7
  import threading
8
+ import time
8
9
  from collections.abc import Callable, Iterable
9
10
  from concurrent.futures import Future
10
11
  from functools import lru_cache
@@ -423,8 +424,9 @@ class SlurmExecutor(ClusterExecutor):
423
424
 
424
425
  # Call `scontrol show jobid=<job_id>` which should return some output including
425
426
  # key=value pairs, such as: "Reason=...", "TimeLimit=...", and "RunTime=..."
426
- stdout, _, exit_code = call(f"scontrol show jobid={job_id_with_index}")
427
+ stdout, stderr, exit_code = call(f"scontrol show jobid={job_id_with_index}")
427
428
 
429
+ properties = None
428
430
  if exit_code == 0:
429
431
  # Parse stdout into a key-value object
430
432
  properties = parse_key_value_pairs(stdout, " ", "=")
@@ -433,18 +435,28 @@ class SlurmExecutor(ClusterExecutor):
433
435
  if time_limit_investigation:
434
436
  return time_limit_investigation
435
437
 
436
- # Call `seff job_id` which should return some output including a line,
437
- # such as: "Memory Efficiency: 25019.18% of 1.00 GB"
438
- stdout, _, exit_code = call(f"seff {job_id_with_index}")
439
- if exit_code != 0:
440
- return None
438
+ # Request gathered job information.
439
+ for _ in range(10):
440
+ stdout, _, exit_code = call(
441
+ f"sacct -P --format=JobID,State,MaxRSS,ReqMem --unit K -j {job_id_with_index}"
442
+ )
443
+ stdout_lines = stdout.splitlines()
441
444
 
442
- # Parse stdout into a key-value object
443
- properties = parse_key_value_pairs(stdout, "\n", ":")
445
+ if exit_code != 0:
446
+ break
447
+ elif len(stdout_lines) <= 1:
448
+ time.sleep(0.2)
449
+ else:
450
+ # Parse stdout into a key-value object
451
+ memory_limit_investigation = self._investigate_memory_consumption(
452
+ stdout
453
+ )
454
+ if memory_limit_investigation:
455
+ return memory_limit_investigation
456
+ break
444
457
 
445
- memory_limit_investigation = self._investigate_memory_consumption(properties)
446
- if memory_limit_investigation:
447
- return memory_limit_investigation
458
+ if properties is None:
459
+ return None
448
460
 
449
461
  return self._investigate_exit_code(properties)
450
462
 
@@ -466,30 +478,33 @@ class SlurmExecutor(ClusterExecutor):
466
478
  return (reason, RemoteTimeLimitException)
467
479
 
468
480
  def _investigate_memory_consumption(
469
- self, properties: dict[str, str]
481
+ self, stdout: str
470
482
  ) -> tuple[str, type[RemoteOutOfMemoryException]] | None:
471
- if not properties.get("Memory Efficiency", None):
472
- return None
473
-
474
- # Extract the "25019.18% of 1.00 GB" part of the line
475
- efficiency_note = properties["Memory Efficiency"]
476
- PERCENTAGE_REGEX = r"([0-9]+(\.[0-9]+)?)%"
477
-
478
- # Extract the percentage to see whether it exceeds 100%.
479
- match = re.search(PERCENTAGE_REGEX, efficiency_note)
480
- percentage = None
481
- if match is None:
482
- return None
483
-
484
- try:
485
- percentage = float(match.group(1))
486
- except ValueError:
487
- return None
488
-
489
- if percentage < 100:
483
+ # Table Format:
484
+ #
485
+ # JobID|State|MaxRSS|ReqMem
486
+ # 91_0|FAILED||30720K
487
+ # 91_0.batch|FAILED|248K|
488
+ # 91_0.0|OUT_OF_MEMORY|164K|
489
+ max_rss = 0
490
+ req_mem = 0
491
+ states = []
492
+ linefilter = re.compile(r"^([^|]*)\|(\w*)\|(\d*)K?\|(\d*)K?$", re.MULTILINE)
493
+ for job_id, state, rss, mem in linefilter.findall(stdout):
494
+ max_rss = max(max_rss, int(rss or 0))
495
+ req_mem = max(req_mem, int(mem or 0))
496
+ states += [state]
497
+
498
+ if "OUT_OF_MEMORY" in states:
499
+ # Check if task plugin killed the job. This is the case if cgroup is used as TaskPlugin and
500
+ # memory limits are enforced.
501
+ reason = f"The job was terminated because it consumed too much memory (Requested: {req_mem / 1000} MB)."
502
+ elif max_rss > req_mem:
503
+ # Check if job accounting canceled the job. This is the case if JobAcctGatherParam=OverMemoryKill
504
+ # is enabled.
505
+ reason = f"The job was probably terminated because it consumed too much memory. Required {max_rss / 1000} MB but requeseted {req_mem / 1000} MB."
506
+ else:
490
507
  return None
491
-
492
- reason = f"The job was probably terminated because it consumed too much memory ({efficiency_note})."
493
508
  return (reason, RemoteOutOfMemoryException)
494
509
 
495
510
  def _investigate_exit_code(
@@ -0,0 +1 @@
1
+ __version__ = '2.3.9'
@@ -1,13 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 2.3.7
3
+ Version: 2.3.9
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
- License: MIT
6
+ License-Expression: MIT
7
7
  Project-URL: Repository, https://github.com/scalableminds/webknossos-libs
8
8
  Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/master/cluster_tools/Changelog.md
9
9
  Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
+ License-File: LICENSE
11
12
  Requires-Dist: typing-extensions~=4.12.0
12
13
  Provides-Extra: kubernetes
13
14
  Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
@@ -16,6 +17,7 @@ Requires-Dist: kubernetes~=27.2.0; extra == "dask"
16
17
  Provides-Extra: all
17
18
  Requires-Dist: cluster_tools[kubernetes]; extra == "all"
18
19
  Requires-Dist: cluster_tools[dask]; extra == "all"
20
+ Dynamic: license-file
19
21
 
20
22
  # Cluster Tools
21
23
 
@@ -1,3 +1,4 @@
1
+ LICENSE
1
2
  README.md
2
3
  pyproject.toml
3
4
  cluster_tools/__init__.py
@@ -1,10 +1,11 @@
1
1
  [project]
2
2
  name = "cluster_tools"
3
- version = "2.3.7" # filled by dunamai
3
+ version = "2.3.9" # filled by dunamai
4
4
  description = "Utility library for easily distributing code execution on clusters"
5
- authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
5
+ authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
6
6
  readme = "README.md"
7
- license = {text = "MIT"}
7
+ license-files = ["LICENSE"]
8
+ license = "MIT"
8
9
  requires-python = ">=3.10"
9
10
  dependencies = [
10
11
  "typing-extensions ~=4.12.0",
@@ -37,11 +38,6 @@ where = ["."]
37
38
  include = ["cluster_tools*"]
38
39
  exclude = ["cluster_tools.tests"]
39
40
 
40
- # This is a fix for an issue in setuptools. See: https://github.com/pypa/setuptools/issues/4759
41
- # This should be removed when the issue is resolved.
42
- [tool.setuptools]
43
- license-files = []
44
-
45
41
  [tool.ruff]
46
42
  # Exclude a variety of commonly ignored directories.
47
43
  exclude = [
@@ -44,12 +44,6 @@ def expect_fork() -> bool:
44
44
  return True
45
45
 
46
46
 
47
- def search_and_replace_in_slurm_config(search_string: str, replace_string: str) -> None:
48
- chcall(
49
- f"sed -ci 's/{search_string}/{replace_string}/g' /etc/slurm/slurm.conf && scontrol reconfigure"
50
- )
51
-
52
-
53
47
  def test_map_with_spawn() -> None:
54
48
  with cluster_tools.get_executor(
55
49
  "slurm", max_workers=5, start_method="spawn"
@@ -273,33 +267,24 @@ def test_slurm_number_of_submitted_jobs() -> None:
273
267
  assert executor.get_number_of_submitted_jobs() == 0
274
268
 
275
269
 
270
+ @pytest.mark.requires_modified_slurm_config
276
271
  def test_slurm_max_array_size() -> None:
277
- max_array_size = 2
272
+ expected_max_array_size = 2
278
273
 
279
274
  executor = cluster_tools.get_executor("slurm", debug=True)
280
- original_max_array_size = executor.get_max_array_size()
281
275
 
282
- command = f"MaxArraySize={max_array_size}"
276
+ max_array_size = executor.get_max_array_size()
277
+ assert max_array_size == expected_max_array_size
283
278
 
284
- try:
285
- chcall(f"echo -e '{command}' >> /etc/slurm/slurm.conf && scontrol reconfigure")
279
+ with executor:
280
+ futures = executor.map_to_futures(square, range(6))
281
+ concurrent.futures.wait(futures)
282
+ job_ids = [fut.cluster_jobid for fut in futures] # type: ignore[attr-defined]
286
283
 
287
- new_max_array_size = executor.get_max_array_size()
288
- assert new_max_array_size == max_array_size
284
+ # Count how often each job_id occurs which corresponds to the array size of the job
285
+ occurrences = list(Counter(job_ids).values())
289
286
 
290
- with executor:
291
- futures = executor.map_to_futures(square, range(6))
292
- concurrent.futures.wait(futures)
293
- job_ids = [fut.cluster_jobid for fut in futures] # type: ignore[attr-defined]
294
-
295
- # Count how often each job_id occurs which corresponds to the array size of the job
296
- occurrences = list(Counter(job_ids).values())
297
-
298
- assert all(array_size <= max_array_size for array_size in occurrences)
299
- finally:
300
- search_and_replace_in_slurm_config(command, "")
301
- reset_max_array_size = executor.get_max_array_size()
302
- assert reset_max_array_size == original_max_array_size
287
+ assert all(array_size <= expected_max_array_size for array_size in occurrences)
303
288
 
304
289
 
305
290
  @pytest.mark.skip(
@@ -323,38 +308,28 @@ def test_slurm_time_limit() -> None:
323
308
  )
324
309
 
325
310
 
311
+ @pytest.mark.requires_modified_slurm_config
326
312
  def test_slurm_memory_limit() -> None:
327
- # Request 1 MB
313
+ # Request 30 MB
328
314
  executor = cluster_tools.get_executor(
329
- "slurm", debug=True, job_resources={"mem": "1M"}
315
+ "slurm",
316
+ debug=True,
317
+ job_resources={"mem": "30M"}, # 30M is the smallest limit enforced by Cgroups
330
318
  )
331
319
 
332
- original_gather_frequency_config = "JobAcctGatherFrequency=30" # from slurm.conf
333
- new_gather_frequency_config = "JobAcctGatherFrequency=1"
334
-
335
- try:
336
- # Increase the frequency at which slurm checks whether a job uses too much memory
337
- search_and_replace_in_slurm_config(
338
- original_gather_frequency_config, new_gather_frequency_config
320
+ with executor:
321
+ # Schedule a job that allocates more than 30 MB and let it run for more than 1 second
322
+ # because the frequency of the memory polling is 1 second
323
+ duration = 3
324
+ futures = executor.map_to_futures(
325
+ partial(allocate, duration), [1024 * 1024 * 50]
339
326
  )
327
+ concurrent.futures.wait(futures)
340
328
 
341
- with executor:
342
- # Schedule a job that allocates more than 1 MB and let it run for more than 1 second
343
- # because the frequency of the memory polling is 1 second
344
- duration = 3
345
- futures = executor.map_to_futures(
346
- partial(allocate, duration), [1024 * 1024 * 2]
347
- )
348
- concurrent.futures.wait(futures)
349
-
350
- # Job should have been killed with a RemoteOutOfMemoryException
351
- assert all(
352
- isinstance(fut.exception(), cluster_tools.RemoteOutOfMemoryException)
353
- for fut in futures
354
- )
355
- finally:
356
- search_and_replace_in_slurm_config(
357
- new_gather_frequency_config, original_gather_frequency_config
329
+ # Job should have been killed with a RemoteOutOfMemoryException
330
+ assert all(
331
+ isinstance(fut.exception(), cluster_tools.RemoteOutOfMemoryException)
332
+ for fut in futures
358
333
  )
359
334
 
360
335
 
@@ -1 +0,0 @@
1
- __version__ = '2.3.7'
File without changes
File without changes