cluster-tools 3.5.2__tar.gz → 3.5.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/PKG-INFO +2 -2
  2. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/cluster_executor.py +24 -6
  3. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/slurm.py +28 -1
  4. cluster_tools-3.5.4/cluster_tools/version.py +1 -0
  5. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/PKG-INFO +2 -2
  6. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/requires.txt +1 -1
  7. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/pyproject.toml +3 -3
  8. cluster_tools-3.5.2/cluster_tools/version.py +0 -1
  9. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/LICENSE +0 -0
  10. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/README.md +0 -0
  11. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/__init__.py +0 -0
  12. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/call.py +0 -0
  13. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/file_wait_thread.py +0 -0
  14. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/multiprocessing_logging_handler.py +0 -0
  15. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/pickling.py +0 -0
  16. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/reflection.py +0 -0
  17. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/string_.py +0 -0
  18. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/tailf.py +0 -0
  19. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/_utils/warning.py +0 -0
  20. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executor_protocol.py +0 -0
  21. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/batching.py +0 -0
  22. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/dask.py +0 -0
  23. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/multiprocessing_.py +0 -0
  24. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
  25. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/sequential.py +0 -0
  26. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/executors/sequential_pickle.py +0 -0
  27. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/remote.py +0 -0
  28. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/__init__.py +0 -0
  29. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/kube.py +0 -0
  30. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools/schedulers/pbs.py +0 -0
  31. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/SOURCES.txt +0 -0
  32. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/dependency_links.txt +0 -0
  33. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/cluster_tools.egg-info/top_level.txt +0 -0
  34. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/setup.cfg +0 -0
  35. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_all.py +0 -0
  36. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_batching.py +0 -0
  37. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_dask.py +0 -0
  38. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_deref_main.py +0 -0
  39. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_kubernetes.py +0 -0
  40. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_multiprocessing.py +0 -0
  41. {cluster_tools-3.5.2 → cluster_tools-3.5.4}/tests/test_slurm.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 3.5.2
3
+ Version: 3.5.4
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
6
  License-Expression: MIT
@@ -9,7 +9,7 @@ Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/ma
9
9
  Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
- Requires-Dist: typing-extensions~=4.12.0
12
+ Requires-Dist: typing-extensions~=4.15.0
13
13
  Provides-Extra: dask
14
14
  Requires-Dist: distributed~=2023.9.1; extra == "dask"
15
15
  Provides-Extra: kubernetes
@@ -1,5 +1,6 @@
1
1
  import atexit
2
2
  import logging
3
+ import math
3
4
  import os
4
5
  import signal
5
6
  import sys
@@ -38,6 +39,9 @@ _T = TypeVar("_T")
38
39
  _P = ParamSpec("_P")
39
40
  _S = TypeVar("_S")
40
41
 
42
+ LOG_FILE_POLLING_INTERVAL_SECONDS = 2
43
+ MAX_LOG_FILE_POLLING_DURATION_SECONDS = 120
44
+
41
45
 
42
46
  def join_messages(strings: list[str]) -> str:
43
47
  return " ".join(x.strip() for x in strings if x.strip())
@@ -776,13 +780,27 @@ class ClusterExecutor(futures.Executor):
776
780
  tailer = Tail(log_path, log_callback)
777
781
  fut.add_done_callback(lambda _: tailer.cancel())
778
782
 
779
- # Poll until the log file exists
780
- while not (os.path.exists(log_path) or tailer.is_cancelled):
781
- time.sleep(2)
783
+ # Wait until the log file exists or the job is completed/cancelled
784
+ log_file_exists = False
785
+ while not (
786
+ (log_file_exists := os.path.exists(log_path)) or tailer.is_cancelled
787
+ ):
788
+ time.sleep(LOG_FILE_POLLING_INTERVAL_SECONDS)
782
789
 
783
- # Log the output of the log file until future is resolved
784
- # by the done_callback we attached earlier.
785
- tailer.follow(2)
790
+ # The job might be done/cancelled but the log file might still not exist (e.g. slow/inconsistent NFS)
791
+ # In this case, we keep polling but with an upper limit to not run into a deadlock.
792
+ retries = 0
793
+ max_retries = math.ceil(
794
+ MAX_LOG_FILE_POLLING_DURATION_SECONDS / LOG_FILE_POLLING_INTERVAL_SECONDS
795
+ )
796
+ while not log_file_exists and retries < max_retries:
797
+ log_file_exists = os.path.exists(log_path)
798
+ retries += 1
799
+ sys.stdout.write(
800
+ f"Job with id {fut.cluster_jobid} is finished but log file couldn't be found at {log_path}. Retrying {retries}/{max_retries}" # type: ignore[attr-defined]
801
+ )
802
+ time.sleep(LOG_FILE_POLLING_INTERVAL_SECONDS)
803
+ tailer.follow(LOG_FILE_POLLING_INTERVAL_SECONDS)
786
804
  return fut.result()
787
805
 
788
806
  @abstractmethod
@@ -54,6 +54,27 @@ SLURM_STATES = {
54
54
  "Unclear": ["SUSPENDED", "REVOKED", "SIGNALING", "SPECIAL_EXIT", "STAGE_OUT"],
55
55
  }
56
56
 
57
+ # Since Slurm 22.05, some resource flags are not automatically inherited from the
58
+ # enclosing sbatch job by srun calls within the script anymore and must be restated
59
+ # explicitly, or srun computes its own (potentially conflicting) resource request
60
+ # leading to failures like:
61
+ # "srun: fatal: cpus-per-task set by two different environment variables ..."
62
+ # Other job_resources (e.g. --mem, --gres) should be inherited by srun automatically.
63
+ # Therefore, we currently explicitely restate potentially non-inherited resource for the srun call.
64
+ # This is most relevant for cpus-per-task, but we have added some others to be on the safe side.
65
+ # It is worth noting that this change was quite disruptive, particularly for MPI implementations.
66
+ # Because of this, in Slurm 23.11, this requirement was largely walked back.
67
+ # SRUN_CPUS_PER_TASK war reverted back to SLURM_CPUS_PER_TASK and an --external-launcher option was introduced to srun
68
+ # to prevent task launchers from being bound to a single CPU.
69
+ SLURM_SRUN_NON_INHERITED_RESOURCE_KEYS = {
70
+ "cpus-per-task",
71
+ "ntasks",
72
+ "gpus",
73
+ "gpus-per-node",
74
+ "gpus-per-socket",
75
+ "gpus-per-task",
76
+ }
77
+
57
78
  SLURM_QUEUE_CHECK_INTERVAL = 1 if "pytest" in sys.modules else 60
58
79
 
59
80
  T = TypeVar("T")
@@ -303,9 +324,15 @@ class SlurmExecutor(ClusterExecutor):
303
324
  log_path = self.format_log_file_path(self.cfut_dir, job_id_string)
304
325
 
305
326
  job_resources_lines = []
327
+ srun_resource_args = []
306
328
  if self.job_resources is not None:
307
329
  for resource, value in self.job_resources.items():
308
330
  job_resources_lines += [f"#SBATCH --{resource}={value}"]
331
+ if resource in SLURM_SRUN_NON_INHERITED_RESOURCE_KEYS:
332
+ srun_resource_args += [f"--{resource}={value}"]
333
+ srun_resource_args_str = (
334
+ " " + " ".join(srun_resource_args) if srun_resource_args else ""
335
+ )
309
336
 
310
337
  max_array_size = self.get_max_array_size()
311
338
  max_submit_jobs = self.get_max_submit_jobs()
@@ -340,7 +367,7 @@ class SlurmExecutor(ClusterExecutor):
340
367
  + job_resources_lines
341
368
  + [
342
369
  *additional_setup_lines,
343
- f"srun {cmdline} {job_index_start}",
370
+ f"srun{srun_resource_args_str} {cmdline} {job_index_start}",
344
371
  ]
345
372
  )
346
373
 
@@ -0,0 +1 @@
1
+ __version__ = '3.5.4'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 3.5.2
3
+ Version: 3.5.4
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
6
  License-Expression: MIT
@@ -9,7 +9,7 @@ Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/ma
9
9
  Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
- Requires-Dist: typing-extensions~=4.12.0
12
+ Requires-Dist: typing-extensions~=4.15.0
13
13
  Provides-Extra: dask
14
14
  Requires-Dist: distributed~=2023.9.1; extra == "dask"
15
15
  Provides-Extra: kubernetes
@@ -1,4 +1,4 @@
1
- typing-extensions~=4.12.0
1
+ typing-extensions~=4.15.0
2
2
 
3
3
  [all]
4
4
  cluster_tools[kubernetes]
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cluster_tools"
3
- version = "3.5.2" # filled by dunamai
3
+ version = "3.5.4" # filled by dunamai
4
4
  description = "Utility library for easily distributing code execution on clusters"
5
5
  authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
6
6
  readme = "README.md"
@@ -8,7 +8,7 @@ license-files = ["LICENSE"]
8
8
  license = "MIT"
9
9
  requires-python = ">=3.10"
10
10
  dependencies = [
11
- "typing-extensions ~=4.12.0",
11
+ "typing-extensions ~=4.15.0",
12
12
  ]
13
13
 
14
14
  [project.urls]
@@ -24,7 +24,7 @@ all=["cluster_tools[kubernetes]", "cluster_tools[dask]"]
24
24
  [dependency-groups]
25
25
  dev = [
26
26
  "icecream ~=2.1.1",
27
- "mypy ~=1.15.0",
27
+ "mypy ~=2.1.0",
28
28
  "pytest ~=8.3.3",
29
29
  "pytest-timeout>=2.4.0",
30
30
  "ruff ~=0.11.0",
@@ -1 +0,0 @@
1
- __version__ = '3.5.2'
File without changes
File without changes
File without changes