fractal-server 2.14.5__py3-none-any.whl → 2.14.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/db/__init__.py +2 -2
  3. fractal_server/app/models/security.py +8 -8
  4. fractal_server/app/models/user_settings.py +8 -10
  5. fractal_server/app/models/v2/accounting.py +2 -3
  6. fractal_server/app/models/v2/dataset.py +1 -2
  7. fractal_server/app/models/v2/history.py +3 -4
  8. fractal_server/app/models/v2/job.py +10 -11
  9. fractal_server/app/models/v2/project.py +1 -2
  10. fractal_server/app/models/v2/task.py +13 -14
  11. fractal_server/app/models/v2/task_group.py +15 -16
  12. fractal_server/app/models/v2/workflow.py +1 -2
  13. fractal_server/app/models/v2/workflowtask.py +6 -7
  14. fractal_server/app/routes/admin/v2/accounting.py +3 -4
  15. fractal_server/app/routes/admin/v2/job.py +13 -14
  16. fractal_server/app/routes/admin/v2/project.py +2 -4
  17. fractal_server/app/routes/admin/v2/task.py +11 -13
  18. fractal_server/app/routes/admin/v2/task_group.py +15 -17
  19. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +5 -8
  20. fractal_server/app/routes/api/v2/__init__.py +2 -0
  21. fractal_server/app/routes/api/v2/_aux_functions.py +7 -9
  22. fractal_server/app/routes/api/v2/_aux_functions_history.py +1 -1
  23. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +1 -3
  24. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -6
  25. fractal_server/app/routes/api/v2/dataset.py +6 -8
  26. fractal_server/app/routes/api/v2/history.py +5 -8
  27. fractal_server/app/routes/api/v2/images.py +2 -3
  28. fractal_server/app/routes/api/v2/job.py +5 -6
  29. fractal_server/app/routes/api/v2/pre_submission_checks.py +1 -3
  30. fractal_server/app/routes/api/v2/project.py +2 -4
  31. fractal_server/app/routes/api/v2/status_legacy.py +2 -4
  32. fractal_server/app/routes/api/v2/submit.py +3 -4
  33. fractal_server/app/routes/api/v2/task.py +6 -7
  34. fractal_server/app/routes/api/v2/task_collection.py +11 -13
  35. fractal_server/app/routes/api/v2/task_collection_custom.py +4 -4
  36. fractal_server/app/routes/api/v2/task_group.py +6 -8
  37. fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -9
  38. fractal_server/app/routes/api/v2/task_version_update.py +270 -0
  39. fractal_server/app/routes/api/v2/workflow.py +5 -6
  40. fractal_server/app/routes/api/v2/workflow_import.py +3 -5
  41. fractal_server/app/routes/api/v2/workflowtask.py +2 -114
  42. fractal_server/app/routes/auth/current_user.py +2 -2
  43. fractal_server/app/routes/pagination.py +2 -3
  44. fractal_server/app/runner/exceptions.py +15 -16
  45. fractal_server/app/runner/executors/base_runner.py +3 -3
  46. fractal_server/app/runner/executors/call_command_wrapper.py +1 -1
  47. fractal_server/app/runner/executors/local/get_local_config.py +2 -3
  48. fractal_server/app/runner/executors/local/runner.py +1 -1
  49. fractal_server/app/runner/executors/slurm_common/_batching.py +2 -3
  50. fractal_server/app/runner/executors/slurm_common/_slurm_config.py +27 -29
  51. fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py +32 -14
  52. fractal_server/app/runner/executors/slurm_common/get_slurm_config.py +2 -3
  53. fractal_server/app/runner/executors/slurm_common/remote.py +2 -2
  54. fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py +2 -3
  55. fractal_server/app/runner/executors/slurm_ssh/run_subprocess.py +2 -3
  56. fractal_server/app/runner/executors/slurm_ssh/runner.py +5 -4
  57. fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -2
  58. fractal_server/app/runner/executors/slurm_sudo/runner.py +7 -8
  59. fractal_server/app/runner/set_start_and_last_task_index.py +2 -5
  60. fractal_server/app/runner/shutdown.py +5 -11
  61. fractal_server/app/runner/task_files.py +3 -5
  62. fractal_server/app/runner/v2/_local.py +3 -4
  63. fractal_server/app/runner/v2/_slurm_ssh.py +8 -7
  64. fractal_server/app/runner/v2/_slurm_sudo.py +8 -9
  65. fractal_server/app/runner/v2/runner.py +4 -5
  66. fractal_server/app/runner/v2/runner_functions.py +4 -5
  67. fractal_server/app/runner/v2/submit_workflow.py +12 -11
  68. fractal_server/app/runner/v2/task_interface.py +2 -3
  69. fractal_server/app/runner/versions.py +1 -2
  70. fractal_server/app/schemas/user.py +2 -4
  71. fractal_server/app/schemas/user_group.py +1 -2
  72. fractal_server/app/schemas/user_settings.py +19 -21
  73. fractal_server/app/schemas/v2/dataset.py +2 -3
  74. fractal_server/app/schemas/v2/dumps.py +13 -15
  75. fractal_server/app/schemas/v2/history.py +6 -7
  76. fractal_server/app/schemas/v2/job.py +17 -18
  77. fractal_server/app/schemas/v2/manifest.py +12 -13
  78. fractal_server/app/schemas/v2/status_legacy.py +2 -2
  79. fractal_server/app/schemas/v2/task.py +29 -30
  80. fractal_server/app/schemas/v2/task_collection.py +8 -9
  81. fractal_server/app/schemas/v2/task_group.py +22 -23
  82. fractal_server/app/schemas/v2/workflow.py +1 -2
  83. fractal_server/app/schemas/v2/workflowtask.py +27 -29
  84. fractal_server/app/security/__init__.py +10 -12
  85. fractal_server/config.py +32 -33
  86. fractal_server/images/models.py +2 -4
  87. fractal_server/images/tools.py +4 -7
  88. fractal_server/logger.py +3 -5
  89. fractal_server/ssh/_fabric.py +37 -12
  90. fractal_server/string_tools.py +2 -2
  91. fractal_server/syringe.py +1 -1
  92. fractal_server/tasks/v2/local/collect.py +2 -3
  93. fractal_server/tasks/v2/local/deactivate.py +1 -1
  94. fractal_server/tasks/v2/local/reactivate.py +1 -1
  95. fractal_server/tasks/v2/ssh/collect.py +256 -245
  96. fractal_server/tasks/v2/ssh/deactivate.py +210 -187
  97. fractal_server/tasks/v2/ssh/reactivate.py +154 -146
  98. fractal_server/tasks/v2/utils_background.py +2 -3
  99. fractal_server/types/__init__.py +1 -2
  100. fractal_server/types/validators/_filter_validators.py +1 -2
  101. fractal_server/utils.py +4 -5
  102. fractal_server/zip_tools.py +1 -1
  103. {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/METADATA +2 -3
  104. {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/RECORD +107 -107
  105. fractal_server/app/history/__init__.py +0 -0
  106. {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/LICENSE +0 -0
  107. {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/WHEEL +0 -0
  108. {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/entry_points.txt +0 -0
@@ -14,7 +14,6 @@ Submodule to handle the local-backend configuration for a WorkflowTask
14
14
  import json
15
15
  from pathlib import Path
16
16
  from typing import Literal
17
- from typing import Optional
18
17
 
19
18
  from pydantic import BaseModel
20
19
  from pydantic import ConfigDict
@@ -45,7 +44,7 @@ class LocalBackendConfig(BaseModel):
45
44
  """
46
45
 
47
46
  model_config = ConfigDict(extra="forbid")
48
- parallel_tasks_per_job: Optional[int] = None
47
+ parallel_tasks_per_job: int | None = None
49
48
 
50
49
  @property
51
50
  def batch_size(self) -> int:
@@ -55,7 +54,7 @@ class LocalBackendConfig(BaseModel):
55
54
  def get_local_backend_config(
56
55
  wftask: WorkflowTaskV2,
57
56
  which_type: Literal["non_parallel", "parallel"],
58
- config_path: Optional[Path] = None,
57
+ config_path: Path | None = None,
59
58
  tot_tasks: int = 1,
60
59
  ) -> LocalBackendConfig:
61
60
  """
@@ -44,7 +44,7 @@ def run_single_task(
44
44
  )
45
45
 
46
46
  try:
47
- with open(task_files.metadiff_file_local, "r") as f:
47
+ with open(task_files.metadiff_file_local) as f:
48
48
  out_meta = json.load(f)
49
49
  return out_meta
50
50
  except FileNotFoundError:
@@ -12,7 +12,6 @@
12
12
  Submodule to determine the number of total/parallel tasks per SLURM job.
13
13
  """
14
14
  import math
15
- from typing import Optional
16
15
 
17
16
  from fractal_server.logger import set_logger
18
17
 
@@ -58,8 +57,8 @@ def heuristics(
58
57
  # Number of parallel components (always known)
59
58
  tot_tasks: int,
60
59
  # Optional WorkflowTask attributes:
61
- tasks_per_job: Optional[int] = None,
62
- parallel_tasks_per_job: Optional[int] = None,
60
+ tasks_per_job: int | None = None,
61
+ parallel_tasks_per_job: int | None = None,
63
62
  # Task requirements (multiple possible sources):
64
63
  cpus_per_task: int,
65
64
  mem_per_task: int,
@@ -14,8 +14,6 @@ Submodule to handle the SLURM configuration for a WorkflowTask
14
14
  """
15
15
  import json
16
16
  from pathlib import Path
17
- from typing import Optional
18
- from typing import Union
19
17
 
20
18
  from pydantic import BaseModel
21
19
  from pydantic import ConfigDict
@@ -56,16 +54,16 @@ class _SlurmConfigSet(BaseModel):
56
54
 
57
55
  model_config = ConfigDict(extra="forbid")
58
56
 
59
- partition: Optional[str] = None
60
- cpus_per_task: Optional[int] = None
61
- mem: Optional[Union[int, str]] = None
62
- constraint: Optional[str] = None
63
- gres: Optional[str] = None
64
- time: Optional[str] = None
65
- account: Optional[str] = None
66
- extra_lines: Optional[list[str]] = None
67
- pre_submission_commands: Optional[list[str]] = None
68
- gpus: Optional[str] = None
57
+ partition: str | None = None
58
+ cpus_per_task: int | None = None
59
+ mem: int | str | None = None
60
+ constraint: str | None = None
61
+ gres: str | None = None
62
+ time: str | None = None
63
+ account: str | None = None
64
+ extra_lines: list[str] | None = None
65
+ pre_submission_commands: list[str] | None = None
66
+ gpus: str | None = None
69
67
 
70
68
 
71
69
  class _BatchingConfigSet(BaseModel):
@@ -89,8 +87,8 @@ class _BatchingConfigSet(BaseModel):
89
87
 
90
88
  target_cpus_per_job: int
91
89
  max_cpus_per_job: int
92
- target_mem_per_job: Union[int, str]
93
- max_mem_per_job: Union[int, str]
90
+ target_mem_per_job: int | str
91
+ max_mem_per_job: int | str
94
92
  target_num_jobs: int
95
93
  max_num_jobs: int
96
94
 
@@ -143,13 +141,13 @@ class SlurmConfigFile(BaseModel):
143
141
  model_config = ConfigDict(extra="forbid")
144
142
 
145
143
  default_slurm_config: _SlurmConfigSet
146
- gpu_slurm_config: Optional[_SlurmConfigSet] = None
144
+ gpu_slurm_config: _SlurmConfigSet | None = None
147
145
  batching_config: _BatchingConfigSet
148
- user_local_exports: Optional[dict[str, str]] = None
146
+ user_local_exports: dict[str, str] | None = None
149
147
 
150
148
 
151
149
  def load_slurm_config_file(
152
- config_path: Optional[Path] = None,
150
+ config_path: Path | None = None,
153
151
  ) -> SlurmConfigFile:
154
152
  """
155
153
  Load a SLURM configuration file and validate its content with
@@ -264,23 +262,23 @@ class SlurmConfig(BaseModel):
264
262
  shebang_line: str = "#!/bin/sh"
265
263
 
266
264
  # Optional SLURM parameters
267
- job_name: Optional[str] = None
268
- constraint: Optional[str] = None
269
- gres: Optional[str] = None
270
- gpus: Optional[str] = None
271
- time: Optional[str] = None
272
- account: Optional[str] = None
265
+ job_name: str | None = None
266
+ constraint: str | None = None
267
+ gres: str | None = None
268
+ gpus: str | None = None
269
+ time: str | None = None
270
+ account: str | None = None
273
271
 
274
272
  # Free-field attribute for extra lines to be added to the SLURM job
275
273
  # preamble
276
- extra_lines: Optional[list[str]] = Field(default_factory=list)
274
+ extra_lines: list[str] | None = Field(default_factory=list)
277
275
 
278
276
  # Variables that will be `export`ed in the SLURM submission script
279
- user_local_exports: Optional[dict[str, str]] = None
277
+ user_local_exports: dict[str, str] | None = None
280
278
 
281
279
  # Metaparameters needed to combine multiple tasks in each SLURM job
282
- tasks_per_job: Optional[int] = None
283
- parallel_tasks_per_job: Optional[int] = None
280
+ tasks_per_job: int | None = None
281
+ parallel_tasks_per_job: int | None = None
284
282
  target_cpus_per_job: int
285
283
  max_cpus_per_job: int
286
284
  target_mem_per_job: int
@@ -328,7 +326,7 @@ class SlurmConfig(BaseModel):
328
326
 
329
327
  def to_sbatch_preamble(
330
328
  self,
331
- remote_export_dir: Optional[str] = None,
329
+ remote_export_dir: str | None = None,
332
330
  ) -> list[str]:
333
331
  """
334
332
  Compile `SlurmConfig` object into the preamble of a SLURM submission
@@ -410,7 +408,7 @@ class SlurmConfig(BaseModel):
410
408
  return self.tasks_per_job
411
409
 
412
410
 
413
- def _parse_mem_value(raw_mem: Union[str, int]) -> int:
411
+ def _parse_mem_value(raw_mem: str | int) -> int:
414
412
  """
415
413
  Convert a memory-specification string into an integer (in MB units), or
416
414
  simply return the input if it is already an integer.
@@ -5,7 +5,6 @@ import time
5
5
  from pathlib import Path
6
6
  from typing import Any
7
7
  from typing import Literal
8
- from typing import Optional
9
8
 
10
9
  from pydantic import BaseModel
11
10
  from pydantic import ConfigDict
@@ -74,6 +73,7 @@ class BaseSlurmRunner(BaseRunner):
74
73
  jobs: dict[str, SlurmJob]
75
74
  python_worker_interpreter: str
76
75
  slurm_runner_type: Literal["ssh", "sudo"]
76
+ slurm_account: str | None = None
77
77
 
78
78
  def __init__(
79
79
  self,
@@ -81,9 +81,10 @@ class BaseSlurmRunner(BaseRunner):
81
81
  root_dir_remote: Path,
82
82
  slurm_runner_type: Literal["ssh", "sudo"],
83
83
  python_worker_interpreter: str,
84
- common_script_lines: Optional[list[str]] = None,
85
- user_cache_dir: Optional[str] = None,
86
- poll_interval: Optional[int] = None,
84
+ common_script_lines: list[str] | None = None,
85
+ user_cache_dir: str | None = None,
86
+ poll_interval: int | None = None,
87
+ slurm_account: str | None = None,
87
88
  ):
88
89
  self.slurm_runner_type = slurm_runner_type
89
90
  self.root_dir_local = root_dir_local
@@ -92,6 +93,7 @@ class BaseSlurmRunner(BaseRunner):
92
93
  self._check_slurm_account()
93
94
  self.user_cache_dir = user_cache_dir
94
95
  self.python_worker_interpreter = python_worker_interpreter
96
+ self.slurm_account = slurm_account
95
97
 
96
98
  settings = Inject(get_settings)
97
99
 
@@ -186,6 +188,24 @@ class BaseSlurmRunner(BaseRunner):
186
188
  ) -> str:
187
189
  logger.debug("[_submit_single_sbatch] START")
188
190
 
191
+ # Include SLURM account in `slurm_config`. Note: we make this change
192
+ # here, rather than exposing a new argument of `get_slurm_config`,
193
+ # because it's a backend-specific argument while `get_slurm_config` has
194
+ # a generic interface.
195
+ if self.slurm_account is not None:
196
+ slurm_config.account = self.slurm_account
197
+
198
+ # Include common_script_lines in extra_lines
199
+ if len(self.common_script_lines) > 0:
200
+ logger.debug(
201
+ f"Add {self.common_script_lines} to "
202
+ f"{slurm_config.extra_lines=}."
203
+ )
204
+ current_extra_lines = slurm_config.extra_lines or []
205
+ slurm_config.extra_lines = (
206
+ current_extra_lines + self.common_script_lines
207
+ )
208
+
189
209
  for task in slurm_job.tasks:
190
210
  # Write input file
191
211
  if self.slurm_runner_type == "ssh":
@@ -241,13 +261,11 @@ class BaseSlurmRunner(BaseRunner):
241
261
  input_file = task.input_file_local
242
262
  output_file = task.output_file_remote
243
263
  cmdlines.append(
244
- (
245
- f"{self.python_worker_interpreter}"
246
- " -m fractal_server.app.runner."
247
- "executors.slurm_common.remote "
248
- f"--input-file {input_file} "
249
- f"--output-file {output_file}"
250
- )
264
+ f"{self.python_worker_interpreter}"
265
+ " -m fractal_server.app.runner."
266
+ "executors.slurm_common.remote "
267
+ f"--input-file {input_file} "
268
+ f"--output-file {output_file}"
251
269
  )
252
270
 
253
271
  # Set ntasks
@@ -390,7 +408,7 @@ class BaseSlurmRunner(BaseRunner):
390
408
  was_job_scancelled: bool = False,
391
409
  ) -> tuple[Any, Exception]:
392
410
  try:
393
- with open(task.output_file_local, "r") as f:
411
+ with open(task.output_file_local) as f:
394
412
  output = json.load(f)
395
413
  success = output[0]
396
414
  if success:
@@ -885,8 +903,8 @@ class BaseSlurmRunner(BaseRunner):
885
903
  """
886
904
  Check that a list of `SlurmJob`s have homogeneous working folders.
887
905
  """
888
- set_workdir_local = set(_job.workdir_local for _job in slurm_jobs)
889
- set_workdir_remote = set(_job.workdir_remote for _job in slurm_jobs)
906
+ set_workdir_local = {_job.workdir_local for _job in slurm_jobs}
907
+ set_workdir_remote = {_job.workdir_remote for _job in slurm_jobs}
890
908
  if len(set_workdir_local) > 1:
891
909
  raise ValueError(f"Non-unique values in {set_workdir_local=}.")
892
910
  if len(set_workdir_remote) > 1:
@@ -1,6 +1,5 @@
1
1
  from pathlib import Path
2
2
  from typing import Literal
3
- from typing import Optional
4
3
 
5
4
  from ._batching import heuristics
6
5
  from ._slurm_config import _parse_mem_value
@@ -14,7 +13,7 @@ from fractal_server.app.models.v2 import WorkflowTaskV2
14
13
  def get_slurm_config_internal(
15
14
  wftask: WorkflowTaskV2,
16
15
  which_type: Literal["non_parallel", "parallel"],
17
- config_path: Optional[Path] = None,
16
+ config_path: Path | None = None,
18
17
  ) -> SlurmConfig:
19
18
  """
20
19
  Prepare a `SlurmConfig` configuration object
@@ -168,7 +167,7 @@ def get_slurm_config_internal(
168
167
  def get_slurm_config(
169
168
  wftask: WorkflowTaskV2,
170
169
  which_type: Literal["non_parallel", "parallel"],
171
- config_path: Optional[Path] = None,
170
+ config_path: Path | None = None,
172
171
  tot_tasks: int = 1,
173
172
  ) -> SlurmConfig:
174
173
  config = get_slurm_config_internal(
@@ -37,7 +37,7 @@ def worker(
37
37
 
38
38
  # Execute the job and capture exceptions
39
39
  try:
40
- with open(in_fname, "r") as f:
40
+ with open(in_fname) as f:
41
41
  input_data = json.load(f)
42
42
 
43
43
  server_python_version = input_data["python_version"]
@@ -68,7 +68,7 @@ def worker(
68
68
  call_command_wrapper(cmd=full_command, log_path=log_path)
69
69
 
70
70
  try:
71
- with open(metadiff_file_remote, "r") as f:
71
+ with open(metadiff_file_remote) as f:
72
72
  out_meta = json.load(f)
73
73
  result = (True, out_meta)
74
74
  except FileNotFoundError:
@@ -1,6 +1,5 @@
1
1
  from pathlib import Path
2
2
  from typing import Any
3
- from typing import Optional
4
3
 
5
4
  from pydantic import BaseModel
6
5
  from pydantic import ConfigDict
@@ -15,7 +14,7 @@ class SlurmTask(BaseModel):
15
14
  workdir_local: Path
16
15
  workdir_remote: Path
17
16
  parameters: dict[str, Any]
18
- zarr_url: Optional[str] = None
17
+ zarr_url: str | None = None
19
18
  task_files: TaskFiles
20
19
  index: int
21
20
 
@@ -65,7 +64,7 @@ class SlurmTask(BaseModel):
65
64
 
66
65
 
67
66
  class SlurmJob(BaseModel):
68
- slurm_job_id: Optional[str] = None
67
+ slurm_job_id: str | None = None
69
68
  prefix: str
70
69
  workdir_local: Path
71
70
  workdir_remote: Path
@@ -1,6 +1,5 @@
1
1
  import shlex
2
2
  import subprocess # nosec
3
- from typing import Optional
4
3
 
5
4
  from fractal_server.logger import get_logger
6
5
  from fractal_server.string_tools import validate_cmd
@@ -8,8 +7,8 @@ from fractal_server.string_tools import validate_cmd
8
7
 
9
8
  def run_subprocess(
10
9
  cmd: str,
11
- allow_char: Optional[str] = None,
12
- logger_name: Optional[str] = None,
10
+ allow_char: str | None = None,
11
+ logger_name: str | None = None,
13
12
  ) -> subprocess.CompletedProcess:
14
13
  validate_cmd(cmd, allow_char=allow_char)
15
14
  logger = get_logger(logger_name)
@@ -1,6 +1,5 @@
1
1
  import time
2
2
  from pathlib import Path
3
- from typing import Optional
4
3
 
5
4
  from ..slurm_common.base_slurm_runner import BaseSlurmRunner
6
5
  from ..slurm_common.slurm_job_task_models import SlurmJob
@@ -27,10 +26,11 @@ class SlurmSSHRunner(BaseSlurmRunner):
27
26
  # Common
28
27
  root_dir_local: Path,
29
28
  root_dir_remote: Path,
30
- common_script_lines: Optional[list[str]] = None,
31
- user_cache_dir: Optional[str] = None,
32
- poll_interval: Optional[int] = None,
29
+ common_script_lines: list[str] | None = None,
30
+ user_cache_dir: str | None = None,
31
+ poll_interval: int | None = None,
33
32
  # Specific
33
+ slurm_account: str | None = None,
34
34
  fractal_ssh: FractalSSH,
35
35
  ) -> None:
36
36
  """
@@ -50,6 +50,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
50
50
  user_cache_dir=user_cache_dir,
51
51
  poll_interval=poll_interval,
52
52
  python_worker_interpreter=settings.FRACTAL_SLURM_WORKER_PYTHON,
53
+ slurm_account=slurm_account,
53
54
  )
54
55
 
55
56
  def _mkdir_local_folder(self, folder: str) -> None:
@@ -17,7 +17,6 @@ another user. Note that this requires appropriate sudo permissions.
17
17
  """
18
18
  import shlex
19
19
  import subprocess # nosec
20
- from typing import Optional
21
20
 
22
21
  from fractal_server.logger import set_logger
23
22
  from fractal_server.string_tools import validate_cmd
@@ -28,7 +27,7 @@ logger = set_logger(__name__)
28
27
  def _run_command_as_user(
29
28
  *,
30
29
  cmd: str,
31
- user: Optional[str] = None,
30
+ user: str | None = None,
32
31
  check: bool = False,
33
32
  ) -> subprocess.CompletedProcess:
34
33
  """
@@ -5,7 +5,6 @@ import subprocess # nosec
5
5
  import sys
6
6
  from concurrent.futures import ThreadPoolExecutor
7
7
  from pathlib import Path
8
- from typing import Optional
9
8
 
10
9
  from ..slurm_common.base_slurm_runner import BaseSlurmRunner
11
10
  from ..slurm_common.slurm_job_task_models import SlurmJob
@@ -21,7 +20,7 @@ logger = set_logger(__name__)
21
20
 
22
21
  def _subprocess_run_or_raise(
23
22
  full_command: str,
24
- ) -> Optional[subprocess.CompletedProcess]:
23
+ ) -> subprocess.CompletedProcess | None:
25
24
  try:
26
25
  output = subprocess.run( # nosec
27
26
  shlex.split(full_command),
@@ -43,7 +42,7 @@ def _subprocess_run_or_raise(
43
42
 
44
43
  class SudoSlurmRunner(BaseSlurmRunner):
45
44
  slurm_user: str
46
- slurm_account: Optional[str] = None
45
+ slurm_account: str | None = None
47
46
 
48
47
  def __init__(
49
48
  self,
@@ -51,11 +50,11 @@ class SudoSlurmRunner(BaseSlurmRunner):
51
50
  # Common
52
51
  root_dir_local: Path,
53
52
  root_dir_remote: Path,
54
- common_script_lines: Optional[list[str]] = None,
55
- user_cache_dir: Optional[str] = None,
56
- poll_interval: Optional[int] = None,
53
+ common_script_lines: list[str] | None = None,
54
+ user_cache_dir: str | None = None,
55
+ poll_interval: int | None = None,
57
56
  # Specific
58
- slurm_account: Optional[str] = None,
57
+ slurm_account: str | None = None,
59
58
  slurm_user: str,
60
59
  ) -> None:
61
60
  """
@@ -64,7 +63,6 @@ class SudoSlurmRunner(BaseSlurmRunner):
64
63
  """
65
64
 
66
65
  self.slurm_user = slurm_user
67
- self.slurm_account = slurm_account
68
66
  settings = Inject(get_settings)
69
67
 
70
68
  super().__init__(
@@ -77,6 +75,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
77
75
  python_worker_interpreter=(
78
76
  settings.FRACTAL_SLURM_WORKER_PYTHON or sys.executable
79
77
  ),
78
+ slurm_account=slurm_account,
80
79
  )
81
80
 
82
81
  def _mkdir_local_folder(self, folder: str) -> None:
@@ -1,10 +1,7 @@
1
- from typing import Optional
2
-
3
-
4
1
  def set_start_and_last_task_index(
5
2
  num_tasks: int,
6
- first_task_index: Optional[int] = None,
7
- last_task_index: Optional[int] = None,
3
+ first_task_index: int | None = None,
4
+ last_task_index: int | None = None,
8
5
  ) -> tuple[int, int]:
9
6
  """
10
7
  Handle `first_task_index` and `last_task_index`, by setting defaults and
@@ -38,21 +38,15 @@ async def cleanup_after_shutdown(*, jobsV2: list[int], logger_name: str):
38
38
 
39
39
  if len(jobsV2_db) == 0:
40
40
  logger.info(
41
- (
42
- "All jobs associated to this app are "
43
- "either done or failed. Exit."
44
- )
41
+ "All jobs associated to this app are "
42
+ "either done or failed. Exit."
45
43
  )
46
44
  return
47
45
  else:
48
- logger.info(
49
- (f"Some jobs are still 'submitted' " f"{jobsV2_db=}")
50
- )
46
+ logger.info(f"Some jobs are still 'submitted' {jobsV2_db=}")
51
47
  logger.info(
52
- (
53
- "Graceful shutdown reached its maximum time, "
54
- "but some jobs are still submitted"
55
- )
48
+ "Graceful shutdown reached its maximum time, "
49
+ "but some jobs are still submitted"
56
50
  )
57
51
 
58
52
  for job in jobsV2_db:
@@ -1,6 +1,4 @@
1
1
  from pathlib import Path
2
- from typing import Optional
3
- from typing import Union
4
2
 
5
3
  from pydantic import BaseModel
6
4
 
@@ -12,7 +10,7 @@ MULTISUBMIT_PREFIX = "par"
12
10
 
13
11
 
14
12
  def task_subfolder_name(
15
- order: Union[int, str],
13
+ order: int | str,
16
14
  task_name: str,
17
15
  ) -> str:
18
16
  """
@@ -48,8 +46,8 @@ class TaskFiles(BaseModel):
48
46
  task_order: int
49
47
 
50
48
  # Per-single-component
51
- component: Optional[str] = None
52
- prefix: Optional[str] = None
49
+ component: str | None = None
50
+ prefix: str | None = None
53
51
 
54
52
  def _check_component(self):
55
53
  if self.component is None:
@@ -1,5 +1,4 @@
1
1
  from pathlib import Path
2
- from typing import Optional
3
2
 
4
3
  from ...models.v2 import DatasetV2
5
4
  from ...models.v2 import WorkflowV2
@@ -16,9 +15,9 @@ def process_workflow(
16
15
  dataset: DatasetV2,
17
16
  workflow_dir_local: Path,
18
17
  job_id: int,
19
- workflow_dir_remote: Optional[Path] = None,
20
- first_task_index: Optional[int] = None,
21
- last_task_index: Optional[int] = None,
18
+ workflow_dir_remote: Path | None = None,
19
+ first_task_index: int | None = None,
20
+ last_task_index: int | None = None,
22
21
  logger_name: str,
23
22
  job_attribute_filters: AttributeFilters,
24
23
  job_type_filters: dict[str, bool],
@@ -16,7 +16,6 @@ Slurm Backend
16
16
  This backend runs fractal workflows in a SLURM cluster.
17
17
  """
18
18
  from pathlib import Path
19
- from typing import Optional
20
19
 
21
20
  from ....ssh._fabric import FractalSSH
22
21
  from ...models.v2 import DatasetV2
@@ -38,16 +37,17 @@ def process_workflow(
38
37
  dataset: DatasetV2,
39
38
  workflow_dir_local: Path,
40
39
  job_id: int,
41
- workflow_dir_remote: Optional[Path] = None,
42
- first_task_index: Optional[int] = None,
43
- last_task_index: Optional[int] = None,
40
+ workflow_dir_remote: Path | None = None,
41
+ first_task_index: int | None = None,
42
+ last_task_index: int | None = None,
44
43
  logger_name: str,
45
44
  job_attribute_filters: AttributeFilters,
46
45
  job_type_filters: dict[str, bool],
47
- fractal_ssh: FractalSSH,
48
- worker_init: Optional[str] = None,
49
46
  user_id: int,
50
- **kwargs, # not used
47
+ # SLURM-ssh-specific
48
+ fractal_ssh: FractalSSH,
49
+ slurm_account: str | None = None,
50
+ worker_init: str | None = None,
51
51
  ) -> None:
52
52
  """
53
53
  Process workflow (SLURM backend public interface)
@@ -80,6 +80,7 @@ def process_workflow(
80
80
  fractal_ssh=fractal_ssh,
81
81
  root_dir_local=workflow_dir_local,
82
82
  root_dir_remote=workflow_dir_remote,
83
+ slurm_account=slurm_account,
83
84
  common_script_lines=worker_init,
84
85
  ) as runner:
85
86
  execute_tasks_v2(
@@ -16,7 +16,6 @@ Slurm Backend
16
16
  This backend runs fractal workflows in a SLURM cluster.
17
17
  """
18
18
  from pathlib import Path
19
- from typing import Optional
20
19
 
21
20
  from ...models.v2 import DatasetV2
22
21
  from ...models.v2 import WorkflowV2
@@ -33,18 +32,18 @@ def process_workflow(
33
32
  dataset: DatasetV2,
34
33
  workflow_dir_local: Path,
35
34
  job_id: int,
36
- workflow_dir_remote: Optional[Path] = None,
37
- first_task_index: Optional[int] = None,
38
- last_task_index: Optional[int] = None,
35
+ workflow_dir_remote: Path | None = None,
36
+ first_task_index: int | None = None,
37
+ last_task_index: int | None = None,
39
38
  logger_name: str,
40
39
  job_attribute_filters: AttributeFilters,
41
40
  job_type_filters: dict[str, bool],
42
41
  user_id: int,
43
- # Slurm-specific
44
- user_cache_dir: Optional[str] = None,
45
- slurm_user: Optional[str] = None,
46
- slurm_account: Optional[str] = None,
47
- worker_init: Optional[str] = None,
42
+ # SLURM-sudo-specific
43
+ user_cache_dir: str | None = None,
44
+ slurm_user: str | None = None,
45
+ slurm_account: str | None = None,
46
+ worker_init: str | None = None,
48
47
  ) -> None:
49
48
  """
50
49
  Process workflow (SLURM backend public interface).
@@ -1,11 +1,10 @@
1
1
  import logging
2
+ from collections.abc import Callable
2
3
  from copy import copy
3
4
  from copy import deepcopy
4
5
  from pathlib import Path
5
6
  from typing import Any
6
- from typing import Callable
7
7
  from typing import Literal
8
- from typing import Optional
9
8
 
10
9
  from sqlalchemy.orm.attributes import flag_modified
11
10
  from sqlmodel import delete
@@ -76,13 +75,13 @@ def execute_tasks_v2(
76
75
  user_id: int,
77
76
  workflow_dir_local: Path,
78
77
  job_id: int,
79
- workflow_dir_remote: Optional[Path] = None,
80
- logger_name: Optional[str] = None,
78
+ workflow_dir_remote: Path | None = None,
79
+ logger_name: str | None = None,
81
80
  get_runner_config: Callable[
82
81
  [
83
82
  WorkflowTaskV2,
84
83
  Literal["non_parallel", "parallel"],
85
- Optional[Path],
84
+ Path | None,
86
85
  ],
87
86
  Any,
88
87
  ],