fractal-server 2.14.5__py3-none-any.whl → 2.14.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/db/__init__.py +2 -2
- fractal_server/app/models/security.py +8 -8
- fractal_server/app/models/user_settings.py +8 -10
- fractal_server/app/models/v2/accounting.py +2 -3
- fractal_server/app/models/v2/dataset.py +1 -2
- fractal_server/app/models/v2/history.py +3 -4
- fractal_server/app/models/v2/job.py +10 -11
- fractal_server/app/models/v2/project.py +1 -2
- fractal_server/app/models/v2/task.py +13 -14
- fractal_server/app/models/v2/task_group.py +15 -16
- fractal_server/app/models/v2/workflow.py +1 -2
- fractal_server/app/models/v2/workflowtask.py +6 -7
- fractal_server/app/routes/admin/v2/accounting.py +3 -4
- fractal_server/app/routes/admin/v2/job.py +13 -14
- fractal_server/app/routes/admin/v2/project.py +2 -4
- fractal_server/app/routes/admin/v2/task.py +11 -13
- fractal_server/app/routes/admin/v2/task_group.py +15 -17
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +5 -8
- fractal_server/app/routes/api/v2/__init__.py +2 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +7 -9
- fractal_server/app/routes/api/v2/_aux_functions_history.py +1 -1
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +1 -3
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -6
- fractal_server/app/routes/api/v2/dataset.py +6 -8
- fractal_server/app/routes/api/v2/history.py +5 -8
- fractal_server/app/routes/api/v2/images.py +2 -3
- fractal_server/app/routes/api/v2/job.py +5 -6
- fractal_server/app/routes/api/v2/pre_submission_checks.py +1 -3
- fractal_server/app/routes/api/v2/project.py +2 -4
- fractal_server/app/routes/api/v2/status_legacy.py +2 -4
- fractal_server/app/routes/api/v2/submit.py +3 -4
- fractal_server/app/routes/api/v2/task.py +6 -7
- fractal_server/app/routes/api/v2/task_collection.py +11 -13
- fractal_server/app/routes/api/v2/task_collection_custom.py +4 -4
- fractal_server/app/routes/api/v2/task_group.py +6 -8
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -9
- fractal_server/app/routes/api/v2/task_version_update.py +270 -0
- fractal_server/app/routes/api/v2/workflow.py +5 -6
- fractal_server/app/routes/api/v2/workflow_import.py +3 -5
- fractal_server/app/routes/api/v2/workflowtask.py +2 -114
- fractal_server/app/routes/auth/current_user.py +2 -2
- fractal_server/app/routes/pagination.py +2 -3
- fractal_server/app/runner/exceptions.py +15 -16
- fractal_server/app/runner/executors/base_runner.py +3 -3
- fractal_server/app/runner/executors/call_command_wrapper.py +1 -1
- fractal_server/app/runner/executors/local/get_local_config.py +2 -3
- fractal_server/app/runner/executors/local/runner.py +1 -1
- fractal_server/app/runner/executors/slurm_common/_batching.py +2 -3
- fractal_server/app/runner/executors/slurm_common/_slurm_config.py +27 -29
- fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py +32 -14
- fractal_server/app/runner/executors/slurm_common/get_slurm_config.py +2 -3
- fractal_server/app/runner/executors/slurm_common/remote.py +2 -2
- fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py +2 -3
- fractal_server/app/runner/executors/slurm_ssh/run_subprocess.py +2 -3
- fractal_server/app/runner/executors/slurm_ssh/runner.py +5 -4
- fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -2
- fractal_server/app/runner/executors/slurm_sudo/runner.py +7 -8
- fractal_server/app/runner/set_start_and_last_task_index.py +2 -5
- fractal_server/app/runner/shutdown.py +5 -11
- fractal_server/app/runner/task_files.py +3 -5
- fractal_server/app/runner/v2/_local.py +3 -4
- fractal_server/app/runner/v2/_slurm_ssh.py +8 -7
- fractal_server/app/runner/v2/_slurm_sudo.py +8 -9
- fractal_server/app/runner/v2/runner.py +4 -5
- fractal_server/app/runner/v2/runner_functions.py +4 -5
- fractal_server/app/runner/v2/submit_workflow.py +12 -11
- fractal_server/app/runner/v2/task_interface.py +2 -3
- fractal_server/app/runner/versions.py +1 -2
- fractal_server/app/schemas/user.py +2 -4
- fractal_server/app/schemas/user_group.py +1 -2
- fractal_server/app/schemas/user_settings.py +19 -21
- fractal_server/app/schemas/v2/dataset.py +2 -3
- fractal_server/app/schemas/v2/dumps.py +13 -15
- fractal_server/app/schemas/v2/history.py +6 -7
- fractal_server/app/schemas/v2/job.py +17 -18
- fractal_server/app/schemas/v2/manifest.py +12 -13
- fractal_server/app/schemas/v2/status_legacy.py +2 -2
- fractal_server/app/schemas/v2/task.py +29 -30
- fractal_server/app/schemas/v2/task_collection.py +8 -9
- fractal_server/app/schemas/v2/task_group.py +22 -23
- fractal_server/app/schemas/v2/workflow.py +1 -2
- fractal_server/app/schemas/v2/workflowtask.py +27 -29
- fractal_server/app/security/__init__.py +10 -12
- fractal_server/config.py +32 -33
- fractal_server/images/models.py +2 -4
- fractal_server/images/tools.py +4 -7
- fractal_server/logger.py +3 -5
- fractal_server/ssh/_fabric.py +37 -12
- fractal_server/string_tools.py +2 -2
- fractal_server/syringe.py +1 -1
- fractal_server/tasks/v2/local/collect.py +2 -3
- fractal_server/tasks/v2/local/deactivate.py +1 -1
- fractal_server/tasks/v2/local/reactivate.py +1 -1
- fractal_server/tasks/v2/ssh/collect.py +256 -245
- fractal_server/tasks/v2/ssh/deactivate.py +210 -187
- fractal_server/tasks/v2/ssh/reactivate.py +154 -146
- fractal_server/tasks/v2/utils_background.py +2 -3
- fractal_server/types/__init__.py +1 -2
- fractal_server/types/validators/_filter_validators.py +1 -2
- fractal_server/utils.py +4 -5
- fractal_server/zip_tools.py +1 -1
- {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/METADATA +2 -3
- {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/RECORD +107 -107
- fractal_server/app/history/__init__.py +0 -0
- {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/LICENSE +0 -0
- {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/WHEEL +0 -0
- {fractal_server-2.14.5.dist-info → fractal_server-2.14.7.dist-info}/entry_points.txt +0 -0
@@ -14,7 +14,6 @@ Submodule to handle the local-backend configuration for a WorkflowTask
|
|
14
14
|
import json
|
15
15
|
from pathlib import Path
|
16
16
|
from typing import Literal
|
17
|
-
from typing import Optional
|
18
17
|
|
19
18
|
from pydantic import BaseModel
|
20
19
|
from pydantic import ConfigDict
|
@@ -45,7 +44,7 @@ class LocalBackendConfig(BaseModel):
|
|
45
44
|
"""
|
46
45
|
|
47
46
|
model_config = ConfigDict(extra="forbid")
|
48
|
-
parallel_tasks_per_job:
|
47
|
+
parallel_tasks_per_job: int | None = None
|
49
48
|
|
50
49
|
@property
|
51
50
|
def batch_size(self) -> int:
|
@@ -55,7 +54,7 @@ class LocalBackendConfig(BaseModel):
|
|
55
54
|
def get_local_backend_config(
|
56
55
|
wftask: WorkflowTaskV2,
|
57
56
|
which_type: Literal["non_parallel", "parallel"],
|
58
|
-
config_path:
|
57
|
+
config_path: Path | None = None,
|
59
58
|
tot_tasks: int = 1,
|
60
59
|
) -> LocalBackendConfig:
|
61
60
|
"""
|
@@ -12,7 +12,6 @@
|
|
12
12
|
Submodule to determine the number of total/parallel tasks per SLURM job.
|
13
13
|
"""
|
14
14
|
import math
|
15
|
-
from typing import Optional
|
16
15
|
|
17
16
|
from fractal_server.logger import set_logger
|
18
17
|
|
@@ -58,8 +57,8 @@ def heuristics(
|
|
58
57
|
# Number of parallel components (always known)
|
59
58
|
tot_tasks: int,
|
60
59
|
# Optional WorkflowTask attributes:
|
61
|
-
tasks_per_job:
|
62
|
-
parallel_tasks_per_job:
|
60
|
+
tasks_per_job: int | None = None,
|
61
|
+
parallel_tasks_per_job: int | None = None,
|
63
62
|
# Task requirements (multiple possible sources):
|
64
63
|
cpus_per_task: int,
|
65
64
|
mem_per_task: int,
|
@@ -14,8 +14,6 @@ Submodule to handle the SLURM configuration for a WorkflowTask
|
|
14
14
|
"""
|
15
15
|
import json
|
16
16
|
from pathlib import Path
|
17
|
-
from typing import Optional
|
18
|
-
from typing import Union
|
19
17
|
|
20
18
|
from pydantic import BaseModel
|
21
19
|
from pydantic import ConfigDict
|
@@ -56,16 +54,16 @@ class _SlurmConfigSet(BaseModel):
|
|
56
54
|
|
57
55
|
model_config = ConfigDict(extra="forbid")
|
58
56
|
|
59
|
-
partition:
|
60
|
-
cpus_per_task:
|
61
|
-
mem:
|
62
|
-
constraint:
|
63
|
-
gres:
|
64
|
-
time:
|
65
|
-
account:
|
66
|
-
extra_lines:
|
67
|
-
pre_submission_commands:
|
68
|
-
gpus:
|
57
|
+
partition: str | None = None
|
58
|
+
cpus_per_task: int | None = None
|
59
|
+
mem: int | str | None = None
|
60
|
+
constraint: str | None = None
|
61
|
+
gres: str | None = None
|
62
|
+
time: str | None = None
|
63
|
+
account: str | None = None
|
64
|
+
extra_lines: list[str] | None = None
|
65
|
+
pre_submission_commands: list[str] | None = None
|
66
|
+
gpus: str | None = None
|
69
67
|
|
70
68
|
|
71
69
|
class _BatchingConfigSet(BaseModel):
|
@@ -89,8 +87,8 @@ class _BatchingConfigSet(BaseModel):
|
|
89
87
|
|
90
88
|
target_cpus_per_job: int
|
91
89
|
max_cpus_per_job: int
|
92
|
-
target_mem_per_job:
|
93
|
-
max_mem_per_job:
|
90
|
+
target_mem_per_job: int | str
|
91
|
+
max_mem_per_job: int | str
|
94
92
|
target_num_jobs: int
|
95
93
|
max_num_jobs: int
|
96
94
|
|
@@ -143,13 +141,13 @@ class SlurmConfigFile(BaseModel):
|
|
143
141
|
model_config = ConfigDict(extra="forbid")
|
144
142
|
|
145
143
|
default_slurm_config: _SlurmConfigSet
|
146
|
-
gpu_slurm_config:
|
144
|
+
gpu_slurm_config: _SlurmConfigSet | None = None
|
147
145
|
batching_config: _BatchingConfigSet
|
148
|
-
user_local_exports:
|
146
|
+
user_local_exports: dict[str, str] | None = None
|
149
147
|
|
150
148
|
|
151
149
|
def load_slurm_config_file(
|
152
|
-
config_path:
|
150
|
+
config_path: Path | None = None,
|
153
151
|
) -> SlurmConfigFile:
|
154
152
|
"""
|
155
153
|
Load a SLURM configuration file and validate its content with
|
@@ -264,23 +262,23 @@ class SlurmConfig(BaseModel):
|
|
264
262
|
shebang_line: str = "#!/bin/sh"
|
265
263
|
|
266
264
|
# Optional SLURM parameters
|
267
|
-
job_name:
|
268
|
-
constraint:
|
269
|
-
gres:
|
270
|
-
gpus:
|
271
|
-
time:
|
272
|
-
account:
|
265
|
+
job_name: str | None = None
|
266
|
+
constraint: str | None = None
|
267
|
+
gres: str | None = None
|
268
|
+
gpus: str | None = None
|
269
|
+
time: str | None = None
|
270
|
+
account: str | None = None
|
273
271
|
|
274
272
|
# Free-field attribute for extra lines to be added to the SLURM job
|
275
273
|
# preamble
|
276
|
-
extra_lines:
|
274
|
+
extra_lines: list[str] | None = Field(default_factory=list)
|
277
275
|
|
278
276
|
# Variables that will be `export`ed in the SLURM submission script
|
279
|
-
user_local_exports:
|
277
|
+
user_local_exports: dict[str, str] | None = None
|
280
278
|
|
281
279
|
# Metaparameters needed to combine multiple tasks in each SLURM job
|
282
|
-
tasks_per_job:
|
283
|
-
parallel_tasks_per_job:
|
280
|
+
tasks_per_job: int | None = None
|
281
|
+
parallel_tasks_per_job: int | None = None
|
284
282
|
target_cpus_per_job: int
|
285
283
|
max_cpus_per_job: int
|
286
284
|
target_mem_per_job: int
|
@@ -328,7 +326,7 @@ class SlurmConfig(BaseModel):
|
|
328
326
|
|
329
327
|
def to_sbatch_preamble(
|
330
328
|
self,
|
331
|
-
remote_export_dir:
|
329
|
+
remote_export_dir: str | None = None,
|
332
330
|
) -> list[str]:
|
333
331
|
"""
|
334
332
|
Compile `SlurmConfig` object into the preamble of a SLURM submission
|
@@ -410,7 +408,7 @@ class SlurmConfig(BaseModel):
|
|
410
408
|
return self.tasks_per_job
|
411
409
|
|
412
410
|
|
413
|
-
def _parse_mem_value(raw_mem:
|
411
|
+
def _parse_mem_value(raw_mem: str | int) -> int:
|
414
412
|
"""
|
415
413
|
Convert a memory-specification string into an integer (in MB units), or
|
416
414
|
simply return the input if it is already an integer.
|
@@ -5,7 +5,6 @@ import time
|
|
5
5
|
from pathlib import Path
|
6
6
|
from typing import Any
|
7
7
|
from typing import Literal
|
8
|
-
from typing import Optional
|
9
8
|
|
10
9
|
from pydantic import BaseModel
|
11
10
|
from pydantic import ConfigDict
|
@@ -74,6 +73,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
74
73
|
jobs: dict[str, SlurmJob]
|
75
74
|
python_worker_interpreter: str
|
76
75
|
slurm_runner_type: Literal["ssh", "sudo"]
|
76
|
+
slurm_account: str | None = None
|
77
77
|
|
78
78
|
def __init__(
|
79
79
|
self,
|
@@ -81,9 +81,10 @@ class BaseSlurmRunner(BaseRunner):
|
|
81
81
|
root_dir_remote: Path,
|
82
82
|
slurm_runner_type: Literal["ssh", "sudo"],
|
83
83
|
python_worker_interpreter: str,
|
84
|
-
common_script_lines:
|
85
|
-
user_cache_dir:
|
86
|
-
poll_interval:
|
84
|
+
common_script_lines: list[str] | None = None,
|
85
|
+
user_cache_dir: str | None = None,
|
86
|
+
poll_interval: int | None = None,
|
87
|
+
slurm_account: str | None = None,
|
87
88
|
):
|
88
89
|
self.slurm_runner_type = slurm_runner_type
|
89
90
|
self.root_dir_local = root_dir_local
|
@@ -92,6 +93,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
92
93
|
self._check_slurm_account()
|
93
94
|
self.user_cache_dir = user_cache_dir
|
94
95
|
self.python_worker_interpreter = python_worker_interpreter
|
96
|
+
self.slurm_account = slurm_account
|
95
97
|
|
96
98
|
settings = Inject(get_settings)
|
97
99
|
|
@@ -186,6 +188,24 @@ class BaseSlurmRunner(BaseRunner):
|
|
186
188
|
) -> str:
|
187
189
|
logger.debug("[_submit_single_sbatch] START")
|
188
190
|
|
191
|
+
# Include SLURM account in `slurm_config`. Note: we make this change
|
192
|
+
# here, rather than exposing a new argument of `get_slurm_config`,
|
193
|
+
# because it's a backend-specific argument while `get_slurm_config` has
|
194
|
+
# a generic interface.
|
195
|
+
if self.slurm_account is not None:
|
196
|
+
slurm_config.account = self.slurm_account
|
197
|
+
|
198
|
+
# Include common_script_lines in extra_lines
|
199
|
+
if len(self.common_script_lines) > 0:
|
200
|
+
logger.debug(
|
201
|
+
f"Add {self.common_script_lines} to "
|
202
|
+
f"{slurm_config.extra_lines=}."
|
203
|
+
)
|
204
|
+
current_extra_lines = slurm_config.extra_lines or []
|
205
|
+
slurm_config.extra_lines = (
|
206
|
+
current_extra_lines + self.common_script_lines
|
207
|
+
)
|
208
|
+
|
189
209
|
for task in slurm_job.tasks:
|
190
210
|
# Write input file
|
191
211
|
if self.slurm_runner_type == "ssh":
|
@@ -241,13 +261,11 @@ class BaseSlurmRunner(BaseRunner):
|
|
241
261
|
input_file = task.input_file_local
|
242
262
|
output_file = task.output_file_remote
|
243
263
|
cmdlines.append(
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
f"--output-file {output_file}"
|
250
|
-
)
|
264
|
+
f"{self.python_worker_interpreter}"
|
265
|
+
" -m fractal_server.app.runner."
|
266
|
+
"executors.slurm_common.remote "
|
267
|
+
f"--input-file {input_file} "
|
268
|
+
f"--output-file {output_file}"
|
251
269
|
)
|
252
270
|
|
253
271
|
# Set ntasks
|
@@ -390,7 +408,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
390
408
|
was_job_scancelled: bool = False,
|
391
409
|
) -> tuple[Any, Exception]:
|
392
410
|
try:
|
393
|
-
with open(task.output_file_local
|
411
|
+
with open(task.output_file_local) as f:
|
394
412
|
output = json.load(f)
|
395
413
|
success = output[0]
|
396
414
|
if success:
|
@@ -885,8 +903,8 @@ class BaseSlurmRunner(BaseRunner):
|
|
885
903
|
"""
|
886
904
|
Check that a list of `SlurmJob`s have homogeneous working folders.
|
887
905
|
"""
|
888
|
-
set_workdir_local =
|
889
|
-
set_workdir_remote =
|
906
|
+
set_workdir_local = {_job.workdir_local for _job in slurm_jobs}
|
907
|
+
set_workdir_remote = {_job.workdir_remote for _job in slurm_jobs}
|
890
908
|
if len(set_workdir_local) > 1:
|
891
909
|
raise ValueError(f"Non-unique values in {set_workdir_local=}.")
|
892
910
|
if len(set_workdir_remote) > 1:
|
@@ -1,6 +1,5 @@
|
|
1
1
|
from pathlib import Path
|
2
2
|
from typing import Literal
|
3
|
-
from typing import Optional
|
4
3
|
|
5
4
|
from ._batching import heuristics
|
6
5
|
from ._slurm_config import _parse_mem_value
|
@@ -14,7 +13,7 @@ from fractal_server.app.models.v2 import WorkflowTaskV2
|
|
14
13
|
def get_slurm_config_internal(
|
15
14
|
wftask: WorkflowTaskV2,
|
16
15
|
which_type: Literal["non_parallel", "parallel"],
|
17
|
-
config_path:
|
16
|
+
config_path: Path | None = None,
|
18
17
|
) -> SlurmConfig:
|
19
18
|
"""
|
20
19
|
Prepare a `SlurmConfig` configuration object
|
@@ -168,7 +167,7 @@ def get_slurm_config_internal(
|
|
168
167
|
def get_slurm_config(
|
169
168
|
wftask: WorkflowTaskV2,
|
170
169
|
which_type: Literal["non_parallel", "parallel"],
|
171
|
-
config_path:
|
170
|
+
config_path: Path | None = None,
|
172
171
|
tot_tasks: int = 1,
|
173
172
|
) -> SlurmConfig:
|
174
173
|
config = get_slurm_config_internal(
|
@@ -37,7 +37,7 @@ def worker(
|
|
37
37
|
|
38
38
|
# Execute the job and capture exceptions
|
39
39
|
try:
|
40
|
-
with open(in_fname
|
40
|
+
with open(in_fname) as f:
|
41
41
|
input_data = json.load(f)
|
42
42
|
|
43
43
|
server_python_version = input_data["python_version"]
|
@@ -68,7 +68,7 @@ def worker(
|
|
68
68
|
call_command_wrapper(cmd=full_command, log_path=log_path)
|
69
69
|
|
70
70
|
try:
|
71
|
-
with open(metadiff_file_remote
|
71
|
+
with open(metadiff_file_remote) as f:
|
72
72
|
out_meta = json.load(f)
|
73
73
|
result = (True, out_meta)
|
74
74
|
except FileNotFoundError:
|
@@ -1,6 +1,5 @@
|
|
1
1
|
from pathlib import Path
|
2
2
|
from typing import Any
|
3
|
-
from typing import Optional
|
4
3
|
|
5
4
|
from pydantic import BaseModel
|
6
5
|
from pydantic import ConfigDict
|
@@ -15,7 +14,7 @@ class SlurmTask(BaseModel):
|
|
15
14
|
workdir_local: Path
|
16
15
|
workdir_remote: Path
|
17
16
|
parameters: dict[str, Any]
|
18
|
-
zarr_url:
|
17
|
+
zarr_url: str | None = None
|
19
18
|
task_files: TaskFiles
|
20
19
|
index: int
|
21
20
|
|
@@ -65,7 +64,7 @@ class SlurmTask(BaseModel):
|
|
65
64
|
|
66
65
|
|
67
66
|
class SlurmJob(BaseModel):
|
68
|
-
slurm_job_id:
|
67
|
+
slurm_job_id: str | None = None
|
69
68
|
prefix: str
|
70
69
|
workdir_local: Path
|
71
70
|
workdir_remote: Path
|
@@ -1,6 +1,5 @@
|
|
1
1
|
import shlex
|
2
2
|
import subprocess # nosec
|
3
|
-
from typing import Optional
|
4
3
|
|
5
4
|
from fractal_server.logger import get_logger
|
6
5
|
from fractal_server.string_tools import validate_cmd
|
@@ -8,8 +7,8 @@ from fractal_server.string_tools import validate_cmd
|
|
8
7
|
|
9
8
|
def run_subprocess(
|
10
9
|
cmd: str,
|
11
|
-
allow_char:
|
12
|
-
logger_name:
|
10
|
+
allow_char: str | None = None,
|
11
|
+
logger_name: str | None = None,
|
13
12
|
) -> subprocess.CompletedProcess:
|
14
13
|
validate_cmd(cmd, allow_char=allow_char)
|
15
14
|
logger = get_logger(logger_name)
|
@@ -1,6 +1,5 @@
|
|
1
1
|
import time
|
2
2
|
from pathlib import Path
|
3
|
-
from typing import Optional
|
4
3
|
|
5
4
|
from ..slurm_common.base_slurm_runner import BaseSlurmRunner
|
6
5
|
from ..slurm_common.slurm_job_task_models import SlurmJob
|
@@ -27,10 +26,11 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
27
26
|
# Common
|
28
27
|
root_dir_local: Path,
|
29
28
|
root_dir_remote: Path,
|
30
|
-
common_script_lines:
|
31
|
-
user_cache_dir:
|
32
|
-
poll_interval:
|
29
|
+
common_script_lines: list[str] | None = None,
|
30
|
+
user_cache_dir: str | None = None,
|
31
|
+
poll_interval: int | None = None,
|
33
32
|
# Specific
|
33
|
+
slurm_account: str | None = None,
|
34
34
|
fractal_ssh: FractalSSH,
|
35
35
|
) -> None:
|
36
36
|
"""
|
@@ -50,6 +50,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
50
50
|
user_cache_dir=user_cache_dir,
|
51
51
|
poll_interval=poll_interval,
|
52
52
|
python_worker_interpreter=settings.FRACTAL_SLURM_WORKER_PYTHON,
|
53
|
+
slurm_account=slurm_account,
|
53
54
|
)
|
54
55
|
|
55
56
|
def _mkdir_local_folder(self, folder: str) -> None:
|
@@ -17,7 +17,6 @@ another user. Note that this requires appropriate sudo permissions.
|
|
17
17
|
"""
|
18
18
|
import shlex
|
19
19
|
import subprocess # nosec
|
20
|
-
from typing import Optional
|
21
20
|
|
22
21
|
from fractal_server.logger import set_logger
|
23
22
|
from fractal_server.string_tools import validate_cmd
|
@@ -28,7 +27,7 @@ logger = set_logger(__name__)
|
|
28
27
|
def _run_command_as_user(
|
29
28
|
*,
|
30
29
|
cmd: str,
|
31
|
-
user:
|
30
|
+
user: str | None = None,
|
32
31
|
check: bool = False,
|
33
32
|
) -> subprocess.CompletedProcess:
|
34
33
|
"""
|
@@ -5,7 +5,6 @@ import subprocess # nosec
|
|
5
5
|
import sys
|
6
6
|
from concurrent.futures import ThreadPoolExecutor
|
7
7
|
from pathlib import Path
|
8
|
-
from typing import Optional
|
9
8
|
|
10
9
|
from ..slurm_common.base_slurm_runner import BaseSlurmRunner
|
11
10
|
from ..slurm_common.slurm_job_task_models import SlurmJob
|
@@ -21,7 +20,7 @@ logger = set_logger(__name__)
|
|
21
20
|
|
22
21
|
def _subprocess_run_or_raise(
|
23
22
|
full_command: str,
|
24
|
-
) ->
|
23
|
+
) -> subprocess.CompletedProcess | None:
|
25
24
|
try:
|
26
25
|
output = subprocess.run( # nosec
|
27
26
|
shlex.split(full_command),
|
@@ -43,7 +42,7 @@ def _subprocess_run_or_raise(
|
|
43
42
|
|
44
43
|
class SudoSlurmRunner(BaseSlurmRunner):
|
45
44
|
slurm_user: str
|
46
|
-
slurm_account:
|
45
|
+
slurm_account: str | None = None
|
47
46
|
|
48
47
|
def __init__(
|
49
48
|
self,
|
@@ -51,11 +50,11 @@ class SudoSlurmRunner(BaseSlurmRunner):
|
|
51
50
|
# Common
|
52
51
|
root_dir_local: Path,
|
53
52
|
root_dir_remote: Path,
|
54
|
-
common_script_lines:
|
55
|
-
user_cache_dir:
|
56
|
-
poll_interval:
|
53
|
+
common_script_lines: list[str] | None = None,
|
54
|
+
user_cache_dir: str | None = None,
|
55
|
+
poll_interval: int | None = None,
|
57
56
|
# Specific
|
58
|
-
slurm_account:
|
57
|
+
slurm_account: str | None = None,
|
59
58
|
slurm_user: str,
|
60
59
|
) -> None:
|
61
60
|
"""
|
@@ -64,7 +63,6 @@ class SudoSlurmRunner(BaseSlurmRunner):
|
|
64
63
|
"""
|
65
64
|
|
66
65
|
self.slurm_user = slurm_user
|
67
|
-
self.slurm_account = slurm_account
|
68
66
|
settings = Inject(get_settings)
|
69
67
|
|
70
68
|
super().__init__(
|
@@ -77,6 +75,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
|
|
77
75
|
python_worker_interpreter=(
|
78
76
|
settings.FRACTAL_SLURM_WORKER_PYTHON or sys.executable
|
79
77
|
),
|
78
|
+
slurm_account=slurm_account,
|
80
79
|
)
|
81
80
|
|
82
81
|
def _mkdir_local_folder(self, folder: str) -> None:
|
@@ -1,10 +1,7 @@
|
|
1
|
-
from typing import Optional
|
2
|
-
|
3
|
-
|
4
1
|
def set_start_and_last_task_index(
|
5
2
|
num_tasks: int,
|
6
|
-
first_task_index:
|
7
|
-
last_task_index:
|
3
|
+
first_task_index: int | None = None,
|
4
|
+
last_task_index: int | None = None,
|
8
5
|
) -> tuple[int, int]:
|
9
6
|
"""
|
10
7
|
Handle `first_task_index` and `last_task_index`, by setting defaults and
|
@@ -38,21 +38,15 @@ async def cleanup_after_shutdown(*, jobsV2: list[int], logger_name: str):
|
|
38
38
|
|
39
39
|
if len(jobsV2_db) == 0:
|
40
40
|
logger.info(
|
41
|
-
|
42
|
-
|
43
|
-
"either done or failed. Exit."
|
44
|
-
)
|
41
|
+
"All jobs associated to this app are "
|
42
|
+
"either done or failed. Exit."
|
45
43
|
)
|
46
44
|
return
|
47
45
|
else:
|
48
|
-
logger.info(
|
49
|
-
(f"Some jobs are still 'submitted' " f"{jobsV2_db=}")
|
50
|
-
)
|
46
|
+
logger.info(f"Some jobs are still 'submitted' {jobsV2_db=}")
|
51
47
|
logger.info(
|
52
|
-
|
53
|
-
|
54
|
-
"but some jobs are still submitted"
|
55
|
-
)
|
48
|
+
"Graceful shutdown reached its maximum time, "
|
49
|
+
"but some jobs are still submitted"
|
56
50
|
)
|
57
51
|
|
58
52
|
for job in jobsV2_db:
|
@@ -1,6 +1,4 @@
|
|
1
1
|
from pathlib import Path
|
2
|
-
from typing import Optional
|
3
|
-
from typing import Union
|
4
2
|
|
5
3
|
from pydantic import BaseModel
|
6
4
|
|
@@ -12,7 +10,7 @@ MULTISUBMIT_PREFIX = "par"
|
|
12
10
|
|
13
11
|
|
14
12
|
def task_subfolder_name(
|
15
|
-
order:
|
13
|
+
order: int | str,
|
16
14
|
task_name: str,
|
17
15
|
) -> str:
|
18
16
|
"""
|
@@ -48,8 +46,8 @@ class TaskFiles(BaseModel):
|
|
48
46
|
task_order: int
|
49
47
|
|
50
48
|
# Per-single-component
|
51
|
-
component:
|
52
|
-
prefix:
|
49
|
+
component: str | None = None
|
50
|
+
prefix: str | None = None
|
53
51
|
|
54
52
|
def _check_component(self):
|
55
53
|
if self.component is None:
|
@@ -1,5 +1,4 @@
|
|
1
1
|
from pathlib import Path
|
2
|
-
from typing import Optional
|
3
2
|
|
4
3
|
from ...models.v2 import DatasetV2
|
5
4
|
from ...models.v2 import WorkflowV2
|
@@ -16,9 +15,9 @@ def process_workflow(
|
|
16
15
|
dataset: DatasetV2,
|
17
16
|
workflow_dir_local: Path,
|
18
17
|
job_id: int,
|
19
|
-
workflow_dir_remote:
|
20
|
-
first_task_index:
|
21
|
-
last_task_index:
|
18
|
+
workflow_dir_remote: Path | None = None,
|
19
|
+
first_task_index: int | None = None,
|
20
|
+
last_task_index: int | None = None,
|
22
21
|
logger_name: str,
|
23
22
|
job_attribute_filters: AttributeFilters,
|
24
23
|
job_type_filters: dict[str, bool],
|
@@ -16,7 +16,6 @@ Slurm Backend
|
|
16
16
|
This backend runs fractal workflows in a SLURM cluster.
|
17
17
|
"""
|
18
18
|
from pathlib import Path
|
19
|
-
from typing import Optional
|
20
19
|
|
21
20
|
from ....ssh._fabric import FractalSSH
|
22
21
|
from ...models.v2 import DatasetV2
|
@@ -38,16 +37,17 @@ def process_workflow(
|
|
38
37
|
dataset: DatasetV2,
|
39
38
|
workflow_dir_local: Path,
|
40
39
|
job_id: int,
|
41
|
-
workflow_dir_remote:
|
42
|
-
first_task_index:
|
43
|
-
last_task_index:
|
40
|
+
workflow_dir_remote: Path | None = None,
|
41
|
+
first_task_index: int | None = None,
|
42
|
+
last_task_index: int | None = None,
|
44
43
|
logger_name: str,
|
45
44
|
job_attribute_filters: AttributeFilters,
|
46
45
|
job_type_filters: dict[str, bool],
|
47
|
-
fractal_ssh: FractalSSH,
|
48
|
-
worker_init: Optional[str] = None,
|
49
46
|
user_id: int,
|
50
|
-
|
47
|
+
# SLURM-ssh-specific
|
48
|
+
fractal_ssh: FractalSSH,
|
49
|
+
slurm_account: str | None = None,
|
50
|
+
worker_init: str | None = None,
|
51
51
|
) -> None:
|
52
52
|
"""
|
53
53
|
Process workflow (SLURM backend public interface)
|
@@ -80,6 +80,7 @@ def process_workflow(
|
|
80
80
|
fractal_ssh=fractal_ssh,
|
81
81
|
root_dir_local=workflow_dir_local,
|
82
82
|
root_dir_remote=workflow_dir_remote,
|
83
|
+
slurm_account=slurm_account,
|
83
84
|
common_script_lines=worker_init,
|
84
85
|
) as runner:
|
85
86
|
execute_tasks_v2(
|
@@ -16,7 +16,6 @@ Slurm Backend
|
|
16
16
|
This backend runs fractal workflows in a SLURM cluster.
|
17
17
|
"""
|
18
18
|
from pathlib import Path
|
19
|
-
from typing import Optional
|
20
19
|
|
21
20
|
from ...models.v2 import DatasetV2
|
22
21
|
from ...models.v2 import WorkflowV2
|
@@ -33,18 +32,18 @@ def process_workflow(
|
|
33
32
|
dataset: DatasetV2,
|
34
33
|
workflow_dir_local: Path,
|
35
34
|
job_id: int,
|
36
|
-
workflow_dir_remote:
|
37
|
-
first_task_index:
|
38
|
-
last_task_index:
|
35
|
+
workflow_dir_remote: Path | None = None,
|
36
|
+
first_task_index: int | None = None,
|
37
|
+
last_task_index: int | None = None,
|
39
38
|
logger_name: str,
|
40
39
|
job_attribute_filters: AttributeFilters,
|
41
40
|
job_type_filters: dict[str, bool],
|
42
41
|
user_id: int,
|
43
|
-
#
|
44
|
-
user_cache_dir:
|
45
|
-
slurm_user:
|
46
|
-
slurm_account:
|
47
|
-
worker_init:
|
42
|
+
# SLURM-sudo-specific
|
43
|
+
user_cache_dir: str | None = None,
|
44
|
+
slurm_user: str | None = None,
|
45
|
+
slurm_account: str | None = None,
|
46
|
+
worker_init: str | None = None,
|
48
47
|
) -> None:
|
49
48
|
"""
|
50
49
|
Process workflow (SLURM backend public interface).
|
@@ -1,11 +1,10 @@
|
|
1
1
|
import logging
|
2
|
+
from collections.abc import Callable
|
2
3
|
from copy import copy
|
3
4
|
from copy import deepcopy
|
4
5
|
from pathlib import Path
|
5
6
|
from typing import Any
|
6
|
-
from typing import Callable
|
7
7
|
from typing import Literal
|
8
|
-
from typing import Optional
|
9
8
|
|
10
9
|
from sqlalchemy.orm.attributes import flag_modified
|
11
10
|
from sqlmodel import delete
|
@@ -76,13 +75,13 @@ def execute_tasks_v2(
|
|
76
75
|
user_id: int,
|
77
76
|
workflow_dir_local: Path,
|
78
77
|
job_id: int,
|
79
|
-
workflow_dir_remote:
|
80
|
-
logger_name:
|
78
|
+
workflow_dir_remote: Path | None = None,
|
79
|
+
logger_name: str | None = None,
|
81
80
|
get_runner_config: Callable[
|
82
81
|
[
|
83
82
|
WorkflowTaskV2,
|
84
83
|
Literal["non_parallel", "parallel"],
|
85
|
-
|
84
|
+
Path | None,
|
86
85
|
],
|
87
86
|
Any,
|
88
87
|
],
|