fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/models/__init__.py +4 -7
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/state.py +1 -1
- fractal_server/app/models/v1/__init__.py +10 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +20 -0
- fractal_server/app/models/v2/dataset.py +55 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +31 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +43 -0
- fractal_server/app/models/v2/workflowtask.py +90 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +275 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +2 -2
- fractal_server/app/routes/api/v1/dataset.py +37 -37
- fractal_server/app/routes/api/v1/job.py +12 -12
- fractal_server/app/routes/api/v1/project.py +23 -21
- fractal_server/app/routes/api/v1/task.py +24 -14
- fractal_server/app/routes/api/v1/task_collection.py +16 -14
- fractal_server/app/routes/api/v1/workflow.py +24 -24
- fractal_server/app/routes/api/v1/workflowtask.py +10 -10
- fractal_server/app/routes/api/v2/__init__.py +28 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +497 -0
- fractal_server/app/routes/api/v2/apply.py +220 -0
- fractal_server/app/routes/api/v2/dataset.py +310 -0
- fractal_server/app/routes/api/v2/images.py +212 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +205 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +229 -0
- fractal_server/app/routes/api/v2/workflow.py +398 -0
- fractal_server/app/routes/api/v2/workflowtask.py +269 -0
- fractal_server/app/routes/aux/_job.py +1 -1
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/local/__init__.py +3 -0
- fractal_server/app/runner/{_local → executors/local}/executor.py +2 -2
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +9 -9
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +105 -0
- fractal_server/app/runner/{__init__.py → v1/__init__.py} +24 -22
- fractal_server/app/runner/{_common.py → v1/_common.py} +13 -120
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +6 -6
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +310 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +3 -9
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +337 -0
- fractal_server/app/runner/v2/_local/__init__.py +169 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_slurm/__init__.py +157 -0
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +83 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +179 -0
- fractal_server/app/runner/v2/components.py +5 -0
- fractal_server/app/runner/v2/deduplicate_list.py +24 -0
- fractal_server/app/runner/v2/handle_failed_job.py +156 -0
- fractal_server/app/runner/v2/merge_outputs.py +41 -0
- fractal_server/app/runner/v2/runner.py +264 -0
- fractal_server/app/runner/v2/runner_functions.py +339 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +134 -0
- fractal_server/app/runner/v2/task_interface.py +43 -0
- fractal_server/app/runner/v2/v1_compat.py +21 -0
- fractal_server/app/schemas/__init__.py +4 -42
- fractal_server/app/schemas/v1/__init__.py +42 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +34 -0
- fractal_server/app/schemas/v2/dataset.py +88 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +113 -0
- fractal_server/app/schemas/v2/manifest.py +109 -0
- fractal_server/app/schemas/v2/project.py +36 -0
- fractal_server/app/schemas/v2/task.py +121 -0
- fractal_server/app/schemas/v2/task_collection.py +105 -0
- fractal_server/app/schemas/v2/workflow.py +78 -0
- fractal_server/app/schemas/v2/workflowtask.py +118 -0
- fractal_server/config.py +5 -4
- fractal_server/images/__init__.py +50 -0
- fractal_server/images/tools.py +86 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/versions/4b35c5cefbe3_tmp_is_v2_compatible.py +39 -0
- fractal_server/migrations/versions/56af171b0159_v2.py +217 -0
- fractal_server/migrations/versions/876f28db9d4e_tmp_split_task_and_wftask_meta.py +68 -0
- fractal_server/migrations/versions/974c802f0dd0_tmp_workflowtaskv2_type_in_db.py +37 -0
- fractal_server/migrations/versions/9cd305cd6023_tmp_workflowtaskv2.py +40 -0
- fractal_server/migrations/versions/a6231ed6273c_tmp_args_schemas_in_taskv2.py +42 -0
- fractal_server/migrations/versions/b9e9eed9d442_tmp_taskv2_type.py +37 -0
- fractal_server/migrations/versions/e3e639454d4b_tmp_make_task_meta_non_optional.py +50 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +18 -50
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/background_operations.py +382 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/METADATA +1 -1
- fractal_server-2.0.0a0.dist-info/RECORD +166 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/_slurm/__init__.py +0 -150
- fractal_server/app/runner/common.py +0 -311
- fractal_server-1.4.10.dist-info/RECORD +0 -98
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,157 @@
|
|
1
|
+
# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
|
2
|
+
# University of Zurich
|
3
|
+
#
|
4
|
+
# Original authors:
|
5
|
+
# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
|
6
|
+
# Tommaso Comparin <tommaso.comparin@exact-lab.it>
|
7
|
+
# Marco Franzon <marco.franzon@exact-lab.it>
|
8
|
+
#
|
9
|
+
# This file is part of Fractal and was originally developed by eXact lab S.r.l.
|
10
|
+
# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
|
11
|
+
# Institute for Biomedical Research and Pelkmans Lab from the University of
|
12
|
+
# Zurich.
|
13
|
+
"""
|
14
|
+
Slurm Bakend
|
15
|
+
|
16
|
+
This backend runs fractal workflows in a SLURM cluster using Clusterfutures
|
17
|
+
Executor objects.
|
18
|
+
"""
|
19
|
+
from pathlib import Path
|
20
|
+
from typing import Any
|
21
|
+
from typing import Optional
|
22
|
+
from typing import Union
|
23
|
+
|
24
|
+
from fractal_server.app.models.v2 import WorkflowV2
|
25
|
+
|
26
|
+
# from ...executors.slurm.executor import FractalSlurmExecutor
|
27
|
+
# from ._submit_setup import _slurm_submit_setup
|
28
|
+
# from .get_slurm_config import get_slurm_config
|
29
|
+
|
30
|
+
# from .._common import execute_tasks
|
31
|
+
# from ..common import async_wrap
|
32
|
+
# from ..common import set_start_and_last_task_index
|
33
|
+
# from ..common import TaskParameters
|
34
|
+
|
35
|
+
|
36
|
+
def _process_workflow(
|
37
|
+
*,
|
38
|
+
workflow: WorkflowV2,
|
39
|
+
input_paths: list[Path],
|
40
|
+
output_path: Path,
|
41
|
+
input_metadata: dict[str, Any],
|
42
|
+
input_history: list[dict[str, Any]],
|
43
|
+
logger_name: str,
|
44
|
+
workflow_dir: Path,
|
45
|
+
workflow_dir_user: Path,
|
46
|
+
first_task_index: int,
|
47
|
+
last_task_index: int,
|
48
|
+
slurm_user: Optional[str] = None,
|
49
|
+
slurm_account: Optional[str] = None,
|
50
|
+
user_cache_dir: str,
|
51
|
+
worker_init: Optional[Union[str, list[str]]] = None,
|
52
|
+
) -> dict[str, Any]:
|
53
|
+
"""
|
54
|
+
Internal processing routine for the SLURM backend
|
55
|
+
|
56
|
+
This function initialises the a FractalSlurmExecutor, setting logging,
|
57
|
+
workflow working dir and user to impersonate. It then schedules the
|
58
|
+
workflow tasks and returns the output dataset metadata.
|
59
|
+
|
60
|
+
Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
|
61
|
+
|
62
|
+
Returns:
|
63
|
+
output_dataset_metadata: Metadata of the output dataset
|
64
|
+
"""
|
65
|
+
|
66
|
+
raise NotImplementedError
|
67
|
+
|
68
|
+
# if not slurm_user:
|
69
|
+
# raise RuntimeError(
|
70
|
+
# "slurm_user argument is required, for slurm backend"
|
71
|
+
# )
|
72
|
+
|
73
|
+
# if isinstance(worker_init, str):
|
74
|
+
# worker_init = worker_init.split("\n")
|
75
|
+
|
76
|
+
# with FractalSlurmExecutor(
|
77
|
+
# debug=True,
|
78
|
+
# keep_logs=True,
|
79
|
+
# slurm_user=slurm_user,
|
80
|
+
# user_cache_dir=user_cache_dir,
|
81
|
+
# working_dir=workflow_dir,
|
82
|
+
# working_dir_user=workflow_dir_user,
|
83
|
+
# common_script_lines=worker_init,
|
84
|
+
# slurm_account=slurm_account,
|
85
|
+
# ) as executor:
|
86
|
+
# output_task_pars = execute_tasks(
|
87
|
+
# executor=executor,
|
88
|
+
# task_list=workflow.task_list[
|
89
|
+
# first_task_index : (last_task_index + 1) # noqa
|
90
|
+
# ], # noqa
|
91
|
+
# task_pars=TaskParameters(
|
92
|
+
# input_paths=input_paths,
|
93
|
+
# output_path=output_path,
|
94
|
+
# metadata=input_metadata,
|
95
|
+
# history=input_history,
|
96
|
+
# ),
|
97
|
+
# workflow_dir=workflow_dir,
|
98
|
+
# workflow_dir_user=workflow_dir_user,
|
99
|
+
# submit_setup_call=_slurm_submit_setup,
|
100
|
+
# logger_name=logger_name,
|
101
|
+
# )
|
102
|
+
# output_dataset_metadata_history = dict(
|
103
|
+
# metadata=output_task_pars.metadata, history=output_task_pars.history
|
104
|
+
# )
|
105
|
+
# return output_dataset_metadata_history
|
106
|
+
|
107
|
+
|
108
|
+
async def process_workflow(
|
109
|
+
*,
|
110
|
+
workflow: WorkflowV2,
|
111
|
+
input_paths: list[Path],
|
112
|
+
output_path: Path,
|
113
|
+
input_metadata: dict[str, Any],
|
114
|
+
input_history: list[dict[str, Any]],
|
115
|
+
logger_name: str,
|
116
|
+
workflow_dir: Path,
|
117
|
+
workflow_dir_user: Optional[Path] = None,
|
118
|
+
user_cache_dir: Optional[str] = None,
|
119
|
+
slurm_user: Optional[str] = None,
|
120
|
+
slurm_account: Optional[str] = None,
|
121
|
+
worker_init: Optional[str] = None,
|
122
|
+
first_task_index: Optional[int] = None,
|
123
|
+
last_task_index: Optional[int] = None,
|
124
|
+
) -> dict[str, Any]:
|
125
|
+
"""
|
126
|
+
Process workflow (SLURM backend public interface)
|
127
|
+
|
128
|
+
Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
|
129
|
+
"""
|
130
|
+
|
131
|
+
raise NotImplementedError
|
132
|
+
|
133
|
+
# # Set values of first_task_index and last_task_index
|
134
|
+
# num_tasks = len(workflow.task_list)
|
135
|
+
# first_task_index, last_task_index = set_start_and_last_task_index(
|
136
|
+
# num_tasks,
|
137
|
+
# first_task_index=first_task_index,
|
138
|
+
# last_task_index=last_task_index,
|
139
|
+
# )
|
140
|
+
|
141
|
+
# output_dataset_metadata_history = await async_wrap(_process_workflow)(
|
142
|
+
# workflow=workflow,
|
143
|
+
# input_paths=input_paths,
|
144
|
+
# output_path=output_path,
|
145
|
+
# input_metadata=input_metadata,
|
146
|
+
# input_history=input_history,
|
147
|
+
# logger_name=logger_name,
|
148
|
+
# workflow_dir=workflow_dir,
|
149
|
+
# workflow_dir_user=workflow_dir_user,
|
150
|
+
# slurm_user=slurm_user,
|
151
|
+
# slurm_account=slurm_account,
|
152
|
+
# user_cache_dir=user_cache_dir,
|
153
|
+
# worker_init=worker_init,
|
154
|
+
# first_task_index=first_task_index,
|
155
|
+
# last_task_index=last_task_index,
|
156
|
+
# )
|
157
|
+
# return output_dataset_metadata_history
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
|
2
|
+
# University of Zurich
|
3
|
+
#
|
4
|
+
# Original authors:
|
5
|
+
# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
|
6
|
+
# Tommaso Comparin <tommaso.comparin@exact-lab.it>
|
7
|
+
#
|
8
|
+
# This file is part of Fractal and was originally developed by eXact lab S.r.l.
|
9
|
+
# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
|
10
|
+
# Institute for Biomedical Research and Pelkmans Lab from the University of
|
11
|
+
# Zurich.
|
12
|
+
"""
|
13
|
+
Submodule to define _slurm_submit_setup, which is also the reference
|
14
|
+
implementation of `submit_setup_call` in
|
15
|
+
[fractal_server.app.runner._common][]).
|
16
|
+
"""
|
17
|
+
from pathlib import Path
|
18
|
+
from typing import Literal
|
19
|
+
|
20
|
+
from ...task_files import get_task_file_paths
|
21
|
+
from .get_slurm_config import get_slurm_config
|
22
|
+
from fractal_server.app.models.v2 import WorkflowTaskV2
|
23
|
+
|
24
|
+
|
25
|
+
def _slurm_submit_setup(
|
26
|
+
*,
|
27
|
+
wftask: WorkflowTaskV2,
|
28
|
+
workflow_dir: Path,
|
29
|
+
workflow_dir_user: Path,
|
30
|
+
which_type: Literal["non_parallel", "parallel"],
|
31
|
+
) -> dict[str, object]:
|
32
|
+
"""
|
33
|
+
Collect WorfklowTask-specific configuration parameters from different
|
34
|
+
sources, and inject them for execution.
|
35
|
+
|
36
|
+
Here goes all the logic for reading attributes from the appropriate sources
|
37
|
+
and transforming them into an appropriate `SlurmConfig` object (encoding
|
38
|
+
SLURM configuration) and `TaskFiles` object (with details e.g. about file
|
39
|
+
paths or filename prefixes).
|
40
|
+
|
41
|
+
For now, this is the reference implementation for the argument
|
42
|
+
`submit_setup_call` of
|
43
|
+
[fractal_server.app.runner._common.execute_tasks][].
|
44
|
+
|
45
|
+
Arguments:
|
46
|
+
wftask:
|
47
|
+
WorkflowTask for which the configuration is to be assembled
|
48
|
+
workflow_dir:
|
49
|
+
Server-owned directory to store all task-execution-related relevant
|
50
|
+
files (inputs, outputs, errors, and all meta files related to the
|
51
|
+
job execution). Note: users cannot write directly to this folder.
|
52
|
+
workflow_dir_user:
|
53
|
+
User-side directory with the same scope as `workflow_dir`, and
|
54
|
+
where a user can write.
|
55
|
+
|
56
|
+
Returns:
|
57
|
+
submit_setup_dict:
|
58
|
+
A dictionary that will be passed on to
|
59
|
+
`FractalSlurmExecutor.submit` and `FractalSlurmExecutor.map`, so
|
60
|
+
as to set extra options.
|
61
|
+
"""
|
62
|
+
|
63
|
+
# Get SlurmConfig object
|
64
|
+
slurm_config = get_slurm_config(
|
65
|
+
wftask=wftask,
|
66
|
+
workflow_dir=workflow_dir,
|
67
|
+
workflow_dir_user=workflow_dir_user,
|
68
|
+
which_type=which_type,
|
69
|
+
)
|
70
|
+
|
71
|
+
# Get TaskFiles object
|
72
|
+
task_files = get_task_file_paths(
|
73
|
+
workflow_dir=workflow_dir,
|
74
|
+
workflow_dir_user=workflow_dir_user,
|
75
|
+
task_order=wftask.order,
|
76
|
+
)
|
77
|
+
|
78
|
+
# Prepare and return output dictionary
|
79
|
+
submit_setup_dict = dict(
|
80
|
+
slurm_config=slurm_config,
|
81
|
+
task_files=task_files,
|
82
|
+
)
|
83
|
+
return submit_setup_dict
|
@@ -0,0 +1,179 @@
|
|
1
|
+
from pathlib import Path
|
2
|
+
from typing import Literal
|
3
|
+
from typing import Optional
|
4
|
+
|
5
|
+
from fractal_server.app.models.v2 import WorkflowTaskV2
|
6
|
+
from fractal_server.app.runner.executors.slurm._slurm_config import (
|
7
|
+
_parse_mem_value,
|
8
|
+
)
|
9
|
+
from fractal_server.app.runner.executors.slurm._slurm_config import (
|
10
|
+
load_slurm_config_file,
|
11
|
+
)
|
12
|
+
from fractal_server.app.runner.executors.slurm._slurm_config import logger
|
13
|
+
from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
|
14
|
+
from fractal_server.app.runner.executors.slurm._slurm_config import (
|
15
|
+
SlurmConfigError,
|
16
|
+
)
|
17
|
+
|
18
|
+
|
19
|
+
def get_slurm_config(
|
20
|
+
wftask: WorkflowTaskV2,
|
21
|
+
workflow_dir: Path,
|
22
|
+
workflow_dir_user: Path,
|
23
|
+
which_type: Literal["non_parallel", "parallel"],
|
24
|
+
config_path: Optional[Path] = None,
|
25
|
+
) -> SlurmConfig:
|
26
|
+
"""
|
27
|
+
Prepare a `SlurmConfig` configuration object
|
28
|
+
|
29
|
+
The argument `which_type` determines whether we use `wftask.meta_parallel`
|
30
|
+
or `wftask.meta_non_parallel`. In the following descritpion, let us assume
|
31
|
+
that `which_type="parallel"`.
|
32
|
+
|
33
|
+
The sources for `SlurmConfig` attributes, in increasing priority order, are
|
34
|
+
|
35
|
+
1. The general content of the Fractal SLURM configuration file.
|
36
|
+
2. The GPU-specific content of the Fractal SLURM configuration file, if
|
37
|
+
appropriate.
|
38
|
+
3. Properties in `wftask.meta_parallel` (which typically include those in
|
39
|
+
`wftask.task.meta_parallel`). Note that `wftask.meta_parallel` may be
|
40
|
+
`None`.
|
41
|
+
|
42
|
+
Arguments:
|
43
|
+
wftask:
|
44
|
+
WorkflowTask for which the SLURM configuration is is to be
|
45
|
+
prepared.
|
46
|
+
workflow_dir:
|
47
|
+
Server-owned directory to store all task-execution-related relevant
|
48
|
+
files (inputs, outputs, errors, and all meta files related to the
|
49
|
+
job execution). Note: users cannot write directly to this folder.
|
50
|
+
workflow_dir_user:
|
51
|
+
User-side directory with the same scope as `workflow_dir`, and
|
52
|
+
where a user can write.
|
53
|
+
config_path:
|
54
|
+
Path of a Fractal SLURM configuration file; if `None`, use
|
55
|
+
`FRACTAL_SLURM_CONFIG_FILE` variable from settings.
|
56
|
+
which_type:
|
57
|
+
Determines whether to use `meta_parallel` or `meta_non_parallel`.
|
58
|
+
|
59
|
+
Returns:
|
60
|
+
slurm_config:
|
61
|
+
The SlurmConfig object
|
62
|
+
"""
|
63
|
+
|
64
|
+
if which_type == "non_parallel":
|
65
|
+
wftask_meta = wftask.meta_non_parallel
|
66
|
+
elif which_type == "parallel":
|
67
|
+
wftask_meta = wftask.meta_parallel
|
68
|
+
else:
|
69
|
+
raise ValueError(
|
70
|
+
f"get_slurm_config received invalid argument {which_type=}."
|
71
|
+
)
|
72
|
+
|
73
|
+
logger.debug(
|
74
|
+
"[get_slurm_config] WorkflowTask meta attribute: {wftask_meta=}"
|
75
|
+
)
|
76
|
+
|
77
|
+
# Incorporate slurm_env.default_slurm_config
|
78
|
+
slurm_env = load_slurm_config_file(config_path=config_path)
|
79
|
+
slurm_dict = slurm_env.default_slurm_config.dict(
|
80
|
+
exclude_unset=True, exclude={"mem"}
|
81
|
+
)
|
82
|
+
if slurm_env.default_slurm_config.mem:
|
83
|
+
slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
|
84
|
+
|
85
|
+
# Incorporate slurm_env.batching_config
|
86
|
+
for key, value in slurm_env.batching_config.dict().items():
|
87
|
+
slurm_dict[key] = value
|
88
|
+
|
89
|
+
# Incorporate slurm_env.user_local_exports
|
90
|
+
slurm_dict["user_local_exports"] = slurm_env.user_local_exports
|
91
|
+
|
92
|
+
logger.debug(
|
93
|
+
"[get_slurm_config] Fractal SLURM configuration file: "
|
94
|
+
f"{slurm_env.dict()=}"
|
95
|
+
)
|
96
|
+
|
97
|
+
# GPU-related options
|
98
|
+
# Notes about priority:
|
99
|
+
# 1. This block of definitions takes priority over other definitions from
|
100
|
+
# slurm_env which are not under the `needs_gpu` subgroup
|
101
|
+
# 2. This block of definitions has lower priority than whatever comes next
|
102
|
+
# (i.e. from WorkflowTask.meta).
|
103
|
+
if wftask_meta is not None:
|
104
|
+
needs_gpu = wftask_meta.get("needs_gpu", False)
|
105
|
+
else:
|
106
|
+
needs_gpu = False
|
107
|
+
logger.debug(f"[get_slurm_config] {needs_gpu=}")
|
108
|
+
if needs_gpu:
|
109
|
+
for key, value in slurm_env.gpu_slurm_config.dict(
|
110
|
+
exclude_unset=True, exclude={"mem"}
|
111
|
+
).items():
|
112
|
+
slurm_dict[key] = value
|
113
|
+
if slurm_env.gpu_slurm_config.mem:
|
114
|
+
slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
|
115
|
+
|
116
|
+
# Number of CPUs per task, for multithreading
|
117
|
+
if wftask_meta is not None and "cpus_per_task" in wftask_meta:
|
118
|
+
cpus_per_task = int(wftask_meta["cpus_per_task"])
|
119
|
+
slurm_dict["cpus_per_task"] = cpus_per_task
|
120
|
+
|
121
|
+
# Required memory per task, in MB
|
122
|
+
if wftask_meta is not None and "mem" in wftask_meta:
|
123
|
+
raw_mem = wftask_meta["mem"]
|
124
|
+
mem_per_task_MB = _parse_mem_value(raw_mem)
|
125
|
+
slurm_dict["mem_per_task_MB"] = mem_per_task_MB
|
126
|
+
|
127
|
+
# Job name
|
128
|
+
job_name = wftask.task.name.replace(" ", "_")
|
129
|
+
slurm_dict["job_name"] = job_name
|
130
|
+
|
131
|
+
# Optional SLURM arguments and extra lines
|
132
|
+
if wftask_meta is not None:
|
133
|
+
account = wftask_meta.get("account", None)
|
134
|
+
if account is not None:
|
135
|
+
error_msg = (
|
136
|
+
f"Invalid {account=} property in WorkflowTask `meta` "
|
137
|
+
"attribute.\n"
|
138
|
+
"SLURM account must be set in the request body of the "
|
139
|
+
"apply-workflow endpoint, or by modifying the user properties."
|
140
|
+
)
|
141
|
+
logger.error(error_msg)
|
142
|
+
raise SlurmConfigError(error_msg)
|
143
|
+
for key in ["time", "gres", "constraint"]:
|
144
|
+
value = wftask_meta.get(key, None)
|
145
|
+
if value:
|
146
|
+
slurm_dict[key] = value
|
147
|
+
if wftask_meta is not None:
|
148
|
+
extra_lines = wftask_meta.get("extra_lines", [])
|
149
|
+
else:
|
150
|
+
extra_lines = []
|
151
|
+
extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
|
152
|
+
if len(set(extra_lines)) != len(extra_lines):
|
153
|
+
logger.debug(
|
154
|
+
"[get_slurm_config] Removing repeated elements "
|
155
|
+
f"from {extra_lines=}."
|
156
|
+
)
|
157
|
+
extra_lines = list(set(extra_lines))
|
158
|
+
slurm_dict["extra_lines"] = extra_lines
|
159
|
+
|
160
|
+
# Job-batching parameters (if None, they will be determined heuristically)
|
161
|
+
if wftask_meta is not None:
|
162
|
+
tasks_per_job = wftask_meta.get("tasks_per_job", None)
|
163
|
+
parallel_tasks_per_job = wftask_meta.get(
|
164
|
+
"parallel_tasks_per_job", None
|
165
|
+
)
|
166
|
+
else:
|
167
|
+
tasks_per_job = None
|
168
|
+
parallel_tasks_per_job = None
|
169
|
+
slurm_dict["tasks_per_job"] = tasks_per_job
|
170
|
+
slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
|
171
|
+
|
172
|
+
# Put everything together
|
173
|
+
logger.debug(
|
174
|
+
"[get_slurm_config] Now create a SlurmConfig object based "
|
175
|
+
f"on {slurm_dict=}"
|
176
|
+
)
|
177
|
+
slurm_config = SlurmConfig(**slurm_dict)
|
178
|
+
|
179
|
+
return slurm_config
|
@@ -0,0 +1,24 @@
|
|
1
|
+
from typing import TypeVar
|
2
|
+
|
3
|
+
from pydantic.main import ModelMetaclass
|
4
|
+
|
5
|
+
from ....images import SingleImage
|
6
|
+
from .task_interface import InitArgsModel
|
7
|
+
|
8
|
+
T = TypeVar("T", SingleImage, InitArgsModel)
|
9
|
+
|
10
|
+
|
11
|
+
def deduplicate_list(
|
12
|
+
this_list: list[T], PydanticModel: ModelMetaclass
|
13
|
+
) -> list[T]:
|
14
|
+
"""
|
15
|
+
Custom replacement for `set(this_list)`, when items are Pydantic-model
|
16
|
+
instances and then non-hashable (e.g. SingleImage or InitArgsModel).
|
17
|
+
"""
|
18
|
+
this_list_dict = [this_item.dict() for this_item in this_list]
|
19
|
+
new_list_dict = []
|
20
|
+
for this_dict in this_list_dict:
|
21
|
+
if this_dict not in new_list_dict:
|
22
|
+
new_list_dict.append(this_dict)
|
23
|
+
new_list = [PydanticModel(**this_dict) for this_dict in new_list_dict]
|
24
|
+
return new_list
|
@@ -0,0 +1,156 @@
|
|
1
|
+
# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
|
2
|
+
# University of Zurich
|
3
|
+
#
|
4
|
+
# Original authors:
|
5
|
+
# Tommaso Comparin <tommaso.comparin@exact-lab.it>
|
6
|
+
# Marco Franzon <marco.franzon@exact-lab.it>
|
7
|
+
#
|
8
|
+
# This file is part of Fractal and was originally developed by eXact lab S.r.l.
|
9
|
+
# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
|
10
|
+
# Institute for Biomedical Research and Pelkmans Lab from the University of
|
11
|
+
# Zurich.
|
12
|
+
"""
|
13
|
+
Helper functions to handle Dataset history.
|
14
|
+
"""
|
15
|
+
import json
|
16
|
+
import logging
|
17
|
+
from pathlib import Path
|
18
|
+
from typing import Any
|
19
|
+
from typing import Optional
|
20
|
+
|
21
|
+
from ...models.v2 import DatasetV2
|
22
|
+
from ...models.v2 import JobV2
|
23
|
+
from ...models.v2 import WorkflowTaskV2
|
24
|
+
from ...models.v2 import WorkflowV2
|
25
|
+
from ...schemas.v2 import WorkflowTaskStatusTypeV2
|
26
|
+
from ..filenames import FILTERS_FILENAME
|
27
|
+
from ..filenames import HISTORY_FILENAME
|
28
|
+
from ..filenames import IMAGES_FILENAME
|
29
|
+
|
30
|
+
|
31
|
+
def assemble_history_failed_job(
|
32
|
+
job: JobV2,
|
33
|
+
dataset: DatasetV2,
|
34
|
+
workflow: WorkflowV2,
|
35
|
+
logger: logging.Logger,
|
36
|
+
failed_wftask: Optional[WorkflowTaskV2] = None,
|
37
|
+
) -> list[dict[str, Any]]:
|
38
|
+
"""
|
39
|
+
Assemble `history` after a workflow-execution job fails.
|
40
|
+
|
41
|
+
Args:
|
42
|
+
job:
|
43
|
+
The failed `ApplyWorkflow` object.
|
44
|
+
output_dataset:
|
45
|
+
The `dataset` associated to `job`.
|
46
|
+
workflow:
|
47
|
+
The `workflow` associated to `job`.
|
48
|
+
logger: A logger instance.
|
49
|
+
failed_wftask:
|
50
|
+
If set, append it to `history` during step 3; if `None`, infer
|
51
|
+
it by comparing the job task list and the one in
|
52
|
+
`HISTORY_FILENAME`.
|
53
|
+
|
54
|
+
Returns:
|
55
|
+
The new value of `history`, to be merged into
|
56
|
+
`output_dataset.meta`.
|
57
|
+
"""
|
58
|
+
|
59
|
+
# The final value of the history attribute should include up to three
|
60
|
+
# parts, coming from: the database, the temporary file, the failed-task
|
61
|
+
# information.
|
62
|
+
|
63
|
+
# Part 1: Read exising history from DB
|
64
|
+
new_history = dataset.history
|
65
|
+
|
66
|
+
# Part 2: Extend history based on temporary-file contents
|
67
|
+
tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME
|
68
|
+
try:
|
69
|
+
with tmp_history_file.open("r") as f:
|
70
|
+
tmp_file_history = json.load(f)
|
71
|
+
new_history.extend(tmp_file_history)
|
72
|
+
except FileNotFoundError:
|
73
|
+
tmp_file_history = []
|
74
|
+
|
75
|
+
# Part 3/A: Identify failed task, if needed
|
76
|
+
if failed_wftask is None:
|
77
|
+
job_wftasks = workflow.task_list[
|
78
|
+
job.first_task_index : (job.last_task_index + 1) # noqa
|
79
|
+
]
|
80
|
+
tmp_file_wftasks = [
|
81
|
+
history_item["workflowtask"] for history_item in tmp_file_history
|
82
|
+
]
|
83
|
+
if len(job_wftasks) <= len(tmp_file_wftasks):
|
84
|
+
n_tasks_job = len(job_wftasks)
|
85
|
+
n_tasks_tmp = len(tmp_file_wftasks)
|
86
|
+
logger.error(
|
87
|
+
"Cannot identify the failed task based on job task list "
|
88
|
+
f"(length {n_tasks_job}) and temporary-file task list "
|
89
|
+
f"(length {n_tasks_tmp})."
|
90
|
+
)
|
91
|
+
logger.error("Failed task not appended to history.")
|
92
|
+
else:
|
93
|
+
failed_wftask = job_wftasks[len(tmp_file_wftasks)]
|
94
|
+
|
95
|
+
# Part 3/B: Append failed task to history
|
96
|
+
if failed_wftask is not None:
|
97
|
+
failed_wftask_dump = failed_wftask.model_dump(exclude={"task"})
|
98
|
+
failed_wftask_dump["task"] = failed_wftask.task.model_dump()
|
99
|
+
new_history_item = dict(
|
100
|
+
workflowtask=failed_wftask_dump,
|
101
|
+
status=WorkflowTaskStatusTypeV2.FAILED,
|
102
|
+
parallelization=dict(), # FIXME: re-include parallelization
|
103
|
+
)
|
104
|
+
new_history.append(new_history_item)
|
105
|
+
|
106
|
+
return new_history
|
107
|
+
|
108
|
+
|
109
|
+
def assemble_images_failed_job(job: JobV2) -> Optional[dict[str, Any]]:
|
110
|
+
"""
|
111
|
+
Assemble `DatasetV2.images` for a failed workflow-execution.
|
112
|
+
|
113
|
+
Assemble new value of `images` based on the last successful task, i.e.
|
114
|
+
based on the content of the temporary `IMAGES_FILENAME` file. If the file
|
115
|
+
is missing, return `None`.
|
116
|
+
|
117
|
+
Argumentss:
|
118
|
+
job:
|
119
|
+
The failed `JobV2` object.
|
120
|
+
|
121
|
+
Returns:
|
122
|
+
The new value of `dataset.images`, or `None` if `IMAGES_FILENAME`
|
123
|
+
is missing.
|
124
|
+
"""
|
125
|
+
tmp_file = Path(job.working_dir) / IMAGES_FILENAME
|
126
|
+
try:
|
127
|
+
with tmp_file.open("r") as f:
|
128
|
+
new_images = json.load(f)
|
129
|
+
return new_images
|
130
|
+
except FileNotFoundError:
|
131
|
+
return None
|
132
|
+
|
133
|
+
|
134
|
+
def assemble_filters_failed_job(job: JobV2) -> Optional[dict[str, Any]]:
|
135
|
+
"""
|
136
|
+
Assemble `DatasetV2.filters` for a failed workflow-execution.
|
137
|
+
|
138
|
+
Assemble new value of `filters` based on the last successful task, i.e.
|
139
|
+
based on the content of the temporary `FILTERS_FILENAME` file. If the file
|
140
|
+
is missing, return `None`.
|
141
|
+
|
142
|
+
Argumentss:
|
143
|
+
job:
|
144
|
+
The failed `JobV2` object.
|
145
|
+
|
146
|
+
Returns:
|
147
|
+
The new value of `dataset.filters`, or `None` if `FILTERS_FILENAME`
|
148
|
+
is missing.
|
149
|
+
"""
|
150
|
+
tmp_file = Path(job.working_dir) / FILTERS_FILENAME
|
151
|
+
try:
|
152
|
+
with tmp_file.open("r") as f:
|
153
|
+
new_filters = json.load(f)
|
154
|
+
return new_filters
|
155
|
+
except FileNotFoundError:
|
156
|
+
return None
|
@@ -0,0 +1,41 @@
|
|
1
|
+
from copy import copy
|
2
|
+
|
3
|
+
from fractal_server.app.runner.v2.deduplicate_list import deduplicate_list
|
4
|
+
from fractal_server.app.runner.v2.task_interface import TaskOutput
|
5
|
+
from fractal_server.images import SingleImage
|
6
|
+
|
7
|
+
|
8
|
+
def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
|
9
|
+
|
10
|
+
final_image_list_updates = []
|
11
|
+
final_image_list_removals = []
|
12
|
+
last_new_filters = None
|
13
|
+
|
14
|
+
for ind, task_output in enumerate(task_outputs):
|
15
|
+
|
16
|
+
final_image_list_updates.extend(task_output.image_list_updates)
|
17
|
+
final_image_list_removals.extend(task_output.image_list_removals)
|
18
|
+
|
19
|
+
# Check that all filters are the same
|
20
|
+
current_new_filters = task_output.filters
|
21
|
+
if ind == 0:
|
22
|
+
last_new_filters = copy(current_new_filters)
|
23
|
+
if current_new_filters != last_new_filters:
|
24
|
+
raise ValueError(f"{current_new_filters=} but {last_new_filters=}")
|
25
|
+
last_new_filters = copy(current_new_filters)
|
26
|
+
|
27
|
+
final_image_list_updates = deduplicate_list(
|
28
|
+
final_image_list_updates, PydanticModel=SingleImage
|
29
|
+
)
|
30
|
+
|
31
|
+
additional_args = {}
|
32
|
+
if last_new_filters is not None:
|
33
|
+
additional_args["filters"] = last_new_filters
|
34
|
+
|
35
|
+
final_output = TaskOutput(
|
36
|
+
image_list_updates=final_image_list_updates,
|
37
|
+
image_list_removals=final_image_list_removals,
|
38
|
+
**additional_args,
|
39
|
+
)
|
40
|
+
|
41
|
+
return final_output
|