fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/models/__init__.py +6 -8
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/v1/__init__.py +12 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{state.py → v1/state.py} +2 -2
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +22 -0
- fractal_server/app/models/v2/collection_state.py +21 -0
- fractal_server/app/models/v2/dataset.py +54 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +30 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +35 -0
- fractal_server/app/models/v2/workflowtask.py +49 -0
- fractal_server/app/routes/admin/__init__.py +0 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +309 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
- fractal_server/app/routes/api/v1/dataset.py +41 -41
- fractal_server/app/routes/api/v1/job.py +14 -14
- fractal_server/app/routes/api/v1/project.py +27 -25
- fractal_server/app/routes/api/v1/task.py +26 -16
- fractal_server/app/routes/api/v1/task_collection.py +28 -16
- fractal_server/app/routes/api/v1/workflow.py +28 -28
- fractal_server/app/routes/api/v1/workflowtask.py +11 -11
- fractal_server/app/routes/api/v2/__init__.py +34 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
- fractal_server/app/routes/api/v2/dataset.py +293 -0
- fractal_server/app/routes/api/v2/images.py +279 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +186 -0
- fractal_server/app/routes/api/v2/status.py +150 -0
- fractal_server/app/routes/api/v2/submit.py +210 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +239 -0
- fractal_server/app/routes/api/v2/task_legacy.py +59 -0
- fractal_server/app/routes/api/v2/workflow.py +380 -0
- fractal_server/app/routes/api/v2/workflowtask.py +265 -0
- fractal_server/app/routes/aux/_job.py +2 -2
- fractal_server/app/runner/__init__.py +0 -364
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/components.py +5 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/__init__.py +0 -0
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +32 -21
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +103 -0
- fractal_server/app/runner/v1/__init__.py +366 -0
- fractal_server/app/runner/{_common.py → v1/_common.py} +14 -121
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +336 -0
- fractal_server/app/runner/v2/_local/__init__.py +162 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_local/executor.py +100 -0
- fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
- fractal_server/app/runner/v2/deduplicate_list.py +23 -0
- fractal_server/app/runner/v2/handle_failed_job.py +165 -0
- fractal_server/app/runner/v2/merge_outputs.py +38 -0
- fractal_server/app/runner/v2/runner.py +343 -0
- fractal_server/app/runner/v2/runner_functions.py +374 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
- fractal_server/app/runner/v2/task_interface.py +62 -0
- fractal_server/app/runner/v2/v1_compat.py +31 -0
- fractal_server/app/schemas/__init__.py +1 -42
- fractal_server/app/schemas/_validators.py +28 -5
- fractal_server/app/schemas/v1/__init__.py +36 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +37 -0
- fractal_server/app/schemas/v2/dataset.py +126 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +114 -0
- fractal_server/app/schemas/v2/manifest.py +159 -0
- fractal_server/app/schemas/v2/project.py +34 -0
- fractal_server/app/schemas/v2/status.py +16 -0
- fractal_server/app/schemas/v2/task.py +151 -0
- fractal_server/app/schemas/v2/task_collection.py +109 -0
- fractal_server/app/schemas/v2/workflow.py +79 -0
- fractal_server/app/schemas/v2/workflowtask.py +208 -0
- fractal_server/config.py +5 -4
- fractal_server/images/__init__.py +4 -0
- fractal_server/images/models.py +136 -0
- fractal_server/images/tools.py +84 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/env.py +0 -2
- fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/v1/__init__.py +0 -0
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/__init__.py +0 -0
- fractal_server/tasks/v2/background_operations.py +381 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- fractal_server/urls.py +13 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/METADATA +10 -10
- fractal_server-2.0.0.dist-info/RECORD +169 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/common.py +0 -311
- fractal_server/app/schemas/json_schemas/manifest.json +0 -81
- fractal_server-1.4.10.dist-info/RECORD +0 -98
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,374 @@
|
|
1
|
+
import functools
|
2
|
+
import logging
|
3
|
+
import traceback
|
4
|
+
from concurrent.futures import Executor
|
5
|
+
from pathlib import Path
|
6
|
+
from typing import Any
|
7
|
+
from typing import Callable
|
8
|
+
from typing import Literal
|
9
|
+
from typing import Optional
|
10
|
+
|
11
|
+
from pydantic import ValidationError
|
12
|
+
|
13
|
+
from ..exceptions import JobExecutionError
|
14
|
+
from .deduplicate_list import deduplicate_list
|
15
|
+
from .merge_outputs import merge_outputs
|
16
|
+
from .runner_functions_low_level import run_single_task
|
17
|
+
from .task_interface import InitTaskOutput
|
18
|
+
from .task_interface import TaskOutput
|
19
|
+
from .v1_compat import convert_v2_args_into_v1
|
20
|
+
from fractal_server.app.models.v1 import Task as TaskV1
|
21
|
+
from fractal_server.app.models.v2 import TaskV2
|
22
|
+
from fractal_server.app.models.v2 import WorkflowTaskV2
|
23
|
+
from fractal_server.app.runner.components import _COMPONENT_KEY_
|
24
|
+
from fractal_server.app.runner.components import _index_to_component
|
25
|
+
|
26
|
+
|
27
|
+
__all__ = [
|
28
|
+
"run_v2_task_non_parallel",
|
29
|
+
"run_v2_task_parallel",
|
30
|
+
"run_v2_task_compound",
|
31
|
+
"run_v1_task_parallel",
|
32
|
+
]
|
33
|
+
|
34
|
+
MAX_PARALLELIZATION_LIST_SIZE = 20_000
|
35
|
+
|
36
|
+
|
37
|
+
def _cast_and_validate_TaskOutput(
|
38
|
+
task_output: dict[str, Any]
|
39
|
+
) -> Optional[TaskOutput]:
|
40
|
+
try:
|
41
|
+
validated_task_output = TaskOutput(**task_output)
|
42
|
+
return validated_task_output
|
43
|
+
except ValidationError as e:
|
44
|
+
raise JobExecutionError(
|
45
|
+
"Validation of task output failed.\n"
|
46
|
+
f"Original error: {str(e)}\n"
|
47
|
+
f"Original data: {task_output}."
|
48
|
+
)
|
49
|
+
|
50
|
+
|
51
|
+
def _cast_and_validate_InitTaskOutput(
|
52
|
+
init_task_output: dict[str, Any],
|
53
|
+
) -> Optional[InitTaskOutput]:
|
54
|
+
try:
|
55
|
+
validated_init_task_output = InitTaskOutput(**init_task_output)
|
56
|
+
return validated_init_task_output
|
57
|
+
except ValidationError as e:
|
58
|
+
raise JobExecutionError(
|
59
|
+
"Validation of init-task output failed.\n"
|
60
|
+
f"Original error: {str(e)}\n"
|
61
|
+
f"Original data: {init_task_output}."
|
62
|
+
)
|
63
|
+
|
64
|
+
|
65
|
+
def no_op_submit_setup_call(
|
66
|
+
*,
|
67
|
+
wftask: WorkflowTaskV2,
|
68
|
+
workflow_dir: Path,
|
69
|
+
workflow_dir_user: Path,
|
70
|
+
which_type: Literal["non_parallel", "parallel"],
|
71
|
+
) -> dict:
|
72
|
+
"""
|
73
|
+
Default (no-operation) interface of submit_setup_call in V2.
|
74
|
+
"""
|
75
|
+
return {}
|
76
|
+
|
77
|
+
|
78
|
+
# Backend-specific configuration
|
79
|
+
def _get_executor_options(
|
80
|
+
*,
|
81
|
+
wftask: WorkflowTaskV2,
|
82
|
+
workflow_dir: Path,
|
83
|
+
workflow_dir_user: Path,
|
84
|
+
submit_setup_call: Callable,
|
85
|
+
which_type: Literal["non_parallel", "parallel"],
|
86
|
+
) -> dict:
|
87
|
+
try:
|
88
|
+
options = submit_setup_call(
|
89
|
+
wftask=wftask,
|
90
|
+
workflow_dir=workflow_dir,
|
91
|
+
workflow_dir_user=workflow_dir_user,
|
92
|
+
which_type=which_type,
|
93
|
+
)
|
94
|
+
except Exception as e:
|
95
|
+
tb = "".join(traceback.format_tb(e.__traceback__))
|
96
|
+
raise RuntimeError(
|
97
|
+
f"{type(e)} error in {submit_setup_call=}\n"
|
98
|
+
f"Original traceback:\n{tb}"
|
99
|
+
)
|
100
|
+
return options
|
101
|
+
|
102
|
+
|
103
|
+
def _check_parallelization_list_size(my_list):
|
104
|
+
if len(my_list) > MAX_PARALLELIZATION_LIST_SIZE:
|
105
|
+
raise JobExecutionError(
|
106
|
+
"Too many parallelization items.\n"
|
107
|
+
f" {len(my_list)}\n"
|
108
|
+
f" {MAX_PARALLELIZATION_LIST_SIZE=}\n"
|
109
|
+
)
|
110
|
+
|
111
|
+
|
112
|
+
def run_v2_task_non_parallel(
|
113
|
+
*,
|
114
|
+
images: list[dict[str, Any]],
|
115
|
+
zarr_dir: str,
|
116
|
+
task: TaskV2,
|
117
|
+
wftask: WorkflowTaskV2,
|
118
|
+
workflow_dir: Path,
|
119
|
+
workflow_dir_user: Optional[Path] = None,
|
120
|
+
executor: Executor,
|
121
|
+
logger_name: Optional[str] = None,
|
122
|
+
submit_setup_call: Callable = no_op_submit_setup_call,
|
123
|
+
) -> TaskOutput:
|
124
|
+
"""
|
125
|
+
This runs server-side (see `executor` argument)
|
126
|
+
"""
|
127
|
+
|
128
|
+
if workflow_dir_user is None:
|
129
|
+
workflow_dir_user = workflow_dir
|
130
|
+
logging.warning(
|
131
|
+
"In `run_single_task`, workflow_dir_user=None. Is this right?"
|
132
|
+
)
|
133
|
+
workflow_dir_user = workflow_dir
|
134
|
+
|
135
|
+
executor_options = _get_executor_options(
|
136
|
+
wftask=wftask,
|
137
|
+
workflow_dir=workflow_dir,
|
138
|
+
workflow_dir_user=workflow_dir_user,
|
139
|
+
submit_setup_call=submit_setup_call,
|
140
|
+
which_type="non_parallel",
|
141
|
+
)
|
142
|
+
|
143
|
+
function_kwargs = dict(
|
144
|
+
zarr_urls=[image["zarr_url"] for image in images],
|
145
|
+
zarr_dir=zarr_dir,
|
146
|
+
**(wftask.args_non_parallel or {}),
|
147
|
+
)
|
148
|
+
future = executor.submit(
|
149
|
+
functools.partial(
|
150
|
+
run_single_task,
|
151
|
+
wftask=wftask,
|
152
|
+
command=task.command_non_parallel,
|
153
|
+
workflow_dir=workflow_dir,
|
154
|
+
workflow_dir_user=workflow_dir_user,
|
155
|
+
),
|
156
|
+
function_kwargs,
|
157
|
+
**executor_options,
|
158
|
+
)
|
159
|
+
output = future.result()
|
160
|
+
if output is None:
|
161
|
+
return TaskOutput()
|
162
|
+
else:
|
163
|
+
return _cast_and_validate_TaskOutput(output)
|
164
|
+
|
165
|
+
|
166
|
+
def run_v2_task_parallel(
|
167
|
+
*,
|
168
|
+
images: list[dict[str, Any]],
|
169
|
+
task: TaskV2,
|
170
|
+
wftask: WorkflowTaskV2,
|
171
|
+
executor: Executor,
|
172
|
+
workflow_dir: Path,
|
173
|
+
workflow_dir_user: Optional[Path] = None,
|
174
|
+
logger_name: Optional[str] = None,
|
175
|
+
submit_setup_call: Callable = no_op_submit_setup_call,
|
176
|
+
) -> TaskOutput:
|
177
|
+
|
178
|
+
if len(images) == 0:
|
179
|
+
return TaskOutput()
|
180
|
+
|
181
|
+
_check_parallelization_list_size(images)
|
182
|
+
|
183
|
+
executor_options = _get_executor_options(
|
184
|
+
wftask=wftask,
|
185
|
+
workflow_dir=workflow_dir,
|
186
|
+
workflow_dir_user=workflow_dir_user,
|
187
|
+
submit_setup_call=submit_setup_call,
|
188
|
+
which_type="parallel",
|
189
|
+
)
|
190
|
+
|
191
|
+
list_function_kwargs = []
|
192
|
+
for ind, image in enumerate(images):
|
193
|
+
list_function_kwargs.append(
|
194
|
+
dict(
|
195
|
+
zarr_url=image["zarr_url"],
|
196
|
+
**(wftask.args_parallel or {}),
|
197
|
+
),
|
198
|
+
)
|
199
|
+
list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
|
200
|
+
|
201
|
+
results_iterator = executor.map(
|
202
|
+
functools.partial(
|
203
|
+
run_single_task,
|
204
|
+
wftask=wftask,
|
205
|
+
command=task.command_parallel,
|
206
|
+
workflow_dir=workflow_dir,
|
207
|
+
workflow_dir_user=workflow_dir_user,
|
208
|
+
),
|
209
|
+
list_function_kwargs,
|
210
|
+
**executor_options,
|
211
|
+
)
|
212
|
+
# Explicitly iterate over the whole list, so that all futures are waited
|
213
|
+
outputs = list(results_iterator)
|
214
|
+
|
215
|
+
# Validate all non-None outputs
|
216
|
+
for ind, output in enumerate(outputs):
|
217
|
+
if output is None:
|
218
|
+
outputs[ind] = TaskOutput()
|
219
|
+
else:
|
220
|
+
outputs[ind] = _cast_and_validate_TaskOutput(output)
|
221
|
+
|
222
|
+
merged_output = merge_outputs(outputs)
|
223
|
+
return merged_output
|
224
|
+
|
225
|
+
|
226
|
+
def run_v2_task_compound(
|
227
|
+
*,
|
228
|
+
images: list[dict[str, Any]],
|
229
|
+
zarr_dir: str,
|
230
|
+
task: TaskV2,
|
231
|
+
wftask: WorkflowTaskV2,
|
232
|
+
executor: Executor,
|
233
|
+
workflow_dir: Path,
|
234
|
+
workflow_dir_user: Optional[Path] = None,
|
235
|
+
logger_name: Optional[str] = None,
|
236
|
+
submit_setup_call: Callable = no_op_submit_setup_call,
|
237
|
+
) -> TaskOutput:
|
238
|
+
|
239
|
+
executor_options_init = _get_executor_options(
|
240
|
+
wftask=wftask,
|
241
|
+
workflow_dir=workflow_dir,
|
242
|
+
workflow_dir_user=workflow_dir_user,
|
243
|
+
submit_setup_call=submit_setup_call,
|
244
|
+
which_type="non_parallel",
|
245
|
+
)
|
246
|
+
executor_options_compute = _get_executor_options(
|
247
|
+
wftask=wftask,
|
248
|
+
workflow_dir=workflow_dir,
|
249
|
+
workflow_dir_user=workflow_dir_user,
|
250
|
+
submit_setup_call=submit_setup_call,
|
251
|
+
which_type="parallel",
|
252
|
+
)
|
253
|
+
|
254
|
+
# 3/A: non-parallel init task
|
255
|
+
function_kwargs = dict(
|
256
|
+
zarr_urls=[image["zarr_url"] for image in images],
|
257
|
+
zarr_dir=zarr_dir,
|
258
|
+
**(wftask.args_non_parallel or {}),
|
259
|
+
)
|
260
|
+
future = executor.submit(
|
261
|
+
functools.partial(
|
262
|
+
run_single_task,
|
263
|
+
wftask=wftask,
|
264
|
+
command=task.command_non_parallel,
|
265
|
+
workflow_dir=workflow_dir,
|
266
|
+
workflow_dir_user=workflow_dir_user,
|
267
|
+
),
|
268
|
+
function_kwargs,
|
269
|
+
**executor_options_init,
|
270
|
+
)
|
271
|
+
output = future.result()
|
272
|
+
if output is None:
|
273
|
+
init_task_output = InitTaskOutput()
|
274
|
+
else:
|
275
|
+
init_task_output = _cast_and_validate_InitTaskOutput(output)
|
276
|
+
parallelization_list = init_task_output.parallelization_list
|
277
|
+
parallelization_list = deduplicate_list(parallelization_list)
|
278
|
+
|
279
|
+
# 3/B: parallel part of a compound task
|
280
|
+
_check_parallelization_list_size(parallelization_list)
|
281
|
+
|
282
|
+
if len(parallelization_list) == 0:
|
283
|
+
return TaskOutput()
|
284
|
+
|
285
|
+
list_function_kwargs = []
|
286
|
+
for ind, parallelization_item in enumerate(parallelization_list):
|
287
|
+
list_function_kwargs.append(
|
288
|
+
dict(
|
289
|
+
zarr_url=parallelization_item.zarr_url,
|
290
|
+
init_args=parallelization_item.init_args,
|
291
|
+
**(wftask.args_parallel or {}),
|
292
|
+
),
|
293
|
+
)
|
294
|
+
list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
|
295
|
+
|
296
|
+
results_iterator = executor.map(
|
297
|
+
functools.partial(
|
298
|
+
run_single_task,
|
299
|
+
wftask=wftask,
|
300
|
+
command=task.command_parallel,
|
301
|
+
workflow_dir=workflow_dir,
|
302
|
+
workflow_dir_user=workflow_dir_user,
|
303
|
+
),
|
304
|
+
list_function_kwargs,
|
305
|
+
**executor_options_compute,
|
306
|
+
)
|
307
|
+
# Explicitly iterate over the whole list, so that all futures are waited
|
308
|
+
outputs = list(results_iterator)
|
309
|
+
|
310
|
+
# Validate all non-None outputs
|
311
|
+
for ind, output in enumerate(outputs):
|
312
|
+
if output is None:
|
313
|
+
outputs[ind] = TaskOutput()
|
314
|
+
else:
|
315
|
+
validated_output = _cast_and_validate_TaskOutput(output)
|
316
|
+
outputs[ind] = validated_output
|
317
|
+
|
318
|
+
merged_output = merge_outputs(outputs)
|
319
|
+
return merged_output
|
320
|
+
|
321
|
+
|
322
|
+
def run_v1_task_parallel(
|
323
|
+
*,
|
324
|
+
images: list[dict[str, Any]],
|
325
|
+
task_legacy: TaskV1,
|
326
|
+
wftask: WorkflowTaskV2,
|
327
|
+
executor: Executor,
|
328
|
+
workflow_dir: Path,
|
329
|
+
workflow_dir_user: Optional[Path] = None,
|
330
|
+
logger_name: Optional[str] = None,
|
331
|
+
submit_setup_call: Callable = no_op_submit_setup_call,
|
332
|
+
) -> TaskOutput:
|
333
|
+
|
334
|
+
_check_parallelization_list_size(images)
|
335
|
+
|
336
|
+
executor_options = _get_executor_options(
|
337
|
+
wftask=wftask,
|
338
|
+
workflow_dir=workflow_dir,
|
339
|
+
workflow_dir_user=workflow_dir_user,
|
340
|
+
submit_setup_call=submit_setup_call,
|
341
|
+
which_type="parallel",
|
342
|
+
)
|
343
|
+
|
344
|
+
list_function_kwargs = []
|
345
|
+
for ind, image in enumerate(images):
|
346
|
+
list_function_kwargs.append(
|
347
|
+
convert_v2_args_into_v1(
|
348
|
+
kwargs_v2=dict(
|
349
|
+
zarr_url=image["zarr_url"],
|
350
|
+
**(wftask.args_parallel or {}),
|
351
|
+
),
|
352
|
+
parallelization_level=task_legacy.parallelization_level,
|
353
|
+
),
|
354
|
+
)
|
355
|
+
list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
|
356
|
+
|
357
|
+
results_iterator = executor.map(
|
358
|
+
functools.partial(
|
359
|
+
run_single_task,
|
360
|
+
wftask=wftask,
|
361
|
+
command=task_legacy.command,
|
362
|
+
workflow_dir=workflow_dir,
|
363
|
+
workflow_dir_user=workflow_dir_user,
|
364
|
+
is_task_v1=True,
|
365
|
+
),
|
366
|
+
list_function_kwargs,
|
367
|
+
**executor_options,
|
368
|
+
)
|
369
|
+
# Explicitly iterate over the whole list, so that all futures are waited
|
370
|
+
list(results_iterator)
|
371
|
+
|
372
|
+
# Ignore any output metadata for V1 tasks, and return an empty object
|
373
|
+
out = TaskOutput()
|
374
|
+
return out
|
@@ -0,0 +1,130 @@
|
|
1
|
+
import json
|
2
|
+
import logging
|
3
|
+
import shutil
|
4
|
+
import subprocess # nosec
|
5
|
+
from pathlib import Path
|
6
|
+
from shlex import split as shlex_split
|
7
|
+
from typing import Any
|
8
|
+
from typing import Optional
|
9
|
+
|
10
|
+
from ..components import _COMPONENT_KEY_
|
11
|
+
from ..exceptions import JobExecutionError
|
12
|
+
from ..exceptions import TaskExecutionError
|
13
|
+
from fractal_server.app.models.v2 import WorkflowTaskV2
|
14
|
+
from fractal_server.app.runner.task_files import get_task_file_paths
|
15
|
+
|
16
|
+
|
17
|
+
def _call_command_wrapper(cmd: str, log_path: Path) -> None:
|
18
|
+
"""
|
19
|
+
Call a command and write its stdout and stderr to files
|
20
|
+
|
21
|
+
Raises:
|
22
|
+
TaskExecutionError: If the `subprocess.run` call returns a positive
|
23
|
+
exit code
|
24
|
+
JobExecutionError: If the `subprocess.run` call returns a negative
|
25
|
+
exit code (e.g. due to the subprocess receiving a
|
26
|
+
TERM or KILL signal)
|
27
|
+
"""
|
28
|
+
|
29
|
+
# Verify that task command is executable
|
30
|
+
if shutil.which(shlex_split(cmd)[0]) is None:
|
31
|
+
msg = (
|
32
|
+
f'Command "{shlex_split(cmd)[0]}" is not valid. '
|
33
|
+
"Hint: make sure that it is executable."
|
34
|
+
)
|
35
|
+
raise TaskExecutionError(msg)
|
36
|
+
|
37
|
+
fp_log = open(log_path, "w")
|
38
|
+
try:
|
39
|
+
result = subprocess.run( # nosec
|
40
|
+
shlex_split(cmd),
|
41
|
+
stderr=fp_log,
|
42
|
+
stdout=fp_log,
|
43
|
+
)
|
44
|
+
except Exception as e:
|
45
|
+
raise e
|
46
|
+
finally:
|
47
|
+
fp_log.close()
|
48
|
+
|
49
|
+
if result.returncode > 0:
|
50
|
+
with log_path.open("r") as fp_stderr:
|
51
|
+
err = fp_stderr.read()
|
52
|
+
raise TaskExecutionError(err)
|
53
|
+
elif result.returncode < 0:
|
54
|
+
raise JobExecutionError(
|
55
|
+
info=f"Task failed with returncode={result.returncode}"
|
56
|
+
)
|
57
|
+
|
58
|
+
|
59
|
+
def run_single_task(
|
60
|
+
args: dict[str, Any],
|
61
|
+
command: str,
|
62
|
+
wftask: WorkflowTaskV2,
|
63
|
+
workflow_dir: Path,
|
64
|
+
workflow_dir_user: Optional[Path] = None,
|
65
|
+
logger_name: Optional[str] = None,
|
66
|
+
is_task_v1: bool = False,
|
67
|
+
) -> dict[str, Any]:
|
68
|
+
"""
|
69
|
+
Runs within an executor.
|
70
|
+
"""
|
71
|
+
|
72
|
+
logger = logging.getLogger(logger_name)
|
73
|
+
logger.debug(f"Now start running {command=}")
|
74
|
+
|
75
|
+
if not workflow_dir_user:
|
76
|
+
workflow_dir_user = workflow_dir
|
77
|
+
|
78
|
+
component = args.pop(_COMPONENT_KEY_, None)
|
79
|
+
task_files = get_task_file_paths(
|
80
|
+
workflow_dir=workflow_dir,
|
81
|
+
workflow_dir_user=workflow_dir_user,
|
82
|
+
task_order=wftask.order,
|
83
|
+
component=component,
|
84
|
+
)
|
85
|
+
|
86
|
+
# Write arguments to args.json file
|
87
|
+
with task_files.args.open("w") as f:
|
88
|
+
json.dump(args, f, indent=2)
|
89
|
+
|
90
|
+
# Assemble full command
|
91
|
+
if is_task_v1:
|
92
|
+
full_command = (
|
93
|
+
f"{command} "
|
94
|
+
f"--json {task_files.args.as_posix()} "
|
95
|
+
f"--metadata-out {task_files.metadiff.as_posix()}"
|
96
|
+
)
|
97
|
+
else:
|
98
|
+
full_command = (
|
99
|
+
f"{command} "
|
100
|
+
f"--args-json {task_files.args.as_posix()} "
|
101
|
+
f"--out-json {task_files.metadiff.as_posix()}"
|
102
|
+
)
|
103
|
+
|
104
|
+
try:
|
105
|
+
_call_command_wrapper(
|
106
|
+
full_command,
|
107
|
+
log_path=task_files.log,
|
108
|
+
)
|
109
|
+
except TaskExecutionError as e:
|
110
|
+
e.workflow_task_order = wftask.order
|
111
|
+
e.workflow_task_id = wftask.id
|
112
|
+
if wftask.is_legacy_task:
|
113
|
+
e.task_name = wftask.task_legacy.name
|
114
|
+
else:
|
115
|
+
e.task_name = wftask.task.name
|
116
|
+
raise e
|
117
|
+
|
118
|
+
try:
|
119
|
+
with task_files.metadiff.open("r") as f:
|
120
|
+
out_meta = json.load(f)
|
121
|
+
except FileNotFoundError as e:
|
122
|
+
logger.debug(
|
123
|
+
"Task did not produce output metadata. "
|
124
|
+
f"Original FileNotFoundError: {str(e)}"
|
125
|
+
)
|
126
|
+
out_meta = None
|
127
|
+
|
128
|
+
if out_meta == {}:
|
129
|
+
return None
|
130
|
+
return out_meta
|
@@ -0,0 +1,62 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
3
|
+
from pydantic import BaseModel
|
4
|
+
from pydantic import Field
|
5
|
+
from pydantic import validator
|
6
|
+
|
7
|
+
from ....images import SingleImageTaskOutput
|
8
|
+
from fractal_server.images import Filters
|
9
|
+
from fractal_server.urls import normalize_url
|
10
|
+
|
11
|
+
|
12
|
+
class TaskOutput(BaseModel):
|
13
|
+
class Config:
|
14
|
+
extra = "forbid"
|
15
|
+
|
16
|
+
image_list_updates: list[SingleImageTaskOutput] = Field(
|
17
|
+
default_factory=list
|
18
|
+
)
|
19
|
+
image_list_removals: list[str] = Field(default_factory=list)
|
20
|
+
filters: Filters = Field(default_factory=Filters)
|
21
|
+
|
22
|
+
def check_zarr_urls_are_unique(self) -> None:
|
23
|
+
zarr_urls = [img.zarr_url for img in self.image_list_updates]
|
24
|
+
zarr_urls.extend(self.image_list_removals)
|
25
|
+
if len(zarr_urls) != len(set(zarr_urls)):
|
26
|
+
duplicates = [
|
27
|
+
zarr_url
|
28
|
+
for zarr_url in set(zarr_urls)
|
29
|
+
if zarr_urls.count(zarr_url) > 1
|
30
|
+
]
|
31
|
+
msg = (
|
32
|
+
"TaskOutput "
|
33
|
+
f"({len(self.image_list_updates)} image_list_updates and "
|
34
|
+
f"{len(self.image_list_removals)} image_list_removals) "
|
35
|
+
"has non-unique zarr_urls:"
|
36
|
+
)
|
37
|
+
for duplicate in duplicates:
|
38
|
+
msg = f"{msg}\n{duplicate}"
|
39
|
+
raise ValueError(msg)
|
40
|
+
|
41
|
+
@validator("image_list_removals")
|
42
|
+
def normalize_paths(cls, v: list[str]) -> list[str]:
|
43
|
+
return [normalize_url(zarr_url) for zarr_url in v]
|
44
|
+
|
45
|
+
|
46
|
+
class InitArgsModel(BaseModel):
|
47
|
+
class Config:
|
48
|
+
extra = "forbid"
|
49
|
+
|
50
|
+
zarr_url: str
|
51
|
+
init_args: dict[str, Any] = Field(default_factory=dict)
|
52
|
+
|
53
|
+
@validator("zarr_url")
|
54
|
+
def normalize_path(cls, v: str) -> str:
|
55
|
+
return normalize_url(v)
|
56
|
+
|
57
|
+
|
58
|
+
class InitTaskOutput(BaseModel):
|
59
|
+
class Config:
|
60
|
+
extra = "forbid"
|
61
|
+
|
62
|
+
parallelization_list: list[InitArgsModel] = Field(default_factory=list)
|
@@ -0,0 +1,31 @@
|
|
1
|
+
from copy import deepcopy
|
2
|
+
from pathlib import Path
|
3
|
+
from typing import Any
|
4
|
+
|
5
|
+
|
6
|
+
def convert_v2_args_into_v1(
|
7
|
+
kwargs_v2: dict[str, Any],
|
8
|
+
parallelization_level: str = "image",
|
9
|
+
) -> dict[str, Any]:
|
10
|
+
kwargs_v1 = deepcopy(kwargs_v2)
|
11
|
+
|
12
|
+
zarr_url = kwargs_v1.pop("zarr_url")
|
13
|
+
input_path = Path(zarr_url).parents[3].as_posix()
|
14
|
+
image_component = zarr_url.replace(input_path, "").lstrip("/")
|
15
|
+
if parallelization_level == "image":
|
16
|
+
component = image_component
|
17
|
+
elif parallelization_level == "well":
|
18
|
+
component = str(Path(image_component).parent)
|
19
|
+
elif parallelization_level == "plate":
|
20
|
+
component = str(Path(image_component).parents[2])
|
21
|
+
else:
|
22
|
+
raise ValueError(f"Invalid {parallelization_level=}.")
|
23
|
+
|
24
|
+
kwargs_v1.update(
|
25
|
+
input_paths=[input_path],
|
26
|
+
output_path=input_path,
|
27
|
+
metadata={},
|
28
|
+
component=component,
|
29
|
+
)
|
30
|
+
|
31
|
+
return kwargs_v1
|
@@ -1,42 +1 @@
|
|
1
|
-
|
2
|
-
Schemas for API request/response bodies
|
3
|
-
"""
|
4
|
-
from .applyworkflow import ApplyWorkflowCreate # noqa: F401
|
5
|
-
from .applyworkflow import ApplyWorkflowRead # noqa: F401
|
6
|
-
from .applyworkflow import ApplyWorkflowUpdate # noqa: F401
|
7
|
-
from .applyworkflow import JobStatusType # noqa: F401
|
8
|
-
from .dataset import DatasetCreate # noqa: F401
|
9
|
-
from .dataset import DatasetRead # noqa: F401
|
10
|
-
from .dataset import DatasetStatusRead # noqa: F401
|
11
|
-
from .dataset import DatasetUpdate # noqa: F401
|
12
|
-
from .dataset import ResourceCreate # noqa: F401
|
13
|
-
from .dataset import ResourceRead # noqa: F401
|
14
|
-
from .dataset import ResourceUpdate # noqa: F401
|
15
|
-
from .manifest import ManifestV1 # noqa: F401
|
16
|
-
from .manifest import TaskManifestV1 # noqa: F401
|
17
|
-
from .project import ProjectCreate # noqa: F401
|
18
|
-
from .project import ProjectRead # noqa: F401
|
19
|
-
from .project import ProjectUpdate # noqa: F401
|
20
|
-
from .state import _StateBase # noqa: F401
|
21
|
-
from .state import StateRead # noqa: F401
|
22
|
-
from .task import TaskCreate # noqa: F401
|
23
|
-
from .task import TaskImport # noqa: F401
|
24
|
-
from .task import TaskRead # noqa: F401
|
25
|
-
from .task import TaskUpdate # noqa: F401
|
26
|
-
from .task_collection import TaskCollectPip # noqa: F401
|
27
|
-
from .task_collection import TaskCollectStatus # noqa: F401
|
28
|
-
from .user import UserCreate # noqa: F401
|
29
|
-
from .user import UserRead # noqa: F401
|
30
|
-
from .user import UserUpdate # noqa: F401
|
31
|
-
from .user import UserUpdateStrict # noqa: F401
|
32
|
-
from .workflow import WorkflowCreate # noqa: F401
|
33
|
-
from .workflow import WorkflowExport # noqa: F401
|
34
|
-
from .workflow import WorkflowImport # noqa: F401
|
35
|
-
from .workflow import WorkflowRead # noqa: F401
|
36
|
-
from .workflow import WorkflowTaskCreate # noqa: F401
|
37
|
-
from .workflow import WorkflowTaskExport # noqa: F401
|
38
|
-
from .workflow import WorkflowTaskImport # noqa: F401
|
39
|
-
from .workflow import WorkflowTaskRead # noqa: F401
|
40
|
-
from .workflow import WorkflowTaskStatusType # noqa: F401
|
41
|
-
from .workflow import WorkflowTaskUpdate # noqa: F401
|
42
|
-
from .workflow import WorkflowUpdate # noqa: F401
|
1
|
+
from .user import * # noqa: F401, F403
|