fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/models/__init__.py +6 -8
  3. fractal_server/app/models/linkuserproject.py +9 -0
  4. fractal_server/app/models/security.py +6 -0
  5. fractal_server/app/models/v1/__init__.py +12 -0
  6. fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
  7. fractal_server/app/models/{job.py → v1/job.py} +5 -5
  8. fractal_server/app/models/{project.py → v1/project.py} +5 -5
  9. fractal_server/app/models/{state.py → v1/state.py} +2 -2
  10. fractal_server/app/models/{task.py → v1/task.py} +7 -2
  11. fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
  12. fractal_server/app/models/v2/__init__.py +22 -0
  13. fractal_server/app/models/v2/collection_state.py +21 -0
  14. fractal_server/app/models/v2/dataset.py +54 -0
  15. fractal_server/app/models/v2/job.py +51 -0
  16. fractal_server/app/models/v2/project.py +30 -0
  17. fractal_server/app/models/v2/task.py +93 -0
  18. fractal_server/app/models/v2/workflow.py +35 -0
  19. fractal_server/app/models/v2/workflowtask.py +49 -0
  20. fractal_server/app/routes/admin/__init__.py +0 -0
  21. fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
  22. fractal_server/app/routes/admin/v2.py +309 -0
  23. fractal_server/app/routes/api/v1/__init__.py +7 -7
  24. fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
  25. fractal_server/app/routes/api/v1/dataset.py +41 -41
  26. fractal_server/app/routes/api/v1/job.py +14 -14
  27. fractal_server/app/routes/api/v1/project.py +27 -25
  28. fractal_server/app/routes/api/v1/task.py +26 -16
  29. fractal_server/app/routes/api/v1/task_collection.py +28 -16
  30. fractal_server/app/routes/api/v1/workflow.py +28 -28
  31. fractal_server/app/routes/api/v1/workflowtask.py +11 -11
  32. fractal_server/app/routes/api/v2/__init__.py +34 -0
  33. fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
  34. fractal_server/app/routes/api/v2/dataset.py +293 -0
  35. fractal_server/app/routes/api/v2/images.py +279 -0
  36. fractal_server/app/routes/api/v2/job.py +200 -0
  37. fractal_server/app/routes/api/v2/project.py +186 -0
  38. fractal_server/app/routes/api/v2/status.py +150 -0
  39. fractal_server/app/routes/api/v2/submit.py +210 -0
  40. fractal_server/app/routes/api/v2/task.py +222 -0
  41. fractal_server/app/routes/api/v2/task_collection.py +239 -0
  42. fractal_server/app/routes/api/v2/task_legacy.py +59 -0
  43. fractal_server/app/routes/api/v2/workflow.py +380 -0
  44. fractal_server/app/routes/api/v2/workflowtask.py +265 -0
  45. fractal_server/app/routes/aux/_job.py +2 -2
  46. fractal_server/app/runner/__init__.py +0 -364
  47. fractal_server/app/runner/async_wrap.py +27 -0
  48. fractal_server/app/runner/components.py +5 -0
  49. fractal_server/app/runner/exceptions.py +129 -0
  50. fractal_server/app/runner/executors/__init__.py +0 -0
  51. fractal_server/app/runner/executors/slurm/__init__.py +3 -0
  52. fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
  53. fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
  54. fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
  55. fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
  56. fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
  57. fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +32 -21
  58. fractal_server/app/runner/filenames.py +6 -0
  59. fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
  60. fractal_server/app/runner/task_files.py +103 -0
  61. fractal_server/app/runner/v1/__init__.py +366 -0
  62. fractal_server/app/runner/{_common.py → v1/_common.py} +14 -121
  63. fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
  64. fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
  65. fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
  66. fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
  67. fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
  68. fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
  69. fractal_server/app/runner/v1/common.py +117 -0
  70. fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
  71. fractal_server/app/runner/v2/__init__.py +336 -0
  72. fractal_server/app/runner/v2/_local/__init__.py +162 -0
  73. fractal_server/app/runner/v2/_local/_local_config.py +118 -0
  74. fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
  75. fractal_server/app/runner/v2/_local/executor.py +100 -0
  76. fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
  77. fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
  78. fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
  79. fractal_server/app/runner/v2/deduplicate_list.py +23 -0
  80. fractal_server/app/runner/v2/handle_failed_job.py +165 -0
  81. fractal_server/app/runner/v2/merge_outputs.py +38 -0
  82. fractal_server/app/runner/v2/runner.py +343 -0
  83. fractal_server/app/runner/v2/runner_functions.py +374 -0
  84. fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
  85. fractal_server/app/runner/v2/task_interface.py +62 -0
  86. fractal_server/app/runner/v2/v1_compat.py +31 -0
  87. fractal_server/app/schemas/__init__.py +1 -42
  88. fractal_server/app/schemas/_validators.py +28 -5
  89. fractal_server/app/schemas/v1/__init__.py +36 -0
  90. fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
  91. fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
  92. fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
  93. fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
  94. fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
  95. fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
  96. fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
  97. fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
  98. fractal_server/app/schemas/v2/__init__.py +37 -0
  99. fractal_server/app/schemas/v2/dataset.py +126 -0
  100. fractal_server/app/schemas/v2/dumps.py +87 -0
  101. fractal_server/app/schemas/v2/job.py +114 -0
  102. fractal_server/app/schemas/v2/manifest.py +159 -0
  103. fractal_server/app/schemas/v2/project.py +34 -0
  104. fractal_server/app/schemas/v2/status.py +16 -0
  105. fractal_server/app/schemas/v2/task.py +151 -0
  106. fractal_server/app/schemas/v2/task_collection.py +109 -0
  107. fractal_server/app/schemas/v2/workflow.py +79 -0
  108. fractal_server/app/schemas/v2/workflowtask.py +208 -0
  109. fractal_server/config.py +5 -4
  110. fractal_server/images/__init__.py +4 -0
  111. fractal_server/images/models.py +136 -0
  112. fractal_server/images/tools.py +84 -0
  113. fractal_server/main.py +11 -3
  114. fractal_server/migrations/env.py +0 -2
  115. fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
  116. fractal_server/tasks/__init__.py +0 -5
  117. fractal_server/tasks/endpoint_operations.py +13 -19
  118. fractal_server/tasks/utils.py +35 -0
  119. fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
  120. fractal_server/tasks/v1/__init__.py +0 -0
  121. fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
  122. fractal_server/tasks/v1/get_collection_data.py +14 -0
  123. fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
  124. fractal_server/tasks/v2/__init__.py +0 -0
  125. fractal_server/tasks/v2/background_operations.py +381 -0
  126. fractal_server/tasks/v2/get_collection_data.py +14 -0
  127. fractal_server/urls.py +13 -0
  128. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/METADATA +10 -10
  129. fractal_server-2.0.0.dist-info/RECORD +169 -0
  130. fractal_server/app/runner/_slurm/.gitignore +0 -2
  131. fractal_server/app/runner/common.py +0 -311
  132. fractal_server/app/schemas/json_schemas/manifest.json +0 -81
  133. fractal_server-1.4.10.dist-info/RECORD +0 -98
  134. /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
  135. /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
  136. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
  137. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
  138. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,374 @@
1
+ import functools
2
+ import logging
3
+ import traceback
4
+ from concurrent.futures import Executor
5
+ from pathlib import Path
6
+ from typing import Any
7
+ from typing import Callable
8
+ from typing import Literal
9
+ from typing import Optional
10
+
11
+ from pydantic import ValidationError
12
+
13
+ from ..exceptions import JobExecutionError
14
+ from .deduplicate_list import deduplicate_list
15
+ from .merge_outputs import merge_outputs
16
+ from .runner_functions_low_level import run_single_task
17
+ from .task_interface import InitTaskOutput
18
+ from .task_interface import TaskOutput
19
+ from .v1_compat import convert_v2_args_into_v1
20
+ from fractal_server.app.models.v1 import Task as TaskV1
21
+ from fractal_server.app.models.v2 import TaskV2
22
+ from fractal_server.app.models.v2 import WorkflowTaskV2
23
+ from fractal_server.app.runner.components import _COMPONENT_KEY_
24
+ from fractal_server.app.runner.components import _index_to_component
25
+
26
+
27
+ __all__ = [
28
+ "run_v2_task_non_parallel",
29
+ "run_v2_task_parallel",
30
+ "run_v2_task_compound",
31
+ "run_v1_task_parallel",
32
+ ]
33
+
34
+ MAX_PARALLELIZATION_LIST_SIZE = 20_000
35
+
36
+
37
+ def _cast_and_validate_TaskOutput(
38
+ task_output: dict[str, Any]
39
+ ) -> Optional[TaskOutput]:
40
+ try:
41
+ validated_task_output = TaskOutput(**task_output)
42
+ return validated_task_output
43
+ except ValidationError as e:
44
+ raise JobExecutionError(
45
+ "Validation of task output failed.\n"
46
+ f"Original error: {str(e)}\n"
47
+ f"Original data: {task_output}."
48
+ )
49
+
50
+
51
+ def _cast_and_validate_InitTaskOutput(
52
+ init_task_output: dict[str, Any],
53
+ ) -> Optional[InitTaskOutput]:
54
+ try:
55
+ validated_init_task_output = InitTaskOutput(**init_task_output)
56
+ return validated_init_task_output
57
+ except ValidationError as e:
58
+ raise JobExecutionError(
59
+ "Validation of init-task output failed.\n"
60
+ f"Original error: {str(e)}\n"
61
+ f"Original data: {init_task_output}."
62
+ )
63
+
64
+
65
+ def no_op_submit_setup_call(
66
+ *,
67
+ wftask: WorkflowTaskV2,
68
+ workflow_dir: Path,
69
+ workflow_dir_user: Path,
70
+ which_type: Literal["non_parallel", "parallel"],
71
+ ) -> dict:
72
+ """
73
+ Default (no-operation) interface of submit_setup_call in V2.
74
+ """
75
+ return {}
76
+
77
+
78
+ # Backend-specific configuration
79
+ def _get_executor_options(
80
+ *,
81
+ wftask: WorkflowTaskV2,
82
+ workflow_dir: Path,
83
+ workflow_dir_user: Path,
84
+ submit_setup_call: Callable,
85
+ which_type: Literal["non_parallel", "parallel"],
86
+ ) -> dict:
87
+ try:
88
+ options = submit_setup_call(
89
+ wftask=wftask,
90
+ workflow_dir=workflow_dir,
91
+ workflow_dir_user=workflow_dir_user,
92
+ which_type=which_type,
93
+ )
94
+ except Exception as e:
95
+ tb = "".join(traceback.format_tb(e.__traceback__))
96
+ raise RuntimeError(
97
+ f"{type(e)} error in {submit_setup_call=}\n"
98
+ f"Original traceback:\n{tb}"
99
+ )
100
+ return options
101
+
102
+
103
+ def _check_parallelization_list_size(my_list):
104
+ if len(my_list) > MAX_PARALLELIZATION_LIST_SIZE:
105
+ raise JobExecutionError(
106
+ "Too many parallelization items.\n"
107
+ f" {len(my_list)}\n"
108
+ f" {MAX_PARALLELIZATION_LIST_SIZE=}\n"
109
+ )
110
+
111
+
112
+ def run_v2_task_non_parallel(
113
+ *,
114
+ images: list[dict[str, Any]],
115
+ zarr_dir: str,
116
+ task: TaskV2,
117
+ wftask: WorkflowTaskV2,
118
+ workflow_dir: Path,
119
+ workflow_dir_user: Optional[Path] = None,
120
+ executor: Executor,
121
+ logger_name: Optional[str] = None,
122
+ submit_setup_call: Callable = no_op_submit_setup_call,
123
+ ) -> TaskOutput:
124
+ """
125
+ This runs server-side (see `executor` argument)
126
+ """
127
+
128
+ if workflow_dir_user is None:
129
+ workflow_dir_user = workflow_dir
130
+ logging.warning(
131
+ "In `run_single_task`, workflow_dir_user=None. Is this right?"
132
+ )
133
+ workflow_dir_user = workflow_dir
134
+
135
+ executor_options = _get_executor_options(
136
+ wftask=wftask,
137
+ workflow_dir=workflow_dir,
138
+ workflow_dir_user=workflow_dir_user,
139
+ submit_setup_call=submit_setup_call,
140
+ which_type="non_parallel",
141
+ )
142
+
143
+ function_kwargs = dict(
144
+ zarr_urls=[image["zarr_url"] for image in images],
145
+ zarr_dir=zarr_dir,
146
+ **(wftask.args_non_parallel or {}),
147
+ )
148
+ future = executor.submit(
149
+ functools.partial(
150
+ run_single_task,
151
+ wftask=wftask,
152
+ command=task.command_non_parallel,
153
+ workflow_dir=workflow_dir,
154
+ workflow_dir_user=workflow_dir_user,
155
+ ),
156
+ function_kwargs,
157
+ **executor_options,
158
+ )
159
+ output = future.result()
160
+ if output is None:
161
+ return TaskOutput()
162
+ else:
163
+ return _cast_and_validate_TaskOutput(output)
164
+
165
+
166
+ def run_v2_task_parallel(
167
+ *,
168
+ images: list[dict[str, Any]],
169
+ task: TaskV2,
170
+ wftask: WorkflowTaskV2,
171
+ executor: Executor,
172
+ workflow_dir: Path,
173
+ workflow_dir_user: Optional[Path] = None,
174
+ logger_name: Optional[str] = None,
175
+ submit_setup_call: Callable = no_op_submit_setup_call,
176
+ ) -> TaskOutput:
177
+
178
+ if len(images) == 0:
179
+ return TaskOutput()
180
+
181
+ _check_parallelization_list_size(images)
182
+
183
+ executor_options = _get_executor_options(
184
+ wftask=wftask,
185
+ workflow_dir=workflow_dir,
186
+ workflow_dir_user=workflow_dir_user,
187
+ submit_setup_call=submit_setup_call,
188
+ which_type="parallel",
189
+ )
190
+
191
+ list_function_kwargs = []
192
+ for ind, image in enumerate(images):
193
+ list_function_kwargs.append(
194
+ dict(
195
+ zarr_url=image["zarr_url"],
196
+ **(wftask.args_parallel or {}),
197
+ ),
198
+ )
199
+ list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
200
+
201
+ results_iterator = executor.map(
202
+ functools.partial(
203
+ run_single_task,
204
+ wftask=wftask,
205
+ command=task.command_parallel,
206
+ workflow_dir=workflow_dir,
207
+ workflow_dir_user=workflow_dir_user,
208
+ ),
209
+ list_function_kwargs,
210
+ **executor_options,
211
+ )
212
+ # Explicitly iterate over the whole list, so that all futures are waited
213
+ outputs = list(results_iterator)
214
+
215
+ # Validate all non-None outputs
216
+ for ind, output in enumerate(outputs):
217
+ if output is None:
218
+ outputs[ind] = TaskOutput()
219
+ else:
220
+ outputs[ind] = _cast_and_validate_TaskOutput(output)
221
+
222
+ merged_output = merge_outputs(outputs)
223
+ return merged_output
224
+
225
+
226
+ def run_v2_task_compound(
227
+ *,
228
+ images: list[dict[str, Any]],
229
+ zarr_dir: str,
230
+ task: TaskV2,
231
+ wftask: WorkflowTaskV2,
232
+ executor: Executor,
233
+ workflow_dir: Path,
234
+ workflow_dir_user: Optional[Path] = None,
235
+ logger_name: Optional[str] = None,
236
+ submit_setup_call: Callable = no_op_submit_setup_call,
237
+ ) -> TaskOutput:
238
+
239
+ executor_options_init = _get_executor_options(
240
+ wftask=wftask,
241
+ workflow_dir=workflow_dir,
242
+ workflow_dir_user=workflow_dir_user,
243
+ submit_setup_call=submit_setup_call,
244
+ which_type="non_parallel",
245
+ )
246
+ executor_options_compute = _get_executor_options(
247
+ wftask=wftask,
248
+ workflow_dir=workflow_dir,
249
+ workflow_dir_user=workflow_dir_user,
250
+ submit_setup_call=submit_setup_call,
251
+ which_type="parallel",
252
+ )
253
+
254
+ # 3/A: non-parallel init task
255
+ function_kwargs = dict(
256
+ zarr_urls=[image["zarr_url"] for image in images],
257
+ zarr_dir=zarr_dir,
258
+ **(wftask.args_non_parallel or {}),
259
+ )
260
+ future = executor.submit(
261
+ functools.partial(
262
+ run_single_task,
263
+ wftask=wftask,
264
+ command=task.command_non_parallel,
265
+ workflow_dir=workflow_dir,
266
+ workflow_dir_user=workflow_dir_user,
267
+ ),
268
+ function_kwargs,
269
+ **executor_options_init,
270
+ )
271
+ output = future.result()
272
+ if output is None:
273
+ init_task_output = InitTaskOutput()
274
+ else:
275
+ init_task_output = _cast_and_validate_InitTaskOutput(output)
276
+ parallelization_list = init_task_output.parallelization_list
277
+ parallelization_list = deduplicate_list(parallelization_list)
278
+
279
+ # 3/B: parallel part of a compound task
280
+ _check_parallelization_list_size(parallelization_list)
281
+
282
+ if len(parallelization_list) == 0:
283
+ return TaskOutput()
284
+
285
+ list_function_kwargs = []
286
+ for ind, parallelization_item in enumerate(parallelization_list):
287
+ list_function_kwargs.append(
288
+ dict(
289
+ zarr_url=parallelization_item.zarr_url,
290
+ init_args=parallelization_item.init_args,
291
+ **(wftask.args_parallel or {}),
292
+ ),
293
+ )
294
+ list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
295
+
296
+ results_iterator = executor.map(
297
+ functools.partial(
298
+ run_single_task,
299
+ wftask=wftask,
300
+ command=task.command_parallel,
301
+ workflow_dir=workflow_dir,
302
+ workflow_dir_user=workflow_dir_user,
303
+ ),
304
+ list_function_kwargs,
305
+ **executor_options_compute,
306
+ )
307
+ # Explicitly iterate over the whole list, so that all futures are waited
308
+ outputs = list(results_iterator)
309
+
310
+ # Validate all non-None outputs
311
+ for ind, output in enumerate(outputs):
312
+ if output is None:
313
+ outputs[ind] = TaskOutput()
314
+ else:
315
+ validated_output = _cast_and_validate_TaskOutput(output)
316
+ outputs[ind] = validated_output
317
+
318
+ merged_output = merge_outputs(outputs)
319
+ return merged_output
320
+
321
+
322
+ def run_v1_task_parallel(
323
+ *,
324
+ images: list[dict[str, Any]],
325
+ task_legacy: TaskV1,
326
+ wftask: WorkflowTaskV2,
327
+ executor: Executor,
328
+ workflow_dir: Path,
329
+ workflow_dir_user: Optional[Path] = None,
330
+ logger_name: Optional[str] = None,
331
+ submit_setup_call: Callable = no_op_submit_setup_call,
332
+ ) -> TaskOutput:
333
+
334
+ _check_parallelization_list_size(images)
335
+
336
+ executor_options = _get_executor_options(
337
+ wftask=wftask,
338
+ workflow_dir=workflow_dir,
339
+ workflow_dir_user=workflow_dir_user,
340
+ submit_setup_call=submit_setup_call,
341
+ which_type="parallel",
342
+ )
343
+
344
+ list_function_kwargs = []
345
+ for ind, image in enumerate(images):
346
+ list_function_kwargs.append(
347
+ convert_v2_args_into_v1(
348
+ kwargs_v2=dict(
349
+ zarr_url=image["zarr_url"],
350
+ **(wftask.args_parallel or {}),
351
+ ),
352
+ parallelization_level=task_legacy.parallelization_level,
353
+ ),
354
+ )
355
+ list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
356
+
357
+ results_iterator = executor.map(
358
+ functools.partial(
359
+ run_single_task,
360
+ wftask=wftask,
361
+ command=task_legacy.command,
362
+ workflow_dir=workflow_dir,
363
+ workflow_dir_user=workflow_dir_user,
364
+ is_task_v1=True,
365
+ ),
366
+ list_function_kwargs,
367
+ **executor_options,
368
+ )
369
+ # Explicitly iterate over the whole list, so that all futures are waited
370
+ list(results_iterator)
371
+
372
+ # Ignore any output metadata for V1 tasks, and return an empty object
373
+ out = TaskOutput()
374
+ return out
@@ -0,0 +1,130 @@
1
+ import json
2
+ import logging
3
+ import shutil
4
+ import subprocess # nosec
5
+ from pathlib import Path
6
+ from shlex import split as shlex_split
7
+ from typing import Any
8
+ from typing import Optional
9
+
10
+ from ..components import _COMPONENT_KEY_
11
+ from ..exceptions import JobExecutionError
12
+ from ..exceptions import TaskExecutionError
13
+ from fractal_server.app.models.v2 import WorkflowTaskV2
14
+ from fractal_server.app.runner.task_files import get_task_file_paths
15
+
16
+
17
+ def _call_command_wrapper(cmd: str, log_path: Path) -> None:
18
+ """
19
+ Call a command and write its stdout and stderr to files
20
+
21
+ Raises:
22
+ TaskExecutionError: If the `subprocess.run` call returns a positive
23
+ exit code
24
+ JobExecutionError: If the `subprocess.run` call returns a negative
25
+ exit code (e.g. due to the subprocess receiving a
26
+ TERM or KILL signal)
27
+ """
28
+
29
+ # Verify that task command is executable
30
+ if shutil.which(shlex_split(cmd)[0]) is None:
31
+ msg = (
32
+ f'Command "{shlex_split(cmd)[0]}" is not valid. '
33
+ "Hint: make sure that it is executable."
34
+ )
35
+ raise TaskExecutionError(msg)
36
+
37
+ fp_log = open(log_path, "w")
38
+ try:
39
+ result = subprocess.run( # nosec
40
+ shlex_split(cmd),
41
+ stderr=fp_log,
42
+ stdout=fp_log,
43
+ )
44
+ except Exception as e:
45
+ raise e
46
+ finally:
47
+ fp_log.close()
48
+
49
+ if result.returncode > 0:
50
+ with log_path.open("r") as fp_stderr:
51
+ err = fp_stderr.read()
52
+ raise TaskExecutionError(err)
53
+ elif result.returncode < 0:
54
+ raise JobExecutionError(
55
+ info=f"Task failed with returncode={result.returncode}"
56
+ )
57
+
58
+
59
+ def run_single_task(
60
+ args: dict[str, Any],
61
+ command: str,
62
+ wftask: WorkflowTaskV2,
63
+ workflow_dir: Path,
64
+ workflow_dir_user: Optional[Path] = None,
65
+ logger_name: Optional[str] = None,
66
+ is_task_v1: bool = False,
67
+ ) -> dict[str, Any]:
68
+ """
69
+ Runs within an executor.
70
+ """
71
+
72
+ logger = logging.getLogger(logger_name)
73
+ logger.debug(f"Now start running {command=}")
74
+
75
+ if not workflow_dir_user:
76
+ workflow_dir_user = workflow_dir
77
+
78
+ component = args.pop(_COMPONENT_KEY_, None)
79
+ task_files = get_task_file_paths(
80
+ workflow_dir=workflow_dir,
81
+ workflow_dir_user=workflow_dir_user,
82
+ task_order=wftask.order,
83
+ component=component,
84
+ )
85
+
86
+ # Write arguments to args.json file
87
+ with task_files.args.open("w") as f:
88
+ json.dump(args, f, indent=2)
89
+
90
+ # Assemble full command
91
+ if is_task_v1:
92
+ full_command = (
93
+ f"{command} "
94
+ f"--json {task_files.args.as_posix()} "
95
+ f"--metadata-out {task_files.metadiff.as_posix()}"
96
+ )
97
+ else:
98
+ full_command = (
99
+ f"{command} "
100
+ f"--args-json {task_files.args.as_posix()} "
101
+ f"--out-json {task_files.metadiff.as_posix()}"
102
+ )
103
+
104
+ try:
105
+ _call_command_wrapper(
106
+ full_command,
107
+ log_path=task_files.log,
108
+ )
109
+ except TaskExecutionError as e:
110
+ e.workflow_task_order = wftask.order
111
+ e.workflow_task_id = wftask.id
112
+ if wftask.is_legacy_task:
113
+ e.task_name = wftask.task_legacy.name
114
+ else:
115
+ e.task_name = wftask.task.name
116
+ raise e
117
+
118
+ try:
119
+ with task_files.metadiff.open("r") as f:
120
+ out_meta = json.load(f)
121
+ except FileNotFoundError as e:
122
+ logger.debug(
123
+ "Task did not produce output metadata. "
124
+ f"Original FileNotFoundError: {str(e)}"
125
+ )
126
+ out_meta = None
127
+
128
+ if out_meta == {}:
129
+ return None
130
+ return out_meta
@@ -0,0 +1,62 @@
1
+ from typing import Any
2
+
3
+ from pydantic import BaseModel
4
+ from pydantic import Field
5
+ from pydantic import validator
6
+
7
+ from ....images import SingleImageTaskOutput
8
+ from fractal_server.images import Filters
9
+ from fractal_server.urls import normalize_url
10
+
11
+
12
+ class TaskOutput(BaseModel):
13
+ class Config:
14
+ extra = "forbid"
15
+
16
+ image_list_updates: list[SingleImageTaskOutput] = Field(
17
+ default_factory=list
18
+ )
19
+ image_list_removals: list[str] = Field(default_factory=list)
20
+ filters: Filters = Field(default_factory=Filters)
21
+
22
+ def check_zarr_urls_are_unique(self) -> None:
23
+ zarr_urls = [img.zarr_url for img in self.image_list_updates]
24
+ zarr_urls.extend(self.image_list_removals)
25
+ if len(zarr_urls) != len(set(zarr_urls)):
26
+ duplicates = [
27
+ zarr_url
28
+ for zarr_url in set(zarr_urls)
29
+ if zarr_urls.count(zarr_url) > 1
30
+ ]
31
+ msg = (
32
+ "TaskOutput "
33
+ f"({len(self.image_list_updates)} image_list_updates and "
34
+ f"{len(self.image_list_removals)} image_list_removals) "
35
+ "has non-unique zarr_urls:"
36
+ )
37
+ for duplicate in duplicates:
38
+ msg = f"{msg}\n{duplicate}"
39
+ raise ValueError(msg)
40
+
41
+ @validator("image_list_removals")
42
+ def normalize_paths(cls, v: list[str]) -> list[str]:
43
+ return [normalize_url(zarr_url) for zarr_url in v]
44
+
45
+
46
+ class InitArgsModel(BaseModel):
47
+ class Config:
48
+ extra = "forbid"
49
+
50
+ zarr_url: str
51
+ init_args: dict[str, Any] = Field(default_factory=dict)
52
+
53
+ @validator("zarr_url")
54
+ def normalize_path(cls, v: str) -> str:
55
+ return normalize_url(v)
56
+
57
+
58
+ class InitTaskOutput(BaseModel):
59
+ class Config:
60
+ extra = "forbid"
61
+
62
+ parallelization_list: list[InitArgsModel] = Field(default_factory=list)
@@ -0,0 +1,31 @@
1
+ from copy import deepcopy
2
+ from pathlib import Path
3
+ from typing import Any
4
+
5
+
6
+ def convert_v2_args_into_v1(
7
+ kwargs_v2: dict[str, Any],
8
+ parallelization_level: str = "image",
9
+ ) -> dict[str, Any]:
10
+ kwargs_v1 = deepcopy(kwargs_v2)
11
+
12
+ zarr_url = kwargs_v1.pop("zarr_url")
13
+ input_path = Path(zarr_url).parents[3].as_posix()
14
+ image_component = zarr_url.replace(input_path, "").lstrip("/")
15
+ if parallelization_level == "image":
16
+ component = image_component
17
+ elif parallelization_level == "well":
18
+ component = str(Path(image_component).parent)
19
+ elif parallelization_level == "plate":
20
+ component = str(Path(image_component).parents[2])
21
+ else:
22
+ raise ValueError(f"Invalid {parallelization_level=}.")
23
+
24
+ kwargs_v1.update(
25
+ input_paths=[input_path],
26
+ output_path=input_path,
27
+ metadata={},
28
+ component=component,
29
+ )
30
+
31
+ return kwargs_v1
@@ -1,42 +1 @@
1
- """
2
- Schemas for API request/response bodies
3
- """
4
- from .applyworkflow import ApplyWorkflowCreate # noqa: F401
5
- from .applyworkflow import ApplyWorkflowRead # noqa: F401
6
- from .applyworkflow import ApplyWorkflowUpdate # noqa: F401
7
- from .applyworkflow import JobStatusType # noqa: F401
8
- from .dataset import DatasetCreate # noqa: F401
9
- from .dataset import DatasetRead # noqa: F401
10
- from .dataset import DatasetStatusRead # noqa: F401
11
- from .dataset import DatasetUpdate # noqa: F401
12
- from .dataset import ResourceCreate # noqa: F401
13
- from .dataset import ResourceRead # noqa: F401
14
- from .dataset import ResourceUpdate # noqa: F401
15
- from .manifest import ManifestV1 # noqa: F401
16
- from .manifest import TaskManifestV1 # noqa: F401
17
- from .project import ProjectCreate # noqa: F401
18
- from .project import ProjectRead # noqa: F401
19
- from .project import ProjectUpdate # noqa: F401
20
- from .state import _StateBase # noqa: F401
21
- from .state import StateRead # noqa: F401
22
- from .task import TaskCreate # noqa: F401
23
- from .task import TaskImport # noqa: F401
24
- from .task import TaskRead # noqa: F401
25
- from .task import TaskUpdate # noqa: F401
26
- from .task_collection import TaskCollectPip # noqa: F401
27
- from .task_collection import TaskCollectStatus # noqa: F401
28
- from .user import UserCreate # noqa: F401
29
- from .user import UserRead # noqa: F401
30
- from .user import UserUpdate # noqa: F401
31
- from .user import UserUpdateStrict # noqa: F401
32
- from .workflow import WorkflowCreate # noqa: F401
33
- from .workflow import WorkflowExport # noqa: F401
34
- from .workflow import WorkflowImport # noqa: F401
35
- from .workflow import WorkflowRead # noqa: F401
36
- from .workflow import WorkflowTaskCreate # noqa: F401
37
- from .workflow import WorkflowTaskExport # noqa: F401
38
- from .workflow import WorkflowTaskImport # noqa: F401
39
- from .workflow import WorkflowTaskRead # noqa: F401
40
- from .workflow import WorkflowTaskStatusType # noqa: F401
41
- from .workflow import WorkflowTaskUpdate # noqa: F401
42
- from .workflow import WorkflowUpdate # noqa: F401
1
+ from .user import * # noqa: F401, F403