fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/models/__init__.py +4 -7
  3. fractal_server/app/models/linkuserproject.py +9 -0
  4. fractal_server/app/models/security.py +6 -0
  5. fractal_server/app/models/state.py +1 -1
  6. fractal_server/app/models/v1/__init__.py +10 -0
  7. fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
  8. fractal_server/app/models/{job.py → v1/job.py} +5 -5
  9. fractal_server/app/models/{project.py → v1/project.py} +5 -5
  10. fractal_server/app/models/{task.py → v1/task.py} +7 -2
  11. fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
  12. fractal_server/app/models/v2/__init__.py +20 -0
  13. fractal_server/app/models/v2/dataset.py +55 -0
  14. fractal_server/app/models/v2/job.py +51 -0
  15. fractal_server/app/models/v2/project.py +31 -0
  16. fractal_server/app/models/v2/task.py +93 -0
  17. fractal_server/app/models/v2/workflow.py +43 -0
  18. fractal_server/app/models/v2/workflowtask.py +90 -0
  19. fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
  20. fractal_server/app/routes/admin/v2.py +275 -0
  21. fractal_server/app/routes/api/v1/__init__.py +7 -7
  22. fractal_server/app/routes/api/v1/_aux_functions.py +2 -2
  23. fractal_server/app/routes/api/v1/dataset.py +37 -37
  24. fractal_server/app/routes/api/v1/job.py +12 -12
  25. fractal_server/app/routes/api/v1/project.py +23 -21
  26. fractal_server/app/routes/api/v1/task.py +24 -14
  27. fractal_server/app/routes/api/v1/task_collection.py +16 -14
  28. fractal_server/app/routes/api/v1/workflow.py +24 -24
  29. fractal_server/app/routes/api/v1/workflowtask.py +10 -10
  30. fractal_server/app/routes/api/v2/__init__.py +28 -0
  31. fractal_server/app/routes/api/v2/_aux_functions.py +497 -0
  32. fractal_server/app/routes/api/v2/apply.py +220 -0
  33. fractal_server/app/routes/api/v2/dataset.py +310 -0
  34. fractal_server/app/routes/api/v2/images.py +212 -0
  35. fractal_server/app/routes/api/v2/job.py +200 -0
  36. fractal_server/app/routes/api/v2/project.py +205 -0
  37. fractal_server/app/routes/api/v2/task.py +222 -0
  38. fractal_server/app/routes/api/v2/task_collection.py +229 -0
  39. fractal_server/app/routes/api/v2/workflow.py +398 -0
  40. fractal_server/app/routes/api/v2/workflowtask.py +269 -0
  41. fractal_server/app/routes/aux/_job.py +1 -1
  42. fractal_server/app/runner/async_wrap.py +27 -0
  43. fractal_server/app/runner/exceptions.py +129 -0
  44. fractal_server/app/runner/executors/local/__init__.py +3 -0
  45. fractal_server/app/runner/{_local → executors/local}/executor.py +2 -2
  46. fractal_server/app/runner/executors/slurm/__init__.py +3 -0
  47. fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
  48. fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
  49. fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
  50. fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
  51. fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
  52. fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +9 -9
  53. fractal_server/app/runner/filenames.py +6 -0
  54. fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
  55. fractal_server/app/runner/task_files.py +105 -0
  56. fractal_server/app/runner/{__init__.py → v1/__init__.py} +24 -22
  57. fractal_server/app/runner/{_common.py → v1/_common.py} +13 -120
  58. fractal_server/app/runner/{_local → v1/_local}/__init__.py +6 -6
  59. fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
  60. fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
  61. fractal_server/app/runner/v1/_slurm/__init__.py +310 -0
  62. fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +3 -9
  63. fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
  64. fractal_server/app/runner/v1/common.py +117 -0
  65. fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
  66. fractal_server/app/runner/v2/__init__.py +337 -0
  67. fractal_server/app/runner/v2/_local/__init__.py +169 -0
  68. fractal_server/app/runner/v2/_local/_local_config.py +118 -0
  69. fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
  70. fractal_server/app/runner/v2/_slurm/__init__.py +157 -0
  71. fractal_server/app/runner/v2/_slurm/_submit_setup.py +83 -0
  72. fractal_server/app/runner/v2/_slurm/get_slurm_config.py +179 -0
  73. fractal_server/app/runner/v2/components.py +5 -0
  74. fractal_server/app/runner/v2/deduplicate_list.py +24 -0
  75. fractal_server/app/runner/v2/handle_failed_job.py +156 -0
  76. fractal_server/app/runner/v2/merge_outputs.py +41 -0
  77. fractal_server/app/runner/v2/runner.py +264 -0
  78. fractal_server/app/runner/v2/runner_functions.py +339 -0
  79. fractal_server/app/runner/v2/runner_functions_low_level.py +134 -0
  80. fractal_server/app/runner/v2/task_interface.py +43 -0
  81. fractal_server/app/runner/v2/v1_compat.py +21 -0
  82. fractal_server/app/schemas/__init__.py +4 -42
  83. fractal_server/app/schemas/v1/__init__.py +42 -0
  84. fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
  85. fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
  86. fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
  87. fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
  88. fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
  89. fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
  90. fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
  91. fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
  92. fractal_server/app/schemas/v2/__init__.py +34 -0
  93. fractal_server/app/schemas/v2/dataset.py +88 -0
  94. fractal_server/app/schemas/v2/dumps.py +87 -0
  95. fractal_server/app/schemas/v2/job.py +113 -0
  96. fractal_server/app/schemas/v2/manifest.py +109 -0
  97. fractal_server/app/schemas/v2/project.py +36 -0
  98. fractal_server/app/schemas/v2/task.py +121 -0
  99. fractal_server/app/schemas/v2/task_collection.py +105 -0
  100. fractal_server/app/schemas/v2/workflow.py +78 -0
  101. fractal_server/app/schemas/v2/workflowtask.py +118 -0
  102. fractal_server/config.py +5 -4
  103. fractal_server/images/__init__.py +50 -0
  104. fractal_server/images/tools.py +86 -0
  105. fractal_server/main.py +11 -3
  106. fractal_server/migrations/versions/4b35c5cefbe3_tmp_is_v2_compatible.py +39 -0
  107. fractal_server/migrations/versions/56af171b0159_v2.py +217 -0
  108. fractal_server/migrations/versions/876f28db9d4e_tmp_split_task_and_wftask_meta.py +68 -0
  109. fractal_server/migrations/versions/974c802f0dd0_tmp_workflowtaskv2_type_in_db.py +37 -0
  110. fractal_server/migrations/versions/9cd305cd6023_tmp_workflowtaskv2.py +40 -0
  111. fractal_server/migrations/versions/a6231ed6273c_tmp_args_schemas_in_taskv2.py +42 -0
  112. fractal_server/migrations/versions/b9e9eed9d442_tmp_taskv2_type.py +37 -0
  113. fractal_server/migrations/versions/e3e639454d4b_tmp_make_task_meta_non_optional.py +50 -0
  114. fractal_server/tasks/__init__.py +0 -5
  115. fractal_server/tasks/endpoint_operations.py +13 -19
  116. fractal_server/tasks/utils.py +35 -0
  117. fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
  118. fractal_server/tasks/{background_operations.py → v1/background_operations.py} +18 -50
  119. fractal_server/tasks/v1/get_collection_data.py +14 -0
  120. fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
  121. fractal_server/tasks/v2/background_operations.py +382 -0
  122. fractal_server/tasks/v2/get_collection_data.py +14 -0
  123. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/METADATA +1 -1
  124. fractal_server-2.0.0a0.dist-info/RECORD +166 -0
  125. fractal_server/app/runner/_slurm/.gitignore +0 -2
  126. fractal_server/app/runner/_slurm/__init__.py +0 -150
  127. fractal_server/app/runner/common.py +0 -311
  128. fractal_server-1.4.10.dist-info/RECORD +0 -98
  129. /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
  130. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/LICENSE +0 -0
  131. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/WHEEL +0 -0
  132. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,310 @@
1
+ # Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
2
+ # University of Zurich
3
+ #
4
+ # Original authors:
5
+ # Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
6
+ # Tommaso Comparin <tommaso.comparin@exact-lab.it>
7
+ # Marco Franzon <marco.franzon@exact-lab.it>
8
+ #
9
+ # This file is part of Fractal and was originally developed by eXact lab S.r.l.
10
+ # <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
11
+ # Institute for Biomedical Research and Pelkmans Lab from the University of
12
+ # Zurich.
13
+ """
14
+ Slurm Bakend
15
+
16
+ This backend runs fractal workflows in a SLURM cluster using Clusterfutures
17
+ Executor objects.
18
+ """
19
+ from pathlib import Path
20
+ from typing import Any
21
+ from typing import Optional
22
+ from typing import Union
23
+
24
+ from ...async_wrap import async_wrap
25
+ from ...executors.slurm.executor import FractalSlurmExecutor
26
+ from ...set_start_and_last_task_index import set_start_and_last_task_index
27
+ from .._common import execute_tasks
28
+ from ..common import TaskParameters
29
+ from ._submit_setup import _slurm_submit_setup
30
+ from fractal_server.app.models.v1 import Workflow
31
+ from fractal_server.app.models.v1 import WorkflowTask
32
+ from fractal_server.app.runner.executors.slurm._slurm_config import (
33
+ _parse_mem_value,
34
+ )
35
+ from fractal_server.app.runner.executors.slurm._slurm_config import (
36
+ load_slurm_config_file,
37
+ )
38
+ from fractal_server.app.runner.executors.slurm._slurm_config import logger
39
+ from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
40
+ from fractal_server.app.runner.executors.slurm._slurm_config import (
41
+ SlurmConfigError,
42
+ )
43
+
44
+
45
+ def _process_workflow(
46
+ *,
47
+ workflow: Workflow,
48
+ input_paths: list[Path],
49
+ output_path: Path,
50
+ input_metadata: dict[str, Any],
51
+ input_history: list[dict[str, Any]],
52
+ logger_name: str,
53
+ workflow_dir: Path,
54
+ workflow_dir_user: Path,
55
+ first_task_index: int,
56
+ last_task_index: int,
57
+ slurm_user: Optional[str] = None,
58
+ slurm_account: Optional[str] = None,
59
+ user_cache_dir: str,
60
+ worker_init: Optional[Union[str, list[str]]] = None,
61
+ ) -> dict[str, Any]:
62
+ """
63
+ Internal processing routine for the SLURM backend
64
+
65
+ This function initialises the a FractalSlurmExecutor, setting logging,
66
+ workflow working dir and user to impersonate. It then schedules the
67
+ workflow tasks and returns the output dataset metadata.
68
+
69
+ Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
70
+
71
+ Returns:
72
+ output_dataset_metadata: Metadata of the output dataset
73
+ """
74
+
75
+ if not slurm_user:
76
+ raise RuntimeError(
77
+ "slurm_user argument is required, for slurm backend"
78
+ )
79
+
80
+ if isinstance(worker_init, str):
81
+ worker_init = worker_init.split("\n")
82
+
83
+ with FractalSlurmExecutor(
84
+ debug=True,
85
+ keep_logs=True,
86
+ slurm_user=slurm_user,
87
+ user_cache_dir=user_cache_dir,
88
+ working_dir=workflow_dir,
89
+ working_dir_user=workflow_dir_user,
90
+ common_script_lines=worker_init,
91
+ slurm_account=slurm_account,
92
+ ) as executor:
93
+ output_task_pars = execute_tasks(
94
+ executor=executor,
95
+ task_list=workflow.task_list[
96
+ first_task_index : (last_task_index + 1) # noqa
97
+ ], # noqa
98
+ task_pars=TaskParameters(
99
+ input_paths=input_paths,
100
+ output_path=output_path,
101
+ metadata=input_metadata,
102
+ history=input_history,
103
+ ),
104
+ workflow_dir=workflow_dir,
105
+ workflow_dir_user=workflow_dir_user,
106
+ submit_setup_call=_slurm_submit_setup,
107
+ logger_name=logger_name,
108
+ )
109
+ output_dataset_metadata_history = dict(
110
+ metadata=output_task_pars.metadata, history=output_task_pars.history
111
+ )
112
+ return output_dataset_metadata_history
113
+
114
+
115
+ async def process_workflow(
116
+ *,
117
+ workflow: Workflow,
118
+ input_paths: list[Path],
119
+ output_path: Path,
120
+ input_metadata: dict[str, Any],
121
+ input_history: list[dict[str, Any]],
122
+ logger_name: str,
123
+ workflow_dir: Path,
124
+ workflow_dir_user: Optional[Path] = None,
125
+ user_cache_dir: Optional[str] = None,
126
+ slurm_user: Optional[str] = None,
127
+ slurm_account: Optional[str] = None,
128
+ worker_init: Optional[str] = None,
129
+ first_task_index: Optional[int] = None,
130
+ last_task_index: Optional[int] = None,
131
+ ) -> dict[str, Any]:
132
+ """
133
+ Process workflow (SLURM backend public interface)
134
+
135
+ Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
136
+ """
137
+
138
+ # Set values of first_task_index and last_task_index
139
+ num_tasks = len(workflow.task_list)
140
+ first_task_index, last_task_index = set_start_and_last_task_index(
141
+ num_tasks,
142
+ first_task_index=first_task_index,
143
+ last_task_index=last_task_index,
144
+ )
145
+
146
+ output_dataset_metadata_history = await async_wrap(_process_workflow)(
147
+ workflow=workflow,
148
+ input_paths=input_paths,
149
+ output_path=output_path,
150
+ input_metadata=input_metadata,
151
+ input_history=input_history,
152
+ logger_name=logger_name,
153
+ workflow_dir=workflow_dir,
154
+ workflow_dir_user=workflow_dir_user,
155
+ slurm_user=slurm_user,
156
+ slurm_account=slurm_account,
157
+ user_cache_dir=user_cache_dir,
158
+ worker_init=worker_init,
159
+ first_task_index=first_task_index,
160
+ last_task_index=last_task_index,
161
+ )
162
+ return output_dataset_metadata_history
163
+
164
+
165
+ def get_slurm_config(
166
+ wftask: WorkflowTask,
167
+ workflow_dir: Path,
168
+ workflow_dir_user: Path,
169
+ config_path: Optional[Path] = None,
170
+ ) -> SlurmConfig:
171
+ """
172
+ Prepare a `SlurmConfig` configuration object
173
+
174
+ The sources for `SlurmConfig` attributes, in increasing priority order, are
175
+
176
+ 1. The general content of the Fractal SLURM configuration file.
177
+ 2. The GPU-specific content of the Fractal SLURM configuration file, if
178
+ appropriate.
179
+ 3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
180
+ `Workflow.insert_task`, also includes `wftask.task.meta`);
181
+
182
+ Note: `wftask.meta` may be `None`.
183
+
184
+ Arguments:
185
+ wftask:
186
+ WorkflowTask for which the SLURM configuration is is to be
187
+ prepared.
188
+ workflow_dir:
189
+ Server-owned directory to store all task-execution-related relevant
190
+ files (inputs, outputs, errors, and all meta files related to the
191
+ job execution). Note: users cannot write directly to this folder.
192
+ workflow_dir_user:
193
+ User-side directory with the same scope as `workflow_dir`, and
194
+ where a user can write.
195
+ config_path:
196
+ Path of aFractal SLURM configuration file; if `None`, use
197
+ `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
198
+
199
+ Returns:
200
+ slurm_config:
201
+ The SlurmConfig object
202
+ """
203
+
204
+ logger.debug(
205
+ "[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
206
+ )
207
+
208
+ # Incorporate slurm_env.default_slurm_config
209
+ slurm_env = load_slurm_config_file(config_path=config_path)
210
+ slurm_dict = slurm_env.default_slurm_config.dict(
211
+ exclude_unset=True, exclude={"mem"}
212
+ )
213
+ if slurm_env.default_slurm_config.mem:
214
+ slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
215
+
216
+ # Incorporate slurm_env.batching_config
217
+ for key, value in slurm_env.batching_config.dict().items():
218
+ slurm_dict[key] = value
219
+
220
+ # Incorporate slurm_env.user_local_exports
221
+ slurm_dict["user_local_exports"] = slurm_env.user_local_exports
222
+
223
+ logger.debug(
224
+ "[get_slurm_config] Fractal SLURM configuration file: "
225
+ f"{slurm_env.dict()=}"
226
+ )
227
+
228
+ # GPU-related options
229
+ # Notes about priority:
230
+ # 1. This block of definitions takes priority over other definitions from
231
+ # slurm_env which are not under the `needs_gpu` subgroup
232
+ # 2. This block of definitions has lower priority than whatever comes next
233
+ # (i.e. from WorkflowTask.meta).
234
+ if wftask.meta is not None:
235
+ needs_gpu = wftask.meta.get("needs_gpu", False)
236
+ else:
237
+ needs_gpu = False
238
+ logger.debug(f"[get_slurm_config] {needs_gpu=}")
239
+ if needs_gpu:
240
+ for key, value in slurm_env.gpu_slurm_config.dict(
241
+ exclude_unset=True, exclude={"mem"}
242
+ ).items():
243
+ slurm_dict[key] = value
244
+ if slurm_env.gpu_slurm_config.mem:
245
+ slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
246
+
247
+ # Number of CPUs per task, for multithreading
248
+ if wftask.meta is not None and "cpus_per_task" in wftask.meta:
249
+ cpus_per_task = int(wftask.meta["cpus_per_task"])
250
+ slurm_dict["cpus_per_task"] = cpus_per_task
251
+
252
+ # Required memory per task, in MB
253
+ if wftask.meta is not None and "mem" in wftask.meta:
254
+ raw_mem = wftask.meta["mem"]
255
+ mem_per_task_MB = _parse_mem_value(raw_mem)
256
+ slurm_dict["mem_per_task_MB"] = mem_per_task_MB
257
+
258
+ # Job name
259
+ job_name = wftask.task.name.replace(" ", "_")
260
+ slurm_dict["job_name"] = job_name
261
+
262
+ # Optional SLURM arguments and extra lines
263
+ if wftask.meta is not None:
264
+ account = wftask.meta.get("account", None)
265
+ if account is not None:
266
+ error_msg = (
267
+ f"Invalid {account=} property in WorkflowTask `meta` "
268
+ "attribute.\n"
269
+ "SLURM account must be set in the request body of the "
270
+ "apply-workflow endpoint, or by modifying the user properties."
271
+ )
272
+ logger.error(error_msg)
273
+ raise SlurmConfigError(error_msg)
274
+ for key in ["time", "gres", "constraint"]:
275
+ value = wftask.meta.get(key, None)
276
+ if value:
277
+ slurm_dict[key] = value
278
+ if wftask.meta is not None:
279
+ extra_lines = wftask.meta.get("extra_lines", [])
280
+ else:
281
+ extra_lines = []
282
+ extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
283
+ if len(set(extra_lines)) != len(extra_lines):
284
+ logger.debug(
285
+ "[get_slurm_config] Removing repeated elements "
286
+ f"from {extra_lines=}."
287
+ )
288
+ extra_lines = list(set(extra_lines))
289
+ slurm_dict["extra_lines"] = extra_lines
290
+
291
+ # Job-batching parameters (if None, they will be determined heuristically)
292
+ if wftask.meta is not None:
293
+ tasks_per_job = wftask.meta.get("tasks_per_job", None)
294
+ parallel_tasks_per_job = wftask.meta.get(
295
+ "parallel_tasks_per_job", None
296
+ )
297
+ else:
298
+ tasks_per_job = None
299
+ parallel_tasks_per_job = None
300
+ slurm_dict["tasks_per_job"] = tasks_per_job
301
+ slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
302
+
303
+ # Put everything together
304
+ logger.debug(
305
+ "[get_slurm_config] Now create a SlurmConfig object based "
306
+ f"on {slurm_dict=}"
307
+ )
308
+ slurm_config = SlurmConfig(**slurm_dict)
309
+
310
+ return slurm_config
@@ -15,12 +15,10 @@ implementation of `submit_setup_call` in
15
15
  [fractal_server.app.runner._common][]).
16
16
  """
17
17
  from pathlib import Path
18
- from typing import Optional
19
18
 
20
- from ...models import WorkflowTask
21
- from .._common import get_task_file_paths
22
- from ..common import TaskParameters
23
- from ._slurm_config import get_slurm_config
19
+ from ...task_files import get_task_file_paths
20
+ from .get_slurm_config import get_slurm_config
21
+ from fractal_server.app.models.v1 import WorkflowTask
24
22
 
25
23
 
26
24
  def _slurm_submit_setup(
@@ -28,7 +26,6 @@ def _slurm_submit_setup(
28
26
  wftask: WorkflowTask,
29
27
  workflow_dir: Path,
30
28
  workflow_dir_user: Path,
31
- task_pars: Optional[TaskParameters] = None,
32
29
  ) -> dict[str, object]:
33
30
  """
34
31
  Collect WorfklowTask-specific configuration parameters from different
@@ -46,9 +43,6 @@ def _slurm_submit_setup(
46
43
  Arguments:
47
44
  wftask:
48
45
  WorkflowTask for which the configuration is to be assembled
49
- task_pars:
50
- Task parameters to be passed to the task
51
- (not used in this function)
52
46
  workflow_dir:
53
47
  Server-owned directory to store all task-execution-related relevant
54
48
  files (inputs, outputs, errors, and all meta files related to the
@@ -0,0 +1,163 @@
1
+ from pathlib import Path
2
+ from typing import Optional
3
+
4
+ from fractal_server.app.models.v1 import WorkflowTask
5
+ from fractal_server.app.runner.executors.slurm._slurm_config import (
6
+ _parse_mem_value,
7
+ )
8
+ from fractal_server.app.runner.executors.slurm._slurm_config import (
9
+ load_slurm_config_file,
10
+ )
11
+ from fractal_server.app.runner.executors.slurm._slurm_config import logger
12
+ from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
13
+ from fractal_server.app.runner.executors.slurm._slurm_config import (
14
+ SlurmConfigError,
15
+ )
16
+
17
+
18
+ def get_slurm_config(
19
+ wftask: WorkflowTask,
20
+ workflow_dir: Path,
21
+ workflow_dir_user: Path,
22
+ config_path: Optional[Path] = None,
23
+ ) -> SlurmConfig:
24
+ """
25
+ Prepare a `SlurmConfig` configuration object
26
+
27
+ The sources for `SlurmConfig` attributes, in increasing priority order, are
28
+
29
+ 1. The general content of the Fractal SLURM configuration file.
30
+ 2. The GPU-specific content of the Fractal SLURM configuration file, if
31
+ appropriate.
32
+ 3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
33
+ `Workflow.insert_task`, also includes `wftask.task.meta`);
34
+
35
+ Note: `wftask.meta` may be `None`.
36
+
37
+ Arguments:
38
+ wftask:
39
+ WorkflowTask for which the SLURM configuration is is to be
40
+ prepared.
41
+ workflow_dir:
42
+ Server-owned directory to store all task-execution-related relevant
43
+ files (inputs, outputs, errors, and all meta files related to the
44
+ job execution). Note: users cannot write directly to this folder.
45
+ workflow_dir_user:
46
+ User-side directory with the same scope as `workflow_dir`, and
47
+ where a user can write.
48
+ config_path:
49
+ Path of aFractal SLURM configuration file; if `None`, use
50
+ `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
51
+
52
+ Returns:
53
+ slurm_config:
54
+ The SlurmConfig object
55
+ """
56
+
57
+ logger.debug(
58
+ "[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
59
+ )
60
+
61
+ # Incorporate slurm_env.default_slurm_config
62
+ slurm_env = load_slurm_config_file(config_path=config_path)
63
+ slurm_dict = slurm_env.default_slurm_config.dict(
64
+ exclude_unset=True, exclude={"mem"}
65
+ )
66
+ if slurm_env.default_slurm_config.mem:
67
+ slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
68
+
69
+ # Incorporate slurm_env.batching_config
70
+ for key, value in slurm_env.batching_config.dict().items():
71
+ slurm_dict[key] = value
72
+
73
+ # Incorporate slurm_env.user_local_exports
74
+ slurm_dict["user_local_exports"] = slurm_env.user_local_exports
75
+
76
+ logger.debug(
77
+ "[get_slurm_config] Fractal SLURM configuration file: "
78
+ f"{slurm_env.dict()=}"
79
+ )
80
+
81
+ # GPU-related options
82
+ # Notes about priority:
83
+ # 1. This block of definitions takes priority over other definitions from
84
+ # slurm_env which are not under the `needs_gpu` subgroup
85
+ # 2. This block of definitions has lower priority than whatever comes next
86
+ # (i.e. from WorkflowTask.meta).
87
+ if wftask.meta is not None:
88
+ needs_gpu = wftask.meta.get("needs_gpu", False)
89
+ else:
90
+ needs_gpu = False
91
+ logger.debug(f"[get_slurm_config] {needs_gpu=}")
92
+ if needs_gpu:
93
+ for key, value in slurm_env.gpu_slurm_config.dict(
94
+ exclude_unset=True, exclude={"mem"}
95
+ ).items():
96
+ slurm_dict[key] = value
97
+ if slurm_env.gpu_slurm_config.mem:
98
+ slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
99
+
100
+ # Number of CPUs per task, for multithreading
101
+ if wftask.meta is not None and "cpus_per_task" in wftask.meta:
102
+ cpus_per_task = int(wftask.meta["cpus_per_task"])
103
+ slurm_dict["cpus_per_task"] = cpus_per_task
104
+
105
+ # Required memory per task, in MB
106
+ if wftask.meta is not None and "mem" in wftask.meta:
107
+ raw_mem = wftask.meta["mem"]
108
+ mem_per_task_MB = _parse_mem_value(raw_mem)
109
+ slurm_dict["mem_per_task_MB"] = mem_per_task_MB
110
+
111
+ # Job name
112
+ job_name = wftask.task.name.replace(" ", "_")
113
+ slurm_dict["job_name"] = job_name
114
+
115
+ # Optional SLURM arguments and extra lines
116
+ if wftask.meta is not None:
117
+ account = wftask.meta.get("account", None)
118
+ if account is not None:
119
+ error_msg = (
120
+ f"Invalid {account=} property in WorkflowTask `meta` "
121
+ "attribute.\n"
122
+ "SLURM account must be set in the request body of the "
123
+ "apply-workflow endpoint, or by modifying the user properties."
124
+ )
125
+ logger.error(error_msg)
126
+ raise SlurmConfigError(error_msg)
127
+ for key in ["time", "gres", "constraint"]:
128
+ value = wftask.meta.get(key, None)
129
+ if value:
130
+ slurm_dict[key] = value
131
+ if wftask.meta is not None:
132
+ extra_lines = wftask.meta.get("extra_lines", [])
133
+ else:
134
+ extra_lines = []
135
+ extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
136
+ if len(set(extra_lines)) != len(extra_lines):
137
+ logger.debug(
138
+ "[get_slurm_config] Removing repeated elements "
139
+ f"from {extra_lines=}."
140
+ )
141
+ extra_lines = list(set(extra_lines))
142
+ slurm_dict["extra_lines"] = extra_lines
143
+
144
+ # Job-batching parameters (if None, they will be determined heuristically)
145
+ if wftask.meta is not None:
146
+ tasks_per_job = wftask.meta.get("tasks_per_job", None)
147
+ parallel_tasks_per_job = wftask.meta.get(
148
+ "parallel_tasks_per_job", None
149
+ )
150
+ else:
151
+ tasks_per_job = None
152
+ parallel_tasks_per_job = None
153
+ slurm_dict["tasks_per_job"] = tasks_per_job
154
+ slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
155
+
156
+ # Put everything together
157
+ logger.debug(
158
+ "[get_slurm_config] Now create a SlurmConfig object based "
159
+ f"on {slurm_dict=}"
160
+ )
161
+ slurm_config = SlurmConfig(**slurm_dict)
162
+
163
+ return slurm_config
@@ -0,0 +1,117 @@
1
+ """
2
+ Common utilities and routines for runner backends (public API)
3
+
4
+ This module includes utilities and routines that are of use to implement
5
+ runner backends but that should also be exposed to the other components of
6
+ `Fractal Server`.
7
+ """
8
+ import json
9
+ from json import JSONEncoder
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from pydantic import BaseModel
14
+
15
+ from ....logger import close_logger as close_job_logger # noqa F401
16
+ from ...models.v1 import Dataset
17
+ from ...models.v1 import Workflow
18
+
19
+
20
+ class TaskParameterEncoder(JSONEncoder):
21
+ """
22
+ Convenience JSONEncoder that serialises `Path`s as strings
23
+ """
24
+
25
+ def default(self, value):
26
+ if isinstance(value, Path):
27
+ return value.as_posix()
28
+ return JSONEncoder.default(self, value)
29
+
30
+
31
+ class TaskParameters(BaseModel):
32
+ """
33
+ Wrapper for task input parameters
34
+
35
+ Instances of this class are used to pass parameters from the output of a
36
+ task to the input of the next one.
37
+
38
+ Attributes:
39
+ input_paths:
40
+ Input paths as derived by the input dataset.
41
+ output_paths:
42
+ Output path as derived from the output dataset.
43
+ metadata:
44
+ Dataset metadata, as found in the input dataset or as updated by
45
+ the previous task.
46
+ history:
47
+ Dataset history, as found in the input dataset or as updated by
48
+ the previous task.
49
+ """
50
+
51
+ input_paths: list[Path]
52
+ output_path: Path
53
+ metadata: dict[str, Any]
54
+ history: list[dict[str, Any]]
55
+
56
+ class Config:
57
+ arbitrary_types_allowed = True
58
+ extra = "forbid"
59
+
60
+
61
+ def validate_workflow_compatibility(
62
+ *,
63
+ input_dataset: Dataset,
64
+ workflow: Workflow,
65
+ output_dataset: Dataset,
66
+ first_task_index: int,
67
+ last_task_index: int,
68
+ ) -> None:
69
+ """
70
+ Check compatibility of workflow and input / ouptut dataset
71
+ """
72
+ # Check input_dataset type
73
+ workflow_input_type = workflow.task_list[first_task_index].task.input_type
74
+ if (
75
+ workflow_input_type != "Any"
76
+ and workflow_input_type != input_dataset.type
77
+ ):
78
+ raise TypeError(
79
+ f"Incompatible types `{workflow_input_type}` of workflow "
80
+ f"`{workflow.name}` and `{input_dataset.type}` of dataset "
81
+ f"`{input_dataset.name}`"
82
+ )
83
+
84
+ # Check output_dataset type
85
+ workflow_output_type = workflow.task_list[last_task_index].task.output_type
86
+ if (
87
+ workflow_output_type != "Any"
88
+ and workflow_output_type != output_dataset.type
89
+ ):
90
+ raise TypeError(
91
+ f"Incompatible types `{workflow_output_type}` of workflow "
92
+ f"`{workflow.name}` and `{output_dataset.type}` of dataset "
93
+ f"`{output_dataset.name}`"
94
+ )
95
+
96
+
97
+ def write_args_file(
98
+ *args: dict[str, Any],
99
+ path: Path,
100
+ ):
101
+ """
102
+ Merge arbitrary dictionaries and write to file
103
+
104
+ Args:
105
+ *args:
106
+ One or more dictionaries that will be merged into one respecting
107
+ the order with which they are passed in, i.e., last in overrides
108
+ previous ones.
109
+ path:
110
+ Destination for serialised file.
111
+ """
112
+ out = {}
113
+ for d in args:
114
+ out.update(d)
115
+
116
+ with open(path, "w") as f:
117
+ json.dump(out, f, cls=TaskParameterEncoder, indent=4)
@@ -19,13 +19,13 @@ from pathlib import Path
19
19
  from typing import Any
20
20
  from typing import Optional
21
21
 
22
- from ..models import ApplyWorkflow
23
- from ..models import Dataset
24
- from ..models import Workflow
25
- from ..models import WorkflowTask
26
- from ..models import WorkflowTaskStatusType
27
- from ._common import HISTORY_FILENAME
28
- from ._common import METADATA_FILENAME
22
+ from ...models.v1 import ApplyWorkflow
23
+ from ...models.v1 import Dataset
24
+ from ...models.v1 import Workflow
25
+ from ...models.v1 import WorkflowTask
26
+ from ...schemas.v1 import WorkflowTaskStatusTypeV1
27
+ from ..filenames import HISTORY_FILENAME
28
+ from ..filenames import METADATA_FILENAME
29
29
 
30
30
 
31
31
  def assemble_history_failed_job(
@@ -98,7 +98,7 @@ def assemble_history_failed_job(
98
98
  failed_wftask_dump["task"] = failed_wftask.task.model_dump()
99
99
  new_history_item = dict(
100
100
  workflowtask=failed_wftask_dump,
101
- status=WorkflowTaskStatusType.FAILED,
101
+ status=WorkflowTaskStatusTypeV1.FAILED,
102
102
  parallelization=dict(
103
103
  parallelization_level=failed_wftask.parallelization_level,
104
104
  ),