fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/models/__init__.py +4 -7
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/state.py +1 -1
- fractal_server/app/models/v1/__init__.py +10 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +20 -0
- fractal_server/app/models/v2/dataset.py +55 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +31 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +43 -0
- fractal_server/app/models/v2/workflowtask.py +90 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +275 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +2 -2
- fractal_server/app/routes/api/v1/dataset.py +44 -37
- fractal_server/app/routes/api/v1/job.py +12 -12
- fractal_server/app/routes/api/v1/project.py +23 -21
- fractal_server/app/routes/api/v1/task.py +24 -14
- fractal_server/app/routes/api/v1/task_collection.py +16 -14
- fractal_server/app/routes/api/v1/workflow.py +24 -24
- fractal_server/app/routes/api/v1/workflowtask.py +10 -10
- fractal_server/app/routes/api/v2/__init__.py +28 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +497 -0
- fractal_server/app/routes/api/v2/apply.py +220 -0
- fractal_server/app/routes/api/v2/dataset.py +310 -0
- fractal_server/app/routes/api/v2/images.py +212 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +205 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +229 -0
- fractal_server/app/routes/api/v2/workflow.py +398 -0
- fractal_server/app/routes/api/v2/workflowtask.py +269 -0
- fractal_server/app/routes/aux/_job.py +1 -1
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/local/__init__.py +3 -0
- fractal_server/app/runner/{_local → executors/local}/executor.py +2 -2
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/executors/slurm/_check_jobs_status.py +72 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +3 -4
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +9 -9
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +105 -0
- fractal_server/app/runner/{__init__.py → v1/__init__.py} +36 -49
- fractal_server/app/runner/{_common.py → v1/_common.py} +13 -120
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +6 -6
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +310 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +3 -9
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +337 -0
- fractal_server/app/runner/v2/_local/__init__.py +169 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_slurm/__init__.py +157 -0
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +83 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +179 -0
- fractal_server/app/runner/v2/components.py +5 -0
- fractal_server/app/runner/v2/deduplicate_list.py +24 -0
- fractal_server/app/runner/v2/handle_failed_job.py +156 -0
- fractal_server/app/runner/v2/merge_outputs.py +41 -0
- fractal_server/app/runner/v2/runner.py +264 -0
- fractal_server/app/runner/v2/runner_functions.py +339 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +134 -0
- fractal_server/app/runner/v2/task_interface.py +43 -0
- fractal_server/app/runner/v2/v1_compat.py +21 -0
- fractal_server/app/schemas/__init__.py +4 -42
- fractal_server/app/schemas/v1/__init__.py +42 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +34 -0
- fractal_server/app/schemas/v2/dataset.py +88 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +113 -0
- fractal_server/app/schemas/v2/manifest.py +109 -0
- fractal_server/app/schemas/v2/project.py +36 -0
- fractal_server/app/schemas/v2/task.py +121 -0
- fractal_server/app/schemas/v2/task_collection.py +105 -0
- fractal_server/app/schemas/v2/workflow.py +78 -0
- fractal_server/app/schemas/v2/workflowtask.py +118 -0
- fractal_server/config.py +5 -10
- fractal_server/images/__init__.py +50 -0
- fractal_server/images/tools.py +86 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/versions/4b35c5cefbe3_tmp_is_v2_compatible.py +39 -0
- fractal_server/migrations/versions/56af171b0159_v2.py +217 -0
- fractal_server/migrations/versions/876f28db9d4e_tmp_split_task_and_wftask_meta.py +68 -0
- fractal_server/migrations/versions/974c802f0dd0_tmp_workflowtaskv2_type_in_db.py +37 -0
- fractal_server/migrations/versions/9cd305cd6023_tmp_workflowtaskv2.py +40 -0
- fractal_server/migrations/versions/a6231ed6273c_tmp_args_schemas_in_taskv2.py +42 -0
- fractal_server/migrations/versions/b9e9eed9d442_tmp_taskv2_type.py +37 -0
- fractal_server/migrations/versions/e3e639454d4b_tmp_make_task_meta_non_optional.py +50 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +18 -50
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/background_operations.py +382 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- {fractal_server-1.4.9.dist-info → fractal_server-2.0.0a0.dist-info}/METADATA +3 -4
- fractal_server-2.0.0a0.dist-info/RECORD +166 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/_slurm/__init__.py +0 -150
- fractal_server/app/runner/common.py +0 -311
- fractal_server-1.4.9.dist-info/RECORD +0 -97
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- {fractal_server-1.4.9.dist-info → fractal_server-2.0.0a0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.9.dist-info → fractal_server-2.0.0a0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.9.dist-info → fractal_server-2.0.0a0.dist-info}/entry_points.txt +0 -0
@@ -1,150 +0,0 @@
|
|
1
|
-
# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
|
2
|
-
# University of Zurich
|
3
|
-
#
|
4
|
-
# Original authors:
|
5
|
-
# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
|
6
|
-
# Tommaso Comparin <tommaso.comparin@exact-lab.it>
|
7
|
-
# Marco Franzon <marco.franzon@exact-lab.it>
|
8
|
-
#
|
9
|
-
# This file is part of Fractal and was originally developed by eXact lab S.r.l.
|
10
|
-
# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
|
11
|
-
# Institute for Biomedical Research and Pelkmans Lab from the University of
|
12
|
-
# Zurich.
|
13
|
-
"""
|
14
|
-
Slurm Bakend
|
15
|
-
|
16
|
-
This backend runs fractal workflows in a SLURM cluster using Clusterfutures
|
17
|
-
Executor objects.
|
18
|
-
"""
|
19
|
-
from pathlib import Path
|
20
|
-
from typing import Any
|
21
|
-
from typing import Optional
|
22
|
-
from typing import Union
|
23
|
-
|
24
|
-
from ...models import Workflow
|
25
|
-
from .._common import execute_tasks
|
26
|
-
from ..common import async_wrap
|
27
|
-
from ..common import set_start_and_last_task_index
|
28
|
-
from ..common import TaskParameters
|
29
|
-
from ._submit_setup import _slurm_submit_setup
|
30
|
-
from .executor import FractalSlurmExecutor
|
31
|
-
|
32
|
-
|
33
|
-
def _process_workflow(
|
34
|
-
*,
|
35
|
-
workflow: Workflow,
|
36
|
-
input_paths: list[Path],
|
37
|
-
output_path: Path,
|
38
|
-
input_metadata: dict[str, Any],
|
39
|
-
input_history: list[dict[str, Any]],
|
40
|
-
logger_name: str,
|
41
|
-
workflow_dir: Path,
|
42
|
-
workflow_dir_user: Path,
|
43
|
-
first_task_index: int,
|
44
|
-
last_task_index: int,
|
45
|
-
slurm_user: Optional[str] = None,
|
46
|
-
slurm_account: Optional[str] = None,
|
47
|
-
user_cache_dir: str,
|
48
|
-
worker_init: Optional[Union[str, list[str]]] = None,
|
49
|
-
) -> dict[str, Any]:
|
50
|
-
"""
|
51
|
-
Internal processing routine for the SLURM backend
|
52
|
-
|
53
|
-
This function initialises the a FractalSlurmExecutor, setting logging,
|
54
|
-
workflow working dir and user to impersonate. It then schedules the
|
55
|
-
workflow tasks and returns the output dataset metadata.
|
56
|
-
|
57
|
-
Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
|
58
|
-
|
59
|
-
Returns:
|
60
|
-
output_dataset_metadata: Metadata of the output dataset
|
61
|
-
"""
|
62
|
-
|
63
|
-
if not slurm_user:
|
64
|
-
raise RuntimeError(
|
65
|
-
"slurm_user argument is required, for slurm backend"
|
66
|
-
)
|
67
|
-
|
68
|
-
if isinstance(worker_init, str):
|
69
|
-
worker_init = worker_init.split("\n")
|
70
|
-
|
71
|
-
with FractalSlurmExecutor(
|
72
|
-
debug=True,
|
73
|
-
keep_logs=True,
|
74
|
-
slurm_user=slurm_user,
|
75
|
-
user_cache_dir=user_cache_dir,
|
76
|
-
working_dir=workflow_dir,
|
77
|
-
working_dir_user=workflow_dir_user,
|
78
|
-
common_script_lines=worker_init,
|
79
|
-
slurm_account=slurm_account,
|
80
|
-
) as executor:
|
81
|
-
output_task_pars = execute_tasks(
|
82
|
-
executor=executor,
|
83
|
-
task_list=workflow.task_list[
|
84
|
-
first_task_index : (last_task_index + 1) # noqa
|
85
|
-
], # noqa
|
86
|
-
task_pars=TaskParameters(
|
87
|
-
input_paths=input_paths,
|
88
|
-
output_path=output_path,
|
89
|
-
metadata=input_metadata,
|
90
|
-
history=input_history,
|
91
|
-
),
|
92
|
-
workflow_dir=workflow_dir,
|
93
|
-
workflow_dir_user=workflow_dir_user,
|
94
|
-
submit_setup_call=_slurm_submit_setup,
|
95
|
-
logger_name=logger_name,
|
96
|
-
)
|
97
|
-
output_dataset_metadata_history = dict(
|
98
|
-
metadata=output_task_pars.metadata, history=output_task_pars.history
|
99
|
-
)
|
100
|
-
return output_dataset_metadata_history
|
101
|
-
|
102
|
-
|
103
|
-
async def process_workflow(
|
104
|
-
*,
|
105
|
-
workflow: Workflow,
|
106
|
-
input_paths: list[Path],
|
107
|
-
output_path: Path,
|
108
|
-
input_metadata: dict[str, Any],
|
109
|
-
input_history: list[dict[str, Any]],
|
110
|
-
logger_name: str,
|
111
|
-
workflow_dir: Path,
|
112
|
-
workflow_dir_user: Optional[Path] = None,
|
113
|
-
user_cache_dir: Optional[str] = None,
|
114
|
-
slurm_user: Optional[str] = None,
|
115
|
-
slurm_account: Optional[str] = None,
|
116
|
-
worker_init: Optional[str] = None,
|
117
|
-
first_task_index: Optional[int] = None,
|
118
|
-
last_task_index: Optional[int] = None,
|
119
|
-
) -> dict[str, Any]:
|
120
|
-
"""
|
121
|
-
Process workflow (SLURM backend public interface)
|
122
|
-
|
123
|
-
Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
|
124
|
-
"""
|
125
|
-
|
126
|
-
# Set values of first_task_index and last_task_index
|
127
|
-
num_tasks = len(workflow.task_list)
|
128
|
-
first_task_index, last_task_index = set_start_and_last_task_index(
|
129
|
-
num_tasks,
|
130
|
-
first_task_index=first_task_index,
|
131
|
-
last_task_index=last_task_index,
|
132
|
-
)
|
133
|
-
|
134
|
-
output_dataset_metadata_history = await async_wrap(_process_workflow)(
|
135
|
-
workflow=workflow,
|
136
|
-
input_paths=input_paths,
|
137
|
-
output_path=output_path,
|
138
|
-
input_metadata=input_metadata,
|
139
|
-
input_history=input_history,
|
140
|
-
logger_name=logger_name,
|
141
|
-
workflow_dir=workflow_dir,
|
142
|
-
workflow_dir_user=workflow_dir_user,
|
143
|
-
slurm_user=slurm_user,
|
144
|
-
slurm_account=slurm_account,
|
145
|
-
user_cache_dir=user_cache_dir,
|
146
|
-
worker_init=worker_init,
|
147
|
-
first_task_index=first_task_index,
|
148
|
-
last_task_index=last_task_index,
|
149
|
-
)
|
150
|
-
return output_dataset_metadata_history
|
@@ -1,311 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
Common utilities and routines for runner backends (public API)
|
3
|
-
|
4
|
-
This module includes utilities and routines that are of use to implement
|
5
|
-
runner backends but that should also be exposed to the other components of
|
6
|
-
`Fractal Server`.
|
7
|
-
"""
|
8
|
-
import asyncio
|
9
|
-
import json
|
10
|
-
import os
|
11
|
-
from functools import partial
|
12
|
-
from functools import wraps
|
13
|
-
from json import JSONEncoder
|
14
|
-
from pathlib import Path
|
15
|
-
from typing import Any
|
16
|
-
from typing import Callable
|
17
|
-
from typing import Optional
|
18
|
-
|
19
|
-
from pydantic import BaseModel
|
20
|
-
|
21
|
-
from ...logger import close_logger as close_job_logger # noqa F401
|
22
|
-
from ..models import Dataset
|
23
|
-
from ..models.workflow import Workflow
|
24
|
-
|
25
|
-
|
26
|
-
class TaskExecutionError(RuntimeError):
|
27
|
-
"""
|
28
|
-
Forwards errors occurred during the execution of a task
|
29
|
-
|
30
|
-
This error wraps and forwards errors occurred during the execution of
|
31
|
-
tasks, when the exit code is larger than 0 (i.e. the error took place
|
32
|
-
within the task). This error also adds information that is useful to track
|
33
|
-
down and debug the failing task within a workflow.
|
34
|
-
|
35
|
-
Attributes:
|
36
|
-
workflow_task_id:
|
37
|
-
ID of the workflow task that failed.
|
38
|
-
workflow_task_order:
|
39
|
-
Order of the task within the workflow.
|
40
|
-
task_name:
|
41
|
-
Human readable name of the failing task.
|
42
|
-
"""
|
43
|
-
|
44
|
-
workflow_task_id: Optional[int] = None
|
45
|
-
workflow_task_order: Optional[int] = None
|
46
|
-
task_name: Optional[str] = None
|
47
|
-
|
48
|
-
def __init__(
|
49
|
-
self,
|
50
|
-
*args,
|
51
|
-
workflow_task_id: Optional[int] = None,
|
52
|
-
workflow_task_order: Optional[int] = None,
|
53
|
-
task_name: Optional[str] = None,
|
54
|
-
):
|
55
|
-
super().__init__(*args)
|
56
|
-
self.workflow_task_id = workflow_task_id
|
57
|
-
self.workflow_task_order = workflow_task_order
|
58
|
-
self.task_name = task_name
|
59
|
-
|
60
|
-
|
61
|
-
class JobExecutionError(RuntimeError):
|
62
|
-
"""
|
63
|
-
Forwards errors in the execution of a task that are due to external factors
|
64
|
-
|
65
|
-
This error wraps and forwards errors occurred during the execution of
|
66
|
-
tasks, but related to external factors like:
|
67
|
-
|
68
|
-
1. A negative exit code (e.g. because the task received a TERM or KILL
|
69
|
-
signal);
|
70
|
-
2. An error on the executor side (e.g. the SLURM executor could not
|
71
|
-
find the pickled file with task output).
|
72
|
-
|
73
|
-
This error also adds information that is useful to track down and debug the
|
74
|
-
failing task within a workflow.
|
75
|
-
|
76
|
-
Attributes:
|
77
|
-
info:
|
78
|
-
A free field for additional information
|
79
|
-
cmd_file:
|
80
|
-
Path to the file of the command that was executed (e.g. a SLURM
|
81
|
-
submission script).
|
82
|
-
stdout_file:
|
83
|
-
Path to the file with the command stdout
|
84
|
-
stderr_file:
|
85
|
-
Path to the file with the command stderr
|
86
|
-
"""
|
87
|
-
|
88
|
-
cmd_file: Optional[str] = None
|
89
|
-
stdout_file: Optional[str] = None
|
90
|
-
stderr_file: Optional[str] = None
|
91
|
-
info: Optional[str] = None
|
92
|
-
|
93
|
-
def __init__(
|
94
|
-
self,
|
95
|
-
*args,
|
96
|
-
cmd_file: Optional[str] = None,
|
97
|
-
stdout_file: Optional[str] = None,
|
98
|
-
stderr_file: Optional[str] = None,
|
99
|
-
info: Optional[str] = None,
|
100
|
-
):
|
101
|
-
super().__init__(*args)
|
102
|
-
self.cmd_file = cmd_file
|
103
|
-
self.stdout_file = stdout_file
|
104
|
-
self.stderr_file = stderr_file
|
105
|
-
self.info = info
|
106
|
-
|
107
|
-
def _read_file(self, filepath: str) -> str:
|
108
|
-
"""
|
109
|
-
Return the content of a text file, and handle the cases where it is
|
110
|
-
empty or missing
|
111
|
-
"""
|
112
|
-
if os.path.exists(filepath):
|
113
|
-
with open(filepath, "r") as f:
|
114
|
-
content = f.read()
|
115
|
-
if content:
|
116
|
-
return f"Content of {filepath}:\n{content}"
|
117
|
-
else:
|
118
|
-
return f"File {filepath} is empty\n"
|
119
|
-
else:
|
120
|
-
return f"File {filepath} is missing\n"
|
121
|
-
|
122
|
-
def assemble_error(self) -> str:
|
123
|
-
"""
|
124
|
-
Read the files that are specified in attributes, and combine them in an
|
125
|
-
error message.
|
126
|
-
"""
|
127
|
-
if self.cmd_file:
|
128
|
-
content = self._read_file(self.cmd_file)
|
129
|
-
cmd_content = f"COMMAND:\n{content}\n\n"
|
130
|
-
else:
|
131
|
-
cmd_content = ""
|
132
|
-
if self.stdout_file:
|
133
|
-
content = self._read_file(self.stdout_file)
|
134
|
-
out_content = f"STDOUT:\n{content}\n\n"
|
135
|
-
else:
|
136
|
-
out_content = ""
|
137
|
-
if self.stderr_file:
|
138
|
-
content = self._read_file(self.stderr_file)
|
139
|
-
err_content = f"STDERR:\n{content}\n\n"
|
140
|
-
else:
|
141
|
-
err_content = ""
|
142
|
-
|
143
|
-
content = f"{cmd_content}{out_content}{err_content}"
|
144
|
-
if self.info:
|
145
|
-
content = f"{content}ADDITIONAL INFO:\n{self.info}\n\n"
|
146
|
-
|
147
|
-
if not content:
|
148
|
-
content = str(self)
|
149
|
-
message = f"JobExecutionError\n\n{content}"
|
150
|
-
return message
|
151
|
-
|
152
|
-
|
153
|
-
class TaskParameterEncoder(JSONEncoder):
|
154
|
-
"""
|
155
|
-
Convenience JSONEncoder that serialises `Path`s as strings
|
156
|
-
"""
|
157
|
-
|
158
|
-
def default(self, value):
|
159
|
-
if isinstance(value, Path):
|
160
|
-
return value.as_posix()
|
161
|
-
return JSONEncoder.default(self, value)
|
162
|
-
|
163
|
-
|
164
|
-
class TaskParameters(BaseModel):
|
165
|
-
"""
|
166
|
-
Wrapper for task input parameters
|
167
|
-
|
168
|
-
Instances of this class are used to pass parameters from the output of a
|
169
|
-
task to the input of the next one.
|
170
|
-
|
171
|
-
Attributes:
|
172
|
-
input_paths:
|
173
|
-
Input paths as derived by the input dataset.
|
174
|
-
output_paths:
|
175
|
-
Output path as derived from the output dataset.
|
176
|
-
metadata:
|
177
|
-
Dataset metadata, as found in the input dataset or as updated by
|
178
|
-
the previous task.
|
179
|
-
history:
|
180
|
-
Dataset history, as found in the input dataset or as updated by
|
181
|
-
the previous task.
|
182
|
-
"""
|
183
|
-
|
184
|
-
input_paths: list[Path]
|
185
|
-
output_path: Path
|
186
|
-
metadata: dict[str, Any]
|
187
|
-
history: list[dict[str, Any]]
|
188
|
-
|
189
|
-
class Config:
|
190
|
-
arbitrary_types_allowed = True
|
191
|
-
extra = "forbid"
|
192
|
-
|
193
|
-
|
194
|
-
def validate_workflow_compatibility(
|
195
|
-
*,
|
196
|
-
input_dataset: Dataset,
|
197
|
-
workflow: Workflow,
|
198
|
-
output_dataset: Dataset,
|
199
|
-
first_task_index: int,
|
200
|
-
last_task_index: int,
|
201
|
-
) -> None:
|
202
|
-
"""
|
203
|
-
Check compatibility of workflow and input / ouptut dataset
|
204
|
-
"""
|
205
|
-
# Check input_dataset type
|
206
|
-
workflow_input_type = workflow.task_list[first_task_index].task.input_type
|
207
|
-
if (
|
208
|
-
workflow_input_type != "Any"
|
209
|
-
and workflow_input_type != input_dataset.type
|
210
|
-
):
|
211
|
-
raise TypeError(
|
212
|
-
f"Incompatible types `{workflow_input_type}` of workflow "
|
213
|
-
f"`{workflow.name}` and `{input_dataset.type}` of dataset "
|
214
|
-
f"`{input_dataset.name}`"
|
215
|
-
)
|
216
|
-
|
217
|
-
# Check output_dataset type
|
218
|
-
workflow_output_type = workflow.task_list[last_task_index].task.output_type
|
219
|
-
if (
|
220
|
-
workflow_output_type != "Any"
|
221
|
-
and workflow_output_type != output_dataset.type
|
222
|
-
):
|
223
|
-
raise TypeError(
|
224
|
-
f"Incompatible types `{workflow_output_type}` of workflow "
|
225
|
-
f"`{workflow.name}` and `{output_dataset.type}` of dataset "
|
226
|
-
f"`{output_dataset.name}`"
|
227
|
-
)
|
228
|
-
|
229
|
-
|
230
|
-
def async_wrap(func: Callable) -> Callable:
|
231
|
-
"""
|
232
|
-
Wrap a synchronous callable in an async task
|
233
|
-
|
234
|
-
Ref: [issue #140](https://github.com/fractal-analytics-platform/fractal-server/issues/140)
|
235
|
-
and [this StackOverflow answer](https://stackoverflow.com/q/43241221/19085332).
|
236
|
-
|
237
|
-
Returns:
|
238
|
-
async_wrapper:
|
239
|
-
A factory that allows wrapping a blocking callable within a
|
240
|
-
coroutine.
|
241
|
-
""" # noqa: E501
|
242
|
-
|
243
|
-
@wraps(func)
|
244
|
-
async def async_wrapper(*args, loop=None, executor=None, **kwargs):
|
245
|
-
if loop is None:
|
246
|
-
loop = asyncio.get_event_loop()
|
247
|
-
pfunc = partial(func, *args, **kwargs)
|
248
|
-
return await loop.run_in_executor(executor, pfunc)
|
249
|
-
|
250
|
-
return async_wrapper
|
251
|
-
|
252
|
-
|
253
|
-
def write_args_file(
|
254
|
-
*args: dict[str, Any],
|
255
|
-
path: Path,
|
256
|
-
):
|
257
|
-
"""
|
258
|
-
Merge arbitrary dictionaries and write to file
|
259
|
-
|
260
|
-
Args:
|
261
|
-
*args:
|
262
|
-
One or more dictionaries that will be merged into one respecting
|
263
|
-
the order with which they are passed in, i.e., last in overrides
|
264
|
-
previous ones.
|
265
|
-
path:
|
266
|
-
Destination for serialised file.
|
267
|
-
"""
|
268
|
-
out = {}
|
269
|
-
for d in args:
|
270
|
-
out.update(d)
|
271
|
-
|
272
|
-
with open(path, "w") as f:
|
273
|
-
json.dump(out, f, cls=TaskParameterEncoder, indent=4)
|
274
|
-
|
275
|
-
|
276
|
-
def set_start_and_last_task_index(
|
277
|
-
num_tasks: int,
|
278
|
-
first_task_index: Optional[int] = None,
|
279
|
-
last_task_index: Optional[int] = None,
|
280
|
-
) -> tuple[int, int]:
|
281
|
-
"""
|
282
|
-
Handle `first_task_index` and `last_task_index`, by setting defaults and
|
283
|
-
validating values.
|
284
|
-
|
285
|
-
num_tasks:
|
286
|
-
Total number of tasks in a workflow task list
|
287
|
-
first_task_index:
|
288
|
-
Positional index of the first task to execute
|
289
|
-
last_task_index:
|
290
|
-
Positional index of the last task to execute
|
291
|
-
"""
|
292
|
-
# Set default values
|
293
|
-
if first_task_index is None:
|
294
|
-
first_task_index = 0
|
295
|
-
if last_task_index is None:
|
296
|
-
last_task_index = num_tasks - 1
|
297
|
-
|
298
|
-
# Perform checks
|
299
|
-
if first_task_index < 0:
|
300
|
-
raise ValueError(f"{first_task_index=} cannot be negative")
|
301
|
-
if last_task_index < 0:
|
302
|
-
raise ValueError(f"{last_task_index=} cannot be negative")
|
303
|
-
if last_task_index > num_tasks - 1:
|
304
|
-
raise ValueError(
|
305
|
-
f"{last_task_index=} cannot be larger than {(num_tasks-1)=}"
|
306
|
-
)
|
307
|
-
if first_task_index > last_task_index:
|
308
|
-
raise ValueError(
|
309
|
-
f"{first_task_index=} cannot be larger than {last_task_index=}"
|
310
|
-
)
|
311
|
-
return (first_task_index, last_task_index)
|
@@ -1,97 +0,0 @@
|
|
1
|
-
fractal_server/__init__.py,sha256=M4MrXRhyDJkCg2BeVrXlq6g_UivxV447cC9KJCkf2Nc,22
|
2
|
-
fractal_server/__main__.py,sha256=CocbzZooX1UtGqPi55GcHGNxnrJXFg5tUU5b3wyFCyo,4958
|
3
|
-
fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
|
4
|
-
fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
-
fractal_server/app/db/__init__.py,sha256=WZEVfdJAX7ZyBM1ngfEGeqWWcjK_NygtCbawpmbwGpU,4042
|
6
|
-
fractal_server/app/models/__init__.py,sha256=RuxWH8fsmkTWsjLhYjrxSt-mvk74coCilAQlX2Q6OO0,353
|
7
|
-
fractal_server/app/models/dataset.py,sha256=nydU9syGVXSVuj3sTsVXIiU2vhTUrdwcUZipM-p00GY,2000
|
8
|
-
fractal_server/app/models/job.py,sha256=t0O9EKGQO4aPuTtc_N9SzLF2vrc-pevjsHumLeCPvM8,3287
|
9
|
-
fractal_server/app/models/linkuserproject.py,sha256=RVtl25Q_N99uoVDE7wx0IN0SgFjc7Id5XbScsgrjv_E,309
|
10
|
-
fractal_server/app/models/project.py,sha256=lK2CObOto_ozeNQ0gzHiioqaMIavCc-Zh_GE8yWbBTQ,848
|
11
|
-
fractal_server/app/models/security.py,sha256=Lvf1Z50oQneDSJeJxYjQcmNTJHAb64EW3hnjfu_ahUY,3135
|
12
|
-
fractal_server/app/models/state.py,sha256=rSTjYPfPZntEfdQudKp6yu5vsdyfHA7nMYNRIBWsiAQ,1087
|
13
|
-
fractal_server/app/models/task.py,sha256=APndtea9A7EF7TtpVK8kWapBM01a6nk3FFCrQbbioI8,2632
|
14
|
-
fractal_server/app/models/workflow.py,sha256=B6v3qqNDb6hvAyDN63n5vkemNueR2aH6zpwSGLlcRNE,3933
|
15
|
-
fractal_server/app/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
16
|
-
fractal_server/app/routes/admin.py,sha256=IfPEUhZA7P__pKiKjkJbOzot3HcvSZ6raJDeM61ml-k,13872
|
17
|
-
fractal_server/app/routes/api/__init__.py,sha256=EVyZrEq3I_1643QGTPCC5lgCp4xH_auYbrFfogTm4pc,315
|
18
|
-
fractal_server/app/routes/api/v1/__init__.py,sha256=V4nhYyMIqhlJxbotLTYikq_ghb6KID0ZKOOYaOq7C-g,944
|
19
|
-
fractal_server/app/routes/api/v1/_aux_functions.py,sha256=wcrYf29PrCrRHAH_8CIOfMge17RGU8iTro4jKvjajDM,11948
|
20
|
-
fractal_server/app/routes/api/v1/dataset.py,sha256=_MRUS4_0kADkuKG0VciBQNFUDFj8PCF9GV9896W4eJc,16553
|
21
|
-
fractal_server/app/routes/api/v1/job.py,sha256=82QMOfSnoO4t3Y90gNif6bBol13018tXQN-KrCkiB2U,5400
|
22
|
-
fractal_server/app/routes/api/v1/project.py,sha256=Z3hqcH6_5H-ddaOxsDNM9qgvKt7ObyuwQKsIFNE-7fE,15673
|
23
|
-
fractal_server/app/routes/api/v1/task.py,sha256=CwGbmlJYoKlX_PuoV273tALAb0WCNuuc9DxqLkDlUtA,5745
|
24
|
-
fractal_server/app/routes/api/v1/task_collection.py,sha256=zKkKd-3hne16hYCaopySvkj1l8HOfWozgjHsQaceGN8,8340
|
25
|
-
fractal_server/app/routes/api/v1/workflow.py,sha256=3dfFBUh0qJ_h4zMEsRgPit7g2Nu7v0CczeyfVA_Q4Fw,10864
|
26
|
-
fractal_server/app/routes/api/v1/workflowtask.py,sha256=9QrsnZatai4PXvRgD7gfT-8QGRu787-2wenN_6gfYuo,5550
|
27
|
-
fractal_server/app/routes/auth.py,sha256=Xv80iqdyfY3lyicYs2Y8B6zEDEnyUu_H6_6psYtv3R4,4885
|
28
|
-
fractal_server/app/routes/aux/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
29
|
-
fractal_server/app/routes/aux/_job.py,sha256=whx2G9PCCt-Hw_lgsZa1ECQlhDKNq4eHvwqgpgvBgwg,1246
|
30
|
-
fractal_server/app/routes/aux/_runner.py,sha256=psW6fsoo_VrAHrD5UQPbqFYikCp0m16VRymC-U1yUTk,675
|
31
|
-
fractal_server/app/runner/.gitignore,sha256=ytzN_oyHWXrGU7iFAtoHSTUbM6Rn6kG0Zkddg0xZk6s,16
|
32
|
-
fractal_server/app/runner/__init__.py,sha256=_GRnrvToLr4UY2RF35oXNknT8rjOfaJ4BNKHd8quxaw,13812
|
33
|
-
fractal_server/app/runner/_common.py,sha256=cRmhAayWLbXrSYV0ZJNAnlZp8hqvLofJmaFLGNChSfE,24363
|
34
|
-
fractal_server/app/runner/_local/__init__.py,sha256=gHsilCnT9VkqVbKpnEIZCnx4BuDydWcKneeWHWb2410,6799
|
35
|
-
fractal_server/app/runner/_local/_local_config.py,sha256=-oNTsjEUmytHlsYpWfw2CrPvSxDFeEhZSdQvI_wf3Mk,3245
|
36
|
-
fractal_server/app/runner/_local/_submit_setup.py,sha256=cP4gjQ_3TFgqglscQacp9dB3aqeXup5tVqqWE7TZl9Q,1631
|
37
|
-
fractal_server/app/runner/_local/executor.py,sha256=QrJlD77G6q4WohoJQO7XXbvi2RlCUsNvMnPDEZIoAqA,3620
|
38
|
-
fractal_server/app/runner/_slurm/.gitignore,sha256=ytzN_oyHWXrGU7iFAtoHSTUbM6Rn6kG0Zkddg0xZk6s,16
|
39
|
-
fractal_server/app/runner/_slurm/__init__.py,sha256=vHuEPhmwZi6c22sAF3rKy3rvSBOA9E1FdwFVQlG3J28,4850
|
40
|
-
fractal_server/app/runner/_slurm/_batching.py,sha256=KE4NrLXRHFZQSLW2vbUyu0X7TE7bTd2WCRrbYhXRTow,8840
|
41
|
-
fractal_server/app/runner/_slurm/_executor_wait_thread.py,sha256=ZGwquq2UHCr84f-b5gH14cmRFgJHB7pYwQSeRkIzxcA,4402
|
42
|
-
fractal_server/app/runner/_slurm/_slurm_config.py,sha256=fNjnqbQgZ2wSMTTUOC9HuPis9MHxX6rl49wd4ro2SKY,21010
|
43
|
-
fractal_server/app/runner/_slurm/_submit_setup.py,sha256=JIPmZEqyLRByQ3SgqiyocQlsHjfm0wKCk7W-KRBGu_0,2930
|
44
|
-
fractal_server/app/runner/_slurm/_subprocess_run_as_user.py,sha256=KYaifaAR8JsQ0OZW2A6JlfL0GsiQ6WFUa5fTvaMgA-g,5122
|
45
|
-
fractal_server/app/runner/_slurm/executor.py,sha256=H5Cpu8ths5XZysvX1dTdEEbWQyku1_C8YNvN-nAeXuw,43886
|
46
|
-
fractal_server/app/runner/_slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77JRK82g_2hx0iBKTiMghuIo,5852
|
47
|
-
fractal_server/app/runner/common.py,sha256=soO9qFWh1Aac13oolk0K1VpP0VIWG5QqTNkcqzHDDUE,9508
|
48
|
-
fractal_server/app/runner/handle_failed_job.py,sha256=Kov_Ha1rcPNdoLuQx8Dq4fz7s2naR25ce4oQaUy-7TI,4653
|
49
|
-
fractal_server/app/schemas/__init__.py,sha256=vjGKGMM45ywNClHV5KZ2u9eGLCa4p7i6ueQqCGtPcSk,2010
|
50
|
-
fractal_server/app/schemas/_validators.py,sha256=s9a6AX4-3Vfoy1Y_HMQA3lXm4FLdmnODYUD4lfsJr6w,2549
|
51
|
-
fractal_server/app/schemas/applyworkflow.py,sha256=hDYB5Oao1uq1RURUBSxFJH7L3AO5YTXCqTxnvICkeZA,4264
|
52
|
-
fractal_server/app/schemas/dataset.py,sha256=e5rM5vyrxWsuWn-rb0BUaGLYS5BtE_Ksq4Vpi8FjDGM,3375
|
53
|
-
fractal_server/app/schemas/dumps.py,sha256=ovxbPB6Zfq1t2R8exBHj-jl0clvI-BcVyGfamU25qtY,1258
|
54
|
-
fractal_server/app/schemas/json_schemas/manifest.json,sha256=yXYKHbYXPYSkSXMTLfTpfCUGBtmQuPTk1xuSXscdba4,1787
|
55
|
-
fractal_server/app/schemas/manifest.py,sha256=xxTd39dAXMK9Ox1y-p3gbyg0zd5udW99pV4JngCUGwM,3819
|
56
|
-
fractal_server/app/schemas/project.py,sha256=NSileJqsKdvRd7wg5nUC-uAJhyAKuTLIZU1FU_w1HqY,1196
|
57
|
-
fractal_server/app/schemas/state.py,sha256=t4XM04aqxeluh8MfvD7LfEc-8-dOmUVluZHhLsfxxkc,692
|
58
|
-
fractal_server/app/schemas/task.py,sha256=2TBE5Ne9tO_-a2-Es0PRXMT8ZddSInTOPMor7u8-gx0,3671
|
59
|
-
fractal_server/app/schemas/task_collection.py,sha256=nkbW076pB0wWYyWkFpplyLBBEWufAP6buYAmEupWV6I,3044
|
60
|
-
fractal_server/app/schemas/user.py,sha256=rE8WgBz-ceVUs0Sz2ZwcjUrSTZTnS0ys5SBtD2XD9r8,3113
|
61
|
-
fractal_server/app/schemas/workflow.py,sha256=sbao4_hWHzby5w7syKB045XLLEwsYv-GHczBSMHM8QU,4525
|
62
|
-
fractal_server/app/security/__init__.py,sha256=wxosoHc3mJYPCdPMyWnRD8w_2OgnKYp2aDkdmwrZh5k,11203
|
63
|
-
fractal_server/config.py,sha256=wsXKDiJIygo1hgdNCRF0XAOEOcxwbfciIabsB1URrEE,15302
|
64
|
-
fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
|
65
|
-
fractal_server/logger.py,sha256=95duXY8eSxf1HWg0CVn8SUGNzgJw9ZR0FlapDDF6WAY,3924
|
66
|
-
fractal_server/main.py,sha256=qz0YrcLCEYVGobHcrntQJicbPiC7y4MFoKTlBkZmCqI,2706
|
67
|
-
fractal_server/migrations/README,sha256=4rQvyDfqodGhpJw74VYijRmgFP49ji5chyEemWGHsuw,59
|
68
|
-
fractal_server/migrations/env.py,sha256=05EoWw0p43ojTNiz7UVG4lsl057B4ImSgXiHmiU-M80,2690
|
69
|
-
fractal_server/migrations/script.py.mako,sha256=oMXw9LC3zRbinWWPPDgeZ4z9FJrV2zhRWiYdS5YgNbI,526
|
70
|
-
fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py,sha256=-wHe-fOffmYeAm0JXVl_lxZ7hhDkaEVqxgxpHkb_uL8,954
|
71
|
-
fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py,sha256=Mob8McGYAcmgvrseyyYOa54E6Gsgr-4SiGdC-r9O4_A,1157
|
72
|
-
fractal_server/migrations/versions/50a13d6138fd_initial_schema.py,sha256=zwXegXs9J40eyCWi3w0c_iIBVJjXNn4VdVnQaT3KxDg,8770
|
73
|
-
fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py,sha256=Q-DsMzG3IcUV2Ol1dhJWosDvKERamBE6QvA2zzS5zpQ,1632
|
74
|
-
fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py,sha256=mbWuCkTpRAdGbRhW7lhXs_e5S6O37UAcCN6JfoY5H8A,1353
|
75
|
-
fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py,sha256=NSCuhANChsg76vBkShBl-9tQ4VEHubOjtAv1etHhlvY,2684
|
76
|
-
fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py,sha256=6pgODDtyAxevZvAJBj9IJ41inhV1RpwbpZr_qfPPu1A,1115
|
77
|
-
fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py,sha256=eKTZm3EgUgapXBxO0RuHkEfTKic-TZG3ADaMpGLuc0k,1057
|
78
|
-
fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py,sha256=0im6TxDr53sKKcjiPgeH4ftVRGnRXZSh2lPbRQ1Ir9w,883
|
79
|
-
fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py,sha256=4l1AHGUsa0ONoJVZlr3fTXw_xbbQ8O7wlD92Az2aRfM,1849
|
80
|
-
fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py,sha256=ekDUML7ILpmdoqEclKbEUdyLi4uw9HSG_sTjG2hp_JE,867
|
81
|
-
fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py,sha256=6cHEZFuTXiQg9yu32Y3RH1XAl71av141WQ6UMbiITIg,949
|
82
|
-
fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py,sha256=lOggSvzGWqQvnxxFuSM6W50Ui49R918A-uBuiZJ0pNM,963
|
83
|
-
fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py,sha256=jilQW3QIqYQ4Q6hCnUiG7UtNMpA41ujqrB3tPFiPM1Q,1221
|
84
|
-
fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=9BwqUS9Gf7UW_KjrzHbtViC880qhD452KAytkHWWZyk,746
|
85
|
-
fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
86
|
-
fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,2786
|
87
|
-
fractal_server/tasks/_TaskCollectPip.py,sha256=Y1YPu0YB0z5abmwyWvBhFVIkP8ORv6lxihg8Q5zsY9I,3765
|
88
|
-
fractal_server/tasks/__init__.py,sha256=k5bhaUOXRrSQSik_riqTDQlWgNHzHMR92AIwmyBrIlw,176
|
89
|
-
fractal_server/tasks/background_operations.py,sha256=GiDIE4s3tVkjJbUle7rSzQsldiFnABes8Vm2zii1WdY,12744
|
90
|
-
fractal_server/tasks/endpoint_operations.py,sha256=PC94y_sNajyGxNFsgxNGB8FDZF8MuCxquL6l63FJeY4,5549
|
91
|
-
fractal_server/tasks/utils.py,sha256=-j8T1VBbjTt5fjP2XdIcs0nBwSkYyuv_yLI1troBg9Q,2274
|
92
|
-
fractal_server/utils.py,sha256=b7WwFdcFZ8unyT65mloFToYuEDXpQoHRcmRNqrhd_dQ,2115
|
93
|
-
fractal_server-1.4.9.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
|
94
|
-
fractal_server-1.4.9.dist-info/METADATA,sha256=16Ky2HoQxuLH-bzaRi-bCG3-eUSX3vcg42yRZbXpaSw,4263
|
95
|
-
fractal_server-1.4.9.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
|
96
|
-
fractal_server-1.4.9.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
|
97
|
-
fractal_server-1.4.9.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|