fractal-server 1.4.6__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/db/__init__.py +0 -1
- fractal_server/app/models/__init__.py +6 -8
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/v1/__init__.py +12 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{state.py → v1/state.py} +2 -2
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +22 -0
- fractal_server/app/models/v2/collection_state.py +21 -0
- fractal_server/app/models/v2/dataset.py +54 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +30 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +35 -0
- fractal_server/app/models/v2/workflowtask.py +49 -0
- fractal_server/app/routes/admin/__init__.py +0 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +309 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
- fractal_server/app/routes/api/v1/dataset.py +48 -41
- fractal_server/app/routes/api/v1/job.py +14 -14
- fractal_server/app/routes/api/v1/project.py +30 -27
- fractal_server/app/routes/api/v1/task.py +26 -16
- fractal_server/app/routes/api/v1/task_collection.py +28 -16
- fractal_server/app/routes/api/v1/workflow.py +28 -28
- fractal_server/app/routes/api/v1/workflowtask.py +11 -11
- fractal_server/app/routes/api/v2/__init__.py +34 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
- fractal_server/app/routes/api/v2/dataset.py +293 -0
- fractal_server/app/routes/api/v2/images.py +279 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +186 -0
- fractal_server/app/routes/api/v2/status.py +150 -0
- fractal_server/app/routes/api/v2/submit.py +210 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +239 -0
- fractal_server/app/routes/api/v2/task_legacy.py +59 -0
- fractal_server/app/routes/api/v2/workflow.py +380 -0
- fractal_server/app/routes/api/v2/workflowtask.py +265 -0
- fractal_server/app/routes/aux/_job.py +2 -2
- fractal_server/app/runner/__init__.py +0 -379
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/components.py +5 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/__init__.py +0 -0
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/executors/slurm/_check_jobs_status.py +72 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +3 -4
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +42 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +46 -27
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +103 -0
- fractal_server/app/runner/v1/__init__.py +366 -0
- fractal_server/app/runner/{_common.py → v1/_common.py} +56 -111
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +336 -0
- fractal_server/app/runner/v2/_local/__init__.py +162 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_local/executor.py +100 -0
- fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
- fractal_server/app/runner/v2/deduplicate_list.py +23 -0
- fractal_server/app/runner/v2/handle_failed_job.py +165 -0
- fractal_server/app/runner/v2/merge_outputs.py +38 -0
- fractal_server/app/runner/v2/runner.py +343 -0
- fractal_server/app/runner/v2/runner_functions.py +374 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
- fractal_server/app/runner/v2/task_interface.py +62 -0
- fractal_server/app/runner/v2/v1_compat.py +31 -0
- fractal_server/app/schemas/__init__.py +1 -42
- fractal_server/app/schemas/_validators.py +28 -5
- fractal_server/app/schemas/v1/__init__.py +36 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +37 -0
- fractal_server/app/schemas/v2/dataset.py +126 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +114 -0
- fractal_server/app/schemas/v2/manifest.py +159 -0
- fractal_server/app/schemas/v2/project.py +34 -0
- fractal_server/app/schemas/v2/status.py +16 -0
- fractal_server/app/schemas/v2/task.py +151 -0
- fractal_server/app/schemas/v2/task_collection.py +109 -0
- fractal_server/app/schemas/v2/workflow.py +79 -0
- fractal_server/app/schemas/v2/workflowtask.py +208 -0
- fractal_server/config.py +13 -10
- fractal_server/images/__init__.py +4 -0
- fractal_server/images/models.py +136 -0
- fractal_server/images/tools.py +84 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/env.py +0 -2
- fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/v1/__init__.py +0 -0
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/__init__.py +0 -0
- fractal_server/tasks/v2/background_operations.py +381 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- fractal_server/urls.py +13 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/METADATA +11 -12
- fractal_server-2.0.0.dist-info/RECORD +169 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/common.py +0 -307
- fractal_server/app/schemas/json_schemas/manifest.json +0 -81
- fractal_server-1.4.6.dist-info/RECORD +0 -97
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -6,30 +6,30 @@ from typing import Optional
|
|
6
6
|
from pydantic import BaseModel
|
7
7
|
from pydantic import validator
|
8
8
|
|
9
|
-
from
|
10
|
-
from
|
11
|
-
from
|
12
|
-
from .project import
|
13
|
-
from .task import
|
14
|
-
from .task import
|
15
|
-
from .task import
|
9
|
+
from .._validators import valint
|
10
|
+
from .._validators import valstr
|
11
|
+
from .._validators import valutc
|
12
|
+
from .project import ProjectReadV1
|
13
|
+
from .task import TaskExportV1
|
14
|
+
from .task import TaskImportV1
|
15
|
+
from .task import TaskReadV1
|
16
16
|
|
17
17
|
__all__ = (
|
18
|
-
"
|
19
|
-
"
|
20
|
-
"
|
21
|
-
"
|
22
|
-
"
|
23
|
-
"
|
24
|
-
"
|
25
|
-
"
|
26
|
-
"
|
27
|
-
"
|
28
|
-
"
|
18
|
+
"WorkflowCreateV1",
|
19
|
+
"WorkflowReadV1",
|
20
|
+
"WorkflowUpdateV1",
|
21
|
+
"WorkflowImportV1",
|
22
|
+
"WorkflowExportV1",
|
23
|
+
"WorkflowTaskCreateV1",
|
24
|
+
"WorkflowTaskImportV1",
|
25
|
+
"WorkflowTaskExportV1",
|
26
|
+
"WorkflowTaskReadV1",
|
27
|
+
"WorkflowTaskUpdateV1",
|
28
|
+
"WorkflowTaskStatusTypeV1",
|
29
29
|
)
|
30
30
|
|
31
31
|
|
32
|
-
class
|
32
|
+
class _WorkflowTaskBaseV1(BaseModel):
|
33
33
|
"""
|
34
34
|
Base class for `WorkflowTask`.
|
35
35
|
"""
|
@@ -38,7 +38,7 @@ class _WorkflowTaskBase(BaseModel):
|
|
38
38
|
args: Optional[dict[str, Any]] = None
|
39
39
|
|
40
40
|
|
41
|
-
class
|
41
|
+
class WorkflowTaskCreateV1(_WorkflowTaskBaseV1):
|
42
42
|
"""
|
43
43
|
Class for `WorkflowTask` creation.
|
44
44
|
|
@@ -51,7 +51,7 @@ class WorkflowTaskCreate(_WorkflowTaskBase):
|
|
51
51
|
_order = validator("order", allow_reuse=True)(valint("order", min_val=0))
|
52
52
|
|
53
53
|
|
54
|
-
class
|
54
|
+
class WorkflowTaskReadV1(_WorkflowTaskBaseV1):
|
55
55
|
"""
|
56
56
|
Class for `WorkflowTask` read from database.
|
57
57
|
|
@@ -67,10 +67,10 @@ class WorkflowTaskRead(_WorkflowTaskBase):
|
|
67
67
|
order: Optional[int]
|
68
68
|
workflow_id: int
|
69
69
|
task_id: int
|
70
|
-
task:
|
70
|
+
task: TaskReadV1
|
71
71
|
|
72
72
|
|
73
|
-
class
|
73
|
+
class WorkflowTaskImportV1(_WorkflowTaskBaseV1):
|
74
74
|
"""
|
75
75
|
Class for `WorkflowTask` import.
|
76
76
|
|
@@ -78,10 +78,10 @@ class WorkflowTaskImport(_WorkflowTaskBase):
|
|
78
78
|
task:
|
79
79
|
"""
|
80
80
|
|
81
|
-
task:
|
81
|
+
task: TaskImportV1
|
82
82
|
|
83
83
|
|
84
|
-
class
|
84
|
+
class WorkflowTaskExportV1(_WorkflowTaskBaseV1):
|
85
85
|
"""
|
86
86
|
Class for `WorkflowTask` export.
|
87
87
|
|
@@ -89,10 +89,10 @@ class WorkflowTaskExport(_WorkflowTaskBase):
|
|
89
89
|
task:
|
90
90
|
"""
|
91
91
|
|
92
|
-
task:
|
92
|
+
task: TaskExportV1
|
93
93
|
|
94
94
|
|
95
|
-
class
|
95
|
+
class WorkflowTaskUpdateV1(_WorkflowTaskBaseV1):
|
96
96
|
"""
|
97
97
|
Class for `WorkflowTask` update.
|
98
98
|
"""
|
@@ -107,7 +107,7 @@ class WorkflowTaskUpdate(_WorkflowTaskBase):
|
|
107
107
|
return m
|
108
108
|
|
109
109
|
|
110
|
-
class
|
110
|
+
class _WorkflowBaseV1(BaseModel):
|
111
111
|
"""
|
112
112
|
Base class for `Workflow`.
|
113
113
|
|
@@ -118,7 +118,7 @@ class _WorkflowBase(BaseModel):
|
|
118
118
|
name: str
|
119
119
|
|
120
120
|
|
121
|
-
class
|
121
|
+
class WorkflowReadV1(_WorkflowBaseV1):
|
122
122
|
"""
|
123
123
|
Task for `Workflow` read from database.
|
124
124
|
|
@@ -131,8 +131,8 @@ class WorkflowRead(_WorkflowBase):
|
|
131
131
|
|
132
132
|
id: int
|
133
133
|
project_id: int
|
134
|
-
task_list: list[
|
135
|
-
project:
|
134
|
+
task_list: list[WorkflowTaskReadV1]
|
135
|
+
project: ProjectReadV1
|
136
136
|
timestamp_created: datetime
|
137
137
|
|
138
138
|
_timestamp_created = validator("timestamp_created", allow_reuse=True)(
|
@@ -140,7 +140,7 @@ class WorkflowRead(_WorkflowBase):
|
|
140
140
|
)
|
141
141
|
|
142
142
|
|
143
|
-
class
|
143
|
+
class WorkflowCreateV1(_WorkflowBaseV1):
|
144
144
|
"""
|
145
145
|
Task for `Workflow` creation.
|
146
146
|
"""
|
@@ -149,7 +149,7 @@ class WorkflowCreate(_WorkflowBase):
|
|
149
149
|
_name = validator("name", allow_reuse=True)(valstr("name"))
|
150
150
|
|
151
151
|
|
152
|
-
class
|
152
|
+
class WorkflowUpdateV1(_WorkflowBaseV1):
|
153
153
|
"""
|
154
154
|
Task for `Workflow` update.
|
155
155
|
|
@@ -173,7 +173,7 @@ class WorkflowUpdate(_WorkflowBase):
|
|
173
173
|
return value
|
174
174
|
|
175
175
|
|
176
|
-
class
|
176
|
+
class WorkflowImportV1(_WorkflowBaseV1):
|
177
177
|
"""
|
178
178
|
Class for `Workflow` import.
|
179
179
|
|
@@ -181,13 +181,13 @@ class WorkflowImport(_WorkflowBase):
|
|
181
181
|
task_list:
|
182
182
|
"""
|
183
183
|
|
184
|
-
task_list: list[
|
184
|
+
task_list: list[WorkflowTaskImportV1]
|
185
185
|
|
186
186
|
# Validators
|
187
187
|
_name = validator("name", allow_reuse=True)(valstr("name"))
|
188
188
|
|
189
189
|
|
190
|
-
class
|
190
|
+
class WorkflowExportV1(_WorkflowBaseV1):
|
191
191
|
"""
|
192
192
|
Class for `Workflow` export.
|
193
193
|
|
@@ -195,10 +195,10 @@ class WorkflowExport(_WorkflowBase):
|
|
195
195
|
task_list:
|
196
196
|
"""
|
197
197
|
|
198
|
-
task_list: list[
|
198
|
+
task_list: list[WorkflowTaskExportV1]
|
199
199
|
|
200
200
|
|
201
|
-
class
|
201
|
+
class WorkflowTaskStatusTypeV1(str, Enum):
|
202
202
|
"""
|
203
203
|
Define the available values for the status of a `WorkflowTask`.
|
204
204
|
|
@@ -0,0 +1,37 @@
|
|
1
|
+
from .dataset import DatasetCreateV2 # noqa F401
|
2
|
+
from .dataset import DatasetExportV2 # noqa F401
|
3
|
+
from .dataset import DatasetImportV2 # noqa F401
|
4
|
+
from .dataset import DatasetReadV2 # noqa F401
|
5
|
+
from .dataset import DatasetUpdateV2 # noqa F401
|
6
|
+
from .dumps import DatasetDumpV2 # noqa F401
|
7
|
+
from .dumps import ProjectDumpV2 # noqa F401
|
8
|
+
from .dumps import TaskDumpV2 # noqa F401
|
9
|
+
from .dumps import WorkflowDumpV2 # noqa F401
|
10
|
+
from .dumps import WorkflowTaskDumpV2 # noqa F401
|
11
|
+
from .job import JobCreateV2 # noqa F401
|
12
|
+
from .job import JobReadV2 # noqa F401
|
13
|
+
from .job import JobStatusTypeV2 # noqa F401
|
14
|
+
from .job import JobUpdateV2 # noqa F401
|
15
|
+
from .manifest import ManifestV2 # noqa F401
|
16
|
+
from .project import ProjectCreateV2 # noqa F401
|
17
|
+
from .project import ProjectReadV2 # noqa F401
|
18
|
+
from .project import ProjectUpdateV2 # noqa F401
|
19
|
+
from .task import TaskCreateV2 # noqa F401
|
20
|
+
from .task import TaskExportV2 # noqa F401
|
21
|
+
from .task import TaskImportV2 # noqa F401
|
22
|
+
from .task import TaskLegacyReadV2 # noqa F401
|
23
|
+
from .task import TaskReadV2 # noqa F401
|
24
|
+
from .task import TaskUpdateV2 # noqa F401
|
25
|
+
from .task_collection import TaskCollectPipV2 # noqa F401
|
26
|
+
from .task_collection import TaskCollectStatusV2 # noqa F401
|
27
|
+
from .workflow import WorkflowCreateV2 # noqa F401
|
28
|
+
from .workflow import WorkflowExportV2 # noqa F401
|
29
|
+
from .workflow import WorkflowImportV2 # noqa F401
|
30
|
+
from .workflow import WorkflowReadV2 # noqa F401
|
31
|
+
from .workflow import WorkflowUpdateV2 # noqa F401
|
32
|
+
from .workflowtask import WorkflowTaskCreateV2 # noqa F401
|
33
|
+
from .workflowtask import WorkflowTaskExportV2 # noqa F401
|
34
|
+
from .workflowtask import WorkflowTaskImportV2 # noqa F401
|
35
|
+
from .workflowtask import WorkflowTaskReadV2 # noqa F401
|
36
|
+
from .workflowtask import WorkflowTaskStatusTypeV2 # noqa F401
|
37
|
+
from .workflowtask import WorkflowTaskUpdateV2 # noqa F401
|
@@ -0,0 +1,126 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from pydantic import BaseModel
|
5
|
+
from pydantic import Extra
|
6
|
+
from pydantic import Field
|
7
|
+
from pydantic import validator
|
8
|
+
|
9
|
+
from .._validators import valstr
|
10
|
+
from .._validators import valutc
|
11
|
+
from .dumps import WorkflowTaskDumpV2
|
12
|
+
from .project import ProjectReadV2
|
13
|
+
from .workflowtask import WorkflowTaskStatusTypeV2
|
14
|
+
from fractal_server.images import Filters
|
15
|
+
from fractal_server.images import SingleImage
|
16
|
+
from fractal_server.urls import normalize_url
|
17
|
+
|
18
|
+
|
19
|
+
class _DatasetHistoryItemV2(BaseModel):
|
20
|
+
"""
|
21
|
+
Class for an item of `Dataset.history`.
|
22
|
+
"""
|
23
|
+
|
24
|
+
workflowtask: WorkflowTaskDumpV2
|
25
|
+
status: WorkflowTaskStatusTypeV2
|
26
|
+
parallelization: Optional[dict]
|
27
|
+
|
28
|
+
|
29
|
+
# CRUD
|
30
|
+
|
31
|
+
|
32
|
+
class DatasetCreateV2(BaseModel, extra=Extra.forbid):
|
33
|
+
|
34
|
+
name: str
|
35
|
+
|
36
|
+
zarr_dir: str
|
37
|
+
|
38
|
+
filters: Filters = Field(default_factory=Filters)
|
39
|
+
|
40
|
+
# Validators
|
41
|
+
@validator("zarr_dir")
|
42
|
+
def normalize_zarr_dir(cls, v: str) -> str:
|
43
|
+
return normalize_url(v)
|
44
|
+
|
45
|
+
_name = validator("name", allow_reuse=True)(valstr("name"))
|
46
|
+
|
47
|
+
|
48
|
+
class DatasetReadV2(BaseModel):
|
49
|
+
|
50
|
+
id: int
|
51
|
+
name: str
|
52
|
+
|
53
|
+
project_id: int
|
54
|
+
project: ProjectReadV2
|
55
|
+
|
56
|
+
history: list[_DatasetHistoryItemV2]
|
57
|
+
|
58
|
+
timestamp_created: datetime
|
59
|
+
|
60
|
+
zarr_dir: str
|
61
|
+
filters: Filters = Field(default_factory=Filters)
|
62
|
+
|
63
|
+
# Validators
|
64
|
+
_timestamp_created = validator("timestamp_created", allow_reuse=True)(
|
65
|
+
valutc("timestamp_created")
|
66
|
+
)
|
67
|
+
|
68
|
+
|
69
|
+
class DatasetUpdateV2(BaseModel):
|
70
|
+
class Config:
|
71
|
+
extra = "forbid"
|
72
|
+
|
73
|
+
name: Optional[str]
|
74
|
+
zarr_dir: Optional[str]
|
75
|
+
filters: Optional[Filters]
|
76
|
+
|
77
|
+
# Validators
|
78
|
+
@validator("zarr_dir")
|
79
|
+
def normalize_zarr_dir(cls, v: Optional[str]) -> Optional[str]:
|
80
|
+
if v is not None:
|
81
|
+
return normalize_url(v)
|
82
|
+
return v
|
83
|
+
|
84
|
+
_name = validator("name", allow_reuse=True)(valstr("name"))
|
85
|
+
|
86
|
+
|
87
|
+
class DatasetImportV2(BaseModel):
|
88
|
+
"""
|
89
|
+
Class for `Dataset` import.
|
90
|
+
|
91
|
+
Attributes:
|
92
|
+
name:
|
93
|
+
zarr_dir:
|
94
|
+
images:
|
95
|
+
filters:
|
96
|
+
"""
|
97
|
+
|
98
|
+
class Config:
|
99
|
+
extra = "forbid"
|
100
|
+
|
101
|
+
name: str
|
102
|
+
zarr_dir: str
|
103
|
+
images: list[SingleImage] = Field(default_factory=[])
|
104
|
+
filters: Filters = Field(default_factory=Filters)
|
105
|
+
|
106
|
+
# Validators
|
107
|
+
@validator("zarr_dir")
|
108
|
+
def normalize_zarr_dir(cls, v: str) -> str:
|
109
|
+
return normalize_url(v)
|
110
|
+
|
111
|
+
|
112
|
+
class DatasetExportV2(BaseModel):
|
113
|
+
"""
|
114
|
+
Class for `Dataset` export.
|
115
|
+
|
116
|
+
Attributes:
|
117
|
+
name:
|
118
|
+
zarr_dir:
|
119
|
+
images:
|
120
|
+
filters:
|
121
|
+
"""
|
122
|
+
|
123
|
+
name: str
|
124
|
+
zarr_dir: str
|
125
|
+
images: list[SingleImage]
|
126
|
+
filters: Filters
|
@@ -0,0 +1,87 @@
|
|
1
|
+
"""
|
2
|
+
|
3
|
+
Dump models differ from their Read counterpart in that:
|
4
|
+
* They are directly JSON-able, without any additional encoder.
|
5
|
+
* They may only include a subset of the Read attributes.
|
6
|
+
|
7
|
+
These models are used in at least two situations:
|
8
|
+
1. In the "*_dump" attributes of Job models;
|
9
|
+
2. In the `_DatasetHistoryItem.workflowtask` model, to trim its size.
|
10
|
+
"""
|
11
|
+
from typing import Optional
|
12
|
+
|
13
|
+
from pydantic import BaseModel
|
14
|
+
from pydantic import Extra
|
15
|
+
from pydantic import root_validator
|
16
|
+
|
17
|
+
from fractal_server.app.schemas.v1.dumps import TaskDumpV1
|
18
|
+
from fractal_server.images import Filters
|
19
|
+
|
20
|
+
|
21
|
+
class ProjectDumpV2(BaseModel, extra=Extra.forbid):
|
22
|
+
|
23
|
+
id: int
|
24
|
+
name: str
|
25
|
+
timestamp_created: str
|
26
|
+
|
27
|
+
|
28
|
+
class TaskDumpV2(BaseModel):
|
29
|
+
id: int
|
30
|
+
name: str
|
31
|
+
type: str
|
32
|
+
|
33
|
+
command_non_parallel: Optional[str]
|
34
|
+
command_parallel: Optional[str]
|
35
|
+
source: str
|
36
|
+
owner: Optional[str]
|
37
|
+
version: Optional[str]
|
38
|
+
|
39
|
+
input_types: dict[str, bool]
|
40
|
+
output_types: dict[str, bool]
|
41
|
+
|
42
|
+
|
43
|
+
class WorkflowTaskDumpV2(BaseModel):
|
44
|
+
id: int
|
45
|
+
workflow_id: int
|
46
|
+
order: Optional[int]
|
47
|
+
|
48
|
+
is_legacy_task: bool
|
49
|
+
|
50
|
+
input_filters: Filters
|
51
|
+
|
52
|
+
task_id: Optional[int]
|
53
|
+
task: Optional[TaskDumpV2]
|
54
|
+
task_legacy_id: Optional[int]
|
55
|
+
task_legacy: Optional[TaskDumpV1]
|
56
|
+
|
57
|
+
# Validators
|
58
|
+
@root_validator
|
59
|
+
def task_v1_or_v2(cls, values):
|
60
|
+
v1 = values.get("task_legacy_id")
|
61
|
+
v2 = values.get("task_id")
|
62
|
+
if ((v1 is not None) and (v2 is not None)) or (
|
63
|
+
(v1 is None) and (v2 is None)
|
64
|
+
):
|
65
|
+
message = "both" if (v1 and v2) else "none"
|
66
|
+
raise ValueError(
|
67
|
+
"One and only one must be provided between "
|
68
|
+
f"'task_legacy_id' and 'task_id' (you provided {message})"
|
69
|
+
)
|
70
|
+
return values
|
71
|
+
|
72
|
+
|
73
|
+
class WorkflowDumpV2(BaseModel):
|
74
|
+
id: int
|
75
|
+
name: str
|
76
|
+
project_id: int
|
77
|
+
timestamp_created: str
|
78
|
+
|
79
|
+
|
80
|
+
class DatasetDumpV2(BaseModel):
|
81
|
+
id: int
|
82
|
+
name: str
|
83
|
+
project_id: int
|
84
|
+
timestamp_created: str
|
85
|
+
|
86
|
+
zarr_dir: str
|
87
|
+
filters: Filters
|
@@ -0,0 +1,114 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from enum import Enum
|
3
|
+
from typing import Optional
|
4
|
+
|
5
|
+
from pydantic import BaseModel
|
6
|
+
from pydantic import Extra
|
7
|
+
from pydantic import validator
|
8
|
+
from pydantic.types import StrictStr
|
9
|
+
|
10
|
+
from .._validators import valstr
|
11
|
+
from .._validators import valutc
|
12
|
+
from .dumps import DatasetDumpV2
|
13
|
+
from .dumps import ProjectDumpV2
|
14
|
+
from .dumps import WorkflowDumpV2
|
15
|
+
|
16
|
+
|
17
|
+
class JobStatusTypeV2(str, Enum):
|
18
|
+
"""
|
19
|
+
Define the available job statuses
|
20
|
+
|
21
|
+
Attributes:
|
22
|
+
SUBMITTED:
|
23
|
+
The job was created. This does not guarantee that it was also
|
24
|
+
submitted to an executor (e.g. other errors could have prevented
|
25
|
+
this), nor that it is actually running (e.g. SLURM jobs could be
|
26
|
+
still in the queue).
|
27
|
+
DONE:
|
28
|
+
The job successfully reached its end.
|
29
|
+
FAILED:
|
30
|
+
The workflow terminated with an error.
|
31
|
+
"""
|
32
|
+
|
33
|
+
SUBMITTED = "submitted"
|
34
|
+
DONE = "done"
|
35
|
+
FAILED = "failed"
|
36
|
+
|
37
|
+
|
38
|
+
class JobCreateV2(BaseModel, extra=Extra.forbid):
|
39
|
+
|
40
|
+
first_task_index: Optional[int] = None
|
41
|
+
last_task_index: Optional[int] = None
|
42
|
+
slurm_account: Optional[StrictStr] = None
|
43
|
+
worker_init: Optional[str]
|
44
|
+
|
45
|
+
# Validators
|
46
|
+
_worker_init = validator("worker_init", allow_reuse=True)(
|
47
|
+
valstr("worker_init")
|
48
|
+
)
|
49
|
+
|
50
|
+
@validator("first_task_index", always=True)
|
51
|
+
def first_task_index_non_negative(cls, v, values):
|
52
|
+
"""
|
53
|
+
Check that `first_task_index` is non-negative.
|
54
|
+
"""
|
55
|
+
if v is not None and v < 0:
|
56
|
+
raise ValueError(
|
57
|
+
f"first_task_index cannot be negative (given: {v})"
|
58
|
+
)
|
59
|
+
return v
|
60
|
+
|
61
|
+
@validator("last_task_index", always=True)
|
62
|
+
def first_last_task_indices(cls, v, values):
|
63
|
+
"""
|
64
|
+
Check that `last_task_index` is non-negative, and that it is not
|
65
|
+
smaller than `first_task_index`.
|
66
|
+
"""
|
67
|
+
if v is not None and v < 0:
|
68
|
+
raise ValueError(
|
69
|
+
f"last_task_index cannot be negative (given: {v})"
|
70
|
+
)
|
71
|
+
|
72
|
+
first_task_index = values.get("first_task_index")
|
73
|
+
last_task_index = v
|
74
|
+
if first_task_index is not None and last_task_index is not None:
|
75
|
+
if first_task_index > last_task_index:
|
76
|
+
raise ValueError(
|
77
|
+
f"{first_task_index=} cannot be larger than "
|
78
|
+
f"{last_task_index=}"
|
79
|
+
)
|
80
|
+
return v
|
81
|
+
|
82
|
+
|
83
|
+
class JobReadV2(BaseModel):
|
84
|
+
|
85
|
+
id: int
|
86
|
+
project_id: Optional[int]
|
87
|
+
project_dump: ProjectDumpV2
|
88
|
+
user_email: str
|
89
|
+
slurm_account: Optional[str]
|
90
|
+
workflow_id: Optional[int]
|
91
|
+
workflow_dump: WorkflowDumpV2
|
92
|
+
dataset_id: Optional[int]
|
93
|
+
dataset_dump: DatasetDumpV2
|
94
|
+
start_timestamp: datetime
|
95
|
+
end_timestamp: Optional[datetime]
|
96
|
+
status: str
|
97
|
+
log: Optional[str]
|
98
|
+
working_dir: Optional[str]
|
99
|
+
working_dir_user: Optional[str]
|
100
|
+
first_task_index: Optional[int]
|
101
|
+
last_task_index: Optional[int]
|
102
|
+
worker_init: Optional[str]
|
103
|
+
|
104
|
+
_start_timestamp = validator("start_timestamp", allow_reuse=True)(
|
105
|
+
valutc("start_timestamp")
|
106
|
+
)
|
107
|
+
_end_timestamp = validator("end_timestamp", allow_reuse=True)(
|
108
|
+
valutc("end_timestamp")
|
109
|
+
)
|
110
|
+
|
111
|
+
|
112
|
+
class JobUpdateV2(BaseModel):
|
113
|
+
|
114
|
+
status: JobStatusTypeV2
|