fractal-server 2.14.13__py3-none-any.whl → 2.14.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -1
- fractal_server/app/routes/api/v2/pre_submission_checks.py +27 -20
- fractal_server/app/routes/api/v2/task.py +5 -4
- fractal_server/app/routes/api/v2/task_group.py +11 -11
- fractal_server/app/routes/api/v2/task_version_update.py +18 -10
- fractal_server/app/routes/api/v2/workflowtask.py +6 -5
- fractal_server/app/runner/executors/base_runner.py +38 -17
- fractal_server/app/runner/executors/local/runner.py +14 -14
- fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py +12 -14
- fractal_server/app/runner/v2/runner.py +19 -8
- fractal_server/app/runner/v2/runner_functions.py +12 -8
- fractal_server/app/schemas/v2/__init__.py +1 -0
- fractal_server/app/schemas/v2/dumps.py +2 -2
- fractal_server/app/schemas/v2/manifest.py +2 -9
- fractal_server/app/schemas/v2/task.py +18 -14
- fractal_server/app/schemas/v2/workflowtask.py +2 -2
- fractal_server/utils.py +0 -49
- {fractal_server-2.14.13.dist-info → fractal_server-2.14.15.dist-info}/METADATA +1 -1
- {fractal_server-2.14.13.dist-info → fractal_server-2.14.15.dist-info}/RECORD +23 -23
- {fractal_server-2.14.13.dist-info → fractal_server-2.14.15.dist-info}/LICENSE +0 -0
- {fractal_server-2.14.13.dist-info → fractal_server-2.14.15.dist-info}/WHEEL +0 -0
- {fractal_server-2.14.13.dist-info → fractal_server-2.14.15.dist-info}/entry_points.txt +0 -0
fractal_server/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__VERSION__ = "2.14.
|
1
|
+
__VERSION__ = "2.14.15"
|
@@ -137,7 +137,7 @@ async def remove_duplicate_task_groups(
|
|
137
137
|
db: AsyncSession,
|
138
138
|
) -> list[TaskGroupV2]:
|
139
139
|
"""
|
140
|
-
Extract
|
140
|
+
Extract an item for each `version` from a *sorted* task-group list.
|
141
141
|
|
142
142
|
Args:
|
143
143
|
task_groups: A list of task groups with identical `pkg_name`
|
@@ -4,7 +4,6 @@ from fastapi import status
|
|
4
4
|
from fastapi.responses import JSONResponse
|
5
5
|
from pydantic import BaseModel
|
6
6
|
from pydantic import Field
|
7
|
-
from sqlmodel import select
|
8
7
|
|
9
8
|
from ._aux_functions import _get_dataset_check_owner
|
10
9
|
from ._aux_functions import _get_workflow_task_check_owner
|
@@ -12,10 +11,11 @@ from .images import ImageQuery
|
|
12
11
|
from fractal_server.app.db import AsyncSession
|
13
12
|
from fractal_server.app.db import get_async_db
|
14
13
|
from fractal_server.app.models import UserOAuth
|
15
|
-
from fractal_server.app.models.v2 import HistoryImageCache
|
16
|
-
from fractal_server.app.models.v2 import HistoryUnit
|
17
14
|
from fractal_server.app.routes.auth import current_active_user
|
18
15
|
from fractal_server.app.schemas.v2 import HistoryUnitStatus
|
16
|
+
from fractal_server.app.schemas.v2 import TaskType
|
17
|
+
from fractal_server.images.status_tools import enrich_images_async
|
18
|
+
from fractal_server.images.status_tools import IMAGE_STATUS_KEY
|
19
19
|
from fractal_server.images.tools import aggregate_types
|
20
20
|
from fractal_server.images.tools import filter_image_list
|
21
21
|
from fractal_server.types import AttributeFilters
|
@@ -30,6 +30,7 @@ router = APIRouter()
|
|
30
30
|
async def verify_unique_types(
|
31
31
|
project_id: int,
|
32
32
|
dataset_id: int,
|
33
|
+
workflowtask_id: int,
|
33
34
|
query: ImageQuery | None = None,
|
34
35
|
user: UserOAuth = Depends(current_active_user),
|
35
36
|
db: AsyncSession = Depends(get_async_db),
|
@@ -44,8 +45,17 @@ async def verify_unique_types(
|
|
44
45
|
if query is None:
|
45
46
|
filtered_images = dataset.images
|
46
47
|
else:
|
48
|
+
if IMAGE_STATUS_KEY in query.attribute_filters.keys():
|
49
|
+
images = await enrich_images_async(
|
50
|
+
dataset_id=dataset_id,
|
51
|
+
workflowtask_id=workflowtask_id,
|
52
|
+
images=dataset.images,
|
53
|
+
db=db,
|
54
|
+
)
|
55
|
+
else:
|
56
|
+
images = dataset.images
|
47
57
|
filtered_images = filter_image_list(
|
48
|
-
images=
|
58
|
+
images=images,
|
49
59
|
attribute_filters=query.attribute_filters,
|
50
60
|
type_filters=query.type_filters,
|
51
61
|
)
|
@@ -77,7 +87,7 @@ class NonProcessedImagesPayload(BaseModel):
|
|
77
87
|
"/project/{project_id}/dataset/{dataset_id}/images/non-processed/",
|
78
88
|
status_code=status.HTTP_200_OK,
|
79
89
|
)
|
80
|
-
async def
|
90
|
+
async def check_non_processed_images(
|
81
91
|
project_id: int,
|
82
92
|
dataset_id: int,
|
83
93
|
workflow_id: int,
|
@@ -105,8 +115,8 @@ async def check_workflowtask(
|
|
105
115
|
# Skip check if previous task has non-trivial `output_types`
|
106
116
|
return JSONResponse(status_code=200, content=[])
|
107
117
|
elif previous_wft.task.type in [
|
108
|
-
|
109
|
-
|
118
|
+
TaskType.CONVERTER_COMPOUND,
|
119
|
+
TaskType.CONVERTER_NON_PARALLEL,
|
110
120
|
]:
|
111
121
|
# Skip check if previous task is converter
|
112
122
|
return JSONResponse(status_code=200, content=[])
|
@@ -124,19 +134,16 @@ async def check_workflowtask(
|
|
124
134
|
attribute_filters=filters.attribute_filters,
|
125
135
|
)
|
126
136
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
.where(HistoryImageCache.zarr_url.in_(filtered_zarr_urls))
|
133
|
-
.where(HistoryImageCache.dataset_id == dataset_id)
|
134
|
-
.where(HistoryImageCache.workflowtask_id == previous_wft.id)
|
135
|
-
.where(HistoryImageCache.latest_history_unit_id == HistoryUnit.id)
|
136
|
-
.where(HistoryUnit.status == HistoryUnitStatus.DONE)
|
137
|
+
filtered_images_with_status = await enrich_images_async(
|
138
|
+
dataset_id=dataset_id,
|
139
|
+
workflowtask_id=previous_wft.id,
|
140
|
+
images=filtered_images,
|
141
|
+
db=db,
|
137
142
|
)
|
138
|
-
|
139
|
-
|
140
|
-
|
143
|
+
missing_zarr_urls = [
|
144
|
+
img["zarr_url"]
|
145
|
+
for img in filtered_images_with_status
|
146
|
+
if img["attributes"][IMAGE_STATUS_KEY] != HistoryUnitStatus.DONE
|
147
|
+
]
|
141
148
|
|
142
149
|
return JSONResponse(status_code=200, content=missing_zarr_urls)
|
@@ -25,6 +25,7 @@ from fractal_server.app.routes.auth import current_active_verified_user
|
|
25
25
|
from fractal_server.app.schemas.v2 import TaskCreateV2
|
26
26
|
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
27
27
|
from fractal_server.app.schemas.v2 import TaskReadV2
|
28
|
+
from fractal_server.app.schemas.v2 import TaskType
|
28
29
|
from fractal_server.app.schemas.v2 import TaskUpdateV2
|
29
30
|
from fractal_server.logger import set_logger
|
30
31
|
|
@@ -109,12 +110,12 @@ async def patch_task(
|
|
109
110
|
update = task_update.model_dump(exclude_unset=True)
|
110
111
|
|
111
112
|
# Forbid changes that set a previously unset command
|
112
|
-
if db_task.type ==
|
113
|
+
if db_task.type == TaskType.NON_PARALLEL and "command_parallel" in update:
|
113
114
|
raise HTTPException(
|
114
115
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
115
116
|
detail="Cannot set an unset `command_parallel`.",
|
116
117
|
)
|
117
|
-
if db_task.type ==
|
118
|
+
if db_task.type == TaskType.PARALLEL and "command_non_parallel" in update:
|
118
119
|
raise HTTPException(
|
119
120
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
120
121
|
detail="Cannot set an unset `command_non_parallel`.",
|
@@ -151,7 +152,7 @@ async def create_task(
|
|
151
152
|
db=db,
|
152
153
|
)
|
153
154
|
|
154
|
-
if task.type ==
|
155
|
+
if task.type == TaskType.PARALLEL and (
|
155
156
|
task.args_schema_non_parallel is not None
|
156
157
|
or task.meta_non_parallel is not None
|
157
158
|
):
|
@@ -162,7 +163,7 @@ async def create_task(
|
|
162
163
|
"`TaskV2.args_schema_non_parallel` if TaskV2 is parallel"
|
163
164
|
),
|
164
165
|
)
|
165
|
-
elif task.type ==
|
166
|
+
elif task.type == TaskType.NON_PARALLEL and (
|
166
167
|
task.args_schema_parallel is not None or task.meta_parallel is not None
|
167
168
|
):
|
168
169
|
raise HTTPException(
|
@@ -163,17 +163,17 @@ async def get_task_group_list(
|
|
163
163
|
grouped_result = [
|
164
164
|
(
|
165
165
|
pkg_name,
|
166
|
-
|
167
|
-
(
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
166
|
+
(
|
167
|
+
await remove_duplicate_task_groups(
|
168
|
+
task_groups=sorted(
|
169
|
+
list(groups),
|
170
|
+
key=_version_sort_key,
|
171
|
+
reverse=True,
|
172
|
+
),
|
173
|
+
user_id=user.id,
|
174
|
+
default_group_id=default_group_id,
|
175
|
+
db=db,
|
176
|
+
)
|
177
177
|
),
|
178
178
|
)
|
179
179
|
for pkg_name, groups in itertools.groupby(
|
@@ -24,21 +24,23 @@ from ._aux_functions_tasks import _get_task_read_access
|
|
24
24
|
from fractal_server.app.models import UserOAuth
|
25
25
|
from fractal_server.app.models.v2 import TaskGroupV2
|
26
26
|
from fractal_server.app.routes.auth import current_active_user
|
27
|
+
from fractal_server.app.schemas.v2 import TaskType
|
27
28
|
from fractal_server.app.schemas.v2 import WorkflowTaskReadV2
|
28
29
|
from fractal_server.app.schemas.v2 import WorkflowTaskReplaceV2
|
29
30
|
|
30
|
-
|
31
31
|
router = APIRouter()
|
32
32
|
|
33
33
|
|
34
34
|
VALID_TYPE_UPDATES = {
|
35
|
-
|
36
|
-
(
|
37
|
-
(
|
38
|
-
|
39
|
-
(
|
40
|
-
(
|
41
|
-
(
|
35
|
+
# Transform into converter
|
36
|
+
(TaskType.NON_PARALLEL, TaskType.CONVERTER_NON_PARALLEL),
|
37
|
+
(TaskType.COMPOUND, TaskType.CONVERTER_COMPOUND),
|
38
|
+
# Keep the same
|
39
|
+
(TaskType.CONVERTER_NON_PARALLEL, TaskType.CONVERTER_NON_PARALLEL),
|
40
|
+
(TaskType.CONVERTER_COMPOUND, TaskType.CONVERTER_COMPOUND),
|
41
|
+
(TaskType.NON_PARALLEL, TaskType.NON_PARALLEL),
|
42
|
+
(TaskType.COMPOUND, TaskType.COMPOUND),
|
43
|
+
(TaskType.PARALLEL, TaskType.PARALLEL),
|
42
44
|
}
|
43
45
|
|
44
46
|
|
@@ -208,12 +210,18 @@ async def replace_workflowtask(
|
|
208
210
|
),
|
209
211
|
)
|
210
212
|
|
211
|
-
if
|
213
|
+
if (
|
214
|
+
replace.args_non_parallel is not None
|
215
|
+
and new_task.type == TaskType.PARALLEL
|
216
|
+
):
|
212
217
|
raise HTTPException(
|
213
218
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
214
219
|
detail="Cannot set 'args_non_parallel' for parallel task.",
|
215
220
|
)
|
216
|
-
if
|
221
|
+
if (
|
222
|
+
replace.args_parallel is not None
|
223
|
+
and new_task.type == TaskType.NON_PARALLEL
|
224
|
+
):
|
217
225
|
raise HTTPException(
|
218
226
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
219
227
|
detail="Cannot set 'args_parallel' for non-parallel task.",
|
@@ -15,6 +15,7 @@ from ._aux_functions_tasks import _check_type_filters_compatibility
|
|
15
15
|
from ._aux_functions_tasks import _get_task_read_access
|
16
16
|
from fractal_server.app.models import UserOAuth
|
17
17
|
from fractal_server.app.routes.auth import current_active_user
|
18
|
+
from fractal_server.app.schemas.v2 import TaskType
|
18
19
|
from fractal_server.app.schemas.v2 import WorkflowTaskCreateV2
|
19
20
|
from fractal_server.app.schemas.v2 import WorkflowTaskReadV2
|
20
21
|
from fractal_server.app.schemas.v2 import WorkflowTaskUpdateV2
|
@@ -47,7 +48,7 @@ async def create_workflowtask(
|
|
47
48
|
task_id=task_id, user_id=user.id, db=db, require_active=True
|
48
49
|
)
|
49
50
|
|
50
|
-
if task.type ==
|
51
|
+
if task.type == TaskType.PARALLEL:
|
51
52
|
if (
|
52
53
|
wftask.meta_non_parallel is not None
|
53
54
|
or wftask.args_non_parallel is not None
|
@@ -60,7 +61,7 @@ async def create_workflowtask(
|
|
60
61
|
"is `parallel`."
|
61
62
|
),
|
62
63
|
)
|
63
|
-
elif task.type ==
|
64
|
+
elif task.type == TaskType.NON_PARALLEL:
|
64
65
|
if (
|
65
66
|
wftask.meta_parallel is not None
|
66
67
|
or wftask.args_parallel is not None
|
@@ -143,7 +144,7 @@ async def update_workflowtask(
|
|
143
144
|
wftask_type_filters=workflow_task_update.type_filters,
|
144
145
|
)
|
145
146
|
|
146
|
-
if db_wf_task.task_type ==
|
147
|
+
if db_wf_task.task_type == TaskType.PARALLEL and (
|
147
148
|
workflow_task_update.args_non_parallel is not None
|
148
149
|
or workflow_task_update.meta_non_parallel is not None
|
149
150
|
):
|
@@ -156,8 +157,8 @@ async def update_workflowtask(
|
|
156
157
|
),
|
157
158
|
)
|
158
159
|
elif db_wf_task.task_type in [
|
159
|
-
|
160
|
-
|
160
|
+
TaskType.NON_PARALLEL,
|
161
|
+
TaskType.CONVERTER_NON_PARALLEL,
|
161
162
|
] and (
|
162
163
|
workflow_task_update.args_parallel is not None
|
163
164
|
or workflow_task_update.meta_parallel is not None
|
@@ -1,19 +1,34 @@
|
|
1
|
+
from enum import StrEnum
|
1
2
|
from typing import Any
|
2
3
|
|
3
4
|
from fractal_server.app.runner.task_files import TaskFiles
|
4
|
-
from fractal_server.app.schemas.v2.task import
|
5
|
+
from fractal_server.app.schemas.v2.task import TaskType
|
5
6
|
from fractal_server.logger import set_logger
|
6
7
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
8
|
+
|
9
|
+
class SubmitTaskType(StrEnum):
|
10
|
+
COMPOUND = TaskType.COMPOUND
|
11
|
+
NON_PARALLEL = TaskType.NON_PARALLEL
|
12
|
+
CONVERTER_NON_PARALLEL = TaskType.CONVERTER_NON_PARALLEL
|
13
|
+
CONVERTER_COMPOUND = TaskType.CONVERTER_COMPOUND
|
14
|
+
|
15
|
+
|
16
|
+
class MultisubmitTaskType(StrEnum):
|
17
|
+
PARALLEL = TaskType.PARALLEL
|
18
|
+
COMPOUND = TaskType.COMPOUND
|
19
|
+
CONVERTER_COMPOUND = TaskType.CONVERTER_COMPOUND
|
20
|
+
|
21
|
+
|
22
|
+
TASK_TYPES_SUBMIT: list[TaskType] = [
|
23
|
+
TaskType.COMPOUND,
|
24
|
+
TaskType.CONVERTER_COMPOUND,
|
25
|
+
TaskType.NON_PARALLEL,
|
26
|
+
TaskType.CONVERTER_NON_PARALLEL,
|
12
27
|
]
|
13
|
-
TASK_TYPES_MULTISUBMIT: list[
|
14
|
-
|
15
|
-
|
16
|
-
|
28
|
+
TASK_TYPES_MULTISUBMIT: list[TaskType] = [
|
29
|
+
TaskType.COMPOUND,
|
30
|
+
TaskType.CONVERTER_COMPOUND,
|
31
|
+
TaskType.PARALLEL,
|
17
32
|
]
|
18
33
|
|
19
34
|
logger = set_logger(__name__)
|
@@ -32,7 +47,7 @@ class BaseRunner:
|
|
32
47
|
task_name: str,
|
33
48
|
parameters: dict[str, Any],
|
34
49
|
history_unit_id: int,
|
35
|
-
task_type:
|
50
|
+
task_type: TaskType,
|
36
51
|
task_files: TaskFiles,
|
37
52
|
config: Any,
|
38
53
|
user_id: int,
|
@@ -64,7 +79,7 @@ class BaseRunner:
|
|
64
79
|
list_parameters: list[dict[str, Any]],
|
65
80
|
history_unit_ids: list[int],
|
66
81
|
list_task_files: list[TaskFiles],
|
67
|
-
task_type:
|
82
|
+
task_type: TaskType,
|
68
83
|
config: Any,
|
69
84
|
user_id: int,
|
70
85
|
) -> tuple[dict[int, Any], dict[int, BaseException]]:
|
@@ -90,7 +105,7 @@ class BaseRunner:
|
|
90
105
|
def validate_submit_parameters(
|
91
106
|
self,
|
92
107
|
parameters: dict[str, Any],
|
93
|
-
task_type:
|
108
|
+
task_type: TaskType,
|
94
109
|
) -> None:
|
95
110
|
"""
|
96
111
|
Validate parameters for `submit` method
|
@@ -104,12 +119,18 @@ class BaseRunner:
|
|
104
119
|
raise ValueError(f"Invalid {task_type=} for `submit`.")
|
105
120
|
if not isinstance(parameters, dict):
|
106
121
|
raise ValueError("`parameters` must be a dictionary.")
|
107
|
-
if task_type in [
|
122
|
+
if task_type in [
|
123
|
+
TaskType.NON_PARALLEL,
|
124
|
+
TaskType.COMPOUND,
|
125
|
+
]:
|
108
126
|
if "zarr_urls" not in parameters.keys():
|
109
127
|
raise ValueError(
|
110
128
|
f"No 'zarr_urls' key in in {list(parameters.keys())}"
|
111
129
|
)
|
112
|
-
elif task_type in [
|
130
|
+
elif task_type in [
|
131
|
+
TaskType.CONVERTER_NON_PARALLEL,
|
132
|
+
TaskType.CONVERTER_COMPOUND,
|
133
|
+
]:
|
113
134
|
if "zarr_urls" in parameters.keys():
|
114
135
|
raise ValueError(
|
115
136
|
f"Forbidden 'zarr_urls' key in {list(parameters.keys())}"
|
@@ -119,7 +140,7 @@ class BaseRunner:
|
|
119
140
|
def validate_multisubmit_parameters(
|
120
141
|
self,
|
121
142
|
*,
|
122
|
-
task_type:
|
143
|
+
task_type: TaskType,
|
123
144
|
list_parameters: list[dict[str, Any]],
|
124
145
|
list_task_files: list[TaskFiles],
|
125
146
|
history_unit_ids: list[int],
|
@@ -163,7 +184,7 @@ class BaseRunner:
|
|
163
184
|
raise ValueError(
|
164
185
|
f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
|
165
186
|
)
|
166
|
-
if task_type ==
|
187
|
+
if task_type == TaskType.PARALLEL:
|
167
188
|
zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
|
168
189
|
if len(zarr_urls) != len(set(zarr_urls)):
|
169
190
|
raise ValueError("Non-unique zarr_urls")
|
@@ -3,19 +3,21 @@ from concurrent.futures import Future
|
|
3
3
|
from concurrent.futures import ThreadPoolExecutor
|
4
4
|
from pathlib import Path
|
5
5
|
from typing import Any
|
6
|
-
from typing import Literal
|
7
6
|
|
8
7
|
from ..call_command_wrapper import call_command_wrapper
|
9
8
|
from .get_local_config import LocalBackendConfig
|
10
9
|
from fractal_server.app.db import get_sync_db
|
11
10
|
from fractal_server.app.runner.exceptions import TaskExecutionError
|
12
11
|
from fractal_server.app.runner.executors.base_runner import BaseRunner
|
12
|
+
from fractal_server.app.runner.executors.base_runner import MultisubmitTaskType
|
13
|
+
from fractal_server.app.runner.executors.base_runner import SubmitTaskType
|
13
14
|
from fractal_server.app.runner.task_files import TaskFiles
|
14
15
|
from fractal_server.app.runner.v2.db_tools import (
|
15
16
|
bulk_update_status_of_history_unit,
|
16
17
|
)
|
17
18
|
from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
|
18
19
|
from fractal_server.app.schemas.v2 import HistoryUnitStatus
|
20
|
+
from fractal_server.app.schemas.v2 import TaskType
|
19
21
|
from fractal_server.logger import set_logger
|
20
22
|
|
21
23
|
logger = set_logger(__name__)
|
@@ -87,12 +89,7 @@ class LocalRunner(BaseRunner):
|
|
87
89
|
history_unit_id: int,
|
88
90
|
task_files: TaskFiles,
|
89
91
|
config: LocalBackendConfig,
|
90
|
-
task_type:
|
91
|
-
"non_parallel",
|
92
|
-
"converter_non_parallel",
|
93
|
-
"compound",
|
94
|
-
"converter_compound",
|
95
|
-
],
|
92
|
+
task_type: SubmitTaskType,
|
96
93
|
user_id: int,
|
97
94
|
) -> tuple[Any, Exception]:
|
98
95
|
logger.debug("[submit] START")
|
@@ -129,7 +126,10 @@ class LocalRunner(BaseRunner):
|
|
129
126
|
try:
|
130
127
|
result = future.result()
|
131
128
|
logger.debug("[submit] END with result")
|
132
|
-
if task_type not in [
|
129
|
+
if task_type not in [
|
130
|
+
TaskType.COMPOUND,
|
131
|
+
TaskType.CONVERTER_COMPOUND,
|
132
|
+
]:
|
133
133
|
update_status_of_history_unit(
|
134
134
|
history_unit_id=history_unit_id,
|
135
135
|
status=HistoryUnitStatus.DONE,
|
@@ -154,7 +154,7 @@ class LocalRunner(BaseRunner):
|
|
154
154
|
list_parameters: list[dict],
|
155
155
|
history_unit_ids: list[int],
|
156
156
|
list_task_files: list[TaskFiles],
|
157
|
-
task_type:
|
157
|
+
task_type: MultisubmitTaskType,
|
158
158
|
config: LocalBackendConfig,
|
159
159
|
user_id: int,
|
160
160
|
) -> tuple[dict[int, Any], dict[int, BaseException]]:
|
@@ -197,7 +197,7 @@ class LocalRunner(BaseRunner):
|
|
197
197
|
exceptions = {
|
198
198
|
ind: exception for ind in range(len(list_parameters))
|
199
199
|
}
|
200
|
-
if task_type ==
|
200
|
+
if task_type == TaskType.PARALLEL:
|
201
201
|
with next(get_sync_db()) as db:
|
202
202
|
bulk_update_status_of_history_unit(
|
203
203
|
history_unit_ids=history_unit_ids,
|
@@ -233,7 +233,7 @@ class LocalRunner(BaseRunner):
|
|
233
233
|
positional_index
|
234
234
|
]
|
235
235
|
exceptions[positional_index] = TaskExecutionError(str(e))
|
236
|
-
if task_type ==
|
236
|
+
if task_type == TaskType.PARALLEL:
|
237
237
|
with next(get_sync_db()) as db:
|
238
238
|
update_status_of_history_unit(
|
239
239
|
history_unit_id=current_history_unit_id,
|
@@ -252,14 +252,14 @@ class LocalRunner(BaseRunner):
|
|
252
252
|
with next(get_sync_db()) as db:
|
253
253
|
for positional_index, fut in finished_futures:
|
254
254
|
active_futures.pop(positional_index)
|
255
|
-
if task_type ==
|
255
|
+
if task_type == TaskType.PARALLEL:
|
256
256
|
current_history_unit_id = history_unit_ids[
|
257
257
|
positional_index
|
258
258
|
]
|
259
259
|
|
260
260
|
try:
|
261
261
|
results[positional_index] = fut.result()
|
262
|
-
if task_type ==
|
262
|
+
if task_type == TaskType.PARALLEL:
|
263
263
|
update_status_of_history_unit(
|
264
264
|
history_unit_id=current_history_unit_id,
|
265
265
|
status=HistoryUnitStatus.DONE,
|
@@ -275,7 +275,7 @@ class LocalRunner(BaseRunner):
|
|
275
275
|
exceptions[positional_index] = TaskExecutionError(
|
276
276
|
str(e)
|
277
277
|
)
|
278
|
-
if task_type ==
|
278
|
+
if task_type == TaskType.PARALLEL:
|
279
279
|
update_status_of_history_unit(
|
280
280
|
history_unit_id=current_history_unit_id,
|
281
281
|
status=HistoryUnitStatus.FAILED,
|
@@ -19,6 +19,8 @@ from fractal_server.app.models.v2 import AccountingRecordSlurm
|
|
19
19
|
from fractal_server.app.runner.exceptions import JobExecutionError
|
20
20
|
from fractal_server.app.runner.exceptions import TaskExecutionError
|
21
21
|
from fractal_server.app.runner.executors.base_runner import BaseRunner
|
22
|
+
from fractal_server.app.runner.executors.base_runner import MultisubmitTaskType
|
23
|
+
from fractal_server.app.runner.executors.base_runner import SubmitTaskType
|
22
24
|
from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
|
23
25
|
from fractal_server.app.runner.task_files import TaskFiles
|
24
26
|
from fractal_server.app.runner.v2.db_tools import (
|
@@ -26,6 +28,7 @@ from fractal_server.app.runner.v2.db_tools import (
|
|
26
28
|
)
|
27
29
|
from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
|
28
30
|
from fractal_server.app.schemas.v2 import HistoryUnitStatus
|
31
|
+
from fractal_server.app.schemas.v2 import TaskType
|
29
32
|
from fractal_server.config import get_settings
|
30
33
|
from fractal_server.logger import set_logger
|
31
34
|
from fractal_server.syringe import Inject
|
@@ -501,12 +504,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
501
504
|
history_unit_id: int,
|
502
505
|
task_files: TaskFiles,
|
503
506
|
config: SlurmConfig,
|
504
|
-
task_type:
|
505
|
-
"non_parallel",
|
506
|
-
"converter_non_parallel",
|
507
|
-
"compound",
|
508
|
-
"converter_compound",
|
509
|
-
],
|
507
|
+
task_type: SubmitTaskType,
|
510
508
|
user_id: int,
|
511
509
|
) -> tuple[Any, Exception]:
|
512
510
|
logger.debug("[submit] START")
|
@@ -604,8 +602,8 @@ class BaseSlurmRunner(BaseRunner):
|
|
604
602
|
)
|
605
603
|
else:
|
606
604
|
if task_type not in [
|
607
|
-
|
608
|
-
|
605
|
+
TaskType.COMPOUND,
|
606
|
+
TaskType.CONVERTER_COMPOUND,
|
609
607
|
]:
|
610
608
|
update_status_of_history_unit(
|
611
609
|
history_unit_id=history_unit_id,
|
@@ -641,7 +639,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
641
639
|
list_parameters: list[dict],
|
642
640
|
history_unit_ids: list[int],
|
643
641
|
list_task_files: list[TaskFiles],
|
644
|
-
task_type:
|
642
|
+
task_type: MultisubmitTaskType,
|
645
643
|
config: SlurmConfig,
|
646
644
|
user_id: int,
|
647
645
|
) -> tuple[dict[int, Any], dict[int, BaseException]]:
|
@@ -654,7 +652,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
654
652
|
logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
|
655
653
|
try:
|
656
654
|
if self.is_shutdown():
|
657
|
-
if task_type ==
|
655
|
+
if task_type == TaskType.PARALLEL:
|
658
656
|
with next(get_sync_db()) as db:
|
659
657
|
bulk_update_status_of_history_unit(
|
660
658
|
history_unit_ids=history_unit_ids,
|
@@ -680,7 +678,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
680
678
|
workdir_remote = list_task_files[0].wftask_subfolder_remote
|
681
679
|
|
682
680
|
# Create local&remote task subfolders
|
683
|
-
if task_type ==
|
681
|
+
if task_type == TaskType.PARALLEL:
|
684
682
|
self._mkdir_local_folder(workdir_local.as_posix())
|
685
683
|
self._mkdir_remote_folder(folder=workdir_remote.as_posix())
|
686
684
|
|
@@ -758,7 +756,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
758
756
|
f" Original error {str(e)}"
|
759
757
|
)
|
760
758
|
self.scancel_jobs()
|
761
|
-
if task_type ==
|
759
|
+
if task_type == TaskType.PARALLEL:
|
762
760
|
with next(get_sync_db()) as db:
|
763
761
|
bulk_update_status_of_history_unit(
|
764
762
|
history_unit_ids=history_unit_ids,
|
@@ -824,7 +822,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
824
822
|
# `result is None` is not relevant for this purpose.
|
825
823
|
if exception is not None:
|
826
824
|
exceptions[task.index] = exception
|
827
|
-
if task_type ==
|
825
|
+
if task_type == TaskType.PARALLEL:
|
828
826
|
update_status_of_history_unit(
|
829
827
|
history_unit_id=history_unit_ids[
|
830
828
|
task.index
|
@@ -834,7 +832,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
834
832
|
)
|
835
833
|
else:
|
836
834
|
results[task.index] = result
|
837
|
-
if task_type ==
|
835
|
+
if task_type == TaskType.PARALLEL:
|
838
836
|
update_status_of_history_unit(
|
839
837
|
history_unit_id=history_unit_ids[
|
840
838
|
task.index
|
@@ -10,10 +10,6 @@ from sqlalchemy.orm.attributes import flag_modified
|
|
10
10
|
from sqlmodel import delete
|
11
11
|
from sqlmodel import update
|
12
12
|
|
13
|
-
from ....images import SingleImage
|
14
|
-
from ....images.tools import filter_image_list
|
15
|
-
from ....images.tools import find_image_by_zarr_url
|
16
|
-
from ..exceptions import JobExecutionError
|
17
13
|
from .merge_outputs import merge_outputs
|
18
14
|
from .runner_functions import run_v2_task_compound
|
19
15
|
from .runner_functions import run_v2_task_non_parallel
|
@@ -28,13 +24,18 @@ from fractal_server.app.models.v2 import HistoryRun
|
|
28
24
|
from fractal_server.app.models.v2 import HistoryUnit
|
29
25
|
from fractal_server.app.models.v2 import TaskGroupV2
|
30
26
|
from fractal_server.app.models.v2 import WorkflowTaskV2
|
27
|
+
from fractal_server.app.runner.exceptions import JobExecutionError
|
31
28
|
from fractal_server.app.runner.executors.base_runner import BaseRunner
|
32
29
|
from fractal_server.app.runner.v2.db_tools import update_status_of_history_run
|
33
30
|
from fractal_server.app.schemas.v2 import HistoryUnitStatus
|
34
31
|
from fractal_server.app.schemas.v2 import TaskDumpV2
|
35
32
|
from fractal_server.app.schemas.v2 import TaskGroupDumpV2
|
33
|
+
from fractal_server.app.schemas.v2 import TaskType
|
34
|
+
from fractal_server.images import SingleImage
|
36
35
|
from fractal_server.images.status_tools import enrich_images_sync
|
37
36
|
from fractal_server.images.status_tools import IMAGE_STATUS_KEY
|
37
|
+
from fractal_server.images.tools import filter_image_list
|
38
|
+
from fractal_server.images.tools import find_image_by_zarr_url
|
38
39
|
from fractal_server.images.tools import merge_type_filters
|
39
40
|
from fractal_server.types import AttributeFilters
|
40
41
|
|
@@ -131,7 +132,11 @@ def execute_tasks_v2(
|
|
131
132
|
# PRE TASK EXECUTION
|
132
133
|
|
133
134
|
# Filter images by types and attributes (in two steps)
|
134
|
-
if wftask.task_type in [
|
135
|
+
if wftask.task_type in [
|
136
|
+
TaskType.COMPOUND,
|
137
|
+
TaskType.PARALLEL,
|
138
|
+
TaskType.NON_PARALLEL,
|
139
|
+
]:
|
135
140
|
# Non-converter task
|
136
141
|
type_filters = copy(current_dataset_type_filters)
|
137
142
|
type_filters_patch = merge_type_filters(
|
@@ -190,7 +195,10 @@ def execute_tasks_v2(
|
|
190
195
|
|
191
196
|
# TASK EXECUTION (V2)
|
192
197
|
try:
|
193
|
-
if task.type in [
|
198
|
+
if task.type in [
|
199
|
+
TaskType.NON_PARALLEL,
|
200
|
+
TaskType.CONVERTER_NON_PARALLEL,
|
201
|
+
]:
|
194
202
|
outcomes_dict, num_tasks = run_v2_task_non_parallel(
|
195
203
|
images=filtered_images,
|
196
204
|
zarr_dir=zarr_dir,
|
@@ -205,7 +213,7 @@ def execute_tasks_v2(
|
|
205
213
|
user_id=user_id,
|
206
214
|
task_type=task.type,
|
207
215
|
)
|
208
|
-
elif task.type ==
|
216
|
+
elif task.type == TaskType.PARALLEL:
|
209
217
|
outcomes_dict, num_tasks = run_v2_task_parallel(
|
210
218
|
images=filtered_images,
|
211
219
|
wftask=wftask,
|
@@ -218,7 +226,10 @@ def execute_tasks_v2(
|
|
218
226
|
dataset_id=dataset.id,
|
219
227
|
user_id=user_id,
|
220
228
|
)
|
221
|
-
elif task.type in [
|
229
|
+
elif task.type in [
|
230
|
+
TaskType.COMPOUND,
|
231
|
+
TaskType.CONVERTER_COMPOUND,
|
232
|
+
]:
|
222
233
|
outcomes_dict, num_tasks = run_v2_task_compound(
|
223
234
|
images=filtered_images,
|
224
235
|
zarr_dir=zarr_dir,
|
@@ -31,6 +31,7 @@ from fractal_server.app.runner.v2.task_interface import (
|
|
31
31
|
_cast_and_validate_TaskOutput,
|
32
32
|
)
|
33
33
|
from fractal_server.app.schemas.v2 import HistoryUnitStatus
|
34
|
+
from fractal_server.app.schemas.v2 import TaskType
|
34
35
|
from fractal_server.logger import set_logger
|
35
36
|
|
36
37
|
__all__ = [
|
@@ -135,14 +136,17 @@ def run_v2_task_non_parallel(
|
|
135
136
|
],
|
136
137
|
dataset_id: int,
|
137
138
|
history_run_id: int,
|
138
|
-
task_type: Literal[
|
139
|
+
task_type: Literal[TaskType.NON_PARALLEL, TaskType.CONVERTER_NON_PARALLEL],
|
139
140
|
user_id: int,
|
140
141
|
) -> tuple[dict[int, SubmissionOutcome], int]:
|
141
142
|
"""
|
142
143
|
This runs server-side (see `executor` argument)
|
143
144
|
"""
|
144
145
|
|
145
|
-
if task_type not in [
|
146
|
+
if task_type not in [
|
147
|
+
TaskType.NON_PARALLEL,
|
148
|
+
TaskType.CONVERTER_NON_PARALLEL,
|
149
|
+
]:
|
146
150
|
raise ValueError(
|
147
151
|
f"Invalid {task_type=} for `run_v2_task_non_parallel`."
|
148
152
|
)
|
@@ -166,14 +170,14 @@ def run_v2_task_non_parallel(
|
|
166
170
|
"zarr_dir": zarr_dir,
|
167
171
|
**(wftask.args_non_parallel or {}),
|
168
172
|
}
|
169
|
-
if task_type ==
|
173
|
+
if task_type == TaskType.NON_PARALLEL:
|
170
174
|
function_kwargs["zarr_urls"] = [img["zarr_url"] for img in images]
|
171
175
|
|
172
176
|
# Database History operations
|
173
177
|
with next(get_sync_db()) as db:
|
174
|
-
if task_type ==
|
178
|
+
if task_type == TaskType.NON_PARALLEL:
|
175
179
|
zarr_urls = function_kwargs["zarr_urls"]
|
176
|
-
elif task_type ==
|
180
|
+
elif task_type == TaskType.CONVERTER_NON_PARALLEL:
|
177
181
|
zarr_urls = []
|
178
182
|
|
179
183
|
history_unit = HistoryUnit(
|
@@ -388,7 +392,7 @@ def run_v2_task_compound(
|
|
388
392
|
],
|
389
393
|
dataset_id: int,
|
390
394
|
history_run_id: int,
|
391
|
-
task_type: Literal[
|
395
|
+
task_type: Literal[TaskType.COMPOUND, TaskType.CONVERTER_COMPOUND],
|
392
396
|
user_id: int,
|
393
397
|
) -> tuple[dict[int, SubmissionOutcome], int]:
|
394
398
|
# Get TaskFiles object
|
@@ -410,10 +414,10 @@ def run_v2_task_compound(
|
|
410
414
|
"zarr_dir": zarr_dir,
|
411
415
|
**(wftask.args_non_parallel or {}),
|
412
416
|
}
|
413
|
-
if task_type ==
|
417
|
+
if task_type == TaskType.COMPOUND:
|
414
418
|
function_kwargs["zarr_urls"] = [img["zarr_url"] for img in images]
|
415
419
|
input_image_zarr_urls = function_kwargs["zarr_urls"]
|
416
|
-
elif task_type ==
|
420
|
+
elif task_type == TaskType.CONVERTER_COMPOUND:
|
417
421
|
input_image_zarr_urls = []
|
418
422
|
|
419
423
|
# Create database History entries
|
@@ -31,6 +31,7 @@ from .task import TaskExportV2 # noqa F401
|
|
31
31
|
from .task import TaskImportV2 # noqa F401
|
32
32
|
from .task import TaskImportV2Legacy # noqa F401
|
33
33
|
from .task import TaskReadV2 # noqa F401
|
34
|
+
from .task import TaskType # noqa F401
|
34
35
|
from .task import TaskUpdateV2 # noqa F401
|
35
36
|
from .task_collection import TaskCollectCustomV2 # noqa F401
|
36
37
|
from .task_collection import TaskCollectPipV2 # noqa F401
|
@@ -11,7 +11,7 @@ from pydantic import BaseModel
|
|
11
11
|
from pydantic import ConfigDict
|
12
12
|
from pydantic import Field
|
13
13
|
|
14
|
-
from .task import
|
14
|
+
from .task import TaskType
|
15
15
|
from .task_group import TaskGroupV2OriginEnum
|
16
16
|
|
17
17
|
|
@@ -25,7 +25,7 @@ class ProjectDumpV2(BaseModel):
|
|
25
25
|
class TaskDumpV2(BaseModel):
|
26
26
|
id: int
|
27
27
|
name: str
|
28
|
-
type:
|
28
|
+
type: TaskType
|
29
29
|
|
30
30
|
command_non_parallel: str | None = None
|
31
31
|
command_parallel: str | None = None
|
@@ -4,6 +4,7 @@ from pydantic import BaseModel
|
|
4
4
|
from pydantic import Field
|
5
5
|
from pydantic import model_validator
|
6
6
|
|
7
|
+
from .task import TaskType
|
7
8
|
from fractal_server.types import DictStrAny
|
8
9
|
from fractal_server.types import HttpUrlStr
|
9
10
|
from fractal_server.types import NonEmptyStr
|
@@ -55,15 +56,7 @@ class TaskManifestV2(BaseModel):
|
|
55
56
|
modality: str | None = None
|
56
57
|
tags: list[str] = Field(default_factory=list)
|
57
58
|
|
58
|
-
type: None |
|
59
|
-
Literal[
|
60
|
-
"compound",
|
61
|
-
"converter_compound",
|
62
|
-
"non_parallel",
|
63
|
-
"converter_non_parallel",
|
64
|
-
"parallel",
|
65
|
-
]
|
66
|
-
) = None
|
59
|
+
type: None | TaskType = None
|
67
60
|
|
68
61
|
@model_validator(mode="after")
|
69
62
|
def validate_executable_args_meta(self):
|
@@ -1,5 +1,5 @@
|
|
1
|
+
from enum import StrEnum
|
1
2
|
from typing import Any
|
2
|
-
from typing import Literal
|
3
3
|
|
4
4
|
from pydantic import BaseModel
|
5
5
|
from pydantic import ConfigDict
|
@@ -14,18 +14,22 @@ from fractal_server.types import ListUniqueNonEmptyString
|
|
14
14
|
from fractal_server.types import NonEmptyStr
|
15
15
|
from fractal_server.types import TypeFilters
|
16
16
|
|
17
|
-
TaskTypeType = Literal[
|
18
|
-
"compound",
|
19
|
-
"converter_compound",
|
20
|
-
"non_parallel",
|
21
|
-
"converter_non_parallel",
|
22
|
-
"parallel",
|
23
|
-
]
|
24
|
-
|
25
17
|
|
26
18
|
logger = set_logger(__name__)
|
27
19
|
|
28
20
|
|
21
|
+
class TaskType(StrEnum):
|
22
|
+
"""
|
23
|
+
Define the available task types.
|
24
|
+
"""
|
25
|
+
|
26
|
+
COMPOUND = "compound"
|
27
|
+
CONVERTER_COMPOUND = "converter_compound"
|
28
|
+
NON_PARALLEL = "non_parallel"
|
29
|
+
CONVERTER_NON_PARALLEL = "converter_non_parallel"
|
30
|
+
PARALLEL = "parallel"
|
31
|
+
|
32
|
+
|
29
33
|
class TaskCreateV2(BaseModel):
|
30
34
|
model_config = ConfigDict(extra="forbid")
|
31
35
|
|
@@ -51,7 +55,7 @@ class TaskCreateV2(BaseModel):
|
|
51
55
|
tags: ListUniqueNonEmptyString = Field(default_factory=list)
|
52
56
|
authors: NonEmptyStr | None = None
|
53
57
|
|
54
|
-
type:
|
58
|
+
type: TaskType | None = None
|
55
59
|
|
56
60
|
@model_validator(mode="after")
|
57
61
|
def validate_commands(self):
|
@@ -78,11 +82,11 @@ class TaskCreateV2(BaseModel):
|
|
78
82
|
"Please move to `fractal-task-tools`."
|
79
83
|
)
|
80
84
|
if self.command_non_parallel is None:
|
81
|
-
self.type =
|
85
|
+
self.type = TaskType.PARALLEL
|
82
86
|
elif self.command_parallel is None:
|
83
|
-
self.type =
|
87
|
+
self.type = TaskType.NON_PARALLEL
|
84
88
|
else:
|
85
|
-
self.type =
|
89
|
+
self.type = TaskType.COMPOUND
|
86
90
|
|
87
91
|
return self
|
88
92
|
|
@@ -90,7 +94,7 @@ class TaskCreateV2(BaseModel):
|
|
90
94
|
class TaskReadV2(BaseModel):
|
91
95
|
id: int
|
92
96
|
name: str
|
93
|
-
type:
|
97
|
+
type: TaskType
|
94
98
|
source: str | None = None
|
95
99
|
version: str | None = None
|
96
100
|
|
@@ -9,7 +9,7 @@ from .task import TaskExportV2
|
|
9
9
|
from .task import TaskImportV2
|
10
10
|
from .task import TaskImportV2Legacy
|
11
11
|
from .task import TaskReadV2
|
12
|
-
from .task import
|
12
|
+
from .task import TaskType
|
13
13
|
from fractal_server.types import DictStrAny
|
14
14
|
from fractal_server.types import TypeFilters
|
15
15
|
from fractal_server.types import WorkflowTaskArgument
|
@@ -45,7 +45,7 @@ class WorkflowTaskReadV2(BaseModel):
|
|
45
45
|
|
46
46
|
type_filters: dict[str, bool]
|
47
47
|
|
48
|
-
task_type:
|
48
|
+
task_type: TaskType
|
49
49
|
task_id: int
|
50
50
|
task: TaskReadV2
|
51
51
|
|
fractal_server/utils.py
CHANGED
@@ -13,12 +13,10 @@
|
|
13
13
|
This module provides general purpose utilities that are not specific to any
|
14
14
|
subsystem.
|
15
15
|
"""
|
16
|
-
import asyncio
|
17
16
|
import shlex
|
18
17
|
import subprocess # nosec
|
19
18
|
from datetime import datetime
|
20
19
|
from datetime import timezone
|
21
|
-
from pathlib import Path
|
22
20
|
|
23
21
|
from .logger import get_logger
|
24
22
|
from .string_tools import validate_cmd
|
@@ -31,53 +29,6 @@ def get_timestamp() -> datetime:
|
|
31
29
|
return datetime.now(tz=timezone.utc)
|
32
30
|
|
33
31
|
|
34
|
-
async def execute_command_async(
|
35
|
-
*,
|
36
|
-
command: str,
|
37
|
-
cwd: Path | None = None,
|
38
|
-
logger_name: str | None = None,
|
39
|
-
) -> str:
|
40
|
-
"""
|
41
|
-
Execute arbitrary command
|
42
|
-
|
43
|
-
If the command returns a return code different from zero, a RuntimeError
|
44
|
-
containing the stderr is raised.
|
45
|
-
|
46
|
-
Args:
|
47
|
-
cwd:
|
48
|
-
The working directory for the command execution.
|
49
|
-
command:
|
50
|
-
The command to execute.
|
51
|
-
|
52
|
-
Returns:
|
53
|
-
stdout:
|
54
|
-
The stdout from the command execution.
|
55
|
-
|
56
|
-
Raises:
|
57
|
-
RuntimeError: if the process exited with non-zero status. The error
|
58
|
-
string is set to the `stderr` of the process.
|
59
|
-
"""
|
60
|
-
command_split = shlex.split(command)
|
61
|
-
cmd, *args = command_split
|
62
|
-
|
63
|
-
logger = get_logger(logger_name)
|
64
|
-
cwd_kwarg = dict() if cwd is None else dict(cwd=cwd)
|
65
|
-
proc = await asyncio.create_subprocess_exec(
|
66
|
-
cmd,
|
67
|
-
*args,
|
68
|
-
stdout=asyncio.subprocess.PIPE,
|
69
|
-
stderr=asyncio.subprocess.PIPE,
|
70
|
-
**cwd_kwarg,
|
71
|
-
)
|
72
|
-
stdout, stderr = await proc.communicate()
|
73
|
-
logger.debug(f"Subprocess call to: {command}")
|
74
|
-
logger.debug(stdout.decode("utf-8"))
|
75
|
-
logger.debug(stderr.decode("utf-8"))
|
76
|
-
if proc.returncode != 0:
|
77
|
-
raise RuntimeError(stderr.decode("utf-8"))
|
78
|
-
return stdout.decode("utf-8")
|
79
|
-
|
80
|
-
|
81
32
|
def execute_command_sync(
|
82
33
|
*,
|
83
34
|
command: str,
|
@@ -1,4 +1,4 @@
|
|
1
|
-
fractal_server/__init__.py,sha256=
|
1
|
+
fractal_server/__init__.py,sha256=1lO5LXhx8fwsKnWQVdnMtEvVmWkDE9cZ0L2RpkoG0O8,24
|
2
2
|
fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
|
3
3
|
fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
|
4
4
|
fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -35,24 +35,24 @@ fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=Z23xwvBaVEEQ5B
|
|
35
35
|
fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=GpKfw9yj01LmOAuNMTOreU1PFkCKpjK5oCt7_wp35-A,6741
|
36
36
|
fractal_server/app/routes/api/v2/_aux_functions_task_version_update.py,sha256=WLDOYCnb6fnS5avKflyx6yN24Vo1n5kJk5ZyiKbzb8Y,1175
|
37
37
|
fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=MFYnyNPBACSHXTDLXe6cSennnpmlpajN84iivOOMW7Y,11599
|
38
|
-
fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py,sha256=
|
38
|
+
fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py,sha256=8x1_q9FyCzItnPmdSdLQuwUTy4B9xCsXscp97_lJcpM,4635
|
39
39
|
fractal_server/app/routes/api/v2/dataset.py,sha256=6u4MFqJ3YZ0Zq6Xx8CRMrTPKW55ZaR63Uno21DqFr4Q,8889
|
40
40
|
fractal_server/app/routes/api/v2/history.py,sha256=BEmf_ENF5HNMy8yXrxRdo4280rWuRUa1Jw4u8R9-LQQ,15477
|
41
41
|
fractal_server/app/routes/api/v2/images.py,sha256=TS1ltUhP0_SaViupdHrSh3MLDi5OVk-lOhE1VCVyZj0,7869
|
42
42
|
fractal_server/app/routes/api/v2/job.py,sha256=8xRTwh_OCHmK9IfI_zUASa2ozewR0qu0zVBl_a4IvHw,6467
|
43
|
-
fractal_server/app/routes/api/v2/pre_submission_checks.py,sha256=
|
43
|
+
fractal_server/app/routes/api/v2/pre_submission_checks.py,sha256=122MlfYwBBIw39bo9b3xuhvzBG20olPvjCzudAxX8H4,4909
|
44
44
|
fractal_server/app/routes/api/v2/project.py,sha256=ldMEyjtwGpX2teu85sCNWaubDFlw-En8U1SA7G1VaIw,4567
|
45
45
|
fractal_server/app/routes/api/v2/status_legacy.py,sha256=ZckHeBy8y21cyQ_OLY-VmkapzMhd3g9ae-qg-r4-uVo,6317
|
46
46
|
fractal_server/app/routes/api/v2/submit.py,sha256=_BDkWtFdo8-p7kZ0Oxaidei04MfuBeaEsWtwJaKZQ_Y,8781
|
47
|
-
fractal_server/app/routes/api/v2/task.py,sha256=
|
47
|
+
fractal_server/app/routes/api/v2/task.py,sha256=ptS47XtxnHzk9bPNZV24Wfroo5sP19RE0-LsfX0ZvOc,7018
|
48
48
|
fractal_server/app/routes/api/v2/task_collection.py,sha256=FGMhTnU88Umd8nMdriUYPtpTtAHcRBRrZIYyOesFhrU,12577
|
49
49
|
fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=EfGpv6W7xDyuYYp6E7XAcXLJiLNAImUHFqMDLgfh-4s,6730
|
50
|
-
fractal_server/app/routes/api/v2/task_group.py,sha256=
|
50
|
+
fractal_server/app/routes/api/v2/task_group.py,sha256=M96VoKcLqOpZlY0RWnsHza8jN0dzAWK9lxw87Om3Fbw,9063
|
51
51
|
fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=C2U2V76YbbqDWmErJ98MH9C2C26Lve2p_35FZ1dNmXg,9095
|
52
|
-
fractal_server/app/routes/api/v2/task_version_update.py,sha256=
|
52
|
+
fractal_server/app/routes/api/v2/task_version_update.py,sha256=o8W_C0I84X0u8gAMnCvi8ChiVAKrb5WzUBuJLSuujCA,8235
|
53
53
|
fractal_server/app/routes/api/v2/workflow.py,sha256=gwMtpfUY_JiTv5_R_q1I9WNkp6nTqEVtYx8jWNJRxcU,10227
|
54
54
|
fractal_server/app/routes/api/v2/workflow_import.py,sha256=kOGDaCj0jCGK1WSYGbnUjtUg2U1YxUY9UMH-2ilqJg4,9027
|
55
|
-
fractal_server/app/routes/api/v2/workflowtask.py,sha256=
|
55
|
+
fractal_server/app/routes/api/v2/workflowtask.py,sha256=KQU9rSQNhc6TRFdUYM09zty8Bu150sKvcLGz_tX4Fgo,7548
|
56
56
|
fractal_server/app/routes/auth/__init__.py,sha256=fao6CS0WiAjHDTvBzgBVV_bSXFpEAeDBF6Z6q7rRkPc,1658
|
57
57
|
fractal_server/app/routes/auth/_aux_auth.py,sha256=UZgauY0V6mSqjte_sYI1cBl2h8bcbLaeWzgpl1jdJlk,4883
|
58
58
|
fractal_server/app/routes/auth/current_user.py,sha256=EjkwMxUA0l6FLbDJdertHGnuOoSS-HEysmm6l5FkAlY,5903
|
@@ -71,16 +71,16 @@ fractal_server/app/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
|
|
71
71
|
fractal_server/app/runner/components.py,sha256=-Ii5l8d_V6f5DFOd-Zsr8VYmOsyqw0Hox9fEFQiuqxY,66
|
72
72
|
fractal_server/app/runner/exceptions.py,sha256=tJxs7WCQ86kjezunFm4o_VAiUAyD70l3GiH6ht0waWA,3958
|
73
73
|
fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
74
|
-
fractal_server/app/runner/executors/base_runner.py,sha256=
|
74
|
+
fractal_server/app/runner/executors/base_runner.py,sha256=7ujN6gN92X9sWAlDwkc07G3FA4Z_Zwew_uBYh2qv0uI,5978
|
75
75
|
fractal_server/app/runner/executors/call_command_wrapper.py,sha256=1BHl-zbXoX2oGUWGAFprVZMmg5QjutPH0-VZJSIC0II,1419
|
76
76
|
fractal_server/app/runner/executors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
77
77
|
fractal_server/app/runner/executors/local/get_local_config.py,sha256=KBYOkcuwpSYl-ZIAwPBxpn59QSyFF8eJ-fLKVIhwwzA,3594
|
78
|
-
fractal_server/app/runner/executors/local/runner.py,sha256=
|
78
|
+
fractal_server/app/runner/executors/local/runner.py,sha256=DZK_oVxjIewyo7tjB7HvTds0nDHxIbpLJ4ABn4Mly8w,10841
|
79
79
|
fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
80
80
|
fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=gbHZIxt90GjUwhB9_UInwVqpX-KdxRQMDeXzUagdL3U,8816
|
81
81
|
fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
|
82
82
|
fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=Zv2l_6X1EfSHGRqcBMj2dbai_kP8hfuMfh-WoIUj0tY,15646
|
83
|
-
fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256
|
83
|
+
fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=2F2zgg3DJKAJ5LecFAzMSGLFmsMiM4lMk4Kh9It35F4,35626
|
84
84
|
fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=VJNryceLzF5_fx9_lS1nGq85EW8rOQ0KrgtMATcfdQc,7271
|
85
85
|
fractal_server/app/runner/executors/slurm_common/remote.py,sha256=xWnI6WktHR_7cxUme72ztIeBb4osnbZNu6J2azWn9K8,3765
|
86
86
|
fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=K4SdJOKsUWzDlnkb8Ug_UmTx6nBMsTqn9_oKqwE4XDI,3520
|
@@ -102,8 +102,8 @@ fractal_server/app/runner/v2/_slurm_sudo.py,sha256=Gvsh4tUlc1_3KdF3B7zEqs-YIntC_
|
|
102
102
|
fractal_server/app/runner/v2/db_tools.py,sha256=du5dKhMMFMErQXbGIgu9JvO_vtMensodyPsyDeqz1yQ,3324
|
103
103
|
fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
|
104
104
|
fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
|
105
|
-
fractal_server/app/runner/v2/runner.py,sha256=
|
106
|
-
fractal_server/app/runner/v2/runner_functions.py,sha256=
|
105
|
+
fractal_server/app/runner/v2/runner.py,sha256=vi5AAHl-MZwq0FMMCfdgzXv4DB8cCDWsAqvTdpaXh9Q,18972
|
106
|
+
fractal_server/app/runner/v2/runner_functions.py,sha256=aEwEDzI2l-QvgfJSj-M2LGvqA89nOiJYIFVryKEq_3M,18988
|
107
107
|
fractal_server/app/runner/v2/submit_workflow.py,sha256=AMnXdozwIGlXD55ch0_SNAG-ntKBO-QRhkbInrvsShU,13140
|
108
108
|
fractal_server/app/runner/v2/task_interface.py,sha256=V2TWBK6tbhycyMrJvFaoJ9IpuKlrLrvmjJbfNMsBBXo,2527
|
109
109
|
fractal_server/app/runner/versions.py,sha256=4BW-8Et8RVgILgpFoUJLWkEnZz53pv8hv_2ucG480ns,398
|
@@ -111,20 +111,20 @@ fractal_server/app/schemas/__init__.py,sha256=stURAU_t3AOBaH0HSUbV-GKhlPKngnnIMo
|
|
111
111
|
fractal_server/app/schemas/user.py,sha256=t9nbyYjGCSOsxm9K97PDG3-9o27CsaFfhWb_L5nrjqA,1910
|
112
112
|
fractal_server/app/schemas/user_group.py,sha256=x3-kqbo0q2wTP7QI0iZ7PU_9Dr957UYrFMKqS7BXLhE,1425
|
113
113
|
fractal_server/app/schemas/user_settings.py,sha256=NpdC0Me0fgwwdfJuTSlFLCnLUjiWWzrJlPn_UPLjXnw,1862
|
114
|
-
fractal_server/app/schemas/v2/__init__.py,sha256
|
114
|
+
fractal_server/app/schemas/v2/__init__.py,sha256=-uKDjFn1esNs7WlhrugvBRl34ABXyo7h1Gm0V5KWk9U,3087
|
115
115
|
fractal_server/app/schemas/v2/accounting.py,sha256=Wylt7uWTiDIFlHJOh4XEtYitk2FjFlmnodDrJDxcr0E,397
|
116
116
|
fractal_server/app/schemas/v2/dataset.py,sha256=NKCjBwGBC7mPiSlXktZAcleJsvlLY6KfNKw7Wx4Zfqk,1728
|
117
|
-
fractal_server/app/schemas/v2/dumps.py,sha256=
|
117
|
+
fractal_server/app/schemas/v2/dumps.py,sha256=xZcUBN3cKADzR37z0X3kjIwizdhPpDHBC_RX1rXxJ5U,2208
|
118
118
|
fractal_server/app/schemas/v2/history.py,sha256=pZiMKfh6nMWbTp5MUtrnGySPKbeRFf5tM1VLFaTgGcw,1784
|
119
119
|
fractal_server/app/schemas/v2/job.py,sha256=fPay7dLSr-skKRdVRoZig8rf_sZwUdVdHZaJ4XM8vMI,3288
|
120
|
-
fractal_server/app/schemas/v2/manifest.py,sha256=
|
120
|
+
fractal_server/app/schemas/v2/manifest.py,sha256=QUpXMDB8WkB1F4UK-yYmm3O8bXoHwDGURnqwn6ayO_I,6674
|
121
121
|
fractal_server/app/schemas/v2/project.py,sha256=l96-3bCfB3knhITaLj1WSyBgbzP_k8CdtvgX_5jO_fU,657
|
122
122
|
fractal_server/app/schemas/v2/status_legacy.py,sha256=eQT1zGxbkzSwd0EqclsOdZ60n1x6J3DB1CZ3m4LYyxc,955
|
123
|
-
fractal_server/app/schemas/v2/task.py,sha256=
|
123
|
+
fractal_server/app/schemas/v2/task.py,sha256=IJv8loB4kx9FBkaIHoiMsswQyq02FxvyAnHK1u074fU,4364
|
124
124
|
fractal_server/app/schemas/v2/task_collection.py,sha256=MyFBr5xltYk8bRSDGEpfHw4eD_LU4287UFSNDP2WPjI,4144
|
125
125
|
fractal_server/app/schemas/v2/task_group.py,sha256=R6u6CB2V62gn28Q_K8AbMHs9rWfyYTd-SPt3J4oNTU0,3172
|
126
126
|
fractal_server/app/schemas/v2/workflow.py,sha256=JtjxbDO52bmY06WUMACUKpFSdJysO4DBv7wezsvODRQ,1775
|
127
|
-
fractal_server/app/schemas/v2/workflowtask.py,sha256=
|
127
|
+
fractal_server/app/schemas/v2/workflowtask.py,sha256=6eweAMyziwaoMT-7R1fVJYunIeZKzT0-7fAVgPO_FEc,3639
|
128
128
|
fractal_server/app/security/__init__.py,sha256=oJ8RVglpOvWPQY4RokiE2YA72Nqo42dZEjywWTt8xr8,14032
|
129
129
|
fractal_server/app/security/signup_email.py,sha256=Xd6QYxcdmg0PHpDwmUE8XQmPcOj3Xjy5oROcIMhmltM,1472
|
130
130
|
fractal_server/app/user_settings.py,sha256=OP1yiYKtPadxwM51_Q0hdPk3z90TCN4z1BLpQsXyWiU,1316
|
@@ -214,10 +214,10 @@ fractal_server/types/validators/_common_validators.py,sha256=MpxyaP2kwgbyCTOaVRj
|
|
214
214
|
fractal_server/types/validators/_filter_validators.py,sha256=_s2dG9aCkiNWyX3ZLLuYlAgNKlQVm-ORdF-HpnyhX10,830
|
215
215
|
fractal_server/types/validators/_workflow_task_arguments_validators.py,sha256=HL7NgV8d56XbcD6gG5PVFUPMHjDm5Q-d7cXhVLdFAGU,387
|
216
216
|
fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
|
217
|
-
fractal_server/utils.py,sha256=
|
217
|
+
fractal_server/utils.py,sha256=Vn35lApt1T1J8nc09sAVqd10Cy0sa3dLipcljI-hkuk,2185
|
218
218
|
fractal_server/zip_tools.py,sha256=tqz_8f-vQ9OBRW-4OQfO6xxY-YInHTyHmZxU7U4PqZo,4885
|
219
|
-
fractal_server-2.14.
|
220
|
-
fractal_server-2.14.
|
221
|
-
fractal_server-2.14.
|
222
|
-
fractal_server-2.14.
|
223
|
-
fractal_server-2.14.
|
219
|
+
fractal_server-2.14.15.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
|
220
|
+
fractal_server-2.14.15.dist-info/METADATA,sha256=Sz4teXD8HvAhlbNjhwBKfOiswHcP47lMAIMnHK6puVU,4244
|
221
|
+
fractal_server-2.14.15.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
|
222
|
+
fractal_server-2.14.15.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
|
223
|
+
fractal_server-2.14.15.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|