fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/models/__init__.py +4 -7
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/state.py +1 -1
- fractal_server/app/models/v1/__init__.py +10 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +20 -0
- fractal_server/app/models/v2/dataset.py +55 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +31 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +43 -0
- fractal_server/app/models/v2/workflowtask.py +90 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +275 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +2 -2
- fractal_server/app/routes/api/v1/dataset.py +37 -37
- fractal_server/app/routes/api/v1/job.py +12 -12
- fractal_server/app/routes/api/v1/project.py +23 -21
- fractal_server/app/routes/api/v1/task.py +24 -14
- fractal_server/app/routes/api/v1/task_collection.py +16 -14
- fractal_server/app/routes/api/v1/workflow.py +24 -24
- fractal_server/app/routes/api/v1/workflowtask.py +10 -10
- fractal_server/app/routes/api/v2/__init__.py +28 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +497 -0
- fractal_server/app/routes/api/v2/apply.py +220 -0
- fractal_server/app/routes/api/v2/dataset.py +310 -0
- fractal_server/app/routes/api/v2/images.py +212 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +205 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +229 -0
- fractal_server/app/routes/api/v2/workflow.py +398 -0
- fractal_server/app/routes/api/v2/workflowtask.py +269 -0
- fractal_server/app/routes/aux/_job.py +1 -1
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/local/__init__.py +3 -0
- fractal_server/app/runner/{_local → executors/local}/executor.py +2 -2
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +9 -9
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +105 -0
- fractal_server/app/runner/{__init__.py → v1/__init__.py} +24 -22
- fractal_server/app/runner/{_common.py → v1/_common.py} +13 -120
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +6 -6
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +310 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +3 -9
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +337 -0
- fractal_server/app/runner/v2/_local/__init__.py +169 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_slurm/__init__.py +157 -0
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +83 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +179 -0
- fractal_server/app/runner/v2/components.py +5 -0
- fractal_server/app/runner/v2/deduplicate_list.py +24 -0
- fractal_server/app/runner/v2/handle_failed_job.py +156 -0
- fractal_server/app/runner/v2/merge_outputs.py +41 -0
- fractal_server/app/runner/v2/runner.py +264 -0
- fractal_server/app/runner/v2/runner_functions.py +339 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +134 -0
- fractal_server/app/runner/v2/task_interface.py +43 -0
- fractal_server/app/runner/v2/v1_compat.py +21 -0
- fractal_server/app/schemas/__init__.py +4 -42
- fractal_server/app/schemas/v1/__init__.py +42 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +34 -0
- fractal_server/app/schemas/v2/dataset.py +88 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +113 -0
- fractal_server/app/schemas/v2/manifest.py +109 -0
- fractal_server/app/schemas/v2/project.py +36 -0
- fractal_server/app/schemas/v2/task.py +121 -0
- fractal_server/app/schemas/v2/task_collection.py +105 -0
- fractal_server/app/schemas/v2/workflow.py +78 -0
- fractal_server/app/schemas/v2/workflowtask.py +118 -0
- fractal_server/config.py +5 -4
- fractal_server/images/__init__.py +50 -0
- fractal_server/images/tools.py +86 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/versions/4b35c5cefbe3_tmp_is_v2_compatible.py +39 -0
- fractal_server/migrations/versions/56af171b0159_v2.py +217 -0
- fractal_server/migrations/versions/876f28db9d4e_tmp_split_task_and_wftask_meta.py +68 -0
- fractal_server/migrations/versions/974c802f0dd0_tmp_workflowtaskv2_type_in_db.py +37 -0
- fractal_server/migrations/versions/9cd305cd6023_tmp_workflowtaskv2.py +40 -0
- fractal_server/migrations/versions/a6231ed6273c_tmp_args_schemas_in_taskv2.py +42 -0
- fractal_server/migrations/versions/b9e9eed9d442_tmp_taskv2_type.py +37 -0
- fractal_server/migrations/versions/e3e639454d4b_tmp_make_task_meta_non_optional.py +50 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +18 -50
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/background_operations.py +382 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/METADATA +1 -1
- fractal_server-2.0.0a0.dist-info/RECORD +166 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/_slurm/__init__.py +0 -150
- fractal_server/app/runner/common.py +0 -311
- fractal_server-1.4.10.dist-info/RECORD +0 -98
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0a0.dist-info}/entry_points.txt +0 -0
@@ -29,14 +29,14 @@ import cloudpickle
|
|
29
29
|
from cfut import SlurmExecutor
|
30
30
|
from cfut.util import random_string
|
31
31
|
|
32
|
-
from
|
33
|
-
from
|
34
|
-
from
|
35
|
-
from
|
36
|
-
from
|
37
|
-
from
|
38
|
-
from
|
39
|
-
from
|
32
|
+
from .....config import get_settings
|
33
|
+
from .....logger import set_logger
|
34
|
+
from .....syringe import Inject
|
35
|
+
from ...exceptions import JobExecutionError
|
36
|
+
from ...exceptions import TaskExecutionError
|
37
|
+
from ...filenames import SHUTDOWN_FILENAME
|
38
|
+
from ...task_files import get_task_file_paths
|
39
|
+
from ...task_files import TaskFiles
|
40
40
|
from ._batching import heuristics
|
41
41
|
from ._executor_wait_thread import FractalSlurmWaitThread
|
42
42
|
from ._slurm_config import get_default_slurm_config
|
@@ -1001,7 +1001,7 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
1001
1001
|
cmdlines.append(
|
1002
1002
|
(
|
1003
1003
|
f"{python_worker_interpreter}"
|
1004
|
-
" -m fractal_server.app.runner.
|
1004
|
+
" -m fractal_server.app.runner.executors.slurm.remote "
|
1005
1005
|
f"--input-file {input_pickle_file} "
|
1006
1006
|
f"--output-file {output_pickle_file}"
|
1007
1007
|
)
|
@@ -0,0 +1,39 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
|
4
|
+
def set_start_and_last_task_index(
|
5
|
+
num_tasks: int,
|
6
|
+
first_task_index: Optional[int] = None,
|
7
|
+
last_task_index: Optional[int] = None,
|
8
|
+
) -> tuple[int, int]:
|
9
|
+
"""
|
10
|
+
Handle `first_task_index` and `last_task_index`, by setting defaults and
|
11
|
+
validating values.
|
12
|
+
|
13
|
+
num_tasks:
|
14
|
+
Total number of tasks in a workflow task list
|
15
|
+
first_task_index:
|
16
|
+
Positional index of the first task to execute
|
17
|
+
last_task_index:
|
18
|
+
Positional index of the last task to execute
|
19
|
+
"""
|
20
|
+
# Set default values
|
21
|
+
if first_task_index is None:
|
22
|
+
first_task_index = 0
|
23
|
+
if last_task_index is None:
|
24
|
+
last_task_index = num_tasks - 1
|
25
|
+
|
26
|
+
# Perform checks
|
27
|
+
if first_task_index < 0:
|
28
|
+
raise ValueError(f"{first_task_index=} cannot be negative")
|
29
|
+
if last_task_index < 0:
|
30
|
+
raise ValueError(f"{last_task_index=} cannot be negative")
|
31
|
+
if last_task_index > num_tasks - 1:
|
32
|
+
raise ValueError(
|
33
|
+
f"{last_task_index=} cannot be larger than {(num_tasks-1)=}"
|
34
|
+
)
|
35
|
+
if first_task_index > last_task_index:
|
36
|
+
raise ValueError(
|
37
|
+
f"{first_task_index=} cannot be larger than {last_task_index=}"
|
38
|
+
)
|
39
|
+
return (first_task_index, last_task_index)
|
@@ -0,0 +1,105 @@
|
|
1
|
+
from functools import lru_cache
|
2
|
+
from pathlib import Path
|
3
|
+
from typing import Optional
|
4
|
+
|
5
|
+
|
6
|
+
def sanitize_component(value: str) -> str:
|
7
|
+
"""
|
8
|
+
Remove {" ", "/", "."} form a string, e.g. going from
|
9
|
+
'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
|
10
|
+
"""
|
11
|
+
return value.replace(" ", "_").replace("/", "_").replace(".", "_")
|
12
|
+
|
13
|
+
|
14
|
+
class TaskFiles:
|
15
|
+
"""
|
16
|
+
Group all file paths pertaining to a task
|
17
|
+
|
18
|
+
Attributes:
|
19
|
+
workflow_dir:
|
20
|
+
Server-owned directory to store all task-execution-related relevant
|
21
|
+
files (inputs, outputs, errors, and all meta files related to the
|
22
|
+
job execution). Note: users cannot write directly to this folder.
|
23
|
+
workflow_dir_user:
|
24
|
+
User-side directory with the same scope as `workflow_dir`, and
|
25
|
+
where a user can write.
|
26
|
+
task_order:
|
27
|
+
Positional order of the task within a workflow.
|
28
|
+
component:
|
29
|
+
Specific component to run the task for (relevant for tasks that
|
30
|
+
will be executed in parallel over many components).
|
31
|
+
file_prefix:
|
32
|
+
Prefix for all task-related files.
|
33
|
+
args:
|
34
|
+
Path for input json file.
|
35
|
+
metadiff:
|
36
|
+
Path for output json file with metadata update.
|
37
|
+
out:
|
38
|
+
Path for task-execution stdout.
|
39
|
+
err:
|
40
|
+
Path for task-execution stderr.
|
41
|
+
"""
|
42
|
+
|
43
|
+
workflow_dir: Path
|
44
|
+
workflow_dir_user: Path
|
45
|
+
task_order: Optional[int] = None
|
46
|
+
component: Optional[str] = None # FIXME: this is actually for V1 only
|
47
|
+
|
48
|
+
file_prefix: str
|
49
|
+
args: Path
|
50
|
+
out: Path
|
51
|
+
err: Path
|
52
|
+
log: Path
|
53
|
+
metadiff: Path
|
54
|
+
|
55
|
+
def __init__(
|
56
|
+
self,
|
57
|
+
workflow_dir: Path,
|
58
|
+
workflow_dir_user: Path,
|
59
|
+
task_order: Optional[int] = None,
|
60
|
+
component: Optional[str] = None,
|
61
|
+
):
|
62
|
+
self.workflow_dir = workflow_dir
|
63
|
+
self.workflow_dir_user = workflow_dir_user
|
64
|
+
self.task_order = task_order
|
65
|
+
self.component = component
|
66
|
+
|
67
|
+
if self.component is not None:
|
68
|
+
component_safe = sanitize_component(str(self.component))
|
69
|
+
component_safe = f"_par_{component_safe}"
|
70
|
+
else:
|
71
|
+
component_safe = ""
|
72
|
+
|
73
|
+
if self.task_order is not None:
|
74
|
+
order = str(self.task_order)
|
75
|
+
else:
|
76
|
+
order = "task"
|
77
|
+
self.file_prefix = f"{order}{component_safe}"
|
78
|
+
self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
|
79
|
+
self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
|
80
|
+
self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
|
81
|
+
self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
|
82
|
+
self.metadiff = (
|
83
|
+
self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
|
84
|
+
)
|
85
|
+
|
86
|
+
|
87
|
+
@lru_cache()
|
88
|
+
def get_task_file_paths(
|
89
|
+
workflow_dir: Path,
|
90
|
+
workflow_dir_user: Path,
|
91
|
+
task_order: Optional[int] = None,
|
92
|
+
component: Optional[str] = None,
|
93
|
+
) -> TaskFiles:
|
94
|
+
"""
|
95
|
+
Return the corrisponding TaskFiles object
|
96
|
+
|
97
|
+
This function is mainly used as a cache to avoid instantiating needless
|
98
|
+
objects.
|
99
|
+
"""
|
100
|
+
return TaskFiles(
|
101
|
+
workflow_dir=workflow_dir,
|
102
|
+
workflow_dir_user=workflow_dir_user,
|
103
|
+
task_order=task_order,
|
104
|
+
component=component,
|
105
|
+
)
|
@@ -22,26 +22,26 @@ import traceback
|
|
22
22
|
from pathlib import Path
|
23
23
|
from typing import Optional
|
24
24
|
|
25
|
-
from
|
26
|
-
from
|
27
|
-
from
|
28
|
-
from ...
|
29
|
-
from ...
|
30
|
-
from
|
31
|
-
from
|
32
|
-
from
|
33
|
-
from
|
34
|
-
from ..
|
35
|
-
from ..
|
36
|
-
from
|
37
|
-
from ._local import process_workflow as local_process_workflow
|
38
|
-
from ._slurm import process_workflow as slurm_process_workflow
|
25
|
+
from ....logger import set_logger
|
26
|
+
from ....syringe import Inject
|
27
|
+
from ....utils import get_timestamp
|
28
|
+
from ...db import DB
|
29
|
+
from ...models import ApplyWorkflow
|
30
|
+
from ...models import Dataset
|
31
|
+
from ...models import Workflow
|
32
|
+
from ...models import WorkflowTask
|
33
|
+
from ...schemas.v1 import JobStatusTypeV1
|
34
|
+
from ..exceptions import JobExecutionError
|
35
|
+
from ..exceptions import TaskExecutionError
|
36
|
+
from ..filenames import WORKFLOW_LOG_FILENAME
|
37
|
+
from ..v1._local import process_workflow as local_process_workflow
|
38
|
+
from ..v1._slurm import process_workflow as slurm_process_workflow
|
39
39
|
from .common import close_job_logger
|
40
|
-
from .common import JobExecutionError
|
41
|
-
from .common import TaskExecutionError
|
42
40
|
from .common import validate_workflow_compatibility # noqa: F401
|
43
41
|
from .handle_failed_job import assemble_history_failed_job
|
44
42
|
from .handle_failed_job import assemble_meta_failed_job
|
43
|
+
from fractal_server import __VERSION__
|
44
|
+
from fractal_server.config import get_settings
|
45
45
|
|
46
46
|
|
47
47
|
_backends = {}
|
@@ -122,7 +122,7 @@ async def submit_workflow(
|
|
122
122
|
log_msg += (
|
123
123
|
f"Cannot fetch workflow {workflow_id} from database\n"
|
124
124
|
)
|
125
|
-
job.status =
|
125
|
+
job.status = JobStatusTypeV1.FAILED
|
126
126
|
job.end_timestamp = get_timestamp()
|
127
127
|
job.log = log_msg
|
128
128
|
db_sync.merge(job)
|
@@ -158,7 +158,9 @@ async def submit_workflow(
|
|
158
158
|
WORKFLOW_DIR_USER = WORKFLOW_DIR
|
159
159
|
elif FRACTAL_RUNNER_BACKEND == "slurm":
|
160
160
|
|
161
|
-
from .
|
161
|
+
from ..executors.slurm._subprocess_run_as_user import (
|
162
|
+
_mkdir_as_user,
|
163
|
+
)
|
162
164
|
|
163
165
|
WORKFLOW_DIR_USER = (
|
164
166
|
Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
|
@@ -259,7 +261,7 @@ async def submit_workflow(
|
|
259
261
|
db_sync.merge(output_dataset)
|
260
262
|
|
261
263
|
# Update job DB entry
|
262
|
-
job.status =
|
264
|
+
job.status = JobStatusTypeV1.DONE
|
263
265
|
job.end_timestamp = get_timestamp()
|
264
266
|
with log_file_path.open("r") as f:
|
265
267
|
logs = f.read()
|
@@ -289,7 +291,7 @@ async def submit_workflow(
|
|
289
291
|
|
290
292
|
db_sync.merge(output_dataset)
|
291
293
|
|
292
|
-
job.status =
|
294
|
+
job.status = JobStatusTypeV1.FAILED
|
293
295
|
job.end_timestamp = get_timestamp()
|
294
296
|
|
295
297
|
exception_args_string = "\n".join(e.args)
|
@@ -322,7 +324,7 @@ async def submit_workflow(
|
|
322
324
|
|
323
325
|
db_sync.merge(output_dataset)
|
324
326
|
|
325
|
-
job.status =
|
327
|
+
job.status = JobStatusTypeV1.FAILED
|
326
328
|
job.end_timestamp = get_timestamp()
|
327
329
|
error = e.assemble_error()
|
328
330
|
job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
|
@@ -351,7 +353,7 @@ async def submit_workflow(
|
|
351
353
|
|
352
354
|
db_sync.merge(output_dataset)
|
353
355
|
|
354
|
-
job.status =
|
356
|
+
job.status = JobStatusTypeV1.FAILED
|
355
357
|
job.end_timestamp = get_timestamp()
|
356
358
|
job.log = (
|
357
359
|
f"UNKNOWN ERROR in Fractal job {job.id}\n"
|
@@ -11,7 +11,6 @@ import subprocess # nosec
|
|
11
11
|
import traceback
|
12
12
|
from concurrent.futures import Executor
|
13
13
|
from copy import deepcopy
|
14
|
-
from functools import lru_cache
|
15
14
|
from functools import partial
|
16
15
|
from pathlib import Path
|
17
16
|
from shlex import split as shlex_split
|
@@ -19,22 +18,19 @@ from typing import Any
|
|
19
18
|
from typing import Callable
|
20
19
|
from typing import Optional
|
21
20
|
|
22
|
-
from
|
23
|
-
from
|
24
|
-
from
|
25
|
-
from
|
26
|
-
from
|
27
|
-
from
|
28
|
-
from
|
29
|
-
from
|
21
|
+
from ....config import get_settings
|
22
|
+
from ....logger import get_logger
|
23
|
+
from ....syringe import Inject
|
24
|
+
from ...models import Task
|
25
|
+
from ...models import WorkflowTask
|
26
|
+
from ...schemas.v1 import WorkflowTaskStatusTypeV1
|
27
|
+
from ..exceptions import JobExecutionError
|
28
|
+
from ..exceptions import TaskExecutionError
|
30
29
|
from .common import TaskParameters
|
31
30
|
from .common import write_args_file
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
METADATA_FILENAME = "metadata.json"
|
36
|
-
SHUTDOWN_FILENAME = "shutdown"
|
37
|
-
WORKFLOW_LOG_FILENAME = "workflow.log"
|
31
|
+
from fractal_server.app.runner.filenames import HISTORY_FILENAME
|
32
|
+
from fractal_server.app.runner.filenames import METADATA_FILENAME
|
33
|
+
from fractal_server.app.runner.task_files import get_task_file_paths
|
38
34
|
|
39
35
|
|
40
36
|
def no_op_submit_setup_call(
|
@@ -42,7 +38,6 @@ def no_op_submit_setup_call(
|
|
42
38
|
wftask: WorkflowTask,
|
43
39
|
workflow_dir: Path,
|
44
40
|
workflow_dir_user: Path,
|
45
|
-
task_pars: TaskParameters,
|
46
41
|
) -> dict:
|
47
42
|
"""
|
48
43
|
Default (no-operation) interface of submit_setup_call.
|
@@ -50,14 +45,6 @@ def no_op_submit_setup_call(
|
|
50
45
|
return {}
|
51
46
|
|
52
47
|
|
53
|
-
def sanitize_component(value: str) -> str:
|
54
|
-
"""
|
55
|
-
Remove {" ", "/", "."} form a string, e.g. going from
|
56
|
-
'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
|
57
|
-
"""
|
58
|
-
return value.replace(" ", "_").replace("/", "_").replace(".", "_")
|
59
|
-
|
60
|
-
|
61
48
|
def _task_needs_image_list(_task: Task) -> bool:
|
62
49
|
"""
|
63
50
|
Whether a task requires `metadata["image"]` in its `args.json` file.
|
@@ -78,98 +65,6 @@ def _task_needs_image_list(_task: Task) -> bool:
|
|
78
65
|
return False
|
79
66
|
|
80
67
|
|
81
|
-
class TaskFiles:
|
82
|
-
"""
|
83
|
-
Group all file paths pertaining to a task
|
84
|
-
|
85
|
-
Attributes:
|
86
|
-
workflow_dir:
|
87
|
-
Server-owned directory to store all task-execution-related relevant
|
88
|
-
files (inputs, outputs, errors, and all meta files related to the
|
89
|
-
job execution). Note: users cannot write directly to this folder.
|
90
|
-
workflow_dir_user:
|
91
|
-
User-side directory with the same scope as `workflow_dir`, and
|
92
|
-
where a user can write.
|
93
|
-
task_order:
|
94
|
-
Positional order of the task within a workflow.
|
95
|
-
component:
|
96
|
-
Specific component to run the task for (relevant for tasks that
|
97
|
-
will be executed in parallel over many components).
|
98
|
-
file_prefix:
|
99
|
-
Prefix for all task-related files.
|
100
|
-
args:
|
101
|
-
Path for input json file.
|
102
|
-
metadiff:
|
103
|
-
Path for output json file with metadata update.
|
104
|
-
out:
|
105
|
-
Path for task-execution stdout.
|
106
|
-
err:
|
107
|
-
Path for task-execution stderr.
|
108
|
-
"""
|
109
|
-
|
110
|
-
workflow_dir: Path
|
111
|
-
workflow_dir_user: Path
|
112
|
-
task_order: Optional[int] = None
|
113
|
-
component: Optional[str] = None
|
114
|
-
|
115
|
-
file_prefix: str
|
116
|
-
args: Path
|
117
|
-
out: Path
|
118
|
-
err: Path
|
119
|
-
metadiff: Path
|
120
|
-
|
121
|
-
def __init__(
|
122
|
-
self,
|
123
|
-
workflow_dir: Path,
|
124
|
-
workflow_dir_user: Path,
|
125
|
-
task_order: Optional[int] = None,
|
126
|
-
component: Optional[str] = None,
|
127
|
-
):
|
128
|
-
self.workflow_dir = workflow_dir
|
129
|
-
self.workflow_dir_user = workflow_dir_user
|
130
|
-
self.task_order = task_order
|
131
|
-
self.component = component
|
132
|
-
|
133
|
-
if self.component is not None:
|
134
|
-
component_safe = sanitize_component(str(self.component))
|
135
|
-
component_safe = f"_par_{component_safe}"
|
136
|
-
else:
|
137
|
-
component_safe = ""
|
138
|
-
|
139
|
-
if self.task_order is not None:
|
140
|
-
order = str(self.task_order)
|
141
|
-
else:
|
142
|
-
order = "task"
|
143
|
-
self.file_prefix = f"{order}{component_safe}"
|
144
|
-
self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
|
145
|
-
self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
|
146
|
-
self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
|
147
|
-
self.metadiff = (
|
148
|
-
self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
|
149
|
-
)
|
150
|
-
|
151
|
-
|
152
|
-
@lru_cache()
|
153
|
-
def get_task_file_paths(
|
154
|
-
workflow_dir: Path,
|
155
|
-
workflow_dir_user: Path,
|
156
|
-
task_order: Optional[int] = None,
|
157
|
-
component: Optional[str] = None,
|
158
|
-
) -> TaskFiles:
|
159
|
-
"""
|
160
|
-
Return the corrisponding TaskFiles object
|
161
|
-
|
162
|
-
This function is mainly used as a cache to avoid instantiating needless
|
163
|
-
objects.
|
164
|
-
"""
|
165
|
-
return TaskFiles(
|
166
|
-
workflow_dir=workflow_dir,
|
167
|
-
workflow_dir_user=workflow_dir_user,
|
168
|
-
task_order=task_order,
|
169
|
-
component=component,
|
170
|
-
)
|
171
|
-
|
172
|
-
|
173
68
|
def _call_command_wrapper(cmd: str, stdout: Path, stderr: Path) -> None:
|
174
69
|
"""
|
175
70
|
Call a command and write its stdout and stderr to files
|
@@ -331,7 +226,7 @@ def call_single_task(
|
|
331
226
|
wftask_dump["task"] = wftask.task.model_dump()
|
332
227
|
new_history_item = dict(
|
333
228
|
workflowtask=wftask_dump,
|
334
|
-
status=
|
229
|
+
status=WorkflowTaskStatusTypeV1.DONE,
|
335
230
|
parallelization=None,
|
336
231
|
)
|
337
232
|
updated_history = task_pars.history.copy()
|
@@ -529,7 +424,6 @@ def call_parallel_task(
|
|
529
424
|
try:
|
530
425
|
extra_setup = submit_setup_call(
|
531
426
|
wftask=wftask,
|
532
|
-
task_pars=task_pars_depend,
|
533
427
|
workflow_dir=workflow_dir,
|
534
428
|
workflow_dir_user=workflow_dir_user,
|
535
429
|
)
|
@@ -592,7 +486,7 @@ def call_parallel_task(
|
|
592
486
|
wftask_dump["task"] = wftask.task.model_dump()
|
593
487
|
new_history_item = dict(
|
594
488
|
workflowtask=wftask_dump,
|
595
|
-
status=
|
489
|
+
status=WorkflowTaskStatusTypeV1.DONE,
|
596
490
|
parallelization=dict(
|
597
491
|
parallelization_level=wftask.parallelization_level,
|
598
492
|
component_list=component_list,
|
@@ -681,7 +575,6 @@ def execute_tasks(
|
|
681
575
|
try:
|
682
576
|
extra_setup = submit_setup_call(
|
683
577
|
wftask=this_wftask,
|
684
|
-
task_pars=current_task_pars,
|
685
578
|
workflow_dir=workflow_dir,
|
686
579
|
workflow_dir_user=workflow_dir_user,
|
687
580
|
)
|
@@ -23,13 +23,13 @@ from pathlib import Path
|
|
23
23
|
from typing import Any
|
24
24
|
from typing import Optional
|
25
25
|
|
26
|
-
from
|
27
|
-
from
|
28
|
-
from
|
29
|
-
from
|
30
|
-
from ..
|
26
|
+
from ....models import Workflow # FIXME: this is v1 specific
|
27
|
+
from ...async_wrap import async_wrap
|
28
|
+
from ...executors.local.executor import FractalThreadPoolExecutor
|
29
|
+
from ...set_start_and_last_task_index import set_start_and_last_task_index
|
30
|
+
from .._common import execute_tasks # FIXME: this is v1 specific
|
31
|
+
from ..common import TaskParameters # FIXME: this is v1 specific
|
31
32
|
from ._submit_setup import _local_submit_setup
|
32
|
-
from .executor import FractalThreadPoolExecutor
|
33
33
|
|
34
34
|
|
35
35
|
def _process_workflow(
|
@@ -19,9 +19,9 @@ from pydantic import BaseModel
|
|
19
19
|
from pydantic import Extra
|
20
20
|
from pydantic.error_wrappers import ValidationError
|
21
21
|
|
22
|
-
from
|
23
|
-
from
|
24
|
-
from
|
22
|
+
from .....config import get_settings
|
23
|
+
from .....syringe import Inject
|
24
|
+
from ....models.v1 import WorkflowTask
|
25
25
|
|
26
26
|
|
27
27
|
class LocalBackendConfigError(ValueError):
|
@@ -63,15 +63,14 @@ def get_local_backend_config(
|
|
63
63
|
The sources for `parallel_tasks_per_job` attributes, starting from the
|
64
64
|
highest-priority one, are
|
65
65
|
|
66
|
-
1. Properties in `wftask.meta
|
67
|
-
`Workflow.insert_task`, also includes `wftask.task.meta`);
|
66
|
+
1. Properties in `wftask.meta`;
|
68
67
|
2. The general content of the local-backend configuration file;
|
69
68
|
3. The default value (`None`).
|
70
69
|
|
71
70
|
Arguments:
|
72
71
|
wftask:
|
73
|
-
WorkflowTask for which the backend configuration
|
74
|
-
prepared.
|
72
|
+
WorkflowTask (V1) for which the backend configuration should
|
73
|
+
be prepared.
|
75
74
|
config_path:
|
76
75
|
Path of local-backend configuration file; if `None`, use
|
77
76
|
`FRACTAL_LOCAL_CONFIG_FILE` variable from settings.
|
@@ -14,8 +14,7 @@ Submodule to define _local_submit_setup
|
|
14
14
|
from pathlib import Path
|
15
15
|
from typing import Optional
|
16
16
|
|
17
|
-
from
|
18
|
-
from ..common import TaskParameters
|
17
|
+
from ....models.v1 import WorkflowTask
|
19
18
|
from ._local_config import get_local_backend_config
|
20
19
|
|
21
20
|
|
@@ -24,7 +23,6 @@ def _local_submit_setup(
|
|
24
23
|
wftask: WorkflowTask,
|
25
24
|
workflow_dir: Optional[Path] = None,
|
26
25
|
workflow_dir_user: Optional[Path] = None,
|
27
|
-
task_pars: Optional[TaskParameters] = None,
|
28
26
|
) -> dict[str, object]:
|
29
27
|
"""
|
30
28
|
Collect WorfklowTask-specific configuration parameters from different
|
@@ -33,8 +31,6 @@ def _local_submit_setup(
|
|
33
31
|
Arguments:
|
34
32
|
wftask:
|
35
33
|
WorkflowTask for which the configuration is to be assembled
|
36
|
-
task_pars:
|
37
|
-
Not used in this function.
|
38
34
|
workflow_dir:
|
39
35
|
Not used in this function.
|
40
36
|
workflow_dir_user:
|