fractal-server 1.4.6__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/db/__init__.py +0 -1
- fractal_server/app/models/__init__.py +6 -8
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/v1/__init__.py +12 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{state.py → v1/state.py} +2 -2
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +22 -0
- fractal_server/app/models/v2/collection_state.py +21 -0
- fractal_server/app/models/v2/dataset.py +54 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +30 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +35 -0
- fractal_server/app/models/v2/workflowtask.py +49 -0
- fractal_server/app/routes/admin/__init__.py +0 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +309 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
- fractal_server/app/routes/api/v1/dataset.py +48 -41
- fractal_server/app/routes/api/v1/job.py +14 -14
- fractal_server/app/routes/api/v1/project.py +30 -27
- fractal_server/app/routes/api/v1/task.py +26 -16
- fractal_server/app/routes/api/v1/task_collection.py +28 -16
- fractal_server/app/routes/api/v1/workflow.py +28 -28
- fractal_server/app/routes/api/v1/workflowtask.py +11 -11
- fractal_server/app/routes/api/v2/__init__.py +34 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
- fractal_server/app/routes/api/v2/dataset.py +293 -0
- fractal_server/app/routes/api/v2/images.py +279 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +186 -0
- fractal_server/app/routes/api/v2/status.py +150 -0
- fractal_server/app/routes/api/v2/submit.py +210 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +239 -0
- fractal_server/app/routes/api/v2/task_legacy.py +59 -0
- fractal_server/app/routes/api/v2/workflow.py +380 -0
- fractal_server/app/routes/api/v2/workflowtask.py +265 -0
- fractal_server/app/routes/aux/_job.py +2 -2
- fractal_server/app/runner/__init__.py +0 -379
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/components.py +5 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/__init__.py +0 -0
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/executors/slurm/_check_jobs_status.py +72 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +3 -4
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +42 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +46 -27
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +103 -0
- fractal_server/app/runner/v1/__init__.py +366 -0
- fractal_server/app/runner/{_common.py → v1/_common.py} +56 -111
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +336 -0
- fractal_server/app/runner/v2/_local/__init__.py +162 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_local/executor.py +100 -0
- fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
- fractal_server/app/runner/v2/deduplicate_list.py +23 -0
- fractal_server/app/runner/v2/handle_failed_job.py +165 -0
- fractal_server/app/runner/v2/merge_outputs.py +38 -0
- fractal_server/app/runner/v2/runner.py +343 -0
- fractal_server/app/runner/v2/runner_functions.py +374 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
- fractal_server/app/runner/v2/task_interface.py +62 -0
- fractal_server/app/runner/v2/v1_compat.py +31 -0
- fractal_server/app/schemas/__init__.py +1 -42
- fractal_server/app/schemas/_validators.py +28 -5
- fractal_server/app/schemas/v1/__init__.py +36 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +37 -0
- fractal_server/app/schemas/v2/dataset.py +126 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +114 -0
- fractal_server/app/schemas/v2/manifest.py +159 -0
- fractal_server/app/schemas/v2/project.py +34 -0
- fractal_server/app/schemas/v2/status.py +16 -0
- fractal_server/app/schemas/v2/task.py +151 -0
- fractal_server/app/schemas/v2/task_collection.py +109 -0
- fractal_server/app/schemas/v2/workflow.py +79 -0
- fractal_server/app/schemas/v2/workflowtask.py +208 -0
- fractal_server/config.py +13 -10
- fractal_server/images/__init__.py +4 -0
- fractal_server/images/models.py +136 -0
- fractal_server/images/tools.py +84 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/env.py +0 -2
- fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/v1/__init__.py +0 -0
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/__init__.py +0 -0
- fractal_server/tasks/v2/background_operations.py +381 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- fractal_server/urls.py +13 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/METADATA +11 -12
- fractal_server-2.0.0.dist-info/RECORD +169 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/common.py +0 -307
- fractal_server/app/schemas/json_schemas/manifest.json +0 -81
- fractal_server-1.4.6.dist-info/RECORD +0 -97
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -22,10 +22,9 @@ from pydantic import Extra
|
|
22
22
|
from pydantic import Field
|
23
23
|
from pydantic.error_wrappers import ValidationError
|
24
24
|
|
25
|
-
from
|
26
|
-
from
|
27
|
-
from
|
28
|
-
from ...models import WorkflowTask
|
25
|
+
from .....config import get_settings
|
26
|
+
from .....logger import set_logger
|
27
|
+
from .....syringe import Inject
|
29
28
|
|
30
29
|
logger = set_logger(__name__)
|
31
30
|
|
@@ -459,151 +458,3 @@ def get_default_slurm_config():
|
|
459
458
|
target_num_jobs=2,
|
460
459
|
max_num_jobs=4,
|
461
460
|
)
|
462
|
-
|
463
|
-
|
464
|
-
def get_slurm_config(
|
465
|
-
wftask: WorkflowTask,
|
466
|
-
workflow_dir: Path,
|
467
|
-
workflow_dir_user: Path,
|
468
|
-
config_path: Optional[Path] = None,
|
469
|
-
) -> SlurmConfig:
|
470
|
-
"""
|
471
|
-
Prepare a `SlurmConfig` configuration object
|
472
|
-
|
473
|
-
The sources for `SlurmConfig` attributes, in increasing priority order, are
|
474
|
-
|
475
|
-
1. The general content of the Fractal SLURM configuration file.
|
476
|
-
2. The GPU-specific content of the Fractal SLURM configuration file, if
|
477
|
-
appropriate.
|
478
|
-
3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
|
479
|
-
`Workflow.insert_task`, also includes `wftask.task.meta`);
|
480
|
-
|
481
|
-
Note: `wftask.meta` may be `None`.
|
482
|
-
|
483
|
-
Arguments:
|
484
|
-
wftask:
|
485
|
-
WorkflowTask for which the SLURM configuration is is to be
|
486
|
-
prepared.
|
487
|
-
workflow_dir:
|
488
|
-
Server-owned directory to store all task-execution-related relevant
|
489
|
-
files (inputs, outputs, errors, and all meta files related to the
|
490
|
-
job execution). Note: users cannot write directly to this folder.
|
491
|
-
workflow_dir_user:
|
492
|
-
User-side directory with the same scope as `workflow_dir`, and
|
493
|
-
where a user can write.
|
494
|
-
config_path:
|
495
|
-
Path of aFractal SLURM configuration file; if `None`, use
|
496
|
-
`FRACTAL_SLURM_CONFIG_FILE` variable from settings.
|
497
|
-
|
498
|
-
Returns:
|
499
|
-
slurm_config:
|
500
|
-
The SlurmConfig object
|
501
|
-
"""
|
502
|
-
|
503
|
-
logger.debug(
|
504
|
-
"[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
|
505
|
-
)
|
506
|
-
|
507
|
-
# Incorporate slurm_env.default_slurm_config
|
508
|
-
slurm_env = load_slurm_config_file(config_path=config_path)
|
509
|
-
slurm_dict = slurm_env.default_slurm_config.dict(
|
510
|
-
exclude_unset=True, exclude={"mem"}
|
511
|
-
)
|
512
|
-
if slurm_env.default_slurm_config.mem:
|
513
|
-
slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
|
514
|
-
|
515
|
-
# Incorporate slurm_env.batching_config
|
516
|
-
for key, value in slurm_env.batching_config.dict().items():
|
517
|
-
slurm_dict[key] = value
|
518
|
-
|
519
|
-
# Incorporate slurm_env.user_local_exports
|
520
|
-
slurm_dict["user_local_exports"] = slurm_env.user_local_exports
|
521
|
-
|
522
|
-
logger.debug(
|
523
|
-
"[get_slurm_config] Fractal SLURM configuration file: "
|
524
|
-
f"{slurm_env.dict()=}"
|
525
|
-
)
|
526
|
-
|
527
|
-
# GPU-related options
|
528
|
-
# Notes about priority:
|
529
|
-
# 1. This block of definitions takes priority over other definitions from
|
530
|
-
# slurm_env which are not under the `needs_gpu` subgroup
|
531
|
-
# 2. This block of definitions has lower priority than whatever comes next
|
532
|
-
# (i.e. from WorkflowTask.meta).
|
533
|
-
if wftask.meta is not None:
|
534
|
-
needs_gpu = wftask.meta.get("needs_gpu", False)
|
535
|
-
else:
|
536
|
-
needs_gpu = False
|
537
|
-
logger.debug(f"[get_slurm_config] {needs_gpu=}")
|
538
|
-
if needs_gpu:
|
539
|
-
for key, value in slurm_env.gpu_slurm_config.dict(
|
540
|
-
exclude_unset=True, exclude={"mem"}
|
541
|
-
).items():
|
542
|
-
slurm_dict[key] = value
|
543
|
-
if slurm_env.gpu_slurm_config.mem:
|
544
|
-
slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
|
545
|
-
|
546
|
-
# Number of CPUs per task, for multithreading
|
547
|
-
if wftask.meta is not None and "cpus_per_task" in wftask.meta:
|
548
|
-
cpus_per_task = int(wftask.meta["cpus_per_task"])
|
549
|
-
slurm_dict["cpus_per_task"] = cpus_per_task
|
550
|
-
|
551
|
-
# Required memory per task, in MB
|
552
|
-
if wftask.meta is not None and "mem" in wftask.meta:
|
553
|
-
raw_mem = wftask.meta["mem"]
|
554
|
-
mem_per_task_MB = _parse_mem_value(raw_mem)
|
555
|
-
slurm_dict["mem_per_task_MB"] = mem_per_task_MB
|
556
|
-
|
557
|
-
# Job name
|
558
|
-
job_name = wftask.task.name.replace(" ", "_")
|
559
|
-
slurm_dict["job_name"] = job_name
|
560
|
-
|
561
|
-
# Optional SLURM arguments and extra lines
|
562
|
-
if wftask.meta is not None:
|
563
|
-
account = wftask.meta.get("account", None)
|
564
|
-
if account is not None:
|
565
|
-
error_msg = (
|
566
|
-
f"Invalid {account=} property in WorkflowTask `meta` "
|
567
|
-
"attribute.\n"
|
568
|
-
"SLURM account must be set in the request body of the "
|
569
|
-
"apply-workflow endpoint, or by modifying the user properties."
|
570
|
-
)
|
571
|
-
logger.error(error_msg)
|
572
|
-
raise SlurmConfigError(error_msg)
|
573
|
-
for key in ["time", "gres", "constraint"]:
|
574
|
-
value = wftask.meta.get(key, None)
|
575
|
-
if value:
|
576
|
-
slurm_dict[key] = value
|
577
|
-
if wftask.meta is not None:
|
578
|
-
extra_lines = wftask.meta.get("extra_lines", [])
|
579
|
-
else:
|
580
|
-
extra_lines = []
|
581
|
-
extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
|
582
|
-
if len(set(extra_lines)) != len(extra_lines):
|
583
|
-
logger.debug(
|
584
|
-
"[get_slurm_config] Removing repeated elements "
|
585
|
-
f"from {extra_lines=}."
|
586
|
-
)
|
587
|
-
extra_lines = list(set(extra_lines))
|
588
|
-
slurm_dict["extra_lines"] = extra_lines
|
589
|
-
|
590
|
-
# Job-batching parameters (if None, they will be determined heuristically)
|
591
|
-
if wftask.meta is not None:
|
592
|
-
tasks_per_job = wftask.meta.get("tasks_per_job", None)
|
593
|
-
parallel_tasks_per_job = wftask.meta.get(
|
594
|
-
"parallel_tasks_per_job", None
|
595
|
-
)
|
596
|
-
else:
|
597
|
-
tasks_per_job = None
|
598
|
-
parallel_tasks_per_job = None
|
599
|
-
slurm_dict["tasks_per_job"] = tasks_per_job
|
600
|
-
slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
|
601
|
-
|
602
|
-
# Put everything together
|
603
|
-
logger.debug(
|
604
|
-
"[get_slurm_config] Now create a SlurmConfig object based "
|
605
|
-
f"on {slurm_dict=}"
|
606
|
-
)
|
607
|
-
slurm_config = SlurmConfig(**slurm_dict)
|
608
|
-
|
609
|
-
return slurm_config
|
@@ -19,7 +19,7 @@ import shlex
|
|
19
19
|
import subprocess # nosec
|
20
20
|
from typing import Optional
|
21
21
|
|
22
|
-
from
|
22
|
+
from .....logger import set_logger
|
23
23
|
|
24
24
|
logger = set_logger(__name__)
|
25
25
|
|
@@ -113,6 +113,47 @@ def _glob_as_user(
|
|
113
113
|
return output
|
114
114
|
|
115
115
|
|
116
|
+
def _glob_as_user_strict(
|
117
|
+
*,
|
118
|
+
folder: str,
|
119
|
+
user: str,
|
120
|
+
startswith: str,
|
121
|
+
) -> list[str]:
|
122
|
+
"""
|
123
|
+
Run `ls` in a folder (as a user) and filter results
|
124
|
+
|
125
|
+
Execute `ls` on a folder (impersonating a user, if `user` is not `None`)
|
126
|
+
and select results that comply with a set of rules. They all start with
|
127
|
+
`startswith` (if not `None`), and they match one of the known filename
|
128
|
+
patterns. See details in
|
129
|
+
https://github.com/fractal-analytics-platform/fractal-server/issues/1240
|
130
|
+
|
131
|
+
|
132
|
+
Arguments:
|
133
|
+
folder: Absolute path to the folder
|
134
|
+
user: If not `None`, the user to be impersonated via `sudo -u`
|
135
|
+
startswith: If not `None`, this is used to filter output of `ls`.
|
136
|
+
"""
|
137
|
+
|
138
|
+
res = _run_command_as_user(cmd=f"ls {folder}", user=user, check=True)
|
139
|
+
output = res.stdout.split()
|
140
|
+
|
141
|
+
new_output = []
|
142
|
+
known_filenames = [
|
143
|
+
f"{startswith}{suffix}"
|
144
|
+
for suffix in [".args.json", ".metadiff.json", ".err", ".out"]
|
145
|
+
]
|
146
|
+
for filename in output:
|
147
|
+
if filename in known_filenames:
|
148
|
+
new_output.append(filename)
|
149
|
+
elif filename.startswith(f"{startswith}_out_") and filename.endswith(
|
150
|
+
".pickle"
|
151
|
+
):
|
152
|
+
new_output.append(filename)
|
153
|
+
|
154
|
+
return new_output
|
155
|
+
|
156
|
+
|
116
157
|
def _path_exists_as_user(*, path: str, user: Optional[str] = None) -> bool:
|
117
158
|
"""
|
118
159
|
Impersonate a user and check if `path` exists via `ls`
|
@@ -29,22 +29,24 @@ import cloudpickle
|
|
29
29
|
from cfut import SlurmExecutor
|
30
30
|
from cfut.util import random_string
|
31
31
|
|
32
|
-
from
|
33
|
-
from
|
34
|
-
from
|
35
|
-
from
|
36
|
-
from
|
37
|
-
from
|
38
|
-
from
|
39
|
-
from
|
32
|
+
from .....config import get_settings
|
33
|
+
from .....logger import set_logger
|
34
|
+
from .....syringe import Inject
|
35
|
+
from ...exceptions import JobExecutionError
|
36
|
+
from ...exceptions import TaskExecutionError
|
37
|
+
from ...filenames import SHUTDOWN_FILENAME
|
38
|
+
from ...task_files import get_task_file_paths
|
39
|
+
from ...task_files import TaskFiles
|
40
40
|
from ._batching import heuristics
|
41
41
|
from ._executor_wait_thread import FractalSlurmWaitThread
|
42
42
|
from ._slurm_config import get_default_slurm_config
|
43
43
|
from ._slurm_config import SlurmConfig
|
44
44
|
from ._subprocess_run_as_user import _glob_as_user
|
45
|
+
from ._subprocess_run_as_user import _glob_as_user_strict
|
45
46
|
from ._subprocess_run_as_user import _path_exists_as_user
|
46
47
|
from ._subprocess_run_as_user import _run_command_as_user
|
47
48
|
from fractal_server import __VERSION__
|
49
|
+
from fractal_server.app.runner.components import _COMPONENT_KEY_
|
48
50
|
|
49
51
|
|
50
52
|
logger = set_logger(__name__)
|
@@ -415,8 +417,6 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
415
417
|
A `TaskFiles` object; if `None`, use
|
416
418
|
`self.get_default_task_files()`.
|
417
419
|
|
418
|
-
Returns:
|
419
|
-
An iterator of results.
|
420
420
|
"""
|
421
421
|
|
422
422
|
def _result_or_cancel(fut):
|
@@ -543,7 +543,7 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
543
543
|
single_task_submission: bool = False,
|
544
544
|
args: Optional[Sequence[Any]] = None,
|
545
545
|
kwargs: Optional[dict] = None,
|
546
|
-
components: list[Any] = None,
|
546
|
+
components: Optional[list[Any]] = None,
|
547
547
|
) -> Future:
|
548
548
|
"""
|
549
549
|
Submit a multi-task job to the pool, where each task is handled via the
|
@@ -579,6 +579,10 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
579
579
|
|
580
580
|
# Define slurm-job-related files
|
581
581
|
if single_task_submission:
|
582
|
+
if components is not None:
|
583
|
+
raise ValueError(
|
584
|
+
f"{single_task_submission=} but components is not None"
|
585
|
+
)
|
582
586
|
job = SlurmJob(
|
583
587
|
slurm_file_prefix=slurm_file_prefix,
|
584
588
|
num_tasks_tot=1,
|
@@ -602,15 +606,23 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
602
606
|
num_tasks_tot=num_tasks_tot,
|
603
607
|
slurm_config=slurm_config,
|
604
608
|
)
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
609
|
+
|
610
|
+
_prefixes = []
|
611
|
+
for component in components:
|
612
|
+
if isinstance(component, dict):
|
613
|
+
# This is needed for V2
|
614
|
+
actual_component = component.get(_COMPONENT_KEY_, None)
|
615
|
+
else:
|
616
|
+
actual_component = component
|
617
|
+
_prefixes.append(
|
618
|
+
get_task_file_paths(
|
619
|
+
workflow_dir=task_files.workflow_dir,
|
620
|
+
workflow_dir_user=task_files.workflow_dir_user,
|
621
|
+
task_order=task_files.task_order,
|
622
|
+
component=actual_component,
|
623
|
+
).file_prefix
|
624
|
+
)
|
625
|
+
job.wftask_file_prefixes = tuple(_prefixes)
|
614
626
|
|
615
627
|
# Define I/O pickle file names/paths
|
616
628
|
job.input_pickle_files = tuple(
|
@@ -930,11 +942,19 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
930
942
|
|
931
943
|
for prefix in prefixes:
|
932
944
|
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
|
945
|
+
if prefix == job.slurm_file_prefix:
|
946
|
+
files_to_copy = _glob_as_user(
|
947
|
+
folder=str(self.working_dir_user),
|
948
|
+
user=self.slurm_user,
|
949
|
+
startswith=prefix,
|
950
|
+
)
|
951
|
+
else:
|
952
|
+
files_to_copy = _glob_as_user_strict(
|
953
|
+
folder=str(self.working_dir_user),
|
954
|
+
user=self.slurm_user,
|
955
|
+
startswith=prefix,
|
956
|
+
)
|
957
|
+
|
938
958
|
logger.debug(
|
939
959
|
"[_copy_files_from_user_to_server] "
|
940
960
|
f"{prefix=}, {len(files_to_copy)=}"
|
@@ -949,7 +969,6 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
949
969
|
source_file_path = str(
|
950
970
|
self.working_dir_user / source_file_name
|
951
971
|
)
|
952
|
-
dest_file_path = str(self.working_dir / source_file_name)
|
953
972
|
|
954
973
|
# Read source_file_path (requires sudo)
|
955
974
|
# NOTE: By setting encoding=None, we read/write bytes instead
|
@@ -993,7 +1012,7 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
993
1012
|
cmdlines.append(
|
994
1013
|
(
|
995
1014
|
f"{python_worker_interpreter}"
|
996
|
-
" -m fractal_server.app.runner.
|
1015
|
+
" -m fractal_server.app.runner.executors.slurm.remote "
|
997
1016
|
f"--input-file {input_pickle_file} "
|
998
1017
|
f"--output-file {output_pickle_file}"
|
999
1018
|
)
|
@@ -0,0 +1,39 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
|
4
|
+
def set_start_and_last_task_index(
|
5
|
+
num_tasks: int,
|
6
|
+
first_task_index: Optional[int] = None,
|
7
|
+
last_task_index: Optional[int] = None,
|
8
|
+
) -> tuple[int, int]:
|
9
|
+
"""
|
10
|
+
Handle `first_task_index` and `last_task_index`, by setting defaults and
|
11
|
+
validating values.
|
12
|
+
|
13
|
+
num_tasks:
|
14
|
+
Total number of tasks in a workflow task list
|
15
|
+
first_task_index:
|
16
|
+
Positional index of the first task to execute
|
17
|
+
last_task_index:
|
18
|
+
Positional index of the last task to execute
|
19
|
+
"""
|
20
|
+
# Set default values
|
21
|
+
if first_task_index is None:
|
22
|
+
first_task_index = 0
|
23
|
+
if last_task_index is None:
|
24
|
+
last_task_index = num_tasks - 1
|
25
|
+
|
26
|
+
# Perform checks
|
27
|
+
if first_task_index < 0:
|
28
|
+
raise ValueError(f"{first_task_index=} cannot be negative")
|
29
|
+
if last_task_index < 0:
|
30
|
+
raise ValueError(f"{last_task_index=} cannot be negative")
|
31
|
+
if last_task_index > num_tasks - 1:
|
32
|
+
raise ValueError(
|
33
|
+
f"{last_task_index=} cannot be larger than {(num_tasks-1)=}"
|
34
|
+
)
|
35
|
+
if first_task_index > last_task_index:
|
36
|
+
raise ValueError(
|
37
|
+
f"{first_task_index=} cannot be larger than {last_task_index=}"
|
38
|
+
)
|
39
|
+
return (first_task_index, last_task_index)
|
@@ -0,0 +1,103 @@
|
|
1
|
+
from pathlib import Path
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
|
5
|
+
def sanitize_component(value: str) -> str:
|
6
|
+
"""
|
7
|
+
Remove {" ", "/", "."} form a string, e.g. going from
|
8
|
+
'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
|
9
|
+
"""
|
10
|
+
return value.replace(" ", "_").replace("/", "_").replace(".", "_")
|
11
|
+
|
12
|
+
|
13
|
+
class TaskFiles:
|
14
|
+
"""
|
15
|
+
Group all file paths pertaining to a task
|
16
|
+
|
17
|
+
Attributes:
|
18
|
+
workflow_dir:
|
19
|
+
Server-owned directory to store all task-execution-related relevant
|
20
|
+
files (inputs, outputs, errors, and all meta files related to the
|
21
|
+
job execution). Note: users cannot write directly to this folder.
|
22
|
+
workflow_dir_user:
|
23
|
+
User-side directory with the same scope as `workflow_dir`, and
|
24
|
+
where a user can write.
|
25
|
+
task_order:
|
26
|
+
Positional order of the task within a workflow.
|
27
|
+
component:
|
28
|
+
Specific component to run the task for (relevant for tasks that
|
29
|
+
will be executed in parallel over many components).
|
30
|
+
file_prefix:
|
31
|
+
Prefix for all task-related files.
|
32
|
+
args:
|
33
|
+
Path for input json file.
|
34
|
+
metadiff:
|
35
|
+
Path for output json file with metadata update.
|
36
|
+
out:
|
37
|
+
Path for task-execution stdout.
|
38
|
+
err:
|
39
|
+
Path for task-execution stderr.
|
40
|
+
"""
|
41
|
+
|
42
|
+
workflow_dir: Path
|
43
|
+
workflow_dir_user: Path
|
44
|
+
task_order: Optional[int] = None
|
45
|
+
component: Optional[str] = None
|
46
|
+
|
47
|
+
file_prefix: str
|
48
|
+
args: Path
|
49
|
+
out: Path
|
50
|
+
err: Path
|
51
|
+
log: Path
|
52
|
+
metadiff: Path
|
53
|
+
|
54
|
+
def __init__(
|
55
|
+
self,
|
56
|
+
workflow_dir: Path,
|
57
|
+
workflow_dir_user: Path,
|
58
|
+
task_order: Optional[int] = None,
|
59
|
+
component: Optional[str] = None,
|
60
|
+
):
|
61
|
+
self.workflow_dir = workflow_dir
|
62
|
+
self.workflow_dir_user = workflow_dir_user
|
63
|
+
self.task_order = task_order
|
64
|
+
self.component = component
|
65
|
+
|
66
|
+
if self.component is not None:
|
67
|
+
component_safe = sanitize_component(str(self.component))
|
68
|
+
component_safe = f"_par_{component_safe}"
|
69
|
+
else:
|
70
|
+
component_safe = ""
|
71
|
+
|
72
|
+
if self.task_order is not None:
|
73
|
+
order = str(self.task_order)
|
74
|
+
else:
|
75
|
+
order = "task"
|
76
|
+
self.file_prefix = f"{order}{component_safe}"
|
77
|
+
self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
|
78
|
+
self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
|
79
|
+
self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
|
80
|
+
self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
|
81
|
+
self.metadiff = (
|
82
|
+
self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
|
83
|
+
)
|
84
|
+
|
85
|
+
|
86
|
+
def get_task_file_paths(
|
87
|
+
workflow_dir: Path,
|
88
|
+
workflow_dir_user: Path,
|
89
|
+
task_order: Optional[int] = None,
|
90
|
+
component: Optional[str] = None,
|
91
|
+
) -> TaskFiles:
|
92
|
+
"""
|
93
|
+
Return the corrisponding TaskFiles object
|
94
|
+
|
95
|
+
This function is mainly used as a cache to avoid instantiating needless
|
96
|
+
objects.
|
97
|
+
"""
|
98
|
+
return TaskFiles(
|
99
|
+
workflow_dir=workflow_dir,
|
100
|
+
workflow_dir_user=workflow_dir_user,
|
101
|
+
task_order=task_order,
|
102
|
+
component=component,
|
103
|
+
)
|