fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/models/__init__.py +6 -8
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/v1/__init__.py +12 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{state.py → v1/state.py} +2 -2
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +22 -0
- fractal_server/app/models/v2/collection_state.py +21 -0
- fractal_server/app/models/v2/dataset.py +54 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +30 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +35 -0
- fractal_server/app/models/v2/workflowtask.py +49 -0
- fractal_server/app/routes/admin/__init__.py +0 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +309 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
- fractal_server/app/routes/api/v1/dataset.py +41 -41
- fractal_server/app/routes/api/v1/job.py +14 -14
- fractal_server/app/routes/api/v1/project.py +27 -25
- fractal_server/app/routes/api/v1/task.py +26 -16
- fractal_server/app/routes/api/v1/task_collection.py +28 -16
- fractal_server/app/routes/api/v1/workflow.py +28 -28
- fractal_server/app/routes/api/v1/workflowtask.py +11 -11
- fractal_server/app/routes/api/v2/__init__.py +34 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
- fractal_server/app/routes/api/v2/dataset.py +293 -0
- fractal_server/app/routes/api/v2/images.py +279 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +186 -0
- fractal_server/app/routes/api/v2/status.py +150 -0
- fractal_server/app/routes/api/v2/submit.py +210 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +239 -0
- fractal_server/app/routes/api/v2/task_legacy.py +59 -0
- fractal_server/app/routes/api/v2/workflow.py +380 -0
- fractal_server/app/routes/api/v2/workflowtask.py +265 -0
- fractal_server/app/routes/aux/_job.py +2 -2
- fractal_server/app/runner/__init__.py +0 -364
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/components.py +5 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/__init__.py +0 -0
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +32 -21
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +103 -0
- fractal_server/app/runner/v1/__init__.py +366 -0
- fractal_server/app/runner/{_common.py → v1/_common.py} +14 -121
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +336 -0
- fractal_server/app/runner/v2/_local/__init__.py +162 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_local/executor.py +100 -0
- fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
- fractal_server/app/runner/v2/deduplicate_list.py +23 -0
- fractal_server/app/runner/v2/handle_failed_job.py +165 -0
- fractal_server/app/runner/v2/merge_outputs.py +38 -0
- fractal_server/app/runner/v2/runner.py +343 -0
- fractal_server/app/runner/v2/runner_functions.py +374 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
- fractal_server/app/runner/v2/task_interface.py +62 -0
- fractal_server/app/runner/v2/v1_compat.py +31 -0
- fractal_server/app/schemas/__init__.py +1 -42
- fractal_server/app/schemas/_validators.py +28 -5
- fractal_server/app/schemas/v1/__init__.py +36 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +37 -0
- fractal_server/app/schemas/v2/dataset.py +126 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +114 -0
- fractal_server/app/schemas/v2/manifest.py +159 -0
- fractal_server/app/schemas/v2/project.py +34 -0
- fractal_server/app/schemas/v2/status.py +16 -0
- fractal_server/app/schemas/v2/task.py +151 -0
- fractal_server/app/schemas/v2/task_collection.py +109 -0
- fractal_server/app/schemas/v2/workflow.py +79 -0
- fractal_server/app/schemas/v2/workflowtask.py +208 -0
- fractal_server/config.py +5 -4
- fractal_server/images/__init__.py +4 -0
- fractal_server/images/models.py +136 -0
- fractal_server/images/tools.py +84 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/env.py +0 -2
- fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/v1/__init__.py +0 -0
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/__init__.py +0 -0
- fractal_server/tasks/v2/background_operations.py +381 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- fractal_server/urls.py +13 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/METADATA +10 -10
- fractal_server-2.0.0.dist-info/RECORD +169 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/common.py +0 -311
- fractal_server/app/schemas/json_schemas/manifest.json +0 -81
- fractal_server-1.4.10.dist-info/RECORD +0 -98
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -22,10 +22,9 @@ from pydantic import Extra
|
|
22
22
|
from pydantic import Field
|
23
23
|
from pydantic.error_wrappers import ValidationError
|
24
24
|
|
25
|
-
from
|
26
|
-
from
|
27
|
-
from
|
28
|
-
from ...models import WorkflowTask
|
25
|
+
from .....config import get_settings
|
26
|
+
from .....logger import set_logger
|
27
|
+
from .....syringe import Inject
|
29
28
|
|
30
29
|
logger = set_logger(__name__)
|
31
30
|
|
@@ -459,151 +458,3 @@ def get_default_slurm_config():
|
|
459
458
|
target_num_jobs=2,
|
460
459
|
max_num_jobs=4,
|
461
460
|
)
|
462
|
-
|
463
|
-
|
464
|
-
def get_slurm_config(
|
465
|
-
wftask: WorkflowTask,
|
466
|
-
workflow_dir: Path,
|
467
|
-
workflow_dir_user: Path,
|
468
|
-
config_path: Optional[Path] = None,
|
469
|
-
) -> SlurmConfig:
|
470
|
-
"""
|
471
|
-
Prepare a `SlurmConfig` configuration object
|
472
|
-
|
473
|
-
The sources for `SlurmConfig` attributes, in increasing priority order, are
|
474
|
-
|
475
|
-
1. The general content of the Fractal SLURM configuration file.
|
476
|
-
2. The GPU-specific content of the Fractal SLURM configuration file, if
|
477
|
-
appropriate.
|
478
|
-
3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
|
479
|
-
`Workflow.insert_task`, also includes `wftask.task.meta`);
|
480
|
-
|
481
|
-
Note: `wftask.meta` may be `None`.
|
482
|
-
|
483
|
-
Arguments:
|
484
|
-
wftask:
|
485
|
-
WorkflowTask for which the SLURM configuration is is to be
|
486
|
-
prepared.
|
487
|
-
workflow_dir:
|
488
|
-
Server-owned directory to store all task-execution-related relevant
|
489
|
-
files (inputs, outputs, errors, and all meta files related to the
|
490
|
-
job execution). Note: users cannot write directly to this folder.
|
491
|
-
workflow_dir_user:
|
492
|
-
User-side directory with the same scope as `workflow_dir`, and
|
493
|
-
where a user can write.
|
494
|
-
config_path:
|
495
|
-
Path of aFractal SLURM configuration file; if `None`, use
|
496
|
-
`FRACTAL_SLURM_CONFIG_FILE` variable from settings.
|
497
|
-
|
498
|
-
Returns:
|
499
|
-
slurm_config:
|
500
|
-
The SlurmConfig object
|
501
|
-
"""
|
502
|
-
|
503
|
-
logger.debug(
|
504
|
-
"[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
|
505
|
-
)
|
506
|
-
|
507
|
-
# Incorporate slurm_env.default_slurm_config
|
508
|
-
slurm_env = load_slurm_config_file(config_path=config_path)
|
509
|
-
slurm_dict = slurm_env.default_slurm_config.dict(
|
510
|
-
exclude_unset=True, exclude={"mem"}
|
511
|
-
)
|
512
|
-
if slurm_env.default_slurm_config.mem:
|
513
|
-
slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
|
514
|
-
|
515
|
-
# Incorporate slurm_env.batching_config
|
516
|
-
for key, value in slurm_env.batching_config.dict().items():
|
517
|
-
slurm_dict[key] = value
|
518
|
-
|
519
|
-
# Incorporate slurm_env.user_local_exports
|
520
|
-
slurm_dict["user_local_exports"] = slurm_env.user_local_exports
|
521
|
-
|
522
|
-
logger.debug(
|
523
|
-
"[get_slurm_config] Fractal SLURM configuration file: "
|
524
|
-
f"{slurm_env.dict()=}"
|
525
|
-
)
|
526
|
-
|
527
|
-
# GPU-related options
|
528
|
-
# Notes about priority:
|
529
|
-
# 1. This block of definitions takes priority over other definitions from
|
530
|
-
# slurm_env which are not under the `needs_gpu` subgroup
|
531
|
-
# 2. This block of definitions has lower priority than whatever comes next
|
532
|
-
# (i.e. from WorkflowTask.meta).
|
533
|
-
if wftask.meta is not None:
|
534
|
-
needs_gpu = wftask.meta.get("needs_gpu", False)
|
535
|
-
else:
|
536
|
-
needs_gpu = False
|
537
|
-
logger.debug(f"[get_slurm_config] {needs_gpu=}")
|
538
|
-
if needs_gpu:
|
539
|
-
for key, value in slurm_env.gpu_slurm_config.dict(
|
540
|
-
exclude_unset=True, exclude={"mem"}
|
541
|
-
).items():
|
542
|
-
slurm_dict[key] = value
|
543
|
-
if slurm_env.gpu_slurm_config.mem:
|
544
|
-
slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
|
545
|
-
|
546
|
-
# Number of CPUs per task, for multithreading
|
547
|
-
if wftask.meta is not None and "cpus_per_task" in wftask.meta:
|
548
|
-
cpus_per_task = int(wftask.meta["cpus_per_task"])
|
549
|
-
slurm_dict["cpus_per_task"] = cpus_per_task
|
550
|
-
|
551
|
-
# Required memory per task, in MB
|
552
|
-
if wftask.meta is not None and "mem" in wftask.meta:
|
553
|
-
raw_mem = wftask.meta["mem"]
|
554
|
-
mem_per_task_MB = _parse_mem_value(raw_mem)
|
555
|
-
slurm_dict["mem_per_task_MB"] = mem_per_task_MB
|
556
|
-
|
557
|
-
# Job name
|
558
|
-
job_name = wftask.task.name.replace(" ", "_")
|
559
|
-
slurm_dict["job_name"] = job_name
|
560
|
-
|
561
|
-
# Optional SLURM arguments and extra lines
|
562
|
-
if wftask.meta is not None:
|
563
|
-
account = wftask.meta.get("account", None)
|
564
|
-
if account is not None:
|
565
|
-
error_msg = (
|
566
|
-
f"Invalid {account=} property in WorkflowTask `meta` "
|
567
|
-
"attribute.\n"
|
568
|
-
"SLURM account must be set in the request body of the "
|
569
|
-
"apply-workflow endpoint, or by modifying the user properties."
|
570
|
-
)
|
571
|
-
logger.error(error_msg)
|
572
|
-
raise SlurmConfigError(error_msg)
|
573
|
-
for key in ["time", "gres", "constraint"]:
|
574
|
-
value = wftask.meta.get(key, None)
|
575
|
-
if value:
|
576
|
-
slurm_dict[key] = value
|
577
|
-
if wftask.meta is not None:
|
578
|
-
extra_lines = wftask.meta.get("extra_lines", [])
|
579
|
-
else:
|
580
|
-
extra_lines = []
|
581
|
-
extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
|
582
|
-
if len(set(extra_lines)) != len(extra_lines):
|
583
|
-
logger.debug(
|
584
|
-
"[get_slurm_config] Removing repeated elements "
|
585
|
-
f"from {extra_lines=}."
|
586
|
-
)
|
587
|
-
extra_lines = list(set(extra_lines))
|
588
|
-
slurm_dict["extra_lines"] = extra_lines
|
589
|
-
|
590
|
-
# Job-batching parameters (if None, they will be determined heuristically)
|
591
|
-
if wftask.meta is not None:
|
592
|
-
tasks_per_job = wftask.meta.get("tasks_per_job", None)
|
593
|
-
parallel_tasks_per_job = wftask.meta.get(
|
594
|
-
"parallel_tasks_per_job", None
|
595
|
-
)
|
596
|
-
else:
|
597
|
-
tasks_per_job = None
|
598
|
-
parallel_tasks_per_job = None
|
599
|
-
slurm_dict["tasks_per_job"] = tasks_per_job
|
600
|
-
slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
|
601
|
-
|
602
|
-
# Put everything together
|
603
|
-
logger.debug(
|
604
|
-
"[get_slurm_config] Now create a SlurmConfig object based "
|
605
|
-
f"on {slurm_dict=}"
|
606
|
-
)
|
607
|
-
slurm_config = SlurmConfig(**slurm_dict)
|
608
|
-
|
609
|
-
return slurm_config
|
@@ -29,14 +29,14 @@ import cloudpickle
|
|
29
29
|
from cfut import SlurmExecutor
|
30
30
|
from cfut.util import random_string
|
31
31
|
|
32
|
-
from
|
33
|
-
from
|
34
|
-
from
|
35
|
-
from
|
36
|
-
from
|
37
|
-
from
|
38
|
-
from
|
39
|
-
from
|
32
|
+
from .....config import get_settings
|
33
|
+
from .....logger import set_logger
|
34
|
+
from .....syringe import Inject
|
35
|
+
from ...exceptions import JobExecutionError
|
36
|
+
from ...exceptions import TaskExecutionError
|
37
|
+
from ...filenames import SHUTDOWN_FILENAME
|
38
|
+
from ...task_files import get_task_file_paths
|
39
|
+
from ...task_files import TaskFiles
|
40
40
|
from ._batching import heuristics
|
41
41
|
from ._executor_wait_thread import FractalSlurmWaitThread
|
42
42
|
from ._slurm_config import get_default_slurm_config
|
@@ -46,6 +46,7 @@ from ._subprocess_run_as_user import _glob_as_user_strict
|
|
46
46
|
from ._subprocess_run_as_user import _path_exists_as_user
|
47
47
|
from ._subprocess_run_as_user import _run_command_as_user
|
48
48
|
from fractal_server import __VERSION__
|
49
|
+
from fractal_server.app.runner.components import _COMPONENT_KEY_
|
49
50
|
|
50
51
|
|
51
52
|
logger = set_logger(__name__)
|
@@ -416,8 +417,6 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
416
417
|
A `TaskFiles` object; if `None`, use
|
417
418
|
`self.get_default_task_files()`.
|
418
419
|
|
419
|
-
Returns:
|
420
|
-
An iterator of results.
|
421
420
|
"""
|
422
421
|
|
423
422
|
def _result_or_cancel(fut):
|
@@ -544,7 +543,7 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
544
543
|
single_task_submission: bool = False,
|
545
544
|
args: Optional[Sequence[Any]] = None,
|
546
545
|
kwargs: Optional[dict] = None,
|
547
|
-
components: list[Any] = None,
|
546
|
+
components: Optional[list[Any]] = None,
|
548
547
|
) -> Future:
|
549
548
|
"""
|
550
549
|
Submit a multi-task job to the pool, where each task is handled via the
|
@@ -580,6 +579,10 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
580
579
|
|
581
580
|
# Define slurm-job-related files
|
582
581
|
if single_task_submission:
|
582
|
+
if components is not None:
|
583
|
+
raise ValueError(
|
584
|
+
f"{single_task_submission=} but components is not None"
|
585
|
+
)
|
583
586
|
job = SlurmJob(
|
584
587
|
slurm_file_prefix=slurm_file_prefix,
|
585
588
|
num_tasks_tot=1,
|
@@ -603,15 +606,23 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
603
606
|
num_tasks_tot=num_tasks_tot,
|
604
607
|
slurm_config=slurm_config,
|
605
608
|
)
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
609
|
+
|
610
|
+
_prefixes = []
|
611
|
+
for component in components:
|
612
|
+
if isinstance(component, dict):
|
613
|
+
# This is needed for V2
|
614
|
+
actual_component = component.get(_COMPONENT_KEY_, None)
|
615
|
+
else:
|
616
|
+
actual_component = component
|
617
|
+
_prefixes.append(
|
618
|
+
get_task_file_paths(
|
619
|
+
workflow_dir=task_files.workflow_dir,
|
620
|
+
workflow_dir_user=task_files.workflow_dir_user,
|
621
|
+
task_order=task_files.task_order,
|
622
|
+
component=actual_component,
|
623
|
+
).file_prefix
|
624
|
+
)
|
625
|
+
job.wftask_file_prefixes = tuple(_prefixes)
|
615
626
|
|
616
627
|
# Define I/O pickle file names/paths
|
617
628
|
job.input_pickle_files = tuple(
|
@@ -1001,7 +1012,7 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
1001
1012
|
cmdlines.append(
|
1002
1013
|
(
|
1003
1014
|
f"{python_worker_interpreter}"
|
1004
|
-
" -m fractal_server.app.runner.
|
1015
|
+
" -m fractal_server.app.runner.executors.slurm.remote "
|
1005
1016
|
f"--input-file {input_pickle_file} "
|
1006
1017
|
f"--output-file {output_pickle_file}"
|
1007
1018
|
)
|
@@ -0,0 +1,39 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
|
4
|
+
def set_start_and_last_task_index(
|
5
|
+
num_tasks: int,
|
6
|
+
first_task_index: Optional[int] = None,
|
7
|
+
last_task_index: Optional[int] = None,
|
8
|
+
) -> tuple[int, int]:
|
9
|
+
"""
|
10
|
+
Handle `first_task_index` and `last_task_index`, by setting defaults and
|
11
|
+
validating values.
|
12
|
+
|
13
|
+
num_tasks:
|
14
|
+
Total number of tasks in a workflow task list
|
15
|
+
first_task_index:
|
16
|
+
Positional index of the first task to execute
|
17
|
+
last_task_index:
|
18
|
+
Positional index of the last task to execute
|
19
|
+
"""
|
20
|
+
# Set default values
|
21
|
+
if first_task_index is None:
|
22
|
+
first_task_index = 0
|
23
|
+
if last_task_index is None:
|
24
|
+
last_task_index = num_tasks - 1
|
25
|
+
|
26
|
+
# Perform checks
|
27
|
+
if first_task_index < 0:
|
28
|
+
raise ValueError(f"{first_task_index=} cannot be negative")
|
29
|
+
if last_task_index < 0:
|
30
|
+
raise ValueError(f"{last_task_index=} cannot be negative")
|
31
|
+
if last_task_index > num_tasks - 1:
|
32
|
+
raise ValueError(
|
33
|
+
f"{last_task_index=} cannot be larger than {(num_tasks-1)=}"
|
34
|
+
)
|
35
|
+
if first_task_index > last_task_index:
|
36
|
+
raise ValueError(
|
37
|
+
f"{first_task_index=} cannot be larger than {last_task_index=}"
|
38
|
+
)
|
39
|
+
return (first_task_index, last_task_index)
|
@@ -0,0 +1,103 @@
|
|
1
|
+
from pathlib import Path
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
|
5
|
+
def sanitize_component(value: str) -> str:
|
6
|
+
"""
|
7
|
+
Remove {" ", "/", "."} form a string, e.g. going from
|
8
|
+
'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
|
9
|
+
"""
|
10
|
+
return value.replace(" ", "_").replace("/", "_").replace(".", "_")
|
11
|
+
|
12
|
+
|
13
|
+
class TaskFiles:
|
14
|
+
"""
|
15
|
+
Group all file paths pertaining to a task
|
16
|
+
|
17
|
+
Attributes:
|
18
|
+
workflow_dir:
|
19
|
+
Server-owned directory to store all task-execution-related relevant
|
20
|
+
files (inputs, outputs, errors, and all meta files related to the
|
21
|
+
job execution). Note: users cannot write directly to this folder.
|
22
|
+
workflow_dir_user:
|
23
|
+
User-side directory with the same scope as `workflow_dir`, and
|
24
|
+
where a user can write.
|
25
|
+
task_order:
|
26
|
+
Positional order of the task within a workflow.
|
27
|
+
component:
|
28
|
+
Specific component to run the task for (relevant for tasks that
|
29
|
+
will be executed in parallel over many components).
|
30
|
+
file_prefix:
|
31
|
+
Prefix for all task-related files.
|
32
|
+
args:
|
33
|
+
Path for input json file.
|
34
|
+
metadiff:
|
35
|
+
Path for output json file with metadata update.
|
36
|
+
out:
|
37
|
+
Path for task-execution stdout.
|
38
|
+
err:
|
39
|
+
Path for task-execution stderr.
|
40
|
+
"""
|
41
|
+
|
42
|
+
workflow_dir: Path
|
43
|
+
workflow_dir_user: Path
|
44
|
+
task_order: Optional[int] = None
|
45
|
+
component: Optional[str] = None
|
46
|
+
|
47
|
+
file_prefix: str
|
48
|
+
args: Path
|
49
|
+
out: Path
|
50
|
+
err: Path
|
51
|
+
log: Path
|
52
|
+
metadiff: Path
|
53
|
+
|
54
|
+
def __init__(
|
55
|
+
self,
|
56
|
+
workflow_dir: Path,
|
57
|
+
workflow_dir_user: Path,
|
58
|
+
task_order: Optional[int] = None,
|
59
|
+
component: Optional[str] = None,
|
60
|
+
):
|
61
|
+
self.workflow_dir = workflow_dir
|
62
|
+
self.workflow_dir_user = workflow_dir_user
|
63
|
+
self.task_order = task_order
|
64
|
+
self.component = component
|
65
|
+
|
66
|
+
if self.component is not None:
|
67
|
+
component_safe = sanitize_component(str(self.component))
|
68
|
+
component_safe = f"_par_{component_safe}"
|
69
|
+
else:
|
70
|
+
component_safe = ""
|
71
|
+
|
72
|
+
if self.task_order is not None:
|
73
|
+
order = str(self.task_order)
|
74
|
+
else:
|
75
|
+
order = "task"
|
76
|
+
self.file_prefix = f"{order}{component_safe}"
|
77
|
+
self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
|
78
|
+
self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
|
79
|
+
self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
|
80
|
+
self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
|
81
|
+
self.metadiff = (
|
82
|
+
self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
|
83
|
+
)
|
84
|
+
|
85
|
+
|
86
|
+
def get_task_file_paths(
|
87
|
+
workflow_dir: Path,
|
88
|
+
workflow_dir_user: Path,
|
89
|
+
task_order: Optional[int] = None,
|
90
|
+
component: Optional[str] = None,
|
91
|
+
) -> TaskFiles:
|
92
|
+
"""
|
93
|
+
Return the corrisponding TaskFiles object
|
94
|
+
|
95
|
+
This function is mainly used as a cache to avoid instantiating needless
|
96
|
+
objects.
|
97
|
+
"""
|
98
|
+
return TaskFiles(
|
99
|
+
workflow_dir=workflow_dir,
|
100
|
+
workflow_dir_user=workflow_dir_user,
|
101
|
+
task_order=task_order,
|
102
|
+
component=component,
|
103
|
+
)
|