fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/models/__init__.py +6 -8
  3. fractal_server/app/models/linkuserproject.py +9 -0
  4. fractal_server/app/models/security.py +6 -0
  5. fractal_server/app/models/v1/__init__.py +12 -0
  6. fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
  7. fractal_server/app/models/{job.py → v1/job.py} +5 -5
  8. fractal_server/app/models/{project.py → v1/project.py} +5 -5
  9. fractal_server/app/models/{state.py → v1/state.py} +2 -2
  10. fractal_server/app/models/{task.py → v1/task.py} +7 -2
  11. fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
  12. fractal_server/app/models/v2/__init__.py +22 -0
  13. fractal_server/app/models/v2/collection_state.py +21 -0
  14. fractal_server/app/models/v2/dataset.py +54 -0
  15. fractal_server/app/models/v2/job.py +51 -0
  16. fractal_server/app/models/v2/project.py +30 -0
  17. fractal_server/app/models/v2/task.py +93 -0
  18. fractal_server/app/models/v2/workflow.py +35 -0
  19. fractal_server/app/models/v2/workflowtask.py +49 -0
  20. fractal_server/app/routes/admin/__init__.py +0 -0
  21. fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
  22. fractal_server/app/routes/admin/v2.py +309 -0
  23. fractal_server/app/routes/api/v1/__init__.py +7 -7
  24. fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
  25. fractal_server/app/routes/api/v1/dataset.py +41 -41
  26. fractal_server/app/routes/api/v1/job.py +14 -14
  27. fractal_server/app/routes/api/v1/project.py +27 -25
  28. fractal_server/app/routes/api/v1/task.py +26 -16
  29. fractal_server/app/routes/api/v1/task_collection.py +28 -16
  30. fractal_server/app/routes/api/v1/workflow.py +28 -28
  31. fractal_server/app/routes/api/v1/workflowtask.py +11 -11
  32. fractal_server/app/routes/api/v2/__init__.py +34 -0
  33. fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
  34. fractal_server/app/routes/api/v2/dataset.py +293 -0
  35. fractal_server/app/routes/api/v2/images.py +279 -0
  36. fractal_server/app/routes/api/v2/job.py +200 -0
  37. fractal_server/app/routes/api/v2/project.py +186 -0
  38. fractal_server/app/routes/api/v2/status.py +150 -0
  39. fractal_server/app/routes/api/v2/submit.py +210 -0
  40. fractal_server/app/routes/api/v2/task.py +222 -0
  41. fractal_server/app/routes/api/v2/task_collection.py +239 -0
  42. fractal_server/app/routes/api/v2/task_legacy.py +59 -0
  43. fractal_server/app/routes/api/v2/workflow.py +380 -0
  44. fractal_server/app/routes/api/v2/workflowtask.py +265 -0
  45. fractal_server/app/routes/aux/_job.py +2 -2
  46. fractal_server/app/runner/__init__.py +0 -364
  47. fractal_server/app/runner/async_wrap.py +27 -0
  48. fractal_server/app/runner/components.py +5 -0
  49. fractal_server/app/runner/exceptions.py +129 -0
  50. fractal_server/app/runner/executors/__init__.py +0 -0
  51. fractal_server/app/runner/executors/slurm/__init__.py +3 -0
  52. fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
  53. fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
  54. fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
  55. fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
  56. fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
  57. fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +32 -21
  58. fractal_server/app/runner/filenames.py +6 -0
  59. fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
  60. fractal_server/app/runner/task_files.py +103 -0
  61. fractal_server/app/runner/v1/__init__.py +366 -0
  62. fractal_server/app/runner/{_common.py → v1/_common.py} +14 -121
  63. fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
  64. fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
  65. fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
  66. fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
  67. fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
  68. fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
  69. fractal_server/app/runner/v1/common.py +117 -0
  70. fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
  71. fractal_server/app/runner/v2/__init__.py +336 -0
  72. fractal_server/app/runner/v2/_local/__init__.py +162 -0
  73. fractal_server/app/runner/v2/_local/_local_config.py +118 -0
  74. fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
  75. fractal_server/app/runner/v2/_local/executor.py +100 -0
  76. fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
  77. fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
  78. fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
  79. fractal_server/app/runner/v2/deduplicate_list.py +23 -0
  80. fractal_server/app/runner/v2/handle_failed_job.py +165 -0
  81. fractal_server/app/runner/v2/merge_outputs.py +38 -0
  82. fractal_server/app/runner/v2/runner.py +343 -0
  83. fractal_server/app/runner/v2/runner_functions.py +374 -0
  84. fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
  85. fractal_server/app/runner/v2/task_interface.py +62 -0
  86. fractal_server/app/runner/v2/v1_compat.py +31 -0
  87. fractal_server/app/schemas/__init__.py +1 -42
  88. fractal_server/app/schemas/_validators.py +28 -5
  89. fractal_server/app/schemas/v1/__init__.py +36 -0
  90. fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
  91. fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
  92. fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
  93. fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
  94. fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
  95. fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
  96. fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
  97. fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
  98. fractal_server/app/schemas/v2/__init__.py +37 -0
  99. fractal_server/app/schemas/v2/dataset.py +126 -0
  100. fractal_server/app/schemas/v2/dumps.py +87 -0
  101. fractal_server/app/schemas/v2/job.py +114 -0
  102. fractal_server/app/schemas/v2/manifest.py +159 -0
  103. fractal_server/app/schemas/v2/project.py +34 -0
  104. fractal_server/app/schemas/v2/status.py +16 -0
  105. fractal_server/app/schemas/v2/task.py +151 -0
  106. fractal_server/app/schemas/v2/task_collection.py +109 -0
  107. fractal_server/app/schemas/v2/workflow.py +79 -0
  108. fractal_server/app/schemas/v2/workflowtask.py +208 -0
  109. fractal_server/config.py +5 -4
  110. fractal_server/images/__init__.py +4 -0
  111. fractal_server/images/models.py +136 -0
  112. fractal_server/images/tools.py +84 -0
  113. fractal_server/main.py +11 -3
  114. fractal_server/migrations/env.py +0 -2
  115. fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
  116. fractal_server/tasks/__init__.py +0 -5
  117. fractal_server/tasks/endpoint_operations.py +13 -19
  118. fractal_server/tasks/utils.py +35 -0
  119. fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
  120. fractal_server/tasks/v1/__init__.py +0 -0
  121. fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
  122. fractal_server/tasks/v1/get_collection_data.py +14 -0
  123. fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
  124. fractal_server/tasks/v2/__init__.py +0 -0
  125. fractal_server/tasks/v2/background_operations.py +381 -0
  126. fractal_server/tasks/v2/get_collection_data.py +14 -0
  127. fractal_server/urls.py +13 -0
  128. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/METADATA +10 -10
  129. fractal_server-2.0.0.dist-info/RECORD +169 -0
  130. fractal_server/app/runner/_slurm/.gitignore +0 -2
  131. fractal_server/app/runner/common.py +0 -311
  132. fractal_server/app/schemas/json_schemas/manifest.json +0 -81
  133. fractal_server-1.4.10.dist-info/RECORD +0 -98
  134. /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
  135. /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
  136. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
  137. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
  138. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -22,10 +22,9 @@ from pydantic import Extra
22
22
  from pydantic import Field
23
23
  from pydantic.error_wrappers import ValidationError
24
24
 
25
- from ....config import get_settings
26
- from ....logger import set_logger
27
- from ....syringe import Inject
28
- from ...models import WorkflowTask
25
+ from .....config import get_settings
26
+ from .....logger import set_logger
27
+ from .....syringe import Inject
29
28
 
30
29
  logger = set_logger(__name__)
31
30
 
@@ -459,151 +458,3 @@ def get_default_slurm_config():
459
458
  target_num_jobs=2,
460
459
  max_num_jobs=4,
461
460
  )
462
-
463
-
464
- def get_slurm_config(
465
- wftask: WorkflowTask,
466
- workflow_dir: Path,
467
- workflow_dir_user: Path,
468
- config_path: Optional[Path] = None,
469
- ) -> SlurmConfig:
470
- """
471
- Prepare a `SlurmConfig` configuration object
472
-
473
- The sources for `SlurmConfig` attributes, in increasing priority order, are
474
-
475
- 1. The general content of the Fractal SLURM configuration file.
476
- 2. The GPU-specific content of the Fractal SLURM configuration file, if
477
- appropriate.
478
- 3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
479
- `Workflow.insert_task`, also includes `wftask.task.meta`);
480
-
481
- Note: `wftask.meta` may be `None`.
482
-
483
- Arguments:
484
- wftask:
485
- WorkflowTask for which the SLURM configuration is is to be
486
- prepared.
487
- workflow_dir:
488
- Server-owned directory to store all task-execution-related relevant
489
- files (inputs, outputs, errors, and all meta files related to the
490
- job execution). Note: users cannot write directly to this folder.
491
- workflow_dir_user:
492
- User-side directory with the same scope as `workflow_dir`, and
493
- where a user can write.
494
- config_path:
495
- Path of aFractal SLURM configuration file; if `None`, use
496
- `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
497
-
498
- Returns:
499
- slurm_config:
500
- The SlurmConfig object
501
- """
502
-
503
- logger.debug(
504
- "[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
505
- )
506
-
507
- # Incorporate slurm_env.default_slurm_config
508
- slurm_env = load_slurm_config_file(config_path=config_path)
509
- slurm_dict = slurm_env.default_slurm_config.dict(
510
- exclude_unset=True, exclude={"mem"}
511
- )
512
- if slurm_env.default_slurm_config.mem:
513
- slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
514
-
515
- # Incorporate slurm_env.batching_config
516
- for key, value in slurm_env.batching_config.dict().items():
517
- slurm_dict[key] = value
518
-
519
- # Incorporate slurm_env.user_local_exports
520
- slurm_dict["user_local_exports"] = slurm_env.user_local_exports
521
-
522
- logger.debug(
523
- "[get_slurm_config] Fractal SLURM configuration file: "
524
- f"{slurm_env.dict()=}"
525
- )
526
-
527
- # GPU-related options
528
- # Notes about priority:
529
- # 1. This block of definitions takes priority over other definitions from
530
- # slurm_env which are not under the `needs_gpu` subgroup
531
- # 2. This block of definitions has lower priority than whatever comes next
532
- # (i.e. from WorkflowTask.meta).
533
- if wftask.meta is not None:
534
- needs_gpu = wftask.meta.get("needs_gpu", False)
535
- else:
536
- needs_gpu = False
537
- logger.debug(f"[get_slurm_config] {needs_gpu=}")
538
- if needs_gpu:
539
- for key, value in slurm_env.gpu_slurm_config.dict(
540
- exclude_unset=True, exclude={"mem"}
541
- ).items():
542
- slurm_dict[key] = value
543
- if slurm_env.gpu_slurm_config.mem:
544
- slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
545
-
546
- # Number of CPUs per task, for multithreading
547
- if wftask.meta is not None and "cpus_per_task" in wftask.meta:
548
- cpus_per_task = int(wftask.meta["cpus_per_task"])
549
- slurm_dict["cpus_per_task"] = cpus_per_task
550
-
551
- # Required memory per task, in MB
552
- if wftask.meta is not None and "mem" in wftask.meta:
553
- raw_mem = wftask.meta["mem"]
554
- mem_per_task_MB = _parse_mem_value(raw_mem)
555
- slurm_dict["mem_per_task_MB"] = mem_per_task_MB
556
-
557
- # Job name
558
- job_name = wftask.task.name.replace(" ", "_")
559
- slurm_dict["job_name"] = job_name
560
-
561
- # Optional SLURM arguments and extra lines
562
- if wftask.meta is not None:
563
- account = wftask.meta.get("account", None)
564
- if account is not None:
565
- error_msg = (
566
- f"Invalid {account=} property in WorkflowTask `meta` "
567
- "attribute.\n"
568
- "SLURM account must be set in the request body of the "
569
- "apply-workflow endpoint, or by modifying the user properties."
570
- )
571
- logger.error(error_msg)
572
- raise SlurmConfigError(error_msg)
573
- for key in ["time", "gres", "constraint"]:
574
- value = wftask.meta.get(key, None)
575
- if value:
576
- slurm_dict[key] = value
577
- if wftask.meta is not None:
578
- extra_lines = wftask.meta.get("extra_lines", [])
579
- else:
580
- extra_lines = []
581
- extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
582
- if len(set(extra_lines)) != len(extra_lines):
583
- logger.debug(
584
- "[get_slurm_config] Removing repeated elements "
585
- f"from {extra_lines=}."
586
- )
587
- extra_lines = list(set(extra_lines))
588
- slurm_dict["extra_lines"] = extra_lines
589
-
590
- # Job-batching parameters (if None, they will be determined heuristically)
591
- if wftask.meta is not None:
592
- tasks_per_job = wftask.meta.get("tasks_per_job", None)
593
- parallel_tasks_per_job = wftask.meta.get(
594
- "parallel_tasks_per_job", None
595
- )
596
- else:
597
- tasks_per_job = None
598
- parallel_tasks_per_job = None
599
- slurm_dict["tasks_per_job"] = tasks_per_job
600
- slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
601
-
602
- # Put everything together
603
- logger.debug(
604
- "[get_slurm_config] Now create a SlurmConfig object based "
605
- f"on {slurm_dict=}"
606
- )
607
- slurm_config = SlurmConfig(**slurm_dict)
608
-
609
- return slurm_config
@@ -19,7 +19,7 @@ import shlex
19
19
  import subprocess # nosec
20
20
  from typing import Optional
21
21
 
22
- from ....logger import set_logger
22
+ from .....logger import set_logger
23
23
 
24
24
  logger = set_logger(__name__)
25
25
 
@@ -29,14 +29,14 @@ import cloudpickle
29
29
  from cfut import SlurmExecutor
30
30
  from cfut.util import random_string
31
31
 
32
- from ....config import get_settings
33
- from ....logger import set_logger
34
- from ....syringe import Inject
35
- from .._common import get_task_file_paths
36
- from .._common import SHUTDOWN_FILENAME
37
- from .._common import TaskFiles
38
- from ..common import JobExecutionError
39
- from ..common import TaskExecutionError
32
+ from .....config import get_settings
33
+ from .....logger import set_logger
34
+ from .....syringe import Inject
35
+ from ...exceptions import JobExecutionError
36
+ from ...exceptions import TaskExecutionError
37
+ from ...filenames import SHUTDOWN_FILENAME
38
+ from ...task_files import get_task_file_paths
39
+ from ...task_files import TaskFiles
40
40
  from ._batching import heuristics
41
41
  from ._executor_wait_thread import FractalSlurmWaitThread
42
42
  from ._slurm_config import get_default_slurm_config
@@ -46,6 +46,7 @@ from ._subprocess_run_as_user import _glob_as_user_strict
46
46
  from ._subprocess_run_as_user import _path_exists_as_user
47
47
  from ._subprocess_run_as_user import _run_command_as_user
48
48
  from fractal_server import __VERSION__
49
+ from fractal_server.app.runner.components import _COMPONENT_KEY_
49
50
 
50
51
 
51
52
  logger = set_logger(__name__)
@@ -416,8 +417,6 @@ class FractalSlurmExecutor(SlurmExecutor):
416
417
  A `TaskFiles` object; if `None`, use
417
418
  `self.get_default_task_files()`.
418
419
 
419
- Returns:
420
- An iterator of results.
421
420
  """
422
421
 
423
422
  def _result_or_cancel(fut):
@@ -544,7 +543,7 @@ class FractalSlurmExecutor(SlurmExecutor):
544
543
  single_task_submission: bool = False,
545
544
  args: Optional[Sequence[Any]] = None,
546
545
  kwargs: Optional[dict] = None,
547
- components: list[Any] = None,
546
+ components: Optional[list[Any]] = None,
548
547
  ) -> Future:
549
548
  """
550
549
  Submit a multi-task job to the pool, where each task is handled via the
@@ -580,6 +579,10 @@ class FractalSlurmExecutor(SlurmExecutor):
580
579
 
581
580
  # Define slurm-job-related files
582
581
  if single_task_submission:
582
+ if components is not None:
583
+ raise ValueError(
584
+ f"{single_task_submission=} but components is not None"
585
+ )
583
586
  job = SlurmJob(
584
587
  slurm_file_prefix=slurm_file_prefix,
585
588
  num_tasks_tot=1,
@@ -603,15 +606,23 @@ class FractalSlurmExecutor(SlurmExecutor):
603
606
  num_tasks_tot=num_tasks_tot,
604
607
  slurm_config=slurm_config,
605
608
  )
606
- job.wftask_file_prefixes = tuple(
607
- get_task_file_paths(
608
- workflow_dir=task_files.workflow_dir,
609
- workflow_dir_user=task_files.workflow_dir_user,
610
- task_order=task_files.task_order,
611
- component=component,
612
- ).file_prefix
613
- for component in components
614
- )
609
+
610
+ _prefixes = []
611
+ for component in components:
612
+ if isinstance(component, dict):
613
+ # This is needed for V2
614
+ actual_component = component.get(_COMPONENT_KEY_, None)
615
+ else:
616
+ actual_component = component
617
+ _prefixes.append(
618
+ get_task_file_paths(
619
+ workflow_dir=task_files.workflow_dir,
620
+ workflow_dir_user=task_files.workflow_dir_user,
621
+ task_order=task_files.task_order,
622
+ component=actual_component,
623
+ ).file_prefix
624
+ )
625
+ job.wftask_file_prefixes = tuple(_prefixes)
615
626
 
616
627
  # Define I/O pickle file names/paths
617
628
  job.input_pickle_files = tuple(
@@ -1001,7 +1012,7 @@ class FractalSlurmExecutor(SlurmExecutor):
1001
1012
  cmdlines.append(
1002
1013
  (
1003
1014
  f"{python_worker_interpreter}"
1004
- " -m fractal_server.app.runner._slurm.remote "
1015
+ " -m fractal_server.app.runner.executors.slurm.remote "
1005
1016
  f"--input-file {input_pickle_file} "
1006
1017
  f"--output-file {output_pickle_file}"
1007
1018
  )
@@ -0,0 +1,6 @@
1
+ HISTORY_FILENAME = "history.json"
2
+ FILTERS_FILENAME = "filters.json"
3
+ IMAGES_FILENAME = "images.json"
4
+ METADATA_FILENAME = "metadata.json"
5
+ SHUTDOWN_FILENAME = "shutdown"
6
+ WORKFLOW_LOG_FILENAME = "workflow.log"
@@ -0,0 +1,39 @@
1
+ from typing import Optional
2
+
3
+
4
+ def set_start_and_last_task_index(
5
+ num_tasks: int,
6
+ first_task_index: Optional[int] = None,
7
+ last_task_index: Optional[int] = None,
8
+ ) -> tuple[int, int]:
9
+ """
10
+ Handle `first_task_index` and `last_task_index`, by setting defaults and
11
+ validating values.
12
+
13
+ num_tasks:
14
+ Total number of tasks in a workflow task list
15
+ first_task_index:
16
+ Positional index of the first task to execute
17
+ last_task_index:
18
+ Positional index of the last task to execute
19
+ """
20
+ # Set default values
21
+ if first_task_index is None:
22
+ first_task_index = 0
23
+ if last_task_index is None:
24
+ last_task_index = num_tasks - 1
25
+
26
+ # Perform checks
27
+ if first_task_index < 0:
28
+ raise ValueError(f"{first_task_index=} cannot be negative")
29
+ if last_task_index < 0:
30
+ raise ValueError(f"{last_task_index=} cannot be negative")
31
+ if last_task_index > num_tasks - 1:
32
+ raise ValueError(
33
+ f"{last_task_index=} cannot be larger than {(num_tasks-1)=}"
34
+ )
35
+ if first_task_index > last_task_index:
36
+ raise ValueError(
37
+ f"{first_task_index=} cannot be larger than {last_task_index=}"
38
+ )
39
+ return (first_task_index, last_task_index)
@@ -0,0 +1,103 @@
1
+ from pathlib import Path
2
+ from typing import Optional
3
+
4
+
5
+ def sanitize_component(value: str) -> str:
6
+ """
7
+ Remove {" ", "/", "."} form a string, e.g. going from
8
+ 'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
9
+ """
10
+ return value.replace(" ", "_").replace("/", "_").replace(".", "_")
11
+
12
+
13
+ class TaskFiles:
14
+ """
15
+ Group all file paths pertaining to a task
16
+
17
+ Attributes:
18
+ workflow_dir:
19
+ Server-owned directory to store all task-execution-related relevant
20
+ files (inputs, outputs, errors, and all meta files related to the
21
+ job execution). Note: users cannot write directly to this folder.
22
+ workflow_dir_user:
23
+ User-side directory with the same scope as `workflow_dir`, and
24
+ where a user can write.
25
+ task_order:
26
+ Positional order of the task within a workflow.
27
+ component:
28
+ Specific component to run the task for (relevant for tasks that
29
+ will be executed in parallel over many components).
30
+ file_prefix:
31
+ Prefix for all task-related files.
32
+ args:
33
+ Path for input json file.
34
+ metadiff:
35
+ Path for output json file with metadata update.
36
+ out:
37
+ Path for task-execution stdout.
38
+ err:
39
+ Path for task-execution stderr.
40
+ """
41
+
42
+ workflow_dir: Path
43
+ workflow_dir_user: Path
44
+ task_order: Optional[int] = None
45
+ component: Optional[str] = None
46
+
47
+ file_prefix: str
48
+ args: Path
49
+ out: Path
50
+ err: Path
51
+ log: Path
52
+ metadiff: Path
53
+
54
+ def __init__(
55
+ self,
56
+ workflow_dir: Path,
57
+ workflow_dir_user: Path,
58
+ task_order: Optional[int] = None,
59
+ component: Optional[str] = None,
60
+ ):
61
+ self.workflow_dir = workflow_dir
62
+ self.workflow_dir_user = workflow_dir_user
63
+ self.task_order = task_order
64
+ self.component = component
65
+
66
+ if self.component is not None:
67
+ component_safe = sanitize_component(str(self.component))
68
+ component_safe = f"_par_{component_safe}"
69
+ else:
70
+ component_safe = ""
71
+
72
+ if self.task_order is not None:
73
+ order = str(self.task_order)
74
+ else:
75
+ order = "task"
76
+ self.file_prefix = f"{order}{component_safe}"
77
+ self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
78
+ self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
79
+ self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
80
+ self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
81
+ self.metadiff = (
82
+ self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
83
+ )
84
+
85
+
86
+ def get_task_file_paths(
87
+ workflow_dir: Path,
88
+ workflow_dir_user: Path,
89
+ task_order: Optional[int] = None,
90
+ component: Optional[str] = None,
91
+ ) -> TaskFiles:
92
+ """
93
+ Return the corrisponding TaskFiles object
94
+
95
+ This function is mainly used as a cache to avoid instantiating needless
96
+ objects.
97
+ """
98
+ return TaskFiles(
99
+ workflow_dir=workflow_dir,
100
+ workflow_dir_user=workflow_dir_user,
101
+ task_order=task_order,
102
+ component=component,
103
+ )