fractal-server 2.17.1a1__py3-none-any.whl → 2.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +19 -18
  3. fractal_server/app/db/__init__.py +3 -3
  4. fractal_server/app/models/__init__.py +1 -0
  5. fractal_server/app/models/linkuserproject.py +3 -1
  6. fractal_server/app/models/security.py +21 -3
  7. fractal_server/app/models/v2/__init__.py +3 -1
  8. fractal_server/app/models/v2/accounting.py +9 -1
  9. fractal_server/app/models/v2/dataset.py +5 -1
  10. fractal_server/app/models/v2/history.py +15 -1
  11. fractal_server/app/models/v2/job.py +4 -0
  12. fractal_server/app/models/v2/profile.py +29 -0
  13. fractal_server/app/models/v2/project.py +4 -10
  14. fractal_server/app/models/v2/resource.py +4 -0
  15. fractal_server/app/models/v2/task_group.py +4 -3
  16. fractal_server/app/models/v2/workflow.py +2 -1
  17. fractal_server/app/routes/admin/v2/__init__.py +1 -2
  18. fractal_server/app/routes/admin/v2/accounting.py +1 -1
  19. fractal_server/app/routes/admin/v2/job.py +9 -9
  20. fractal_server/app/routes/admin/v2/profile.py +3 -2
  21. fractal_server/app/routes/admin/v2/resource.py +5 -5
  22. fractal_server/app/routes/admin/v2/task.py +28 -18
  23. fractal_server/app/routes/admin/v2/task_group.py +0 -1
  24. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +1 -2
  25. fractal_server/app/routes/api/__init__.py +1 -0
  26. fractal_server/app/routes/api/v2/__init__.py +5 -6
  27. fractal_server/app/routes/api/v2/_aux_functions.py +70 -63
  28. fractal_server/app/routes/api/v2/_aux_functions_history.py +43 -20
  29. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +2 -4
  30. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -7
  31. fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
  32. fractal_server/app/routes/api/v2/dataset.py +13 -32
  33. fractal_server/app/routes/api/v2/history.py +35 -21
  34. fractal_server/app/routes/api/v2/images.py +3 -2
  35. fractal_server/app/routes/api/v2/job.py +17 -14
  36. fractal_server/app/routes/api/v2/pre_submission_checks.py +5 -4
  37. fractal_server/app/routes/api/v2/project.py +22 -17
  38. fractal_server/app/routes/api/v2/status_legacy.py +12 -11
  39. fractal_server/app/routes/api/v2/submit.py +11 -12
  40. fractal_server/app/routes/api/v2/task.py +4 -3
  41. fractal_server/app/routes/api/v2/task_collection.py +28 -30
  42. fractal_server/app/routes/api/v2/task_collection_custom.py +8 -7
  43. fractal_server/app/routes/api/v2/task_collection_pixi.py +1 -2
  44. fractal_server/app/routes/api/v2/task_group.py +7 -6
  45. fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -6
  46. fractal_server/app/routes/api/v2/task_version_update.py +13 -12
  47. fractal_server/app/routes/api/v2/workflow.py +14 -31
  48. fractal_server/app/routes/api/v2/workflow_import.py +17 -19
  49. fractal_server/app/routes/api/v2/workflowtask.py +10 -12
  50. fractal_server/app/routes/auth/__init__.py +1 -3
  51. fractal_server/app/routes/auth/_aux_auth.py +1 -2
  52. fractal_server/app/routes/auth/current_user.py +4 -5
  53. fractal_server/app/routes/auth/group.py +7 -5
  54. fractal_server/app/routes/auth/login.py +1 -0
  55. fractal_server/app/routes/auth/oauth.py +4 -3
  56. fractal_server/app/routes/auth/register.py +4 -2
  57. fractal_server/app/routes/auth/users.py +10 -10
  58. fractal_server/app/routes/aux/_job.py +1 -1
  59. fractal_server/app/routes/aux/_runner.py +2 -2
  60. fractal_server/app/routes/pagination.py +1 -1
  61. fractal_server/app/schemas/user.py +3 -3
  62. fractal_server/app/schemas/v2/accounting.py +11 -0
  63. fractal_server/app/schemas/v2/dataset.py +28 -4
  64. fractal_server/app/schemas/v2/dumps.py +1 -0
  65. fractal_server/app/schemas/v2/manifest.py +4 -3
  66. fractal_server/app/schemas/v2/profile.py +53 -2
  67. fractal_server/app/schemas/v2/resource.py +109 -13
  68. fractal_server/app/schemas/v2/task.py +0 -1
  69. fractal_server/app/schemas/v2/task_collection.py +1 -1
  70. fractal_server/app/schemas/v2/workflowtask.py +4 -3
  71. fractal_server/app/security/__init__.py +4 -7
  72. fractal_server/app/security/signup_email.py +4 -5
  73. fractal_server/config/_data.py +36 -25
  74. fractal_server/config/_database.py +19 -20
  75. fractal_server/config/_email.py +30 -38
  76. fractal_server/config/_main.py +33 -52
  77. fractal_server/config/_oauth.py +17 -21
  78. fractal_server/exceptions.py +4 -0
  79. fractal_server/images/models.py +3 -3
  80. fractal_server/images/status_tools.py +4 -2
  81. fractal_server/logger.py +1 -1
  82. fractal_server/main.py +4 -3
  83. fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
  84. fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
  85. fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
  86. fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
  87. fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
  88. fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
  89. fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
  90. fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
  91. fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
  92. fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
  93. fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
  94. fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
  95. fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
  96. fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
  97. fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
  98. fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
  99. fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
  100. fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
  101. fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
  102. fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
  103. fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
  104. fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
  105. fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
  106. fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
  107. fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
  108. fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
  109. fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
  110. fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
  111. fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
  112. fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
  113. fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
  114. fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
  115. fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
  116. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
  117. fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
  118. fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
  119. fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
  120. fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
  121. fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
  122. fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
  123. fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
  124. fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
  125. fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
  126. fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
  127. fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
  128. fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
  129. fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
  130. fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
  131. fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
  132. fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
  133. fractal_server/runner/config/_local.py +8 -5
  134. fractal_server/runner/config/_slurm.py +37 -33
  135. fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
  136. fractal_server/runner/executors/base_runner.py +29 -4
  137. fractal_server/runner/executors/local/get_local_config.py +1 -0
  138. fractal_server/runner/executors/local/runner.py +14 -13
  139. fractal_server/runner/executors/slurm_common/_batching.py +5 -10
  140. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
  141. fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
  142. fractal_server/runner/executors/slurm_common/remote.py +3 -1
  143. fractal_server/runner/executors/slurm_common/slurm_config.py +1 -0
  144. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
  145. fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
  146. fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
  147. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
  148. fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
  149. fractal_server/runner/task_files.py +9 -3
  150. fractal_server/runner/v2/_local.py +9 -4
  151. fractal_server/runner/v2/_slurm_ssh.py +11 -5
  152. fractal_server/runner/v2/_slurm_sudo.py +11 -5
  153. fractal_server/runner/v2/db_tools.py +0 -1
  154. fractal_server/runner/v2/deduplicate_list.py +2 -1
  155. fractal_server/runner/v2/runner.py +11 -14
  156. fractal_server/runner/v2/runner_functions.py +11 -14
  157. fractal_server/runner/v2/submit_workflow.py +7 -6
  158. fractal_server/ssh/_fabric.py +6 -13
  159. fractal_server/string_tools.py +0 -1
  160. fractal_server/syringe.py +1 -1
  161. fractal_server/tasks/config/_pixi.py +1 -1
  162. fractal_server/tasks/config/_python.py +16 -9
  163. fractal_server/tasks/utils.py +0 -1
  164. fractal_server/tasks/v2/local/_utils.py +1 -1
  165. fractal_server/tasks/v2/local/collect.py +10 -12
  166. fractal_server/tasks/v2/local/collect_pixi.py +9 -10
  167. fractal_server/tasks/v2/local/deactivate.py +7 -8
  168. fractal_server/tasks/v2/local/deactivate_pixi.py +4 -4
  169. fractal_server/tasks/v2/local/delete.py +1 -3
  170. fractal_server/tasks/v2/local/reactivate.py +7 -7
  171. fractal_server/tasks/v2/local/reactivate_pixi.py +7 -7
  172. fractal_server/tasks/v2/ssh/_utils.py +3 -3
  173. fractal_server/tasks/v2/ssh/collect.py +14 -19
  174. fractal_server/tasks/v2/ssh/collect_pixi.py +17 -19
  175. fractal_server/tasks/v2/ssh/deactivate.py +10 -8
  176. fractal_server/tasks/v2/ssh/deactivate_pixi.py +6 -5
  177. fractal_server/tasks/v2/ssh/delete.py +7 -5
  178. fractal_server/tasks/v2/ssh/reactivate.py +11 -11
  179. fractal_server/tasks/v2/ssh/reactivate_pixi.py +8 -9
  180. fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
  181. fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
  182. fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
  183. fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
  184. fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
  185. fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
  186. fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
  187. fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
  188. fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
  189. fractal_server/tasks/v2/utils_background.py +3 -3
  190. fractal_server/tasks/v2/utils_package_names.py +1 -2
  191. fractal_server/tasks/v2/utils_pixi.py +1 -3
  192. fractal_server/types/__init__.py +76 -1
  193. fractal_server/types/validators/_common_validators.py +1 -3
  194. fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
  195. fractal_server/utils.py +1 -0
  196. fractal_server/zip_tools.py +34 -0
  197. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/METADATA +1 -1
  198. fractal_server-2.17.2.dist-info/RECORD +265 -0
  199. fractal_server/app/routes/admin/v2/project.py +0 -41
  200. fractal_server-2.17.1a1.dist-info/RECORD +0 -264
  201. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/WHEEL +0 -0
  202. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/entry_points.txt +0 -0
  203. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,14 +1,15 @@
1
1
  from typing import Literal
2
2
 
3
- from ._batching import heuristics
4
- from .slurm_config import logger
5
- from .slurm_config import SlurmConfig
6
3
  from fractal_server.app.models.v2 import WorkflowTaskV2
7
4
  from fractal_server.runner.config import JobRunnerConfigSLURM
8
5
  from fractal_server.runner.config.slurm_mem_to_MB import slurm_mem_to_MB
9
6
  from fractal_server.runner.exceptions import SlurmConfigError
10
7
  from fractal_server.string_tools import interpret_as_bool
11
8
 
9
+ from ._batching import heuristics
10
+ from .slurm_config import SlurmConfig
11
+ from .slurm_config import logger
12
+
12
13
 
13
14
  def _get_slurm_config_internal(
14
15
  shared_config: JobRunnerConfigSLURM,
@@ -33,7 +34,6 @@ def _get_slurm_config_internal(
33
34
  which_type:
34
35
  Whether we should look at the non-parallel or parallel part
35
36
  of `wftask`.
36
- tot_tasks: Not used here, only present as a common interface.
37
37
 
38
38
  Returns:
39
39
  A ready-to-use `SlurmConfig` object.
@@ -138,9 +138,7 @@ def _get_slurm_config_internal(
138
138
  # Job-batching parameters (if None, they will be determined heuristically)
139
139
  if wftask_meta is not None:
140
140
  tasks_per_job = wftask_meta.get("tasks_per_job", None)
141
- parallel_tasks_per_job = wftask_meta.get(
142
- "parallel_tasks_per_job", None
143
- )
141
+ parallel_tasks_per_job = wftask_meta.get("parallel_tasks_per_job", None)
144
142
  else:
145
143
  tasks_per_job = None
146
144
  parallel_tasks_per_job = None
@@ -162,6 +160,15 @@ def get_slurm_config(
162
160
  which_type: Literal["non_parallel", "parallel"],
163
161
  tot_tasks: int = 1,
164
162
  ) -> SlurmConfig:
163
+ """
164
+ Get `SlurmConfig` object.
165
+
166
+ Args:
167
+ shared_config:
168
+ wftask:
169
+ which_type:
170
+ tot_tasks:
171
+ """
165
172
  config = _get_slurm_config_internal(
166
173
  shared_config=shared_config,
167
174
  wftask=wftask,
@@ -3,8 +3,10 @@ import json
3
3
  import os
4
4
  import sys
5
5
 
6
- from ..call_command_wrapper import call_command_wrapper
7
6
  from fractal_server import __VERSION__
7
+ from fractal_server.runner.executors.call_command_wrapper import (
8
+ call_command_wrapper,
9
+ )
8
10
 
9
11
 
10
12
  class FractalVersionMismatch(RuntimeError):
@@ -1,6 +1,7 @@
1
1
  """
2
2
  Submodule to handle the SLURM configuration for a WorkflowTask
3
3
  """
4
+
4
5
  from pathlib import Path
5
6
 
6
7
  from pydantic import BaseModel
@@ -24,9 +24,7 @@ class SlurmTask(BaseModel):
24
24
 
25
25
  @property
26
26
  def input_file_local_path(self) -> Path:
27
- return (
28
- self.workdir_local / f"{self.prefix}-{self.component}-input.json"
29
- )
27
+ return self.workdir_local / f"{self.prefix}-{self.component}-input.json"
30
28
 
31
29
  @property
32
30
  def input_file_remote_path(self) -> Path:
@@ -1,23 +1,32 @@
1
1
  import time
2
2
  from pathlib import Path
3
3
 
4
- from ..slurm_common.base_slurm_runner import BaseSlurmRunner
5
- from ..slurm_common.slurm_job_task_models import SlurmJob
6
- from .run_subprocess import run_subprocess
7
- from .tar_commands import get_tar_compression_cmd
8
- from .tar_commands import get_tar_extraction_cmd
9
4
  from fractal_server.app.models import Profile
10
5
  from fractal_server.app.models import Resource
11
6
  from fractal_server.logger import set_logger
12
7
  from fractal_server.runner.config import JobRunnerConfigSLURM
8
+ from fractal_server.runner.executors.slurm_common.base_slurm_runner import (
9
+ BaseSlurmRunner,
10
+ )
11
+ from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
12
+ SlurmJob,
13
+ )
13
14
  from fractal_server.ssh._fabric import FractalSSH
14
15
  from fractal_server.ssh._fabric import FractalSSHCommandError
15
16
  from fractal_server.ssh._fabric import FractalSSHTimeoutError
16
17
 
18
+ from .run_subprocess import run_subprocess
19
+ from .tar_commands import get_tar_compression_cmd
20
+ from .tar_commands import get_tar_extraction_cmd
21
+
17
22
  logger = set_logger(__name__)
18
23
 
19
24
 
20
25
  class SlurmSSHRunner(BaseSlurmRunner):
26
+ """
27
+ Runner implementation for a computational `sudo_slurm` resource.
28
+ """
29
+
21
30
  fractal_ssh: FractalSSH
22
31
 
23
32
  def __init__(
@@ -39,9 +48,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
39
48
  different SLURM jobs/tasks.
40
49
  """
41
50
  self.fractal_ssh = fractal_ssh
42
- self.shared_config = JobRunnerConfigSLURM(
43
- **resource.jobs_runner_config
44
- )
51
+ self.shared_config = JobRunnerConfigSLURM(**resource.jobs_runner_config)
45
52
  logger.warning(self.fractal_ssh)
46
53
 
47
54
  # Check SSH connection and try to recover from a closed-socket error
@@ -80,9 +87,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
80
87
  return None
81
88
 
82
89
  t_0 = time.perf_counter()
83
- logger.debug(
84
- f"[_fetch_artifacts] START ({len(finished_slurm_jobs)=})."
85
- )
90
+ logger.debug(f"[_fetch_artifacts] START ({len(finished_slurm_jobs)=}).")
86
91
 
87
92
  # Extract `workdir_remote` and `workdir_local`
88
93
  self.validate_slurm_jobs_workdirs(finished_slurm_jobs)
@@ -1,6 +1,7 @@
1
1
  """
2
2
  Prepare tar commands for task-subfolder compression/extraction.
3
3
  """
4
+
4
5
  from pathlib import Path
5
6
 
6
7
 
@@ -15,6 +15,7 @@ This module provides a set of tools similar to `subprocess.run`, `glob.glob` or
15
15
  `os.path.exists`, but extended so that they can be executed on behalf of
16
16
  another user. Note that this requires appropriate sudo permissions.
17
17
  """
18
+
18
19
  import shlex
19
20
  import subprocess # nosec
20
21
 
@@ -5,15 +5,20 @@ import subprocess # nosec
5
5
  from concurrent.futures import ThreadPoolExecutor
6
6
  from pathlib import Path
7
7
 
8
- from ..slurm_common.base_slurm_runner import BaseSlurmRunner
9
- from ..slurm_common.slurm_job_task_models import SlurmJob
10
- from ._subprocess_run_as_user import _mkdir_as_user
11
- from ._subprocess_run_as_user import _run_command_as_user
12
8
  from fractal_server.app.models import Profile
13
9
  from fractal_server.app.models import Resource
14
10
  from fractal_server.logger import set_logger
15
11
  from fractal_server.runner.config import JobRunnerConfigSLURM
16
12
  from fractal_server.runner.exceptions import JobExecutionError
13
+ from fractal_server.runner.executors.slurm_common.base_slurm_runner import (
14
+ BaseSlurmRunner,
15
+ )
16
+ from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
17
+ SlurmJob,
18
+ )
19
+
20
+ from ._subprocess_run_as_user import _mkdir_as_user
21
+ from ._subprocess_run_as_user import _run_command_as_user
17
22
 
18
23
  logger = set_logger(__name__)
19
24
 
@@ -40,7 +45,11 @@ def _subprocess_run_or_raise(
40
45
  raise JobExecutionError(info=error_msg)
41
46
 
42
47
 
43
- class SudoSlurmRunner(BaseSlurmRunner):
48
+ class SlurmSudoRunner(BaseSlurmRunner):
49
+ """
50
+ Runner implementation for a computational `sudo_slurm` resource.
51
+ """
52
+
44
53
  slurm_user: str
45
54
  slurm_account: str | None = None
46
55
 
@@ -63,9 +72,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
63
72
  """
64
73
 
65
74
  self.slurm_user = profile.username
66
- self.shared_config = JobRunnerConfigSLURM(
67
- **resource.jobs_runner_config
68
- )
75
+ self.shared_config = JobRunnerConfigSLURM(**resource.jobs_runner_config)
69
76
 
70
77
  super().__init__(
71
78
  slurm_runner_type="sudo",
@@ -90,9 +97,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
90
97
  """
91
98
  Fetch artifacts for a single SLURM jobs.
92
99
  """
93
- logger.debug(
94
- f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START"
95
- )
100
+ logger.debug(f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START")
96
101
  source_target_list = [
97
102
  (job.slurm_stdout_remote, job.slurm_stdout_local),
98
103
  (job.slurm_stderr_remote, job.slurm_stderr_local),
@@ -86,9 +86,7 @@ class TaskFiles(BaseModel):
86
86
  @property
87
87
  def log_file_remote_path(self) -> Path:
88
88
  self._check_component()
89
- return (
90
- self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
91
- )
89
+ return self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
92
90
 
93
91
  @property
94
92
  def log_file_remote(self) -> str:
@@ -141,8 +139,16 @@ def enrich_task_files_multisubmit(
141
139
  ) -> list[TaskFiles]:
142
140
  """
143
141
  Expand `TaskFiles` objects with `component` and `prefix`.
142
+
143
+ Args:
144
+ tot_tasks: Total number of images to process.
145
+ batch_size: Batch size, where `0` means `batch_size=tot_tasks`.
146
+ base_task_files: Original `TaskFiles` object to be enriched.
144
147
  """
145
148
 
149
+ # Replace `batch_size=0` with `batch_size=tot_tasks`
150
+ batch_size = batch_size or tot_tasks
151
+
146
152
  new_list_task_files: list[TaskFiles] = []
147
153
  for absolute_index in range(tot_tasks):
148
154
  ind_batch = absolute_index // batch_size
@@ -1,16 +1,21 @@
1
1
  from pathlib import Path
2
2
 
3
- from ..executors.local.get_local_config import get_local_backend_config
4
- from ..executors.local.runner import LocalRunner
5
- from ..set_start_and_last_task_index import set_start_and_last_task_index
6
- from .runner import execute_tasks_v2
7
3
  from fractal_server.app.models.v2 import DatasetV2
8
4
  from fractal_server.app.models.v2 import Profile
9
5
  from fractal_server.app.models.v2 import Resource
10
6
  from fractal_server.app.models.v2 import WorkflowV2
7
+ from fractal_server.runner.executors.local.get_local_config import (
8
+ get_local_backend_config,
9
+ )
10
+ from fractal_server.runner.executors.local.runner import LocalRunner
11
+ from fractal_server.runner.set_start_and_last_task_index import (
12
+ set_start_and_last_task_index,
13
+ )
11
14
  from fractal_server.ssh._fabric import FractalSSH
12
15
  from fractal_server.types import AttributeFilters
13
16
 
17
+ from .runner import execute_tasks_v2
18
+
14
19
 
15
20
  def process_workflow(
16
21
  *,
@@ -15,20 +15,26 @@ Slurm Backend
15
15
 
16
16
  This backend runs fractal workflows in a SLURM cluster.
17
17
  """
18
+
18
19
  from pathlib import Path
19
20
 
20
- from ...ssh._fabric import FractalSSH
21
- from ..executors.slurm_common.get_slurm_config import get_slurm_config
22
- from ..executors.slurm_ssh.runner import SlurmSSHRunner
23
- from ..set_start_and_last_task_index import set_start_and_last_task_index
24
- from .runner import execute_tasks_v2
25
21
  from fractal_server.app.models.v2 import DatasetV2
26
22
  from fractal_server.app.models.v2 import Profile
27
23
  from fractal_server.app.models.v2 import Resource
28
24
  from fractal_server.app.models.v2 import WorkflowV2
29
25
  from fractal_server.logger import set_logger
26
+ from fractal_server.runner.executors.slurm_common.get_slurm_config import (
27
+ get_slurm_config,
28
+ )
29
+ from fractal_server.runner.executors.slurm_ssh.runner import SlurmSSHRunner
30
+ from fractal_server.runner.set_start_and_last_task_index import (
31
+ set_start_and_last_task_index,
32
+ )
33
+ from fractal_server.ssh._fabric import FractalSSH
30
34
  from fractal_server.types import AttributeFilters
31
35
 
36
+ from .runner import execute_tasks_v2
37
+
32
38
  logger = set_logger(__name__)
33
39
 
34
40
 
@@ -15,19 +15,25 @@ Slurm Backend
15
15
 
16
16
  This backend runs fractal workflows in a SLURM cluster.
17
17
  """
18
+
18
19
  from pathlib import Path
19
20
 
20
- from ..executors.slurm_common.get_slurm_config import get_slurm_config
21
- from ..executors.slurm_sudo.runner import SudoSlurmRunner
22
- from ..set_start_and_last_task_index import set_start_and_last_task_index
23
- from .runner import execute_tasks_v2
24
21
  from fractal_server.app.models.v2 import DatasetV2
25
22
  from fractal_server.app.models.v2 import Profile
26
23
  from fractal_server.app.models.v2 import Resource
27
24
  from fractal_server.app.models.v2 import WorkflowV2
25
+ from fractal_server.runner.executors.slurm_common.get_slurm_config import (
26
+ get_slurm_config,
27
+ )
28
+ from fractal_server.runner.executors.slurm_sudo.runner import SlurmSudoRunner
29
+ from fractal_server.runner.set_start_and_last_task_index import (
30
+ set_start_and_last_task_index,
31
+ )
28
32
  from fractal_server.ssh._fabric import FractalSSH
29
33
  from fractal_server.types import AttributeFilters
30
34
 
35
+ from .runner import execute_tasks_v2
36
+
31
37
 
32
38
  def process_workflow(
33
39
  *,
@@ -94,7 +100,7 @@ def process_workflow(
94
100
  if isinstance(worker_init, str):
95
101
  worker_init = worker_init.split("\n")
96
102
 
97
- with SudoSlurmRunner(
103
+ with SlurmSudoRunner(
98
104
  root_dir_local=workflow_dir_local,
99
105
  root_dir_remote=workflow_dir_remote,
100
106
  common_script_lines=worker_init,
@@ -10,7 +10,6 @@ from fractal_server.app.models.v2 import HistoryUnit
10
10
  from fractal_server.app.schemas.v2 import HistoryUnitStatus
11
11
  from fractal_server.logger import set_logger
12
12
 
13
-
14
13
  _CHUNK_SIZE = 2_000
15
14
 
16
15
  logger = set_logger(__name__)
@@ -1,9 +1,10 @@
1
1
  from typing import TypeVar
2
2
 
3
- from .task_interface import InitArgsModel
4
3
  from fractal_server.images import SingleImage
5
4
  from fractal_server.images import SingleImageTaskOutput
6
5
 
6
+ from .task_interface import InitArgsModel
7
+
7
8
  T = TypeVar("T", SingleImage, SingleImageTaskOutput, InitArgsModel)
8
9
 
9
10
 
@@ -7,13 +7,6 @@ from sqlalchemy.orm.attributes import flag_modified
7
7
  from sqlmodel import delete
8
8
  from sqlmodel import update
9
9
 
10
- from .merge_outputs import merge_outputs
11
- from .runner_functions import GetRunnerConfigType
12
- from .runner_functions import run_v2_task_compound
13
- from .runner_functions import run_v2_task_non_parallel
14
- from .runner_functions import run_v2_task_parallel
15
- from .runner_functions import SubmissionOutcome
16
- from .task_interface import TaskOutput
17
10
  from fractal_server.app.db import get_sync_db
18
11
  from fractal_server.app.models.v2 import AccountingRecord
19
12
  from fractal_server.app.models.v2 import DatasetV2
@@ -28,8 +21,8 @@ from fractal_server.app.schemas.v2 import TaskDumpV2
28
21
  from fractal_server.app.schemas.v2 import TaskGroupDumpV2
29
22
  from fractal_server.app.schemas.v2 import TaskType
30
23
  from fractal_server.images import SingleImage
31
- from fractal_server.images.status_tools import enrich_images_unsorted_sync
32
24
  from fractal_server.images.status_tools import IMAGE_STATUS_KEY
25
+ from fractal_server.images.status_tools import enrich_images_unsorted_sync
33
26
  from fractal_server.images.tools import filter_image_list
34
27
  from fractal_server.images.tools import find_image_by_zarr_url
35
28
  from fractal_server.images.tools import merge_type_filters
@@ -39,6 +32,14 @@ from fractal_server.runner.executors.base_runner import BaseRunner
39
32
  from fractal_server.runner.v2.db_tools import update_status_of_history_run
40
33
  from fractal_server.types import AttributeFilters
41
34
 
35
+ from .merge_outputs import merge_outputs
36
+ from .runner_functions import GetRunnerConfigType
37
+ from .runner_functions import SubmissionOutcome
38
+ from .runner_functions import run_v2_task_compound
39
+ from .runner_functions import run_v2_task_non_parallel
40
+ from .runner_functions import run_v2_task_parallel
41
+ from .task_interface import TaskOutput
42
+
42
43
 
43
44
  def _remove_status_from_attributes(
44
45
  images: list[dict[str, Any]],
@@ -312,9 +313,7 @@ def execute_tasks_v2(
312
313
  # history status.
313
314
  for image_obj in current_task_output.image_list_updates:
314
315
  image = image_obj.model_dump()
315
- if image["zarr_url"] in [
316
- img["zarr_url"] for img in tmp_images
317
- ]:
316
+ if image["zarr_url"] in [img["zarr_url"] for img in tmp_images]:
318
317
  img_search = find_image_by_zarr_url(
319
318
  images=tmp_images,
320
319
  zarr_url=image["zarr_url"],
@@ -418,9 +417,7 @@ def execute_tasks_v2(
418
417
 
419
418
  # Update type_filters based on task-manifest output_types
420
419
  type_filters_from_task_manifest = task.output_types
421
- current_dataset_type_filters.update(
422
- type_filters_from_task_manifest
423
- )
420
+ current_dataset_type_filters.update(type_filters_from_task_manifest)
424
421
  except Exception as e:
425
422
  logger.error(
426
423
  "Unexpected error in post-task-execution block. "
@@ -6,12 +6,6 @@ from typing import Protocol
6
6
  from pydantic import BaseModel
7
7
  from pydantic import ConfigDict
8
8
 
9
- from ..exceptions import JobExecutionError
10
- from ..exceptions import TaskOutputValidationError
11
- from .db_tools import update_status_of_history_unit
12
- from .deduplicate_list import deduplicate_list
13
- from .task_interface import InitTaskOutput
14
- from .task_interface import TaskOutput
15
9
  from fractal_server.app.db import get_sync_db
16
10
  from fractal_server.app.models.v2 import HistoryUnit
17
11
  from fractal_server.app.models.v2 import TaskV2
@@ -22,16 +16,16 @@ from fractal_server.exceptions import UnreachableBranchError
22
16
  from fractal_server.logger import set_logger
23
17
  from fractal_server.runner.config import JobRunnerConfigLocal
24
18
  from fractal_server.runner.config import JobRunnerConfigSLURM
19
+ from fractal_server.runner.exceptions import JobExecutionError
20
+ from fractal_server.runner.exceptions import TaskOutputValidationError
25
21
  from fractal_server.runner.executors.base_runner import BaseRunner
26
22
  from fractal_server.runner.executors.slurm_common.slurm_config import (
27
23
  SlurmConfig,
28
24
  )
29
- from fractal_server.runner.task_files import enrich_task_files_multisubmit
30
25
  from fractal_server.runner.task_files import SUBMIT_PREFIX
31
26
  from fractal_server.runner.task_files import TaskFiles
32
- from fractal_server.runner.v2.db_tools import (
33
- bulk_update_status_of_history_unit,
34
- )
27
+ from fractal_server.runner.task_files import enrich_task_files_multisubmit
28
+ from fractal_server.runner.v2.db_tools import bulk_update_status_of_history_unit
35
29
  from fractal_server.runner.v2.db_tools import bulk_upsert_image_cache_fast
36
30
  from fractal_server.runner.v2.task_interface import (
37
31
  _cast_and_validate_InitTaskOutput,
@@ -40,6 +34,11 @@ from fractal_server.runner.v2.task_interface import (
40
34
  _cast_and_validate_TaskOutput,
41
35
  )
42
36
 
37
+ from .db_tools import update_status_of_history_unit
38
+ from .deduplicate_list import deduplicate_list
39
+ from .task_interface import InitTaskOutput
40
+ from .task_interface import TaskOutput
41
+
43
42
 
44
43
  class GetRunnerConfigTypeLocal(Protocol):
45
44
  def __call__(
@@ -48,8 +47,7 @@ class GetRunnerConfigTypeLocal(Protocol):
48
47
  wftask: WorkflowTaskV2,
49
48
  which_type: Literal["non_parallel", "parallel"],
50
49
  tot_tasks: int,
51
- ) -> JobRunnerConfigLocal:
52
- ...
50
+ ) -> JobRunnerConfigLocal: ...
53
51
 
54
52
 
55
53
  class GetRunnerConfigTypeSLURM(Protocol):
@@ -59,8 +57,7 @@ class GetRunnerConfigTypeSLURM(Protocol):
59
57
  wftask: WorkflowTaskV2,
60
58
  which_type: Literal["non_parallel", "parallel"],
61
59
  tot_tasks: int,
62
- ) -> SlurmConfig:
63
- ...
60
+ ) -> SlurmConfig: ...
64
61
 
65
62
 
66
63
  GetRunnerConfigType = GetRunnerConfigTypeLocal | GetRunnerConfigTypeSLURM
@@ -5,6 +5,7 @@ This module is the single entry point to the runner backend subsystem V2.
5
5
  Other subsystems should only import this module and not its submodules or
6
6
  the individual backends.
7
7
  """
8
+
8
9
  import os
9
10
  import traceback
10
11
  from pathlib import Path
@@ -12,9 +13,6 @@ from typing import Protocol
12
13
 
13
14
  from sqlalchemy.orm import Session as DBSyncSession
14
15
 
15
- from ._local import process_workflow as local_process_workflow
16
- from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
17
- from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
18
16
  from fractal_server import __VERSION__
19
17
  from fractal_server.app.db import DB
20
18
  from fractal_server.app.models.v2 import DatasetV2
@@ -34,6 +32,10 @@ from fractal_server.types import AttributeFilters
34
32
  from fractal_server.utils import get_timestamp
35
33
  from fractal_server.zip_tools import _zip_folder_to_file_and_remove
36
34
 
35
+ from ._local import process_workflow as local_process_workflow
36
+ from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
37
+ from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
38
+
37
39
 
38
40
  class ProcessWorkflowType(Protocol):
39
41
  def __call__(
@@ -53,8 +55,7 @@ class ProcessWorkflowType(Protocol):
53
55
  resource: Resource,
54
56
  profile: Profile,
55
57
  user_cache_dir: str,
56
- ) -> None:
57
- ...
58
+ ) -> None: ...
58
59
 
59
60
 
60
61
  def fail_job(
@@ -119,7 +120,7 @@ def submit_workflow(
119
120
  Computational resource to be used for this job (e.g. a SLURM
120
121
  cluster).
121
122
  profile:
122
- Computational profile to be used for this job.
123
+ Computational profile to be used for this job.
123
124
  fractal_ssh: SSH object, for when `resource.type = "slurm_ssh"`.
124
125
  """
125
126
  # Declare runner backend and set `process_workflow` function
@@ -16,12 +16,11 @@ from invoke import UnexpectedExit
16
16
  from paramiko.ssh_exception import NoValidConnectionsError
17
17
  from pydantic import BaseModel
18
18
 
19
- from ..logger import close_logger
20
- from ..logger import get_logger
21
- from ..logger import set_logger
19
+ from fractal_server.logger import close_logger
20
+ from fractal_server.logger import get_logger
21
+ from fractal_server.logger import set_logger
22
22
  from fractal_server.string_tools import validate_cmd
23
23
 
24
-
25
24
  SSH_MONITORING_LOGGER_NAME = "ssh-log"
26
25
 
27
26
 
@@ -99,8 +98,7 @@ def _acquire_lock_with_timeout(
99
98
  if not result:
100
99
  logger.error(f"Lock for '{label}' was *not* acquired.")
101
100
  raise FractalSSHTimeoutError(
102
- f"Failed to acquire lock for '{label}' within "
103
- f"{timeout} seconds"
101
+ f"Failed to acquire lock for '{label}' within {timeout} seconds"
104
102
  )
105
103
  t_lock_acquisition = time.perf_counter()
106
104
  elapsed = t_lock_acquisition - t_lock_request
@@ -374,8 +372,6 @@ class FractalSSH:
374
372
  Args:
375
373
  cmd: Command to be run
376
374
  allow_char: Forbidden chars to allow for this command
377
- max_attempts:
378
- base_interval:
379
375
  lock_timeout:
380
376
 
381
377
  Returns:
@@ -458,9 +454,7 @@ class FractalSSH:
458
454
  logger_name=self.logger_name,
459
455
  ):
460
456
  self._sftp_unsafe().put(local, remote)
461
- self.logger.info(
462
- f"[send_file] END transfer of '{local}' over SSH."
463
- )
457
+ self.logger.info(f"[send_file] END transfer of '{local}' over SSH.")
464
458
  except Exception as e:
465
459
  self.log_and_raise(
466
460
  e=e,
@@ -766,8 +760,7 @@ class FractalSSHList:
766
760
  logger_name=self._logger_name,
767
761
  ):
768
762
  self.logger.info(
769
- f"Removing FractalSSH object for {user}@{host} "
770
- "from collection."
763
+ f"Removing FractalSSH object for {user}@{host} from collection."
771
764
  )
772
765
  fractal_ssh_obj = self._data.pop(key)
773
766
  self.logger.info(
@@ -1,6 +1,5 @@
1
1
  import string
2
2
 
3
-
4
3
  __SPECIAL_CHARACTERS__ = f"{string.punctuation}{string.whitespace}"
5
4
 
6
5
  # List of invalid characters discussed here:
fractal_server/syringe.py CHANGED
@@ -34,11 +34,11 @@ or popped from the directory.
34
34
  >>> bar()
35
35
  42
36
36
  """
37
+
37
38
  from collections.abc import Callable
38
39
  from typing import Any
39
40
  from typing import TypeVar
40
41
 
41
-
42
42
  T = TypeVar("T")
43
43
  _instance_count = 0
44
44
 
@@ -3,8 +3,8 @@ from typing import Annotated
3
3
 
4
4
  from pydantic import AfterValidator
5
5
  from pydantic import BaseModel
6
- from pydantic import model_validator
7
6
  from pydantic import PositiveInt
7
+ from pydantic import model_validator
8
8
 
9
9
  from fractal_server.types import DictStrStr
10
10
  from fractal_server.types import NonEmptyStr
@@ -14,12 +14,25 @@ class TasksPythonSettings(BaseModel):
14
14
 
15
15
  For task collection to work, there must be one or more base Python
16
16
  interpreters available on your system.
17
+
18
+ Attributes:
19
+ default_version:
20
+ Default task-collection Python version (must be a key of
21
+ `versions`).
22
+ versions:
23
+ Dictionary mapping Python versions to the corresponding
24
+ interpreters. Example:
25
+ ```json
26
+ {
27
+ "3.11": "/path/to/python3.11",
28
+ "3.13": "/path/to/python3.13"
29
+ }
30
+ ```
31
+ pip_cache_dir:
32
+ Argument for `--cache-dir` option of `pip install`, if set.
17
33
  """
18
34
 
19
35
  default_version: NonEmptyStr
20
- """
21
- Default task-collection Python version (must be a key of `versions`).
22
- """
23
36
  versions: dict[
24
37
  Literal[
25
38
  "3.9",
@@ -31,14 +44,8 @@ class TasksPythonSettings(BaseModel):
31
44
  ],
32
45
  AbsolutePathStr,
33
46
  ]
34
- """
35
- Dictionary mapping Python versions to the corresponding interpreters.
36
- """
37
47
 
38
48
  pip_cache_dir: AbsolutePathStr | None = None
39
- """
40
- Argument for `--cache-dir` option of `pip install`, if set.
41
- """
42
49
 
43
50
  @model_validator(mode="after")
44
51
  def _validate_versions(self) -> Self: