fractal-server 2.17.1a0__py3-none-any.whl → 2.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +19 -18
  3. fractal_server/app/db/__init__.py +3 -3
  4. fractal_server/app/models/__init__.py +1 -1
  5. fractal_server/app/models/linkuserproject.py +3 -1
  6. fractal_server/app/models/security.py +22 -17
  7. fractal_server/app/models/v2/__init__.py +3 -1
  8. fractal_server/app/models/v2/accounting.py +9 -1
  9. fractal_server/app/models/v2/dataset.py +5 -1
  10. fractal_server/app/models/v2/history.py +15 -1
  11. fractal_server/app/models/v2/job.py +4 -0
  12. fractal_server/app/models/v2/profile.py +29 -0
  13. fractal_server/app/models/v2/project.py +5 -14
  14. fractal_server/app/models/v2/resource.py +4 -0
  15. fractal_server/app/models/v2/task_group.py +5 -7
  16. fractal_server/app/models/v2/workflow.py +2 -1
  17. fractal_server/app/routes/admin/v2/__init__.py +1 -2
  18. fractal_server/app/routes/admin/v2/accounting.py +1 -1
  19. fractal_server/app/routes/admin/v2/job.py +9 -9
  20. fractal_server/app/routes/admin/v2/profile.py +3 -2
  21. fractal_server/app/routes/admin/v2/resource.py +5 -5
  22. fractal_server/app/routes/admin/v2/task.py +28 -18
  23. fractal_server/app/routes/admin/v2/task_group.py +0 -1
  24. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +1 -2
  25. fractal_server/app/routes/api/__init__.py +1 -0
  26. fractal_server/app/routes/api/v2/__init__.py +5 -6
  27. fractal_server/app/routes/api/v2/_aux_functions.py +70 -63
  28. fractal_server/app/routes/api/v2/_aux_functions_history.py +43 -20
  29. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +2 -4
  30. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -7
  31. fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
  32. fractal_server/app/routes/api/v2/dataset.py +13 -32
  33. fractal_server/app/routes/api/v2/history.py +35 -21
  34. fractal_server/app/routes/api/v2/images.py +3 -2
  35. fractal_server/app/routes/api/v2/job.py +17 -14
  36. fractal_server/app/routes/api/v2/pre_submission_checks.py +5 -4
  37. fractal_server/app/routes/api/v2/project.py +22 -17
  38. fractal_server/app/routes/api/v2/status_legacy.py +12 -11
  39. fractal_server/app/routes/api/v2/submit.py +11 -12
  40. fractal_server/app/routes/api/v2/task.py +4 -3
  41. fractal_server/app/routes/api/v2/task_collection.py +28 -30
  42. fractal_server/app/routes/api/v2/task_collection_custom.py +8 -7
  43. fractal_server/app/routes/api/v2/task_collection_pixi.py +1 -2
  44. fractal_server/app/routes/api/v2/task_group.py +7 -6
  45. fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -6
  46. fractal_server/app/routes/api/v2/task_version_update.py +13 -12
  47. fractal_server/app/routes/api/v2/workflow.py +14 -31
  48. fractal_server/app/routes/api/v2/workflow_import.py +17 -19
  49. fractal_server/app/routes/api/v2/workflowtask.py +10 -12
  50. fractal_server/app/routes/auth/__init__.py +1 -3
  51. fractal_server/app/routes/auth/_aux_auth.py +1 -2
  52. fractal_server/app/routes/auth/current_user.py +4 -5
  53. fractal_server/app/routes/auth/group.py +7 -5
  54. fractal_server/app/routes/auth/login.py +1 -0
  55. fractal_server/app/routes/auth/oauth.py +4 -3
  56. fractal_server/app/routes/auth/register.py +4 -2
  57. fractal_server/app/routes/auth/users.py +10 -10
  58. fractal_server/app/routes/aux/_job.py +1 -1
  59. fractal_server/app/routes/aux/_runner.py +2 -2
  60. fractal_server/app/routes/pagination.py +1 -1
  61. fractal_server/app/schemas/user.py +3 -3
  62. fractal_server/app/schemas/v2/accounting.py +11 -0
  63. fractal_server/app/schemas/v2/dataset.py +28 -4
  64. fractal_server/app/schemas/v2/dumps.py +1 -0
  65. fractal_server/app/schemas/v2/manifest.py +4 -3
  66. fractal_server/app/schemas/v2/profile.py +53 -2
  67. fractal_server/app/schemas/v2/resource.py +109 -13
  68. fractal_server/app/schemas/v2/task.py +0 -1
  69. fractal_server/app/schemas/v2/task_collection.py +1 -1
  70. fractal_server/app/schemas/v2/workflowtask.py +4 -3
  71. fractal_server/app/security/__init__.py +4 -7
  72. fractal_server/app/security/signup_email.py +4 -5
  73. fractal_server/app/shutdown.py +23 -19
  74. fractal_server/config/_data.py +36 -25
  75. fractal_server/config/_database.py +19 -20
  76. fractal_server/config/_email.py +30 -38
  77. fractal_server/config/_main.py +34 -53
  78. fractal_server/config/_oauth.py +17 -21
  79. fractal_server/exceptions.py +4 -0
  80. fractal_server/images/models.py +3 -3
  81. fractal_server/images/status_tools.py +4 -2
  82. fractal_server/logger.py +1 -1
  83. fractal_server/main.py +4 -3
  84. fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
  85. fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
  86. fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
  87. fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
  88. fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
  89. fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
  90. fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
  91. fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
  92. fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +46 -0
  93. fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
  94. fractal_server/migrations/versions/49d0856e9569_drop_table.py +62 -0
  95. fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
  96. fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
  97. fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
  98. fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
  99. fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
  100. fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
  101. fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
  102. fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +29 -0
  103. fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
  104. fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
  105. fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
  106. fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
  107. fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
  108. fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
  109. fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
  110. fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
  111. fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
  112. fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
  113. fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
  114. fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
  115. fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
  116. fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
  117. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
  118. fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
  119. fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
  120. fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
  121. fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
  122. fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +49 -0
  123. fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
  124. fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
  125. fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
  126. fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
  127. fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
  128. fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
  129. fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
  130. fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
  131. fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
  132. fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
  133. fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
  134. fractal_server/runner/config/_local.py +8 -5
  135. fractal_server/runner/config/_slurm.py +37 -33
  136. fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
  137. fractal_server/runner/executors/base_runner.py +29 -4
  138. fractal_server/runner/executors/local/get_local_config.py +1 -0
  139. fractal_server/runner/executors/local/runner.py +14 -13
  140. fractal_server/runner/executors/slurm_common/_batching.py +5 -10
  141. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
  142. fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
  143. fractal_server/runner/executors/slurm_common/remote.py +3 -1
  144. fractal_server/runner/executors/slurm_common/slurm_config.py +1 -0
  145. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
  146. fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
  147. fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
  148. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
  149. fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
  150. fractal_server/runner/task_files.py +9 -3
  151. fractal_server/runner/v2/_local.py +9 -4
  152. fractal_server/runner/v2/_slurm_ssh.py +11 -5
  153. fractal_server/runner/v2/_slurm_sudo.py +11 -5
  154. fractal_server/runner/v2/db_tools.py +0 -1
  155. fractal_server/runner/v2/deduplicate_list.py +2 -1
  156. fractal_server/runner/v2/runner.py +11 -14
  157. fractal_server/runner/v2/runner_functions.py +11 -14
  158. fractal_server/runner/v2/submit_workflow.py +7 -6
  159. fractal_server/ssh/_fabric.py +6 -13
  160. fractal_server/string_tools.py +0 -1
  161. fractal_server/syringe.py +1 -1
  162. fractal_server/tasks/config/_pixi.py +1 -1
  163. fractal_server/tasks/config/_python.py +16 -9
  164. fractal_server/tasks/utils.py +0 -1
  165. fractal_server/tasks/v2/local/_utils.py +1 -1
  166. fractal_server/tasks/v2/local/collect.py +10 -12
  167. fractal_server/tasks/v2/local/collect_pixi.py +9 -10
  168. fractal_server/tasks/v2/local/deactivate.py +7 -8
  169. fractal_server/tasks/v2/local/deactivate_pixi.py +4 -4
  170. fractal_server/tasks/v2/local/delete.py +1 -3
  171. fractal_server/tasks/v2/local/reactivate.py +7 -7
  172. fractal_server/tasks/v2/local/reactivate_pixi.py +7 -7
  173. fractal_server/tasks/v2/ssh/_utils.py +3 -3
  174. fractal_server/tasks/v2/ssh/collect.py +14 -19
  175. fractal_server/tasks/v2/ssh/collect_pixi.py +17 -19
  176. fractal_server/tasks/v2/ssh/deactivate.py +10 -8
  177. fractal_server/tasks/v2/ssh/deactivate_pixi.py +6 -5
  178. fractal_server/tasks/v2/ssh/delete.py +7 -5
  179. fractal_server/tasks/v2/ssh/reactivate.py +11 -11
  180. fractal_server/tasks/v2/ssh/reactivate_pixi.py +8 -9
  181. fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
  182. fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
  183. fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
  184. fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
  185. fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
  186. fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
  187. fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
  188. fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
  189. fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
  190. fractal_server/tasks/v2/utils_background.py +3 -3
  191. fractal_server/tasks/v2/utils_package_names.py +1 -2
  192. fractal_server/tasks/v2/utils_pixi.py +1 -3
  193. fractal_server/types/__init__.py +76 -1
  194. fractal_server/types/validators/_common_validators.py +1 -3
  195. fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
  196. fractal_server/utils.py +1 -0
  197. fractal_server/zip_tools.py +34 -0
  198. {fractal_server-2.17.1a0.dist-info → fractal_server-2.17.2.dist-info}/METADATA +1 -1
  199. fractal_server-2.17.2.dist-info/RECORD +265 -0
  200. fractal_server/app/models/user_settings.py +0 -37
  201. fractal_server/app/routes/admin/v2/project.py +0 -41
  202. fractal_server/data_migrations/2_17_0.py +0 -339
  203. fractal_server-2.17.1a0.dist-info/RECORD +0 -262
  204. {fractal_server-2.17.1a0.dist-info → fractal_server-2.17.2.dist-info}/WHEEL +0 -0
  205. {fractal_server-2.17.1a0.dist-info → fractal_server-2.17.2.dist-info}/entry_points.txt +0 -0
  206. {fractal_server-2.17.1a0.dist-info → fractal_server-2.17.2.dist-info}/licenses/LICENSE +0 -0
@@ -5,15 +5,20 @@ import subprocess # nosec
5
5
  from concurrent.futures import ThreadPoolExecutor
6
6
  from pathlib import Path
7
7
 
8
- from ..slurm_common.base_slurm_runner import BaseSlurmRunner
9
- from ..slurm_common.slurm_job_task_models import SlurmJob
10
- from ._subprocess_run_as_user import _mkdir_as_user
11
- from ._subprocess_run_as_user import _run_command_as_user
12
8
  from fractal_server.app.models import Profile
13
9
  from fractal_server.app.models import Resource
14
10
  from fractal_server.logger import set_logger
15
11
  from fractal_server.runner.config import JobRunnerConfigSLURM
16
12
  from fractal_server.runner.exceptions import JobExecutionError
13
+ from fractal_server.runner.executors.slurm_common.base_slurm_runner import (
14
+ BaseSlurmRunner,
15
+ )
16
+ from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
17
+ SlurmJob,
18
+ )
19
+
20
+ from ._subprocess_run_as_user import _mkdir_as_user
21
+ from ._subprocess_run_as_user import _run_command_as_user
17
22
 
18
23
  logger = set_logger(__name__)
19
24
 
@@ -40,7 +45,11 @@ def _subprocess_run_or_raise(
40
45
  raise JobExecutionError(info=error_msg)
41
46
 
42
47
 
43
- class SudoSlurmRunner(BaseSlurmRunner):
48
+ class SlurmSudoRunner(BaseSlurmRunner):
49
+ """
50
+ Runner implementation for a computational `sudo_slurm` resource.
51
+ """
52
+
44
53
  slurm_user: str
45
54
  slurm_account: str | None = None
46
55
 
@@ -63,9 +72,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
63
72
  """
64
73
 
65
74
  self.slurm_user = profile.username
66
- self.shared_config = JobRunnerConfigSLURM(
67
- **resource.jobs_runner_config
68
- )
75
+ self.shared_config = JobRunnerConfigSLURM(**resource.jobs_runner_config)
69
76
 
70
77
  super().__init__(
71
78
  slurm_runner_type="sudo",
@@ -90,9 +97,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
90
97
  """
91
98
  Fetch artifacts for a single SLURM jobs.
92
99
  """
93
- logger.debug(
94
- f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START"
95
- )
100
+ logger.debug(f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START")
96
101
  source_target_list = [
97
102
  (job.slurm_stdout_remote, job.slurm_stdout_local),
98
103
  (job.slurm_stderr_remote, job.slurm_stderr_local),
@@ -86,9 +86,7 @@ class TaskFiles(BaseModel):
86
86
  @property
87
87
  def log_file_remote_path(self) -> Path:
88
88
  self._check_component()
89
- return (
90
- self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
91
- )
89
+ return self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
92
90
 
93
91
  @property
94
92
  def log_file_remote(self) -> str:
@@ -141,8 +139,16 @@ def enrich_task_files_multisubmit(
141
139
  ) -> list[TaskFiles]:
142
140
  """
143
141
  Expand `TaskFiles` objects with `component` and `prefix`.
142
+
143
+ Args:
144
+ tot_tasks: Total number of images to process.
145
+ batch_size: Batch size, where `0` means `batch_size=tot_tasks`.
146
+ base_task_files: Original `TaskFiles` object to be enriched.
144
147
  """
145
148
 
149
+ # Replace `batch_size=0` with `batch_size=tot_tasks`
150
+ batch_size = batch_size or tot_tasks
151
+
146
152
  new_list_task_files: list[TaskFiles] = []
147
153
  for absolute_index in range(tot_tasks):
148
154
  ind_batch = absolute_index // batch_size
@@ -1,16 +1,21 @@
1
1
  from pathlib import Path
2
2
 
3
- from ..executors.local.get_local_config import get_local_backend_config
4
- from ..executors.local.runner import LocalRunner
5
- from ..set_start_and_last_task_index import set_start_and_last_task_index
6
- from .runner import execute_tasks_v2
7
3
  from fractal_server.app.models.v2 import DatasetV2
8
4
  from fractal_server.app.models.v2 import Profile
9
5
  from fractal_server.app.models.v2 import Resource
10
6
  from fractal_server.app.models.v2 import WorkflowV2
7
+ from fractal_server.runner.executors.local.get_local_config import (
8
+ get_local_backend_config,
9
+ )
10
+ from fractal_server.runner.executors.local.runner import LocalRunner
11
+ from fractal_server.runner.set_start_and_last_task_index import (
12
+ set_start_and_last_task_index,
13
+ )
11
14
  from fractal_server.ssh._fabric import FractalSSH
12
15
  from fractal_server.types import AttributeFilters
13
16
 
17
+ from .runner import execute_tasks_v2
18
+
14
19
 
15
20
  def process_workflow(
16
21
  *,
@@ -15,20 +15,26 @@ Slurm Backend
15
15
 
16
16
  This backend runs fractal workflows in a SLURM cluster.
17
17
  """
18
+
18
19
  from pathlib import Path
19
20
 
20
- from ...ssh._fabric import FractalSSH
21
- from ..executors.slurm_common.get_slurm_config import get_slurm_config
22
- from ..executors.slurm_ssh.runner import SlurmSSHRunner
23
- from ..set_start_and_last_task_index import set_start_and_last_task_index
24
- from .runner import execute_tasks_v2
25
21
  from fractal_server.app.models.v2 import DatasetV2
26
22
  from fractal_server.app.models.v2 import Profile
27
23
  from fractal_server.app.models.v2 import Resource
28
24
  from fractal_server.app.models.v2 import WorkflowV2
29
25
  from fractal_server.logger import set_logger
26
+ from fractal_server.runner.executors.slurm_common.get_slurm_config import (
27
+ get_slurm_config,
28
+ )
29
+ from fractal_server.runner.executors.slurm_ssh.runner import SlurmSSHRunner
30
+ from fractal_server.runner.set_start_and_last_task_index import (
31
+ set_start_and_last_task_index,
32
+ )
33
+ from fractal_server.ssh._fabric import FractalSSH
30
34
  from fractal_server.types import AttributeFilters
31
35
 
36
+ from .runner import execute_tasks_v2
37
+
32
38
  logger = set_logger(__name__)
33
39
 
34
40
 
@@ -15,19 +15,25 @@ Slurm Backend
15
15
 
16
16
  This backend runs fractal workflows in a SLURM cluster.
17
17
  """
18
+
18
19
  from pathlib import Path
19
20
 
20
- from ..executors.slurm_common.get_slurm_config import get_slurm_config
21
- from ..executors.slurm_sudo.runner import SudoSlurmRunner
22
- from ..set_start_and_last_task_index import set_start_and_last_task_index
23
- from .runner import execute_tasks_v2
24
21
  from fractal_server.app.models.v2 import DatasetV2
25
22
  from fractal_server.app.models.v2 import Profile
26
23
  from fractal_server.app.models.v2 import Resource
27
24
  from fractal_server.app.models.v2 import WorkflowV2
25
+ from fractal_server.runner.executors.slurm_common.get_slurm_config import (
26
+ get_slurm_config,
27
+ )
28
+ from fractal_server.runner.executors.slurm_sudo.runner import SlurmSudoRunner
29
+ from fractal_server.runner.set_start_and_last_task_index import (
30
+ set_start_and_last_task_index,
31
+ )
28
32
  from fractal_server.ssh._fabric import FractalSSH
29
33
  from fractal_server.types import AttributeFilters
30
34
 
35
+ from .runner import execute_tasks_v2
36
+
31
37
 
32
38
  def process_workflow(
33
39
  *,
@@ -94,7 +100,7 @@ def process_workflow(
94
100
  if isinstance(worker_init, str):
95
101
  worker_init = worker_init.split("\n")
96
102
 
97
- with SudoSlurmRunner(
103
+ with SlurmSudoRunner(
98
104
  root_dir_local=workflow_dir_local,
99
105
  root_dir_remote=workflow_dir_remote,
100
106
  common_script_lines=worker_init,
@@ -10,7 +10,6 @@ from fractal_server.app.models.v2 import HistoryUnit
10
10
  from fractal_server.app.schemas.v2 import HistoryUnitStatus
11
11
  from fractal_server.logger import set_logger
12
12
 
13
-
14
13
  _CHUNK_SIZE = 2_000
15
14
 
16
15
  logger = set_logger(__name__)
@@ -1,9 +1,10 @@
1
1
  from typing import TypeVar
2
2
 
3
- from .task_interface import InitArgsModel
4
3
  from fractal_server.images import SingleImage
5
4
  from fractal_server.images import SingleImageTaskOutput
6
5
 
6
+ from .task_interface import InitArgsModel
7
+
7
8
  T = TypeVar("T", SingleImage, SingleImageTaskOutput, InitArgsModel)
8
9
 
9
10
 
@@ -7,13 +7,6 @@ from sqlalchemy.orm.attributes import flag_modified
7
7
  from sqlmodel import delete
8
8
  from sqlmodel import update
9
9
 
10
- from .merge_outputs import merge_outputs
11
- from .runner_functions import GetRunnerConfigType
12
- from .runner_functions import run_v2_task_compound
13
- from .runner_functions import run_v2_task_non_parallel
14
- from .runner_functions import run_v2_task_parallel
15
- from .runner_functions import SubmissionOutcome
16
- from .task_interface import TaskOutput
17
10
  from fractal_server.app.db import get_sync_db
18
11
  from fractal_server.app.models.v2 import AccountingRecord
19
12
  from fractal_server.app.models.v2 import DatasetV2
@@ -28,8 +21,8 @@ from fractal_server.app.schemas.v2 import TaskDumpV2
28
21
  from fractal_server.app.schemas.v2 import TaskGroupDumpV2
29
22
  from fractal_server.app.schemas.v2 import TaskType
30
23
  from fractal_server.images import SingleImage
31
- from fractal_server.images.status_tools import enrich_images_unsorted_sync
32
24
  from fractal_server.images.status_tools import IMAGE_STATUS_KEY
25
+ from fractal_server.images.status_tools import enrich_images_unsorted_sync
33
26
  from fractal_server.images.tools import filter_image_list
34
27
  from fractal_server.images.tools import find_image_by_zarr_url
35
28
  from fractal_server.images.tools import merge_type_filters
@@ -39,6 +32,14 @@ from fractal_server.runner.executors.base_runner import BaseRunner
39
32
  from fractal_server.runner.v2.db_tools import update_status_of_history_run
40
33
  from fractal_server.types import AttributeFilters
41
34
 
35
+ from .merge_outputs import merge_outputs
36
+ from .runner_functions import GetRunnerConfigType
37
+ from .runner_functions import SubmissionOutcome
38
+ from .runner_functions import run_v2_task_compound
39
+ from .runner_functions import run_v2_task_non_parallel
40
+ from .runner_functions import run_v2_task_parallel
41
+ from .task_interface import TaskOutput
42
+
42
43
 
43
44
  def _remove_status_from_attributes(
44
45
  images: list[dict[str, Any]],
@@ -312,9 +313,7 @@ def execute_tasks_v2(
312
313
  # history status.
313
314
  for image_obj in current_task_output.image_list_updates:
314
315
  image = image_obj.model_dump()
315
- if image["zarr_url"] in [
316
- img["zarr_url"] for img in tmp_images
317
- ]:
316
+ if image["zarr_url"] in [img["zarr_url"] for img in tmp_images]:
318
317
  img_search = find_image_by_zarr_url(
319
318
  images=tmp_images,
320
319
  zarr_url=image["zarr_url"],
@@ -418,9 +417,7 @@ def execute_tasks_v2(
418
417
 
419
418
  # Update type_filters based on task-manifest output_types
420
419
  type_filters_from_task_manifest = task.output_types
421
- current_dataset_type_filters.update(
422
- type_filters_from_task_manifest
423
- )
420
+ current_dataset_type_filters.update(type_filters_from_task_manifest)
424
421
  except Exception as e:
425
422
  logger.error(
426
423
  "Unexpected error in post-task-execution block. "
@@ -6,12 +6,6 @@ from typing import Protocol
6
6
  from pydantic import BaseModel
7
7
  from pydantic import ConfigDict
8
8
 
9
- from ..exceptions import JobExecutionError
10
- from ..exceptions import TaskOutputValidationError
11
- from .db_tools import update_status_of_history_unit
12
- from .deduplicate_list import deduplicate_list
13
- from .task_interface import InitTaskOutput
14
- from .task_interface import TaskOutput
15
9
  from fractal_server.app.db import get_sync_db
16
10
  from fractal_server.app.models.v2 import HistoryUnit
17
11
  from fractal_server.app.models.v2 import TaskV2
@@ -22,16 +16,16 @@ from fractal_server.exceptions import UnreachableBranchError
22
16
  from fractal_server.logger import set_logger
23
17
  from fractal_server.runner.config import JobRunnerConfigLocal
24
18
  from fractal_server.runner.config import JobRunnerConfigSLURM
19
+ from fractal_server.runner.exceptions import JobExecutionError
20
+ from fractal_server.runner.exceptions import TaskOutputValidationError
25
21
  from fractal_server.runner.executors.base_runner import BaseRunner
26
22
  from fractal_server.runner.executors.slurm_common.slurm_config import (
27
23
  SlurmConfig,
28
24
  )
29
- from fractal_server.runner.task_files import enrich_task_files_multisubmit
30
25
  from fractal_server.runner.task_files import SUBMIT_PREFIX
31
26
  from fractal_server.runner.task_files import TaskFiles
32
- from fractal_server.runner.v2.db_tools import (
33
- bulk_update_status_of_history_unit,
34
- )
27
+ from fractal_server.runner.task_files import enrich_task_files_multisubmit
28
+ from fractal_server.runner.v2.db_tools import bulk_update_status_of_history_unit
35
29
  from fractal_server.runner.v2.db_tools import bulk_upsert_image_cache_fast
36
30
  from fractal_server.runner.v2.task_interface import (
37
31
  _cast_and_validate_InitTaskOutput,
@@ -40,6 +34,11 @@ from fractal_server.runner.v2.task_interface import (
40
34
  _cast_and_validate_TaskOutput,
41
35
  )
42
36
 
37
+ from .db_tools import update_status_of_history_unit
38
+ from .deduplicate_list import deduplicate_list
39
+ from .task_interface import InitTaskOutput
40
+ from .task_interface import TaskOutput
41
+
43
42
 
44
43
  class GetRunnerConfigTypeLocal(Protocol):
45
44
  def __call__(
@@ -48,8 +47,7 @@ class GetRunnerConfigTypeLocal(Protocol):
48
47
  wftask: WorkflowTaskV2,
49
48
  which_type: Literal["non_parallel", "parallel"],
50
49
  tot_tasks: int,
51
- ) -> JobRunnerConfigLocal:
52
- ...
50
+ ) -> JobRunnerConfigLocal: ...
53
51
 
54
52
 
55
53
  class GetRunnerConfigTypeSLURM(Protocol):
@@ -59,8 +57,7 @@ class GetRunnerConfigTypeSLURM(Protocol):
59
57
  wftask: WorkflowTaskV2,
60
58
  which_type: Literal["non_parallel", "parallel"],
61
59
  tot_tasks: int,
62
- ) -> SlurmConfig:
63
- ...
60
+ ) -> SlurmConfig: ...
64
61
 
65
62
 
66
63
  GetRunnerConfigType = GetRunnerConfigTypeLocal | GetRunnerConfigTypeSLURM
@@ -5,6 +5,7 @@ This module is the single entry point to the runner backend subsystem V2.
5
5
  Other subsystems should only import this module and not its submodules or
6
6
  the individual backends.
7
7
  """
8
+
8
9
  import os
9
10
  import traceback
10
11
  from pathlib import Path
@@ -12,9 +13,6 @@ from typing import Protocol
12
13
 
13
14
  from sqlalchemy.orm import Session as DBSyncSession
14
15
 
15
- from ._local import process_workflow as local_process_workflow
16
- from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
17
- from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
18
16
  from fractal_server import __VERSION__
19
17
  from fractal_server.app.db import DB
20
18
  from fractal_server.app.models.v2 import DatasetV2
@@ -34,6 +32,10 @@ from fractal_server.types import AttributeFilters
34
32
  from fractal_server.utils import get_timestamp
35
33
  from fractal_server.zip_tools import _zip_folder_to_file_and_remove
36
34
 
35
+ from ._local import process_workflow as local_process_workflow
36
+ from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
37
+ from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
38
+
37
39
 
38
40
  class ProcessWorkflowType(Protocol):
39
41
  def __call__(
@@ -53,8 +55,7 @@ class ProcessWorkflowType(Protocol):
53
55
  resource: Resource,
54
56
  profile: Profile,
55
57
  user_cache_dir: str,
56
- ) -> None:
57
- ...
58
+ ) -> None: ...
58
59
 
59
60
 
60
61
  def fail_job(
@@ -119,7 +120,7 @@ def submit_workflow(
119
120
  Computational resource to be used for this job (e.g. a SLURM
120
121
  cluster).
121
122
  profile:
122
- Computational profile to be used for this job.
123
+ Computational profile to be used for this job.
123
124
  fractal_ssh: SSH object, for when `resource.type = "slurm_ssh"`.
124
125
  """
125
126
  # Declare runner backend and set `process_workflow` function
@@ -16,12 +16,11 @@ from invoke import UnexpectedExit
16
16
  from paramiko.ssh_exception import NoValidConnectionsError
17
17
  from pydantic import BaseModel
18
18
 
19
- from ..logger import close_logger
20
- from ..logger import get_logger
21
- from ..logger import set_logger
19
+ from fractal_server.logger import close_logger
20
+ from fractal_server.logger import get_logger
21
+ from fractal_server.logger import set_logger
22
22
  from fractal_server.string_tools import validate_cmd
23
23
 
24
-
25
24
  SSH_MONITORING_LOGGER_NAME = "ssh-log"
26
25
 
27
26
 
@@ -99,8 +98,7 @@ def _acquire_lock_with_timeout(
99
98
  if not result:
100
99
  logger.error(f"Lock for '{label}' was *not* acquired.")
101
100
  raise FractalSSHTimeoutError(
102
- f"Failed to acquire lock for '{label}' within "
103
- f"{timeout} seconds"
101
+ f"Failed to acquire lock for '{label}' within {timeout} seconds"
104
102
  )
105
103
  t_lock_acquisition = time.perf_counter()
106
104
  elapsed = t_lock_acquisition - t_lock_request
@@ -374,8 +372,6 @@ class FractalSSH:
374
372
  Args:
375
373
  cmd: Command to be run
376
374
  allow_char: Forbidden chars to allow for this command
377
- max_attempts:
378
- base_interval:
379
375
  lock_timeout:
380
376
 
381
377
  Returns:
@@ -458,9 +454,7 @@ class FractalSSH:
458
454
  logger_name=self.logger_name,
459
455
  ):
460
456
  self._sftp_unsafe().put(local, remote)
461
- self.logger.info(
462
- f"[send_file] END transfer of '{local}' over SSH."
463
- )
457
+ self.logger.info(f"[send_file] END transfer of '{local}' over SSH.")
464
458
  except Exception as e:
465
459
  self.log_and_raise(
466
460
  e=e,
@@ -766,8 +760,7 @@ class FractalSSHList:
766
760
  logger_name=self._logger_name,
767
761
  ):
768
762
  self.logger.info(
769
- f"Removing FractalSSH object for {user}@{host} "
770
- "from collection."
763
+ f"Removing FractalSSH object for {user}@{host} from collection."
771
764
  )
772
765
  fractal_ssh_obj = self._data.pop(key)
773
766
  self.logger.info(
@@ -1,6 +1,5 @@
1
1
  import string
2
2
 
3
-
4
3
  __SPECIAL_CHARACTERS__ = f"{string.punctuation}{string.whitespace}"
5
4
 
6
5
  # List of invalid characters discussed here:
fractal_server/syringe.py CHANGED
@@ -34,11 +34,11 @@ or popped from the directory.
34
34
  >>> bar()
35
35
  42
36
36
  """
37
+
37
38
  from collections.abc import Callable
38
39
  from typing import Any
39
40
  from typing import TypeVar
40
41
 
41
-
42
42
  T = TypeVar("T")
43
43
  _instance_count = 0
44
44
 
@@ -3,8 +3,8 @@ from typing import Annotated
3
3
 
4
4
  from pydantic import AfterValidator
5
5
  from pydantic import BaseModel
6
- from pydantic import model_validator
7
6
  from pydantic import PositiveInt
7
+ from pydantic import model_validator
8
8
 
9
9
  from fractal_server.types import DictStrStr
10
10
  from fractal_server.types import NonEmptyStr
@@ -14,12 +14,25 @@ class TasksPythonSettings(BaseModel):
14
14
 
15
15
  For task collection to work, there must be one or more base Python
16
16
  interpreters available on your system.
17
+
18
+ Attributes:
19
+ default_version:
20
+ Default task-collection Python version (must be a key of
21
+ `versions`).
22
+ versions:
23
+ Dictionary mapping Python versions to the corresponding
24
+ interpreters. Example:
25
+ ```json
26
+ {
27
+ "3.11": "/path/to/python3.11",
28
+ "3.13": "/path/to/python3.13"
29
+ }
30
+ ```
31
+ pip_cache_dir:
32
+ Argument for `--cache-dir` option of `pip install`, if set.
17
33
  """
18
34
 
19
35
  default_version: NonEmptyStr
20
- """
21
- Default task-collection Python version (must be a key of `versions`).
22
- """
23
36
  versions: dict[
24
37
  Literal[
25
38
  "3.9",
@@ -31,14 +44,8 @@ class TasksPythonSettings(BaseModel):
31
44
  ],
32
45
  AbsolutePathStr,
33
46
  ]
34
- """
35
- Dictionary mapping Python versions to the corresponding interpreters.
36
- """
37
47
 
38
48
  pip_cache_dir: AbsolutePathStr | None = None
39
- """
40
- Argument for `--cache-dir` option of `pip install`, if set.
41
- """
42
49
 
43
50
  @model_validator(mode="after")
44
51
  def _validate_versions(self) -> Self:
@@ -1,6 +1,5 @@
1
1
  from pathlib import Path
2
2
 
3
-
4
3
  COLLECTION_FILENAME = "collection.json"
5
4
  COLLECTION_LOG_FILENAME = "collection.log"
6
5
  COLLECTION_FREEZE_FILENAME = "collection_freeze.txt"
@@ -1,10 +1,10 @@
1
1
  from pathlib import Path
2
2
 
3
- from ..utils_pixi import simplify_pyproject_toml
4
3
  from fractal_server.app.models import Resource
5
4
  from fractal_server.app.schemas.v2 import TaskCreateV2
6
5
  from fractal_server.logger import get_logger
7
6
  from fractal_server.logger import set_logger
7
+ from fractal_server.tasks.v2.utils_pixi import simplify_pyproject_toml
8
8
  from fractal_server.tasks.v2.utils_templates import customize_template
9
9
  from fractal_server.utils import execute_command_sync
10
10
 
@@ -4,8 +4,6 @@ import time
4
4
  from pathlib import Path
5
5
  from tempfile import TemporaryDirectory
6
6
 
7
- from ..utils_database import create_db_tasks_and_update_task_group_sync
8
- from ._utils import _customize_and_run_template
9
7
  from fractal_server.app.db import get_sync_db
10
8
  from fractal_server.app.models import Profile
11
9
  from fractal_server.app.models import Resource
@@ -20,22 +18,23 @@ from fractal_server.tasks.utils import get_log_path
20
18
  from fractal_server.tasks.v2.local._utils import check_task_files_exist
21
19
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
22
20
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
23
- from fractal_server.tasks.v2.utils_background import (
24
- get_activity_and_task_group,
25
- )
21
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
26
22
  from fractal_server.tasks.v2.utils_background import get_current_log
27
23
  from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
24
+ from fractal_server.tasks.v2.utils_database import (
25
+ create_db_tasks_and_update_task_group_sync,
26
+ )
28
27
  from fractal_server.tasks.v2.utils_package_names import compare_package_names
29
28
  from fractal_server.tasks.v2.utils_python_interpreter import (
30
29
  get_python_interpreter,
31
30
  )
32
- from fractal_server.tasks.v2.utils_templates import get_collection_replacements
33
- from fractal_server.tasks.v2.utils_templates import (
34
- parse_script_pip_show_stdout,
35
- )
36
31
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
32
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
33
+ from fractal_server.tasks.v2.utils_templates import parse_script_pip_show_stdout
37
34
  from fractal_server.utils import get_timestamp
38
35
 
36
+ from ._utils import _customize_and_run_template
37
+
39
38
 
40
39
  def collect_local(
41
40
  *,
@@ -234,7 +233,7 @@ def collect_local(
234
233
  activity.log = get_current_log(log_file_path)
235
234
  activity = add_commit_refresh(obj=activity, db=db)
236
235
 
237
- logger.info("create_db_tasks_and_update_task_group - " "start")
236
+ logger.info("create_db_tasks_and_update_task_group - start")
238
237
  create_db_tasks_and_update_task_group_sync(
239
238
  task_list=task_list,
240
239
  task_group_id=task_group.id,
@@ -274,8 +273,7 @@ def collect_local(
274
273
  logger.info(f"Deleted folder {task_group.path}")
275
274
  except Exception as rm_e:
276
275
  logger.error(
277
- "Removing folder failed.\n"
278
- f"Original error:\n{str(rm_e)}"
276
+ f"Removing folder failed.\nOriginal error:\n{str(rm_e)}"
279
277
  )
280
278
 
281
279
  fail_and_cleanup(
@@ -4,10 +4,6 @@ import time
4
4
  from pathlib import Path
5
5
  from tempfile import TemporaryDirectory
6
6
 
7
- from ..utils_database import create_db_tasks_and_update_task_group_sync
8
- from ..utils_pixi import parse_collect_stdout
9
- from ..utils_pixi import SOURCE_DIR_NAME
10
- from ._utils import edit_pyproject_toml_in_place_local
11
7
  from fractal_server.app.db import get_sync_db
12
8
  from fractal_server.app.models import Profile
13
9
  from fractal_server.app.models import Resource
@@ -22,15 +18,20 @@ from fractal_server.tasks.v2.local._utils import _customize_and_run_template
22
18
  from fractal_server.tasks.v2.local._utils import check_task_files_exist
23
19
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
24
20
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
25
- from fractal_server.tasks.v2.utils_background import (
26
- get_activity_and_task_group,
27
- )
21
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
28
22
  from fractal_server.tasks.v2.utils_background import get_current_log
29
23
  from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
24
+ from fractal_server.tasks.v2.utils_database import (
25
+ create_db_tasks_and_update_task_group_sync,
26
+ )
27
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
28
+ from fractal_server.tasks.v2.utils_pixi import parse_collect_stdout
30
29
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
31
30
  from fractal_server.utils import execute_command_sync
32
31
  from fractal_server.utils import get_timestamp
33
32
 
33
+ from ._utils import edit_pyproject_toml_in_place_local
34
+
34
35
 
35
36
  def collect_local_pixi(
36
37
  *,
@@ -177,9 +178,7 @@ def collect_local_pixi(
177
178
  package_root = parsed_output["package_root"]
178
179
  venv_size = parsed_output["venv_size"]
179
180
  venv_file_number = parsed_output["venv_file_number"]
180
- project_python_wrapper = parsed_output[
181
- "project_python_wrapper"
182
- ]
181
+ project_python_wrapper = parsed_output["project_python_wrapper"]
183
182
 
184
183
  # Make task folder 755
185
184
  command = f"chmod -R 755 {source_dir}"