fractal-server 2.17.1a1__py3-none-any.whl → 2.18.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +21 -19
  3. fractal_server/app/db/__init__.py +3 -3
  4. fractal_server/app/models/__init__.py +1 -0
  5. fractal_server/app/models/linkuserproject.py +43 -1
  6. fractal_server/app/models/security.py +28 -8
  7. fractal_server/app/models/v2/__init__.py +3 -1
  8. fractal_server/app/models/v2/accounting.py +9 -1
  9. fractal_server/app/models/v2/dataset.py +5 -1
  10. fractal_server/app/models/v2/history.py +15 -1
  11. fractal_server/app/models/v2/job.py +17 -2
  12. fractal_server/app/models/v2/profile.py +29 -0
  13. fractal_server/app/models/v2/project.py +4 -10
  14. fractal_server/app/models/v2/resource.py +17 -0
  15. fractal_server/app/models/v2/task_group.py +4 -3
  16. fractal_server/app/models/v2/workflow.py +2 -1
  17. fractal_server/app/routes/admin/v2/__init__.py +12 -13
  18. fractal_server/app/routes/admin/v2/accounting.py +3 -3
  19. fractal_server/app/routes/admin/v2/job.py +35 -24
  20. fractal_server/app/routes/admin/v2/profile.py +3 -2
  21. fractal_server/app/routes/admin/v2/resource.py +5 -5
  22. fractal_server/app/routes/admin/v2/sharing.py +103 -0
  23. fractal_server/app/routes/admin/v2/task.py +37 -26
  24. fractal_server/app/routes/admin/v2/task_group.py +94 -17
  25. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +21 -22
  26. fractal_server/app/routes/api/__init__.py +1 -9
  27. fractal_server/app/routes/api/v2/__init__.py +49 -50
  28. fractal_server/app/routes/api/v2/_aux_functions.py +132 -124
  29. fractal_server/app/routes/api/v2/_aux_functions_history.py +51 -23
  30. fractal_server/app/routes/api/v2/_aux_functions_sharing.py +97 -0
  31. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +6 -8
  32. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +7 -9
  33. fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
  34. fractal_server/app/routes/api/v2/dataset.py +95 -102
  35. fractal_server/app/routes/api/v2/history.py +59 -33
  36. fractal_server/app/routes/api/v2/images.py +24 -9
  37. fractal_server/app/routes/api/v2/job.py +52 -33
  38. fractal_server/app/routes/api/v2/pre_submission_checks.py +16 -8
  39. fractal_server/app/routes/api/v2/project.py +65 -37
  40. fractal_server/app/routes/api/v2/sharing.py +311 -0
  41. fractal_server/app/routes/api/v2/status_legacy.py +31 -41
  42. fractal_server/app/routes/api/v2/submit.py +82 -78
  43. fractal_server/app/routes/api/v2/task.py +19 -20
  44. fractal_server/app/routes/api/v2/task_collection.py +41 -43
  45. fractal_server/app/routes/api/v2/task_collection_custom.py +19 -20
  46. fractal_server/app/routes/api/v2/task_collection_pixi.py +10 -11
  47. fractal_server/app/routes/api/v2/task_group.py +25 -24
  48. fractal_server/app/routes/api/v2/task_group_lifecycle.py +32 -32
  49. fractal_server/app/routes/api/v2/task_version_update.py +23 -19
  50. fractal_server/app/routes/api/v2/workflow.py +50 -55
  51. fractal_server/app/routes/api/v2/workflow_import.py +37 -37
  52. fractal_server/app/routes/api/v2/workflowtask.py +32 -26
  53. fractal_server/app/routes/auth/__init__.py +1 -3
  54. fractal_server/app/routes/auth/_aux_auth.py +101 -2
  55. fractal_server/app/routes/auth/current_user.py +2 -66
  56. fractal_server/app/routes/auth/group.py +8 -35
  57. fractal_server/app/routes/auth/login.py +1 -0
  58. fractal_server/app/routes/auth/oauth.py +4 -3
  59. fractal_server/app/routes/auth/register.py +4 -2
  60. fractal_server/app/routes/auth/router.py +2 -0
  61. fractal_server/app/routes/auth/users.py +19 -10
  62. fractal_server/app/routes/auth/viewer_paths.py +43 -0
  63. fractal_server/app/routes/aux/_job.py +1 -1
  64. fractal_server/app/routes/aux/_runner.py +2 -2
  65. fractal_server/app/routes/pagination.py +1 -1
  66. fractal_server/app/schemas/user.py +29 -12
  67. fractal_server/app/schemas/user_group.py +0 -15
  68. fractal_server/app/schemas/v2/__init__.py +55 -48
  69. fractal_server/app/schemas/v2/accounting.py +11 -0
  70. fractal_server/app/schemas/v2/dataset.py +57 -11
  71. fractal_server/app/schemas/v2/dumps.py +10 -9
  72. fractal_server/app/schemas/v2/job.py +11 -11
  73. fractal_server/app/schemas/v2/manifest.py +4 -3
  74. fractal_server/app/schemas/v2/profile.py +53 -2
  75. fractal_server/app/schemas/v2/project.py +3 -3
  76. fractal_server/app/schemas/v2/resource.py +121 -16
  77. fractal_server/app/schemas/v2/sharing.py +99 -0
  78. fractal_server/app/schemas/v2/status_legacy.py +3 -3
  79. fractal_server/app/schemas/v2/task.py +6 -7
  80. fractal_server/app/schemas/v2/task_collection.py +5 -5
  81. fractal_server/app/schemas/v2/task_group.py +16 -16
  82. fractal_server/app/schemas/v2/workflow.py +16 -16
  83. fractal_server/app/schemas/v2/workflowtask.py +16 -15
  84. fractal_server/app/security/__init__.py +5 -8
  85. fractal_server/app/security/signup_email.py +4 -5
  86. fractal_server/app/shutdown.py +6 -6
  87. fractal_server/config/__init__.py +0 -6
  88. fractal_server/config/_data.py +0 -68
  89. fractal_server/config/_database.py +19 -20
  90. fractal_server/config/_email.py +30 -38
  91. fractal_server/config/_main.py +38 -52
  92. fractal_server/config/_oauth.py +17 -21
  93. fractal_server/data_migrations/2_18_0.py +30 -0
  94. fractal_server/exceptions.py +4 -0
  95. fractal_server/images/models.py +4 -5
  96. fractal_server/images/status_tools.py +4 -2
  97. fractal_server/logger.py +1 -1
  98. fractal_server/main.py +75 -13
  99. fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
  100. fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
  101. fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
  102. fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
  103. fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
  104. fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
  105. fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
  106. fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
  107. fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
  108. fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
  109. fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
  110. fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
  111. fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
  112. fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
  113. fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
  114. fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
  115. fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
  116. fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
  117. fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
  118. fractal_server/migrations/versions/7910eed4cf97_user_project_dirs_and_usergroup_viewer_.py +60 -0
  119. fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
  120. fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
  121. fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
  122. fractal_server/migrations/versions/88270f589c9b_add_prevent_new_submissions.py +39 -0
  123. fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
  124. fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
  125. fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
  126. fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
  127. fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
  128. fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
  129. fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
  130. fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
  131. fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
  132. fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
  133. fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
  134. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
  135. fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
  136. fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
  137. fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
  138. fractal_server/migrations/versions/bc0e8b3327a7_project_sharing.py +72 -0
  139. fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
  140. fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
  141. fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
  142. fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
  143. fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
  144. fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
  145. fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
  146. fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
  147. fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
  148. fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
  149. fractal_server/migrations/versions/f0702066b007_one_submitted_job_per_dataset.py +40 -0
  150. fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
  151. fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
  152. fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
  153. fractal_server/runner/config/_local.py +8 -5
  154. fractal_server/runner/config/_slurm.py +39 -33
  155. fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
  156. fractal_server/runner/executors/base_runner.py +29 -4
  157. fractal_server/runner/executors/local/get_local_config.py +1 -0
  158. fractal_server/runner/executors/local/runner.py +14 -13
  159. fractal_server/runner/executors/slurm_common/_batching.py +9 -20
  160. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
  161. fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
  162. fractal_server/runner/executors/slurm_common/remote.py +3 -1
  163. fractal_server/runner/executors/slurm_common/slurm_config.py +2 -0
  164. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
  165. fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
  166. fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
  167. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
  168. fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
  169. fractal_server/runner/task_files.py +9 -3
  170. fractal_server/runner/v2/_local.py +12 -6
  171. fractal_server/runner/v2/_slurm_ssh.py +14 -7
  172. fractal_server/runner/v2/_slurm_sudo.py +14 -7
  173. fractal_server/runner/v2/db_tools.py +0 -1
  174. fractal_server/runner/v2/deduplicate_list.py +2 -1
  175. fractal_server/runner/v2/runner.py +44 -28
  176. fractal_server/runner/v2/runner_functions.py +22 -28
  177. fractal_server/runner/v2/submit_workflow.py +29 -15
  178. fractal_server/ssh/_fabric.py +6 -13
  179. fractal_server/string_tools.py +0 -1
  180. fractal_server/syringe.py +1 -1
  181. fractal_server/tasks/config/_pixi.py +1 -1
  182. fractal_server/tasks/config/_python.py +16 -9
  183. fractal_server/tasks/utils.py +0 -1
  184. fractal_server/tasks/v2/local/_utils.py +3 -3
  185. fractal_server/tasks/v2/local/collect.py +15 -18
  186. fractal_server/tasks/v2/local/collect_pixi.py +14 -16
  187. fractal_server/tasks/v2/local/deactivate.py +14 -15
  188. fractal_server/tasks/v2/local/deactivate_pixi.py +7 -7
  189. fractal_server/tasks/v2/local/delete.py +6 -8
  190. fractal_server/tasks/v2/local/reactivate.py +12 -12
  191. fractal_server/tasks/v2/local/reactivate_pixi.py +12 -12
  192. fractal_server/tasks/v2/ssh/_utils.py +3 -3
  193. fractal_server/tasks/v2/ssh/collect.py +19 -24
  194. fractal_server/tasks/v2/ssh/collect_pixi.py +22 -24
  195. fractal_server/tasks/v2/ssh/deactivate.py +17 -15
  196. fractal_server/tasks/v2/ssh/deactivate_pixi.py +8 -7
  197. fractal_server/tasks/v2/ssh/delete.py +12 -10
  198. fractal_server/tasks/v2/ssh/reactivate.py +16 -16
  199. fractal_server/tasks/v2/ssh/reactivate_pixi.py +13 -14
  200. fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
  201. fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
  202. fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
  203. fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
  204. fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
  205. fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
  206. fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
  207. fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
  208. fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
  209. fractal_server/tasks/v2/utils_background.py +10 -10
  210. fractal_server/tasks/v2/utils_database.py +5 -5
  211. fractal_server/tasks/v2/utils_package_names.py +1 -2
  212. fractal_server/tasks/v2/utils_pixi.py +1 -3
  213. fractal_server/types/__init__.py +98 -1
  214. fractal_server/types/validators/__init__.py +3 -0
  215. fractal_server/types/validators/_common_validators.py +33 -3
  216. fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
  217. fractal_server/utils.py +1 -0
  218. fractal_server/zip_tools.py +34 -0
  219. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/METADATA +3 -2
  220. fractal_server-2.18.0.dist-info/RECORD +275 -0
  221. fractal_server/app/routes/admin/v2/project.py +0 -41
  222. fractal_server-2.17.1a1.dist-info/RECORD +0 -264
  223. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/WHEEL +0 -0
  224. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/entry_points.txt +0 -0
  225. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/licenses/LICENSE +0 -0
@@ -5,6 +5,7 @@ This module is the single entry point to the runner backend subsystem V2.
5
5
  Other subsystems should only import this module and not its submodules or
6
6
  the individual backends.
7
7
  """
8
+
8
9
  import os
9
10
  import traceback
10
11
  from pathlib import Path
@@ -12,9 +13,6 @@ from typing import Protocol
12
13
 
13
14
  from sqlalchemy.orm import Session as DBSyncSession
14
15
 
15
- from ._local import process_workflow as local_process_workflow
16
- from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
17
- from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
18
16
  from fractal_server import __VERSION__
19
17
  from fractal_server.app.db import DB
20
18
  from fractal_server.app.models.v2 import DatasetV2
@@ -22,7 +20,7 @@ from fractal_server.app.models.v2 import JobV2
22
20
  from fractal_server.app.models.v2 import Profile
23
21
  from fractal_server.app.models.v2 import Resource
24
22
  from fractal_server.app.models.v2 import WorkflowV2
25
- from fractal_server.app.schemas.v2 import JobStatusTypeV2
23
+ from fractal_server.app.schemas.v2 import JobStatusType
26
24
  from fractal_server.app.schemas.v2 import ResourceType
27
25
  from fractal_server.logger import get_logger
28
26
  from fractal_server.logger import reset_logger_handlers
@@ -34,6 +32,10 @@ from fractal_server.types import AttributeFilters
34
32
  from fractal_server.utils import get_timestamp
35
33
  from fractal_server.zip_tools import _zip_folder_to_file_and_remove
36
34
 
35
+ from ._local import process_workflow as local_process_workflow
36
+ from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
37
+ from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
38
+
37
39
 
38
40
  class ProcessWorkflowType(Protocol):
39
41
  def __call__(
@@ -53,8 +55,7 @@ class ProcessWorkflowType(Protocol):
53
55
  resource: Resource,
54
56
  profile: Profile,
55
57
  user_cache_dir: str,
56
- ) -> None:
57
- ...
58
+ ) -> None: ...
58
59
 
59
60
 
60
61
  def fail_job(
@@ -70,7 +71,7 @@ def fail_job(
70
71
  logger.error(log_msg)
71
72
  reset_logger_handlers(logger)
72
73
  job = db.get(JobV2, job.id) # refetch, in case it was updated
73
- job.status = JobStatusTypeV2.FAILED
74
+ job.status = JobStatusType.FAILED
74
75
  job.end_timestamp = get_timestamp()
75
76
  job.log = log_msg
76
77
  db.merge(job)
@@ -119,7 +120,7 @@ def submit_workflow(
119
120
  Computational resource to be used for this job (e.g. a SLURM
120
121
  cluster).
121
122
  profile:
122
- Computational profile to be used for this job.
123
+ Computational profile to be used for this job.
123
124
  fractal_ssh: SSH object, for when `resource.type = "slurm_ssh"`.
124
125
  """
125
126
  # Declare runner backend and set `process_workflow` function
@@ -144,14 +145,27 @@ def submit_workflow(
144
145
  return
145
146
  if dataset is None or workflow is None:
146
147
  log_msg = ""
147
- if not dataset:
148
- log_msg += f"Cannot fetch dataset {dataset_id} from database\n"
149
- if not workflow:
150
- log_msg += (
151
- f"Cannot fetch workflow {workflow_id} from database\n"
148
+ if dataset is None:
149
+ current_log_msg = (
150
+ f"Cannot fetch dataset {dataset_id} from database "
151
+ f"(as part of job {job_id})."
152
+ )
153
+ logger.error(current_log_msg)
154
+ log_msg += f"{current_log_msg}\n"
155
+ if workflow is None:
156
+ current_log_msg += (
157
+ f"Cannot fetch workflow {workflow_id} from database "
158
+ f"(as part of job {job_id})."
152
159
  )
160
+ logger.error(current_log_msg)
161
+ log_msg += f"{current_log_msg}\n"
162
+
153
163
  fail_job(
154
- db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
164
+ db=db_sync,
165
+ job=job,
166
+ log_msg=log_msg,
167
+ logger_name=logger_name,
168
+ emit_log=False,
155
169
  )
156
170
  return
157
171
 
@@ -272,7 +286,7 @@ def submit_workflow(
272
286
  # Update job DB entry
273
287
  with next(DB.get_sync_db()) as db_sync:
274
288
  job = db_sync.get(JobV2, job_id)
275
- job.status = JobStatusTypeV2.DONE
289
+ job.status = JobStatusType.DONE
276
290
  job.end_timestamp = get_timestamp()
277
291
  with log_file_path.open("r") as f:
278
292
  logs = f.read()
@@ -16,12 +16,11 @@ from invoke import UnexpectedExit
16
16
  from paramiko.ssh_exception import NoValidConnectionsError
17
17
  from pydantic import BaseModel
18
18
 
19
- from ..logger import close_logger
20
- from ..logger import get_logger
21
- from ..logger import set_logger
19
+ from fractal_server.logger import close_logger
20
+ from fractal_server.logger import get_logger
21
+ from fractal_server.logger import set_logger
22
22
  from fractal_server.string_tools import validate_cmd
23
23
 
24
-
25
24
  SSH_MONITORING_LOGGER_NAME = "ssh-log"
26
25
 
27
26
 
@@ -99,8 +98,7 @@ def _acquire_lock_with_timeout(
99
98
  if not result:
100
99
  logger.error(f"Lock for '{label}' was *not* acquired.")
101
100
  raise FractalSSHTimeoutError(
102
- f"Failed to acquire lock for '{label}' within "
103
- f"{timeout} seconds"
101
+ f"Failed to acquire lock for '{label}' within {timeout} seconds"
104
102
  )
105
103
  t_lock_acquisition = time.perf_counter()
106
104
  elapsed = t_lock_acquisition - t_lock_request
@@ -374,8 +372,6 @@ class FractalSSH:
374
372
  Args:
375
373
  cmd: Command to be run
376
374
  allow_char: Forbidden chars to allow for this command
377
- max_attempts:
378
- base_interval:
379
375
  lock_timeout:
380
376
 
381
377
  Returns:
@@ -458,9 +454,7 @@ class FractalSSH:
458
454
  logger_name=self.logger_name,
459
455
  ):
460
456
  self._sftp_unsafe().put(local, remote)
461
- self.logger.info(
462
- f"[send_file] END transfer of '{local}' over SSH."
463
- )
457
+ self.logger.info(f"[send_file] END transfer of '{local}' over SSH.")
464
458
  except Exception as e:
465
459
  self.log_and_raise(
466
460
  e=e,
@@ -766,8 +760,7 @@ class FractalSSHList:
766
760
  logger_name=self._logger_name,
767
761
  ):
768
762
  self.logger.info(
769
- f"Removing FractalSSH object for {user}@{host} "
770
- "from collection."
763
+ f"Removing FractalSSH object for {user}@{host} from collection."
771
764
  )
772
765
  fractal_ssh_obj = self._data.pop(key)
773
766
  self.logger.info(
@@ -1,6 +1,5 @@
1
1
  import string
2
2
 
3
-
4
3
  __SPECIAL_CHARACTERS__ = f"{string.punctuation}{string.whitespace}"
5
4
 
6
5
  # List of invalid characters discussed here:
fractal_server/syringe.py CHANGED
@@ -34,11 +34,11 @@ or popped from the directory.
34
34
  >>> bar()
35
35
  42
36
36
  """
37
+
37
38
  from collections.abc import Callable
38
39
  from typing import Any
39
40
  from typing import TypeVar
40
41
 
41
-
42
42
  T = TypeVar("T")
43
43
  _instance_count = 0
44
44
 
@@ -3,8 +3,8 @@ from typing import Annotated
3
3
 
4
4
  from pydantic import AfterValidator
5
5
  from pydantic import BaseModel
6
- from pydantic import model_validator
7
6
  from pydantic import PositiveInt
7
+ from pydantic import model_validator
8
8
 
9
9
  from fractal_server.types import DictStrStr
10
10
  from fractal_server.types import NonEmptyStr
@@ -14,12 +14,25 @@ class TasksPythonSettings(BaseModel):
14
14
 
15
15
  For task collection to work, there must be one or more base Python
16
16
  interpreters available on your system.
17
+
18
+ Attributes:
19
+ default_version:
20
+ Default task-collection Python version (must be a key of
21
+ `versions`).
22
+ versions:
23
+ Dictionary mapping Python versions to the corresponding
24
+ interpreters. Example:
25
+ ```json
26
+ {
27
+ "3.11": "/path/to/python3.11",
28
+ "3.13": "/path/to/python3.13"
29
+ }
30
+ ```
31
+ pip_cache_dir:
32
+ Argument for `--cache-dir` option of `pip install`, if set.
17
33
  """
18
34
 
19
35
  default_version: NonEmptyStr
20
- """
21
- Default task-collection Python version (must be a key of `versions`).
22
- """
23
36
  versions: dict[
24
37
  Literal[
25
38
  "3.9",
@@ -31,14 +44,8 @@ class TasksPythonSettings(BaseModel):
31
44
  ],
32
45
  AbsolutePathStr,
33
46
  ]
34
- """
35
- Dictionary mapping Python versions to the corresponding interpreters.
36
- """
37
47
 
38
48
  pip_cache_dir: AbsolutePathStr | None = None
39
- """
40
- Argument for `--cache-dir` option of `pip install`, if set.
41
- """
42
49
 
43
50
  @model_validator(mode="after")
44
51
  def _validate_versions(self) -> Self:
@@ -1,6 +1,5 @@
1
1
  from pathlib import Path
2
2
 
3
-
4
3
  COLLECTION_FILENAME = "collection.json"
5
4
  COLLECTION_LOG_FILENAME = "collection.log"
6
5
  COLLECTION_FREEZE_FILENAME = "collection_freeze.txt"
@@ -1,10 +1,10 @@
1
1
  from pathlib import Path
2
2
 
3
- from ..utils_pixi import simplify_pyproject_toml
4
3
  from fractal_server.app.models import Resource
5
- from fractal_server.app.schemas.v2 import TaskCreateV2
4
+ from fractal_server.app.schemas.v2 import TaskCreate
6
5
  from fractal_server.logger import get_logger
7
6
  from fractal_server.logger import set_logger
7
+ from fractal_server.tasks.v2.utils_pixi import simplify_pyproject_toml
8
8
  from fractal_server.tasks.v2.utils_templates import customize_template
9
9
  from fractal_server.utils import execute_command_sync
10
10
 
@@ -51,7 +51,7 @@ def _customize_and_run_template(
51
51
  return stdout
52
52
 
53
53
 
54
- def check_task_files_exist(task_list: list[TaskCreateV2]) -> None:
54
+ def check_task_files_exist(task_list: list[TaskCreate]) -> None:
55
55
  """
56
56
  Check that the modules listed in task commands point to existing files.
57
57
 
@@ -4,15 +4,13 @@ import time
4
4
  from pathlib import Path
5
5
  from tempfile import TemporaryDirectory
6
6
 
7
- from ..utils_database import create_db_tasks_and_update_task_group_sync
8
- from ._utils import _customize_and_run_template
9
7
  from fractal_server.app.db import get_sync_db
10
8
  from fractal_server.app.models import Profile
11
9
  from fractal_server.app.models import Resource
12
10
  from fractal_server.app.models.v2 import TaskGroupV2
13
11
  from fractal_server.app.schemas.v2 import FractalUploadedFile
14
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
15
- from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
12
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
13
+ from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
16
14
  from fractal_server.app.schemas.v2.manifest import ManifestV2
17
15
  from fractal_server.logger import reset_logger_handlers
18
16
  from fractal_server.logger import set_logger
@@ -20,22 +18,23 @@ from fractal_server.tasks.utils import get_log_path
20
18
  from fractal_server.tasks.v2.local._utils import check_task_files_exist
21
19
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
22
20
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
23
- from fractal_server.tasks.v2.utils_background import (
24
- get_activity_and_task_group,
25
- )
21
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
26
22
  from fractal_server.tasks.v2.utils_background import get_current_log
27
23
  from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
24
+ from fractal_server.tasks.v2.utils_database import (
25
+ create_db_tasks_and_update_task_group_sync,
26
+ )
28
27
  from fractal_server.tasks.v2.utils_package_names import compare_package_names
29
28
  from fractal_server.tasks.v2.utils_python_interpreter import (
30
29
  get_python_interpreter,
31
30
  )
32
- from fractal_server.tasks.v2.utils_templates import get_collection_replacements
33
- from fractal_server.tasks.v2.utils_templates import (
34
- parse_script_pip_show_stdout,
35
- )
36
31
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
32
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
33
+ from fractal_server.tasks.v2.utils_templates import parse_script_pip_show_stdout
37
34
  from fractal_server.utils import get_timestamp
38
35
 
36
+ from ._utils import _customize_and_run_template
37
+
39
38
 
40
39
  def collect_local(
41
40
  *,
@@ -133,14 +132,13 @@ def collect_local(
133
132
  Path(task_group.path) / SCRIPTS_SUBFOLDER
134
133
  ).as_posix(),
135
134
  prefix=(
136
- f"{int(time.time())}_"
137
- f"{TaskGroupActivityActionV2.COLLECT}"
135
+ f"{int(time.time())}_{TaskGroupActivityAction.COLLECT}"
138
136
  ),
139
137
  logger_name=LOGGER_NAME,
140
138
  )
141
139
 
142
140
  # Set status to ONGOING and refresh logs
143
- activity.status = TaskGroupActivityStatusV2.ONGOING
141
+ activity.status = TaskGroupActivityStatus.ONGOING
144
142
  activity.log = get_current_log(log_file_path)
145
143
  activity = add_commit_refresh(obj=activity, db=db)
146
144
 
@@ -234,7 +232,7 @@ def collect_local(
234
232
  activity.log = get_current_log(log_file_path)
235
233
  activity = add_commit_refresh(obj=activity, db=db)
236
234
 
237
- logger.info("create_db_tasks_and_update_task_group - " "start")
235
+ logger.info("create_db_tasks_and_update_task_group - start")
238
236
  create_db_tasks_and_update_task_group_sync(
239
237
  task_list=task_list,
240
238
  task_group_id=task_group.id,
@@ -258,7 +256,7 @@ def collect_local(
258
256
 
259
257
  # Finalize (write metadata to DB)
260
258
  logger.info("finalising - START")
261
- activity.status = TaskGroupActivityStatusV2.OK
259
+ activity.status = TaskGroupActivityStatus.OK
262
260
  activity.timestamp_ended = get_timestamp()
263
261
  activity = add_commit_refresh(obj=activity, db=db)
264
262
  logger.info("finalising - END")
@@ -274,8 +272,7 @@ def collect_local(
274
272
  logger.info(f"Deleted folder {task_group.path}")
275
273
  except Exception as rm_e:
276
274
  logger.error(
277
- "Removing folder failed.\n"
278
- f"Original error:\n{str(rm_e)}"
275
+ f"Removing folder failed.\nOriginal error:\n{str(rm_e)}"
279
276
  )
280
277
 
281
278
  fail_and_cleanup(
@@ -4,16 +4,12 @@ import time
4
4
  from pathlib import Path
5
5
  from tempfile import TemporaryDirectory
6
6
 
7
- from ..utils_database import create_db_tasks_and_update_task_group_sync
8
- from ..utils_pixi import parse_collect_stdout
9
- from ..utils_pixi import SOURCE_DIR_NAME
10
- from ._utils import edit_pyproject_toml_in_place_local
11
7
  from fractal_server.app.db import get_sync_db
12
8
  from fractal_server.app.models import Profile
13
9
  from fractal_server.app.models import Resource
14
10
  from fractal_server.app.schemas.v2 import FractalUploadedFile
15
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
16
- from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
11
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
12
+ from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
17
13
  from fractal_server.app.schemas.v2.manifest import ManifestV2
18
14
  from fractal_server.logger import reset_logger_handlers
19
15
  from fractal_server.logger import set_logger
@@ -22,15 +18,20 @@ from fractal_server.tasks.v2.local._utils import _customize_and_run_template
22
18
  from fractal_server.tasks.v2.local._utils import check_task_files_exist
23
19
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
24
20
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
25
- from fractal_server.tasks.v2.utils_background import (
26
- get_activity_and_task_group,
27
- )
21
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
28
22
  from fractal_server.tasks.v2.utils_background import get_current_log
29
23
  from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
24
+ from fractal_server.tasks.v2.utils_database import (
25
+ create_db_tasks_and_update_task_group_sync,
26
+ )
27
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
28
+ from fractal_server.tasks.v2.utils_pixi import parse_collect_stdout
30
29
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
31
30
  from fractal_server.utils import execute_command_sync
32
31
  from fractal_server.utils import get_timestamp
33
32
 
33
+ from ._utils import edit_pyproject_toml_in_place_local
34
+
34
35
 
35
36
  def collect_local_pixi(
36
37
  *,
@@ -131,13 +132,12 @@ def collect_local_pixi(
131
132
  task_group.path, SCRIPTS_SUBFOLDER
132
133
  ).as_posix(),
133
134
  prefix=(
134
- f"{int(time.time())}_"
135
- f"{TaskGroupActivityActionV2.COLLECT}"
135
+ f"{int(time.time())}_{TaskGroupActivityAction.COLLECT}"
136
136
  ),
137
137
  logger_name=LOGGER_NAME,
138
138
  )
139
139
 
140
- activity.status = TaskGroupActivityStatusV2.ONGOING
140
+ activity.status = TaskGroupActivityStatus.ONGOING
141
141
  activity.log = get_current_log(log_file_path)
142
142
  activity = add_commit_refresh(obj=activity, db=db)
143
143
 
@@ -177,9 +177,7 @@ def collect_local_pixi(
177
177
  package_root = parsed_output["package_root"]
178
178
  venv_size = parsed_output["venv_size"]
179
179
  venv_file_number = parsed_output["venv_file_number"]
180
- project_python_wrapper = parsed_output[
181
- "project_python_wrapper"
182
- ]
180
+ project_python_wrapper = parsed_output["project_python_wrapper"]
183
181
 
184
182
  # Make task folder 755
185
183
  command = f"chmod -R 755 {source_dir}"
@@ -242,7 +240,7 @@ def collect_local_pixi(
242
240
 
243
241
  # Finalize (write metadata to DB)
244
242
  logger.info("finalising - START")
245
- activity.status = TaskGroupActivityStatusV2.OK
243
+ activity.status = TaskGroupActivityStatus.OK
246
244
  activity.timestamp_ended = get_timestamp()
247
245
  activity = add_commit_refresh(obj=activity, db=db)
248
246
  logger.info("finalising - END")
@@ -3,25 +3,26 @@ import time
3
3
  from pathlib import Path
4
4
  from tempfile import TemporaryDirectory
5
5
 
6
- from ..utils_background import add_commit_refresh
7
- from ..utils_background import fail_and_cleanup
8
- from ..utils_background import get_activity_and_task_group
9
- from ..utils_templates import get_collection_replacements
10
- from ._utils import _customize_and_run_template
11
6
  from fractal_server.app.db import get_sync_db
12
7
  from fractal_server.app.models import Profile
13
8
  from fractal_server.app.models import Resource
14
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
15
- from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
16
- from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
9
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
10
+ from fractal_server.app.schemas.v2 import TaskGroupOriginEnum
11
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatus
17
12
  from fractal_server.logger import reset_logger_handlers
18
13
  from fractal_server.logger import set_logger
19
14
  from fractal_server.tasks.utils import FORBIDDEN_DEPENDENCY_STRINGS
20
15
  from fractal_server.tasks.utils import get_log_path
16
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
17
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
18
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
21
19
  from fractal_server.tasks.v2.utils_background import get_current_log
22
20
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
21
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
23
22
  from fractal_server.utils import get_timestamp
24
23
 
24
+ from ._utils import _customize_and_run_template
25
+
25
26
 
26
27
  def deactivate_local(
27
28
  *,
@@ -77,7 +78,7 @@ def deactivate_local(
77
78
  return
78
79
 
79
80
  try:
80
- activity.status = TaskGroupActivityStatusV2.ONGOING
81
+ activity.status = TaskGroupActivityStatus.ONGOING
81
82
  activity = add_commit_refresh(obj=activity, db=db)
82
83
 
83
84
  if task_group.env_info is None:
@@ -101,7 +102,7 @@ def deactivate_local(
101
102
  ).as_posix(),
102
103
  prefix=(
103
104
  f"{int(time.time())}_"
104
- f"{TaskGroupActivityActionV2.DEACTIVATE}"
105
+ f"{TaskGroupActivityAction.DEACTIVATE}"
105
106
  ),
106
107
  logger_name=LOGGER_NAME,
107
108
  )
@@ -119,7 +120,7 @@ def deactivate_local(
119
120
  logger.info("Add pip freeze stdout to TaskGroupV2 - end")
120
121
 
121
122
  # Handle some specific cases for wheel-file case
122
- if task_group.origin == TaskGroupV2OriginEnum.WHEELFILE:
123
+ if task_group.origin == TaskGroupOriginEnum.WHEELFILE:
123
124
  logger.info(
124
125
  f"Handle specific cases for {task_group.origin=}."
125
126
  )
@@ -172,9 +173,7 @@ def deactivate_local(
172
173
  / Path(task_group.archive_path).name
173
174
  ).as_posix()
174
175
  shutil.copy(task_group.archive_path, new_archive_path)
175
- logger.info(
176
- f"Copied wheel file to {new_archive_path}."
177
- )
176
+ logger.info(f"Copied wheel file to {new_archive_path}.")
178
177
 
179
178
  task_group.archive_path = new_archive_path
180
179
  new_pip_freeze = task_group.env_info.replace(
@@ -210,7 +209,7 @@ def deactivate_local(
210
209
  logger.info(f"Now removing {task_group.venv_path}.")
211
210
  shutil.rmtree(task_group.venv_path)
212
211
  logger.info(f"All good, {task_group.venv_path} removed.")
213
- activity.status = TaskGroupActivityStatusV2.OK
212
+ activity.status = TaskGroupActivityStatus.OK
214
213
  activity.log = get_current_log(log_file_path)
215
214
  activity.timestamp_ended = get_timestamp()
216
215
  activity = add_commit_refresh(obj=activity, db=db)
@@ -2,18 +2,18 @@ import shutil
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import add_commit_refresh
6
- from ..utils_background import fail_and_cleanup
7
- from ..utils_background import get_activity_and_task_group
8
- from ..utils_pixi import SOURCE_DIR_NAME
9
5
  from fractal_server.app.db import get_sync_db
10
6
  from fractal_server.app.models import Profile
11
7
  from fractal_server.app.models import Resource
12
- from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
8
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatus
13
9
  from fractal_server.logger import reset_logger_handlers
14
10
  from fractal_server.logger import set_logger
15
11
  from fractal_server.tasks.utils import get_log_path
12
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
13
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
14
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
16
15
  from fractal_server.tasks.v2.utils_background import get_current_log
16
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
17
17
  from fractal_server.utils import get_timestamp
18
18
 
19
19
 
@@ -70,7 +70,7 @@ def deactivate_local_pixi(
70
70
  return
71
71
 
72
72
  try:
73
- activity.status = TaskGroupActivityStatusV2.ONGOING
73
+ activity.status = TaskGroupActivityStatus.ONGOING
74
74
  activity = add_commit_refresh(obj=activity, db=db)
75
75
 
76
76
  # Actually mark the task group as non-active
@@ -82,7 +82,7 @@ def deactivate_local_pixi(
82
82
  logger.info(f"Now removing '{source_dir.as_posix()}'.")
83
83
  shutil.rmtree(source_dir)
84
84
  logger.info(f"All good, '{source_dir.as_posix()}' removed.")
85
- activity.status = TaskGroupActivityStatusV2.OK
85
+ activity.status = TaskGroupActivityStatus.OK
86
86
  activity.log = get_current_log(log_file_path)
87
87
  activity.timestamp_ended = get_timestamp()
88
88
  activity = add_commit_refresh(obj=activity, db=db)
@@ -5,16 +5,14 @@ from tempfile import TemporaryDirectory
5
5
  from fractal_server.app.db import get_sync_db
6
6
  from fractal_server.app.models import Profile
7
7
  from fractal_server.app.models import Resource
8
- from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
9
- from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
8
+ from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
9
+ from fractal_server.app.schemas.v2 import TaskGroupOriginEnum
10
10
  from fractal_server.logger import reset_logger_handlers
11
11
  from fractal_server.logger import set_logger
12
12
  from fractal_server.tasks.utils import get_log_path
13
13
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
14
14
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
15
- from fractal_server.tasks.v2.utils_background import (
16
- get_activity_and_task_group,
17
- )
15
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
18
16
  from fractal_server.tasks.v2.utils_background import get_current_log
19
17
  from fractal_server.utils import get_timestamp
20
18
 
@@ -47,7 +45,7 @@ def delete_local(
47
45
  return
48
46
 
49
47
  try:
50
- activity.status = TaskGroupActivityStatusV2.ONGOING
48
+ activity.status = TaskGroupActivityStatus.ONGOING
51
49
  activity.log = get_current_log(log_file_path)
52
50
  activity = add_commit_refresh(obj=activity, db=db)
53
51
 
@@ -55,12 +53,12 @@ def delete_local(
55
53
  db.commit()
56
54
  logger.debug("Task group removed from database.")
57
55
 
58
- if task_group.origin != TaskGroupV2OriginEnum.OTHER:
56
+ if task_group.origin != TaskGroupOriginEnum.OTHER:
59
57
  logger.debug(f"Removing {task_group.path=}.")
60
58
  shutil.rmtree(task_group.path)
61
59
  logger.debug(f"{task_group.path=} removed.")
62
60
 
63
- activity.status = TaskGroupActivityStatusV2.OK
61
+ activity.status = TaskGroupActivityStatus.OK
64
62
  activity.log = get_current_log(log_file_path)
65
63
  activity.timestamp_ended = get_timestamp()
66
64
  activity = add_commit_refresh(obj=activity, db=db)