fractal-server 2.17.1a1__py3-none-any.whl → 2.18.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +21 -19
  3. fractal_server/app/db/__init__.py +3 -3
  4. fractal_server/app/models/__init__.py +1 -0
  5. fractal_server/app/models/linkuserproject.py +43 -1
  6. fractal_server/app/models/security.py +28 -8
  7. fractal_server/app/models/v2/__init__.py +3 -1
  8. fractal_server/app/models/v2/accounting.py +9 -1
  9. fractal_server/app/models/v2/dataset.py +5 -1
  10. fractal_server/app/models/v2/history.py +15 -1
  11. fractal_server/app/models/v2/job.py +17 -2
  12. fractal_server/app/models/v2/profile.py +29 -0
  13. fractal_server/app/models/v2/project.py +4 -10
  14. fractal_server/app/models/v2/resource.py +17 -0
  15. fractal_server/app/models/v2/task_group.py +4 -3
  16. fractal_server/app/models/v2/workflow.py +2 -1
  17. fractal_server/app/routes/admin/v2/__init__.py +12 -13
  18. fractal_server/app/routes/admin/v2/accounting.py +3 -3
  19. fractal_server/app/routes/admin/v2/job.py +35 -24
  20. fractal_server/app/routes/admin/v2/profile.py +3 -2
  21. fractal_server/app/routes/admin/v2/resource.py +5 -5
  22. fractal_server/app/routes/admin/v2/sharing.py +103 -0
  23. fractal_server/app/routes/admin/v2/task.py +37 -26
  24. fractal_server/app/routes/admin/v2/task_group.py +94 -17
  25. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +21 -22
  26. fractal_server/app/routes/api/__init__.py +1 -9
  27. fractal_server/app/routes/api/v2/__init__.py +49 -50
  28. fractal_server/app/routes/api/v2/_aux_functions.py +132 -124
  29. fractal_server/app/routes/api/v2/_aux_functions_history.py +51 -23
  30. fractal_server/app/routes/api/v2/_aux_functions_sharing.py +97 -0
  31. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +6 -8
  32. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +7 -9
  33. fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
  34. fractal_server/app/routes/api/v2/dataset.py +95 -102
  35. fractal_server/app/routes/api/v2/history.py +59 -33
  36. fractal_server/app/routes/api/v2/images.py +24 -9
  37. fractal_server/app/routes/api/v2/job.py +52 -33
  38. fractal_server/app/routes/api/v2/pre_submission_checks.py +16 -8
  39. fractal_server/app/routes/api/v2/project.py +65 -37
  40. fractal_server/app/routes/api/v2/sharing.py +311 -0
  41. fractal_server/app/routes/api/v2/status_legacy.py +31 -41
  42. fractal_server/app/routes/api/v2/submit.py +82 -78
  43. fractal_server/app/routes/api/v2/task.py +19 -20
  44. fractal_server/app/routes/api/v2/task_collection.py +41 -43
  45. fractal_server/app/routes/api/v2/task_collection_custom.py +19 -20
  46. fractal_server/app/routes/api/v2/task_collection_pixi.py +10 -11
  47. fractal_server/app/routes/api/v2/task_group.py +25 -24
  48. fractal_server/app/routes/api/v2/task_group_lifecycle.py +32 -32
  49. fractal_server/app/routes/api/v2/task_version_update.py +23 -19
  50. fractal_server/app/routes/api/v2/workflow.py +50 -55
  51. fractal_server/app/routes/api/v2/workflow_import.py +37 -37
  52. fractal_server/app/routes/api/v2/workflowtask.py +32 -26
  53. fractal_server/app/routes/auth/__init__.py +1 -3
  54. fractal_server/app/routes/auth/_aux_auth.py +101 -2
  55. fractal_server/app/routes/auth/current_user.py +2 -66
  56. fractal_server/app/routes/auth/group.py +8 -35
  57. fractal_server/app/routes/auth/login.py +1 -0
  58. fractal_server/app/routes/auth/oauth.py +4 -3
  59. fractal_server/app/routes/auth/register.py +4 -2
  60. fractal_server/app/routes/auth/router.py +2 -0
  61. fractal_server/app/routes/auth/users.py +19 -10
  62. fractal_server/app/routes/auth/viewer_paths.py +43 -0
  63. fractal_server/app/routes/aux/_job.py +1 -1
  64. fractal_server/app/routes/aux/_runner.py +2 -2
  65. fractal_server/app/routes/pagination.py +1 -1
  66. fractal_server/app/schemas/user.py +29 -12
  67. fractal_server/app/schemas/user_group.py +0 -15
  68. fractal_server/app/schemas/v2/__init__.py +55 -48
  69. fractal_server/app/schemas/v2/accounting.py +11 -0
  70. fractal_server/app/schemas/v2/dataset.py +57 -11
  71. fractal_server/app/schemas/v2/dumps.py +10 -9
  72. fractal_server/app/schemas/v2/job.py +11 -11
  73. fractal_server/app/schemas/v2/manifest.py +4 -3
  74. fractal_server/app/schemas/v2/profile.py +53 -2
  75. fractal_server/app/schemas/v2/project.py +3 -3
  76. fractal_server/app/schemas/v2/resource.py +121 -16
  77. fractal_server/app/schemas/v2/sharing.py +99 -0
  78. fractal_server/app/schemas/v2/status_legacy.py +3 -3
  79. fractal_server/app/schemas/v2/task.py +6 -7
  80. fractal_server/app/schemas/v2/task_collection.py +5 -5
  81. fractal_server/app/schemas/v2/task_group.py +16 -16
  82. fractal_server/app/schemas/v2/workflow.py +16 -16
  83. fractal_server/app/schemas/v2/workflowtask.py +16 -15
  84. fractal_server/app/security/__init__.py +5 -8
  85. fractal_server/app/security/signup_email.py +4 -5
  86. fractal_server/app/shutdown.py +6 -6
  87. fractal_server/config/__init__.py +0 -6
  88. fractal_server/config/_data.py +0 -68
  89. fractal_server/config/_database.py +19 -20
  90. fractal_server/config/_email.py +30 -38
  91. fractal_server/config/_main.py +38 -52
  92. fractal_server/config/_oauth.py +17 -21
  93. fractal_server/data_migrations/2_18_0.py +30 -0
  94. fractal_server/exceptions.py +4 -0
  95. fractal_server/images/models.py +4 -5
  96. fractal_server/images/status_tools.py +4 -2
  97. fractal_server/logger.py +1 -1
  98. fractal_server/main.py +75 -13
  99. fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
  100. fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
  101. fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
  102. fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
  103. fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
  104. fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
  105. fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
  106. fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
  107. fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
  108. fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
  109. fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
  110. fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
  111. fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
  112. fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
  113. fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
  114. fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
  115. fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
  116. fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
  117. fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
  118. fractal_server/migrations/versions/7910eed4cf97_user_project_dirs_and_usergroup_viewer_.py +60 -0
  119. fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
  120. fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
  121. fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
  122. fractal_server/migrations/versions/88270f589c9b_add_prevent_new_submissions.py +39 -0
  123. fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
  124. fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
  125. fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
  126. fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
  127. fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
  128. fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
  129. fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
  130. fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
  131. fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
  132. fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
  133. fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
  134. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
  135. fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
  136. fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
  137. fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
  138. fractal_server/migrations/versions/bc0e8b3327a7_project_sharing.py +72 -0
  139. fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
  140. fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
  141. fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
  142. fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
  143. fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
  144. fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
  145. fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
  146. fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
  147. fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
  148. fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
  149. fractal_server/migrations/versions/f0702066b007_one_submitted_job_per_dataset.py +40 -0
  150. fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
  151. fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
  152. fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
  153. fractal_server/runner/config/_local.py +8 -5
  154. fractal_server/runner/config/_slurm.py +39 -33
  155. fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
  156. fractal_server/runner/executors/base_runner.py +29 -4
  157. fractal_server/runner/executors/local/get_local_config.py +1 -0
  158. fractal_server/runner/executors/local/runner.py +14 -13
  159. fractal_server/runner/executors/slurm_common/_batching.py +9 -20
  160. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
  161. fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
  162. fractal_server/runner/executors/slurm_common/remote.py +3 -1
  163. fractal_server/runner/executors/slurm_common/slurm_config.py +2 -0
  164. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
  165. fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
  166. fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
  167. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
  168. fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
  169. fractal_server/runner/task_files.py +9 -3
  170. fractal_server/runner/v2/_local.py +12 -6
  171. fractal_server/runner/v2/_slurm_ssh.py +14 -7
  172. fractal_server/runner/v2/_slurm_sudo.py +14 -7
  173. fractal_server/runner/v2/db_tools.py +0 -1
  174. fractal_server/runner/v2/deduplicate_list.py +2 -1
  175. fractal_server/runner/v2/runner.py +44 -28
  176. fractal_server/runner/v2/runner_functions.py +22 -28
  177. fractal_server/runner/v2/submit_workflow.py +29 -15
  178. fractal_server/ssh/_fabric.py +6 -13
  179. fractal_server/string_tools.py +0 -1
  180. fractal_server/syringe.py +1 -1
  181. fractal_server/tasks/config/_pixi.py +1 -1
  182. fractal_server/tasks/config/_python.py +16 -9
  183. fractal_server/tasks/utils.py +0 -1
  184. fractal_server/tasks/v2/local/_utils.py +3 -3
  185. fractal_server/tasks/v2/local/collect.py +15 -18
  186. fractal_server/tasks/v2/local/collect_pixi.py +14 -16
  187. fractal_server/tasks/v2/local/deactivate.py +14 -15
  188. fractal_server/tasks/v2/local/deactivate_pixi.py +7 -7
  189. fractal_server/tasks/v2/local/delete.py +6 -8
  190. fractal_server/tasks/v2/local/reactivate.py +12 -12
  191. fractal_server/tasks/v2/local/reactivate_pixi.py +12 -12
  192. fractal_server/tasks/v2/ssh/_utils.py +3 -3
  193. fractal_server/tasks/v2/ssh/collect.py +19 -24
  194. fractal_server/tasks/v2/ssh/collect_pixi.py +22 -24
  195. fractal_server/tasks/v2/ssh/deactivate.py +17 -15
  196. fractal_server/tasks/v2/ssh/deactivate_pixi.py +8 -7
  197. fractal_server/tasks/v2/ssh/delete.py +12 -10
  198. fractal_server/tasks/v2/ssh/reactivate.py +16 -16
  199. fractal_server/tasks/v2/ssh/reactivate_pixi.py +13 -14
  200. fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
  201. fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
  202. fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
  203. fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
  204. fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
  205. fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
  206. fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
  207. fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
  208. fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
  209. fractal_server/tasks/v2/utils_background.py +10 -10
  210. fractal_server/tasks/v2/utils_database.py +5 -5
  211. fractal_server/tasks/v2/utils_package_names.py +1 -2
  212. fractal_server/tasks/v2/utils_pixi.py +1 -3
  213. fractal_server/types/__init__.py +98 -1
  214. fractal_server/types/validators/__init__.py +3 -0
  215. fractal_server/types/validators/_common_validators.py +33 -3
  216. fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
  217. fractal_server/utils.py +1 -0
  218. fractal_server/zip_tools.py +34 -0
  219. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/METADATA +3 -2
  220. fractal_server-2.18.0.dist-info/RECORD +275 -0
  221. fractal_server/app/routes/admin/v2/project.py +0 -41
  222. fractal_server-2.17.1a1.dist-info/RECORD +0 -264
  223. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/WHEEL +0 -0
  224. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/entry_points.txt +0 -0
  225. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/licenses/LICENSE +0 -0
@@ -3,26 +3,27 @@ import time
3
3
  from pathlib import Path
4
4
  from tempfile import TemporaryDirectory
5
5
 
6
- from ..utils_background import add_commit_refresh
7
- from ..utils_background import fail_and_cleanup
8
- from ..utils_background import get_activity_and_task_group
9
- from ..utils_templates import get_collection_replacements
10
- from ._utils import _customize_and_run_template
11
6
  from fractal_server.app.db import get_sync_db
12
7
  from fractal_server.app.models import Profile
13
8
  from fractal_server.app.models import Resource
14
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
15
- from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
9
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
10
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatus
16
11
  from fractal_server.logger import reset_logger_handlers
17
12
  from fractal_server.logger import set_logger
18
13
  from fractal_server.tasks.utils import get_log_path
14
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
15
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
16
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
19
17
  from fractal_server.tasks.v2.utils_background import get_current_log
20
18
  from fractal_server.tasks.v2.utils_python_interpreter import (
21
19
  get_python_interpreter,
22
20
  )
23
21
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
22
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
24
23
  from fractal_server.utils import get_timestamp
25
24
 
25
+ from ._utils import _customize_and_run_template
26
+
26
27
 
27
28
  def reactivate_local(
28
29
  *,
@@ -78,7 +79,7 @@ def reactivate_local(
78
79
  return
79
80
 
80
81
  try:
81
- activity.status = TaskGroupActivityStatusV2.ONGOING
82
+ activity.status = TaskGroupActivityStatus.ONGOING
82
83
  activity = add_commit_refresh(obj=activity, db=db)
83
84
 
84
85
  # Prepare replacements for templates
@@ -104,7 +105,7 @@ def reactivate_local(
104
105
  ).as_posix(),
105
106
  prefix=(
106
107
  f"{int(time.time())}_"
107
- f"{TaskGroupActivityActionV2.REACTIVATE}"
108
+ f"{TaskGroupActivityAction.REACTIVATE}"
108
109
  ),
109
110
  logger_name=LOGGER_NAME,
110
111
  )
@@ -125,7 +126,7 @@ def reactivate_local(
125
126
  )
126
127
  logger.debug("end - install from pip freeze")
127
128
  activity.log = get_current_log(log_file_path)
128
- activity.status = TaskGroupActivityStatusV2.OK
129
+ activity.status = TaskGroupActivityStatus.OK
129
130
  activity.timestamp_ended = get_timestamp()
130
131
  activity = add_commit_refresh(obj=activity, db=db)
131
132
  task_group.active = True
@@ -142,8 +143,7 @@ def reactivate_local(
142
143
  logger.info(f"Deleted folder {task_group.venv_path}")
143
144
  except Exception as rm_e:
144
145
  logger.error(
145
- "Removing folder failed.\n"
146
- f"Original error:\n{str(rm_e)}"
146
+ f"Removing folder failed.\nOriginal error:\n{str(rm_e)}"
147
147
  )
148
148
 
149
149
  fail_and_cleanup(
@@ -3,25 +3,26 @@ import time
3
3
  from pathlib import Path
4
4
  from tempfile import TemporaryDirectory
5
5
 
6
- from ..utils_background import add_commit_refresh
7
- from ..utils_background import fail_and_cleanup
8
- from ..utils_background import get_activity_and_task_group
9
- from ..utils_pixi import SOURCE_DIR_NAME
10
- from ._utils import edit_pyproject_toml_in_place_local
11
6
  from fractal_server.app.db import get_sync_db
12
7
  from fractal_server.app.models import Profile
13
8
  from fractal_server.app.models import Resource
14
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
15
- from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
9
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
10
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatus
16
11
  from fractal_server.logger import reset_logger_handlers
17
12
  from fractal_server.logger import set_logger
18
13
  from fractal_server.tasks.utils import get_log_path
19
14
  from fractal_server.tasks.v2.local._utils import _customize_and_run_template
15
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
16
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
17
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
20
18
  from fractal_server.tasks.v2.utils_background import get_current_log
19
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
21
20
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
22
21
  from fractal_server.utils import execute_command_sync
23
22
  from fractal_server.utils import get_timestamp
24
23
 
24
+ from ._utils import edit_pyproject_toml_in_place_local
25
+
25
26
 
26
27
  def reactivate_local_pixi(
27
28
  *,
@@ -76,7 +77,7 @@ def reactivate_local_pixi(
76
77
  return
77
78
 
78
79
  try:
79
- activity.status = TaskGroupActivityStatusV2.ONGOING
80
+ activity.status = TaskGroupActivityStatus.ONGOING
80
81
  activity = add_commit_refresh(obj=activity, db=db)
81
82
 
82
83
  common_args = dict(
@@ -125,7 +126,7 @@ def reactivate_local_pixi(
125
126
  ).as_posix(),
126
127
  prefix=(
127
128
  f"{int(time.time())}_"
128
- f"{TaskGroupActivityActionV2.REACTIVATE}"
129
+ f"{TaskGroupActivityAction.REACTIVATE}"
129
130
  ),
130
131
  logger_name=LOGGER_NAME,
131
132
  )
@@ -175,7 +176,7 @@ def reactivate_local_pixi(
175
176
  )
176
177
 
177
178
  activity.log = get_current_log(log_file_path)
178
- activity.status = TaskGroupActivityStatusV2.OK
179
+ activity.status = TaskGroupActivityStatus.OK
179
180
  activity.timestamp_ended = get_timestamp()
180
181
  activity = add_commit_refresh(obj=activity, db=db)
181
182
  task_group.active = True
@@ -192,8 +193,7 @@ def reactivate_local_pixi(
192
193
  logger.info(f"Deleted folder {source_dir}")
193
194
  except Exception as rm_e:
194
195
  logger.error(
195
- "Removing folder failed. "
196
- f"Original error: {str(rm_e)}"
196
+ f"Removing folder failed. Original error: {str(rm_e)}"
197
197
  )
198
198
 
199
199
  fail_and_cleanup(
@@ -3,14 +3,14 @@ from pathlib import Path
3
3
 
4
4
  from sqlalchemy.orm import Session
5
5
 
6
- from ..utils_background import fail_and_cleanup
7
- from ..utils_pixi import simplify_pyproject_toml
8
6
  from fractal_server.app.models import Resource
9
7
  from fractal_server.app.models.v2 import TaskGroupActivityV2
10
8
  from fractal_server.app.models.v2 import TaskGroupV2
11
9
  from fractal_server.logger import get_logger
12
10
  from fractal_server.logger import set_logger
13
11
  from fractal_server.ssh._fabric import FractalSSH
12
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
13
+ from fractal_server.tasks.v2.utils_pixi import simplify_pyproject_toml
14
14
  from fractal_server.tasks.v2.utils_templates import customize_template
15
15
 
16
16
  logger = set_logger(__name__)
@@ -156,7 +156,7 @@ def check_ssh_or_fail_and_cleanup(
156
156
  except Exception as e:
157
157
  logger = get_logger(logger_name=logger_name)
158
158
  logger.error(
159
- "Cannot establish SSH connection. " f"Original error: {str(e)}"
159
+ f"Cannot establish SSH connection. Original error: {str(e)}"
160
160
  )
161
161
  fail_and_cleanup(
162
162
  task_group=task_group,
@@ -2,17 +2,12 @@ import time
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ..utils_background import prepare_tasks_metadata
8
- from ..utils_database import create_db_tasks_and_update_task_group_sync
9
- from ._utils import check_ssh_or_fail_and_cleanup
10
5
  from fractal_server.app.db import get_sync_db
11
6
  from fractal_server.app.models import Profile
12
7
  from fractal_server.app.models import Resource
13
8
  from fractal_server.app.schemas.v2 import FractalUploadedFile
14
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
15
- from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
9
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
10
+ from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
16
11
  from fractal_server.app.schemas.v2.manifest import ManifestV2
17
12
  from fractal_server.logger import reset_logger_handlers
18
13
  from fractal_server.logger import set_logger
@@ -20,18 +15,24 @@ from fractal_server.ssh._fabric import SingleUseFractalSSH
20
15
  from fractal_server.ssh._fabric import SSHConfig
21
16
  from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
22
17
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
18
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
19
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
23
20
  from fractal_server.tasks.v2.utils_background import get_current_log
21
+ from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
22
+ from fractal_server.tasks.v2.utils_database import (
23
+ create_db_tasks_and_update_task_group_sync,
24
+ )
24
25
  from fractal_server.tasks.v2.utils_package_names import compare_package_names
25
26
  from fractal_server.tasks.v2.utils_python_interpreter import (
26
27
  get_python_interpreter,
27
28
  )
28
- from fractal_server.tasks.v2.utils_templates import get_collection_replacements
29
- from fractal_server.tasks.v2.utils_templates import (
30
- parse_script_pip_show_stdout,
31
- )
32
29
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
30
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
31
+ from fractal_server.tasks.v2.utils_templates import parse_script_pip_show_stdout
33
32
  from fractal_server.utils import get_timestamp
34
33
 
34
+ from ._utils import check_ssh_or_fail_and_cleanup
35
+
35
36
 
36
37
  def collect_ssh(
37
38
  *,
@@ -55,8 +56,8 @@ def collect_ssh(
55
56
  Args:
56
57
  task_group_id:
57
58
  task_group_activity_id:
58
- ssh_config:
59
59
  resource:
60
+ profile:
60
61
  wheel_file:
61
62
  """
62
63
 
@@ -136,9 +137,7 @@ def collect_ssh(
136
137
  tmp_archive_path = (
137
138
  Path(tmpdir) / wheel_filename
138
139
  ).as_posix()
139
- logger.info(
140
- f"Write wheel file into {tmp_archive_path}"
141
- )
140
+ logger.info(f"Write wheel file into {tmp_archive_path}")
142
141
  with open(tmp_archive_path, "wb") as f:
143
142
  f.write(wheel_file.contents)
144
143
  fractal_ssh.send_file(
@@ -167,7 +166,7 @@ def collect_ssh(
167
166
  script_dir_remote=script_dir_remote,
168
167
  prefix=(
169
168
  f"{int(time.time())}_"
170
- f"{TaskGroupActivityActionV2.COLLECT}"
169
+ f"{TaskGroupActivityAction.COLLECT}"
171
170
  ),
172
171
  fractal_ssh=fractal_ssh,
173
172
  logger_name=LOGGER_NAME,
@@ -176,7 +175,7 @@ def collect_ssh(
176
175
  logger.info("installing - START")
177
176
 
178
177
  # Set status to ONGOING and refresh logs
179
- activity.status = TaskGroupActivityStatusV2.ONGOING
178
+ activity.status = TaskGroupActivityStatus.ONGOING
180
179
  activity.log = get_current_log(log_file_path)
181
180
  activity = add_commit_refresh(obj=activity, db=db)
182
181
 
@@ -267,9 +266,7 @@ def collect_ssh(
267
266
  )
268
267
  logger.info("_prepare_tasks_metadata - end")
269
268
 
270
- logger.info(
271
- "create_db_tasks_and_update_task_group - " "start"
272
- )
269
+ logger.info("create_db_tasks_and_update_task_group - start")
273
270
  create_db_tasks_and_update_task_group_sync(
274
271
  task_list=task_list,
275
272
  task_group_id=task_group.id,
@@ -293,7 +290,7 @@ def collect_ssh(
293
290
 
294
291
  # Finalize (write metadata to DB)
295
292
  logger.info("finalising - START")
296
- activity.status = TaskGroupActivityStatusV2.OK
293
+ activity.status = TaskGroupActivityStatus.OK
297
294
  activity.timestamp_ended = get_timestamp()
298
295
  activity = add_commit_refresh(obj=activity, db=db)
299
296
  logger.info("finalising - END")
@@ -310,9 +307,7 @@ def collect_ssh(
310
307
  folder=task_group.path,
311
308
  safe_root=profile.tasks_remote_dir,
312
309
  )
313
- logger.info(
314
- f"Deleted remoted folder {task_group.path}"
315
- )
310
+ logger.info(f"Deleted remoted folder {task_group.path}")
316
311
  except Exception as e_rm:
317
312
  logger.error(
318
313
  "Removing folder failed. "
@@ -2,21 +2,12 @@ import time
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ..utils_background import prepare_tasks_metadata
8
- from ..utils_database import create_db_tasks_and_update_task_group_sync
9
- from ..utils_pixi import parse_collect_stdout
10
- from ..utils_pixi import SOURCE_DIR_NAME
11
- from ._pixi_slurm_ssh import run_script_on_remote_slurm
12
- from ._utils import check_ssh_or_fail_and_cleanup
13
- from ._utils import edit_pyproject_toml_in_place_ssh
14
5
  from fractal_server.app.db import get_sync_db
15
6
  from fractal_server.app.models import Profile
16
7
  from fractal_server.app.models import Resource
17
8
  from fractal_server.app.schemas.v2 import FractalUploadedFile
18
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
19
- from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
9
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
10
+ from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
20
11
  from fractal_server.app.schemas.v2.manifest import ManifestV2
21
12
  from fractal_server.logger import reset_logger_handlers
22
13
  from fractal_server.logger import set_logger
@@ -25,10 +16,22 @@ from fractal_server.ssh._fabric import SSHConfig
25
16
  from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
26
17
  from fractal_server.tasks.v2.ssh._utils import _customize_and_send_template
27
18
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
19
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
20
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
28
21
  from fractal_server.tasks.v2.utils_background import get_current_log
22
+ from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
23
+ from fractal_server.tasks.v2.utils_database import (
24
+ create_db_tasks_and_update_task_group_sync,
25
+ )
26
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
27
+ from fractal_server.tasks.v2.utils_pixi import parse_collect_stdout
29
28
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
30
29
  from fractal_server.utils import get_timestamp
31
30
 
31
+ from ._pixi_slurm_ssh import run_script_on_remote_slurm
32
+ from ._utils import check_ssh_or_fail_and_cleanup
33
+ from ._utils import edit_pyproject_toml_in_place_ssh
34
+
32
35
 
33
36
  def collect_ssh_pixi(
34
37
  *,
@@ -52,8 +55,9 @@ def collect_ssh_pixi(
52
55
  Args:
53
56
  task_group_id:
54
57
  task_group_activity_id:
55
- ssh_config:
56
58
  tar_gz_file:
59
+ resource:
60
+ profile:
57
61
  """
58
62
 
59
63
  LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -180,7 +184,7 @@ def collect_ssh_pixi(
180
184
  logger.info("installing - START")
181
185
 
182
186
  # Set status to ONGOING and refresh logs
183
- activity.status = TaskGroupActivityStatusV2.ONGOING
187
+ activity.status = TaskGroupActivityStatus.ONGOING
184
188
  activity.log = get_current_log(log_file_path)
185
189
  activity = add_commit_refresh(obj=activity, db=db)
186
190
 
@@ -191,7 +195,7 @@ def collect_ssh_pixi(
191
195
  script_dir_remote=script_dir_remote,
192
196
  prefix=(
193
197
  f"{int(time.time())}_"
194
- f"{TaskGroupActivityActionV2.COLLECT}"
198
+ f"{TaskGroupActivityAction.COLLECT}"
195
199
  ),
196
200
  logger_name=LOGGER_NAME,
197
201
  fractal_ssh=fractal_ssh,
@@ -247,9 +251,7 @@ def collect_ssh_pixi(
247
251
  remote_script3_path,
248
252
  f"chmod -R 755 {source_dir}",
249
253
  ],
250
- slurm_config=resource.tasks_pixi_config[
251
- "SLURM_CONFIG"
252
- ],
254
+ slurm_config=resource.tasks_pixi_config["SLURM_CONFIG"],
253
255
  fractal_ssh=fractal_ssh,
254
256
  logger_name=LOGGER_NAME,
255
257
  prefix=common_args["prefix"],
@@ -290,9 +292,7 @@ def collect_ssh_pixi(
290
292
  )
291
293
  logger.info("_prepare_tasks_metadata - end")
292
294
 
293
- logger.info(
294
- "create_db_tasks_and_update_task_group - " "start"
295
- )
295
+ logger.info("create_db_tasks_and_update_task_group - start")
296
296
  create_db_tasks_and_update_task_group_sync(
297
297
  task_list=task_list,
298
298
  task_group_id=task_group.id,
@@ -327,7 +327,7 @@ def collect_ssh_pixi(
327
327
 
328
328
  # Finalize (write metadata to DB)
329
329
  logger.info("finalising - START")
330
- activity.status = TaskGroupActivityStatusV2.OK
330
+ activity.status = TaskGroupActivityStatus.OK
331
331
  activity.timestamp_ended = get_timestamp()
332
332
  activity = add_commit_refresh(obj=activity, db=db)
333
333
  logger.info("finalising - END")
@@ -344,9 +344,7 @@ def collect_ssh_pixi(
344
344
  folder=task_group.path,
345
345
  safe_root=profile.tasks_remote_dir,
346
346
  )
347
- logger.info(
348
- f"Deleted remoted folder {task_group.path}"
349
- )
347
+ logger.info(f"Deleted remoted folder {task_group.path}")
350
348
  except Exception as e_rm:
351
349
  logger.error(
352
350
  "Removing folder failed. "
@@ -2,29 +2,30 @@ import time
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import add_commit_refresh
6
- from ..utils_background import fail_and_cleanup
7
- from ..utils_background import get_activity_and_task_group
8
- from ..utils_templates import get_collection_replacements
9
- from ._utils import _copy_wheel_file_ssh
10
- from ._utils import _customize_and_run_template
11
- from ._utils import check_ssh_or_fail_and_cleanup
12
5
  from fractal_server.app.db import get_sync_db
13
6
  from fractal_server.app.models import Profile
14
7
  from fractal_server.app.models import Resource
15
- from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
16
- from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
17
- from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
8
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
9
+ from fractal_server.app.schemas.v2 import TaskGroupOriginEnum
10
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatus
18
11
  from fractal_server.logger import reset_logger_handlers
19
12
  from fractal_server.logger import set_logger
20
13
  from fractal_server.ssh._fabric import SingleUseFractalSSH
21
14
  from fractal_server.ssh._fabric import SSHConfig
22
15
  from fractal_server.tasks.utils import FORBIDDEN_DEPENDENCY_STRINGS
23
16
  from fractal_server.tasks.utils import get_log_path
17
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
18
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
19
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
24
20
  from fractal_server.tasks.v2.utils_background import get_current_log
25
21
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
22
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
26
23
  from fractal_server.utils import get_timestamp
27
24
 
25
+ from ._utils import _copy_wheel_file_ssh
26
+ from ._utils import _customize_and_run_template
27
+ from ._utils import check_ssh_or_fail_and_cleanup
28
+
28
29
 
29
30
  def deactivate_ssh(
30
31
  *,
@@ -42,7 +43,8 @@ def deactivate_ssh(
42
43
  Args:
43
44
  task_group_id:
44
45
  task_group_activity_id:
45
- ssh_config:
46
+ resource:
47
+ profile:
46
48
  """
47
49
 
48
50
  LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -99,7 +101,7 @@ def deactivate_ssh(
99
101
  )
100
102
  return
101
103
 
102
- activity.status = TaskGroupActivityStatusV2.ONGOING
104
+ activity.status = TaskGroupActivityStatus.ONGOING
103
105
  activity = add_commit_refresh(obj=activity, db=db)
104
106
 
105
107
  if task_group.env_info is None:
@@ -133,7 +135,7 @@ def deactivate_ssh(
133
135
  script_dir_remote=script_dir_remote,
134
136
  prefix=(
135
137
  f"{int(time.time())}_"
136
- f"{TaskGroupActivityActionV2.DEACTIVATE}"
138
+ f"{TaskGroupActivityAction.DEACTIVATE}"
137
139
  ),
138
140
  fractal_ssh=fractal_ssh,
139
141
  logger_name=LOGGER_NAME,
@@ -158,7 +160,7 @@ def deactivate_ssh(
158
160
  )
159
161
 
160
162
  # Handle some specific cases for wheel-file case
161
- if task_group.origin == TaskGroupV2OriginEnum.WHEELFILE:
163
+ if task_group.origin == TaskGroupOriginEnum.WHEELFILE:
162
164
  logger.info(
163
165
  f"Handle specific cases for {task_group.origin=}."
164
166
  )
@@ -262,7 +264,7 @@ def deactivate_ssh(
262
264
  safe_root=profile.tasks_remote_dir,
263
265
  )
264
266
  logger.info(f"All good, {task_group.venv_path} removed.")
265
- activity.status = TaskGroupActivityStatusV2.OK
267
+ activity.status = TaskGroupActivityStatus.OK
266
268
  activity.log = get_current_log(log_file_path)
267
269
  activity.timestamp_ended = get_timestamp()
268
270
  activity = add_commit_refresh(obj=activity, db=db)
@@ -1,23 +1,24 @@
1
1
  from pathlib import Path
2
2
  from tempfile import TemporaryDirectory
3
3
 
4
- from ..utils_background import add_commit_refresh
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ..utils_pixi import SOURCE_DIR_NAME
8
- from ._utils import check_ssh_or_fail_and_cleanup
9
4
  from fractal_server.app.db import get_sync_db
10
5
  from fractal_server.app.models import Profile
11
6
  from fractal_server.app.models import Resource
12
- from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
7
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatus
13
8
  from fractal_server.logger import reset_logger_handlers
14
9
  from fractal_server.logger import set_logger
15
10
  from fractal_server.ssh._fabric import SingleUseFractalSSH
16
11
  from fractal_server.ssh._fabric import SSHConfig
17
12
  from fractal_server.tasks.utils import get_log_path
13
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
14
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
15
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
18
16
  from fractal_server.tasks.v2.utils_background import get_current_log
17
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
19
18
  from fractal_server.utils import get_timestamp
20
19
 
20
+ from ._utils import check_ssh_or_fail_and_cleanup
21
+
21
22
 
22
23
  def deactivate_ssh_pixi(
23
24
  *,
@@ -108,7 +109,7 @@ def deactivate_ssh_pixi(
108
109
  safe_root=profile.tasks_remote_dir,
109
110
  )
110
111
  logger.info(f"All good, {source_dir} removed.")
111
- activity.status = TaskGroupActivityStatusV2.OK
112
+ activity.status = TaskGroupActivityStatus.OK
112
113
  activity.log = get_current_log(log_file_path)
113
114
  activity.timestamp_ended = get_timestamp()
114
115
  activity = add_commit_refresh(obj=activity, db=db)
@@ -1,23 +1,24 @@
1
1
  from pathlib import Path
2
2
  from tempfile import TemporaryDirectory
3
3
 
4
- from ..utils_background import add_commit_refresh
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ._utils import check_ssh_or_fail_and_cleanup
8
4
  from fractal_server.app.db import get_sync_db
9
5
  from fractal_server.app.models import Profile
10
6
  from fractal_server.app.models import Resource
11
- from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
12
- from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
7
+ from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
8
+ from fractal_server.app.schemas.v2 import TaskGroupOriginEnum
13
9
  from fractal_server.logger import reset_logger_handlers
14
10
  from fractal_server.logger import set_logger
15
11
  from fractal_server.ssh._fabric import SingleUseFractalSSH
16
12
  from fractal_server.ssh._fabric import SSHConfig
17
13
  from fractal_server.tasks.utils import get_log_path
14
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
15
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
16
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
18
17
  from fractal_server.tasks.v2.utils_background import get_current_log
19
18
  from fractal_server.utils import get_timestamp
20
19
 
20
+ from ._utils import check_ssh_or_fail_and_cleanup
21
+
21
22
 
22
23
  def delete_ssh(
23
24
  *,
@@ -35,7 +36,8 @@ def delete_ssh(
35
36
  Args:
36
37
  task_group_id:
37
38
  task_group_activity_id:
38
- ssh_config:
39
+ resource:
40
+ profile:
39
41
  """
40
42
 
41
43
  LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -78,7 +80,7 @@ def delete_ssh(
78
80
  if not ssh_ok:
79
81
  return
80
82
 
81
- activity.status = TaskGroupActivityStatusV2.ONGOING
83
+ activity.status = TaskGroupActivityStatus.ONGOING
82
84
  activity.log = get_current_log(log_file_path)
83
85
  activity = add_commit_refresh(obj=activity, db=db)
84
86
 
@@ -86,7 +88,7 @@ def delete_ssh(
86
88
  db.commit()
87
89
  logger.debug("Task group removed from database.")
88
90
 
89
- if task_group.origin != TaskGroupV2OriginEnum.OTHER:
91
+ if task_group.origin != TaskGroupOriginEnum.OTHER:
90
92
  logger.debug(
91
93
  f"Removing remote {task_group.path=} "
92
94
  f"(with {profile.tasks_remote_dir=})."
@@ -97,7 +99,7 @@ def delete_ssh(
97
99
  )
98
100
  logger.debug(f"Remote {task_group.path=} removed.")
99
101
 
100
- activity.status = TaskGroupActivityStatusV2.OK
102
+ activity.status = TaskGroupActivityStatus.OK
101
103
  activity.log = get_current_log(log_file_path)
102
104
  activity.timestamp_ended = get_timestamp()
103
105
  activity = add_commit_refresh(obj=activity, db=db)