fractal-server 2.17.1a1__py3-none-any.whl → 2.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +19 -18
  3. fractal_server/app/db/__init__.py +3 -3
  4. fractal_server/app/models/__init__.py +1 -0
  5. fractal_server/app/models/linkuserproject.py +3 -1
  6. fractal_server/app/models/security.py +21 -3
  7. fractal_server/app/models/v2/__init__.py +3 -1
  8. fractal_server/app/models/v2/accounting.py +9 -1
  9. fractal_server/app/models/v2/dataset.py +5 -1
  10. fractal_server/app/models/v2/history.py +15 -1
  11. fractal_server/app/models/v2/job.py +4 -0
  12. fractal_server/app/models/v2/profile.py +29 -0
  13. fractal_server/app/models/v2/project.py +4 -10
  14. fractal_server/app/models/v2/resource.py +4 -0
  15. fractal_server/app/models/v2/task_group.py +4 -3
  16. fractal_server/app/models/v2/workflow.py +2 -1
  17. fractal_server/app/routes/admin/v2/__init__.py +1 -2
  18. fractal_server/app/routes/admin/v2/accounting.py +1 -1
  19. fractal_server/app/routes/admin/v2/job.py +9 -9
  20. fractal_server/app/routes/admin/v2/profile.py +3 -2
  21. fractal_server/app/routes/admin/v2/resource.py +5 -5
  22. fractal_server/app/routes/admin/v2/task.py +28 -18
  23. fractal_server/app/routes/admin/v2/task_group.py +0 -1
  24. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +1 -2
  25. fractal_server/app/routes/api/__init__.py +1 -0
  26. fractal_server/app/routes/api/v2/__init__.py +5 -6
  27. fractal_server/app/routes/api/v2/_aux_functions.py +70 -63
  28. fractal_server/app/routes/api/v2/_aux_functions_history.py +43 -20
  29. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +2 -4
  30. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -7
  31. fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
  32. fractal_server/app/routes/api/v2/dataset.py +13 -32
  33. fractal_server/app/routes/api/v2/history.py +35 -21
  34. fractal_server/app/routes/api/v2/images.py +3 -2
  35. fractal_server/app/routes/api/v2/job.py +17 -14
  36. fractal_server/app/routes/api/v2/pre_submission_checks.py +5 -4
  37. fractal_server/app/routes/api/v2/project.py +22 -17
  38. fractal_server/app/routes/api/v2/status_legacy.py +12 -11
  39. fractal_server/app/routes/api/v2/submit.py +11 -12
  40. fractal_server/app/routes/api/v2/task.py +4 -3
  41. fractal_server/app/routes/api/v2/task_collection.py +28 -30
  42. fractal_server/app/routes/api/v2/task_collection_custom.py +8 -7
  43. fractal_server/app/routes/api/v2/task_collection_pixi.py +1 -2
  44. fractal_server/app/routes/api/v2/task_group.py +7 -6
  45. fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -6
  46. fractal_server/app/routes/api/v2/task_version_update.py +13 -12
  47. fractal_server/app/routes/api/v2/workflow.py +14 -31
  48. fractal_server/app/routes/api/v2/workflow_import.py +17 -19
  49. fractal_server/app/routes/api/v2/workflowtask.py +10 -12
  50. fractal_server/app/routes/auth/__init__.py +1 -3
  51. fractal_server/app/routes/auth/_aux_auth.py +1 -2
  52. fractal_server/app/routes/auth/current_user.py +4 -5
  53. fractal_server/app/routes/auth/group.py +7 -5
  54. fractal_server/app/routes/auth/login.py +1 -0
  55. fractal_server/app/routes/auth/oauth.py +4 -3
  56. fractal_server/app/routes/auth/register.py +4 -2
  57. fractal_server/app/routes/auth/users.py +10 -10
  58. fractal_server/app/routes/aux/_job.py +1 -1
  59. fractal_server/app/routes/aux/_runner.py +2 -2
  60. fractal_server/app/routes/pagination.py +1 -1
  61. fractal_server/app/schemas/user.py +3 -3
  62. fractal_server/app/schemas/v2/accounting.py +11 -0
  63. fractal_server/app/schemas/v2/dataset.py +28 -4
  64. fractal_server/app/schemas/v2/dumps.py +1 -0
  65. fractal_server/app/schemas/v2/manifest.py +4 -3
  66. fractal_server/app/schemas/v2/profile.py +53 -2
  67. fractal_server/app/schemas/v2/resource.py +109 -13
  68. fractal_server/app/schemas/v2/task.py +0 -1
  69. fractal_server/app/schemas/v2/task_collection.py +1 -1
  70. fractal_server/app/schemas/v2/workflowtask.py +4 -3
  71. fractal_server/app/security/__init__.py +4 -7
  72. fractal_server/app/security/signup_email.py +4 -5
  73. fractal_server/config/_data.py +36 -25
  74. fractal_server/config/_database.py +19 -20
  75. fractal_server/config/_email.py +30 -38
  76. fractal_server/config/_main.py +33 -52
  77. fractal_server/config/_oauth.py +17 -21
  78. fractal_server/exceptions.py +4 -0
  79. fractal_server/images/models.py +3 -3
  80. fractal_server/images/status_tools.py +4 -2
  81. fractal_server/logger.py +1 -1
  82. fractal_server/main.py +4 -3
  83. fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
  84. fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
  85. fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
  86. fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
  87. fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
  88. fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
  89. fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
  90. fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
  91. fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
  92. fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
  93. fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
  94. fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
  95. fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
  96. fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
  97. fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
  98. fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
  99. fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
  100. fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
  101. fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
  102. fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
  103. fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
  104. fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
  105. fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
  106. fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
  107. fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
  108. fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
  109. fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
  110. fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
  111. fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
  112. fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
  113. fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
  114. fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
  115. fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
  116. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
  117. fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
  118. fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
  119. fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
  120. fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
  121. fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
  122. fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
  123. fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
  124. fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
  125. fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
  126. fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
  127. fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
  128. fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
  129. fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
  130. fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
  131. fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
  132. fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
  133. fractal_server/runner/config/_local.py +8 -5
  134. fractal_server/runner/config/_slurm.py +37 -33
  135. fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
  136. fractal_server/runner/executors/base_runner.py +29 -4
  137. fractal_server/runner/executors/local/get_local_config.py +1 -0
  138. fractal_server/runner/executors/local/runner.py +14 -13
  139. fractal_server/runner/executors/slurm_common/_batching.py +5 -10
  140. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
  141. fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
  142. fractal_server/runner/executors/slurm_common/remote.py +3 -1
  143. fractal_server/runner/executors/slurm_common/slurm_config.py +1 -0
  144. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
  145. fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
  146. fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
  147. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
  148. fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
  149. fractal_server/runner/task_files.py +9 -3
  150. fractal_server/runner/v2/_local.py +9 -4
  151. fractal_server/runner/v2/_slurm_ssh.py +11 -5
  152. fractal_server/runner/v2/_slurm_sudo.py +11 -5
  153. fractal_server/runner/v2/db_tools.py +0 -1
  154. fractal_server/runner/v2/deduplicate_list.py +2 -1
  155. fractal_server/runner/v2/runner.py +11 -14
  156. fractal_server/runner/v2/runner_functions.py +11 -14
  157. fractal_server/runner/v2/submit_workflow.py +7 -6
  158. fractal_server/ssh/_fabric.py +6 -13
  159. fractal_server/string_tools.py +0 -1
  160. fractal_server/syringe.py +1 -1
  161. fractal_server/tasks/config/_pixi.py +1 -1
  162. fractal_server/tasks/config/_python.py +16 -9
  163. fractal_server/tasks/utils.py +0 -1
  164. fractal_server/tasks/v2/local/_utils.py +1 -1
  165. fractal_server/tasks/v2/local/collect.py +10 -12
  166. fractal_server/tasks/v2/local/collect_pixi.py +9 -10
  167. fractal_server/tasks/v2/local/deactivate.py +7 -8
  168. fractal_server/tasks/v2/local/deactivate_pixi.py +4 -4
  169. fractal_server/tasks/v2/local/delete.py +1 -3
  170. fractal_server/tasks/v2/local/reactivate.py +7 -7
  171. fractal_server/tasks/v2/local/reactivate_pixi.py +7 -7
  172. fractal_server/tasks/v2/ssh/_utils.py +3 -3
  173. fractal_server/tasks/v2/ssh/collect.py +14 -19
  174. fractal_server/tasks/v2/ssh/collect_pixi.py +17 -19
  175. fractal_server/tasks/v2/ssh/deactivate.py +10 -8
  176. fractal_server/tasks/v2/ssh/deactivate_pixi.py +6 -5
  177. fractal_server/tasks/v2/ssh/delete.py +7 -5
  178. fractal_server/tasks/v2/ssh/reactivate.py +11 -11
  179. fractal_server/tasks/v2/ssh/reactivate_pixi.py +8 -9
  180. fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
  181. fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
  182. fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
  183. fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
  184. fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
  185. fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
  186. fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
  187. fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
  188. fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
  189. fractal_server/tasks/v2/utils_background.py +3 -3
  190. fractal_server/tasks/v2/utils_package_names.py +1 -2
  191. fractal_server/tasks/v2/utils_pixi.py +1 -3
  192. fractal_server/types/__init__.py +76 -1
  193. fractal_server/types/validators/_common_validators.py +1 -3
  194. fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
  195. fractal_server/utils.py +1 -0
  196. fractal_server/zip_tools.py +34 -0
  197. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/METADATA +1 -1
  198. fractal_server-2.17.2.dist-info/RECORD +265 -0
  199. fractal_server/app/routes/admin/v2/project.py +0 -41
  200. fractal_server-2.17.1a1.dist-info/RECORD +0 -264
  201. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/WHEEL +0 -0
  202. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/entry_points.txt +0 -0
  203. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,6 +1,5 @@
1
1
  from pathlib import Path
2
2
 
3
-
4
3
  COLLECTION_FILENAME = "collection.json"
5
4
  COLLECTION_LOG_FILENAME = "collection.log"
6
5
  COLLECTION_FREEZE_FILENAME = "collection_freeze.txt"
@@ -1,10 +1,10 @@
1
1
  from pathlib import Path
2
2
 
3
- from ..utils_pixi import simplify_pyproject_toml
4
3
  from fractal_server.app.models import Resource
5
4
  from fractal_server.app.schemas.v2 import TaskCreateV2
6
5
  from fractal_server.logger import get_logger
7
6
  from fractal_server.logger import set_logger
7
+ from fractal_server.tasks.v2.utils_pixi import simplify_pyproject_toml
8
8
  from fractal_server.tasks.v2.utils_templates import customize_template
9
9
  from fractal_server.utils import execute_command_sync
10
10
 
@@ -4,8 +4,6 @@ import time
4
4
  from pathlib import Path
5
5
  from tempfile import TemporaryDirectory
6
6
 
7
- from ..utils_database import create_db_tasks_and_update_task_group_sync
8
- from ._utils import _customize_and_run_template
9
7
  from fractal_server.app.db import get_sync_db
10
8
  from fractal_server.app.models import Profile
11
9
  from fractal_server.app.models import Resource
@@ -20,22 +18,23 @@ from fractal_server.tasks.utils import get_log_path
20
18
  from fractal_server.tasks.v2.local._utils import check_task_files_exist
21
19
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
22
20
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
23
- from fractal_server.tasks.v2.utils_background import (
24
- get_activity_and_task_group,
25
- )
21
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
26
22
  from fractal_server.tasks.v2.utils_background import get_current_log
27
23
  from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
24
+ from fractal_server.tasks.v2.utils_database import (
25
+ create_db_tasks_and_update_task_group_sync,
26
+ )
28
27
  from fractal_server.tasks.v2.utils_package_names import compare_package_names
29
28
  from fractal_server.tasks.v2.utils_python_interpreter import (
30
29
  get_python_interpreter,
31
30
  )
32
- from fractal_server.tasks.v2.utils_templates import get_collection_replacements
33
- from fractal_server.tasks.v2.utils_templates import (
34
- parse_script_pip_show_stdout,
35
- )
36
31
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
32
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
33
+ from fractal_server.tasks.v2.utils_templates import parse_script_pip_show_stdout
37
34
  from fractal_server.utils import get_timestamp
38
35
 
36
+ from ._utils import _customize_and_run_template
37
+
39
38
 
40
39
  def collect_local(
41
40
  *,
@@ -234,7 +233,7 @@ def collect_local(
234
233
  activity.log = get_current_log(log_file_path)
235
234
  activity = add_commit_refresh(obj=activity, db=db)
236
235
 
237
- logger.info("create_db_tasks_and_update_task_group - " "start")
236
+ logger.info("create_db_tasks_and_update_task_group - start")
238
237
  create_db_tasks_and_update_task_group_sync(
239
238
  task_list=task_list,
240
239
  task_group_id=task_group.id,
@@ -274,8 +273,7 @@ def collect_local(
274
273
  logger.info(f"Deleted folder {task_group.path}")
275
274
  except Exception as rm_e:
276
275
  logger.error(
277
- "Removing folder failed.\n"
278
- f"Original error:\n{str(rm_e)}"
276
+ f"Removing folder failed.\nOriginal error:\n{str(rm_e)}"
279
277
  )
280
278
 
281
279
  fail_and_cleanup(
@@ -4,10 +4,6 @@ import time
4
4
  from pathlib import Path
5
5
  from tempfile import TemporaryDirectory
6
6
 
7
- from ..utils_database import create_db_tasks_and_update_task_group_sync
8
- from ..utils_pixi import parse_collect_stdout
9
- from ..utils_pixi import SOURCE_DIR_NAME
10
- from ._utils import edit_pyproject_toml_in_place_local
11
7
  from fractal_server.app.db import get_sync_db
12
8
  from fractal_server.app.models import Profile
13
9
  from fractal_server.app.models import Resource
@@ -22,15 +18,20 @@ from fractal_server.tasks.v2.local._utils import _customize_and_run_template
22
18
  from fractal_server.tasks.v2.local._utils import check_task_files_exist
23
19
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
24
20
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
25
- from fractal_server.tasks.v2.utils_background import (
26
- get_activity_and_task_group,
27
- )
21
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
28
22
  from fractal_server.tasks.v2.utils_background import get_current_log
29
23
  from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
24
+ from fractal_server.tasks.v2.utils_database import (
25
+ create_db_tasks_and_update_task_group_sync,
26
+ )
27
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
28
+ from fractal_server.tasks.v2.utils_pixi import parse_collect_stdout
30
29
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
31
30
  from fractal_server.utils import execute_command_sync
32
31
  from fractal_server.utils import get_timestamp
33
32
 
33
+ from ._utils import edit_pyproject_toml_in_place_local
34
+
34
35
 
35
36
  def collect_local_pixi(
36
37
  *,
@@ -177,9 +178,7 @@ def collect_local_pixi(
177
178
  package_root = parsed_output["package_root"]
178
179
  venv_size = parsed_output["venv_size"]
179
180
  venv_file_number = parsed_output["venv_file_number"]
180
- project_python_wrapper = parsed_output[
181
- "project_python_wrapper"
182
- ]
181
+ project_python_wrapper = parsed_output["project_python_wrapper"]
183
182
 
184
183
  # Make task folder 755
185
184
  command = f"chmod -R 755 {source_dir}"
@@ -3,11 +3,6 @@ import time
3
3
  from pathlib import Path
4
4
  from tempfile import TemporaryDirectory
5
5
 
6
- from ..utils_background import add_commit_refresh
7
- from ..utils_background import fail_and_cleanup
8
- from ..utils_background import get_activity_and_task_group
9
- from ..utils_templates import get_collection_replacements
10
- from ._utils import _customize_and_run_template
11
6
  from fractal_server.app.db import get_sync_db
12
7
  from fractal_server.app.models import Profile
13
8
  from fractal_server.app.models import Resource
@@ -18,10 +13,16 @@ from fractal_server.logger import reset_logger_handlers
18
13
  from fractal_server.logger import set_logger
19
14
  from fractal_server.tasks.utils import FORBIDDEN_DEPENDENCY_STRINGS
20
15
  from fractal_server.tasks.utils import get_log_path
16
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
17
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
18
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
21
19
  from fractal_server.tasks.v2.utils_background import get_current_log
22
20
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
21
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
23
22
  from fractal_server.utils import get_timestamp
24
23
 
24
+ from ._utils import _customize_and_run_template
25
+
25
26
 
26
27
  def deactivate_local(
27
28
  *,
@@ -172,9 +173,7 @@ def deactivate_local(
172
173
  / Path(task_group.archive_path).name
173
174
  ).as_posix()
174
175
  shutil.copy(task_group.archive_path, new_archive_path)
175
- logger.info(
176
- f"Copied wheel file to {new_archive_path}."
177
- )
176
+ logger.info(f"Copied wheel file to {new_archive_path}.")
178
177
 
179
178
  task_group.archive_path = new_archive_path
180
179
  new_pip_freeze = task_group.env_info.replace(
@@ -2,10 +2,6 @@ import shutil
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import add_commit_refresh
6
- from ..utils_background import fail_and_cleanup
7
- from ..utils_background import get_activity_and_task_group
8
- from ..utils_pixi import SOURCE_DIR_NAME
9
5
  from fractal_server.app.db import get_sync_db
10
6
  from fractal_server.app.models import Profile
11
7
  from fractal_server.app.models import Resource
@@ -13,7 +9,11 @@ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
13
9
  from fractal_server.logger import reset_logger_handlers
14
10
  from fractal_server.logger import set_logger
15
11
  from fractal_server.tasks.utils import get_log_path
12
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
13
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
14
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
16
15
  from fractal_server.tasks.v2.utils_background import get_current_log
16
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
17
17
  from fractal_server.utils import get_timestamp
18
18
 
19
19
 
@@ -12,9 +12,7 @@ from fractal_server.logger import set_logger
12
12
  from fractal_server.tasks.utils import get_log_path
13
13
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
14
14
  from fractal_server.tasks.v2.utils_background import fail_and_cleanup
15
- from fractal_server.tasks.v2.utils_background import (
16
- get_activity_and_task_group,
17
- )
15
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
18
16
  from fractal_server.tasks.v2.utils_background import get_current_log
19
17
  from fractal_server.utils import get_timestamp
20
18
 
@@ -3,11 +3,6 @@ import time
3
3
  from pathlib import Path
4
4
  from tempfile import TemporaryDirectory
5
5
 
6
- from ..utils_background import add_commit_refresh
7
- from ..utils_background import fail_and_cleanup
8
- from ..utils_background import get_activity_and_task_group
9
- from ..utils_templates import get_collection_replacements
10
- from ._utils import _customize_and_run_template
11
6
  from fractal_server.app.db import get_sync_db
12
7
  from fractal_server.app.models import Profile
13
8
  from fractal_server.app.models import Resource
@@ -16,13 +11,19 @@ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
16
11
  from fractal_server.logger import reset_logger_handlers
17
12
  from fractal_server.logger import set_logger
18
13
  from fractal_server.tasks.utils import get_log_path
14
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
15
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
16
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
19
17
  from fractal_server.tasks.v2.utils_background import get_current_log
20
18
  from fractal_server.tasks.v2.utils_python_interpreter import (
21
19
  get_python_interpreter,
22
20
  )
23
21
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
22
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
24
23
  from fractal_server.utils import get_timestamp
25
24
 
25
+ from ._utils import _customize_and_run_template
26
+
26
27
 
27
28
  def reactivate_local(
28
29
  *,
@@ -142,8 +143,7 @@ def reactivate_local(
142
143
  logger.info(f"Deleted folder {task_group.venv_path}")
143
144
  except Exception as rm_e:
144
145
  logger.error(
145
- "Removing folder failed.\n"
146
- f"Original error:\n{str(rm_e)}"
146
+ f"Removing folder failed.\nOriginal error:\n{str(rm_e)}"
147
147
  )
148
148
 
149
149
  fail_and_cleanup(
@@ -3,11 +3,6 @@ import time
3
3
  from pathlib import Path
4
4
  from tempfile import TemporaryDirectory
5
5
 
6
- from ..utils_background import add_commit_refresh
7
- from ..utils_background import fail_and_cleanup
8
- from ..utils_background import get_activity_and_task_group
9
- from ..utils_pixi import SOURCE_DIR_NAME
10
- from ._utils import edit_pyproject_toml_in_place_local
11
6
  from fractal_server.app.db import get_sync_db
12
7
  from fractal_server.app.models import Profile
13
8
  from fractal_server.app.models import Resource
@@ -17,11 +12,17 @@ from fractal_server.logger import reset_logger_handlers
17
12
  from fractal_server.logger import set_logger
18
13
  from fractal_server.tasks.utils import get_log_path
19
14
  from fractal_server.tasks.v2.local._utils import _customize_and_run_template
15
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
16
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
17
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
20
18
  from fractal_server.tasks.v2.utils_background import get_current_log
19
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
21
20
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
22
21
  from fractal_server.utils import execute_command_sync
23
22
  from fractal_server.utils import get_timestamp
24
23
 
24
+ from ._utils import edit_pyproject_toml_in_place_local
25
+
25
26
 
26
27
  def reactivate_local_pixi(
27
28
  *,
@@ -192,8 +193,7 @@ def reactivate_local_pixi(
192
193
  logger.info(f"Deleted folder {source_dir}")
193
194
  except Exception as rm_e:
194
195
  logger.error(
195
- "Removing folder failed. "
196
- f"Original error: {str(rm_e)}"
196
+ f"Removing folder failed. Original error: {str(rm_e)}"
197
197
  )
198
198
 
199
199
  fail_and_cleanup(
@@ -3,14 +3,14 @@ from pathlib import Path
3
3
 
4
4
  from sqlalchemy.orm import Session
5
5
 
6
- from ..utils_background import fail_and_cleanup
7
- from ..utils_pixi import simplify_pyproject_toml
8
6
  from fractal_server.app.models import Resource
9
7
  from fractal_server.app.models.v2 import TaskGroupActivityV2
10
8
  from fractal_server.app.models.v2 import TaskGroupV2
11
9
  from fractal_server.logger import get_logger
12
10
  from fractal_server.logger import set_logger
13
11
  from fractal_server.ssh._fabric import FractalSSH
12
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
13
+ from fractal_server.tasks.v2.utils_pixi import simplify_pyproject_toml
14
14
  from fractal_server.tasks.v2.utils_templates import customize_template
15
15
 
16
16
  logger = set_logger(__name__)
@@ -156,7 +156,7 @@ def check_ssh_or_fail_and_cleanup(
156
156
  except Exception as e:
157
157
  logger = get_logger(logger_name=logger_name)
158
158
  logger.error(
159
- "Cannot establish SSH connection. " f"Original error: {str(e)}"
159
+ f"Cannot establish SSH connection. Original error: {str(e)}"
160
160
  )
161
161
  fail_and_cleanup(
162
162
  task_group=task_group,
@@ -2,11 +2,6 @@ import time
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ..utils_background import prepare_tasks_metadata
8
- from ..utils_database import create_db_tasks_and_update_task_group_sync
9
- from ._utils import check_ssh_or_fail_and_cleanup
10
5
  from fractal_server.app.db import get_sync_db
11
6
  from fractal_server.app.models import Profile
12
7
  from fractal_server.app.models import Resource
@@ -20,18 +15,24 @@ from fractal_server.ssh._fabric import SingleUseFractalSSH
20
15
  from fractal_server.ssh._fabric import SSHConfig
21
16
  from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
22
17
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
18
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
19
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
23
20
  from fractal_server.tasks.v2.utils_background import get_current_log
21
+ from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
22
+ from fractal_server.tasks.v2.utils_database import (
23
+ create_db_tasks_and_update_task_group_sync,
24
+ )
24
25
  from fractal_server.tasks.v2.utils_package_names import compare_package_names
25
26
  from fractal_server.tasks.v2.utils_python_interpreter import (
26
27
  get_python_interpreter,
27
28
  )
28
- from fractal_server.tasks.v2.utils_templates import get_collection_replacements
29
- from fractal_server.tasks.v2.utils_templates import (
30
- parse_script_pip_show_stdout,
31
- )
32
29
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
30
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
31
+ from fractal_server.tasks.v2.utils_templates import parse_script_pip_show_stdout
33
32
  from fractal_server.utils import get_timestamp
34
33
 
34
+ from ._utils import check_ssh_or_fail_and_cleanup
35
+
35
36
 
36
37
  def collect_ssh(
37
38
  *,
@@ -55,8 +56,8 @@ def collect_ssh(
55
56
  Args:
56
57
  task_group_id:
57
58
  task_group_activity_id:
58
- ssh_config:
59
59
  resource:
60
+ profile:
60
61
  wheel_file:
61
62
  """
62
63
 
@@ -136,9 +137,7 @@ def collect_ssh(
136
137
  tmp_archive_path = (
137
138
  Path(tmpdir) / wheel_filename
138
139
  ).as_posix()
139
- logger.info(
140
- f"Write wheel file into {tmp_archive_path}"
141
- )
140
+ logger.info(f"Write wheel file into {tmp_archive_path}")
142
141
  with open(tmp_archive_path, "wb") as f:
143
142
  f.write(wheel_file.contents)
144
143
  fractal_ssh.send_file(
@@ -267,9 +266,7 @@ def collect_ssh(
267
266
  )
268
267
  logger.info("_prepare_tasks_metadata - end")
269
268
 
270
- logger.info(
271
- "create_db_tasks_and_update_task_group - " "start"
272
- )
269
+ logger.info("create_db_tasks_and_update_task_group - start")
273
270
  create_db_tasks_and_update_task_group_sync(
274
271
  task_list=task_list,
275
272
  task_group_id=task_group.id,
@@ -310,9 +307,7 @@ def collect_ssh(
310
307
  folder=task_group.path,
311
308
  safe_root=profile.tasks_remote_dir,
312
309
  )
313
- logger.info(
314
- f"Deleted remoted folder {task_group.path}"
315
- )
310
+ logger.info(f"Deleted remoted folder {task_group.path}")
316
311
  except Exception as e_rm:
317
312
  logger.error(
318
313
  "Removing folder failed. "
@@ -2,15 +2,6 @@ import time
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ..utils_background import prepare_tasks_metadata
8
- from ..utils_database import create_db_tasks_and_update_task_group_sync
9
- from ..utils_pixi import parse_collect_stdout
10
- from ..utils_pixi import SOURCE_DIR_NAME
11
- from ._pixi_slurm_ssh import run_script_on_remote_slurm
12
- from ._utils import check_ssh_or_fail_and_cleanup
13
- from ._utils import edit_pyproject_toml_in_place_ssh
14
5
  from fractal_server.app.db import get_sync_db
15
6
  from fractal_server.app.models import Profile
16
7
  from fractal_server.app.models import Resource
@@ -25,10 +16,22 @@ from fractal_server.ssh._fabric import SSHConfig
25
16
  from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
26
17
  from fractal_server.tasks.v2.ssh._utils import _customize_and_send_template
27
18
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
19
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
20
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
28
21
  from fractal_server.tasks.v2.utils_background import get_current_log
22
+ from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
23
+ from fractal_server.tasks.v2.utils_database import (
24
+ create_db_tasks_and_update_task_group_sync,
25
+ )
26
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
27
+ from fractal_server.tasks.v2.utils_pixi import parse_collect_stdout
29
28
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
30
29
  from fractal_server.utils import get_timestamp
31
30
 
31
+ from ._pixi_slurm_ssh import run_script_on_remote_slurm
32
+ from ._utils import check_ssh_or_fail_and_cleanup
33
+ from ._utils import edit_pyproject_toml_in_place_ssh
34
+
32
35
 
33
36
  def collect_ssh_pixi(
34
37
  *,
@@ -52,8 +55,9 @@ def collect_ssh_pixi(
52
55
  Args:
53
56
  task_group_id:
54
57
  task_group_activity_id:
55
- ssh_config:
56
58
  tar_gz_file:
59
+ resource:
60
+ profile:
57
61
  """
58
62
 
59
63
  LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -247,9 +251,7 @@ def collect_ssh_pixi(
247
251
  remote_script3_path,
248
252
  f"chmod -R 755 {source_dir}",
249
253
  ],
250
- slurm_config=resource.tasks_pixi_config[
251
- "SLURM_CONFIG"
252
- ],
254
+ slurm_config=resource.tasks_pixi_config["SLURM_CONFIG"],
253
255
  fractal_ssh=fractal_ssh,
254
256
  logger_name=LOGGER_NAME,
255
257
  prefix=common_args["prefix"],
@@ -290,9 +292,7 @@ def collect_ssh_pixi(
290
292
  )
291
293
  logger.info("_prepare_tasks_metadata - end")
292
294
 
293
- logger.info(
294
- "create_db_tasks_and_update_task_group - " "start"
295
- )
295
+ logger.info("create_db_tasks_and_update_task_group - start")
296
296
  create_db_tasks_and_update_task_group_sync(
297
297
  task_list=task_list,
298
298
  task_group_id=task_group.id,
@@ -344,9 +344,7 @@ def collect_ssh_pixi(
344
344
  folder=task_group.path,
345
345
  safe_root=profile.tasks_remote_dir,
346
346
  )
347
- logger.info(
348
- f"Deleted remoted folder {task_group.path}"
349
- )
347
+ logger.info(f"Deleted remoted folder {task_group.path}")
350
348
  except Exception as e_rm:
351
349
  logger.error(
352
350
  "Removing folder failed. "
@@ -2,13 +2,6 @@ import time
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import add_commit_refresh
6
- from ..utils_background import fail_and_cleanup
7
- from ..utils_background import get_activity_and_task_group
8
- from ..utils_templates import get_collection_replacements
9
- from ._utils import _copy_wheel_file_ssh
10
- from ._utils import _customize_and_run_template
11
- from ._utils import check_ssh_or_fail_and_cleanup
12
5
  from fractal_server.app.db import get_sync_db
13
6
  from fractal_server.app.models import Profile
14
7
  from fractal_server.app.models import Resource
@@ -21,10 +14,18 @@ from fractal_server.ssh._fabric import SingleUseFractalSSH
21
14
  from fractal_server.ssh._fabric import SSHConfig
22
15
  from fractal_server.tasks.utils import FORBIDDEN_DEPENDENCY_STRINGS
23
16
  from fractal_server.tasks.utils import get_log_path
17
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
18
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
19
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
24
20
  from fractal_server.tasks.v2.utils_background import get_current_log
25
21
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
22
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
26
23
  from fractal_server.utils import get_timestamp
27
24
 
25
+ from ._utils import _copy_wheel_file_ssh
26
+ from ._utils import _customize_and_run_template
27
+ from ._utils import check_ssh_or_fail_and_cleanup
28
+
28
29
 
29
30
  def deactivate_ssh(
30
31
  *,
@@ -42,7 +43,8 @@ def deactivate_ssh(
42
43
  Args:
43
44
  task_group_id:
44
45
  task_group_activity_id:
45
- ssh_config:
46
+ resource:
47
+ profile:
46
48
  """
47
49
 
48
50
  LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -1,11 +1,6 @@
1
1
  from pathlib import Path
2
2
  from tempfile import TemporaryDirectory
3
3
 
4
- from ..utils_background import add_commit_refresh
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ..utils_pixi import SOURCE_DIR_NAME
8
- from ._utils import check_ssh_or_fail_and_cleanup
9
4
  from fractal_server.app.db import get_sync_db
10
5
  from fractal_server.app.models import Profile
11
6
  from fractal_server.app.models import Resource
@@ -15,9 +10,15 @@ from fractal_server.logger import set_logger
15
10
  from fractal_server.ssh._fabric import SingleUseFractalSSH
16
11
  from fractal_server.ssh._fabric import SSHConfig
17
12
  from fractal_server.tasks.utils import get_log_path
13
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
14
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
15
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
18
16
  from fractal_server.tasks.v2.utils_background import get_current_log
17
+ from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
19
18
  from fractal_server.utils import get_timestamp
20
19
 
20
+ from ._utils import check_ssh_or_fail_and_cleanup
21
+
21
22
 
22
23
  def deactivate_ssh_pixi(
23
24
  *,
@@ -1,10 +1,6 @@
1
1
  from pathlib import Path
2
2
  from tempfile import TemporaryDirectory
3
3
 
4
- from ..utils_background import add_commit_refresh
5
- from ..utils_background import fail_and_cleanup
6
- from ..utils_background import get_activity_and_task_group
7
- from ._utils import check_ssh_or_fail_and_cleanup
8
4
  from fractal_server.app.db import get_sync_db
9
5
  from fractal_server.app.models import Profile
10
6
  from fractal_server.app.models import Resource
@@ -15,9 +11,14 @@ from fractal_server.logger import set_logger
15
11
  from fractal_server.ssh._fabric import SingleUseFractalSSH
16
12
  from fractal_server.ssh._fabric import SSHConfig
17
13
  from fractal_server.tasks.utils import get_log_path
14
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
15
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
16
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
18
17
  from fractal_server.tasks.v2.utils_background import get_current_log
19
18
  from fractal_server.utils import get_timestamp
20
19
 
20
+ from ._utils import check_ssh_or_fail_and_cleanup
21
+
21
22
 
22
23
  def delete_ssh(
23
24
  *,
@@ -35,7 +36,8 @@ def delete_ssh(
35
36
  Args:
36
37
  task_group_id:
37
38
  task_group_activity_id:
38
- ssh_config:
39
+ resource:
40
+ profile:
39
41
  """
40
42
 
41
43
  LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -2,12 +2,6 @@ import time
2
2
  from pathlib import Path
3
3
  from tempfile import TemporaryDirectory
4
4
 
5
- from ..utils_background import add_commit_refresh
6
- from ..utils_background import fail_and_cleanup
7
- from ..utils_background import get_activity_and_task_group
8
- from ..utils_templates import get_collection_replacements
9
- from ._utils import _customize_and_run_template
10
- from ._utils import check_ssh_or_fail_and_cleanup
11
5
  from fractal_server.app.db import get_sync_db
12
6
  from fractal_server.app.models import Profile
13
7
  from fractal_server.app.models import Resource
@@ -18,13 +12,20 @@ from fractal_server.logger import set_logger
18
12
  from fractal_server.ssh._fabric import SingleUseFractalSSH
19
13
  from fractal_server.ssh._fabric import SSHConfig
20
14
  from fractal_server.tasks.utils import get_log_path
15
+ from fractal_server.tasks.v2.utils_background import add_commit_refresh
16
+ from fractal_server.tasks.v2.utils_background import fail_and_cleanup
17
+ from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
21
18
  from fractal_server.tasks.v2.utils_background import get_current_log
22
19
  from fractal_server.tasks.v2.utils_python_interpreter import (
23
20
  get_python_interpreter,
24
21
  )
25
22
  from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
23
+ from fractal_server.tasks.v2.utils_templates import get_collection_replacements
26
24
  from fractal_server.utils import get_timestamp
27
25
 
26
+ from ._utils import _customize_and_run_template
27
+ from ._utils import check_ssh_or_fail_and_cleanup
28
+
28
29
 
29
30
  def reactivate_ssh(
30
31
  *,
@@ -40,9 +41,10 @@ def reactivate_ssh(
40
41
  handled.
41
42
 
42
43
  Args:
43
- task_group_id:
44
44
  task_group_activity_id:
45
- ssh_config:
45
+ task_group_id:
46
+ resource:
47
+ profile:
46
48
  """
47
49
 
48
50
  LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -182,9 +184,7 @@ def reactivate_ssh(
182
184
  except Exception as reactivate_e:
183
185
  # Delete corrupted venv_path
184
186
  try:
185
- logger.info(
186
- f"Now delete folder {task_group.venv_path}"
187
- )
187
+ logger.info(f"Now delete folder {task_group.venv_path}")
188
188
  fractal_ssh.remove_folder(
189
189
  folder=task_group.venv_path,
190
190
  safe_root=profile.tasks_remote_dir,