fractal-server 2.17.1a1__py3-none-any.whl → 2.18.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +21 -19
  3. fractal_server/app/db/__init__.py +3 -3
  4. fractal_server/app/models/__init__.py +1 -0
  5. fractal_server/app/models/linkuserproject.py +43 -1
  6. fractal_server/app/models/security.py +28 -8
  7. fractal_server/app/models/v2/__init__.py +3 -1
  8. fractal_server/app/models/v2/accounting.py +9 -1
  9. fractal_server/app/models/v2/dataset.py +5 -1
  10. fractal_server/app/models/v2/history.py +15 -1
  11. fractal_server/app/models/v2/job.py +17 -2
  12. fractal_server/app/models/v2/profile.py +29 -0
  13. fractal_server/app/models/v2/project.py +4 -10
  14. fractal_server/app/models/v2/resource.py +17 -0
  15. fractal_server/app/models/v2/task_group.py +4 -3
  16. fractal_server/app/models/v2/workflow.py +2 -1
  17. fractal_server/app/routes/admin/v2/__init__.py +12 -13
  18. fractal_server/app/routes/admin/v2/accounting.py +3 -3
  19. fractal_server/app/routes/admin/v2/job.py +35 -24
  20. fractal_server/app/routes/admin/v2/profile.py +3 -2
  21. fractal_server/app/routes/admin/v2/resource.py +5 -5
  22. fractal_server/app/routes/admin/v2/sharing.py +103 -0
  23. fractal_server/app/routes/admin/v2/task.py +37 -26
  24. fractal_server/app/routes/admin/v2/task_group.py +94 -17
  25. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +21 -22
  26. fractal_server/app/routes/api/__init__.py +1 -9
  27. fractal_server/app/routes/api/v2/__init__.py +49 -50
  28. fractal_server/app/routes/api/v2/_aux_functions.py +132 -124
  29. fractal_server/app/routes/api/v2/_aux_functions_history.py +51 -23
  30. fractal_server/app/routes/api/v2/_aux_functions_sharing.py +97 -0
  31. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +6 -8
  32. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +7 -9
  33. fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
  34. fractal_server/app/routes/api/v2/dataset.py +95 -102
  35. fractal_server/app/routes/api/v2/history.py +59 -33
  36. fractal_server/app/routes/api/v2/images.py +24 -9
  37. fractal_server/app/routes/api/v2/job.py +52 -33
  38. fractal_server/app/routes/api/v2/pre_submission_checks.py +16 -8
  39. fractal_server/app/routes/api/v2/project.py +65 -37
  40. fractal_server/app/routes/api/v2/sharing.py +311 -0
  41. fractal_server/app/routes/api/v2/status_legacy.py +31 -41
  42. fractal_server/app/routes/api/v2/submit.py +82 -78
  43. fractal_server/app/routes/api/v2/task.py +19 -20
  44. fractal_server/app/routes/api/v2/task_collection.py +41 -43
  45. fractal_server/app/routes/api/v2/task_collection_custom.py +19 -20
  46. fractal_server/app/routes/api/v2/task_collection_pixi.py +10 -11
  47. fractal_server/app/routes/api/v2/task_group.py +25 -24
  48. fractal_server/app/routes/api/v2/task_group_lifecycle.py +32 -32
  49. fractal_server/app/routes/api/v2/task_version_update.py +23 -19
  50. fractal_server/app/routes/api/v2/workflow.py +50 -55
  51. fractal_server/app/routes/api/v2/workflow_import.py +37 -37
  52. fractal_server/app/routes/api/v2/workflowtask.py +32 -26
  53. fractal_server/app/routes/auth/__init__.py +1 -3
  54. fractal_server/app/routes/auth/_aux_auth.py +101 -2
  55. fractal_server/app/routes/auth/current_user.py +2 -66
  56. fractal_server/app/routes/auth/group.py +8 -35
  57. fractal_server/app/routes/auth/login.py +1 -0
  58. fractal_server/app/routes/auth/oauth.py +4 -3
  59. fractal_server/app/routes/auth/register.py +4 -2
  60. fractal_server/app/routes/auth/router.py +2 -0
  61. fractal_server/app/routes/auth/users.py +19 -10
  62. fractal_server/app/routes/auth/viewer_paths.py +43 -0
  63. fractal_server/app/routes/aux/_job.py +1 -1
  64. fractal_server/app/routes/aux/_runner.py +2 -2
  65. fractal_server/app/routes/pagination.py +1 -1
  66. fractal_server/app/schemas/user.py +29 -12
  67. fractal_server/app/schemas/user_group.py +0 -15
  68. fractal_server/app/schemas/v2/__init__.py +55 -48
  69. fractal_server/app/schemas/v2/accounting.py +11 -0
  70. fractal_server/app/schemas/v2/dataset.py +57 -11
  71. fractal_server/app/schemas/v2/dumps.py +10 -9
  72. fractal_server/app/schemas/v2/job.py +11 -11
  73. fractal_server/app/schemas/v2/manifest.py +4 -3
  74. fractal_server/app/schemas/v2/profile.py +53 -2
  75. fractal_server/app/schemas/v2/project.py +3 -3
  76. fractal_server/app/schemas/v2/resource.py +121 -16
  77. fractal_server/app/schemas/v2/sharing.py +99 -0
  78. fractal_server/app/schemas/v2/status_legacy.py +3 -3
  79. fractal_server/app/schemas/v2/task.py +6 -7
  80. fractal_server/app/schemas/v2/task_collection.py +5 -5
  81. fractal_server/app/schemas/v2/task_group.py +16 -16
  82. fractal_server/app/schemas/v2/workflow.py +16 -16
  83. fractal_server/app/schemas/v2/workflowtask.py +16 -15
  84. fractal_server/app/security/__init__.py +5 -8
  85. fractal_server/app/security/signup_email.py +4 -5
  86. fractal_server/app/shutdown.py +6 -6
  87. fractal_server/config/__init__.py +0 -6
  88. fractal_server/config/_data.py +0 -68
  89. fractal_server/config/_database.py +19 -20
  90. fractal_server/config/_email.py +30 -38
  91. fractal_server/config/_main.py +38 -52
  92. fractal_server/config/_oauth.py +17 -21
  93. fractal_server/data_migrations/2_18_0.py +30 -0
  94. fractal_server/exceptions.py +4 -0
  95. fractal_server/images/models.py +4 -5
  96. fractal_server/images/status_tools.py +4 -2
  97. fractal_server/logger.py +1 -1
  98. fractal_server/main.py +75 -13
  99. fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
  100. fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
  101. fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
  102. fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
  103. fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
  104. fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
  105. fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
  106. fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
  107. fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
  108. fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
  109. fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
  110. fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
  111. fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
  112. fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
  113. fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
  114. fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
  115. fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
  116. fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
  117. fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
  118. fractal_server/migrations/versions/7910eed4cf97_user_project_dirs_and_usergroup_viewer_.py +60 -0
  119. fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
  120. fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
  121. fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
  122. fractal_server/migrations/versions/88270f589c9b_add_prevent_new_submissions.py +39 -0
  123. fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
  124. fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
  125. fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
  126. fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
  127. fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
  128. fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
  129. fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
  130. fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
  131. fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
  132. fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
  133. fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
  134. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
  135. fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
  136. fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
  137. fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
  138. fractal_server/migrations/versions/bc0e8b3327a7_project_sharing.py +72 -0
  139. fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
  140. fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
  141. fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
  142. fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
  143. fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
  144. fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
  145. fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
  146. fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
  147. fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
  148. fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
  149. fractal_server/migrations/versions/f0702066b007_one_submitted_job_per_dataset.py +40 -0
  150. fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
  151. fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
  152. fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
  153. fractal_server/runner/config/_local.py +8 -5
  154. fractal_server/runner/config/_slurm.py +39 -33
  155. fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
  156. fractal_server/runner/executors/base_runner.py +29 -4
  157. fractal_server/runner/executors/local/get_local_config.py +1 -0
  158. fractal_server/runner/executors/local/runner.py +14 -13
  159. fractal_server/runner/executors/slurm_common/_batching.py +9 -20
  160. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
  161. fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
  162. fractal_server/runner/executors/slurm_common/remote.py +3 -1
  163. fractal_server/runner/executors/slurm_common/slurm_config.py +2 -0
  164. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
  165. fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
  166. fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
  167. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
  168. fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
  169. fractal_server/runner/task_files.py +9 -3
  170. fractal_server/runner/v2/_local.py +12 -6
  171. fractal_server/runner/v2/_slurm_ssh.py +14 -7
  172. fractal_server/runner/v2/_slurm_sudo.py +14 -7
  173. fractal_server/runner/v2/db_tools.py +0 -1
  174. fractal_server/runner/v2/deduplicate_list.py +2 -1
  175. fractal_server/runner/v2/runner.py +44 -28
  176. fractal_server/runner/v2/runner_functions.py +22 -28
  177. fractal_server/runner/v2/submit_workflow.py +29 -15
  178. fractal_server/ssh/_fabric.py +6 -13
  179. fractal_server/string_tools.py +0 -1
  180. fractal_server/syringe.py +1 -1
  181. fractal_server/tasks/config/_pixi.py +1 -1
  182. fractal_server/tasks/config/_python.py +16 -9
  183. fractal_server/tasks/utils.py +0 -1
  184. fractal_server/tasks/v2/local/_utils.py +3 -3
  185. fractal_server/tasks/v2/local/collect.py +15 -18
  186. fractal_server/tasks/v2/local/collect_pixi.py +14 -16
  187. fractal_server/tasks/v2/local/deactivate.py +14 -15
  188. fractal_server/tasks/v2/local/deactivate_pixi.py +7 -7
  189. fractal_server/tasks/v2/local/delete.py +6 -8
  190. fractal_server/tasks/v2/local/reactivate.py +12 -12
  191. fractal_server/tasks/v2/local/reactivate_pixi.py +12 -12
  192. fractal_server/tasks/v2/ssh/_utils.py +3 -3
  193. fractal_server/tasks/v2/ssh/collect.py +19 -24
  194. fractal_server/tasks/v2/ssh/collect_pixi.py +22 -24
  195. fractal_server/tasks/v2/ssh/deactivate.py +17 -15
  196. fractal_server/tasks/v2/ssh/deactivate_pixi.py +8 -7
  197. fractal_server/tasks/v2/ssh/delete.py +12 -10
  198. fractal_server/tasks/v2/ssh/reactivate.py +16 -16
  199. fractal_server/tasks/v2/ssh/reactivate_pixi.py +13 -14
  200. fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
  201. fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
  202. fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
  203. fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
  204. fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
  205. fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
  206. fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
  207. fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
  208. fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
  209. fractal_server/tasks/v2/utils_background.py +10 -10
  210. fractal_server/tasks/v2/utils_database.py +5 -5
  211. fractal_server/tasks/v2/utils_package_names.py +1 -2
  212. fractal_server/tasks/v2/utils_pixi.py +1 -3
  213. fractal_server/types/__init__.py +98 -1
  214. fractal_server/types/validators/__init__.py +3 -0
  215. fractal_server/types/validators/_common_validators.py +33 -3
  216. fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
  217. fractal_server/utils.py +1 -0
  218. fractal_server/zip_tools.py +34 -0
  219. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/METADATA +3 -2
  220. fractal_server-2.18.0.dist-info/RECORD +275 -0
  221. fractal_server/app/routes/admin/v2/project.py +0 -41
  222. fractal_server-2.17.1a1.dist-info/RECORD +0 -264
  223. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/WHEEL +0 -0
  224. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/entry_points.txt +0 -0
  225. {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/licenses/LICENSE +0 -0
@@ -9,12 +9,8 @@ from fastapi import HTTPException
9
9
  from fastapi import Request
10
10
  from fastapi import status
11
11
  from sqlmodel import select
12
+ from sqlmodel import update
12
13
 
13
- from ...aux.validate_user_profile import validate_user_profile
14
- from ._aux_functions import _get_dataset_check_owner
15
- from ._aux_functions import _get_workflow_check_owner
16
- from ._aux_functions import clean_app_job_list_v2
17
- from ._aux_functions_tasks import _check_type_filters_compatibility
18
14
  from fractal_server.app.db import AsyncSession
19
15
  from fractal_server.app.db import get_async_db
20
16
  from fractal_server.app.models import Profile
@@ -25,10 +21,14 @@ from fractal_server.app.routes.api.v2._aux_functions_tasks import (
25
21
  _get_task_read_access,
26
22
  )
27
23
  from fractal_server.app.routes.auth import current_user_act_ver_prof
28
- from fractal_server.app.schemas.v2 import JobCreateV2
29
- from fractal_server.app.schemas.v2 import JobReadV2
30
- from fractal_server.app.schemas.v2 import JobStatusTypeV2
24
+ from fractal_server.app.routes.aux.validate_user_profile import (
25
+ validate_user_profile,
26
+ )
27
+ from fractal_server.app.schemas.v2 import JobCreate
28
+ from fractal_server.app.schemas.v2 import JobRead
29
+ from fractal_server.app.schemas.v2 import JobStatusType
31
30
  from fractal_server.app.schemas.v2 import ResourceType
31
+ from fractal_server.app.schemas.v2.sharing import ProjectPermissions
32
32
  from fractal_server.config import get_settings
33
33
  from fractal_server.logger import set_logger
34
34
  from fractal_server.runner.set_start_and_last_task_index import (
@@ -37,6 +37,11 @@ from fractal_server.runner.set_start_and_last_task_index import (
37
37
  from fractal_server.runner.v2.submit_workflow import submit_workflow
38
38
  from fractal_server.syringe import Inject
39
39
 
40
+ from ._aux_functions import _get_dataset_check_access
41
+ from ._aux_functions import _get_workflow_check_access
42
+ from ._aux_functions import clean_app_job_list
43
+ from ._aux_functions_tasks import _check_type_filters_compatibility
44
+
40
45
  FRACTAL_CACHE_DIR = ".fractal_cache"
41
46
  router = APIRouter()
42
47
  logger = set_logger(__name__)
@@ -45,37 +50,33 @@ logger = set_logger(__name__)
45
50
  @router.post(
46
51
  "/project/{project_id}/job/submit/",
47
52
  status_code=status.HTTP_202_ACCEPTED,
48
- response_model=JobReadV2,
53
+ response_model=JobRead,
49
54
  )
50
- async def apply_workflow(
55
+ async def submit_job(
51
56
  project_id: int,
52
57
  workflow_id: int,
53
58
  dataset_id: int,
54
- job_create: JobCreateV2,
59
+ job_create: JobCreate,
55
60
  background_tasks: BackgroundTasks,
56
61
  request: Request,
57
62
  user: UserOAuth = Depends(current_user_act_ver_prof),
58
63
  db: AsyncSession = Depends(get_async_db),
59
- ) -> JobReadV2 | None:
60
- # Remove non-submitted V2 jobs from the app state when the list grows
64
+ ) -> JobRead | None:
65
+ # Remove non-submitted Jobs from the app state when the list grows
61
66
  # beyond a threshold
62
- # NOTE: this may lead to a race condition on `app.state.jobsV2` if two
63
- # requests take place at the same time and `clean_app_job_list_v2` is
67
+ # NOTE: this may lead to a race condition on `app.state.jobs` if two
68
+ # requests take place at the same time and `clean_app_job_list` is
64
69
  # somewhat slow.
65
70
  settings = Inject(get_settings)
66
- if (
67
- len(request.app.state.jobsV2)
68
- > settings.FRACTAL_API_MAX_JOB_LIST_LENGTH
69
- ):
70
- new_jobs_list = await clean_app_job_list_v2(
71
- db, request.app.state.jobsV2
72
- )
73
- request.app.state.jobsV2 = new_jobs_list
71
+ if len(request.app.state.jobs) > settings.FRACTAL_API_MAX_JOB_LIST_LENGTH:
72
+ new_jobs_list = await clean_app_job_list(db, request.app.state.jobs)
73
+ request.app.state.jobs = new_jobs_list
74
74
 
75
- output = await _get_dataset_check_owner(
75
+ output = await _get_dataset_check_access(
76
76
  project_id=project_id,
77
77
  dataset_id=dataset_id,
78
78
  user_id=user.id,
79
+ required_permissions=ProjectPermissions.EXECUTE,
79
80
  db=db,
80
81
  )
81
82
  project = output["project"]
@@ -92,8 +93,12 @@ async def apply_workflow(
92
93
  detail="Project resource does not match with user's resource",
93
94
  )
94
95
 
95
- workflow = await _get_workflow_check_owner(
96
- project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
96
+ workflow = await _get_workflow_check_access(
97
+ project_id=project_id,
98
+ workflow_id=workflow_id,
99
+ user_id=user.id,
100
+ required_permissions=ProjectPermissions.EXECUTE,
101
+ db=db,
97
102
  )
98
103
  num_tasks = len(workflow.task_list)
99
104
  if num_tasks == 0:
@@ -141,36 +146,15 @@ async def apply_workflow(
141
146
  user=user,
142
147
  db=db,
143
148
  )
144
-
145
- # Check that no other job with the same dataset_id is SUBMITTED
146
- stm = (
147
- select(JobV2)
148
- .where(JobV2.dataset_id == dataset_id)
149
- .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
150
- )
151
- res = await db.execute(stm)
152
- if res.scalars().all():
149
+ if resource.prevent_new_submissions:
153
150
  raise HTTPException(
154
151
  status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
155
152
  detail=(
156
- f"Dataset {dataset_id} is already in use "
157
- "in submitted job(s)."
153
+ f"The '{resource.name}' resource does not currently accept "
154
+ "new job submissions."
158
155
  ),
159
156
  )
160
157
 
161
- if job_create.slurm_account is not None:
162
- if job_create.slurm_account not in user.slurm_accounts:
163
- raise HTTPException(
164
- status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
165
- detail=(
166
- f"SLURM account '{job_create.slurm_account}' is not "
167
- "among those available to the current user"
168
- ),
169
- )
170
- else:
171
- if len(user.slurm_accounts) > 0:
172
- job_create.slurm_account = user.slurm_accounts[0]
173
-
174
158
  # User appropriate FractalSSH object
175
159
  if resource.type == ResourceType.SLURM_SSH:
176
160
  ssh_config = dict(
@@ -193,6 +177,35 @@ async def apply_workflow(
193
177
  else:
194
178
  fractal_ssh = None
195
179
 
180
+ # Assign `job_create.slurm_account`
181
+ if job_create.slurm_account is not None:
182
+ if job_create.slurm_account not in user.slurm_accounts:
183
+ raise HTTPException(
184
+ status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
185
+ detail=(
186
+ f"SLURM account '{job_create.slurm_account}' is not "
187
+ "among those available to the current user"
188
+ ),
189
+ )
190
+ else:
191
+ if len(user.slurm_accounts) > 0:
192
+ job_create.slurm_account = user.slurm_accounts[0]
193
+
194
+ # Check that no other job with the same dataset_id is SUBMITTED
195
+ stm = (
196
+ select(JobV2)
197
+ .where(JobV2.dataset_id == dataset_id)
198
+ .where(JobV2.status == JobStatusType.SUBMITTED)
199
+ )
200
+ res = await db.execute(stm)
201
+ if res.scalars().all():
202
+ raise HTTPException(
203
+ status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
204
+ detail=(
205
+ f"Dataset {dataset_id} is already in use in submitted job(s)."
206
+ ),
207
+ )
208
+
196
209
  # Add new Job object to DB
197
210
  job = JobV2(
198
211
  project_id=project_id,
@@ -206,7 +219,7 @@ async def apply_workflow(
206
219
  workflow.model_dump_json(exclude={"task_list"})
207
220
  ),
208
221
  project_dump=json.loads(
209
- project.model_dump_json(exclude={"user_list", "resource_id"})
222
+ project.model_dump_json(exclude={"resource_id"})
210
223
  ),
211
224
  **job_create.model_dump(),
212
225
  )
@@ -216,38 +229,31 @@ async def apply_workflow(
216
229
  await db.refresh(job)
217
230
 
218
231
  # Update TaskGroupV2.timestamp_last_used
219
- res = await db.execute(
220
- select(TaskGroupV2).where(TaskGroupV2.id.in_(used_task_group_ids))
232
+ await db.execute(
233
+ update(TaskGroupV2)
234
+ .where(TaskGroupV2.id.in_(used_task_group_ids))
235
+ .values(timestamp_last_used=job.start_timestamp)
221
236
  )
222
- used_task_groups = res.scalars().all()
223
- for used_task_group in used_task_groups:
224
- used_task_group.timestamp_last_used = job.start_timestamp
225
- db.add(used_task_group)
226
237
  await db.commit()
227
238
 
228
- # Define server-side job directory
229
- timestamp_string = job.start_timestamp.strftime("%Y%m%d_%H%M%S")
230
- WORKFLOW_DIR_LOCAL = Path(resource.jobs_local_dir) / (
239
+ # Define `cache_dir`
240
+ cache_dir = Path(user.project_dirs[0], FRACTAL_CACHE_DIR)
241
+
242
+ # Define server-side and user-side job directories
243
+ timestamp_string = job.start_timestamp.strftime(r"%Y%m%d_%H%M%S")
244
+ working_dir = Path(resource.jobs_local_dir) / (
231
245
  f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
232
246
  f"_{timestamp_string}"
233
247
  )
234
-
235
- # Define user-side job directory
236
- cache_dir = Path(user.project_dir, FRACTAL_CACHE_DIR)
237
248
  match resource.type:
238
249
  case ResourceType.LOCAL:
239
- WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
250
+ working_dir_user = working_dir
240
251
  case ResourceType.SLURM_SUDO:
241
- WORKFLOW_DIR_REMOTE = cache_dir / WORKFLOW_DIR_LOCAL.name
252
+ working_dir_user = cache_dir / working_dir.name
242
253
  case ResourceType.SLURM_SSH:
243
- WORKFLOW_DIR_REMOTE = Path(
244
- profile.jobs_remote_dir,
245
- WORKFLOW_DIR_LOCAL.name,
246
- )
247
-
248
- # Update job folders in the db
249
- job.working_dir = WORKFLOW_DIR_LOCAL.as_posix()
250
- job.working_dir_user = WORKFLOW_DIR_REMOTE.as_posix()
254
+ working_dir_user = Path(profile.jobs_remote_dir, working_dir.name)
255
+ job.working_dir = working_dir.as_posix()
256
+ job.working_dir_user = working_dir_user.as_posix()
251
257
  await db.merge(job)
252
258
  await db.commit()
253
259
 
@@ -263,11 +269,9 @@ async def apply_workflow(
263
269
  resource=resource,
264
270
  profile=profile,
265
271
  )
266
- request.app.state.jobsV2.append(job.id)
272
+ request.app.state.jobs.append(job.id)
267
273
  logger.info(
268
- f"Current worker's pid is {os.getpid()}. "
269
- f"Current status of worker job's list "
270
- f"{request.app.state.jobsV2}"
274
+ f"Job {job.id}, worker with pid {os.getpid()}. "
275
+ f"Worker jobs list: {request.app.state.jobs}."
271
276
  )
272
- await db.close()
273
277
  return job
@@ -9,7 +9,9 @@ from sqlmodel import func
9
9
  from sqlmodel import or_
10
10
  from sqlmodel import select
11
11
 
12
- from ...aux.validate_user_profile import validate_user_profile
12
+ from fractal_server.app.routes.aux.validate_user_profile import (
13
+ validate_user_profile,
14
+ )
13
15
  from ._aux_functions import _get_user_resource_id
14
16
  from ._aux_functions_tasks import _get_task_full_access
15
17
  from ._aux_functions_tasks import _get_task_read_access
@@ -23,11 +25,11 @@ from fractal_server.app.models import UserOAuth
23
25
  from fractal_server.app.models.v2 import TaskGroupV2
24
26
  from fractal_server.app.models.v2 import TaskV2
25
27
  from fractal_server.app.routes.auth import current_user_act_ver_prof
26
- from fractal_server.app.schemas.v2 import TaskCreateV2
27
- from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
28
- from fractal_server.app.schemas.v2 import TaskReadV2
28
+ from fractal_server.app.schemas.v2 import TaskCreate
29
+ from fractal_server.app.schemas.v2 import TaskGroupOriginEnum
30
+ from fractal_server.app.schemas.v2 import TaskRead
29
31
  from fractal_server.app.schemas.v2 import TaskType
30
- from fractal_server.app.schemas.v2 import TaskUpdateV2
32
+ from fractal_server.app.schemas.v2 import TaskUpdate
31
33
  from fractal_server.logger import set_logger
32
34
 
33
35
  router = APIRouter()
@@ -35,7 +37,7 @@ router = APIRouter()
35
37
  logger = set_logger(__name__)
36
38
 
37
39
 
38
- @router.get("/", response_model=list[TaskReadV2])
40
+ @router.get("/", response_model=list[TaskRead])
39
41
  async def get_list_task(
40
42
  args_schema: bool = True,
41
43
  category: str | None = None,
@@ -43,7 +45,7 @@ async def get_list_task(
43
45
  author: str | None = None,
44
46
  user: UserOAuth = Depends(current_user_act_ver_prof),
45
47
  db: AsyncSession = Depends(get_async_db),
46
- ) -> list[TaskReadV2]:
48
+ ) -> list[TaskRead]:
47
49
  """
48
50
  Get list of available tasks
49
51
  """
@@ -52,8 +54,7 @@ async def get_list_task(
52
54
 
53
55
  stm = (
54
56
  select(TaskV2)
55
- .join(TaskGroupV2)
56
- .where(TaskGroupV2.id == TaskV2.taskgroupv2_id)
57
+ .join(TaskGroupV2, TaskGroupV2.id == TaskV2.taskgroupv2_id)
57
58
  .where(TaskGroupV2.resource_id == user_resource_id)
58
59
  .where(
59
60
  or_(
@@ -85,12 +86,12 @@ async def get_list_task(
85
86
  return task_list
86
87
 
87
88
 
88
- @router.get("/{task_id}/", response_model=TaskReadV2)
89
+ @router.get("/{task_id}/", response_model=TaskRead)
89
90
  async def get_task(
90
91
  task_id: int,
91
92
  user: UserOAuth = Depends(current_user_act_ver_prof),
92
93
  db: AsyncSession = Depends(get_async_db),
93
- ) -> TaskReadV2:
94
+ ) -> TaskRead:
94
95
  """
95
96
  Get info on a specific task
96
97
  """
@@ -98,13 +99,13 @@ async def get_task(
98
99
  return task
99
100
 
100
101
 
101
- @router.patch("/{task_id}/", response_model=TaskReadV2)
102
+ @router.patch("/{task_id}/", response_model=TaskRead)
102
103
  async def patch_task(
103
104
  task_id: int,
104
- task_update: TaskUpdateV2,
105
+ task_update: TaskUpdate,
105
106
  user: UserOAuth = Depends(current_user_act_ver_prof),
106
107
  db: AsyncSession = Depends(get_async_db),
107
- ) -> TaskReadV2 | None:
108
+ ) -> TaskRead | None:
108
109
  """
109
110
  Edit a specific task (restricted to task owner)
110
111
  """
@@ -136,16 +137,14 @@ async def patch_task(
136
137
  return db_task
137
138
 
138
139
 
139
- @router.post(
140
- "/", response_model=TaskReadV2, status_code=status.HTTP_201_CREATED
141
- )
140
+ @router.post("/", response_model=TaskRead, status_code=status.HTTP_201_CREATED)
142
141
  async def create_task(
143
- task: TaskCreateV2,
142
+ task: TaskCreate,
144
143
  user_group_id: int | None = None,
145
144
  private: bool = False,
146
145
  user: UserOAuth = Depends(current_user_act_ver_prof),
147
146
  db: AsyncSession = Depends(get_async_db),
148
- ) -> TaskReadV2 | None:
147
+ ) -> TaskRead | None:
149
148
  """
150
149
  Create a new task
151
150
  """
@@ -210,7 +209,7 @@ async def create_task(
210
209
  resource_id=resource_id,
211
210
  active=True,
212
211
  task_list=[db_task],
213
- origin=TaskGroupV2OriginEnum.OTHER,
212
+ origin=TaskGroupOriginEnum.OTHER,
214
213
  version=db_task.version,
215
214
  pkg_name=pkg_name,
216
215
  )
@@ -8,39 +8,32 @@ from fastapi import File
8
8
  from fastapi import Form
9
9
  from fastapi import HTTPException
10
10
  from fastapi import Response
11
- from fastapi import status
12
11
  from fastapi import UploadFile
12
+ from fastapi import status
13
13
  from pydantic import BaseModel
14
- from pydantic import model_validator
15
14
  from pydantic import ValidationError
15
+ from pydantic import model_validator
16
16
 
17
- from .....logger import reset_logger_handlers
18
- from .....logger import set_logger
19
- from ....db import AsyncSession
20
- from ....db import get_async_db
21
- from ....models.v2 import TaskGroupV2
22
- from ....schemas.v2 import FractalUploadedFile
23
- from ....schemas.v2 import TaskCollectPipV2
24
- from ....schemas.v2 import TaskGroupActivityStatusV2
25
- from ....schemas.v2 import TaskGroupActivityV2Read
26
- from ....schemas.v2 import TaskGroupCreateV2Strict
27
- from ...aux.validate_user_profile import validate_user_profile
28
- from ._aux_functions_task_lifecycle import get_package_version_from_pypi
29
- from ._aux_functions_tasks import _get_valid_user_group_id
30
- from ._aux_functions_tasks import _verify_non_duplication_group_constraint
31
- from ._aux_functions_tasks import _verify_non_duplication_group_path
32
- from ._aux_functions_tasks import _verify_non_duplication_user_constraint
17
+ from fractal_server.app.db import AsyncSession
18
+ from fractal_server.app.db import get_async_db
33
19
  from fractal_server.app.models import UserOAuth
34
20
  from fractal_server.app.models.v2 import TaskGroupActivityV2
21
+ from fractal_server.app.models.v2 import TaskGroupV2
35
22
  from fractal_server.app.routes.auth import current_user_act_ver_prof
36
- from fractal_server.app.schemas.v2 import ResourceType
37
- from fractal_server.app.schemas.v2 import (
38
- TaskGroupActivityActionV2,
39
- )
40
- from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
41
- from fractal_server.tasks.v2.local.collect import (
42
- collect_local,
23
+ from fractal_server.app.routes.aux.validate_user_profile import (
24
+ validate_user_profile,
43
25
  )
26
+ from fractal_server.app.schemas.v2 import FractalUploadedFile
27
+ from fractal_server.app.schemas.v2 import ResourceType
28
+ from fractal_server.app.schemas.v2 import TaskCollectPip
29
+ from fractal_server.app.schemas.v2 import TaskGroupActivityAction
30
+ from fractal_server.app.schemas.v2 import TaskGroupActivityRead
31
+ from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
32
+ from fractal_server.app.schemas.v2 import TaskGroupCreateStrict
33
+ from fractal_server.app.schemas.v2 import TaskGroupOriginEnum
34
+ from fractal_server.logger import reset_logger_handlers
35
+ from fractal_server.logger import set_logger
36
+ from fractal_server.tasks.v2.local.collect import collect_local
44
37
  from fractal_server.tasks.v2.ssh import collect_ssh
45
38
  from fractal_server.tasks.v2.utils_package_names import _parse_wheel_filename
46
39
  from fractal_server.tasks.v2.utils_package_names import normalize_package_name
@@ -48,6 +41,11 @@ from fractal_server.tasks.v2.utils_python_interpreter import (
48
41
  get_python_interpreter,
49
42
  )
50
43
 
44
+ from ._aux_functions_task_lifecycle import get_package_version_from_pypi
45
+ from ._aux_functions_tasks import _get_valid_user_group_id
46
+ from ._aux_functions_tasks import _verify_non_duplication_group_constraint
47
+ from ._aux_functions_tasks import _verify_non_duplication_group_path
48
+ from ._aux_functions_tasks import _verify_non_duplication_user_constraint
51
49
 
52
50
  router = APIRouter()
53
51
 
@@ -61,9 +59,9 @@ class CollectionRequestData(BaseModel):
61
59
  Validate form data _and_ wheel file.
62
60
  """
63
61
 
64
- task_collect: TaskCollectPipV2
62
+ task_collect: TaskCollectPip
65
63
  file: UploadFile | None = None
66
- origin: TaskGroupV2OriginEnum
64
+ origin: TaskGroupOriginEnum
67
65
 
68
66
  @model_validator(mode="before")
69
67
  @classmethod
@@ -77,7 +75,7 @@ class CollectionRequestData(BaseModel):
77
75
  raise ValueError(
78
76
  "When no `file` is provided, `package` is required."
79
77
  )
80
- values["origin"] = TaskGroupV2OriginEnum.PYPI
78
+ values["origin"] = TaskGroupOriginEnum.PYPI
81
79
  else:
82
80
  if package is not None:
83
81
  raise ValueError(
@@ -89,7 +87,7 @@ class CollectionRequestData(BaseModel):
89
87
  "Cannot set `package_version` when `file` is "
90
88
  f"provided (given package_version='{package_version}')."
91
89
  )
92
- values["origin"] = TaskGroupV2OriginEnum.WHEELFILE
90
+ values["origin"] = TaskGroupOriginEnum.WHEELFILE
93
91
 
94
92
  for forbidden_char in FORBIDDEN_CHAR_WHEEL:
95
93
  if forbidden_char in file.filename:
@@ -127,7 +125,7 @@ def parse_request_data(
127
125
  else None
128
126
  )
129
127
  # Validate and coerce form data
130
- task_collect_pip = TaskCollectPipV2(
128
+ task_collect_pip = TaskCollectPip(
131
129
  package=package,
132
130
  package_version=package_version,
133
131
  package_extras=package_extras,
@@ -152,7 +150,7 @@ def parse_request_data(
152
150
 
153
151
  @router.post(
154
152
  "/collect/pip/",
155
- response_model=TaskGroupActivityV2Read,
153
+ response_model=TaskGroupActivityRead,
156
154
  )
157
155
  async def collect_tasks_pip(
158
156
  response: Response,
@@ -162,7 +160,7 @@ async def collect_tasks_pip(
162
160
  user_group_id: int | None = None,
163
161
  user: UserOAuth = Depends(current_user_act_ver_prof),
164
162
  db: AsyncSession = Depends(get_async_db),
165
- ) -> TaskGroupActivityV2Read:
163
+ ) -> TaskGroupActivityRead:
166
164
  """
167
165
  Task-collection endpoint
168
166
  """
@@ -211,19 +209,19 @@ async def collect_tasks_pip(
211
209
 
212
210
  # Set pinned_package_versions
213
211
  if task_collect.pinned_package_versions_pre is not None:
214
- task_group_attrs[
215
- "pinned_package_versions_pre"
216
- ] = task_collect.pinned_package_versions_pre
212
+ task_group_attrs["pinned_package_versions_pre"] = (
213
+ task_collect.pinned_package_versions_pre
214
+ )
217
215
  if task_collect.pinned_package_versions_post is not None:
218
- task_group_attrs[
219
- "pinned_package_versions_post"
220
- ] = task_collect.pinned_package_versions_post
216
+ task_group_attrs["pinned_package_versions_post"] = (
217
+ task_collect.pinned_package_versions_post
218
+ )
221
219
 
222
220
  # Initialize wheel_file_content as None
223
221
  wheel_file = None
224
222
 
225
223
  # Set pkg_name, version, origin and archive_path
226
- if request_data.origin == TaskGroupV2OriginEnum.WHEELFILE:
224
+ if request_data.origin == TaskGroupOriginEnum.WHEELFILE:
227
225
  try:
228
226
  wheel_filename = request_data.file.filename
229
227
  wheel_info = _parse_wheel_filename(wheel_filename)
@@ -244,7 +242,7 @@ async def collect_tasks_pip(
244
242
  wheel_info["distribution"]
245
243
  )
246
244
  task_group_attrs["version"] = wheel_info["version"]
247
- elif request_data.origin == TaskGroupV2OriginEnum.PYPI:
245
+ elif request_data.origin == TaskGroupOriginEnum.PYPI:
248
246
  pkg_name = task_collect.package
249
247
  task_group_attrs["pkg_name"] = normalize_package_name(pkg_name)
250
248
  latest_version = await get_package_version_from_pypi(
@@ -280,7 +278,7 @@ async def collect_tasks_pip(
280
278
 
281
279
  # Validate TaskGroupV2 attributes
282
280
  try:
283
- TaskGroupCreateV2Strict(**task_group_attrs)
281
+ TaskGroupCreateStrict(**task_group_attrs)
284
282
  except ValidationError as e:
285
283
  raise HTTPException(
286
284
  status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
@@ -330,8 +328,8 @@ async def collect_tasks_pip(
330
328
  task_group_activity = TaskGroupActivityV2(
331
329
  user_id=task_group.user_id,
332
330
  taskgroupv2_id=task_group.id,
333
- status=TaskGroupActivityStatusV2.PENDING,
334
- action=TaskGroupActivityActionV2.COLLECT,
331
+ status=TaskGroupActivityStatus.PENDING,
332
+ action=TaskGroupActivityAction.COLLECT,
335
333
  pkg_name=task_group.pkg_name,
336
334
  version=task_group.version,
337
335
  )
@@ -9,44 +9,43 @@ from fastapi import HTTPException
9
9
  from fastapi import status
10
10
  from sqlalchemy.ext.asyncio import AsyncSession
11
11
 
12
- from ...aux.validate_user_profile import validate_user_profile
13
- from ._aux_functions_tasks import _get_valid_user_group_id
14
- from ._aux_functions_tasks import _verify_non_duplication_group_constraint
15
- from ._aux_functions_tasks import _verify_non_duplication_user_constraint
16
12
  from fractal_server.app.db import get_async_db
17
13
  from fractal_server.app.models import UserOAuth
18
14
  from fractal_server.app.models.v2 import TaskGroupV2
19
15
  from fractal_server.app.routes.auth import current_user_act_ver_prof
16
+ from fractal_server.app.routes.aux.validate_user_profile import (
17
+ validate_user_profile,
18
+ )
20
19
  from fractal_server.app.schemas.v2 import ResourceType
21
- from fractal_server.app.schemas.v2 import TaskCollectCustomV2
22
- from fractal_server.app.schemas.v2 import TaskCreateV2
23
- from fractal_server.app.schemas.v2 import TaskGroupCreateV2
24
- from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
25
- from fractal_server.app.schemas.v2 import TaskReadV2
20
+ from fractal_server.app.schemas.v2 import TaskCollectCustom
21
+ from fractal_server.app.schemas.v2 import TaskCreate
22
+ from fractal_server.app.schemas.v2 import TaskGroupCreate
23
+ from fractal_server.app.schemas.v2 import TaskGroupOriginEnum
24
+ from fractal_server.app.schemas.v2 import TaskRead
26
25
  from fractal_server.logger import set_logger
27
26
  from fractal_server.string_tools import validate_cmd
28
- from fractal_server.tasks.v2.utils_background import (
29
- prepare_tasks_metadata,
30
- )
27
+ from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
31
28
  from fractal_server.tasks.v2.utils_database import (
32
29
  create_db_tasks_and_update_task_group_async,
33
30
  )
34
31
 
32
+ from ._aux_functions_tasks import _get_valid_user_group_id
33
+ from ._aux_functions_tasks import _verify_non_duplication_group_constraint
34
+ from ._aux_functions_tasks import _verify_non_duplication_user_constraint
35
+
35
36
  router = APIRouter()
36
37
 
37
38
  logger = set_logger(__name__)
38
39
 
39
40
 
40
- @router.post(
41
- "/collect/custom/", status_code=201, response_model=list[TaskReadV2]
42
- )
41
+ @router.post("/collect/custom/", status_code=201, response_model=list[TaskRead])
43
42
  async def collect_task_custom(
44
- task_collect: TaskCollectCustomV2,
43
+ task_collect: TaskCollectCustom,
45
44
  private: bool = False,
46
45
  user_group_id: int | None = None,
47
46
  user: UserOAuth = Depends(current_user_act_ver_prof),
48
47
  db: AsyncSession = Depends(get_async_db),
49
- ) -> list[TaskReadV2]:
48
+ ) -> list[TaskRead]:
50
49
  # Get validated resource and profile
51
50
  resource, profile = await validate_user_profile(user=user, db=db)
52
51
  resource_id = resource.id
@@ -138,7 +137,7 @@ async def collect_task_custom(
138
137
  else:
139
138
  package_root = Path(task_collect.package_root)
140
139
 
141
- task_list: list[TaskCreateV2] = prepare_tasks_metadata(
140
+ task_list: list[TaskCreate] = prepare_tasks_metadata(
142
141
  package_manifest=task_collect.manifest,
143
142
  python_bin=Path(task_collect.python_interpreter),
144
143
  package_root=package_root,
@@ -147,14 +146,14 @@ async def collect_task_custom(
147
146
 
148
147
  # Prepare task-group attributes
149
148
  task_group_attrs = dict(
150
- origin=TaskGroupV2OriginEnum.OTHER,
149
+ origin=TaskGroupOriginEnum.OTHER,
151
150
  pkg_name=task_collect.label,
152
151
  user_id=user.id,
153
152
  user_group_id=user_group_id,
154
153
  version=task_collect.version,
155
154
  resource_id=resource_id,
156
155
  )
157
- TaskGroupCreateV2(**task_group_attrs)
156
+ TaskGroupCreate(**task_group_attrs)
158
157
 
159
158
  # Verify non-duplication constraints
160
159
  await _verify_non_duplication_user_constraint(