fractal-server 2.17.1a1__py3-none-any.whl → 2.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +19 -18
  3. fractal_server/app/db/__init__.py +3 -3
  4. fractal_server/app/models/__init__.py +1 -0
  5. fractal_server/app/models/linkuserproject.py +3 -1
  6. fractal_server/app/models/security.py +21 -3
  7. fractal_server/app/models/v2/__init__.py +3 -1
  8. fractal_server/app/models/v2/accounting.py +9 -1
  9. fractal_server/app/models/v2/dataset.py +5 -1
  10. fractal_server/app/models/v2/history.py +15 -1
  11. fractal_server/app/models/v2/job.py +4 -0
  12. fractal_server/app/models/v2/profile.py +29 -0
  13. fractal_server/app/models/v2/project.py +4 -10
  14. fractal_server/app/models/v2/resource.py +4 -0
  15. fractal_server/app/models/v2/task_group.py +4 -3
  16. fractal_server/app/models/v2/workflow.py +2 -1
  17. fractal_server/app/routes/admin/v2/__init__.py +1 -2
  18. fractal_server/app/routes/admin/v2/accounting.py +1 -1
  19. fractal_server/app/routes/admin/v2/job.py +9 -9
  20. fractal_server/app/routes/admin/v2/profile.py +3 -2
  21. fractal_server/app/routes/admin/v2/resource.py +5 -5
  22. fractal_server/app/routes/admin/v2/task.py +28 -18
  23. fractal_server/app/routes/admin/v2/task_group.py +0 -1
  24. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +1 -2
  25. fractal_server/app/routes/api/__init__.py +1 -0
  26. fractal_server/app/routes/api/v2/__init__.py +5 -6
  27. fractal_server/app/routes/api/v2/_aux_functions.py +70 -63
  28. fractal_server/app/routes/api/v2/_aux_functions_history.py +43 -20
  29. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +2 -4
  30. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -7
  31. fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
  32. fractal_server/app/routes/api/v2/dataset.py +13 -32
  33. fractal_server/app/routes/api/v2/history.py +35 -21
  34. fractal_server/app/routes/api/v2/images.py +3 -2
  35. fractal_server/app/routes/api/v2/job.py +17 -14
  36. fractal_server/app/routes/api/v2/pre_submission_checks.py +5 -4
  37. fractal_server/app/routes/api/v2/project.py +22 -17
  38. fractal_server/app/routes/api/v2/status_legacy.py +12 -11
  39. fractal_server/app/routes/api/v2/submit.py +11 -12
  40. fractal_server/app/routes/api/v2/task.py +4 -3
  41. fractal_server/app/routes/api/v2/task_collection.py +28 -30
  42. fractal_server/app/routes/api/v2/task_collection_custom.py +8 -7
  43. fractal_server/app/routes/api/v2/task_collection_pixi.py +1 -2
  44. fractal_server/app/routes/api/v2/task_group.py +7 -6
  45. fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -6
  46. fractal_server/app/routes/api/v2/task_version_update.py +13 -12
  47. fractal_server/app/routes/api/v2/workflow.py +14 -31
  48. fractal_server/app/routes/api/v2/workflow_import.py +17 -19
  49. fractal_server/app/routes/api/v2/workflowtask.py +10 -12
  50. fractal_server/app/routes/auth/__init__.py +1 -3
  51. fractal_server/app/routes/auth/_aux_auth.py +1 -2
  52. fractal_server/app/routes/auth/current_user.py +4 -5
  53. fractal_server/app/routes/auth/group.py +7 -5
  54. fractal_server/app/routes/auth/login.py +1 -0
  55. fractal_server/app/routes/auth/oauth.py +4 -3
  56. fractal_server/app/routes/auth/register.py +4 -2
  57. fractal_server/app/routes/auth/users.py +10 -10
  58. fractal_server/app/routes/aux/_job.py +1 -1
  59. fractal_server/app/routes/aux/_runner.py +2 -2
  60. fractal_server/app/routes/pagination.py +1 -1
  61. fractal_server/app/schemas/user.py +3 -3
  62. fractal_server/app/schemas/v2/accounting.py +11 -0
  63. fractal_server/app/schemas/v2/dataset.py +28 -4
  64. fractal_server/app/schemas/v2/dumps.py +1 -0
  65. fractal_server/app/schemas/v2/manifest.py +4 -3
  66. fractal_server/app/schemas/v2/profile.py +53 -2
  67. fractal_server/app/schemas/v2/resource.py +109 -13
  68. fractal_server/app/schemas/v2/task.py +0 -1
  69. fractal_server/app/schemas/v2/task_collection.py +1 -1
  70. fractal_server/app/schemas/v2/workflowtask.py +4 -3
  71. fractal_server/app/security/__init__.py +4 -7
  72. fractal_server/app/security/signup_email.py +4 -5
  73. fractal_server/config/_data.py +36 -25
  74. fractal_server/config/_database.py +19 -20
  75. fractal_server/config/_email.py +30 -38
  76. fractal_server/config/_main.py +33 -52
  77. fractal_server/config/_oauth.py +17 -21
  78. fractal_server/exceptions.py +4 -0
  79. fractal_server/images/models.py +3 -3
  80. fractal_server/images/status_tools.py +4 -2
  81. fractal_server/logger.py +1 -1
  82. fractal_server/main.py +4 -3
  83. fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
  84. fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
  85. fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
  86. fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
  87. fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
  88. fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
  89. fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
  90. fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
  91. fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
  92. fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
  93. fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
  94. fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
  95. fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
  96. fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
  97. fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
  98. fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
  99. fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
  100. fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
  101. fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
  102. fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
  103. fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
  104. fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
  105. fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
  106. fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
  107. fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
  108. fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
  109. fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
  110. fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
  111. fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
  112. fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
  113. fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
  114. fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
  115. fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
  116. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
  117. fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
  118. fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
  119. fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
  120. fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
  121. fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
  122. fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
  123. fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
  124. fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
  125. fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
  126. fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
  127. fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
  128. fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
  129. fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
  130. fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
  131. fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
  132. fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
  133. fractal_server/runner/config/_local.py +8 -5
  134. fractal_server/runner/config/_slurm.py +37 -33
  135. fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
  136. fractal_server/runner/executors/base_runner.py +29 -4
  137. fractal_server/runner/executors/local/get_local_config.py +1 -0
  138. fractal_server/runner/executors/local/runner.py +14 -13
  139. fractal_server/runner/executors/slurm_common/_batching.py +5 -10
  140. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
  141. fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
  142. fractal_server/runner/executors/slurm_common/remote.py +3 -1
  143. fractal_server/runner/executors/slurm_common/slurm_config.py +1 -0
  144. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
  145. fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
  146. fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
  147. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
  148. fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
  149. fractal_server/runner/task_files.py +9 -3
  150. fractal_server/runner/v2/_local.py +9 -4
  151. fractal_server/runner/v2/_slurm_ssh.py +11 -5
  152. fractal_server/runner/v2/_slurm_sudo.py +11 -5
  153. fractal_server/runner/v2/db_tools.py +0 -1
  154. fractal_server/runner/v2/deduplicate_list.py +2 -1
  155. fractal_server/runner/v2/runner.py +11 -14
  156. fractal_server/runner/v2/runner_functions.py +11 -14
  157. fractal_server/runner/v2/submit_workflow.py +7 -6
  158. fractal_server/ssh/_fabric.py +6 -13
  159. fractal_server/string_tools.py +0 -1
  160. fractal_server/syringe.py +1 -1
  161. fractal_server/tasks/config/_pixi.py +1 -1
  162. fractal_server/tasks/config/_python.py +16 -9
  163. fractal_server/tasks/utils.py +0 -1
  164. fractal_server/tasks/v2/local/_utils.py +1 -1
  165. fractal_server/tasks/v2/local/collect.py +10 -12
  166. fractal_server/tasks/v2/local/collect_pixi.py +9 -10
  167. fractal_server/tasks/v2/local/deactivate.py +7 -8
  168. fractal_server/tasks/v2/local/deactivate_pixi.py +4 -4
  169. fractal_server/tasks/v2/local/delete.py +1 -3
  170. fractal_server/tasks/v2/local/reactivate.py +7 -7
  171. fractal_server/tasks/v2/local/reactivate_pixi.py +7 -7
  172. fractal_server/tasks/v2/ssh/_utils.py +3 -3
  173. fractal_server/tasks/v2/ssh/collect.py +14 -19
  174. fractal_server/tasks/v2/ssh/collect_pixi.py +17 -19
  175. fractal_server/tasks/v2/ssh/deactivate.py +10 -8
  176. fractal_server/tasks/v2/ssh/deactivate_pixi.py +6 -5
  177. fractal_server/tasks/v2/ssh/delete.py +7 -5
  178. fractal_server/tasks/v2/ssh/reactivate.py +11 -11
  179. fractal_server/tasks/v2/ssh/reactivate_pixi.py +8 -9
  180. fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
  181. fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
  182. fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
  183. fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
  184. fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
  185. fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
  186. fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
  187. fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
  188. fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
  189. fractal_server/tasks/v2/utils_background.py +3 -3
  190. fractal_server/tasks/v2/utils_package_names.py +1 -2
  191. fractal_server/tasks/v2/utils_pixi.py +1 -3
  192. fractal_server/types/__init__.py +76 -1
  193. fractal_server/types/validators/_common_validators.py +1 -3
  194. fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
  195. fractal_server/utils.py +1 -0
  196. fractal_server/zip_tools.py +34 -0
  197. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/METADATA +1 -1
  198. fractal_server-2.17.2.dist-info/RECORD +265 -0
  199. fractal_server/app/routes/admin/v2/project.py +0 -41
  200. fractal_server-2.17.1a1.dist-info/RECORD +0 -264
  201. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/WHEEL +0 -0
  202. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/entry_points.txt +0 -0
  203. {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,6 +1,7 @@
1
1
  """
2
2
  `api` module
3
3
  """
4
+
4
5
  from fastapi import APIRouter
5
6
  from fastapi import Depends
6
7
 
@@ -1,8 +1,12 @@
1
1
  """
2
2
  `api/v2` module
3
3
  """
4
+
4
5
  from fastapi import APIRouter
5
6
 
7
+ from fractal_server.config import get_settings
8
+ from fractal_server.syringe import Inject
9
+
6
10
  from .dataset import router as dataset_router_v2
7
11
  from .history import router as history_router_v2
8
12
  from .images import router as images_routes_v2
@@ -21,9 +25,6 @@ from .task_version_update import router as task_version_update_router_v2
21
25
  from .workflow import router as workflow_router_v2
22
26
  from .workflow_import import router as workflow_import_router_v2
23
27
  from .workflowtask import router as workflowtask_router_v2
24
- from fractal_server.config import get_settings
25
- from fractal_server.syringe import Inject
26
-
27
28
 
28
29
  router_api_v2 = APIRouter()
29
30
 
@@ -34,9 +35,7 @@ router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
34
35
  router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
35
36
  router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
36
37
  router_api_v2.include_router(history_router_v2, tags=["V2 History"])
37
- router_api_v2.include_router(
38
- status_legacy_router_v2, tags=["V2 Status Legacy"]
39
- )
38
+ router_api_v2.include_router(status_legacy_router_v2, tags=["V2 Status Legacy"])
40
39
 
41
40
 
42
41
  settings = Inject(get_settings)
@@ -1,6 +1,7 @@
1
1
  """
2
2
  Auxiliary functions to get object from the database or perform simple checks
3
3
  """
4
+
4
5
  from typing import Any
5
6
  from typing import Literal
6
7
 
@@ -11,18 +12,18 @@ from sqlalchemy.orm.attributes import flag_modified
11
12
  from sqlmodel import select
12
13
  from sqlmodel.sql.expression import SelectOfScalar
13
14
 
14
- from ....models.v2 import DatasetV2
15
- from ....models.v2 import JobV2
16
- from ....models.v2 import LinkUserProjectV2
17
- from ....models.v2 import ProjectV2
18
- from ....models.v2 import TaskV2
19
- from ....models.v2 import WorkflowTaskV2
20
- from ....models.v2 import WorkflowV2
21
- from ....schemas.v2 import JobStatusTypeV2
22
15
  from fractal_server.app.db import AsyncSession
23
16
  from fractal_server.app.models import Profile
24
17
  from fractal_server.app.models import Resource
25
18
  from fractal_server.app.models import UserOAuth
19
+ from fractal_server.app.models.v2 import DatasetV2
20
+ from fractal_server.app.models.v2 import JobV2
21
+ from fractal_server.app.models.v2 import LinkUserProjectV2
22
+ from fractal_server.app.models.v2 import ProjectV2
23
+ from fractal_server.app.models.v2 import TaskV2
24
+ from fractal_server.app.models.v2 import WorkflowTaskV2
25
+ from fractal_server.app.models.v2 import WorkflowV2
26
+ from fractal_server.app.schemas.v2 import JobStatusTypeV2
26
27
  from fractal_server.logger import set_logger
27
28
 
28
29
  logger = set_logger(__name__)
@@ -88,29 +89,29 @@ async def _get_workflow_check_owner(
88
89
 
89
90
  Raises:
90
91
  HTTPException(status_code=404_NOT_FOUND):
91
- If the workflow does not exist
92
- HTTPException(status_code=422_UNPROCESSABLE_ENTITY):
93
- If the workflow is not associated to the project
92
+ If the project or the workflow do not exist or if they are not
93
+ associated
94
+ HTTPException(status_code=403_FORBIDDEN):
95
+ If the user is not a member of the project
94
96
  """
95
97
 
96
98
  # Access control for project
97
- project = await _get_project_check_owner(
99
+ await _get_project_check_owner(
98
100
  project_id=project_id, user_id=user_id, db=db
99
101
  )
100
102
 
101
- # Get workflow
102
- # (See issue 1087 for 'populate_existing=True')
103
- workflow = await db.get(WorkflowV2, workflow_id, populate_existing=True)
103
+ res = await db.execute(
104
+ select(WorkflowV2)
105
+ .where(WorkflowV2.id == workflow_id)
106
+ .where(WorkflowV2.project_id == project_id)
107
+ .execution_options(populate_existing=True) # See issue 1087
108
+ )
109
+ workflow = res.scalars().one_or_none()
104
110
 
105
111
  if not workflow:
106
112
  raise HTTPException(
107
113
  status_code=status.HTTP_404_NOT_FOUND, detail="Workflow not found"
108
114
  )
109
- if workflow.project_id != project.id:
110
- raise HTTPException(
111
- status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
112
- detail=(f"Invalid {project_id=} for {workflow_id=}."),
113
- )
114
115
 
115
116
  return workflow
116
117
 
@@ -138,9 +139,10 @@ async def _get_workflow_task_check_owner(
138
139
 
139
140
  Raises:
140
141
  HTTPException(status_code=404_NOT_FOUND):
141
- If the WorkflowTask does not exist
142
- HTTPException(status_code=422_UNPROCESSABLE_ENTITY):
143
- If the WorkflowTask is not associated to the Workflow
142
+ If the project, the workflow or the workflowtask do not exist or
143
+ if they are not associated
144
+ HTTPException(status_code=403_FORBIDDEN):
145
+ If the user is not a member of the project
144
146
  """
145
147
 
146
148
  # Access control for workflow
@@ -151,21 +153,19 @@ async def _get_workflow_task_check_owner(
151
153
  db=db,
152
154
  )
153
155
 
154
- # If WorkflowTask is not in the db, exit
155
- workflow_task = await db.get(WorkflowTaskV2, workflow_task_id)
156
- if not workflow_task:
156
+ res = await db.execute(
157
+ select(WorkflowTaskV2)
158
+ .where(WorkflowTaskV2.id == workflow_task_id)
159
+ .where(WorkflowTaskV2.workflow_id == workflow_id)
160
+ )
161
+ workflow_task = res.scalars().one_or_none()
162
+
163
+ if workflow_task is None:
157
164
  raise HTTPException(
158
165
  status_code=status.HTTP_404_NOT_FOUND,
159
166
  detail="WorkflowTask not found",
160
167
  )
161
168
 
162
- # If WorkflowTask is not part of the expected Workflow, exit
163
- if workflow_id != workflow_task.workflow_id:
164
- raise HTTPException(
165
- status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
166
- detail=f"Invalid {workflow_id=} for {workflow_task_id=}",
167
- )
168
-
169
169
  return workflow_task, workflow
170
170
 
171
171
 
@@ -220,7 +220,7 @@ async def _check_project_exists(
220
220
  """
221
221
  stm = (
222
222
  select(ProjectV2)
223
- .join(LinkUserProjectV2)
223
+ .join(LinkUserProjectV2, LinkUserProjectV2.project_id == ProjectV2.id)
224
224
  .where(ProjectV2.name == project_name)
225
225
  .where(LinkUserProjectV2.user_id == user_id)
226
226
  )
@@ -253,27 +253,29 @@ async def _get_dataset_check_owner(
253
253
  `project`).
254
254
 
255
255
  Raises:
256
- HTTPException(status_code=422_UNPROCESSABLE_ENTITY):
257
- If the dataset is not associated to the project
256
+ HTTPException(status_code=404_UNPROCESSABLE_ENTITY):
257
+ If the project or the dataset do not exist or if they are not
258
+ associated
259
+ HTTPException(status_code=403_FORBIDDEN):
260
+ If the user is not a member of the project
258
261
  """
259
262
  # Access control for project
260
263
  project = await _get_project_check_owner(
261
264
  project_id=project_id, user_id=user_id, db=db
262
265
  )
263
266
 
264
- # Get dataset
265
- # (See issue 1087 for 'populate_existing=True')
266
- dataset = await db.get(DatasetV2, dataset_id, populate_existing=True)
267
+ res = await db.execute(
268
+ select(DatasetV2)
269
+ .where(DatasetV2.id == dataset_id)
270
+ .where(DatasetV2.project_id == project_id)
271
+ .execution_options(populate_existing=True) # See issue 1087
272
+ )
273
+ dataset = res.scalars().one_or_none()
267
274
 
268
- if not dataset:
275
+ if dataset is None:
269
276
  raise HTTPException(
270
277
  status_code=status.HTTP_404_NOT_FOUND, detail="Dataset not found"
271
278
  )
272
- if dataset.project_id != project_id:
273
- raise HTTPException(
274
- status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
275
- detail=f"Invalid {project_id=} for {dataset_id=}",
276
- )
277
279
 
278
280
  return dict(dataset=dataset, project=project)
279
281
 
@@ -299,8 +301,11 @@ async def _get_job_check_owner(
299
301
  `project`).
300
302
 
301
303
  Raises:
302
- HTTPException(status_code=422_UNPROCESSABLE_ENTITY):
303
- If the job is not associated to the project
304
+ HTTPException(status_code=404_UNPROCESSABLE_ENTITY):
305
+ If the project or the job do not exist or if they are not
306
+ associated
307
+ HTTPException(status_code=403_FORBIDDEN):
308
+ If the user is not a member of the project
304
309
  """
305
310
  # Access control for project
306
311
  project = await _get_project_check_owner(
@@ -308,17 +313,19 @@ async def _get_job_check_owner(
308
313
  user_id=user_id,
309
314
  db=db,
310
315
  )
311
- # Get dataset
312
- job = await db.get(JobV2, job_id)
313
- if not job:
316
+
317
+ res = await db.execute(
318
+ select(JobV2)
319
+ .where(JobV2.id == job_id)
320
+ .where(JobV2.project_id == project_id)
321
+ )
322
+ job = res.scalars().one_or_none()
323
+
324
+ if job is None:
314
325
  raise HTTPException(
315
326
  status_code=status.HTTP_404_NOT_FOUND, detail="Job not found"
316
327
  )
317
- if job.project_id != project_id:
318
- raise HTTPException(
319
- status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
320
- detail=f"Invalid {project_id=} for {job_id=}",
321
- )
328
+
322
329
  return dict(job=job, project=project)
323
330
 
324
331
 
@@ -411,8 +418,11 @@ async def _workflow_insert_task(
411
418
  flag_modified(db_workflow, "task_list")
412
419
  await db.commit()
413
420
 
414
- # See issue 1087 for 'populate_existing=True'
415
- wf_task = await db.get(WorkflowTaskV2, wf_task.id, populate_existing=True)
421
+ wf_task = await db.get(
422
+ WorkflowTaskV2,
423
+ wf_task.id,
424
+ populate_existing=True, # See issue 1087
425
+ )
416
426
 
417
427
  return wf_task
418
428
 
@@ -533,8 +543,7 @@ async def _get_submitted_job_or_none(
533
543
  return res.scalars().one_or_none()
534
544
  except MultipleResultsFound as e:
535
545
  error_msg = (
536
- "Multiple running jobs found for "
537
- f"{dataset_id=} and {workflow_id=}."
546
+ f"Multiple running jobs found for {dataset_id=} and {workflow_id=}."
538
547
  )
539
548
  logger.error(f"{error_msg} Original error: {str(e)}.")
540
549
  raise HTTPException(
@@ -546,10 +555,8 @@ async def _get_submitted_job_or_none(
546
555
  async def _get_user_resource_id(user_id: int, db: AsyncSession) -> int | None:
547
556
  res = await db.execute(
548
557
  select(Resource.id)
549
- .join(Profile)
550
- .join(UserOAuth)
551
- .where(Resource.id == Profile.resource_id)
552
- .where(Profile.id == UserOAuth.profile_id)
558
+ .join(Profile, Resource.id == Profile.resource_id)
559
+ .join(UserOAuth, Profile.id == UserOAuth.profile_id)
553
560
  .where(UserOAuth.id == user_id)
554
561
  )
555
562
  resource_id = res.scalar_one_or_none()
@@ -1,3 +1,4 @@
1
+ import os
1
2
  from pathlib import Path
2
3
  from typing import Literal
3
4
 
@@ -14,14 +15,12 @@ from fractal_server.app.routes.api.v2._aux_functions import _get_dataset_or_404
14
15
  from fractal_server.app.routes.api.v2._aux_functions import (
15
16
  _get_project_check_owner,
16
17
  )
17
- from fractal_server.app.routes.api.v2._aux_functions import (
18
- _get_workflow_or_404,
19
- )
18
+ from fractal_server.app.routes.api.v2._aux_functions import _get_workflow_or_404
20
19
  from fractal_server.app.routes.api.v2._aux_functions import (
21
20
  _get_workflowtask_or_404,
22
21
  )
23
22
  from fractal_server.logger import set_logger
24
-
23
+ from fractal_server.zip_tools import _read_single_file_from_zip
25
24
 
26
25
  logger = set_logger(__name__)
27
26
 
@@ -66,27 +65,51 @@ async def get_history_run_or_404(
66
65
 
67
66
  def read_log_file(
68
67
  *,
69
- logfile: str | None,
70
- wftask: WorkflowTaskV2,
68
+ task_name: str,
71
69
  dataset_id: int,
72
- ):
73
- if logfile is None or not Path(logfile).exists():
74
- logger.debug(
75
- f"Logs for task '{wftask.task.name}' in dataset "
76
- f"{dataset_id} are not available ({logfile=})."
77
- )
78
- return (
79
- f"Logs for task '{wftask.task.name}' in dataset "
80
- f"{dataset_id} are not available."
81
- )
70
+ logfile: str,
71
+ job_working_dir: str,
72
+ ) -> str:
73
+ """
74
+ Returns the contents of a Job's log file, either directly from the working
75
+ directory or from the corresponding ZIP archive.
76
+
77
+ The function first checks if `logfile` exists on disk.
82
78
 
79
+ If not, it checks if the Job working directory has been zipped and tries to
80
+ read `logfile` from within the archive.
81
+ (Note: it is assumed that `logfile` is relative to `job_working_dir`)
82
+ """
83
+ archive_path = os.path.normpath(job_working_dir) + ".zip"
83
84
  try:
84
- with open(logfile) as f:
85
- return f.read()
85
+ if Path(logfile).exists():
86
+ with open(logfile) as f:
87
+ return f.read()
88
+ elif Path(archive_path).exists():
89
+ relative_logfile = (
90
+ Path(logfile).relative_to(job_working_dir).as_posix()
91
+ )
92
+ return _read_single_file_from_zip(
93
+ file_path=relative_logfile, archive_path=archive_path
94
+ )
95
+
96
+ else:
97
+ logger.error(
98
+ f"Error while retrieving logs for {logfile=} and "
99
+ f"{archive_path=}: both files do not exist."
100
+ )
101
+ return (
102
+ f"Logs for task '{task_name}' in dataset "
103
+ f"{dataset_id} are not available."
104
+ )
86
105
  except Exception as e:
106
+ logger.error(
107
+ f"Error while retrieving logs for {logfile=} and {archive_path=}. "
108
+ f"Original error: {str(e)}"
109
+ )
87
110
  return (
88
- f"Error while retrieving logs for task '{wftask.task.name}' "
89
- f"in dataset {dataset_id}. Original error: {str(e)}."
111
+ f"Error while retrieving logs for task '{task_name}' "
112
+ f"in dataset {dataset_id}."
90
113
  )
91
114
 
92
115
 
@@ -99,8 +99,7 @@ async def get_package_version_from_pypi(
99
99
  raise HTTPException(
100
100
  status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
101
101
  detail=(
102
- f"Could not get {url} (status_code {res.status_code})."
103
- f"\n{hint}"
102
+ f"Could not get {url} (status_code {res.status_code}).\n{hint}"
104
103
  ),
105
104
  )
106
105
  try:
@@ -144,8 +143,7 @@ async def get_package_version_from_pypi(
144
143
  raise HTTPException(
145
144
  status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
146
145
  detail=(
147
- f"No version starting with {version} found.\n"
148
- f"{hint}"
146
+ f"No version starting with {version} found.\n{hint}"
149
147
  ),
150
148
  )
151
149
  else:
@@ -2,6 +2,7 @@
2
2
  Auxiliary functions to get task and task-group object from the database or
3
3
  perform simple checks
4
4
  """
5
+
5
6
  from typing import Any
6
7
 
7
8
  from fastapi import HTTPException
@@ -74,8 +75,7 @@ async def _get_task_group_read_access(
74
75
  forbidden_exception = HTTPException(
75
76
  status_code=status.HTTP_403_FORBIDDEN,
76
77
  detail=(
77
- "Current user has no read access to TaskGroupV2 "
78
- f"{task_group_id}."
78
+ f"Current user has no read access to TaskGroupV2 {task_group_id}."
79
79
  ),
80
80
  )
81
81
 
@@ -86,13 +86,11 @@ async def _get_task_group_read_access(
86
86
  else:
87
87
  stm = (
88
88
  select(LinkUserGroup)
89
- .join(UserOAuth)
90
- .join(Profile)
89
+ .join(UserOAuth, UserOAuth.id == LinkUserGroup.user_id)
90
+ .join(Profile, Profile.id == UserOAuth.profile_id)
91
91
  .where(LinkUserGroup.group_id == task_group.user_group_id)
92
- .where(LinkUserGroup.user_id == user_id)
93
92
  .where(UserOAuth.id == user_id)
94
- .where(Profile.id == UserOAuth.profile_id)
95
- .where(task_group.resource_id == Profile.resource_id)
93
+ .where(Profile.resource_id == task_group.resource_id)
96
94
  )
97
95
  res = await db.execute(stm)
98
96
  link = res.unique().scalars().one_or_none()
@@ -10,7 +10,6 @@ from fractal_server.exceptions import UnreachableBranchError
10
10
  from fractal_server.logger import set_logger
11
11
  from fractal_server.syringe import Inject
12
12
 
13
-
14
13
  logger = set_logger(__name__)
15
14
 
16
15
 
@@ -84,7 +83,7 @@ async def _disambiguate_task_groups(
84
83
  res = await db.execute(stm)
85
84
  oldest_user_group_id = res.scalars().first()
86
85
  logger.debug(
87
- "[_disambiguate_task_groups] " f"Result: {oldest_user_group_id=}."
86
+ f"[_disambiguate_task_groups] Result: {oldest_user_group_id=}."
88
87
  )
89
88
  task_group = next(
90
89
  iter(
@@ -5,24 +5,24 @@ from fastapi import Response
5
5
  from fastapi import status
6
6
  from sqlmodel import select
7
7
 
8
- from ....db import AsyncSession
9
- from ....db import get_async_db
10
- from ....models.v2 import DatasetV2
11
- from ....models.v2 import JobV2
12
- from ....models.v2 import ProjectV2
13
- from ....schemas.v2 import DatasetCreateV2
14
- from ....schemas.v2 import DatasetReadV2
15
- from ....schemas.v2 import DatasetUpdateV2
16
- from ....schemas.v2.dataset import DatasetExportV2
17
- from ....schemas.v2.dataset import DatasetImportV2
18
- from ._aux_functions import _get_dataset_check_owner
19
- from ._aux_functions import _get_project_check_owner
20
- from ._aux_functions import _get_submitted_jobs_statement
8
+ from fractal_server.app.db import AsyncSession
9
+ from fractal_server.app.db import get_async_db
21
10
  from fractal_server.app.models import UserOAuth
11
+ from fractal_server.app.models.v2 import DatasetV2
12
+ from fractal_server.app.models.v2 import JobV2
22
13
  from fractal_server.app.routes.auth import current_user_act_ver_prof
14
+ from fractal_server.app.schemas.v2 import DatasetCreateV2
15
+ from fractal_server.app.schemas.v2 import DatasetReadV2
16
+ from fractal_server.app.schemas.v2 import DatasetUpdateV2
17
+ from fractal_server.app.schemas.v2.dataset import DatasetExportV2
18
+ from fractal_server.app.schemas.v2.dataset import DatasetImportV2
23
19
  from fractal_server.string_tools import sanitize_string
24
20
  from fractal_server.urls import normalize_url
25
21
 
22
+ from ._aux_functions import _get_dataset_check_owner
23
+ from ._aux_functions import _get_project_check_owner
24
+ from ._aux_functions import _get_submitted_jobs_statement
25
+
26
26
  router = APIRouter()
27
27
 
28
28
 
@@ -208,25 +208,6 @@ async def delete_dataset(
208
208
  return Response(status_code=status.HTTP_204_NO_CONTENT)
209
209
 
210
210
 
211
- @router.get("/dataset/", response_model=list[DatasetReadV2])
212
- async def get_user_datasets(
213
- user: UserOAuth = Depends(current_user_act_ver_prof),
214
- db: AsyncSession = Depends(get_async_db),
215
- ) -> list[DatasetReadV2]:
216
- """
217
- Returns all the datasets of the current user
218
- """
219
- stm = select(DatasetV2)
220
- stm = stm.join(ProjectV2).where(
221
- ProjectV2.user_list.any(UserOAuth.id == user.id)
222
- )
223
-
224
- res = await db.execute(stm)
225
- dataset_list = res.scalars().all()
226
- await db.close()
227
- return dataset_list
228
-
229
-
230
211
  @router.get(
231
212
  "/project/{project_id}/dataset/{dataset_id}/export/",
232
213
  response_model=DatasetExportV2,
@@ -6,40 +6,42 @@ from fastapi.responses import JSONResponse
6
6
  from sqlmodel import func
7
7
  from sqlmodel import select
8
8
 
9
- from ._aux_functions import _get_dataset_check_owner
10
- from ._aux_functions import _get_submitted_job_or_none
11
- from ._aux_functions import _get_workflow_check_owner
12
- from ._aux_functions_history import _verify_workflow_and_dataset_access
13
- from ._aux_functions_history import get_history_run_or_404
14
- from ._aux_functions_history import get_history_unit_or_404
15
- from ._aux_functions_history import get_wftask_check_owner
16
- from ._aux_functions_history import read_log_file
17
- from .images import ImagePage
18
- from .images import ImageQuery
19
9
  from fractal_server.app.db import AsyncSession
20
10
  from fractal_server.app.db import get_async_db
21
11
  from fractal_server.app.models import UserOAuth
22
12
  from fractal_server.app.models.v2 import HistoryImageCache
23
13
  from fractal_server.app.models.v2 import HistoryRun
24
14
  from fractal_server.app.models.v2 import HistoryUnit
15
+ from fractal_server.app.models.v2 import JobV2
25
16
  from fractal_server.app.models.v2 import TaskV2
26
17
  from fractal_server.app.routes.auth import current_user_act_ver_prof
27
- from fractal_server.app.routes.pagination import get_pagination_params
28
18
  from fractal_server.app.routes.pagination import PaginationRequest
29
19
  from fractal_server.app.routes.pagination import PaginationResponse
20
+ from fractal_server.app.routes.pagination import get_pagination_params
30
21
  from fractal_server.app.schemas.v2 import HistoryRunRead
31
22
  from fractal_server.app.schemas.v2 import HistoryRunReadAggregated
32
23
  from fractal_server.app.schemas.v2 import HistoryUnitRead
33
24
  from fractal_server.app.schemas.v2 import HistoryUnitStatus
34
25
  from fractal_server.app.schemas.v2 import HistoryUnitStatusWithUnset
35
26
  from fractal_server.app.schemas.v2 import ImageLogsRequest
36
- from fractal_server.images.status_tools import enrich_images_unsorted_async
37
27
  from fractal_server.images.status_tools import IMAGE_STATUS_KEY
28
+ from fractal_server.images.status_tools import enrich_images_unsorted_async
38
29
  from fractal_server.images.tools import aggregate_attributes
39
30
  from fractal_server.images.tools import aggregate_types
40
31
  from fractal_server.images.tools import filter_image_list
41
32
  from fractal_server.logger import set_logger
42
33
 
34
+ from ._aux_functions import _get_dataset_check_owner
35
+ from ._aux_functions import _get_submitted_job_or_none
36
+ from ._aux_functions import _get_workflow_check_owner
37
+ from ._aux_functions_history import _verify_workflow_and_dataset_access
38
+ from ._aux_functions_history import get_history_run_or_404
39
+ from ._aux_functions_history import get_history_unit_or_404
40
+ from ._aux_functions_history import get_wftask_check_owner
41
+ from ._aux_functions_history import read_log_file
42
+ from .images import ImagePage
43
+ from .images import ImageQuery
44
+
43
45
 
44
46
  def check_historyrun_related_to_dataset_and_wftask(
45
47
  history_run: HistoryRun,
@@ -135,19 +137,19 @@ async def get_workflow_tasks_statuses(
135
137
  logger.debug(f"C1: {wftask.id=} not in {running_wftask_ids=}.")
136
138
  response[wftask.id] = dict(status=latest_run.status)
137
139
 
138
- response[wftask.id][
139
- "num_available_images"
140
- ] = latest_run.num_available_images
140
+ response[wftask.id]["num_available_images"] = (
141
+ latest_run.num_available_images
142
+ )
141
143
 
142
144
  for target_status in HistoryUnitStatus:
143
145
  stm = (
144
146
  select(func.count(HistoryImageCache.zarr_url))
145
- .join(HistoryUnit)
147
+ .join(
148
+ HistoryUnit,
149
+ HistoryImageCache.latest_history_unit_id == HistoryUnit.id,
150
+ )
146
151
  .where(HistoryImageCache.dataset_id == dataset_id)
147
152
  .where(HistoryImageCache.workflowtask_id == wftask.id)
148
- .where(
149
- HistoryImageCache.latest_history_unit_id == HistoryUnit.id
150
- )
151
153
  .where(HistoryUnit.status == target_status)
152
154
  )
153
155
  res = await db.execute(stm)
@@ -444,11 +446,20 @@ async def get_image_log(
444
446
  db=db,
445
447
  )
446
448
 
449
+ # Get job.working_dir
450
+ res = await db.execute(
451
+ select(JobV2.working_dir)
452
+ .join(HistoryRun, HistoryRun.job_id == JobV2.id)
453
+ .where(HistoryRun.id == history_unit.history_run_id)
454
+ )
455
+ job_working_dir = res.scalar_one_or_none()
456
+
447
457
  # Get log or placeholder text
448
458
  log = read_log_file(
449
459
  logfile=history_unit.logfile,
450
- wftask=wftask,
460
+ task_name=wftask.task.name,
451
461
  dataset_id=request_data.dataset_id,
462
+ job_working_dir=job_working_dir,
452
463
  )
453
464
  return JSONResponse(content=log)
454
465
 
@@ -495,11 +506,14 @@ async def get_history_unit_log(
495
506
  workflowtask_id=workflowtask_id,
496
507
  )
497
508
 
509
+ job = await db.get(JobV2, history_run.job_id)
510
+
498
511
  # Get log or placeholder text
499
512
  log = read_log_file(
500
513
  logfile=history_unit.logfile,
501
- wftask=wftask,
514
+ task_name=wftask.task.name,
502
515
  dataset_id=dataset_id,
516
+ job_working_dir=job.working_dir,
503
517
  )
504
518
  return JSONResponse(content=log)
505
519
 
@@ -8,15 +8,14 @@ from pydantic import Field
8
8
  from sqlalchemy.orm.attributes import flag_modified
9
9
  from sqlmodel import delete
10
10
 
11
- from ._aux_functions import _get_dataset_check_owner
12
11
  from fractal_server.app.db import AsyncSession
13
12
  from fractal_server.app.db import get_async_db
14
13
  from fractal_server.app.models import HistoryImageCache
15
14
  from fractal_server.app.models import UserOAuth
16
15
  from fractal_server.app.routes.auth import current_user_act_ver_prof
17
- from fractal_server.app.routes.pagination import get_pagination_params
18
16
  from fractal_server.app.routes.pagination import PaginationRequest
19
17
  from fractal_server.app.routes.pagination import PaginationResponse
18
+ from fractal_server.app.routes.pagination import get_pagination_params
20
19
  from fractal_server.images import SingleImage
21
20
  from fractal_server.images import SingleImageUpdate
22
21
  from fractal_server.images.tools import aggregate_attributes
@@ -27,6 +26,8 @@ from fractal_server.types import AttributeFilters
27
26
  from fractal_server.types import ImageAttributeValue
28
27
  from fractal_server.types import TypeFilters
29
28
 
29
+ from ._aux_functions import _get_dataset_check_owner
30
+
30
31
  router = APIRouter()
31
32
 
32
33