fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/models/__init__.py +6 -8
  3. fractal_server/app/models/linkuserproject.py +9 -0
  4. fractal_server/app/models/security.py +6 -0
  5. fractal_server/app/models/v1/__init__.py +12 -0
  6. fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
  7. fractal_server/app/models/{job.py → v1/job.py} +5 -5
  8. fractal_server/app/models/{project.py → v1/project.py} +5 -5
  9. fractal_server/app/models/{state.py → v1/state.py} +2 -2
  10. fractal_server/app/models/{task.py → v1/task.py} +7 -2
  11. fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
  12. fractal_server/app/models/v2/__init__.py +22 -0
  13. fractal_server/app/models/v2/collection_state.py +21 -0
  14. fractal_server/app/models/v2/dataset.py +54 -0
  15. fractal_server/app/models/v2/job.py +51 -0
  16. fractal_server/app/models/v2/project.py +30 -0
  17. fractal_server/app/models/v2/task.py +93 -0
  18. fractal_server/app/models/v2/workflow.py +35 -0
  19. fractal_server/app/models/v2/workflowtask.py +49 -0
  20. fractal_server/app/routes/admin/__init__.py +0 -0
  21. fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
  22. fractal_server/app/routes/admin/v2.py +309 -0
  23. fractal_server/app/routes/api/v1/__init__.py +7 -7
  24. fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
  25. fractal_server/app/routes/api/v1/dataset.py +41 -41
  26. fractal_server/app/routes/api/v1/job.py +14 -14
  27. fractal_server/app/routes/api/v1/project.py +27 -25
  28. fractal_server/app/routes/api/v1/task.py +26 -16
  29. fractal_server/app/routes/api/v1/task_collection.py +28 -16
  30. fractal_server/app/routes/api/v1/workflow.py +28 -28
  31. fractal_server/app/routes/api/v1/workflowtask.py +11 -11
  32. fractal_server/app/routes/api/v2/__init__.py +34 -0
  33. fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
  34. fractal_server/app/routes/api/v2/dataset.py +293 -0
  35. fractal_server/app/routes/api/v2/images.py +279 -0
  36. fractal_server/app/routes/api/v2/job.py +200 -0
  37. fractal_server/app/routes/api/v2/project.py +186 -0
  38. fractal_server/app/routes/api/v2/status.py +150 -0
  39. fractal_server/app/routes/api/v2/submit.py +210 -0
  40. fractal_server/app/routes/api/v2/task.py +222 -0
  41. fractal_server/app/routes/api/v2/task_collection.py +239 -0
  42. fractal_server/app/routes/api/v2/task_legacy.py +59 -0
  43. fractal_server/app/routes/api/v2/workflow.py +380 -0
  44. fractal_server/app/routes/api/v2/workflowtask.py +265 -0
  45. fractal_server/app/routes/aux/_job.py +2 -2
  46. fractal_server/app/runner/__init__.py +0 -364
  47. fractal_server/app/runner/async_wrap.py +27 -0
  48. fractal_server/app/runner/components.py +5 -0
  49. fractal_server/app/runner/exceptions.py +129 -0
  50. fractal_server/app/runner/executors/__init__.py +0 -0
  51. fractal_server/app/runner/executors/slurm/__init__.py +3 -0
  52. fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
  53. fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py +1 -1
  54. fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +1 -1
  55. fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
  56. fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +1 -1
  57. fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +32 -21
  58. fractal_server/app/runner/filenames.py +6 -0
  59. fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
  60. fractal_server/app/runner/task_files.py +103 -0
  61. fractal_server/app/runner/v1/__init__.py +366 -0
  62. fractal_server/app/runner/{_common.py → v1/_common.py} +14 -121
  63. fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
  64. fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
  65. fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
  66. fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
  67. fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
  68. fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
  69. fractal_server/app/runner/v1/common.py +117 -0
  70. fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
  71. fractal_server/app/runner/v2/__init__.py +336 -0
  72. fractal_server/app/runner/v2/_local/__init__.py +162 -0
  73. fractal_server/app/runner/v2/_local/_local_config.py +118 -0
  74. fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
  75. fractal_server/app/runner/v2/_local/executor.py +100 -0
  76. fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
  77. fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
  78. fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
  79. fractal_server/app/runner/v2/deduplicate_list.py +23 -0
  80. fractal_server/app/runner/v2/handle_failed_job.py +165 -0
  81. fractal_server/app/runner/v2/merge_outputs.py +38 -0
  82. fractal_server/app/runner/v2/runner.py +343 -0
  83. fractal_server/app/runner/v2/runner_functions.py +374 -0
  84. fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
  85. fractal_server/app/runner/v2/task_interface.py +62 -0
  86. fractal_server/app/runner/v2/v1_compat.py +31 -0
  87. fractal_server/app/schemas/__init__.py +1 -42
  88. fractal_server/app/schemas/_validators.py +28 -5
  89. fractal_server/app/schemas/v1/__init__.py +36 -0
  90. fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
  91. fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
  92. fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
  93. fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
  94. fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
  95. fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
  96. fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
  97. fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
  98. fractal_server/app/schemas/v2/__init__.py +37 -0
  99. fractal_server/app/schemas/v2/dataset.py +126 -0
  100. fractal_server/app/schemas/v2/dumps.py +87 -0
  101. fractal_server/app/schemas/v2/job.py +114 -0
  102. fractal_server/app/schemas/v2/manifest.py +159 -0
  103. fractal_server/app/schemas/v2/project.py +34 -0
  104. fractal_server/app/schemas/v2/status.py +16 -0
  105. fractal_server/app/schemas/v2/task.py +151 -0
  106. fractal_server/app/schemas/v2/task_collection.py +109 -0
  107. fractal_server/app/schemas/v2/workflow.py +79 -0
  108. fractal_server/app/schemas/v2/workflowtask.py +208 -0
  109. fractal_server/config.py +5 -4
  110. fractal_server/images/__init__.py +4 -0
  111. fractal_server/images/models.py +136 -0
  112. fractal_server/images/tools.py +84 -0
  113. fractal_server/main.py +11 -3
  114. fractal_server/migrations/env.py +0 -2
  115. fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
  116. fractal_server/tasks/__init__.py +0 -5
  117. fractal_server/tasks/endpoint_operations.py +13 -19
  118. fractal_server/tasks/utils.py +35 -0
  119. fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
  120. fractal_server/tasks/v1/__init__.py +0 -0
  121. fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
  122. fractal_server/tasks/v1/get_collection_data.py +14 -0
  123. fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
  124. fractal_server/tasks/v2/__init__.py +0 -0
  125. fractal_server/tasks/v2/background_operations.py +381 -0
  126. fractal_server/tasks/v2/get_collection_data.py +14 -0
  127. fractal_server/urls.py +13 -0
  128. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/METADATA +10 -10
  129. fractal_server-2.0.0.dist-info/RECORD +169 -0
  130. fractal_server/app/runner/_slurm/.gitignore +0 -2
  131. fractal_server/app/runner/common.py +0 -311
  132. fractal_server/app/schemas/json_schemas/manifest.json +0 -81
  133. fractal_server-1.4.10.dist-info/RECORD +0 -98
  134. /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
  135. /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
  136. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
  137. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
  138. {fractal_server-1.4.10.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,200 @@
1
+ from pathlib import Path
2
+ from typing import Optional
3
+
4
+ from fastapi import APIRouter
5
+ from fastapi import Depends
6
+ from fastapi import Response
7
+ from fastapi import status
8
+ from fastapi.responses import StreamingResponse
9
+ from sqlmodel import select
10
+
11
+ from ....db import AsyncSession
12
+ from ....db import get_async_db
13
+ from ....models.v2 import JobV2
14
+ from ....models.v2 import ProjectV2
15
+ from ....runner.filenames import WORKFLOW_LOG_FILENAME
16
+ from ....schemas.v2 import JobReadV2
17
+ from ....schemas.v2 import JobStatusTypeV2
18
+ from ....security import current_active_user
19
+ from ....security import User
20
+ from ...aux._job import _write_shutdown_file
21
+ from ...aux._job import _zip_folder_to_byte_stream
22
+ from ...aux._runner import _check_backend_is_slurm
23
+ from ._aux_functions import _get_job_check_owner
24
+ from ._aux_functions import _get_project_check_owner
25
+ from ._aux_functions import _get_workflow_check_owner
26
+
27
+ router = APIRouter()
28
+
29
+
30
+ @router.get("/job/", response_model=list[JobReadV2])
31
+ async def get_user_jobs(
32
+ user: User = Depends(current_active_user),
33
+ log: bool = True,
34
+ db: AsyncSession = Depends(get_async_db),
35
+ ) -> list[JobReadV2]:
36
+ """
37
+ Returns all the jobs of the current user
38
+ """
39
+ stm = (
40
+ select(JobV2)
41
+ .join(ProjectV2)
42
+ .where(ProjectV2.user_list.any(User.id == user.id))
43
+ )
44
+ res = await db.execute(stm)
45
+ job_list = res.scalars().all()
46
+ await db.close()
47
+ if not log:
48
+ for job in job_list:
49
+ setattr(job, "log", None)
50
+
51
+ return job_list
52
+
53
+
54
+ @router.get(
55
+ "/project/{project_id}/workflow/{workflow_id}/job/",
56
+ response_model=list[JobReadV2],
57
+ )
58
+ async def get_workflow_jobs(
59
+ project_id: int,
60
+ workflow_id: int,
61
+ user: User = Depends(current_active_user),
62
+ db: AsyncSession = Depends(get_async_db),
63
+ ) -> Optional[list[JobReadV2]]:
64
+ """
65
+ Returns all the jobs related to a specific workflow
66
+ """
67
+ await _get_workflow_check_owner(
68
+ project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
69
+ )
70
+ stm = select(JobV2).where(JobV2.workflow_id == workflow_id)
71
+ res = await db.execute(stm)
72
+ job_list = res.scalars().all()
73
+ return job_list
74
+
75
+
76
+ @router.get(
77
+ "/project/{project_id}/job/{job_id}/",
78
+ response_model=JobReadV2,
79
+ )
80
+ async def read_job(
81
+ project_id: int,
82
+ job_id: int,
83
+ show_tmp_logs: bool = False,
84
+ user: User = Depends(current_active_user),
85
+ db: AsyncSession = Depends(get_async_db),
86
+ ) -> Optional[JobReadV2]:
87
+ """
88
+ Return info on an existing job
89
+ """
90
+
91
+ output = await _get_job_check_owner(
92
+ project_id=project_id,
93
+ job_id=job_id,
94
+ user_id=user.id,
95
+ db=db,
96
+ )
97
+ job = output["job"]
98
+ await db.close()
99
+
100
+ if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
101
+ try:
102
+ with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}", "r") as f:
103
+ job.log = f.read()
104
+ except FileNotFoundError:
105
+ pass
106
+
107
+ return job
108
+
109
+
110
+ @router.get(
111
+ "/project/{project_id}/job/{job_id}/download/",
112
+ response_class=StreamingResponse,
113
+ )
114
+ async def download_job_logs(
115
+ project_id: int,
116
+ job_id: int,
117
+ user: User = Depends(current_active_user),
118
+ db: AsyncSession = Depends(get_async_db),
119
+ ) -> StreamingResponse:
120
+ """
121
+ Download job folder
122
+ """
123
+ output = await _get_job_check_owner(
124
+ project_id=project_id,
125
+ job_id=job_id,
126
+ user_id=user.id,
127
+ db=db,
128
+ )
129
+ job = output["job"]
130
+
131
+ # Create and return byte stream for zipped log folder
132
+ PREFIX_ZIP = Path(job.working_dir).name
133
+ zip_filename = f"{PREFIX_ZIP}_archive.zip"
134
+ byte_stream = _zip_folder_to_byte_stream(
135
+ folder=job.working_dir, zip_filename=zip_filename
136
+ )
137
+ return StreamingResponse(
138
+ iter([byte_stream.getvalue()]),
139
+ media_type="application/x-zip-compressed",
140
+ headers={"Content-Disposition": f"attachment;filename={zip_filename}"},
141
+ )
142
+
143
+
144
+ @router.get(
145
+ "/project/{project_id}/job/",
146
+ response_model=list[JobReadV2],
147
+ )
148
+ async def get_job_list(
149
+ project_id: int,
150
+ user: User = Depends(current_active_user),
151
+ log: bool = True,
152
+ db: AsyncSession = Depends(get_async_db),
153
+ ) -> Optional[list[JobReadV2]]:
154
+ """
155
+ Get job list for given project
156
+ """
157
+ project = await _get_project_check_owner(
158
+ project_id=project_id, user_id=user.id, db=db
159
+ )
160
+
161
+ stm = select(JobV2).where(JobV2.project_id == project.id)
162
+ res = await db.execute(stm)
163
+ job_list = res.scalars().all()
164
+ await db.close()
165
+ if not log:
166
+ for job in job_list:
167
+ setattr(job, "log", None)
168
+
169
+ return job_list
170
+
171
+
172
+ @router.get(
173
+ "/project/{project_id}/job/{job_id}/stop/",
174
+ status_code=202,
175
+ )
176
+ async def stop_job(
177
+ project_id: int,
178
+ job_id: int,
179
+ user: User = Depends(current_active_user),
180
+ db: AsyncSession = Depends(get_async_db),
181
+ ) -> Response:
182
+ """
183
+ Stop execution of a workflow job (only available for slurm backend)
184
+ """
185
+
186
+ # This endpoint is only implemented for SLURM backend
187
+ _check_backend_is_slurm()
188
+
189
+ # Get job from DB
190
+ output = await _get_job_check_owner(
191
+ project_id=project_id,
192
+ job_id=job_id,
193
+ user_id=user.id,
194
+ db=db,
195
+ )
196
+ job = output["job"]
197
+
198
+ _write_shutdown_file(job=job)
199
+
200
+ return Response(status_code=status.HTTP_202_ACCEPTED)
@@ -0,0 +1,186 @@
1
+ from typing import Optional
2
+
3
+ from fastapi import APIRouter
4
+ from fastapi import Depends
5
+ from fastapi import HTTPException
6
+ from fastapi import Response
7
+ from fastapi import status
8
+ from sqlmodel import select
9
+
10
+ from ....db import AsyncSession
11
+ from ....db import get_async_db
12
+ from ....models.v2 import DatasetV2
13
+ from ....models.v2 import JobV2
14
+ from ....models.v2 import LinkUserProjectV2
15
+ from ....models.v2 import ProjectV2
16
+ from ....models.v2 import WorkflowV2
17
+ from ....schemas.v2 import ProjectCreateV2
18
+ from ....schemas.v2 import ProjectReadV2
19
+ from ....schemas.v2 import ProjectUpdateV2
20
+ from ....security import current_active_user
21
+ from ....security import User
22
+ from ._aux_functions import _check_project_exists
23
+ from ._aux_functions import _get_project_check_owner
24
+ from ._aux_functions import _get_submitted_jobs_statement
25
+
26
+ router = APIRouter()
27
+
28
+
29
+ @router.get("/project/", response_model=list[ProjectReadV2])
30
+ async def get_list_project(
31
+ user: User = Depends(current_active_user),
32
+ db: AsyncSession = Depends(get_async_db),
33
+ ) -> list[ProjectV2]:
34
+ """
35
+ Return list of projects user is member of
36
+ """
37
+ stm = (
38
+ select(ProjectV2)
39
+ .join(LinkUserProjectV2)
40
+ .where(LinkUserProjectV2.user_id == user.id)
41
+ )
42
+ res = await db.execute(stm)
43
+ project_list = res.scalars().all()
44
+ await db.close()
45
+ return project_list
46
+
47
+
48
+ @router.post("/project/", response_model=ProjectReadV2, status_code=201)
49
+ async def create_project(
50
+ project: ProjectCreateV2,
51
+ user: User = Depends(current_active_user),
52
+ db: AsyncSession = Depends(get_async_db),
53
+ ) -> Optional[ProjectReadV2]:
54
+ """
55
+ Create new poject
56
+ """
57
+
58
+ # Check that there is no project with the same user and name
59
+ await _check_project_exists(
60
+ project_name=project.name, user_id=user.id, db=db
61
+ )
62
+
63
+ db_project = ProjectV2(**project.dict())
64
+ db_project.user_list.append(user)
65
+
66
+ db.add(db_project)
67
+ await db.commit()
68
+ await db.refresh(db_project)
69
+ await db.close()
70
+
71
+ return db_project
72
+
73
+
74
+ @router.get("/project/{project_id}/", response_model=ProjectReadV2)
75
+ async def read_project(
76
+ project_id: int,
77
+ user: User = Depends(current_active_user),
78
+ db: AsyncSession = Depends(get_async_db),
79
+ ) -> Optional[ProjectReadV2]:
80
+ """
81
+ Return info on an existing project
82
+ """
83
+ project = await _get_project_check_owner(
84
+ project_id=project_id, user_id=user.id, db=db
85
+ )
86
+ await db.close()
87
+ return project
88
+
89
+
90
+ @router.patch("/project/{project_id}/", response_model=ProjectReadV2)
91
+ async def update_project(
92
+ project_id: int,
93
+ project_update: ProjectUpdateV2,
94
+ user: User = Depends(current_active_user),
95
+ db: AsyncSession = Depends(get_async_db),
96
+ ):
97
+ project = await _get_project_check_owner(
98
+ project_id=project_id, user_id=user.id, db=db
99
+ )
100
+
101
+ # Check that there is no project with the same user and name
102
+ if project_update.name is not None:
103
+ await _check_project_exists(
104
+ project_name=project_update.name, user_id=user.id, db=db
105
+ )
106
+
107
+ for key, value in project_update.dict(exclude_unset=True).items():
108
+ setattr(project, key, value)
109
+
110
+ await db.commit()
111
+ await db.refresh(project)
112
+ await db.close()
113
+ return project
114
+
115
+
116
+ @router.delete("/project/{project_id}/", status_code=204)
117
+ async def delete_project(
118
+ project_id: int,
119
+ user: User = Depends(current_active_user),
120
+ db: AsyncSession = Depends(get_async_db),
121
+ ) -> Response:
122
+ """
123
+ Delete project
124
+ """
125
+ project = await _get_project_check_owner(
126
+ project_id=project_id, user_id=user.id, db=db
127
+ )
128
+
129
+ # Fail if there exist jobs that are submitted and in relation with the
130
+ # current project.
131
+ stm = _get_submitted_jobs_statement().where(JobV2.project_id == project_id)
132
+ res = await db.execute(stm)
133
+ jobs = res.scalars().all()
134
+ if jobs:
135
+ string_ids = str([job.id for job in jobs])[1:-1]
136
+ raise HTTPException(
137
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
138
+ detail=(
139
+ f"Cannot delete project {project.id} because it "
140
+ f"is linked to active job(s) {string_ids}."
141
+ ),
142
+ )
143
+
144
+ # Cascade operations
145
+
146
+ # Workflows
147
+ stm = select(WorkflowV2).where(WorkflowV2.project_id == project_id)
148
+ res = await db.execute(stm)
149
+ workflows = res.scalars().all()
150
+ for wf in workflows:
151
+ # Cascade operations: set foreign-keys to null for jobs which are in
152
+ # relationship with the current workflow
153
+ stm = select(JobV2).where(JobV2.workflow_id == wf.id)
154
+ res = await db.execute(stm)
155
+ jobs = res.scalars().all()
156
+ for job in jobs:
157
+ job.workflow_id = None
158
+ # Delete workflow
159
+ await db.delete(wf)
160
+
161
+ # Dataset
162
+ stm = select(DatasetV2).where(DatasetV2.project_id == project_id)
163
+ res = await db.execute(stm)
164
+ datasets = res.scalars().all()
165
+ for ds in datasets:
166
+ # Cascade operations: set foreign-keys to null for jobs which are in
167
+ # relationship with the current dataset
168
+ stm = select(JobV2).where(JobV2.dataset_id == ds.id)
169
+ res = await db.execute(stm)
170
+ jobs = res.scalars().all()
171
+ for job in jobs:
172
+ job.dataset_id = None
173
+ # Delete dataset
174
+ await db.delete(ds)
175
+
176
+ # Job
177
+ stm = select(JobV2).where(JobV2.project_id == project_id)
178
+ res = await db.execute(stm)
179
+ jobs = res.scalars().all()
180
+ for job in jobs:
181
+ job.project_id = None
182
+
183
+ await db.delete(project)
184
+ await db.commit()
185
+
186
+ return Response(status_code=status.HTTP_204_NO_CONTENT)
@@ -0,0 +1,150 @@
1
+ import json
2
+ from pathlib import Path
3
+ from typing import Optional
4
+
5
+ from fastapi import APIRouter
6
+ from fastapi import Depends
7
+ from fastapi import HTTPException
8
+ from fastapi import status
9
+
10
+ from ....db import AsyncSession
11
+ from ....db import get_async_db
12
+ from ....models.v2 import JobV2
13
+ from ....schemas.v2.dataset import WorkflowTaskStatusTypeV2
14
+ from ....schemas.v2.status import StatusReadV2
15
+ from ....security import current_active_user
16
+ from ....security import User
17
+ from ._aux_functions import _get_dataset_check_owner
18
+ from ._aux_functions import _get_submitted_jobs_statement
19
+ from ._aux_functions import _get_workflow_check_owner
20
+ from fractal_server.app.runner.filenames import HISTORY_FILENAME
21
+
22
+ router = APIRouter()
23
+
24
+
25
+ @router.get(
26
+ "/project/{project_id}/status/",
27
+ response_model=StatusReadV2,
28
+ )
29
+ async def get_workflowtask_status(
30
+ project_id: int,
31
+ dataset_id: int,
32
+ workflow_id: int,
33
+ user: User = Depends(current_active_user),
34
+ db: AsyncSession = Depends(get_async_db),
35
+ ) -> Optional[StatusReadV2]:
36
+ """
37
+ Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
38
+ on a given `DatasetV2`.
39
+
40
+ *NOTE*: the current endpoint is not guaranteed to provide consistent
41
+ results if the workflow task list is modified in a non-trivial way
42
+ (that is, by adding intermediate tasks, removing tasks, or changing their
43
+ order). See fractal-server GitHub issues: 793, 1083.
44
+ """
45
+ # Get the dataset DB entry
46
+ output = await _get_dataset_check_owner(
47
+ project_id=project_id,
48
+ dataset_id=dataset_id,
49
+ user_id=user.id,
50
+ db=db,
51
+ )
52
+ dataset = output["dataset"]
53
+
54
+ # Get the workflow DB entry
55
+ workflow = await _get_workflow_check_owner(
56
+ project_id=project_id,
57
+ workflow_id=workflow_id,
58
+ user_id=user.id,
59
+ db=db,
60
+ )
61
+
62
+ # Check whether there exists a submitted job associated to this
63
+ # workflow/dataset pair. If it does exist, it will be used later.
64
+ # If there are multiple jobs, raise an error.
65
+ stm = _get_submitted_jobs_statement()
66
+ stm = stm.where(JobV2.dataset_id == dataset_id)
67
+ stm = stm.where(JobV2.workflow_id == workflow_id)
68
+ res = await db.execute(stm)
69
+ running_jobs = res.scalars().all()
70
+ if len(running_jobs) == 0:
71
+ running_job = None
72
+ elif len(running_jobs) == 1:
73
+ running_job = running_jobs[0]
74
+ else:
75
+ string_ids = str([job.id for job in running_jobs])[1:-1]
76
+ raise HTTPException(
77
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
78
+ detail=(
79
+ f"Cannot get WorkflowTaskV2 statuses as DatasetV2 {dataset.id}"
80
+ f" is linked to multiple active jobs: {string_ids}."
81
+ ),
82
+ )
83
+
84
+ # Initialize empty dictionary for WorkflowTaskV2 status
85
+ workflow_tasks_status_dict: dict = {}
86
+
87
+ # Lowest priority: read status from DB, which corresponds to jobs that are
88
+ # not running
89
+ history = dataset.history
90
+ for history_item in history:
91
+ wftask_id = history_item["workflowtask"]["id"]
92
+ wftask_status = history_item["status"]
93
+ workflow_tasks_status_dict[wftask_id] = wftask_status
94
+
95
+ if running_job is None:
96
+ # If no job is running, the chronological-last history item is also the
97
+ # positional-last workflow task to be included in the response.
98
+ if len(dataset.history) > 0:
99
+ last_valid_wftask_id = dataset.history[-1]["workflowtask"]["id"]
100
+ else:
101
+ last_valid_wftask_id = None
102
+ else:
103
+ # If a job is running, then gather more up-to-date information
104
+
105
+ # Mid priority: Set all WorkflowTask's that are part of the running job
106
+ # as "submitted"
107
+ start = running_job.first_task_index
108
+ end = running_job.last_task_index + 1
109
+ for wftask in workflow.task_list[start:end]:
110
+ workflow_tasks_status_dict[
111
+ wftask.id
112
+ ] = WorkflowTaskStatusTypeV2.SUBMITTED
113
+
114
+ # The last workflow task that is included in the submitted job is also
115
+ # the positional-last workflow task to be included in the response.
116
+ last_valid_wftask_id = workflow.task_list[end - 1]
117
+
118
+ # Highest priority: Read status updates coming from the running-job
119
+ # temporary file. Note: this file only contains information on
120
+ # WorkflowTask's that ran through successfully.
121
+ tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
122
+ try:
123
+ with tmp_file.open("r") as f:
124
+ history = json.load(f)
125
+ except FileNotFoundError:
126
+ history = []
127
+ for history_item in history:
128
+ wftask_id = history_item["workflowtask"]["id"]
129
+ wftask_status = history_item["status"]
130
+ workflow_tasks_status_dict[wftask_id] = wftask_status
131
+
132
+ # Based on previously-gathered information, clean up the response body
133
+ clean_workflow_tasks_status_dict = {}
134
+ for wf_task in workflow.task_list:
135
+ wf_task_status = workflow_tasks_status_dict.get(wf_task.id)
136
+ if wf_task_status is None:
137
+ # If a wftask ID was not found, ignore it and continue
138
+ continue
139
+ clean_workflow_tasks_status_dict[wf_task.id] = wf_task_status
140
+ if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
141
+ # Starting from the beginning of `workflow.task_list`, stop the
142
+ # first time that you hit a failed job
143
+ break
144
+ if wf_task.id == last_valid_wftask_id:
145
+ # Starting from the beginning of `workflow.task_list`, stop the
146
+ # first time that you hit `last_valid_wftask_id``
147
+ break
148
+
149
+ response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
150
+ return response_body