fractal-server 1.4.6__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/db/__init__.py +0 -1
- fractal_server/app/models/__init__.py +6 -8
- fractal_server/app/models/linkuserproject.py +9 -0
- fractal_server/app/models/security.py +6 -0
- fractal_server/app/models/v1/__init__.py +12 -0
- fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
- fractal_server/app/models/{job.py → v1/job.py} +5 -5
- fractal_server/app/models/{project.py → v1/project.py} +5 -5
- fractal_server/app/models/{state.py → v1/state.py} +2 -2
- fractal_server/app/models/{task.py → v1/task.py} +7 -2
- fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
- fractal_server/app/models/v2/__init__.py +22 -0
- fractal_server/app/models/v2/collection_state.py +21 -0
- fractal_server/app/models/v2/dataset.py +54 -0
- fractal_server/app/models/v2/job.py +51 -0
- fractal_server/app/models/v2/project.py +30 -0
- fractal_server/app/models/v2/task.py +93 -0
- fractal_server/app/models/v2/workflow.py +35 -0
- fractal_server/app/models/v2/workflowtask.py +49 -0
- fractal_server/app/routes/admin/__init__.py +0 -0
- fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
- fractal_server/app/routes/admin/v2.py +309 -0
- fractal_server/app/routes/api/v1/__init__.py +7 -7
- fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
- fractal_server/app/routes/api/v1/dataset.py +48 -41
- fractal_server/app/routes/api/v1/job.py +14 -14
- fractal_server/app/routes/api/v1/project.py +30 -27
- fractal_server/app/routes/api/v1/task.py +26 -16
- fractal_server/app/routes/api/v1/task_collection.py +28 -16
- fractal_server/app/routes/api/v1/workflow.py +28 -28
- fractal_server/app/routes/api/v1/workflowtask.py +11 -11
- fractal_server/app/routes/api/v2/__init__.py +34 -0
- fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
- fractal_server/app/routes/api/v2/dataset.py +293 -0
- fractal_server/app/routes/api/v2/images.py +279 -0
- fractal_server/app/routes/api/v2/job.py +200 -0
- fractal_server/app/routes/api/v2/project.py +186 -0
- fractal_server/app/routes/api/v2/status.py +150 -0
- fractal_server/app/routes/api/v2/submit.py +210 -0
- fractal_server/app/routes/api/v2/task.py +222 -0
- fractal_server/app/routes/api/v2/task_collection.py +239 -0
- fractal_server/app/routes/api/v2/task_legacy.py +59 -0
- fractal_server/app/routes/api/v2/workflow.py +380 -0
- fractal_server/app/routes/api/v2/workflowtask.py +265 -0
- fractal_server/app/routes/aux/_job.py +2 -2
- fractal_server/app/runner/__init__.py +0 -379
- fractal_server/app/runner/async_wrap.py +27 -0
- fractal_server/app/runner/components.py +5 -0
- fractal_server/app/runner/exceptions.py +129 -0
- fractal_server/app/runner/executors/__init__.py +0 -0
- fractal_server/app/runner/executors/slurm/__init__.py +3 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
- fractal_server/app/runner/executors/slurm/_check_jobs_status.py +72 -0
- fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +3 -4
- fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
- fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +42 -1
- fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +46 -27
- fractal_server/app/runner/filenames.py +6 -0
- fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
- fractal_server/app/runner/task_files.py +103 -0
- fractal_server/app/runner/v1/__init__.py +366 -0
- fractal_server/app/runner/{_common.py → v1/_common.py} +56 -111
- fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
- fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
- fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
- fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
- fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
- fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
- fractal_server/app/runner/v1/common.py +117 -0
- fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
- fractal_server/app/runner/v2/__init__.py +336 -0
- fractal_server/app/runner/v2/_local/__init__.py +162 -0
- fractal_server/app/runner/v2/_local/_local_config.py +118 -0
- fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
- fractal_server/app/runner/v2/_local/executor.py +100 -0
- fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
- fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
- fractal_server/app/runner/v2/deduplicate_list.py +23 -0
- fractal_server/app/runner/v2/handle_failed_job.py +165 -0
- fractal_server/app/runner/v2/merge_outputs.py +38 -0
- fractal_server/app/runner/v2/runner.py +343 -0
- fractal_server/app/runner/v2/runner_functions.py +374 -0
- fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
- fractal_server/app/runner/v2/task_interface.py +62 -0
- fractal_server/app/runner/v2/v1_compat.py +31 -0
- fractal_server/app/schemas/__init__.py +1 -42
- fractal_server/app/schemas/_validators.py +28 -5
- fractal_server/app/schemas/v1/__init__.py +36 -0
- fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
- fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
- fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
- fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
- fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
- fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
- fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
- fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
- fractal_server/app/schemas/v2/__init__.py +37 -0
- fractal_server/app/schemas/v2/dataset.py +126 -0
- fractal_server/app/schemas/v2/dumps.py +87 -0
- fractal_server/app/schemas/v2/job.py +114 -0
- fractal_server/app/schemas/v2/manifest.py +159 -0
- fractal_server/app/schemas/v2/project.py +34 -0
- fractal_server/app/schemas/v2/status.py +16 -0
- fractal_server/app/schemas/v2/task.py +151 -0
- fractal_server/app/schemas/v2/task_collection.py +109 -0
- fractal_server/app/schemas/v2/workflow.py +79 -0
- fractal_server/app/schemas/v2/workflowtask.py +208 -0
- fractal_server/config.py +13 -10
- fractal_server/images/__init__.py +4 -0
- fractal_server/images/models.py +136 -0
- fractal_server/images/tools.py +84 -0
- fractal_server/main.py +11 -3
- fractal_server/migrations/env.py +0 -2
- fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
- fractal_server/tasks/__init__.py +0 -5
- fractal_server/tasks/endpoint_operations.py +13 -19
- fractal_server/tasks/utils.py +35 -0
- fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
- fractal_server/tasks/v1/__init__.py +0 -0
- fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
- fractal_server/tasks/v1/get_collection_data.py +14 -0
- fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
- fractal_server/tasks/v2/__init__.py +0 -0
- fractal_server/tasks/v2/background_operations.py +381 -0
- fractal_server/tasks/v2/get_collection_data.py +14 -0
- fractal_server/urls.py +13 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/METADATA +11 -12
- fractal_server-2.0.0.dist-info/RECORD +169 -0
- fractal_server/app/runner/_slurm/.gitignore +0 -2
- fractal_server/app/runner/common.py +0 -307
- fractal_server/app/schemas/json_schemas/manifest.json +0 -81
- fractal_server-1.4.6.dist-info/RECORD +0 -97
- /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
- /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,200 @@
|
|
1
|
+
from pathlib import Path
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from fastapi import APIRouter
|
5
|
+
from fastapi import Depends
|
6
|
+
from fastapi import Response
|
7
|
+
from fastapi import status
|
8
|
+
from fastapi.responses import StreamingResponse
|
9
|
+
from sqlmodel import select
|
10
|
+
|
11
|
+
from ....db import AsyncSession
|
12
|
+
from ....db import get_async_db
|
13
|
+
from ....models.v2 import JobV2
|
14
|
+
from ....models.v2 import ProjectV2
|
15
|
+
from ....runner.filenames import WORKFLOW_LOG_FILENAME
|
16
|
+
from ....schemas.v2 import JobReadV2
|
17
|
+
from ....schemas.v2 import JobStatusTypeV2
|
18
|
+
from ....security import current_active_user
|
19
|
+
from ....security import User
|
20
|
+
from ...aux._job import _write_shutdown_file
|
21
|
+
from ...aux._job import _zip_folder_to_byte_stream
|
22
|
+
from ...aux._runner import _check_backend_is_slurm
|
23
|
+
from ._aux_functions import _get_job_check_owner
|
24
|
+
from ._aux_functions import _get_project_check_owner
|
25
|
+
from ._aux_functions import _get_workflow_check_owner
|
26
|
+
|
27
|
+
router = APIRouter()
|
28
|
+
|
29
|
+
|
30
|
+
@router.get("/job/", response_model=list[JobReadV2])
|
31
|
+
async def get_user_jobs(
|
32
|
+
user: User = Depends(current_active_user),
|
33
|
+
log: bool = True,
|
34
|
+
db: AsyncSession = Depends(get_async_db),
|
35
|
+
) -> list[JobReadV2]:
|
36
|
+
"""
|
37
|
+
Returns all the jobs of the current user
|
38
|
+
"""
|
39
|
+
stm = (
|
40
|
+
select(JobV2)
|
41
|
+
.join(ProjectV2)
|
42
|
+
.where(ProjectV2.user_list.any(User.id == user.id))
|
43
|
+
)
|
44
|
+
res = await db.execute(stm)
|
45
|
+
job_list = res.scalars().all()
|
46
|
+
await db.close()
|
47
|
+
if not log:
|
48
|
+
for job in job_list:
|
49
|
+
setattr(job, "log", None)
|
50
|
+
|
51
|
+
return job_list
|
52
|
+
|
53
|
+
|
54
|
+
@router.get(
|
55
|
+
"/project/{project_id}/workflow/{workflow_id}/job/",
|
56
|
+
response_model=list[JobReadV2],
|
57
|
+
)
|
58
|
+
async def get_workflow_jobs(
|
59
|
+
project_id: int,
|
60
|
+
workflow_id: int,
|
61
|
+
user: User = Depends(current_active_user),
|
62
|
+
db: AsyncSession = Depends(get_async_db),
|
63
|
+
) -> Optional[list[JobReadV2]]:
|
64
|
+
"""
|
65
|
+
Returns all the jobs related to a specific workflow
|
66
|
+
"""
|
67
|
+
await _get_workflow_check_owner(
|
68
|
+
project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
|
69
|
+
)
|
70
|
+
stm = select(JobV2).where(JobV2.workflow_id == workflow_id)
|
71
|
+
res = await db.execute(stm)
|
72
|
+
job_list = res.scalars().all()
|
73
|
+
return job_list
|
74
|
+
|
75
|
+
|
76
|
+
@router.get(
|
77
|
+
"/project/{project_id}/job/{job_id}/",
|
78
|
+
response_model=JobReadV2,
|
79
|
+
)
|
80
|
+
async def read_job(
|
81
|
+
project_id: int,
|
82
|
+
job_id: int,
|
83
|
+
show_tmp_logs: bool = False,
|
84
|
+
user: User = Depends(current_active_user),
|
85
|
+
db: AsyncSession = Depends(get_async_db),
|
86
|
+
) -> Optional[JobReadV2]:
|
87
|
+
"""
|
88
|
+
Return info on an existing job
|
89
|
+
"""
|
90
|
+
|
91
|
+
output = await _get_job_check_owner(
|
92
|
+
project_id=project_id,
|
93
|
+
job_id=job_id,
|
94
|
+
user_id=user.id,
|
95
|
+
db=db,
|
96
|
+
)
|
97
|
+
job = output["job"]
|
98
|
+
await db.close()
|
99
|
+
|
100
|
+
if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
|
101
|
+
try:
|
102
|
+
with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}", "r") as f:
|
103
|
+
job.log = f.read()
|
104
|
+
except FileNotFoundError:
|
105
|
+
pass
|
106
|
+
|
107
|
+
return job
|
108
|
+
|
109
|
+
|
110
|
+
@router.get(
|
111
|
+
"/project/{project_id}/job/{job_id}/download/",
|
112
|
+
response_class=StreamingResponse,
|
113
|
+
)
|
114
|
+
async def download_job_logs(
|
115
|
+
project_id: int,
|
116
|
+
job_id: int,
|
117
|
+
user: User = Depends(current_active_user),
|
118
|
+
db: AsyncSession = Depends(get_async_db),
|
119
|
+
) -> StreamingResponse:
|
120
|
+
"""
|
121
|
+
Download job folder
|
122
|
+
"""
|
123
|
+
output = await _get_job_check_owner(
|
124
|
+
project_id=project_id,
|
125
|
+
job_id=job_id,
|
126
|
+
user_id=user.id,
|
127
|
+
db=db,
|
128
|
+
)
|
129
|
+
job = output["job"]
|
130
|
+
|
131
|
+
# Create and return byte stream for zipped log folder
|
132
|
+
PREFIX_ZIP = Path(job.working_dir).name
|
133
|
+
zip_filename = f"{PREFIX_ZIP}_archive.zip"
|
134
|
+
byte_stream = _zip_folder_to_byte_stream(
|
135
|
+
folder=job.working_dir, zip_filename=zip_filename
|
136
|
+
)
|
137
|
+
return StreamingResponse(
|
138
|
+
iter([byte_stream.getvalue()]),
|
139
|
+
media_type="application/x-zip-compressed",
|
140
|
+
headers={"Content-Disposition": f"attachment;filename={zip_filename}"},
|
141
|
+
)
|
142
|
+
|
143
|
+
|
144
|
+
@router.get(
|
145
|
+
"/project/{project_id}/job/",
|
146
|
+
response_model=list[JobReadV2],
|
147
|
+
)
|
148
|
+
async def get_job_list(
|
149
|
+
project_id: int,
|
150
|
+
user: User = Depends(current_active_user),
|
151
|
+
log: bool = True,
|
152
|
+
db: AsyncSession = Depends(get_async_db),
|
153
|
+
) -> Optional[list[JobReadV2]]:
|
154
|
+
"""
|
155
|
+
Get job list for given project
|
156
|
+
"""
|
157
|
+
project = await _get_project_check_owner(
|
158
|
+
project_id=project_id, user_id=user.id, db=db
|
159
|
+
)
|
160
|
+
|
161
|
+
stm = select(JobV2).where(JobV2.project_id == project.id)
|
162
|
+
res = await db.execute(stm)
|
163
|
+
job_list = res.scalars().all()
|
164
|
+
await db.close()
|
165
|
+
if not log:
|
166
|
+
for job in job_list:
|
167
|
+
setattr(job, "log", None)
|
168
|
+
|
169
|
+
return job_list
|
170
|
+
|
171
|
+
|
172
|
+
@router.get(
|
173
|
+
"/project/{project_id}/job/{job_id}/stop/",
|
174
|
+
status_code=202,
|
175
|
+
)
|
176
|
+
async def stop_job(
|
177
|
+
project_id: int,
|
178
|
+
job_id: int,
|
179
|
+
user: User = Depends(current_active_user),
|
180
|
+
db: AsyncSession = Depends(get_async_db),
|
181
|
+
) -> Response:
|
182
|
+
"""
|
183
|
+
Stop execution of a workflow job (only available for slurm backend)
|
184
|
+
"""
|
185
|
+
|
186
|
+
# This endpoint is only implemented for SLURM backend
|
187
|
+
_check_backend_is_slurm()
|
188
|
+
|
189
|
+
# Get job from DB
|
190
|
+
output = await _get_job_check_owner(
|
191
|
+
project_id=project_id,
|
192
|
+
job_id=job_id,
|
193
|
+
user_id=user.id,
|
194
|
+
db=db,
|
195
|
+
)
|
196
|
+
job = output["job"]
|
197
|
+
|
198
|
+
_write_shutdown_file(job=job)
|
199
|
+
|
200
|
+
return Response(status_code=status.HTTP_202_ACCEPTED)
|
@@ -0,0 +1,186 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
from fastapi import APIRouter
|
4
|
+
from fastapi import Depends
|
5
|
+
from fastapi import HTTPException
|
6
|
+
from fastapi import Response
|
7
|
+
from fastapi import status
|
8
|
+
from sqlmodel import select
|
9
|
+
|
10
|
+
from ....db import AsyncSession
|
11
|
+
from ....db import get_async_db
|
12
|
+
from ....models.v2 import DatasetV2
|
13
|
+
from ....models.v2 import JobV2
|
14
|
+
from ....models.v2 import LinkUserProjectV2
|
15
|
+
from ....models.v2 import ProjectV2
|
16
|
+
from ....models.v2 import WorkflowV2
|
17
|
+
from ....schemas.v2 import ProjectCreateV2
|
18
|
+
from ....schemas.v2 import ProjectReadV2
|
19
|
+
from ....schemas.v2 import ProjectUpdateV2
|
20
|
+
from ....security import current_active_user
|
21
|
+
from ....security import User
|
22
|
+
from ._aux_functions import _check_project_exists
|
23
|
+
from ._aux_functions import _get_project_check_owner
|
24
|
+
from ._aux_functions import _get_submitted_jobs_statement
|
25
|
+
|
26
|
+
router = APIRouter()
|
27
|
+
|
28
|
+
|
29
|
+
@router.get("/project/", response_model=list[ProjectReadV2])
|
30
|
+
async def get_list_project(
|
31
|
+
user: User = Depends(current_active_user),
|
32
|
+
db: AsyncSession = Depends(get_async_db),
|
33
|
+
) -> list[ProjectV2]:
|
34
|
+
"""
|
35
|
+
Return list of projects user is member of
|
36
|
+
"""
|
37
|
+
stm = (
|
38
|
+
select(ProjectV2)
|
39
|
+
.join(LinkUserProjectV2)
|
40
|
+
.where(LinkUserProjectV2.user_id == user.id)
|
41
|
+
)
|
42
|
+
res = await db.execute(stm)
|
43
|
+
project_list = res.scalars().all()
|
44
|
+
await db.close()
|
45
|
+
return project_list
|
46
|
+
|
47
|
+
|
48
|
+
@router.post("/project/", response_model=ProjectReadV2, status_code=201)
|
49
|
+
async def create_project(
|
50
|
+
project: ProjectCreateV2,
|
51
|
+
user: User = Depends(current_active_user),
|
52
|
+
db: AsyncSession = Depends(get_async_db),
|
53
|
+
) -> Optional[ProjectReadV2]:
|
54
|
+
"""
|
55
|
+
Create new poject
|
56
|
+
"""
|
57
|
+
|
58
|
+
# Check that there is no project with the same user and name
|
59
|
+
await _check_project_exists(
|
60
|
+
project_name=project.name, user_id=user.id, db=db
|
61
|
+
)
|
62
|
+
|
63
|
+
db_project = ProjectV2(**project.dict())
|
64
|
+
db_project.user_list.append(user)
|
65
|
+
|
66
|
+
db.add(db_project)
|
67
|
+
await db.commit()
|
68
|
+
await db.refresh(db_project)
|
69
|
+
await db.close()
|
70
|
+
|
71
|
+
return db_project
|
72
|
+
|
73
|
+
|
74
|
+
@router.get("/project/{project_id}/", response_model=ProjectReadV2)
|
75
|
+
async def read_project(
|
76
|
+
project_id: int,
|
77
|
+
user: User = Depends(current_active_user),
|
78
|
+
db: AsyncSession = Depends(get_async_db),
|
79
|
+
) -> Optional[ProjectReadV2]:
|
80
|
+
"""
|
81
|
+
Return info on an existing project
|
82
|
+
"""
|
83
|
+
project = await _get_project_check_owner(
|
84
|
+
project_id=project_id, user_id=user.id, db=db
|
85
|
+
)
|
86
|
+
await db.close()
|
87
|
+
return project
|
88
|
+
|
89
|
+
|
90
|
+
@router.patch("/project/{project_id}/", response_model=ProjectReadV2)
|
91
|
+
async def update_project(
|
92
|
+
project_id: int,
|
93
|
+
project_update: ProjectUpdateV2,
|
94
|
+
user: User = Depends(current_active_user),
|
95
|
+
db: AsyncSession = Depends(get_async_db),
|
96
|
+
):
|
97
|
+
project = await _get_project_check_owner(
|
98
|
+
project_id=project_id, user_id=user.id, db=db
|
99
|
+
)
|
100
|
+
|
101
|
+
# Check that there is no project with the same user and name
|
102
|
+
if project_update.name is not None:
|
103
|
+
await _check_project_exists(
|
104
|
+
project_name=project_update.name, user_id=user.id, db=db
|
105
|
+
)
|
106
|
+
|
107
|
+
for key, value in project_update.dict(exclude_unset=True).items():
|
108
|
+
setattr(project, key, value)
|
109
|
+
|
110
|
+
await db.commit()
|
111
|
+
await db.refresh(project)
|
112
|
+
await db.close()
|
113
|
+
return project
|
114
|
+
|
115
|
+
|
116
|
+
@router.delete("/project/{project_id}/", status_code=204)
|
117
|
+
async def delete_project(
|
118
|
+
project_id: int,
|
119
|
+
user: User = Depends(current_active_user),
|
120
|
+
db: AsyncSession = Depends(get_async_db),
|
121
|
+
) -> Response:
|
122
|
+
"""
|
123
|
+
Delete project
|
124
|
+
"""
|
125
|
+
project = await _get_project_check_owner(
|
126
|
+
project_id=project_id, user_id=user.id, db=db
|
127
|
+
)
|
128
|
+
|
129
|
+
# Fail if there exist jobs that are submitted and in relation with the
|
130
|
+
# current project.
|
131
|
+
stm = _get_submitted_jobs_statement().where(JobV2.project_id == project_id)
|
132
|
+
res = await db.execute(stm)
|
133
|
+
jobs = res.scalars().all()
|
134
|
+
if jobs:
|
135
|
+
string_ids = str([job.id for job in jobs])[1:-1]
|
136
|
+
raise HTTPException(
|
137
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
138
|
+
detail=(
|
139
|
+
f"Cannot delete project {project.id} because it "
|
140
|
+
f"is linked to active job(s) {string_ids}."
|
141
|
+
),
|
142
|
+
)
|
143
|
+
|
144
|
+
# Cascade operations
|
145
|
+
|
146
|
+
# Workflows
|
147
|
+
stm = select(WorkflowV2).where(WorkflowV2.project_id == project_id)
|
148
|
+
res = await db.execute(stm)
|
149
|
+
workflows = res.scalars().all()
|
150
|
+
for wf in workflows:
|
151
|
+
# Cascade operations: set foreign-keys to null for jobs which are in
|
152
|
+
# relationship with the current workflow
|
153
|
+
stm = select(JobV2).where(JobV2.workflow_id == wf.id)
|
154
|
+
res = await db.execute(stm)
|
155
|
+
jobs = res.scalars().all()
|
156
|
+
for job in jobs:
|
157
|
+
job.workflow_id = None
|
158
|
+
# Delete workflow
|
159
|
+
await db.delete(wf)
|
160
|
+
|
161
|
+
# Dataset
|
162
|
+
stm = select(DatasetV2).where(DatasetV2.project_id == project_id)
|
163
|
+
res = await db.execute(stm)
|
164
|
+
datasets = res.scalars().all()
|
165
|
+
for ds in datasets:
|
166
|
+
# Cascade operations: set foreign-keys to null for jobs which are in
|
167
|
+
# relationship with the current dataset
|
168
|
+
stm = select(JobV2).where(JobV2.dataset_id == ds.id)
|
169
|
+
res = await db.execute(stm)
|
170
|
+
jobs = res.scalars().all()
|
171
|
+
for job in jobs:
|
172
|
+
job.dataset_id = None
|
173
|
+
# Delete dataset
|
174
|
+
await db.delete(ds)
|
175
|
+
|
176
|
+
# Job
|
177
|
+
stm = select(JobV2).where(JobV2.project_id == project_id)
|
178
|
+
res = await db.execute(stm)
|
179
|
+
jobs = res.scalars().all()
|
180
|
+
for job in jobs:
|
181
|
+
job.project_id = None
|
182
|
+
|
183
|
+
await db.delete(project)
|
184
|
+
await db.commit()
|
185
|
+
|
186
|
+
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
@@ -0,0 +1,150 @@
|
|
1
|
+
import json
|
2
|
+
from pathlib import Path
|
3
|
+
from typing import Optional
|
4
|
+
|
5
|
+
from fastapi import APIRouter
|
6
|
+
from fastapi import Depends
|
7
|
+
from fastapi import HTTPException
|
8
|
+
from fastapi import status
|
9
|
+
|
10
|
+
from ....db import AsyncSession
|
11
|
+
from ....db import get_async_db
|
12
|
+
from ....models.v2 import JobV2
|
13
|
+
from ....schemas.v2.dataset import WorkflowTaskStatusTypeV2
|
14
|
+
from ....schemas.v2.status import StatusReadV2
|
15
|
+
from ....security import current_active_user
|
16
|
+
from ....security import User
|
17
|
+
from ._aux_functions import _get_dataset_check_owner
|
18
|
+
from ._aux_functions import _get_submitted_jobs_statement
|
19
|
+
from ._aux_functions import _get_workflow_check_owner
|
20
|
+
from fractal_server.app.runner.filenames import HISTORY_FILENAME
|
21
|
+
|
22
|
+
router = APIRouter()
|
23
|
+
|
24
|
+
|
25
|
+
@router.get(
|
26
|
+
"/project/{project_id}/status/",
|
27
|
+
response_model=StatusReadV2,
|
28
|
+
)
|
29
|
+
async def get_workflowtask_status(
|
30
|
+
project_id: int,
|
31
|
+
dataset_id: int,
|
32
|
+
workflow_id: int,
|
33
|
+
user: User = Depends(current_active_user),
|
34
|
+
db: AsyncSession = Depends(get_async_db),
|
35
|
+
) -> Optional[StatusReadV2]:
|
36
|
+
"""
|
37
|
+
Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
|
38
|
+
on a given `DatasetV2`.
|
39
|
+
|
40
|
+
*NOTE*: the current endpoint is not guaranteed to provide consistent
|
41
|
+
results if the workflow task list is modified in a non-trivial way
|
42
|
+
(that is, by adding intermediate tasks, removing tasks, or changing their
|
43
|
+
order). See fractal-server GitHub issues: 793, 1083.
|
44
|
+
"""
|
45
|
+
# Get the dataset DB entry
|
46
|
+
output = await _get_dataset_check_owner(
|
47
|
+
project_id=project_id,
|
48
|
+
dataset_id=dataset_id,
|
49
|
+
user_id=user.id,
|
50
|
+
db=db,
|
51
|
+
)
|
52
|
+
dataset = output["dataset"]
|
53
|
+
|
54
|
+
# Get the workflow DB entry
|
55
|
+
workflow = await _get_workflow_check_owner(
|
56
|
+
project_id=project_id,
|
57
|
+
workflow_id=workflow_id,
|
58
|
+
user_id=user.id,
|
59
|
+
db=db,
|
60
|
+
)
|
61
|
+
|
62
|
+
# Check whether there exists a submitted job associated to this
|
63
|
+
# workflow/dataset pair. If it does exist, it will be used later.
|
64
|
+
# If there are multiple jobs, raise an error.
|
65
|
+
stm = _get_submitted_jobs_statement()
|
66
|
+
stm = stm.where(JobV2.dataset_id == dataset_id)
|
67
|
+
stm = stm.where(JobV2.workflow_id == workflow_id)
|
68
|
+
res = await db.execute(stm)
|
69
|
+
running_jobs = res.scalars().all()
|
70
|
+
if len(running_jobs) == 0:
|
71
|
+
running_job = None
|
72
|
+
elif len(running_jobs) == 1:
|
73
|
+
running_job = running_jobs[0]
|
74
|
+
else:
|
75
|
+
string_ids = str([job.id for job in running_jobs])[1:-1]
|
76
|
+
raise HTTPException(
|
77
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
78
|
+
detail=(
|
79
|
+
f"Cannot get WorkflowTaskV2 statuses as DatasetV2 {dataset.id}"
|
80
|
+
f" is linked to multiple active jobs: {string_ids}."
|
81
|
+
),
|
82
|
+
)
|
83
|
+
|
84
|
+
# Initialize empty dictionary for WorkflowTaskV2 status
|
85
|
+
workflow_tasks_status_dict: dict = {}
|
86
|
+
|
87
|
+
# Lowest priority: read status from DB, which corresponds to jobs that are
|
88
|
+
# not running
|
89
|
+
history = dataset.history
|
90
|
+
for history_item in history:
|
91
|
+
wftask_id = history_item["workflowtask"]["id"]
|
92
|
+
wftask_status = history_item["status"]
|
93
|
+
workflow_tasks_status_dict[wftask_id] = wftask_status
|
94
|
+
|
95
|
+
if running_job is None:
|
96
|
+
# If no job is running, the chronological-last history item is also the
|
97
|
+
# positional-last workflow task to be included in the response.
|
98
|
+
if len(dataset.history) > 0:
|
99
|
+
last_valid_wftask_id = dataset.history[-1]["workflowtask"]["id"]
|
100
|
+
else:
|
101
|
+
last_valid_wftask_id = None
|
102
|
+
else:
|
103
|
+
# If a job is running, then gather more up-to-date information
|
104
|
+
|
105
|
+
# Mid priority: Set all WorkflowTask's that are part of the running job
|
106
|
+
# as "submitted"
|
107
|
+
start = running_job.first_task_index
|
108
|
+
end = running_job.last_task_index + 1
|
109
|
+
for wftask in workflow.task_list[start:end]:
|
110
|
+
workflow_tasks_status_dict[
|
111
|
+
wftask.id
|
112
|
+
] = WorkflowTaskStatusTypeV2.SUBMITTED
|
113
|
+
|
114
|
+
# The last workflow task that is included in the submitted job is also
|
115
|
+
# the positional-last workflow task to be included in the response.
|
116
|
+
last_valid_wftask_id = workflow.task_list[end - 1]
|
117
|
+
|
118
|
+
# Highest priority: Read status updates coming from the running-job
|
119
|
+
# temporary file. Note: this file only contains information on
|
120
|
+
# WorkflowTask's that ran through successfully.
|
121
|
+
tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
|
122
|
+
try:
|
123
|
+
with tmp_file.open("r") as f:
|
124
|
+
history = json.load(f)
|
125
|
+
except FileNotFoundError:
|
126
|
+
history = []
|
127
|
+
for history_item in history:
|
128
|
+
wftask_id = history_item["workflowtask"]["id"]
|
129
|
+
wftask_status = history_item["status"]
|
130
|
+
workflow_tasks_status_dict[wftask_id] = wftask_status
|
131
|
+
|
132
|
+
# Based on previously-gathered information, clean up the response body
|
133
|
+
clean_workflow_tasks_status_dict = {}
|
134
|
+
for wf_task in workflow.task_list:
|
135
|
+
wf_task_status = workflow_tasks_status_dict.get(wf_task.id)
|
136
|
+
if wf_task_status is None:
|
137
|
+
# If a wftask ID was not found, ignore it and continue
|
138
|
+
continue
|
139
|
+
clean_workflow_tasks_status_dict[wf_task.id] = wf_task_status
|
140
|
+
if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
|
141
|
+
# Starting from the beginning of `workflow.task_list`, stop the
|
142
|
+
# first time that you hit a failed job
|
143
|
+
break
|
144
|
+
if wf_task.id == last_valid_wftask_id:
|
145
|
+
# Starting from the beginning of `workflow.task_list`, stop the
|
146
|
+
# first time that you hit `last_valid_wftask_id``
|
147
|
+
break
|
148
|
+
|
149
|
+
response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
|
150
|
+
return response_body
|