fractal-server 1.4.2a5__py3-none-any.whl → 1.4.3a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/db/__init__.py +36 -25
- fractal_server/app/models/dataset.py +8 -0
- fractal_server/app/models/workflow.py +9 -55
- fractal_server/app/routes/admin.py +8 -8
- fractal_server/app/routes/api/v1/_aux_functions.py +64 -5
- fractal_server/app/routes/api/v1/dataset.py +24 -23
- fractal_server/app/routes/api/v1/job.py +7 -7
- fractal_server/app/routes/api/v1/project.py +29 -26
- fractal_server/app/routes/api/v1/task.py +6 -6
- fractal_server/app/routes/api/v1/task_collection.py +12 -126
- fractal_server/app/routes/api/v1/workflow.py +16 -14
- fractal_server/app/routes/api/v1/workflowtask.py +8 -6
- fractal_server/app/routes/auth.py +2 -2
- fractal_server/app/runner/__init__.py +0 -1
- fractal_server/app/schemas/__init__.py +1 -0
- fractal_server/app/schemas/applyworkflow.py +9 -13
- fractal_server/app/schemas/dataset.py +2 -0
- fractal_server/app/schemas/dumps.py +2 -0
- fractal_server/app/schemas/task_collection.py +2 -10
- fractal_server/app/schemas/user.py +7 -3
- fractal_server/app/schemas/workflow.py +2 -0
- fractal_server/app/security/__init__.py +3 -3
- fractal_server/config.py +14 -0
- fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +42 -0
- fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +60 -0
- fractal_server/tasks/_TaskCollectPip.py +103 -0
- fractal_server/tasks/__init__.py +3 -1
- fractal_server/tasks/background_operations.py +384 -0
- fractal_server/tasks/endpoint_operations.py +167 -0
- fractal_server/tasks/utils.py +86 -0
- {fractal_server-1.4.2a5.dist-info → fractal_server-1.4.3a1.dist-info}/METADATA +2 -2
- {fractal_server-1.4.2a5.dist-info → fractal_server-1.4.3a1.dist-info}/RECORD +36 -31
- fractal_server/tasks/collection.py +0 -556
- {fractal_server-1.4.2a5.dist-info → fractal_server-1.4.3a1.dist-info}/LICENSE +0 -0
- {fractal_server-1.4.2a5.dist-info → fractal_server-1.4.3a1.dist-info}/WHEEL +0 -0
- {fractal_server-1.4.2a5.dist-info → fractal_server-1.4.3a1.dist-info}/entry_points.txt +0 -0
@@ -1,4 +1,3 @@
|
|
1
|
-
import json
|
2
1
|
from typing import Optional
|
3
2
|
|
4
3
|
from fastapi import APIRouter
|
@@ -15,7 +14,7 @@ from .....logger import close_logger
|
|
15
14
|
from .....logger import set_logger
|
16
15
|
from .....syringe import Inject
|
17
16
|
from ....db import AsyncSession
|
18
|
-
from ....db import
|
17
|
+
from ....db import get_async_db
|
19
18
|
from ....models import ApplyWorkflow
|
20
19
|
from ....models import Dataset
|
21
20
|
from ....models import LinkUserProject
|
@@ -34,9 +33,9 @@ from ....security import current_active_user
|
|
34
33
|
from ....security import current_active_verified_user
|
35
34
|
from ....security import User
|
36
35
|
from ._aux_functions import _check_project_exists
|
37
|
-
from ._aux_functions import _get_active_jobs_statement
|
38
36
|
from ._aux_functions import _get_dataset_check_owner
|
39
37
|
from ._aux_functions import _get_project_check_owner
|
38
|
+
from ._aux_functions import _get_submitted_jobs_statement
|
40
39
|
from ._aux_functions import _get_workflow_check_owner
|
41
40
|
|
42
41
|
|
@@ -46,7 +45,7 @@ router = APIRouter()
|
|
46
45
|
@router.get("/", response_model=list[ProjectRead])
|
47
46
|
async def get_list_project(
|
48
47
|
user: User = Depends(current_active_user),
|
49
|
-
db: AsyncSession = Depends(
|
48
|
+
db: AsyncSession = Depends(get_async_db),
|
50
49
|
) -> list[Project]:
|
51
50
|
"""
|
52
51
|
Return list of projects user is member of
|
@@ -66,7 +65,7 @@ async def get_list_project(
|
|
66
65
|
async def create_project(
|
67
66
|
project: ProjectCreate,
|
68
67
|
user: User = Depends(current_active_user),
|
69
|
-
db: AsyncSession = Depends(
|
68
|
+
db: AsyncSession = Depends(get_async_db),
|
70
69
|
) -> Optional[ProjectRead]:
|
71
70
|
"""
|
72
71
|
Create new poject
|
@@ -101,7 +100,7 @@ async def create_project(
|
|
101
100
|
async def read_project(
|
102
101
|
project_id: int,
|
103
102
|
user: User = Depends(current_active_user),
|
104
|
-
db: AsyncSession = Depends(
|
103
|
+
db: AsyncSession = Depends(get_async_db),
|
105
104
|
) -> Optional[ProjectRead]:
|
106
105
|
"""
|
107
106
|
Return info on an existing project
|
@@ -118,7 +117,7 @@ async def update_project(
|
|
118
117
|
project_id: int,
|
119
118
|
project_update: ProjectUpdate,
|
120
119
|
user: User = Depends(current_active_user),
|
121
|
-
db: AsyncSession = Depends(
|
120
|
+
db: AsyncSession = Depends(get_async_db),
|
122
121
|
):
|
123
122
|
project = await _get_project_check_owner(
|
124
123
|
project_id=project_id, user_id=user.id, db=db
|
@@ -143,7 +142,7 @@ async def update_project(
|
|
143
142
|
async def delete_project(
|
144
143
|
project_id: int,
|
145
144
|
user: User = Depends(current_active_user),
|
146
|
-
db: AsyncSession = Depends(
|
145
|
+
db: AsyncSession = Depends(get_async_db),
|
147
146
|
) -> Response:
|
148
147
|
"""
|
149
148
|
Delete project
|
@@ -152,9 +151,9 @@ async def delete_project(
|
|
152
151
|
project_id=project_id, user_id=user.id, db=db
|
153
152
|
)
|
154
153
|
|
155
|
-
# Fail if there exist jobs that are
|
156
|
-
#
|
157
|
-
stm =
|
154
|
+
# Fail if there exist jobs that are submitted and in relation with the
|
155
|
+
# current project.
|
156
|
+
stm = _get_submitted_jobs_statement().where(
|
158
157
|
ApplyWorkflow.project_id == project_id
|
159
158
|
)
|
160
159
|
res = await db.execute(stm)
|
@@ -246,7 +245,7 @@ async def apply_workflow(
|
|
246
245
|
input_dataset_id: int,
|
247
246
|
output_dataset_id: int,
|
248
247
|
user: User = Depends(current_active_verified_user),
|
249
|
-
db: AsyncSession = Depends(
|
248
|
+
db: AsyncSession = Depends(get_async_db),
|
250
249
|
) -> Optional[ApplyWorkflowRead]:
|
251
250
|
|
252
251
|
output = await _get_dataset_check_owner(
|
@@ -353,16 +352,11 @@ async def apply_workflow(
|
|
353
352
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)
|
354
353
|
)
|
355
354
|
|
356
|
-
# Check that no other job with the same output_dataset_id is
|
357
|
-
# SUBMITTED or RUNNING
|
355
|
+
# Check that no other job with the same output_dataset_id is SUBMITTED
|
358
356
|
stm = (
|
359
357
|
select(ApplyWorkflow)
|
360
358
|
.where(ApplyWorkflow.output_dataset_id == output_dataset_id)
|
361
|
-
.where(
|
362
|
-
ApplyWorkflow.status.in_(
|
363
|
-
[JobStatusType.SUBMITTED, JobStatusType.RUNNING]
|
364
|
-
)
|
365
|
-
)
|
359
|
+
.where(ApplyWorkflow.status == JobStatusType.SUBMITTED)
|
366
360
|
)
|
367
361
|
res = await db.execute(stm)
|
368
362
|
if res.scalars().all():
|
@@ -370,7 +364,7 @@ async def apply_workflow(
|
|
370
364
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
371
365
|
detail=(
|
372
366
|
f"Output dataset {output_dataset_id} is already in use "
|
373
|
-
"in
|
367
|
+
"in submitted job(s)."
|
374
368
|
),
|
375
369
|
)
|
376
370
|
|
@@ -395,31 +389,40 @@ async def apply_workflow(
|
|
395
389
|
workflow_id=workflow_id,
|
396
390
|
user_email=user.email,
|
397
391
|
input_dataset_dump=dict(
|
398
|
-
input_dataset.model_dump(
|
392
|
+
**input_dataset.model_dump(
|
393
|
+
exclude={"resource_list", "timestamp_created"}
|
394
|
+
),
|
395
|
+
timestamp_created=str(input_dataset.timestamp_created),
|
399
396
|
resource_list=[
|
400
397
|
resource.model_dump()
|
401
398
|
for resource in input_dataset.resource_list
|
402
399
|
],
|
403
400
|
),
|
404
401
|
output_dataset_dump=dict(
|
405
|
-
output_dataset.model_dump(
|
402
|
+
**output_dataset.model_dump(
|
403
|
+
exclude={"resource_list", "timestamp_created"}
|
404
|
+
),
|
405
|
+
timestamp_created=str(output_dataset.timestamp_created),
|
406
406
|
resource_list=[
|
407
407
|
resource.model_dump()
|
408
408
|
for resource in output_dataset.resource_list
|
409
409
|
],
|
410
410
|
),
|
411
411
|
workflow_dump=dict(
|
412
|
-
workflow.model_dump(exclude={"task_list"}),
|
412
|
+
**workflow.model_dump(exclude={"task_list", "timestamp_created"}),
|
413
|
+
timestamp_created=str(workflow.timestamp_created),
|
413
414
|
task_list=[
|
414
415
|
dict(
|
415
|
-
wf_task.model_dump(exclude={"task"}),
|
416
|
+
**wf_task.model_dump(exclude={"task"}),
|
416
417
|
task=wf_task.task.model_dump(),
|
417
418
|
)
|
418
419
|
for wf_task in workflow.task_list
|
419
420
|
],
|
420
421
|
),
|
421
|
-
|
422
|
-
|
422
|
+
project_dump=dict(
|
423
|
+
**project.model_dump(exclude={"user_list", "timestamp_created"}),
|
424
|
+
timestamp_created=str(project.timestamp_created),
|
425
|
+
),
|
423
426
|
**apply_workflow.dict(),
|
424
427
|
)
|
425
428
|
db.add(job)
|
@@ -10,7 +10,7 @@ from sqlmodel import select
|
|
10
10
|
|
11
11
|
from .....logger import set_logger
|
12
12
|
from ....db import AsyncSession
|
13
|
-
from ....db import
|
13
|
+
from ....db import get_async_db
|
14
14
|
from ....models import Task
|
15
15
|
from ....models import WorkflowTask
|
16
16
|
from ....schemas import TaskCreate
|
@@ -29,7 +29,7 @@ logger = set_logger(__name__)
|
|
29
29
|
@router.get("/", response_model=list[TaskRead])
|
30
30
|
async def get_list_task(
|
31
31
|
user: User = Depends(current_active_user),
|
32
|
-
db: AsyncSession = Depends(
|
32
|
+
db: AsyncSession = Depends(get_async_db),
|
33
33
|
) -> list[TaskRead]:
|
34
34
|
"""
|
35
35
|
Get list of available tasks
|
@@ -45,7 +45,7 @@ async def get_list_task(
|
|
45
45
|
async def get_task(
|
46
46
|
task_id: int,
|
47
47
|
user: User = Depends(current_active_user),
|
48
|
-
db: AsyncSession = Depends(
|
48
|
+
db: AsyncSession = Depends(get_async_db),
|
49
49
|
) -> TaskRead:
|
50
50
|
"""
|
51
51
|
Get info on a specific task
|
@@ -64,7 +64,7 @@ async def patch_task(
|
|
64
64
|
task_id: int,
|
65
65
|
task_update: TaskUpdate,
|
66
66
|
user: User = Depends(current_active_verified_user),
|
67
|
-
db: AsyncSession = Depends(
|
67
|
+
db: AsyncSession = Depends(get_async_db),
|
68
68
|
) -> Optional[TaskRead]:
|
69
69
|
"""
|
70
70
|
Edit a specific task (restricted to superusers and task owner)
|
@@ -108,7 +108,7 @@ async def patch_task(
|
|
108
108
|
async def create_task(
|
109
109
|
task: TaskCreate,
|
110
110
|
user: User = Depends(current_active_verified_user),
|
111
|
-
db: AsyncSession = Depends(
|
111
|
+
db: AsyncSession = Depends(get_async_db),
|
112
112
|
) -> Optional[TaskRead]:
|
113
113
|
"""
|
114
114
|
Create a new task
|
@@ -154,7 +154,7 @@ async def create_task(
|
|
154
154
|
async def delete_task(
|
155
155
|
task_id: int,
|
156
156
|
user: User = Depends(current_active_user),
|
157
|
-
db: AsyncSession = Depends(
|
157
|
+
db: AsyncSession = Depends(get_async_db),
|
158
158
|
) -> Response:
|
159
159
|
"""
|
160
160
|
Delete a task
|
@@ -1,7 +1,5 @@
|
|
1
|
-
import json
|
2
1
|
from pathlib import Path
|
3
2
|
from shutil import copy as shell_copy
|
4
|
-
from shutil import rmtree as shell_rmtree
|
5
3
|
from tempfile import TemporaryDirectory
|
6
4
|
|
7
5
|
from fastapi import APIRouter
|
@@ -17,27 +15,21 @@ from .....config import get_settings
|
|
17
15
|
from .....logger import close_logger
|
18
16
|
from .....logger import set_logger
|
19
17
|
from .....syringe import Inject
|
20
|
-
from .....tasks.
|
21
|
-
from .....tasks.
|
22
|
-
from .....tasks.
|
23
|
-
from .....tasks.
|
24
|
-
from .....tasks.
|
25
|
-
from .....tasks.
|
26
|
-
from .....tasks.
|
27
|
-
from .....tasks.
|
28
|
-
from .....tasks.collection import inspect_package
|
29
|
-
from .....tasks.collection import slugify_task_name
|
18
|
+
from .....tasks._TaskCollectPip import _TaskCollectPip
|
19
|
+
from .....tasks.background_operations import background_collect_pip
|
20
|
+
from .....tasks.endpoint_operations import create_package_dir_pip
|
21
|
+
from .....tasks.endpoint_operations import download_package
|
22
|
+
from .....tasks.endpoint_operations import get_collection_data
|
23
|
+
from .....tasks.endpoint_operations import inspect_package
|
24
|
+
from .....tasks.utils import get_collection_log
|
25
|
+
from .....tasks.utils import slugify_task_name
|
30
26
|
from ....db import AsyncSession
|
31
|
-
from ....db import
|
32
|
-
from ....db import get_db
|
33
|
-
from ....db import get_sync_db
|
27
|
+
from ....db import get_async_db
|
34
28
|
from ....models import State
|
35
29
|
from ....models import Task
|
36
30
|
from ....schemas import StateRead
|
37
31
|
from ....schemas import TaskCollectPip
|
38
32
|
from ....schemas import TaskCollectStatus
|
39
|
-
from ....schemas import TaskCreate
|
40
|
-
from ....schemas import TaskRead
|
41
33
|
from ....security import current_active_user
|
42
34
|
from ....security import current_active_verified_user
|
43
35
|
from ....security import User
|
@@ -47,112 +39,6 @@ router = APIRouter()
|
|
47
39
|
logger = set_logger(__name__)
|
48
40
|
|
49
41
|
|
50
|
-
async def _background_collect_pip(
|
51
|
-
state_id: int,
|
52
|
-
venv_path: Path,
|
53
|
-
task_pkg: _TaskCollectPip,
|
54
|
-
) -> None:
|
55
|
-
"""
|
56
|
-
Install package and collect tasks
|
57
|
-
|
58
|
-
Install a python package and collect the tasks it provides according to
|
59
|
-
the manifest.
|
60
|
-
|
61
|
-
In case of error, copy the log into the state and delete the package
|
62
|
-
directory.
|
63
|
-
"""
|
64
|
-
logger_name = task_pkg.package.replace("/", "_")
|
65
|
-
logger = set_logger(
|
66
|
-
logger_name=logger_name,
|
67
|
-
log_file_path=get_log_path(venv_path),
|
68
|
-
)
|
69
|
-
logger.debug("Start background task collection")
|
70
|
-
for key, value in task_pkg.dict(exclude={"package_manifest"}).items():
|
71
|
-
logger.debug(f"{key}: {value}")
|
72
|
-
|
73
|
-
with next(get_sync_db()) as db:
|
74
|
-
state: State = db.get(State, state_id)
|
75
|
-
data = TaskCollectStatus(**state.data)
|
76
|
-
data.info = None
|
77
|
-
|
78
|
-
try:
|
79
|
-
# install
|
80
|
-
logger.debug("Task-collection status: installing")
|
81
|
-
data.status = "installing"
|
82
|
-
|
83
|
-
state.data = data.sanitised_dict()
|
84
|
-
db.merge(state)
|
85
|
-
db.commit()
|
86
|
-
task_list = await create_package_environment_pip(
|
87
|
-
venv_path=venv_path,
|
88
|
-
task_pkg=task_pkg,
|
89
|
-
logger_name=logger_name,
|
90
|
-
)
|
91
|
-
|
92
|
-
# collect
|
93
|
-
logger.debug("Task-collection status: collecting")
|
94
|
-
data.status = "collecting"
|
95
|
-
state.data = data.sanitised_dict()
|
96
|
-
db.merge(state)
|
97
|
-
db.commit()
|
98
|
-
tasks = await _insert_tasks(task_list=task_list, db=db)
|
99
|
-
|
100
|
-
# finalise
|
101
|
-
logger.debug("Task-collection status: finalising")
|
102
|
-
collection_path = get_collection_path(venv_path)
|
103
|
-
data.task_list = [TaskRead(**task.model_dump()) for task in tasks]
|
104
|
-
with collection_path.open("w") as f:
|
105
|
-
json.dump(data.sanitised_dict(), f)
|
106
|
-
|
107
|
-
# Update DB
|
108
|
-
data.status = "OK"
|
109
|
-
data.log = get_collection_log(venv_path)
|
110
|
-
state.data = data.sanitised_dict()
|
111
|
-
db.add(state)
|
112
|
-
db.merge(state)
|
113
|
-
db.commit()
|
114
|
-
|
115
|
-
# Write last logs to file
|
116
|
-
logger.debug("Task-collection status: OK")
|
117
|
-
logger.info("Background task collection completed successfully")
|
118
|
-
close_logger(logger)
|
119
|
-
db.close()
|
120
|
-
|
121
|
-
except Exception as e:
|
122
|
-
# Write last logs to file
|
123
|
-
logger.debug("Task-collection status: fail")
|
124
|
-
logger.info(f"Background collection failed. Original error: {e}")
|
125
|
-
close_logger(logger)
|
126
|
-
|
127
|
-
# Update db
|
128
|
-
data.status = "fail"
|
129
|
-
data.info = f"Original error: {e}"
|
130
|
-
data.log = get_collection_log(venv_path)
|
131
|
-
state.data = data.sanitised_dict()
|
132
|
-
db.merge(state)
|
133
|
-
db.commit()
|
134
|
-
db.close()
|
135
|
-
|
136
|
-
# Delete corrupted package dir
|
137
|
-
shell_rmtree(venv_path)
|
138
|
-
|
139
|
-
|
140
|
-
async def _insert_tasks(
|
141
|
-
task_list: list[TaskCreate],
|
142
|
-
db: DBSyncSession,
|
143
|
-
) -> list[Task]:
|
144
|
-
"""
|
145
|
-
Insert tasks into database
|
146
|
-
"""
|
147
|
-
task_db_list = [Task(**t.dict()) for t in task_list]
|
148
|
-
db.add_all(task_db_list)
|
149
|
-
db.commit()
|
150
|
-
for t in task_db_list:
|
151
|
-
db.refresh(t)
|
152
|
-
db.close()
|
153
|
-
return task_db_list
|
154
|
-
|
155
|
-
|
156
42
|
@router.post(
|
157
43
|
"/collect/pip/",
|
158
44
|
response_model=StateRead,
|
@@ -175,7 +61,7 @@ async def collect_tasks_pip(
|
|
175
61
|
background_tasks: BackgroundTasks,
|
176
62
|
response: Response,
|
177
63
|
user: User = Depends(current_active_verified_user),
|
178
|
-
db: AsyncSession = Depends(
|
64
|
+
db: AsyncSession = Depends(get_async_db),
|
179
65
|
) -> StateRead: # State[TaskCollectStatus]
|
180
66
|
"""
|
181
67
|
Task collection endpoint
|
@@ -289,7 +175,7 @@ async def collect_tasks_pip(
|
|
289
175
|
await db.refresh(state)
|
290
176
|
|
291
177
|
background_tasks.add_task(
|
292
|
-
|
178
|
+
background_collect_pip,
|
293
179
|
state_id=state.id,
|
294
180
|
venv_path=venv_path,
|
295
181
|
task_pkg=task_pkg,
|
@@ -314,7 +200,7 @@ async def check_collection_status(
|
|
314
200
|
state_id: int,
|
315
201
|
user: User = Depends(current_active_user),
|
316
202
|
verbose: bool = False,
|
317
|
-
db: AsyncSession = Depends(
|
203
|
+
db: AsyncSession = Depends(get_async_db),
|
318
204
|
) -> StateRead: # State[TaskCollectStatus]
|
319
205
|
"""
|
320
206
|
Check status of background task collection
|
@@ -22,7 +22,7 @@ from sqlmodel import select
|
|
22
22
|
from .....logger import close_logger
|
23
23
|
from .....logger import set_logger
|
24
24
|
from ....db import AsyncSession
|
25
|
-
from ....db import
|
25
|
+
from ....db import get_async_db
|
26
26
|
from ....models import ApplyWorkflow
|
27
27
|
from ....models import Project
|
28
28
|
from ....models import Task
|
@@ -36,9 +36,10 @@ from ....schemas import WorkflowUpdate
|
|
36
36
|
from ....security import current_active_user
|
37
37
|
from ....security import User
|
38
38
|
from ._aux_functions import _check_workflow_exists
|
39
|
-
from ._aux_functions import _get_active_jobs_statement
|
40
39
|
from ._aux_functions import _get_project_check_owner
|
40
|
+
from ._aux_functions import _get_submitted_jobs_statement
|
41
41
|
from ._aux_functions import _get_workflow_check_owner
|
42
|
+
from ._aux_functions import _workflow_insert_task
|
42
43
|
|
43
44
|
|
44
45
|
router = APIRouter()
|
@@ -51,7 +52,7 @@ router = APIRouter()
|
|
51
52
|
async def get_workflow_list(
|
52
53
|
project_id: int,
|
53
54
|
user: User = Depends(current_active_user),
|
54
|
-
db: AsyncSession = Depends(
|
55
|
+
db: AsyncSession = Depends(get_async_db),
|
55
56
|
) -> Optional[list[WorkflowRead]]:
|
56
57
|
"""
|
57
58
|
Get workflow list for given project
|
@@ -78,7 +79,7 @@ async def create_workflow(
|
|
78
79
|
project_id: int,
|
79
80
|
workflow: WorkflowCreate,
|
80
81
|
user: User = Depends(current_active_user),
|
81
|
-
db: AsyncSession = Depends(
|
82
|
+
db: AsyncSession = Depends(get_async_db),
|
82
83
|
) -> Optional[WorkflowRead]:
|
83
84
|
"""
|
84
85
|
Create a workflow, associate to a project
|
@@ -108,7 +109,7 @@ async def read_workflow(
|
|
108
109
|
project_id: int,
|
109
110
|
workflow_id: int,
|
110
111
|
user: User = Depends(current_active_user),
|
111
|
-
db: AsyncSession = Depends(
|
112
|
+
db: AsyncSession = Depends(get_async_db),
|
112
113
|
) -> Optional[WorkflowRead]:
|
113
114
|
"""
|
114
115
|
Get info on an existing workflow
|
@@ -130,7 +131,7 @@ async def update_workflow(
|
|
130
131
|
workflow_id: int,
|
131
132
|
patch: WorkflowUpdate,
|
132
133
|
user: User = Depends(current_active_user),
|
133
|
-
db: AsyncSession = Depends(
|
134
|
+
db: AsyncSession = Depends(get_async_db),
|
134
135
|
) -> Optional[WorkflowRead]:
|
135
136
|
"""
|
136
137
|
Edit a workflow
|
@@ -181,7 +182,7 @@ async def delete_workflow(
|
|
181
182
|
project_id: int,
|
182
183
|
workflow_id: int,
|
183
184
|
user: User = Depends(current_active_user),
|
184
|
-
db: AsyncSession = Depends(
|
185
|
+
db: AsyncSession = Depends(get_async_db),
|
185
186
|
) -> Response:
|
186
187
|
"""
|
187
188
|
Delete a workflow
|
@@ -191,9 +192,9 @@ async def delete_workflow(
|
|
191
192
|
project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
|
192
193
|
)
|
193
194
|
|
194
|
-
# Fail if there exist jobs that are
|
195
|
-
#
|
196
|
-
stm =
|
195
|
+
# Fail if there exist jobs that are submitted and in relation with the
|
196
|
+
# current workflow.
|
197
|
+
stm = _get_submitted_jobs_statement().where(
|
197
198
|
ApplyWorkflow.workflow_id == workflow.id
|
198
199
|
)
|
199
200
|
res = await db.execute(stm)
|
@@ -233,7 +234,7 @@ async def export_worfklow(
|
|
233
234
|
project_id: int,
|
234
235
|
workflow_id: int,
|
235
236
|
user: User = Depends(current_active_user),
|
236
|
-
db: AsyncSession = Depends(
|
237
|
+
db: AsyncSession = Depends(get_async_db),
|
237
238
|
) -> Optional[WorkflowExport]:
|
238
239
|
"""
|
239
240
|
Export an existing workflow, after stripping all IDs
|
@@ -266,7 +267,7 @@ async def import_workflow(
|
|
266
267
|
project_id: int,
|
267
268
|
workflow: WorkflowImport,
|
268
269
|
user: User = Depends(current_active_user),
|
269
|
-
db: AsyncSession = Depends(
|
270
|
+
db: AsyncSession = Depends(get_async_db),
|
270
271
|
) -> Optional[WorkflowRead]:
|
271
272
|
"""
|
272
273
|
Import an existing workflow into a project
|
@@ -323,8 +324,9 @@ async def import_workflow(
|
|
323
324
|
**wf_task.dict(exclude_none=True),
|
324
325
|
)
|
325
326
|
# Insert task
|
326
|
-
await
|
327
|
+
await _workflow_insert_task(
|
327
328
|
**new_wf_task.dict(),
|
329
|
+
workflow_id=db_workflow.id,
|
328
330
|
task_id=task_id,
|
329
331
|
db=db,
|
330
332
|
)
|
@@ -336,7 +338,7 @@ async def import_workflow(
|
|
336
338
|
@router.get("/workflow/", response_model=list[WorkflowRead])
|
337
339
|
async def get_user_workflows(
|
338
340
|
user: User = Depends(current_active_user),
|
339
|
-
db: AsyncSession = Depends(
|
341
|
+
db: AsyncSession = Depends(get_async_db),
|
340
342
|
) -> list[WorkflowRead]:
|
341
343
|
"""
|
342
344
|
Returns all the workflows of the current user
|
@@ -21,7 +21,7 @@ from fastapi import Response
|
|
21
21
|
from fastapi import status
|
22
22
|
|
23
23
|
from ....db import AsyncSession
|
24
|
-
from ....db import
|
24
|
+
from ....db import get_async_db
|
25
25
|
from ....models import Task
|
26
26
|
from ....schemas import WorkflowTaskCreate
|
27
27
|
from ....schemas import WorkflowTaskRead
|
@@ -30,6 +30,7 @@ from ....security import current_active_user
|
|
30
30
|
from ....security import User
|
31
31
|
from ._aux_functions import _get_workflow_check_owner
|
32
32
|
from ._aux_functions import _get_workflow_task_check_owner
|
33
|
+
from ._aux_functions import _workflow_insert_task
|
33
34
|
|
34
35
|
router = APIRouter()
|
35
36
|
|
@@ -45,7 +46,7 @@ async def create_workflowtask(
|
|
45
46
|
task_id: int,
|
46
47
|
new_task: WorkflowTaskCreate,
|
47
48
|
user: User = Depends(current_active_user),
|
48
|
-
db: AsyncSession = Depends(
|
49
|
+
db: AsyncSession = Depends(get_async_db),
|
49
50
|
) -> Optional[WorkflowTaskRead]:
|
50
51
|
"""
|
51
52
|
Add a WorkflowTask to a Workflow
|
@@ -64,8 +65,9 @@ async def create_workflowtask(
|
|
64
65
|
)
|
65
66
|
|
66
67
|
async with db:
|
67
|
-
workflow_task = await
|
68
|
+
workflow_task = await _workflow_insert_task(
|
68
69
|
**new_task.dict(),
|
70
|
+
workflow_id=workflow.id,
|
69
71
|
task_id=task_id,
|
70
72
|
db=db,
|
71
73
|
)
|
@@ -83,7 +85,7 @@ async def read_workflowtask(
|
|
83
85
|
workflow_id: int,
|
84
86
|
workflow_task_id: int,
|
85
87
|
user: User = Depends(current_active_user),
|
86
|
-
db: AsyncSession = Depends(
|
88
|
+
db: AsyncSession = Depends(get_async_db),
|
87
89
|
):
|
88
90
|
workflow_task, _ = await _get_workflow_task_check_owner(
|
89
91
|
project_id=project_id,
|
@@ -105,7 +107,7 @@ async def update_workflowtask(
|
|
105
107
|
workflow_task_id: int,
|
106
108
|
workflow_task_update: WorkflowTaskUpdate,
|
107
109
|
user: User = Depends(current_active_user),
|
108
|
-
db: AsyncSession = Depends(
|
110
|
+
db: AsyncSession = Depends(get_async_db),
|
109
111
|
) -> Optional[WorkflowTaskRead]:
|
110
112
|
"""
|
111
113
|
Edit a WorkflowTask of a Workflow
|
@@ -160,7 +162,7 @@ async def delete_workflowtask(
|
|
160
162
|
workflow_id: int,
|
161
163
|
workflow_task_id: int,
|
162
164
|
user: User = Depends(current_active_user),
|
163
|
-
db: AsyncSession = Depends(
|
165
|
+
db: AsyncSession = Depends(get_async_db),
|
164
166
|
) -> Response:
|
165
167
|
"""
|
166
168
|
Delete a WorkflowTask of a Workflow
|
@@ -13,7 +13,7 @@ from sqlmodel import select
|
|
13
13
|
|
14
14
|
from ...config import get_settings
|
15
15
|
from ...syringe import Inject
|
16
|
-
from ..db import
|
16
|
+
from ..db import get_async_db
|
17
17
|
from ..models.security import UserOAuth as User
|
18
18
|
from ..schemas.user import UserCreate
|
19
19
|
from ..schemas.user import UserRead
|
@@ -97,7 +97,7 @@ async def get_current_user(user: User = Depends(current_active_user)):
|
|
97
97
|
@router_auth.get("/users/", response_model=list[UserRead])
|
98
98
|
async def list_users(
|
99
99
|
user: User = Depends(current_active_superuser),
|
100
|
-
db: AsyncSession = Depends(
|
100
|
+
db: AsyncSession = Depends(get_async_db),
|
101
101
|
):
|
102
102
|
"""
|
103
103
|
Return list of all users
|
@@ -26,6 +26,7 @@ from .task import TaskUpdate # noqa: F401
|
|
26
26
|
from .task_collection import TaskCollectPip # noqa: F401
|
27
27
|
from .task_collection import TaskCollectStatus # noqa: F401
|
28
28
|
from .user import UserCreate # noqa: F401
|
29
|
+
from .user import UserRead # noqa: F401
|
29
30
|
from .user import UserUpdate # noqa: F401
|
30
31
|
from .user import UserUpdateStrict # noqa: F401
|
31
32
|
from .workflow import WorkflowCreate # noqa: F401
|
@@ -25,21 +25,17 @@ class JobStatusType(str, Enum):
|
|
25
25
|
|
26
26
|
Attributes:
|
27
27
|
SUBMITTED:
|
28
|
-
The
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
The workflow was scheduled with an executor. Note that it might not
|
33
|
-
yet be running within the executor, e.g., jobs could still be
|
34
|
-
pending within a SLURM executor.
|
28
|
+
The job was created. This does not guarantee that it was also
|
29
|
+
submitted to an executor (e.g. other errors could have prevented
|
30
|
+
this), nor that it is actually running (e.g. SLURM jobs could be
|
31
|
+
still in the queue).
|
35
32
|
DONE:
|
36
|
-
The
|
33
|
+
The job successfully reached its end.
|
37
34
|
FAILED:
|
38
35
|
The workflow terminated with an error.
|
39
36
|
"""
|
40
37
|
|
41
38
|
SUBMITTED = "submitted"
|
42
|
-
RUNNING = "running"
|
43
39
|
DONE = "done"
|
44
40
|
FAILED = "failed"
|
45
41
|
|
@@ -135,15 +131,15 @@ class ApplyWorkflowRead(_ApplyWorkflowBase):
|
|
135
131
|
|
136
132
|
id: int
|
137
133
|
project_id: Optional[int]
|
138
|
-
project_dump:
|
134
|
+
project_dump: ProjectDump
|
139
135
|
user_email: str
|
140
136
|
slurm_account: Optional[str]
|
141
137
|
workflow_id: Optional[int]
|
142
|
-
workflow_dump:
|
138
|
+
workflow_dump: WorkflowDump
|
143
139
|
input_dataset_id: Optional[int]
|
144
|
-
input_dataset_dump:
|
140
|
+
input_dataset_dump: DatasetDump
|
145
141
|
output_dataset_id: Optional[int]
|
146
|
-
output_dataset_dump:
|
142
|
+
output_dataset_dump: DatasetDump
|
147
143
|
start_timestamp: datetime
|
148
144
|
end_timestamp: Optional[datetime]
|
149
145
|
status: str
|
@@ -1,3 +1,4 @@
|
|
1
|
+
from datetime import datetime
|
1
2
|
from typing import Any
|
2
3
|
from typing import Optional
|
3
4
|
|
@@ -147,6 +148,7 @@ class DatasetRead(_DatasetBase):
|
|
147
148
|
project_id: int
|
148
149
|
read_only: bool
|
149
150
|
project: ProjectRead
|
151
|
+
timestamp_created: datetime
|
150
152
|
|
151
153
|
|
152
154
|
class DatasetStatusRead(BaseModel):
|
@@ -46,6 +46,7 @@ class WorkflowDump(BaseModel):
|
|
46
46
|
name: str
|
47
47
|
project_id: int
|
48
48
|
task_list: list[WorkflowTaskDump]
|
49
|
+
timestamp_created: str
|
49
50
|
|
50
51
|
|
51
52
|
class ResourceDump(BaseModel):
|
@@ -61,3 +62,4 @@ class DatasetDump(BaseModel):
|
|
61
62
|
read_only: bool
|
62
63
|
resource_list: list[ResourceDump]
|
63
64
|
project_id: int
|
65
|
+
timestamp_created: str
|