fractal-server 2.18.0__py3-none-any.whl → 2.18.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +1 -2
- fractal_server/app/models/security.py +5 -7
- fractal_server/app/models/v2/job.py +2 -13
- fractal_server/app/models/v2/resource.py +0 -13
- fractal_server/app/routes/admin/v2/__init__.py +12 -10
- fractal_server/app/routes/admin/v2/accounting.py +2 -2
- fractal_server/app/routes/admin/v2/job.py +17 -17
- fractal_server/app/routes/admin/v2/task.py +8 -8
- fractal_server/app/routes/admin/v2/task_group.py +16 -94
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +20 -20
- fractal_server/app/routes/api/__init__.py +9 -0
- fractal_server/app/routes/api/v2/__init__.py +49 -47
- fractal_server/app/routes/api/v2/_aux_functions.py +47 -22
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +4 -4
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +2 -2
- fractal_server/app/routes/api/v2/dataset.py +60 -66
- fractal_server/app/routes/api/v2/history.py +5 -7
- fractal_server/app/routes/api/v2/job.py +12 -12
- fractal_server/app/routes/api/v2/project.py +11 -11
- fractal_server/app/routes/api/v2/sharing.py +2 -1
- fractal_server/app/routes/api/v2/status_legacy.py +29 -15
- fractal_server/app/routes/api/v2/submit.py +66 -65
- fractal_server/app/routes/api/v2/task.py +17 -15
- fractal_server/app/routes/api/v2/task_collection.py +18 -18
- fractal_server/app/routes/api/v2/task_collection_custom.py +13 -11
- fractal_server/app/routes/api/v2/task_collection_pixi.py +9 -9
- fractal_server/app/routes/api/v2/task_group.py +18 -18
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +26 -26
- fractal_server/app/routes/api/v2/task_version_update.py +5 -5
- fractal_server/app/routes/api/v2/workflow.py +18 -18
- fractal_server/app/routes/api/v2/workflow_import.py +11 -11
- fractal_server/app/routes/api/v2/workflowtask.py +36 -10
- fractal_server/app/routes/auth/_aux_auth.py +0 -100
- fractal_server/app/routes/auth/current_user.py +63 -0
- fractal_server/app/routes/auth/group.py +30 -1
- fractal_server/app/routes/auth/router.py +0 -2
- fractal_server/app/routes/auth/users.py +0 -9
- fractal_server/app/schemas/user.py +12 -29
- fractal_server/app/schemas/user_group.py +15 -0
- fractal_server/app/schemas/v2/__init__.py +48 -48
- fractal_server/app/schemas/v2/dataset.py +13 -35
- fractal_server/app/schemas/v2/dumps.py +9 -9
- fractal_server/app/schemas/v2/job.py +11 -11
- fractal_server/app/schemas/v2/project.py +3 -3
- fractal_server/app/schemas/v2/resource.py +4 -13
- fractal_server/app/schemas/v2/status_legacy.py +3 -3
- fractal_server/app/schemas/v2/task.py +6 -6
- fractal_server/app/schemas/v2/task_collection.py +4 -4
- fractal_server/app/schemas/v2/task_group.py +16 -16
- fractal_server/app/schemas/v2/workflow.py +16 -16
- fractal_server/app/schemas/v2/workflowtask.py +14 -14
- fractal_server/app/security/__init__.py +1 -1
- fractal_server/app/shutdown.py +6 -6
- fractal_server/config/__init__.py +6 -0
- fractal_server/config/_data.py +79 -0
- fractal_server/config/_main.py +1 -6
- fractal_server/images/models.py +2 -1
- fractal_server/main.py +11 -72
- fractal_server/runner/config/_slurm.py +0 -2
- fractal_server/runner/executors/slurm_common/slurm_config.py +0 -1
- fractal_server/runner/v2/_local.py +3 -4
- fractal_server/runner/v2/_slurm_ssh.py +3 -4
- fractal_server/runner/v2/_slurm_sudo.py +3 -4
- fractal_server/runner/v2/runner.py +17 -36
- fractal_server/runner/v2/runner_functions.py +14 -11
- fractal_server/runner/v2/submit_workflow.py +9 -22
- fractal_server/tasks/v2/local/_utils.py +2 -2
- fractal_server/tasks/v2/local/collect.py +6 -5
- fractal_server/tasks/v2/local/collect_pixi.py +6 -5
- fractal_server/tasks/v2/local/deactivate.py +7 -7
- fractal_server/tasks/v2/local/deactivate_pixi.py +3 -3
- fractal_server/tasks/v2/local/delete.py +5 -5
- fractal_server/tasks/v2/local/reactivate.py +5 -5
- fractal_server/tasks/v2/local/reactivate_pixi.py +5 -5
- fractal_server/tasks/v2/ssh/collect.py +5 -5
- fractal_server/tasks/v2/ssh/collect_pixi.py +5 -5
- fractal_server/tasks/v2/ssh/deactivate.py +7 -7
- fractal_server/tasks/v2/ssh/deactivate_pixi.py +2 -2
- fractal_server/tasks/v2/ssh/delete.py +5 -5
- fractal_server/tasks/v2/ssh/reactivate.py +5 -5
- fractal_server/tasks/v2/ssh/reactivate_pixi.py +5 -5
- fractal_server/tasks/v2/utils_background.py +7 -7
- fractal_server/tasks/v2/utils_database.py +5 -5
- fractal_server/types/__init__.py +0 -22
- fractal_server/types/validators/__init__.py +0 -3
- fractal_server/types/validators/_common_validators.py +0 -32
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a0.dist-info}/METADATA +1 -1
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a0.dist-info}/RECORD +92 -97
- fractal_server/app/routes/auth/viewer_paths.py +0 -43
- fractal_server/data_migrations/2_18_0.py +0 -30
- fractal_server/migrations/versions/7910eed4cf97_user_project_dirs_and_usergroup_viewer_.py +0 -60
- fractal_server/migrations/versions/88270f589c9b_add_prevent_new_submissions.py +0 -39
- fractal_server/migrations/versions/f0702066b007_one_submitted_job_per_dataset.py +0 -40
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a0.dist-info}/WHEEL +0 -0
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a0.dist-info}/entry_points.txt +0 -0
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a0.dist-info}/licenses/LICENSE +0 -0
|
@@ -7,66 +7,68 @@ from fastapi import APIRouter
|
|
|
7
7
|
from fractal_server.config import get_settings
|
|
8
8
|
from fractal_server.syringe import Inject
|
|
9
9
|
|
|
10
|
-
from .dataset import router as
|
|
11
|
-
from .history import router as
|
|
12
|
-
from .images import router as
|
|
13
|
-
from .job import router as
|
|
10
|
+
from .dataset import router as dataset_router_v2
|
|
11
|
+
from .history import router as history_router_v2
|
|
12
|
+
from .images import router as images_routes_v2
|
|
13
|
+
from .job import router as job_router_v2
|
|
14
14
|
from .pre_submission_checks import router as pre_submission_checks_router
|
|
15
|
-
from .project import router as
|
|
16
|
-
from .sharing import router as
|
|
17
|
-
from .status_legacy import router as
|
|
18
|
-
from .submit import router as
|
|
19
|
-
from .task import router as
|
|
20
|
-
from .task_collection import router as
|
|
21
|
-
from .task_collection_custom import router as
|
|
22
|
-
from .task_collection_pixi import router as
|
|
23
|
-
from .task_group import router as
|
|
24
|
-
from .task_group_lifecycle import router as
|
|
25
|
-
from .task_version_update import router as
|
|
26
|
-
from .workflow import router as
|
|
27
|
-
from .workflow_import import router as
|
|
28
|
-
from .workflowtask import router as
|
|
15
|
+
from .project import router as project_router_v2
|
|
16
|
+
from .sharing import router as sharing_router_v2
|
|
17
|
+
from .status_legacy import router as status_legacy_router_v2
|
|
18
|
+
from .submit import router as submit_job_router_v2
|
|
19
|
+
from .task import router as task_router_v2
|
|
20
|
+
from .task_collection import router as task_collection_router_v2
|
|
21
|
+
from .task_collection_custom import router as task_collection_router_v2_custom
|
|
22
|
+
from .task_collection_pixi import router as task_collection_pixi_router_v2
|
|
23
|
+
from .task_group import router as task_group_router_v2
|
|
24
|
+
from .task_group_lifecycle import router as task_group_lifecycle_router_v2
|
|
25
|
+
from .task_version_update import router as task_version_update_router_v2
|
|
26
|
+
from .workflow import router as workflow_router_v2
|
|
27
|
+
from .workflow_import import router as workflow_import_router_v2
|
|
28
|
+
from .workflowtask import router as workflowtask_router_v2
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
router_api_v2 = APIRouter()
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
32
|
+
router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
|
|
33
|
+
router_api_v2.include_router(pre_submission_checks_router, tags=["V2 Job"])
|
|
34
|
+
router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
|
|
35
|
+
router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
|
|
36
|
+
router_api_v2.include_router(sharing_router_v2, tags=["Project Sharing"])
|
|
37
|
+
router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
|
|
38
|
+
router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
|
|
39
|
+
router_api_v2.include_router(history_router_v2, tags=["V2 History"])
|
|
40
|
+
router_api_v2.include_router(status_legacy_router_v2, tags=["V2 Status Legacy"])
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
settings = Inject(get_settings)
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
router_api_v2.include_router(
|
|
45
|
+
task_collection_router_v2,
|
|
46
46
|
prefix="/task",
|
|
47
|
-
tags=["Task Lifecycle"],
|
|
47
|
+
tags=["V2 Task Lifecycle"],
|
|
48
48
|
)
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
router_api_v2.include_router(
|
|
50
|
+
task_collection_router_v2_custom,
|
|
51
51
|
prefix="/task",
|
|
52
|
-
tags=["Task Lifecycle"],
|
|
52
|
+
tags=["V2 Task Lifecycle"],
|
|
53
53
|
)
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
router_api_v2.include_router(
|
|
55
|
+
task_collection_pixi_router_v2,
|
|
56
56
|
prefix="/task",
|
|
57
|
-
tags=["Task Lifecycle"],
|
|
57
|
+
tags=["V2 Task Lifecycle"],
|
|
58
58
|
)
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
router_api_v2.include_router(
|
|
60
|
+
task_group_lifecycle_router_v2,
|
|
61
61
|
prefix="/task-group",
|
|
62
|
-
tags=["Task Lifecycle"],
|
|
62
|
+
tags=["V2 Task Lifecycle"],
|
|
63
63
|
)
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
router_api_v2.include_router(task_router_v2, prefix="/task", tags=["V2 Task"])
|
|
66
|
+
router_api_v2.include_router(task_version_update_router_v2, tags=["V2 Task"])
|
|
67
|
+
router_api_v2.include_router(
|
|
68
|
+
task_group_router_v2, prefix="/task-group", tags=["V2 TaskGroup"]
|
|
69
69
|
)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
router_api_v2.include_router(workflow_router_v2, tags=["V2 Workflow"])
|
|
71
|
+
router_api_v2.include_router(
|
|
72
|
+
workflow_import_router_v2, tags=["V2 Workflow Import"]
|
|
73
|
+
)
|
|
74
|
+
router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
|
|
@@ -3,10 +3,11 @@ Auxiliary functions to get object from the database or perform simple checks
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
from typing import Any
|
|
6
|
-
from typing import
|
|
6
|
+
from typing import Literal
|
|
7
7
|
|
|
8
8
|
from fastapi import HTTPException
|
|
9
9
|
from fastapi import status
|
|
10
|
+
from sqlalchemy.exc import MultipleResultsFound
|
|
10
11
|
from sqlalchemy.orm.attributes import flag_modified
|
|
11
12
|
from sqlmodel import select
|
|
12
13
|
from sqlmodel.sql.expression import SelectOfScalar
|
|
@@ -22,7 +23,7 @@ from fractal_server.app.models.v2 import ProjectV2
|
|
|
22
23
|
from fractal_server.app.models.v2 import TaskV2
|
|
23
24
|
from fractal_server.app.models.v2 import WorkflowTaskV2
|
|
24
25
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
25
|
-
from fractal_server.app.schemas.v2 import
|
|
26
|
+
from fractal_server.app.schemas.v2 import JobStatusTypeV2
|
|
26
27
|
from fractal_server.app.schemas.v2 import ProjectPermissions
|
|
27
28
|
from fractal_server.logger import set_logger
|
|
28
29
|
|
|
@@ -251,11 +252,6 @@ async def _check_project_exists(
|
|
|
251
252
|
)
|
|
252
253
|
|
|
253
254
|
|
|
254
|
-
class DatasetOrProject(TypedDict):
|
|
255
|
-
dataset: DatasetV2
|
|
256
|
-
project: ProjectV2
|
|
257
|
-
|
|
258
|
-
|
|
259
255
|
async def _get_dataset_check_access(
|
|
260
256
|
*,
|
|
261
257
|
project_id: int,
|
|
@@ -263,7 +259,7 @@ async def _get_dataset_check_access(
|
|
|
263
259
|
user_id: int,
|
|
264
260
|
required_permissions: ProjectPermissions,
|
|
265
261
|
db: AsyncSession,
|
|
266
|
-
) ->
|
|
262
|
+
) -> dict[Literal["dataset", "project"], DatasetV2 | ProjectV2]:
|
|
267
263
|
"""
|
|
268
264
|
Get a dataset and a project, after access control on the project
|
|
269
265
|
|
|
@@ -308,11 +304,6 @@ async def _get_dataset_check_access(
|
|
|
308
304
|
return dict(dataset=dataset, project=project)
|
|
309
305
|
|
|
310
306
|
|
|
311
|
-
class JobAndProject(TypedDict):
|
|
312
|
-
job: JobV2
|
|
313
|
-
project: ProjectV2
|
|
314
|
-
|
|
315
|
-
|
|
316
307
|
async def _get_job_check_access(
|
|
317
308
|
*,
|
|
318
309
|
project_id: int,
|
|
@@ -320,7 +311,7 @@ async def _get_job_check_access(
|
|
|
320
311
|
user_id: int,
|
|
321
312
|
required_permissions: ProjectPermissions,
|
|
322
313
|
db: AsyncSession,
|
|
323
|
-
) ->
|
|
314
|
+
) -> dict[Literal["job", "project"], JobV2 | ProjectV2]:
|
|
324
315
|
"""
|
|
325
316
|
Get a job and a project, after access control on the project
|
|
326
317
|
|
|
@@ -370,7 +361,7 @@ def _get_submitted_jobs_statement() -> SelectOfScalar:
|
|
|
370
361
|
A sqlmodel statement that selects all `Job`s with
|
|
371
362
|
`Job.status` equal to `submitted`.
|
|
372
363
|
"""
|
|
373
|
-
stm = select(JobV2).where(JobV2.status ==
|
|
364
|
+
stm = select(JobV2).where(JobV2.status == JobStatusTypeV2.SUBMITTED)
|
|
374
365
|
return stm
|
|
375
366
|
|
|
376
367
|
|
|
@@ -380,7 +371,7 @@ async def _workflow_has_submitted_job(
|
|
|
380
371
|
) -> bool:
|
|
381
372
|
res = await db.execute(
|
|
382
373
|
select(JobV2.id)
|
|
383
|
-
.where(JobV2.status ==
|
|
374
|
+
.where(JobV2.status == JobStatusTypeV2.SUBMITTED)
|
|
384
375
|
.where(JobV2.workflow_id == workflow_id)
|
|
385
376
|
.limit(1)
|
|
386
377
|
)
|
|
@@ -462,9 +453,8 @@ async def _workflow_insert_task(
|
|
|
462
453
|
return wf_task
|
|
463
454
|
|
|
464
455
|
|
|
465
|
-
async def
|
|
466
|
-
db: AsyncSession,
|
|
467
|
-
jobs_list: list[int],
|
|
456
|
+
async def clean_app_job_list_v2(
|
|
457
|
+
db: AsyncSession, jobs_list: list[int]
|
|
468
458
|
) -> list[int]:
|
|
469
459
|
"""
|
|
470
460
|
Remove from a job list all jobs with status different from submitted.
|
|
@@ -476,14 +466,14 @@ async def clean_app_job_list(
|
|
|
476
466
|
Return:
|
|
477
467
|
List of IDs for submitted jobs.
|
|
478
468
|
"""
|
|
479
|
-
logger.info(f"[clean_app_job_list] START - {jobs_list=}.")
|
|
480
469
|
stmt = select(JobV2).where(JobV2.id.in_(jobs_list))
|
|
481
470
|
result = await db.execute(stmt)
|
|
482
471
|
db_jobs_list = result.scalars().all()
|
|
483
472
|
submitted_job_ids = [
|
|
484
|
-
job.id
|
|
473
|
+
job.id
|
|
474
|
+
for job in db_jobs_list
|
|
475
|
+
if job.status == JobStatusTypeV2.SUBMITTED
|
|
485
476
|
]
|
|
486
|
-
logger.info(f"[clean_app_job_list] END - {submitted_job_ids=}.")
|
|
487
477
|
return submitted_job_ids
|
|
488
478
|
|
|
489
479
|
|
|
@@ -553,6 +543,41 @@ async def _get_workflowtask_or_404(
|
|
|
553
543
|
return wftask
|
|
554
544
|
|
|
555
545
|
|
|
546
|
+
async def _get_submitted_job_or_none(
|
|
547
|
+
*,
|
|
548
|
+
dataset_id: int,
|
|
549
|
+
workflow_id: int,
|
|
550
|
+
db: AsyncSession,
|
|
551
|
+
) -> JobV2 | None:
|
|
552
|
+
"""
|
|
553
|
+
Get the submitted job for given dataset/workflow, if any.
|
|
554
|
+
|
|
555
|
+
This function also handles the invalid branch where more than one job
|
|
556
|
+
is found.
|
|
557
|
+
|
|
558
|
+
Args:
|
|
559
|
+
dataset_id:
|
|
560
|
+
workflow_id:
|
|
561
|
+
db:
|
|
562
|
+
"""
|
|
563
|
+
res = await db.execute(
|
|
564
|
+
_get_submitted_jobs_statement()
|
|
565
|
+
.where(JobV2.dataset_id == dataset_id)
|
|
566
|
+
.where(JobV2.workflow_id == workflow_id)
|
|
567
|
+
)
|
|
568
|
+
try:
|
|
569
|
+
return res.scalars().one_or_none()
|
|
570
|
+
except MultipleResultsFound as e:
|
|
571
|
+
error_msg = (
|
|
572
|
+
f"Multiple running jobs found for {dataset_id=} and {workflow_id=}."
|
|
573
|
+
)
|
|
574
|
+
logger.error(f"{error_msg} Original error: {str(e)}.")
|
|
575
|
+
raise HTTPException(
|
|
576
|
+
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
577
|
+
detail=error_msg,
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
|
|
556
581
|
async def _get_user_resource_id(user_id: int, db: AsyncSession) -> int | None:
|
|
557
582
|
res = await db.execute(
|
|
558
583
|
select(Resource.id)
|
|
@@ -14,8 +14,8 @@ from fractal_server.app.models.v2 import TaskGroupV2
|
|
|
14
14
|
from fractal_server.app.models.v2 import TaskV2
|
|
15
15
|
from fractal_server.app.models.v2 import WorkflowTaskV2
|
|
16
16
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
17
|
-
from fractal_server.app.schemas.v2 import
|
|
18
|
-
from fractal_server.app.schemas.v2 import
|
|
17
|
+
from fractal_server.app.schemas.v2 import JobStatusTypeV2
|
|
18
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
19
19
|
from fractal_server.logger import set_logger
|
|
20
20
|
from fractal_server.tasks.v2.utils_package_names import normalize_package_name
|
|
21
21
|
|
|
@@ -171,7 +171,7 @@ async def check_no_ongoing_activity(
|
|
|
171
171
|
stm = (
|
|
172
172
|
select(TaskGroupActivityV2)
|
|
173
173
|
.where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
|
|
174
|
-
.where(TaskGroupActivityV2.status ==
|
|
174
|
+
.where(TaskGroupActivityV2.status == TaskGroupActivityStatusV2.ONGOING)
|
|
175
175
|
)
|
|
176
176
|
res = await db.execute(stm)
|
|
177
177
|
ongoing_activities = res.scalars().all()
|
|
@@ -213,7 +213,7 @@ async def check_no_submitted_job(
|
|
|
213
213
|
.join(TaskV2, WorkflowTaskV2.task_id == TaskV2.id)
|
|
214
214
|
.where(WorkflowTaskV2.order >= JobV2.first_task_index)
|
|
215
215
|
.where(WorkflowTaskV2.order <= JobV2.last_task_index)
|
|
216
|
-
.where(JobV2.status ==
|
|
216
|
+
.where(JobV2.status == JobStatusTypeV2.SUBMITTED)
|
|
217
217
|
.where(TaskV2.taskgroupv2_id == task_group_id)
|
|
218
218
|
)
|
|
219
219
|
res = await db.execute(stm)
|
|
@@ -27,7 +27,7 @@ from fractal_server.app.routes.auth._aux_auth import (
|
|
|
27
27
|
from fractal_server.app.routes.auth._aux_auth import (
|
|
28
28
|
_verify_user_belongs_to_group,
|
|
29
29
|
)
|
|
30
|
-
from fractal_server.app.schemas.v2 import
|
|
30
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
31
31
|
from fractal_server.images.tools import merge_type_filters
|
|
32
32
|
from fractal_server.logger import set_logger
|
|
33
33
|
|
|
@@ -252,7 +252,7 @@ async def _get_collection_task_group_activity_status_message(
|
|
|
252
252
|
res = await db.execute(
|
|
253
253
|
select(TaskGroupActivityV2)
|
|
254
254
|
.where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
|
|
255
|
-
.where(TaskGroupActivityV2.action ==
|
|
255
|
+
.where(TaskGroupActivityV2.action == TaskGroupActivityActionV2.COLLECT)
|
|
256
256
|
)
|
|
257
257
|
task_group_activity_list = res.scalars().all()
|
|
258
258
|
if len(task_group_activity_list) > 1:
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
1
|
from fastapi import APIRouter
|
|
5
2
|
from fastapi import Depends
|
|
6
3
|
from fastapi import HTTPException
|
|
@@ -14,11 +11,11 @@ from fractal_server.app.models import UserOAuth
|
|
|
14
11
|
from fractal_server.app.models.v2 import DatasetV2
|
|
15
12
|
from fractal_server.app.models.v2 import JobV2
|
|
16
13
|
from fractal_server.app.routes.auth import current_user_act_ver_prof
|
|
17
|
-
from fractal_server.app.schemas.v2 import
|
|
18
|
-
from fractal_server.app.schemas.v2 import
|
|
19
|
-
from fractal_server.app.schemas.v2 import
|
|
20
|
-
from fractal_server.app.schemas.v2.dataset import
|
|
21
|
-
from fractal_server.app.schemas.v2.dataset import
|
|
14
|
+
from fractal_server.app.schemas.v2 import DatasetCreateV2
|
|
15
|
+
from fractal_server.app.schemas.v2 import DatasetReadV2
|
|
16
|
+
from fractal_server.app.schemas.v2 import DatasetUpdateV2
|
|
17
|
+
from fractal_server.app.schemas.v2.dataset import DatasetExportV2
|
|
18
|
+
from fractal_server.app.schemas.v2.dataset import DatasetImportV2
|
|
22
19
|
from fractal_server.app.schemas.v2.sharing import ProjectPermissions
|
|
23
20
|
from fractal_server.string_tools import sanitize_string
|
|
24
21
|
from fractal_server.urls import normalize_url
|
|
@@ -32,15 +29,15 @@ router = APIRouter()
|
|
|
32
29
|
|
|
33
30
|
@router.post(
|
|
34
31
|
"/project/{project_id}/dataset/",
|
|
35
|
-
response_model=
|
|
32
|
+
response_model=DatasetReadV2,
|
|
36
33
|
status_code=status.HTTP_201_CREATED,
|
|
37
34
|
)
|
|
38
35
|
async def create_dataset(
|
|
39
36
|
project_id: int,
|
|
40
|
-
dataset:
|
|
37
|
+
dataset: DatasetCreateV2,
|
|
41
38
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
42
39
|
db: AsyncSession = Depends(get_async_db),
|
|
43
|
-
) ->
|
|
40
|
+
) -> DatasetReadV2 | None:
|
|
44
41
|
"""
|
|
45
42
|
Add new dataset to current project
|
|
46
43
|
"""
|
|
@@ -51,54 +48,44 @@ async def create_dataset(
|
|
|
51
48
|
db=db,
|
|
52
49
|
)
|
|
53
50
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
if dataset.project_dir not in user.project_dirs:
|
|
67
|
-
await db.delete(db_dataset)
|
|
68
|
-
await db.commit()
|
|
69
|
-
raise HTTPException(
|
|
70
|
-
status_code=status.HTTP_403_FORBIDDEN,
|
|
71
|
-
detail=f"You are not allowed to use {dataset.project_dir=}.",
|
|
72
|
-
)
|
|
73
|
-
project_dir = dataset.project_dir
|
|
74
|
-
|
|
75
|
-
if dataset.zarr_subfolder is None:
|
|
76
|
-
zarr_subfolder = (
|
|
77
|
-
f"fractal/{project_id}_{sanitize_string(project.name)}/"
|
|
51
|
+
if dataset.zarr_dir is None:
|
|
52
|
+
db_dataset = DatasetV2(
|
|
53
|
+
project_id=project_id,
|
|
54
|
+
zarr_dir="__PLACEHOLDER__",
|
|
55
|
+
**dataset.model_dump(exclude={"zarr_dir"}),
|
|
56
|
+
)
|
|
57
|
+
db.add(db_dataset)
|
|
58
|
+
await db.commit()
|
|
59
|
+
await db.refresh(db_dataset)
|
|
60
|
+
path = (
|
|
61
|
+
f"{user.project_dir}/fractal/"
|
|
62
|
+
f"{project_id}_{sanitize_string(project.name)}/"
|
|
78
63
|
f"{db_dataset.id}_{sanitize_string(db_dataset.name)}"
|
|
79
64
|
)
|
|
80
|
-
|
|
81
|
-
|
|
65
|
+
normalized_path = normalize_url(path)
|
|
66
|
+
db_dataset.zarr_dir = normalized_path
|
|
82
67
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
68
|
+
db.add(db_dataset)
|
|
69
|
+
await db.commit()
|
|
70
|
+
await db.refresh(db_dataset)
|
|
71
|
+
else:
|
|
72
|
+
db_dataset = DatasetV2(project_id=project_id, **dataset.model_dump())
|
|
73
|
+
db.add(db_dataset)
|
|
74
|
+
await db.commit()
|
|
75
|
+
await db.refresh(db_dataset)
|
|
89
76
|
|
|
90
77
|
return db_dataset
|
|
91
78
|
|
|
92
79
|
|
|
93
80
|
@router.get(
|
|
94
81
|
"/project/{project_id}/dataset/",
|
|
95
|
-
response_model=list[
|
|
82
|
+
response_model=list[DatasetReadV2],
|
|
96
83
|
)
|
|
97
84
|
async def read_dataset_list(
|
|
98
85
|
project_id: int,
|
|
99
86
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
100
87
|
db: AsyncSession = Depends(get_async_db),
|
|
101
|
-
) -> list[
|
|
88
|
+
) -> list[DatasetReadV2] | None:
|
|
102
89
|
"""
|
|
103
90
|
Get dataset list for given project
|
|
104
91
|
"""
|
|
@@ -121,14 +108,14 @@ async def read_dataset_list(
|
|
|
121
108
|
|
|
122
109
|
@router.get(
|
|
123
110
|
"/project/{project_id}/dataset/{dataset_id}/",
|
|
124
|
-
response_model=
|
|
111
|
+
response_model=DatasetReadV2,
|
|
125
112
|
)
|
|
126
113
|
async def read_dataset(
|
|
127
114
|
project_id: int,
|
|
128
115
|
dataset_id: int,
|
|
129
116
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
130
117
|
db: AsyncSession = Depends(get_async_db),
|
|
131
|
-
) ->
|
|
118
|
+
) -> DatasetReadV2 | None:
|
|
132
119
|
"""
|
|
133
120
|
Get info on a dataset associated to the current project
|
|
134
121
|
"""
|
|
@@ -145,15 +132,15 @@ async def read_dataset(
|
|
|
145
132
|
|
|
146
133
|
@router.patch(
|
|
147
134
|
"/project/{project_id}/dataset/{dataset_id}/",
|
|
148
|
-
response_model=
|
|
135
|
+
response_model=DatasetReadV2,
|
|
149
136
|
)
|
|
150
137
|
async def update_dataset(
|
|
151
138
|
project_id: int,
|
|
152
139
|
dataset_id: int,
|
|
153
|
-
dataset_update:
|
|
140
|
+
dataset_update: DatasetUpdateV2,
|
|
154
141
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
155
142
|
db: AsyncSession = Depends(get_async_db),
|
|
156
|
-
) ->
|
|
143
|
+
) -> DatasetReadV2 | None:
|
|
157
144
|
"""
|
|
158
145
|
Edit a dataset associated to the current project
|
|
159
146
|
"""
|
|
@@ -167,6 +154,15 @@ async def update_dataset(
|
|
|
167
154
|
)
|
|
168
155
|
db_dataset = output["dataset"]
|
|
169
156
|
|
|
157
|
+
if (dataset_update.zarr_dir is not None) and (len(db_dataset.images) != 0):
|
|
158
|
+
raise HTTPException(
|
|
159
|
+
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
160
|
+
detail=(
|
|
161
|
+
"Cannot modify `zarr_dir` because the dataset has a non-empty "
|
|
162
|
+
"image list."
|
|
163
|
+
),
|
|
164
|
+
)
|
|
165
|
+
|
|
170
166
|
for key, value in dataset_update.model_dump(exclude_unset=True).items():
|
|
171
167
|
setattr(db_dataset, key, value)
|
|
172
168
|
|
|
@@ -221,14 +217,14 @@ async def delete_dataset(
|
|
|
221
217
|
|
|
222
218
|
@router.get(
|
|
223
219
|
"/project/{project_id}/dataset/{dataset_id}/export/",
|
|
224
|
-
response_model=
|
|
220
|
+
response_model=DatasetExportV2,
|
|
225
221
|
)
|
|
226
222
|
async def export_dataset(
|
|
227
223
|
project_id: int,
|
|
228
224
|
dataset_id: int,
|
|
229
225
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
230
226
|
db: AsyncSession = Depends(get_async_db),
|
|
231
|
-
) ->
|
|
227
|
+
) -> DatasetExportV2 | None:
|
|
232
228
|
"""
|
|
233
229
|
Export an existing dataset
|
|
234
230
|
"""
|
|
@@ -246,15 +242,15 @@ async def export_dataset(
|
|
|
246
242
|
|
|
247
243
|
@router.post(
|
|
248
244
|
"/project/{project_id}/dataset/import/",
|
|
249
|
-
response_model=
|
|
245
|
+
response_model=DatasetReadV2,
|
|
250
246
|
status_code=status.HTTP_201_CREATED,
|
|
251
247
|
)
|
|
252
248
|
async def import_dataset(
|
|
253
249
|
project_id: int,
|
|
254
|
-
dataset:
|
|
250
|
+
dataset: DatasetImportV2,
|
|
255
251
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
256
252
|
db: AsyncSession = Depends(get_async_db),
|
|
257
|
-
) ->
|
|
253
|
+
) -> DatasetReadV2 | None:
|
|
258
254
|
"""
|
|
259
255
|
Import an existing dataset into a project
|
|
260
256
|
"""
|
|
@@ -267,17 +263,15 @@ async def import_dataset(
|
|
|
267
263
|
db=db,
|
|
268
264
|
)
|
|
269
265
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
),
|
|
280
|
-
)
|
|
266
|
+
for image in dataset.images:
|
|
267
|
+
if not image.zarr_url.startswith(dataset.zarr_dir):
|
|
268
|
+
raise HTTPException(
|
|
269
|
+
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
270
|
+
detail=(
|
|
271
|
+
f"Cannot import dataset: zarr_url {image.zarr_url} is not "
|
|
272
|
+
f"relative to zarr_dir={dataset.zarr_dir}."
|
|
273
|
+
),
|
|
274
|
+
)
|
|
281
275
|
|
|
282
276
|
# Create new Dataset
|
|
283
277
|
db_dataset = DatasetV2(
|
|
@@ -33,7 +33,7 @@ from fractal_server.images.tools import filter_image_list
|
|
|
33
33
|
from fractal_server.logger import set_logger
|
|
34
34
|
|
|
35
35
|
from ._aux_functions import _get_dataset_check_access
|
|
36
|
-
from ._aux_functions import
|
|
36
|
+
from ._aux_functions import _get_submitted_job_or_none
|
|
37
37
|
from ._aux_functions import _get_workflow_check_access
|
|
38
38
|
from ._aux_functions_history import _verify_workflow_and_dataset_access
|
|
39
39
|
from ._aux_functions_history import get_history_run_or_404
|
|
@@ -90,13 +90,11 @@ async def get_workflow_tasks_statuses(
|
|
|
90
90
|
db=db,
|
|
91
91
|
)
|
|
92
92
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
93
|
+
running_job = await _get_submitted_job_or_none(
|
|
94
|
+
db=db,
|
|
95
|
+
dataset_id=dataset_id,
|
|
96
|
+
workflow_id=workflow_id,
|
|
97
97
|
)
|
|
98
|
-
running_job = res.scalars().one_or_none()
|
|
99
|
-
|
|
100
98
|
if running_job is not None:
|
|
101
99
|
running_wftasks = workflow.task_list[
|
|
102
100
|
running_job.first_task_index : running_job.last_task_index + 1
|
|
@@ -18,8 +18,8 @@ from fractal_server.app.models.v2 import LinkUserProjectV2
|
|
|
18
18
|
from fractal_server.app.routes.auth import current_user_act_ver_prof
|
|
19
19
|
from fractal_server.app.routes.aux._job import _write_shutdown_file
|
|
20
20
|
from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
|
|
21
|
-
from fractal_server.app.schemas.v2 import
|
|
22
|
-
from fractal_server.app.schemas.v2 import
|
|
21
|
+
from fractal_server.app.schemas.v2 import JobReadV2
|
|
22
|
+
from fractal_server.app.schemas.v2 import JobStatusTypeV2
|
|
23
23
|
from fractal_server.app.schemas.v2.sharing import ProjectPermissions
|
|
24
24
|
from fractal_server.runner.filenames import WORKFLOW_LOG_FILENAME
|
|
25
25
|
from fractal_server.zip_tools import _zip_folder_to_byte_stream_iterator
|
|
@@ -39,12 +39,12 @@ async def zip_folder_threaded(folder: str) -> Iterator[bytes]:
|
|
|
39
39
|
router = APIRouter()
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
@router.get("/job/", response_model=list[
|
|
42
|
+
@router.get("/job/", response_model=list[JobReadV2])
|
|
43
43
|
async def get_user_jobs(
|
|
44
44
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
45
45
|
log: bool = True,
|
|
46
46
|
db: AsyncSession = Depends(get_async_db),
|
|
47
|
-
) -> list[
|
|
47
|
+
) -> list[JobReadV2]:
|
|
48
48
|
"""
|
|
49
49
|
Returns all the jobs of the current user
|
|
50
50
|
"""
|
|
@@ -68,14 +68,14 @@ async def get_user_jobs(
|
|
|
68
68
|
|
|
69
69
|
@router.get(
|
|
70
70
|
"/project/{project_id}/workflow/{workflow_id}/job/",
|
|
71
|
-
response_model=list[
|
|
71
|
+
response_model=list[JobReadV2],
|
|
72
72
|
)
|
|
73
73
|
async def get_workflow_jobs(
|
|
74
74
|
project_id: int,
|
|
75
75
|
workflow_id: int,
|
|
76
76
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
77
77
|
db: AsyncSession = Depends(get_async_db),
|
|
78
|
-
) -> list[
|
|
78
|
+
) -> list[JobReadV2] | None:
|
|
79
79
|
"""
|
|
80
80
|
Returns all the jobs related to a specific workflow
|
|
81
81
|
"""
|
|
@@ -99,7 +99,7 @@ async def get_latest_job(
|
|
|
99
99
|
dataset_id: int,
|
|
100
100
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
101
101
|
db: AsyncSession = Depends(get_async_db),
|
|
102
|
-
) ->
|
|
102
|
+
) -> JobReadV2:
|
|
103
103
|
await _get_workflow_check_access(
|
|
104
104
|
project_id=project_id,
|
|
105
105
|
workflow_id=workflow_id,
|
|
@@ -127,7 +127,7 @@ async def get_latest_job(
|
|
|
127
127
|
|
|
128
128
|
@router.get(
|
|
129
129
|
"/project/{project_id}/job/{job_id}/",
|
|
130
|
-
response_model=
|
|
130
|
+
response_model=JobReadV2,
|
|
131
131
|
)
|
|
132
132
|
async def read_job(
|
|
133
133
|
project_id: int,
|
|
@@ -135,7 +135,7 @@ async def read_job(
|
|
|
135
135
|
show_tmp_logs: bool = False,
|
|
136
136
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
137
137
|
db: AsyncSession = Depends(get_async_db),
|
|
138
|
-
) ->
|
|
138
|
+
) -> JobReadV2 | None:
|
|
139
139
|
"""
|
|
140
140
|
Return info on an existing job
|
|
141
141
|
"""
|
|
@@ -150,7 +150,7 @@ async def read_job(
|
|
|
150
150
|
job = output["job"]
|
|
151
151
|
await db.close()
|
|
152
152
|
|
|
153
|
-
if show_tmp_logs and (job.status ==
|
|
153
|
+
if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
|
|
154
154
|
try:
|
|
155
155
|
with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}") as f:
|
|
156
156
|
job.log = f.read()
|
|
@@ -194,14 +194,14 @@ async def download_job_logs(
|
|
|
194
194
|
|
|
195
195
|
@router.get(
|
|
196
196
|
"/project/{project_id}/job/",
|
|
197
|
-
response_model=list[
|
|
197
|
+
response_model=list[JobReadV2],
|
|
198
198
|
)
|
|
199
199
|
async def get_job_list(
|
|
200
200
|
project_id: int,
|
|
201
201
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
202
202
|
log: bool = True,
|
|
203
203
|
db: AsyncSession = Depends(get_async_db),
|
|
204
|
-
) -> list[
|
|
204
|
+
) -> list[JobReadV2] | None:
|
|
205
205
|
"""
|
|
206
206
|
Get job list for given project
|
|
207
207
|
"""
|