fractal-server 2.18.0__py3-none-any.whl → 2.18.0a1__py3-none-any.whl

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -3,10 +3,11 @@ Auxiliary functions to get object from the database or perform simple checks
 """
 
 from typing import Any
-from typing import TypedDict
+from typing import Literal
 
 from fastapi import HTTPException
 from fastapi import status
+from sqlalchemy.exc import MultipleResultsFound
 from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import select
 from sqlmodel.sql.expression import SelectOfScalar
@@ -22,7 +23,7 @@ from fractal_server.app.models.v2 import ProjectV2
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.models.v2 import WorkflowV2
-from fractal_server.app.schemas.v2 import JobStatusType
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
 from fractal_server.app.schemas.v2 import ProjectPermissions
 from fractal_server.logger import set_logger
 
@@ -251,11 +252,6 @@ async def _check_project_exists(
         )
 
 
-class DatasetOrProject(TypedDict):
-    dataset: DatasetV2
-    project: ProjectV2
-
-
 async def _get_dataset_check_access(
     *,
     project_id: int,
@@ -263,7 +259,7 @@ async def _get_dataset_check_access(
     user_id: int,
     required_permissions: ProjectPermissions,
     db: AsyncSession,
-) -> DatasetOrProject:
+) -> dict[Literal["dataset", "project"], DatasetV2 | ProjectV2]:
     """
     Get a dataset and a project, after access control on the project
 
@@ -308,11 +304,6 @@ async def _get_dataset_check_access(
     return dict(dataset=dataset, project=project)
 
 
-class JobAndProject(TypedDict):
-    job: JobV2
-    project: ProjectV2
-
-
 async def _get_job_check_access(
     *,
     project_id: int,
@@ -320,7 +311,7 @@ async def _get_job_check_access(
     user_id: int,
     required_permissions: ProjectPermissions,
     db: AsyncSession,
-) -> JobAndProject:
+) -> dict[Literal["job", "project"], JobV2 | ProjectV2]:
     """
     Get a job and a project, after access control on the project
 
@@ -370,7 +361,7 @@ def _get_submitted_jobs_statement() -> SelectOfScalar:
         A sqlmodel statement that selects all `Job`s with
         `Job.status` equal to `submitted`.
     """
-    stm = select(JobV2).where(JobV2.status == JobStatusType.SUBMITTED)
+    stm = select(JobV2).where(JobV2.status == JobStatusTypeV2.SUBMITTED)
     return stm
 
 
@@ -380,7 +371,7 @@ async def _workflow_has_submitted_job(
 ) -> bool:
     res = await db.execute(
         select(JobV2.id)
-        .where(JobV2.status == JobStatusType.SUBMITTED)
+        .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
         .where(JobV2.workflow_id == workflow_id)
         .limit(1)
     )
@@ -462,9 +453,8 @@ async def _workflow_insert_task(
     return wf_task
 
 
-async def clean_app_job_list(
-    db: AsyncSession,
-    jobs_list: list[int],
+async def clean_app_job_list_v2(
+    db: AsyncSession, jobs_list: list[int]
 ) -> list[int]:
     """
     Remove from a job list all jobs with status different from submitted.
@@ -476,14 +466,14 @@ async def clean_app_job_list(
     Return:
         List of IDs for submitted jobs.
     """
-    logger.info(f"[clean_app_job_list] START - {jobs_list=}.")
     stmt = select(JobV2).where(JobV2.id.in_(jobs_list))
     result = await db.execute(stmt)
     db_jobs_list = result.scalars().all()
     submitted_job_ids = [
-        job.id for job in db_jobs_list if job.status == JobStatusType.SUBMITTED
+        job.id
+        for job in db_jobs_list
+        if job.status == JobStatusTypeV2.SUBMITTED
     ]
-    logger.info(f"[clean_app_job_list] END - {submitted_job_ids=}.")
     return submitted_job_ids
 
 
@@ -553,6 +543,41 @@ async def _get_workflowtask_or_404(
         return wftask
 
 
+async def _get_submitted_job_or_none(
+    *,
+    dataset_id: int,
+    workflow_id: int,
+    db: AsyncSession,
+) -> JobV2 | None:
+    """
+    Get the submitted job for given dataset/workflow, if any.
+
+    This function also handles the invalid branch where more than one job
+    is found.
+
+    Args:
+        dataset_id:
+        workflow_id:
+        db:
+    """
+    res = await db.execute(
+        _get_submitted_jobs_statement()
+        .where(JobV2.dataset_id == dataset_id)
+        .where(JobV2.workflow_id == workflow_id)
+    )
+    try:
+        return res.scalars().one_or_none()
+    except MultipleResultsFound as e:
+        error_msg = (
+            f"Multiple running jobs found for {dataset_id=} and {workflow_id=}."
+        )
+        logger.error(f"{error_msg} Original error: {str(e)}.")
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=error_msg,
+        )
+
+
 async def _get_user_resource_id(user_id: int, db: AsyncSession) -> int | None:
     res = await db.execute(
         select(Resource.id)

fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py

@@ -14,8 +14,8 @@ from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.models.v2 import WorkflowV2
-from fractal_server.app.schemas.v2 import JobStatusType
-from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
 from fractal_server.logger import set_logger
 from fractal_server.tasks.v2.utils_package_names import normalize_package_name
 
@@ -171,7 +171,7 @@ async def check_no_ongoing_activity(
     stm = (
         select(TaskGroupActivityV2)
         .where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
-        .where(TaskGroupActivityV2.status == TaskGroupActivityStatus.ONGOING)
+        .where(TaskGroupActivityV2.status == TaskGroupActivityStatusV2.ONGOING)
     )
     res = await db.execute(stm)
     ongoing_activities = res.scalars().all()
@@ -213,7 +213,7 @@ async def check_no_submitted_job(
         .join(TaskV2, WorkflowTaskV2.task_id == TaskV2.id)
         .where(WorkflowTaskV2.order >= JobV2.first_task_index)
         .where(WorkflowTaskV2.order <= JobV2.last_task_index)
-        .where(JobV2.status == JobStatusType.SUBMITTED)
+        .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
         .where(TaskV2.taskgroupv2_id == task_group_id)
     )
     res = await db.execute(stm)

fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -27,7 +27,7 @@ from fractal_server.app.routes.auth._aux_auth import (
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
-from fractal_server.app.schemas.v2 import TaskGroupActivityAction
+from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
 
@@ -252,7 +252,7 @@ async def _get_collection_task_group_activity_status_message(
     res = await db.execute(
         select(TaskGroupActivityV2)
         .where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
-        .where(TaskGroupActivityV2.action == TaskGroupActivityAction.COLLECT)
+        .where(TaskGroupActivityV2.action == TaskGroupActivityActionV2.COLLECT)
     )
     task_group_activity_list = res.scalars().all()
     if len(task_group_activity_list) > 1:

fractal_server/app/routes/api/v2/dataset.py

@@ -1,6 +1,3 @@
-import os
-from pathlib import Path
-
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -14,11 +11,11 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.routes.auth import current_user_act_ver_prof
-from fractal_server.app.schemas.v2 import DatasetCreate
-from fractal_server.app.schemas.v2 import DatasetRead
-from fractal_server.app.schemas.v2 import DatasetUpdate
-from fractal_server.app.schemas.v2.dataset import DatasetExport
-from fractal_server.app.schemas.v2.dataset import DatasetImport
+from fractal_server.app.schemas.v2 import DatasetCreateV2
+from fractal_server.app.schemas.v2 import DatasetReadV2
+from fractal_server.app.schemas.v2 import DatasetUpdateV2
+from fractal_server.app.schemas.v2.dataset import DatasetExportV2
+from fractal_server.app.schemas.v2.dataset import DatasetImportV2
 from fractal_server.app.schemas.v2.sharing import ProjectPermissions
 from fractal_server.string_tools import sanitize_string
 from fractal_server.urls import normalize_url
@@ -32,15 +29,15 @@ router = APIRouter()
 
 @router.post(
     "/project/{project_id}/dataset/",
-    response_model=DatasetRead,
+    response_model=DatasetReadV2,
     status_code=status.HTTP_201_CREATED,
 )
 async def create_dataset(
     project_id: int,
-    dataset: DatasetCreate,
+    dataset: DatasetCreateV2,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetRead | None:
+) -> DatasetReadV2 | None:
     """
     Add new dataset to current project
     """
@@ -51,54 +48,44 @@ async def create_dataset(
         db=db,
     )
 
-    db_dataset = DatasetV2(
-        project_id=project_id,
-        zarr_dir="__PLACEHOLDER__",
-        **dataset.model_dump(exclude={"project_dir", "zarr_subfolder"}),
-    )
-    db.add(db_dataset)
-    await db.commit()
-    await db.refresh(db_dataset)
-
-    if dataset.project_dir is None:
-        project_dir = user.project_dirs[0]
-    else:
-        if dataset.project_dir not in user.project_dirs:
-            await db.delete(db_dataset)
-            await db.commit()
-            raise HTTPException(
-                status_code=status.HTTP_403_FORBIDDEN,
-                detail=f"You are not allowed to use {dataset.project_dir=}.",
-            )
-        project_dir = dataset.project_dir
-
-    if dataset.zarr_subfolder is None:
-        zarr_subfolder = (
-            f"fractal/{project_id}_{sanitize_string(project.name)}/"
+    if dataset.zarr_dir is None:
+        db_dataset = DatasetV2(
+            project_id=project_id,
+            zarr_dir="__PLACEHOLDER__",
+            **dataset.model_dump(exclude={"zarr_dir"}),
+        )
+        db.add(db_dataset)
+        await db.commit()
+        await db.refresh(db_dataset)
+        path = (
+            f"{user.project_dir}/fractal/"
+            f"{project_id}_{sanitize_string(project.name)}/"
             f"{db_dataset.id}_{sanitize_string(db_dataset.name)}"
         )
-    else:
-        zarr_subfolder = dataset.zarr_subfolder
+        normalized_path = normalize_url(path)
+        db_dataset.zarr_dir = normalized_path
 
-    zarr_dir = os.path.join(project_dir, zarr_subfolder)
-    db_dataset.zarr_dir = normalize_url(zarr_dir)
-
-    db.add(db_dataset)
-    await db.commit()
-    await db.refresh(db_dataset)
+        db.add(db_dataset)
+        await db.commit()
+        await db.refresh(db_dataset)
+    else:
+        db_dataset = DatasetV2(project_id=project_id, **dataset.model_dump())
+        db.add(db_dataset)
+        await db.commit()
+        await db.refresh(db_dataset)
 
     return db_dataset
 
 
 @router.get(
     "/project/{project_id}/dataset/",
-    response_model=list[DatasetRead],
+    response_model=list[DatasetReadV2],
 )
 async def read_dataset_list(
     project_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> list[DatasetRead] | None:
+) -> list[DatasetReadV2] | None:
     """
     Get dataset list for given project
     """
@@ -121,14 +108,14 @@ async def read_dataset_list(
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/",
-    response_model=DatasetRead,
+    response_model=DatasetReadV2,
 )
 async def read_dataset(
     project_id: int,
     dataset_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetRead | None:
+) -> DatasetReadV2 | None:
     """
     Get info on a dataset associated to the current project
     """
@@ -145,15 +132,15 @@ async def read_dataset(
 
 @router.patch(
     "/project/{project_id}/dataset/{dataset_id}/",
-    response_model=DatasetRead,
+    response_model=DatasetReadV2,
 )
 async def update_dataset(
     project_id: int,
     dataset_id: int,
-    dataset_update: DatasetUpdate,
+    dataset_update: DatasetUpdateV2,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetRead | None:
+) -> DatasetReadV2 | None:
     """
     Edit a dataset associated to the current project
     """
@@ -167,6 +154,15 @@ async def update_dataset(
     )
     db_dataset = output["dataset"]
 
+    if (dataset_update.zarr_dir is not None) and (len(db_dataset.images) != 0):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=(
+                "Cannot modify `zarr_dir` because the dataset has a non-empty "
+                "image list."
+            ),
+        )
+
     for key, value in dataset_update.model_dump(exclude_unset=True).items():
         setattr(db_dataset, key, value)
 
@@ -221,14 +217,14 @@ async def delete_dataset(
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/export/",
-    response_model=DatasetExport,
+    response_model=DatasetExportV2,
 )
 async def export_dataset(
     project_id: int,
     dataset_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetExport | None:
+) -> DatasetExportV2 | None:
     """
     Export an existing dataset
     """
@@ -246,15 +242,15 @@ async def export_dataset(
 
 @router.post(
     "/project/{project_id}/dataset/import/",
-    response_model=DatasetRead,
+    response_model=DatasetReadV2,
     status_code=status.HTTP_201_CREATED,
 )
 async def import_dataset(
     project_id: int,
-    dataset: DatasetImport,
+    dataset: DatasetImportV2,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetRead | None:
+) -> DatasetReadV2 | None:
     """
     Import an existing dataset into a project
     """
@@ -267,17 +263,15 @@ async def import_dataset(
         db=db,
     )
 
-    if not any(
-        Path(dataset.zarr_dir).is_relative_to(project_dir)
-        for project_dir in user.project_dirs
-    ):
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
-            detail=(
-                f"{dataset.zarr_dir=} is not relative to any of user's project "
-                "dirs."
-            ),
-        )
+    for image in dataset.images:
+        if not image.zarr_url.startswith(dataset.zarr_dir):
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+                detail=(
+                    f"Cannot import dataset: zarr_url {image.zarr_url} is not "
+                    f"relative to zarr_dir={dataset.zarr_dir}."
+                ),
+            )
 
     # Create new Dataset
     db_dataset = DatasetV2(

fractal_server/app/routes/api/v2/history.py

@@ -33,7 +33,7 @@ from fractal_server.images.tools import filter_image_list
 from fractal_server.logger import set_logger
 
 from ._aux_functions import _get_dataset_check_access
-from ._aux_functions import _get_submitted_jobs_statement
+from ._aux_functions import _get_submitted_job_or_none
 from ._aux_functions import _get_workflow_check_access
 from ._aux_functions_history import _verify_workflow_and_dataset_access
 from ._aux_functions_history import get_history_run_or_404
@@ -90,13 +90,11 @@ async def get_workflow_tasks_statuses(
         db=db,
     )
 
-    res = await db.execute(
-        _get_submitted_jobs_statement()
-        .where(JobV2.dataset_id == dataset_id)
-        .where(JobV2.workflow_id == workflow_id)
+    running_job = await _get_submitted_job_or_none(
+        db=db,
+        dataset_id=dataset_id,
+        workflow_id=workflow_id,
     )
-    running_job = res.scalars().one_or_none()
-
     if running_job is not None:
         running_wftasks = workflow.task_list[
             running_job.first_task_index : running_job.last_task_index + 1

fractal_server/app/routes/api/v2/job.py

@@ -18,8 +18,8 @@ from fractal_server.app.models.v2 import LinkUserProjectV2
 from fractal_server.app.routes.auth import current_user_act_ver_prof
 from fractal_server.app.routes.aux._job import _write_shutdown_file
 from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
-from fractal_server.app.schemas.v2 import JobRead
-from fractal_server.app.schemas.v2 import JobStatusType
+from fractal_server.app.schemas.v2 import JobReadV2
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
 from fractal_server.app.schemas.v2.sharing import ProjectPermissions
 from fractal_server.runner.filenames import WORKFLOW_LOG_FILENAME
 from fractal_server.zip_tools import _zip_folder_to_byte_stream_iterator
@@ -39,12 +39,12 @@ async def zip_folder_threaded(folder: str) -> Iterator[bytes]:
 router = APIRouter()
 
 
-@router.get("/job/", response_model=list[JobRead])
+@router.get("/job/", response_model=list[JobReadV2])
 async def get_user_jobs(
     user: UserOAuth = Depends(current_user_act_ver_prof),
     log: bool = True,
     db: AsyncSession = Depends(get_async_db),
-) -> list[JobRead]:
+) -> list[JobReadV2]:
     """
     Returns all the jobs of the current user
     """
@@ -68,14 +68,14 @@ async def get_user_jobs(
 
 @router.get(
     "/project/{project_id}/workflow/{workflow_id}/job/",
-    response_model=list[JobRead],
+    response_model=list[JobReadV2],
 )
 async def get_workflow_jobs(
     project_id: int,
     workflow_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> list[JobRead] | None:
+) -> list[JobReadV2] | None:
     """
     Returns all the jobs related to a specific workflow
     """
@@ -99,7 +99,7 @@ async def get_latest_job(
     dataset_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> JobRead:
+) -> JobReadV2:
     await _get_workflow_check_access(
         project_id=project_id,
         workflow_id=workflow_id,
@@ -127,7 +127,7 @@ async def get_latest_job(
 
 @router.get(
     "/project/{project_id}/job/{job_id}/",
-    response_model=JobRead,
+    response_model=JobReadV2,
 )
 async def read_job(
     project_id: int,
@@ -135,7 +135,7 @@ async def read_job(
     show_tmp_logs: bool = False,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> JobRead | None:
+) -> JobReadV2 | None:
     """
     Return info on an existing job
     """
@@ -150,7 +150,7 @@ async def read_job(
     job = output["job"]
     await db.close()
 
-    if show_tmp_logs and (job.status == JobStatusType.SUBMITTED):
+    if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
         try:
             with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}") as f:
                 job.log = f.read()
@@ -194,14 +194,14 @@ async def download_job_logs(
 
 @router.get(
     "/project/{project_id}/job/",
-    response_model=list[JobRead],
+    response_model=list[JobReadV2],
 )
 async def get_job_list(
     project_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     log: bool = True,
     db: AsyncSession = Depends(get_async_db),
-) -> list[JobRead] | None:
+) -> list[JobReadV2] | None:
     """
     Get job list for given project
     """

fractal_server/app/routes/api/v2/project.py

@@ -15,10 +15,10 @@ from fractal_server.app.routes.auth import current_user_act_ver_prof
 from fractal_server.app.routes.aux.validate_user_profile import (
     validate_user_profile,
 )
-from fractal_server.app.schemas.v2 import ProjectCreate
+from fractal_server.app.schemas.v2 import ProjectCreateV2
 from fractal_server.app.schemas.v2 import ProjectPermissions
-from fractal_server.app.schemas.v2 import ProjectRead
-from fractal_server.app.schemas.v2 import ProjectUpdate
+from fractal_server.app.schemas.v2 import ProjectReadV2
+from fractal_server.app.schemas.v2 import ProjectUpdateV2
 from fractal_server.logger import set_logger
 
 from ._aux_functions import _check_project_exists
@@ -29,7 +29,7 @@ logger = set_logger(__name__)
 router = APIRouter()
 
 
-@router.get("/project/", response_model=list[ProjectRead])
+@router.get("/project/", response_model=list[ProjectReadV2])
 async def get_list_project(
     is_owner: bool = True,
     user: UserOAuth = Depends(current_user_act_ver_prof),
@@ -51,12 +51,12 @@ async def get_list_project(
     return project_list
 
 
-@router.post("/project/", response_model=ProjectRead, status_code=201)
+@router.post("/project/", response_model=ProjectReadV2, status_code=201)
 async def create_project(
-    project: ProjectCreate,
+    project: ProjectCreateV2,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> ProjectRead | None:
+) -> ProjectReadV2 | None:
     """
     Create new project
     """
@@ -92,12 +92,12 @@ async def create_project(
     return db_project
 
 
-@router.get("/project/{project_id}/", response_model=ProjectRead)
+@router.get("/project/{project_id}/", response_model=ProjectReadV2)
 async def read_project(
     project_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> ProjectRead | None:
+) -> ProjectReadV2 | None:
     """
     Return info on an existing project
     """
@@ -111,10 +111,10 @@ async def read_project(
     return project
 
 
-@router.patch("/project/{project_id}/", response_model=ProjectRead)
+@router.patch("/project/{project_id}/", response_model=ProjectReadV2)
 async def update_project(
     project_id: int,
-    project_update: ProjectUpdate,
+    project_update: ProjectUpdateV2,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
 ):