fractal-server 2.18.0a3__py3-none-any.whl → 2.18.0a5__py3-none-any.whl

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -3,11 +3,10 @@ Auxiliary functions to get object from the database or perform simple checks
 """
 
 from typing import Any
-from typing import Literal
+from typing import TypedDict
 
 from fastapi import HTTPException
 from fastapi import status
-from sqlalchemy.exc import MultipleResultsFound
 from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import select
 from sqlmodel.sql.expression import SelectOfScalar
@@ -23,7 +22,7 @@ from fractal_server.app.models.v2 import ProjectV2
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.models.v2 import WorkflowV2
-from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.app.schemas.v2 import JobStatusType
 from fractal_server.app.schemas.v2 import ProjectPermissions
 from fractal_server.logger import set_logger
 
@@ -252,6 +251,11 @@ async def _check_project_exists(
         )
 
 
+class DatasetOrProject(TypedDict):
+    dataset: DatasetV2
+    project: ProjectV2
+
+
 async def _get_dataset_check_access(
     *,
     project_id: int,
@@ -259,7 +263,7 @@ async def _get_dataset_check_access(
     user_id: int,
     required_permissions: ProjectPermissions,
     db: AsyncSession,
-) -> dict[Literal["dataset", "project"], DatasetV2 | ProjectV2]:
+) -> DatasetOrProject:
     """
     Get a dataset and a project, after access control on the project
 
@@ -304,6 +308,11 @@ async def _get_dataset_check_access(
     return dict(dataset=dataset, project=project)
 
 
+class JobAndProject(TypedDict):
+    job: JobV2
+    project: ProjectV2
+
+
 async def _get_job_check_access(
     *,
     project_id: int,
@@ -311,7 +320,7 @@ async def _get_job_check_access(
     user_id: int,
     required_permissions: ProjectPermissions,
     db: AsyncSession,
-) -> dict[Literal["job", "project"], JobV2 | ProjectV2]:
+) -> JobAndProject:
     """
     Get a job and a project, after access control on the project
 
@@ -361,7 +370,7 @@ def _get_submitted_jobs_statement() -> SelectOfScalar:
         A sqlmodel statement that selects all `Job`s with
         `Job.status` equal to `submitted`.
     """
-    stm = select(JobV2).where(JobV2.status == JobStatusTypeV2.SUBMITTED)
+    stm = select(JobV2).where(JobV2.status == JobStatusType.SUBMITTED)
     return stm
 
 
@@ -371,7 +380,7 @@ async def _workflow_has_submitted_job(
 ) -> bool:
     res = await db.execute(
         select(JobV2.id)
-        .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
+        .where(JobV2.status == JobStatusType.SUBMITTED)
         .where(JobV2.workflow_id == workflow_id)
         .limit(1)
     )
@@ -453,8 +462,9 @@ async def _workflow_insert_task(
     return wf_task
 
 
-async def clean_app_job_list_v2(
-    db: AsyncSession, jobs_list: list[int]
+async def clean_app_job_list(
+    db: AsyncSession,
+    jobs_list: list[int],
 ) -> list[int]:
     """
     Remove from a job list all jobs with status different from submitted.
@@ -466,14 +476,14 @@ async def clean_app_job_list_v2(
     Return:
         List of IDs for submitted jobs.
     """
+    logger.info(f"[clean_app_job_list] START - {jobs_list=}.")
     stmt = select(JobV2).where(JobV2.id.in_(jobs_list))
     result = await db.execute(stmt)
     db_jobs_list = result.scalars().all()
     submitted_job_ids = [
-        job.id
-        for job in db_jobs_list
-        if job.status == JobStatusTypeV2.SUBMITTED
+        job.id for job in db_jobs_list if job.status == JobStatusType.SUBMITTED
     ]
+    logger.info(f"[clean_app_job_list] END - {submitted_job_ids=}.")
     return submitted_job_ids
 
 
@@ -543,41 +553,6 @@ async def _get_workflowtask_or_404(
         return wftask
 
 
-async def _get_submitted_job_or_none(
-    *,
-    dataset_id: int,
-    workflow_id: int,
-    db: AsyncSession,
-) -> JobV2 | None:
-    """
-    Get the submitted job for given dataset/workflow, if any.
-
-    This function also handles the invalid branch where more than one job
-    is found.
-
-    Args:
-        dataset_id:
-        workflow_id:
-        db:
-    """
-    res = await db.execute(
-        _get_submitted_jobs_statement()
-        .where(JobV2.dataset_id == dataset_id)
-        .where(JobV2.workflow_id == workflow_id)
-    )
-    try:
-        return res.scalars().one_or_none()
-    except MultipleResultsFound as e:
-        error_msg = (
-            f"Multiple running jobs found for {dataset_id=} and {workflow_id=}."
-        )
-        logger.error(f"{error_msg} Original error: {str(e)}.")
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
-            detail=error_msg,
-        )
-
-
 async def _get_user_resource_id(user_id: int, db: AsyncSession) -> int | None:
     res = await db.execute(
         select(Resource.id)

fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py

@@ -14,8 +14,8 @@ from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.models.v2 import WorkflowV2
-from fractal_server.app.schemas.v2 import JobStatusTypeV2
-from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
+from fractal_server.app.schemas.v2 import JobStatusType
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
 from fractal_server.logger import set_logger
 from fractal_server.tasks.v2.utils_package_names import normalize_package_name
 
@@ -171,7 +171,7 @@ async def check_no_ongoing_activity(
     stm = (
         select(TaskGroupActivityV2)
         .where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
-        .where(TaskGroupActivityV2.status == TaskGroupActivityStatusV2.ONGOING)
+        .where(TaskGroupActivityV2.status == TaskGroupActivityStatus.ONGOING)
     )
     res = await db.execute(stm)
     ongoing_activities = res.scalars().all()
@@ -213,7 +213,7 @@ async def check_no_submitted_job(
         .join(TaskV2, WorkflowTaskV2.task_id == TaskV2.id)
         .where(WorkflowTaskV2.order >= JobV2.first_task_index)
         .where(WorkflowTaskV2.order <= JobV2.last_task_index)
-        .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
+        .where(JobV2.status == JobStatusType.SUBMITTED)
         .where(TaskV2.taskgroupv2_id == task_group_id)
     )
     res = await db.execute(stm)

fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -27,7 +27,7 @@ from fractal_server.app.routes.auth._aux_auth import (
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
-from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityAction
 from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
 
@@ -252,7 +252,7 @@ async def _get_collection_task_group_activity_status_message(
     res = await db.execute(
         select(TaskGroupActivityV2)
         .where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
-        .where(TaskGroupActivityV2.action == TaskGroupActivityActionV2.COLLECT)
+        .where(TaskGroupActivityV2.action == TaskGroupActivityAction.COLLECT)
     )
     task_group_activity_list = res.scalars().all()
     if len(task_group_activity_list) > 1:

fractal_server/app/routes/api/v2/dataset.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 from fastapi import APIRouter
@@ -13,11 +14,11 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.routes.auth import current_user_act_ver_prof
-from fractal_server.app.schemas.v2 import DatasetCreateV2
-from fractal_server.app.schemas.v2 import DatasetReadV2
-from fractal_server.app.schemas.v2 import DatasetUpdateV2
-from fractal_server.app.schemas.v2.dataset import DatasetExportV2
-from fractal_server.app.schemas.v2.dataset import DatasetImportV2
+from fractal_server.app.schemas.v2 import DatasetCreate
+from fractal_server.app.schemas.v2 import DatasetRead
+from fractal_server.app.schemas.v2 import DatasetUpdate
+from fractal_server.app.schemas.v2.dataset import DatasetExport
+from fractal_server.app.schemas.v2.dataset import DatasetImport
 from fractal_server.app.schemas.v2.sharing import ProjectPermissions
 from fractal_server.string_tools import sanitize_string
 from fractal_server.urls import normalize_url
@@ -31,15 +32,15 @@ router = APIRouter()
 
 @router.post(
     "/project/{project_id}/dataset/",
-    response_model=DatasetReadV2,
+    response_model=DatasetRead,
     status_code=status.HTTP_201_CREATED,
 )
 async def create_dataset(
     project_id: int,
-    dataset: DatasetCreateV2,
+    dataset: DatasetCreate,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetReadV2 | None:
+) -> DatasetRead | None:
     """
     Add new dataset to current project
     """
@@ -50,56 +51,54 @@ async def create_dataset(
         db=db,
     )
 
-    if dataset.zarr_dir is None:
-        db_dataset = DatasetV2(
-            project_id=project_id,
-            zarr_dir="__PLACEHOLDER__",
-            **dataset.model_dump(exclude={"zarr_dir"}),
-        )
-        db.add(db_dataset)
-        await db.commit()
-        await db.refresh(db_dataset)
-        path = (
-            f"{user.project_dirs[0]}/fractal/"
-            f"{project_id}_{sanitize_string(project.name)}/"
-            f"{db_dataset.id}_{sanitize_string(db_dataset.name)}"
-        )
-        normalized_path = normalize_url(path)
-        db_dataset.zarr_dir = normalized_path
+    db_dataset = DatasetV2(
+        project_id=project_id,
+        zarr_dir="__PLACEHOLDER__",
+        **dataset.model_dump(exclude={"project_dir", "zarr_subfolder"}),
+    )
+    db.add(db_dataset)
+    await db.commit()
+    await db.refresh(db_dataset)
 
-        db.add(db_dataset)
-        await db.commit()
-        await db.refresh(db_dataset)
+    if dataset.project_dir is None:
+        project_dir = user.project_dirs[0]
     else:
-        if not any(
-            Path(dataset.zarr_dir).is_relative_to(project_dir)
-            for project_dir in user.project_dirs
-        ):
+        if dataset.project_dir not in user.project_dirs:
+            await db.delete(db_dataset)
+            await db.commit()
             raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
-                detail=(
-                    "Dataset zarr_dir is not relative to any of the user "
-                    "project directories."
-                ),
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=f"You are not allowed to use {dataset.project_dir=}.",
             )
+        project_dir = dataset.project_dir
 
-        db_dataset = DatasetV2(project_id=project_id, **dataset.model_dump())
-        db.add(db_dataset)
-        await db.commit()
-        await db.refresh(db_dataset)
+    if dataset.zarr_subfolder is None:
+        zarr_subfolder = (
+            f"fractal/{project_id}_{sanitize_string(project.name)}/"
+            f"{db_dataset.id}_{sanitize_string(db_dataset.name)}"
+        )
+    else:
+        zarr_subfolder = dataset.zarr_subfolder
+
+    zarr_dir = os.path.join(project_dir, zarr_subfolder)
+    db_dataset.zarr_dir = normalize_url(zarr_dir)
+
+    db.add(db_dataset)
+    await db.commit()
+    await db.refresh(db_dataset)
 
     return db_dataset
 
 
 @router.get(
     "/project/{project_id}/dataset/",
-    response_model=list[DatasetReadV2],
+    response_model=list[DatasetRead],
 )
 async def read_dataset_list(
     project_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> list[DatasetReadV2] | None:
+) -> list[DatasetRead] | None:
     """
     Get dataset list for given project
     """
@@ -122,14 +121,14 @@ async def read_dataset_list(
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/",
-    response_model=DatasetReadV2,
+    response_model=DatasetRead,
 )
 async def read_dataset(
     project_id: int,
     dataset_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetReadV2 | None:
+) -> DatasetRead | None:
     """
     Get info on a dataset associated to the current project
     """
@@ -146,15 +145,15 @@ async def read_dataset(
 
 @router.patch(
     "/project/{project_id}/dataset/{dataset_id}/",
-    response_model=DatasetReadV2,
+    response_model=DatasetRead,
 )
 async def update_dataset(
     project_id: int,
     dataset_id: int,
-    dataset_update: DatasetUpdateV2,
+    dataset_update: DatasetUpdate,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetReadV2 | None:
+) -> DatasetRead | None:
     """
     Edit a dataset associated to the current project
     """
@@ -168,27 +167,6 @@ async def update_dataset(
     )
     db_dataset = output["dataset"]
 
-    if dataset_update.zarr_dir is not None:
-        if db_dataset.images:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
-                detail=(
-                    "Cannot modify `zarr_dir` because the dataset has a "
-                    "non-empty image list."
-                ),
-            )
-        if not any(
-            Path(dataset_update.zarr_dir).is_relative_to(project_dir)
-            for project_dir in user.project_dirs
-        ):
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
-                detail=(
-                    "Dataset zarr_dir is not relative to any of the user "
-                    "project directories."
-                ),
-            )
-
     for key, value in dataset_update.model_dump(exclude_unset=True).items():
         setattr(db_dataset, key, value)
 
@@ -243,14 +221,14 @@ async def delete_dataset(
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/export/",
-    response_model=DatasetExportV2,
+    response_model=DatasetExport,
 )
 async def export_dataset(
     project_id: int,
     dataset_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetExportV2 | None:
+) -> DatasetExport | None:
     """
     Export an existing dataset
     """
@@ -268,15 +246,15 @@ async def export_dataset(
 
 @router.post(
     "/project/{project_id}/dataset/import/",
-    response_model=DatasetReadV2,
+    response_model=DatasetRead,
     status_code=status.HTTP_201_CREATED,
 )
 async def import_dataset(
     project_id: int,
-    dataset: DatasetImportV2,
+    dataset: DatasetImport,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> DatasetReadV2 | None:
+) -> DatasetRead | None:
     """
     Import an existing dataset into a project
     """
@@ -289,6 +267,18 @@ async def import_dataset(
         db=db,
     )
 
+    if not any(
+        Path(dataset.zarr_dir).is_relative_to(project_dir)
+        for project_dir in user.project_dirs
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=(
+                f"{dataset.zarr_dir=} is not relative to any of user's project "
+                "dirs."
+            ),
+        )
+
     for image in dataset.images:
         if not image.zarr_url.startswith(dataset.zarr_dir):
             raise HTTPException(

fractal_server/app/routes/api/v2/history.py

@@ -33,7 +33,7 @@ from fractal_server.images.tools import filter_image_list
 from fractal_server.logger import set_logger
 
 from ._aux_functions import _get_dataset_check_access
-from ._aux_functions import _get_submitted_job_or_none
+from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_access
 from ._aux_functions_history import _verify_workflow_and_dataset_access
 from ._aux_functions_history import get_history_run_or_404
@@ -90,11 +90,13 @@ async def get_workflow_tasks_statuses(
         db=db,
     )
 
-    running_job = await _get_submitted_job_or_none(
-        db=db,
-        dataset_id=dataset_id,
-        workflow_id=workflow_id,
+    res = await db.execute(
+        _get_submitted_jobs_statement()
+        .where(JobV2.dataset_id == dataset_id)
+        .where(JobV2.workflow_id == workflow_id)
     )
+    running_job = res.scalars().one_or_none()
+
     if running_job is not None:
         running_wftasks = workflow.task_list[
             running_job.first_task_index : running_job.last_task_index + 1

fractal_server/app/routes/api/v2/job.py

@@ -18,8 +18,8 @@ from fractal_server.app.models.v2 import LinkUserProjectV2
 from fractal_server.app.routes.auth import current_user_act_ver_prof
 from fractal_server.app.routes.aux._job import _write_shutdown_file
 from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
-from fractal_server.app.schemas.v2 import JobReadV2
-from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.app.schemas.v2 import JobRead
+from fractal_server.app.schemas.v2 import JobStatusType
 from fractal_server.app.schemas.v2.sharing import ProjectPermissions
 from fractal_server.runner.filenames import WORKFLOW_LOG_FILENAME
 from fractal_server.zip_tools import _zip_folder_to_byte_stream_iterator
@@ -39,12 +39,12 @@ async def zip_folder_threaded(folder: str) -> Iterator[bytes]:
 router = APIRouter()
 
 
-@router.get("/job/", response_model=list[JobReadV2])
+@router.get("/job/", response_model=list[JobRead])
 async def get_user_jobs(
     user: UserOAuth = Depends(current_user_act_ver_prof),
     log: bool = True,
     db: AsyncSession = Depends(get_async_db),
-) -> list[JobReadV2]:
+) -> list[JobRead]:
     """
     Returns all the jobs of the current user
     """
@@ -68,14 +68,14 @@ async def get_user_jobs(
 
 @router.get(
     "/project/{project_id}/workflow/{workflow_id}/job/",
-    response_model=list[JobReadV2],
+    response_model=list[JobRead],
 )
 async def get_workflow_jobs(
     project_id: int,
     workflow_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> list[JobReadV2] | None:
+) -> list[JobRead] | None:
     """
     Returns all the jobs related to a specific workflow
     """
@@ -99,7 +99,7 @@ async def get_latest_job(
     dataset_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> JobReadV2:
+) -> JobRead:
     await _get_workflow_check_access(
         project_id=project_id,
         workflow_id=workflow_id,
@@ -127,7 +127,7 @@ async def get_latest_job(
 
 @router.get(
     "/project/{project_id}/job/{job_id}/",
-    response_model=JobReadV2,
+    response_model=JobRead,
 )
 async def read_job(
     project_id: int,
@@ -135,7 +135,7 @@ async def read_job(
     show_tmp_logs: bool = False,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> JobReadV2 | None:
+) -> JobRead | None:
     """
     Return info on an existing job
     """
@@ -150,7 +150,7 @@ async def read_job(
     job = output["job"]
     await db.close()
 
-    if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
+    if show_tmp_logs and (job.status == JobStatusType.SUBMITTED):
         try:
             with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}") as f:
                 job.log = f.read()
@@ -194,14 +194,14 @@ async def download_job_logs(
 
 @router.get(
     "/project/{project_id}/job/",
-    response_model=list[JobReadV2],
+    response_model=list[JobRead],
 )
 async def get_job_list(
     project_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     log: bool = True,
     db: AsyncSession = Depends(get_async_db),
-) -> list[JobReadV2] | None:
+) -> list[JobRead] | None:
     """
     Get job list for given project
     """

fractal_server/app/routes/api/v2/project.py

@@ -15,10 +15,10 @@ from fractal_server.app.routes.auth import current_user_act_ver_prof
 from fractal_server.app.routes.aux.validate_user_profile import (
     validate_user_profile,
 )
-from fractal_server.app.schemas.v2 import ProjectCreateV2
+from fractal_server.app.schemas.v2 import ProjectCreate
 from fractal_server.app.schemas.v2 import ProjectPermissions
-from fractal_server.app.schemas.v2 import ProjectReadV2
-from fractal_server.app.schemas.v2 import ProjectUpdateV2
+from fractal_server.app.schemas.v2 import ProjectRead
+from fractal_server.app.schemas.v2 import ProjectUpdate
 from fractal_server.logger import set_logger
 
 from ._aux_functions import _check_project_exists
@@ -29,7 +29,7 @@ logger = set_logger(__name__)
 router = APIRouter()
 
 
-@router.get("/project/", response_model=list[ProjectReadV2])
+@router.get("/project/", response_model=list[ProjectRead])
 async def get_list_project(
     is_owner: bool = True,
     user: UserOAuth = Depends(current_user_act_ver_prof),
@@ -51,12 +51,12 @@ async def get_list_project(
     return project_list
 
 
-@router.post("/project/", response_model=ProjectReadV2, status_code=201)
+@router.post("/project/", response_model=ProjectRead, status_code=201)
 async def create_project(
-    project: ProjectCreateV2,
+    project: ProjectCreate,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> ProjectReadV2 | None:
+) -> ProjectRead | None:
     """
     Create new project
     """
@@ -92,12 +92,12 @@ async def create_project(
     return db_project
 
 
-@router.get("/project/{project_id}/", response_model=ProjectReadV2)
+@router.get("/project/{project_id}/", response_model=ProjectRead)
 async def read_project(
     project_id: int,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
-) -> ProjectReadV2 | None:
+) -> ProjectRead | None:
     """
     Return info on an existing project
     """
@@ -111,10 +111,10 @@ async def read_project(
     return project
 
 
-@router.patch("/project/{project_id}/", response_model=ProjectReadV2)
+@router.patch("/project/{project_id}/", response_model=ProjectRead)
 async def update_project(
     project_id: int,
-    project_update: ProjectUpdateV2,
+    project_update: ProjectUpdate,
     user: UserOAuth = Depends(current_user_act_ver_prof),
     db: AsyncSession = Depends(get_async_db),
 ):