fractal-server 2.13.1__tar.gz → 2.14.0a0__tar.gz

{fractal_server-2.13.1 → fractal_server-2.14.0a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.13.1
+Version: 2.14.0a0
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.14.0a0/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a0"

fractal_server-2.14.0a0/fractal_server/app/history/__init__.py

@@ -0,0 +1,4 @@
+from .image_updates import update_all_images  # noqa: F401
+from .image_updates import update_single_image  # noqa
+from .image_updates import update_single_image_logfile  # noqa
+from .status_enum import HistoryItemImageStatus  # noqa: F401

fractal_server-2.14.0a0/fractal_server/app/history/image_updates.py

@@ -0,0 +1,142 @@
+from typing import Optional
+
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.history.status_enum import HistoryItemImageStatus
+from fractal_server.app.models.v2 import HistoryItemV2
+from fractal_server.app.models.v2 import ImageStatus
+from fractal_server.logger import set_logger
+
+logger = set_logger(__name__)
+
+
+def _update_single_image_status(
+    *,
+    zarr_url: str,
+    workflowtask_id: int,
+    dataset_id: int,
+    status: HistoryItemImageStatus,
+    db: Session,
+    commit: bool = True,
+    logfile: Optional[str] = None,
+) -> None:
+    image_status = db.get(
+        ImageStatus,
+        (
+            zarr_url,
+            workflowtask_id,
+            dataset_id,
+        ),
+    )
+    if image_status is None:
+        raise RuntimeError("This should have not happened")
+    image_status.status = status
+    if logfile is not None:
+        image_status.logfile = logfile
+    db.add(image_status)
+    if commit:
+        db.commit()
+
+
+def update_single_image(
+    *,
+    history_item_id: int,
+    zarr_url: str,
+    status: HistoryItemImageStatus,
+) -> None:
+
+    logger.debug(
+        f"[update_single_image] {history_item_id=}, {status=}, {zarr_url=}"
+    )
+
+    # Note: thanks to `with_for_update`, a lock is acquired and kept
+    # until `db.commit()`
+    with next(get_sync_db()) as db:
+        stm = (
+            select(HistoryItemV2)
+            .where(HistoryItemV2.id == history_item_id)
+            .with_for_update(nowait=False)
+        )
+        history_item = db.execute(stm).scalar_one()
+        history_item.images[zarr_url] = status
+        flag_modified(history_item, "images")
+        db.commit()
+
+        _update_single_image_status(
+            zarr_url=zarr_url,
+            dataset_id=history_item.dataset_id,
+            workflowtask_id=history_item.workflowtask_id,
+            commit=True,
+            status=status,
+            db=db,
+        )
+
+
+def update_single_image_logfile(
+    *,
+    history_item_id: int,
+    zarr_url: str,
+    logfile: str,
+) -> None:
+
+    logger.debug(
+        f"[update_single_image_logfile] {history_item_id=}, {logfile=}, {zarr_url=}"
+    )
+
+    with next(get_sync_db()) as db:
+        history_item = db.get(HistoryItemV2, history_item_id)
+        image_status = db.get(
+            ImageStatus,
+            (
+                zarr_url,
+                history_item.workflowtask_id,
+                history_item.dataset_id,
+            ),
+        )
+        if image_status is None:
+            raise RuntimeError("This should have not happened")
+        image_status.logfile = logfile
+        db.merge(image_status)
+        db.commit()
+
+
+def update_all_images(
+    *,
+    history_item_id: int,
+    status: HistoryItemImageStatus,
+    logfile: Optional[str] = None,
+) -> None:
+
+    logger.debug(f"[update_all_images] {history_item_id=}, {status=}")
+
+    # Note: thanks to `with_for_update`, a lock is acquired and kept
+    # until `db.commit()`
+    stm = (
+        select(HistoryItemV2)
+        .where(HistoryItemV2.id == history_item_id)
+        .with_for_update(nowait=False)
+    )
+    with next(get_sync_db()) as db:
+        history_item = db.execute(stm).scalar_one()
+        new_images = {
+            zarr_url: status for zarr_url in history_item.images.keys()
+        }
+        history_item.images = new_images
+        flag_modified(history_item, "images")
+        db.commit()
+
+        # FIXME: Make this a bulk edit, if possible
+        for ind, zarr_url in enumerate(history_item.images.keys()):
+            _update_single_image_status(
+                zarr_url=zarr_url,
+                dataset_id=history_item.dataset_id,
+                workflowtask_id=history_item.workflowtask_id,
+                commit=False,
+                status=status,
+                logfile=logfile,
+                db=db,
+            )
+        db.commit()

fractal_server-2.14.0a0/fractal_server/app/history/status_enum.py

@@ -0,0 +1,16 @@
+from enum import Enum
+
+
+class HistoryItemImageStatus(str, Enum):
+    """
+    Available image-status values within a `HistoryItemV2`
+
+    Attributes:
+        SUBMITTED:
+        DONE:
+        FAILED:
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"

{fractal_server-2.13.1 → fractal_server-2.14.0a0}/fractal_server/app/models/v2/__init__.py

@@ -5,6 +5,8 @@ from ..linkuserproject import LinkUserProjectV2
 from .accounting import AccountingRecord
 from .accounting import AccountingRecordSlurm
 from .dataset import DatasetV2
+from .history import HistoryItemV2
+from .history import ImageStatus
 from .job import JobV2
 from .project import ProjectV2
 from .task import TaskV2
@@ -23,6 +25,8 @@ __all__ = [
     "TaskGroupV2",
     "TaskGroupActivityV2",
     "TaskV2",
-    "WorkflowTaskV2",
     "WorkflowV2",
+    "WorkflowTaskV2",
+    "HistoryItemV2",
+    "ImageStatus",
 ]

fractal_server-2.14.0a0/fractal_server/app/models/v2/history.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+from typing import Any
+from typing import Optional
+
+from pydantic import ConfigDict
+from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.types import DateTime
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+
+
+class HistoryItemV2(SQLModel, table=True):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    dataset_id: int = Field(foreign_key="datasetv2.id")
+    workflowtask_id: Optional[int] = Field(
+        foreign_key="workflowtaskv2.id",
+        default=None,
+    )
+    timestamp_started: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(
+            DateTime(timezone=True),
+            nullable=False,
+        ),
+    )
+    workflowtask_dump: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False)
+    )
+    task_group_dump: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False)
+    )
+    parameters_hash: str
+    num_available_images: int
+    num_current_images: int
+    images: dict[str, str] = Field(sa_column=Column(JSONB, nullable=False))
+
+
+class ImageStatus(SQLModel, table=True):
+
+    zarr_url: str = Field(primary_key=True)
+    workflowtask_id: int = Field(
+        primary_key=True, foreign_key="workflowtaskv2.id"
+    )
+    dataset_id: int = Field(primary_key=True, foreign_key="datasetv2.id")
+
+    parameters_hash: str
+    status: str
+    logfile: str

{fractal_server-2.13.1 → fractal_server-2.14.0a0}/fractal_server/app/routes/api/v2/__init__.py

@@ -4,10 +4,10 @@
 from fastapi import APIRouter
 
 from .dataset import router as dataset_router_v2
+from .history import router as history_router_v2
 from .images import router as images_routes_v2
 from .job import router as job_router_v2
 from .project import router as project_router_v2
-from .status import router as status_router_v2
 from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
@@ -28,6 +28,7 @@ router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
 router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
+router_api_v2.include_router(history_router_v2, tags=["V2 History"])
 
 
 settings = Inject(get_settings)
@@ -56,4 +57,3 @@ router_api_v2.include_router(
     workflow_import_router_v2, tags=["V2 Workflow Import"]
 )
 router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
-router_api_v2.include_router(status_router_v2, tags=["V2 Status"])

{fractal_server-2.13.1 → fractal_server-2.14.0a0}/fractal_server/app/routes/api/v2/_aux_functions.py

@@ -417,3 +417,81 @@ async def clean_app_job_list_v2(
         if job.status == JobStatusTypeV2.SUBMITTED
     ]
     return submitted_job_ids
+
+
+async def _get_workflow_check_history_owner(
+    *,
+    workflow_id: int,
+    dataset_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> list[int]:
+    """
+    Verify user access for the history of this dataset and workflowtask.
+
+    Args:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+
+    Returns:
+        List of WorkflowTask IDs
+    """
+    workflow = await db.get(WorkflowV2, workflow_id)
+    if workflow is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Workflow not found.",
+        )
+    await _get_project_check_owner(
+        project_id=workflow.project_id,
+        user_id=user_id,
+        db=db,
+    )
+    dataset = await db.get(DatasetV2, dataset_id)
+    if dataset is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Dataset not found.",
+        )
+    if workflow.project_id != dataset.project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Dataset and workflow belong to different projects.",
+        )
+
+    return [wftask.id for wftask in workflow.task_list]
+
+
+async def _get_workflowtask_check_history_owner(
+    *,
+    workflowtask_id: int,
+    dataset_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> list[int]:
+    """
+    Verify user access for the history of this dataset and workflowtask.
+
+    Args:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+
+    Returns:
+        List of WorkflowTask IDs
+    """
+    workflowtask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if workflowtask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="WorkflowTask not found.",
+        )
+    await _get_workflow_check_history_owner(
+        workflow_id=workflowtask.workflow_id,
+        dataset_id=dataset_id,
+        user_id=user_id,
+        db=db,
+    )

{fractal_server-2.13.1 → fractal_server-2.14.0a0}/fractal_server/app/routes/api/v2/dataset.py

@@ -5,11 +5,14 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from sqlmodel import delete
 from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import DatasetV2
+from ....models.v2 import HistoryItemV2
+from ....models.v2 import ImageStatus
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....schemas.v2 import DatasetCreateV2
@@ -47,7 +50,6 @@ async def create_dataset(
     )
 
     if dataset.zarr_dir is None:
-
         if user.settings.project_dir is None:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -91,7 +93,6 @@ async def create_dataset(
 )
 async def read_dataset_list(
     project_id: int,
-    history: bool = True,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[list[DatasetReadV2]]:
@@ -110,9 +111,6 @@ async def read_dataset_list(
     res = await db.execute(stm)
     dataset_list = res.scalars().all()
     await db.close()
-    if not history:
-        for ds in dataset_list:
-            setattr(ds, "history", [])
     return dataset_list
 
 
@@ -225,6 +223,15 @@ async def delete_dataset(
     for job in jobs:
         job.dataset_id = None
 
+    # Cascade operations: delete history items and image status which are in
+    # relationship with the current dataset
+
+    stm = delete(HistoryItemV2).where(HistoryItemV2.dataset_id == dataset_id)
+    await db.execute(stm)
+
+    stm = delete(ImageStatus).where(ImageStatus.dataset_id == dataset_id)
+    await db.execute(stm)
+
     # Delete dataset
     await db.delete(dataset)
     await db.commit()
@@ -234,7 +241,6 @@ async def delete_dataset(
 
 @router.get("/dataset/", response_model=list[DatasetReadV2])
 async def get_user_datasets(
-    history: bool = True,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[DatasetReadV2]:
@@ -249,9 +255,6 @@ async def get_user_datasets(
     res = await db.execute(stm)
     dataset_list = res.scalars().all()
     await db.close()
-    if not history:
-        for ds in dataset_list:
-            setattr(ds, "history", [])
     return dataset_list
 
 

fractal_server-2.14.0a0/fractal_server/app/routes/api/v2/history.py

@@ -0,0 +1,247 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Query
+from fastapi import status
+from fastapi.responses import JSONResponse
+from sqlmodel import func
+from sqlmodel import select
+
+from ._aux_functions import _get_dataset_check_owner
+from ._aux_functions import _get_workflow_check_owner
+from ._aux_functions import _get_workflow_task_check_owner
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.history.status_enum import HistoryItemImageStatus
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import HistoryItemV2
+from fractal_server.app.models.v2 import ImageStatus
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.schemas.v2.history import HistoryItemV2Read
+
+router = APIRouter()
+
+
+@router.get(
+    "/project/{project_id}/dataset/{dataset_id}/history/",
+    response_model=list[HistoryItemV2Read],
+)
+async def get_dataset_history(
+    project_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[HistoryItemV2Read]:
+    await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    stm = (
+        select(HistoryItemV2)
+        .where(HistoryItemV2.dataset_id == dataset_id)
+        .order_by(HistoryItemV2.timestamp_started)
+    )
+    res = await db.execute(stm)
+    items = res.scalars().all()
+    return items
+
+
+@router.get("/project/{project_id}/status/")
+async def get_per_workflow_aggregated_info(
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    wft_ids = [wftask.id for wftask in workflow.task_list]
+
+    # num_available_images
+    stm = (
+        select(
+            HistoryItemV2.workflowtask_id, HistoryItemV2.num_available_images
+        )
+        .where(HistoryItemV2.dataset_id == dataset_id)
+        .where(HistoryItemV2.workflowtask_id.in_(wft_ids))
+        .order_by(
+            HistoryItemV2.workflowtask_id,
+            HistoryItemV2.timestamp_started.desc(),
+        )
+        # https://www.postgresql.org/docs/current/sql-select.html#SQL-DISTINCT
+        .distinct(HistoryItemV2.workflowtask_id)
+    )
+    res = await db.execute(stm)
+    num_available_images = {k: v for k, v in res.all()}
+
+    count = {}
+    for _status in HistoryItemImageStatus:
+        stm = (
+            select(ImageStatus.workflowtask_id, func.count())
+            .where(ImageStatus.dataset_id == dataset_id)
+            .where(ImageStatus.workflowtask_id.in_(wft_ids))
+            .where(ImageStatus.status == _status)
+            # https://docs.sqlalchemy.org/en/20/tutorial/data_select.html#tutorial-group-by-w-aggregates
+            .group_by(ImageStatus.workflowtask_id)
+        )
+        res = await db.execute(stm)
+        count[_status] = {k: v for k, v in res.all()}
+
+    result = {
+        str(_id): None
+        if _id not in num_available_images
+        else {
+            "num_available_images": num_available_images[_id],
+            "num_done_images": count["done"].get(_id, 0),
+            "num_submitted_images": count["submitted"].get(_id, 0),
+            "num_failed_images": count["failed"].get(_id, 0),
+        }
+        for _id in wft_ids
+    }
+
+    return JSONResponse(content=result, status_code=200)
+
+
+@router.get("/project/{project_id}/status/subsets/")
+async def get_per_workflowtask_subsets_aggregated_info(
+    project_id: int,
+    workflowtask_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if wftask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="WorkflowTask not found",
+        )
+    await _get_workflow_task_check_owner(
+        project_id=project_id,
+        workflow_id=wftask.workflow_id,
+        workflow_task_id=workflowtask_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    stm = (
+        select(ImageStatus.parameters_hash, func.array_agg(ImageStatus.status))
+        .where(ImageStatus.dataset_id == dataset_id)
+        .where(ImageStatus.workflowtask_id == workflowtask_id)
+        .group_by(ImageStatus.parameters_hash)
+    )
+    res = await db.execute(stm)
+    hash_statuses = res.all()
+
+    result = []
+    for _hash, statuses in hash_statuses:
+        dump = await db.execute(
+            select(HistoryItemV2.workflowtask_dump)
+            .where(HistoryItemV2.workflowtask_id == workflowtask_id)
+            .where(HistoryItemV2.dataset_id == dataset_id)
+            .where(HistoryItemV2.parameters_hash == _hash)
+        )
+        result.append(
+            {
+                "workflowtask_dump": dump.scalar_one(),
+                "parameters_hash": _hash,
+                "info": {
+                    "num_done_images": statuses.count(
+                        HistoryItemImageStatus.DONE
+                    ),
+                    "num_failed_images": statuses.count(
+                        HistoryItemImageStatus.FAILED
+                    ),
+                    "num_submitted_images": statuses.count(
+                        HistoryItemImageStatus.SUBMITTED
+                    ),
+                },
+            }
+        )
+
+    return JSONResponse(content=result, status_code=200)
+
+
+@router.get("/project/{project_id}/status/images/")
+async def get_per_workflowtask_images(
+    project_id: int,
+    workflowtask_id: int,
+    dataset_id: int,
+    status: HistoryItemImageStatus,
+    parameters_hash: Optional[str] = None,
+    # Pagination
+    page: int = Query(default=1, ge=1),
+    page_size: Optional[int] = Query(default=None, ge=1),
+    # Dependencies
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+
+    if page_size is None and page > 1:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(f"Invalid pagination parameters: {page=}, {page_size=}."),
+        )
+
+    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if wftask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="WorkflowTask not found",
+        )
+    await _get_workflow_task_check_owner(
+        project_id=project_id,
+        workflow_id=wftask.workflow_id,
+        workflow_task_id=workflowtask_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    total_count_stm = (
+        select(func.count(ImageStatus.zarr_url))
+        .where(ImageStatus.dataset_id == dataset_id)
+        .where(ImageStatus.workflowtask_id == workflowtask_id)
+        .where(ImageStatus.status == status)
+    )
+    query = (
+        select(ImageStatus.zarr_url)
+        .where(ImageStatus.dataset_id == dataset_id)
+        .where(ImageStatus.workflowtask_id == workflowtask_id)
+        .where(ImageStatus.status == status)
+    )
+
+    if parameters_hash is not None:
+        total_count_stm = total_count_stm.where(
+            ImageStatus.parameters_hash == parameters_hash
+        )
+        query = query.where(ImageStatus.parameters_hash == parameters_hash)
+
+    if page_size is not None:
+        query = query.limit(page_size)
+    if page > 1:
+        query = query.offset((page - 1) * page_size)
+
+    res_total_count = await db.execute(total_count_stm)
+    total_count = res_total_count.scalar()
+
+    res = await db.execute(query)
+    images = res.scalars().all()
+
+    return {
+        "total_count": total_count,
+        "page_size": page_size,
+        "current_page": page,
+        "images": images,
+    }