fractal-server 2.14.0a3__py3-none-any.whl → 2.14.0a5__py3-none-any.whl

fractal_server/app/routes/api/v2/history.py

@@ -1,4 +1,5 @@
-from pathlib import Path
+from datetime import datetime
+from typing import Any
 from typing import Optional
 
 from fastapi import APIRouter
@@ -6,126 +7,196 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import status
 from fastapi.responses import JSONResponse
+from pydantic import AwareDatetime
 from pydantic import BaseModel
+from pydantic import field_serializer
 from sqlmodel import func
 from sqlmodel import select
 
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import _get_workflowtask_check_history_owner
+from ._aux_functions_history import get_history_unit_or_404
+from ._aux_functions_history import read_log_file
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
-from fractal_server.app.history.status_enum import HistoryItemImageStatus
+from fractal_server.app.history.status_enum import XXXStatus
 from fractal_server.app.models import UserOAuth
-from fractal_server.app.models.v2 import HistoryItemV2
-from fractal_server.app.models.v2 import ImageStatus
+from fractal_server.app.models.v2 import HistoryImageCache
+from fractal_server.app.models.v2 import HistoryRun
+from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.pagination import get_pagination_params
 from fractal_server.app.routes.pagination import PaginationRequest
 from fractal_server.app.routes.pagination import PaginationResponse
-from fractal_server.app.schemas.v2.history import HistoryItemV2Read
+from fractal_server.images.tools import filter_image_list
+from fractal_server.images.tools import merge_type_filters
+from fractal_server.logger import set_logger
 
 router = APIRouter()
+logger = set_logger(__name__)
 
 
-@router.get(
-    "/project/{project_id}/dataset/{dataset_id}/history/",
-    response_model=list[HistoryItemV2Read],
-)
-async def get_dataset_history(
+@router.get("/project/{project_id}/status/")
+async def get_workflow_tasks_statuses(
     project_id: int,
     dataset_id: int,
+    workflow_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> list[HistoryItemV2Read]:
-    await _get_dataset_check_owner(
+) -> JSONResponse:
+    workflow = await _get_workflow_check_owner(
         project_id=project_id,
-        dataset_id=dataset_id,
+        workflow_id=workflow_id,
         user_id=user.id,
         db=db,
     )
+    response = {}
+    for wftask in workflow.task_list:
+        res = await db.execute(
+            select(HistoryRun)
+            .where(HistoryRun.dataset_id == dataset_id)
+            .where(HistoryRun.workflowtask_id == wftask.id)
+            .order_by(HistoryRun.timestamp_started.desc())
+            .limit(1)
+        )
+        latest_history_run = res.scalar()
+        if not latest_history_run:
+            response[wftask.id] = None
+            continue
+        response[wftask.id] = dict(
+            status=latest_history_run.status,
+            num_available_images=latest_history_run.num_available_images,
+        )
 
-    stm = (
-        select(HistoryItemV2)
-        .where(HistoryItemV2.dataset_id == dataset_id)
-        .order_by(HistoryItemV2.timestamp_started)
-    )
-    res = await db.execute(stm)
-    items = res.scalars().all()
-    return items
+        for target_status in XXXStatus:
+            stm = (
+                select(func.count(HistoryImageCache.zarr_url))
+                .join(HistoryUnit)
+                .where(HistoryImageCache.dataset_id == dataset_id)
+                .where(HistoryImageCache.workflowtask_id == wftask.id)
+                .where(
+                    HistoryImageCache.latest_history_unit_id == HistoryUnit.id
+                )
+                .where(HistoryUnit.status == target_status.value)
+            )
+            res = await db.execute(stm)
+            num_images = res.scalar()
+            response[wftask.id][
+                f"num_{target_status.value}_images"
+            ] = num_images
 
+    return JSONResponse(content=response, status_code=200)
 
-@router.get("/project/{project_id}/status/")
-async def get_per_workflow_aggregated_info(
+
+# FIXME MOVE TO SCHEMAS
+
+
+class HistoryUnitRead(BaseModel):
+    id: int
+    logfile: Optional[str] = None
+    status: XXXStatus
+    zarr_urls: list[str]
+
+
+class HistoryRunReadAggregated(BaseModel):
+    id: int
+    timestamp_started: AwareDatetime
+    workflowtask_dump: dict[str, Any]
+    num_submitted_units: int
+    num_done_units: int
+    num_failed_units: int
+
+    @field_serializer("timestamp_started")
+    def serialize_datetime(v: datetime) -> str:
+        return v.isoformat()
+
+
+class ImageLogsRequest(BaseModel):
+    workflowtask_id: int
+    dataset_id: int
+    zarr_url: str
+
+
+class ImageWithStatus(BaseModel):
+    zarr_url: str
+    status: Optional[XXXStatus] = None
+
+
+# end FIXME
+
+
+@router.get("/project/{project_id}/status/run/")
+async def get_history_run_list(
     project_id: int,
-    workflow_id: int,
     dataset_id: int,
+    workflowtask_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> JSONResponse:
-    workflow = await _get_workflow_check_owner(
-        project_id=project_id,
-        workflow_id=workflow_id,
+) -> list[HistoryRunReadAggregated]:
+    # Access control
+    await _get_workflowtask_check_history_owner(
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
 
-    wft_ids = [wftask.id for wftask in workflow.task_list]
+    # Get all runs
+    stm = (
+        select(HistoryRun)
+        .where(HistoryRun.dataset_id == dataset_id)
+        .where(HistoryRun.workflowtask_id == workflowtask_id)
+        .order_by(HistoryRun.timestamp_started)
+    )
+    res = await db.execute(stm)
+    runs = res.scalars().all()
+
+    # Add units count by status
+
+    if not runs:
+        return []
 
-    # num_available_images
+    run_ids = [run.id for run in runs]
     stm = (
         select(
-            HistoryItemV2.workflowtask_id, HistoryItemV2.num_available_images
+            HistoryUnit.history_run_id,
+            HistoryUnit.status,
+            func.count(HistoryUnit.id),
         )
-        .where(HistoryItemV2.dataset_id == dataset_id)
-        .where(HistoryItemV2.workflowtask_id.in_(wft_ids))
-        .order_by(
-            HistoryItemV2.workflowtask_id,
-            HistoryItemV2.timestamp_started.desc(),
-        )
-        # https://www.postgresql.org/docs/current/sql-select.html#SQL-DISTINCT
-        .distinct(HistoryItemV2.workflowtask_id)
+        .where(HistoryUnit.history_run_id.in_(run_ids))
+        .group_by(HistoryUnit.history_run_id, HistoryUnit.status)
     )
     res = await db.execute(stm)
-    num_available_images = {k: v for k, v in res.all()}
-
-    count = {}
-    for _status in HistoryItemImageStatus:
-        stm = (
-            select(ImageStatus.workflowtask_id, func.count())
-            .where(ImageStatus.dataset_id == dataset_id)
-            .where(ImageStatus.workflowtask_id.in_(wft_ids))
-            .where(ImageStatus.status == _status)
-            # https://docs.sqlalchemy.org/en/20/tutorial/data_select.html#tutorial-group-by-w-aggregates
-            .group_by(ImageStatus.workflowtask_id)
-        )
-        res = await db.execute(stm)
-        count[_status] = {k: v for k, v in res.all()}
-
-    result = {
-        str(_id): None
-        if _id not in num_available_images
-        else {
-            "num_available_images": num_available_images[_id],
-            "num_done_images": count["done"].get(_id, 0),
-            "num_submitted_images": count["submitted"].get(_id, 0),
-            "num_failed_images": count["failed"].get(_id, 0),
+    unit_counts = res.all()
+
+    count_map = {
+        run_id: {
+            "num_done_units": 0,
+            "num_submitted_units": 0,
+            "num_failed_units": 0,
         }
-        for _id in wft_ids
+        for run_id in run_ids
     }
+    for run_id, unit_status, count in unit_counts:
+        count_map[run_id][f"num_{unit_status}_units"] = count
+
+    runs = [dict(**run.model_dump(), **count_map[run.id]) for run in runs]
 
-    return JSONResponse(content=result, status_code=200)
+    return runs
 
 
-@router.get("/project/{project_id}/status/subsets/")
-async def get_per_workflowtask_subsets_aggregated_info(
+@router.get("/project/{project_id}/status/run/{history_run_id}/units/")
+async def get_history_run_units(
     project_id: int,
-    workflowtask_id: int,
     dataset_id: int,
+    workflowtask_id: int,
+    history_run_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> JSONResponse:
-
+    pagination: PaginationRequest = Depends(get_pagination_params),
+) -> PaginationResponse[HistoryUnitRead]:
+    # Access control
     await _get_workflowtask_check_history_owner(
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
@@ -133,133 +204,161 @@ async def get_per_workflowtask_subsets_aggregated_info(
         db=db,
     )
 
-    stm = (
-        select(ImageStatus.parameters_hash, func.array_agg(ImageStatus.status))
-        .where(ImageStatus.dataset_id == dataset_id)
-        .where(ImageStatus.workflowtask_id == workflowtask_id)
-        .group_by(ImageStatus.parameters_hash)
-    )
-    res = await db.execute(stm)
-    hash_to_statuses = res.all()
-
-    subsets = []
-    for parameters_hash, statuses in hash_to_statuses:
-        # Get the oldest HistoryItemV2 matching with `parameters_hash`
-        stm = (
-            select(HistoryItemV2)
-            .where(HistoryItemV2.workflowtask_id == workflowtask_id)
-            .where(HistoryItemV2.dataset_id == dataset_id)
-            .where(HistoryItemV2.parameters_hash == parameters_hash)
-            .order_by(HistoryItemV2.timestamp_started)
-            .limit(1)
+    history_run = await db.get(HistoryRun, history_run_id)
+    if history_run is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"HistoryRun {history_run_id} not found",
         )
-        res = await db.execute(stm)
-        oldest_history_item = res.scalar_one()
-
-        subsets.append(
-            {
-                "_timestamp": oldest_history_item.timestamp_started,
-                "workflowtask_dump": oldest_history_item.workflowtask_dump,
-                "parameters_hash": parameters_hash,
-                "info": {
-                    "num_done_images": statuses.count(
-                        HistoryItemImageStatus.DONE
-                    ),
-                    "num_failed_images": statuses.count(
-                        HistoryItemImageStatus.FAILED
-                    ),
-                    "num_submitted_images": statuses.count(
-                        HistoryItemImageStatus.SUBMITTED
-                    ),
-                },
-            }
+
+    res = await db.execute(
+        select(func.count(HistoryUnit.id)).where(
+            HistoryUnit.history_run_id == history_run_id
         )
+    )
+    total_count = res.scalar()
+
+    page_size = pagination.page_size or total_count
 
-    # Use `_timestamp` values for sorting, and then drop them from the response
-    sorted_results = sorted(subsets, key=lambda obj: obj["_timestamp"])
-    [item.pop("_timestamp") for item in sorted_results]
+    res = await db.execute(
+        select(HistoryUnit)
+        .where(HistoryUnit.history_run_id == history_run_id)
+        .offset((pagination.page - 1) * page_size)
+        .limit(page_size)
+    )
+    units = res.scalars().all()
 
-    return JSONResponse(content=sorted_results, status_code=200)
+    return dict(
+        current_page=pagination.page,
+        page_size=page_size,
+        total_count=total_count,
+        items=units,
+    )
 
 
 @router.get("/project/{project_id}/status/images/")
-async def get_per_workflowtask_images(
+async def get_history_images(
     project_id: int,
-    workflowtask_id: int,
     dataset_id: int,
-    status: HistoryItemImageStatus,
-    parameters_hash: Optional[str] = None,
-    # Dependencies
-    pagination: PaginationRequest = Depends(get_pagination_params),
+    workflowtask_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> PaginationResponse[str]:
-
-    page = pagination.page
-    page_size = pagination.page_size
-
-    await _get_workflowtask_check_history_owner(
+    pagination: PaginationRequest = Depends(get_pagination_params),
+) -> PaginationResponse[ImageWithStatus]:
+    # Access control and object retrieval
+    # FIXME: Provide a single function that checks/gets what is needed
+    res = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    dataset = res["dataset"]
+    wftask = await _get_workflowtask_check_history_owner(
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id,
+        workflow_id=wftask.workflow_id,
+        user_id=user.id,
+        db=db,
+    )
 
-    total_count_stm = (
-        select(func.count(ImageStatus.zarr_url))
-        .where(ImageStatus.dataset_id == dataset_id)
-        .where(ImageStatus.workflowtask_id == workflowtask_id)
-        .where(ImageStatus.status == status)
+    # FIXME reduce logging?
+    prefix = f"[DS{dataset.id}-WFT{wftask.id}-images]"
+
+    # (1) Get the type-filtered list of dataset images
+
+    # (1A) Reconstruct dataset type filters by starting from {} and making
+    # incremental updates with `output_types` of all previous tasks
+    inferred_dataset_type_filters = {}
+    for current_wftask in workflow.task_list[0 : wftask.order]:
+        inferred_dataset_type_filters.update(current_wftask.task.output_types)
+    logger.debug(f"{prefix} {inferred_dataset_type_filters=}")
+    # (1B) Compute type filters for the current wftask
+    type_filters_patch = merge_type_filters(
+        task_input_types=wftask.task.input_types,
+        wftask_type_filters=wftask.type_filters,
     )
-    query = (
-        select(ImageStatus.zarr_url)
-        .where(ImageStatus.dataset_id == dataset_id)
-        .where(ImageStatus.workflowtask_id == workflowtask_id)
-        .where(ImageStatus.status == status)
+    logger.debug(f"{prefix} {type_filters_patch=}")
+    # (1C) Combine dataset type filters (lower priority) and current-wftask
+    # filters (higher priority)
+    actual_filters = inferred_dataset_type_filters
+    actual_filters.update(type_filters_patch)
+    logger.debug(f"{prefix} {actual_filters=}")
+    # (1D) Get all matching images from the dataset
+    filtered_dataset_images = filter_image_list(
+        images=dataset.images,
+        type_filters=inferred_dataset_type_filters,
+    )
+    logger.debug(f"{prefix} {len(dataset.images)=}")
+    logger.debug(f"{prefix} {len(filtered_dataset_images)=}")
+    # (1E) Extract the list of URLs for filtered images
+    filtered_dataset_images_url = list(
+        img["zarr_url"] for img in filtered_dataset_images
     )
 
-    if parameters_hash is not None:
-        total_count_stm = total_count_stm.where(
-            ImageStatus.parameters_hash == parameters_hash
-        )
-        query = query.where(ImageStatus.parameters_hash == parameters_hash)
-
-    res_total_count = await db.execute(total_count_stm)
-    total_count = res_total_count.scalar()
+    # (2) Get `(zarr_url, status)` pairs for all images that have already
+    # been processed
+    res = await db.execute(
+        select(HistoryImageCache.zarr_url, HistoryUnit.status)
+        .join(HistoryUnit)
+        .where(HistoryImageCache.dataset_id == dataset_id)
+        .where(HistoryImageCache.workflowtask_id == workflowtask_id)
+        .where(HistoryImageCache.latest_history_unit_id == HistoryUnit.id)
+        .where(HistoryImageCache.zarr_url.in_(filtered_dataset_images_url))
+        .order_by(HistoryImageCache.zarr_url)
+    )
+    list_processed_url_status = res.all()
+    logger.debug(f"{prefix} {len(list_processed_url_status)=}")
 
-    if page_size is not None:
-        query = query.limit(page_size)
-    else:
-        page_size = total_count
+    # (3) Combine outputs from 1 and 2
+    list_processed_url = list(item[0] for item in list_processed_url_status)
+    logger.debug(f"{prefix} {len(list_processed_url)=}")
 
-    if page > 1:
-        query = query.offset((page - 1) * page_size)
+    list_non_processed_url_status = list(
+        (url, None)
+        for url in filtered_dataset_images_url
+        if url not in list_processed_url
+    )
+    logger.debug(f"{prefix} {len(list_non_processed_url_status)=}")
 
-    res = await db.execute(query)
-    images = res.scalars().all()
+    sorted_list_url_status = sorted(
+        list_processed_url_status + list_non_processed_url_status,
+        key=lambda url_status: url_status[0],
+    )
+    logger.debug(f"{prefix} {len(sorted_list_url_status)=}")
 
-    return PaginationResponse[str](
-        total_count=total_count,
-        page_size=page_size,
-        current_page=page,
-        items=images,
+    # Final list of objects
+    sorted_list_objects = list(
+        dict(zarr_url=url_status[0], status=url_status[1])
+        for url_status in sorted_list_url_status
     )
 
+    total_count = len(sorted_list_objects)
+    page_size = pagination.page_size or total_count
 
-class ImageLogsRequest(BaseModel):
-    workflowtask_id: int
-    dataset_id: int
-    zarr_url: str
+    return dict(
+        current_page=pagination.page,
+        page_size=page_size,
+        total_count=total_count,
+        items=sorted_list_objects[
+            (pagination.page - 1) * page_size : pagination.page * page_size
+        ],
+    )
 
 
-@router.post("/project/{project_id}/status/image-logs/")
-async def get_image_logs(
+@router.post("/project/{project_id}/status/image-log/")
+async def get_image_log(
     project_id: int,
     request_data: ImageLogsRequest,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
-
+    # Access control
     wftask = await _get_workflowtask_check_history_owner(
         dataset_id=request_data.dataset_id,
         workflowtask_id=request_data.workflowtask_id,
@@ -267,39 +366,63 @@ async def get_image_logs(
         db=db,
     )
 
-    image_status = await db.get(
-        ImageStatus,
+    # Get HistoryImageCache
+    history_image_cache = await db.get(
+        HistoryImageCache,
         (
             request_data.zarr_url,
-            request_data.workflowtask_id,
             request_data.dataset_id,
+            request_data.workflowtask_id,
         ),
     )
-    if image_status is None:
+    if history_image_cache is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
-            detail="ImageStatus not found",
+            detail="HistoryImageCache not found",
         )
+    # Get history unit
+    history_unit = await get_history_unit_or_404(
+        history_unit_id=history_image_cache.latest_history_unit_id,
+        db=db,
+    )
 
-    if image_status.logfile is None:
-        return JSONResponse(
-            content=(
-                f"Logs for task '{wftask.task.name}' in dataset "
-                f"{request_data.dataset_id} are not yet available."
-            )
-        )
+    # Get log or placeholder text
+    log = read_log_file(
+        logfile=history_unit.logfile,
+        wftask=wftask,
+        dataset_id=request_data.dataset_id,
+    )
+    return JSONResponse(content=log)
 
-    logfile = Path(image_status.logfile)
-    if not logfile.exists():
-        return JSONResponse(
-            content=(
-                f"Error while retrieving logs for task '{wftask.task.name}' "
-                f"in dataset {request_data.dataset_id}: "
-                f"file '{logfile}' is not available."
-            )
-        )
 
-    with logfile.open("r") as f:
-        file_contents = f.read()
+@router.get("/project/{project_id}/status/unit-log/")
+async def get_history_unit_log(
+    project_id: int,
+    history_run_id: int,
+    history_unit_id: int,
+    workflowtask_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+    # Access control
+    wftask = await _get_workflowtask_check_history_owner(
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    # Get history unit
+    history_unit = await get_history_unit_or_404(
+        history_unit_id=history_unit_id,
+        db=db,
+    )
 
-    return JSONResponse(content=file_contents)
+    # Get log or placeholder text
+    log = read_log_file(
+        logfile=history_unit.logfile,
+        wftask=wftask,
+        dataset_id=dataset_id,
+    )
+    return JSONResponse(content=log)

fractal_server/app/routes/api/v2/project.py

@@ -5,7 +5,6 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
-from sqlmodel import delete
 from sqlmodel import select
 
 from .....logger import reset_logger_handlers
@@ -13,8 +12,6 @@ from .....logger import set_logger
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import DatasetV2
-from ....models.v2 import HistoryItemV2
-from ....models.v2 import ImageStatus
 from ....models.v2 import JobV2
 from ....models.v2 import LinkUserProjectV2
 from ....models.v2 import ProjectV2
@@ -164,22 +161,6 @@ async def delete_project(
         for job in jobs:
             logger.info(f"Setting Job[{job.id}].workflow_id to None.")
             job.workflow_id = None
-        # Cascade operations: set foreign-keys to null for history items
-        # which are in relationship with the current workflow
-        wft_ids = [wft.id for wft in wf.task_list]
-        stm = select(HistoryItemV2).where(
-            HistoryItemV2.workflowtask_id.in_(wft_ids)
-        )
-        res = await db.execute(stm)
-        history_items = res.scalars().all()
-        for history_item in history_items:
-            history_item.workflowtask_id = None
-        # Cascade operations: delete all image status which are in relationship
-        # with the current workflow
-        stm = delete(ImageStatus).where(
-            ImageStatus.workflowtask_id.in_(wft_ids)
-        )
-        await db.execute(stm)
         # Delete workflow
         logger.info(f"Adding Workflow[{wf.id}] to deletion.")
         await db.delete(wf)
@@ -199,12 +180,6 @@ async def delete_project(
         for job in jobs:
             logger.info(f"Setting Job[{job.id}].dataset_id to None.")
             job.dataset_id = None
-        # Cascade operations: delete history items and image statuses which are
-        # in relationship with the current dataset
-        stm = delete(HistoryItemV2).where(HistoryItemV2.dataset_id == ds.id)
-        await db.execute(stm)
-        stm = delete(ImageStatus).where(ImageStatus.dataset_id == ds.id)
-        await db.execute(stm)
         # Delete dataset
         logger.info(f"Adding Dataset[{ds.id}] to deletion.")
         await db.delete(ds)