fractal-server 2.14.0a6__tar.gz → 2.14.0a8__tar.gz

{fractal_server-2.14.0a6 → fractal_server-2.14.0a8}/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a6
+Version: 2.14.0a8
 Summary: Backend component of the Fractal analytics platform
-Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause
 Author: Tommaso Comparin
 Author-email: tommaso.comparin@exact-lab.it
@@ -29,6 +28,7 @@ Requires-Dist: sqlmodel (==0.0.22)
 Requires-Dist: uvicorn (>=0.29.0,<0.35.0)
 Requires-Dist: uvicorn-worker (==0.3.0)
 Project-URL: Documentation, https://fractal-analytics-platform.github.io/fractal-server
+Project-URL: Homepage, https://github.com/fractal-analytics-platform/fractal-server
 Project-URL: Repository, https://github.com/fractal-analytics-platform/fractal-server
 Project-URL: changelog, https://github.com/fractal-analytics-platform/fractal-server/blob/main/CHANGELOG.md
 Description-Content-Type: text/markdown

fractal_server-2.14.0a8/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a8"

{fractal_server-2.14.0a6 → fractal_server-2.14.0a8}/fractal_server/app/routes/api/v2/_aux_functions.py

@@ -419,77 +419,67 @@ async def clean_app_job_list_v2(
     return submitted_job_ids
 
 
-async def _get_workflow_check_history_owner(
+async def _get_dataset_or_404(
     *,
-    workflow_id: int,
     dataset_id: int,
-    user_id: int,
     db: AsyncSession,
-) -> list[int]:
+) -> DatasetV2:
     """
-    Verify user access for the history of this dataset and workflowtask.
+    Get a dataset or raise 404.
 
     Args:
         dataset_id:
-        workflow_task_id:
-        user_id:
         db:
-
-    Returns:
-        List of WorkflowTask IDs
     """
-    workflow = await db.get(WorkflowV2, workflow_id)
-    if workflow is None:
+    ds = await db.get(DatasetV2, dataset_id)
+    if ds is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
-            detail="Workflow not found.",
+            detail=f"Dataset {dataset_id} not found.",
         )
-    await _get_project_check_owner(
-        project_id=workflow.project_id,
-        user_id=user_id,
-        db=db,
-    )
-    dataset = await db.get(DatasetV2, dataset_id)
-    if dataset is None:
+    else:
+        return ds
+
+
+async def _get_workflow_or_404(
+    *,
+    workflow_id: int,
+    db: AsyncSession,
+) -> WorkflowV2:
+    """
+    Get a workflow or raise 404.
+
+    Args:
+        workflow_id:
+        db:
+    """
+    wf = await db.get(WorkflowV2, workflow_id)
+    if wf is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
-            detail="Dataset not found.",
+            detail=f"Workflow {workflow_id} not found.",
         )
-    if workflow.project_id != dataset.project_id:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail="Dataset and workflow belong to different projects.",
-        )
-
-    return [wftask.id for wftask in workflow.task_list]
+    else:
+        return wf
 
 
-async def _get_workflowtask_check_history_owner(
+async def _get_workflowtask_or_404(
     *,
     workflowtask_id: int,
-    dataset_id: int,
-    user_id: int,
     db: AsyncSession,
 ) -> WorkflowTaskV2:
     """
-    Verify user access for the history of this dataset and workflowtask.
+    Get a workflow task or raise 404.
 
     Args:
-        dataset_id:
-        workflow_task_id:
-        user_id:
+        workflowtask_id:
         db:
     """
-    workflowtask = await db.get(WorkflowTaskV2, workflowtask_id)
-    if workflowtask is None:
+    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if wftask is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
-            detail="WorkflowTask not found.",
+            detail=f"WorkflowTask {workflowtask_id} not found.",
         )
-    await _get_workflow_check_history_owner(
-        workflow_id=workflowtask.workflow_id,
-        dataset_id=dataset_id,
-        user_id=user_id,
-        db=db,
-    )
-    return workflowtask
+    else:
+        return wftask

fractal_server-2.14.0a8/fractal_server/app/routes/api/v2/_aux_functions_history.py

@@ -0,0 +1,158 @@
+from pathlib import Path
+from typing import Literal
+
+from fastapi import HTTPException
+from fastapi import status
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models import WorkflowTaskV2
+from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import HistoryRun
+from fractal_server.app.models.v2 import HistoryUnit
+from fractal_server.app.models.v2 import WorkflowV2
+from fractal_server.app.routes.api.v2._aux_functions import _get_dataset_or_404
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_project_check_owner,
+)
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_workflow_or_404,
+)
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_workflowtask_or_404,
+)
+
+
+async def get_history_unit_or_404(
+    *, history_unit_id: int, db: AsyncSession
+) -> HistoryUnit:
+    """
+    Get an existing HistoryUnit  or raise a 404.
+
+    Arguments:
+        history_unit_id: The `HistoryUnit` id
+        db: An asynchronous db session
+    """
+    history_unit = await db.get(HistoryUnit, history_unit_id)
+    if history_unit is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"HistoryUnit {history_unit_id} not found",
+        )
+    return history_unit
+
+
+async def get_history_run_or_404(
+    *, history_run_id: int, db: AsyncSession
+) -> HistoryRun:
+    """
+    Get an existing HistoryRun  or raise a 404.
+
+    Arguments:
+        history_run_id:
+        db:
+    """
+    history_run = await db.get(HistoryRun, history_run_id)
+    if history_run is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"HistoryRun {history_run_id} not found",
+        )
+    return history_run
+
+
+def read_log_file(
+    *,
+    logfile: str | None,
+    wftask: WorkflowTaskV2,
+    dataset_id: int,
+):
+    if logfile is None or not Path(logfile).exists():
+        return (
+            f"Logs for task '{wftask.task.name}' in dataset "
+            f"{dataset_id} are not available."
+        )
+
+    try:
+        with open(logfile, "r") as f:
+            return f.read()
+    except Exception as e:
+        return (
+            f"Error while retrieving logs for task '{wftask.task.name}' "
+            f"in dataset {dataset_id}. Original error: {str(e)}."
+        )
+
+
+async def _verify_workflow_and_dataset_access(
+    *,
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> dict[Literal["dataset", "workflow"], DatasetV2 | WorkflowV2]:
+    """
+    Verify user access to a dataset/workflow pair.
+
+    Args:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+    """
+    await _get_project_check_owner(
+        project_id=project_id,
+        user_id=user_id,
+        db=db,
+    )
+    workflow = await _get_workflow_or_404(
+        workflow_id=workflow_id,
+        db=db,
+    )
+    if workflow.project_id != project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Workflow does not belong to expected project.",
+        )
+    dataset = await _get_dataset_or_404(
+        dataset_id=dataset_id,
+        db=db,
+    )
+    if dataset.project_id != project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Dataset does not belong to expected project.",
+        )
+
+    return dict(dataset=dataset, workflow=workflow)
+
+
+async def get_wftask_check_owner(
+    *,
+    project_id: int,
+    dataset_id: int,
+    workflowtask_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> WorkflowTaskV2:
+    """
+    Verify user access for the history of this dataset and workflowtask.
+
+    Args:
+        project_id:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+    """
+    wftask = await _get_workflowtask_or_404(
+        workflowtask_id=workflowtask_id,
+        db=db,
+    )
+    await _verify_workflow_and_dataset_access(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        workflow_id=wftask.workflow_id,
+        user_id=user_id,
+        db=db,
+    )
+    return wftask

{fractal_server-2.14.0a6 → fractal_server-2.14.0a8}/fractal_server/app/routes/api/v2/history.py

@@ -1,26 +1,20 @@
-from datetime import datetime
-from typing import Any
-from typing import Optional
-
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import status
 from fastapi.responses import JSONResponse
-from pydantic import AwareDatetime
-from pydantic import BaseModel
-from pydantic import field_serializer
 from sqlmodel import func
 from sqlmodel import select
 
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
-from ._aux_functions import _get_workflowtask_check_history_owner
+from ._aux_functions_history import _verify_workflow_and_dataset_access
+from ._aux_functions_history import get_history_run_or_404
 from ._aux_functions_history import get_history_unit_or_404
+from ._aux_functions_history import get_wftask_check_owner
 from ._aux_functions_history import read_log_file
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
-from fractal_server.app.history.status_enum import XXXStatus
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
@@ -29,6 +23,11 @@ from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.pagination import get_pagination_params
 from fractal_server.app.routes.pagination import PaginationRequest
 from fractal_server.app.routes.pagination import PaginationResponse
+from fractal_server.app.schemas.v2 import HistoryRunReadAggregated
+from fractal_server.app.schemas.v2 import HistoryUnitRead
+from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.app.schemas.v2 import ImageLogsRequest
+from fractal_server.app.schemas.v2 import ZarrUrlAndStatus
 from fractal_server.images.tools import filter_image_list
 from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
@@ -45,12 +44,21 @@ async def get_workflow_tasks_statuses(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
+
+    # Access control
     workflow = await _get_workflow_check_owner(
         project_id=project_id,
         workflow_id=workflow_id,
         user_id=user.id,
         db=db,
     )
+    await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+
     response = {}
     for wftask in workflow.task_list:
         res = await db.execute(
@@ -69,7 +77,7 @@ async def get_workflow_tasks_statuses(
             num_available_images=latest_history_run.num_available_images,
         )
 
-        for target_status in XXXStatus:
+        for target_status in HistoryUnitStatus:
             stm = (
                 select(func.count(HistoryImageCache.zarr_url))
                 .join(HistoryUnit)
@@ -89,43 +97,6 @@ async def get_workflow_tasks_statuses(
     return JSONResponse(content=response, status_code=200)
 
 
-# FIXME MOVE TO SCHEMAS
-
-
-class HistoryUnitRead(BaseModel):
-    id: int
-    logfile: Optional[str] = None
-    status: XXXStatus
-    zarr_urls: list[str]
-
-
-class HistoryRunReadAggregated(BaseModel):
-    id: int
-    timestamp_started: AwareDatetime
-    workflowtask_dump: dict[str, Any]
-    num_submitted_units: int
-    num_done_units: int
-    num_failed_units: int
-
-    @field_serializer("timestamp_started")
-    def serialize_datetime(v: datetime) -> str:
-        return v.isoformat()
-
-
-class ImageLogsRequest(BaseModel):
-    workflowtask_id: int
-    dataset_id: int
-    zarr_url: str
-
-
-class ImageWithStatus(BaseModel):
-    zarr_url: str
-    status: Optional[XXXStatus] = None
-
-
-# end FIXME
-
-
 @router.get("/project/{project_id}/status/run/")
 async def get_history_run_list(
     project_id: int,
@@ -134,8 +105,10 @@ async def get_history_run_list(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[HistoryRunReadAggregated]:
+
     # Access control
-    await _get_workflowtask_check_history_owner(
+    await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,
@@ -152,11 +125,11 @@ async def get_history_run_list(
     res = await db.execute(stm)
     runs = res.scalars().all()
 
-    # Add units count by status
-
+    # Respond early if there are no runs
     if not runs:
         return []
 
+    # Add units count by status
     run_ids = [run.id for run in runs]
     stm = (
         select(
@@ -196,30 +169,29 @@ async def get_history_run_units(
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
 ) -> PaginationResponse[HistoryUnitRead]:
+
     # Access control
-    await _get_workflowtask_check_history_owner(
+    await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
 
-    history_run = await db.get(HistoryRun, history_run_id)
-    if history_run is None:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"HistoryRun {history_run_id} not found",
-        )
+    # Check that `HistoryRun` exists
+    await get_history_run_or_404(history_run_id=history_run_id, db=db)
 
+    # Count `HistoryUnit`s
     res = await db.execute(
         select(func.count(HistoryUnit.id)).where(
             HistoryUnit.history_run_id == history_run_id
         )
     )
     total_count = res.scalar()
-
     page_size = pagination.page_size or total_count
 
+    # Query `HistoryUnit`s
     res = await db.execute(
         select(HistoryUnit)
         .where(HistoryUnit.history_run_id == history_run_id)
@@ -244,30 +216,27 @@ async def get_history_images(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
-) -> PaginationResponse[ImageWithStatus]:
+) -> PaginationResponse[ZarrUrlAndStatus]:
+
     # Access control and object retrieval
-    # FIXME: Provide a single function that checks/gets what is needed
-    res = await _get_dataset_check_owner(
+    wftask = await get_wftask_check_owner(
         project_id=project_id,
-        dataset_id=dataset_id,
-        user_id=user.id,
-        db=db,
-    )
-    dataset = res["dataset"]
-    wftask = await _get_workflowtask_check_history_owner(
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
-    workflow = await _get_workflow_check_owner(
+    res = await _verify_workflow_and_dataset_access(
         project_id=project_id,
         workflow_id=wftask.workflow_id,
+        dataset_id=dataset_id,
         user_id=user.id,
         db=db,
     )
+    dataset = res["dataset"]
+    workflow = res["workflow"]
 
-    # FIXME reduce logging?
+    # Setup prefix for logging
     prefix = f"[DS{dataset.id}-WFT{wftask.id}-images]"
 
     # (1) Get the type-filtered list of dataset images
@@ -359,7 +328,8 @@ async def get_image_log(
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
     # Access control
-    wftask = await _get_workflowtask_check_history_owner(
+    wftask = await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=request_data.dataset_id,
         workflowtask_id=request_data.workflowtask_id,
         user_id=user.id,
@@ -406,7 +376,8 @@ async def get_history_unit_log(
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
     # Access control
-    wftask = await _get_workflowtask_check_history_owner(
+    wftask = await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,

{fractal_server-2.14.0a6 → fractal_server-2.14.0a8}/fractal_server/app/routes/api/v2/job.py

@@ -5,6 +5,7 @@ from typing import Optional
 
 from fastapi import APIRouter
 from fastapi import Depends
+from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from fastapi.responses import StreamingResponse
@@ -83,6 +84,35 @@ async def get_workflow_jobs(
     return job_list
 
 
+@router.get("/project/{project_id}/latest-job/")
+async def get_latest_job(
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JobReadV2:
+    await _get_workflow_check_owner(
+        project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
+    )
+    stm = (
+        select(JobV2)
+        .where(JobV2.project_id == project_id)
+        .where(JobV2.workflow_id == workflow_id)
+        .where(JobV2.dataset_id == dataset_id)
+        .order_by(JobV2.start_timestamp.desc())
+        .limit(1)
+    )
+    res = await db.execute(stm)
+    latest_job = res.scalar_one_or_none()
+    if latest_job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job with {workflow_id=} and {dataset_id=} not found.",
+        )
+    return latest_job
+
+
 @router.get(
     "/project/{project_id}/job/{job_id}/",
     response_model=JobReadV2,

{fractal_server-2.14.0a6 → fractal_server-2.14.0a8}/fractal_server/app/routes/api/v2/status_legacy.py

@@ -9,8 +9,8 @@ from .....logger import set_logger
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import JobV2
-from ....schemas.v2.status import StatusReadV2
-from ....schemas.v2.workflowtask import WorkflowTaskStatusTypeV2
+from ....schemas.v2.status_legacy import LegacyStatusReadV2
+from ....schemas.v2.status_legacy import WorkflowTaskStatusTypeV2
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
@@ -24,7 +24,7 @@ logger = set_logger(__name__)
 
 @router.get(
     "/project/{project_id}/status-legacy/",
-    response_model=StatusReadV2,
+    response_model=LegacyStatusReadV2,
 )
 async def get_workflowtask_status(
     project_id: int,
@@ -32,7 +32,7 @@ async def get_workflowtask_status(
     workflow_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[StatusReadV2]:
+) -> Optional[LegacyStatusReadV2]:
     """
     Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
     on a given `DatasetV2`.
@@ -164,5 +164,5 @@ async def get_workflowtask_status(
             # first time that you hit `last_valid_wftask_id``
             break
 
-    response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
+    response_body = LegacyStatusReadV2(status=clean_workflow_tasks_status_dict)
     return response_body

{fractal_server-2.14.0a6 → fractal_server-2.14.0a8}/fractal_server/app/routes/api/v2/task.py

@@ -152,14 +152,7 @@ async def create_task(
         db=db,
     )
 
-    if task.command_non_parallel is None:
-        task_type = "parallel"
-    elif task.command_parallel is None:
-        task_type = "non_parallel"
-    else:
-        task_type = "compound"
-
-    if task_type == "parallel" and (
+    if task.type == "parallel" and (
         task.args_schema_non_parallel is not None
         or task.meta_non_parallel is not None
     ):
@@ -170,7 +163,7 @@ async def create_task(
                 "`TaskV2.args_schema_non_parallel` if TaskV2 is parallel"
             ),
         )
-    elif task_type == "non_parallel" and (
+    elif task.type == "non_parallel" and (
         task.args_schema_parallel is not None or task.meta_parallel is not None
     ):
         raise HTTPException(
@@ -183,7 +176,7 @@ async def create_task(
 
     # Add task
 
-    db_task = TaskV2(**task.model_dump(exclude_unset=True), type=task_type)
+    db_task = TaskV2(**task.model_dump(exclude_unset=True))
     pkg_name = db_task.name
     await _verify_non_duplication_user_constraint(
         db=db, pkg_name=pkg_name, user_id=user.id, version=db_task.version

{fractal_server-2.14.0a6 → fractal_server-2.14.0a8}/fractal_server/app/routes/api/v2/workflowtask.py

@@ -56,7 +56,14 @@ async def replace_workflowtask(
     )
 
     # Preliminary checks
-    if old_wftask.task_type != new_task.type:
+    EQUIVALENT_TASK_TYPES = [
+        {"non_parallel", "converter_non_parallel"},
+        {"compound", "converter_compound"},
+    ]
+    if (
+        old_wftask.task_type != new_task.type
+        and {old_wftask.task_type, new_task.type} not in EQUIVALENT_TASK_TYPES
+    ):
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
@@ -64,6 +71,7 @@ async def replace_workflowtask(
                 f"{old_wftask.task_type} to {new_task.type}."
             ),
         )
+
     if replace.args_non_parallel is not None and new_task.type == "parallel":
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -259,7 +267,10 @@ async def update_workflowtask(
                 "parallel."
             ),
         )
-    elif db_wf_task.task_type == "non_parallel" and (
+    elif db_wf_task.task_type in [
+        "non_parallel",
+        "converter_non_parallel",
+    ] and (
         workflow_task_update.args_parallel is not None
         or workflow_task_update.meta_parallel is not None
     ):