fractal-server 2.14.0a7__py3-none-any.whl → 2.14.0a8__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a7"
+__VERSION__ = "2.14.0a8"

fractal_server/app/history/__init__.py

@@ -1,4 +0,0 @@
-# from .image_updates import update_all_images  # noqa: F401
-# from .image_updates import update_single_image  # noqa
-# from .image_updates import update_single_image_logfile  # noqa
-# from .status_enum import XXXStatus  # noqa: F401

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -419,77 +419,67 @@ async def clean_app_job_list_v2(
     return submitted_job_ids
 
 
-async def _get_workflow_check_history_owner(
+async def _get_dataset_or_404(
     *,
-    workflow_id: int,
     dataset_id: int,
-    user_id: int,
     db: AsyncSession,
-) -> list[int]:
+) -> DatasetV2:
     """
-    Verify user access for the history of this dataset and workflowtask.
+    Get a dataset or raise 404.
 
     Args:
         dataset_id:
-        workflow_task_id:
-        user_id:
         db:
-
-    Returns:
-        List of WorkflowTask IDs
     """
-    workflow = await db.get(WorkflowV2, workflow_id)
-    if workflow is None:
+    ds = await db.get(DatasetV2, dataset_id)
+    if ds is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
-            detail="Workflow not found.",
+            detail=f"Dataset {dataset_id} not found.",
         )
-    await _get_project_check_owner(
-        project_id=workflow.project_id,
-        user_id=user_id,
-        db=db,
-    )
-    dataset = await db.get(DatasetV2, dataset_id)
-    if dataset is None:
+    else:
+        return ds
+
+
+async def _get_workflow_or_404(
+    *,
+    workflow_id: int,
+    db: AsyncSession,
+) -> WorkflowV2:
+    """
+    Get a workflow or raise 404.
+
+    Args:
+        workflow_id:
+        db:
+    """
+    wf = await db.get(WorkflowV2, workflow_id)
+    if wf is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
-            detail="Dataset not found.",
+            detail=f"Workflow {workflow_id} not found.",
         )
-    if workflow.project_id != dataset.project_id:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail="Dataset and workflow belong to different projects.",
-        )
-
-    return [wftask.id for wftask in workflow.task_list]
+    else:
+        return wf
 
 
-async def _get_workflowtask_check_history_owner(
+async def _get_workflowtask_or_404(
     *,
     workflowtask_id: int,
-    dataset_id: int,
-    user_id: int,
     db: AsyncSession,
 ) -> WorkflowTaskV2:
     """
-    Verify user access for the history of this dataset and workflowtask.
+    Get a workflow task or raise 404.
 
     Args:
-        dataset_id:
-        workflow_task_id:
-        user_id:
+        workflowtask_id:
         db:
     """
-    workflowtask = await db.get(WorkflowTaskV2, workflowtask_id)
-    if workflowtask is None:
+    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if wftask is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
-            detail="WorkflowTask not found.",
+            detail=f"WorkflowTask {workflowtask_id} not found.",
         )
-    await _get_workflow_check_history_owner(
-        workflow_id=workflowtask.workflow_id,
-        dataset_id=dataset_id,
-        user_id=user_id,
-        db=db,
-    )
-    return workflowtask
+    else:
+        return wftask

fractal_server/app/routes/api/v2/_aux_functions_history.py

@@ -1,11 +1,25 @@
 from pathlib import Path
+from typing import Literal
 
 from fastapi import HTTPException
 from fastapi import status
 
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.models import WorkflowTaskV2
+from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
+from fractal_server.app.models.v2 import WorkflowV2
+from fractal_server.app.routes.api.v2._aux_functions import _get_dataset_or_404
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_project_check_owner,
+)
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_workflow_or_404,
+)
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_workflowtask_or_404,
+)
 
 
 async def get_history_unit_or_404(
@@ -27,6 +41,25 @@ async def get_history_unit_or_404(
     return history_unit
 
 
+async def get_history_run_or_404(
+    *, history_run_id: int, db: AsyncSession
+) -> HistoryRun:
+    """
+    Get an existing HistoryRun  or raise a 404.
+
+    Arguments:
+        history_run_id:
+        db:
+    """
+    history_run = await db.get(HistoryRun, history_run_id)
+    if history_run is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"HistoryRun {history_run_id} not found",
+        )
+    return history_run
+
+
 def read_log_file(
     *,
     logfile: str | None,
@@ -47,3 +80,79 @@ def read_log_file(
             f"Error while retrieving logs for task '{wftask.task.name}' "
             f"in dataset {dataset_id}. Original error: {str(e)}."
         )
+
+
+async def _verify_workflow_and_dataset_access(
+    *,
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> dict[Literal["dataset", "workflow"], DatasetV2 | WorkflowV2]:
+    """
+    Verify user access to a dataset/workflow pair.
+
+    Args:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+    """
+    await _get_project_check_owner(
+        project_id=project_id,
+        user_id=user_id,
+        db=db,
+    )
+    workflow = await _get_workflow_or_404(
+        workflow_id=workflow_id,
+        db=db,
+    )
+    if workflow.project_id != project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Workflow does not belong to expected project.",
+        )
+    dataset = await _get_dataset_or_404(
+        dataset_id=dataset_id,
+        db=db,
+    )
+    if dataset.project_id != project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Dataset does not belong to expected project.",
+        )
+
+    return dict(dataset=dataset, workflow=workflow)
+
+
+async def get_wftask_check_owner(
+    *,
+    project_id: int,
+    dataset_id: int,
+    workflowtask_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> WorkflowTaskV2:
+    """
+    Verify user access for the history of this dataset and workflowtask.
+
+    Args:
+        project_id:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+    """
+    wftask = await _get_workflowtask_or_404(
+        workflowtask_id=workflowtask_id,
+        db=db,
+    )
+    await _verify_workflow_and_dataset_access(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        workflow_id=wftask.workflow_id,
+        user_id=user_id,
+        db=db,
+    )
+    return wftask

fractal_server/app/routes/api/v2/history.py

@@ -1,26 +1,20 @@
-from datetime import datetime
-from typing import Any
-from typing import Optional
-
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import status
 from fastapi.responses import JSONResponse
-from pydantic import AwareDatetime
-from pydantic import BaseModel
-from pydantic import field_serializer
 from sqlmodel import func
 from sqlmodel import select
 
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
-from ._aux_functions import _get_workflowtask_check_history_owner
+from ._aux_functions_history import _verify_workflow_and_dataset_access
+from ._aux_functions_history import get_history_run_or_404
 from ._aux_functions_history import get_history_unit_or_404
+from ._aux_functions_history import get_wftask_check_owner
 from ._aux_functions_history import read_log_file
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
-from fractal_server.app.history.status_enum import XXXStatus
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
@@ -29,6 +23,11 @@ from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.pagination import get_pagination_params
 from fractal_server.app.routes.pagination import PaginationRequest
 from fractal_server.app.routes.pagination import PaginationResponse
+from fractal_server.app.schemas.v2 import HistoryRunReadAggregated
+from fractal_server.app.schemas.v2 import HistoryUnitRead
+from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.app.schemas.v2 import ImageLogsRequest
+from fractal_server.app.schemas.v2 import ZarrUrlAndStatus
 from fractal_server.images.tools import filter_image_list
 from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
@@ -45,12 +44,21 @@ async def get_workflow_tasks_statuses(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
+
+    # Access control
     workflow = await _get_workflow_check_owner(
         project_id=project_id,
         workflow_id=workflow_id,
         user_id=user.id,
         db=db,
     )
+    await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+
     response = {}
     for wftask in workflow.task_list:
         res = await db.execute(
@@ -69,7 +77,7 @@ async def get_workflow_tasks_statuses(
             num_available_images=latest_history_run.num_available_images,
         )
 
-        for target_status in XXXStatus:
+        for target_status in HistoryUnitStatus:
             stm = (
                 select(func.count(HistoryImageCache.zarr_url))
                 .join(HistoryUnit)
@@ -89,43 +97,6 @@ async def get_workflow_tasks_statuses(
     return JSONResponse(content=response, status_code=200)
 
 
-# FIXME MOVE TO SCHEMAS
-
-
-class HistoryUnitRead(BaseModel):
-    id: int
-    logfile: Optional[str] = None
-    status: XXXStatus
-    zarr_urls: list[str]
-
-
-class HistoryRunReadAggregated(BaseModel):
-    id: int
-    timestamp_started: AwareDatetime
-    workflowtask_dump: dict[str, Any]
-    num_submitted_units: int
-    num_done_units: int
-    num_failed_units: int
-
-    @field_serializer("timestamp_started")
-    def serialize_datetime(v: datetime) -> str:
-        return v.isoformat()
-
-
-class ImageLogsRequest(BaseModel):
-    workflowtask_id: int
-    dataset_id: int
-    zarr_url: str
-
-
-class ImageWithStatus(BaseModel):
-    zarr_url: str
-    status: Optional[XXXStatus] = None
-
-
-# end FIXME
-
-
 @router.get("/project/{project_id}/status/run/")
 async def get_history_run_list(
     project_id: int,
@@ -134,8 +105,10 @@ async def get_history_run_list(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[HistoryRunReadAggregated]:
+
     # Access control
-    await _get_workflowtask_check_history_owner(
+    await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,
@@ -152,11 +125,11 @@ async def get_history_run_list(
     res = await db.execute(stm)
     runs = res.scalars().all()
 
-    # Add units count by status
-
+    # Respond early if there are no runs
     if not runs:
         return []
 
+    # Add units count by status
     run_ids = [run.id for run in runs]
     stm = (
         select(
@@ -196,30 +169,29 @@ async def get_history_run_units(
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
 ) -> PaginationResponse[HistoryUnitRead]:
+
     # Access control
-    await _get_workflowtask_check_history_owner(
+    await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
 
-    history_run = await db.get(HistoryRun, history_run_id)
-    if history_run is None:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"HistoryRun {history_run_id} not found",
-        )
+    # Check that `HistoryRun` exists
+    await get_history_run_or_404(history_run_id=history_run_id, db=db)
 
+    # Count `HistoryUnit`s
     res = await db.execute(
         select(func.count(HistoryUnit.id)).where(
             HistoryUnit.history_run_id == history_run_id
         )
     )
     total_count = res.scalar()
-
     page_size = pagination.page_size or total_count
 
+    # Query `HistoryUnit`s
     res = await db.execute(
         select(HistoryUnit)
         .where(HistoryUnit.history_run_id == history_run_id)
@@ -244,30 +216,27 @@ async def get_history_images(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
-) -> PaginationResponse[ImageWithStatus]:
+) -> PaginationResponse[ZarrUrlAndStatus]:
+
     # Access control and object retrieval
-    # FIXME: Provide a single function that checks/gets what is needed
-    res = await _get_dataset_check_owner(
+    wftask = await get_wftask_check_owner(
         project_id=project_id,
-        dataset_id=dataset_id,
-        user_id=user.id,
-        db=db,
-    )
-    dataset = res["dataset"]
-    wftask = await _get_workflowtask_check_history_owner(
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
-    workflow = await _get_workflow_check_owner(
+    res = await _verify_workflow_and_dataset_access(
         project_id=project_id,
         workflow_id=wftask.workflow_id,
+        dataset_id=dataset_id,
         user_id=user.id,
         db=db,
     )
+    dataset = res["dataset"]
+    workflow = res["workflow"]
 
-    # FIXME reduce logging?
+    # Setup prefix for logging
     prefix = f"[DS{dataset.id}-WFT{wftask.id}-images]"
 
     # (1) Get the type-filtered list of dataset images
@@ -359,7 +328,8 @@ async def get_image_log(
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
     # Access control
-    wftask = await _get_workflowtask_check_history_owner(
+    wftask = await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=request_data.dataset_id,
         workflowtask_id=request_data.workflowtask_id,
         user_id=user.id,
@@ -406,7 +376,8 @@ async def get_history_unit_log(
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
     # Access control
-    wftask = await _get_workflowtask_check_history_owner(
+    wftask = await get_wftask_check_owner(
+        project_id=project_id,
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
         user_id=user.id,

fractal_server/app/routes/api/v2/job.py

@@ -5,6 +5,7 @@ from typing import Optional
 
 from fastapi import APIRouter
 from fastapi import Depends
+from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from fastapi.responses import StreamingResponse
@@ -83,6 +84,35 @@ async def get_workflow_jobs(
     return job_list
 
 
+@router.get("/project/{project_id}/latest-job/")
+async def get_latest_job(
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JobReadV2:
+    await _get_workflow_check_owner(
+        project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
+    )
+    stm = (
+        select(JobV2)
+        .where(JobV2.project_id == project_id)
+        .where(JobV2.workflow_id == workflow_id)
+        .where(JobV2.dataset_id == dataset_id)
+        .order_by(JobV2.start_timestamp.desc())
+        .limit(1)
+    )
+    res = await db.execute(stm)
+    latest_job = res.scalar_one_or_none()
+    if latest_job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job with {workflow_id=} and {dataset_id=} not found.",
+        )
+    return latest_job
+
+
 @router.get(
     "/project/{project_id}/job/{job_id}/",
     response_model=JobReadV2,

fractal_server/app/routes/api/v2/status_legacy.py

@@ -9,8 +9,8 @@ from .....logger import set_logger
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import JobV2
-from ....schemas.v2.status import StatusReadV2
-from ....schemas.v2.workflowtask import WorkflowTaskStatusTypeV2
+from ....schemas.v2.status_legacy import LegacyStatusReadV2
+from ....schemas.v2.status_legacy import WorkflowTaskStatusTypeV2
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
@@ -24,7 +24,7 @@ logger = set_logger(__name__)
 
 @router.get(
     "/project/{project_id}/status-legacy/",
-    response_model=StatusReadV2,
+    response_model=LegacyStatusReadV2,
 )
 async def get_workflowtask_status(
     project_id: int,
@@ -32,7 +32,7 @@ async def get_workflowtask_status(
     workflow_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[StatusReadV2]:
+) -> Optional[LegacyStatusReadV2]:
     """
     Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
     on a given `DatasetV2`.
@@ -164,5 +164,5 @@ async def get_workflowtask_status(
             # first time that you hit `last_valid_wftask_id``
             break
 
-    response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
+    response_body = LegacyStatusReadV2(status=clean_workflow_tasks_status_dict)
     return response_body

fractal_server/app/runner/executors/slurm_ssh/executor.py

@@ -9,7 +9,6 @@ from concurrent.futures import InvalidStateError
 from copy import copy
 from pathlib import Path
 from typing import Any
-from typing import Callable
 from typing import Optional
 from typing import Sequence
 
@@ -216,7 +215,7 @@ class FractalSlurmSSHExecutor(Executor):
 
     def submit(
         self,
-        fun: Callable[..., Any],
+        fun: callable,
         *fun_args: Sequence[Any],
         slurm_config: SlurmConfig,
         task_files: TaskFiles,
@@ -278,7 +277,7 @@ class FractalSlurmSSHExecutor(Executor):
 
     def map(
         self,
-        fn: Callable[..., Any],
+        fn: callable,
         iterable: list[Sequence[Any]],
         *,
         slurm_config: SlurmConfig,
@@ -446,7 +445,7 @@ class FractalSlurmSSHExecutor(Executor):
 
     def _prepare_job(
         self,
-        fun: Callable[..., Any],
+        fun: callable,
         slurm_file_prefix: str,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,

fractal_server/app/runner/v2/runner.py

@@ -2,7 +2,6 @@ import logging
 from copy import copy
 from copy import deepcopy
 from pathlib import Path
-from typing import Callable
 from typing import Optional
 
 from sqlalchemy.orm.attributes import flag_modified
@@ -20,13 +19,15 @@ from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
 from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.history.status_enum import XXXStatus
 from fractal_server.app.models.v2 import AccountingRecord
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.runner.executors.base_runner import BaseRunner
+from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.app.schemas.v2 import TaskDumpV2
+from fractal_server.app.schemas.v2 import TaskGroupDumpV2
 from fractal_server.images.models import AttributeFiltersType
 from fractal_server.images.tools import merge_type_filters
 
@@ -40,7 +41,7 @@ def execute_tasks_v2(
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
-    submit_setup_call: Callable = no_op_submit_setup_call,
+    submit_setup_call: callable = no_op_submit_setup_call,
     job_attribute_filters: AttributeFiltersType,
 ) -> None:
     logger = logging.getLogger(logger_name)
@@ -86,28 +87,24 @@ def execute_tasks_v2(
         else:
             num_available_images = 0
 
-        # Create history item
         with next(get_sync_db()) as db:
+            # Create dumps for workflowtask and taskgroup
             workflowtask_dump = dict(
                 **wftask.model_dump(exclude={"task"}),
-                task=wftask.task.model_dump(),
+                task=TaskDumpV2(**wftask.task.model_dump()).model_dump(),
             )
-
-            # Exclude timestamps since they'd need to be serialized properly
             task_group = db.get(TaskGroupV2, wftask.task.taskgroupv2_id)
-            task_group_dump = task_group.model_dump(
-                exclude={
-                    "timestamp_created",
-                    "timestamp_last_used",
-                }
-            )
+            task_group_dump = TaskGroupDumpV2(
+                **task_group.model_dump()
+            ).model_dump()
+            # Create HistoryRun
             history_run = HistoryRun(
                 dataset_id=dataset.id,
                 workflowtask_id=wftask.id,
                 workflowtask_dump=workflowtask_dump,
                 task_group_dump=task_group_dump,
                 num_available_images=num_available_images,
-                status=XXXStatus.SUBMITTED,
+                status=HistoryUnitStatus.SUBMITTED,
             )
             db.add(history_run)
             db.commit()
@@ -361,14 +358,14 @@ def execute_tasks_v2(
                 db.execute(
                     update(HistoryRun)
                     .where(HistoryRun.id == history_run_id)
-                    .values(status=XXXStatus.DONE)
+                    .values(status=HistoryUnitStatus.DONE)
                 )
                 db.commit()
             else:
                 db.execute(
                     update(HistoryRun)
                     .where(HistoryRun.id == history_run_id)
-                    .values(status=XXXStatus.FAILED)
+                    .values(status=HistoryUnitStatus.FAILED)
                 )
                 db.commit()
                 logger.error(