fractal-server 2.14.0a1__tar.gz → 2.14.0a3__tar.gz

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a1
+Version: 2.14.0a3
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.14.0a3/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a3"

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/history/image_updates.py

@@ -83,7 +83,8 @@ def update_single_image_logfile(
 ) -> None:
 
     logger.debug(
-        f"[update_single_image_logfile] {history_item_id=}, {logfile=}, {zarr_url=}"
+        "[update_single_image_logfile] "
+        f"{history_item_id=}, {logfile=}, {zarr_url=}"
     )
 
     with next(get_sync_db()) as db:

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/models/v2/history.py

@@ -50,4 +50,4 @@ class ImageStatus(SQLModel, table=True):
 
     parameters_hash: str
     status: str
-    logfile: str
+    logfile: Optional[str]

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/routes/admin/v2/accounting.py

@@ -3,9 +3,6 @@ from typing import Optional
 
 from fastapi import APIRouter
 from fastapi import Depends
-from fastapi import HTTPException
-from fastapi import Query
-from fastapi import status
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from pydantic.types import AwareDatetime
@@ -18,6 +15,9 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import AccountingRecord
 from fractal_server.app.models.v2 import AccountingRecordSlurm
 from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.routes.pagination import get_pagination_params
+from fractal_server.app.routes.pagination import PaginationRequest
+from fractal_server.app.routes.pagination import PaginationResponse
 from fractal_server.app.schemas.v2 import AccountingRecordRead
 
 
@@ -27,32 +27,19 @@ class AccountingQuery(BaseModel):
     timestamp_max: Optional[AwareDatetime] = None
 
 
-class AccountingPage(BaseModel):
-    total_count: int
-    page_size: int
-    current_page: int
-    records: list[AccountingRecordRead]
-
-
 router = APIRouter()
 
 
-@router.post("/", response_model=AccountingPage)
+@router.post("/", response_model=PaginationResponse[AccountingRecordRead])
 async def query_accounting(
     query: AccountingQuery,
-    # pagination
-    page: int = Query(default=1, ge=1),
-    page_size: Optional[int] = Query(default=None, ge=1),
-    # dependencies
+    # Dependencies
+    pagination: PaginationRequest = Depends(get_pagination_params),
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> AccountingPage:
-
-    if page_size is None and page > 1:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(f"Invalid pagination parameters: {page=}, {page_size=}."),
-        )
+) -> PaginationResponse[AccountingRecordRead]:
+    page = pagination.page
+    page_size = pagination.page_size
 
     stm = select(AccountingRecord).order_by(AccountingRecord.id)
     stm_count = select(func.count(AccountingRecord.id))
@@ -69,20 +56,23 @@ async def query_accounting(
         stm_count = stm_count.where(
             AccountingRecord.timestamp <= query.timestamp_max
         )
+
+    res_total_count = await db.execute(stm_count)
+    total_count = res_total_count.scalar()
+
     if page_size is not None:
         stm = stm.offset((page - 1) * page_size).limit(page_size)
+    else:
+        page_size = total_count
 
     res = await db.execute(stm)
     records = res.scalars().all()
-    res_total_count = await db.execute(stm_count)
-    total_count = res_total_count.scalar()
 
-    actual_page_size = page_size or len(records)
-    return AccountingPage(
+    return PaginationResponse[AccountingRecordRead](
         total_count=total_count,
-        page_size=actual_page_size,
+        page_size=page_size,
         current_page=page,
-        records=[record.model_dump() for record in records],
+        items=[record.model_dump() for record in records],
     )
 
 

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/routes/admin/v2/task.py

@@ -68,7 +68,7 @@ async def query_tasks(
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskV2Info]:
     """
-    Query `TaskV2` table and get informations about related items
+    Query `TaskV2` table and get information about related items
     (WorkflowV2s and ProjectV2s)
 
     Args:

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/routes/api/v2/_aux_functions.py

@@ -470,7 +470,7 @@ async def _get_workflowtask_check_history_owner(
     dataset_id: int,
     user_id: int,
     db: AsyncSession,
-) -> list[int]:
+) -> WorkflowTaskV2:
     """
     Verify user access for the history of this dataset and workflowtask.
 
@@ -479,9 +479,6 @@ async def _get_workflowtask_check_history_owner(
         workflow_task_id:
         user_id:
         db:
-
-    Returns:
-        List of WorkflowTask IDs
     """
     workflowtask = await db.get(WorkflowTaskV2, workflowtask_id)
     if workflowtask is None:
@@ -495,3 +492,4 @@ async def _get_workflowtask_check_history_owner(
         user_id=user_id,
         db=db,
     )
+    return workflowtask

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/routes/api/v2/history.py

@@ -1,25 +1,28 @@
+from pathlib import Path
 from typing import Optional
 
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
-from fastapi import Query
 from fastapi import status
 from fastapi.responses import JSONResponse
+from pydantic import BaseModel
 from sqlmodel import func
 from sqlmodel import select
 
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
-from ._aux_functions import _get_workflow_task_check_owner
+from ._aux_functions import _get_workflowtask_check_history_owner
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.history.status_enum import HistoryItemImageStatus
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import HistoryItemV2
 from fractal_server.app.models.v2 import ImageStatus
-from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.routes.pagination import get_pagination_params
+from fractal_server.app.routes.pagination import PaginationRequest
+from fractal_server.app.routes.pagination import PaginationResponse
 from fractal_server.app.schemas.v2.history import HistoryItemV2Read
 
 router = APIRouter()
@@ -122,16 +125,10 @@ async def get_per_workflowtask_subsets_aggregated_info(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
-    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
-    if wftask is None:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail="WorkflowTask not found",
-        )
-    await _get_workflow_task_check_owner(
-        project_id=project_id,
-        workflow_id=wftask.workflow_id,
-        workflow_task_id=workflowtask_id,
+
+    await _get_workflowtask_check_history_owner(
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
@@ -143,20 +140,27 @@ async def get_per_workflowtask_subsets_aggregated_info(
         .group_by(ImageStatus.parameters_hash)
     )
     res = await db.execute(stm)
-    hash_statuses = res.all()
+    hash_to_statuses = res.all()
 
-    result = []
-    for _hash, statuses in hash_statuses:
-        dump = await db.execute(
-            select(HistoryItemV2.workflowtask_dump)
+    subsets = []
+    for parameters_hash, statuses in hash_to_statuses:
+        # Get the oldest HistoryItemV2 matching with `parameters_hash`
+        stm = (
+            select(HistoryItemV2)
             .where(HistoryItemV2.workflowtask_id == workflowtask_id)
             .where(HistoryItemV2.dataset_id == dataset_id)
-            .where(HistoryItemV2.parameters_hash == _hash)
+            .where(HistoryItemV2.parameters_hash == parameters_hash)
+            .order_by(HistoryItemV2.timestamp_started)
+            .limit(1)
         )
-        result.append(
+        res = await db.execute(stm)
+        oldest_history_item = res.scalar_one()
+
+        subsets.append(
             {
-                "workflowtask_dump": dump.scalar_one(),
-                "parameters_hash": _hash,
+                "_timestamp": oldest_history_item.timestamp_started,
+                "workflowtask_dump": oldest_history_item.workflowtask_dump,
+                "parameters_hash": parameters_hash,
                 "info": {
                     "num_done_images": statuses.count(
                         HistoryItemImageStatus.DONE
@@ -171,7 +175,11 @@ async def get_per_workflowtask_subsets_aggregated_info(
             }
         )
 
-    return JSONResponse(content=result, status_code=200)
+    # Use `_timestamp` values for sorting, and then drop them from the response
+    sorted_results = sorted(subsets, key=lambda obj: obj["_timestamp"])
+    [item.pop("_timestamp") for item in sorted_results]
+
+    return JSONResponse(content=sorted_results, status_code=200)
 
 
 @router.get("/project/{project_id}/status/images/")
@@ -181,30 +189,18 @@ async def get_per_workflowtask_images(
     dataset_id: int,
     status: HistoryItemImageStatus,
     parameters_hash: Optional[str] = None,
-    # Pagination
-    page: int = Query(default=1, ge=1),
-    page_size: Optional[int] = Query(default=None, ge=1),
     # Dependencies
+    pagination: PaginationRequest = Depends(get_pagination_params),
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> JSONResponse:
+) -> PaginationResponse[str]:
 
-    if page_size is None and page > 1:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(f"Invalid pagination parameters: {page=}, {page_size=}."),
-        )
+    page = pagination.page
+    page_size = pagination.page_size
 
-    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
-    if wftask is None:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail="WorkflowTask not found",
-        )
-    await _get_workflow_task_check_owner(
-        project_id=project_id,
-        workflow_id=wftask.workflow_id,
-        workflow_task_id=workflowtask_id,
+    await _get_workflowtask_check_history_owner(
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
         user_id=user.id,
         db=db,
     )
@@ -228,20 +224,82 @@ async def get_per_workflowtask_images(
         )
         query = query.where(ImageStatus.parameters_hash == parameters_hash)
 
+    res_total_count = await db.execute(total_count_stm)
+    total_count = res_total_count.scalar()
+
     if page_size is not None:
         query = query.limit(page_size)
+    else:
+        page_size = total_count
+
     if page > 1:
         query = query.offset((page - 1) * page_size)
 
-    res_total_count = await db.execute(total_count_stm)
-    total_count = res_total_count.scalar()
-
     res = await db.execute(query)
     images = res.scalars().all()
 
-    return {
-        "total_count": total_count,
-        "page_size": page_size,
-        "current_page": page,
-        "images": images,
-    }
+    return PaginationResponse[str](
+        total_count=total_count,
+        page_size=page_size,
+        current_page=page,
+        items=images,
+    )
+
+
+class ImageLogsRequest(BaseModel):
+    workflowtask_id: int
+    dataset_id: int
+    zarr_url: str
+
+
+@router.post("/project/{project_id}/status/image-logs/")
+async def get_image_logs(
+    project_id: int,
+    request_data: ImageLogsRequest,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+
+    wftask = await _get_workflowtask_check_history_owner(
+        dataset_id=request_data.dataset_id,
+        workflowtask_id=request_data.workflowtask_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    image_status = await db.get(
+        ImageStatus,
+        (
+            request_data.zarr_url,
+            request_data.workflowtask_id,
+            request_data.dataset_id,
+        ),
+    )
+    if image_status is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="ImageStatus not found",
+        )
+
+    if image_status.logfile is None:
+        return JSONResponse(
+            content=(
+                f"Logs for task '{wftask.task.name}' in dataset "
+                f"{request_data.dataset_id} are not yet available."
+            )
+        )
+
+    logfile = Path(image_status.logfile)
+    if not logfile.exists():
+        return JSONResponse(
+            content=(
+                f"Error while retrieving logs for task '{wftask.task.name}' "
+                f"in dataset {request_data.dataset_id}: "
+                f"file '{logfile}' is not available."
+            )
+        )
+
+    with logfile.open("r") as f:
+        file_contents = f.read()
+
+    return JSONResponse(content=file_contents)

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/routes/api/v2/images.py

@@ -17,6 +17,9 @@ from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.routes.pagination import get_pagination_params
+from fractal_server.app.routes.pagination import PaginationRequest
+from fractal_server.app.routes.pagination import PaginationResponse
 from fractal_server.app.schemas._filter_validators import (
     validate_attribute_filters,
 )
@@ -31,17 +34,11 @@ from fractal_server.images.tools import match_filter
 router = APIRouter()
 
 
-class ImagePage(BaseModel):
-
-    total_count: int
-    page_size: int
-    current_page: int
+class ImagePage(PaginationResponse[SingleImage]):
 
     attributes: dict[str, list[Any]]
     types: list[str]
 
-    images: list[SingleImage]
-
 
 class ImageQuery(BaseModel):
     zarr_url: Optional[str] = None
@@ -118,18 +115,14 @@ async def post_new_image(
 async def query_dataset_images(
     project_id: int,
     dataset_id: int,
-    page: int = 1,  # query param
-    page_size: Optional[int] = None,  # query param
-    query: Optional[ImageQuery] = None,  # body
+    query: Optional[ImageQuery] = None,
+    pagination: PaginationRequest = Depends(get_pagination_params),
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> ImagePage:
 
-    if page < 1:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=f"Invalid pagination parameter: page={page} < 1",
-        )
+    page = pagination.page
+    page_size = pagination.page_size
 
     output = await _get_dataset_check_owner(
         project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
@@ -177,20 +170,10 @@ async def query_dataset_images(
 
     total_count = len(images)
 
-    if page_size is not None:
-        if page_size <= 0:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=(
-                    f"Invalid pagination parameter: page_size={page_size} <= 0"
-                ),
-            )
-    else:
+    if page_size is None:
         page_size = total_count
 
-    if total_count == 0:
-        page = 1
-    else:
+    if total_count > 0:
         last_page = (total_count // page_size) + (total_count % page_size > 0)
         if page > last_page:
             page = last_page
@@ -201,9 +184,9 @@ async def query_dataset_images(
         total_count=total_count,
         current_page=page,
         page_size=page_size,
+        items=images,
         attributes=attributes,
         types=types,
-        images=images,
     )
 
 

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/routes/api/v2/project.py

@@ -57,7 +57,7 @@ async def create_project(
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[ProjectReadV2]:
     """
-    Create new poject
+    Create new project
     """
 
     # Check that there is no project with the same user and name

fractal_server-2.14.0a3/fractal_server/app/routes/pagination.py

@@ -0,0 +1,47 @@
+from typing import Generic
+from typing import Optional
+from typing import TypeVar
+
+from fastapi import HTTPException
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import model_validator
+from pydantic import ValidationError
+
+T = TypeVar("T")
+
+
+class PaginationRequest(BaseModel):
+
+    page: int = Field(ge=1)
+    page_size: Optional[int] = Field(ge=1)
+
+    @model_validator(mode="after")
+    def valid_pagination_parameters(self):
+        if self.page_size is None and self.page > 1:
+            raise ValueError(
+                f"page_size is None but page={self.page} is greater than 1."
+            )
+        return self
+
+
+def get_pagination_params(
+    page: int = 1, page_size: Optional[int] = None
+) -> PaginationRequest:
+    try:
+        pagination = PaginationRequest(page=page, page_size=page_size)
+    except ValidationError as e:
+        raise HTTPException(
+            status_code=422,
+            detail=f"Invalid pagination parameters. Original error: '{e}'.",
+        )
+    return pagination
+
+
+class PaginationResponse(BaseModel, Generic[T]):
+
+    current_page: int = Field(ge=1)
+    page_size: int = Field(ge=0)
+    total_count: int = Field(ge=0)
+
+    items: list[T]

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/executors/base_runner.py

@@ -106,7 +106,8 @@ class BaseRunner(object):
                 )
             if _COMPONENT_KEY_ not in single_kwargs.keys():
                 raise ValueError(
-                    f"No '{_COMPONENT_KEY_}' key in in {list(single_kwargs.keys())}"
+                    f"No '{_COMPONENT_KEY_}' key "
+                    f"in {list(single_kwargs.keys())}"
                 )
         if not in_compound_task:
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/executors/local/_submit_setup.py

@@ -21,18 +21,10 @@ def _local_submit_setup(
     FIXME
 
     Arguments:
-        wftask:
-            WorkflowTask for which the configuration is to be assembled
-        workflow_dir_local:
-            Not used in this function.
-        workflow_dir_remote:
-            Not used in this function.
-
-    Returns:
-        submit_setup_dict:
-            A dictionary that will be passed on to
-            `FractalThreadPoolExecutor.submit` and
-            `FractalThreadPoolExecutor.map`, so as to set extra options.
+        wftask: WorkflowTask for which the configuration is to be assembled
+        root_dir_local:
+        root_dir_rempote: Not used in this function.
+        which_type: Whether it is a parallel or non-parallel task.
     """
 
     local_backend_config = get_local_backend_config(
@@ -43,7 +35,7 @@ def _local_submit_setup(
     # Get TaskFiles object
     task_files = TaskFiles(
         root_dir_local=root_dir_local,
-        root_dir_remote=root_dir_remote,
+        root_dir_remote=root_dir_local,
         task_order=wftask.order,
         task_name=wftask.task.name,
     )

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/executors/slurm_common/_slurm_config.py

@@ -213,7 +213,7 @@ class SlurmConfig(BaseModel):
     expected file content are defined in
     [`SlurmConfigFile`](./#fractal_server.app.runner._slurm._slurm_config.SlurmConfigFile)).
 
-    Part of the attributes map directly to some of the SLURM attribues (see
+    Part of the attributes map directly to some of the SLURM attributes (see
     https://slurm.schedmd.com/sbatch.html), e.g. `partition`. Other attributes
     are metaparameters which are needed in fractal-server to combine multiple
     tasks in the same SLURM job (e.g. `parallel_tasks_per_job` or

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/executors/slurm_common/get_slurm_config.py

@@ -19,7 +19,7 @@ def get_slurm_config(
     Prepare a `SlurmConfig` configuration object
 
     The argument `which_type` determines whether we use `wftask.meta_parallel`
-    or `wftask.meta_non_parallel`. In the following descritpion, let us assume
+    or `wftask.meta_non_parallel`. In the following description, let us assume
     that `which_type="parallel"`.
 
     The sources for `SlurmConfig` attributes, in increasing priority order, are

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/executors/slurm_common/remote.py

@@ -134,7 +134,7 @@ def worker(
         _extra_import_paths = extra_import_paths.split(":")
         sys.path[:0] = _extra_import_paths
 
-    # Execute the job and catpure exceptions
+    # Execute the job and capture exceptions
     try:
         with open(in_fname, "rb") as f:
             indata = f.read()

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/v2/__init__.py

@@ -2,7 +2,7 @@
 Runner backend subsystem root V2
 
 This module is the single entry point to the runner backend subsystem V2.
-Other subystems should only import this module and not its submodules or
+Other subsystems should only import this module and not its submodules or
 the individual backends.
 """
 import os
@@ -118,7 +118,7 @@ def submit_workflow(
             )
         except Exception as e:
             logger.error(
-                f"Error conneting to the database. Original error: {str(e)}"
+                f"Error connecting to the database. Original error: {str(e)}"
             )
             reset_logger_handlers(logger)
             return

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/v2/_slurm_ssh.py

@@ -11,7 +11,7 @@
 # Institute for Biomedical Research and Pelkmans Lab from the University of
 # Zurich.
 """
-Slurm Bakend
+Slurm Backend
 
 This backend runs fractal workflows in a SLURM cluster using Clusterfutures
 Executor objects.

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/v2/_slurm_sudo.py

@@ -11,7 +11,7 @@
 # Institute for Biomedical Research and Pelkmans Lab from the University of
 # Zurich.
 """
-Slurm Bakend
+Slurm Backend
 
 This backend runs fractal workflows in a SLURM cluster using Clusterfutures
 Executor objects.

{fractal_server-2.14.0a1 → fractal_server-2.14.0a3}/fractal_server/app/runner/v2/runner.py

@@ -127,7 +127,7 @@ def execute_tasks_v2(
                         dataset_id=dataset.id,
                         parameters_hash=parameters_hash,
                         status=HistoryItemImageStatus.SUBMITTED,
-                        logfile="/placeholder",
+                        logfile=None,
                     )
                 )
             db.commit()
@@ -198,7 +198,8 @@ def execute_tasks_v2(
         # Update image list
         num_new_images = 0
         current_task_output.check_zarr_urls_are_unique()
-        # FIXME: Introduce for loop over task outputs, and processe them sequentially
+        # FIXME: Introduce for loop over task outputs, and processe them
+        # sequentially
         # each failure should lead to an update of the specific image status
         for image_obj in current_task_output.image_list_updates:
             image = image_obj.model_dump()