fractal-server 2.13.0__py3-none-any.whl → 2.14.0a0__py3-none-any.whl

fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py

@@ -55,7 +55,7 @@ async def get_package_version_from_pypi(
             f"A TimeoutException occurred while getting {url}.\n"
             f"Original error: {str(e)}."
         )
-        logger.error(error_msg)
+        logger.warning(error_msg)
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=error_msg,
@@ -65,7 +65,7 @@ async def get_package_version_from_pypi(
             f"An unknown error occurred while getting {url}. "
             f"Original error: {str(e)}."
         )
-        logger.error(error_msg)
+        logger.warning(error_msg)
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=error_msg,
@@ -85,7 +85,7 @@ async def get_package_version_from_pypi(
         latest_version = response_data["info"]["version"]
         available_releases = response_data["releases"].keys()
     except KeyError as e:
-        logger.error(
+        logger.warning(
             f"A KeyError occurred while getting {url}. "
             f"Original error: {str(e)}."
         )

fractal_server/app/routes/api/v2/dataset.py

@@ -5,11 +5,14 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from sqlmodel import delete
 from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import DatasetV2
+from ....models.v2 import HistoryItemV2
+from ....models.v2 import ImageStatus
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....schemas.v2 import DatasetCreateV2
@@ -47,7 +50,6 @@ async def create_dataset(
     )
 
     if dataset.zarr_dir is None:
-
         if user.settings.project_dir is None:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -91,7 +93,6 @@ async def create_dataset(
 )
 async def read_dataset_list(
     project_id: int,
-    history: bool = True,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[list[DatasetReadV2]]:
@@ -110,9 +111,6 @@ async def read_dataset_list(
     res = await db.execute(stm)
     dataset_list = res.scalars().all()
     await db.close()
-    if not history:
-        for ds in dataset_list:
-            setattr(ds, "history", [])
     return dataset_list
 
 
@@ -225,6 +223,15 @@ async def delete_dataset(
     for job in jobs:
         job.dataset_id = None
 
+    # Cascade operations: delete history items and image status which are in
+    # relationship with the current dataset
+
+    stm = delete(HistoryItemV2).where(HistoryItemV2.dataset_id == dataset_id)
+    await db.execute(stm)
+
+    stm = delete(ImageStatus).where(ImageStatus.dataset_id == dataset_id)
+    await db.execute(stm)
+
     # Delete dataset
     await db.delete(dataset)
     await db.commit()
@@ -234,7 +241,6 @@ async def delete_dataset(
 
 @router.get("/dataset/", response_model=list[DatasetReadV2])
 async def get_user_datasets(
-    history: bool = True,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[DatasetReadV2]:
@@ -249,9 +255,6 @@ async def get_user_datasets(
     res = await db.execute(stm)
     dataset_list = res.scalars().all()
     await db.close()
-    if not history:
-        for ds in dataset_list:
-            setattr(ds, "history", [])
     return dataset_list
 
 

fractal_server/app/routes/api/v2/history.py

@@ -0,0 +1,247 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Query
+from fastapi import status
+from fastapi.responses import JSONResponse
+from sqlmodel import func
+from sqlmodel import select
+
+from ._aux_functions import _get_dataset_check_owner
+from ._aux_functions import _get_workflow_check_owner
+from ._aux_functions import _get_workflow_task_check_owner
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.history.status_enum import HistoryItemImageStatus
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import HistoryItemV2
+from fractal_server.app.models.v2 import ImageStatus
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.schemas.v2.history import HistoryItemV2Read
+
+router = APIRouter()
+
+
+@router.get(
+    "/project/{project_id}/dataset/{dataset_id}/history/",
+    response_model=list[HistoryItemV2Read],
+)
+async def get_dataset_history(
+    project_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[HistoryItemV2Read]:
+    await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    stm = (
+        select(HistoryItemV2)
+        .where(HistoryItemV2.dataset_id == dataset_id)
+        .order_by(HistoryItemV2.timestamp_started)
+    )
+    res = await db.execute(stm)
+    items = res.scalars().all()
+    return items
+
+
+@router.get("/project/{project_id}/status/")
+async def get_per_workflow_aggregated_info(
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    wft_ids = [wftask.id for wftask in workflow.task_list]
+
+    # num_available_images
+    stm = (
+        select(
+            HistoryItemV2.workflowtask_id, HistoryItemV2.num_available_images
+        )
+        .where(HistoryItemV2.dataset_id == dataset_id)
+        .where(HistoryItemV2.workflowtask_id.in_(wft_ids))
+        .order_by(
+            HistoryItemV2.workflowtask_id,
+            HistoryItemV2.timestamp_started.desc(),
+        )
+        # https://www.postgresql.org/docs/current/sql-select.html#SQL-DISTINCT
+        .distinct(HistoryItemV2.workflowtask_id)
+    )
+    res = await db.execute(stm)
+    num_available_images = {k: v for k, v in res.all()}
+
+    count = {}
+    for _status in HistoryItemImageStatus:
+        stm = (
+            select(ImageStatus.workflowtask_id, func.count())
+            .where(ImageStatus.dataset_id == dataset_id)
+            .where(ImageStatus.workflowtask_id.in_(wft_ids))
+            .where(ImageStatus.status == _status)
+            # https://docs.sqlalchemy.org/en/20/tutorial/data_select.html#tutorial-group-by-w-aggregates
+            .group_by(ImageStatus.workflowtask_id)
+        )
+        res = await db.execute(stm)
+        count[_status] = {k: v for k, v in res.all()}
+
+    result = {
+        str(_id): None
+        if _id not in num_available_images
+        else {
+            "num_available_images": num_available_images[_id],
+            "num_done_images": count["done"].get(_id, 0),
+            "num_submitted_images": count["submitted"].get(_id, 0),
+            "num_failed_images": count["failed"].get(_id, 0),
+        }
+        for _id in wft_ids
+    }
+
+    return JSONResponse(content=result, status_code=200)
+
+
+@router.get("/project/{project_id}/status/subsets/")
+async def get_per_workflowtask_subsets_aggregated_info(
+    project_id: int,
+    workflowtask_id: int,
+    dataset_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if wftask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="WorkflowTask not found",
+        )
+    await _get_workflow_task_check_owner(
+        project_id=project_id,
+        workflow_id=wftask.workflow_id,
+        workflow_task_id=workflowtask_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    stm = (
+        select(ImageStatus.parameters_hash, func.array_agg(ImageStatus.status))
+        .where(ImageStatus.dataset_id == dataset_id)
+        .where(ImageStatus.workflowtask_id == workflowtask_id)
+        .group_by(ImageStatus.parameters_hash)
+    )
+    res = await db.execute(stm)
+    hash_statuses = res.all()
+
+    result = []
+    for _hash, statuses in hash_statuses:
+        dump = await db.execute(
+            select(HistoryItemV2.workflowtask_dump)
+            .where(HistoryItemV2.workflowtask_id == workflowtask_id)
+            .where(HistoryItemV2.dataset_id == dataset_id)
+            .where(HistoryItemV2.parameters_hash == _hash)
+        )
+        result.append(
+            {
+                "workflowtask_dump": dump.scalar_one(),
+                "parameters_hash": _hash,
+                "info": {
+                    "num_done_images": statuses.count(
+                        HistoryItemImageStatus.DONE
+                    ),
+                    "num_failed_images": statuses.count(
+                        HistoryItemImageStatus.FAILED
+                    ),
+                    "num_submitted_images": statuses.count(
+                        HistoryItemImageStatus.SUBMITTED
+                    ),
+                },
+            }
+        )
+
+    return JSONResponse(content=result, status_code=200)
+
+
+@router.get("/project/{project_id}/status/images/")
+async def get_per_workflowtask_images(
+    project_id: int,
+    workflowtask_id: int,
+    dataset_id: int,
+    status: HistoryItemImageStatus,
+    parameters_hash: Optional[str] = None,
+    # Pagination
+    page: int = Query(default=1, ge=1),
+    page_size: Optional[int] = Query(default=None, ge=1),
+    # Dependencies
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+
+    if page_size is None and page > 1:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(f"Invalid pagination parameters: {page=}, {page_size=}."),
+        )
+
+    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if wftask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="WorkflowTask not found",
+        )
+    await _get_workflow_task_check_owner(
+        project_id=project_id,
+        workflow_id=wftask.workflow_id,
+        workflow_task_id=workflowtask_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    total_count_stm = (
+        select(func.count(ImageStatus.zarr_url))
+        .where(ImageStatus.dataset_id == dataset_id)
+        .where(ImageStatus.workflowtask_id == workflowtask_id)
+        .where(ImageStatus.status == status)
+    )
+    query = (
+        select(ImageStatus.zarr_url)
+        .where(ImageStatus.dataset_id == dataset_id)
+        .where(ImageStatus.workflowtask_id == workflowtask_id)
+        .where(ImageStatus.status == status)
+    )
+
+    if parameters_hash is not None:
+        total_count_stm = total_count_stm.where(
+            ImageStatus.parameters_hash == parameters_hash
+        )
+        query = query.where(ImageStatus.parameters_hash == parameters_hash)
+
+    if page_size is not None:
+        query = query.limit(page_size)
+    if page > 1:
+        query = query.offset((page - 1) * page_size)
+
+    res_total_count = await db.execute(total_count_stm)
+    total_count = res_total_count.scalar()
+
+    res = await db.execute(query)
+    images = res.scalars().all()
+
+    return {
+        "total_count": total_count,
+        "page_size": page_size,
+        "current_page": page,
+        "images": images,
+    }

fractal_server/app/routes/api/v2/submit.py

@@ -239,6 +239,7 @@ async def apply_workflow(
         workflow_id=workflow.id,
         dataset_id=dataset.id,
         job_id=job.id,
+        user_id=user.id,
         user_settings=user_settings,
         worker_init=job.worker_init,
         slurm_user=user_settings.slurm_user,

fractal_server/app/routes/api/v2/task_group.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from typing import Optional
 
 from fastapi import APIRouter
@@ -6,6 +5,7 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from pydantic.types import AwareDatetime
 from sqlmodel import or_
 from sqlmodel import select
 
@@ -23,7 +23,6 @@ from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
-from fractal_server.app.routes.aux import _raise_if_naive_datetime
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
@@ -43,13 +42,11 @@ async def get_task_group_activity_list(
     pkg_name: Optional[str] = None,
     status: Optional[TaskGroupActivityStatusV2] = None,
     action: Optional[TaskGroupActivityActionV2] = None,
-    timestamp_started_min: Optional[datetime] = None,
+    timestamp_started_min: Optional[AwareDatetime] = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskGroupActivityV2Read]:
 
-    _raise_if_naive_datetime(timestamp_started_min)
-
     stm = select(TaskGroupActivityV2).where(
         TaskGroupActivityV2.user_id == user.id
     )

fractal_server/app/routes/api/v2/workflow.py

@@ -7,10 +7,13 @@ from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from pydantic import BaseModel
+from sqlmodel import delete
 from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
+from ....models.v2 import HistoryItemV2
+from ....models.v2 import ImageStatus
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....models.v2 import WorkflowV2
@@ -225,14 +228,26 @@ async def delete_workflow(
             ),
         )
 
-    # Cascade operations: set foreign-keys to null for jobs which are in
-    # relationship with the current workflow
+    # Cascade operation: set foreign-keys to null for jobs and history items
+    # which are in relationship with the current workflow.
     stm = select(JobV2).where(JobV2.workflow_id == workflow_id)
     res = await db.execute(stm)
     jobs = res.scalars().all()
     for job in jobs:
         job.workflow_id = None
 
+    wft_ids = [wft.id for wft in workflow.task_list]
+    stm = select(HistoryItemV2).where(
+        HistoryItemV2.workflowtask_id.in_(wft_ids)
+    )
+    res = await db.execute(stm)
+    history_items = res.scalars().all()
+    for history_item in history_items:
+        history_item.workflowtask_id = None
+
+    stm = delete(ImageStatus).where(ImageStatus.workflowtask_id.in_(wft_ids))
+    await db.execute(stm)
+
     # Delete workflow
     await db.delete(workflow)
     await db.commit()
@@ -244,7 +259,7 @@ async def delete_workflow(
     "/project/{project_id}/workflow/{workflow_id}/export/",
     response_model=WorkflowExportV2,
 )
-async def export_worfklow(
+async def export_workflow(
     project_id: int,
     workflow_id: int,
     user: UserOAuth = Depends(current_active_user),

fractal_server/app/routes/api/v2/workflowtask.py

@@ -6,6 +6,8 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from sqlmodel import delete
+from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
@@ -15,6 +17,8 @@ from ._aux_functions import _workflow_insert_task
 from ._aux_functions_tasks import _check_type_filters_compatibility
 from ._aux_functions_tasks import _get_task_read_access
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import HistoryItemV2
+from fractal_server.app.models.v2 import ImageStatus
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.schemas.v2 import WorkflowTaskCreateV2
@@ -333,6 +337,24 @@ async def delete_workflowtask(
         db=db,
     )
 
+    # Cascade operations:
+    # * set foreign-keys to null for history items which are in relationship
+    #   with the current workflowtask;
+    # * delete ImageStatus in relationship with the current workflowtask.
+    stm = select(HistoryItemV2).where(
+        HistoryItemV2.workflowtask_id == db_workflow_task.id
+    )
+    res = await db.execute(stm)
+    history_items = res.scalars().all()
+    for history_item in history_items:
+        history_item.workflowtask_id = None
+
+    stm = delete(ImageStatus).where(
+        ImageStatus.workflowtask_id == db_workflow_task.id
+    )
+    await db.execute(stm)
+
+    # Delete WorkflowTask
     await db.delete(db_workflow_task)
     await db.commit()
 

fractal_server/app/routes/aux/__init__.py

@@ -1,20 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from fastapi import HTTPException
-from fastapi import status
-
-
-def _raise_if_naive_datetime(*timestamps: tuple[Optional[datetime]]) -> None:
-    """
-    Raise 422 if any not-null argument is a naive `datetime` object:
-    https://docs.python.org/3/library/datetime.html#determining-if-an-object-is-aware-or-naive
-    """
-    for timestamp in filter(None, timestamps):
-        if (timestamp.tzinfo is None) or (
-            timestamp.tzinfo.utcoffset(timestamp) is None
-        ):
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=f"{timestamp=} is naive. You must provide a timezone.",
-            )

fractal_server/app/runner/executors/base_runner.py

@@ -0,0 +1,114 @@
+from typing import Any
+
+from fractal_server.app.runner.components import _COMPONENT_KEY_
+
+
+class BaseRunner(object):
+    """
+    Base class for Fractal runners.
+    """
+
+    def shutdown(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def submit(
+        self,
+        func: callable,
+        parameters: dict[str, Any],
+        history_item_id: int,
+        in_compound_task: bool,
+        **kwargs,
+    ) -> tuple[Any, BaseException]:
+        """
+        Run a single fractal task.
+
+        # FIXME: Describe more in detail
+
+        Args:
+            func:
+                Function to be executed.
+            parameters:
+                Dictionary of parameters. Must include `zarr_urls` key.
+            history_item_id:
+                Database ID of the corresponding `HistoryItemV2` entry.
+            in_compound_task:
+                Whether this is the init part of a compound task.
+            kwargs:
+                Runner-specific parameters.
+        """
+        raise NotImplementedError()
+
+    def multisubmit(
+        self,
+        func: callable,
+        list_parameters: list[dict[str, Any]],
+        history_item_id: int,
+        in_compound_task: bool,
+        **kwargs,
+    ) -> tuple[dict[int, Any], dict[int, BaseException]]:
+        """
+        Run a parallel fractal task.
+
+        # FIXME: Describe more in detail
+
+        Args:
+            func:
+                Function to be executed.
+            list_parameters:
+                List of dictionaries of parameters. Each one must include a
+                `zarr_url` key.
+            history_item_id:
+                Database ID of the corresponding `HistoryItemV2` entry.
+            in_compound_task:
+                Whether this is the compute part of a compound task.
+            kwargs:
+                Runner-specific parameters.
+        """
+        raise NotImplementedError()
+
+    def validate_submit_parameters(self, parameters: dict[str, Any]) -> None:
+        """
+        Validate parameters for `submit` method
+
+        Args:
+            parameters: Parameters dictionary.
+        """
+        if not isinstance(parameters, dict):
+            raise ValueError("`parameters` must be a dictionary.")
+        if "zarr_urls" not in parameters.keys():
+            raise ValueError(
+                f"No 'zarr_urls' key in in {list(parameters.keys())}"
+            )
+        if _COMPONENT_KEY_ not in parameters.keys():
+            raise ValueError(
+                f"No '{_COMPONENT_KEY_}' key in in {list(parameters.keys())}"
+            )
+
+    def validate_multisubmit_parameters(
+        self,
+        list_parameters: list[dict[str, Any]],
+        in_compound_task: bool,
+    ) -> None:
+        """
+        Validate parameters for `multi_submit` method
+
+        Args:
+            list_parameters: List of parameters dictionaries.
+            in_compound_task:
+               Whether this is the compute part of a compound task.
+        """
+        for single_kwargs in list_parameters:
+            if not isinstance(single_kwargs, dict):
+                raise RuntimeError("kwargs itemt must be a dictionary.")
+            if "zarr_url" not in single_kwargs.keys():
+                raise RuntimeError(
+                    f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
+                )
+            if _COMPONENT_KEY_ not in single_kwargs.keys():
+                raise ValueError(
+                    f"No '{_COMPONENT_KEY_}' key in in {list(single_kwargs.keys())}"
+                )
+        if not in_compound_task:
+            zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
+            if len(zarr_urls) != len(set(zarr_urls)):
+                raise RuntimeError("Non-unique zarr_urls")

fractal_server/app/runner/{v2/_local → executors/local}/_local_config.py

@@ -20,9 +20,9 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import ValidationError
 
-from .....config import get_settings
-from .....syringe import Inject
-from ....models.v2 import WorkflowTaskV2
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.config import get_settings
+from fractal_server.syringe import Inject
 
 
 class LocalBackendConfigError(ValueError):

fractal_server/app/runner/executors/local/_submit_setup.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+from typing import Any
+from typing import Literal
+
+from ...task_files import TaskFiles
+from ._local_config import get_local_backend_config
+from fractal_server.app.models.v2 import WorkflowTaskV2
+
+
+def _local_submit_setup(
+    *,
+    wftask: WorkflowTaskV2,
+    root_dir_local: Path,
+    root_dir_remote: Path,
+    which_type: Literal["non_parallel", "parallel"],
+) -> dict[str, Any]:
+    """
+    Collect WorkflowTask-specific configuration parameters from different
+    sources, and inject them for execution.
+
+    FIXME
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the configuration is to be assembled
+        workflow_dir_local:
+            Not used in this function.
+        workflow_dir_remote:
+            Not used in this function.
+
+    Returns:
+        submit_setup_dict:
+            A dictionary that will be passed on to
+            `FractalThreadPoolExecutor.submit` and
+            `FractalThreadPoolExecutor.map`, so as to set extra options.
+    """
+
+    local_backend_config = get_local_backend_config(
+        wftask=wftask,
+        which_type=which_type,
+    )
+
+    # Get TaskFiles object
+    task_files = TaskFiles(
+        root_dir_local=root_dir_local,
+        root_dir_remote=root_dir_remote,
+        task_order=wftask.order,
+        task_name=wftask.task.name,
+    )
+
+    return dict(
+        local_backend_config=local_backend_config,
+        task_files=task_files,
+    )