fractal-server 2.14.11__tar.gz → 2.14.13__tar.gz

{fractal_server-2.14.11 → fractal_server-2.14.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.11
+Version: 2.14.13
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.13/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.13"

fractal_server-2.14.13/fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py

@@ -0,0 +1,163 @@
+import itertools
+
+from sqlmodel import select
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models import LinkUserGroup
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.exceptions import UnreachableBranchError
+from fractal_server.logger import set_logger
+
+
+logger = set_logger(__name__)
+
+
+async def _disambiguate_task_groups(
+    *,
+    matching_task_groups: list[TaskGroupV2],
+    user_id: int,
+    default_group_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2 | None:
+    """
+    Find ownership-based top-priority task group, if any.
+
+    Args:
+        matching_task_groups:
+        user_id:
+        default_group_id:
+        db:
+
+    Returns:
+        The task group or `None`.
+    """
+
+    # Highest priority: task groups created by user
+    list_user_ids = [tg.user_id for tg in matching_task_groups]
+    try:
+        ind_user_id = list_user_ids.index(user_id)
+        task_group = matching_task_groups[ind_user_id]
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            f"Found task group {task_group.id} with {user_id=}, return."
+        )
+        return task_group
+    except ValueError:
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            f"No task group with {user_id=}, continue."
+        )
+
+    # Medium priority: task groups owned by default user group
+    list_user_group_ids = [tg.user_group_id for tg in matching_task_groups]
+    try:
+        ind_user_group_id = list_user_group_ids.index(default_group_id)
+        task_group = matching_task_groups[ind_user_group_id]
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            f"Found task group {task_group.id} with {user_id=}, return."
+        )
+        return task_group
+    except ValueError:
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            "No task group with user_group_id="
+            f"{default_group_id}, continue."
+        )
+
+    # Lowest priority: task groups owned by other groups, sorted
+    # according to age of the user/usergroup link
+    logger.debug(
+        "[_disambiguate_task_groups] "
+        "Sort remaining task groups by oldest-user-link."
+    )
+    stm = (
+        select(LinkUserGroup.group_id)
+        .where(LinkUserGroup.user_id == user_id)
+        .where(LinkUserGroup.group_id.in_(list_user_group_ids))
+        .order_by(LinkUserGroup.timestamp_created.asc())
+    )
+    res = await db.execute(stm)
+    oldest_user_group_id = res.scalars().first()
+    logger.debug(
+        "[_disambiguate_task_groups] " f"Result: {oldest_user_group_id=}."
+    )
+    task_group = next(
+        iter(
+            task_group
+            for task_group in matching_task_groups
+            if task_group.user_group_id == oldest_user_group_id
+        ),
+        None,
+    )
+    return task_group
+
+
+async def _disambiguate_task_groups_not_none(
+    *,
+    matching_task_groups: list[TaskGroupV2],
+    user_id: int,
+    default_group_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2:
+    """
+    Find ownership-based top-priority task group, and fail otherwise.
+
+    Args:
+        matching_task_groups:
+        user_id:
+        default_group_id:
+        db:
+
+    Returns:
+        The top-priority task group.
+    """
+    task_group = await _disambiguate_task_groups(
+        matching_task_groups=matching_task_groups,
+        user_id=user_id,
+        default_group_id=default_group_id,
+        db=db,
+    )
+    if task_group is None:
+        error_msg = (
+            "[_disambiguate_task_groups_not_none] Could not find a task "
+            f"group ({user_id=}, {default_group_id=})."
+        )
+        logger.error(f"UnreachableBranchError {error_msg}")
+        raise UnreachableBranchError(error_msg)
+    else:
+        return task_group
+
+
+async def remove_duplicate_task_groups(
+    *,
+    task_groups: list[TaskGroupV2],
+    user_id: int,
+    default_group_id: int,
+    db: AsyncSession,
+) -> list[TaskGroupV2]:
+    """
+    Extract a single task group for each `version`.
+
+    Args:
+        task_groups: A list of task groups with identical `pkg_name`
+        user_id: User ID
+
+    Returns:
+        New list of task groups with no duplicate `(pkg_name,version)` entries
+    """
+
+    new_task_groups = [
+        (
+            await _disambiguate_task_groups_not_none(
+                matching_task_groups=list(groups),
+                user_id=user_id,
+                default_group_id=default_group_id,
+                db=db,
+            )
+        )
+        for version, groups in itertools.groupby(
+            task_groups, key=lambda tg: tg.version
+        )
+    ]
+    return new_task_groups

{fractal_server-2.14.11 → fractal_server-2.14.13}/fractal_server/app/routes/api/v2/history.py

@@ -1,5 +1,4 @@
 from copy import deepcopy
-from typing import Any
 
 from fastapi import APIRouter
 from fastapi import Depends
@@ -16,6 +15,7 @@ from ._aux_functions_history import get_history_run_or_404
 from ._aux_functions_history import get_history_unit_or_404
 from ._aux_functions_history import get_wftask_check_owner
 from ._aux_functions_history import read_log_file
+from .images import ImagePage
 from .images import ImageQuery
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
@@ -34,13 +34,11 @@ from fractal_server.app.schemas.v2 import HistoryUnitRead
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import HistoryUnitStatusWithUnset
 from fractal_server.app.schemas.v2 import ImageLogsRequest
-from fractal_server.app.schemas.v2 import SingleImageWithStatus
-from fractal_server.images.image_status import enrich_image_list
-from fractal_server.images.image_status import IMAGE_STATUS_KEY
+from fractal_server.images.status_tools import enrich_images_async
+from fractal_server.images.status_tools import IMAGE_STATUS_KEY
 from fractal_server.images.tools import aggregate_attributes
 from fractal_server.images.tools import aggregate_types
 from fractal_server.images.tools import filter_image_list
-from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
 
 
@@ -62,11 +60,6 @@ def check_historyrun_related_to_dataset_and_wftask(
         )
 
 
-class ImageWithStatusPage(PaginationResponse[SingleImageWithStatus]):
-    attributes: dict[str, list[Any]]
-    types: list[str]
-
-
 router = APIRouter()
 logger = set_logger(__name__)
 
@@ -299,11 +292,11 @@ async def get_history_images(
     dataset_id: int,
     workflowtask_id: int,
     request_body: ImageQuery,
-    unit_status: HistoryUnitStatusWithUnset | None = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
-) -> ImageWithStatusPage:
+) -> ImagePage:
+
     # Access control and object retrieval
     wftask = await get_wftask_check_owner(
         project_id=project_id,
@@ -320,82 +313,54 @@ async def get_history_images(
         db=db,
     )
     dataset = res["dataset"]
-    workflow = res["workflow"]
 
     # Setup prefix for logging
-    prefix = f"[DS{dataset.id}-WFT{wftask.id}-images]"
-
-    # (1) Get the type-filtered list of dataset images
-
-    # (1A) Reconstruct dataset type filters by starting from {} and making
-    # incremental updates with `output_types` of all previous tasks
-    inferred_dataset_type_filters = {}
-    for current_wftask in workflow.task_list[0 : wftask.order]:
-        inferred_dataset_type_filters.update(current_wftask.task.output_types)
-    logger.debug(f"{prefix} {inferred_dataset_type_filters=}")
-    # (1B) Compute type filters for the current wftask
-    type_filters_patch = merge_type_filters(
-        task_input_types=wftask.task.input_types,
-        wftask_type_filters=wftask.type_filters,
-    )
-    logger.debug(f"{prefix} {type_filters_patch=}")
-    # (1C) Combine dataset type filters (lower priority) and current-wftask
-    # filters (higher priority)
-    actual_filters = inferred_dataset_type_filters
-    actual_filters.update(type_filters_patch)
-    logger.debug(f"{prefix} {actual_filters=}")
-    # (1D) Get all matching images from the dataset
-
-    pre_filtered_dataset_images = filter_image_list(
+    prefix = f"[DS{dataset.id}-WFT{workflowtask_id}-images]"
+
+    # (1) Apply type filters
+    type_filtered_images = filter_image_list(
         images=dataset.images,
-        type_filters=inferred_dataset_type_filters,
+        type_filters=request_body.type_filters,
     )
 
-    full_images_list = await enrich_image_list(
+    # (2) Extract valid values for attributes and types
+    attributes = aggregate_attributes(type_filtered_images)
+    attributes[IMAGE_STATUS_KEY] = [
+        HistoryUnitStatusWithUnset.DONE,
+        HistoryUnitStatusWithUnset.SUBMITTED,
+        HistoryUnitStatusWithUnset.FAILED,
+        HistoryUnitStatusWithUnset.UNSET,
+    ]
+    types = aggregate_types(type_filtered_images)
+
+    # (3) Enrich images with status attribute
+    type_filtered_images_with_status = await enrich_images_async(
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
-        images=pre_filtered_dataset_images,
+        images=type_filtered_images,
         db=db,
     )
 
-    if unit_status is not None:
-        request_body.attribute_filters[IMAGE_STATUS_KEY] = unit_status
-
-    filtered_dataset_images = filter_image_list(
-        full_images_list,
-        type_filters=request_body.type_filters,
+    # (4) Apply attribute filters
+    final_images_with_status = filter_image_list(
+        type_filtered_images_with_status,
         attribute_filters=request_body.attribute_filters,
     )
-    logger.debug(f"{prefix} {len(dataset.images)=}")
-    logger.debug(f"{prefix} {len(filtered_dataset_images)=}")
-
-    attributes = aggregate_attributes(pre_filtered_dataset_images)
-    types = aggregate_types(pre_filtered_dataset_images)
 
-    # Final list of objects
+    logger.debug(f"{prefix} {len(dataset.images)=}")
+    logger.debug(f"{prefix} {len(final_images_with_status)=}")
 
-    total_count = len(filtered_dataset_images)
+    # (5) Apply pagination logic
+    total_count = len(final_images_with_status)
     page_size = pagination.page_size or total_count
     sorted_images_list = sorted(
-        filtered_dataset_images,
+        final_images_with_status,
         key=lambda image: image["zarr_url"],
     )
     paginated_images_list = sorted_images_list[
         (pagination.page - 1) * page_size : pagination.page * page_size
     ]
 
-    # FIXME: This is only for backwards-compatibility. To remove when we
-    # update the webclient
-    paginated_images_list = [
-        {
-            **img,
-            "status": (
-                lambda x: None if x == HistoryUnitStatusWithUnset.UNSET else x
-            )(img["attributes"].pop(IMAGE_STATUS_KEY)),
-        }
-        for img in paginated_images_list
-    ]
-
     return dict(
         current_page=pagination.page,
         page_size=page_size,

{fractal_server-2.14.11 → fractal_server-2.14.13}/fractal_server/app/routes/api/v2/images.py

@@ -1,5 +1,3 @@
-from typing import Any
-
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -26,6 +24,7 @@ from fractal_server.images.tools import aggregate_types
 from fractal_server.images.tools import find_image_by_zarr_url
 from fractal_server.images.tools import match_filter
 from fractal_server.types import AttributeFilters
+from fractal_server.types import ImageAttributeValue
 from fractal_server.types import TypeFilters
 
 router = APIRouter()
@@ -33,11 +32,19 @@ router = APIRouter()
 
 class ImagePage(PaginationResponse[SingleImage]):
 
-    attributes: dict[str, list[Any]]
+    attributes: dict[str, list[ImageAttributeValue]]
     types: list[str]
 
 
 class ImageQuery(BaseModel):
+    """
+    Query for a list of images.
+
+    Attributes:
+        type_filters:
+        attribute_filters:
+    """
+
     type_filters: TypeFilters = Field(default_factory=dict)
     attribute_filters: AttributeFilters = Field(default_factory=dict)
 

{fractal_server-2.14.11 → fractal_server-2.14.13}/fractal_server/app/routes/api/v2/task_group.py

@@ -1,8 +1,13 @@
+import itertools
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from packaging.version import InvalidVersion
+from packaging.version import parse
+from packaging.version import Version
 from pydantic.types import AwareDatetime
 from sqlmodel import or_
 from sqlmodel import select
@@ -10,6 +15,7 @@ from sqlmodel import select
 from ._aux_functions_tasks import _get_task_group_full_access
 from ._aux_functions_tasks import _get_task_group_read_access
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
+from ._aux_task_group_disambiguation import remove_duplicate_task_groups
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import LinkUserGroup
@@ -18,6 +24,7 @@ from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
@@ -33,6 +40,26 @@ router = APIRouter()
 logger = set_logger(__name__)
 
 
+def _version_sort_key(
+    task_group: TaskGroupV2,
+) -> tuple[int, Version | str | None]:
+    """
+    Returns a tuple used as (reverse) ordering key for TaskGroups in
+    `get_task_group_list`.
+    The TaskGroups with a parsable versions are the first in order,
+    sorted according to the sorting rules of packaging.version.Version.
+    Next in order we have the TaskGroups with non-null non-parsable versions,
+    sorted alphabetically.
+    Last we have the TaskGroups with null version.
+    """
+    if task_group.version is None:
+        return (0, task_group.version)
+    try:
+        return (2, parse(task_group.version))
+    except InvalidVersion:
+        return (1, task_group.version)
+
+
 @router.get("/activity/", response_model=list[TaskGroupActivityV2Read])
 async def get_task_group_activity_list(
     task_group_activity_id: int | None = None,
@@ -97,14 +124,14 @@ async def get_task_group_activity(
     return activity
 
 
-@router.get("/", response_model=list[TaskGroupReadV2])
+@router.get("/", response_model=list[tuple[str, list[TaskGroupReadV2]]])
 async def get_task_group_list(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     only_active: bool = False,
     only_owner: bool = False,
     args_schema: bool = True,
-) -> list[TaskGroupReadV2]:
+) -> list[tuple[str, list[TaskGroupReadV2]]]:
     """
     Get all accessible TaskGroups
     """
@@ -119,7 +146,7 @@ async def get_task_group_list(
                 )
             ),
         )
-    stm = select(TaskGroupV2).where(condition)
+    stm = select(TaskGroupV2).where(condition).order_by(TaskGroupV2.pkg_name)
     if only_active:
         stm = stm.where(TaskGroupV2.active)
 
@@ -132,7 +159,28 @@ async def get_task_group_list(
                 setattr(task, "args_schema_non_parallel", None)
                 setattr(task, "args_schema_parallel", None)
 
-    return task_groups
+    default_group_id = await _get_default_usergroup_id(db)
+    grouped_result = [
+        (
+            pkg_name,
+            sorted(
+                (
+                    await remove_duplicate_task_groups(
+                        task_groups=list(groups),
+                        user_id=user.id,
+                        default_group_id=default_group_id,
+                        db=db,
+                    )
+                ),
+                key=_version_sort_key,
+                reverse=True,
+            ),
+        )
+        for pkg_name, groups in itertools.groupby(
+            task_groups, key=lambda tg: tg.pkg_name
+        )
+    ]
+    return grouped_result
 
 
 @router.get("/{task_group_id}/", response_model=TaskGroupReadV2)

{fractal_server-2.14.11 → fractal_server-2.14.13}/fractal_server/app/routes/api/v2/workflow_import.py

@@ -21,6 +21,9 @@ from ._aux_functions_tasks import _check_type_filters_compatibility
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.routes.api.v2._aux_task_group_disambiguation import (
+    _disambiguate_task_groups,
+)
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
 from fractal_server.app.schemas.v2 import TaskImportV2
@@ -85,76 +88,6 @@ async def _get_task_by_source(
     return task_id
 
 
-async def _disambiguate_task_groups(
-    *,
-    matching_task_groups: list[TaskGroupV2],
-    user_id: int,
-    db: AsyncSession,
-    default_group_id: int,
-) -> TaskV2 | None:
-    """
-    Disambiguate task groups based on ownership information.
-    """
-    # Highest priority: task groups created by user
-    for task_group in matching_task_groups:
-        if task_group.user_id == user_id:
-            logger.info(
-                "[_disambiguate_task_groups] "
-                f"Found task group {task_group.id} with {user_id=}, return."
-            )
-            return task_group
-    logger.info(
-        "[_disambiguate_task_groups] "
-        f"No task group found with {user_id=}, continue."
-    )
-
-    # Medium priority: task groups owned by default user group
-    for task_group in matching_task_groups:
-        if task_group.user_group_id == default_group_id:
-            logger.info(
-                "[_disambiguate_task_groups] "
-                f"Found task group {task_group.id} with user_group_id="
-                f"{default_group_id}, return."
-            )
-            return task_group
-    logger.info(
-        "[_disambiguate_task_groups] "
-        "No task group found with user_group_id="
-        f"{default_group_id}, continue."
-    )
-
-    # Lowest priority: task groups owned by other groups, sorted
-    # according to age of the user/usergroup link
-    logger.info(
-        "[_disambiguate_task_groups] "
-        "Now sorting remaining task groups by oldest-user-link."
-    )
-    user_group_ids = [
-        task_group.user_group_id for task_group in matching_task_groups
-    ]
-    stm = (
-        select(LinkUserGroup.group_id)
-        .where(LinkUserGroup.user_id == user_id)
-        .where(LinkUserGroup.group_id.in_(user_group_ids))
-        .order_by(LinkUserGroup.timestamp_created.asc())
-    )
-    res = await db.execute(stm)
-    oldest_user_group_id = res.scalars().first()
-    logger.info(
-        "[_disambiguate_task_groups] "
-        f"Result of sorting: {oldest_user_group_id=}."
-    )
-    task_group = next(
-        iter(
-            task_group
-            for task_group in matching_task_groups
-            if task_group.user_group_id == oldest_user_group_id
-        ),
-        None,
-    )
-    return task_group
-
-
 async def _get_task_by_taskimport(
     *,
     task_import: TaskImportV2,

{fractal_server-2.14.11 → fractal_server-2.14.13}/fractal_server/app/runner/v2/runner.py

@@ -33,10 +33,23 @@ from fractal_server.app.runner.v2.db_tools import update_status_of_history_run
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import TaskDumpV2
 from fractal_server.app.schemas.v2 import TaskGroupDumpV2
+from fractal_server.images.status_tools import enrich_images_sync
+from fractal_server.images.status_tools import IMAGE_STATUS_KEY
 from fractal_server.images.tools import merge_type_filters
 from fractal_server.types import AttributeFilters
 
 
+def _remove_status_from_attributes(
+    images: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """
+    Drop attribute `IMAGE_STATUS_KEY` from all images.
+    """
+    images_copy = deepcopy(images)
+    [img["attributes"].pop(IMAGE_STATUS_KEY) for img in images_copy]
+    return images_copy
+
+
 def drop_none_attributes(attributes: dict[str, Any]) -> dict[str, Any]:
     # Unset attributes with `None` value
     non_none_attributes = {
@@ -106,7 +119,11 @@ def execute_tasks_v2(
     tmp_images = deepcopy(dataset.images)
     current_dataset_type_filters = copy(job_type_filters)
 
-    for wftask in wf_task_list:
+    ENRICH_IMAGES_WITH_STATUS: bool = (
+        IMAGE_STATUS_KEY in job_attribute_filters.keys()
+    )
+
+    for ind_wftask, wftask in enumerate(wf_task_list):
         task = wftask.task
         task_name = task.name
         logger.debug(f'SUBMIT {wftask.order}-th task (name="{task_name}")')
@@ -122,15 +139,22 @@ def execute_tasks_v2(
                 wftask_type_filters=wftask.type_filters,
             )
             type_filters.update(type_filters_patch)
+
+            if ind_wftask == 0 and ENRICH_IMAGES_WITH_STATUS:
+                # FIXME: Could this be done on `type_filtered_images`?
+                tmp_images = enrich_images_sync(
+                    images=tmp_images,
+                    dataset_id=dataset.id,
+                    workflowtask_id=wftask.id,
+                )
             type_filtered_images = filter_image_list(
                 images=tmp_images,
                 type_filters=type_filters,
-                attribute_filters=None,
             )
             num_available_images = len(type_filtered_images)
+
             filtered_images = filter_image_list(
                 images=type_filtered_images,
-                type_filters=None,
                 attribute_filters=job_attribute_filters,
             )
         else:
@@ -379,7 +403,11 @@ def execute_tasks_v2(
         with next(get_sync_db()) as db:
             # Write current dataset images into the database.
             db_dataset = db.get(DatasetV2, dataset.id)
-            db_dataset.images = tmp_images
+            if ENRICH_IMAGES_WITH_STATUS:
+
+                db_dataset.images = _remove_status_from_attributes(tmp_images)
+            else:
+                db_dataset.images = tmp_images
             flag_modified(db_dataset, "images")
             db.merge(db_dataset)
 

{fractal_server-2.14.11 → fractal_server-2.14.13}/fractal_server/app/schemas/v2/__init__.py

@@ -16,7 +16,6 @@ from .history import HistoryUnitRead  # noqa F401
 from .history import HistoryUnitStatus  # noqa F401
 from .history import HistoryUnitStatusWithUnset  # noqa F401
 from .history import ImageLogsRequest  # noqa F401
-from .history import SingleImageWithStatus  # noqa F401
 from .job import JobCreateV2  # noqa F401
 from .job import JobReadV2  # noqa F401
 from .job import JobStatusTypeV2  # noqa F401

{fractal_server-2.14.11 → fractal_server-2.14.13}/fractal_server/app/schemas/v2/history.py

@@ -6,8 +6,6 @@ from pydantic import AwareDatetime
 from pydantic import BaseModel
 from pydantic import field_serializer
 
-from ....images import SingleImage
-
 
 class HistoryUnitStatus(StrEnum):
     """
@@ -83,8 +81,3 @@ class ImageLogsRequest(BaseModel):
     workflowtask_id: int
     dataset_id: int
     zarr_url: str
-
-
-# FIXME: remove this when we update the webclient
-class SingleImageWithStatus(SingleImage):
-    status: HistoryUnitStatus | None = None

fractal_server-2.14.13/fractal_server/exceptions.py

@@ -0,0 +1,2 @@
+class UnreachableBranchError(RuntimeError):
+    pass