fractal-server 2.14.11__py3-none-any.whl → 2.14.13__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.11"
+__VERSION__ = "2.14.13"

fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py

@@ -0,0 +1,163 @@
+import itertools
+
+from sqlmodel import select
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models import LinkUserGroup
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.exceptions import UnreachableBranchError
+from fractal_server.logger import set_logger
+
+
+logger = set_logger(__name__)
+
+
+async def _disambiguate_task_groups(
+    *,
+    matching_task_groups: list[TaskGroupV2],
+    user_id: int,
+    default_group_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2 | None:
+    """
+    Find ownership-based top-priority task group, if any.
+
+    Args:
+        matching_task_groups:
+        user_id:
+        default_group_id:
+        db:
+
+    Returns:
+        The task group or `None`.
+    """
+
+    # Highest priority: task groups created by user
+    list_user_ids = [tg.user_id for tg in matching_task_groups]
+    try:
+        ind_user_id = list_user_ids.index(user_id)
+        task_group = matching_task_groups[ind_user_id]
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            f"Found task group {task_group.id} with {user_id=}, return."
+        )
+        return task_group
+    except ValueError:
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            f"No task group with {user_id=}, continue."
+        )
+
+    # Medium priority: task groups owned by default user group
+    list_user_group_ids = [tg.user_group_id for tg in matching_task_groups]
+    try:
+        ind_user_group_id = list_user_group_ids.index(default_group_id)
+        task_group = matching_task_groups[ind_user_group_id]
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            f"Found task group {task_group.id} with {user_id=}, return."
+        )
+        return task_group
+    except ValueError:
+        logger.debug(
+            "[_disambiguate_task_groups] "
+            "No task group with user_group_id="
+            f"{default_group_id}, continue."
+        )
+
+    # Lowest priority: task groups owned by other groups, sorted
+    # according to age of the user/usergroup link
+    logger.debug(
+        "[_disambiguate_task_groups] "
+        "Sort remaining task groups by oldest-user-link."
+    )
+    stm = (
+        select(LinkUserGroup.group_id)
+        .where(LinkUserGroup.user_id == user_id)
+        .where(LinkUserGroup.group_id.in_(list_user_group_ids))
+        .order_by(LinkUserGroup.timestamp_created.asc())
+    )
+    res = await db.execute(stm)
+    oldest_user_group_id = res.scalars().first()
+    logger.debug(
+        "[_disambiguate_task_groups] " f"Result: {oldest_user_group_id=}."
+    )
+    task_group = next(
+        iter(
+            task_group
+            for task_group in matching_task_groups
+            if task_group.user_group_id == oldest_user_group_id
+        ),
+        None,
+    )
+    return task_group
+
+
+async def _disambiguate_task_groups_not_none(
+    *,
+    matching_task_groups: list[TaskGroupV2],
+    user_id: int,
+    default_group_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2:
+    """
+    Find ownership-based top-priority task group, and fail otherwise.
+
+    Args:
+        matching_task_groups:
+        user_id:
+        default_group_id:
+        db:
+
+    Returns:
+        The top-priority task group.
+    """
+    task_group = await _disambiguate_task_groups(
+        matching_task_groups=matching_task_groups,
+        user_id=user_id,
+        default_group_id=default_group_id,
+        db=db,
+    )
+    if task_group is None:
+        error_msg = (
+            "[_disambiguate_task_groups_not_none] Could not find a task "
+            f"group ({user_id=}, {default_group_id=})."
+        )
+        logger.error(f"UnreachableBranchError {error_msg}")
+        raise UnreachableBranchError(error_msg)
+    else:
+        return task_group
+
+
+async def remove_duplicate_task_groups(
+    *,
+    task_groups: list[TaskGroupV2],
+    user_id: int,
+    default_group_id: int,
+    db: AsyncSession,
+) -> list[TaskGroupV2]:
+    """
+    Extract a single task group for each `version`.
+
+    Args:
+        task_groups: A list of task groups with identical `pkg_name`
+        user_id: User ID
+
+    Returns:
+        New list of task groups with no duplicate `(pkg_name,version)` entries
+    """
+
+    new_task_groups = [
+        (
+            await _disambiguate_task_groups_not_none(
+                matching_task_groups=list(groups),
+                user_id=user_id,
+                default_group_id=default_group_id,
+                db=db,
+            )
+        )
+        for version, groups in itertools.groupby(
+            task_groups, key=lambda tg: tg.version
+        )
+    ]
+    return new_task_groups

fractal_server/app/routes/api/v2/history.py

@@ -1,5 +1,4 @@
 from copy import deepcopy
-from typing import Any
 
 from fastapi import APIRouter
 from fastapi import Depends
@@ -16,6 +15,7 @@ from ._aux_functions_history import get_history_run_or_404
 from ._aux_functions_history import get_history_unit_or_404
 from ._aux_functions_history import get_wftask_check_owner
 from ._aux_functions_history import read_log_file
+from .images import ImagePage
 from .images import ImageQuery
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
@@ -34,13 +34,11 @@ from fractal_server.app.schemas.v2 import HistoryUnitRead
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import HistoryUnitStatusWithUnset
 from fractal_server.app.schemas.v2 import ImageLogsRequest
-from fractal_server.app.schemas.v2 import SingleImageWithStatus
-from fractal_server.images.image_status import enrich_image_list
-from fractal_server.images.image_status import IMAGE_STATUS_KEY
+from fractal_server.images.status_tools import enrich_images_async
+from fractal_server.images.status_tools import IMAGE_STATUS_KEY
 from fractal_server.images.tools import aggregate_attributes
 from fractal_server.images.tools import aggregate_types
 from fractal_server.images.tools import filter_image_list
-from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
 
 
@@ -62,11 +60,6 @@ def check_historyrun_related_to_dataset_and_wftask(
         )
 
 
-class ImageWithStatusPage(PaginationResponse[SingleImageWithStatus]):
-    attributes: dict[str, list[Any]]
-    types: list[str]
-
-
 router = APIRouter()
 logger = set_logger(__name__)
 
@@ -299,11 +292,11 @@ async def get_history_images(
     dataset_id: int,
     workflowtask_id: int,
     request_body: ImageQuery,
-    unit_status: HistoryUnitStatusWithUnset | None = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
-) -> ImageWithStatusPage:
+) -> ImagePage:
+
     # Access control and object retrieval
     wftask = await get_wftask_check_owner(
         project_id=project_id,
@@ -320,82 +313,54 @@ async def get_history_images(
         db=db,
     )
     dataset = res["dataset"]
-    workflow = res["workflow"]
 
     # Setup prefix for logging
-    prefix = f"[DS{dataset.id}-WFT{wftask.id}-images]"
-
-    # (1) Get the type-filtered list of dataset images
-
-    # (1A) Reconstruct dataset type filters by starting from {} and making
-    # incremental updates with `output_types` of all previous tasks
-    inferred_dataset_type_filters = {}
-    for current_wftask in workflow.task_list[0 : wftask.order]:
-        inferred_dataset_type_filters.update(current_wftask.task.output_types)
-    logger.debug(f"{prefix} {inferred_dataset_type_filters=}")
-    # (1B) Compute type filters for the current wftask
-    type_filters_patch = merge_type_filters(
-        task_input_types=wftask.task.input_types,
-        wftask_type_filters=wftask.type_filters,
-    )
-    logger.debug(f"{prefix} {type_filters_patch=}")
-    # (1C) Combine dataset type filters (lower priority) and current-wftask
-    # filters (higher priority)
-    actual_filters = inferred_dataset_type_filters
-    actual_filters.update(type_filters_patch)
-    logger.debug(f"{prefix} {actual_filters=}")
-    # (1D) Get all matching images from the dataset
-
-    pre_filtered_dataset_images = filter_image_list(
+    prefix = f"[DS{dataset.id}-WFT{workflowtask_id}-images]"
+
+    # (1) Apply type filters
+    type_filtered_images = filter_image_list(
         images=dataset.images,
-        type_filters=inferred_dataset_type_filters,
+        type_filters=request_body.type_filters,
     )
 
-    full_images_list = await enrich_image_list(
+    # (2) Extract valid values for attributes and types
+    attributes = aggregate_attributes(type_filtered_images)
+    attributes[IMAGE_STATUS_KEY] = [
+        HistoryUnitStatusWithUnset.DONE,
+        HistoryUnitStatusWithUnset.SUBMITTED,
+        HistoryUnitStatusWithUnset.FAILED,
+        HistoryUnitStatusWithUnset.UNSET,
+    ]
+    types = aggregate_types(type_filtered_images)
+
+    # (3) Enrich images with status attribute
+    type_filtered_images_with_status = await enrich_images_async(
         dataset_id=dataset_id,
         workflowtask_id=workflowtask_id,
-        images=pre_filtered_dataset_images,
+        images=type_filtered_images,
         db=db,
     )
 
-    if unit_status is not None:
-        request_body.attribute_filters[IMAGE_STATUS_KEY] = unit_status
-
-    filtered_dataset_images = filter_image_list(
-        full_images_list,
-        type_filters=request_body.type_filters,
+    # (4) Apply attribute filters
+    final_images_with_status = filter_image_list(
+        type_filtered_images_with_status,
         attribute_filters=request_body.attribute_filters,
     )
-    logger.debug(f"{prefix} {len(dataset.images)=}")
-    logger.debug(f"{prefix} {len(filtered_dataset_images)=}")
-
-    attributes = aggregate_attributes(pre_filtered_dataset_images)
-    types = aggregate_types(pre_filtered_dataset_images)
 
-    # Final list of objects
+    logger.debug(f"{prefix} {len(dataset.images)=}")
+    logger.debug(f"{prefix} {len(final_images_with_status)=}")
 
-    total_count = len(filtered_dataset_images)
+    # (5) Apply pagination logic
+    total_count = len(final_images_with_status)
     page_size = pagination.page_size or total_count
     sorted_images_list = sorted(
-        filtered_dataset_images,
+        final_images_with_status,
         key=lambda image: image["zarr_url"],
     )
     paginated_images_list = sorted_images_list[
         (pagination.page - 1) * page_size : pagination.page * page_size
     ]
 
-    # FIXME: This is only for backwards-compatibility. To remove when we
-    # update the webclient
-    paginated_images_list = [
-        {
-            **img,
-            "status": (
-                lambda x: None if x == HistoryUnitStatusWithUnset.UNSET else x
-            )(img["attributes"].pop(IMAGE_STATUS_KEY)),
-        }
-        for img in paginated_images_list
-    ]
-
     return dict(
         current_page=pagination.page,
         page_size=page_size,

fractal_server/app/routes/api/v2/images.py

@@ -1,5 +1,3 @@
-from typing import Any
-
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -26,6 +24,7 @@ from fractal_server.images.tools import aggregate_types
 from fractal_server.images.tools import find_image_by_zarr_url
 from fractal_server.images.tools import match_filter
 from fractal_server.types import AttributeFilters
+from fractal_server.types import ImageAttributeValue
 from fractal_server.types import TypeFilters
 
 router = APIRouter()
@@ -33,11 +32,19 @@ router = APIRouter()
 
 class ImagePage(PaginationResponse[SingleImage]):
 
-    attributes: dict[str, list[Any]]
+    attributes: dict[str, list[ImageAttributeValue]]
     types: list[str]
 
 
 class ImageQuery(BaseModel):
+    """
+    Query for a list of images.
+
+    Attributes:
+        type_filters:
+        attribute_filters:
+    """
+
     type_filters: TypeFilters = Field(default_factory=dict)
     attribute_filters: AttributeFilters = Field(default_factory=dict)
 

fractal_server/app/routes/api/v2/task_group.py

@@ -1,8 +1,13 @@
+import itertools
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from packaging.version import InvalidVersion
+from packaging.version import parse
+from packaging.version import Version
 from pydantic.types import AwareDatetime
 from sqlmodel import or_
 from sqlmodel import select
@@ -10,6 +15,7 @@ from sqlmodel import select
 from ._aux_functions_tasks import _get_task_group_full_access
 from ._aux_functions_tasks import _get_task_group_read_access
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
+from ._aux_task_group_disambiguation import remove_duplicate_task_groups
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import LinkUserGroup
@@ -18,6 +24,7 @@ from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
@@ -33,6 +40,26 @@ router = APIRouter()
 logger = set_logger(__name__)
 
 
+def _version_sort_key(
+    task_group: TaskGroupV2,
+) -> tuple[int, Version | str | None]:
+    """
+    Returns a tuple used as (reverse) ordering key for TaskGroups in
+    `get_task_group_list`.
+    The TaskGroups with a parsable versions are the first in order,
+    sorted according to the sorting rules of packaging.version.Version.
+    Next in order we have the TaskGroups with non-null non-parsable versions,
+    sorted alphabetically.
+    Last we have the TaskGroups with null version.
+    """
+    if task_group.version is None:
+        return (0, task_group.version)
+    try:
+        return (2, parse(task_group.version))
+    except InvalidVersion:
+        return (1, task_group.version)
+
+
 @router.get("/activity/", response_model=list[TaskGroupActivityV2Read])
 async def get_task_group_activity_list(
     task_group_activity_id: int | None = None,
@@ -97,14 +124,14 @@ async def get_task_group_activity(
     return activity
 
 
-@router.get("/", response_model=list[TaskGroupReadV2])
+@router.get("/", response_model=list[tuple[str, list[TaskGroupReadV2]]])
 async def get_task_group_list(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     only_active: bool = False,
     only_owner: bool = False,
     args_schema: bool = True,
-) -> list[TaskGroupReadV2]:
+) -> list[tuple[str, list[TaskGroupReadV2]]]:
     """
     Get all accessible TaskGroups
     """
@@ -119,7 +146,7 @@ async def get_task_group_list(
                 )
             ),
         )
-    stm = select(TaskGroupV2).where(condition)
+    stm = select(TaskGroupV2).where(condition).order_by(TaskGroupV2.pkg_name)
     if only_active:
         stm = stm.where(TaskGroupV2.active)
 
@@ -132,7 +159,28 @@ async def get_task_group_list(
                 setattr(task, "args_schema_non_parallel", None)
                 setattr(task, "args_schema_parallel", None)
 
-    return task_groups
+    default_group_id = await _get_default_usergroup_id(db)
+    grouped_result = [
+        (
+            pkg_name,
+            sorted(
+                (
+                    await remove_duplicate_task_groups(
+                        task_groups=list(groups),
+                        user_id=user.id,
+                        default_group_id=default_group_id,
+                        db=db,
+                    )
+                ),
+                key=_version_sort_key,
+                reverse=True,
+            ),
+        )
+        for pkg_name, groups in itertools.groupby(
+            task_groups, key=lambda tg: tg.pkg_name
+        )
+    ]
+    return grouped_result
 
 
 @router.get("/{task_group_id}/", response_model=TaskGroupReadV2)

fractal_server/app/routes/api/v2/workflow_import.py

@@ -21,6 +21,9 @@ from ._aux_functions_tasks import _check_type_filters_compatibility
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.routes.api.v2._aux_task_group_disambiguation import (
+    _disambiguate_task_groups,
+)
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
 from fractal_server.app.schemas.v2 import TaskImportV2
@@ -85,76 +88,6 @@ async def _get_task_by_source(
     return task_id
 
 
-async def _disambiguate_task_groups(
-    *,
-    matching_task_groups: list[TaskGroupV2],
-    user_id: int,
-    db: AsyncSession,
-    default_group_id: int,
-) -> TaskV2 | None:
-    """
-    Disambiguate task groups based on ownership information.
-    """
-    # Highest priority: task groups created by user
-    for task_group in matching_task_groups:
-        if task_group.user_id == user_id:
-            logger.info(
-                "[_disambiguate_task_groups] "
-                f"Found task group {task_group.id} with {user_id=}, return."
-            )
-            return task_group
-    logger.info(
-        "[_disambiguate_task_groups] "
-        f"No task group found with {user_id=}, continue."
-    )
-
-    # Medium priority: task groups owned by default user group
-    for task_group in matching_task_groups:
-        if task_group.user_group_id == default_group_id:
-            logger.info(
-                "[_disambiguate_task_groups] "
-                f"Found task group {task_group.id} with user_group_id="
-                f"{default_group_id}, return."
-            )
-            return task_group
-    logger.info(
-        "[_disambiguate_task_groups] "
-        "No task group found with user_group_id="
-        f"{default_group_id}, continue."
-    )
-
-    # Lowest priority: task groups owned by other groups, sorted
-    # according to age of the user/usergroup link
-    logger.info(
-        "[_disambiguate_task_groups] "
-        "Now sorting remaining task groups by oldest-user-link."
-    )
-    user_group_ids = [
-        task_group.user_group_id for task_group in matching_task_groups
-    ]
-    stm = (
-        select(LinkUserGroup.group_id)
-        .where(LinkUserGroup.user_id == user_id)
-        .where(LinkUserGroup.group_id.in_(user_group_ids))
-        .order_by(LinkUserGroup.timestamp_created.asc())
-    )
-    res = await db.execute(stm)
-    oldest_user_group_id = res.scalars().first()
-    logger.info(
-        "[_disambiguate_task_groups] "
-        f"Result of sorting: {oldest_user_group_id=}."
-    )
-    task_group = next(
-        iter(
-            task_group
-            for task_group in matching_task_groups
-            if task_group.user_group_id == oldest_user_group_id
-        ),
-        None,
-    )
-    return task_group
-
-
 async def _get_task_by_taskimport(
     *,
     task_import: TaskImportV2,

fractal_server/app/runner/v2/runner.py

@@ -33,10 +33,23 @@ from fractal_server.app.runner.v2.db_tools import update_status_of_history_run
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import TaskDumpV2
 from fractal_server.app.schemas.v2 import TaskGroupDumpV2
+from fractal_server.images.status_tools import enrich_images_sync
+from fractal_server.images.status_tools import IMAGE_STATUS_KEY
 from fractal_server.images.tools import merge_type_filters
 from fractal_server.types import AttributeFilters
 
 
+def _remove_status_from_attributes(
+    images: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """
+    Drop attribute `IMAGE_STATUS_KEY` from all images.
+    """
+    images_copy = deepcopy(images)
+    [img["attributes"].pop(IMAGE_STATUS_KEY) for img in images_copy]
+    return images_copy
+
+
 def drop_none_attributes(attributes: dict[str, Any]) -> dict[str, Any]:
     # Unset attributes with `None` value
     non_none_attributes = {
@@ -106,7 +119,11 @@ def execute_tasks_v2(
     tmp_images = deepcopy(dataset.images)
     current_dataset_type_filters = copy(job_type_filters)
 
-    for wftask in wf_task_list:
+    ENRICH_IMAGES_WITH_STATUS: bool = (
+        IMAGE_STATUS_KEY in job_attribute_filters.keys()
+    )
+
+    for ind_wftask, wftask in enumerate(wf_task_list):
         task = wftask.task
         task_name = task.name
         logger.debug(f'SUBMIT {wftask.order}-th task (name="{task_name}")')
@@ -122,15 +139,22 @@ def execute_tasks_v2(
                 wftask_type_filters=wftask.type_filters,
             )
             type_filters.update(type_filters_patch)
+
+            if ind_wftask == 0 and ENRICH_IMAGES_WITH_STATUS:
+                # FIXME: Could this be done on `type_filtered_images`?
+                tmp_images = enrich_images_sync(
+                    images=tmp_images,
+                    dataset_id=dataset.id,
+                    workflowtask_id=wftask.id,
+                )
             type_filtered_images = filter_image_list(
                 images=tmp_images,
                 type_filters=type_filters,
-                attribute_filters=None,
             )
             num_available_images = len(type_filtered_images)
+
             filtered_images = filter_image_list(
                 images=type_filtered_images,
-                type_filters=None,
                 attribute_filters=job_attribute_filters,
             )
         else:
@@ -379,7 +403,11 @@ def execute_tasks_v2(
         with next(get_sync_db()) as db:
             # Write current dataset images into the database.
             db_dataset = db.get(DatasetV2, dataset.id)
-            db_dataset.images = tmp_images
+            if ENRICH_IMAGES_WITH_STATUS:
+
+                db_dataset.images = _remove_status_from_attributes(tmp_images)
+            else:
+                db_dataset.images = tmp_images
             flag_modified(db_dataset, "images")
             db.merge(db_dataset)
 

fractal_server/app/schemas/v2/__init__.py

@@ -16,7 +16,6 @@ from .history import HistoryUnitRead  # noqa F401
 from .history import HistoryUnitStatus  # noqa F401
 from .history import HistoryUnitStatusWithUnset  # noqa F401
 from .history import ImageLogsRequest  # noqa F401
-from .history import SingleImageWithStatus  # noqa F401
 from .job import JobCreateV2  # noqa F401
 from .job import JobReadV2  # noqa F401
 from .job import JobStatusTypeV2  # noqa F401

fractal_server/app/schemas/v2/history.py

@@ -6,8 +6,6 @@ from pydantic import AwareDatetime
 from pydantic import BaseModel
 from pydantic import field_serializer
 
-from ....images import SingleImage
-
 
 class HistoryUnitStatus(StrEnum):
     """
@@ -83,8 +81,3 @@ class ImageLogsRequest(BaseModel):
     workflowtask_id: int
     dataset_id: int
     zarr_url: str
-
-
-# FIXME: remove this when we update the webclient
-class SingleImageWithStatus(SingleImage):
-    status: HistoryUnitStatus | None = None

fractal_server/exceptions.py

@@ -0,0 +1,2 @@
+class UnreachableBranchError(RuntimeError):
+    pass

fractal_server/images/status_tools.py

@@ -0,0 +1,174 @@
+import time
+from copy import deepcopy
+from typing import Any
+
+from sqlalchemy import Select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models.v2 import HistoryImageCache
+from fractal_server.app.models.v2 import HistoryUnit
+from fractal_server.app.schemas.v2 import HistoryUnitStatusWithUnset
+from fractal_server.logger import set_logger
+from fractal_server.types import ImageAttributeValue
+
+logger = set_logger(__name__)
+
+
+IMAGE_STATUS_KEY = "__wftask_dataset_image_status__"
+
+
+def _enriched_image(*, img: dict[str, Any], status: str) -> dict[str, Any]:
+    img["attributes"][IMAGE_STATUS_KEY] = status
+    return img
+
+
+def _prepare_query(
+    *,
+    dataset_id: int,
+    workflowtask_id: int,
+    zarr_urls: list[str],
+) -> Select:
+    stm = (
+        select(HistoryImageCache.zarr_url, HistoryUnit.status)
+        .join(HistoryUnit)
+        .where(HistoryImageCache.dataset_id == dataset_id)
+        .where(HistoryImageCache.workflowtask_id == workflowtask_id)
+        .where(HistoryImageCache.latest_history_unit_id == HistoryUnit.id)
+        .where(HistoryImageCache.zarr_url.in_(zarr_urls))
+        .order_by(HistoryImageCache.zarr_url)
+    )
+    return stm
+
+
+async def enrich_images_async(
+    *,
+    images: list[dict[str, Any]],
+    dataset_id: int,
+    workflowtask_id: int,
+    db: AsyncSession,
+) -> list[dict[str, ImageAttributeValue]]:
+    """
+    Enrich images with a status-related attribute.
+
+    Args:
+        images: The input image list
+        dataset_id: The dataset ID
+        workflowtask_id: The workflow-task ID
+        db: An async db session
+
+    Returns:
+        The list of enriched images
+    """
+    t_0 = time.perf_counter()
+    logger.info(
+        f"[enrich_images_async] START, {dataset_id=}, {workflowtask_id=}"
+    )
+
+    zarr_url_to_image = {img["zarr_url"]: deepcopy(img) for img in images}
+
+    res = await db.execute(
+        _prepare_query(
+            dataset_id=dataset_id,
+            workflowtask_id=workflowtask_id,
+            zarr_urls=zarr_url_to_image.keys(),
+        )
+    )
+    list_processed_url_status = res.all()
+    t_1 = time.perf_counter()
+    logger.debug(f"[enrich_images_async] db-query, elapsed={t_1 - t_0:.3f} s")
+
+    set_processed_urls = set(item[0] for item in list_processed_url_status)
+    processed_images_with_status = [
+        _enriched_image(
+            img=zarr_url_to_image[item[0]],
+            status=item[1],
+        )
+        for item in list_processed_url_status
+    ]
+    t_2 = time.perf_counter()
+    logger.debug(
+        "[enrich_images_async] processed-images, " f"elapsed={t_2 - t_1:.3f} s"
+    )
+
+    non_processed_urls = zarr_url_to_image.keys() - set_processed_urls
+    non_processed_images_with_status = [
+        _enriched_image(
+            img=zarr_url_to_image[zarr_url],
+            status=HistoryUnitStatusWithUnset.UNSET,
+        )
+        for zarr_url in non_processed_urls
+    ]
+    t_3 = time.perf_counter()
+    logger.debug(
+        "[enrich_images_async] non-processed-images, "
+        f"elapsed={t_3 - t_2:.3f} s"
+    )
+
+    return processed_images_with_status + non_processed_images_with_status
+
+
+def enrich_images_sync(
+    *,
+    images: list[dict[str, Any]],
+    dataset_id: int,
+    workflowtask_id: int,
+) -> list[dict[str, ImageAttributeValue]]:
+    """
+    Enrich images with a status-related attribute.
+
+    Args:
+        images: The input image list
+        dataset_id: The dataset ID
+        workflowtask_id: The workflow-task ID
+
+    Returns:
+        The list of enriched images
+    """
+    t_0 = time.perf_counter()
+    logger.info(
+        f"[enrich_images_async] START, {dataset_id=}, {workflowtask_id=}"
+    )
+
+    zarr_url_to_image = {img["zarr_url"]: deepcopy(img) for img in images}
+    with next(get_sync_db()) as db:
+        res = db.execute(
+            _prepare_query(
+                dataset_id=dataset_id,
+                workflowtask_id=workflowtask_id,
+                zarr_urls=zarr_url_to_image.keys(),
+            )
+        )
+    list_processed_url_status = res.all()
+    t_1 = time.perf_counter()
+    logger.debug(f"[enrich_images_async] db-query, elapsed={t_1 - t_0:.3f} s")
+
+    set_processed_urls = set(item[0] for item in list_processed_url_status)
+    processed_images_with_status = [
+        _enriched_image(
+            img=zarr_url_to_image[item[0]],
+            status=item[1],
+        )
+        for item in list_processed_url_status
+    ]
+    t_2 = time.perf_counter()
+    logger.debug(
+        "[enrich_images_async] processed-images, " f"elapsed={t_2 - t_1:.3f} s"
+    )
+
+    non_processed_urls = zarr_url_to_image.keys() - set_processed_urls
+    non_processed_images_with_status = [
+        _enriched_image(
+            img=zarr_url_to_image[zarr_url],
+            status=HistoryUnitStatusWithUnset.UNSET,
+        )
+        for zarr_url in non_processed_urls
+    ]
+    t_3 = time.perf_counter()
+    logger.debug(
+        "[enrich_images_async] non-processed-images, "
+        f"elapsed={t_3 - t_2:.3f} s"
+    )
+
+    return processed_images_with_status + non_processed_images_with_status

fractal_server/images/tools.py

@@ -125,7 +125,7 @@ def merge_type_filters(
 def aggregate_attributes(images: list[dict[str, Any]]) -> dict[str, list[Any]]:
     """
     Given a list of images, this function returns a dictionary of all image
-    attributes, each mapped to a list of present values.
+    attributes, each mapped to a sorted list of existing values.
     """
     attributes = {}
     for image in images:
@@ -133,7 +133,10 @@ def aggregate_attributes(images: list[dict[str, Any]]) -> dict[str, list[Any]]:
             attributes.setdefault(k, []).append(v)
         for k, v in attributes.items():
             attributes[k] = list(set(v))
-    return attributes
+    sorted_attributes = {
+        key: sorted(value) for key, value in attributes.items()
+    }
+    return sorted_attributes
 
 
 def aggregate_types(images: list[dict[str, Any]]) -> list[str]:

fractal_server/tasks/v2/templates/2_pip_install.sh

@@ -24,11 +24,21 @@ write_log "END   upgrade pip and install setuptools"
 echo
 
 # Install package
-write_log "START install with INSTALL_STRING=${INSTALL_STRING} and PINNED_PACKAGE_LIST=${PINNED_PACKAGE_LIST}"
-"$VENVPYTHON" -m pip install ${FRACTAL_PIP_CACHE_DIR_ARG} "$INSTALL_STRING" $PINNED_PACKAGE_LIST
-write_log "END   install with INSTALL_STRING=${INSTALL_STRING} and PINNED_PACKAGE_LIST=${PINNED_PACKAGE_LIST}"
+write_log "START install with INSTALL_STRING=${INSTALL_STRING}"
+"$VENVPYTHON" -m pip install ${FRACTAL_PIP_CACHE_DIR_ARG} "$INSTALL_STRING"
+write_log "END   install with INSTALL_STRING=${INSTALL_STRING}"
 echo
 
+# Install pinned packages (note: do not quote $PINNED_PACKAGE_LIST since it could be e.g. "numpy==1.2.3 torch=3.2.1")
+if [ "$PINNED_PACKAGE_LIST" != "" ]; then
+    write_log "START install with PINNED_PACKAGE_LIST=${PINNED_PACKAGE_LIST}"
+    "$VENVPYTHON" -m pip install ${FRACTAL_PIP_CACHE_DIR_ARG} $PINNED_PACKAGE_LIST
+    write_log "END install with PINNED_PACKAGE_LIST=${PINNED_PACKAGE_LIST}"
+    echo
+else
+    write_log "SKIP installing pinned versions $PINNED_PACKAGE_LIST (empty list)"
+fi
+
 # End
 TIME_END=$(date +%s)
 write_log "All good up to here."

{fractal_server-2.14.11.dist-info → fractal_server-2.14.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.11
+Version: 2.14.13
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.14.11.dist-info → fractal_server-2.14.13.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=_nZVfV9ubhO5D3aYI-vZIXltETQy27u2BcyxMBu66Ik,24
+fractal_server/__init__.py,sha256=6bdBE5bJoA01yAEh1heVK-Ta-FFnpeeh531ZgEO8YXY,24
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,9 +35,10 @@ fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=Z23xwvBaVEEQ5B
 fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=GpKfw9yj01LmOAuNMTOreU1PFkCKpjK5oCt7_wp35-A,6741
 fractal_server/app/routes/api/v2/_aux_functions_task_version_update.py,sha256=WLDOYCnb6fnS5avKflyx6yN24Vo1n5kJk5ZyiKbzb8Y,1175
 fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=MFYnyNPBACSHXTDLXe6cSennnpmlpajN84iivOOMW7Y,11599
+fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py,sha256=2sK7-bZzcl3-2mkx62tw0MPxeUYVDch30DSWgdhouHI,4615
 fractal_server/app/routes/api/v2/dataset.py,sha256=6u4MFqJ3YZ0Zq6Xx8CRMrTPKW55ZaR63Uno21DqFr4Q,8889
-fractal_server/app/routes/api/v2/history.py,sha256=OHy3Y4lreGyGXk9v5iud73tzoGV-YAT027gMWJH-5p4,16958
-fractal_server/app/routes/api/v2/images.py,sha256=tJn0ANv4Tz2KHyb41sPbBRmSCpt632m8HEcgC3u-rHk,7709
+fractal_server/app/routes/api/v2/history.py,sha256=BEmf_ENF5HNMy8yXrxRdo4280rWuRUa1Jw4u8R9-LQQ,15477
+fractal_server/app/routes/api/v2/images.py,sha256=TS1ltUhP0_SaViupdHrSh3MLDi5OVk-lOhE1VCVyZj0,7869
 fractal_server/app/routes/api/v2/job.py,sha256=8xRTwh_OCHmK9IfI_zUASa2ozewR0qu0zVBl_a4IvHw,6467
 fractal_server/app/routes/api/v2/pre_submission_checks.py,sha256=MmjvSQ0pNAWEw5BavR16zIZ4h13py302AmToaz8Vvco,4768
 fractal_server/app/routes/api/v2/project.py,sha256=ldMEyjtwGpX2teu85sCNWaubDFlw-En8U1SA7G1VaIw,4567
@@ -46,11 +47,11 @@ fractal_server/app/routes/api/v2/submit.py,sha256=_BDkWtFdo8-p7kZ0Oxaidei04MfuBe
 fractal_server/app/routes/api/v2/task.py,sha256=cUFrCxFOLGlRV7UCbUMHs4Xy4tIc3pqwG8gEqVP5GcU,6939
 fractal_server/app/routes/api/v2/task_collection.py,sha256=FGMhTnU88Umd8nMdriUYPtpTtAHcRBRrZIYyOesFhrU,12577
 fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=EfGpv6W7xDyuYYp6E7XAcXLJiLNAImUHFqMDLgfh-4s,6730
-fractal_server/app/routes/api/v2/task_group.py,sha256=iShTvM9nJQhQLwR8ZpQRucVwYhJ7t00Lbesqh3M6mY4,7361
+fractal_server/app/routes/api/v2/task_group.py,sha256=Xfsj5Wy0NOIkbeYsdqyFke4mkaeq0riJeTrGCHbt-eM,9059
 fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=C2U2V76YbbqDWmErJ98MH9C2C26Lve2p_35FZ1dNmXg,9095
 fractal_server/app/routes/api/v2/task_version_update.py,sha256=h2c6aTLXj0_ZyBuHVsD5-ZTNMGEUpS96qZ4Ot1jlb74,7974
 fractal_server/app/routes/api/v2/workflow.py,sha256=gwMtpfUY_JiTv5_R_q1I9WNkp6nTqEVtYx8jWNJRxcU,10227
-fractal_server/app/routes/api/v2/workflow_import.py,sha256=Q4CnkSV47F11j6DkNT_U3AhwBK-LSsWWegItfdoOJ6c,11167
+fractal_server/app/routes/api/v2/workflow_import.py,sha256=kOGDaCj0jCGK1WSYGbnUjtUg2U1YxUY9UMH-2ilqJg4,9027
 fractal_server/app/routes/api/v2/workflowtask.py,sha256=vVqEoJa3lrMl2CU94WoxFaqO3U0QImPgvrkkUNdqDOU,7462
 fractal_server/app/routes/auth/__init__.py,sha256=fao6CS0WiAjHDTvBzgBVV_bSXFpEAeDBF6Z6q7rRkPc,1658
 fractal_server/app/routes/auth/_aux_auth.py,sha256=UZgauY0V6mSqjte_sYI1cBl2h8bcbLaeWzgpl1jdJlk,4883
@@ -101,7 +102,7 @@ fractal_server/app/runner/v2/_slurm_sudo.py,sha256=Gvsh4tUlc1_3KdF3B7zEqs-YIntC_
 fractal_server/app/runner/v2/db_tools.py,sha256=du5dKhMMFMErQXbGIgu9JvO_vtMensodyPsyDeqz1yQ,3324
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
 fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
-fractal_server/app/runner/v2/runner.py,sha256=YkxHhWA25Mv7HzviCaYMTxkyuP-JFHj73H9aPO6Fxm0,17713
+fractal_server/app/runner/v2/runner.py,sha256=_vmFdJCqNZVK9sQa7MoAIemkRX-hakLm5RLCBz8QxDg,18667
 fractal_server/app/runner/v2/runner_functions.py,sha256=Q9AVIR2NEBfRpfqW1wtQTTQfks_R1TnwRFBRro2fvjQ,18837
 fractal_server/app/runner/v2/submit_workflow.py,sha256=AMnXdozwIGlXD55ch0_SNAG-ntKBO-QRhkbInrvsShU,13140
 fractal_server/app/runner/v2/task_interface.py,sha256=V2TWBK6tbhycyMrJvFaoJ9IpuKlrLrvmjJbfNMsBBXo,2527
@@ -110,11 +111,11 @@ fractal_server/app/schemas/__init__.py,sha256=stURAU_t3AOBaH0HSUbV-GKhlPKngnnIMo
 fractal_server/app/schemas/user.py,sha256=t9nbyYjGCSOsxm9K97PDG3-9o27CsaFfhWb_L5nrjqA,1910
 fractal_server/app/schemas/user_group.py,sha256=x3-kqbo0q2wTP7QI0iZ7PU_9Dr957UYrFMKqS7BXLhE,1425
 fractal_server/app/schemas/user_settings.py,sha256=NpdC0Me0fgwwdfJuTSlFLCnLUjiWWzrJlPn_UPLjXnw,1862
-fractal_server/app/schemas/v2/__init__.py,sha256=M49RJ8SKcVoSfSTuiTCcbexSo8JMtLQTVFltCW4CuGQ,3103
+fractal_server/app/schemas/v2/__init__.py,sha256=ft9gFmLLClNbWE8pwmG81fuLl1hlfdSsb8TvLoX3Sqk,3047
 fractal_server/app/schemas/v2/accounting.py,sha256=Wylt7uWTiDIFlHJOh4XEtYitk2FjFlmnodDrJDxcr0E,397
 fractal_server/app/schemas/v2/dataset.py,sha256=NKCjBwGBC7mPiSlXktZAcleJsvlLY6KfNKw7Wx4Zfqk,1728
 fractal_server/app/schemas/v2/dumps.py,sha256=o4RiWoSmQ8UPoWxgKoeORykGNIdczeNmm-ng-dBRD7k,2216
-fractal_server/app/schemas/v2/history.py,sha256=BCOii7GW7OvcDr1mLZl5-kYxtczzefQciuAxp95zrFk,1958
+fractal_server/app/schemas/v2/history.py,sha256=pZiMKfh6nMWbTp5MUtrnGySPKbeRFf5tM1VLFaTgGcw,1784
 fractal_server/app/schemas/v2/job.py,sha256=fPay7dLSr-skKRdVRoZig8rf_sZwUdVdHZaJ4XM8vMI,3288
 fractal_server/app/schemas/v2/manifest.py,sha256=sZhj99iDgjE2MWXeTxnXSb6pFdKwRnFpCVQzcnpoTrI,6821
 fractal_server/app/schemas/v2/project.py,sha256=l96-3bCfB3knhITaLj1WSyBgbzP_k8CdtvgX_5jO_fU,657
@@ -131,11 +132,12 @@ fractal_server/config.py,sha256=ldI9VzEWmwU75Z7zVku6I-rXGKS3bJDdCifZnwad9-4,2592
 fractal_server/data_migrations/2_14_10.py,sha256=gMRR5QB0SDv0ToEiXVLg1VrHprM_Ii-9O1Kg-ZF-YhY,1599
 fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
 fractal_server/data_migrations/tools.py,sha256=LeMeASwYGtEqd-3wOLle6WARdTGAimoyMmRbbJl-hAM,572
+fractal_server/exceptions.py,sha256=7ftpWwNsTQmNonWCynhH5ErUh1haPPhIaVPrNHla7-o,53
 fractal_server/gunicorn_fractal.py,sha256=u6U01TLGlXgq1v8QmEpLih3QnsInZD7CqphgJ_GrGzc,1230
 fractal_server/images/__init__.py,sha256=-_wjoKtSX02P1KjDxDP_EXKvmbONTRmbf7iGVTsyBpM,154
-fractal_server/images/image_status.py,sha256=2EUxvuYEFKSKoICEiQntT5P3Un7FS8gv2Ef1HZFUfF4,2667
 fractal_server/images/models.py,sha256=6WchcIzLLLwdkLNRfg71Dl4Y-9UFLPyrrzh1lWgjuP0,1245
-fractal_server/images/tools.py,sha256=XKhbdjfWZpTSe1akK1bSQl4gzEQlj9ETDbELkuwayVg,4066
+fractal_server/images/status_tools.py,sha256=tLp-Sojlhf-eQ97O1hj-2fg2zmgHfED9EXkec3Jjz_0,5141
+fractal_server/images/tools.py,sha256=92kmt2Fnyp8ycTbyuar9_U8kJTi0wKpBk8ZagARWl9Y,4177
 fractal_server/logger.py,sha256=QIeVn3QpZsiIL2jDdrKotr-MLyDcZYgiPiTluFU46lE,5317
 fractal_server/main.py,sha256=FD9KzTTsXTQnTW0z3Hu7y0Nj_oAkBeZEInKDXFd4hjE,4561
 fractal_server/migrations/env.py,sha256=nfyBpMIOT3kny6t-b-tUjyRjZ4k906bb1_wCQ7me1BI,1353
@@ -196,7 +198,7 @@ fractal_server/tasks/v2/ssh/collect.py,sha256=bClq8hB04igrUXk1Mgc7pRWQws77mpYX8K
 fractal_server/tasks/v2/ssh/deactivate.py,sha256=YO2PJ0VV-LhVW-6O-t-d6BQciO2fYAkYbz5Y9UBiXaA,12928
 fractal_server/tasks/v2/ssh/reactivate.py,sha256=1DIQduhqZLbrIeoVyyp54vemBWZu94tFDvjpmDsZZI0,8818
 fractal_server/tasks/v2/templates/1_create_venv.sh,sha256=PK0jdHKtQpda1zULebBaVPORt4t6V17wa4N1ohcj5ac,548
-fractal_server/tasks/v2/templates/2_pip_install.sh,sha256=Enm1NxXqUglpi3R4_8_ojVq14ozjCc1XPvHXWKPLeS8,1262
+fractal_server/tasks/v2/templates/2_pip_install.sh,sha256=jMJPQJXHKznO6fxOOXtFXKPdCmTf1VLLWj_JL_ZdKxo,1644
 fractal_server/tasks/v2/templates/3_pip_freeze.sh,sha256=JldREScEBI4cD_qjfX4UK7V4aI-FnX9ZvVNxgpSOBFc,168
 fractal_server/tasks/v2/templates/4_pip_show.sh,sha256=qm1vPy6AkKhWDjCJGXS8LqCLYO3KsAyRK325ZsFcF6U,1747
 fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh,sha256=q-6ZUvA6w6FDVEoSd9O63LaJ9tKZc7qAFH72SGPrd_k,284
@@ -214,8 +216,8 @@ fractal_server/types/validators/_workflow_task_arguments_validators.py,sha256=HL
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=FCY6HUsRnnbsWkT2kwQ2izijiHuCrCD3Kh50G0QudxE,3531
 fractal_server/zip_tools.py,sha256=tqz_8f-vQ9OBRW-4OQfO6xxY-YInHTyHmZxU7U4PqZo,4885
-fractal_server-2.14.11.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.14.11.dist-info/METADATA,sha256=q7pnKsWb8a7g-nCtAe7PSCPQU-RkyHVSTj1Amo_LBLE,4244
-fractal_server-2.14.11.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.14.11.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.14.11.dist-info/RECORD,,
+fractal_server-2.14.13.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.14.13.dist-info/METADATA,sha256=-h8oCvhvIAs4Sp0G5zxavNQkpS6NkBsM8XHHmCT47EE,4244
+fractal_server-2.14.13.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.14.13.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.14.13.dist-info/RECORD,,

fractal_server/images/image_status.py

@@ -1,85 +0,0 @@
-import time
-from copy import deepcopy
-from typing import Any
-
-from fastapi import Depends
-from sqlmodel import select
-
-from fractal_server.app.db import AsyncSession
-from fractal_server.app.db import get_async_db
-from fractal_server.app.models.v2 import HistoryImageCache
-from fractal_server.app.models.v2 import HistoryUnit
-from fractal_server.app.schemas.v2 import HistoryUnitStatusWithUnset
-from fractal_server.logger import set_logger
-
-IMAGE_STATUS_KEY = "__wftask_dataset_image_status__"
-
-logger = set_logger(__name__)
-
-
-def _enriched_image(*, img: dict[str, Any], status: str) -> dict[str, Any]:
-    img["attributes"][IMAGE_STATUS_KEY] = status
-    return img
-
-
-async def enrich_image_list(
-    *,
-    images: list[dict[str, Any]],
-    dataset_id: int,
-    workflowtask_id: int,
-    db: AsyncSession = Depends(get_async_db),
-) -> list[dict[str, Any]]:
-    start_time = time.perf_counter()
-    logger.info(
-        f"START {enrich_image_list.__name__} for {dataset_id=}, "
-        f"{workflowtask_id=}"
-    )
-
-    zarr_url_to_image = {img["zarr_url"]: deepcopy(img) for img in images}
-
-    stm = (
-        select(HistoryImageCache.zarr_url, HistoryUnit.status)
-        .join(HistoryUnit)
-        .where(HistoryImageCache.dataset_id == dataset_id)
-        .where(HistoryImageCache.workflowtask_id == workflowtask_id)
-        .where(HistoryImageCache.latest_history_unit_id == HistoryUnit.id)
-        .where(HistoryImageCache.zarr_url.in_(zarr_url_to_image.keys()))
-        .order_by(HistoryImageCache.zarr_url)
-    )
-    res = await db.execute(stm)
-    list_processed_url_status = res.all()
-    logger.debug(
-        f"POST db query, "
-        f"elapsed={time.perf_counter() - start_time:.3f} "
-        "seconds"
-    )
-
-    set_processed_urls = set(item[0] for item in list_processed_url_status)
-    processed_images_with_status = [
-        _enriched_image(
-            img=zarr_url_to_image[item[0]],
-            status=item[1],
-        )
-        for item in list_processed_url_status
-    ]
-    logger.debug(
-        f"POST processed_images_with_status, "
-        f"elapsed={time.perf_counter() - start_time:.3f} "
-        "seconds"
-    )
-
-    non_processed_urls = zarr_url_to_image.keys() - set_processed_urls
-    non_processed_images_with_status = [
-        _enriched_image(
-            img=zarr_url_to_image[zarr_url],
-            status=HistoryUnitStatusWithUnset.UNSET,
-        )
-        for zarr_url in non_processed_urls
-    ]
-    logger.debug(
-        f"POST non_processed_images_with_status, "
-        f"elapsed={time.perf_counter() - start_time:.3f} "
-        "seconds"
-    )
-
-    return processed_images_with_status + non_processed_images_with_status