hexkit 8.2.1__tar.gz → 8.4.0__tar.gz

{hexkit-8.2.1/src/hexkit.egg-info → hexkit-8.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexkit
-Version: 8.2.1
+Version: 8.4.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0

{hexkit-8.2.1 → hexkit-8.4.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "8.2.1"
+version = "8.4.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "opentelemetry-api >=1.39, <2",

{hexkit-8.2.1 → hexkit-8.4.0}/src/hexkit/protocols/dao.py

@@ -20,7 +20,7 @@ with the database.
 
 import typing
 from abc import ABC, abstractmethod
-from collections.abc import AsyncIterator, Collection, Mapping
+from collections.abc import AsyncIterator, Awaitable, Callable, Collection, Mapping
 from contextlib import AbstractAsyncContextManager
 from functools import partial
 from typing import Any, Generic, TypeVar
@@ -35,6 +35,7 @@ __all__ = [
     "Dao",
     "DaoFactoryProtocol",
     "FindError",
+    "FindResult",
     "MultipleHitsFoundError",
     "ResourceAlreadyExistsError",
     "ResourceNotFoundError",
@@ -137,6 +138,47 @@ class DbTimeoutError(DaoError):
 UUID4Field = partial(Field, default_factory=uuid4)
 
 
+class FindResult(AsyncIterator[Dto]):
+    """An async-iterable result from find_all() that also exposes pagination metadata.
+
+    The `total_count()` method performs an additional count query on first call (lazily)
+    and caches the result for subsequent calls.
+    """
+
+    def __init__(
+        self,
+        *,
+        results_iterator: AsyncIterator[Dto],
+        get_total_count: Callable[[], Awaitable[int]],
+    ) -> None:
+        self._iterator = results_iterator
+        self._get_total_count = get_total_count
+        self._cached_total: int | None = None
+
+    def __aiter__(self) -> "FindResult[Dto]":  # noqa: D105
+        return self
+
+    async def __anext__(self) -> Dto:  # noqa: D105
+        return await self._iterator.__anext__()
+
+    async def total_count(self) -> int:
+        """Total number of matching documents, ignoring skip and limit.
+
+        The result is cached after the first call.
+        """
+        if self._cached_total is None:
+            self._cached_total = await self._get_total_count()
+        return self._cached_total
+
+    async def to_list(self) -> list[Dto]:
+        """Collect all results into a list.
+
+        Calling this function will consume the iterator. Subsequent cals will return
+        an empty list.
+        """
+        return [item async for item in self]
+
+
 class Dao(typing.Protocol[Dto]):
     """A duck type with methods common to all DAOs."""
 
@@ -223,7 +265,14 @@ class Dao(typing.Protocol[Dto]):
         """
         ...
 
-    def find_all(self, *, mapping: Mapping[str, Any]) -> AsyncIterator[Dto]:
+    def find_all(
+        self,
+        *,
+        mapping: Mapping[str, Any],
+        skip: int | None = None,
+        limit: int | None = None,
+        sort: list[str] | None = None,
+    ) -> "FindResult[Dto]":
         """Find all resources that match the specified mapping.
 
         The values in the mapping are used to filter the resources, these are
@@ -235,10 +284,27 @@ class Dao(typing.Protocol[Dto]):
             mapping:
                 A mapping where the keys correspond to the names of resource fields
                 and the values correspond to the actual values of the resource fields.
+            skip:
+                Number of matching resources to skip before yielding results.
+                Defaults to None (no skipping).
+            limit:
+                Maximum number of resources to yield. Defaults to None (no limit).
+                Specifying 0 is interpreted literally and returns no results.
+            sort:
+                A list of field names defining the sort order, where field names
+                prefixed with "-" indicate *descending* order. For example, if sort is
+                specified as `["name", "-age"]`, the results would be sorted first by
+                "name" in ascending order, then by "age" in descending order. When
+                paginating with skip/limit, providing a sort order is strongly
+                recommended to ensure consistent, deterministic results. Defaults to
+                None (no sort).
 
         Returns:
-            An AsyncIterator of hits. All hits are in the form of the respective DTO
-            model.
+            A FindResult that is async-iterable and also provides total_count().
+
+        Raises:
+            InvalidMappingError: If `mapping` doesn't pass validation.
+            ValueError: if `skip` or `limit` are less than 0.
         """
         ...
 

{hexkit-8.2.1 → hexkit-8.4.0}/src/hexkit/providers/mongodb/provider/dao.py

@@ -33,6 +33,7 @@ from hexkit.protocols.dao import (
     Dao,
     DaoFactoryProtocol,
     Dto,
+    FindResult,
     IndexBase,
     InvalidFindMappingError,
     MultipleHitsFoundError,
@@ -65,7 +66,7 @@ def validate_find_mapping(mapping: Mapping[str, Any], *, dto_model: type[Dto]):
     by checking if the mapping keys exist as model fields.
 
     Raises:
-        InvalidMappingError: If validation fails.
+        InvalidMappingError: If `mapping` doesn't pass validation.
     """
     try:
         validate_fields_in_model(model=dto_model, fields=set(mapping))
@@ -310,7 +311,14 @@ class MongoDbDao(Generic[Dto]):
         hits = self.find_all(mapping=mapping)
         return await get_single_hit(hits=hits, mapping=mapping)
 
-    async def find_all(self, *, mapping: Mapping[str, Any]) -> AsyncIterator[Dto]:
+    def find_all(  # noqa: C901
+        self,
+        *,
+        mapping: Mapping[str, Any],
+        skip: int | None = None,
+        limit: int | None = None,
+        sort: list[str] | None = None,
+    ) -> FindResult[Dto]:
         """Find all resources that match the specified mapping.
 
         The values in the mapping are used to filter the resources, these are
@@ -322,19 +330,71 @@ class MongoDbDao(Generic[Dto]):
             mapping:
                 A mapping where the keys correspond to the names of resource fields
                 and the values correspond to the actual values of the resource fields.
+            skip:
+                Number of matching resources to skip before yielding results.
+                Defaults to None (no skipping).
+            limit:
+                Maximum number of resources to yield. Defaults to None (no limit).
+                Specifying 0 is interpreted literally and returns no results.
+            sort:
+                A list of field names defining the sort order, where field names
+                prefixed with "-" indicate *descending* order. For example, if sort is
+                specified as `["name", "-age"]`, the results would be sorted first by
+                "name" in ascending order, then by "age" in descending order. When
+                paginating with skip/limit, providing a sort order is strongly
+                recommended to ensure consistent, deterministic results. Defaults to
+                None (no sort).
 
         Returns:
-            An AsyncIterator of hits. All hits are in the form of the respective DTO
-            model.
+            A FindResult that is async-iterable and also provides total_count().
         """
+        skip = skip or 0
+
+        if skip < 0:
+            raise ValueError("skip must be >= 0")
+        if limit is not None and limit < 0:
+            raise ValueError("limit must be >= 0")
+
         validate_find_mapping(mapping, dto_model=self._dto_model)
         mapping = replace_id_field_in_find_mapping(mapping, self._id_field)
 
-        with translate_pymongo_errors():
-            cursor = self._collection.find(filter=mapping)
+        # Convert generic sort spec to MongoDB-specific sort spec
+        mongodb_sort: list[tuple[str, int]] = []
+        for spec in sort or []:
+            field = spec.removeprefix("-")
+            order = 1 if field == spec else -1
+            if field == self._id_field:
+                field = "_id"
+            mongodb_sort.append((field, order))
+
+        collection = self._collection
+
+        async def _total_count() -> int:
+            with translate_pymongo_errors():
+                return await collection.count_documents(filter=mapping)
+
+        if limit == 0:
+
+            async def _empty_iter() -> AsyncIterator[Dto]:
+                return
+                yield  # turns this into an async generator
+
+            return FindResult(
+                results_iterator=_empty_iter(), get_total_count=_total_count
+            )
 
-            async for document in cursor:
-                yield self._document_to_dto(document)
+        async def _iter() -> AsyncIterator[Dto]:
+            with translate_pymongo_errors():
+                cursor = collection.find(filter=mapping)
+                if sort:
+                    cursor = cursor.sort(mongodb_sort)
+                cursor = cursor.skip(skip)
+                if limit:
+                    cursor = cursor.limit(limit)
+                async for document in cursor:
+                    yield self._document_to_dto(document)
+
+        return FindResult(results_iterator=_iter(), get_total_count=_total_count)
 
     @classmethod
     def with_transaction(cls) -> AbstractAsyncContextManager["Dao[Dto]"]:

{hexkit-8.2.1 → hexkit-8.4.0}/src/hexkit/providers/mongokafka/provider/daopub.py

@@ -42,6 +42,7 @@ from hexkit.custom_types import ID, JsonObject
 from hexkit.protocols.dao import (
     Dao,
     Dto,
+    FindResult,
     ResourceNotFoundError,
     UniqueConstraintViolationError,
 )
@@ -369,7 +370,14 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
         hits = self.find_all(mapping=mapping)
         return await get_single_hit(hits=hits, mapping=mapping)
 
-    async def find_all(self, *, mapping: Mapping[str, Any]) -> AsyncIterator[Dto]:
+    def find_all(  # noqa: C901
+        self,
+        *,
+        mapping: Mapping[str, Any],
+        skip: int | None = None,
+        limit: int | None = None,
+        sort: list[str] | None = None,
+    ) -> FindResult[Dto]:
         """Find all resources that match the specified mapping.
 
         The values in the mapping are used to filter the resources, these are
@@ -381,23 +389,91 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
             mapping:
                 A mapping where the keys correspond to the names of resource fields
                 and the values correspond to the actual values of the resource fields.
+            skip:
+                Number of matching resources to skip before yielding results.
+                Defaults to None (no skipping).
+            limit:
+                Maximum number of resources to yield. Defaults to None (no limit).
+                Specifying 0 is interpreted literally and returns no results.
+            sort:
+                A list of field names defining the sort order, where field names
+                prefixed with "-" indicate *descending* order. For example, if sort is
+                specified as `["name", "-age"]`, the results would be sorted first by
+                "name" in ascending order, then by "age" in descending order. When
+                paginating with skip/limit, providing a sort order is strongly
+                recommended to ensure consistent, deterministic results. Defaults to
+                None (no sort).
 
         Returns:
-            An AsyncIterator of hits. All hits are in the form of the respective DTO
-            model.
+            A FindResult that is async-iterable and also provides total_count().
+
+        Raises:
+            InvalidMappingError: If `mapping` doesn't pass validation.
+            ValueError: if `skip` or `limit` are less than 0.
         """
+        skip = skip or 0
+        if skip < 0:
+            raise ValueError("skip must be >= 0")
+        if limit is not None and limit < 0:
+            raise ValueError("limit must be >= 0")
+
         validate_find_mapping(mapping, dto_model=self._dto_model)
         mapping = replace_id_field_in_find_mapping(mapping, self._id_field)
 
-        with translate_pymongo_errors():
-            cursor = self._collection.find(filter=mapping)
+        # Ensure we don't retrieve deleted docs. Documents lacking outbox metadata are
+        # treated as valid (matching update/delete), and the $and wrapper avoids
+        # clobbering any caller-supplied $or in the mapping.
+        mapping_without_deleted = {
+            "$and": [
+                dict(mapping),
+                {
+                    "$or": [
+                        {"__metadata__": {"$exists": False}},
+                        {"__metadata__.deleted": False},
+                    ]
+                },
+            ]
+        }
 
-            async for document in cursor:
-                if document.get("__metadata__", {}).get("deleted", False):
-                    continue
-                yield document_to_dto(
-                    document, id_field=self._id_field, dto_model=self._dto_model
-                )
+        # Convert generic sort spec to MongoDB-specific sort spec
+        mongodb_sort: list[tuple[str, int]] = []
+        for spec in sort or []:
+            field = spec.removeprefix("-")
+            order = 1 if field == spec else -1
+            if field == self._id_field:
+                field = "_id"
+            mongodb_sort.append((field, order))
+
+        collection = self._collection
+
+        async def _total_count() -> int:
+            with translate_pymongo_errors():
+                return await collection.count_documents(filter=mapping_without_deleted)
+
+        if limit == 0:
+
+            async def _empty_iter() -> AsyncIterator[Dto]:
+                return
+                yield  # turns this into an async generator
+
+            return FindResult(
+                results_iterator=_empty_iter(), get_total_count=_total_count
+            )
+
+        async def _iter() -> AsyncIterator[Dto]:
+            with translate_pymongo_errors():
+                cursor = collection.find(filter=mapping_without_deleted)
+                if sort:
+                    cursor = cursor.sort(mongodb_sort)
+                cursor = cursor.skip(skip)
+                if limit:
+                    cursor = cursor.limit(limit)
+                async for document in cursor:
+                    yield document_to_dto(
+                        document, id_field=self._id_field, dto_model=self._dto_model
+                    )
+
+        return FindResult(results_iterator=_iter(), get_total_count=_total_count)
 
     async def insert(self, dto: Dto) -> None:
         """Create a new resource.

{hexkit-8.2.1 → hexkit-8.4.0}/src/hexkit/providers/testing/dao.py

@@ -16,15 +16,17 @@
 """A mock (in-memory) DAO"""
 
 from collections.abc import AsyncIterator, Callable, Mapping
-from contextlib import suppress
+from contextlib import AbstractAsyncContextManager, suppress
 from copy import deepcopy
+from functools import partial
 from typing import Any, Generic, Protocol, TypeVar
-from unittest.mock import AsyncMock, Mock
+from unittest.mock import AsyncMock
 
 from pydantic import BaseModel
 
 from hexkit.custom_types import ID
 from hexkit.protocols.dao import (
+    FindResult,
     MultipleHitsFoundError,
     NoHitsFoundError,
     ResourceAlreadyExistsError,
@@ -86,6 +88,18 @@ def _get_nested_value(resource: Document, field_path: str) -> Any:
     return value
 
 
+def _sort_key(resource: Document, *, field_path: str) -> tuple[bool, Any]:
+    """Build a sort key for a (possibly nested) field path of a resource.
+
+    Missing values (None) are grouped together and sorted ahead of present values,
+    mirroring MongoDB's handling of missing/null fields. The leading boolean keeps
+    None values from being compared against present values (or one another), which
+    would otherwise raise a TypeError.
+    """
+    value = _get_nested_value(resource, field_path)
+    return (value is not None, value if value is not None else 0)
+
+
 class Predicate(Protocol):
     def evaluate(self, resource: dict[str, Any]) -> bool:
         """Determine if the supplied dict value satisfies the predicate."""
@@ -345,7 +359,10 @@ class BaseInMemDao(Generic[DTO]):
     _deserialize: Callable
     publish_pending = AsyncMock()
     republish = AsyncMock()
-    with_transaction = Mock()
+
+    @classmethod
+    def with_transaction(cls) -> AbstractAsyncContextManager["BaseInMemDao[DTO]"]:
+        raise NotImplementedError()
 
     def __init__(self) -> None:
         self.resources: dict[ID, Document] = {}
@@ -391,27 +408,83 @@ class BaseInMemDao(Generic[DTO]):
 
         raise MultipleHitsFoundError(mapping=mapping)
 
-    async def find_all(self, *, mapping: Mapping[str, Any]) -> AsyncIterator[DTO]:
+    def _resource_matches(
+        self,
+        resource: Document,
+        mapping: Mapping[str, Any],
+        predicates: list["Predicate"],
+    ) -> bool:
+        """Return whether a resource matches the given mapping or predicates."""
+        if self._handle_mql:
+            return all(p.evaluate(resource=resource) for p in predicates)
+        for key, expected_value in mapping.items():
+            if _get_nested_value(resource, key) != expected_value:
+                return False
+        return True
+
+    def find_all(
+        self,
+        *,
+        mapping: Mapping[str, Any],
+        skip: int | None = None,
+        limit: int | None = None,
+        sort: list[str] | None = None,
+    ) -> "FindResult[DTO]":
         """Find all resources that match the specified mapping."""
+        skip = skip or 0
+        if skip < 0:
+            raise ValueError("skip must be >= 0")
+        if limit is not None and limit < 0:
+            raise ValueError("limit must be >= 0")
+
         if "" in mapping:
             raise KeyError("Query mappings can't contain empty-string keys")
 
         _mapping = replace_id_field_in_find_mapping(mapping, self._id_field)
         predicates = build_predicates(_mapping) if self._handle_mql else []
 
-        for resource in self.resources.values():
-            if self._handle_mql:
-                matches = all(p.evaluate(resource=resource) for p in predicates)
-            else:
-                matches = True
-                for key, expected_value in mapping.items():
-                    actual_value = _get_nested_value(resource, key)
-                    if actual_value != expected_value:
-                        matches = False
-                        break
-            if matches:
+        matching = [
+            resource
+            for resource in self.resources.values()
+            if self._resource_matches(resource, mapping, predicates)
+        ]
+
+        # Interpret the sorting specification and sort our resources accordingly
+        for spec in reversed(sort or []):
+            desc = spec.startswith("-")
+            field = spec.removeprefix("-")
+            # Only the exact ID field is translated to the internal "_id" field, matching
+            # the MongoDB provider. Other fields (including nested sub-fields of the ID,
+            # which MongoDB cannot sort by) are looked up as-is and resolve to None when
+            # absent, sorting consistently rather than raising - again like MongoDB.
+            doc_field = "_id" if field == self._id_field else field
+            matching.sort(key=partial(_sort_key, field_path=doc_field), reverse=desc)
+
+        total = len(matching)
+
+        async def _total_count() -> int:
+            return total
+
+        if limit == 0:
+
+            async def _empty_iter() -> AsyncIterator[DTO]:
+                return
+                yield  # makes this an async generator
+
+            return FindResult(
+                results_iterator=_empty_iter(), get_total_count=_total_count
+            )
+
+        # Manually apply pagination params
+        end = (skip + limit) if limit is not None else None
+        page = matching[skip:end]
+
+        async def _iter() -> AsyncIterator[DTO]:
+            for resource in page:
                 yield self._deserialize(resource)
 
+        return FindResult(results_iterator=_iter(), get_total_count=_total_count)
+
     async def insert(self, dto: DTO) -> None:
         """Insert a resource.
 

{hexkit-8.2.1 → hexkit-8.4.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexkit
-Version: 8.2.1
+Version: 8.4.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0