GeneralManager 0.57.3__tar.gz → 0.58.0__tar.gz

{generalmanager-0.57.3/src/GeneralManager.egg-info → generalmanager-0.58.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.57.3
+Version: 0.58.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.57.3 → generalmanager-0.58.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "GeneralManager"
-version = "0.57.3"
+version = "0.58.0"
 description = "Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching."
 readme = "README.md"
 authors = [{ name = "Tim Kleindick", email = "tkleindick@yahoo.de" }]

{generalmanager-0.57.3 → generalmanager-0.58.0/src/GeneralManager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.57.3
+Version: 0.58.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.57.3 → generalmanager-0.58.0}/src/general_manager/api/graphql_search.py

@@ -14,6 +14,7 @@ from datetime import date, datetime
 from decimal import Decimal
 from typing import (
     TYPE_CHECKING,
+    Literal,
     cast,
     get_args,
 )
@@ -23,6 +24,7 @@ from graphql import GraphQLError
 
 from django.utils import timezone
 
+from general_manager.conf import get_setting
 from general_manager.logging import get_logger
 from general_manager.manager.general_manager import GeneralManager
 from general_manager.measurement.measurement import Measurement
@@ -69,9 +71,20 @@ GrapheneReadMapper = Callable[
 SearchResolverPayload = dict[str, object]
 SearchHitEntry = tuple[float | None, SearchHit, GeneralManager]
 SortableValue = float | datetime | str | None
+SearchTotalMode = Literal["exact", "bounded"]
 _BAD_USER_INPUT_CODE = "BAD_USER_INPUT"
 _PAGE_MUST_BE_POSITIVE = "page must be a positive integer."
 _PAGE_SIZE_MUST_BE_POSITIVE = "pageSize must be a positive integer."
+DEFAULT_GRAPHQL_SEARCH_TOTAL_SCAN_LIMIT = 1000
+SEARCH_TOTAL_MODE_ERROR = "must be one of: exact, bounded."
+SEARCH_TOTAL_SCAN_LIMIT_ERROR = (
+    "GRAPHQL_SEARCH_TOTAL_SCAN_LIMIT must be a positive integer."
+)
+
+
+def _bad_user_input_error(message: str) -> GraphQLError:
+    """Build a GraphQL user-input error with the shared extension code."""
+    return GraphQLError(message, extensions={"code": _BAD_USER_INPUT_CODE})
 
 
 def _resolve_search_pagination(
@@ -96,6 +109,106 @@ def _resolve_search_pagination(
     return current_page, limit
 
 
+def _invalid_search_total_mode_error(setting_label: str) -> GraphQLError:
+    """Build the validation error for an invalid total-mode value."""
+    message = f"{setting_label} {SEARCH_TOTAL_MODE_ERROR}"
+    return _bad_user_input_error(message)
+
+
+def normalize_search_total_mode(total_mode: str | None = None) -> SearchTotalMode:
+    """Resolve and validate the requested GraphQL search total-count mode."""
+    if total_mode is None:
+        raw_mode = get_setting("GRAPHQL_SEARCH_TOTAL_MODE", "exact")
+        setting_label = 'GENERAL_MANAGER["GRAPHQL_SEARCH_TOTAL_MODE"]'
+    else:
+        raw_mode = total_mode
+        setting_label = "totalMode"
+
+    if not isinstance(raw_mode, str):
+        raise _invalid_search_total_mode_error(setting_label)
+
+    normalized = raw_mode.strip().lower()
+    if normalized not in {"exact", "bounded"}:
+        raise _invalid_search_total_mode_error(setting_label)
+    return cast(SearchTotalMode, normalized)
+
+
+def get_graphql_search_total_scan_limit() -> int:
+    """Return the configured positive scan limit for bounded total counts."""
+    raw_limit = get_setting(
+        "GRAPHQL_SEARCH_TOTAL_SCAN_LIMIT",
+        DEFAULT_GRAPHQL_SEARCH_TOTAL_SCAN_LIMIT,
+    )
+    if not isinstance(raw_limit, int) or isinstance(raw_limit, bool) or raw_limit <= 0:
+        raise _bad_user_input_error(SEARCH_TOTAL_SCAN_LIMIT_ERROR)
+    return raw_limit
+
+
+def normalize_search_sort_value(value: object) -> SortableValue:
+    """Normalise a search sort value to a comparable type."""
+    if value is None:
+        return None
+    if isinstance(value, (int, float, Decimal)):
+        return float(value)
+    if isinstance(value, datetime):
+        if timezone.is_naive(value):
+            return timezone.make_aware(value)
+        return value
+    if isinstance(value, date):
+        return timezone.make_aware(datetime.combine(value, datetime.min.time()))
+    if isinstance(value, str):
+        try:
+            parsed = datetime.fromisoformat(value)
+        except ValueError:
+            return value
+        if timezone.is_naive(parsed):
+            parsed = timezone.make_aware(parsed)
+        return parsed
+    return str(value)
+
+
+def sort_search_hit_entries(
+    entries: list[SearchHitEntry],
+    *,
+    sort_by: str | None,
+    sort_desc: bool,
+) -> None:
+    """Sort retained search entries using GraphQL search response semantics."""
+    if sort_by:
+
+        def _sort_key(
+            item: SearchHitEntry,
+        ) -> tuple[bool, SortableValue]:
+            """Return a null-last comparable key for an indexed search hit."""
+            value = item[1].data.get(sort_by) if item[1].data else None
+            normalized = normalize_search_sort_value(value)
+            return (normalized is None, normalized)
+
+        entries.sort(key=_sort_key)
+        if sort_desc:
+            null_start = next(
+                (index for index, item in enumerate(entries) if _sort_key(item)[0]),
+                len(entries),
+            )
+            entries[:null_start] = reversed(entries[:null_start])
+    else:
+        entries.sort(key=lambda item: item[0] or 0, reverse=True)
+
+
+def trim_search_hit_entries_to_window(
+    entries: list[SearchHitEntry],
+    *,
+    requested_count: int,
+    sort_by: str | None,
+    sort_desc: bool,
+) -> None:
+    """Keep only entries that can still appear in the requested global page."""
+    if len(entries) <= requested_count:
+        return
+    sort_search_hit_entries(entries, sort_by=sort_by, sort_desc=sort_desc)
+    del entries[requested_count:]
+
+
 # ---------------------------------------------------------------------------
 # Filter normalisation
 # ---------------------------------------------------------------------------
@@ -343,7 +456,7 @@ def create_search_result_type(
 
     Returns:
         A ``SearchResult`` Graphene ObjectType with ``results``, ``total``,
-        ``took_ms``, and ``raw`` fields.
+        ``total_is_exact``, ``took_ms``, and ``raw`` fields.
     """
     return type(
         "SearchResult",
@@ -351,6 +464,7 @@ def create_search_result_type(
         {
             "results": graphene.List(union_type),
             "total": graphene.Int(),
+            "total_is_exact": graphene.Boolean(),
             "took_ms": graphene.Int(),
             "raw": graphene.JSONString(),
         },
@@ -767,13 +881,15 @@ def register_search_query(
 
     Manager search order follows ``manager_registry.values()`` filtered to
     managers with search configuration, unless the resolver receives ``types``;
-    unknown names in ``types`` are ignored. ``total`` in the resolver payload is
-    the post-permission authorized total, not the backend raw total. Sorting
-    uses hit ``data[sort_by]`` when available and does not validate the sort
-    field ahead of comparison. Configured filter-key validation runs on the
-    parsed top-level search filters before permission filters or relation
-    normalizers are applied; exclude-derived permission keys are not validated
-    by this search helper.
+    unknown names in ``types`` are ignored. In exact total mode, ``total`` in
+    the resolver payload is the post-permission authorized total, not the
+    backend raw total. In bounded mode, ``total`` is the authorized count found
+    before the per-manager scan cap and ``total_is_exact`` tells callers whether
+    that count is exact. Sorting uses hit ``data[sort_by]`` when available and
+    does not validate the sort field ahead of comparison. Configured filter-key
+    validation runs on the parsed top-level search filters before permission
+    filters or relation normalizers are applied; exclude-derived permission keys
+    are not validated by this search helper.
 
     Parameters:
         query_fields: Mutable dict of registered query fields; will be updated.
@@ -819,6 +935,7 @@ def register_search_query(
         filters: GraphQLSearchFilterInput = None,
         sort_by: str | None = None,
         sort_desc: bool = False,
+        total_mode: str | None = None,
         page: int | None = None,
         page_size: int | None = None,
     ) -> SearchResolverPayload:
@@ -855,10 +972,13 @@ def register_search_query(
         raw: list[object] = []
         requested_count = offset + limit
         fetch_limit = max(requested_count, limit)
-        # Tracks how many authorized hits have been appended globally so far.
-        # This enforces a single cross-manager cap on collected results while
-        # still letting the loop continue counting for an accurate ``total``.
-        global_appended_hits = 0
+        resolved_total_mode = normalize_search_total_mode(total_mode)
+        total_scan_limit = (
+            max(get_graphql_search_total_scan_limit(), requested_count)
+            if resolved_total_mode == "bounded"
+            else None
+        )
+        total_is_exact = True
 
         for manager_class in manager_classes:
             type_label = manager_class.__name__
@@ -872,17 +992,24 @@ def register_search_query(
                 parsed_filters,
                 permission_plan.filters,
             )
-            authorized_hits: list[SearchHitEntry] = []
             total_hits_for_manager = 0
             candidate_hits_for_manager = 0
-            appended_hits_for_manager = 0
             offset_cursor = 0
+            scanned_hits_for_manager = 0
+            manager_total_is_exact = True
             while True:
+                query_limit = fetch_limit
+                if total_scan_limit is not None:
+                    remaining_scan_budget = total_scan_limit - scanned_hits_for_manager
+                    if remaining_scan_budget <= 0:
+                        manager_total_is_exact = False
+                        break
+                    query_limit = min(fetch_limit, remaining_scan_budget)
                 result = backend.search(
                     index_name,
                     query,
                     filters=filter_groups,
-                    limit=fetch_limit,
+                    limit=query_limit,
                     offset=offset_cursor,
                     types=[type_label],
                     sort_by=sort_by,
@@ -896,6 +1023,7 @@ def register_search_query(
                 raw.append(result.raw)
                 if not result.hits:
                     break
+                scanned_hits_for_manager += len(result.hits)
                 offset_cursor += len(result.hits)
                 for hit in result.hits:
                     try:
@@ -918,17 +1046,24 @@ def register_search_query(
                     ):
                         continue
                     total_hits_for_manager += 1
-                    # Apply a single global cap across all managers so that
-                    # ``authorized_hits`` never grows beyond ``requested_count``
-                    # items, regardless of how many managers are searched.
-                    if global_appended_hits < requested_count:
-                        authorized_hits.append((hit.score, hit, instance))
-                        appended_hits_for_manager += 1
-                        global_appended_hits += 1
-                if len(result.hits) < fetch_limit:
+                    hits.append((hit.score, hit, instance))
+                trim_search_hit_entries_to_window(
+                    hits,
+                    requested_count=requested_count,
+                    sort_by=sort_by,
+                    sort_desc=sort_desc,
+                )
+                if len(result.hits) < query_limit:
+                    break
+                if (
+                    total_scan_limit is not None
+                    and scanned_hits_for_manager >= total_scan_limit
+                ):
+                    manager_total_is_exact = False
                     break
             total += total_hits_for_manager
-            hits.extend(authorized_hits)
+            if not manager_total_is_exact:
+                total_is_exact = False
             if permission_plan.requires_instance_check:
                 logger.info(
                     "graphql read authorization summary",
@@ -947,43 +1082,7 @@ def register_search_query(
                     },
                 )
 
-        if sort_by:
-
-            def _normalize_sort_value(value: object) -> SortableValue:
-                """Normalise a sort value to a comparable type."""
-                if value is None:
-                    return None
-                if isinstance(value, (int, float, Decimal)):
-                    return float(value)
-                if isinstance(value, datetime):
-                    if timezone.is_naive(value):
-                        return timezone.make_aware(value)
-                    return value
-                if isinstance(value, date):
-                    return timezone.make_aware(
-                        datetime.combine(value, datetime.min.time())
-                    )
-                if isinstance(value, str):
-                    try:
-                        parsed = datetime.fromisoformat(value)
-                    except ValueError:
-                        return value
-                    if timezone.is_naive(parsed):
-                        parsed = timezone.make_aware(parsed)
-                    return parsed
-                return str(value)
-
-            def _sort_key(
-                item: SearchHitEntry,
-            ) -> tuple[bool, SortableValue]:
-                """Sort key placing None-valued items last."""
-                value = item[1].data.get(sort_by) if item[1].data else None
-                normalized = _normalize_sort_value(value)
-                return (normalized is None, normalized)
-
-            hits.sort(key=_sort_key, reverse=sort_desc)
-        else:
-            hits.sort(key=lambda item: item[0] or 0, reverse=True)
+        sort_search_hit_entries(hits, sort_by=sort_by, sort_desc=sort_desc)
 
         items: list[GeneralManager] = []
         for _, _hit, instance in hits[offset : offset + limit]:
@@ -992,6 +1091,7 @@ def register_search_query(
         return {
             "results": items,
             "total": total,
+            "total_is_exact": total_is_exact,
             "took_ms": took_ms,
             "raw": raw,
         }
@@ -1004,6 +1104,7 @@ def register_search_query(
         filters=graphene.JSONString(),
         sort_by=graphene.String(),
         sort_desc=graphene.Boolean(),
+        total_mode=graphene.String(),
         page=graphene.Int(),
         page_size=graphene.Int(),
         resolver=resolver,

{generalmanager-0.57.3 → generalmanager-0.58.0}/src/general_manager/bucket/calculation_bucket.py

@@ -1,8 +1,9 @@
 """Bucket implementation that enumerates calculation interface combinations."""
 
 from __future__ import annotations
-from collections.abc import Hashable, Iterable
+from collections.abc import Hashable, Iterable, Iterator
 from types import UnionType
+from itertools import islice
 from typing import (
     Type,
     TYPE_CHECKING,
@@ -25,6 +26,7 @@ from general_manager.bucket.base_bucket import Bucket
 from general_manager.bucket.indexing import freeze_bucket_index_value
 from general_manager.manager.input import Input, InputDomain
 from general_manager.utils.filter_parser import (
+    FilterFunction,
     ParsedFilters,
     parse_filters,
 )
@@ -47,6 +49,15 @@ class SortedFilters(TypedDict):
     input_excludes: ParsedFilters
 
 
+def _build_exclude_filter(exclude_func: FilterFunction) -> FilterFunction:
+    """Create a lazy predicate that keeps values an exclude does not reject."""
+
+    def includes_value(value: object) -> bool:
+        return not exclude_func(value)
+
+    return includes_value
+
+
 class InvalidCalculationInterfaceError(TypeError):
     """Raised when a CalculationBucket is initialized with a non-CalculationInterface manager."""
 
@@ -308,25 +319,95 @@ class CalculationBucket(Bucket[GeneralManagerType]):
 
     def __str__(self) -> str:
         """
-        Return a compact preview of the generated combinations.
+        Return a compact preview of generated combinations.
+
+        Cached buckets include the exact combination count. Uncached buckets avoid
+        materializing all combinations for string formatting; when more than the
+        preview limit exists, the count is reported as a lower-bound label.
 
         Returns:
             str: Human-readable summary of up to five combinations.
         """
         PRINT_MAX = 5
-        combinations = self.generate_combinations()
-        prefix = f"CalculationBucket ({len(combinations)})["
+        combinations, count_label, has_more = self._str_combinations_preview(PRINT_MAX)
+        prefix = f"CalculationBucket ({count_label})["
         main = ",".join(
-            [
-                f"{self._manager_class.__name__}(**{comb})"
-                for comb in combinations[:PRINT_MAX]
-            ]
+            [f"{self._manager_class.__name__}(**{comb})" for comb in combinations]
+        )
+        suffix = "]"
+        if has_more:
+            suffix = ", ...]"
+
+        return f"{prefix}{main}{suffix}"
+
+    def _str_combinations_preview(
+        self, limit: int
+    ) -> tuple[list[Combination], str, bool]:
+        """
+        Return combinations, count label, and overflow flag for ``__str__``.
+
+        Sorted or reversed buckets use normal materialization so the preview
+        reflects the final global ordering. Unsorted uncached buckets read at
+        most ``limit + 1`` matching combinations and leave ``_data`` untouched.
+        """
+        if self._data is not None:
+            return self._data[:limit], str(len(self._data)), len(self._data) > limit
+
+        if self._normalized_sort_key() is not None or self.reverse:
+            combinations = self.generate_combinations()
+            return (
+                combinations[:limit],
+                str(len(combinations)),
+                len(combinations) > limit,
+            )
+
+        from general_manager.cache.run_context import ensure_calculation_run_context
+
+        with ensure_calculation_run_context():
+            sorted_inputs = self.topological_sort_inputs()
+            sorted_filters = self._sort_filters(sorted_inputs)
+            if self._uses_static_iterator_possible_values(sorted_inputs):
+                combinations = self.generate_combinations()
+                return (
+                    combinations[:limit],
+                    str(len(combinations)),
+                    len(combinations) > limit,
+                )
+            snapshot_iterables = self._uses_dependent_possible_values(sorted_inputs)
+            preview_iterator = self._iter_input_combinations(
+                sorted_inputs,
+                sorted_filters["input_filters"],
+                sorted_filters["input_excludes"],
+                snapshot_iterables=snapshot_iterables,
+            )
+            if sorted_filters["prop_filters"] or sorted_filters["prop_excludes"]:
+                preview_iterator = self._iter_prop_filtered_identifications(
+                    preview_iterator,
+                    sorted_filters["prop_filters"],
+                    sorted_filters["prop_excludes"],
+                )
+            preview = list(islice(preview_iterator, limit + 1))
+
+        has_more = len(preview) > limit
+        if has_more:
+            preview = preview[:limit]
+        count_label = f"{limit}+" if has_more else str(len(preview))
+        return preview, count_label, has_more
+
+    def _uses_static_iterator_possible_values(self, sorted_inputs: list[str]) -> bool:
+        """Return whether previewing would consume a one-shot static iterator."""
+        return any(
+            isinstance(self.input_fields[input_name].possible_values, Iterator)
+            for input_name in sorted_inputs
         )
-        sufix = "]"
-        if len(combinations) > PRINT_MAX:
-            sufix = ", ...]"
 
-        return f"{prefix}{main}{sufix}"
+    def _uses_dependent_possible_values(self, sorted_inputs: list[str]) -> bool:
+        """Return whether previewing should snapshot values before dependencies."""
+        return any(
+            bool(self.input_fields[input_name].depends_on)
+            and self.input_fields[input_name].possible_values is not None
+            for input_name in sorted_inputs
+        )
 
     def __repr__(self) -> str:
         """
@@ -549,12 +630,14 @@ class CalculationBucket(Bucket[GeneralManagerType]):
         self,
         combinations: list[Combination],
     ) -> list[GeneralManagerType]:
+        """Instantiate managers for each raw input-combination dictionary."""
         return [self._manager_class(**combo) for combo in combinations]
 
     @staticmethod
     def _manager_identifications(
         managers: list[GeneralManagerType],
     ) -> list[Combination]:
+        """Return the identification dictionaries from manager instances."""
         return [manager.identification for manager in managers]
 
     def generate_combinations(self) -> List[Combination]:
@@ -730,22 +813,25 @@ class CalculationBucket(Bucket[GeneralManagerType]):
             raise InvalidPossibleValuesError(key_name)
         return possible_values
 
-    def _generate_input_combinations(
+    def _iter_input_combinations(
         self,
         sorted_inputs: List[str],
         filters: ParsedFilters,
         excludes: ParsedFilters,
-    ) -> List[Combination]:
+        *,
+        snapshot_iterables: bool,
+    ) -> Generator[Combination, None, None]:
         """
-        Generate all valid assignments of input fields that satisfy the provided per-field filters and exclusions.
+        Yield valid assignments of input fields satisfying filters and excludes.
 
         Parameters:
             sorted_inputs (list[str]): Input names in dependency-respecting order.
             filters (dict[str, dict]): Per-input filter definitions (may include `filter_funcs` or `filter_kwargs`).
             excludes (dict[str, dict]): Per-input exclusion definitions (may include `filter_funcs` or `filter_kwargs`).
 
-        Returns:
-            list[Combination]: Completed input-to-value mappings that meet the filters and excludes.
+        Yields:
+            Combination: Completed input-to-value mappings that meet the
+                filters and excludes.
         """
 
         def input_passes_filters(
@@ -812,10 +898,10 @@ class CalculationBucket(Bucket[GeneralManagerType]):
                 exclude_funcs = field_excludes.get("filter_funcs", [])
                 for exclude_func in exclude_funcs:
                     possible_values = filter(
-                        lambda x: not exclude_func(x), possible_values
+                        _build_exclude_filter(exclude_func), possible_values
                     )
-
-                possible_values = list(possible_values)
+                if snapshot_iterables:
+                    possible_values = list(possible_values)
 
             for value in possible_values:
                 if not isinstance(value, input_field.type):
@@ -824,7 +910,52 @@ class CalculationBucket(Bucket[GeneralManagerType]):
                 yield from helper(index + 1, current_combo)
                 del current_combo[input_name]
 
-        return list(helper(0, {}))
+        yield from helper(0, {})
+
+    def _generate_input_combinations(
+        self,
+        sorted_inputs: List[str],
+        filters: ParsedFilters,
+        excludes: ParsedFilters,
+    ) -> List[Combination]:
+        """
+        Generate all valid assignments of input fields that satisfy filters.
+
+        Parameters:
+            sorted_inputs (list[str]): Input names in dependency-respecting order.
+            filters (dict[str, dict]): Per-input filter definitions.
+            excludes (dict[str, dict]): Per-input exclusion definitions.
+
+        Returns:
+            list[Combination]: Completed input-to-value mappings that meet the
+                filters and excludes.
+        """
+        return list(
+            self._iter_input_combinations(
+                sorted_inputs,
+                filters,
+                excludes,
+                snapshot_iterables=True,
+            )
+        )
+
+    def _iter_prop_filtered_identifications(
+        self,
+        combinations: Iterable[Combination],
+        prop_filters: ParsedFilters,
+        prop_excludes: ParsedFilters,
+    ) -> Generator[Combination, None, None]:
+        """
+        Lazily apply property filters and yield manager identifications.
+
+        This mirrors the property-filter materialization path used by
+        :meth:`generate_combinations`, but lets ``__str__`` stop after enough
+        matching combinations have been found.
+        """
+        for combo in combinations:
+            manager = self._manager_class(**combo)
+            if self._filter_prop_combinations([manager], prop_filters, prop_excludes):
+                yield manager.identification
 
     def _filter_prop_combinations(
         self,