GeneralManager 0.47.0__tar.gz → 0.49.0__tar.gz

{generalmanager-0.47.0/src/GeneralManager.egg-info → generalmanager-0.49.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.47.0
+Version: 0.49.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.47.0 → generalmanager-0.49.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "GeneralManager"
-version = "0.47.0"
+version = "0.49.0"
 description = "Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching."
 readme = "README.md"
 authors = [{ name = "Tim Kleindick", email = "tkleindick@yahoo.de" }]

{generalmanager-0.47.0 → generalmanager-0.49.0/src/GeneralManager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.47.0
+Version: 0.49.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.47.0 → generalmanager-0.49.0}/src/general_manager/bucket/database_bucket.py

@@ -1,20 +1,24 @@
 """Database-backed bucket implementation for GeneralManager collections."""
 
 from __future__ import annotations
+from collections.abc import Hashable
 from datetime import date, datetime
 from typing import Any, Generator, Type, TypeVar
 
-from django.core.exceptions import FieldError
+from django.core.exceptions import EmptyResultSet, FieldError
 from django.db import models
+from django.db.models.sql.query import Query
 
 from general_manager.bucket.base_bucket import Bucket
 from general_manager.cache.cache_tracker import DependencyTracker
 from general_manager.cache.dependency_index import serialize_dependency_identifier
+from general_manager.cache.run_context import current_calculation_run_context
 from general_manager.manager.general_manager import GeneralManager
 from general_manager.utils.filter_parser import create_filter_function
 
 modelsModel = TypeVar("modelsModel", bound=models.Model)
 GeneralManagerType = TypeVar("GeneralManagerType", bound=GeneralManager)
+MAX_RUN_SCOPED_BUCKET_RESULT_ROWS = 1000
 
 
 class DatabaseBucketTypeMismatchError(TypeError):
@@ -204,6 +208,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         search_date: datetime | date | None = None,
         sort_keys: tuple[str, ...] | None = None,
         sort_reverse: bool = False,
+        run_scoped_cacheable: bool = True,
     ) -> None:
         """
         Instantiate a database-backed bucket with optional filter state.
@@ -214,6 +219,9 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             filter_definitions (dict[str, list[Any]] | None): Pre-existing filter expressions captured from parent buckets.
             exclude_definitions (dict[str, list[Any]] | None): Pre-existing exclusion expressions captured from parent buckets.
             search_date (datetime | date | None): Optional timestamp applied when instantiating manager instances.
+            sort_keys (tuple[str, ...] | None): Property names used by a previous bucket sort operation.
+            sort_reverse (bool): Whether sorted keys should be interpreted in descending order for dependency tracking.
+            run_scoped_cacheable (bool): Whether terminal results from this bucket are safe to reuse inside the active calculation run.
 
         Returns:
             None
@@ -225,6 +233,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         self._search_date = search_date
         self._sort_keys = sort_keys
         self._sort_reverse = sort_reverse
+        self._run_scoped_cacheable = run_scoped_cacheable
 
     def __reduce__(self) -> str | tuple[Any, ...]:
         """
@@ -281,6 +290,127 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             for key, values in definitions.items()
         }
 
+    def _query_signature(self) -> tuple[Hashable, ...] | None:
+        """
+        Return a conservative run-scoped cache signature for this queryset.
+
+        The signature is only produced for queryset shapes whose SQL, database
+        alias, manager class, model, search date, and sort state can safely
+        distinguish equivalent terminal results. Unsupported or risky query
+        forms return ``None`` so callers fall back to normal ORM evaluation.
+
+        Returns:
+            tuple[Hashable, ...] | None: Cache key components for equivalent
+            queryset results, or ``None`` when reuse should be bypassed.
+        """
+        if not self._run_scoped_cacheable:
+            return None
+        query = self._data.query
+        if not isinstance(query, Query):
+            return None
+        if query.select_for_update:
+            return None
+        if query.combinator:
+            return None
+        if query.distinct:
+            return None
+        try:
+            sql, params = self._data.query.sql_with_params()
+        except (EmptyResultSet, FieldError, TypeError, ValueError):
+            return None
+        return (
+            self._manager_class,
+            self._data.model,
+            self._data.db,
+            sql,
+            tuple(params),
+            self._search_date,
+            self._sort_keys,
+            self._sort_reverse,
+        )
+
+    def _get_run_scoped_primary_keys(self) -> tuple[Any, ...] | None:
+        """
+        Load or reuse a bounded primary-key snapshot for this bucket.
+
+        The method materializes at most ``MAX_RUN_SCOPED_BUCKET_RESULT_ROWS``
+        primary keys and stores them in the active
+        :class:`CalculationRunContext`. Buckets without an active context,
+        without a safe signature, or above the materialization guardrail return
+        ``None`` so terminal operations continue through the database.
+
+        Returns:
+            tuple[Any, ...] | None: Cached or newly materialized primary keys,
+            or ``None`` when run-scoped reuse is unavailable.
+        """
+        context = current_calculation_run_context()
+        if context is None:
+            return None
+        signature = self._query_signature()
+        if signature is None:
+            return None
+        cached = context.get_orm_bucket_result(signature)
+        if cached is not None:
+            return cached  # type: ignore[return-value]
+
+        primary_keys = tuple(
+            self._data.values_list("pk", flat=True)[
+                : MAX_RUN_SCOPED_BUCKET_RESULT_ROWS + 1
+            ]
+        )
+        if len(primary_keys) > MAX_RUN_SCOPED_BUCKET_RESULT_ROWS:
+            return None
+        context.set_orm_bucket_result(signature, primary_keys)
+        return primary_keys
+
+    def _peek_run_scoped_primary_keys(self) -> tuple[Any, ...] | None:
+        """
+        Return an already cached primary-key snapshot without evaluating the ORM.
+
+        Terminal operations such as ``count()`` and membership checks call this
+        helper so they can reuse a snapshot created by iteration without forcing
+        materialization on their own.
+
+        Returns:
+            tuple[Any, ...] | None: Cached primary keys for this bucket, or
+            ``None`` when no snapshot exists.
+        """
+        context = current_calculation_run_context()
+        if context is None:
+            return None
+        signature = self._query_signature()
+        if signature is None:
+            return None
+        cached = context.get_orm_bucket_result(signature)
+        if cached is None:
+            return None
+        return cached  # type: ignore[return-value]
+
+    @staticmethod
+    def _snapshot_get_primary_key(kwargs: dict[str, Any]) -> Any | None:
+        """
+        Extract the primary key from a snapshot-safe ``get()`` lookup.
+
+        Only single-key ``pk`` and ``id`` lookups can be answered from the
+        cached primary-key tuple while preserving Django's ``QuerySet.get()``
+        semantics. All other lookup shapes return ``None`` and use the normal
+        ORM path.
+
+        Parameters:
+            kwargs (dict[str, Any]): Lookup arguments passed to ``get()``.
+
+        Returns:
+            Any | None: Requested primary key when the lookup is snapshot-safe,
+            otherwise ``None``.
+        """
+        if len(kwargs) != 1:
+            return None
+        if "pk" in kwargs:
+            return kwargs["pk"]
+        if "id" in kwargs:
+            return kwargs["id"]
+        return None
+
     def _track_effective_dependencies(self) -> None:
         """Record the bucket's effective filter/exclude state when it is evaluated."""
         manager_name = self._manager_class.__name__
@@ -321,6 +451,11 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             GeneralManagerType: Manager instance for each primary key in the queryset.
         """
         self._track_effective_dependencies()
+        primary_keys = self._get_run_scoped_primary_keys()
+        if primary_keys is not None:
+            for primary_key in primary_keys:
+                yield self._build_manager(primary_key)
+            return
         for item in self._data:
             yield self._build_manager(item.pk)
 
@@ -365,6 +500,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             search_date=self._search_date,
             sort_keys=self._sort_keys,
             sort_reverse=self._sort_reverse,
+            run_scoped_cacheable=False,
         )
 
     def __merge_filter_definitions(
@@ -502,6 +638,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             search_date=search_date,
             sort_keys=self._sort_keys,
             sort_reverse=self._sort_reverse,
+            run_scoped_cacheable=self._run_scoped_cacheable,
         )
 
     def exclude(self, **kwargs: Any) -> DatabaseBucket[GeneralManagerType]:
@@ -549,6 +686,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             search_date=search_date,
             sort_keys=self._sort_keys,
             sort_reverse=self._sort_reverse,
+            run_scoped_cacheable=self._run_scoped_cacheable,
         )
 
     def first(self) -> GeneralManagerType | None:
@@ -559,6 +697,11 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             GeneralManagerType | None: First manager instance if available.
         """
         self._track_effective_dependencies()
+        primary_keys = self._peek_run_scoped_primary_keys()
+        if primary_keys is not None:
+            if not primary_keys:
+                return None
+            return self._build_manager(primary_keys[0])
         first_element = self._data.first()
         if first_element is None:
             return None
@@ -572,6 +715,11 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             GeneralManagerType | None: Last manager instance if available.
         """
         self._track_effective_dependencies()
+        primary_keys = self._peek_run_scoped_primary_keys()
+        if primary_keys is not None:
+            if not primary_keys:
+                return None
+            return self._build_manager(primary_keys[-1])
         first_element = self._data.last()
         if first_element is None:
             return None
@@ -585,6 +733,9 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             int: Number of queryset rows.
         """
         self._track_effective_dependencies()
+        primary_keys = self._peek_run_scoped_primary_keys()
+        if primary_keys is not None:
+            return len(primary_keys)
         return self._data.count()
 
     def all(self) -> DatabaseBucket[GeneralManagerType]:
@@ -602,6 +753,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             search_date=self._search_date,
             sort_keys=self._sort_keys,
             sort_reverse=self._sort_reverse,
+            run_scoped_cacheable=self._run_scoped_cacheable,
         )
 
     def get(self, **kwargs: Any) -> GeneralManagerType:
@@ -619,6 +771,16 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             models.MultipleObjectsReturned: Propagated when multiple rows satisfy the lookup.
         """
         self._track_effective_dependencies()
+        primary_keys = self._peek_run_scoped_primary_keys()
+        if primary_keys is not None:
+            requested_primary_key = self._snapshot_get_primary_key(kwargs)
+            if requested_primary_key is not None:
+                match_count = primary_keys.count(requested_primary_key)
+                if match_count == 1:
+                    return self._build_manager(requested_primary_key)
+                if match_count > 1:
+                    raise self._data.model.MultipleObjectsReturned
+                raise self._data.model.DoesNotExist
         element = self._data.get(**kwargs)
         return self._build_manager(element.pk)
 
@@ -643,8 +805,14 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
                 search_date=self._search_date,
                 sort_keys=self._sort_keys,
                 sort_reverse=self._sort_reverse,
+                run_scoped_cacheable=self._run_scoped_cacheable,
             )
         self._track_effective_dependencies()
+        primary_keys = self._peek_run_scoped_primary_keys()
+        if primary_keys is not None:
+            if item < 0:
+                raise ValueError
+            return self._build_manager(primary_keys[item])
         return self._build_manager(self._data[item].pk)
 
     def __len__(self) -> int:
@@ -655,6 +823,9 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             int: Size of the queryset.
         """
         self._track_effective_dependencies()
+        primary_keys = self._peek_run_scoped_primary_keys()
+        if primary_keys is not None:
+            return len(primary_keys)
         return self._data.count()
 
     def __str__(self) -> str:
@@ -694,6 +865,9 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             pk = item.pk
         if pk is None:
             return False
+        primary_keys = self._peek_run_scoped_primary_keys()
+        if primary_keys is not None:
+            return pk in primary_keys
         return self._data.filter(pk=pk).exists()
 
     def sort(
@@ -779,6 +953,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             search_date=self._search_date,
             sort_keys=key,
             sort_reverse=reverse,
+            run_scoped_cacheable=self._run_scoped_cacheable,
         )
 
     def none(self) -> DatabaseBucket[GeneralManagerType]:

{generalmanager-0.47.0 → generalmanager-0.49.0}/src/general_manager/cache/cache_decorator.py

@@ -28,6 +28,7 @@ from general_manager.cache.dependency_index import (
 )
 from general_manager.cache.dependency_publish import (
     CachePublishAborted,
+    PendingDependencyCachePublication,
     acquire_compute_lease,
     publish_dependency_cache_entry,
     release_compute_lease,
@@ -232,6 +233,7 @@ def cached(
                     )
                 lease = acquire_compute_lease(key)
 
+            lease_transferred_to_context = False
             try:
                 cached_hit = read_dependency_cache_hit(
                     cache_backend,
@@ -252,29 +254,47 @@ def cached(
                     for entry_key, entry_dependencies in entries:
                         record_fn(entry_key, set(entry_dependencies))
 
-                try:
-                    publish_dependency_cache_entry(
-                        cache_key=key,
-                        result=result,
-                        dependencies=dependencies,
-                        cache_backend=cache_backend,
-                        timeout=timeout,
-                        started_generation=started_generation,
-                        record_many_fn=(
-                            None if record_fn is record_dependencies else record_many
-                        ),
-                    )
-                except CachePublishAborted:
-                    logger.debug(
-                        "dependency cache publish aborted",
-                        context={
-                            "function": func.__qualname__,
-                            "key": key,
-                            "scope": scope,
-                        },
+                publish_context = current_calculation_run_context()
+                if publish_context is not None and record_fn is record_dependencies:
+                    publish_context.buffer_dependency_cache_publication(
+                        PendingDependencyCachePublication(
+                            cache_key=key,
+                            result=result,
+                            dependencies=frozenset(dependencies),
+                            cache_backend=cache_backend,
+                            timeout=timeout,
+                            started_generation=started_generation,
+                            lease=lease,
+                        )
                     )
+                    lease_transferred_to_context = True
+                else:
+                    try:
+                        publish_dependency_cache_entry(
+                            cache_key=key,
+                            result=result,
+                            dependencies=dependencies,
+                            cache_backend=cache_backend,
+                            timeout=timeout,
+                            started_generation=started_generation,
+                            record_many_fn=(
+                                None
+                                if record_fn is record_dependencies
+                                else record_many
+                            ),
+                        )
+                    except CachePublishAborted:
+                        logger.debug(
+                            "dependency cache publish aborted",
+                            context={
+                                "function": func.__qualname__,
+                                "key": key,
+                                "scope": scope,
+                            },
+                        )
             finally:
-                release_compute_lease(lease)
+                if not lease_transferred_to_context:
+                    release_compute_lease(lease)
 
             logger.debug(
                 "cache miss recorded",

{generalmanager-0.47.0 → generalmanager-0.49.0}/src/general_manager/cache/dependency_cache.py

@@ -45,6 +45,16 @@ class DependencyCacheGetManyBackend(DependencyCacheBackend, Protocol):
         ...
 
 
+class DependencyCacheSetManyBackend(DependencyCacheBackend, Protocol):
+    def set_many(
+        self,
+        data: Mapping[str, Any],
+        timeout: int | None = None,
+    ) -> Any:
+        """Store cached values for many keys."""
+        ...
+
+
 def make_dependency_cache_entry(
     value: Any,
     dependencies: Iterable[Dependency],

{generalmanager-0.47.0 → generalmanager-0.49.0}/src/general_manager/cache/dependency_index.py

@@ -124,6 +124,14 @@ def _set_dependency_data_change_count(count: int) -> int:
     return count
 
 
+def _discard_active_context_dependency_cache_state() -> None:
+    from general_manager.cache.run_context import current_calculation_run_context
+
+    context = current_calculation_run_context()
+    if context is not None:
+        context.discard_dependency_cache_state()
+
+
 def begin_dependency_data_change() -> int:
     """
     Mark a data change as active and bump the dependency generation.
@@ -136,9 +144,17 @@ def begin_dependency_data_change() -> int:
         generation = _set_dependency_generation(get_dependency_generation() + 1)
         _set_dependency_data_change_count(_get_dependency_data_change_count() + 1)
         cache.set(DATA_CHANGE_LOCK_KEY, "1", None)
-        return generation
     finally:
         release_lock()
+    try:
+        _discard_active_context_dependency_cache_state()
+    except Exception:
+        try:
+            end_dependency_data_change()
+        except Exception:
+            logger.exception("dependency data-change cleanup rollback failed")
+        raise
+    return generation
 
 
 def end_dependency_data_change() -> None:

{generalmanager-0.47.0 → generalmanager-0.49.0}/src/general_manager/cache/dependency_publish.py

@@ -4,13 +4,15 @@ from __future__ import annotations
 
 import time
 import uuid
+from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Callable, Iterable
+from typing import Any, Callable, Iterable, TypeGuard
 
 from django.core.cache import cache as coordination_cache
 
 from general_manager.cache.dependency_cache import (
     DependencyCacheBackend,
+    DependencyCacheSetManyBackend,
     make_dependency_cache_entry,
     read_dependency_cache_hit,
 )
@@ -39,6 +41,19 @@ class CacheComputeLease:
     token: str
 
 
+@dataclass(frozen=True)
+class PendingDependencyCachePublication:
+    """A dependency-cache miss waiting for guarded publication."""
+
+    cache_key: str
+    result: Any
+    dependencies: frozenset[Dependency]
+    cache_backend: DependencyCacheBackend
+    timeout: int | None
+    started_generation: int
+    lease: CacheComputeLease
+
+
 RecordManyDependenciesFn = Callable[[Iterable[tuple[str, Iterable[Dependency]]]], None]
 
 COMPUTE_LOCK_PREFIX = "dependency_cache_compute_lock"
@@ -111,6 +126,81 @@ def _ensure_publish_current(started_generation: int) -> None:
         raise CachePublishAborted()
 
 
+def _supports_set_many(
+    cache_backend: DependencyCacheBackend,
+) -> TypeGuard[DependencyCacheSetManyBackend]:
+    return callable(getattr(cache_backend, "set_many", None))
+
+
+def _set_dependency_cache_entries(
+    entries: Iterable[PendingDependencyCachePublication],
+) -> None:
+    grouped: dict[
+        tuple[int, int | None],
+        list[PendingDependencyCachePublication],
+    ] = defaultdict(list)
+    for entry in entries:
+        grouped[(id(entry.cache_backend), entry.timeout)].append(entry)
+
+    for group_entries in grouped.values():
+        cache_backend = group_entries[0].cache_backend
+        timeout = group_entries[0].timeout
+        payloads = {
+            entry.cache_key: make_dependency_cache_entry(
+                entry.result,
+                entry.dependencies,
+            )
+            for entry in group_entries
+        }
+        if _supports_set_many(cache_backend):
+            failed_keys = cache_backend.set_many(payloads, timeout) or ()
+            for key in failed_keys:
+                if key in payloads:
+                    cache_backend.set(key, payloads[key], timeout)
+            continue
+        for key, payload in payloads.items():
+            cache_backend.set(key, payload, timeout)
+
+
+def publish_dependency_cache_entries(
+    entries: Iterable[PendingDependencyCachePublication],
+) -> None:
+    """Publish dependency metadata and values for current entries in one batch."""
+    pending_entries = tuple(entries)
+    if not pending_entries:
+        return
+
+    acquire_lock_with_retry("publish_dependency_cache_entries")
+    try:
+        if is_dependency_data_change_active():
+            raise CachePublishAborted()
+
+        current_generation = get_dependency_generation()
+        publishable_entries = tuple(
+            entry
+            for entry in pending_entries
+            if entry.started_generation == current_generation
+        )
+        if not publishable_entries:
+            return
+
+        idx = get_full_index()
+        for entry in publishable_entries:
+            if entry.dependencies:
+                _record_dependencies_locked(
+                    idx,
+                    entry.cache_key,
+                    entry.dependencies,
+                )
+
+        _ensure_publish_current(current_generation)
+        set_full_index(idx)
+        _ensure_publish_current(current_generation)
+        _set_dependency_cache_entries(publishable_entries)
+    finally:
+        release_lock()
+
+
 def publish_dependency_cache_entry(
     *,
     cache_key: str,