GeneralManager 0.68.3__tar.gz → 0.69.1__tar.gz

{generalmanager-0.68.3/src/GeneralManager.egg-info → generalmanager-0.69.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.68.3
+Version: 0.69.1
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.68.3 → generalmanager-0.69.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "GeneralManager"
-version = "0.68.3"
+version = "0.69.1"
 description = "Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching."
 readme = "README.md"
 authors = [{ name = "Tim Kleindick", email = "tkleindick@yahoo.de" }]

{generalmanager-0.68.3 → generalmanager-0.69.1/src/GeneralManager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.68.3
+Version: 0.69.1
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.68.3 → generalmanager-0.69.1}/src/general_manager/api/graphql.py

@@ -6,6 +6,7 @@ import asyncio
 from contextlib import asynccontextmanager
 from copy import deepcopy
 import re
+from time import perf_counter
 from types import UnionType
 from typing import (
     AsyncIterator,
@@ -37,6 +38,7 @@ from general_manager.bucket.group_bucket import GroupBucket
 from general_manager.cache.dependency_index import (
     Dependency,
 )
+from general_manager.cache.data_change_context import record_data_change_phase
 from general_manager.cache.signals import post_data_change
 from general_manager.conf import get_setting
 from general_manager.interface.base_interface import InterfaceBase
@@ -1784,38 +1786,44 @@ class GraphQL:
             instance (GeneralManager | None): The GeneralManager instance that changed.
             action (str): A string describing the change action (e.g., "created", "updated", "deleted").
         """
-        event_instance = instance if instance is not None else previous_instance
-        if event_instance is None or not isinstance(event_instance, GeneralManager):
-            return
+        started = perf_counter()
+        try:
+            event_instance = instance if instance is not None else previous_instance
+            if event_instance is None or not isinstance(event_instance, GeneralManager):
+                return
 
-        if isinstance(sender, type) and issubclass(sender, GeneralManager):
-            manager_class: type[GeneralManager] = sender
-        else:
-            manager_class = event_instance.__class__
+            if isinstance(sender, type) and issubclass(sender, GeneralManager):
+                manager_class: type[GeneralManager] = sender
+            else:
+                manager_class = event_instance.__class__
 
-        if manager_class.__name__ not in cls.manager_registry:
-            logger.debug(
-                "skipping subscription event for unregistered manager",
-                context={
-                    "manager": manager_class.__name__,
-                    "action": action,
-                },
-            )
-            return
+            if manager_class.__name__ not in cls.manager_registry:
+                logger.debug(
+                    "skipping subscription event for unregistered manager",
+                    context={
+                        "manager": manager_class.__name__,
+                        "action": action,
+                    },
+                )
+                return
 
-        event_identification = deepcopy(
-            identification
-            if identification is not None
-            else event_instance.identification
-        )
-        transaction.on_commit(
-            lambda: cls._publish_data_change(
-                manager_class,
-                action,
-                event_identification,
-            ),
-            using=database_alias,
-        )
+            event_identification = deepcopy(
+                identification
+                if identification is not None
+                else event_instance.identification
+            )
+            transaction.on_commit(
+                lambda: cls._publish_data_change(
+                    manager_class,
+                    action,
+                    event_identification,
+                ),
+                using=database_alias,
+            )
+        finally:
+            record_data_change_phase(
+                "subscription", perf_counter() - started, database_alias
+            )
 
     @classmethod
     def _publish_data_change(

{generalmanager-0.68.3 → generalmanager-0.69.1}/src/general_manager/api/remote_invalidation.py

@@ -8,6 +8,7 @@ from datetime import date, datetime
 import json
 from importlib import import_module
 import re
+from time import perf_counter
 from uuid import UUID, uuid4
 from collections.abc import Callable, Coroutine, Mapping, MutableSequence, Sequence
 from types import ModuleType
@@ -25,6 +26,7 @@ from general_manager.api.remote_api import (
     get_remote_api_config,
 )
 from general_manager.api.notification_batching import _queue_notification
+from general_manager.cache.data_change_context import record_data_change_phase
 from general_manager.cache.signals import post_data_change
 from general_manager.logging import get_logger
 
@@ -200,24 +202,30 @@ def emit_remote_invalidation(
     originating database alias and delegates channel lookup, batching, payload
     creation, and delivery to `_publish_remote_invalidation` after commit.
     """
-    config = get_remote_api_config(sender)
-    if config is None or not config.websocket_invalidation:
-        return
+    started = perf_counter()
+    try:
+        config = get_remote_api_config(sender)
+        if config is None or not config.websocket_invalidation:
+            return
 
-    event_identification = identification
-    if event_identification is None and instance is not None:
-        event_identification = dict(instance.identification)
-    captured_identification = (
-        deepcopy(event_identification) if event_identification is not None else None
-    )
-    transaction.on_commit(
-        lambda: _publish_remote_invalidation(
-            config,
-            action,
-            captured_identification,
-        ),
-        using=database_alias,
-    )
+        event_identification = identification
+        if event_identification is None and instance is not None:
+            event_identification = dict(instance.identification)
+        captured_identification = (
+            deepcopy(event_identification) if event_identification is not None else None
+        )
+        transaction.on_commit(
+            lambda: _publish_remote_invalidation(
+                config,
+                action,
+                captured_identification,
+            ),
+            using=database_alias,
+        )
+    finally:
+        record_data_change_phase(
+            "subscription", perf_counter() - started, database_alias
+        )
 
 
 post_data_change.connect(emit_remote_invalidation, weak=False)

{generalmanager-0.68.3 → generalmanager-0.69.1}/src/general_manager/cache/data_change_context.py

@@ -5,17 +5,45 @@ from __future__ import annotations
 from collections.abc import Iterator
 from contextlib import contextmanager
 from contextvars import ContextVar
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from math import isfinite
+from typing import Literal
+
+
+@dataclass(frozen=True)
+class DataChangeTransactionContext:
+    """Mutable state owned by one framework-managed data-change transaction."""
+
+    database_alias: str
+    caller_in_atomic_block: bool
+    changed_classes: set[str] = field(default_factory=set)
+    metadata: dict[str, object] = field(default_factory=dict)
+    phase_seconds: dict[str, float] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class DataChangeTransactionScope:
+    """Public view of a context-manager entry's transaction state."""
+
+    transaction: DataChangeTransactionContext
+    is_outermost: bool
 
 
 @dataclass
 class _DataChangeTransaction:
     """Live marker for one framework-owned transaction envelope."""
 
-    database_alias: str
-    caller_in_atomic_block: bool
+    context: DataChangeTransactionContext
     active: bool = True
 
+    @property
+    def database_alias(self) -> str:
+        return self.context.database_alias
+
+    @property
+    def caller_in_atomic_block(self) -> bool:
+        return self.context.caller_in_atomic_block
+
 
 _owned_transactions: ContextVar[tuple[_DataChangeTransaction, ...]] = ContextVar(
     "owned_data_change_transactions",
@@ -34,25 +62,68 @@ def _current_owner(database_alias: str) -> _DataChangeTransaction | None:
     return None
 
 
+def current_data_change_transaction(
+    database_alias: str,
+) -> DataChangeTransactionContext | None:
+    """Return the live framework-owned context for a database alias, if any."""
+    owner = _current_owner(database_alias)
+    return owner.context if owner is not None else None
+
+
 @contextmanager
 def own_data_change_transaction(
     database_alias: str,
     *,
     caller_in_atomic_block: bool,
-) -> Iterator[None]:
+) -> Iterator[DataChangeTransactionScope]:
     """Record a live framework envelope without inspecting Django internals."""
-    owner = _DataChangeTransaction(
+    existing_owner = _current_owner(database_alias)
+    if existing_owner is not None:
+        yield DataChangeTransactionScope(
+            transaction=existing_owner.context,
+            is_outermost=False,
+        )
+        return
+
+    context = DataChangeTransactionContext(
         database_alias=database_alias,
         caller_in_atomic_block=caller_in_atomic_block,
     )
+    owner = _DataChangeTransaction(
+        context=context,
+    )
     token = _owned_transactions.set((*_owned_transactions.get(), owner))
     try:
-        yield
+        yield DataChangeTransactionScope(transaction=context, is_outermost=True)
     finally:
         owner.active = False
         _owned_transactions.reset(token)
 
 
+def register_data_change_class(class_name: str, database_alias: str) -> bool:
+    """Record a changed class when the framework owns the live transaction."""
+    context = current_data_change_transaction(database_alias)
+    if context is None or context.caller_in_atomic_block:
+        return False
+    context.changed_classes.add(class_name)
+    return True
+
+
+def record_data_change_phase(
+    phase: Literal["database", "invalidation", "subscription", "search", "workflow"],
+    duration_seconds: float,
+    database_alias: str,
+) -> None:
+    """Accumulate a non-negative duration for a phase in the live context."""
+    if not isfinite(duration_seconds):
+        return
+    context = current_data_change_transaction(database_alias)
+    if context is not None:
+        context.phase_seconds[phase] = context.phase_seconds.get(phase, 0.0) + max(
+            duration_seconds, 0.0
+        )
+
+
 @contextmanager
 def authorize_data_change_operation(database_alias: str) -> Iterator[None]:
     """Authorize only the decorated manager method to reuse its envelope."""

{generalmanager-0.68.3 → generalmanager-0.69.1}/src/general_manager/cache/dependency_index.py

@@ -8,9 +8,11 @@ import time
 import uuid
 from contextvars import ContextVar
 from datetime import date, datetime
+from time import perf_counter
 from typing import TYPE_CHECKING, Callable, Iterable, Literal, Tuple, Type, cast
 
 from django.core.cache import cache
+from django.db import DEFAULT_DB_ALIAS
 from django.dispatch import receiver
 
 from general_manager.cache.dependency_matching import (
@@ -34,6 +36,7 @@ from general_manager.cache.dependency_shards import (
     reverse_memberships,
     tracked_lookup_names,
 )
+from general_manager.cache.data_change_context import record_data_change_phase
 from general_manager.cache.signals import post_data_change, pre_data_change
 from general_manager.logging import get_logger
 
@@ -1384,6 +1387,7 @@ def generic_cache_invalidation(
     sender: type[GeneralManager],
     instance: GeneralManager,
     old_relevant_values: dict[str, object],
+    database_alias: str = DEFAULT_DB_ALIAS,
     **kwargs: object,
 ) -> None:
     """
@@ -1395,27 +1399,34 @@ def generic_cache_invalidation(
         sender (type[GeneralManager]): Manager class that emitted the signal.
         instance (GeneralManager): The manager instance that was changed.
         old_relevant_values (dict[str, object]): Mapping of lookup paths (joined by "__") to their values as captured before the change; used to compare old vs. new values for invalidation decisions.
+        database_alias (str): Database alias used to attribute invalidation timing.
     """
-    manager_name = sender.__name__
-    invalidated_cache_keys: set[str] = set()
-    lock_token = acquire_lock_with_retry("generic_cache_invalidation")
+    started = perf_counter()
     try:
-        if legacy_dependency_index_exists():
-            idx = get_full_index()
-            invalidated_cache_keys = _generic_cache_invalidation_locked(
-                idx,
-                manager_name,
-                instance,
-                old_relevant_values,
-            )
-            set_full_index(idx)
-        else:
-            invalidated_cache_keys = _generic_cache_invalidation_from_shards(
-                manager_name,
-                instance,
-                old_relevant_values,
-            )
+        manager_name = sender.__name__
+        invalidated_cache_keys: set[str] = set()
+        lock_token = acquire_lock_with_retry("generic_cache_invalidation")
+        try:
+            if legacy_dependency_index_exists():
+                idx = get_full_index()
+                invalidated_cache_keys = _generic_cache_invalidation_locked(
+                    idx,
+                    manager_name,
+                    instance,
+                    old_relevant_values,
+                )
+                set_full_index(idx)
+            else:
+                invalidated_cache_keys = _generic_cache_invalidation_from_shards(
+                    manager_name,
+                    instance,
+                    old_relevant_values,
+                )
+        finally:
+            release_lock(lock_token)
+        if invalidated_cache_keys:
+            record_invalidated_cache_keys_for_graphql_rewarm(invalidated_cache_keys)
     finally:
-        release_lock(lock_token)
-    if invalidated_cache_keys:
-        record_invalidated_cache_keys_for_graphql_rewarm(invalidated_cache_keys)
+        record_data_change_phase(
+            "invalidation", perf_counter() - started, database_alias
+        )

{generalmanager-0.68.3 → generalmanager-0.69.1}/src/general_manager/cache/run_context.py

@@ -6,6 +6,7 @@ from collections import defaultdict
 from collections.abc import Callable, Hashable, Iterable, Mapping
 from contextvars import ContextVar, Token
 from dataclasses import dataclass
+from threading import RLock, get_ident
 from types import TracebackType
 from typing import TYPE_CHECKING, Optional, TypeVar, cast
 
@@ -43,8 +44,10 @@ ORM_BUCKET_EXISTS_PREFIX = "orm_bucket_exists"
 BUCKET_INDEX_PREFIX = "bucket_index"
 TRUSTED_ORM_MANAGER_PREFIX = "trusted_orm_manager"
 DEFAULT_DEPENDENCY_CACHE_PUBLISH_BATCH_SIZE = 1000
+RUN_CONTEXT_TOUCH_BATCH_SIZE = 64
 logger = get_logger("cache.run_context")
 OrmModelRowKey = tuple[Hashable, Hashable | None]
+PendingRunCacheTouch = tuple[RunCacheNamespace, Hashable]
 
 
 @dataclass(frozen=True)
@@ -104,9 +107,16 @@ class CalculationRunContext:
         ] = {}
         self._dependency_cache_publish_batch_size = dependency_cache_publish_batch_size
         self._tokens: list[Token[CalculationRunContext | None]] = []
+        self._run_cache_touch_lock = RLock()
+        self._pending_run_cache_touches: dict[PendingRunCacheTouch, None] = {}
+        self._last_pending_run_cache_touch: PendingRunCacheTouch | None = None
+        self._run_cache_touch_count = 0
+        self._run_cache_touch_thread_id: int | None = None
+        max_bytes = resolve_run_context_cache_max_bytes()
+        self._run_cache_budget_enabled = max_bytes is not None
         run_context_cache_budget.register(
             self,
-            resolve_run_context_cache_max_bytes(),
+            max_bytes,
         )
 
     def _iter_run_cache_entries(
@@ -119,31 +129,45 @@ class CalculationRunContext:
             if key not in self._dependency_cache_pending_publications:
                 yield "dependency_hits", key, hit
 
+    def _set_run_cache_budget_enabled(self, enabled: bool) -> None:
+        """Refresh the owner hint after a coordinator transition."""
+        with self._run_cache_touch_lock:
+            self._run_cache_budget_enabled = enabled
+            if not enabled:
+                self._pending_run_cache_touches.clear()
+                self._last_pending_run_cache_touch = None
+                self._run_cache_touch_count = 0
+
     def _evict_run_cache_entry(
         self,
         namespace: RunCacheNamespace,
         key: Hashable,
     ) -> None:
         """Remove one entry selected by the process-wide coordinator."""
+        self._discard_run_cache_touch((namespace, key))
         if namespace == "values":
             self._values.pop(key, None)
         elif key not in self._dependency_cache_pending_publications:
             self._dependency_cache_hits.pop(cast(str, key), None)
 
     def _store_run_value(self, scoped_key: Hashable, value: object) -> None:
+        self._flush_run_cache_touches()
         self._values[scoped_key] = value
         run_context_cache_budget.track(self, "values", scoped_key, value)
 
     def _remove_run_value(self, scoped_key: Hashable) -> None:
+        self._discard_run_cache_touch(("values", scoped_key))
         if scoped_key in self._values:
             del self._values[scoped_key]
             run_context_cache_budget.remove(self, "values", scoped_key)
 
     def _refresh_run_value(self, scoped_key: Hashable) -> None:
+        self._discard_run_cache_touch(("values", scoped_key))
         try:
             value = self._values[scoped_key]
         except KeyError:
             return
+        self._flush_run_cache_touches()
         run_context_cache_budget.refresh(self, "values", scoped_key, value)
 
     def _store_dependency_cache_hit(
@@ -151,26 +175,109 @@ class CalculationRunContext:
         key: str,
         hit: DependencyCacheHit,
     ) -> None:
+        is_pending_publication = key in self._dependency_cache_pending_publications
+        if not is_pending_publication:
+            self._flush_run_cache_touches()
         self._dependency_cache_hits[key] = hit
-        if key in self._dependency_cache_pending_publications:
+        if is_pending_publication:
+            self._discard_run_cache_touch(("dependency_hits", key))
             run_context_cache_budget.remove(self, "dependency_hits", key)
         else:
             run_context_cache_budget.track(self, "dependency_hits", key, hit)
 
     def _remove_dependency_cache_hit(self, key: str) -> None:
+        self._discard_run_cache_touch(("dependency_hits", key))
         if key in self._dependency_cache_hits:
             del self._dependency_cache_hits[key]
             run_context_cache_budget.remove(self, "dependency_hits", key)
 
     def _refresh_dependency_cache_hit(self, key: str) -> None:
+        self._discard_run_cache_touch(("dependency_hits", key))
         if key in self._dependency_cache_pending_publications:
             return
         try:
             hit = self._dependency_cache_hits[key]
         except KeyError:
             return
+        self._flush_run_cache_touches()
         run_context_cache_budget.refresh(self, "dependency_hits", key, hit)
 
+    def _queue_run_cache_touch(
+        self,
+        namespace: RunCacheNamespace,
+        key: Hashable,
+    ) -> None:
+        pending_key = (namespace, key)
+        with self._run_cache_touch_lock:
+            if not self._run_cache_budget_enabled:
+                return
+            if pending_key != self._last_pending_run_cache_touch:
+                self._pending_run_cache_touches.pop(pending_key, None)
+                self._pending_run_cache_touches[pending_key] = None
+                self._last_pending_run_cache_touch = pending_key
+        self._queue_repeated_run_cache_touch(pending_key)
+
+    def _queue_repeated_run_cache_touch(
+        self,
+        pending_key: PendingRunCacheTouch,
+    ) -> bool:
+        touch_immediately = False
+        touches_to_flush: tuple[PendingRunCacheTouch, ...] = ()
+        with self._run_cache_touch_lock:
+            if not self._run_cache_budget_enabled:
+                return False
+            if self._run_cache_touch_thread_id != get_ident():
+                touch_immediately = True
+            elif pending_key != self._last_pending_run_cache_touch:
+                return False
+            else:
+                count = self._run_cache_touch_count + 1
+                if count < RUN_CONTEXT_TOUCH_BATCH_SIZE:
+                    self._run_cache_touch_count = count
+                    return True
+                touches_to_flush = self._take_run_cache_touches_locked()
+        if touch_immediately:
+            run_context_cache_budget.touch(self, *pending_key)
+            return True
+        run_context_cache_budget.touch_many(self, touches_to_flush)
+        return True
+
+    def _touch_run_cache_entry(
+        self,
+        namespace: RunCacheNamespace,
+        key: Hashable,
+    ) -> None:
+        pending_key: PendingRunCacheTouch = (namespace, key)
+        if not self._queue_repeated_run_cache_touch(pending_key):
+            self._queue_run_cache_touch(namespace, key)
+
+    def _discard_run_cache_touch(self, pending_key: PendingRunCacheTouch) -> None:
+        with self._run_cache_touch_lock:
+            self._pending_run_cache_touches.pop(pending_key, None)
+            if pending_key == self._last_pending_run_cache_touch:
+                self._last_pending_run_cache_touch = next(
+                    reversed(self._pending_run_cache_touches),
+                    None,
+                )
+
+    def _take_run_cache_touches_locked(
+        self,
+    ) -> tuple[PendingRunCacheTouch, ...]:
+        touches = tuple(self._pending_run_cache_touches)
+        self._pending_run_cache_touches.clear()
+        self._last_pending_run_cache_touch = None
+        self._run_cache_touch_count = 0
+        return touches
+
+    def _flush_run_cache_touches(self) -> None:
+        with self._run_cache_touch_lock:
+            if not self._pending_run_cache_touches:
+                self._last_pending_run_cache_touch = None
+                self._run_cache_touch_count = 0
+                return
+            touches = self._take_run_cache_touches_locked()
+        run_context_cache_budget.touch_many(self, touches)
+
     @staticmethod
     def _scoped_key(key: Hashable) -> Hashable:
         """Namespace a run value by the active historical snapshot."""
@@ -182,9 +289,16 @@ class CalculationRunContext:
     def __enter__(self) -> "CalculationRunContext":
         """Activate this context and return it for the `with` block."""
         if not self._tokens:
+            max_bytes = resolve_run_context_cache_max_bytes()
+            with self._run_cache_touch_lock:
+                self._pending_run_cache_touches.clear()
+                self._last_pending_run_cache_touch = None
+                self._run_cache_touch_count = 0
+                self._run_cache_touch_thread_id = get_ident()
+                self._run_cache_budget_enabled = max_bytes is not None
             run_context_cache_budget.register(
                 self,
-                resolve_run_context_cache_max_bytes(),
+                max_bytes,
             )
         self._tokens.append(_active_context.set(self))
         return self
@@ -219,6 +333,12 @@ class CalculationRunContext:
 
         token = self._tokens.pop()
         is_outermost_exit = not self._tokens
+        if is_outermost_exit:
+            with self._run_cache_touch_lock:
+                self._pending_run_cache_touches.clear()
+                self._last_pending_run_cache_touch = None
+                self._run_cache_touch_count = 0
+                self._run_cache_touch_thread_id = None
         try:
             if is_outermost_exit:
                 if exc_type is None:
@@ -253,7 +373,7 @@ class CalculationRunContext:
             value = loader()
             self._store_run_value(scoped_key, value)
             return value
-        run_context_cache_budget.touch(self, "values", scoped_key)
+        self._touch_run_cache_entry("values", scoped_key)
         return cast(T, value)
 
     def get(self, key: Hashable, default: object = None) -> object:
@@ -263,7 +383,7 @@ class CalculationRunContext:
             value = self._values[scoped_key]
         except KeyError:
             return default
-        run_context_cache_budget.touch(self, "values", scoped_key)
+        self._touch_run_cache_entry("values", scoped_key)
         return value
 
     def set(self, key: Hashable, value: object) -> None:
@@ -295,7 +415,7 @@ class CalculationRunContext:
         except KeyError:
             return default
         if key not in self._dependency_cache_pending_publications:
-            run_context_cache_budget.touch(self, "dependency_hits", key)
+            self._touch_run_cache_entry("dependency_hits", key)
         return hit
 
     def buffer_dependency_cache_publication(
@@ -718,7 +838,7 @@ class CalculationRunContext:
         scoped_key = self._scoped_key(key)
         if scoped_key not in self._values:
             return False
-        run_context_cache_budget.touch(self, "values", scoped_key)
+        self._touch_run_cache_entry("values", scoped_key)
         return True
 
     def __contains__(self, key: Hashable) -> bool: