GeneralManager 0.37.0__tar.gz → 0.37.2__tar.gz

{generalmanager-0.37.0/src/GeneralManager.egg-info → generalmanager-0.37.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.37.0
+Version: 0.37.2
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.37.0 → generalmanager-0.37.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "GeneralManager"
-version = "0.37.0"
+version = "0.37.2"
 description = "Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching."
 readme = "README.md"
 authors = [{ name = "Tim Kleindick", email = "tkleindick@yahoo.de" }]

{generalmanager-0.37.0 → generalmanager-0.37.2/src/GeneralManager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.37.0
+Version: 0.37.2
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.37.0 → generalmanager-0.37.2}/src/general_manager/bucket/database_bucket.py

@@ -8,6 +8,8 @@ from django.core.exceptions import FieldError
 from django.db import models
 
 from general_manager.bucket.base_bucket import Bucket
+from general_manager.cache.cache_tracker import DependencyTracker
+from general_manager.cache.dependency_index import serialize_dependency_identifier
 from general_manager.manager.general_manager import GeneralManager
 from general_manager.utils.filter_parser import create_filter_function
 
@@ -149,6 +151,8 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         exclude_definitions: dict[str, list[Any]] | None = None,
         *,
         search_date: datetime | date | None = None,
+        sort_keys: tuple[str, ...] | None = None,
+        sort_reverse: bool = False,
     ) -> None:
         """
         Instantiate a database-backed bucket with optional filter state.
@@ -165,15 +169,77 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         """
         self._data = data
         self._manager_class = manager_class
-        self.filters = {**(filter_definitions or {})}
-        self.excludes = {**(exclude_definitions or {})}
+        self.filters = self._copy_filter_definitions(filter_definitions)
+        self.excludes = self._copy_filter_definitions(exclude_definitions)
         self._search_date = search_date
+        self._sort_keys = sort_keys
+        self._sort_reverse = sort_reverse
 
     def _build_manager(self, pk: Any) -> GeneralManagerType:
         if self._search_date is None:
             return self._manager_class(pk)
         return self._manager_class(pk, search_date=self._search_date)
 
+    @staticmethod
+    def _copy_filter_definitions(
+        definitions: dict[str, Any] | None,
+    ) -> dict[str, list[Any]]:
+        """
+        Return a copy of filter/exclude definitions without sharing nested lists.
+        """
+        copied: dict[str, list[Any]] = {}
+        for key, values in (definitions or {}).items():
+            if isinstance(values, list):
+                copied[key] = list(values)
+            elif isinstance(values, tuple):
+                copied[key] = list(values)
+            else:
+                copied[key] = [values]
+        return copied
+
+    @staticmethod
+    def _normalize_dependency_mapping(
+        definitions: dict[str, Any],
+    ) -> dict[str, Any]:
+        return {
+            key: values[0]
+            if isinstance(values, (list, tuple)) and len(values) == 1
+            else values
+            for key, values in definitions.items()
+        }
+
+    def _track_effective_dependencies(self) -> None:
+        """Record the bucket's effective filter/exclude state when it is evaluated."""
+        manager_name = self._manager_class.__name__
+        normalized_filters = self._normalize_dependency_mapping(self.filters)
+        normalized_excludes = self._normalize_dependency_mapping(self.excludes)
+        if self.filters:
+            DependencyTracker.track(
+                manager_name,
+                "filter",
+                serialize_dependency_identifier(normalized_filters),
+            )
+        else:
+            DependencyTracker.track(manager_name, "all", "")
+        if self.excludes:
+            DependencyTracker.track(
+                manager_name,
+                "exclude",
+                serialize_dependency_identifier(normalized_excludes),
+            )
+        if self._sort_keys:
+            for sort_key in self._sort_keys:
+                payload = {
+                    "filters": normalized_filters,
+                    "excludes": normalized_excludes,
+                    "reverse": self._sort_reverse,
+                }
+                DependencyTracker.track(
+                    manager_name,
+                    "filter",
+                    serialize_dependency_identifier({f"__sort__{sort_key}": payload}),
+                )
+
     def __iter__(self) -> Generator[GeneralManagerType, None, None]:
         """
         Iterate over manager instances corresponding to the queryset rows.
@@ -181,6 +247,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         Yields:
             GeneralManagerType: Manager instance for each primary key in the queryset.
         """
+        self._track_effective_dependencies()
         for item in self._data:
             yield self._build_manager(item.pk)
 
@@ -223,6 +290,8 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             self._manager_class,
             {},
             search_date=self._search_date,
+            sort_keys=self._sort_keys,
+            sort_reverse=self._sort_reverse,
         )
 
     def __merge_filter_definitions(
@@ -238,9 +307,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         Returns:
             dict[str, list[Any]]: Combined mapping of lookups to value lists.
         """
-        kwarg_filter: dict[str, list[Any]] = {}
-        for key, value in basis.items():
-            kwarg_filter[key] = value
+        kwarg_filter = self._copy_filter_definitions(basis)
         for key, value in kwargs.items():
             if key not in kwarg_filter:
                 kwarg_filter[key] = []
@@ -360,6 +427,8 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             merged_filter,
             self.excludes,
             search_date=search_date,
+            sort_keys=self._sort_keys,
+            sort_reverse=self._sort_reverse,
         )
 
     def exclude(self, **kwargs: Any) -> DatabaseBucket[GeneralManagerType]:
@@ -405,6 +474,8 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             self.filters,
             merged_exclude,
             search_date=search_date,
+            sort_keys=self._sort_keys,
+            sort_reverse=self._sort_reverse,
         )
 
     def first(self) -> GeneralManagerType | None:
@@ -414,6 +485,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         Returns:
             GeneralManagerType | None: First manager instance if available.
         """
+        self._track_effective_dependencies()
         first_element = self._data.first()
         if first_element is None:
             return None
@@ -426,6 +498,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         Returns:
             GeneralManagerType | None: Last manager instance if available.
         """
+        self._track_effective_dependencies()
         first_element = self._data.last()
         if first_element is None:
             return None
@@ -438,6 +511,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         Returns:
             int: Number of queryset rows.
         """
+        self._track_effective_dependencies()
         return self._data.count()
 
     def all(self) -> DatabaseBucket[GeneralManagerType]:
@@ -450,7 +524,11 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         return self.__class__(
             self._data.all(),
             self._manager_class,
+            self.filters,
+            self.excludes,
             search_date=self._search_date,
+            sort_keys=self._sort_keys,
+            sort_reverse=self._sort_reverse,
         )
 
     def get(self, **kwargs: Any) -> GeneralManagerType:
@@ -467,6 +545,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             models.ObjectDoesNotExist: Propagated from the underlying queryset when no row matches.
             models.MultipleObjectsReturned: Propagated when multiple rows satisfy the lookup.
         """
+        self._track_effective_dependencies()
         element = self._data.get(**kwargs)
         return self._build_manager(element.pk)
 
@@ -486,8 +565,13 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             return self.__class__(
                 self._data[item],
                 self._manager_class,
+                self.filters,
+                self.excludes,
                 search_date=self._search_date,
+                sort_keys=self._sort_keys,
+                sort_reverse=self._sort_reverse,
             )
+        self._track_effective_dependencies()
         return self._build_manager(self._data[item].pk)
 
     def __len__(self) -> int:
@@ -497,6 +581,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         Returns:
             int: Size of the queryset.
         """
+        self._track_effective_dependencies()
         return self._data.count()
 
     def __str__(self) -> str:
@@ -529,6 +614,7 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
         """
         from general_manager.manager.general_manager import GeneralManager
 
+        self._track_effective_dependencies()
         if isinstance(item, GeneralManager):
             return item.identification.get("id", None) in self._data.values_list(
                 "pk", flat=True
@@ -610,7 +696,15 @@ class DatabaseBucket(Bucket[GeneralManagerType]):
             except (FieldError, TypeError, ValueError) as error:
                 raise QuerysetOrderingError(error) from error
 
-        return self.__class__(qs, self._manager_class, search_date=self._search_date)
+        return self.__class__(
+            qs,
+            self._manager_class,
+            self.filters,
+            self.excludes,
+            search_date=self._search_date,
+            sort_keys=key,
+            sort_reverse=reverse,
+        )
 
     def none(self) -> DatabaseBucket[GeneralManagerType]:
         """

{generalmanager-0.37.0 → generalmanager-0.37.2}/src/general_manager/cache/dependency_index.py

@@ -24,7 +24,7 @@ type lookup = str  # e.g. "field__gt", "field__in", "field__contains", "field"
 type cache_keys = set[str]  # e.g. "cache_key_1", "cache_key_2"
 type identifier = str  # e.g. "{'id': 1}"", "{'project': Project(**{'id': 1})}", ...
 type dependency_index = dict[
-    Literal["filter", "exclude", "request_query"],
+    Literal["filter", "exclude", "request_query", "all"],
     dict[
         general_manager_name,
         dict[attribute, dict[lookup, cache_keys]] | dict[identifier, cache_keys],
@@ -34,7 +34,9 @@ type lookup_dependency_map = dict[lookup, cache_keys]
 type manager_dependency_section = dict[attribute, lookup_dependency_map]
 type request_query_manager_section = dict[identifier, cache_keys]
 
-type filter_type = Literal["filter", "exclude", "identification", "request_query"]
+type filter_type = Literal[
+    "filter", "exclude", "identification", "request_query", "all"
+]
 type Dependency = Tuple[general_manager_name, filter_type, str]
 
 logger = get_logger("cache.dependency_index")
@@ -64,6 +66,8 @@ LOCK_TIMEOUT = 5  # Lock TTL in seconds
 UNDEFINED = object()  # Sentinel for undefined values
 ACTIONS: tuple[Literal["filter"], Literal["exclude"]] = ("filter", "exclude")
 REQUEST_QUERY_ACTION: Literal["request_query"] = "request_query"
+ALL_RECORDS_LOOKUP = "__all__"
+ALL_RECORDS_VALUE = "__all__"
 
 
 # -----------------------------------------------------------------------------
@@ -130,15 +134,31 @@ def get_full_index() -> dependency_index:
     Fetch the dependency index from cache, initialising it on first access.
 
     Returns:
-        dependency_index: Mapping of tracked `filter`, `exclude`, and
+        dependency_index: Mapping of tracked `filter`, `exclude`, `all`, and
         `request_query` dependencies keyed by manager name.
     """
     cached_index = cache.get(INDEX_KEY, None)
     if cached_index is None:
-        idx: dependency_index = {"filter": {}, "exclude": {}, "request_query": {}}
+        idx: dependency_index = {
+            "filter": {},
+            "exclude": {},
+            "request_query": {},
+            "all": {},
+        }
         cache.set(INDEX_KEY, idx, None)
         return idx
-    return cast(dependency_index, cached_index)
+    idx = cast(dependency_index, cached_index)
+    changed = False
+    for key in cast(
+        tuple[Literal["filter", "exclude", "request_query", "all"], ...],
+        ("filter", "exclude", "request_query", "all"),
+    ):
+        if key not in idx:
+            idx[key] = {}
+            changed = True
+    if changed:
+        cache.set(INDEX_KEY, idx, None)
+    return idx
 
 
 def set_full_index(idx: dependency_index) -> None:
@@ -223,6 +243,10 @@ def record_dependencies(
                     idx[action_key],
                 )
                 section = action_section.setdefault(model_name, {})
+                if not params:
+                    lookup_map = section.setdefault(ALL_RECORDS_LOOKUP, {})
+                    lookup_map.setdefault(ALL_RECORDS_VALUE, set()).add(cache_key)
+                    continue
                 if len(params) > 1:
                     cache_dependencies = section.setdefault(
                         "__cache_dependencies__", {}
@@ -243,6 +267,13 @@ def record_dependencies(
                 request_section = request_index.setdefault(model_name, {})
                 request_section.setdefault(identifier, set()).add(cache_key)
 
+            elif action == "all":
+                all_index = cast(
+                    dict[str, set[str]],
+                    idx.setdefault("all", {}),
+                )
+                all_index.setdefault(model_name, set()).add(cache_key)
+
             else:
                 # Treat identification lookups as a simple filter on `id`
                 filter_section = cast(
@@ -278,6 +309,12 @@ def remove_cache_key_from_index(cache_key: str) -> None:
     acquire_lock_with_retry("remove_cache_key_from_index")
     try:
         idx = get_full_index()
+        all_section = cast(dict[str, set[str]], idx.get("all", {}))
+        for mname, key_set in list(all_section.items()):
+            if cache_key in key_set:
+                key_set.remove(cache_key)
+                if not key_set:
+                    del all_section[mname]
         for action in ACTIONS:
             action_section = cast(
                 dict[general_manager_name, manager_dependency_section],
@@ -286,7 +323,7 @@ def remove_cache_key_from_index(cache_key: str) -> None:
             for mname, model_section in list(action_section.items()):
                 cache_dependencies = model_section.get("__cache_dependencies__", {})
                 for lookup, lookup_map in list(model_section.items()):
-                    if lookup.startswith("__"):
+                    if lookup == "__cache_dependencies__":
                         continue
                     lookup_map = cast(lookup_dependency_map, lookup_map)
                     for val_key, key_set in list(lookup_map.items()):
@@ -340,6 +377,12 @@ def _remove_cache_keys_from_index_locked(
     cache_keys: tuple[str, ...],
 ) -> None:
     """Remove cache keys from all dependency-index sections while the lock is held."""
+    all_section = cast(dict[str, set[str]], idx.get("all", {}))
+    for mname, key_set in list(all_section.items()):
+        for cache_key in cache_keys:
+            key_set.discard(cache_key)
+        if not key_set:
+            del all_section[mname]
     for action in ACTIONS:
         action_section = cast(
             dict[general_manager_name, manager_dependency_section],
@@ -348,7 +391,7 @@ def _remove_cache_keys_from_index_locked(
         for mname, model_section in list(action_section.items()):
             cache_dependencies = model_section.get("__cache_dependencies__", {})
             for lookup, lookup_map in list(model_section.items()):
-                if lookup.startswith("__"):
+                if lookup == "__cache_dependencies__":
                     continue
                 lookup_map = cast(lookup_dependency_map, lookup_map)
                 for val_key, key_set in list(lookup_map.items()):
@@ -453,11 +496,15 @@ def capture_old_values(
     for action in ACTIONS:
         model_section = idx[action].get(manager_name)
         if isinstance(model_section, dict):
-            lookups |= {
-                lookup
-                for lookup in model_section.keys()
-                if isinstance(lookup, str) and not lookup.startswith("__")
-            }
+            for lookup in model_section.keys():
+                if not isinstance(lookup, str):
+                    continue
+                if lookup.startswith("__sort__"):
+                    lookups.add(lookup.removeprefix("__sort__"))
+                    continue
+                if lookup.startswith("__"):
+                    continue
+                lookups.add(lookup)
         elif isinstance(model_section, list):
             lookups |= set(model_section)
     if lookups and instance.identification:
@@ -504,6 +551,20 @@ def generic_cache_invalidation(
             },
         )
     idx = get_full_index()
+    all_cache_keys = tuple(
+        cast(dict[str, set[str]], idx.get("all", {})).get(manager_name, set())
+    )
+    for cache_key in all_cache_keys:
+        logger.info(
+            "invalidating cache key",
+            context={
+                "manager": manager_name,
+                "key": cache_key,
+                "action": "all",
+            },
+        )
+        invalidate_cache_key(cache_key)
+        remove_cache_key_from_index(cache_key)
 
     def _json_loads_val_key(val_key: Any) -> Any:
         if isinstance(val_key, str):
@@ -783,6 +844,74 @@ def generic_cache_invalidation(
                     return True
         return False
 
+    def bucket_membership_matches(
+        params: dict[str, Any],
+        *,
+        use_old_values: bool,
+    ) -> bool:
+        """
+        Check whether the changed row belongs to a bucket described by filters/excludes.
+        """
+        filters = params.get("filters", {})
+        excludes = params.get("excludes", {})
+        if not isinstance(filters, dict) or not isinstance(excludes, dict):
+            return False
+
+        def value_for_lookup(attr_path: list[str]) -> Any:
+            if use_old_values:
+                return old_relevant_values.get("__".join(attr_path))
+            return current_value_for_path(attr_path)
+
+        for lookup, expected in filters.items():
+            parts = lookup.split("__")
+            if parts[-1] in (
+                "gt",
+                "gte",
+                "lt",
+                "lte",
+                "in",
+                "contains",
+                "startswith",
+                "endswith",
+                "regex",
+            ):
+                op = parts[-1]
+                attr_path = parts[:-1]
+            else:
+                op = "eq"
+                attr_path = parts
+            expected_key = json.dumps(
+                _normalize_dependency_identifier(expected), sort_keys=True
+            )
+            if not matches(op, value_for_lookup(attr_path), expected_key):
+                return False
+
+        for lookup, expected in excludes.items():
+            parts = lookup.split("__")
+            if parts[-1] in (
+                "gt",
+                "gte",
+                "lt",
+                "lte",
+                "in",
+                "contains",
+                "startswith",
+                "endswith",
+                "regex",
+            ):
+                op = parts[-1]
+                attr_path = parts[:-1]
+            else:
+                op = "eq"
+                attr_path = parts
+            expected_key = json.dumps(
+                _normalize_dependency_identifier(expected), sort_keys=True
+            )
+            if matches(op, value_for_lookup(attr_path), expected_key):
+                return False
+
+        return True
+
     for action in ACTIONS:
         action_section = cast(
             dict[general_manager_name, manager_dependency_section],
@@ -793,6 +922,57 @@ def generic_cache_invalidation(
             continue
         for lookup, lookup_map in model_section.items():
             if lookup.startswith("__"):
+                if lookup == ALL_RECORDS_LOOKUP:
+                    for cache_keys in cast(lookup_dependency_map, lookup_map).values():
+                        for ck in list(cache_keys):
+                            logger.info(
+                                "invalidating cache key",
+                                context={
+                                    "manager": manager_name,
+                                    "key": ck,
+                                    "lookup": lookup,
+                                    "action": action,
+                                    "value": ALL_RECORDS_VALUE,
+                                },
+                            )
+                            invalidate_cache_key(ck)
+                            remove_cache_key_from_index(ck)
+                elif lookup.startswith("__sort__"):
+                    sort_lookup = lookup.removeprefix("__sort__")
+                    attr_path = sort_lookup.split("__")
+                    old_sort_value = old_relevant_values.get(sort_lookup)
+                    new_sort_value = current_value_for_path(attr_path)
+                    if old_sort_value == new_sort_value:
+                        continue
+                    for val_key, cache_keys in list(
+                        cast(lookup_dependency_map, lookup_map).items()
+                    ):
+                        payload = _json_loads_val_key(val_key)
+                        if not isinstance(payload, dict):
+                            continue
+                        old_in_bucket = bucket_membership_matches(
+                            payload,
+                            use_old_values=True,
+                        )
+                        new_in_bucket = bucket_membership_matches(
+                            payload,
+                            use_old_values=False,
+                        )
+                        if not (old_in_bucket or new_in_bucket):
+                            continue
+                        for ck in list(cache_keys):
+                            logger.info(
+                                "invalidating cache key",
+                                context={
+                                    "manager": manager_name,
+                                    "key": ck,
+                                    "lookup": lookup,
+                                    "action": action,
+                                    "value": val_key,
+                                },
+                            )
+                            invalidate_cache_key(ck)
+                            remove_cache_key_from_index(ck)
                 continue
             lookup_map = cast(lookup_dependency_map, lookup_map)
             # 1) get operator and attribute path