GeneralManager 0.63.2__tar.gz → 0.64.0__tar.gz

{generalmanager-0.63.2/src/GeneralManager.egg-info → generalmanager-0.64.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.63.2
+Version: 0.64.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.63.2 → generalmanager-0.64.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "GeneralManager"
-version = "0.63.2"
+version = "0.64.0"
 description = "Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching."
 readme = "README.md"
 authors = [{ name = "Tim Kleindick", email = "tkleindick@yahoo.de" }]

{generalmanager-0.63.2 → generalmanager-0.64.0/src/GeneralManager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GeneralManager
-Version: 0.63.2
+Version: 0.64.0
 Summary: Modular Django-based data management framework with ORM, GraphQL, fine-grained permissions, rule validation, calculations and caching.
 Author-email: Tim Kleindick <tkleindick@yahoo.de>
 License: MIT License

{generalmanager-0.63.2 → generalmanager-0.64.0}/src/GeneralManager.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ src/general_manager/api/graphql_warmup.py
 src/general_manager/api/graphql_warmup_registry.py
 src/general_manager/api/graphql_warmup_tasks.py
 src/general_manager/api/mutation.py
+src/general_manager/api/notification_batching.py
 src/general_manager/api/property.py
 src/general_manager/api/registry.py
 src/general_manager/api/remote_api.py

{generalmanager-0.63.2 → generalmanager-0.64.0}/src/general_manager/_types/api.py

@@ -56,6 +56,7 @@ __all__ = [
     "UploadTokenInvalidError",
     "UploadTransferConflictError",
     "UploadTransport",
+    "bulk_data_change_notifications",
     "graph_ql_mutation",
     "graph_ql_property",
     "register_upload_adapter",
@@ -115,5 +116,6 @@ from general_manager.uploads.errors import UploadTokenInvalidError
 from general_manager.uploads.errors import UploadTransferConflictError
 from general_manager.uploads.types import UploadTransport
 from general_manager.api.mutation import graph_ql_mutation
+from general_manager.api.notification_batching import bulk_data_change_notifications
 from general_manager.api.property import graph_ql_property
 from general_manager.uploads.public import register_upload_adapter

{generalmanager-0.63.2 → generalmanager-0.64.0}/src/general_manager/api/graphql.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import asyncio
-from contextlib import suppress
+from contextlib import asynccontextmanager
 from copy import deepcopy
 import re
 from types import UnionType
@@ -113,10 +113,13 @@ from general_manager.api.graphql_search import (
     create_filter_options as _create_filter_options_fn,
     normalize_filter_input as _normalize_filter_input_fn,
 )
+from general_manager.api.notification_batching import _queue_notification
 from general_manager.api.graphql_subscriptions import (
     get_channel_layer_safe as _get_channel_layer_fn,
     group_name as _group_name_fn,
     class_group_name as _class_group_name_fn,
+    refresh_group_name as _refresh_group_name_fn,
+    dispatch_subscription_event as _dispatch_subscription_event_fn,
     channel_listener as _channel_listener_fn,
     channel_message_listener as _channel_message_listener_fn,
     prime_graphql_properties as _prime_graphql_properties_fn,
@@ -155,6 +158,101 @@ GraphQLFilterNormalizer = Callable[
     dict[str, GraphQLFilterMapping],
 ]
 GraphQLMutationMap = dict[str, type[graphene.Mutation]]
+_SUBSCRIPTION_CLEANUP_FAILURE_MESSAGE = "subscription cleanup failed"
+_SUBSCRIPTION_ROLLBACK_FAILURE_MESSAGE = "subscription setup and rollback failed"
+_SUBSCRIPTION_STREAM_CLEANUP_FAILURE_MESSAGE = "subscription stream and cleanup failed"
+
+
+async def _cleanup_subscription_resources(
+    channel_layer: BaseChannelLayer,
+    channel_name: str,
+    joined_groups: Iterable[str],
+    listener_task: asyncio.Task[None] | None = None,
+) -> None:
+    """Stop a listener and attempt to discard every successfully joined group."""
+    cleanup_errors: list[BaseException] = []
+    if listener_task is not None:
+        listener_task.cancel()
+        try:
+            await listener_task
+        except asyncio.CancelledError:
+            pass
+        except BaseException as exc:  # noqa: BLE001
+            cleanup_errors.append(exc)
+
+    for group in joined_groups:
+        try:
+            await channel_layer.group_discard(group, channel_name)
+        except BaseException as exc:  # noqa: BLE001
+            cleanup_errors.append(exc)
+
+    if len(cleanup_errors) == 1:
+        raise cleanup_errors[0]
+    if cleanup_errors:
+        raise BaseExceptionGroup(
+            _SUBSCRIPTION_CLEANUP_FAILURE_MESSAGE,
+            cleanup_errors,
+        )
+
+
+async def _join_subscription_groups(
+    channel_layer: BaseChannelLayer,
+    channel_name: str,
+    group_names: Iterable[str],
+) -> list[str]:
+    """Join groups and roll back every successful join if a later join fails."""
+    joined_groups: list[str] = []
+    try:
+        for group in group_names:
+            await channel_layer.group_add(group, channel_name)
+            joined_groups.append(group)
+    except BaseException as setup_error:
+        try:
+            await _cleanup_subscription_resources(
+                channel_layer,
+                channel_name,
+                joined_groups,
+            )
+        except BaseException as cleanup_error:  # noqa: BLE001
+            raise BaseExceptionGroup(
+                _SUBSCRIPTION_ROLLBACK_FAILURE_MESSAGE,
+                [setup_error, cleanup_error],
+            ) from None
+        raise
+    return joined_groups
+
+
+@asynccontextmanager
+async def _subscription_cleanup_scope(
+    channel_layer: BaseChannelLayer,
+    channel_name: str,
+    joined_groups: Iterable[str],
+    listener_task: asyncio.Task[None],
+) -> AsyncIterator[None]:
+    """Clean subscription resources without masking an active stream failure."""
+    try:
+        yield
+    except BaseException as stream_error:
+        try:
+            await _cleanup_subscription_resources(
+                channel_layer,
+                channel_name,
+                joined_groups,
+                listener_task,
+            )
+        except BaseException as cleanup_error:  # noqa: BLE001
+            raise BaseExceptionGroup(
+                _SUBSCRIPTION_STREAM_CLEANUP_FAILURE_MESSAGE,
+                [stream_error, cleanup_error],
+            ) from None
+        raise
+    else:
+        await _cleanup_subscription_resources(
+            channel_layer,
+            channel_name,
+            joined_groups,
+            listener_task,
+        )
 
 
 class GraphQL:
@@ -281,6 +379,15 @@ class GraphQL:
         """
         return _class_group_name_fn(manager_class)
 
+    @staticmethod
+    def _refresh_group_name(manager_class: type[GeneralManager]) -> str:
+        """
+        Compatibility wrapper for ``graphql_subscriptions.refresh_group_name(manager_class)``.
+
+        Returns the deterministic manager-wide refresh group name.
+        """
+        return _refresh_group_name_fn(manager_class)
+
     @staticmethod
     async def _channel_listener(
         channel_layer: BaseChannelLayer,
@@ -1352,7 +1459,7 @@ class GraphQL:
             """
             Stream subscription events for a specific manager instance identified by the provided arguments.
 
-            Yields an initial `SubscriptionEvent` with `action` set to `"snapshot"` containing the current manager instance, then yields `SubscriptionEvent`s for each subsequent action. For update events, `item` will be the re-instantiated manager instance or `None` if instantiation fails. The subscriber is registered on the manager's channel groups (including dependent managers' groups) and the channel subscriptions and background listener are cleaned up when the iterator is closed or cancelled.
+            Yields an initial `SubscriptionEvent` with `action` set to `"snapshot"` containing the current manager instance, then yields `SubscriptionEvent`s for each subsequent action. For subsequent events, `item` will be the re-instantiated manager instance or `None` if instantiation fails. The subscriber is registered on the manager's channel groups (including dependent managers' groups) and the channel subscriptions and background listener are cleaned up when the iterator is closed or cancelled.
 
             Parameters:
                 identification: Identification fields required to locate the manager instance (maps to the manager's identification signature).
@@ -1379,14 +1486,13 @@ class GraphQL:
                 )
             except RuntimeError as exc:
                 raise GraphQLError(str(exc)) from exc
-            channel_name = cast(str, await channel_layer.new_channel())
-            queue: asyncio.Queue[str] = asyncio.Queue[str]()
 
             group_names = {
                 cls._group_name(
                     generalManagerClass,
                     instance.identification,
-                )
+                ),
+                cls._refresh_group_name(generalManagerClass),
             }
             dependencies = cls._resolve_subscription_dependencies(
                 generalManagerClass,
@@ -1397,13 +1503,7 @@ class GraphQL:
                 group_names.add(
                     cls._group_name(dependency_class, dependency_identification)
                 )
-
-            for group in group_names:
-                await channel_layer.group_add(group, channel_name)
-
-            listener_task = asyncio.create_task(
-                cls._channel_listener(channel_layer, channel_name, queue)
-            )
+                group_names.add(cls._refresh_group_name(dependency_class))
 
             async def event_stream() -> AsyncIterator[SubscriptionEvent]:
                 """
@@ -1415,7 +1515,22 @@ class GraphQL:
                 Notes:
                     When the iterator is closed or exits, the background listener task is cancelled and the subscription's channel group memberships are discarded.
                 """
-                try:
+                channel_name = cast(str, await channel_layer.new_channel())
+                queue: asyncio.Queue[str] = asyncio.Queue[str]()
+                joined_groups = await _join_subscription_groups(
+                    channel_layer,
+                    channel_name,
+                    group_names,
+                )
+                listener_task = asyncio.create_task(
+                    cls._channel_listener(channel_layer, channel_name, queue)
+                )
+                async with _subscription_cleanup_scope(
+                    channel_layer,
+                    channel_name,
+                    joined_groups,
+                    listener_task,
+                ):
                     clear_capability_context(info)
                     yield SubscriptionEvent(item=instance, action="snapshot")
                     while True:
@@ -1431,12 +1546,6 @@ class GraphQL:
                             item = None
                         clear_capability_context(info)
                         yield SubscriptionEvent(item=item, action=action)
-                finally:
-                    listener_task.cancel()
-                    with suppress(asyncio.CancelledError):
-                        await listener_task
-                    for group in group_names:
-                        await channel_layer.group_discard(group, channel_name)
 
             return event_stream()
 
@@ -1466,9 +1575,10 @@ class GraphQL:
             Stream future authorized change events for any instance of a manager class.
 
             Unlike single-entity subscriptions, class-wide subscriptions do not
-            yield an initial snapshot. Each channel event is hydrated and
-            checked against the request user's object-level read permission
-            before it is yielded.
+            yield an initial snapshot. Aggregate refresh events without an
+            identification carry no item; identified row events, including
+            row-level refresh actions, are hydrated and checked against the
+            request user's object-level read permission before they are yielded.
             """
             try:
                 channel_layer = cast(
@@ -1476,25 +1586,41 @@ class GraphQL:
                 )
             except RuntimeError as exc:
                 raise GraphQLError(str(exc)) from exc
-            channel_name = cast(str, await channel_layer.new_channel())
-            queue: asyncio.Queue[GraphQLFieldMap] = asyncio.Queue()
-            group_name = cls._class_group_name(generalManagerClass)
-            await channel_layer.group_add(group_name, channel_name)
-            listener_task = asyncio.create_task(
-                cls._channel_message_listener(channel_layer, channel_name, queue)
-            )
+            group_names = {
+                cls._class_group_name(generalManagerClass),
+                cls._refresh_group_name(generalManagerClass),
+            }
 
             async def event_stream() -> AsyncIterator[SubscriptionEvent]:
-                try:
+                channel_name = cast(str, await channel_layer.new_channel())
+                queue: asyncio.Queue[GraphQLFieldMap] = asyncio.Queue()
+                joined_groups = await _join_subscription_groups(
+                    channel_layer,
+                    channel_name,
+                    group_names,
+                )
+                listener_task = asyncio.create_task(
+                    cls._channel_message_listener(channel_layer, channel_name, queue)
+                )
+                async with _subscription_cleanup_scope(
+                    channel_layer,
+                    channel_name,
+                    joined_groups,
+                    listener_task,
+                ):
                     while True:
                         message = await queue.get()
                         if message.get("manager") != generalManagerClass.__name__:
                             continue
                         action = message.get("action")
+                        if not isinstance(action, str):
+                            continue
                         identification = message.get("identification")
-                        if not isinstance(action, str) or not isinstance(
-                            identification, dict
-                        ):
+                        if action == "refresh" and identification is None:
+                            clear_capability_context(info)
+                            yield SubscriptionEvent(item=None, action=action)
+                            continue
+                        if not isinstance(identification, dict):
                             continue
                         identification_copy = deepcopy(identification)
                         try:
@@ -1509,11 +1635,6 @@ class GraphQL:
                             continue
                         clear_capability_context(info)
                         yield SubscriptionEvent(item=item, action=action)
-                finally:
-                    listener_task.cancel()
-                    with suppress(asyncio.CancelledError):
-                        await listener_task
-                    await channel_layer.group_discard(group_name, channel_name)
 
             return event_stream()
 
@@ -1595,14 +1716,15 @@ class GraphQL:
         Send a "gm.subscription.event" message to the channel group corresponding to a changed GeneralManager instance.
 
         If the provided instance is a registered GeneralManager and a channel
-        layer is configured, publish a message containing the given action to the
-        instance channel group and the class-wide channel group. Identification
-        comes from the provided `identification` mapping when supplied,
-        otherwise from the changed instance, and is deep-copied before dispatch.
-        If the instance is `None`, the manager type is not registered, or no
-        channel layer is available, the function returns without dispatching.
-        Group dispatch failures are logged and do not stop attempts for the
-        remaining target groups.
+        layer is configured, queue one manager-wide refresh while notification
+        batching is active. Outside a batch, publish the row-level message to
+        the instance and class-wide groups through one async bridge.
+        Identification comes from the provided `identification` mapping when
+        supplied, otherwise from the changed instance, and is deep-copied
+        before dispatch. If the instance is `None`, the manager type is not
+        registered, or no channel layer is available, the function returns
+        without dispatching. Ordinary group dispatch failures are logged and
+        do not stop attempts for remaining groups.
 
         Parameters:
             sender (type[GeneralManager] | GeneralManager): The signal sender; either a GeneralManager subclass or an instance.
@@ -1646,35 +1768,57 @@ class GraphQL:
         )
         group_name = cls._group_name(manager_class, event_identification)
         class_group_name = cls._class_group_name(manager_class)
-        message = {
+        message: dict[str, object] = {
             "type": "gm.subscription.event",
             "action": action,
             "manager": manager_class.__name__,
             "identification": event_identification,
         }
-        group_send = async_to_sync(channel_layer.group_send)
-        for target_group_name in (group_name, class_group_name):
-            try:
-                group_send(target_group_name, message)
-            except Exception as exc:  # noqa: BLE001
-                logger.warning(
-                    "failed to dispatch subscription event",
-                    context={
-                        "manager": manager_class.__name__,
-                        "action": action,
-                        "group": target_group_name,
-                        "message": message,
-                    },
-                    exc_info=exc,
-                )
-        logger.debug(
-            "dispatched subscription event",
-            context={
-                "manager": manager_class.__name__,
-                "action": action,
-                "groups": [group_name, class_group_name],
-            },
-        )
+        refresh_group_name = cls._refresh_group_name(manager_class)
+        refresh_message: dict[str, object] = {
+            "type": "gm.subscription.event",
+            "action": "refresh",
+            "manager": manager_class.__name__,
+        }
+        if _queue_notification(
+            key=("graphql", refresh_group_name),
+            group_send=channel_layer.group_send,
+            group=refresh_group_name,
+            message=refresh_message,
+        ):
+            return
+
+        target_groups = (group_name, class_group_name)
+        try:
+            dispatched = async_to_sync(_dispatch_subscription_event_fn)(
+                channel_layer,
+                target_groups,
+                message,
+            )
+        except MemoryError:
+            raise
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "failed to dispatch subscription event",
+                context={
+                    "manager": manager_class.__name__,
+                    "action": action,
+                    "target_groups": list(target_groups),
+                    "message": message,
+                },
+                exc_info=exc,
+            )
+            return
+        if dispatched > 0:
+            logger.debug(
+                "dispatched subscription event",
+                context={
+                    "manager": manager_class.__name__,
+                    "action": action,
+                    "target_groups": list(target_groups),
+                    "successful_group_count": dispatched,
+                },
+            )
 
 
 post_data_change.connect(GraphQL._handle_data_change, weak=False)

{generalmanager-0.63.2 → generalmanager-0.64.0}/src/general_manager/api/graphql_subscriptions.py

@@ -110,6 +110,56 @@ def class_group_name(manager_class: type[GeneralManager]) -> str:
     return f"gm_subscriptions.{manager_class.__name__}.__class__"
 
 
+def refresh_group_name(manager_class: type[GeneralManager]) -> str:
+    """
+    Build the channel-group name for batched refreshes of a manager class.
+
+    Parameters:
+        manager_class: Manager class used to namespace the refresh group.
+
+    Returns:
+        A stable group identifier for aggregate manager refresh events.
+    """
+    return f"gm_subscriptions.{manager_class.__name__}.__refresh__"
+
+
+async def dispatch_subscription_event(
+    channel_layer: BaseChannelLayer,
+    group_names: Iterable[str],
+    message: SubscriptionMessage,
+) -> int:
+    """
+    Send one subscription event sequentially to each requested group.
+
+    Ordinary dispatch failures are logged for their target and do not prevent
+    later sends. ``MemoryError`` propagates immediately because continuing
+    dispatch during memory exhaustion is unsafe.
+
+    Returns:
+        The number of groups that accepted the event successfully.
+    """
+    dispatched = 0
+    for target_group_name in group_names:
+        try:
+            await channel_layer.group_send(target_group_name, message)
+        except MemoryError:
+            raise
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "failed to dispatch subscription event",
+                context={
+                    "group": target_group_name,
+                    "event_type": message.get("type"),
+                    "action": message.get("action"),
+                    "manager": message.get("manager"),
+                },
+                exc_info=exc,
+            )
+        else:
+            dispatched += 1
+    return dispatched
+
+
 async def channel_listener(
     channel_layer: BaseChannelLayer,
     channel_name: str,

generalmanager-0.64.0/src/general_manager/api/notification_batching.py

@@ -0,0 +1,121 @@
+"""Batch data-change notifications across synchronous bulk operations."""
+
+from __future__ import annotations
+
+import threading
+from _thread import LockType
+from collections.abc import Awaitable, Callable, Iterator, Mapping
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+
+from asgiref.sync import async_to_sync
+
+from general_manager.logging import get_logger
+
+type NotificationKey = tuple[str, ...]
+type NotificationMessage = Mapping[str, object]
+type GroupSend = Callable[[str, dict[str, object]], Awaitable[None]]
+
+logger = get_logger("api.notification_batching")
+_COMBINED_FAILURE_MESSAGE = "bulk data change and notification flush failed"
+
+
+@dataclass(slots=True)
+class _PendingNotification:
+    key: NotificationKey
+    group_send: GroupSend = field(repr=False)
+    group: str
+    message: dict[str, object]
+
+
+@dataclass(slots=True)
+class _BatchState:
+    targets: dict[NotificationKey, _PendingNotification] = field(default_factory=dict)
+    accepting: bool = True
+    lock: LockType = field(default_factory=threading.Lock, repr=False)
+
+
+_active_batch: ContextVar[_BatchState | None] = ContextVar(
+    "general_manager_notification_batch",
+    default=None,
+)
+
+
+def _queue_notification(
+    *,
+    key: NotificationKey,
+    group_send: GroupSend,
+    group: str,
+    message: NotificationMessage,
+) -> bool:
+    """Queue a target when batching is active."""
+    state = _active_batch.get()
+    if state is None:
+        return False
+    with state.lock:
+        if not state.accepting:
+            return False
+        if key in state.targets:
+            return True
+        state.targets[key] = _PendingNotification(
+            key,
+            group_send,
+            group,
+            dict(message),
+        )
+        return True
+
+
+def _close_batch(state: _BatchState) -> None:
+    with state.lock:
+        state.accepting = False
+
+
+async def _flush_notifications(state: _BatchState) -> None:
+    for key in sorted(state.targets):
+        target = state.targets[key]
+        try:
+            await target.group_send(target.group, target.message)
+        except MemoryError:
+            raise
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "failed to dispatch batched notification",
+                context={"key": key, "group": target.group},
+                exc_info=exc,
+            )
+
+
+def _flush_sync(state: _BatchState) -> None:
+    if state.targets:
+        async_to_sync(_flush_notifications)(state)
+
+
+@contextmanager
+def bulk_data_change_notifications() -> Iterator[None]:
+    """Collect and flush data-change notifications for a bulk operation."""
+    current = _active_batch.get()
+    if current is not None:
+        yield
+        return
+
+    state = _BatchState()
+    token = _active_batch.set(state)
+    try:
+        yield
+    except BaseException as body_error:
+        _close_batch(state)
+        _active_batch.reset(token)
+        try:
+            _flush_sync(state)
+        except BaseException as flush_error:  # noqa: BLE001
+            raise BaseExceptionGroup(
+                _COMBINED_FAILURE_MESSAGE,
+                [body_error, flush_error],
+            ) from None
+        raise
+    else:
+        _close_batch(state)
+        _active_batch.reset(token)
+        _flush_sync(state)