pycrdt 0.12.12__cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl → 0.12.45__cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl

pycrdt/__init__.py

@@ -1,16 +1,27 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
 from ._array import Array as Array
 from ._array import ArrayEvent as ArrayEvent
 from ._array import TypedArray as TypedArray
 from ._awareness import Awareness as Awareness
+from ._awareness import is_awareness_disconnect_message as is_awareness_disconnect_message
 from ._doc import Doc as Doc
 from ._doc import TypedDoc as TypedDoc
 from ._map import Map as Map
 from ._map import MapEvent as MapEvent
 from ._map import TypedMap as TypedMap
+from ._provider import Channel as Channel
+from ._provider import Provider as Provider
+from ._pycrdt import DeleteSet as DeleteSet
 from ._pycrdt import StackItem as StackItem
 from ._pycrdt import SubdocsEvent as SubdocsEvent
 from ._pycrdt import Subscription as Subscription
 from ._pycrdt import TransactionEvent as TransactionEvent
+from ._snapshot import Snapshot as Snapshot
+from ._sticky_index import Assoc as Assoc
+from ._sticky_index import StickyIndex as StickyIndex
 from ._sync import Decoder as Decoder
 from ._sync import Encoder as Encoder
 from ._sync import YMessageType as YMessageType

pycrdt/_array.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, cast, overload
 
-from ._base import BaseDoc, BaseEvent, BaseType, Typed, base_types, event_types
+from ._base import BaseDoc, BaseEvent, BaseType, Sequence, Typed, base_types, event_types
 from ._pycrdt import Array as _Array
 from ._pycrdt import ArrayEvent as _ArrayEvent
 from ._pycrdt import Subscription
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
 T = TypeVar("T")
 
 
-class Array(BaseType, Generic[T]):
+class Array(Sequence, Generic[T]):
     """
     A collection used to store data in an indexed sequence structure, similar to a Python `list`.
     """
@@ -399,7 +399,7 @@ class ArrayIterator:
         self.idx = 0
 
     def __iter__(self) -> ArrayIterator:
-        return self
+        return self  # pragma: nocover
 
     def __next__(self) -> Any:
         if self.idx == self.length:

pycrdt/_awareness.py

@@ -10,7 +10,7 @@ from anyio import TASK_STATUS_IGNORED, create_task_group, sleep
 from anyio.abc import TaskGroup, TaskStatus
 
 from ._doc import Doc
-from ._sync import Decoder, Encoder
+from ._sync import Decoder, Encoder, read_message
 
 
 class Awareness:
@@ -278,3 +278,27 @@ class Awareness:
             id: The subscription ID to unregister.
         """
         del self._subscriptions[id]
+
+
+def is_awareness_disconnect_message(message: bytes) -> bool:
+    """
+    Check if the message is null, which means that it is a disconnection message
+    from the client.
+
+    Args:
+        message: The message received from the client.
+
+    Returns:
+        Whether the message is a disconnection message or not.
+    """
+    decoder = Decoder(read_message(message))
+    length = decoder.read_var_uint()
+    # A disconnection message should be a single message
+    if length == 1:
+        # Remove client_id and clock information from message (not used)
+        for _ in range(2):
+            decoder.read_var_uint()
+        state = decoder.read_var_string()
+        if state == "null":
+            return True
+    return False

pycrdt/_base.py

@@ -4,6 +4,7 @@ import threading
 from abc import ABC, abstractmethod
 from functools import lru_cache, partial
 from inspect import signature
+from types import UnionType
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -12,17 +13,21 @@ from typing import (
     Type,
     Union,
     cast,
+    get_args,
+    get_origin,
     get_type_hints,
     overload,
 )
 
 import anyio
 from anyio import BrokenResourceError, create_memory_object_stream
+from anyio.abc import TaskGroup
 from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream
 
 from ._pycrdt import Doc as _Doc
 from ._pycrdt import Subscription
 from ._pycrdt import Transaction as _Transaction
+from ._sticky_index import Assoc, StickyIndex
 from ._transaction import ReadTransaction, Transaction
 
 if TYPE_CHECKING:
@@ -42,17 +47,20 @@ class BaseDoc:
     _doc: _Doc
     _twin_doc: BaseDoc | None
     _txn: Transaction | None
+    _exceptions: list[Exception]
     _txn_lock: threading.Lock
     _txn_async_lock: anyio.Lock
     _allow_multithreading: bool
     _Model: Any
     _subscriptions: list[Subscription]
     _origins: dict[int, Any]
+    _task_group: TaskGroup | None
 
     def __init__(
         self,
         *,
         client_id: int | None = None,
+        skip_gc: bool | None = None,
         doc: _Doc | None = None,
         Model=None,
         allow_multithreading: bool = False,
@@ -60,15 +68,17 @@ class BaseDoc:
     ) -> None:
         super().__init__(**data)
         if doc is None:
-            doc = _Doc(client_id)
+            doc = _Doc(client_id, skip_gc)
         self._doc = doc
         self._txn = None
+        self._exceptions = []
         self._txn_lock = threading.Lock()
         self._txn_async_lock = anyio.Lock()
         self._Model = Model
         self._subscriptions = []
         self._origins = {}
         self._allow_multithreading = allow_multithreading
+        self._task_group = None
 
 
 class BaseType(ABC):
@@ -261,11 +271,30 @@ class BaseType(ABC):
             except BrokenResourceError:
                 to_remove.append(send_stream)
         for send_stream in to_remove:
+            send_stream.close()
             send_streams.remove(send_stream)
         if not send_streams:
             self.unobserve(self._event_subscription[deep])
 
 
+class Sequence(BaseType):
+    def sticky_index(self, index: int, assoc: Assoc = Assoc.AFTER) -> StickyIndex:
+        """
+        A permanent position that sticks to the same place even when
+        concurrent updates are made.
+
+        Args:
+            index: The index at which to stick.
+            assoc: The [Assoc][pycrdt.Assoc] specifying whether to stick to the location
+                before or after the index.
+
+        Returns:
+            A [StickyIndex][pycrdt.StickyIndex] that can be used to retrieve the index after
+            an update was applied.
+        """
+        return StickyIndex.new(self, index, assoc)
+
+
 def observe_callback(
     callback: Callable[[], None] | Callable[[Any], None] | Callable[[Any, ReadTransaction], None],
     doc: Doc,
@@ -275,7 +304,10 @@ def observe_callback(
     _event = event_types[type(event)](event, doc)
     with doc._read_transaction(event.transaction) as txn:
         params = (_event, txn)
-        callback(*params[:param_nb])  # type: ignore[arg-type]
+        try:
+            callback(*params[:param_nb])  # type: ignore[arg-type]
+        except Exception as exc:
+            doc._exceptions.append(exc)
 
 
 def observe_deep_callback(
@@ -288,7 +320,10 @@ def observe_deep_callback(
         events[idx] = event_types[type(event)](event, doc)
     with doc._read_transaction(event.transaction) as txn:
         params = (events, txn)
-        callback(*params[:param_nb])  # type: ignore[arg-type]
+        try:
+            callback(*params[:param_nb])  # type: ignore[arg-type]
+        except Exception as exc:
+            doc._exceptions.append(exc)
 
 
 class BaseEvent:
@@ -364,10 +399,12 @@ class Typed:
             if key not in annotations:
                 raise AttributeError(f'"{type(self).mro()[0]}" has no attribute "{key}"')
             expected_type = annotations[key]
-            if hasattr(expected_type, "__origin__"):
-                expected_type = expected_type.__origin__
-            if hasattr(expected_type, "__args__"):
-                expected_types = expected_type.__args__
+            origin = get_origin(expected_type)
+            if origin in (Union, UnionType):
+                expected_types = get_args(expected_type)
+            elif origin is not None:
+                expected_type = origin
+                expected_types = (expected_type,)
             else:
                 expected_types = (expected_type,)
             if type(value) not in expected_types:

pycrdt/_doc.py

@@ -1,7 +1,20 @@
 from __future__ import annotations
 
 from functools import partial
-from typing import Any, Callable, Generic, Iterable, Literal, Type, TypeVar, Union, cast, overload
+from inspect import iscoroutinefunction
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Generic,
+    Iterable,
+    Literal,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+)
 
 from anyio import BrokenResourceError, create_memory_object_stream
 from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream
@@ -10,9 +23,13 @@ from ._base import BaseDoc, BaseType, Typed, base_types, forbid_read_transaction
 from ._pycrdt import Doc as _Doc
 from ._pycrdt import SubdocsEvent, Subscription, TransactionEvent
 from ._pycrdt import Transaction as _Transaction
+from ._snapshot import Snapshot
 from ._transaction import NewTransaction, ReadTransaction, Transaction
 
 T = TypeVar("T", bound=BaseType)
+TransactionOrSubdocsEvent = TypeVar(
+    "TransactionOrSubdocsEvent", bound=TransactionEvent | SubdocsEvent
+)
 
 
 class Doc(BaseDoc, Generic[T]):
@@ -30,6 +47,7 @@ class Doc(BaseDoc, Generic[T]):
         init: dict[str, T] = {},
         *,
         client_id: int | None = None,
+        skip_gc: bool | None = None,
         doc: _Doc | None = None,
         Model=None,
         allow_multithreading: bool = False,
@@ -38,10 +56,16 @@ class Doc(BaseDoc, Generic[T]):
         Args:
             init: The initial root types of the document.
             client_id: An optional client ID for the document.
+            skip_gc: Whether to skip garbage collection on deleted collections
+                on transaction commit.
             allow_multithreading: Whether to allow the document to be used in different threads.
         """
         super().__init__(
-            client_id=client_id, doc=doc, Model=Model, allow_multithreading=allow_multithreading
+            client_id=client_id,
+            skip_gc=skip_gc,
+            doc=doc,
+            Model=Model,
+            allow_multithreading=allow_multithreading,
         )
         for k, v in init.items():
             self[k] = v
@@ -140,7 +164,9 @@ class Doc(BaseDoc, Generic[T]):
         Returns:
             The current document state.
         """
-        return self._doc.get_state()
+        with self.transaction() as txn:
+            assert txn._txn is not None
+            return self._doc.get_state(txn._txn)
 
     def get_update(self, state: bytes | None = None) -> bytes:
         """
@@ -152,7 +178,9 @@ class Doc(BaseDoc, Generic[T]):
         """
         if state is None:
             state = b"\x00"
-        return self._doc.get_update(state)
+        with self.transaction() as txn:
+            assert txn._txn is not None
+            return self._doc.get_update(txn._txn, state)
 
     def apply_update(self, update: bytes) -> None:
         """
@@ -173,6 +201,19 @@ class Doc(BaseDoc, Generic[T]):
             assert txn._txn is not None
             self._doc.apply_update(txn._txn, update)
 
+    @staticmethod
+    def from_snapshot(snapshot: "Snapshot", doc: "Doc") -> "Doc":
+        """
+        Create a new Doc from a Snapshot and an original Doc.
+        Args:
+            snapshot: The Snapshot to restore to.
+            doc: The original Doc to use for options/state.
+        Returns:
+            A new Doc instance restored to the snapshot state.
+        """
+        new_doc = _Doc.from_snapshot(snapshot._snapshot, doc._doc)
+        return Doc(doc=new_doc)
+
     def __setitem__(self, key: str, value: T) -> None:
         """
         Sets a document root type:
@@ -267,21 +308,44 @@ class Doc(BaseDoc, Generic[T]):
                 for key, val in self._doc.roots(txn._txn).items()
             }
 
-    def observe(self, callback: Callable[[TransactionEvent], None]) -> Subscription:
+    def observe(
+        self,
+        callback: Callable[[TransactionEvent], None]
+        | Callable[[TransactionEvent], Awaitable[None]],
+    ) -> Subscription:
         """
         Subscribes a callback to be called with the document change event.
 
         Args:
             callback: The callback to call with the [TransactionEvent][pycrdt.TransactionEvent].
+                If the callback is async, async transactions must be used.
 
         Returns:
             The subscription that can be used to [unobserve()][pycrdt.Doc.unobserve].
         """
-        subscription = self._doc.observe(callback)
+        if iscoroutinefunction(callback):
+            cb = self._async_callback_to_sync(callback)
+        else:
+            cb = partial(observe_callback, cast(Callable[[TransactionEvent], None], callback), self)
+        subscription = self._doc.observe(cb)
         self._subscriptions.append(subscription)
         return subscription
 
-    def observe_subdocs(self, callback: Callable[[SubdocsEvent], None]) -> Subscription:
+    def _async_callback_to_sync(
+        self,
+        async_callback: Callable[[TransactionOrSubdocsEvent], Awaitable[None]],
+    ) -> Callable[[TransactionOrSubdocsEvent], None]:
+        def callback(event: TransactionOrSubdocsEvent) -> None:
+            if self._task_group is None:
+                raise RuntimeError("Async callback in non-async transaction")
+            self._task_group.start_soon(async_callback, event)
+
+        return callback
+
+    def observe_subdocs(
+        self,
+        callback: Callable[[SubdocsEvent], None] | Callable[[SubdocsEvent], Awaitable[None]],
+    ) -> Subscription:
         """
         Subscribes a callback to be called with the document subdoc change event.
 
@@ -291,7 +355,11 @@ class Doc(BaseDoc, Generic[T]):
         Returns:
             The subscription that can be used to [unobserve()][pycrdt.Doc.unobserve].
         """
-        subscription = self._doc.observe_subdocs(callback)
+        if iscoroutinefunction(callback):
+            cb = self._async_callback_to_sync(callback)
+        else:
+            cb = partial(observe_callback, cast(Callable[[SubdocsEvent], None], callback), self)
+        subscription = self._doc.observe_subdocs(cb)
         self._subscriptions.append(subscription)
         return subscription
 
@@ -308,21 +376,24 @@ class Doc(BaseDoc, Generic[T]):
     @overload
     def events(
         self,
-        subdocs: Literal[False],
+        subdocs: Literal[False] = False,
         max_buffer_size: float = float("inf"),
+        async_transactions: bool = False,
     ) -> MemoryObjectReceiveStream[TransactionEvent]: ...
 
     @overload
     def events(
         self,
-        subdocs: Literal[True],
+        subdocs: Literal[True] = True,
         max_buffer_size: float = float("inf"),
+        async_transactions: bool = False,
     ) -> MemoryObjectReceiveStream[list[SubdocsEvent]]: ...
 
     def events(
         self,
         subdocs: bool = False,
         max_buffer_size: float = float("inf"),
+        async_transactions: bool = False,
     ):
         """
         Allows to asynchronously iterate over the document events, without using a callback.
@@ -343,13 +414,21 @@ class Doc(BaseDoc, Generic[T]):
             subdocs: Whether to iterate over the [SubdocsEvent][pycrdt.SubdocsEvent] events
                 (default is [TransactionEvent][pycrdt.TransactionEvent]).
             max_buffer_size: Maximum number of events that can be buffered.
+            async_transactions: Whether async transactions are used for this document,
+                in which case iterating over the events can put back-pressure on the
+                transactions (don't use an infinite `max_buffer_size` in this case).
 
         Returns:
             An async iterator over the document events.
         """
         observe = self.observe_subdocs if subdocs else self.observe
         if not self._send_streams[subdocs]:
-            self._event_subscription[subdocs] = observe(partial(self._send_event, subdocs))
+            if async_transactions:
+                self._event_subscription[subdocs] = observe(
+                    partial(self._async_send_event, subdocs)
+                )
+            else:
+                self._event_subscription[subdocs] = observe(partial(self._send_event, subdocs))
         send_stream, receive_stream = create_memory_object_stream[
             Union[TransactionEvent, SubdocsEvent]
         ](max_buffer_size=max_buffer_size)
@@ -365,6 +444,21 @@ class Doc(BaseDoc, Generic[T]):
             except BrokenResourceError:
                 to_remove.append(send_stream)
         for send_stream in to_remove:
+            send_stream.close()
+            send_streams.remove(send_stream)
+        if not send_streams:
+            self.unobserve(self._event_subscription[subdocs])
+
+    async def _async_send_event(self, subdocs: bool, event: TransactionEvent | SubdocsEvent):
+        to_remove: list[MemoryObjectSendStream[TransactionEvent | SubdocsEvent]] = []
+        send_streams = self._send_streams[subdocs]
+        for send_stream in send_streams:
+            try:
+                await send_stream.send(event)
+            except BrokenResourceError:
+                to_remove.append(send_stream)
+        for send_stream in to_remove:
+            send_stream.close()
             send_streams.remove(send_stream)
         if not send_streams:
             self.unobserve(self._event_subscription[subdocs])
@@ -409,4 +503,15 @@ class TypedDoc(Typed):
             doc[name] = root_type
 
 
+def observe_callback(
+    callback: Callable[[TransactionEvent], None] | Callable[[SubdocsEvent], None],
+    doc: Doc,
+    event: Any,
+) -> None:
+    try:
+        callback(event)
+    except Exception as exc:
+        doc._exceptions.append(exc)
+
+
 base_types[_Doc] = Doc

pycrdt/_map.py

@@ -203,7 +203,9 @@ class Map(BaseType, Generic[T]):
         Returns:
             True if the key was found.
         """
-        return item in self.keys()
+        # use integrated.has to avoid fetching all keys
+        with self.doc.transaction() as txn:
+            return self.integrated.has(txn._txn, item)
 
     @overload
     def get(self, key: str) -> T | None: ...
@@ -224,7 +226,7 @@ class Map(BaseType, Generic[T]):
         """
         key, *default_value = args
         with self.doc.transaction():
-            if key in self.keys():
+            if self.__contains__(key):
                 return self[key]
             if not default_value:
                 return None
@@ -248,7 +250,7 @@ class Map(BaseType, Generic[T]):
         """
         key, *default_value = args
         with self.doc.transaction():
-            if key not in self.keys():
+            if not self.__contains__(key):
                 if not default_value:
                     raise KeyError
                 return default_value[0]
@@ -261,7 +263,7 @@ class Map(BaseType, Generic[T]):
     def _check_key(self, key: str) -> None:
         if not isinstance(key, str):
             raise RuntimeError("Key must be of type string")
-        if key not in self.keys():
+        if not self.__contains__(key):
             raise KeyError(key)
 
     def keys(self) -> Iterable[str]:

pycrdt/_provider.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from contextlib import AsyncExitStack, asynccontextmanager
+from logging import Logger, getLogger
+from typing import AsyncIterator, Protocol
+
+from anyio import (
+    TASK_STATUS_IGNORED,
+    Event,
+    Lock,
+    create_task_group,
+)
+from anyio.abc import TaskGroup, TaskStatus
+
+from ._doc import Doc
+from ._sync import (
+    YMessageType,
+    YSyncMessageType,
+    create_sync_message,
+    create_update_message,
+    handle_sync_message,
+)
+
+
+class Channel(Protocol):
+    """A transport-agnostic stream used to synchronize a document through a provider.
+    An example of a channel is a WebSocket.
+
+    Messages can be received through the channel using an async iterator,
+    until the connection is closed:
+    ```py
+    async for message in channel:
+        ...
+    ```
+    Or directly by calling `recv()`:
+    ```py
+    message = await channel.recv()
+    ```
+    Sending messages is done with `send()`:
+    ```py
+    await channel.send(message)
+    ```
+    """
+
+    @property
+    def path(self) -> str:
+        """The channel path."""
+        ...  # pragma: nocover
+
+    def __aiter__(self) -> "Channel":
+        return self
+
+    async def __anext__(self) -> bytes:
+        return await self.recv()
+
+    async def send(self, message: bytes) -> None:
+        """Send a message.
+
+        Args:
+            message: The message to send.
+        """
+        ...  # pragma: nocover
+
+    async def recv(self) -> bytes:
+        """Receive a message.
+
+        Returns:
+            The received message.
+        """
+        ...  # pragma: nocover
+
+
+class Provider:
+    def __init__(self, doc: Doc, channel: Channel, log: Logger | None = None) -> None:
+        """A provider synchronizes a document through a channel.
+
+        The provider should preferably be used with an async context manager:
+        ```py
+        async with provider:
+            ...
+        ```
+        However, a lower-level API can also be used:
+        ```py
+        task = asyncio.create_task(provider.start())
+        await provider.started.wait()
+        ...
+        await provider.stop()
+        ```
+
+        Arguments:
+            doc: The `Doc` to connect through the `Channel`.
+            channel: The `Channel` through which to connect the `Doc`.
+            log: An optional logger.
+        """
+        self._doc = doc
+        self._channel = channel
+        self.log = log or getLogger(__name__)
+        self.started = Event()
+        self._start_lock = Lock()
+        self._task_group: TaskGroup | None = None
+
+    async def _run(self):
+        sync_message = create_sync_message(self._doc)
+        self.log.debug(
+            "Sending %s message to endpoint: %s",
+            YSyncMessageType.SYNC_STEP1.name,
+            self._channel.path,
+        )
+        await self._channel.send(sync_message)
+        assert self._task_group is not None
+        self._task_group.start_soon(self._send_updates)
+        async for message in self._channel:
+            if message[0] == YMessageType.SYNC:
+                self.log.debug(
+                    "Received %s message from endpoint: %s",
+                    YSyncMessageType(message[1]).name,
+                    self._channel.path,
+                )
+                reply = handle_sync_message(message[1:], self._doc)
+                if reply is not None:
+                    self.log.debug(
+                        "Sending %s message to endpoint: %s",
+                        YSyncMessageType.SYNC_STEP2.name,
+                        self._channel.path,
+                    )
+                    await self._channel.send(reply)
+
+    async def _send_updates(self):
+        async with self._doc.events() as events:
+            async for event in events:
+                message = create_update_message(event.update)
+                await self._channel.send(message)
+
+    async def __aenter__(self) -> Provider:
+        async with AsyncExitStack() as exit_stack:
+            self._task_group = await exit_stack.enter_async_context(
+                self._get_or_create_task_group()
+            )
+            await self._task_group.start(self.start)
+            self._exit_stack = exit_stack.pop_all()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, exc_tb):
+        await self.stop()
+        return await self._exit_stack.__aexit__(exc_type, exc_value, exc_tb)
+
+    @asynccontextmanager
+    async def _get_or_create_task_group(self) -> AsyncIterator[TaskGroup]:
+        if self._task_group is not None:
+            yield self._task_group
+            return
+
+        async with create_task_group() as tg:
+            yield tg
+
+    async def start(
+        self,
+        *,
+        task_status: TaskStatus[None] = TASK_STATUS_IGNORED,
+    ) -> None:
+        """Start the provider.
+
+        Args:
+            task_status: The status to set when the task has started.
+        """
+        async with self._start_lock:
+            async with self._get_or_create_task_group() as self._task_group:
+                task_status.started()
+                self.started.set()
+                self._task_group.start_soon(self._run)
+
+    async def stop(self) -> None:
+        """Stop the provider."""
+        assert self._task_group is not None
+        self._task_group.cancel_scope.cancel()
+        self._task_group = None