beaver-db 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

beaver/channels.py

@@ -1,54 +1,185 @@
 import json
 import sqlite3
+import threading
 import time
-from typing import Any, Iterator
+from queue import Empty, Queue
+from typing import Any, Iterator, Set
 
+# A special message object used to signal the listener to gracefully shut down.
+_SHUTDOWN_SENTINEL = object()
 
-class ChannelManager(Iterator):
+
+class Subscriber:
     """
-    A synchronous, blocking iterator that polls a channel for new messages.
+    A thread-safe message receiver for a specific channel subscription.
+
+    This object is designed to be used as a context manager (`with` statement).
+    It holds a dedicated in-memory queue that receives messages from the
+    channel's central polling thread, ensuring that a slow listener does not
+    impact others.
     """
 
-    def __init__(
-        self, conn: sqlite3.Connection, channel_name: str, poll_interval: float = 0.1
-    ):
+    def __init__(self, channel: "ChannelManager"):
+        self._channel = channel
+        self._queue: Queue = Queue()
+
+    def __enter__(self) -> "Subscriber":
+        """Registers the listener's queue with the channel to start receiving messages."""
+        self._channel._register(self._queue)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Unregisters the listener's queue from the channel to stop receiving messages."""
+        self._channel._unregister(self._queue)
+
+    def listen(self, timeout: float | None = None) -> Iterator[Any]:
         """
-        Initializes the synchronous subscriber.
+        Returns a blocking iterator that yields messages as they arrive.
+
+        This method pulls messages from the listener's dedicated, thread-safe
+        in-memory queue. It performs no database operations itself.
 
         Args:
-            conn: The SQLite database connection.
-            channel_name: The name of the channel to subscribe to.
-            poll_interval: The time in seconds to wait between polling for new messages.
+            timeout: If provided, the iterator will raise `queue.Empty` if no message is
+                     received within this many seconds.
         """
+        while True:
+            try:
+                msg = self._queue.get(timeout=timeout)
+
+                if msg is _SHUTDOWN_SENTINEL:
+                    break
+
+                yield msg
+            except Empty:
+                raise TimeoutError(f"Timeout {timeout}s expired.")
+
+
+class ChannelManager:
+    """
+    The central hub for a named pub/sub channel.
+
+    This object manages all active listeners for the channel and runs a single,
+    efficient background thread to poll the database for new messages. It then
+    "fans out" these messages to all subscribed listeners.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        conn: sqlite3.Connection,
+        db_path: str,
+        poll_interval: float = 0.1,
+    ):
+        self._name = name
         self._conn = conn
-        self._channel = channel_name
+        self._db_path = db_path
         self._poll_interval = poll_interval
-        self._last_seen_timestamp = time.time()
+        self._listeners: Set[Queue] = set()
+        self._lock = threading.Lock()
+        self._polling_thread: threading.Thread | None = None
+        self._stop_event = threading.Event()
 
-    def __iter__(self) -> "ChannelManager":
-        """Returns the iterator object itself."""
-        return self
+    def _register(self, queue: Queue):
+        """Adds a listener's queue and starts the poller if it's the first one."""
+
+        with self._lock:
+            self._listeners.add(queue)
+            # If the polling thread isn't running, start it.
+            if self._polling_thread is None or not self._polling_thread.is_alive():
+                self._start_polling()
 
-    def __next__(self) -> Any:
+    def _unregister(self, queue: Queue):
+        """Removes a listener's queue and stops the poller if it's the last one."""
+
+        with self._lock:
+            self._listeners.discard(queue)
+            # If there are no more listeners, stop the polling thread to save resources.
+            if not self._listeners:
+                self._stop_polling()
+
+    def _start_polling(self):
+        """Starts the background polling thread."""
+        self._stop_event.clear()
+        self._polling_thread = threading.Thread(target=self._polling_loop, daemon=True)
+        self._polling_thread.start()
+
+    def _stop_polling(self):
+        """Signals the background polling thread to stop."""
+        if self._polling_thread and self._polling_thread.is_alive():
+            self._stop_event.set()
+            self._polling_thread.join()
+            self._polling_thread = None
+
+    def close(self):
+        """Reliable close this channel and removes listeners."""
+        self._stop_polling()
+
+        with self._lock:
+            for listener in self._listeners:
+                listener.put(_SHUTDOWN_SENTINEL)
+
+        self._listeners.clear()
+
+    def _polling_loop(self):
         """
-        Blocks until a new message is available on the channel and returns it.
-        This polling mechanism is simple but can introduce a slight latency
-        equivalent to the poll_interval.
+        The main loop for the background thread.
+
+        This function polls the database for new messages and fans them out
+        to all registered listener queues.
         """
-        while True:
-            # Fetch the next available message from the database
-            cursor = self._conn.cursor()
+        # A separate SQLite connection is required for each thread.
+        thread_conn = sqlite3.connect(self._db_path, check_same_thread=False)
+        thread_conn.row_factory = sqlite3.Row
+
+        # The poller starts listening for messages from this moment forward.
+        last_seen_timestamp = time.time()
+
+
+        while not self._stop_event.is_set():
+            cursor = thread_conn.cursor()
             cursor.execute(
-                "SELECT timestamp, message_payload FROM beaver_pubsub_log WHERE channel_name = ? AND timestamp > ? ORDER BY timestamp ASC LIMIT 1",
-                (self._channel, self._last_seen_timestamp),
+                "SELECT timestamp, message_payload FROM beaver_pubsub_log WHERE channel_name = ? AND timestamp > ? ORDER BY timestamp ASC",
+                (self._name, last_seen_timestamp),
             )
-            result = cursor.fetchone()
+            messages = cursor.fetchall()
             cursor.close()
 
-            if result:
-                # If a message is found, update the timestamp and return the payload
-                self._last_seen_timestamp = result["timestamp"]
-                return json.loads(result["message_payload"])
-            else:
-                # If no new messages, wait for the poll interval before trying again
-                time.sleep(self._poll_interval)
+            if messages:
+                # Update the timestamp to the last message we've seen.
+                last_seen_timestamp = messages[-1]["timestamp"]
+
+                # The "fan-out": Push messages to all active listener queues.
+                # This block is locked to prevent modification of the listeners set
+                # while we are iterating over it.
+                with self._lock:
+                    for queue in self._listeners:
+                        for row in messages:
+                            queue.put(json.loads(row["message_payload"]))
+
+            # Wait for the poll interval before checking for new messages again.
+            time.sleep(self._poll_interval)
+
+        thread_conn.close()
+
+    def subscribe(self) -> Subscriber:
+        """Creates a new subscription, returning a Listener context manager."""
+        return Subscriber(self)
+
+    def publish(self, payload: Any):
+        """
+        Publishes a JSON-serializable message to the channel.
+
+        This is a synchronous operation that performs a fast, atomic INSERT
+        into the database's pub/sub log.
+        """
+        try:
+            json_payload = json.dumps(payload)
+        except TypeError as e:
+            raise TypeError("Message payload must be JSON-serializable.") from e
+
+        with self._conn:
+            self._conn.execute(
+                "INSERT INTO beaver_pubsub_log (timestamp, channel_name, message_payload) VALUES (?, ?, ?)",
+                (time.time(), self._name, json_payload),
+            )

beaver/core.py

@@ -1,7 +1,5 @@
-import json
 import sqlite3
-import time
-from typing import Any
+import threading
 
 from .dicts import DictManager
 from .lists import ListManager
@@ -29,6 +27,8 @@ class BeaverDB:
         self._conn.execute("PRAGMA journal_mode=WAL;")
         self._conn.row_factory = sqlite3.Row
         self._create_all_tables()
+        self._channels: dict[str, ChannelManager] = {}
+        self._channels_lock = threading.Lock()
 
     def _create_all_tables(self):
         """Initializes all required tables in the database file."""
@@ -170,6 +170,10 @@ class BeaverDB:
     def close(self):
         """Closes the database connection."""
         if self._conn:
+            # Cleanly shut down any active polling threads before closing
+            with self._channels_lock:
+                for channel in self._channels.values():
+                    channel.close()
             self._conn.close()
 
     # --- Factory and Passthrough Methods ---
@@ -178,41 +182,39 @@ class BeaverDB:
         """Returns a wrapper object for interacting with a named dictionary."""
         if not isinstance(name, str) or not name:
             raise TypeError("Dictionary name must be a non-empty string.")
+
         return DictManager(name, self._conn)
 
     def list(self, name: str) -> ListManager:
         """Returns a wrapper object for interacting with a named list."""
         if not isinstance(name, str) or not name:
             raise TypeError("List name must be a non-empty string.")
+
         return ListManager(name, self._conn)
 
     def queue(self, name: str) -> QueueManager:
         """Returns a wrapper object for interacting with a persistent priority queue."""
         if not isinstance(name, str) or not name:
             raise TypeError("Queue name must be a non-empty string.")
+
         return QueueManager(name, self._conn)
 
     def collection(self, name: str) -> CollectionManager:
         """Returns a wrapper for interacting with a document collection."""
         if not isinstance(name, str) or not name:
             raise TypeError("Collection name must be a non-empty string.")
+
         return CollectionManager(name, self._conn)
 
-    def publish(self, channel_name: str, payload: Any):
-        """Publishes a JSON-serializable message to a channel. This is synchronous."""
-        if not isinstance(channel_name, str) or not channel_name:
+    def channel(self, name: str) -> ChannelManager:
+        """
+        Returns a singleton Channel instance for high-efficiency pub/sub.
+        """
+        if not isinstance(name, str) or not name:
             raise ValueError("Channel name must be a non-empty string.")
-        try:
-            json_payload = json.dumps(payload)
-        except TypeError as e:
-            raise TypeError("Message payload must be JSON-serializable.") from e
-
-        with self._conn:
-            self._conn.execute(
-                "INSERT INTO beaver_pubsub_log (timestamp, channel_name, message_payload) VALUES (?, ?, ?)",
-                (time.time(), channel_name, json_payload),
-            )
 
-    def subscribe(self, channel_name: str) -> ChannelManager:
-        """Subscribes to a channel, returning a synchronous iterator."""
-        return ChannelManager(self._conn, channel_name)
+        # Use a thread-safe lock to ensure only one Channel object is created per name.
+        with self._channels_lock:
+            if name not in self._channels:
+                self._channels[name] = ChannelManager(name, self._conn, self._db_path)
+            return self._channels[name]

{beaver_db-0.8.0.dist-info → beaver_db-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: beaver-db
-Version: 0.8.0
+Version: 0.9.0
 Summary: Fast, embedded, and multi-modal DB based on SQLite for AI-powered applications.
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
@@ -9,6 +9,10 @@ Requires-Dist: numpy>=2.3.3
 Requires-Dist: scipy>=1.16.2
 Dynamic: license-file
 
+I've updated the README to highlight the new high-efficiency, thread-safe, and process-safe pub/sub system. I've also added an example of how you can use it to build real-time, event-driven applications.
+
+Here are the changes:
+
 # beaver 🦫
 
 A fast, single-file, multi-modal database for Python, built with the standard `sqlite3` library.
@@ -27,7 +31,7 @@ A fast, single-file, multi-modal database for Python, built with the standard `s
 
 ## Core Features
 
-  - **Synchronous Pub/Sub**: A simple, thread-safe, Redis-like publish-subscribe system for real-time messaging.
+  - **High-Efficiency Pub/Sub**: A powerful, thread and process-safe publish-subscribe system for real-time messaging with a fan-out architecture.
   - **Namespaced Key-Value Dictionaries**: A Pythonic, dictionary-like interface for storing any JSON-serializable object within separate namespaces with optional TTL for cache implementations.
   - **Pythonic List Management**: A fluent, Redis-like interface for managing persistent, ordered lists.
   - **Persistent Priority Queue**: A high-performance, persistent queue that always returns the item with the highest priority, perfect for task management.
@@ -163,19 +167,36 @@ if response is None:
     api_cache.set("weather_new_york", response, ttl_seconds=3600)
 ```
 
+### 6. Real-time Event-Driven Systems
+
+Use the **high-efficiency pub/sub system** to build applications where different components react to events in real-time. This is perfect for decoupled systems, real-time UIs, or monitoring services.
+
+```python
+# In one process or thread (e.g., a monitoring service)
+system_events = db.channel("system_events")
+system_events.publish({"event": "user_login", "user_id": "alice"})
+
+# In another process or thread (e.g., a UI updater or logger)
+with db.channel("system_events").subscribe() as listener:
+    for message in listener.listen():
+        print(f"Event received: {message}")
+        # >> Event received: {'event': 'user_login', 'user_id': 'alice'}
+```
+
 ## More Examples
 
 For more in-depth examples, check out the scripts in the `examples/` directory:
 
-  - [`examples/kvstore.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/kvstore.py%5D\(https://www.google.com/search%3Fq%3Dexamples/kvstore.py\)): A comprehensive demo of the namespaced dictionary feature.
-  - [`examples/list.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/list.py%5D\(https://www.google.com/search%3Fq%3Dexamples/list.py\)): Shows the full capabilities of the persistent list, including slicing and in-place updates.
-  - [`examples/queue.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/queue.py%5D\(https://www.google.com/search%3Fq%3Dexamples/queue.py\)): A practical example of using the persistent priority queue for task management.
-  - [`examples/vector.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/vector.py%5D\(https://www.google.com/search%3Fq%3Dexamples/vector.py\)): Demonstrates how to index and search vector embeddings, including upserts.
-  - [`examples/fts.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/fts.py%5D\(https://www.google.com/search%3Fq%3Dexamples/fts.py\)): A detailed look at full-text search, including targeted searches on specific metadata fields.
-  - [`examples/graph.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/graph.py%5D\(https://www.google.com/search%3Fq%3Dexamples/graph.py\)): Shows how to create relationships between documents and perform multi-hop graph traversals.
-  - [`examples/pubsub.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/pubsub.py%5D\(https://www.google.com/search%3Fq%3Dexamples/pubsub.py\)): A demonstration of the synchronous, thread-safe publish/subscribe system.
-  - [`examples/cache.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/cache.py%5D\(https://www.google.com/search%3Fq%3Dexamples/cache.py\)): A practical example of using a dictionary with TTL as a cache for API calls.
-  - [`examples/rerank.py`](https://www.google.com/search?q=%5Bhttps://www.google.com/search%3Fq%3Dexamples/rerank.py%5D\(https://www.google.com/search%3Fq%3Dexamples/rerank.py\)): Shows how to combine results from vector and text search for more refined results.
+  - [`examples/kvstore.py`](examples/kvstore.py): A comprehensive demo of the namespaced dictionary feature.
+  - [`examples/list.py`](examples/list.py): Shows the full capabilities of the persistent list, including slicing and in-place updates.
+  - [`examples/queue.py`](examples/queue.py): A practical example of using the persistent priority queue for task management.
+  - [`examples/vector.py`](examples/vector.py): Demonstrates how to index and search vector embeddings, including upserts.
+  - [`examples/fts.py`](examples/fts.py): A detailed look at full-text search, including targeted searches on specific metadata fields.
+  - [`examples/graph.py`](examples/graph.py): Shows how to create relationships between documents and perform multi-hop graph traversals.
+  - [`examples/pubsub.py`](examples/pubsub.py): A demonstration of the synchronous, thread-safe publish/subscribe system in a single process.
+  - [`examples/publisher.py`](examples/publisher.py) and [`examples/subscriber.py`](examples/subscriber.py): A pair of examples demonstrating inter-process message passing with the publish/subscribe system.
+  - [`examples/cache.py`](examples/cache.py): A practical example of using a dictionary with TTL as a cache for API calls.
+  - [`examples/rerank.py`](examples/rerank.py): Shows how to combine results from vector and text search for more refined results.
 
 ## Roadmap
 
@@ -183,10 +204,9 @@ These are some of the features and improvements planned for future releases:
 
   - **Fuzzy search**: Implement fuzzy matching capabilities for text search.
   - **Faster ANN**: Explore integrating more advanced ANN libraries like `faiss` for improved vector search performance.
-  - **Improved Pub/Sub**: Fan-out implementation with a more Pythonic API.
   - **Async API**: Comprehensive async support with on-demand wrappers for all collections.
 
-Check out the [roadmap](https://www.google.com/search?q=roadmap.md) for a detailed list of upcoming features and design ideas.
+Check out the [roadmap](roadmap.md) for a detailed list of upcoming features and design ideas.
 
 ## License
 

beaver_db-0.9.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+beaver/__init__.py,sha256=-z5Gj6YKMOswpJOOn5Gej8z5i6k3c0Xs00DIYLA-bMI,75
+beaver/channels.py,sha256=6Fr6uyp_1TtVzNzQobgCa_-c8KS285Z2fyC7OzUOd34,6611
+beaver/collections.py,sha256=R4bVmP37s_ZnCkb3Jdck2H8dRvD3-ihFV4mEsA14YeE,15716
+beaver/core.py,sha256=l5hI55vc2VlF1b_a6CP7ZP5r7H-MQdTNV4zr2lSumcs,7864
+beaver/dicts.py,sha256=y4z632XKWU29ekP_vdFSOP-MAG9Z8b79kBEHA88gO7E,4463
+beaver/lists.py,sha256=jFlDWwyaYycG0ZFVm58rMChefUaVZhaP1UeQ-hVo3Sg,9082
+beaver/queues.py,sha256=WKpBzlXr9Hp_rOKEs_Y1Tjyj_hWx6ql1uBRKBV7rw8w,2780
+beaver_db-0.9.0.dist-info/licenses/LICENSE,sha256=1xrIY5JnMk_QDQzsqmVzPIIyCgZAkWCC8kF2Ddo1UT0,1071
+beaver_db-0.9.0.dist-info/METADATA,sha256=Fd_Qvcs5a_wbCVMCvfmxOXAKBX4FbtQKJwrWwFAxjrE,9732
+beaver_db-0.9.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+beaver_db-0.9.0.dist-info/top_level.txt,sha256=FxA4XnX5Qm5VudEXCduFriqi4dQmDWpQ64d7g69VQKI,7
+beaver_db-0.9.0.dist-info/RECORD,,

beaver_db-0.8.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-beaver/__init__.py,sha256=-z5Gj6YKMOswpJOOn5Gej8z5i6k3c0Xs00DIYLA-bMI,75
-beaver/channels.py,sha256=2lem2_yEFMc7cjdSx4FbFZQOaeT-HtbSvon3WYYRYzY,1952
-beaver/collections.py,sha256=R4bVmP37s_ZnCkb3Jdck2H8dRvD3-ihFV4mEsA14YeE,15716
-beaver/core.py,sha256=GpQ5HVJd2yBGq_RGC6TKlhRpGvNdFg88c59oAkltgqY,7904
-beaver/dicts.py,sha256=y4z632XKWU29ekP_vdFSOP-MAG9Z8b79kBEHA88gO7E,4463
-beaver/lists.py,sha256=jFlDWwyaYycG0ZFVm58rMChefUaVZhaP1UeQ-hVo3Sg,9082
-beaver/queues.py,sha256=WKpBzlXr9Hp_rOKEs_Y1Tjyj_hWx6ql1uBRKBV7rw8w,2780
-beaver_db-0.8.0.dist-info/licenses/LICENSE,sha256=1xrIY5JnMk_QDQzsqmVzPIIyCgZAkWCC8kF2Ddo1UT0,1071
-beaver_db-0.8.0.dist-info/METADATA,sha256=hB3ZWilwbPhfq64laoxiwxx99KfT-WIFOk1Nj2jK28c,9833
-beaver_db-0.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-beaver_db-0.8.0.dist-info/top_level.txt,sha256=FxA4XnX5Qm5VudEXCduFriqi4dQmDWpQ64d7g69VQKI,7
-beaver_db-0.8.0.dist-info/RECORD,,