beaver-db 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

beaver/collections.py

@@ -1,11 +1,13 @@
 import json
 import sqlite3
+import threading
 import uuid
 from enum import Enum
-from typing import Any, List, Literal, Set
+from typing import Any, List, Literal, Tuple
 
 import numpy as np
-from scipy.spatial import KDTree
+
+from .vectors import VectorIndex
 
 
 # --- Fuzzy Search Helper Functions ---
@@ -103,16 +105,18 @@ class Document:
 
 class CollectionManager:
     """
-    A wrapper for multi-modal collection operations with an in-memory ANN index,
-    FTS, and graph capabilities.
+    A wrapper for multi-modal collection operations, including document storage,
+    FTS, fuzzy search, graph traversal, and persistent vector search.
     """
 
     def __init__(self, name: str, conn: sqlite3.Connection):
         self._name = name
         self._conn = conn
-        self._kdtree: KDTree | None = None
-        self._doc_ids: List[str] = []
-        self._local_index_version = -1  # Version of the in-memory index
+        # All vector-related operations are now delegated to the VectorIndex class.
+        self._vector_index = VectorIndex(name, conn)
+        # A lock to ensure only one compaction thread runs at a time for this collection.
+        self._compaction_lock = threading.Lock()
+        self._compaction_thread: threading.Thread | None = None
 
     def _flatten_metadata(self, metadata: dict, prefix: str = "") -> dict[str, Any]:
         """Flattens a nested dictionary for indexing."""
@@ -125,21 +129,58 @@ class CollectionManager:
                 flat_dict[new_key] = value
         return flat_dict
 
-    def _get_db_version(self) -> int:
-        """Gets the current version of the collection from the database."""
+    def _needs_compaction(self, threshold: int = 1000) -> bool:
+        """Checks if the total number of pending vector operations exceeds the threshold."""
         cursor = self._conn.cursor()
         cursor.execute(
-            "SELECT version FROM beaver_collection_versions WHERE collection_name = ?",
-            (self._name,),
+            "SELECT COUNT(*) FROM _beaver_ann_pending_log WHERE collection_name = ?",
+            (self._name,)
+        )
+        pending_count = cursor.fetchone()[0]
+        cursor.execute(
+            "SELECT COUNT(*) FROM _beaver_ann_deletions_log WHERE collection_name = ?",
+            (self._name,)
         )
-        result = cursor.fetchone()
-        return result[0] if result else 0
+        deletion_count = cursor.fetchone()[0]
+        return (pending_count + deletion_count) >= threshold
+
+    def _run_compaction_and_release_lock(self):
+        """
+        A target function for the background thread that runs the compaction
+        and ensures the lock is always released, even if errors occur.
+        """
+        try:
+            self._vector_index.compact()
+        finally:
+            self._compaction_lock.release()
+
+    def compact(self, block: bool = False):
+        """
+        Triggers a non-blocking background compaction of the vector index.
+
+        If a compaction is already running for this collection, this method returns
+        immediately without starting a new one.
 
-    def _is_index_stale(self) -> bool:
-        """Checks if the in-memory index is out of sync with the DB."""
-        if self._local_index_version == -1:
-            return True
-        return self._local_index_version < self._get_db_version()
+        Args:
+            block: If True, this method will wait for the compaction to complete
+                   before returning. Defaults to False (non-blocking).
+        """
+        # Use a non-blocking lock acquire to check if a compaction is already running.
+        if self._compaction_lock.acquire(blocking=False):
+            try:
+                # If we get the lock, start a new background thread.
+                self._compaction_thread = threading.Thread(
+                    target=self._run_compaction_and_release_lock,
+                    daemon=True  # Daemon threads don't block program exit.
+                )
+                self._compaction_thread.start()
+                if block:
+                    self._compaction_thread.join()
+            except Exception:
+                # If something goes wrong during thread creation, release the lock.
+                self._compaction_lock.release()
+                raise
+        # If acquire fails, it means another thread holds the lock, so we do nothing.
 
     def index(
         self,
@@ -152,9 +193,14 @@ class CollectionManager:
         Indexes a Document, including vector, FTS, and fuzzy search data.
         The entire operation is performed in a single atomic transaction.
         """
+        if not isinstance(document, Document):
+            raise TypeError("Item to index must be a Document object.")
+
         with self._conn:
-            # Step 1: Core Document and Vector Storage (Unaffected by FTS/Fuzzy)
-            self._conn.execute(
+            cursor = self._conn.cursor()
+
+            # Step 1: Core Document and Vector Storage
+            cursor.execute(
                 "INSERT OR REPLACE INTO beaver_collections (collection, item_id, item_vector, metadata) VALUES (?, ?, ?, ?)",
                 (
                     self._name,
@@ -164,12 +210,14 @@ class CollectionManager:
                 ),
             )
 
-            # Step 2: FTS and Fuzzy Indexing
-            # First, clean up old index data for this document
-            self._conn.execute("DELETE FROM beaver_fts_index WHERE collection = ? AND item_id = ?", (self._name, document.id))
-            self._conn.execute("DELETE FROM beaver_trigrams WHERE collection = ? AND item_id = ?", (self._name, document.id))
+            # Step 2: Delegate to the VectorIndex if an embedding exists.
+            if document.embedding is not None:
+                self._vector_index.index(document.id, document.embedding, cursor)
+
+            # Step 3: FTS and Fuzzy Indexing
+            cursor.execute("DELETE FROM beaver_fts_index WHERE collection = ? AND item_id = ?", (self._name, document.id))
+            cursor.execute("DELETE FROM beaver_trigrams WHERE collection = ? AND item_id = ?", (self._name, document.id))
 
-            # Determine which string fields to index
             flat_metadata = self._flatten_metadata(document.to_dict())
             fields_to_index: dict[str, str] = {}
             if isinstance(fts, list):
@@ -178,63 +226,46 @@ class CollectionManager:
                 fields_to_index = {k: v for k, v in flat_metadata.items() if isinstance(v, str)}
 
             if fields_to_index:
-                # FTS indexing
                 fts_data = [(self._name, document.id, path, content) for path, content in fields_to_index.items()]
-                self._conn.executemany(
-                    "INSERT INTO beaver_fts_index (collection, item_id, field_path, field_content) VALUES (?, ?, ?, ?)",
-                    fts_data,
-                )
-
-                # Fuzzy indexing (if enabled)
+                cursor.executemany("INSERT INTO beaver_fts_index (collection, item_id, field_path, field_content) VALUES (?, ?, ?, ?)", fts_data)
                 if fuzzy:
                     trigram_data = []
                     for path, content in fields_to_index.items():
                         for trigram in _get_trigrams(content.lower()):
                             trigram_data.append((self._name, document.id, path, trigram))
                     if trigram_data:
-                        self._conn.executemany(
-                            "INSERT INTO beaver_trigrams (collection, item_id, field_path, trigram) VALUES (?, ?, ?, ?)",
-                            trigram_data,
-                        )
+                        cursor.executemany("INSERT INTO beaver_trigrams (collection, item_id, field_path, trigram) VALUES (?, ?, ?, ?)", trigram_data)
 
-            # Step 3: Update Collection Version
-            self._conn.execute(
-                """
-                INSERT INTO beaver_collection_versions (collection_name, version) VALUES (?, 1)
-                ON CONFLICT(collection_name) DO UPDATE SET version = version + 1
-                """,
+            # Step 4: Update Collection Version to signal a change.
+            cursor.execute(
+                "INSERT INTO beaver_collection_versions (collection_name, version) VALUES (?, 1) ON CONFLICT(collection_name) DO UPDATE SET version = version + 1",
                 (self._name,),
             )
 
+        # After the transaction commits, check if auto-compaction is needed.
+        if self._needs_compaction():
+            self.compact()
+
     def drop(self, document: Document):
         """Removes a document and all its associated data from the collection."""
         if not isinstance(document, Document):
             raise TypeError("Item to drop must be a Document object.")
         with self._conn:
-            self._conn.execute(
-                "DELETE FROM beaver_collections WHERE collection = ? AND item_id = ?",
-                (self._name, document.id),
-            )
-            self._conn.execute(
-                "DELETE FROM beaver_fts_index WHERE collection = ? AND item_id = ?",
-                (self._name, document.id),
-            )
-            self._conn.execute(
-                "DELETE FROM beaver_trigrams WHERE collection = ? AND item_id = ?",
-                (self._name, document.id),
-            )
-            self._conn.execute(
-                "DELETE FROM beaver_edges WHERE collection = ? AND (source_item_id = ? OR target_item_id = ?)",
-                (self._name, document.id, document.id),
-            )
-            self._conn.execute(
-                """
-                INSERT INTO beaver_collection_versions (collection_name, version) VALUES (?, 1)
-                ON CONFLICT(collection_name) DO UPDATE SET version = version + 1
-                """,
+            cursor = self._conn.cursor()
+            cursor.execute("DELETE FROM beaver_collections WHERE collection = ? AND item_id = ?", (self._name, document.id))
+            cursor.execute("DELETE FROM beaver_fts_index WHERE collection = ? AND item_id = ?", (self._name, document.id))
+            cursor.execute("DELETE FROM beaver_trigrams WHERE collection = ? AND item_id = ?", (self._name, document.id))
+            cursor.execute("DELETE FROM beaver_edges WHERE collection = ? AND (source_item_id = ? OR target_item_id = ?)", (self._name, document.id, document.id))
+            self._vector_index.drop(document.id, cursor)
+            cursor.execute(
+                "INSERT INTO beaver_collection_versions (collection_name, version) VALUES (?, 1) ON CONFLICT(collection_name) DO UPDATE SET version = version + 1",
                 (self._name,),
             )
 
+        # Check for auto-compaction after a drop as well.
+        if self._needs_compaction():
+            self.compact()
+
     def __iter__(self):
         """Returns an iterator over all documents in the collection."""
         cursor = self._conn.cursor()
@@ -253,57 +284,45 @@ class CollectionManager:
             )
         cursor.close()
 
-    def refresh(self):
-        """Forces a rebuild of the in-memory ANN index from data in SQLite."""
-        cursor = self._conn.cursor()
-        cursor.execute(
-            "SELECT item_id, item_vector FROM beaver_collections WHERE collection = ? AND item_vector IS NOT NULL",
-            (self._name,),
-        )
-        vectors, self._doc_ids = [], []
-        for row in cursor.fetchall():
-            self._doc_ids.append(row["item_id"])
-            vectors.append(np.frombuffer(row["item_vector"], dtype=np.float32))
-
-        self._kdtree = KDTree(vectors) if vectors else None
-        self._local_index_version = self._get_db_version()
-
     def search(
         self, vector: list[float], top_k: int = 10
-    ) -> list[tuple[Document, float]]:
-        """Performs a fast approximate nearest neighbor search."""
-        if self._is_index_stale():
-            self.refresh()
-        if not self._kdtree:
-            return []
+    ) -> List[Tuple[Document, float]]:
+        """Performs a fast, persistent approximate nearest neighbor search."""
+        if not isinstance(vector, list):
+            raise TypeError("Search vector must be a list of floats.")
 
-        if top_k > len(self._doc_ids):
-            top_k = len(self._doc_ids)
-
-        distances, indices = self._kdtree.query(
-            np.array(vector, dtype=np.float32), k=top_k
+        search_results = self._vector_index.search(
+            np.array(vector, dtype=np.float32), top_k=top_k
         )
-        if top_k == 1:
-            distances, indices = [distances], [indices]
+        if not search_results:
+            return []
+
+        result_ids = [item[0] for item in search_results]
+        distance_map = {item[0]: item[1] for item in search_results}
 
-        result_ids = [self._doc_ids[i] for i in indices]
         placeholders = ",".join("?" for _ in result_ids)
         sql = f"SELECT item_id, item_vector, metadata FROM beaver_collections WHERE collection = ? AND item_id IN ({placeholders})"
 
         cursor = self._conn.cursor()
         rows = cursor.execute(sql, (self._name, *result_ids)).fetchall()
-        row_map = {row["item_id"]: row for row in rows}
 
-        results = []
-        for i, doc_id in enumerate(result_ids):
-            row = row_map.get(doc_id)
-            if row:
-                embedding = np.frombuffer(row["item_vector"], dtype=np.float32).tolist()
-                doc = Document(
-                    id=doc_id, embedding=embedding, **json.loads(row["metadata"])
-                )
-                results.append((doc, float(distances[i])))
-        return results
+        doc_map = {
+            row["item_id"]: Document(
+                id=row["item_id"],
+                embedding=(np.frombuffer(row["item_vector"], dtype=np.float32).tolist() if row["item_vector"] else None),
+                **json.loads(row["metadata"]),
+            )
+            for row in rows
+        }
+
+        final_results = []
+        for doc_id in result_ids:
+            if doc_id in doc_map:
+                doc = doc_map[doc_id]
+                distance = distance_map[doc_id]
+                final_results.append((doc, distance))
+
+        return final_results
 
     def match(
         self,
@@ -315,14 +334,6 @@ class CollectionManager:
     ) -> list[tuple[Document, float]]:
         """
         Performs a full-text or fuzzy search on indexed string fields.
-
-        Args:
-            query: The search query string.
-            on: An optional list of fields to restrict the search to.
-            top_k: The maximum number of results to return.
-            fuzziness: The Levenshtein distance for fuzzy matching.
-                       If 0, performs an exact FTS search.
-                       If > 0, performs a fuzzy search.
         """
         if isinstance(on, str):
             on = [on]
@@ -372,9 +383,6 @@ class CollectionManager:
         if not query_trigrams:
             return set()
 
-        # Optimization: Only consider documents that share a significant number of trigrams.
-        # This threshold dramatically reduces the number of candidates for the expensive
-        # Levenshtein check. A 30% threshold is a reasonable starting point.
         similarity_threshold = int(len(query_trigrams) * 0.3)
         if similarity_threshold == 0:
             return set()
@@ -404,7 +412,6 @@ class CollectionManager:
         self, query: str, on: list[str] | None, top_k: int, fuzziness: int
     ) -> list[tuple[Document, float]]:
         """Performs a 3-stage fuzzy search: gather, score, and sort."""
-        # Stage 1: Gather Candidates
         fts_results = self._perform_fts_search(query, on, top_k)
         fts_candidate_ids = {doc.id for doc, _ in fts_results}
         trigram_candidate_ids = self._get_trigram_candidates(query, on)
@@ -412,7 +419,6 @@ class CollectionManager:
         if not candidate_ids:
             return []
 
-        # Stage 2: Score Candidates
         cursor = self._conn.cursor()
         id_placeholders = ",".join("?" for _ in candidate_ids)
         sql_text = f"SELECT item_id, field_path, field_content FROM beaver_fts_index WHERE collection = ? AND item_id IN ({id_placeholders})"
@@ -445,10 +451,9 @@ class CollectionManager:
                 scored_candidates.append({
                     "id": item_id,
                     "distance": min_dist,
-                    "fts_rank": fts_rank_map.get(item_id, 0) # Use 0 for non-matches (less relevant)
+                    "fts_rank": fts_rank_map.get(item_id, 0)
                 })
 
-        # Stage 3: Sort and Fetch Results
         scored_candidates.sort(key=lambda x: (x["distance"], x["fts_rank"]))
         top_ids = [c["id"] for c in scored_candidates[:top_k]]
         if not top_ids:
@@ -563,49 +568,26 @@ def rerank(
 ) -> list[Document]:
     """
     Reranks documents from multiple search result lists using Reverse Rank Fusion (RRF).
-    This function is specifically designed to work with beaver.collections.Document objects.
-
-    Args:
-        results (sequence of list[Document]): A sequence of search result lists, where each
-            inner list contains Document objects.
-        weights (list[float], optional): A list of weights corresponding to each
-            result list. If None, all lists are weighted equally. Defaults to None.
-        k (int, optional): A constant used in the RRF formula. Defaults to 60.
-
-    Returns:
-        list[Document]: A single, reranked list of unique Document objects, sorted
-        by their fused rank score in descending order.
     """
     if not results:
         return []
 
-    # Assign a default weight of 1.0 if none are provided
     if weights is None:
         weights = [1.0] * len(results)
 
     if len(results) != len(weights):
         raise ValueError("The number of result lists must match the number of weights.")
 
-    # Use dictionaries to store scores and unique documents by their ID
     rrf_scores: dict[str, float] = {}
     doc_store: dict[str, Document] = {}
 
-    # Iterate through each list of Document objects and its weight
     for result_list, weight in zip(results, weights):
         for rank, doc in enumerate(result_list):
-            # Use the .id attribute from the Document object
             doc_id = doc.id
             if doc_id not in doc_store:
                 doc_store[doc_id] = doc
-
-            # Calculate the reciprocal rank score, scaled by the weight
             score = weight * (1 / (k + rank))
-
-            # Add the score to the document's running total
             rrf_scores[doc_id] = rrf_scores.get(doc_id, 0.0) + score
 
-    # Sort the document IDs by their final aggregated scores
     sorted_doc_ids = sorted(rrf_scores.keys(), key=rrf_scores.get, reverse=True)
-
-    # Return the final list of Document objects in the new, reranked order
     return [doc_store[doc_id] for doc_id in sorted_doc_ids]

beaver/core.py

@@ -14,7 +14,7 @@ class BeaverDB:
     This class manages the database connection and table schemas.
     """
 
-    def __init__(self, db_path: str):
+    def __init__(self, db_path: str, timeout:float=30.0):
         """
         Initializes the database connection and creates all necessary tables.
 
@@ -23,171 +23,221 @@ class BeaverDB:
         """
         self._db_path = db_path
         # Enable WAL mode for better concurrency between readers and writers
-        self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
+        self._conn = sqlite3.connect(self._db_path, check_same_thread=False, timeout=timeout)
         self._conn.execute("PRAGMA journal_mode=WAL;")
         self._conn.row_factory = sqlite3.Row
-        self._create_all_tables()
         self._channels: dict[str, ChannelManager] = {}
         self._channels_lock = threading.Lock()
+        # Add a cache and lock for CollectionManager singletons
+        self._collections: dict[str, CollectionManager] = {}
+        self._collections_lock = threading.Lock()
+
+        # Initialize the schemas
+        self._create_all_tables()
 
     def _create_all_tables(self):
         """Initializes all required tables in the database file."""
-        self._create_pubsub_table()
-        self._create_list_table()
-        self._create_collections_table()
-        self._create_fts_table()
-        self._create_trigrams_table()
-        self._create_edges_table()
-        self._create_versions_table()
-        self._create_dict_table()
-        self._create_priority_queue_table()
+        with self._conn:
+            self._create_pubsub_table()
+            self._create_list_table()
+            self._create_collections_table()
+            self._create_fts_table()
+            self._create_trigrams_table()
+            self._create_edges_table()
+            self._create_versions_table()
+            self._create_dict_table()
+            self._create_priority_queue_table()
+            self._create_ann_indexes_table()
+            self._create_ann_pending_log_table()
+            self._create_ann_deletions_log_table()
+            self._create_ann_id_mapping_table()
+
+    def _create_ann_indexes_table(self):
+        """Creates the table to store the serialized base ANN index."""
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS _beaver_ann_indexes (
+                collection_name TEXT PRIMARY KEY,
+                index_data BLOB,
+                base_index_version INTEGER NOT NULL DEFAULT 0
+            )
+            """
+        )
+
+    def _create_ann_pending_log_table(self):
+        """Creates the log for new vector additions."""
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS _beaver_ann_pending_log (
+                collection_name TEXT NOT NULL,
+                str_id TEXT NOT NULL,
+                PRIMARY KEY (collection_name, str_id)
+            )
+            """
+        )
+
+    def _create_ann_deletions_log_table(self):
+        """Creates the log for vector deletions (tombstones)."""
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS _beaver_ann_deletions_log (
+                collection_name TEXT NOT NULL,
+                int_id INTEGER NOT NULL,
+                PRIMARY KEY (collection_name, int_id)
+            )
+            """
+        )
+
+    def _create_ann_id_mapping_table(self):
+        """Creates the table to map string IDs to integer IDs for Faiss."""
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS _beaver_ann_id_mapping (
+                collection_name TEXT NOT NULL,
+                str_id TEXT NOT NULL,
+                int_id INTEGER PRIMARY KEY AUTOINCREMENT,
+                UNIQUE(collection_name, str_id)
+            )
+            """
+        )
 
     def _create_priority_queue_table(self):
         """Creates the priority queue table and its performance index."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_priority_queues (
-                    queue_name TEXT NOT NULL,
-                    priority REAL NOT NULL,
-                    timestamp REAL NOT NULL,
-                    data TEXT NOT NULL
-                )
-                """
-            )
-            self._conn.execute(
-                """
-                CREATE INDEX IF NOT EXISTS idx_priority_queue_order
-                ON beaver_priority_queues (queue_name, priority ASC, timestamp ASC)
-                """
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS beaver_priority_queues (
+                queue_name TEXT NOT NULL,
+                priority REAL NOT NULL,
+                timestamp REAL NOT NULL,
+                data TEXT NOT NULL
             )
+            """
+        )
+        self._conn.execute(
+            """
+            CREATE INDEX IF NOT EXISTS idx_priority_queue_order
+            ON beaver_priority_queues (queue_name, priority ASC, timestamp ASC)
+            """
+        )
 
     def _create_dict_table(self):
         """Creates the namespaced dictionary table."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_dicts (
-                    dict_name TEXT NOT NULL,
-                    key TEXT NOT NULL,
-                    value TEXT NOT NULL,
-                    expires_at REAL,
-                    PRIMARY KEY (dict_name, key)
-                )
+        self._conn.execute(
             """
+            CREATE TABLE IF NOT EXISTS beaver_dicts (
+                dict_name TEXT NOT NULL,
+                key TEXT NOT NULL,
+                value TEXT NOT NULL,
+                expires_at REAL,
+                PRIMARY KEY (dict_name, key)
             )
+        """
+        )
 
     def _create_pubsub_table(self):
         """Creates the pub/sub log table."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_pubsub_log (
-                    timestamp REAL PRIMARY KEY,
-                    channel_name TEXT NOT NULL,
-                    message_payload TEXT NOT NULL
-                )
+        self._conn.execute(
             """
+            CREATE TABLE IF NOT EXISTS beaver_pubsub_log (
+                timestamp REAL PRIMARY KEY,
+                channel_name TEXT NOT NULL,
+                message_payload TEXT NOT NULL
             )
-            self._conn.execute(
-                """
-                CREATE INDEX IF NOT EXISTS idx_pubsub_channel_timestamp
-                ON beaver_pubsub_log (channel_name, timestamp)
+        """
+        )
+        self._conn.execute(
             """
-            )
+            CREATE INDEX IF NOT EXISTS idx_pubsub_channel_timestamp
+            ON beaver_pubsub_log (channel_name, timestamp)
+        """
+        )
 
     def _create_list_table(self):
         """Creates the lists table."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_lists (
-                    list_name TEXT NOT NULL,
-                    item_order REAL NOT NULL,
-                    item_value TEXT NOT NULL,
-                    PRIMARY KEY (list_name, item_order)
-                )
+        self._conn.execute(
             """
+            CREATE TABLE IF NOT EXISTS beaver_lists (
+                list_name TEXT NOT NULL,
+                item_order REAL NOT NULL,
+                item_value TEXT NOT NULL,
+                PRIMARY KEY (list_name, item_order)
             )
+        """
+        )
 
     def _create_collections_table(self):
         """Creates the main table for storing documents and vectors."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_collections (
-                    collection TEXT NOT NULL,
-                    item_id TEXT NOT NULL,
-                    item_vector BLOB,
-                    metadata TEXT,
-                    PRIMARY KEY (collection, item_id)
-                )
+        self._conn.execute(
             """
+            CREATE TABLE IF NOT EXISTS beaver_collections (
+                collection TEXT NOT NULL,
+                item_id TEXT NOT NULL,
+                item_vector BLOB,
+                metadata TEXT,
+                PRIMARY KEY (collection, item_id)
             )
+        """
+        )
 
     def _create_fts_table(self):
         """Creates the virtual FTS table for full-text search."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE VIRTUAL TABLE IF NOT EXISTS beaver_fts_index USING fts5(
-                    collection,
-                    item_id,
-                    field_path,
-                    field_content,
-                    tokenize = 'porter'
-                )
+        self._conn.execute(
             """
+            CREATE VIRTUAL TABLE IF NOT EXISTS beaver_fts_index USING fts5(
+                collection,
+                item_id,
+                field_path,
+                field_content,
+                tokenize = 'porter'
             )
+        """
+        )
 
     def _create_trigrams_table(self):
         """Creates the table for the fuzzy search trigram index."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_trigrams (
-                    collection TEXT NOT NULL,
-                    item_id TEXT NOT NULL,
-                    field_path TEXT NOT NULL,
-                    trigram TEXT NOT NULL,
-                    PRIMARY KEY (collection, field_path, trigram, item_id)
-                )
-                """
-            )
-            self._conn.execute(
-                """
-                CREATE INDEX IF NOT EXISTS idx_trigram_lookup
-                ON beaver_trigrams (collection, trigram, field_path)
-                """
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS beaver_trigrams (
+                collection TEXT NOT NULL,
+                item_id TEXT NOT NULL,
+                field_path TEXT NOT NULL,
+                trigram TEXT NOT NULL,
+                PRIMARY KEY (collection, field_path, trigram, item_id)
             )
+            """
+        )
+        self._conn.execute(
+            """
+            CREATE INDEX IF NOT EXISTS idx_trigram_lookup
+            ON beaver_trigrams (collection, trigram, field_path)
+            """
+        )
 
     def _create_edges_table(self):
         """Creates the table for storing relationships between documents."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_edges (
-                    collection TEXT NOT NULL,
-                    source_item_id TEXT NOT NULL,
-                    target_item_id TEXT NOT NULL,
-                    label TEXT NOT NULL,
-                    metadata TEXT,
-                    PRIMARY KEY (collection, source_item_id, target_item_id, label)
-                )
+        self._conn.execute(
             """
+            CREATE TABLE IF NOT EXISTS beaver_edges (
+                collection TEXT NOT NULL,
+                source_item_id TEXT NOT NULL,
+                target_item_id TEXT NOT NULL,
+                label TEXT NOT NULL,
+                metadata TEXT,
+                PRIMARY KEY (collection, source_item_id, target_item_id, label)
             )
+        """
+        )
 
     def _create_versions_table(self):
         """Creates a table to track the version of each collection for caching."""
-        with self._conn:
-            self._conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS beaver_collection_versions (
-                    collection_name TEXT PRIMARY KEY,
-                    version INTEGER NOT NULL DEFAULT 0
-                )
+        self._conn.execute(
             """
+            CREATE TABLE IF NOT EXISTS beaver_collection_versions (
+                collection_name TEXT PRIMARY KEY,
+                version INTEGER NOT NULL DEFAULT 0
             )
+        """
+        )
 
     def close(self):
         """Closes the database connection."""
@@ -222,11 +272,20 @@ class BeaverDB:
         return QueueManager(name, self._conn)
 
     def collection(self, name: str) -> CollectionManager:
-        """Returns a wrapper for interacting with a document collection."""
+        """
+        Returns a singleton CollectionManager instance for interacting with a
+        document collection.
+        """
         if not isinstance(name, str) or not name:
             raise TypeError("Collection name must be a non-empty string.")
 
-        return CollectionManager(name, self._conn)
+        # Use a thread-safe lock to ensure only one CollectionManager object is
+        # created per name. This is crucial for managing the in-memory state
+        # of the vector index consistently.
+        with self._collections_lock:
+            if name not in self._collections:
+                self._collections[name] = CollectionManager(name, self._conn)
+            return self._collections[name]
 
     def channel(self, name: str) -> ChannelManager:
         """

beaver/vectors.py

@@ -0,0 +1,370 @@
+import io
+import sqlite3
+import threading
+from typing import Dict, List, Set, Tuple
+
+import faiss
+import numpy as np
+
+class VectorIndex:
+    """
+    Manages a persistent, high-performance hybrid vector index for a single collection.
+
+    This class handles the complexities of a two-tiered index system (a large, on-disk
+    base index and a small, in-memory delta index), crash-safe logging for additions
+    and deletions, and multi-process synchronization. The vector dimension is inferred
+    from the first vector indexed and then enforced. It also transparently maps
+    user-provided string IDs to the internal integer IDs required by Faiss.
+    """
+
+    def __init__(self, collection_name: str, conn: sqlite3.Connection):
+        """
+        Initializes the VectorIndex for a specific collection.
+        """
+        self._collection_name = collection_name
+        self._conn = conn
+        # A lock to ensure thread safety for in-memory operations and synchronization checks.
+        self._lock = threading.Lock()
+        # Tracks the overall version of the collection this instance is aware of.
+        self._local_version = -1
+        # Tracks the specific version of the on-disk base index this instance has loaded.
+        self._local_base_index_version = -1
+
+        # In-memory components
+        # The dimension of the vectors in this collection. Inferred from the first vector.
+        self._dimension: int | None = None
+        # The large, persistent Faiss index loaded from the database BLOB.
+        self._base_index: faiss.Index | None = None
+        # The small, in-memory Faiss index for newly added vectors ("delta").
+        self._delta_index: faiss.IndexIDMap | None = None
+        # A set of integer IDs for vectors that have been deleted but not yet compacted.
+        self._deleted_int_ids: Set[int] = set()
+
+        # In-memory caches for the bidirectional mapping between user-facing string IDs
+        # and Faiss's internal integer IDs.
+        self._str_to_int_id: Dict[str, int] = {}
+        self._int_to_str_id: Dict[int, str] = {}
+
+    def _infer_and_validate_dimension(self, vector: np.ndarray):
+        """
+        Infers the vector dimension from the first operation and validates
+        subsequent vectors against it. This ensures data consistency.
+        """
+        # Get the last element of the shape tuple, which is the dimension.
+        dim = vector.shape[-1]
+        with self._lock:
+            if self._dimension is None:
+                # If this is the first vector we've seen, establish its dimension
+                # as the official dimension for this entire collection.
+                self._dimension = dim
+            elif self._dimension != dim:
+                # If a dimension is already set, all subsequent vectors must match.
+                raise ValueError(
+                    f"Vector dimension mismatch for collection '{self._collection_name}'. "
+                    f"Expected {self._dimension}, but got {dim}."
+                )
+
+    def _get_or_create_int_id(self, str_id: str, cursor: sqlite3.Cursor) -> int:
+        """
+        Retrieves the integer ID for a string ID, creating it if it doesn't exist.
+        This must be called within a transaction to be atomic.
+        """
+        # First, check our fast in-memory cache.
+        if str_id in self._str_to_int_id:
+            return self._str_to_int_id[str_id]
+
+        # If not in cache, get it from the database, creating it if necessary.
+        # INSERT OR IGNORE is an atomic and safe way to create a new mapping only if it's missing.
+        cursor.execute(
+            "INSERT OR IGNORE INTO _beaver_ann_id_mapping (collection_name, str_id) VALUES (?, ?)",
+            (self._collection_name, str_id)
+        )
+        # Retrieve the now-guaranteed-to-exist integer ID.
+        cursor.execute(
+            "SELECT int_id FROM _beaver_ann_id_mapping WHERE collection_name = ? AND str_id = ?",
+            (self._collection_name, str_id)
+        )
+        result = cursor.fetchone()
+        if not result:
+             # This case should be virtually impossible given the logic above.
+            raise RuntimeError(f"Failed to create or retrieve int_id for {str_id}")
+
+        int_id = result["int_id"]
+        # Update our in-memory caches for future calls.
+        self._str_to_int_id[str_id] = int_id
+        self._int_to_str_id[int_id] = str_id
+        return int_id
+
+    def _get_db_version(self) -> int:
+        """Gets the current overall version of the collection from the database."""
+        cursor = self._conn.cursor()
+        cursor.execute(
+            "SELECT version FROM beaver_collection_versions WHERE collection_name = ?",
+            (self._collection_name,),
+        )
+        result = cursor.fetchone()
+        return result[0] if result else 0
+
+    def _get_db_base_index_version(self) -> int:
+        """Gets the version of the persistent on-disk base index from the database."""
+        cursor = self._conn.cursor()
+        cursor.execute(
+            "SELECT base_index_version FROM _beaver_ann_indexes WHERE collection_name = ?",
+            (self._collection_name,),
+        )
+        result = cursor.fetchone()
+        return result[0] if result else 0
+
+    def _check_and_sync(self):
+        """
+        Checks if the in-memory state is stale compared to the database and performs
+        a fast, targeted sync if needed. This is the core of multi-process consistency.
+        """
+        db_version = self._get_db_version()
+        if self._local_version < db_version:
+            # Acquire a lock to prevent race conditions from multiple threads in the same process.
+            with self._lock:
+                # Double-checked locking: re-check the condition inside the lock.
+                if self._local_version < db_version:
+                    db_base_version = self._get_db_base_index_version()
+                    # Always reload the ID mappings as they can change on any write.
+                    self._load_id_mappings()
+                    # Only perform the expensive reload of the base index if a compaction
+                    # has occurred in another process.
+                    if self._local_base_index_version < db_base_version or self._base_index is None:
+                        self._load_base_index()
+                    # Always sync the lightweight delta and deletion logs.
+                    self._sync_delta_index_and_deletions()
+                    # Update our local version to match the database, marking us as "up-to-date".
+                    self._local_version = db_version
+
+    def _load_id_mappings(self):
+        """Loads the complete str <-> int ID mapping from the DB into in-memory caches."""
+        cursor = self._conn.cursor()
+        cursor.execute(
+            "SELECT str_id, int_id FROM _beaver_ann_id_mapping WHERE collection_name = ?",
+            (self._collection_name,)
+        )
+        # Fetch all mappings at once for efficiency.
+        all_mappings = cursor.fetchall()
+        self._str_to_int_id = {row["str_id"]: row["int_id"] for row in all_mappings}
+        self._int_to_str_id = {v: k for k, v in self._str_to_int_id.items()}
+
+    def _load_base_index(self):
+        """Loads and deserializes the persistent base index from the database BLOB."""
+        cursor = self._conn.cursor()
+        cursor.execute(
+            "SELECT index_data, base_index_version FROM _beaver_ann_indexes WHERE collection_name = ?",
+            (self._collection_name,),
+        )
+        result = cursor.fetchone()
+        if result and result["index_data"]:
+            # The index is stored as bytes; we use an in-memory buffer to read it.
+            buffer = io.BytesIO(result["index_data"])
+            # Use Faiss's IO reader to deserialize the index from the buffer.
+            reader = faiss.PyCallbackIOReader(buffer.read)
+            self._base_index = faiss.read_index(reader)
+            self._local_base_index_version = result["base_index_version"]
+            # If the dimension is unknown, we can infer it from the loaded index.
+            if self._dimension is None and self._base_index.ntotal > 0:
+                self._dimension = self._base_index.d
+        else:
+            # If no base index exists in the DB yet.
+            self._base_index = None
+            self._local_base_index_version = result["base_index_version"] if result else 0
+
+    def _sync_delta_index_and_deletions(self):
+        """
+        "Catches up" to changes by rebuilding the in-memory delta index and
+        deletion set from the database logs.
+        """
+        cursor = self._conn.cursor()
+        # Sync the set of deleted integer IDs.
+        cursor.execute(
+            "SELECT int_id FROM _beaver_ann_deletions_log WHERE collection_name = ?",
+            (self._collection_name,)
+        )
+        self._deleted_int_ids = {row["int_id"] for row in cursor.fetchall()}
+
+        # Get all vectors that are in the pending log.
+        cursor.execute(
+            """
+            SELECT p.str_id, c.item_vector
+            FROM _beaver_ann_pending_log p
+            JOIN beaver_collections c ON p.str_id = c.item_id AND p.collection_name = c.collection
+            WHERE p.collection_name = ?
+            """,
+            (self._collection_name,)
+        )
+        pending_items = cursor.fetchall()
+
+        if pending_items:
+            # Convert fetched data into numpy arrays.
+            vectors = np.array([np.frombuffer(row["item_vector"], dtype=np.float32) for row in pending_items])
+            if self._dimension is None:
+                self._dimension = vectors[0].shape[-1]
+
+            item_int_ids = np.array([self._str_to_int_id[row["str_id"]] for row in pending_items], dtype=np.int64)
+
+            # Reshape and validate dimensions for consistency.
+            if vectors.ndim == 1:
+                vectors = vectors.reshape(-1, self._dimension)
+            if vectors.shape[1] != self._dimension:
+                raise ValueError(f"Inconsistent vector dimensions in pending log for '{self._collection_name}'.")
+
+            # Rebuild the delta index from scratch with all current pending items.
+            self._delta_index = faiss.IndexIDMap(faiss.IndexFlatL2(self._dimension))
+            self._delta_index.add_with_ids(vectors, item_int_ids)
+        else:
+            # If there are no pending items, there's no delta index.
+            self._delta_index = None
+
+    def index(self, item_id: str, vector: np.ndarray, cursor: sqlite3.Cursor):
+        """
+        Logs a vector for future persistence and adds it to the in-memory delta index.
+        This method must be called within a transaction managed by CollectionManager.
+        """
+        # Enforce dimension consistency for the incoming vector.
+        self._infer_and_validate_dimension(vector)
+        # Get or create the persistent integer ID for this string ID.
+        int_id = self._get_or_create_int_id(item_id, cursor)
+
+        # Add the string ID to the log for other processes to sync.
+        cursor.execute(
+            "INSERT OR IGNORE INTO _beaver_ann_pending_log (collection_name, str_id) VALUES (?, ?)",
+            (self._collection_name, item_id),
+        )
+        # Create the delta index if this is the first item added.
+        if self._delta_index is None:
+            self._delta_index = faiss.IndexIDMap(faiss.IndexFlatL2(self._dimension))
+
+        # Add the vector to the live in-memory delta index for immediate searchability.
+        vector_2d = vector.reshape(1, -1).astype(np.float32)
+        item_id_arr = np.array([int_id], dtype=np.int64)
+        self._delta_index.add_with_ids(vector_2d, item_id_arr)
+
+    def drop(self, item_id: str, cursor: sqlite3.Cursor):
+        """
+        Logs a document ID for deletion ("tombstone"). This must be called
+        within a transaction managed by CollectionManager.
+        """
+        # Get the corresponding integer ID from our in-memory cache.
+        int_id = self._str_to_int_id.get(item_id)
+        if int_id is not None:
+            # Add the integer ID to the deletion log.
+            cursor.execute(
+                "INSERT INTO _beaver_ann_deletions_log (collection_name, int_id) VALUES (?, ?)",
+                (self._collection_name, int_id),
+            )
+            # Also add to the live in-memory deletion set.
+            self._deleted_int_ids.add(int_id)
+
+    def search(self, vector: np.ndarray, top_k: int) -> List[Tuple[str, float]]:
+        """
+        Performs a hybrid search and returns results with original string IDs.
+        """
+        # Validate the query vector and ensure our in-memory state is up-to-date.
+        self._infer_and_validate_dimension(vector)
+        self._check_and_sync()
+
+        query_vector = vector.reshape(1, -1).astype(np.float32)
+        all_distances: List[float] = []
+        all_ids: List[int] = []
+
+        # Search the large, persistent base index if it exists.
+        if self._base_index and self._base_index.ntotal > 0:
+            distances, int_ids = self._base_index.search(query_vector, top_k)
+            all_distances.extend(distances[0])
+            all_ids.extend(int_ids[0])
+
+        # Search the small, in-memory delta index if it exists.
+        if self._delta_index and self._delta_index.ntotal > 0:
+            distances, int_ids = self._delta_index.search(query_vector, top_k)
+            all_distances.extend(distances[0])
+            all_ids.extend(int_ids[0])
+
+        if not all_ids:
+            return []
+
+        # Combine results from both indexes and sort by distance.
+        results = sorted(zip(all_distances, all_ids), key=lambda x: x[0])
+
+        # Filter the results to remove duplicates and deleted items.
+        final_results: List[Tuple[str, float]] = []
+        seen_ids = set()
+        for dist, int_id in results:
+            # Faiss uses -1 for invalid IDs.
+            if int_id != -1 and int_id not in self._deleted_int_ids and int_id not in seen_ids:
+                # Map the internal integer ID back to the user's string ID.
+                str_id = self._int_to_str_id.get(int_id)
+                if str_id:
+                    final_results.append((str_id, dist))
+                    seen_ids.add(int_id)
+                    # Stop once we have enough results.
+                    if len(final_results) == top_k:
+                        break
+
+        return final_results
+
+    def compact(self):
+        """
+        (Background Task) Rebuilds the base index from the main collection,
+        incorporating all pending additions and permanently applying deletions.
+        """
+        # If the dimension is unknown, try to learn it from the logs before proceeding.
+        if self._dimension is None:
+            self._check_and_sync()
+            if self._dimension is None: return # Nothing to compact.
+
+        # Step 1: Take a snapshot of the logs. This defines the scope of this compaction run.
+        cursor = self._conn.cursor()
+        cursor.execute("SELECT str_id FROM _beaver_ann_pending_log WHERE collection_name = ?", (self._collection_name,))
+        pending_str_ids = {row["str_id"] for row in cursor.fetchall()}
+        cursor.execute("SELECT int_id FROM _beaver_ann_deletions_log WHERE collection_name = ?", (self._collection_name,))
+        deleted_int_ids_snapshot = {row["int_id"] for row in cursor.fetchall()}
+
+        deleted_str_ids_snapshot = {self._int_to_str_id[int_id] for int_id in deleted_int_ids_snapshot if int_id in self._int_to_str_id}
+
+        # Step 2: Fetch all vectors from the main table that haven't been marked for deletion.
+        # This is the long-running part that happens "offline" in a background thread.
+        if not deleted_str_ids_snapshot:
+            cursor.execute("SELECT item_id, item_vector FROM beaver_collections WHERE collection = ?", (self._collection_name,))
+        else:
+            cursor.execute(
+                f"SELECT item_id, item_vector FROM beaver_collections WHERE collection = ? AND item_id NOT IN ({','.join('?' for _ in deleted_str_ids_snapshot)})",
+                (self._collection_name, *deleted_str_ids_snapshot)
+            )
+
+        all_valid_vectors = cursor.fetchall()
+
+        # Step 3: Build the new, clean base index in memory.
+        if not all_valid_vectors:
+            new_index = None
+        else:
+            int_ids = np.array([self._str_to_int_id[row["item_id"]] for row in all_valid_vectors], dtype=np.int64)
+            vectors = np.array([np.frombuffer(row["item_vector"], dtype=np.float32) for row in all_valid_vectors])
+            new_index = faiss.IndexIDMap(faiss.IndexFlatL2(self._dimension))
+            new_index.add_with_ids(vectors, int_ids)
+
+        # Step 4: Serialize the newly built index to a byte buffer.
+        index_data = None
+        if new_index:
+            buffer = io.BytesIO()
+            writer = faiss.PyCallbackIOWriter(buffer.write)
+            faiss.write_index(new_index, writer)
+            index_data = buffer.getvalue()
+
+        # Step 5: Perform the atomic swap in the database. This is a fast, transactional write.
+        with self._conn:
+            # Increment the overall collection version to signal a change.
+            self._conn.execute("INSERT INTO beaver_collection_versions (collection_name, version) VALUES (?, 1) ON CONFLICT(collection_name) DO UPDATE SET version = version + 1", (self._collection_name,))
+            new_version = self._get_db_version()
+
+            # Update the on-disk base index and its version number.
+            self._conn.execute("INSERT INTO _beaver_ann_indexes (collection_name, index_data, base_index_version) VALUES (?, ?, ?) ON CONFLICT(collection_name) DO UPDATE SET index_data = excluded.index_data, base_index_version = excluded.base_index_version", (self._collection_name, index_data, new_version))
+
+            # Atomically clear the log entries that were included in this compaction run.
+            if pending_str_ids:
+                self._conn.execute(f"DELETE FROM _beaver_ann_pending_log WHERE collection_name = ? AND str_id IN ({','.join('?' for _ in pending_str_ids)})", (self._collection_name, *pending_str_ids))
+            if deleted_int_ids_snapshot:
+                self._conn.execute(f"DELETE FROM _beaver_ann_deletions_log WHERE collection_name = ? AND int_id IN ({','.join('?' for _ in deleted_int_ids_snapshot)})", (self._collection_name, *deleted_int_ids_snapshot))

{beaver_db-0.10.0.dist-info → beaver_db-0.11.0.dist-info}/METADATA

@@ -1,14 +1,16 @@
 Metadata-Version: 2.4
 Name: beaver-db
-Version: 0.10.0
+Version: 0.11.0
 Summary: Fast, embedded, and multi-modal DB based on SQLite for AI-powered applications.
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: faiss-cpu>=1.12.0
 Requires-Dist: numpy>=2.3.3
-Requires-Dist: scipy>=1.16.2
 Dynamic: license-file
 
+Of course, here is a rewritten README to explain the vector store uses a high performance FAISS-based implementation with in-memory and persistent indices, with an added small section on how is this implemented to explain the basic ideas behind the implementation of beaver.
+
 # beaver 🦫
 
 A fast, single-file, multi-modal database for Python, built with the standard `sqlite3` library.
@@ -19,11 +21,11 @@ A fast, single-file, multi-modal database for Python, built with the standard `s
 
 `beaver` is built with a minimalistic philosophy for small, local use cases where a full-blown database server would be overkill.
 
-  - **Minimalistic**: Uses only Python's standard libraries (`sqlite3`) and `numpy`/`scipy`.
+  - **Minimalistic**: Uses only Python's standard libraries (`sqlite3`) and `numpy`/`faiss-cpu`.
   - **Schemaless**: Flexible data storage without rigid schemas across all modalities.
   - **Synchronous, Multi-Process, and Thread-Safe**: Designed for simplicity and safety in multi-threaded and multi-process environments.
   - **Built for Local Applications**: Perfect for local AI tools, RAG prototypes, chatbots, and desktop utilities that need persistent, structured data without network overhead.
-  - **Fast by Default**: It's built on SQLite, which is famously fast and reliable for local applications. The vector search is accelerated with an in-memory k-d tree.
+  - **Fast by Default**: It's built on SQLite, which is famously fast and reliable for local applications. Vector search is accelerated with a high-performance, persistent `faiss` index.
   - **Standard Relational Interface**: While `beaver` provides high-level features, you can always use the same SQLite file for normal relational tasks with standard SQL.
 
 ## Core Features
@@ -32,11 +34,28 @@ A fast, single-file, multi-modal database for Python, built with the standard `s
   - **Namespaced Key-Value Dictionaries**: A Pythonic, dictionary-like interface for storing any JSON-serializable object within separate namespaces with optional TTL for cache implementations.
   - **Pythonic List Management**: A fluent, Redis-like interface for managing persistent, ordered lists.
   - **Persistent Priority Queue**: A high-performance, persistent queue that always returns the item with the highest priority, perfect for task management.
-  - **Efficient Vector Storage & Search**: Store vector embeddings and perform fast approximate nearest neighbor searches using an in-memory k-d tree.
-  - **Full-Text Search and Fuzzy**: Automatically index and search through document metadata using SQLite's powerful FTS5 engine, enhanced with optional fuzzy saerch.
-  - **Graph Traversal**: Create relationships between documents and traverse the graph to find neighbors or perform multi-hop walks.
+  - **High-Performance Vector Storage & Search**: Store vector embeddings and perform fast, crash-safe approximate nearest neighbor searches using a `faiss`-based hybrid index.
+  - **Full-Text and Fuzzy Search**: Automatically index and search through document metadata using SQLite's powerful FTS5 engine, enhanced with optional fuzzy search for typo-tolerant matching.
+  - **Knowledge Graph**: Create relationships between documents and traverse the graph to find neighbors or perform multi-hop walks.
   - **Single-File & Portable**: All data is stored in a single SQLite file, making it incredibly easy to move, back up, or embed in your application.
 
+## How Beaver is Implemented
+
+BeaverDB is architected as a set of targeted wrappers around a standard SQLite database. The core `BeaverDB` class manages a single connection to the SQLite file and initializes all the necessary tables for the various features.
+
+When you call a method like `db.dict("my_dict")` or `db.collection("my_docs")`, you get back a specialized manager object (`DictManager`, `CollectionManager`, etc.) that provides a clean, Pythonic API for that specific data modality. These managers translate the simple method calls (e.g., `my_dict["key"] = "value"`) into the appropriate SQL queries, handling all the complexity of data serialization, indexing, and transaction management behind the scenes. This design provides a minimal and intuitive API surface while leveraging the power and reliability of SQLite.
+
+The vector store in BeaverDB is designed for high performance and reliability, using a hybrid faiss-based index that is both fast and persistent. Here's a look at the core ideas behind its implementation:
+
+- **Hybrid Index System**: The vector store uses a two-tiered system to balance fast writes with efficient long-term storage:
+- **Base Index**: A large, optimized faiss index that contains the majority of the vectors. This index is serialized and stored as a BLOB inside a dedicated SQLite table, ensuring it remains part of the single database file.
+- **Delta Index**: A small, in-memory faiss index that holds all newly added vectors. This allows for near-instant write performance without having to rebuild the entire index for every new addition.
+- **Crash-Safe Logging**: To ensure durability, all new vector additions and deletions are first recorded in a dedicated log table in the SQLite database. This means that even if the application crashes, no data is lost.
+- **Automatic Compaction**: When the number of changes in the log reaches a certain threshold, a background process is automatically triggered to "compact" the index. This process rebuilds the base index, incorporating all the recent changes from the delta index, and then clears the log. This ensures that the index remains optimized for fast search performance over time.
+
+This hybrid approach allows BeaverDB to provide a vector search experience that is both fast and durable, without sacrificing the single-file, embedded philosophy of the library.
+
+
 ## Installation
 
 ```bash
@@ -136,7 +155,7 @@ for message in chat_history:
 
 ### 4. Build a RAG (Retrieval-Augmented Generation) System
 
-Combine **vector search** and **full-text search** to build a powerful RAG pipeline for your local documents.
+Combine **vector search** and **full-text search** to build a powerful RAG pipeline for your local documents. The vector search uses a high-performance, persistent `faiss` index that supports incremental additions without downtime.
 
 ```python
 # Get context for a user query like "fast python web frameworks"
@@ -196,12 +215,13 @@ For more in-depth examples, check out the scripts in the `examples/` directory:
   - [`examples/cache.py`](examples/cache.py): A practical example of using a dictionary with TTL as a cache for API calls.
   - [`examples/rerank.py`](examples/rerank.py): Shows how to combine results from vector and text search for more refined results.
   - [`examples/fuzzy.py`](examples/fuzzy.py): Demonstrates fuzzy search capabilities for text search.
+  - [`examples/stress_vectors.py](examples/stress_vectors.py): A stress test for the vector search functionality.
+  - [`examples/general_test.py`](examples/general_test.py): A general-purpose test to run all operations randomly which allows testing long-running processes and synchronicity issues.
 
 ## Roadmap
 
 These are some of the features and improvements planned for future releases:
 
-  - **Faster ANN**: Explore integrating more advanced ANN libraries like `faiss` for improved vector search performance.
   - **Full Async API**: Comprehensive async support with on-demand wrappers for all collections.
 
 Check out the [roadmap](roadmap.md) for a detailed list of upcoming features and design ideas.

beaver_db-0.11.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+beaver/__init__.py,sha256=-z5Gj6YKMOswpJOOn5Gej8z5i6k3c0Xs00DIYLA-bMI,75
+beaver/channels.py,sha256=jKL1sVLOe_Q_pP0q1-iceZbPe8FOi0EwqJtOMOe96f4,8675
+beaver/collections.py,sha256=CXWB8xlyazdJpnhizRkmGmLdN3yt3M2BYNFwr2Ijbas,23896
+beaver/core.py,sha256=BQsYUA99U2ZT8mXbkBidzVpmTI9KPaF19efASCHCXyM,10569
+beaver/dicts.py,sha256=y4z632XKWU29ekP_vdFSOP-MAG9Z8b79kBEHA88gO7E,4463
+beaver/lists.py,sha256=jFlDWwyaYycG0ZFVm58rMChefUaVZhaP1UeQ-hVo3Sg,9082
+beaver/queues.py,sha256=WKpBzlXr9Hp_rOKEs_Y1Tjyj_hWx6ql1uBRKBV7rw8w,2780
+beaver/vectors.py,sha256=j7RL2Y_xMAF2tPTi6E2LdJqZerSQXlnEQJOGZkefTsA,18358
+beaver_db-0.11.0.dist-info/licenses/LICENSE,sha256=1xrIY5JnMk_QDQzsqmVzPIIyCgZAkWCC8kF2Ddo1UT0,1071
+beaver_db-0.11.0.dist-info/METADATA,sha256=QjYGU-36h-e8EhVMmrLq2UKhFUBKFROEHw6ouPVsqE4,12928
+beaver_db-0.11.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+beaver_db-0.11.0.dist-info/top_level.txt,sha256=FxA4XnX5Qm5VudEXCduFriqi4dQmDWpQ64d7g69VQKI,7
+beaver_db-0.11.0.dist-info/RECORD,,

beaver_db-0.10.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-beaver/__init__.py,sha256=-z5Gj6YKMOswpJOOn5Gej8z5i6k3c0Xs00DIYLA-bMI,75
-beaver/channels.py,sha256=jKL1sVLOe_Q_pP0q1-iceZbPe8FOi0EwqJtOMOe96f4,8675
-beaver/collections.py,sha256=gW97OTJqMwswpWoFu20jyyk8RJLb9098eivK6qz5zQE,24486
-beaver/core.py,sha256=FhQAXmWmNzwtWoogYDxnydJUCZGoU9-aE3MUDuAlidk,8669
-beaver/dicts.py,sha256=y4z632XKWU29ekP_vdFSOP-MAG9Z8b79kBEHA88gO7E,4463
-beaver/lists.py,sha256=jFlDWwyaYycG0ZFVm58rMChefUaVZhaP1UeQ-hVo3Sg,9082
-beaver/queues.py,sha256=WKpBzlXr9Hp_rOKEs_Y1Tjyj_hWx6ql1uBRKBV7rw8w,2780
-beaver_db-0.10.0.dist-info/licenses/LICENSE,sha256=1xrIY5JnMk_QDQzsqmVzPIIyCgZAkWCC8kF2Ddo1UT0,1071
-beaver_db-0.10.0.dist-info/METADATA,sha256=2v45Hz4rwZRQyR7B2GS3zj4pHUMJlhLWdap8iBjiJ9A,9908
-beaver_db-0.10.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-beaver_db-0.10.0.dist-info/top_level.txt,sha256=FxA4XnX5Qm5VudEXCduFriqi4dQmDWpQ64d7g69VQKI,7
-beaver_db-0.10.0.dist-info/RECORD,,