haystack-velesdb 3.12.0__tar.gz → 4.0.0__tar.gz

{haystack_velesdb-3.12.0/src/haystack_velesdb.egg-info → haystack_velesdb-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-velesdb
-Version: 3.12.0
+Version: 4.0.0
 Summary: Haystack 2.x DocumentStore for VelesDB: The Local AI Memory Database.
 Author-email: VelesDB Team <contact@wiscale.fr>
 License: MIT
@@ -29,7 +29,9 @@ Dynamic: license-file
 # haystack-velesdb
 
 A Haystack 2.x `DocumentStore` backed by [VelesDB](https://github.com/cyberlife-coder/VelesDB) —
-the local-first, microsecond-latency vector database.
+**the explainable, local-first memory engine for AI agents** (microsecond vector
+search is the proof, not the pitch). For the connected `why()` recall trail
+across typed links, see [velesdb-memory](../../crates/velesdb-memory/README.md).
 
 This integration joins the existing [LangChain](../langchain/) and [LlamaIndex](../llamaindex/)
 connectors, completing the trio of major Python RAG frameworks supported by VelesDB.
@@ -102,26 +104,15 @@ indexer.connect("embedder", "writer")
 indexer.run({"converter": {"sources": ["paper.pdf"]}})
 
 # Query pipeline. `InMemoryEmbeddingRetriever` is bound to `InMemoryDocumentStore`
-# and would NOT work against a custom DocumentStore — wrap `embedding_retrieval`
-# in a thin Haystack component that forwards the call. Full working example in
-# `integrations/haystack/examples/rag_pipeline.py` (`_VelesRetriever`).
-from haystack import component
-from haystack.dataclasses import Document
-from typing import List
-
-@component
-class VelesRetriever:
-    def __init__(self, document_store, top_k: int = 10):
-        self._store = document_store
-        self._top_k = top_k
-
-    @component.output_types(documents=List[Document])
-    def run(self, query_embedding: List[float]):
-        return {"documents": self._store.embedding_retrieval(query_embedding, top_k=self._top_k)}
+# and would NOT work against a custom DocumentStore — use the shipped
+# `VelesDBEmbeddingRetriever` instead (like `QdrantEmbeddingRetriever` in the
+# Qdrant integration, no hand-rolled `@component` wrapper needed). Full working
+# example in `integrations/haystack/examples/rag_pipeline.py`.
+from haystack_velesdb import VelesDBEmbeddingRetriever
 
 querier = Pipeline()
 querier.add_component("embedder", SentenceTransformersTextEmbedder(model="all-MiniLM-L6-v2"))
-querier.add_component("retriever", VelesRetriever(document_store=store))
+querier.add_component("retriever", VelesDBEmbeddingRetriever(document_store=store))
 querier.connect("embedder.embedding", "retriever.query_embedding")
 result = querier.run({"embedder": {"text": "What is VelesDB?"}})
 print(result["retriever"]["documents"])
@@ -137,16 +128,25 @@ print(result["retriever"]["documents"])
 | `collection_name` | `"haystack_documents"` | VelesDB collection name |
 | `embedding_dim` | `768` | Embedding vector dimension |
 | `metric` | `"cosine"` | Distance metric: `"cosine"`, `"euclidean"`, `"dot"`, `"hamming"`, or `"jaccard"` |
+| `scroll_limit` | `10_000` | Max documents returned by `filter_documents()`; raise for collections bigger than this |
 
 ### Methods
 
 | Method | Description |
 |--------|-------------|
-| `write_documents(documents, policy)` | Upsert documents; returns count written |
-| `filter_documents(filters)` | Scroll documents matching a VelesDB filter dict |
-| `embedding_retrieval(query_embedding, top_k, filters, scale_score)` | Vector similarity search |
+| `write_documents(documents, policy, sparse_vectors=None)` | Upsert documents; returns count written. `sparse_vectors` is an optional list aligned with `documents` — each entry a flat `dict[int, float]` or a named `dict[str, dict[int, float]]` mapping (e.g. `{"bge_m3": {0: 1.5}}`); a named mapping creates that sparse index for later hybrid retrieval |
+| `stream_insert(documents, sparse_vectors=None)` | Insert documents through VelesDB's streaming ingestion channel in one call (append-only, no `DuplicatePolicy` check); returns count inserted. See "Note on streaming ingestion" below |
+| `write_documents_streaming(documents, policy, sparse_vectors=None, batch_size=100)` | Same `DuplicatePolicy` semantics as `write_documents`, but sent through the streaming channel in `batch_size`-sized chunks — better throughput for large bulk loads. See "Note on streaming ingestion" below |
+| `flush()` | Flush pending changes to disk. See "Note on streaming ingestion" below |
+| `filter_documents(filters)` | Scroll documents matching a VelesDB filter dict, up to `scroll_limit` |
+| `embedding_retrieval(query_embedding, top_k, filters, scale_score, fusion=None, fusion_params=None)` | Vector similarity search. `fusion` selects a `velesdb.FusionStrategy` (`"average"`, `"maximum"`, `"rrf"`, `"weighted"`, `"relative_score"` / `"rsf"`) applied via `Collection.multi_query_search`, changing the ranking; `fusion_params` configures it (e.g. `{"dense_weight": 0.7, "sparse_weight": 0.3}` for `"rsf"`). `filters` cannot be combined with `fusion` |
 | `count_documents()` | Total document count |
 | `delete_documents(document_ids)` | Delete by Haystack string IDs |
+| `train_pq(m=8, k=256, opq=False)` | Train Product Quantization on the collection |
+| `analyze_collection()` | Compute and persist collection statistics (point/row counts, column stats) |
+| `get_collection_stats()` | Return cached statistics from the last `analyze_collection()` call, or `None` |
+| `is_metadata_only()` | Whether the current collection holds no vectors |
+| `create_metadata_collection(name)` | Create a metadata-only collection (no vectors), joinable with vector collections |
 | `to_dict()` / `from_dict()` | Haystack pipeline serialisation |
 
 **Note on `DuplicatePolicy`:** `NONE` and `OVERWRITE` use VelesDB upsert semantics
@@ -164,6 +164,16 @@ if a collision is detected between a new document and an existing one.
 are normalised from `[-1, 1]` to `[0, 1]` so they behave like probabilities
 in downstream re-ranking.
 
+**Note on streaming ingestion:** `stream_insert()` and `write_documents_streaming()`
+forward to the underlying `velesdb.Collection.stream_insert`, which requires
+the collection to have `enable_streaming()` called on it first — the same
+caller-managed contract as the LangChain and LlamaIndex integrations (see
+`docs/reference/ECOSYSTEM_PARITY.md`). The channel batches asynchronously on
+its own background interval (engine default ~50ms), so a point may not be
+immediately visible to reads right after either call returns; `flush()`
+flushes pending changes to disk but does **not** guarantee streaming-channel
+visibility.
+
 ## Running tests
 
 ```bash

{haystack_velesdb-3.12.0 → haystack_velesdb-4.0.0}/README.md

@@ -1,7 +1,9 @@
 # haystack-velesdb
 
 A Haystack 2.x `DocumentStore` backed by [VelesDB](https://github.com/cyberlife-coder/VelesDB) —
-the local-first, microsecond-latency vector database.
+**the explainable, local-first memory engine for AI agents** (microsecond vector
+search is the proof, not the pitch). For the connected `why()` recall trail
+across typed links, see [velesdb-memory](../../crates/velesdb-memory/README.md).
 
 This integration joins the existing [LangChain](../langchain/) and [LlamaIndex](../llamaindex/)
 connectors, completing the trio of major Python RAG frameworks supported by VelesDB.
@@ -74,26 +76,15 @@ indexer.connect("embedder", "writer")
 indexer.run({"converter": {"sources": ["paper.pdf"]}})
 
 # Query pipeline. `InMemoryEmbeddingRetriever` is bound to `InMemoryDocumentStore`
-# and would NOT work against a custom DocumentStore — wrap `embedding_retrieval`
-# in a thin Haystack component that forwards the call. Full working example in
-# `integrations/haystack/examples/rag_pipeline.py` (`_VelesRetriever`).
-from haystack import component
-from haystack.dataclasses import Document
-from typing import List
-
-@component
-class VelesRetriever:
-    def __init__(self, document_store, top_k: int = 10):
-        self._store = document_store
-        self._top_k = top_k
-
-    @component.output_types(documents=List[Document])
-    def run(self, query_embedding: List[float]):
-        return {"documents": self._store.embedding_retrieval(query_embedding, top_k=self._top_k)}
+# and would NOT work against a custom DocumentStore — use the shipped
+# `VelesDBEmbeddingRetriever` instead (like `QdrantEmbeddingRetriever` in the
+# Qdrant integration, no hand-rolled `@component` wrapper needed). Full working
+# example in `integrations/haystack/examples/rag_pipeline.py`.
+from haystack_velesdb import VelesDBEmbeddingRetriever
 
 querier = Pipeline()
 querier.add_component("embedder", SentenceTransformersTextEmbedder(model="all-MiniLM-L6-v2"))
-querier.add_component("retriever", VelesRetriever(document_store=store))
+querier.add_component("retriever", VelesDBEmbeddingRetriever(document_store=store))
 querier.connect("embedder.embedding", "retriever.query_embedding")
 result = querier.run({"embedder": {"text": "What is VelesDB?"}})
 print(result["retriever"]["documents"])
@@ -109,16 +100,25 @@ print(result["retriever"]["documents"])
 | `collection_name` | `"haystack_documents"` | VelesDB collection name |
 | `embedding_dim` | `768` | Embedding vector dimension |
 | `metric` | `"cosine"` | Distance metric: `"cosine"`, `"euclidean"`, `"dot"`, `"hamming"`, or `"jaccard"` |
+| `scroll_limit` | `10_000` | Max documents returned by `filter_documents()`; raise for collections bigger than this |
 
 ### Methods
 
 | Method | Description |
 |--------|-------------|
-| `write_documents(documents, policy)` | Upsert documents; returns count written |
-| `filter_documents(filters)` | Scroll documents matching a VelesDB filter dict |
-| `embedding_retrieval(query_embedding, top_k, filters, scale_score)` | Vector similarity search |
+| `write_documents(documents, policy, sparse_vectors=None)` | Upsert documents; returns count written. `sparse_vectors` is an optional list aligned with `documents` — each entry a flat `dict[int, float]` or a named `dict[str, dict[int, float]]` mapping (e.g. `{"bge_m3": {0: 1.5}}`); a named mapping creates that sparse index for later hybrid retrieval |
+| `stream_insert(documents, sparse_vectors=None)` | Insert documents through VelesDB's streaming ingestion channel in one call (append-only, no `DuplicatePolicy` check); returns count inserted. See "Note on streaming ingestion" below |
+| `write_documents_streaming(documents, policy, sparse_vectors=None, batch_size=100)` | Same `DuplicatePolicy` semantics as `write_documents`, but sent through the streaming channel in `batch_size`-sized chunks — better throughput for large bulk loads. See "Note on streaming ingestion" below |
+| `flush()` | Flush pending changes to disk. See "Note on streaming ingestion" below |
+| `filter_documents(filters)` | Scroll documents matching a VelesDB filter dict, up to `scroll_limit` |
+| `embedding_retrieval(query_embedding, top_k, filters, scale_score, fusion=None, fusion_params=None)` | Vector similarity search. `fusion` selects a `velesdb.FusionStrategy` (`"average"`, `"maximum"`, `"rrf"`, `"weighted"`, `"relative_score"` / `"rsf"`) applied via `Collection.multi_query_search`, changing the ranking; `fusion_params` configures it (e.g. `{"dense_weight": 0.7, "sparse_weight": 0.3}` for `"rsf"`). `filters` cannot be combined with `fusion` |
 | `count_documents()` | Total document count |
 | `delete_documents(document_ids)` | Delete by Haystack string IDs |
+| `train_pq(m=8, k=256, opq=False)` | Train Product Quantization on the collection |
+| `analyze_collection()` | Compute and persist collection statistics (point/row counts, column stats) |
+| `get_collection_stats()` | Return cached statistics from the last `analyze_collection()` call, or `None` |
+| `is_metadata_only()` | Whether the current collection holds no vectors |
+| `create_metadata_collection(name)` | Create a metadata-only collection (no vectors), joinable with vector collections |
 | `to_dict()` / `from_dict()` | Haystack pipeline serialisation |
 
 **Note on `DuplicatePolicy`:** `NONE` and `OVERWRITE` use VelesDB upsert semantics
@@ -136,6 +136,16 @@ if a collision is detected between a new document and an existing one.
 are normalised from `[-1, 1]` to `[0, 1]` so they behave like probabilities
 in downstream re-ranking.
 
+**Note on streaming ingestion:** `stream_insert()` and `write_documents_streaming()`
+forward to the underlying `velesdb.Collection.stream_insert`, which requires
+the collection to have `enable_streaming()` called on it first — the same
+caller-managed contract as the LangChain and LlamaIndex integrations (see
+`docs/reference/ECOSYSTEM_PARITY.md`). The channel batches asynchronously on
+its own background interval (engine default ~50ms), so a point may not be
+immediately visible to reads right after either call returns; `flush()`
+flushes pending changes to disk but does **not** guarantee streaming-channel
+visibility.
+
 ## Running tests
 
 ```bash

{haystack_velesdb-3.12.0 → haystack_velesdb-4.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "haystack-velesdb"
-version = "3.12.0"
+version = "4.0.0"
 description = "Haystack 2.x DocumentStore for VelesDB: The Local AI Memory Database."
 readme = "README.md"
 license = {text = "MIT"}

{haystack_velesdb-3.12.0 → haystack_velesdb-4.0.0}/src/haystack_velesdb/__init__.py

@@ -4,4 +4,4 @@ from haystack_velesdb.document_store import VelesDBDocumentStore
 from haystack_velesdb.retriever import VelesDBEmbeddingRetriever
 
 __all__ = ["VelesDBDocumentStore", "VelesDBEmbeddingRetriever"]
-__version__ = "3.12.0"
+__version__ = "4.0.0"

{haystack_velesdb-3.12.0 → haystack_velesdb-4.0.0}/src/haystack_velesdb/document_store.py

@@ -14,6 +14,7 @@ from haystack.document_stores.errors import DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 
 import velesdb
+from velesdb_common.collection_admin import CollectionAdminMixin
 from velesdb_common.fusion import build_fusion_strategy
 from velesdb_common.ids import stable_hash_id
 from velesdb_common.security import (
@@ -381,7 +382,48 @@ def _documents_to_points(
     return points
 
 
-class VelesDBDocumentStore:
+def _resolve_survivors_for_policy(
+    col: Any,
+    documents: List[Document],
+    int_id_map: Dict[int, str],
+    policy: DuplicatePolicy,
+) -> Optional[List[Document]]:
+    """Apply *policy* to *documents* against the collection's existing points.
+
+    Shared by :meth:`VelesDBDocumentStore.write_documents` and
+    :meth:`VelesDBDocumentStore.write_documents_streaming` — both need the
+    exact same FAIL/SKIP/NONE·OVERWRITE resolution before choosing how to
+    send the surviving points (single ``upsert`` vs. batched
+    ``stream_insert``).
+
+    Returns:
+        The list of documents to write, or ``None`` when ``policy`` is
+        ``SKIP`` and every incoming document already exists (caller should
+        return 0 without touching the collection).
+    """
+    if policy == DuplicatePolicy.FAIL:
+        _enforce_fail_policy(col, int_id_map)
+        return documents
+    if policy == DuplicatePolicy.SKIP:
+        survivors = _filter_skip_policy(col, int_id_map, documents)
+        return survivors or None
+    return documents
+
+
+def _flush_stream_batches(collection: Any, points: List[dict], batch_size: int) -> None:
+    """Send *points* to *collection* in batches via the streaming channel.
+
+    Mirrors ``langchain_velesdb.point_builder.flush_stream_batches`` and
+    ``llamaindex_velesdb.node_builder.flush_in_batches`` — kept local (rather
+    than imported cross-package) so this MIT-licensed integration does not
+    take on a dependency on the sibling LangChain/LlamaIndex distributions.
+    Only calls the ``velesdb`` binding's ``stream_insert``; no core logic.
+    """
+    for start in range(0, len(points), batch_size):
+        collection.stream_insert(points[start : start + batch_size])
+
+
+class VelesDBDocumentStore(CollectionAdminMixin):
     """Haystack 2.x DocumentStore backed by a local VelesDB collection.
 
     Stores documents (with optional embeddings) in VelesDB and exposes the
@@ -418,18 +460,32 @@ class VelesDBDocumentStore:
     # Internal connection management
     # ------------------------------------------------------------------
 
-    def _get_collection(self) -> Any:
-        """Return the VelesDB collection, opening or creating it as needed."""
+    @property
+    def collection_name(self) -> str:
+        """Return the collection name (satisfies the CollectionAdminMixin contract)."""
+        return self._collection_name
+
+    def _get_db(self) -> Any:
+        """Return the VelesDB database handle, opening it as needed.
+
+        Required by ``CollectionAdminMixin`` (``get_stats``/``analyze``/
+        ``train_pq``/``create_metadata_collection`` all go through this).
+        """
         if self._db is None:
             self._db = velesdb.Database(self._path)
+        return self._db
+
+    def _get_collection(self) -> Any:
+        """Return the VelesDB collection, opening or creating it as needed."""
+        db = self._get_db()
         if self._collection is None:
             col: Optional[Any] = None
             try:
-                col = self._db.get_collection(self._collection_name)
+                col = db.get_collection(self._collection_name)
             except KeyError:
                 pass
             if col is None:
-                col = self._db.create_collection(
+                col = db.create_collection(
                     self._collection_name,
                     dimension=self._embedding_dim,
                     metric=self._metric,
@@ -522,19 +578,90 @@ class VelesDBDocumentStore:
         sparse_by_id = _build_sparse_by_id(documents, sparse_vectors)
         int_id_map = _build_int_id_map(documents)
         col = self._get_collection()
-        if policy == DuplicatePolicy.FAIL:
-            _enforce_fail_policy(col, int_id_map)
-            survivors = documents
-        elif policy == DuplicatePolicy.SKIP:
-            survivors = _filter_skip_policy(col, int_id_map, documents)
-            if not survivors:
-                return 0
-        else:
-            survivors = documents
+        survivors = _resolve_survivors_for_policy(col, documents, int_id_map, policy)
+        if survivors is None:
+            return 0
         points = _documents_to_points(survivors, sparse_by_id)
         result = col.upsert(points)
         return result if isinstance(result, int) else len(points)
 
+    def stream_insert(
+        self,
+        documents: List[Document],
+        sparse_vectors: Optional[List[dict]] = None,
+    ) -> int:
+        """Insert *documents* via VelesDB's streaming ingestion channel.
+
+        Sends every point through ``Collection.stream_insert`` in a single
+        call. Unlike :meth:`write_documents`, this bypasses the
+        ``DuplicatePolicy`` checks (streaming is an append-only bulk-loading
+        path) — pre-dedupe on the caller side if that matters. For very
+        large batches where the bounded streaming channel's backpressure
+        should be respected across multiple sends, use
+        :meth:`write_documents_streaming` instead.
+
+        Mirrors ``langchain_velesdb.VelesDBVectorStore.stream_insert`` and
+        ``llamaindex_velesdb.VelesDBVectorStore.stream_insert``, which
+        forward the same way to ``collection.stream_insert``.
+
+        Args:
+            documents: Documents to insert.
+            sparse_vectors: Optional list aligned with *documents*; see
+                :meth:`write_documents` for the accepted shapes.
+
+        Returns:
+            Number of points inserted.
+        """
+        if not documents:
+            return 0
+        sparse_by_id = _build_sparse_by_id(documents, sparse_vectors)
+        points = _documents_to_points(documents, sparse_by_id)
+        self._get_collection().stream_insert(points)
+        return len(points)
+
+    def write_documents_streaming(
+        self,
+        documents: List[Document],
+        policy: DuplicatePolicy = DuplicatePolicy.NONE,
+        sparse_vectors: Optional[List[dict]] = None,
+        batch_size: int = 100,
+    ) -> int:
+        """Write *documents* using the streaming ingestion channel, batched.
+
+        Same ``DuplicatePolicy`` semantics as :meth:`write_documents`, but
+        points are sent through ``Collection.stream_insert`` in
+        ``batch_size``-sized chunks instead of one ``upsert`` call — better
+        throughput for large bulk loads, with built-in backpressure. Mirrors
+        the ``add_texts_streaming`` (LangChain) / ``add_streaming``
+        (LlamaIndex) pattern.
+
+        Args:
+            documents: Documents to write.
+            policy: Duplicate-handling policy (see :meth:`write_documents`).
+            sparse_vectors: Optional list aligned with *documents*; see
+                :meth:`write_documents` for the accepted shapes.
+            batch_size: Number of points per streaming batch. Defaults to 100.
+
+        Returns:
+            Number of points written.
+
+        Raises:
+            DuplicateDocumentError: When *policy* is ``FAIL`` and at least
+                one document already exists in the store.
+            ValueError: When a SHA-256 hash collision is detected.
+        """
+        if not documents:
+            return 0
+        sparse_by_id = _build_sparse_by_id(documents, sparse_vectors)
+        int_id_map = _build_int_id_map(documents)
+        col = self._get_collection()
+        survivors = _resolve_survivors_for_policy(col, documents, int_id_map, policy)
+        if survivors is None:
+            return 0
+        points = _documents_to_points(survivors, sparse_by_id)
+        _flush_stream_batches(col, points, batch_size)
+        return len(points)
+
     def delete_documents(
         self,
         document_ids: Optional[List[str]] = None,
@@ -545,6 +672,21 @@ class VelesDBDocumentStore:
         int_ids = [stable_hash_id(did) for did in document_ids]
         self._get_collection().delete(int_ids)
 
+    def flush(self) -> None:
+        """Flush pending changes to disk.
+
+        Mirrors ``VelesDBVectorStore.flush()`` in the LangChain and
+        LlamaIndex integrations. Note this does **not** guarantee
+        read-your-writes visibility for :meth:`stream_insert` /
+        :meth:`write_documents_streaming`: those go through a bounded async
+        channel with its own background flush interval (engine default
+        ~50ms) that is independent of this disk flush. A point inserted via
+        the streaming channel may only become visible to reads after that
+        interval elapses.
+        """
+        if self._collection is not None:
+            self._collection.flush()
+
     def embedding_retrieval(
         self,
         query_embedding: List[float],

{haystack_velesdb-3.12.0 → haystack_velesdb-4.0.0/src/haystack_velesdb.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-velesdb
-Version: 3.12.0
+Version: 4.0.0
 Summary: Haystack 2.x DocumentStore for VelesDB: The Local AI Memory Database.
 Author-email: VelesDB Team <contact@wiscale.fr>
 License: MIT
@@ -29,7 +29,9 @@ Dynamic: license-file
 # haystack-velesdb
 
 A Haystack 2.x `DocumentStore` backed by [VelesDB](https://github.com/cyberlife-coder/VelesDB) —
-the local-first, microsecond-latency vector database.
+**the explainable, local-first memory engine for AI agents** (microsecond vector
+search is the proof, not the pitch). For the connected `why()` recall trail
+across typed links, see [velesdb-memory](../../crates/velesdb-memory/README.md).
 
 This integration joins the existing [LangChain](../langchain/) and [LlamaIndex](../llamaindex/)
 connectors, completing the trio of major Python RAG frameworks supported by VelesDB.
@@ -102,26 +104,15 @@ indexer.connect("embedder", "writer")
 indexer.run({"converter": {"sources": ["paper.pdf"]}})
 
 # Query pipeline. `InMemoryEmbeddingRetriever` is bound to `InMemoryDocumentStore`
-# and would NOT work against a custom DocumentStore — wrap `embedding_retrieval`
-# in a thin Haystack component that forwards the call. Full working example in
-# `integrations/haystack/examples/rag_pipeline.py` (`_VelesRetriever`).
-from haystack import component
-from haystack.dataclasses import Document
-from typing import List
-
-@component
-class VelesRetriever:
-    def __init__(self, document_store, top_k: int = 10):
-        self._store = document_store
-        self._top_k = top_k
-
-    @component.output_types(documents=List[Document])
-    def run(self, query_embedding: List[float]):
-        return {"documents": self._store.embedding_retrieval(query_embedding, top_k=self._top_k)}
+# and would NOT work against a custom DocumentStore — use the shipped
+# `VelesDBEmbeddingRetriever` instead (like `QdrantEmbeddingRetriever` in the
+# Qdrant integration, no hand-rolled `@component` wrapper needed). Full working
+# example in `integrations/haystack/examples/rag_pipeline.py`.
+from haystack_velesdb import VelesDBEmbeddingRetriever
 
 querier = Pipeline()
 querier.add_component("embedder", SentenceTransformersTextEmbedder(model="all-MiniLM-L6-v2"))
-querier.add_component("retriever", VelesRetriever(document_store=store))
+querier.add_component("retriever", VelesDBEmbeddingRetriever(document_store=store))
 querier.connect("embedder.embedding", "retriever.query_embedding")
 result = querier.run({"embedder": {"text": "What is VelesDB?"}})
 print(result["retriever"]["documents"])
@@ -137,16 +128,25 @@ print(result["retriever"]["documents"])
 | `collection_name` | `"haystack_documents"` | VelesDB collection name |
 | `embedding_dim` | `768` | Embedding vector dimension |
 | `metric` | `"cosine"` | Distance metric: `"cosine"`, `"euclidean"`, `"dot"`, `"hamming"`, or `"jaccard"` |
+| `scroll_limit` | `10_000` | Max documents returned by `filter_documents()`; raise for collections bigger than this |
 
 ### Methods
 
 | Method | Description |
 |--------|-------------|
-| `write_documents(documents, policy)` | Upsert documents; returns count written |
-| `filter_documents(filters)` | Scroll documents matching a VelesDB filter dict |
-| `embedding_retrieval(query_embedding, top_k, filters, scale_score)` | Vector similarity search |
+| `write_documents(documents, policy, sparse_vectors=None)` | Upsert documents; returns count written. `sparse_vectors` is an optional list aligned with `documents` — each entry a flat `dict[int, float]` or a named `dict[str, dict[int, float]]` mapping (e.g. `{"bge_m3": {0: 1.5}}`); a named mapping creates that sparse index for later hybrid retrieval |
+| `stream_insert(documents, sparse_vectors=None)` | Insert documents through VelesDB's streaming ingestion channel in one call (append-only, no `DuplicatePolicy` check); returns count inserted. See "Note on streaming ingestion" below |
+| `write_documents_streaming(documents, policy, sparse_vectors=None, batch_size=100)` | Same `DuplicatePolicy` semantics as `write_documents`, but sent through the streaming channel in `batch_size`-sized chunks — better throughput for large bulk loads. See "Note on streaming ingestion" below |
+| `flush()` | Flush pending changes to disk. See "Note on streaming ingestion" below |
+| `filter_documents(filters)` | Scroll documents matching a VelesDB filter dict, up to `scroll_limit` |
+| `embedding_retrieval(query_embedding, top_k, filters, scale_score, fusion=None, fusion_params=None)` | Vector similarity search. `fusion` selects a `velesdb.FusionStrategy` (`"average"`, `"maximum"`, `"rrf"`, `"weighted"`, `"relative_score"` / `"rsf"`) applied via `Collection.multi_query_search`, changing the ranking; `fusion_params` configures it (e.g. `{"dense_weight": 0.7, "sparse_weight": 0.3}` for `"rsf"`). `filters` cannot be combined with `fusion` |
 | `count_documents()` | Total document count |
 | `delete_documents(document_ids)` | Delete by Haystack string IDs |
+| `train_pq(m=8, k=256, opq=False)` | Train Product Quantization on the collection |
+| `analyze_collection()` | Compute and persist collection statistics (point/row counts, column stats) |
+| `get_collection_stats()` | Return cached statistics from the last `analyze_collection()` call, or `None` |
+| `is_metadata_only()` | Whether the current collection holds no vectors |
+| `create_metadata_collection(name)` | Create a metadata-only collection (no vectors), joinable with vector collections |
 | `to_dict()` / `from_dict()` | Haystack pipeline serialisation |
 
 **Note on `DuplicatePolicy`:** `NONE` and `OVERWRITE` use VelesDB upsert semantics
@@ -164,6 +164,16 @@ if a collision is detected between a new document and an existing one.
 are normalised from `[-1, 1]` to `[0, 1]` so they behave like probabilities
 in downstream re-ranking.
 
+**Note on streaming ingestion:** `stream_insert()` and `write_documents_streaming()`
+forward to the underlying `velesdb.Collection.stream_insert`, which requires
+the collection to have `enable_streaming()` called on it first — the same
+caller-managed contract as the LangChain and LlamaIndex integrations (see
+`docs/reference/ECOSYSTEM_PARITY.md`). The channel batches asynchronously on
+its own background interval (engine default ~50ms), so a point may not be
+immediately visible to reads right after either call returns; `flush()`
+flushes pending changes to disk but does **not** guarantee streaming-channel
+visibility.
+
 ## Running tests
 
 ```bash

{haystack_velesdb-3.12.0 → haystack_velesdb-4.0.0}/tests/test_document_store.py

@@ -221,6 +221,20 @@ class _FakeCollection:
     def count(self) -> int:
         return len(self._points)
 
+    def stream_insert(self, points: list) -> None:
+        """Fake streaming ingestion channel — mirrors ``upsert`` semantics
+        (the real SDK forwards through a bounded channel with backpressure;
+        this fake is synchronous and deterministic for tests).
+        """
+        for p in points:
+            self._points[p["id"]] = p
+
+    def is_metadata_only(self) -> bool:
+        return False
+
+    def flush(self) -> None:
+        pass
+
 
 class _FakeDatabase:
     def __init__(self, path: str) -> None:
@@ -293,6 +307,20 @@ def _load_module() -> types.ModuleType:
     sys.modules["velesdb_common.ids"] = vc_ids
     _ids_spec.loader.exec_module(vc_ids)
 
+    # Load the REAL velesdb_common.collection_admin (pure stdlib, same reasoning
+    # as ids.py above) so the store exercises the canonical CollectionAdminMixin
+    # rather than a forked copy — single-source-of-truth + license hygiene.
+    _admin_path = (
+        Path(__file__).resolve().parents[2] / "common" / "src" / "velesdb_common" / "collection_admin.py"
+    )
+    _admin_spec = importlib.util.spec_from_file_location(
+        "velesdb_common.collection_admin", _admin_path
+    )
+    assert _admin_spec and _admin_spec.loader
+    vc_admin = importlib.util.module_from_spec(_admin_spec)
+    sys.modules["velesdb_common.collection_admin"] = vc_admin
+    _admin_spec.loader.exec_module(vc_admin)
+
     vc_sec = types.ModuleType("velesdb_common.security")
     vc_sec.validate_path = _passthrough  # type: ignore[attr-defined]
     vc_sec.validate_collection_name = _passthrough  # type: ignore[attr-defined]
@@ -1052,3 +1080,237 @@ def test_id_hashing_uses_canonical_stable_hash_id():
         )
         assert _MOD.stable_hash_id(doc_id) == expected
         assert 0 <= _MOD.stable_hash_id(doc_id) <= 0x7FFFFFFFFFFFFFFF
+
+
+# ---------------------------------------------------------------------------
+# Streaming ingestion — stream_insert() / write_documents_streaming()
+#
+# Mirrors the pattern already shipped by langchain_velesdb.stream_insert /
+# add_texts_streaming and llamaindex_velesdb.stream_insert / add_streaming:
+# both forward to Collection.stream_insert and are tested with mocked
+# collections that record the points they receive (issue #1548).
+# ---------------------------------------------------------------------------
+
+
+def test_stream_insert_calls_collection_stream_insert() -> None:
+    """stream_insert must build points and forward them in one call to
+    Collection.stream_insert, returning the number of points inserted."""
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stream")
+    inserted_points: List[dict] = []
+
+    class _MockCollection:
+        def stream_insert(self, points: list) -> None:
+            inserted_points.extend(points)
+
+    store._collection = _MockCollection()
+    docs = [
+        Document(id="a", content="hello", embedding=[0.1, 0.2]),
+        Document(id="b", content="world", embedding=[0.3, 0.4]),
+    ]
+    count = store.stream_insert(docs)
+
+    assert count == 2
+    assert len(inserted_points) == 2
+    assert inserted_points[0]["payload"]["_doc_id"] == "a"
+    assert inserted_points[0]["payload"]["content"] == "hello"
+    assert inserted_points[0]["vector"] == [0.1, 0.2]
+
+
+def test_stream_insert_empty_list_returns_zero() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stream_empty")
+    assert store.stream_insert([]) == 0
+
+
+def test_stream_insert_forwards_named_sparse_vectors() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stream_sparse")
+    inserted_points: List[dict] = []
+
+    class _MockCollection:
+        def stream_insert(self, points: list) -> None:
+            inserted_points.extend(points)
+
+    store._collection = _MockCollection()
+    store.stream_insert(
+        [Document(id="s1", content="hi", embedding=[0.5])],
+        sparse_vectors=[{"bge_m3": {0: 1.5, 7: 0.8}}],
+    )
+    assert inserted_points[0]["sparse_vector"] == {"bge_m3": {0: 1.5, 7: 0.8}}
+
+
+def test_write_documents_streaming_batches_via_stream_insert() -> None:
+    """write_documents_streaming must chunk points into batch_size-sized
+    calls to Collection.stream_insert (backpressure-friendly bulk loading)."""
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stream_batches")
+    call_batches: List[list] = []
+
+    class _MockCollection:
+        def stream_insert(self, points: list) -> None:
+            call_batches.append(list(points))
+
+    store._collection = _MockCollection()
+    docs = [
+        Document(id=f"d{i}", content=f"text {i}", embedding=[float(i)])
+        for i in range(5)
+    ]
+    written = store.write_documents_streaming(docs, batch_size=2)
+
+    assert written == 5
+    assert len(call_batches) == 3  # 2 + 2 + 1
+    assert [len(b) for b in call_batches] == [2, 2, 1]
+    assert sum(len(b) for b in call_batches) == 5
+
+
+def test_write_documents_streaming_empty_returns_zero() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stream_batches_empty")
+    assert store.write_documents_streaming([]) == 0
+
+
+def test_write_documents_streaming_skip_policy_does_not_overwrite_existing() -> None:
+    """write_documents_streaming must honour DuplicatePolicy.SKIP exactly like
+    write_documents: existing documents are left untouched."""
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stream_skip")
+    original = Document(id="dup", content="original content", embedding=[0.1, 0.2])
+    store.write_documents([original])
+
+    new = Document(id="dup", content="REPLACED CONTENT", embedding=[0.9, 0.9])
+    fresh = Document(id="brand_new", content="fresh", embedding=[0.5, 0.5])
+    written = store.write_documents_streaming([new, fresh], policy=DuplicatePolicy.SKIP)
+
+    assert written == 1, "SKIP should report only the genuinely-new doc as written"
+    docs = {d.id: d for d in store.filter_documents()}
+    assert docs["dup"].content == "original content"
+    assert docs["brand_new"].content == "fresh"
+
+
+def test_write_documents_streaming_forwards_sparse_vectors() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stream_sparse_batch")
+    call_batches: List[list] = []
+
+    class _MockCollection:
+        def stream_insert(self, points: list) -> None:
+            call_batches.append(list(points))
+
+    store._collection = _MockCollection()
+    store.write_documents_streaming(
+        [Document(id="p1", content="x", embedding=[0.5])],
+        sparse_vectors=[{"bge_m3": {0: 1.5}}],
+    )
+    assert call_batches[0][0]["sparse_vector"] == {"bge_m3": {0: 1.5}}
+
+
+# ---------------------------------------------------------------------------
+# CollectionAdminMixin — get_stats / analyze / train_pq / metadata-only
+#
+# Mirrors the mixin already shared with the LangChain and LlamaIndex stores
+# (velesdb_common.collection_admin.CollectionAdminMixin), now also mixed
+# into VelesDBDocumentStore so all three integrations expose the same
+# admin surface (issue #1548).
+# ---------------------------------------------------------------------------
+
+
+def test_collection_name_property_exposes_constructor_arg() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="my_collection")
+    assert store.collection_name == "my_collection"
+
+
+def test_train_pq_calls_db_train_pq() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_pq")
+    calls: List[dict] = []
+
+    class _MockDb:
+        def train_pq(self, collection_name: str, m: int, k: int, opq: bool) -> str:
+            calls.append({"collection_name": collection_name, "m": m, "k": k, "opq": opq})
+            return "trained"
+
+    store._db = _MockDb()
+    result = store.train_pq(m=16, k=128, opq=True)
+
+    assert result == "trained"
+    assert calls == [{"collection_name": "t_pq", "m": 16, "k": 128, "opq": True}]
+
+
+def test_analyze_collection_calls_db_analyze_collection() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_analyze")
+    calls: List[str] = []
+
+    class _MockDb:
+        def analyze_collection(self, collection_name: str) -> dict:
+            calls.append(collection_name)
+            return {"total_points": 3, "row_count": 3}
+
+    store._db = _MockDb()
+    result = store.analyze_collection()
+
+    assert result == {"total_points": 3, "row_count": 3}
+    assert calls == ["t_analyze"]
+
+
+def test_get_collection_stats_calls_db_get_collection_stats() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_stats")
+    calls: List[str] = []
+
+    class _MockDb:
+        def get_collection_stats(self, collection_name: str) -> Optional[dict]:
+            calls.append(collection_name)
+            return None
+
+    store._db = _MockDb()
+    assert store.get_collection_stats() is None
+    assert calls == ["t_stats"]
+
+
+def test_create_metadata_collection_calls_db_create_metadata_collection() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_meta_coll")
+    calls: List[str] = []
+
+    class _MockDb:
+        def create_metadata_collection(self, name: str) -> None:
+            calls.append(name)
+
+    store._db = _MockDb()
+    store.create_metadata_collection("refs")
+    assert calls == ["refs"]
+
+
+def test_is_metadata_only_false_when_no_collection_yet() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_meta_only_none")
+    assert store.is_metadata_only() is False
+
+
+def test_is_metadata_only_delegates_to_collection() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_meta_only")
+
+    class _MockCollection:
+        def is_metadata_only(self) -> bool:
+            return True
+
+    store._collection = _MockCollection()
+    assert store.is_metadata_only() is True
+
+
+def test_flush_delegates_to_collection() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_flush")
+    calls: List[bool] = []
+
+    class _MockCollection:
+        def flush(self) -> None:
+            calls.append(True)
+
+    store._collection = _MockCollection()
+    store.flush()
+    assert calls == [True]
+
+
+def test_flush_is_noop_without_collection() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_flush_none")
+    store.flush()  # must not raise
+
+
+def test_get_db_reuses_existing_connection() -> None:
+    """_get_db() must return the cached ``self._db`` instead of opening a new
+    ``velesdb.Database`` when one is already set (same contract LangChain and
+    LlamaIndex rely on for the CollectionAdminMixin)."""
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_get_db")
+    sentinel = object()
+    store._db = sentinel
+    assert store._get_db() is sentinel

{haystack_velesdb-3.12.0 → haystack_velesdb-4.0.0}/tests/test_real_haystack_integration.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 
 import importlib
 import sys
+import time
 from typing import List
 
 import pytest
@@ -136,6 +137,23 @@ def _store(tmp_path) -> VelesDBDocumentStore:
     )
 
 
+def _wait_until(predicate, timeout_s: float = 2.0, interval_s: float = 0.02):
+    """Poll *predicate* until it returns truthy or *timeout_s* elapses.
+
+    The real streaming ingestion channel flushes asynchronously on its own
+    background interval (engine default ~50ms), independent of
+    ``DocumentStore.flush()``. Tests that exercise ``stream_insert`` /
+    ``write_documents_streaming`` need to wait for that eventual visibility
+    instead of asserting immediately after the call returns.
+    """
+    deadline = time.monotonic() + timeout_s
+    while time.monotonic() < deadline:
+        if predicate():
+            return True
+        time.sleep(interval_s)
+    return predicate()
+
+
 def _docs() -> List[Document]:
     return [
         Document(
@@ -245,14 +263,95 @@ def test_duplicate_policy_skip_with_real_haystack(tmp_path) -> None:
 
 
 # ---------------------------------------------------------------------------
-# Pipeline integration — exercises the @component decorator on VelesRetriever
+# Streaming ingestion — real velesdb Collection.stream_insert (issue #1548)
+#
+# The real binding requires ``Collection.enable_streaming()`` before
+# ``stream_insert`` will accept points (it raises "streaming is not
+# configured on this collection" otherwise) — same caller-managed contract
+# already documented for LangChain/LlamaIndex in ECOSYSTEM_PARITY.md.
+# ---------------------------------------------------------------------------
+
+
+def test_stream_insert_round_trips_via_real_velesdb(tmp_path) -> None:
+    """The real streaming channel is async (bounded queue, own background
+    flush interval, default ~50ms) — poll for eventual visibility instead of
+    asserting immediately after ``stream_insert`` returns."""
+    store = _store(tmp_path)
+    store._get_collection().enable_streaming()
+
+    count = store.stream_insert(_docs())
+    assert count == 3
+
+    assert _wait_until(lambda: store.count_documents() == 3)
+    assert {d.id for d in store.filter_documents()} == {
+        "doc-en-1", "doc-en-2", "doc-fr-1",
+    }
+
+
+def test_write_documents_streaming_round_trips_via_real_velesdb(tmp_path) -> None:
+    store = _store(tmp_path)
+    store._get_collection().enable_streaming()
+
+    written = store.write_documents_streaming(_docs(), batch_size=2)
+    assert written == 3
+
+    assert _wait_until(lambda: store.count_documents() == 3)
+    # count() and vector-searchability can land at slightly different times
+    # (HNSW indexing trails the raw point store) — poll the search itself.
+    results_holder: dict = {}
+
+    def _search_ready() -> bool:
+        found = store.embedding_retrieval(
+            query_embedding=[1.0, 0.0, 0.0, 0.0], top_k=1
+        )
+        if found:
+            results_holder["docs"] = found
+        return bool(found)
+
+    assert _wait_until(_search_ready)
+    assert results_holder["docs"][0].id == "doc-en-1"
+
+
+# ---------------------------------------------------------------------------
+# CollectionAdminMixin — real velesdb Database.train_pq / analyze_collection
+# (issue #1548)
+# ---------------------------------------------------------------------------
+
+
+def test_analyze_and_get_collection_stats_via_real_velesdb(tmp_path) -> None:
+    store = _store(tmp_path)
+    store.write_documents(_docs())
+
+    stats = store.analyze_collection()
+    assert stats["total_points"] == 3
+
+    cached = store.get_collection_stats()
+    assert cached is not None
+    assert cached["total_points"] == 3
+
+
+def test_is_metadata_only_false_for_vector_collection_via_real_velesdb(tmp_path) -> None:
+    store = _store(tmp_path)
+    store.write_documents(_docs())
+    assert store.is_metadata_only() is False
+
+
+# ---------------------------------------------------------------------------
+# Pipeline integration — exercises the @component decorator contract
 # ---------------------------------------------------------------------------
 
 
 @component
 class VelesRetriever:
-    """Mirror of the canonical pattern from the README. The decorator is
-    REQUIRED — without it, ``Pipeline.add_component`` raises.
+    """A hand-rolled ``@component`` wrapper around ``embedding_retrieval``.
+
+    The README and ``examples/rag_pipeline.py`` now use the shipped
+    ``VelesDBEmbeddingRetriever`` instead (see
+    ``test_shipped_embedding_retriever_component_runs`` below); this class
+    stays only to exercise the generic Haystack 2.x ``@component`` contract
+    for callers who need custom retrieval logic beyond what the shipped
+    component offers. The decorator is REQUIRED — without it,
+    ``Pipeline.add_component`` raises.
     """
 
     def __init__(self, document_store: VelesDBDocumentStore, top_k: int = 5) -> None:
@@ -269,11 +368,10 @@ class VelesRetriever:
 
 
 def test_pipeline_with_decorated_retriever(tmp_path) -> None:
-    """A real Haystack Pipeline accepts the decorated VelesRetriever and runs.
+    """A real Haystack Pipeline accepts a custom decorated component and runs.
 
     Without the @component decorator, ``add_component`` raises
-    ``PipelineError`` — this test would fail at construction. Acts as
-    the canary for the rag_pipeline.py example fix.
+    ``PipelineError`` — this test would fail at construction.
     """
     store = _store(tmp_path)
     store.write_documents(_docs())