haystack-velesdb 1.14.1__py3-none-any.whl

haystack_velesdb/__init__.py

@@ -0,0 +1,6 @@
+"""Haystack 2.x DocumentStore integration for VelesDB."""
+
+from haystack_velesdb.document_store import VelesDBDocumentStore
+
+__all__ = ["VelesDBDocumentStore"]
+__version__ = "1.0.0"

haystack_velesdb/document_store.py

@@ -0,0 +1,350 @@
+"""Haystack 2.x DocumentStore backed by VelesDB.
+
+Implements the Haystack ``DocumentStore`` protocol so VelesDB can be used
+as the vector backend in any Haystack 2.x indexing or retrieval pipeline.
+"""
+from __future__ import annotations
+
+import hashlib
+import logging
+from typing import Any, Dict, List, Optional
+
+from haystack import default_from_dict, default_to_dict
+from haystack.dataclasses import Document
+from haystack.document_stores.errors import DuplicateDocumentError
+from haystack.document_stores.types import DuplicatePolicy
+
+import velesdb
+from velesdb_common.security import (
+    validate_collection_name,
+    validate_metric,
+    validate_path,
+)
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["VelesDBDocumentStore"]
+
+_DEFAULT_COLLECTION = "haystack_documents"
+_DEFAULT_DIMENSION = 768
+_DEFAULT_METRIC = "cosine"
+_DEFAULT_SCROLL_LIMIT = 10_000
+_INT63_MASK = (1 << 63) - 1
+# Reserved keys stored by this integration in the VelesDB payload.
+_RESERVED_PAYLOAD_KEYS = frozenset({"_doc_id", "content"})
+
+
+def _str_id_to_int(doc_id: str) -> int:
+    """Map a Haystack string document ID to a stable positive 63-bit integer.
+
+    Uses the first 8 bytes of SHA-256, masked to 63 bits (~9.2 × 10¹⁸ slots).
+    Collision probability for a 1 M-document collection is roughly 5 × 10⁻¹⁴ —
+    negligible for typical RAG workloads but not zero.  If two distinct string
+    IDs produce the same integer ID, :meth:`write_documents` raises
+    :class:`ValueError` rather than silently overwriting the existing document.
+    """
+    return int.from_bytes(hashlib.sha256(doc_id.encode()).digest()[:8], "big") & _INT63_MASK
+
+
+def _doc_to_point(doc: Document) -> dict:
+    """Convert a Haystack Document to a VelesDB point dict.
+
+    Reserved payload keys (``_doc_id``, ``content``) are always written from
+    the document's canonical fields, not from ``doc.meta``.  Any meta entry
+    that shares a reserved name is silently dropped from the payload to
+    prevent round-trip corruption.
+    """
+    payload: dict = {}
+    # Merge meta first; reserved keys are excluded so they cannot
+    # clobber the canonical doc identity written below.
+    if doc.meta:
+        for k, v in doc.meta.items():
+            if k not in _RESERVED_PAYLOAD_KEYS:
+                payload[k] = v
+    payload["_doc_id"] = doc.id
+    if doc.content is not None:
+        payload["content"] = doc.content
+    point: dict = {"id": _str_id_to_int(doc.id), "payload": payload}
+    if doc.embedding is not None:
+        point["vector"] = list(doc.embedding)
+    return point
+
+
+def _result_to_doc(
+    result: dict, *, scale_score: bool = False, metric: str = "cosine"
+) -> Document:
+    """Convert a VelesDB search or scroll result to a Haystack Document.
+
+    Requires ``_doc_id`` to be present in the payload. Points written by
+    :meth:`VelesDBDocumentStore.write_documents` always carry that key, so
+    a missing ``_doc_id`` means the underlying VelesDB collection was
+    populated by a different code path (raw ``col.upsert``, migration
+    scripts, mixed tooling). Falling back to the stringified integer ID
+    would silently corrupt :meth:`delete_documents`: the integer-as-string
+    re-hashes through SHA-256 to a *different* integer, so the delete
+    would no-op without raising. We fail fast instead.
+
+    Raises:
+        ValueError: When ``_doc_id`` is missing from the payload.
+    """
+    payload = result.get("payload", {})
+    doc_id = payload.get("_doc_id")
+    if doc_id is None:
+        raise ValueError(
+            f"VelesDB point id={result.get('id')} has no '_doc_id' field in "
+            "its payload. VelesDBDocumentStore requires every point in the "
+            "underlying collection to be written via write_documents(); "
+            "points populated by raw col.upsert() or external migration "
+            "scripts cannot be round-tripped because the stringified "
+            "integer ID would re-hash to a different integer and break "
+            "delete_documents()."
+        )
+    content = payload.get("content")
+    meta = {k: v for k, v in payload.items() if k not in _RESERVED_PAYLOAD_KEYS}
+    raw_score: Optional[float] = result.get("score")
+    if scale_score and raw_score is not None and metric == "cosine":
+        # Normalise cosine similarity from [-1, 1] to [0, 1].
+        # Only meaningful for cosine; l2 and dot scores have different ranges.
+        score: Optional[float] = (raw_score + 1.0) / 2.0
+    else:
+        score = raw_score
+    return Document(id=doc_id, content=content, meta=meta, score=score)
+
+
+def _build_int_id_map(documents: List[Document]) -> Dict[int, str]:
+    """Map every document's integer ID back to its string ID, raising on
+    in-batch SHA-256 collisions.
+
+    Two distinct string IDs that hash to the same 63-bit integer would
+    silently overwrite each other on upsert. This helper is the first
+    line of defence: it detects collisions inside a single
+    ``write_documents`` batch before any state hits the collection.
+    """
+    int_id_map: Dict[int, str] = {}
+    for doc in documents:
+        iid = _str_id_to_int(doc.id)
+        if iid in int_id_map and int_id_map[iid] != doc.id:
+            raise ValueError(
+                f"SHA-256 collision in write batch: '{int_id_map[iid]}' and "
+                f"'{doc.id}' map to the same integer ID {iid}. "
+                "Rename one of the documents."
+            )
+        int_id_map[iid] = doc.id
+    return int_id_map
+
+
+def _enforce_fail_policy(col: Any, int_id_map: Dict[int, str]) -> None:
+    """For ``DuplicatePolicy.FAIL``, raise if any incoming integer ID
+    already exists in the collection, or if a stored point points to a
+    different string ID (cross-store SHA-256 collision).
+
+    Uses point-by-point ``col.get(int_ids)`` — O(batch_size) — instead of
+    a full scroll, so collections larger than ``scroll_limit`` are still
+    correctly enforced.
+    """
+    existing_points: List[Any] = col.get(list(int_id_map.keys()))
+    conflicts: List[str] = []
+    for point in existing_points:
+        if point is None:
+            continue
+        iid = point["id"]
+        existing_str = point.get("payload", {}).get("_doc_id", str(iid))
+        str_id = int_id_map[iid]
+        if existing_str != str_id:
+            raise ValueError(
+                f"SHA-256 collision on write: incoming document '{str_id}' "
+                f"maps to the same integer ID {iid} as existing document "
+                f"'{existing_str}'. Rename one of the documents."
+            )
+        conflicts.append(str_id)
+    if conflicts:
+        raise DuplicateDocumentError(
+            f"Documents already exist (policy=FAIL): {conflicts}"
+        )
+
+
+def _documents_to_points(documents: List[Document]) -> List[dict]:
+    """Convert each document to its VelesDB point dict, logging documents
+    that lack an embedding so the caller still gets feedback when the
+    underlying SDK accepts vector-less points.
+    """
+    points: List[dict] = []
+    for doc in documents:
+        if doc.embedding is None:
+            logger.warning(
+                "Document '%s' has no embedding; stored without vector.", doc.id
+            )
+        points.append(_doc_to_point(doc))
+    return points
+
+
+class VelesDBDocumentStore:
+    """Haystack 2.x DocumentStore backed by a local VelesDB collection.
+
+    Stores documents (with optional embeddings) in VelesDB and exposes the
+    standard Haystack retrieval interface so this store works as a drop-in
+    backend for ``EmbeddingRetriever`` and similar pipeline components.
+
+    Args:
+        path: Directory path where VelesDB persists data.
+        collection_name: Name of the VelesDB collection to use.
+        embedding_dim: Dimensionality of the embedding vectors.
+        metric: Distance metric: ``"cosine"``, ``"euclidean"``, or ``"dot"``.
+        scroll_limit: Maximum documents returned by :meth:`filter_documents`.
+            Increase this value when your collection exceeds 10 000 documents.
+    """
+
+    def __init__(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+        self,
+        path: str = "./velesdb_haystack",
+        collection_name: str = _DEFAULT_COLLECTION,
+        embedding_dim: int = _DEFAULT_DIMENSION,
+        metric: str = _DEFAULT_METRIC,
+        scroll_limit: int = _DEFAULT_SCROLL_LIMIT,
+    ) -> None:
+        self._path = validate_path(path)
+        self._collection_name = validate_collection_name(collection_name)
+        self._embedding_dim = embedding_dim
+        self._metric = validate_metric(metric)
+        self._scroll_limit = scroll_limit
+        self._db: Optional[Any] = None
+        self._collection: Optional[Any] = None
+
+    # ------------------------------------------------------------------
+    # Internal connection management
+    # ------------------------------------------------------------------
+
+    def _get_collection(self) -> Any:
+        """Return the VelesDB collection, opening or creating it as needed."""
+        if self._db is None:
+            self._db = velesdb.Database(self._path)
+        if self._collection is None:
+            col: Optional[Any] = None
+            try:
+                col = self._db.get_collection(self._collection_name)
+            except KeyError:
+                pass
+            if col is None:
+                col = self._db.create_collection(
+                    self._collection_name,
+                    dimension=self._embedding_dim,
+                    metric=self._metric,
+                )
+            self._collection = col
+        return self._collection
+
+    # ------------------------------------------------------------------
+    # DocumentStore protocol
+    # ------------------------------------------------------------------
+
+    def count_documents(self) -> int:
+        """Return the total number of documents in the store."""
+        result = self._get_collection().count()
+        return result if isinstance(result, int) else 0
+
+    def filter_documents(
+        self,
+        filters: Optional[Dict[str, Any]] = None,
+    ) -> List[Document]:
+        """Return documents matching *filters*, or all documents when *None*.
+
+        Passes *filters* directly to VelesDB's scroll operation. The real
+        SDK returns ``Iterator[List[Dict]]`` and has no ``limit`` kwarg, so
+        we drive the iterator ourselves and stop once ``self._scroll_limit``
+        documents have been collected. Increase ``scroll_limit`` on the
+        constructor for collections larger than the default 10 000.
+        """
+        col = self._get_collection()
+        documents: List[Document] = []
+        for batch in col.scroll(filter=filters):
+            for raw in batch:
+                if len(documents) >= self._scroll_limit:
+                    return documents
+                documents.append(_result_to_doc(raw))
+        return documents
+
+    def write_documents(
+        self,
+        documents: List[Document],
+        policy: DuplicatePolicy = DuplicatePolicy.NONE,
+    ) -> int:
+        """Write *documents* to VelesDB and return the number written.
+
+        VelesDB upsert semantics apply for policies other than ``FAIL``:
+        an existing point with the same integer ID is overwritten.
+
+        When *policy* is ``DuplicatePolicy.FAIL`` this method scans the
+        collection before writing and raises :class:`DuplicateDocumentError`
+        if any incoming document already exists.  For large collections
+        prefer ``OVERWRITE`` or ``NONE`` to avoid the pre-scan cost.
+
+        Raises:
+            DuplicateDocumentError: When *policy* is ``FAIL`` and at least
+                one document already exists in the store.
+            ValueError: When a SHA-256 hash collision is detected — two
+                distinct string IDs that map to the same integer ID.
+        """
+        if not documents:
+            return 0
+        int_id_map = _build_int_id_map(documents)
+        col = self._get_collection()
+        if policy == DuplicatePolicy.FAIL:
+            _enforce_fail_policy(col, int_id_map)
+        points = _documents_to_points(documents)
+        result = col.upsert(points)
+        return result if isinstance(result, int) else len(points)
+
+    def delete_documents(
+        self,
+        document_ids: Optional[List[str]] = None,
+    ) -> None:
+        """Delete documents identified by their Haystack string IDs."""
+        if not document_ids:
+            return
+        int_ids = [_str_id_to_int(did) for did in document_ids]
+        self._get_collection().delete(int_ids)
+
+    def embedding_retrieval(
+        self,
+        query_embedding: List[float],
+        *,
+        top_k: int = 10,
+        filters: Optional[Dict[str, Any]] = None,
+        scale_score: bool = True,
+    ) -> List[Document]:
+        """Return the *top_k* documents most similar to *query_embedding*.
+
+        Args:
+            query_embedding: Dense query vector.
+            top_k: Maximum number of documents to return.
+            filters: Optional VelesDB filter dict to restrict the search space.
+            scale_score: When ``True`` and ``metric="cosine"``, scores are
+                normalised from ``[-1, 1]`` to ``[0, 1]``. Ignored for other
+                metrics, where raw scores are returned unchanged.
+        """
+        results: List[dict] = self._get_collection().search(
+            vector=query_embedding,
+            top_k=top_k,
+            filter=filters,
+        )
+        return [_result_to_doc(r, scale_score=scale_score, metric=self._metric) for r in results]
+
+    # ------------------------------------------------------------------
+    # Haystack pipeline serialisation
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialise the store configuration for Haystack pipeline YAML."""
+        return default_to_dict(
+            self,
+            path=self._path,
+            collection_name=self._collection_name,
+            embedding_dim=self._embedding_dim,
+            metric=self._metric,
+            scroll_limit=self._scroll_limit,
+        )
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "VelesDBDocumentStore":
+        """Restore a store instance from a Haystack pipeline config dict."""
+        return default_from_dict(cls, data)

haystack_velesdb-1.14.1.dist-info/METADATA

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: haystack-velesdb
+Version: 1.14.1
+Summary: Haystack 2.x DocumentStore for VelesDB: The Local AI Memory Database.
+Author-email: VelesDB Team <contact@wiscale.fr>
+License: MIT
+Project-URL: Homepage, https://github.com/cyberlife-coder/VelesDB
+Project-URL: Documentation, https://velesdb.com/docs/integrations/haystack
+Project-URL: Repository, https://github.com/cyberlife-coder/VelesDB
+Keywords: haystack,velesdb,vector-database,embeddings,rag,local-first,semantic-search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: haystack-ai>=2.0.0
+Requires-Dist: velesdb>=1.13.2
+Requires-Dist: velesdb-common>=1.13.2
+Provides-Extra: dev
+Requires-Dist: pytest<9.0,>=7.0; extra == "dev"
+Dynamic: license-file
+
+# haystack-velesdb
+
+A Haystack 2.x `DocumentStore` backed by [VelesDB](https://github.com/cyberlife-coder/VelesDB) —
+the local-first, microsecond-latency vector database.
+
+This integration joins the existing [LangChain](../langchain/) and [LlamaIndex](../llamaindex/)
+connectors, completing the trio of major Python RAG frameworks supported by VelesDB.
+
+## Installation
+
+```bash
+pip install haystack-velesdb
+```
+
+For development:
+
+```bash
+pip install -e "integrations/haystack[dev]"
+```
+
+## Quick start
+
+```python
+from haystack_velesdb import VelesDBDocumentStore
+from haystack.dataclasses import Document
+
+store = VelesDBDocumentStore(
+    path="./my_docs",
+    collection_name="knowledge_base",
+    embedding_dim=768,
+    metric="cosine",
+)
+
+# Write pre-embedded documents
+documents = [
+    Document(id="doc1", content="VelesDB is fast.", embedding=[0.1, 0.2, ...]),
+    Document(id="doc2", content="Local-first AI memory.", embedding=[0.3, 0.4, ...]),
+]
+store.write_documents(documents)
+
+# Retrieve by vector
+results = store.embedding_retrieval(query_embedding=[0.1, 0.2, ...], top_k=5)
+for doc in results:
+    print(doc.content, doc.score)
+```
+
+## Full RAG pipeline
+
+See [`examples/rag_pipeline.py`](examples/rag_pipeline.py) for a complete PDF ingestion
+and semantic search example using `SentenceTransformersDocumentEmbedder`.
+
+```python
+from haystack import Pipeline
+from haystack.components.converters import PyPDFToDocument
+from haystack.components.embedders import (
+    SentenceTransformersDocumentEmbedder,
+    SentenceTransformersTextEmbedder,
+)
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack_velesdb import VelesDBDocumentStore
+
+store = VelesDBDocumentStore(path="./rag_store", embedding_dim=384)
+
+# Indexing pipeline
+indexer = Pipeline()
+indexer.add_component("converter", PyPDFToDocument())
+indexer.add_component("splitter", DocumentSplitter(split_by="sentence", split_length=3))
+indexer.add_component("embedder", SentenceTransformersDocumentEmbedder(model="all-MiniLM-L6-v2"))
+indexer.add_component("writer", DocumentWriter(document_store=store))
+indexer.connect("converter", "splitter")
+indexer.connect("splitter", "embedder")
+indexer.connect("embedder", "writer")
+indexer.run({"converter": {"sources": ["paper.pdf"]}})
+
+# Query pipeline
+from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
+
+querier = Pipeline()
+querier.add_component("embedder", SentenceTransformersTextEmbedder(model="all-MiniLM-L6-v2"))
+querier.add_component("retriever", InMemoryEmbeddingRetriever(document_store=store))
+querier.connect("embedder.embedding", "retriever.query_embedding")
+result = querier.run({"embedder": {"text": "What is VelesDB?"}})
+print(result["retriever"]["documents"])
+```
+
+## API reference
+
+### `VelesDBDocumentStore`
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `path` | `"./velesdb_haystack"` | Directory where VelesDB persists data |
+| `collection_name` | `"haystack_documents"` | VelesDB collection name |
+| `embedding_dim` | `768` | Embedding vector dimension |
+| `metric` | `"cosine"` | Distance metric: `"cosine"`, `"euclidean"`, or `"dot"` |
+
+### Methods
+
+| Method | Description |
+|--------|-------------|
+| `write_documents(documents, policy)` | Upsert documents; returns count written |
+| `filter_documents(filters)` | Scroll documents matching a VelesDB filter dict |
+| `embedding_retrieval(query_embedding, top_k, filters, scale_score)` | Vector similarity search |
+| `count_documents()` | Total document count |
+| `delete_documents(document_ids)` | Delete by Haystack string IDs |
+| `to_dict()` / `from_dict()` | Haystack pipeline serialisation |
+
+**Note on `DuplicatePolicy`:** `NONE` and `OVERWRITE` use VelesDB upsert semantics
+and always overwrite on collision.  `FAIL` is fully enforced: a pre-scan is
+performed before writing and `DuplicateDocumentError` is raised if any document
+already exists (prefer `OVERWRITE` or `NONE` for bulk loads to skip the scan cost).
+
+**Note on document IDs and SHA-256:** Haystack string IDs are mapped to 63-bit
+integers using the first 8 bytes of SHA-256 (~9.2 × 10¹⁸ slots).  For a
+1 M-document collection the collision probability is roughly 5 × 10⁻¹⁴, which
+is negligible for typical RAG workloads.  A `ValueError` is raised at write time
+if a collision is detected between a new document and an existing one.
+
+**Note on `scale_score`:** When `True` (default), cosine similarity scores
+are normalised from `[-1, 1]` to `[0, 1]` so they behave like probabilities
+in downstream re-ranking.
+
+## Running tests
+
+```bash
+cd integrations/haystack
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+Tests use lightweight fake VelesDB objects — no running server required.

haystack_velesdb-1.14.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+haystack_velesdb/__init__.py,sha256=cLohq27QUAJIgaJI6lbULnleSLBsF5bpvdcYSzgHFAU,182
+haystack_velesdb/document_store.py,sha256=osxoJYH7pQIz6ZPGT1xz3eXaaVLB9mRT5fetHJFqLhg,14283
+haystack_velesdb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+haystack_velesdb-1.14.1.dist-info/licenses/LICENSE,sha256=YpLk4nyLjvWm2Ls3Tel-oI1iTgFlNpxJN-6kb5Ck1zY,1069
+haystack_velesdb-1.14.1.dist-info/METADATA,sha256=by3EY-zqOCoagq-K-j_NQh1UFVIZ3E-HOwi6HDAAFsw,6024
+haystack_velesdb-1.14.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+haystack_velesdb-1.14.1.dist-info/top_level.txt,sha256=t5iFDRUDDh5niR6Dc_3gGe-4HnUHITvo6F1hIALXUns,17
+haystack_velesdb-1.14.1.dist-info/RECORD,,

haystack_velesdb-1.14.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

haystack_velesdb-1.14.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 VelesDB Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

haystack_velesdb-1.14.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+haystack_velesdb