ragleap-vectorstores 0.1.0__tar.gz

ragleap_vectorstores-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+# Environment
+.env
+*.env.local
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+venv/
+.venv/
+
+# Node
+node_modules/
+
+# Docker
+docker-compose.override.yml
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+.env
+
+# Package build artifacts
+dist/
+build/

ragleap_vectorstores-0.1.0/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+All notable changes to `ragleap-vectorstores` are documented here. Format
+loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-08-29
+### Added
+- Initial package scaffold - pyproject.toml, package layout matching
+  ragleap-graph's src/ragleap_vectorstores convention.
+- `ChromaBackend` - first vector backend, implementing the full
+  `VectorBackend` interface via chromadb's embedded PersistentClient
+  (no server required). Available via the `chroma` optional extra
+  (`pip install ragleap-vectorstores[chroma]`).
+- `supports_sparse()` correctly reports `False` for Chroma - no native
+  keyword/BM25 search as of chromadb 1.5.9, so hybrid search honestly
+  falls back to dense-only rather than claiming unimplemented capability.

ragleap_vectorstores-0.1.0/PKG-INFO

@@ -0,0 +1,63 @@
+Metadata-Version: 2.5
+Name: ragleap-vectorstores
+Version: 0.1.0
+Summary: Pluggable vector backends beyond ragleap-rag core's 6 (PgVector, FAISS, Pinecone, Weaviate, Qdrant, Milvus) - additional VectorBackend implementations as optional extras, no vendor lock-in, no bloated core dependency footprint.
+Project-URL: Homepage, https://github.com/antonyrag/ragleap-core
+Project-URL: Repository, https://github.com/antonyrag/ragleap-core
+Project-URL: Documentation, https://packages.ragleap.com/docs/ragleap-vectorstores.html
+Project-URL: Issues, https://github.com/antonyrag/ragleap-core/issues
+Author-email: Antony <antony@ragleap.com>
+License-Expression: MIT
+Keywords: rag,ragleap,vector-database,vector-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: ragleap-rag>=0.12.4
+Provides-Extra: chroma
+Requires-Dist: chromadb>=1.0.0; extra == 'chroma'
+Provides-Extra: test
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'test'
+Requires-Dist: pytest>=8.0.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# ragleap-vectorstores
+
+Pluggable vector backends beyond [ragleap-rag](https://pypi.org/project/ragleap-rag/)'s
+built-in 6 (PgVector, FAISS, Pinecone, Weaviate, Qdrant, Milvus).
+
+This package is under active development - more backends will be added
+over time. See the
+[roadmap](https://github.com/antonyrag/ragleap-core/wiki/Roadmap) for status.
+
+## Available backends
+
+| Backend | Extra | Notes |
+|---|---|---|
+| Chroma | `chroma` | Embedded/local via chromadb's PersistentClient - no server required. No native sparse/keyword search (`supports_sparse()` is `False`); hybrid search falls back to dense-only. |
+
+## Design
+
+Every backend here implements ragleap-rag's `VectorBackend` interface, so it
+can be passed directly to `RagLeap(vector_backend=...)`. Each backend's real
+client SDK is an optional extra - installing `ragleap-vectorstores` alone
+pulls in no heavy dependencies beyond `ragleap-rag` itself.
+
+## Install
+
+```bash
+pip install ragleap-vectorstores[chroma]
+```
+
+## Usage
+
+```python
+from ragleap_vectorstores import ChromaBackend
+
+backend = ChromaBackend(persist_directory="./chroma_data")
+```

ragleap_vectorstores-0.1.0/README.md

@@ -0,0 +1,35 @@
+# ragleap-vectorstores
+
+Pluggable vector backends beyond [ragleap-rag](https://pypi.org/project/ragleap-rag/)'s
+built-in 6 (PgVector, FAISS, Pinecone, Weaviate, Qdrant, Milvus).
+
+This package is under active development - more backends will be added
+over time. See the
+[roadmap](https://github.com/antonyrag/ragleap-core/wiki/Roadmap) for status.
+
+## Available backends
+
+| Backend | Extra | Notes |
+|---|---|---|
+| Chroma | `chroma` | Embedded/local via chromadb's PersistentClient - no server required. No native sparse/keyword search (`supports_sparse()` is `False`); hybrid search falls back to dense-only. |
+
+## Design
+
+Every backend here implements ragleap-rag's `VectorBackend` interface, so it
+can be passed directly to `RagLeap(vector_backend=...)`. Each backend's real
+client SDK is an optional extra - installing `ragleap-vectorstores` alone
+pulls in no heavy dependencies beyond `ragleap-rag` itself.
+
+## Install
+
+```bash
+pip install ragleap-vectorstores[chroma]
+```
+
+## Usage
+
+```python
+from ragleap_vectorstores import ChromaBackend
+
+backend = ChromaBackend(persist_directory="./chroma_data")
+```

ragleap_vectorstores-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ragleap-vectorstores"
+version = "0.1.0"
+description = "Pluggable vector backends beyond ragleap-rag core's 6 (PgVector, FAISS, Pinecone, Weaviate, Qdrant, Milvus) - additional VectorBackend implementations as optional extras, no vendor lock-in, no bloated core dependency footprint."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "Antony", email = "antony@ragleap.com" }
+]
+keywords = ["rag", "vector-search", "vector-database", "ragleap"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "ragleap-rag>=0.12.4",
+]
+
+[project.optional-dependencies]
+test = ["pytest>=8.0.0", "pytest-asyncio>=0.24.0"]
+chroma = ["chromadb>=1.0.0"]
+
+[project.urls]
+Homepage = "https://github.com/antonyrag/ragleap-core"
+Repository = "https://github.com/antonyrag/ragleap-core"
+Documentation = "https://packages.ragleap.com/docs/ragleap-vectorstores.html"
+Issues = "https://github.com/antonyrag/ragleap-core/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ragleap_vectorstores"]

ragleap_vectorstores-0.1.0/src/ragleap_vectorstores/__init__.py

@@ -0,0 +1,19 @@
+"""
+ragleap-vectorstores: pluggable vector backends beyond ragleap-rag core's 6.
+
+Each backend import is wrapped in try/except ImportError, matching
+ragleap-rag's own vectorstores/__init__.py pattern - a missing optional
+extra simply makes that one backend unavailable rather than breaking
+the package.
+"""
+from ragleap.vectorstores.base import VectorBackend
+
+__version__ = "0.1.0"
+
+__all__ = ["VectorBackend", "__version__"]
+
+try:
+    from ragleap_vectorstores.chroma_backend import ChromaBackend
+    __all__.append("ChromaBackend")
+except ImportError:
+    pass  # chroma extra not installed - ChromaBackend simply unavailable, not an error

ragleap_vectorstores-0.1.0/src/ragleap_vectorstores/chroma_backend.py

@@ -0,0 +1,203 @@
+"""
+Chroma-backed vector storage for ragleap-vectorstores.
+
+Live-verified against the actually installed chromadb==1.5.9 package:
+every method signature and return shape used below was introspected and
+exercised against a real local PersistentClient during development, not
+assumed from documentation.
+
+Design notes:
+- persist_directory is REQUIRED. Chroma's PersistentClient already stores
+  chunk text and metadata durably on disk under this path, so - unlike
+  QdrantBackend/PineconeBackend/WeaviateBackend - no SQLite sidecar is
+  needed for chunk data. A small SQLite sidecar is still used, but only
+  for the document registry (filename, uploaded_at, metadata), because
+  Chroma has no native "parent document" concept - it only knows about
+  individual (id, embedding, metadata, document-text) chunk records.
+- Chroma's `where=` filter requires exactly one top-level operator per
+  clause - live-verified: a plain multi-key dict like
+  {"document_id": "d1", "tag": "x"} raises ValueError ("Expected where to
+  have exactly one operator"). Multiple equality conditions must be
+  wrapped in {"$and": [...]}. _build_where() below handles this.
+- similarity_score is derived as (1 - distance). Verified live with
+  "hnsw:space": "cosine" (the default configured here): querying with an
+  embedding identical to a stored one returns a distance ~0, consistent
+  with Chroma's documented cosine distance = 1 - cosine_similarity.
+- supports_sparse() is False. Chroma has no native BM25/keyword search
+  surface as of 1.5.9 - only vector similarity - so hybrid search falls
+  back to the VectorBackend default (dense-only), matching the honest
+  "don't claim capability that isn't there" rule QdrantBackend follows.
+"""
+import json
+import logging
+import os
+import sqlite3
+import threading
+from typing import Dict, List, Optional
+
+from ragleap.vectorstores.base import VectorBackend
+
+logger = logging.getLogger(__name__)
+
+
+class ChromaBackend(VectorBackend):
+    def __init__(
+        self,
+        persist_directory: str,
+        collection_name: str = "ragleap",
+    ):
+        try:
+            import chromadb  # noqa: F401
+        except ImportError as e:
+            raise ImportError(
+                "ChromaBackend requires the 'chroma' extra: "
+                "pip install ragleap-vectorstores[chroma]"
+            ) from e
+
+        if not persist_directory:
+            raise ValueError(
+                "ChromaBackend requires persist_directory= - this is where "
+                "both Chroma's own on-disk index and the document-registry "
+                "SQLite sidecar are stored."
+            )
+
+        self.persist_directory = persist_directory
+        self.collection_name = collection_name
+        self._client = None
+        self._collection = None
+        self._dimensions = None
+        self._lock = threading.Lock()
+
+        os.makedirs(persist_directory, exist_ok=True)
+        self._sqlite_path = os.path.join(persist_directory, "chroma_documents.sqlite3")
+        self._conn = sqlite3.connect(self._sqlite_path, check_same_thread=False)
+        self._conn.execute(
+            """CREATE TABLE IF NOT EXISTS documents (
+                id TEXT PRIMARY KEY, filename TEXT NOT NULL,
+                metadata TEXT NOT NULL, uploaded_at TEXT NOT NULL
+            )"""
+        )
+        self._conn.commit()
+
+    def _vector_key(self, document_id: str, chunk_index: int) -> str:
+        return f"{document_id}:{chunk_index}"
+
+    def _build_where(self, metadata_filter: Optional[Dict]):
+        """Chroma requires exactly one top-level operator per where-clause
+        (live-verified) - a plain multi-key dict raises ValueError. Wrap
+        multiple equality conditions in $and; pass single conditions
+        through directly."""
+        if not metadata_filter:
+            return None
+        if len(metadata_filter) == 1:
+            return dict(metadata_filter)
+        return {"$and": [{k: v} for k, v in metadata_filter.items()]}
+
+    def init_schema(self, dimensions: int) -> None:
+        import chromadb
+
+        self._dimensions = dimensions
+        self._client = chromadb.PersistentClient(path=self.persist_directory)
+        self._collection = self._client.get_or_create_collection(
+            name=self.collection_name,
+            metadata={"hnsw:space": "cosine"},
+        )
+        logger.info(
+            f"ChromaBackend: using collection '{self.collection_name}' "
+            f"(dimensions={dimensions}) at {self.persist_directory}"
+        )
+
+    def insert_document(self, document_id: str, filename: str, metadata: Dict) -> None:
+        import datetime
+        with self._lock:
+            self._conn.execute(
+                "INSERT INTO documents (id, filename, metadata, uploaded_at) VALUES (?, ?, ?, ?)",
+                (document_id, filename, json.dumps(metadata or {}), datetime.datetime.utcnow().isoformat()),
+            )
+            self._conn.commit()
+
+    def insert_chunk(
+        self, document_id: str, document_name: str, chunk_index: int,
+        text: str, token_count: int, embedding: List[float], metadata: Dict,
+    ) -> None:
+        vector_key = self._vector_key(document_id, chunk_index)
+        payload = {
+            "document_id": document_id, "document_name": document_name,
+            "chunk_index": chunk_index, "token_count": token_count or 0,
+            **(metadata or {}),
+        }
+        with self._lock:
+            self._collection.upsert(
+                ids=[vector_key],
+                embeddings=[embedding],
+                metadatas=[payload],
+                documents=[text],
+            )
+
+    def search_dense(self, embedding: List[float], top_k: int, metadata_filter: Optional[Dict] = None) -> List[Dict]:
+        if not embedding or self._collection is None:
+            return []
+
+        result = self._collection.query(
+            query_embeddings=[embedding],
+            n_results=top_k,
+            where=self._build_where(metadata_filter),
+            include=["documents", "metadatas", "distances"],
+        )
+
+        ids = result.get("ids") or [[]]
+        docs = result.get("documents") or [[]]
+        metas = result.get("metadatas") or [[]]
+        dists = result.get("distances") or [[]]
+        if not ids or not ids[0]:
+            return []
+
+        results = []
+        for chunk_id, text, meta, distance in zip(ids[0], docs[0], metas[0], dists[0]):
+            meta = meta or {}
+            results.append({
+                "chunk_id": chunk_id,
+                "text": text,
+                "similarity_score": round(1.0 - float(distance), 4),
+                "document_id": meta.get("document_id"),
+                "document_name": meta.get("document_name"),
+                "chunk_index": meta.get("chunk_index"),
+            })
+        return results
+
+    def supports_sparse(self) -> bool:
+        return False
+
+    def list_documents(self, limit: int, offset: int) -> List[Dict]:
+        rows = self._conn.execute(
+            "SELECT id, filename, uploaded_at, metadata FROM documents "
+            "ORDER BY uploaded_at DESC LIMIT ? OFFSET ?",
+            (limit, offset),
+        ).fetchall()
+
+        results = []
+        for doc_id, filename, uploaded_at, metadata_json in rows:
+            chunk_count = 0
+            if self._collection is not None:
+                chunk_rows = self._collection.get(where={"document_id": doc_id}, include=[])
+                chunk_count = len((chunk_rows or {}).get("ids", []))
+            results.append({
+                "document_id": doc_id, "filename": filename, "uploaded_at": uploaded_at,
+                "metadata": json.loads(metadata_json), "chunk_count": chunk_count,
+            })
+        return results
+
+    def delete_document(self, document_id: str) -> bool:
+        with self._lock:
+            if self._collection is not None:
+                self._collection.delete(where={"document_id": document_id})
+            cur = self._conn.execute("DELETE FROM documents WHERE id = ?", (document_id,))
+            deleted = cur.rowcount > 0
+            self._conn.commit()
+        return deleted
+
+    def get_document_filename(self, document_id: str) -> Optional[str]:
+        row = self._conn.execute(
+            "SELECT filename FROM documents WHERE id = ?", (document_id,)
+        ).fetchone()
+        return row[0] if row else None

ragleap_vectorstores-0.1.0/tests/test_chroma_backend.py

@@ -0,0 +1,109 @@
+"""
+Tests for ChromaBackend. Uses a real local chromadb PersistentClient
+against a temp directory (Chroma has no meaningful "fake" - it's already
+fully local/embedded, so there's no live-credentials gap to skip around,
+unlike QdrantBackend/PineconeBackend/WeaviateBackend's tests).
+"""
+import shutil
+
+import pytest
+
+from ragleap_vectorstores.chroma_backend import ChromaBackend
+
+
+@pytest.fixture
+def backend(tmp_path):
+    b = ChromaBackend(persist_directory=str(tmp_path / "chroma_data"))
+    b.init_schema(dimensions=3)
+    yield b
+
+
+def _seed(backend):
+    backend.insert_document("doc1", "report.pdf", {"source": "upload"})
+    backend.insert_chunk("doc1", "report.pdf", 0, "The cat sat on the mat.", 6, [0.1, 0.2, 0.3], {"page": 1})
+    backend.insert_chunk("doc1", "report.pdf", 1, "Dogs bark loudly outside.", 5, [0.9, 0.1, 0.1], {"page": 2})
+    backend.insert_document("doc2", "notes.txt", {"source": "manual"})
+    backend.insert_chunk("doc2", "notes.txt", 0, "Unrelated content here.", 4, [0.5, 0.5, 0.5], {"page": 1})
+
+
+def test_requires_persist_directory():
+    with pytest.raises(ValueError):
+        ChromaBackend(persist_directory="")
+
+
+def test_insert_and_search_dense_shape(backend):
+    _seed(backend)
+    results = backend.search_dense([0.1, 0.2, 0.3], top_k=5)
+    assert results, "expected at least one result"
+    assert set(results[0].keys()) == {
+        "chunk_id", "text", "similarity_score", "document_id", "document_name", "chunk_index",
+    }
+    assert results[0]["chunk_id"] == "doc1:0"
+    assert results[0]["text"] == "The cat sat on the mat."
+
+
+def test_search_dense_single_key_filter(backend):
+    _seed(backend)
+    results = backend.search_dense([0.1, 0.2, 0.3], top_k=5, metadata_filter={"document_id": "doc2"})
+    assert results
+    assert all(r["document_id"] == "doc2" for r in results)
+
+
+def test_search_dense_multi_key_filter(backend):
+    """Regression guard for the real Chroma constraint (live-verified): a
+    plain multi-key `where=` dict raises ValueError - multiple conditions
+    must be wrapped in $and. This test would fail loudly if _build_where()
+    regressed to a naive dict pass-through."""
+    _seed(backend)
+    results = backend.search_dense([0.1, 0.2, 0.3], top_k=5, metadata_filter={"document_id": "doc1", "page": 2})
+    assert len(results) == 1
+    assert results[0]["chunk_id"] == "doc1:1"
+
+
+def test_search_dense_empty_embedding_returns_empty(backend):
+    assert backend.search_dense([], top_k=5) == []
+
+
+def test_search_sparse_not_supported(backend):
+    _seed(backend)
+    assert backend.search_sparse("cat", top_k=5) == []
+
+
+def test_search_hybrid_falls_back_to_dense(backend):
+    _seed(backend)
+    hybrid = backend.search_hybrid("cat", [0.1, 0.2, 0.3], top_k=5)
+    dense = backend.search_dense([0.1, 0.2, 0.3], top_k=5)
+    assert [r["chunk_id"] for r in hybrid] == [r["chunk_id"] for r in dense]
+
+
+def test_supports_sparse_is_false(backend):
+    assert backend.supports_sparse() is False
+
+
+def test_list_documents(backend):
+    _seed(backend)
+    docs = backend.list_documents(limit=10, offset=0)
+    assert len(docs) == 2
+    by_id = {d["document_id"]: d for d in docs}
+    assert by_id["doc1"]["chunk_count"] == 2
+    assert by_id["doc2"]["chunk_count"] == 1
+    assert by_id["doc1"]["filename"] == "report.pdf"
+    assert by_id["doc1"]["metadata"] == {"source": "upload"}
+
+
+def test_get_document_filename(backend):
+    _seed(backend)
+    assert backend.get_document_filename("doc1") == "report.pdf"
+    assert backend.get_document_filename("nonexistent") is None
+
+
+def test_delete_document_removes_registry_entry_and_vectors(backend):
+    _seed(backend)
+    assert backend.delete_document("doc1") is True
+    assert backend.delete_document("doc1") is False  # already gone
+
+    docs = backend.list_documents(limit=10, offset=0)
+    assert [d["document_id"] for d in docs] == ["doc2"]
+
+    remaining = backend.search_dense([0.1, 0.2, 0.3], top_k=5)
+    assert all(r["document_id"] != "doc1" for r in remaining)

ragleap_vectorstores-0.1.0/tests/test_smoke.py

@@ -0,0 +1,7 @@
+"""Smoke test - the package imports cleanly and VectorBackend is always
+re-exported regardless of which optional backend extras are installed."""
+import ragleap_vectorstores
+
+
+def test_import():
+    assert "VectorBackend" in ragleap_vectorstores.__all__