omendb 0.0.23__cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

omendb/llamaindex.py

@@ -0,0 +1,336 @@
+"""LlamaIndex VectorStore integration for OmenDB.
+
+This module provides a LlamaIndex-compatible VectorStore implementation
+that wraps OmenDB for seamless integration with LlamaIndex RAG pipelines.
+
+Example:
+    >>> from llama_index.core import VectorStoreIndex, StorageContext
+    >>> from llama_index.embeddings.openai import OpenAIEmbedding
+    >>> from omendb.llamaindex import OmenDBVectorStore
+    >>>
+    >>> # Create vector store
+    >>> vector_store = OmenDBVectorStore(path="./my_vectors")
+    >>> storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    >>>
+    >>> # Build index from documents
+    >>> index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)
+    >>>
+    >>> # Query
+    >>> query_engine = index.as_query_engine()
+    >>> response = query_engine.query("What is the main topic?")
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from llama_index.core.schema import BaseNode, TextNode
+from llama_index.core.vector_stores.types import (
+    BasePydanticVectorStore,
+    VectorStoreQuery,
+    VectorStoreQueryResult,
+)
+
+
+class OmenDBVectorStore(BasePydanticVectorStore):
+    """LlamaIndex VectorStore implementation using OmenDB.
+
+    OmenDB is a fast embedded vector database with HNSW + ACORN-1
+    that provides ~19,000 QPS @ 10K vectors with 100% recall.
+
+    Features:
+        - HNSW index with adaptive parameters
+        - Extended RaBitQ quantization (8x compression)
+        - ACORN-1 filtered search (37.79x speedup)
+        - MongoDB-style metadata filtering
+        - Automatic persistence with persistent storage
+
+    Args:
+        path: Path to database directory. Uses persistent persistent storage.
+        dimensions: Vector dimensionality. If None, auto-detected on first insert.
+        **kwargs: Additional arguments passed to omendb.open().
+
+    Example:
+        >>> from llama_index.core import VectorStoreIndex, StorageContext
+        >>> from omendb.llamaindex import OmenDBVectorStore
+        >>>
+        >>> vector_store = OmenDBVectorStore(path="./my_vectors")
+        >>> storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        >>> index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)
+    """
+
+    stores_text: bool = True
+    flat_metadata: bool = True
+
+    # Pydantic fields
+    path: str = "./omendb-vectors"
+    dimensions: int | None = None
+
+    # Private attributes (not serialized)
+    _db: Any = None
+    _initialized: bool = False
+
+    def __init__(
+        self,
+        path: str = "./omendb-vectors",
+        dimensions: int | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize OmenDBVectorStore.
+
+        Args:
+            path: Path to database directory.
+            dimensions: Vector dimensionality (auto-detected on first insert if None).
+            **kwargs: Additional arguments for omendb.open().
+        """
+        super().__init__(path=path, dimensions=dimensions)
+        self._kwargs = kwargs
+        self._db = None
+        self._initialized = False
+
+    def _ensure_db(self, dimensions: int | None = None) -> None:
+        """Ensure database is initialized."""
+        if self._db is not None:
+            return
+
+        import omendb
+
+        # Use provided dimensions or fall back to stored
+        dims = dimensions or self.dimensions or 1536  # Default to OpenAI dimensions
+        self._db = omendb.open(self.path, dimensions=dims, **getattr(self, "_kwargs", {}))
+        self._initialized = True
+
+    @classmethod
+    def class_name(cls) -> str:
+        """Return class name for serialization."""
+        return "OmenDBVectorStore"
+
+    @property
+    def client(self) -> Any:
+        """Return the underlying OmenDB database client."""
+        self._ensure_db()
+        return self._db
+
+    def add(
+        self,
+        nodes: list[BaseNode],
+        **kwargs: Any,
+    ) -> list[str]:
+        """Add nodes to the vector store.
+
+        Args:
+            nodes: List of nodes to add.
+            **kwargs: Additional arguments (unused).
+
+        Returns:
+            List of node IDs that were added.
+        """
+        if not nodes:
+            return []
+
+        # Get dimensions from first node's embedding
+        first_embedding = nodes[0].get_embedding()
+        if first_embedding:
+            self._ensure_db(dimensions=len(first_embedding))
+        else:
+            self._ensure_db()
+
+        ids = []
+        items = []
+
+        for node in nodes:
+            node_id = node.node_id
+            embedding = node.get_embedding()
+
+            if embedding is None:
+                continue
+
+            # Build metadata from node
+            metadata = node.metadata.copy() if node.metadata else {}
+
+            # Store text content in metadata for retrieval
+            text = node.get_content()
+            if text:
+                metadata["_text"] = text
+
+            # Store node type info
+            metadata["_node_type"] = node.class_name()
+
+            items.append(
+                {
+                    "id": node_id,
+                    "vector": embedding,
+                    "metadata": metadata,
+                }
+            )
+            ids.append(node_id)
+
+        if items:
+            self._db.set(items)
+
+        return ids
+
+    def delete(self, ref_doc_id: str, **kwargs: Any) -> None:
+        """Delete nodes by reference document ID.
+
+        Args:
+            ref_doc_id: The document ID to delete.
+            **kwargs: Additional arguments (unused).
+        """
+        self._ensure_db()
+        # OmenDB delete expects a list of IDs
+        self._db.delete([ref_doc_id])
+
+    def delete_nodes(
+        self,
+        node_ids: list[str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Delete specific nodes by their IDs.
+
+        Args:
+            node_ids: List of node IDs to delete.
+            **kwargs: Additional arguments (unused).
+        """
+        if not node_ids:
+            return
+
+        self._ensure_db()
+        self._db.delete(node_ids)
+
+    def query(
+        self,
+        query: VectorStoreQuery,
+        **kwargs: Any,
+    ) -> VectorStoreQueryResult:
+        """Query the vector store.
+
+        Args:
+            query: VectorStoreQuery containing query embedding and parameters.
+            **kwargs: Additional arguments (unused).
+
+        Returns:
+            VectorStoreQueryResult with matching nodes, similarities, and IDs.
+        """
+        self._ensure_db()
+
+        if query.query_embedding is None:
+            return VectorStoreQueryResult(nodes=[], similarities=[], ids=[])
+
+        # Build filter dict from LlamaIndex filters
+        filter_dict = None
+        if query.filters is not None:
+            filter_dict = self._convert_filters(query.filters)
+
+        # Query OmenDB
+        k = query.similarity_top_k or 10
+        results = self._db.search(
+            query=query.query_embedding,
+            k=k,
+            filter=filter_dict,
+        )
+
+        # Convert results to LlamaIndex format
+        nodes = []
+        similarities = []
+        ids = []
+
+        for result in results:
+            node_id = result.get("id", "")
+            distance = result.get("distance", 0.0)
+            metadata = result.get("metadata", {})
+
+            # Extract text from metadata
+            text = metadata.pop("_text", "")
+            metadata.pop("_node_type", None)
+
+            # Create TextNode
+            node = TextNode(
+                id_=node_id,
+                text=text,
+                metadata=metadata,
+                embedding=result.get("vector"),
+            )
+
+            nodes.append(node)
+            # Convert distance to similarity (assuming L2 distance)
+            # For L2: similarity = 1 / (1 + distance)
+            similarity = 1.0 / (1.0 + distance) if distance >= 0 else 0.0
+            similarities.append(similarity)
+            ids.append(node_id)
+
+        return VectorStoreQueryResult(
+            nodes=nodes,
+            similarities=similarities,
+            ids=ids,
+        )
+
+    def _convert_filters(self, filters: Any) -> dict[str, Any] | None:
+        """Convert LlamaIndex MetadataFilters to OmenDB filter format.
+
+        Args:
+            filters: LlamaIndex MetadataFilters object.
+
+        Returns:
+            OmenDB-compatible filter dictionary.
+        """
+        if filters is None:
+            return None
+
+        from llama_index.core.vector_stores.types import (
+            FilterCondition,
+            FilterOperator,
+            MetadataFilter,
+            MetadataFilters,
+        )
+
+        if not isinstance(filters, MetadataFilters):
+            return None
+
+        filter_list = []
+
+        for f in filters.filters:
+            if not isinstance(f, MetadataFilter):
+                continue
+
+            key = f.key
+            value = f.value
+            op = f.operator
+
+            # Map LlamaIndex operators to OmenDB operators
+            if op == FilterOperator.EQ:
+                filter_list.append({key: value})
+            elif op == FilterOperator.NE:
+                filter_list.append({key: {"$ne": value}})
+            elif op == FilterOperator.GT:
+                filter_list.append({key: {"$gt": value}})
+            elif op == FilterOperator.GTE:
+                filter_list.append({key: {"$gte": value}})
+            elif op == FilterOperator.LT:
+                filter_list.append({key: {"$lt": value}})
+            elif op == FilterOperator.LTE:
+                filter_list.append({key: {"$lte": value}})
+            elif op == FilterOperator.IN:
+                filter_list.append({key: {"$in": value}})
+            elif op == FilterOperator.CONTAINS:
+                filter_list.append({key: {"$contains": value}})
+            else:
+                # Default to equality
+                filter_list.append({key: value})
+
+        if not filter_list:
+            return None
+
+        # Combine filters based on condition
+        if len(filter_list) == 1:
+            return filter_list[0]
+
+        condition = getattr(filters, "condition", FilterCondition.AND)
+        if condition == FilterCondition.OR:
+            return {"$or": filter_list}
+        else:
+            return {"$and": filter_list}
+
+    def flush(self) -> None:
+        """Flush data to disk for persistence."""
+        self._db.flush()

omendb-0.0.23.dist-info/METADATA

@@ -0,0 +1,280 @@
+Metadata-Version: 2.4
+Name: omendb
+Version: 0.0.23
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Rust
+Requires-Dist: numpy>=1.24.4
+Requires-Dist: langchain-core>=0.2.0 ; extra == 'langchain'
+Requires-Dist: llama-index-core>=0.10.0 ; extra == 'llamaindex'
+Provides-Extra: langchain
+Provides-Extra: llamaindex
+Summary: Fast embedded vector database with HNSW + ACORN-1 filtered search
+Author: OmenDB Team
+License: Elastic-2.0
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/omendb/omendb
+Project-URL: Repository, https://github.com/omendb/omendb
+
+# OmenDB
+
+[![PyPI](https://img.shields.io/pypi/v/omendb)](https://pypi.org/project/omendb/)
+[![License](https://img.shields.io/badge/License-Elastic_2.0-blue.svg)](https://github.com/omendb/omendb/blob/main/LICENSE)
+
+Embedded vector database for Python and Node.js. No server, no setup, just install.
+
+```bash
+pip install omendb
+```
+
+## Quick Start
+
+```python
+import omendb
+
+# Create database (persistent) - creates ./mydb.omen file
+db = omendb.open("./mydb", dimensions=128)
+
+# Add vectors with metadata
+db.set([
+    {"id": "doc1", "vector": [0.1] * 128, "metadata": {"category": "science"}},
+    {"id": "doc2", "vector": [0.2] * 128, "metadata": {"category": "history"}},
+])
+
+# Search
+results = db.search([0.1] * 128, k=5)
+
+# Filtered search
+results = db.search([0.1] * 128, k=5, filter={"category": "science"})
+```
+
+## Features
+
+- **Embedded** - Runs in-process, no server needed
+- **Persistent** - Data survives restarts automatically
+- **Filtered search** - Query by metadata with JSON-style filters
+- **Hybrid search** - Combine vector similarity with BM25 text search
+- **Quantization** - 4-8x smaller indexes with minimal recall loss
+
+## Platforms
+
+| Platform                     | Status       |
+| ---------------------------- | ------------ |
+| Linux (x86_64, ARM64)        | Supported    |
+| macOS (Intel, Apple Silicon) | Supported    |
+| Windows (x86_64)             | Experimental |
+
+## API
+
+```python
+# Database
+db = omendb.open(path, dimensions)      # Open or create
+db = omendb.open(":memory:", dimensions)  # In-memory (ephemeral)
+
+# CRUD
+db.set(items)                           # Insert/update vectors
+db.get(id)                              # Get by ID
+db.get_batch(ids)                       # Batch get by IDs
+db.delete(ids)                          # Delete by IDs
+db.delete_by_filter(filter)             # Delete by metadata filter
+db.update(id, metadata)                 # Update metadata only
+
+# Iteration
+len(db)                                 # Number of vectors
+db.count()                              # Same as len(db)
+db.count(filter={...})                  # Count matching filter
+db.ids()                                # Iterate all IDs (lazy)
+db.items()                              # Get all items as list
+db.exists(id)                           # Check if ID exists
+"id" in db                              # Same as exists()
+for item in db: ...                     # Iterate all items (lazy)
+
+# Search
+db.search(query, k)                     # Vector search
+db.search(query, k, filter={...})       # Filtered search
+db.search(query, k, max_distance=0.5)   # Only results with distance <= 0.5
+db.search_batch(queries, k)             # Batch search (parallel)
+
+# Hybrid search (requires text field in vectors)
+db.search_hybrid(query_vector, query_text, k)
+db.search_hybrid(query_vector, query_text, k, alpha=0.7)  # 70% vector, 30% text
+db.search_hybrid(query_vector, query_text, k, subscores=True)  # Return separate scores
+db.search_text(query_text, k)           # Text-only BM25
+
+# Persistence
+db.flush()                              # Flush to disk
+```
+
+## Distance Filtering
+
+Use `max_distance` to filter out low-relevance results (prevents "context rot" in RAG):
+
+```python
+# Only return results with distance <= 0.5
+results = db.search(query, k=10, max_distance=0.5)
+
+# Combine with metadata filter
+results = db.search(query, k=10, filter={"type": "doc"}, max_distance=0.5)
+```
+
+This ensures your RAG pipeline only receives highly relevant context, avoiding distractors that can hurt LLM performance.
+
+## Filters
+
+```python
+# Equality
+{"field": "value"}                      # Shorthand
+{"field": {"$eq": "value"}}             # Explicit
+
+# Comparison
+{"field": {"$ne": "value"}}             # Not equal
+{"field": {"$gt": 10}}                  # Greater than
+{"field": {"$gte": 10}}                 # Greater or equal
+{"field": {"$lt": 10}}                  # Less than
+{"field": {"$lte": 10}}                 # Less or equal
+
+# Membership
+{"field": {"$in": ["a", "b"]}}          # In list
+{"field": {"$contains": "sub"}}         # String contains
+
+# Logical
+{"$and": [{...}, {...}]}                # AND
+{"$or": [{...}, {...}]}                 # OR
+```
+
+## Configuration
+
+```python
+db = omendb.open(
+    "./mydb",              # Creates ./mydb.omen + ./mydb.wal
+    dimensions=384,
+    m=16,                # HNSW connections per node (default: 16)
+    ef_construction=200, # Index build quality (default: 100)
+    ef_search=100,       # Search quality (default: 100)
+    quantization=True,   # SQ8 quantization (default: None)
+    metric="cosine",     # Distance metric (default: "l2")
+)
+
+# Quantization options:
+# - True or "sq8": SQ8 ~4x smaller, ~99% recall (recommended)
+# - "rabitq": RaBitQ ~8x smaller, ~98% recall
+# - None/False: Full precision (default)
+
+# Distance metric options:
+# - "l2" or "euclidean": Euclidean distance (default)
+# - "cosine": Cosine distance (1 - cosine similarity)
+# - "dot" or "ip": Inner product (for MIPS)
+
+# Context manager (auto-flush on exit)
+with omendb.open("./db", dimensions=768) as db:
+    db.set([...])
+
+# Hybrid search with alpha (0=text, 1=vector, default=0.5)
+db.search_hybrid(query_vec, "query text", k=10, alpha=0.7)
+
+# Get separate keyword and semantic scores for debugging/tuning
+results = db.search_hybrid(query_vec, "query text", k=10, subscores=True)
+# Returns: {"id": "...", "score": 0.85, "keyword_score": 0.92, "semantic_score": 0.78}
+```
+
+## Performance
+
+**10K vectors, Apple M3 Max** (m=16, ef=100, k=10):
+
+| Dimension | Single QPS | Batch QPS | Speedup |
+| --------- | ---------- | --------- | ------- |
+| 128D      | 12,000+    | 87,000+   | 7.2x    |
+| 768D      | 3,800+     | 20,500+   | 5.4x    |
+| 1536D     | 1,600+     | 6,200+    | 3.8x    |
+
+**SIFT-1M** (1M vectors, 128D, m=16, ef=100, k=10):
+
+| Machine      | QPS   | Recall |
+| ------------ | ----- | ------ |
+| i9-13900KF   | 4,591 | 98.6%  |
+| Apple M3 Max | 3,216 | 98.4%  |
+
+**Quantization** reduces memory with minimal recall loss:
+
+| Mode   | Compression | Use Case                       |
+| ------ | ----------- | ------------------------------ |
+| f32    | 1x          | Default, highest recall        |
+| sq8    | 4x          | Recommended for most users     |
+| rabitq | 8x          | Large datasets, cost-sensitive |
+
+```python
+db = omendb.open("./db", dimensions=768, quantization=True)  # Enable SQ8
+```
+
+<details>
+<summary>Benchmark methodology</summary>
+
+- **Parameters**: m=16, ef_construction=100, ef_search=100
+- **Batch**: Uses Rayon for parallel search across all cores
+- **Recall**: Validated against brute-force ground truth on SIFT/GloVe
+- **Reproduce**:
+  - Quick (10K): `uv run python benchmarks/run.py`
+  - SIFT-1M: `uv run python benchmarks/ann_dataset_test.py --dataset sift-128-euclidean`
+
+</details>
+
+## Examples
+
+See [`python/examples/`](python/examples/) for complete working examples:
+
+- `quickstart.py` - Minimal working example
+- `basic.py` - CRUD operations and persistence
+- `filters.py` - All filter operators
+- `rag.py` - RAG workflow with mock embeddings
+
+## Integrations
+
+### LangChain
+
+```bash
+pip install omendb[langchain]
+```
+
+```python
+from langchain_openai import OpenAIEmbeddings
+from omendb.langchain import OmenDBVectorStore
+
+store = OmenDBVectorStore.from_texts(
+    texts=["Paris is the capital of France"],
+    embedding=OpenAIEmbeddings(),
+    path="./langchain_vectors",
+)
+docs = store.similarity_search("capital of France", k=1)
+```
+
+### LlamaIndex
+
+```bash
+pip install omendb[llamaindex]
+```
+
+```python
+from llama_index.core import VectorStoreIndex, Document, StorageContext
+from omendb.llamaindex import OmenDBVectorStore
+
+vector_store = OmenDBVectorStore(path="./llama_vectors")
+storage_context = StorageContext.from_defaults(vector_store=vector_store)
+index = VectorStoreIndex.from_documents(
+    [Document(text="OmenDB is fast")],
+    storage_context=storage_context,
+)
+response = index.as_query_engine().query("What is OmenDB?")
+```
+
+## License
+
+[Elastic License 2.0](LICENSE) - Free to use, modify, and embed. The only restriction: you can't offer OmenDB as a managed service to third parties.
+

omendb-0.0.23.dist-info/RECORD

@@ -0,0 +1,8 @@
+omendb/__init__.py,sha256=u92FKzI-Sr3Usd9LLi-XCul0QC4QgKhzICaDBIBOtAE,834
+omendb/__init__.pyi,sha256=oWtFhkFRi-5FvR0a4gayZ9W2Fu8iP60vlwlqR9Ru3U8,17118
+omendb/langchain.py,sha256=w0W5W17ZlSzxoor1LYIzrZYrRvoKK0WW1oQBNcZ_7mk,12184
+omendb/llamaindex.py,sha256=3aKSUufpn33uSK6h61BdKEZvmXNPbz-FexRmlALOtLs,10328
+omendb/omendb.cpython-313t-aarch64-linux-gnu.so,sha256=wS-gTAdJc0-7yyVeuqVQkExy7oM-s6sIMZ2ODpVaSZ0,81094304
+omendb-0.0.23.dist-info/METADATA,sha256=-xyaRPNp1moyGIbD1JJOtH7wODIY7fEVrqhZtUedRls,9186
+omendb-0.0.23.dist-info/WHEEL,sha256=q835leI5tUqUNzGLT6_2V4Cxb6Anw8rRDNKWlUcsVqo,151
+omendb-0.0.23.dist-info/RECORD,,

omendb-0.0.23.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: maturin (1.11.5)
+Root-Is-Purelib: false
+Tag: cp313-cp313t-manylinux_2_17_aarch64
+Tag: cp313-cp313t-manylinux2014_aarch64