haiku.rag 0.9.2__py3-none-any.whl → 0.14.0__py3-none-any.whl

haiku/rag/store/repositories/document.py

@@ -1,234 +0,0 @@
-import json
-from datetime import datetime
-from typing import TYPE_CHECKING
-from uuid import uuid4
-
-from docling_core.types.doc.document import DoclingDocument
-
-from haiku.rag.store.engine import DocumentRecord, Store
-from haiku.rag.store.models.document import Document
-
-if TYPE_CHECKING:
-    from haiku.rag.store.models.chunk import Chunk
-
-
-class DocumentRepository:
-    """Repository for Document operations."""
-
-    def __init__(self, store: Store) -> None:
-        self.store = store
-        self._chunk_repository = None
-
-    @property
-    def chunk_repository(self):
-        """Lazy-load ChunkRepository when needed."""
-        if self._chunk_repository is None:
-            from haiku.rag.store.repositories.chunk import ChunkRepository
-
-            self._chunk_repository = ChunkRepository(self.store)
-        return self._chunk_repository
-
-    def _record_to_document(self, record: DocumentRecord) -> Document:
-        """Convert a DocumentRecord to a Document model."""
-        return Document(
-            id=record.id,
-            content=record.content,
-            uri=record.uri,
-            metadata=json.loads(record.metadata) if record.metadata else {},
-            created_at=datetime.fromisoformat(record.created_at)
-            if record.created_at
-            else datetime.now(),
-            updated_at=datetime.fromisoformat(record.updated_at)
-            if record.updated_at
-            else datetime.now(),
-        )
-
-    async def create(self, entity: Document) -> Document:
-        """Create a document in the database."""
-        # Generate new UUID
-        doc_id = str(uuid4())
-
-        # Create timestamp
-        now = datetime.now().isoformat()
-
-        # Create document record
-        doc_record = DocumentRecord(
-            id=doc_id,
-            content=entity.content,
-            uri=entity.uri,
-            metadata=json.dumps(entity.metadata),
-            created_at=now,
-            updated_at=now,
-        )
-
-        # Add to table
-        self.store.documents_table.add([doc_record])
-
-        entity.id = doc_id
-        entity.created_at = datetime.fromisoformat(now)
-        entity.updated_at = datetime.fromisoformat(now)
-        return entity
-
-    async def get_by_id(self, entity_id: str) -> Document | None:
-        """Get a document by its ID."""
-        results = list(
-            self.store.documents_table.search()
-            .where(f"id = '{entity_id}'")
-            .limit(1)
-            .to_pydantic(DocumentRecord)
-        )
-
-        if not results:
-            return None
-
-        return self._record_to_document(results[0])
-
-    async def update(self, entity: Document) -> Document:
-        """Update an existing document."""
-        assert entity.id, "Document ID is required for update"
-
-        # Update timestamp
-        now = datetime.now().isoformat()
-        entity.updated_at = datetime.fromisoformat(now)
-
-        # Update the record
-        self.store.documents_table.update(
-            where=f"id = '{entity.id}'",
-            values={
-                "content": entity.content,
-                "uri": entity.uri,
-                "metadata": json.dumps(entity.metadata),
-                "updated_at": now,
-            },
-        )
-
-        return entity
-
-    async def delete(self, entity_id: str) -> bool:
-        """Delete a document by its ID."""
-        # Check if document exists
-        doc = await self.get_by_id(entity_id)
-        if doc is None:
-            return False
-
-        # Delete associated chunks first
-        await self.chunk_repository.delete_by_document_id(entity_id)
-
-        # Delete the document
-        self.store.documents_table.delete(f"id = '{entity_id}'")
-        return True
-
-    async def list_all(
-        self, limit: int | None = None, offset: int | None = None
-    ) -> list[Document]:
-        """List all documents with optional pagination."""
-        query = self.store.documents_table.search()
-
-        if offset is not None:
-            query = query.offset(offset)
-        if limit is not None:
-            query = query.limit(limit)
-
-        results = list(query.to_pydantic(DocumentRecord))
-        return [self._record_to_document(doc) for doc in results]
-
-    async def get_by_uri(self, uri: str) -> Document | None:
-        """Get a document by its URI."""
-        results = list(
-            self.store.documents_table.search()
-            .where(f"uri = '{uri}'")
-            .limit(1)
-            .to_pydantic(DocumentRecord)
-        )
-
-        if not results:
-            return None
-
-        return self._record_to_document(results[0])
-
-    async def delete_all(self) -> None:
-        """Delete all documents from the database."""
-        # Delete all chunks first
-        await self.chunk_repository.delete_all()
-
-        # Get count before deletion
-        count = len(
-            list(
-                self.store.documents_table.search().limit(1).to_pydantic(DocumentRecord)
-            )
-        )
-        if count > 0:
-            # Drop and recreate table to clear all data
-            self.store.db.drop_table("documents")
-            self.store.documents_table = self.store.db.create_table(
-                "documents", schema=DocumentRecord
-            )
-
-    async def _create_with_docling(
-        self,
-        entity: Document,
-        docling_document: DoclingDocument,
-        chunks: list["Chunk"] | None = None,
-    ) -> Document:
-        """Create a document with its chunks and embeddings."""
-        # Snapshot table versions for versioned rollback (if supported)
-        versions = self.store.current_table_versions()
-
-        # Create the document
-        created_doc = await self.create(entity)
-
-        # Attempt to create chunks; on failure, prefer version rollback
-        try:
-            # Create chunks if not provided
-            if chunks is None:
-                assert created_doc.id is not None, (
-                    "Document ID should not be None after creation"
-                )
-                await self.chunk_repository.create_chunks_for_document(
-                    created_doc.id, docling_document
-                )
-            else:
-                # Use provided chunks, set order from list position
-                assert created_doc.id is not None, (
-                    "Document ID should not be None after creation"
-                )
-                for order, chunk in enumerate(chunks):
-                    chunk.document_id = created_doc.id
-                    chunk.metadata["order"] = order
-                    await self.chunk_repository.create(chunk)
-
-            return created_doc
-        except Exception:
-            # Roll back to the captured versions and re-raise
-            self.store.restore_table_versions(versions)
-            raise
-
-    async def _update_with_docling(
-        self, entity: Document, docling_document: DoclingDocument
-    ) -> Document:
-        """Update a document and regenerate its chunks."""
-        assert entity.id is not None, "Document ID is required for update"
-
-        # Snapshot table versions for versioned rollback
-        versions = self.store.current_table_versions()
-
-        # Delete existing chunks before writing new ones
-        await self.chunk_repository.delete_by_document_id(entity.id)
-
-        try:
-            # Update the document
-            updated_doc = await self.update(entity)
-
-            # Create new chunks
-            assert updated_doc.id is not None, (
-                "Document ID should not be None after update"
-            )
-            await self.chunk_repository.create_chunks_for_document(
-                updated_doc.id, docling_document
-            )
-
-            return updated_doc
-        except Exception:
-            # Roll back to the captured versions and re-raise
-            self.store.restore_table_versions(versions)
-            raise

haiku/rag/store/repositories/settings.py

@@ -1,148 +0,0 @@
-import json
-
-from haiku.rag.config import Config
-from haiku.rag.store.engine import SettingsRecord, Store
-
-
-class ConfigMismatchError(Exception):
-    """Raised when stored config doesn't match current config."""
-
-    pass
-
-
-class SettingsRepository:
-    """Repository for Settings operations."""
-
-    def __init__(self, store: Store) -> None:
-        self.store = store
-
-    async def create(self, entity: dict) -> dict:
-        """Create settings in the database."""
-        settings_record = SettingsRecord(id="settings", settings=json.dumps(entity))
-        self.store.settings_table.add([settings_record])
-        return entity
-
-    async def get_by_id(self, entity_id: str) -> dict | None:
-        """Get settings by ID."""
-        results = list(
-            self.store.settings_table.search()
-            .where(f"id = '{entity_id}'")
-            .limit(1)
-            .to_pydantic(SettingsRecord)
-        )
-
-        if not results:
-            return None
-
-        return json.loads(results[0].settings) if results[0].settings else {}
-
-    async def update(self, entity: dict) -> dict:
-        """Update existing settings."""
-        self.store.settings_table.update(
-            where="id = 'settings'", values={"settings": json.dumps(entity)}
-        )
-        return entity
-
-    async def delete(self, entity_id: str) -> bool:
-        """Delete settings by ID."""
-        self.store.settings_table.delete(f"id = '{entity_id}'")
-        return True
-
-    async def list_all(
-        self, limit: int | None = None, offset: int | None = None
-    ) -> list[dict]:
-        """List all settings."""
-        results = list(self.store.settings_table.search().to_pydantic(SettingsRecord))
-        return [
-            json.loads(record.settings) if record.settings else {} for record in results
-        ]
-
-    def get_current_settings(self) -> dict:
-        """Get the current settings."""
-        results = list(
-            self.store.settings_table.search()
-            .where("id = 'settings'")
-            .limit(1)
-            .to_pydantic(SettingsRecord)
-        )
-
-        if not results:
-            return {}
-
-        return json.loads(results[0].settings) if results[0].settings else {}
-
-    def save_current_settings(self) -> None:
-        """Save the current configuration to the database."""
-        current_config = Config.model_dump(mode="json")
-
-        # Check if settings exist
-        existing = list(
-            self.store.settings_table.search()
-            .where("id = 'settings'")
-            .limit(1)
-            .to_pydantic(SettingsRecord)
-        )
-
-        if existing:
-            # Only update when configuration actually changed to avoid needless new versions
-            existing_payload = (
-                json.loads(existing[0].settings) if existing[0].settings else {}
-            )
-            if existing_payload != current_config:
-                self.store.settings_table.update(
-                    where="id = 'settings'",
-                    values={"settings": json.dumps(current_config)},
-                )
-        else:
-            # Create new settings
-            settings_record = SettingsRecord(
-                id="settings", settings=json.dumps(current_config)
-            )
-            self.store.settings_table.add([settings_record])
-
-    def validate_config_compatibility(self) -> None:
-        """Validate that the current configuration is compatible with stored settings."""
-        stored_settings = self.get_current_settings()
-
-        # If no stored settings, this is a new database - save current config and return
-        if not stored_settings:
-            self.save_current_settings()
-            return
-
-        current_config = Config.model_dump(mode="json")
-
-        # Check if embedding provider or model has changed
-        stored_provider = stored_settings.get("EMBEDDINGS_PROVIDER")
-        current_provider = current_config.get("EMBEDDINGS_PROVIDER")
-
-        stored_model = stored_settings.get("EMBEDDINGS_MODEL")
-        current_model = current_config.get("EMBEDDINGS_MODEL")
-
-        stored_vector_dim = stored_settings.get("EMBEDDINGS_VECTOR_DIM")
-        current_vector_dim = current_config.get("EMBEDDINGS_VECTOR_DIM")
-
-        # Check for incompatible changes
-        incompatible_changes = []
-
-        if stored_provider and stored_provider != current_provider:
-            incompatible_changes.append(
-                f"Embedding provider changed from '{stored_provider}' to '{current_provider}'"
-            )
-
-        if stored_model and stored_model != current_model:
-            incompatible_changes.append(
-                f"Embedding model changed from '{stored_model}' to '{current_model}'"
-            )
-
-        if stored_vector_dim and stored_vector_dim != current_vector_dim:
-            incompatible_changes.append(
-                f"Vector dimension changed from {stored_vector_dim} to {current_vector_dim}"
-            )
-
-        if incompatible_changes:
-            error_msg = (
-                "Database configuration is incompatible with current settings:\n"
-                + "\n".join(f"  - {change}" for change in incompatible_changes)
-            )
-            error_msg += "\n\nPlease rebuild the database using: haiku-rag rebuild"
-            raise ConfigMismatchError(error_msg)

haiku/rag/store/upgrades/__init__.py

@@ -1 +0,0 @@
-upgrades = []

haiku/rag/utils.py

@@ -1,162 +0,0 @@
-import asyncio
-import importlib
-import importlib.util
-import sys
-from collections.abc import Callable
-from functools import wraps
-from importlib import metadata
-from io import BytesIO
-from pathlib import Path
-from types import ModuleType
-
-import httpx
-from docling.document_converter import DocumentConverter
-from docling_core.types.doc.document import DoclingDocument
-from docling_core.types.io import DocumentStream
-from packaging.version import Version, parse
-
-
-def debounce(wait: float) -> Callable:
-    """
-    A decorator to debounce a function, ensuring it is called only after a specified delay
-    and always executes after the last call.
-
-    Args:
-        wait (float): The debounce delay in seconds.
-
-    Returns:
-        Callable: The decorated function.
-    """
-
-    def decorator(func: Callable) -> Callable:
-        last_call = None
-        task = None
-
-        @wraps(func)
-        async def debounced(*args, **kwargs):
-            nonlocal last_call, task
-            last_call = asyncio.get_event_loop().time()
-
-            if task:
-                task.cancel()
-
-            async def call_func():
-                await asyncio.sleep(wait)
-                if asyncio.get_event_loop().time() - last_call >= wait:  # type: ignore
-                    await func(*args, **kwargs)
-
-            task = asyncio.create_task(call_func())
-
-        return debounced
-
-    return decorator
-
-
-def get_default_data_dir() -> Path:
-    """Get the user data directory for the current system platform.
-
-    Linux: ~/.local/share/haiku.rag
-    macOS: ~/Library/Application Support/haiku.rag
-    Windows: C:/Users/<USER>/AppData/Roaming/haiku.rag
-
-    Returns:
-        User Data Path.
-    """
-    home = Path.home()
-
-    system_paths = {
-        "win32": home / "AppData/Roaming/haiku.rag",
-        "linux": home / ".local/share/haiku.rag",
-        "darwin": home / "Library/Application Support/haiku.rag",
-    }
-
-    data_path = system_paths[sys.platform]
-    return data_path
-
-
-async def is_up_to_date() -> tuple[bool, Version, Version]:
-    """Check whether haiku.rag is current.
-
-    Returns:
-        A tuple containing a boolean indicating whether haiku.rag is current,
-        the running version and the latest version.
-    """
-
-    async with httpx.AsyncClient() as client:
-        running_version = parse(metadata.version("haiku.rag"))
-        try:
-            response = await client.get("https://pypi.org/pypi/haiku.rag/json")
-            data = response.json()
-            pypi_version = parse(data["info"]["version"])
-        except Exception:
-            # If no network connection, do not raise alarms.
-            pypi_version = running_version
-    return running_version >= pypi_version, running_version, pypi_version
-
-
-def text_to_docling_document(text: str, name: str = "content.md") -> DoclingDocument:
-    """Convert text content to a DoclingDocument.
-
-    Args:
-        text: The text content to convert.
-        name: The name to use for the document stream (defaults to "content.md").
-
-    Returns:
-        A DoclingDocument created from the text content.
-    """
-    bytes_io = BytesIO(text.encode("utf-8"))
-    doc_stream = DocumentStream(name=name, stream=bytes_io)
-    converter = DocumentConverter()
-    result = converter.convert(doc_stream)
-    return result.document
-
-
-def load_callable(path: str):
-    """Load a callable from a dotted path or file path.
-
-    Supported formats:
-    - "package.module:func" or "package.module.func"
-    - "path/to/file.py:func"
-
-    Returns the loaded callable. Raises ValueError on failure.
-    """
-    if not path:
-        raise ValueError("Empty callable path provided")
-
-    module_part = None
-    func_name = None
-
-    if ":" in path:
-        module_part, func_name = path.split(":", 1)
-    else:
-        # split by last dot for module.attr
-        if "." in path:
-            module_part, func_name = path.rsplit(".", 1)
-        else:
-            raise ValueError(
-                "Invalid callable path format. Use 'module:func' or 'module.func' or 'file.py:func'."
-            )
-
-    # Try file path first
-    mod: ModuleType | None = None
-    module_path = Path(module_part)
-    if module_path.suffix == ".py" and module_path.exists():
-        spec = importlib.util.spec_from_file_location(module_path.stem, module_path)
-        if spec and spec.loader:
-            mod = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(mod)
-    else:
-        # Import as a module path
-        try:
-            mod = importlib.import_module(module_part)
-        except Exception as e:
-            raise ValueError(f"Failed to import module '{module_part}': {e}")
-
-    if not hasattr(mod, func_name):
-        raise ValueError(f"Callable '{func_name}' not found in module '{module_part}'")
-    func = getattr(mod, func_name)
-    if not callable(func):
-        raise ValueError(
-            f"Attribute '{func_name}' in module '{module_part}' is not callable"
-        )
-    return func

haiku_rag-0.9.2.dist-info/METADATA

@@ -1,131 +0,0 @@
-Metadata-Version: 2.4
-Name: haiku.rag
-Version: 0.9.2
-Summary: Agentic Retrieval Augmented Generation (RAG) with LanceDB
-Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
-License: MIT
-License-File: LICENSE
-Keywords: RAG,lancedb,mcp,ml,vector-database
-Classifier: Development Status :: 4 - Beta
-Classifier: Environment :: Console
-Classifier: Intended Audience :: Developers
-Classifier: Operating System :: MacOS
-Classifier: Operating System :: Microsoft :: Windows :: Windows 10
-Classifier: Operating System :: Microsoft :: Windows :: Windows 11
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Typing :: Typed
-Requires-Python: >=3.12
-Requires-Dist: docling>=2.52.0
-Requires-Dist: fastmcp>=2.12.3
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: lancedb>=0.25.0
-Requires-Dist: pydantic-ai>=1.0.8
-Requires-Dist: pydantic>=2.11.9
-Requires-Dist: python-dotenv>=1.1.1
-Requires-Dist: rich>=14.1.0
-Requires-Dist: tiktoken>=0.11.0
-Requires-Dist: typer>=0.16.1
-Requires-Dist: watchfiles>=1.1.0
-Provides-Extra: mxbai
-Requires-Dist: mxbai-rerank>=0.1.6; extra == 'mxbai'
-Provides-Extra: voyageai
-Requires-Dist: voyageai>=0.3.5; extra == 'voyageai'
-Description-Content-Type: text/markdown
-
-# Haiku RAG
-
-Retrieval-Augmented Generation (RAG) library built on LanceDB.
-
-`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
-
-> **Note**: Starting with version 0.7.0, haiku.rag uses LanceDB instead of SQLite. If you have an existing SQLite database, use `haiku-rag migrate old_database.sqlite` to migrate your data safely.
-
-## Features
-
-- **Local LanceDB**: No external servers required, supports also LanceDB cloud storage, S3, Google Cloud & Azure
-- **Multiple embedding providers**: Ollama, VoyageAI, OpenAI, vLLM
-- **Multiple QA providers**: Any provider/model supported by Pydantic AI
-- **Native hybrid search**: Vector + full-text search with native LanceDB RRF reranking
-- **Reranking**: Default search result reranking with MixedBread AI, Cohere, or vLLM
-- **Question answering**: Built-in QA agents on your documents
-- **File monitoring**: Auto-index files when run as server
-- **40+ file formats**: PDF, DOCX, HTML, Markdown, code files, URLs
-- **MCP server**: Expose as tools for AI assistants
-- **CLI & Python API**: Use from command line or Python
-
-## Quick Start
-
-```bash
-# Install
-uv pip install haiku.rag
-
-# Add documents
-haiku-rag add "Your content here"
-haiku-rag add-src document.pdf
-
-# Search
-haiku-rag search "query"
-
-# Ask questions
-haiku-rag ask "Who is the author of haiku.rag?"
-
-# Ask questions with citations
-haiku-rag ask "Who is the author of haiku.rag?" --cite
-
-# Rebuild database (re-chunk and re-embed all documents)
-haiku-rag rebuild
-
-# Migrate from SQLite to LanceDB
-haiku-rag migrate old_database.sqlite
-
-# Start server with file monitoring
-export MONITOR_DIRECTORIES="/path/to/docs"
-haiku-rag serve
-```
-
-## Python Usage
-
-```python
-from haiku.rag.client import HaikuRAG
-
-async with HaikuRAG("database.lancedb") as client:
-    # Add document
-    doc = await client.create_document("Your content")
-
-    # Search (reranking enabled by default)
-    results = await client.search("query")
-    for chunk, score in results:
-        print(f"{score:.3f}: {chunk.content}")
-
-    # Ask questions
-    answer = await client.ask("Who is the author of haiku.rag?")
-    print(answer)
-
-    # Ask questions with citations
-    answer = await client.ask("Who is the author of haiku.rag?", cite=True)
-    print(answer)
-```
-
-## MCP Server
-
-Use with AI assistants like Claude Desktop:
-
-```bash
-haiku-rag serve --stdio
-```
-
-Provides tools for document management and search directly in your AI assistant.
-
-## Documentation
-
-Full documentation at: https://ggozad.github.io/haiku.rag/
-
-- [Installation](https://ggozad.github.io/haiku.rag/installation/) - Provider setup
-- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
-- [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
-- [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs
-- [Agents](https://ggozad.github.io/haiku.rag/agents/) - QA agent and multi-agent research
-- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks