haiku.rag-slim 0.16.0__py3-none-any.whl

haiku/rag/store/repositories/settings.py

@@ -0,0 +1,165 @@
+import json
+
+from haiku.rag.store.engine import SettingsRecord, Store
+
+
+class ConfigMismatchError(Exception):
+    """Raised when stored config doesn't match current config."""
+
+    pass
+
+
+class SettingsRepository:
+    """Repository for Settings operations."""
+
+    def __init__(self, store: Store) -> None:
+        self.store = store
+
+    async def create(self, entity: dict) -> dict:
+        """Create settings in the database."""
+        settings_record = SettingsRecord(id="settings", settings=json.dumps(entity))
+        self.store.settings_table.add([settings_record])
+        return entity
+
+    async def get_by_id(self, entity_id: str) -> dict | None:
+        """Get settings by ID."""
+        results = list(
+            self.store.settings_table.search()
+            .where(f"id = '{entity_id}'")
+            .limit(1)
+            .to_pydantic(SettingsRecord)
+        )
+
+        if not results:
+            return None
+
+        return json.loads(results[0].settings) if results[0].settings else {}
+
+    async def update(self, entity: dict) -> dict:
+        """Update existing settings."""
+        self.store.settings_table.update(
+            where="id = 'settings'", values={"settings": json.dumps(entity)}
+        )
+        return entity
+
+    async def delete(self, entity_id: str) -> bool:
+        """Delete settings by ID."""
+        self.store.settings_table.delete(f"id = '{entity_id}'")
+        return True
+
+    async def list_all(
+        self, limit: int | None = None, offset: int | None = None
+    ) -> list[dict]:
+        """List all settings."""
+        results = list(self.store.settings_table.search().to_pydantic(SettingsRecord))
+        return [
+            json.loads(record.settings) if record.settings else {} for record in results
+        ]
+
+    def get_current_settings(self) -> dict:
+        """Get the current settings."""
+        results = list(
+            self.store.settings_table.search()
+            .where("id = 'settings'")
+            .limit(1)
+            .to_pydantic(SettingsRecord)
+        )
+
+        if not results:
+            return {}
+
+        return json.loads(results[0].settings) if results[0].settings else {}
+
+    def save_current_settings(self) -> None:
+        """Save the current configuration to the database."""
+        current_config = self.store._config.model_dump(mode="json")
+
+        # Check if settings exist
+        existing = list(
+            self.store.settings_table.search()
+            .where("id = 'settings'")
+            .limit(1)
+            .to_pydantic(SettingsRecord)
+        )
+
+        if existing:
+            # Preserve existing version if present to avoid interfering with upgrade flow
+            try:
+                existing_settings = (
+                    json.loads(existing[0].settings) if existing[0].settings else {}
+                )
+            except Exception:
+                existing_settings = {}
+            if "version" in existing_settings:
+                current_config["version"] = existing_settings["version"]
+
+            # Update existing settings
+            if existing_settings != current_config:
+                self.store.settings_table.update(
+                    where="id = 'settings'",
+                    values={"settings": json.dumps(current_config)},
+                )
+        else:
+            # Create new settings
+            settings_record = SettingsRecord(
+                id="settings", settings=json.dumps(current_config)
+            )
+            self.store.settings_table.add([settings_record])
+
+    def validate_config_compatibility(self) -> None:
+        """Validate that the current configuration is compatible with stored settings."""
+        stored_settings = self.get_current_settings()
+
+        # If no stored settings, this is a new database - save current config and return
+        if not stored_settings:
+            self.save_current_settings()
+            return
+
+        current_config = self.store._config.model_dump(mode="json")
+
+        # Check if embedding provider or model has changed
+        # Support both old flat structure and new nested structure for backward compatibility
+        stored_embeddings = stored_settings.get("embeddings", {})
+        current_embeddings = current_config.get("embeddings", {})
+
+        # Try nested structure first, fall back to flat for old databases
+        stored_provider = stored_embeddings.get("provider") or stored_settings.get(
+            "EMBEDDINGS_PROVIDER"
+        )
+        current_provider = current_embeddings.get("provider")
+
+        stored_model = stored_embeddings.get("model") or stored_settings.get(
+            "EMBEDDINGS_MODEL"
+        )
+        current_model = current_embeddings.get("model")
+
+        stored_vector_dim = stored_embeddings.get("vector_dim") or stored_settings.get(
+            "EMBEDDINGS_VECTOR_DIM"
+        )
+        current_vector_dim = current_embeddings.get("vector_dim")
+
+        # Check for incompatible changes
+        incompatible_changes = []
+
+        if stored_provider and stored_provider != current_provider:
+            incompatible_changes.append(
+                f"Stored (db) embedding provider: '{stored_provider}' -> Environment (current) embedding provider: '{current_provider}'"
+            )
+
+        if stored_model and stored_model != current_model:
+            incompatible_changes.append(
+                f"Stored (db) embedding model '{stored_model}' -> Environment (current) embedding model '{current_model}'"
+            )
+
+        if stored_vector_dim and stored_vector_dim != current_vector_dim:
+            incompatible_changes.append(
+                f"Stored (db) embedding vector dimension {stored_vector_dim} -> Environment (current) embedding vector dimension {current_vector_dim}"
+            )
+
+        if incompatible_changes:
+            error_msg = (
+                "Database configuration is incompatible with current settings:\n"
+                + "\n".join(f"  - {change}" for change in incompatible_changes)
+            )
+            error_msg += "\n\nPlease rebuild the database using: haiku-rag rebuild"
+            raise ConfigMismatchError(error_msg)

haiku/rag/store/upgrades/__init__.py

@@ -0,0 +1,62 @@
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from packaging.version import Version, parse
+
+from haiku.rag.store.engine import Store
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Upgrade:
+    """Represents a database upgrade step."""
+
+    version: str
+    apply: Callable[[Store], None]
+    description: str = ""
+
+
+# Registry of upgrade steps (ordered by version)
+upgrades: list[Upgrade] = []
+
+
+def run_pending_upgrades(store: Store, from_version: str, to_version: str) -> None:
+    """Run upgrades where from_version < step.version <= to_version."""
+    v_from: Version = parse(from_version)
+    v_to: Version = parse(to_version)
+
+    # Ensure that tests/development run available code upgrades even if the
+    # installed package version hasn't been bumped to include them yet.
+    if upgrades:
+        highest_step_version: Version = max(parse(u.version) for u in upgrades)
+        if highest_step_version > v_to:
+            v_to = highest_step_version
+
+    # Determine applicable steps
+    sorted_steps = sorted(upgrades, key=lambda u: parse(u.version))
+    applicable = [s for s in sorted_steps if v_from < parse(s.version) <= v_to]
+    if applicable:
+        logger.info("%d upgrade step(s) pending", len(applicable))
+
+    # Apply in ascending order
+    for idx, step in enumerate(applicable, start=1):
+        logger.info(
+            "Applying upgrade %s: %s (%d/%d)",
+            step.version,
+            step.description or "",
+            idx,
+            len(applicable),
+        )
+        step.apply(store)
+        logger.info("Completed upgrade %s", step.version)
+
+
+from .v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts  # noqa: E402
+from .v0_9_3 import upgrade_order as upgrade_0_9_3_order  # noqa: E402
+from .v0_10_1 import upgrade_add_title as upgrade_0_10_1_add_title  # noqa: E402
+
+upgrades.append(upgrade_0_9_3_order)
+upgrades.append(upgrade_0_9_3_fts)
+upgrades.append(upgrade_0_10_1_add_title)

haiku/rag/store/upgrades/v0_10_1.py

@@ -0,0 +1,64 @@
+import json
+
+from lancedb.pydantic import LanceModel
+from pydantic import Field
+
+from haiku.rag.store.engine import Store
+from haiku.rag.store.upgrades import Upgrade
+
+
+def _apply_add_document_title(store: Store) -> None:
+    """Add a nullable 'title' column to the documents table."""
+
+    # Read existing rows using Arrow for schema-agnostic access
+    try:
+        docs_arrow = store.documents_table.search().to_arrow()
+        rows = docs_arrow.to_pylist()
+    except Exception:
+        rows = []
+
+    class DocumentRecordV2(LanceModel):
+        id: str
+        content: str
+        uri: str | None = None
+        title: str | None = None
+        metadata: str = Field(default="{}")
+        created_at: str = Field(default_factory=lambda: "")
+        updated_at: str = Field(default_factory=lambda: "")
+
+    # Drop and recreate documents table with the new schema
+    try:
+        store.db.drop_table("documents")
+    except Exception:
+        pass
+
+    store.documents_table = store.db.create_table("documents", schema=DocumentRecordV2)
+
+    # Reinsert previous rows with title=None
+    if rows:
+        backfilled = []
+        for row in rows:
+            backfilled.append(
+                DocumentRecordV2(
+                    id=row.get("id"),
+                    content=row.get("content", ""),
+                    uri=row.get("uri"),
+                    title=None,
+                    metadata=(
+                        row.get("metadata")
+                        if isinstance(row.get("metadata"), str)
+                        else json.dumps(row.get("metadata") or {})
+                    ),
+                    created_at=row.get("created_at", ""),
+                    updated_at=row.get("updated_at", ""),
+                )
+            )
+
+        store.documents_table.add(backfilled)
+
+
+upgrade_add_title = Upgrade(
+    version="0.10.1",
+    apply=_apply_add_document_title,
+    description="Add nullable 'title' column to documents table",
+)

haiku/rag/store/upgrades/v0_9_3.py

@@ -0,0 +1,112 @@
+import json
+
+from lancedb.pydantic import LanceModel, Vector
+from pydantic import Field
+
+from haiku.rag.store.engine import Store
+from haiku.rag.store.upgrades import Upgrade
+
+
+def _infer_vector_dim(store: Store) -> int:
+    """Infer vector dimension from existing data; fallback to embedder config."""
+    try:
+        arrow = store.chunks_table.search().limit(1).to_arrow()
+        rows = arrow.to_pylist()
+        if rows:
+            vec = rows[0].get("vector")
+            if isinstance(vec, list) and vec:
+                return len(vec)
+    except Exception:
+        pass
+    # Fallback to configured embedder vector dim
+    return getattr(store.embedder, "_vector_dim", 1024)
+
+
+def _apply_chunk_order(store: Store) -> None:
+    """Add integer 'order' column to chunks and backfill from metadata."""
+
+    vector_dim = _infer_vector_dim(store)
+
+    class ChunkRecordV2(LanceModel):
+        id: str
+        document_id: str
+        content: str
+        metadata: str = Field(default="{}")
+        order: int = Field(default=0)
+        vector: Vector(vector_dim) = Field(  # type: ignore
+            default_factory=lambda: [0.0] * vector_dim
+        )
+
+    # Read existing chunks
+    try:
+        chunks_arrow = store.chunks_table.search().to_arrow()
+        rows = chunks_arrow.to_pylist()
+    except Exception:
+        rows = []
+
+    new_chunk_records: list[ChunkRecordV2] = []
+    for row in rows:
+        md_raw = row.get("metadata") or "{}"
+        try:
+            md = json.loads(md_raw) if isinstance(md_raw, str) else md_raw
+        except Exception:
+            md = {}
+        # Extract and normalize order
+        order_val = 0
+        try:
+            if isinstance(md, dict) and "order" in md:
+                order_val = int(md["order"])  # type: ignore[arg-type]
+        except Exception:
+            order_val = 0
+
+        if isinstance(md, dict) and "order" in md:
+            md = {k: v for k, v in md.items() if k != "order"}
+
+        vec = row.get("vector") or [0.0] * vector_dim
+
+        new_chunk_records.append(
+            ChunkRecordV2(
+                id=row.get("id"),
+                document_id=row.get("document_id"),
+                content=row.get("content", ""),
+                metadata=json.dumps(md),
+                order=order_val,
+                vector=vec,
+            )
+        )
+
+    # Recreate chunks table with new schema
+    try:
+        store.db.drop_table("chunks")
+    except Exception:
+        pass
+
+    store.chunks_table = store.db.create_table("chunks", schema=ChunkRecordV2)
+    store.chunks_table.create_fts_index("content", replace=True)
+
+    if new_chunk_records:
+        store.chunks_table.add(new_chunk_records)
+
+
+upgrade_order = Upgrade(
+    version="0.9.3",
+    apply=_apply_chunk_order,
+    description="Add 'order' column to chunks and backfill from metadata",
+)
+
+
+def _apply_fts_phrase_support(store: Store) -> None:
+    """Recreate FTS index with phrase query support and no stop-word removal."""
+    try:
+        store.chunks_table.create_fts_index(
+            "content", replace=True, with_position=True, remove_stop_words=False
+        )
+    except Exception:
+        pass
+
+
+upgrade_fts_phrase = Upgrade(
+    version="0.9.3",
+    apply=_apply_fts_phrase_support,
+    description="Enable FTS phrase queries (with positions) and keep stop-words",
+)

haiku/rag/utils.py

@@ -0,0 +1,211 @@
+import asyncio
+import importlib
+import importlib.util
+import sys
+from collections.abc import Callable
+from functools import wraps
+from importlib import metadata
+from io import BytesIO
+from pathlib import Path
+from types import ModuleType
+
+from packaging.version import Version, parse
+
+
+def debounce(wait: float) -> Callable:
+    """
+    A decorator to debounce a function, ensuring it is called only after a specified delay
+    and always executes after the last call.
+
+    Args:
+        wait (float): The debounce delay in seconds.
+
+    Returns:
+        Callable: The decorated function.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        last_call = None
+        task = None
+
+        @wraps(func)
+        async def debounced(*args, **kwargs):
+            nonlocal last_call, task
+            last_call = asyncio.get_event_loop().time()
+
+            if task:
+                task.cancel()
+
+            async def call_func():
+                await asyncio.sleep(wait)
+                if asyncio.get_event_loop().time() - last_call >= wait:  # type: ignore
+                    await func(*args, **kwargs)
+
+            task = asyncio.create_task(call_func())
+
+        return debounced
+
+    return decorator
+
+
+def get_default_data_dir() -> Path:
+    """Get the user data directory for the current system platform.
+
+    Linux: ~/.local/share/haiku.rag
+    macOS: ~/Library/Application Support/haiku.rag
+    Windows: C:/Users/<USER>/AppData/Roaming/haiku.rag
+
+    Returns:
+        User Data Path.
+    """
+    home = Path.home()
+
+    system_paths = {
+        "win32": home / "AppData/Roaming/haiku.rag",
+        "linux": home / ".local/share/haiku.rag",
+        "darwin": home / "Library/Application Support/haiku.rag",
+    }
+
+    data_path = system_paths[sys.platform]
+    return data_path
+
+
+async def is_up_to_date() -> tuple[bool, Version, Version]:
+    """Check whether haiku.rag is current.
+
+    Returns:
+        A tuple containing a boolean indicating whether haiku.rag is current,
+        the running version and the latest version.
+    """
+
+    # Lazy import to avoid pulling httpx (and its deps) on module import
+    import httpx
+
+    async with httpx.AsyncClient() as client:
+        running_version = parse(metadata.version("haiku.rag-slim"))
+        try:
+            response = await client.get("https://pypi.org/pypi/haiku.rag/json")
+            data = response.json()
+            pypi_version = parse(data["info"]["version"])
+        except Exception:
+            # If no network connection, do not raise alarms.
+            pypi_version = running_version
+    return running_version >= pypi_version, running_version, pypi_version
+
+
+def text_to_docling_document(text: str, name: str = "content.md"):
+    """Convert text content to a DoclingDocument.
+
+    Args:
+        text: The text content to convert.
+        name: The name to use for the document stream (defaults to "content.md").
+
+    Returns:
+        A DoclingDocument created from the text content.
+    """
+    try:
+        import docling  # noqa: F401
+    except ImportError as e:
+        raise ImportError(
+            "Docling is required for document conversion. "
+            "Install with: pip install haiku.rag-slim[docling]"
+        ) from e
+
+    from docling.document_converter import DocumentConverter
+    from docling_core.types.io import DocumentStream
+
+    bytes_io = BytesIO(text.encode("utf-8"))
+    doc_stream = DocumentStream(name=name, stream=bytes_io)
+    converter = DocumentConverter()
+    result = converter.convert(doc_stream)
+    return result.document
+
+
+def load_callable(path: str):
+    """Load a callable from a dotted path or file path.
+
+    Supported formats:
+    - "package.module:func" or "package.module.func"
+    - "path/to/file.py:func"
+
+    Returns the loaded callable. Raises ValueError on failure.
+    """
+    if not path:
+        raise ValueError("Empty callable path provided")
+
+    module_part = None
+    func_name = None
+
+    if ":" in path:
+        module_part, func_name = path.split(":", 1)
+    else:
+        # split by last dot for module.attr
+        if "." in path:
+            module_part, func_name = path.rsplit(".", 1)
+        else:
+            raise ValueError(
+                "Invalid callable path format. Use 'module:func' or 'module.func' or 'file.py:func'."
+            )
+
+    # Try file path first
+    mod: ModuleType | None = None
+    module_path = Path(module_part)
+    if module_path.suffix == ".py" and module_path.exists():
+        spec = importlib.util.spec_from_file_location(module_path.stem, module_path)
+        if spec and spec.loader:
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+    else:
+        # Import as a module path
+        try:
+            mod = importlib.import_module(module_part)
+        except Exception as e:
+            raise ValueError(f"Failed to import module '{module_part}': {e}")
+
+    if not hasattr(mod, func_name):
+        raise ValueError(f"Callable '{func_name}' not found in module '{module_part}'")
+    func = getattr(mod, func_name)
+    if not callable(func):
+        raise ValueError(
+            f"Attribute '{func_name}' in module '{module_part}' is not callable"
+        )
+    return func
+
+
+def prefetch_models():
+    """Prefetch runtime models (Docling + Ollama as configured)."""
+    import httpx
+
+    from haiku.rag.config import Config
+
+    try:
+        from docling.utils.model_downloader import download_models
+
+        download_models()
+    except ImportError:
+        # Docling not installed, skip downloading docling models
+        pass
+
+    # Collect Ollama models from config
+    required_models: set[str] = set()
+    if Config.embeddings.provider == "ollama":
+        required_models.add(Config.embeddings.model)
+    if Config.qa.provider == "ollama":
+        required_models.add(Config.qa.model)
+    if Config.research.provider == "ollama":
+        required_models.add(Config.research.model)
+    if Config.reranking.provider == "ollama":
+        required_models.add(Config.reranking.model)
+
+    if not required_models:
+        return
+
+    base_url = Config.providers.ollama.base_url
+
+    with httpx.Client(timeout=None) as client:
+        for model in sorted(required_models):
+            with client.stream(
+                "POST", f"{base_url}/api/pull", json={"model": model}
+            ) as r:
+                for _ in r.iter_lines():
+                    pass