haiku.rag 0.6.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

haiku/rag/utils.py

@@ -1,4 +1,7 @@
+import asyncio
 import sys
+from collections.abc import Callable
+from functools import wraps
 from importlib import metadata
 from io import BytesIO
 from pathlib import Path
@@ -10,6 +13,42 @@ from docling_core.types.io import DocumentStream
 from packaging.version import Version, parse
 
 
+def debounce(wait: float) -> Callable:
+    """
+    A decorator to debounce a function, ensuring it is called only after a specified delay
+    and always executes after the last call.
+
+    Args:
+        wait (float): The debounce delay in seconds.
+
+    Returns:
+        Callable: The decorated function.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        last_call = None
+        task = None
+
+        @wraps(func)
+        async def debounced(*args, **kwargs):
+            nonlocal last_call, task
+            last_call = asyncio.get_event_loop().time()
+
+            if task:
+                task.cancel()
+
+            async def call_func():
+                await asyncio.sleep(wait)
+                if asyncio.get_event_loop().time() - last_call >= wait:  # type: ignore
+                    await func(*args, **kwargs)
+
+            task = asyncio.create_task(call_func())
+
+        return debounced
+
+    return decorator
+
+
 def get_default_data_dir() -> Path:
     """Get the user data directory for the current system platform.
 
@@ -32,37 +71,6 @@ def get_default_data_dir() -> Path:
     return data_path
 
 
-def semantic_version_to_int(version: str) -> int:
-    """Convert a semantic version string to an integer.
-
-    Args:
-        version: Semantic version string.
-
-    Returns:
-        Integer representation of semantic version.
-    """
-    major, minor, patch = version.split(".")
-    major = int(major) << 16
-    minor = int(minor) << 8
-    patch = int(patch)
-    return major + minor + patch
-
-
-def int_to_semantic_version(version: int) -> str:
-    """Convert an integer to a semantic version string.
-
-    Args:
-        version: Integer representation of semantic version.
-
-    Returns:
-        Semantic version string.
-    """
-    major = version >> 16
-    minor = (version >> 8) & 255
-    patch = version & 255
-    return f"{major}.{minor}.{patch}"
-
-
 async def is_up_to_date() -> tuple[bool, Version, Version]:
     """Check whether haiku.rag is current.
 

{haiku_rag-0.6.0.dist-info → haiku_rag-0.7.1.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.6.0
-Summary: Retrieval Augmented Generation (RAG) with SQLite
+Version: 0.7.1
+Summary: Retrieval Augmented Generation (RAG) with LanceDB
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
 License-File: LICENSE
-Keywords: RAG,mcp,ml,sqlite,sqlite-vec
+Keywords: RAG,lancedb,mcp,ml,vector-database
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -17,18 +17,18 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Typing :: Typed
-Requires-Python: >=3.11
-Requires-Dist: docling>=2.15.0
+Requires-Python: >=3.12
+Requires-Dist: docling>=2.49.0
 Requires-Dist: fastmcp>=2.8.1
 Requires-Dist: httpx>=0.28.1
+Requires-Dist: lancedb>=0.24.3
 Requires-Dist: ollama>=0.5.3
-Requires-Dist: pydantic-ai>=0.7.2
+Requires-Dist: pydantic-ai>=0.8.1
 Requires-Dist: pydantic>=2.11.7
 Requires-Dist: python-dotenv>=1.1.0
-Requires-Dist: rich>=14.0.0
-Requires-Dist: sqlite-vec>=0.1.6
-Requires-Dist: tiktoken>=0.9.0
-Requires-Dist: typer>=0.16.0
+Requires-Dist: rich>=14.1.0
+Requires-Dist: tiktoken>=0.11.0
+Requires-Dist: typer>=0.16.1
 Requires-Dist: watchfiles>=1.1.0
 Provides-Extra: mxbai
 Requires-Dist: mxbai-rerank>=0.1.6; extra == 'mxbai'
@@ -36,18 +36,20 @@ Provides-Extra: voyageai
 Requires-Dist: voyageai>=0.3.2; extra == 'voyageai'
 Description-Content-Type: text/markdown
 
-# Haiku SQLite RAG
+# Haiku RAG
 
-Retrieval-Augmented Generation (RAG) library on SQLite.
+Retrieval-Augmented Generation (RAG) library built on LanceDB.
 
-`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work on SQLite alone without the need for external vector databases. It uses [sqlite-vec](https://github.com/asg017/sqlite-vec) for storing the embeddings and performs semantic (vector) search as well as full-text search combined through Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
+`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
+
+> **Note**: Starting with version 0.7.0, haiku.rag uses LanceDB instead of SQLite. If you have an existing SQLite database, use `haiku-rag migrate old_database.sqlite` to migrate your data safely.
 
 ## Features
 
-- **Local SQLite**: No external servers required
+- **Local LanceDB**: No external servers required, supports also LanceDB cloud storage, S3, Google Cloud & Azure
 - **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
 - **Multiple QA providers**: Any provider/model supported by Pydantic AI
-- **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
+- **Native hybrid search**: Vector + full-text search with native LanceDB RRF reranking
 - **Reranking**: Default search result reranking with MixedBread AI or Cohere
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
@@ -77,6 +79,9 @@ haiku-rag ask "Who is the author of haiku.rag?" --cite
 # Rebuild database (re-chunk and re-embed all documents)
 haiku-rag rebuild
 
+# Migrate from SQLite to LanceDB
+haiku-rag migrate old_database.sqlite
+
 # Start server with file monitoring
 export MONITOR_DIRECTORIES="/path/to/docs"
 haiku-rag serve
@@ -87,7 +92,7 @@ haiku-rag serve
 ```python
 from haiku.rag.client import HaikuRAG
 
-async with HaikuRAG("database.db") as client:
+async with HaikuRAG("database.lancedb") as client:
     # Add document
     doc = await client.create_document("Your content")
 

haiku_rag-0.7.1.dist-info/RECORD

@@ -0,0 +1,39 @@
+haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+haiku/rag/app.py,sha256=GmuZxH7BMutWt8Mdu0RSateRBaKiqXh7Z9tV7cZX6n0,7655
+haiku/rag/chunker.py,sha256=PVe6ysv8UlacUd4Zb3_8RFWIaWDXnzBAy2VDJ4TaUsE,1555
+haiku/rag/cli.py,sha256=UY9Vh5RsIxSCV14eQbNOiwToKmbFAvqTOAnxjieaYBs,6399
+haiku/rag/client.py,sha256=N4zkWjE9Rsw9YgPvNo83xptHUQR2ognfOnjkoV_w6hc,20999
+haiku/rag/config.py,sha256=9Mv0QJ3c6VF1oVRSXJlSsG234dCd_sKnJO-ybMaTpDQ,1690
+haiku/rag/logging.py,sha256=DOQi9QMpQRl8h17Vu4nQh8HxpHdeIu29n8-HZaT3SRQ,786
+haiku/rag/mcp.py,sha256=bR9Y-Nz-hvjiql20Y0KE0hwNGwyjmPGX8K9d-qmXptY,4683
+haiku/rag/migration.py,sha256=gWxQwiKo0YulRhogYz4K8N98kHN9LQXIx9FeTmT24v4,10915
+haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
+haiku/rag/reader.py,sha256=qkPTMJuQ_o4sK-8zpDl9WFYe_MJ7aL_gUw6rczIpW-g,3274
+haiku/rag/utils.py,sha256=c8F0ECsFSqvQxzxINAOAnvShoOnJPLsOaNE3JEY2JSc,3230
+haiku/rag/embeddings/__init__.py,sha256=n7aHW3BxHlpGxU4ze4YYDOsljzFpEep8dwVE2n45JoE,1218
+haiku/rag/embeddings/base.py,sha256=NTQvuzbZPu0LBo5wAu3qGyJ4xXUaRAt1fjBO0ygWn_Y,465
+haiku/rag/embeddings/ollama.py,sha256=y6-lp0XpbnyIjoOEdtSzMdEVkU5glOwnWQ1FkpUZnpI,370
+haiku/rag/embeddings/openai.py,sha256=iA-DewCOSip8PLU_RhEJHFHBle4DtmCCIGNfGs58Wvk,357
+haiku/rag/embeddings/voyageai.py,sha256=0hiRTIqu-bpl-4OaCtMHvWfPdgbrzhnfZJowSV8pLRA,415
+haiku/rag/qa/__init__.py,sha256=Sl7Kzrg9CuBOcMF01wc1NtQhUNWjJI0MhIHfCWrb8V4,434
+haiku/rag/qa/agent.py,sha256=gZ12vLUSHHCMl0HyPoLlPDbhUWoyEUydXG7u8lG1eqg,2602
+haiku/rag/qa/prompts.py,sha256=xdT4cyrOrAK9UDgVqyev1wHF49jD57Bh40gx2sH4NPI,3341
+haiku/rag/reranking/__init__.py,sha256=IRXHs4qPu6VbGJQpzSwhgtVWWumURH_vEoVFE-extlo,894
+haiku/rag/reranking/base.py,sha256=LM9yUSSJ414UgBZhFTgxGprlRqzfTe4I1vgjricz2JY,405
+haiku/rag/reranking/cohere.py,sha256=1iTdiaa8vvb6oHVB2qpWzUOVkyfUcimVSZp6Qr4aq4c,1049
+haiku/rag/reranking/mxbai.py,sha256=46sVTsTIkzIX9THgM3u8HaEmgY7evvEyB-N54JTHvK8,867
+haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
+haiku/rag/store/engine.py,sha256=XHGo5Xl-dCFdQHrOdMo64xVK5n0k8-LoUl5V-tlA0HI,7131
+haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
+haiku/rag/store/models/chunk.py,sha256=ZNyTfO6lh3rXWLVYO3TZcitbL4LSUGr42fR6jQQ5iQc,364
+haiku/rag/store/models/document.py,sha256=zSSpt6pyrMJAIXGQvIcqojcqUzwZnhp3WxVokaWxNRc,396
+haiku/rag/store/repositories/__init__.py,sha256=Olv5dLfBQINRV3HrsfUpjzkZ7Qm7goEYyMNykgo_DaY,291
+haiku/rag/store/repositories/chunk.py,sha256=5S77mGh6pWxPHjaXriJGmvbSOhoNM8tLwygE2GXPlbU,13586
+haiku/rag/store/repositories/document.py,sha256=lP8Lo82KTP-qwXFRpYZ46WjeAdAsHwZ5pJcrXdz4g0U,6988
+haiku/rag/store/repositories/settings.py,sha256=dqnAvm-98nQrWpLBbf9QghJw673QD80-iqQhRMP5t0c,5025
+haiku/rag/store/upgrades/__init__.py,sha256=wUiEoSiHTahvuagx93E4FB07v123AhdbOjwUkPusiIg,14
+haiku_rag-0.7.1.dist-info/METADATA,sha256=yIz4nATa_b2vDstrCZe9CzV_77Vw74QIhRWqCZCdaes,4597
+haiku_rag-0.7.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.7.1.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.7.1.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.7.1.dist-info/RECORD,,

haiku/rag/reranking/ollama.py

@@ -1,81 +0,0 @@
-from pydantic import BaseModel
-from pydantic_ai import Agent
-from pydantic_ai.models.openai import OpenAIModel
-from pydantic_ai.providers.ollama import OllamaProvider
-
-from haiku.rag.config import Config
-from haiku.rag.reranking.base import RerankerBase
-from haiku.rag.store.models.chunk import Chunk
-
-
-class RerankResult(BaseModel):
-    """Individual rerank result with index and relevance score."""
-
-    index: int
-    relevance_score: float
-
-
-class RerankResponse(BaseModel):
-    """Response from the reranking model containing ranked results."""
-
-    results: list[RerankResult]
-
-
-class OllamaReranker(RerankerBase):
-    def __init__(self, model: str = Config.RERANK_MODEL):
-        self._model = model
-
-        # Create the reranking prompt
-        system_prompt = """You are a document reranking assistant. Given a query and a list of document chunks, you must rank them by relevance to the query.
-
-Return your response as a JSON object with a "results" array. Each result should have:
-- "index": the original index of the document (integer)
-- "relevance_score": a score between 0.0 and 1.0 indicating relevance (float, where 1.0 is most relevant)
-
-Only return the top documents up to the requested limit, ordered by decreasing relevance score.
-/no_think
-"""
-
-        model_obj = OpenAIModel(
-            model_name=model,
-            provider=OllamaProvider(base_url=f"{Config.OLLAMA_BASE_URL}/v1"),
-        )
-
-        self._agent = Agent(
-            model=model_obj,
-            output_type=RerankResponse,
-            system_prompt=system_prompt,
-        )
-
-    async def rerank(
-        self, query: str, chunks: list[Chunk], top_n: int = 10
-    ) -> list[tuple[Chunk, float]]:
-        if not chunks:
-            return []
-
-        documents = []
-        for i, chunk in enumerate(chunks):
-            documents.append({"index": i, "content": chunk.content})
-
-        documents_text = ""
-        for doc in documents:
-            documents_text += f"Index {doc['index']}: {doc['content']}\n\n"
-
-        user_prompt = f"""Query: {query}
-
-Documents to rerank:
-{documents_text.strip()}
-
-Rank these documents by relevance to the query and return the top {top_n} results as JSON."""
-
-        try:
-            result = await self._agent.run(user_prompt)
-
-            return [
-                (chunks[result_item.index], result_item.relevance_score)
-                for result_item in result.output.results[:top_n]
-            ]
-
-        except Exception:
-            # Fallback: return chunks in original order with same score
-            return [(chunks[i], 1.0) for i in range(min(top_n, len(chunks)))]

haiku/rag/store/repositories/base.py

@@ -1,40 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Generic, TypeVar
-
-from haiku.rag.store.engine import Store
-
-T = TypeVar("T")
-
-
-class BaseRepository(ABC, Generic[T]):
-    """Base repository interface for database operations."""
-
-    def __init__(self, store: Store):
-        self.store = store
-
-    @abstractmethod
-    async def create(self, entity: T) -> T:
-        """Create a new entity in the database."""
-        pass
-
-    @abstractmethod
-    async def get_by_id(self, entity_id: int) -> T | None:
-        """Get an entity by its ID."""
-        pass
-
-    @abstractmethod
-    async def update(self, entity: T) -> T:
-        """Update an existing entity."""
-        pass
-
-    @abstractmethod
-    async def delete(self, entity_id: int) -> bool:
-        """Delete an entity by its ID."""
-        pass
-
-    @abstractmethod
-    async def list_all(
-        self, limit: int | None = None, offset: int | None = None
-    ) -> list[T]:
-        """List all entities with optional pagination."""
-        pass

haiku/rag/store/upgrades/v0_3_4.py

@@ -1,26 +0,0 @@
-from collections.abc import Callable
-from sqlite3 import Connection
-
-from haiku.rag.config import Config
-
-
-def add_settings_table(db: Connection) -> None:
-    """Create settings table for storing current configuration"""
-    db.execute("""
-        CREATE TABLE settings (
-            id INTEGER PRIMARY KEY DEFAULT 1,
-            settings TEXT NOT NULL DEFAULT '{}'
-        )
-    """)
-
-    settings_json = Config.model_dump_json()
-    db.execute(
-        "INSERT INTO settings (id, settings) VALUES (1, ?)",
-        (settings_json,),
-    )
-    db.commit()
-
-
-upgrades: list[tuple[str, list[Callable[[Connection], None]]]] = [
-    ("0.3.4", [add_settings_table])
-]

haiku_rag-0.6.0.dist-info/RECORD

@@ -1,41 +0,0 @@
-haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-haiku/rag/app.py,sha256=k45EOz-rbYg_8RSII3btqsZo2TpGqj3ysamFehhaCGo,7673
-haiku/rag/chunker.py,sha256=PVe6ysv8UlacUd4Zb3_8RFWIaWDXnzBAy2VDJ4TaUsE,1555
-haiku/rag/cli.py,sha256=mGpdnEH8rS-rZLGmE4MbcDci8uexci7UkGTdCxrz1Lg,5987
-haiku/rag/client.py,sha256=rpMKVxtb3Q2kuDM1pHaZsGX_w0TC1zxotsooNwI3Jbg,21129
-haiku/rag/config.py,sha256=oLrmwGp1OjcKPpJFnf9GgTpoBSOXalFWO6PCKFwQe0w,1615
-haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
-haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
-haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
-haiku/rag/reader.py,sha256=qkPTMJuQ_o4sK-8zpDl9WFYe_MJ7aL_gUw6rczIpW-g,3274
-haiku/rag/utils.py,sha256=g-uNTG60iBLgkeHHuah6eVZEkX3NFLs-LZU1YnzJzLQ,2967
-haiku/rag/embeddings/__init__.py,sha256=n7aHW3BxHlpGxU4ze4YYDOsljzFpEep8dwVE2n45JoE,1218
-haiku/rag/embeddings/base.py,sha256=NTQvuzbZPu0LBo5wAu3qGyJ4xXUaRAt1fjBO0ygWn_Y,465
-haiku/rag/embeddings/ollama.py,sha256=y6-lp0XpbnyIjoOEdtSzMdEVkU5glOwnWQ1FkpUZnpI,370
-haiku/rag/embeddings/openai.py,sha256=iA-DewCOSip8PLU_RhEJHFHBle4DtmCCIGNfGs58Wvk,357
-haiku/rag/embeddings/voyageai.py,sha256=0hiRTIqu-bpl-4OaCtMHvWfPdgbrzhnfZJowSV8pLRA,415
-haiku/rag/qa/__init__.py,sha256=Sl7Kzrg9CuBOcMF01wc1NtQhUNWjJI0MhIHfCWrb8V4,434
-haiku/rag/qa/agent.py,sha256=r6tYKvOW4W1HxBRHH1kmzlzb1bIJcQSuHd6cG9ANqXY,2594
-haiku/rag/qa/prompts.py,sha256=xdT4cyrOrAK9UDgVqyev1wHF49jD57Bh40gx2sH4NPI,3341
-haiku/rag/reranking/__init__.py,sha256=fwC3pauteJwh9Ulm2270QvwAdwr4NMr4RUEuolC-wKU,1063
-haiku/rag/reranking/base.py,sha256=LM9yUSSJ414UgBZhFTgxGprlRqzfTe4I1vgjricz2JY,405
-haiku/rag/reranking/cohere.py,sha256=1iTdiaa8vvb6oHVB2qpWzUOVkyfUcimVSZp6Qr4aq4c,1049
-haiku/rag/reranking/mxbai.py,sha256=46sVTsTIkzIX9THgM3u8HaEmgY7evvEyB-N54JTHvK8,867
-haiku/rag/reranking/ollama.py,sha256=Q3dJepxFyB9CRCtrZvcwX-Drrpa2-8TMO7YGhxD1Qcs,2610
-haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
-haiku/rag/store/engine.py,sha256=cOMBToLilI1Di1qQrFzGLqtRMsuvtiX0Q5RNIEzQy9w,6232
-haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
-haiku/rag/store/models/chunk.py,sha256=9-vIxW75-kMTelIhgVIMd_WhP-Drc1q65vjaWMP8w1E,364
-haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
-haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPvrDIHVhY5T1A,263
-haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
-haiku/rag/store/repositories/chunk.py,sha256=R8dvNy3po2FspZvkWKZTGlqccbekLjY39GroXRfAU18,18808
-haiku/rag/store/repositories/document.py,sha256=ki8LiDukwU1469Yw51i0rQFvBzUQeYkFYWs3Ly83akc,8815
-haiku/rag/store/repositories/settings.py,sha256=qZLXvLsErnCWL0nBQQNfRnatHzCKhtUDLvUK9k-W_fU,2463
-haiku/rag/store/upgrades/__init__.py,sha256=kKS1YWT_P-CYKhKtokOLTIFNKf9jlfjFFr8lyIMeogM,100
-haiku/rag/store/upgrades/v0_3_4.py,sha256=GLogKZdZ40NX1vBHKdOJju7fFzNUCHoEnjSZg17Hm2U,663
-haiku_rag-0.6.0.dist-info/METADATA,sha256=oLxNtf0pFMyLwc9sVsiztYbrpiyVNkg0wsX0TZdUYFw,4283
-haiku_rag-0.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.6.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.6.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.6.0.dist-info/RECORD,,