haiku.rag 0.3.3__py3-none-any.whl → 0.4.0__py3-none-any.whl

haiku/rag/app.py

@@ -74,7 +74,7 @@ class HaikuRAGApp:
                 self.console.print(f"[red]Error: {e}[/red]")
 
     async def rebuild(self):
-        async with HaikuRAG(db_path=self.db_path) as client:
+        async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client:
             try:
                 documents = await client.list_documents()
                 total_docs = len(documents)

haiku/rag/chunker.py

@@ -6,15 +6,11 @@ from haiku.rag.config import Config
 
 
 class Chunker:
-    """
-    A class that chunks text into smaller pieces for embedding and retrieval.
-
-    Parameters
-    ----------
-    chunk_size : int
-        The maximum size of a chunk in characters.
-    chunk_overlap : int
-        The number of characters of overlap between chunks.
+    """A class that chunks text into smaller pieces for embedding and retrieval.
+
+    Args:
+        chunk_size: The maximum size of a chunk in tokens.
+        chunk_overlap: The number of tokens of overlap between chunks.
     """
 
     encoder: ClassVar[tiktoken.Encoding] = tiktoken.encoding_for_model("gpt-4o")
@@ -28,18 +24,13 @@ class Chunker:
         self.chunk_overlap = chunk_overlap
 
     async def chunk(self, text: str) -> list[str]:
-        """
-        Split the text into chunks.
+        """Split the text into chunks based on token boundaries.
 
-        Parameters
-        ----------
-        text : str
-            The text to be split into chunks.
+        Args:
+            text: The text to be split into chunks.
 
-        Returns
-        -------
-        list
-            A list of text chunks.
+        Returns:
+            A list of text chunks with token-based boundaries and overlap.
         """
         if not text:
             return []

haiku/rag/cli.py

@@ -5,7 +5,7 @@ import typer
 from rich.console import Console
 
 from haiku.rag.app import HaikuRAGApp
-from haiku.rag.utils import get_default_data_dir
+from haiku.rag.utils import get_default_data_dir, is_up_to_date
 
 cli = typer.Typer(
     context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
@@ -15,6 +15,23 @@ console = Console()
 event_loop = asyncio.get_event_loop()
 
 
+async def check_version():
+    """Check if haiku.rag is up to date and show warning if not."""
+    up_to_date, current_version, latest_version = await is_up_to_date()
+    if not up_to_date:
+        console.print(
+            f"[yellow]Warning: haiku.rag is outdated. Current: {current_version}, Latest: {latest_version}[/yellow]"
+        )
+        console.print("[yellow]Please update.[/yellow]")
+
+
+@cli.callback()
+def main():
+    """haiku.rag CLI - SQLite-based RAG system"""
+    # Run version check before any command
+    event_loop.run_until_complete(check_version())
+
+
 @cli.command("list", help="List all stored documents")
 def list_documents(
     db: Path = typer.Option(

haiku/rag/client.py

@@ -10,6 +10,7 @@ import httpx
 
 from haiku.rag.config import Config
 from haiku.rag.reader import FileReader
+from haiku.rag.reranking import get_reranker
 from haiku.rag.store.engine import Store
 from haiku.rag.store.models.chunk import Chunk
 from haiku.rag.store.models.document import Document
@@ -24,12 +25,18 @@ class HaikuRAG:
         self,
         db_path: Path | Literal[":memory:"] = Config.DEFAULT_DATA_DIR
         / "haiku.rag.sqlite",
+        skip_validation: bool = False,
     ):
-        """Initialize the RAG client with a database path."""
+        """Initialize the RAG client with a database path.
+
+        Args:
+            db_path: Path to the SQLite database file or ":memory:" for in-memory database.
+            skip_validation: Whether to skip configuration validation on database load.
+        """
         if isinstance(db_path, Path):
             if not db_path.parent.exists():
                 Path.mkdir(db_path.parent, parents=True)
-        self.store = Store(db_path)
+        self.store = Store(db_path, skip_validation=skip_validation)
         self.document_repository = DocumentRepository(self.store)
         self.chunk_repository = ChunkRepository(self.store)
 
@@ -45,7 +52,16 @@ class HaikuRAG:
     async def create_document(
         self, content: str, uri: str | None = None, metadata: dict | None = None
     ) -> Document:
-        """Create a new document with optional URI and metadata."""
+        """Create a new document with optional URI and metadata.
+
+        Args:
+            content: The text content of the document.
+            uri: Optional URI identifier for the document.
+            metadata: Optional metadata dictionary.
+
+        Returns:
+            The created Document instance.
+        """
         document = Document(
             content=content,
             uri=uri,
@@ -165,29 +181,26 @@ class HaikuRAG:
 
             # Create a temporary file with the appropriate extension
             with tempfile.NamedTemporaryFile(
-                mode="wb", suffix=file_extension, delete=False
+                mode="wb", suffix=file_extension
             ) as temp_file:
                 temp_file.write(response.content)
+                temp_file.flush()  # Ensure content is written to disk
                 temp_path = Path(temp_file.name)
 
-            try:
                 # Parse the content using FileReader
                 content = FileReader.parse_file(temp_path)
 
-                # Merge metadata with contentType and md5
-                metadata.update({"contentType": content_type, "md5": md5_hash})
-
-                if existing_doc:
-                    existing_doc.content = content
-                    existing_doc.metadata = metadata
-                    return await self.update_document(existing_doc)
-                else:
-                    return await self.create_document(
-                        content=content, uri=url, metadata=metadata
-                    )
-            finally:
-                # Clean up temporary file
-                temp_path.unlink(missing_ok=True)
+            # Merge metadata with contentType and md5
+            metadata.update({"contentType": content_type, "md5": md5_hash})
+
+            if existing_doc:
+                existing_doc.content = content
+                existing_doc.metadata = metadata
+                return await self.update_document(existing_doc)
+            else:
+                return await self.create_document(
+                    content=content, uri=url, metadata=metadata
+                )
 
     def _get_extension_from_content_type_or_url(
         self, url: str, content_type: str
@@ -221,11 +234,25 @@ class HaikuRAG:
         return ".html"
 
     async def get_document_by_id(self, document_id: int) -> Document | None:
-        """Get a document by its ID."""
+        """Get a document by its ID.
+
+        Args:
+            document_id: The unique identifier of the document.
+
+        Returns:
+            The Document instance if found, None otherwise.
+        """
         return await self.document_repository.get_by_id(document_id)
 
     async def get_document_by_uri(self, uri: str) -> Document | None:
-        """Get a document by its URI."""
+        """Get a document by its URI.
+
+        Args:
+            uri: The URI identifier of the document.
+
+        Returns:
+            The Document instance if found, None otherwise.
+        """
         return await self.document_repository.get_by_uri(uri)
 
     async def update_document(self, document: Document) -> Document:
@@ -239,32 +266,55 @@ class HaikuRAG:
     async def list_documents(
         self, limit: int | None = None, offset: int | None = None
     ) -> list[Document]:
-        """List all documents with optional pagination."""
+        """List all documents with optional pagination.
+
+        Args:
+            limit: Maximum number of documents to return.
+            offset: Number of documents to skip.
+
+        Returns:
+            List of Document instances.
+        """
         return await self.document_repository.list_all(limit=limit, offset=offset)
 
     async def search(
-        self, query: str, limit: int = 5, k: int = 60
+        self, query: str, limit: int = 3, k: int = 60, rerank=Config.RERANK
     ) -> list[tuple[Chunk, float]]:
-        """Search for relevant chunks using hybrid search (vector similarity + full-text search).
+        """Search for relevant chunks using hybrid search (vector similarity + full-text search) with reranking.
 
         Args:
-            query: The search query string
-            limit: Maximum number of results to return
-            k: Parameter for Reciprocal Rank Fusion (default: 60)
+            query: The search query string.
+            limit: Maximum number of results to return.
+            k: Parameter for Reciprocal Rank Fusion (default: 60).
 
         Returns:
-            List of (chunk, score) tuples ordered by relevance
+            List of (chunk, score) tuples ordered by relevance.
         """
-        return await self.chunk_repository.search_chunks_hybrid(query, limit, k)
+
+        if not rerank:
+            return await self.chunk_repository.search_chunks_hybrid(query, limit, k)
+
+        # Get more initial results (3X) for reranking
+        search_results = await self.chunk_repository.search_chunks_hybrid(
+            query, limit * 3, k
+        )
+
+        # Apply reranking
+        reranker = get_reranker()
+        chunks = [chunk for chunk, _ in search_results]
+        reranked_results = await reranker.rerank(query, chunks, top_n=limit)
+
+        # Return reranked results with scores from reranker
+        return reranked_results
 
     async def ask(self, question: str) -> str:
         """Ask a question using the configured QA agent.
 
         Args:
-            question: The question to ask
+            question: The question to ask.
 
         Returns:
-            The generated answer as a string
+            The generated answer as a string.
         """
         from haiku.rag.qa import get_qa_agent
 
@@ -277,12 +327,16 @@ class HaikuRAG:
         Yields:
             int: The ID of the document currently being processed
         """
-        documents = await self.list_documents()
+        await self.chunk_repository.delete_all()
+        self.store.recreate_embeddings_table()
 
-        if not documents:
-            return
+        # Update settings to current config
+        from haiku.rag.store.repositories.settings import SettingsRepository
 
-        await self.chunk_repository.delete_all()
+        settings_repo = SettingsRepository(self.store)
+        settings_repo.save()
+
+        documents = await self.list_documents()
 
         for doc in documents:
             if doc.id is not None:

haiku/rag/config.py

@@ -19,6 +19,10 @@ class AppConfig(BaseModel):
     EMBEDDINGS_MODEL: str = "mxbai-embed-large"
     EMBEDDINGS_VECTOR_DIM: int = 1024
 
+    RERANK: bool = True
+    RERANK_PROVIDER: str = "mxbai"
+    RERANK_MODEL: str = "mixedbread-ai/mxbai-rerank-base-v2"
+
     QA_PROVIDER: str = "ollama"
     QA_MODEL: str = "qwen3"
 
@@ -31,6 +35,7 @@ class AppConfig(BaseModel):
     VOYAGE_API_KEY: str = ""
     OPENAI_API_KEY: str = ""
     ANTHROPIC_API_KEY: str = ""
+    COHERE_API_KEY: str = ""
 
     @field_validator("MONITOR_DIRECTORIES", mode="before")
     @classmethod
@@ -52,3 +57,5 @@ if Config.VOYAGE_API_KEY:
     os.environ["VOYAGE_API_KEY"] = Config.VOYAGE_API_KEY
 if Config.ANTHROPIC_API_KEY:
     os.environ["ANTHROPIC_API_KEY"] = Config.ANTHROPIC_API_KEY
+if Config.COHERE_API_KEY:
+    os.environ["CO_API_KEY"] = Config.COHERE_API_KEY

haiku/rag/embeddings/base.py

@@ -1,6 +1,9 @@
+from haiku.rag.config import Config
+
+
 class EmbedderBase:
-    _model: str = ""
-    _vector_dim: int = 0
+    _model: str = Config.EMBEDDINGS_MODEL
+    _vector_dim: int = Config.EMBEDDINGS_VECTOR_DIM
 
     def __init__(self, model: str, vector_dim: int):
         self._model = model

haiku/rag/embeddings/ollama.py

@@ -5,9 +5,6 @@ from haiku.rag.embeddings.base import EmbedderBase
 
 
 class Embedder(EmbedderBase):
-    _model: str = Config.EMBEDDINGS_MODEL
-    _vector_dim: int = 1024
-
     async def embed(self, text: str) -> list[float]:
         client = AsyncClient(host=Config.OLLAMA_BASE_URL)
         res = await client.embeddings(model=self._model, prompt=text)

haiku/rag/embeddings/openai.py

@@ -1,13 +1,9 @@
 try:
     from openai import AsyncOpenAI
 
-    from haiku.rag.config import Config
     from haiku.rag.embeddings.base import EmbedderBase
 
     class Embedder(EmbedderBase):
-        _model: str = Config.EMBEDDINGS_MODEL
-        _vector_dim: int = 1536
-
         async def embed(self, text: str) -> list[float]:
             client = AsyncOpenAI()
             response = await client.embeddings.create(

haiku/rag/embeddings/voyageai.py

@@ -1,13 +1,9 @@
 try:
     from voyageai.client import Client  # type: ignore
 
-    from haiku.rag.config import Config
     from haiku.rag.embeddings.base import EmbedderBase
 
     class Embedder(EmbedderBase):
-        _model: str = Config.EMBEDDINGS_MODEL
-        _vector_dim: int = 1024
-
         async def embed(self, text: str) -> list[float]:
             client = Client()
             res = client.embed([text], model=self._model, output_dtype="float")

haiku/rag/qa/prompts.py

@@ -6,7 +6,7 @@ Your process:
 2. Search with specific keywords and phrases from the user's question
 3. Review the search results and their relevance scores
 4. If you need additional context, perform follow-up searches with different keywords
-5. Provide a comprehensive answer based only on the retrieved documents
+5. Provide a short and to the point comprehensive answer based only on the retrieved documents
 
 Guidelines:
 - Base your answers strictly on the provided document content
@@ -15,6 +15,7 @@ Guidelines:
 - Indicate when information is incomplete or when you need to search for additional context
 - If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question."
 - For complex questions, consider breaking them down and performing multiple searches
+- Stick to the answer, do not ellaborate or provde context unless asked for it.
 
 Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents.
 """

haiku/rag/reranking/__init__.py

@@ -0,0 +1,37 @@
+from haiku.rag.config import Config
+from haiku.rag.reranking.base import RerankerBase
+
+try:
+    from haiku.rag.reranking.cohere import CohereReranker
+except ImportError:
+    pass
+
+_reranker: RerankerBase | None = None
+
+
+def get_reranker() -> RerankerBase:
+    """
+    Factory function to get the appropriate reranker based on the configuration.
+    """
+    global _reranker
+    if _reranker is not None:
+        return _reranker
+    if Config.RERANK_PROVIDER == "mxbai":
+        from haiku.rag.reranking.mxbai import MxBAIReranker
+
+        _reranker = MxBAIReranker()
+        return _reranker
+
+    if Config.RERANK_PROVIDER == "cohere":
+        try:
+            from haiku.rag.reranking.cohere import CohereReranker
+        except ImportError:
+            raise ImportError(
+                "Cohere reranker requires the 'cohere' package. "
+                "Please install haiku.rag with the 'cohere' extra:"
+                "uv pip install haiku.rag --extra cohere"
+            )
+        _reranker = CohereReranker()
+        return _reranker
+
+    raise ValueError(f"Unsupported reranker provider: {Config.RERANK_PROVIDER}")

haiku/rag/reranking/base.py

@@ -0,0 +1,13 @@
+from haiku.rag.config import Config
+from haiku.rag.store.models.chunk import Chunk
+
+
+class RerankerBase:
+    _model: str = Config.RERANK_MODEL
+
+    async def rerank(
+        self, query: str, chunks: list[Chunk], top_n: int = 10
+    ) -> list[tuple[Chunk, float]]:
+        raise NotImplementedError(
+            "Reranker is an abstract class. Please implement the rerank method in a subclass."
+        )

haiku/rag/reranking/cohere.py

@@ -0,0 +1,34 @@
+from haiku.rag.config import Config
+from haiku.rag.reranking.base import RerankerBase
+from haiku.rag.store.models.chunk import Chunk
+
+try:
+    import cohere
+except ImportError as e:
+    raise ImportError(
+        "cohere is not installed. Please install it with `pip install cohere` or use the cohere optional dependency."
+    ) from e
+
+
+class CohereReranker(RerankerBase):
+    def __init__(self):
+        self._client = cohere.ClientV2(api_key=Config.COHERE_API_KEY)
+
+    async def rerank(
+        self, query: str, chunks: list[Chunk], top_n: int = 10
+    ) -> list[tuple[Chunk, float]]:
+        if not chunks:
+            return []
+
+        documents = [chunk.content for chunk in chunks]
+
+        response = self._client.rerank(
+            model=self._model, query=query, documents=documents, top_n=top_n
+        )
+
+        reranked_chunks = []
+        for result in response.results:
+            original_chunk = chunks[result.index]
+            reranked_chunks.append((original_chunk, result.relevance_score))
+
+        return reranked_chunks

haiku/rag/reranking/mxbai.py

@@ -0,0 +1,28 @@
+from mxbai_rerank import MxbaiRerankV2
+
+from haiku.rag.config import Config
+from haiku.rag.reranking.base import RerankerBase
+from haiku.rag.store.models.chunk import Chunk
+
+
+class MxBAIReranker(RerankerBase):
+    def __init__(self):
+        self._client = MxbaiRerankV2(
+            Config.RERANK_MODEL, disable_transformers_warnings=True
+        )
+
+    async def rerank(
+        self, query: str, chunks: list[Chunk], top_n: int = 10
+    ) -> list[tuple[Chunk, float]]:
+        if not chunks:
+            return []
+
+        documents = [chunk.content for chunk in chunks]
+
+        results = self._client.rank(query=query, documents=documents, top_k=top_n)
+        reranked_chunks = []
+        for result in results:
+            original_chunk = chunks[result.index]
+            reranked_chunks.append((original_chunk, result.score))
+
+        return reranked_chunks

haiku/rag/store/engine.py

@@ -1,23 +1,65 @@
 import sqlite3
 import struct
+from importlib import metadata
 from pathlib import Path
 from typing import Literal
 
 import sqlite_vec
+from packaging.version import parse
+from rich.console import Console
 
+from haiku.rag.config import Config
 from haiku.rag.embeddings import get_embedder
+from haiku.rag.store.upgrades import upgrades
+from haiku.rag.utils import int_to_semantic_version, semantic_version_to_int
 
 
 class Store:
-    def __init__(self, db_path: Path | Literal[":memory:"]):
+    def __init__(
+        self, db_path: Path | Literal[":memory:"], skip_validation: bool = False
+    ):
         self.db_path: Path | Literal[":memory:"] = db_path
-        self._connection = self.create_db()
+        self.create_or_update_db()
 
-    def create_db(self) -> sqlite3.Connection:
+        # Validate config compatibility after connection is established
+        if not skip_validation:
+            from haiku.rag.store.repositories.settings import SettingsRepository
+
+            settings_repo = SettingsRepository(self)
+            settings_repo.validate_config_compatibility()
+        current_version = metadata.version("haiku.rag")
+        self.set_user_version(current_version)
+
+    def create_or_update_db(self):
         """Create the database and tables with sqlite-vec support for embeddings."""
+        current_version = metadata.version("haiku.rag")
+
         db = sqlite3.connect(self.db_path)
         db.enable_load_extension(True)
         sqlite_vec.load(db)
+        self._connection = db
+        existing_tables = [
+            row[0]
+            for row in db.execute(
+                "SELECT name FROM sqlite_master WHERE type='table';"
+            ).fetchall()
+        ]
+
+        # If we have a db already, perform upgrades and return
+        if self.db_path != ":memory:" and "documents" in existing_tables:
+            # Upgrade database
+            console = Console()
+            db_version = self.get_user_version()
+            for version, steps in upgrades:
+                if parse(current_version) >= parse(version) and parse(version) > parse(
+                    db_version
+                ):
+                    for step in steps:
+                        step(db)
+                        console.print(
+                            f"[green][b]DB Upgrade: [/b]{step.__doc__}[/green]"
+                        )
+            return
 
         # Create documents table
         db.execute("""
@@ -30,7 +72,6 @@ class Store:
                 updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
             )
         """)
-
         # Create chunks table
         db.execute("""
             CREATE TABLE IF NOT EXISTS chunks (
@@ -41,7 +82,6 @@ class Store:
                 FOREIGN KEY (document_id) REFERENCES documents (id) ON DELETE CASCADE
             )
         """)
-
         # Create vector table for chunk embeddings
         embedder = get_embedder()
         db.execute(f"""
@@ -50,7 +90,6 @@ class Store:
                 embedding FLOAT[{embedder._vector_dim}]
             )
         """)
-
         # Create FTS5 table for full-text search
         db.execute("""
             CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
@@ -59,14 +98,61 @@ class Store:
                 content_rowid='id'
             )
         """)
-
+        # Create settings table for storing current configuration
+        db.execute("""
+            CREATE TABLE IF NOT EXISTS settings (
+                id INTEGER PRIMARY KEY DEFAULT 1,
+                settings TEXT NOT NULL DEFAULT '{}'
+            )
+        """)
+        # Save current settings to the new database
+        settings_json = Config.model_dump_json()
+        db.execute(
+            "INSERT OR IGNORE INTO settings (id, settings) VALUES (1, ?)",
+            (settings_json,),
+        )
         # Create indexes for better performance
         db.execute(
             "CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)"
         )
-
         db.commit()
-        return db
+
+    def get_user_version(self) -> str:
+        """Returns the SQLite user version"""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self._connection.execute("PRAGMA user_version;")
+        version = cursor.fetchone()
+        return int_to_semantic_version(version[0])
+
+    def set_user_version(self, version: str) -> None:
+        """Updates the SQLite user version"""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        self._connection.execute(
+            f"PRAGMA user_version = {semantic_version_to_int(version)};"
+        )
+
+    def recreate_embeddings_table(self) -> None:
+        """Recreate the embeddings table with current vector dimensions."""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        # Drop existing embeddings table
+        self._connection.execute("DROP TABLE IF EXISTS chunk_embeddings")
+
+        # Recreate with current dimensions
+        embedder = get_embedder()
+        self._connection.execute(f"""
+            CREATE VIRTUAL TABLE chunk_embeddings USING vec0(
+                chunk_id INTEGER PRIMARY KEY,
+                embedding FLOAT[{embedder._vector_dim}]
+            )
+        """)
+
+        self._connection.commit()
 
     @staticmethod
     def serialize_embedding(embedding: list[float]) -> bytes:

haiku/rag/store/repositories/settings.py

@@ -0,0 +1,78 @@
+import json
+from typing import Any
+
+from haiku.rag.store.engine import Store
+
+
+class ConfigMismatchError(Exception):
+    """Raised when current config doesn't match stored settings."""
+
+    pass
+
+
+class SettingsRepository:
+    def __init__(self, store: Store):
+        self.store = store
+
+    def get(self) -> dict[str, Any]:
+        """Get all settings from the database."""
+        if self.store._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self.store._connection.execute("SELECT settings FROM settings LIMIT 1")
+        row = cursor.fetchone()
+        if row:
+            return json.loads(row[0])
+        return {}
+
+    def save(self) -> None:
+        """Sync settings from the current AppConfig to database."""
+        if self.store._connection is None:
+            raise ValueError("Store connection is not available")
+
+        from haiku.rag.config import Config
+
+        settings_json = Config.model_dump_json()
+
+        self.store._connection.execute(
+            "INSERT INTO settings (id, settings) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET settings = excluded.settings",
+            (settings_json,),
+        )
+
+        self.store._connection.commit()
+
+    def validate_config_compatibility(self) -> None:
+        """Check if current config is compatible with stored settings.
+
+        Raises ConfigMismatchError if there are incompatible differences.
+        If no settings exist, saves current config.
+        """
+        db_settings = self.get()
+        if not db_settings:
+            # No settings in DB, save current config
+            self.save()
+            return
+
+        from haiku.rag.config import Config
+
+        current_config = Config.model_dump(mode="json")
+
+        # Critical settings that must match
+        critical_settings = [
+            "EMBEDDINGS_PROVIDER",
+            "EMBEDDINGS_MODEL",
+            "EMBEDDINGS_VECTOR_DIM",
+            "CHUNK_SIZE",
+            "CHUNK_OVERLAP",
+        ]
+
+        errors = []
+        for setting in critical_settings:
+            if db_settings.get(setting) != current_config.get(setting):
+                errors.append(
+                    f"{setting}: current={current_config.get(setting)}, stored={db_settings.get(setting)}"
+                )
+
+        if errors:
+            error_msg = f"Config mismatch detected: {'; '.join(errors)}. Consider rebuilding the database with the current configuration."
+            raise ConfigMismatchError(error_msg)

haiku/rag/store/upgrades/__init__.py

@@ -0,0 +1,3 @@
+from haiku.rag.store.upgrades.v0_3_4 import upgrades as v0_3_4_upgrades
+
+upgrades = v0_3_4_upgrades

haiku/rag/store/upgrades/v0_3_4.py

@@ -0,0 +1,26 @@
+from collections.abc import Callable
+from sqlite3 import Connection
+
+from haiku.rag.config import Config
+
+
+def add_settings_table(db: Connection) -> None:
+    """Create settings table for storing current configuration"""
+    db.execute("""
+        CREATE TABLE settings (
+            id INTEGER PRIMARY KEY DEFAULT 1,
+            settings TEXT NOT NULL DEFAULT '{}'
+        )
+    """)
+
+    settings_json = Config.model_dump_json()
+    db.execute(
+        "INSERT INTO settings (id, settings) VALUES (1, ?)",
+        (settings_json,),
+    )
+    db.commit()
+
+
+upgrades: list[tuple[str, list[Callable[[Connection], None]]]] = [
+    ("0.3.4", [add_settings_table])
+]

haiku/rag/utils.py

@@ -1,17 +1,20 @@
 import sys
+from importlib import metadata
 from pathlib import Path
 
+import httpx
+from packaging.version import Version, parse
+
 
 def get_default_data_dir() -> Path:
-    """
-    Get the user data directory for the current system platform.
+    """Get the user data directory for the current system platform.
 
     Linux: ~/.local/share/haiku.rag
     macOS: ~/Library/Application Support/haiku.rag
     Windows: C:/Users/<USER>/AppData/Roaming/haiku.rag
 
-    :return: User Data Path
-    :rtype: Path
+    Returns:
+        User Data Path.
     """
     home = Path.home()
 
@@ -23,3 +26,54 @@ def get_default_data_dir() -> Path:
 
     data_path = system_paths[sys.platform]
     return data_path
+
+
+def semantic_version_to_int(version: str) -> int:
+    """Convert a semantic version string to an integer.
+
+    Args:
+        version: Semantic version string.
+
+    Returns:
+        Integer representation of semantic version.
+    """
+    major, minor, patch = version.split(".")
+    major = int(major) << 16
+    minor = int(minor) << 8
+    patch = int(patch)
+    return major + minor + patch
+
+
+def int_to_semantic_version(version: int) -> str:
+    """Convert an integer to a semantic version string.
+
+    Args:
+        version: Integer representation of semantic version.
+
+    Returns:
+        Semantic version string.
+    """
+    major = version >> 16
+    minor = (version >> 8) & 255
+    patch = version & 255
+    return f"{major}.{minor}.{patch}"
+
+
+async def is_up_to_date() -> tuple[bool, Version, Version]:
+    """Check whether haiku.rag is current.
+
+    Returns:
+        A tuple containing a boolean indicating whether haiku.rag is current,
+        the running version and the latest version.
+    """
+
+    async with httpx.AsyncClient() as client:
+        running_version = parse(metadata.version("haiku.rag"))
+        try:
+            response = await client.get("https://pypi.org/pypi/haiku.rag/json")
+            data = response.json()
+            pypi_version = parse(data["info"]["version"])
+        except Exception:
+            # If no network connection, do not raise alarms.
+            pypi_version = running_version
+    return running_version >= pypi_version, running_version, pypi_version

{haiku_rag-0.3.3.dist-info → haiku_rag-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.3.3
+Version: 0.4.0
 Summary: Retrieval Augmented Generation (RAG) with SQLite
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
@@ -21,6 +21,7 @@ Requires-Python: >=3.10
 Requires-Dist: fastmcp>=2.8.1
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: markitdown[audio-transcription,docx,pdf,pptx,xlsx]>=0.1.2
+Requires-Dist: mxbai-rerank>=0.1.6
 Requires-Dist: ollama>=0.5.1
 Requires-Dist: pydantic>=2.11.7
 Requires-Dist: python-dotenv>=1.1.0
@@ -31,6 +32,8 @@ Requires-Dist: typer>=0.16.0
 Requires-Dist: watchfiles>=1.1.0
 Provides-Extra: anthropic
 Requires-Dist: anthropic>=0.56.0; extra == 'anthropic'
+Provides-Extra: cohere
+Requires-Dist: cohere>=5.16.1; extra == 'cohere'
 Provides-Extra: openai
 Requires-Dist: openai>=1.0.0; extra == 'openai'
 Provides-Extra: voyageai
@@ -49,6 +52,7 @@ Retrieval-Augmented Generation (RAG) library on SQLite.
 - **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
 - **Multiple QA providers**: Ollama, OpenAI, Anthropic
 - **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
+- **Reranking**: Default search result reranking with MixedBread AI or Cohere
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
 - **40+ file formats**: PDF, DOCX, HTML, Markdown, audio, URLs
@@ -88,7 +92,7 @@ async with HaikuRAG("database.db") as client:
     # Add document
     doc = await client.create_document("Your content")
 
-    # Search
+    # Search (reranking enabled by default)
     results = await client.search("query")
     for chunk, score in results:
         print(f"{score:.3f}: {chunk.content}")

haiku_rag-0.4.0.dist-info/RECORD

@@ -0,0 +1,43 @@
+haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+haiku/rag/app.py,sha256=FpLVyP1-zAq_XPmU8CPVLkuIAeuhBOGvMqhYS8RbN40,7649
+haiku/rag/chunker.py,sha256=MbCtP66OfTFoIBvqmVT9T9c87fozsYYzAQzJJEfPBVI,1812
+haiku/rag/cli.py,sha256=8PC7r5odIVLyksSm_BXor2rznIZ2KDug-YhzqbFPvms,5605
+haiku/rag/client.py,sha256=W8iw22v9Muoq37e3uGww2DpbQnQhQzaPZiP9MVMRKJE,12554
+haiku/rag/config.py,sha256=_Ss54kmfxVAJupExLKaYjYUlFxJgb7hEEdbG4-isapY,1662
+haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
+haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
+haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
+haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
+haiku/rag/utils.py,sha256=Ez_tvNlRO_D8c2CBZ83Hs9Gmzcqdq4cmw_V5GBdKy_8,2214
+haiku/rag/embeddings/__init__.py,sha256=4jUPe2FyIf8BGZ7AncWSlBdNXG3URejBbnkhQf3JiD0,1505
+haiku/rag/embeddings/base.py,sha256=NTQvuzbZPu0LBo5wAu3qGyJ4xXUaRAt1fjBO0ygWn_Y,465
+haiku/rag/embeddings/ollama.py,sha256=y6-lp0XpbnyIjoOEdtSzMdEVkU5glOwnWQ1FkpUZnpI,370
+haiku/rag/embeddings/openai.py,sha256=i4Ui5hAJkcKqJkH9L3jJo7fuGYHn07td532w-ksg_T8,431
+haiku/rag/embeddings/voyageai.py,sha256=0hiRTIqu-bpl-4OaCtMHvWfPdgbrzhnfZJowSV8pLRA,415
+haiku/rag/qa/__init__.py,sha256=oso98Ypti7mBLTJ6Zk71YaSJ9Rgc89QXp9RSB6zSpYs,1501
+haiku/rag/qa/anthropic.py,sha256=6I6cf6ySNkYbmDFdy22sA8r3GO5moiiH75tJnHcgJQA,4448
+haiku/rag/qa/base.py,sha256=4ZTM_l5FAZ9cA0f8NeqRJiUAmjatwCTmSoclFw0gTFQ,1349
+haiku/rag/qa/ollama.py,sha256=-UtNFErYlA_66g3WLU6lK38a1Y5zhAL6s_uZ5AP0TFs,2381
+haiku/rag/qa/openai.py,sha256=dF32sGgVt8mZi5oVxByaeECs9NqLjvDiZnnpJBsrHm8,3968
+haiku/rag/qa/prompts.py,sha256=W6QYvqIGcW_VmnTA88quJqCi6h6bafEP4pyrcGze9TA,1303
+haiku/rag/reranking/__init__.py,sha256=6tuQCrk5CEfyGaUjB7uaskWxsB15GaV8t9rm7VVSMrg,1125
+haiku/rag/reranking/base.py,sha256=LM9yUSSJ414UgBZhFTgxGprlRqzfTe4I1vgjricz2JY,405
+haiku/rag/reranking/cohere.py,sha256=1iTdiaa8vvb6oHVB2qpWzUOVkyfUcimVSZp6Qr4aq4c,1049
+haiku/rag/reranking/mxbai.py,sha256=46sVTsTIkzIX9THgM3u8HaEmgY7evvEyB-N54JTHvK8,867
+haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
+haiku/rag/store/engine.py,sha256=4ouAD0s-TFwEoEHjVVw_KnV6aaw5nwhe9fdT8PRXfok,6061
+haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
+haiku/rag/store/models/chunk.py,sha256=lmbPOOTz-N4PXhrA5XCUxyRcSTZBo135fqkV1mwnGcE,309
+haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
+haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPvrDIHVhY5T1A,263
+haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
+haiku/rag/store/repositories/chunk.py,sha256=gik7ZPOK3gCoG6tU1pGueAZBPmJxIb7obYFUhwINrYg,16497
+haiku/rag/store/repositories/document.py,sha256=xpWOpjHFbhVwNJ1gpusEKNY6l_Qyibg9y_bdHCwcfpk,7133
+haiku/rag/store/repositories/settings.py,sha256=dme3_ulQdQvyF9daavSjAd-SjZ5hh0MJoxP7iXgap-A,2492
+haiku/rag/store/upgrades/__init__.py,sha256=kKS1YWT_P-CYKhKtokOLTIFNKf9jlfjFFr8lyIMeogM,100
+haiku/rag/store/upgrades/v0_3_4.py,sha256=GLogKZdZ40NX1vBHKdOJju7fFzNUCHoEnjSZg17Hm2U,663
+haiku_rag-0.4.0.dist-info/METADATA,sha256=59rxYHim3hZeKPkQhEmnNR_Tj6DuF9hLBj2GA2T3T_s,4235
+haiku_rag-0.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.4.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.4.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.4.0.dist-info/RECORD,,

haiku_rag-0.3.3.dist-info/RECORD

@@ -1,36 +0,0 @@
-haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-haiku/rag/app.py,sha256=Foi_K-sAqHWsIAAaxY2Tb0hyXnMCi06LqIFCPiBS5n0,7627
-haiku/rag/chunker.py,sha256=lSSPWgNAe7gNZL_yNLmDtqxJix4YclOiG7gbARcEpV8,1871
-haiku/rag/cli.py,sha256=9F64IIm2c1nBKn7p9D5yYkVZr8HcjDemrzjF9SRGIY8,5017
-haiku/rag/client.py,sha256=qoVgdsP_MH8wVcDTvPIcMgW7323tTjOXH8JKugz5snY,10847
-haiku/rag/config.py,sha256=ctD_pu7nDOieirJofhNMO-OJIONLC5myvcru9iTm_ps,1433
-haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
-haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
-haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
-haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
-haiku/rag/utils.py,sha256=6xVM6z2OmhzB4FEDlPbMsr_ZBBmCbMQb83nP6E2UdxY,629
-haiku/rag/embeddings/__init__.py,sha256=4jUPe2FyIf8BGZ7AncWSlBdNXG3URejBbnkhQf3JiD0,1505
-haiku/rag/embeddings/base.py,sha256=PTAWKTU-Q-hXIhbRK1o6pIdpaW7DFdzJXQ0Nzc6VI-w,379
-haiku/rag/embeddings/ollama.py,sha256=hWdrTiuJwNSRYCqP0WP-z6XXA3RBGkAiknZMsPLH0qU,441
-haiku/rag/embeddings/openai.py,sha256=reh8AykG2f9f5hhRDmqSsjiuCPi9SsXfe2YEZFlxXk8,550
-haiku/rag/embeddings/voyageai.py,sha256=jc0JywdLJD3Ee1MUv1m8MhWCEo0enNnVcrIBtUvD-Ss,534
-haiku/rag/qa/__init__.py,sha256=oso98Ypti7mBLTJ6Zk71YaSJ9Rgc89QXp9RSB6zSpYs,1501
-haiku/rag/qa/anthropic.py,sha256=6I6cf6ySNkYbmDFdy22sA8r3GO5moiiH75tJnHcgJQA,4448
-haiku/rag/qa/base.py,sha256=4ZTM_l5FAZ9cA0f8NeqRJiUAmjatwCTmSoclFw0gTFQ,1349
-haiku/rag/qa/ollama.py,sha256=-UtNFErYlA_66g3WLU6lK38a1Y5zhAL6s_uZ5AP0TFs,2381
-haiku/rag/qa/openai.py,sha256=dF32sGgVt8mZi5oVxByaeECs9NqLjvDiZnnpJBsrHm8,3968
-haiku/rag/qa/prompts.py,sha256=578LJGZJ0LQ_q7ccyj5hLabtHo8Zcfw5-DiLGN9lC-w,1200
-haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
-haiku/rag/store/engine.py,sha256=BeYZRZ08zaYeeu375ysnAL3tGz4roA3GzP7WRNwznCo,2603
-haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
-haiku/rag/store/models/chunk.py,sha256=lmbPOOTz-N4PXhrA5XCUxyRcSTZBo135fqkV1mwnGcE,309
-haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
-haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPvrDIHVhY5T1A,263
-haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
-haiku/rag/store/repositories/chunk.py,sha256=gik7ZPOK3gCoG6tU1pGueAZBPmJxIb7obYFUhwINrYg,16497
-haiku/rag/store/repositories/document.py,sha256=xpWOpjHFbhVwNJ1gpusEKNY6l_Qyibg9y_bdHCwcfpk,7133
-haiku_rag-0.3.3.dist-info/METADATA,sha256=nDI-sy2F8h7qr9hK1S7VQLOMRcWYP1clxJYxNVB1AaA,4019
-haiku_rag-0.3.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.3.3.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.3.3.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.3.3.dist-info/RECORD,,