haiku.rag 0.3.2__tar.gz → 0.3.4__tar.gz

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.3.2
+Version: 0.3.4
 Summary: Retrieval Augmented Generation (RAG) with SQLite
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
@@ -116,3 +116,4 @@ Full documentation at: https://ggozad.github.io/haiku.rag/
 - [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
 - [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
 - [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs
+- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/README.md

@@ -77,3 +77,4 @@ Full documentation at: https://ggozad.github.io/haiku.rag/
 - [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
 - [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
 - [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs
+- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks

haiku_rag-0.3.4/docs/benchmarks.md

@@ -0,0 +1,28 @@
+# Benchmarks
+
+We use the [repliqa](https://huggingface.co/datasets/ServiceNow/repliqa) dataset for the evaluation of `haiku.rag`.
+
+You can perform your own evaluations using as example the script found at
+`tests/generate_benchmark_db.py`.
+
+## Recall
+
+In order to calculate recall, we load the `News Stories` from `repliqa_3` which is 1035 documents and index them in a sqlite db. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question.
+
+
+The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 for the top 3 results.
+
+| Model                                 | Document in top 1 | Document in top 3 |
+|---------------------------------------|-------------------|-------------------|
+| Ollama / `mxbai-embed-large`          | 0.77              | 0.89              |
+| Ollama / `nomic-embed-text`           | 0.74              | 0.88              |
+| OpenAI / `text-embeddings-3-small`    | 0.75              | 0.88              |
+
+## Question/Answer evaluation
+
+Again using the same dataset, we use a QA agent to answer the question. In addition we use an LLM judge (using the Ollama `qwen3`) to evaluate whether the answer is correct or not. The obtained accuracy is as follows:
+
+| Embedding Model              | QA Model                          | Accuracy  |
+|------------------------------|-----------------------------------|-----------|
+| Ollama / `mxbai-embed-large` | Ollama / `qwen3`                  | 0.64      |
+| Ollama / `mxbai-embed-large` | Anthropic / `Claude Sonnet 3.7`   | 0.79      |

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/docs/configuration.md

@@ -2,6 +2,9 @@
 
 Configuration is done through the use of environment variables.
 
+!!! note
+    If you create a db with certain settings and later change them, `haiku.rag` will detect incompatibilities (for example, if you change embedding provider) and will exit. You can **rebuild** the database to apply the new settings, see [Rebuild Database](./cli.md#rebuild-database).
+
 ## File Monitoring
 
 Set directories to monitor for automatic indexing:

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/mkdocs.yml

@@ -63,6 +63,7 @@ nav:
       - Server: server.md
       - MCP: mcp.md
       - Python: python.md
+      - Benchmarks: benchmarks.md
 markdown_extensions:
   - admonition
   - attr_list

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "haiku.rag"
-version = "0.3.2"
+version = "0.3.4"
 description = "Retrieval Augmented Generation (RAG) with SQLite"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
@@ -56,7 +56,7 @@ dev = [
     "mkdocs>=1.6.1",
     "mkdocs-material>=9.6.14",
     "pre-commit>=4.2.0",
-    "pyright>=1.1.402",
+    "pyright>=1.1.403",
     "pytest>=8.4.0",
     "pytest-asyncio>=1.0.0",
     "pytest-cov>=6.2.1",

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/src/haiku/rag/app.py

@@ -74,7 +74,7 @@ class HaikuRAGApp:
                 self.console.print(f"[red]Error: {e}[/red]")
 
     async def rebuild(self):
-        async with HaikuRAG(db_path=self.db_path) as client:
+        async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client:
             try:
                 documents = await client.list_documents()
                 total_docs = len(documents)

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/src/haiku/rag/cli.py

@@ -5,7 +5,7 @@ import typer
 from rich.console import Console
 
 from haiku.rag.app import HaikuRAGApp
-from haiku.rag.utils import get_default_data_dir
+from haiku.rag.utils import get_default_data_dir, is_up_to_date
 
 cli = typer.Typer(
     context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
@@ -15,6 +15,23 @@ console = Console()
 event_loop = asyncio.get_event_loop()
 
 
+async def check_version():
+    """Check if haiku.rag is up to date and show warning if not."""
+    up_to_date, current_version, latest_version = await is_up_to_date()
+    if not up_to_date:
+        console.print(
+            f"[yellow]Warning: haiku.rag is outdated. Current: {current_version}, Latest: {latest_version}[/yellow]"
+        )
+        console.print("[yellow]Please update.[/yellow]")
+
+
+@cli.callback()
+def main():
+    """haiku.rag CLI - SQLite-based RAG system"""
+    # Run version check before any command
+    event_loop.run_until_complete(check_version())
+
+
 @cli.command("list", help="List all stored documents")
 def list_documents(
     db: Path = typer.Option(

{haiku_rag-0.3.2 → haiku_rag-0.3.4}/src/haiku/rag/client.py

@@ -24,12 +24,13 @@ class HaikuRAG:
         self,
         db_path: Path | Literal[":memory:"] = Config.DEFAULT_DATA_DIR
         / "haiku.rag.sqlite",
+        skip_validation: bool = False,
     ):
         """Initialize the RAG client with a database path."""
         if isinstance(db_path, Path):
             if not db_path.parent.exists():
                 Path.mkdir(db_path.parent, parents=True)
-        self.store = Store(db_path)
+        self.store = Store(db_path, skip_validation=skip_validation)
         self.document_repository = DocumentRepository(self.store)
         self.chunk_repository = ChunkRepository(self.store)
 
@@ -165,29 +166,26 @@ class HaikuRAG:
 
             # Create a temporary file with the appropriate extension
             with tempfile.NamedTemporaryFile(
-                mode="wb", suffix=file_extension, delete=False
+                mode="wb", suffix=file_extension
             ) as temp_file:
                 temp_file.write(response.content)
+                temp_file.flush()  # Ensure content is written to disk
                 temp_path = Path(temp_file.name)
 
-            try:
                 # Parse the content using FileReader
                 content = FileReader.parse_file(temp_path)
 
-                # Merge metadata with contentType and md5
-                metadata.update({"contentType": content_type, "md5": md5_hash})
-
-                if existing_doc:
-                    existing_doc.content = content
-                    existing_doc.metadata = metadata
-                    return await self.update_document(existing_doc)
-                else:
-                    return await self.create_document(
-                        content=content, uri=url, metadata=metadata
-                    )
-            finally:
-                # Clean up temporary file
-                temp_path.unlink(missing_ok=True)
+            # Merge metadata with contentType and md5
+            metadata.update({"contentType": content_type, "md5": md5_hash})
+
+            if existing_doc:
+                existing_doc.content = content
+                existing_doc.metadata = metadata
+                return await self.update_document(existing_doc)
+            else:
+                return await self.create_document(
+                    content=content, uri=url, metadata=metadata
+                )
 
     def _get_extension_from_content_type_or_url(
         self, url: str, content_type: str
@@ -277,12 +275,16 @@ class HaikuRAG:
         Yields:
             int: The ID of the document currently being processed
         """
-        documents = await self.list_documents()
+        await self.chunk_repository.delete_all()
+        self.store.recreate_embeddings_table()
 
-        if not documents:
-            return
+        # Update settings to current config
+        from haiku.rag.store.repositories.settings import SettingsRepository
 
-        await self.chunk_repository.delete_all()
+        settings_repo = SettingsRepository(self.store)
+        settings_repo.save()
+
+        documents = await self.list_documents()
 
         for doc in documents:
             if doc.id is not None:

haiku_rag-0.3.4/src/haiku/rag/qa/anthropic.py

@@ -0,0 +1,106 @@
+from collections.abc import Sequence
+
+try:
+    from anthropic import AsyncAnthropic
+    from anthropic.types import MessageParam, TextBlock, ToolParam, ToolUseBlock
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerAnthropicAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "claude-3-5-haiku-20241022"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ToolParam] = [
+                ToolParam(
+                    name="search_documents",
+                    description="Search the knowledge base for relevant documents",
+                    input_schema={
+                        "type": "object",
+                        "properties": {
+                            "query": {
+                                "type": "string",
+                                "description": "The search query to find relevant documents",
+                            },
+                            "limit": {
+                                "type": "integer",
+                                "description": "Maximum number of results to return",
+                                "default": 3,
+                            },
+                        },
+                        "required": ["query"],
+                    },
+                )
+            ]
+
+        async def answer(self, question: str) -> str:
+            anthropic_client = AsyncAnthropic()
+
+            messages: list[MessageParam] = [{"role": "user", "content": question}]
+
+            max_rounds = 5  # Prevent infinite loops
+
+            for _ in range(max_rounds):
+                response = await anthropic_client.messages.create(
+                    model=self._model,
+                    max_tokens=4096,
+                    system=self._system_prompt,
+                    messages=messages,
+                    tools=self.tools,
+                    temperature=0.0,
+                )
+
+                if response.stop_reason == "tool_use":
+                    messages.append({"role": "assistant", "content": response.content})
+
+                    # Process tool calls
+                    tool_results = []
+                    for content_block in response.content:
+                        if isinstance(content_block, ToolUseBlock):
+                            if content_block.name == "search_documents":
+                                args = content_block.input
+                                query = (
+                                    args.get("query", question)
+                                    if isinstance(args, dict)
+                                    else question
+                                )
+                                limit = (
+                                    int(args.get("limit", 3))
+                                    if isinstance(args, dict)
+                                    else 3
+                                )
+
+                                search_results = await self._client.search(
+                                    query, limit=limit
+                                )
+
+                                context_chunks = []
+                                for chunk, score in search_results:
+                                    context_chunks.append(
+                                        f"Content: {chunk.content}\nScore: {score:.4f}"
+                                    )
+
+                                context = "\n\n".join(context_chunks)
+
+                                tool_results.append(
+                                    {
+                                        "type": "tool_result",
+                                        "tool_use_id": content_block.id,
+                                        "content": context,
+                                    }
+                                )
+
+                    if tool_results:
+                        messages.append({"role": "user", "content": tool_results})
+                else:
+                    # No tool use, return the response
+                    if response.content:
+                        first_content = response.content[0]
+                        if isinstance(first_content, TextBlock):
+                            return first_content.text
+                    return ""
+
+            # If we've exhausted max rounds, return empty string
+            return ""
+
+except ImportError:
+    pass

haiku_rag-0.3.4/src/haiku/rag/qa/ollama.py

@@ -0,0 +1,64 @@
+from ollama import AsyncClient
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+OLLAMA_OPTIONS = {"temperature": 0.0, "seed": 42, "num_ctx": 64000}
+
+
+class QuestionAnswerOllamaAgent(QuestionAnswerAgentBase):
+    def __init__(self, client: HaikuRAG, model: str = Config.QA_MODEL):
+        super().__init__(client, model or self._model)
+
+    async def answer(self, question: str) -> str:
+        ollama_client = AsyncClient(host=Config.OLLAMA_BASE_URL)
+
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": question},
+        ]
+
+        max_rounds = 5  # Prevent infinite loops
+
+        for _ in range(max_rounds):
+            response = await ollama_client.chat(
+                model=self._model,
+                messages=messages,
+                tools=self.tools,
+                options=OLLAMA_OPTIONS,
+                think=False,
+            )
+
+            if response.get("message", {}).get("tool_calls"):
+                messages.append(response["message"])
+
+                for tool_call in response["message"]["tool_calls"]:
+                    if tool_call["function"]["name"] == "search_documents":
+                        args = tool_call["function"]["arguments"]
+                        query = args.get("query", question)
+                        limit = int(args.get("limit", 3))
+
+                        search_results = await self._client.search(query, limit=limit)
+
+                        context_chunks = []
+                        for chunk, score in search_results:
+                            context_chunks.append(
+                                f"Content: {chunk.content}\nScore: {score:.4f}"
+                            )
+
+                        context = "\n\n".join(context_chunks)
+
+                        messages.append(
+                            {
+                                "role": "tool",
+                                "content": context,
+                                "tool_call_id": tool_call.get("id", "search_tool"),
+                            }
+                        )
+            else:
+                # No tool calls, return the response
+                return response["message"]["content"]
+
+        # If we've exhausted max rounds, return empty string
+        return ""

haiku_rag-0.3.4/src/haiku/rag/qa/openai.py

@@ -0,0 +1,100 @@
+from collections.abc import Sequence
+
+try:
+    from openai import AsyncOpenAI
+    from openai.types.chat import (
+        ChatCompletionAssistantMessageParam,
+        ChatCompletionMessageParam,
+        ChatCompletionSystemMessageParam,
+        ChatCompletionToolMessageParam,
+        ChatCompletionUserMessageParam,
+    )
+    from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerOpenAIAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "gpt-4o-mini"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ChatCompletionToolParam] = [
+                ChatCompletionToolParam(tool) for tool in self.tools
+            ]
+
+        async def answer(self, question: str) -> str:
+            openai_client = AsyncOpenAI()
+
+            messages: list[ChatCompletionMessageParam] = [
+                ChatCompletionSystemMessageParam(
+                    role="system", content=self._system_prompt
+                ),
+                ChatCompletionUserMessageParam(role="user", content=question),
+            ]
+
+            max_rounds = 5  # Prevent infinite loops
+
+            for _ in range(max_rounds):
+                response = await openai_client.chat.completions.create(
+                    model=self._model,
+                    messages=messages,
+                    tools=self.tools,
+                    temperature=0.0,
+                )
+
+                response_message = response.choices[0].message
+
+                if response_message.tool_calls:
+                    messages.append(
+                        ChatCompletionAssistantMessageParam(
+                            role="assistant",
+                            content=response_message.content,
+                            tool_calls=[
+                                {
+                                    "id": tc.id,
+                                    "type": "function",
+                                    "function": {
+                                        "name": tc.function.name,
+                                        "arguments": tc.function.arguments,
+                                    },
+                                }
+                                for tc in response_message.tool_calls
+                            ],
+                        )
+                    )
+
+                    for tool_call in response_message.tool_calls:
+                        if tool_call.function.name == "search_documents":
+                            import json
+
+                            args = json.loads(tool_call.function.arguments)
+                            query = args.get("query", question)
+                            limit = int(args.get("limit", 3))
+
+                            search_results = await self._client.search(
+                                query, limit=limit
+                            )
+
+                            context_chunks = []
+                            for chunk, score in search_results:
+                                context_chunks.append(
+                                    f"Content: {chunk.content}\nScore: {score:.4f}"
+                                )
+
+                            context = "\n\n".join(context_chunks)
+
+                            messages.append(
+                                ChatCompletionToolMessageParam(
+                                    role="tool",
+                                    content=context,
+                                    tool_call_id=tool_call.id,
+                                )
+                            )
+                else:
+                    # No tool calls, return the response
+                    return response_message.content or ""
+
+            # If we've exhausted max rounds, return empty string
+            return ""
+
+except ImportError:
+    pass

haiku_rag-0.3.4/src/haiku/rag/qa/prompts.py

@@ -0,0 +1,20 @@
+SYSTEM_PROMPT = """
+You are a knowledgeable assistant that helps users find information from a document knowledge base.
+
+Your process:
+1. When a user asks a question, use the search_documents tool to find relevant information
+2. Search with specific keywords and phrases from the user's question
+3. Review the search results and their relevance scores
+4. If you need additional context, perform follow-up searches with different keywords
+5. Provide a comprehensive answer based only on the retrieved documents
+
+Guidelines:
+- Base your answers strictly on the provided document content
+- Quote or reference specific information when possible
+- If multiple documents contain relevant information, synthesize them coherently
+- Indicate when information is incomplete or when you need to search for additional context
+- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question."
+- For complex questions, consider breaking them down and performing multiple searches
+
+Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents.
+"""

haiku_rag-0.3.4/src/haiku/rag/store/engine.py

@@ -0,0 +1,166 @@
+import sqlite3
+import struct
+from importlib import metadata
+from pathlib import Path
+from typing import Literal
+
+import sqlite_vec
+from packaging.version import parse
+from rich.console import Console
+
+from haiku.rag.config import Config
+from haiku.rag.embeddings import get_embedder
+from haiku.rag.store.upgrades import upgrades
+from haiku.rag.utils import int_to_semantic_version, semantic_version_to_int
+
+
+class Store:
+    def __init__(
+        self, db_path: Path | Literal[":memory:"], skip_validation: bool = False
+    ):
+        self.db_path: Path | Literal[":memory:"] = db_path
+        self.create_or_update_db()
+
+        # Validate config compatibility after connection is established
+        if not skip_validation:
+            from haiku.rag.store.repositories.settings import SettingsRepository
+
+            settings_repo = SettingsRepository(self)
+            settings_repo.validate_config_compatibility()
+        current_version = metadata.version("haiku.rag")
+        self.set_user_version(current_version)
+
+    def create_or_update_db(self):
+        """Create the database and tables with sqlite-vec support for embeddings."""
+        current_version = metadata.version("haiku.rag")
+
+        db = sqlite3.connect(self.db_path)
+        db.enable_load_extension(True)
+        sqlite_vec.load(db)
+        self._connection = db
+        existing_tables = [
+            row[0]
+            for row in db.execute(
+                "SELECT name FROM sqlite_master WHERE type='table';"
+            ).fetchall()
+        ]
+
+        # If we have a db already, perform upgrades and return
+        if self.db_path != ":memory:" and "documents" in existing_tables:
+            # Upgrade database
+            console = Console()
+            db_version = self.get_user_version()
+            for version, steps in upgrades:
+                if parse(current_version) >= parse(version) and parse(version) > parse(
+                    db_version
+                ):
+                    for step in steps:
+                        step(db)
+                        console.print(
+                            f"[green][b]DB Upgrade: [/b]{step.__doc__}[/green]"
+                        )
+            return
+
+        # Create documents table
+        db.execute("""
+            CREATE TABLE IF NOT EXISTS documents (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                content TEXT NOT NULL,
+                uri TEXT,
+                metadata TEXT DEFAULT '{}',
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        # Create chunks table
+        db.execute("""
+            CREATE TABLE IF NOT EXISTS chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                document_id INTEGER NOT NULL,
+                content TEXT NOT NULL,
+                metadata TEXT DEFAULT '{}',
+                FOREIGN KEY (document_id) REFERENCES documents (id) ON DELETE CASCADE
+            )
+        """)
+        # Create vector table for chunk embeddings
+        embedder = get_embedder()
+        db.execute(f"""
+            CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0(
+                chunk_id INTEGER PRIMARY KEY,
+                embedding FLOAT[{embedder._vector_dim}]
+            )
+        """)
+        # Create FTS5 table for full-text search
+        db.execute("""
+            CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+                content,
+                content='chunks',
+                content_rowid='id'
+            )
+        """)
+        # Create settings table for storing current configuration
+        db.execute("""
+            CREATE TABLE IF NOT EXISTS settings (
+                id INTEGER PRIMARY KEY DEFAULT 1,
+                settings TEXT NOT NULL DEFAULT '{}'
+            )
+        """)
+        # Save current settings to the new database
+        settings_json = Config.model_dump_json()
+        db.execute(
+            "INSERT OR IGNORE INTO settings (id, settings) VALUES (1, ?)",
+            (settings_json,),
+        )
+        # Create indexes for better performance
+        db.execute(
+            "CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)"
+        )
+        db.commit()
+
+    def get_user_version(self) -> str:
+        """Returns the SQLite user version"""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self._connection.execute("PRAGMA user_version;")
+        version = cursor.fetchone()
+        return int_to_semantic_version(version[0])
+
+    def set_user_version(self, version: str) -> None:
+        """Updates the SQLite user version"""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        self._connection.execute(
+            f"PRAGMA user_version = {semantic_version_to_int(version)};"
+        )
+
+    def recreate_embeddings_table(self) -> None:
+        """Recreate the embeddings table with current vector dimensions."""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        # Drop existing embeddings table
+        self._connection.execute("DROP TABLE IF EXISTS chunk_embeddings")
+
+        # Recreate with current dimensions
+        embedder = get_embedder()
+        self._connection.execute(f"""
+            CREATE VIRTUAL TABLE chunk_embeddings USING vec0(
+                chunk_id INTEGER PRIMARY KEY,
+                embedding FLOAT[{embedder._vector_dim}]
+            )
+        """)
+
+        self._connection.commit()
+
+    @staticmethod
+    def serialize_embedding(embedding: list[float]) -> bytes:
+        """Serialize a list of floats to bytes for sqlite-vec storage."""
+        return struct.pack(f"{len(embedding)}f", *embedding)
+
+    def close(self):
+        """Close the database connection if it's an in-memory database."""
+        if self._connection is not None:
+            self._connection.close()
+            self._connection = None