haiku.rag 0.2.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

haiku/rag/app.py

@@ -3,6 +3,7 @@ from pathlib import Path
 
 from rich.console import Console
 from rich.markdown import Markdown
+from rich.progress import Progress
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.config import Config
@@ -61,6 +62,61 @@ class HaikuRAGApp:
             for chunk, score in results:
                 self._rich_print_search_result(chunk, score)
 
+    async def ask(self, question: str):
+        async with HaikuRAG(db_path=self.db_path) as self.client:
+            try:
+                answer = await self.client.ask(question)
+                self.console.print(f"[bold blue]Question:[/bold blue] {question}")
+                self.console.print()
+                self.console.print("[bold green]Answer:[/bold green]")
+                self.console.print(Markdown(answer))
+            except Exception as e:
+                self.console.print(f"[red]Error: {e}[/red]")
+
+    async def rebuild(self):
+        async with HaikuRAG(db_path=self.db_path) as client:
+            try:
+                documents = await client.list_documents()
+                total_docs = len(documents)
+
+                if total_docs == 0:
+                    self.console.print(
+                        "[yellow]No documents found in database.[/yellow]"
+                    )
+                    return
+
+                self.console.print(
+                    f"[b]Rebuilding database with {total_docs} documents...[/b]"
+                )
+                with Progress() as progress:
+                    task = progress.add_task("Rebuilding...", total=total_docs)
+                    async for _ in client.rebuild_database():
+                        progress.update(task, advance=1)
+
+                self.console.print("[b]Database rebuild completed successfully.[/b]")
+            except Exception as e:
+                self.console.print(f"[red]Error rebuilding database: {e}[/red]")
+
+    def show_settings(self):
+        """Display current configuration settings."""
+        self.console.print("[bold]haiku.rag configuration[/bold]")
+        self.console.print()
+
+        # Get all config fields dynamically
+        for field_name, field_value in Config.model_dump().items():
+            # Format the display value
+            if isinstance(field_value, str) and (
+                "key" in field_name.lower()
+                or "password" in field_name.lower()
+                or "token" in field_name.lower()
+            ):
+                # Hide sensitive values but show if they're set
+                display_value = "✓ Set" if field_value else "✗ Not set"
+            else:
+                display_value = field_value
+
+            self.console.print(f"  [cyan]{field_name}[/cyan]: {display_value}")
+
     def _rich_print_document(self, doc: Document, truncate: bool = False):
         """Format a document for display."""
         if truncate:
@@ -88,6 +144,12 @@ class HaikuRAGApp:
             f"[repr.attrib_name]document_id[/repr.attrib_name]: {chunk.document_id} "
             f"[repr.attrib_name]score[/repr.attrib_name]: {score:.4f}"
         )
+        if chunk.document_uri:
+            self.console.print("[repr.attrib_name]document uri[/repr.attrib_name]:")
+            self.console.print(chunk.document_uri)
+        if chunk.document_meta:
+            self.console.print("[repr.attrib_name]document meta[/repr.attrib_name]:")
+            self.console.print(chunk.document_meta)
         self.console.print("[repr.attrib_name]content[/repr.attrib_name]:")
         self.console.print(content)
         self.console.rule()

haiku/rag/cli.py

@@ -113,6 +113,42 @@ def search(
     event_loop.run_until_complete(app.search(query=query, limit=limit, k=k))
 
 
+@cli.command("ask", help="Ask a question using the QA agent")
+def ask(
+    question: str = typer.Argument(
+        help="The question to ask",
+    ),
+    db: Path = typer.Option(
+        get_default_data_dir() / "haiku.rag.sqlite",
+        "--db",
+        help="Path to the SQLite database file",
+    ),
+):
+    app = HaikuRAGApp(db_path=db)
+    event_loop.run_until_complete(app.ask(question=question))
+
+
+@cli.command("settings", help="Display current configuration settings")
+def settings():
+    app = HaikuRAGApp(db_path=Path())  # Don't need actual DB for settings
+    app.show_settings()
+
+
+@cli.command(
+    "rebuild",
+    help="Rebuild the database by deleting all chunks and re-indexing all documents",
+)
+def rebuild(
+    db: Path = typer.Option(
+        get_default_data_dir() / "haiku.rag.sqlite",
+        "--db",
+        help="Path to the SQLite database file",
+    ),
+):
+    app = HaikuRAGApp(db_path=db)
+    event_loop.run_until_complete(app.rebuild())
+
+
 @cli.command(
     "serve", help="Start the haiku.rag MCP server (by default in streamable HTTP mode)"
 )

haiku/rag/client.py

@@ -1,6 +1,7 @@
 import hashlib
 import mimetypes
 import tempfile
+from collections.abc import AsyncGenerator
 from pathlib import Path
 from typing import Literal
 from urllib.parse import urlparse
@@ -36,7 +37,7 @@ class HaikuRAG:
         """Async context manager entry."""
         return self
 
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
+    async def __aexit__(self, exc_type, exc_val, exc_tb):  # noqa: ARG002
         """Async context manager exit."""
         self.close()
         return False
@@ -256,6 +257,43 @@ class HaikuRAG:
         """
         return await self.chunk_repository.search_chunks_hybrid(query, limit, k)
 
+    async def ask(self, question: str) -> str:
+        """Ask a question using the configured QA agent.
+
+        Args:
+            question: The question to ask
+
+        Returns:
+            The generated answer as a string
+        """
+        from haiku.rag.qa import get_qa_agent
+
+        qa_agent = get_qa_agent(self)
+        return await qa_agent.answer(question)
+
+    async def rebuild_database(self) -> AsyncGenerator[int, None]:
+        """Rebuild the database by deleting all chunks and re-indexing all documents.
+
+        Yields:
+            int: The ID of the document currently being processed
+        """
+        documents = await self.list_documents()
+
+        if not documents:
+            return
+
+        await self.chunk_repository.delete_all()
+
+        for doc in documents:
+            if doc.id is not None:
+                await self.chunk_repository.create_chunks_for_document(
+                    doc.id, doc.content, commit=False
+                )
+                yield doc.id
+
+        if self.store._connection:
+            self.store._connection.commit()
+
     def close(self):
         """Close the underlying store connection."""
         self.store.close()

haiku/rag/config.py

@@ -19,11 +19,19 @@ class AppConfig(BaseModel):
     EMBEDDINGS_MODEL: str = "mxbai-embed-large"
     EMBEDDINGS_VECTOR_DIM: int = 1024
 
+    QA_PROVIDER: str = "ollama"
+    QA_MODEL: str = "qwen3"
+
     CHUNK_SIZE: int = 256
     CHUNK_OVERLAP: int = 32
 
     OLLAMA_BASE_URL: str = "http://localhost:11434"
 
+    # Provider keys
+    VOYAGE_API_KEY: str = ""
+    OPENAI_API_KEY: str = ""
+    ANTHROPIC_API_KEY: str = ""
+
     @field_validator("MONITOR_DIRECTORIES", mode="before")
     @classmethod
     def parse_monitor_directories(cls, v):
@@ -38,3 +46,9 @@ class AppConfig(BaseModel):
 
 # Expose Config object for app to import
 Config = AppConfig.model_validate(os.environ)
+if Config.OPENAI_API_KEY:
+    os.environ["OPENAI_API_KEY"] = Config.OPENAI_API_KEY
+if Config.VOYAGE_API_KEY:
+    os.environ["VOYAGE_API_KEY"] = Config.VOYAGE_API_KEY
+if Config.ANTHROPIC_API_KEY:
+    os.environ["ANTHROPIC_API_KEY"] = Config.ANTHROPIC_API_KEY

haiku/rag/monitor.py

@@ -49,7 +49,6 @@ class FileWatcher:
         try:
             uri = file.as_uri()
             existing_doc = await self.client.get_document_by_uri(uri)
-            print(uri)
             if existing_doc:
                 doc = await self.client.create_document_from_source(str(file))
                 logger.info(f"Updated document {existing_doc.id} from {file}")

haiku/rag/qa/__init__.py

@@ -0,0 +1,39 @@
+from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.qa.base import QuestionAnswerAgentBase
+from haiku.rag.qa.ollama import QuestionAnswerOllamaAgent
+
+
+def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
+    """
+    Factory function to get the appropriate QA agent based on the configuration.
+    """
+
+    if Config.QA_PROVIDER == "ollama":
+        return QuestionAnswerOllamaAgent(client, model or Config.QA_MODEL)
+
+    if Config.QA_PROVIDER == "openai":
+        try:
+            from haiku.rag.qa.openai import QuestionAnswerOpenAIAgent
+        except ImportError:
+            raise ImportError(
+                "OpenAI QA agent requires the 'openai' package. "
+                "Please install haiku.rag with the 'openai' extra:"
+                "uv pip install haiku.rag --extra openai"
+            )
+        return QuestionAnswerOpenAIAgent(client, model or "gpt-4o-mini")
+
+    if Config.QA_PROVIDER == "anthropic":
+        try:
+            from haiku.rag.qa.anthropic import QuestionAnswerAnthropicAgent
+        except ImportError:
+            raise ImportError(
+                "Anthropic QA agent requires the 'anthropic' package. "
+                "Please install haiku.rag with the 'anthropic' extra:"
+                "uv pip install haiku.rag --extra anthropic"
+            )
+        return QuestionAnswerAnthropicAgent(
+            client, model or "claude-3-5-haiku-20241022"
+        )
+
+    raise ValueError(f"Unsupported QA provider: {Config.QA_PROVIDER}")

haiku/rag/qa/anthropic.py

@@ -0,0 +1,112 @@
+from collections.abc import Sequence
+
+try:
+    from anthropic import AsyncAnthropic
+    from anthropic.types import MessageParam, TextBlock, ToolParam, ToolUseBlock
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerAnthropicAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "claude-3-5-haiku-20241022"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ToolParam] = [
+                ToolParam(
+                    name="search_documents",
+                    description="Search the knowledge base for relevant documents",
+                    input_schema={
+                        "type": "object",
+                        "properties": {
+                            "query": {
+                                "type": "string",
+                                "description": "The search query to find relevant documents",
+                            },
+                            "limit": {
+                                "type": "integer",
+                                "description": "Maximum number of results to return",
+                                "default": 3,
+                            },
+                        },
+                        "required": ["query"],
+                    },
+                )
+            ]
+
+        async def answer(self, question: str) -> str:
+            anthropic_client = AsyncAnthropic()
+
+            messages: list[MessageParam] = [{"role": "user", "content": question}]
+
+            response = await anthropic_client.messages.create(
+                model=self._model,
+                max_tokens=4096,
+                system=self._system_prompt,
+                messages=messages,
+                tools=self.tools,
+                temperature=0.0,
+            )
+
+            if response.stop_reason == "tool_use":
+                messages.append({"role": "assistant", "content": response.content})
+
+                # Process tool calls
+                tool_results = []
+                for content_block in response.content:
+                    if isinstance(content_block, ToolUseBlock):
+                        if content_block.name == "search_documents":
+                            args = content_block.input
+                            query = (
+                                args.get("query", question)
+                                if isinstance(args, dict)
+                                else question
+                            )
+                            limit = (
+                                int(args.get("limit", 3))
+                                if isinstance(args, dict)
+                                else 3
+                            )
+
+                            search_results = await self._client.search(
+                                query, limit=limit
+                            )
+
+                            context_chunks = []
+                            for chunk, score in search_results:
+                                context_chunks.append(
+                                    f"Content: {chunk.content}\nScore: {score:.4f}"
+                                )
+
+                            context = "\n\n".join(context_chunks)
+
+                            tool_results.append(
+                                {
+                                    "type": "tool_result",
+                                    "tool_use_id": content_block.id,
+                                    "content": context,
+                                }
+                            )
+
+                if tool_results:
+                    messages.append({"role": "user", "content": tool_results})
+
+                    final_response = await anthropic_client.messages.create(
+                        model=self._model,
+                        max_tokens=4096,
+                        system=self._system_prompt,
+                        messages=messages,
+                        temperature=0.0,
+                    )
+                    if final_response.content:
+                        first_content = final_response.content[0]
+                        if isinstance(first_content, TextBlock):
+                            return first_content.text
+                    return ""
+
+            if response.content:
+                first_content = response.content[0]
+                if isinstance(first_content, TextBlock):
+                    return first_content.text
+            return ""
+
+except ImportError:
+    pass

haiku/rag/qa/base.py

@@ -0,0 +1,41 @@
+from haiku.rag.client import HaikuRAG
+from haiku.rag.qa.prompts import SYSTEM_PROMPT
+
+
+class QuestionAnswerAgentBase:
+    _model: str = ""
+    _system_prompt: str = SYSTEM_PROMPT
+
+    def __init__(self, client: HaikuRAG, model: str = ""):
+        self._model = model
+        self._client = client
+
+    async def answer(self, question: str) -> str:
+        raise NotImplementedError(
+            "QABase is an abstract class. Please implement the answer method in a subclass."
+        )
+
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "search_documents",
+                "description": "Search the knowledge base for relevant documents",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query to find relevant documents",
+                        },
+                        "limit": {
+                            "type": "integer",
+                            "description": "Maximum number of results to return",
+                            "default": 3,
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+        }
+    ]

haiku/rag/qa/ollama.py

@@ -0,0 +1,67 @@
+from ollama import AsyncClient
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+OLLAMA_OPTIONS = {"temperature": 0.0, "seed": 42, "num_ctx": 64000}
+
+
+class QuestionAnswerOllamaAgent(QuestionAnswerAgentBase):
+    def __init__(self, client: HaikuRAG, model: str = Config.QA_MODEL):
+        super().__init__(client, model or self._model)
+
+    async def answer(self, question: str) -> str:
+        ollama_client = AsyncClient(host=Config.OLLAMA_BASE_URL)
+
+        # Define the search tool
+
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": question},
+        ]
+
+        # Initial response with tool calling
+        response = await ollama_client.chat(
+            model=self._model,
+            messages=messages,
+            tools=self.tools,
+            options=OLLAMA_OPTIONS,
+            think=False,
+        )
+
+        if response.get("message", {}).get("tool_calls"):
+            for tool_call in response["message"]["tool_calls"]:
+                if tool_call["function"]["name"] == "search_documents":
+                    args = tool_call["function"]["arguments"]
+                    query = args.get("query", question)
+                    limit = int(args.get("limit", 3))
+
+                    search_results = await self._client.search(query, limit=limit)
+
+                    context_chunks = []
+                    for chunk, score in search_results:
+                        context_chunks.append(
+                            f"Content: {chunk.content}\nScore: {score:.4f}"
+                        )
+
+                    context = "\n\n".join(context_chunks)
+
+                    messages.append(response["message"])
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "content": context,
+                            "tool_call_id": tool_call.get("id", "search_tool"),
+                        }
+                    )
+
+            final_response = await ollama_client.chat(
+                model=self._model,
+                messages=messages,
+                think=False,
+                options=OLLAMA_OPTIONS,
+            )
+            return final_response["message"]["content"]
+        else:
+            return response["message"]["content"]

haiku/rag/qa/openai.py

@@ -0,0 +1,101 @@
+from collections.abc import Sequence
+
+try:
+    from openai import AsyncOpenAI
+    from openai.types.chat import (
+        ChatCompletionAssistantMessageParam,
+        ChatCompletionMessageParam,
+        ChatCompletionSystemMessageParam,
+        ChatCompletionToolMessageParam,
+        ChatCompletionUserMessageParam,
+    )
+    from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerOpenAIAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "gpt-4o-mini"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ChatCompletionToolParam] = [
+                ChatCompletionToolParam(tool) for tool in self.tools
+            ]
+
+        async def answer(self, question: str) -> str:
+            openai_client = AsyncOpenAI()
+
+            # Define the search tool
+
+            messages: list[ChatCompletionMessageParam] = [
+                ChatCompletionSystemMessageParam(
+                    role="system", content=self._system_prompt
+                ),
+                ChatCompletionUserMessageParam(role="user", content=question),
+            ]
+
+            # Initial response with tool calling
+            response = await openai_client.chat.completions.create(
+                model=self._model,
+                messages=messages,
+                tools=self.tools,
+                temperature=0.0,
+            )
+
+            response_message = response.choices[0].message
+
+            if response_message.tool_calls:
+                messages.append(
+                    ChatCompletionAssistantMessageParam(
+                        role="assistant",
+                        content=response_message.content,
+                        tool_calls=[
+                            {
+                                "id": tc.id,
+                                "type": "function",
+                                "function": {
+                                    "name": tc.function.name,
+                                    "arguments": tc.function.arguments,
+                                },
+                            }
+                            for tc in response_message.tool_calls
+                        ],
+                    )
+                )
+
+                for tool_call in response_message.tool_calls:
+                    if tool_call.function.name == "search_documents":
+                        import json
+
+                        args = json.loads(tool_call.function.arguments)
+                        query = args.get("query", question)
+                        limit = int(args.get("limit", 3))
+
+                        search_results = await self._client.search(query, limit=limit)
+
+                        context_chunks = []
+                        for chunk, score in search_results:
+                            context_chunks.append(
+                                f"Content: {chunk.content}\nScore: {score:.4f}"
+                            )
+
+                        context = "\n\n".join(context_chunks)
+
+                        messages.append(
+                            ChatCompletionToolMessageParam(
+                                role="tool",
+                                content=context,
+                                tool_call_id=tool_call.id,
+                            )
+                        )
+
+                final_response = await openai_client.chat.completions.create(
+                    model=self._model,
+                    messages=messages,
+                    temperature=0.0,
+                )
+                return final_response.choices[0].message.content or ""
+            else:
+                return response_message.content or ""
+
+except ImportError:
+    pass

haiku/rag/qa/prompts.py

@@ -0,0 +1,7 @@
+SYSTEM_PROMPT = """
+You are a helpful assistant that uses a RAG library to answer the user's prompt.
+Your task is to provide a concise and accurate answer based on the provided context.
+You should ask the provided tools to find relevant documents and then use the content of those documents to answer the question.
+Never make up information, always use the context to answer the question.
+If the context does not contain enough information to answer the question, respond with "I cannot answer that based on the provided context."
+"""

haiku/rag/store/models/chunk.py

@@ -3,10 +3,12 @@ from pydantic import BaseModel
 
 class Chunk(BaseModel):
     """
-    Represents a document with an ID, content, and metadata.
+    Represents a chunk with content, metadata, and optional document information.
     """
 
     id: int | None = None
     document_id: int
     content: str
     metadata: dict = {}
+    document_uri: str | None = None
+    document_meta: dict = {}

haiku/rag/store/repositories/chunk.py

@@ -208,6 +208,22 @@ class ChunkRepository(BaseRepository[Chunk]):
 
         return created_chunks
 
+    async def delete_all(self, commit: bool = True) -> bool:
+        """Delete all chunks from the database."""
+        if self.store._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self.store._connection.cursor()
+
+        cursor.execute("DELETE FROM chunks_fts")
+        cursor.execute("DELETE FROM chunk_embeddings")
+        cursor.execute("DELETE FROM chunks")
+
+        deleted = cursor.rowcount > 0
+        if commit:
+            self.store._connection.commit()
+        return deleted
+
     async def delete_by_document_id(
         self, document_id: int, commit: bool = True
     ) -> bool:
@@ -240,9 +256,10 @@ class ChunkRepository(BaseRepository[Chunk]):
         # Search for similar chunks using sqlite-vec
         cursor.execute(
             """
-            SELECT c.id, c.document_id, c.content, c.metadata, distance
+            SELECT c.id, c.document_id, c.content, c.metadata, distance, d.uri, d.metadata as document_metadata
             FROM chunk_embeddings
             JOIN chunks c ON c.id = chunk_embeddings.chunk_id
+            JOIN documents d ON c.document_id = d.id
             WHERE embedding MATCH :embedding AND k = :k
             ORDER BY distance
             """,
@@ -257,10 +274,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 1.0 / (1.0 + distance),
             )
-            for chunk_id, document_id, content, metadata_json, distance in results
+            for chunk_id, document_id, content, metadata_json, distance, document_uri, document_metadata_json in results
         ]
 
     async def search_chunks_fts(
@@ -281,9 +302,10 @@ class ChunkRepository(BaseRepository[Chunk]):
         # Search using FTS5
         cursor.execute(
             """
-            SELECT c.id, c.document_id, c.content, c.metadata, rank
+            SELECT c.id, c.document_id, c.content, c.metadata, rank, d.uri, d.metadata as document_metadata
             FROM chunks_fts
             JOIN chunks c ON c.id = chunks_fts.rowid
+            JOIN documents d ON c.document_id = d.id
             WHERE chunks_fts MATCH :query
             ORDER BY rank
             LIMIT :limit
@@ -300,10 +322,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 -rank,
             )
-            for chunk_id, document_id, content, metadata_json, rank in results
+            for chunk_id, document_id, content, metadata_json, rank, document_uri, document_metadata_json in results
             # FTS5 rank is negative BM25 score
         ]
 
@@ -325,7 +351,6 @@ class ChunkRepository(BaseRepository[Chunk]):
         words = re.findall(r"\b\w+\b", query.lower())
         # Join with OR to find chunks containing any of the keywords
         fts_query = " OR ".join(words) if words else query
-
         # Perform hybrid search using RRF (Reciprocal Rank Fusion)
         cursor.execute(
             """
@@ -369,9 +394,10 @@ class ChunkRepository(BaseRepository[Chunk]):
                 LEFT JOIN vector_search v ON a.id = v.id
                 LEFT JOIN fts_search f ON a.id = f.id
             )
-            SELECT id, document_id, content, metadata, rrf_score
-            FROM rrf_scores
-            ORDER BY rrf_score DESC
+            SELECT r.id, r.document_id, r.content, r.metadata, r.rrf_score, d.uri, d.metadata as document_metadata
+            FROM rrf_scores r
+            JOIN documents d ON r.document_id = d.id
+            ORDER BY r.rrf_score DESC
             LIMIT :limit
             """,
             {
@@ -391,10 +417,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 rrf_score,
             )
-            for chunk_id, document_id, content, metadata_json, rrf_score in results
+            for chunk_id, document_id, content, metadata_json, rrf_score, document_uri, document_metadata_json in results
         ]
 
     async def get_by_document_id(self, document_id: int) -> list[Chunk]:
@@ -405,9 +435,11 @@ class ChunkRepository(BaseRepository[Chunk]):
         cursor = self.store._connection.cursor()
         cursor.execute(
             """
-            SELECT id, document_id, content, metadata
-            FROM chunks WHERE document_id = :document_id
-            ORDER BY JSON_EXTRACT(metadata, '$.order')
+            SELECT c.id, c.document_id, c.content, c.metadata, d.uri, d.metadata as document_metadata
+            FROM chunks c
+            JOIN documents d ON c.document_id = d.id
+            WHERE c.document_id = :document_id
+            ORDER BY JSON_EXTRACT(c.metadata, '$.order')
             """,
             {"document_id": document_id},
         )
@@ -419,6 +451,10 @@ class ChunkRepository(BaseRepository[Chunk]):
                 document_id=document_id,
                 content=content,
                 metadata=json.loads(metadata_json) if metadata_json else {},
+                document_uri=document_uri,
+                document_meta=json.loads(document_metadata_json)
+                if document_metadata_json
+                else {},
             )
-            for chunk_id, document_id, content, metadata_json in rows
+            for chunk_id, document_id, content, metadata_json, document_uri, document_metadata_json in rows
         ]

haiku_rag-0.3.1.dist-info/METADATA

@@ -0,0 +1,118 @@
+Metadata-Version: 2.4
+Name: haiku.rag
+Version: 0.3.1
+Summary: Retrieval Augmented Generation (RAG) with SQLite
+Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: RAG,mcp,ml,sqlite,sqlite-vec
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows :: Windows 10
+Classifier: Operating System :: Microsoft :: Windows :: Windows 11
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: fastmcp>=2.8.1
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: markitdown[audio-transcription,docx,pdf,pptx,xlsx]>=0.1.2
+Requires-Dist: ollama>=0.5.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: rich>=14.0.0
+Requires-Dist: sqlite-vec>=0.1.6
+Requires-Dist: tiktoken>=0.9.0
+Requires-Dist: typer>=0.16.0
+Requires-Dist: watchfiles>=1.1.0
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.56.0; extra == 'anthropic'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
+Provides-Extra: voyageai
+Requires-Dist: voyageai>=0.3.2; extra == 'voyageai'
+Description-Content-Type: text/markdown
+
+# Haiku SQLite RAG
+
+Retrieval-Augmented Generation (RAG) library on SQLite.
+
+`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work on SQLite alone without the need for external vector databases. It uses [sqlite-vec](https://github.com/asg017/sqlite-vec) for storing the embeddings and performs semantic (vector) search as well as full-text search combined through Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
+
+## Features
+
+- **Local SQLite**: No external servers required
+- **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
+- **Multiple QA providers**: Ollama, OpenAI, Anthropic
+- **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
+- **Question answering**: Built-in QA agents on your documents
+- **File monitoring**: Auto-index files when run as server
+- **40+ file formats**: PDF, DOCX, HTML, Markdown, audio, URLs
+- **MCP server**: Expose as tools for AI assistants
+- **CLI & Python API**: Use from command line or Python
+
+## Quick Start
+
+```bash
+# Install
+uv pip install haiku.rag
+
+# Add documents
+haiku-rag add "Your content here"
+haiku-rag add-src document.pdf
+
+# Search
+haiku-rag search "query"
+
+# Ask questions
+haiku-rag ask "Who is the author of haiku.rag?"
+
+# Rebuild database (re-chunk and re-embed all documents)
+haiku-rag rebuild
+
+# Start server with file monitoring
+export MONITOR_DIRECTORIES="/path/to/docs"
+haiku-rag serve
+```
+
+## Python Usage
+
+```python
+from haiku.rag.client import HaikuRAG
+
+async with HaikuRAG("database.db") as client:
+    # Add document
+    doc = await client.create_document("Your content")
+
+    # Search
+    results = await client.search("query")
+    for chunk, score in results:
+        print(f"{score:.3f}: {chunk.content}")
+
+    # Ask questions
+    answer = await client.ask("Who is the author of haiku.rag?")
+    print(answer)
+```
+
+## MCP Server
+
+Use with AI assistants like Claude Desktop:
+
+```bash
+haiku-rag serve --stdio
+```
+
+Provides tools for document management and search directly in your AI assistant.
+
+## Documentation
+
+Full documentation at: https://ggozad.github.io/haiku.rag/
+
+- [Installation](https://ggozad.github.io/haiku.rag/installation/) - Provider setup
+- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
+- [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
+- [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs

{haiku_rag-0.2.0.dist-info → haiku_rag-0.3.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-haiku/rag/app.py,sha256=FedUvIxPXCi7SmxUi9zJcxmoZBTQZJO00XIkoD-k87s,4915
+haiku/rag/app.py,sha256=Foi_K-sAqHWsIAAaxY2Tb0hyXnMCi06LqIFCPiBS5n0,7627
 haiku/rag/chunker.py,sha256=lSSPWgNAe7gNZL_yNLmDtqxJix4YclOiG7gbARcEpV8,1871
-haiku/rag/cli.py,sha256=SvDPYHHdjPu8bEF8PgE4agUo-5j3Kuq_rS9Cav6xch0,4051
-haiku/rag/client.py,sha256=uWqcowc8J2Yv-liGBGSJnuQkFw4CDlf_ivOxp6E5C1M,9707
-haiku/rag/config.py,sha256=b346EQ7HCFy-OU3K-SzSSoOLMuQseHFoiysYZMO1zCc,1003
+haiku/rag/cli.py,sha256=9F64IIm2c1nBKn7p9D5yYkVZr8HcjDemrzjF9SRGIY8,5017
+haiku/rag/client.py,sha256=qoVgdsP_MH8wVcDTvPIcMgW7323tTjOXH8JKugz5snY,10847
+haiku/rag/config.py,sha256=ctD_pu7nDOieirJofhNMO-OJIONLC5myvcru9iTm_ps,1433
 haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
 haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
-haiku/rag/monitor.py,sha256=aFJb5cnesEBIGyVzt8IXYrlTujiC1QSPczPuAam2yXw,2793
+haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
 haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
 haiku/rag/utils.py,sha256=6xVM6z2OmhzB4FEDlPbMsr_ZBBmCbMQb83nP6E2UdxY,629
 haiku/rag/embeddings/__init__.py,sha256=4jUPe2FyIf8BGZ7AncWSlBdNXG3URejBbnkhQf3JiD0,1505
@@ -14,17 +14,23 @@ haiku/rag/embeddings/base.py,sha256=PTAWKTU-Q-hXIhbRK1o6pIdpaW7DFdzJXQ0Nzc6VI-w,
 haiku/rag/embeddings/ollama.py,sha256=hWdrTiuJwNSRYCqP0WP-z6XXA3RBGkAiknZMsPLH0qU,441
 haiku/rag/embeddings/openai.py,sha256=reh8AykG2f9f5hhRDmqSsjiuCPi9SsXfe2YEZFlxXk8,550
 haiku/rag/embeddings/voyageai.py,sha256=jc0JywdLJD3Ee1MUv1m8MhWCEo0enNnVcrIBtUvD-Ss,534
+haiku/rag/qa/__init__.py,sha256=xN36Sw5xj3rHiI3D9YGSoX4ywK0sSkmHnG0uf_3bj08,1534
+haiku/rag/qa/anthropic.py,sha256=lzHRQxpEv6Qd6iBIqexUgWnq-ITqytppwkfOuRGWdDs,4556
+haiku/rag/qa/base.py,sha256=4ZTM_l5FAZ9cA0f8NeqRJiUAmjatwCTmSoclFw0gTFQ,1349
+haiku/rag/qa/ollama.py,sha256=poShrse-RgLTwa5gbVzoERNTrn5QRpovJCZKYkIpOZI,2393
+haiku/rag/qa/openai.py,sha256=yBbSjGlG4Lo5p2B2NOTa5C6JceX0OJ1jXar_ABFZYYI,3849
+haiku/rag/qa/prompts.py,sha256=dAz2HjD4eJ8tcW534Tx7EuFOs6pSv2kPr7yrHnHtS0E,535
 haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
 haiku/rag/store/engine.py,sha256=BeYZRZ08zaYeeu375ysnAL3tGz4roA3GzP7WRNwznCo,2603
 haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
-haiku/rag/store/models/chunk.py,sha256=D-fLHXtItXXyClj_KaE1OV-QQ-urDGS7lTE-qv2VHjw,223
+haiku/rag/store/models/chunk.py,sha256=lmbPOOTz-N4PXhrA5XCUxyRcSTZBo135fqkV1mwnGcE,309
 haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
 haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPvrDIHVhY5T1A,263
 haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
-haiku/rag/store/repositories/chunk.py,sha256=6zABVlb5zbMQ4s50z9qb53ieHYaiv4CjgxpbsXxs814,14639
+haiku/rag/store/repositories/chunk.py,sha256=gik7ZPOK3gCoG6tU1pGueAZBPmJxIb7obYFUhwINrYg,16497
 haiku/rag/store/repositories/document.py,sha256=xpWOpjHFbhVwNJ1gpusEKNY6l_Qyibg9y_bdHCwcfpk,7133
-haiku_rag-0.2.0.dist-info/METADATA,sha256=o9PPoiXU7VIRAuQVwFvfQg4w-8ufz5aLo9PuG0ykWuI,7468
-haiku_rag-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.2.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.2.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.2.0.dist-info/RECORD,,
+haiku_rag-0.3.1.dist-info/METADATA,sha256=_1rJ4s0aq82EkBRPfaPmRZ84QGYTfACyV5V_hk3F118,3931
+haiku_rag-0.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.3.1.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.3.1.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.3.1.dist-info/RECORD,,

haiku_rag-0.2.0.dist-info/METADATA

@@ -1,230 +0,0 @@
-Metadata-Version: 2.4
-Name: haiku.rag
-Version: 0.2.0
-Summary: Retrieval Augmented Generation (RAG) with SQLite
-Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
-License: MIT
-License-File: LICENSE
-Keywords: RAG,mcp,ml,sqlite,sqlite-vec
-Classifier: Development Status :: 4 - Beta
-Classifier: Environment :: Console
-Classifier: Intended Audience :: Developers
-Classifier: Operating System :: MacOS
-Classifier: Operating System :: Microsoft :: Windows :: Windows 10
-Classifier: Operating System :: Microsoft :: Windows :: Windows 11
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Typing :: Typed
-Requires-Python: >=3.10
-Requires-Dist: fastmcp>=2.8.1
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: markitdown[audio-transcription,docx,pdf,pptx,xlsx]>=0.1.2
-Requires-Dist: ollama>=0.5.1
-Requires-Dist: pydantic>=2.11.7
-Requires-Dist: python-dotenv>=1.1.0
-Requires-Dist: rich>=14.0.0
-Requires-Dist: sqlite-vec>=0.1.6
-Requires-Dist: tiktoken>=0.9.0
-Requires-Dist: typer>=0.16.0
-Requires-Dist: watchfiles>=1.1.0
-Provides-Extra: openai
-Requires-Dist: openai>=1.0.0; extra == 'openai'
-Provides-Extra: voyageai
-Requires-Dist: voyageai>=0.3.2; extra == 'voyageai'
-Description-Content-Type: text/markdown
-
-# Haiku SQLite RAG
-
-A SQLite-based Retrieval-Augmented Generation (RAG) system built for efficient document storage, chunking, and hybrid search capabilities.
-
-## Features
-- **Local SQLite**: No need to run additional servers
-- **Support for various embedding providers**: You can use Ollama, VoyageAI, OpenAI or add your own
-- **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
-- **Multi-format Support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a url!
-- **File monitoring** when run as a server automatically indexing your files
-- **MCP server** Exposes functionality as MCP tools.
-- **Python client** Call `haiku.rag` from your own python applications.
-
-## Installation
-
-```bash
-uv pip install haiku.rag
-```
-
-By default Ollama (with the `mxbai-embed-large` model) is used for the embeddings.
-For other providers use:
-
-- **VoyageAI**: `uv pip install haiku.rag --extra voyageai`
-- **OpenAI**: `uv pip install haiku.rag --extra openai`
-
-## Configuration
-
-You can set the directories to monitor using the `MONITOR_DIRECTORIES` environment variable (as comma separated values) :
-
-```bash
-# Monitor single directory
-export MONITOR_DIRECTORIES="/path/to/documents,/another_path/to/documents"
-```
-
-If you want to use an alternative embeddings provider (Ollama being the default) you will need to set the provider details through environment variables:
-
-By default:
-
-```bash
-EMBEDDINGS_PROVIDER="ollama"
-EMBEDDINGS_MODEL="mxbai-embed-large" # or any other model
-EMBEDDINGS_VECTOR_DIM=1024
-```
-
-For VoyageAI:
-```bash
-EMBEDDINGS_PROVIDER="voyageai"
-EMBEDDINGS_MODEL="voyage-3.5" # or any other model
-EMBEDDINGS_VECTOR_DIM=1024
-VOYAGE_API_KEY="your-api-key"
-```
-
-For OpenAI:
-```bash
-EMBEDDINGS_PROVIDER="openai"
-EMBEDDINGS_MODEL="text-embedding-3-small" # or text-embedding-3-large
-EMBEDDINGS_VECTOR_DIM=1536
-OPENAI_API_KEY="your-api-key"
-```
-
-## Command Line Interface
-
-`haiku.rag` includes a CLI application for managing documents and performing searches from the command line:
-
-### Available Commands
-
-```bash
-# List all documents
-haiku-rag list
-
-# Add document from text
-haiku-rag add "Your document content here"
-
-# Add document from file or URL
-haiku-rag add-src /path/to/document.pdf
-haiku-rag add-src https://example.com/article.html
-
-# Get and display a specific document
-haiku-rag get 1
-
-# Delete a document by ID
-haiku-rag delete 1
-
-# Search documents
-haiku-rag search "machine learning"
-
-# Search with custom options
-haiku-rag search "python programming" --limit 10 --k 100
-
-# Start file monitoring & MCP server (default HTTP transport)
-haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
-```
-
-All commands support the `--db` option to specify a custom database path. Run
-```bash
-haiku-rag command -h
-```
-to see additional parameters for a command.
-
-## File Monitoring & MCP server
-
-You can start the server (using Streamble HTTP, stdio or SSE transports) with:
-
-```bash
-# Start with default HTTP transport
-haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
-```
-
-You need to have set the `MONITOR_DIRECTORIES` environment variable for monitoring to take place.
-
-### File monitoring
-
-`haiku.rag` can watch directories for changes and automatically update the document store:
-
-- **Startup**: Scan all monitored directories and add any new files
-- **File Added/Modified**: Automatically parse and add/update the document in the database
-- **File Deleted**: Remove the corresponding document from the database
-
-### MCP Server
-
-`haiku.rag` includes a Model Context Protocol (MCP) server that exposes RAG functionality as tools for AI assistants like Claude Desktop. The MCP server provides the following tools:
-
-- `add_document_from_file` - Add documents from local file paths
-- `add_document_from_url` - Add documents from URLs
-- `add_document_from_text` - Add documents from raw text content
-- `search_documents` - Search documents using hybrid search
-- `get_document` - Retrieve specific documents by ID
-- `list_documents` - List all documents with pagination
-- `delete_document` - Delete documents by ID
-
-## Using `haiku.rag` from python
-
-### Managing documents
-
-```python
-from pathlib import Path
-from haiku.rag.client import HaikuRAG
-
-# Use as async context manager (recommended)
-async with HaikuRAG("path/to/database.db") as client:
-    # Create document from text
-    doc = await client.create_document(
-        content="Your document content here",
-        uri="doc://example",
-        metadata={"source": "manual", "topic": "example"}
-    )
-
-    # Create document from file (auto-parses content)
-    doc = await client.create_document_from_source("path/to/document.pdf")
-
-    # Create document from URL
-    doc = await client.create_document_from_source("https://example.com/article.html")
-
-    # Retrieve documents
-    doc = await client.get_document_by_id(1)
-    doc = await client.get_document_by_uri("file:///path/to/document.pdf")
-
-    # List all documents with pagination
-    docs = await client.list_documents(limit=10, offset=0)
-
-    # Update document content
-    doc.content = "Updated content"
-    await client.update_document(doc)
-
-    # Delete document
-    await client.delete_document(doc.id)
-
-    # Search documents using hybrid search (vector + full-text)
-    results = await client.search("machine learning algorithms", limit=5)
-    for chunk, score in results:
-        print(f"Score: {score:.3f}")
-        print(f"Content: {chunk.content}")
-        print(f"Document ID: {chunk.document_id}")
-        print("---")
-```
-
-## Searching documents
-
-```python
-async with HaikuRAG("database.db") as client:
-
-    results = await client.search(
-        query="machine learning",
-        limit=5,  # Maximum results to return, defaults to 5
-        k=60      # RRF parameter for reciprocal rank fusion, defaults to 60
-    )
-
-    # Process results
-    for chunk, relevance_score in results:
-        print(f"Relevance: {relevance_score:.3f}")
-        print(f"Content: {chunk.content}")
-        print(f"From document: {chunk.document_id}")
-```