haiku.rag 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

haiku/rag/app.py

@@ -1,9 +1,13 @@
+import asyncio
 from pathlib import Path
 
 from rich.console import Console
 from rich.markdown import Markdown
 
 from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.mcp import create_mcp_server
+from haiku.rag.monitor import FileWatcher
 from haiku.rag.store.models.chunk import Chunk
 from haiku.rag.store.models.document import Document
 
@@ -57,6 +61,17 @@ class HaikuRAGApp:
             for chunk, score in results:
                 self._rich_print_search_result(chunk, score)
 
+    async def ask(self, question: str):
+        async with HaikuRAG(db_path=self.db_path) as self.client:
+            try:
+                answer = await self.client.ask(question)
+                self.console.print(f"[bold blue]Question:[/bold blue] {question}")
+                self.console.print()
+                self.console.print("[bold green]Answer:[/bold green]")
+                self.console.print(Markdown(answer))
+            except Exception as e:
+                self.console.print(f"[red]Error: {e}[/red]")
+
     def _rich_print_document(self, doc: Document, truncate: bool = False):
         """Format a document for display."""
         if truncate:
@@ -88,20 +103,25 @@ class HaikuRAGApp:
         self.console.print(content)
         self.console.rule()
 
-    def serve(self, transport: str | None = None):
+    async def serve(self, transport: str | None = None):
         """Start the MCP server."""
-        from haiku.rag.mcp import create_mcp_server
-
-        server = create_mcp_server(self.db_path)
+        async with HaikuRAG(self.db_path) as client:
+            monitor = FileWatcher(paths=Config.MONITOR_DIRECTORIES, client=client)
+            monitor_task = asyncio.create_task(monitor.observe())
+            server = create_mcp_server(self.db_path)
 
-        if transport == "stdio":
-            self.console.print("[green]Starting MCP server on stdio...[/green]")
-            server.run("stdio")
-        elif transport == "sse":
-            self.console.print(
-                "[green]Starting MCP server with streamable HTTP...[/green]"
-            )
-            server.run("sse")
-        else:
-            self.console.print("[green]Starting MCP server with HTTP...[/green]")
-            server.run("streamable-http")
+            try:
+                if transport == "stdio":
+                    await server.run_stdio_async()
+                elif transport == "sse":
+                    await server.run_sse_async("sse")
+                else:
+                    await server.run_http_async("streamable-http")
+            except KeyboardInterrupt:
+                pass
+            finally:
+                monitor_task.cancel()
+                try:
+                    await monitor_task
+                except asyncio.CancelledError:
+                    pass

haiku/rag/cli.py

@@ -113,6 +113,21 @@ def search(
     event_loop.run_until_complete(app.search(query=query, limit=limit, k=k))
 
 
+@cli.command("ask", help="Ask a question using the QA agent")
+def ask(
+    question: str = typer.Argument(
+        help="The question to ask",
+    ),
+    db: Path = typer.Option(
+        get_default_data_dir() / "haiku.rag.sqlite",
+        "--db",
+        help="Path to the SQLite database file",
+    ),
+):
+    app = HaikuRAGApp(db_path=db)
+    event_loop.run_until_complete(app.ask(question=question))
+
+
 @cli.command(
     "serve", help="Start the haiku.rag MCP server (by default in streamable HTTP mode)"
 )
@@ -146,7 +161,7 @@ def serve(
     elif sse:
         transport = "sse"
 
-    app.serve(transport=transport)
+    event_loop.run_until_complete(app.serve(transport=transport))
 
 
 if __name__ == "__main__":

haiku/rag/client.py

@@ -36,7 +36,7 @@ class HaikuRAG:
         """Async context manager entry."""
         return self
 
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
+    async def __aexit__(self, exc_type, exc_val, exc_tb):  # noqa: ARG002
         """Async context manager exit."""
         self.close()
         return False
@@ -88,7 +88,7 @@ class HaikuRAG:
         if not source_path.exists():
             raise ValueError(f"File does not exist: {source_path}")
 
-        uri = str(source_path.resolve())
+        uri = source_path.as_uri()
         md5_hash = hashlib.md5(source_path.read_bytes()).hexdigest()
 
         # Check if document already exists
@@ -256,6 +256,20 @@ class HaikuRAG:
         """
         return await self.chunk_repository.search_chunks_hybrid(query, limit, k)
 
+    async def ask(self, question: str) -> str:
+        """Ask a question using the configured QA agent.
+
+        Args:
+            question: The question to ask
+
+        Returns:
+            The generated answer as a string
+        """
+        from haiku.rag.qa import get_qa_agent
+
+        qa_agent = get_qa_agent(self)
+        return await qa_agent.answer(question)
+
     def close(self):
         """Close the underlying store connection."""
         self.store.close()

haiku/rag/config.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 
 from dotenv import load_dotenv
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 from haiku.rag.utils import get_default_data_dir
 
@@ -13,16 +13,39 @@ class AppConfig(BaseModel):
     ENV: str = "development"
 
     DEFAULT_DATA_DIR: Path = get_default_data_dir()
+    MONITOR_DIRECTORIES: list[Path] = []
 
-    EMBEDDING_PROVIDER: str = "ollama"
-    EMBEDDING_MODEL: str = "mxbai-embed-large"
-    EMBEDDING_VECTOR_DIM: int = 1024
+    EMBEDDINGS_PROVIDER: str = "ollama"
+    EMBEDDINGS_MODEL: str = "mxbai-embed-large"
+    EMBEDDINGS_VECTOR_DIM: int = 1024
+
+    QA_PROVIDER: str = "ollama"
+    QA_MODEL: str = "qwen3"
 
     CHUNK_SIZE: int = 256
     CHUNK_OVERLAP: int = 32
 
     OLLAMA_BASE_URL: str = "http://localhost:11434"
 
+    # Provider keys
+    VOYAGE_API_KEY: str = ""
+    OPENAI_API_KEY: str = ""
+
+    @field_validator("MONITOR_DIRECTORIES", mode="before")
+    @classmethod
+    def parse_monitor_directories(cls, v):
+        if isinstance(v, str):
+            if not v.strip():
+                return []
+            return [
+                Path(path.strip()).absolute() for path in v.split(",") if path.strip()
+            ]
+        return v
+
 
 # Expose Config object for app to import
 Config = AppConfig.model_validate(os.environ)
+if Config.OPENAI_API_KEY:
+    os.environ["OPENAI_API_KEY"] = Config.OPENAI_API_KEY
+if Config.VOYAGE_API_KEY:
+    os.environ["VOYAGE_API_KEY"] = Config.VOYAGE_API_KEY

haiku/rag/embeddings/__init__.py

@@ -8,10 +8,10 @@ def get_embedder() -> EmbedderBase:
     Factory function to get the appropriate embedder based on the configuration.
     """
 
-    if Config.EMBEDDING_PROVIDER == "ollama":
-        return OllamaEmbedder(Config.EMBEDDING_MODEL, Config.EMBEDDING_VECTOR_DIM)
+    if Config.EMBEDDINGS_PROVIDER == "ollama":
+        return OllamaEmbedder(Config.EMBEDDINGS_MODEL, Config.EMBEDDINGS_VECTOR_DIM)
 
-    if Config.EMBEDDING_PROVIDER == "voyageai":
+    if Config.EMBEDDINGS_PROVIDER == "voyageai":
         try:
             from haiku.rag.embeddings.voyageai import Embedder as VoyageAIEmbedder
         except ImportError:
@@ -20,5 +20,17 @@ def get_embedder() -> EmbedderBase:
                 "Please install haiku.rag with the 'voyageai' extra:"
                 "uv pip install haiku.rag --extra voyageai"
             )
-        return VoyageAIEmbedder(Config.EMBEDDING_MODEL, Config.EMBEDDING_VECTOR_DIM)
-    raise ValueError(f"Unsupported embedding provider: {Config.EMBEDDING_PROVIDER}")
+        return VoyageAIEmbedder(Config.EMBEDDINGS_MODEL, Config.EMBEDDINGS_VECTOR_DIM)
+
+    if Config.EMBEDDINGS_PROVIDER == "openai":
+        try:
+            from haiku.rag.embeddings.openai import Embedder as OpenAIEmbedder
+        except ImportError:
+            raise ImportError(
+                "OpenAI embedder requires the 'openai' package. "
+                "Please install haiku.rag with the 'openai' extra:"
+                "uv pip install haiku.rag --extra openai"
+            )
+        return OpenAIEmbedder(Config.EMBEDDINGS_MODEL, Config.EMBEDDINGS_VECTOR_DIM)
+
+    raise ValueError(f"Unsupported embedding provider: {Config.EMBEDDINGS_PROVIDER}")

haiku/rag/embeddings/ollama.py

@@ -5,7 +5,7 @@ from haiku.rag.embeddings.base import EmbedderBase
 
 
 class Embedder(EmbedderBase):
-    _model: str = Config.EMBEDDING_MODEL
+    _model: str = Config.EMBEDDINGS_MODEL
     _vector_dim: int = 1024
 
     async def embed(self, text: str) -> list[float]:

haiku/rag/embeddings/openai.py

@@ -0,0 +1,20 @@
+try:
+    from openai import AsyncOpenAI
+
+    from haiku.rag.config import Config
+    from haiku.rag.embeddings.base import EmbedderBase
+
+    class Embedder(EmbedderBase):
+        _model: str = Config.EMBEDDINGS_MODEL
+        _vector_dim: int = 1536
+
+        async def embed(self, text: str) -> list[float]:
+            client = AsyncOpenAI()
+            response = await client.embeddings.create(
+                model=self._model,
+                input=text,
+            )
+            return response.data[0].embedding
+
+except ImportError:
+    pass

haiku/rag/embeddings/voyageai.py

@@ -5,7 +5,7 @@ try:
     from haiku.rag.embeddings.base import EmbedderBase
 
     class Embedder(EmbedderBase):
-        _model: str = Config.EMBEDDING_MODEL
+        _model: str = Config.EMBEDDINGS_MODEL
         _vector_dim: int = 1024
 
         async def embed(self, text: str) -> list[float]:

haiku/rag/logging.py

@@ -0,0 +1,24 @@
+import logging
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+
+def get_logger() -> logging.Logger:
+    logger = logging.getLogger("haiku.rag")
+
+    handler = RichHandler(
+        console=Console(stderr=True),
+        rich_tracebacks=True,
+    )
+    formatter = logging.Formatter("%(message)s")
+    handler.setFormatter(formatter)
+
+    logger.setLevel("INFO")
+
+    # Remove any existing handlers to avoid duplicates on reconfiguration
+    for hdlr in logger.handlers[:]:
+        logger.removeHandler(hdlr)
+
+    logger.addHandler(handler)
+    return logger

haiku/rag/monitor.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+from watchfiles import Change, DefaultFilter, awatch
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.logging import get_logger
+from haiku.rag.reader import FileReader
+from haiku.rag.store.models.document import Document
+
+logger = get_logger()
+
+
+class FileFilter(DefaultFilter):
+    def __init__(self, *, ignore_paths: list[Path] | None = None) -> None:
+        self.extensions = tuple(FileReader.extensions)
+        super().__init__(ignore_paths=ignore_paths)
+
+    def __call__(self, change: "Change", path: str) -> bool:
+        return path.endswith(self.extensions) and super().__call__(change, path)
+
+
+class FileWatcher:
+    def __init__(self, paths: list[Path], client: HaikuRAG):
+        self.paths = paths
+        self.client = client
+
+    async def observe(self):
+        logger.info(f"Watching files in {self.paths}")
+        filter = FileFilter()
+        await self.refresh()
+
+        async for changes in awatch(*self.paths, watch_filter=filter):
+            await self.handler(changes)
+
+    async def handler(self, changes: set[tuple[Change, str]]):
+        for change, path in changes:
+            if change == Change.added or change == Change.modified:
+                await self._upsert_document(Path(path))
+            elif change == Change.deleted:
+                await self._delete_document(Path(path))
+
+    async def refresh(self):
+        for path in self.paths:
+            for f in Path(path).rglob("**/*"):
+                if f.is_file() and f.suffix in FileReader.extensions:
+                    await self._upsert_document(f)
+
+    async def _upsert_document(self, file: Path) -> Document | None:
+        try:
+            uri = file.as_uri()
+            existing_doc = await self.client.get_document_by_uri(uri)
+            if existing_doc:
+                doc = await self.client.create_document_from_source(str(file))
+                logger.info(f"Updated document {existing_doc.id} from {file}")
+                return doc
+            else:
+                doc = await self.client.create_document_from_source(str(file))
+                logger.info(f"Created new document {doc.id} from {file}")
+                return doc
+        except Exception as e:
+            logger.error(f"Failed to upsert document from {file}: {e}")
+            return None
+
+    async def _delete_document(self, file: Path):
+        try:
+            uri = file.as_uri()
+            existing_doc = await self.client.get_document_by_uri(uri)
+
+            if existing_doc and existing_doc.id:
+                await self.client.delete_document(existing_doc.id)
+                logger.info(f"Deleted document {existing_doc.id} for {file}")
+        except Exception as e:
+            logger.error(f"Failed to delete document for {file}: {e}")

haiku/rag/qa/__init__.py

@@ -0,0 +1,26 @@
+from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.qa.base import QuestionAnswerAgentBase
+from haiku.rag.qa.ollama import QuestionAnswerOllamaAgent
+
+
+def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
+    """
+    Factory function to get the appropriate QA agent based on the configuration.
+    """
+
+    if Config.QA_PROVIDER == "ollama":
+        return QuestionAnswerOllamaAgent(client, model or Config.QA_MODEL)
+
+    if Config.QA_PROVIDER == "openai":
+        try:
+            from haiku.rag.qa.openai import QuestionAnswerOpenAIAgent
+        except ImportError:
+            raise ImportError(
+                "OpenAI QA agent requires the 'openai' package. "
+                "Please install haiku.rag with the 'openai' extra:"
+                "uv pip install haiku.rag --extra openai"
+            )
+        return QuestionAnswerOpenAIAgent(client, model or "gpt-4o-mini")
+
+    raise ValueError(f"Unsupported QA provider: {Config.QA_PROVIDER}")

haiku/rag/qa/base.py

@@ -0,0 +1,41 @@
+from haiku.rag.client import HaikuRAG
+from haiku.rag.qa.prompts import SYSTEM_PROMPT
+
+
+class QuestionAnswerAgentBase:
+    _model: str = ""
+    _system_prompt: str = SYSTEM_PROMPT
+
+    def __init__(self, client: HaikuRAG, model: str = ""):
+        self._model = model
+        self._client = client
+
+    async def answer(self, question: str) -> str:
+        raise NotImplementedError(
+            "QABase is an abstract class. Please implement the answer method in a subclass."
+        )
+
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "search_documents",
+                "description": "Search the knowledge base for relevant documents",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query to find relevant documents",
+                        },
+                        "limit": {
+                            "type": "integer",
+                            "description": "Maximum number of results to return",
+                            "default": 3,
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+        }
+    ]

haiku/rag/qa/ollama.py

@@ -0,0 +1,67 @@
+from ollama import AsyncClient
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+OLLAMA_OPTIONS = {"temperature": 0.0, "seed": 42, "num_ctx": 64000}
+
+
+class QuestionAnswerOllamaAgent(QuestionAnswerAgentBase):
+    def __init__(self, client: HaikuRAG, model: str = Config.QA_MODEL):
+        super().__init__(client, model or self._model)
+
+    async def answer(self, question: str) -> str:
+        ollama_client = AsyncClient(host=Config.OLLAMA_BASE_URL)
+
+        # Define the search tool
+
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": question},
+        ]
+
+        # Initial response with tool calling
+        response = await ollama_client.chat(
+            model=self._model,
+            messages=messages,
+            tools=self.tools,
+            options=OLLAMA_OPTIONS,
+            think=False,
+        )
+
+        if response.get("message", {}).get("tool_calls"):
+            for tool_call in response["message"]["tool_calls"]:
+                if tool_call["function"]["name"] == "search_documents":
+                    args = tool_call["function"]["arguments"]
+                    query = args.get("query", question)
+                    limit = int(args.get("limit", 3))
+
+                    search_results = await self._client.search(query, limit=limit)
+
+                    context_chunks = []
+                    for chunk, score in search_results:
+                        context_chunks.append(
+                            f"Content: {chunk.content}\nScore: {score:.4f}"
+                        )
+
+                    context = "\n\n".join(context_chunks)
+
+                    messages.append(response["message"])
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "content": context,
+                            "tool_call_id": tool_call.get("id", "search_tool"),
+                        }
+                    )
+
+            final_response = await ollama_client.chat(
+                model=self._model,
+                messages=messages,
+                think=False,
+                options=OLLAMA_OPTIONS,
+            )
+            return final_response["message"]["content"]
+        else:
+            return response["message"]["content"]

haiku/rag/qa/openai.py

@@ -0,0 +1,101 @@
+from collections.abc import Sequence
+
+try:
+    from openai import AsyncOpenAI
+    from openai.types.chat import (
+        ChatCompletionAssistantMessageParam,
+        ChatCompletionMessageParam,
+        ChatCompletionSystemMessageParam,
+        ChatCompletionToolMessageParam,
+        ChatCompletionUserMessageParam,
+    )
+    from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerOpenAIAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "gpt-4o-mini"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ChatCompletionToolParam] = [
+                ChatCompletionToolParam(tool) for tool in self.tools
+            ]
+
+        async def answer(self, question: str) -> str:
+            openai_client = AsyncOpenAI()
+
+            # Define the search tool
+
+            messages: list[ChatCompletionMessageParam] = [
+                ChatCompletionSystemMessageParam(
+                    role="system", content=self._system_prompt
+                ),
+                ChatCompletionUserMessageParam(role="user", content=question),
+            ]
+
+            # Initial response with tool calling
+            response = await openai_client.chat.completions.create(
+                model=self._model,
+                messages=messages,
+                tools=self.tools,
+                temperature=0.0,
+            )
+
+            response_message = response.choices[0].message
+
+            if response_message.tool_calls:
+                messages.append(
+                    ChatCompletionAssistantMessageParam(
+                        role="assistant",
+                        content=response_message.content,
+                        tool_calls=[
+                            {
+                                "id": tc.id,
+                                "type": "function",
+                                "function": {
+                                    "name": tc.function.name,
+                                    "arguments": tc.function.arguments,
+                                },
+                            }
+                            for tc in response_message.tool_calls
+                        ],
+                    )
+                )
+
+                for tool_call in response_message.tool_calls:
+                    if tool_call.function.name == "search_documents":
+                        import json
+
+                        args = json.loads(tool_call.function.arguments)
+                        query = args.get("query", question)
+                        limit = int(args.get("limit", 3))
+
+                        search_results = await self._client.search(query, limit=limit)
+
+                        context_chunks = []
+                        for chunk, score in search_results:
+                            context_chunks.append(
+                                f"Content: {chunk.content}\nScore: {score:.4f}"
+                            )
+
+                        context = "\n\n".join(context_chunks)
+
+                        messages.append(
+                            ChatCompletionToolMessageParam(
+                                role="tool",
+                                content=context,
+                                tool_call_id=tool_call.id,
+                            )
+                        )
+
+                final_response = await openai_client.chat.completions.create(
+                    model=self._model,
+                    messages=messages,
+                    temperature=0.0,
+                )
+                return final_response.choices[0].message.content or ""
+            else:
+                return response_message.content or ""
+
+except ImportError:
+    pass

haiku/rag/qa/prompts.py

@@ -0,0 +1,7 @@
+SYSTEM_PROMPT = """
+You are a helpful assistant that uses a RAG library to answer the user's prompt.
+Your task is to provide a concise and accurate answer based on the provided context.
+You should ask the provided tools to find relevant documents and then use the content of those documents to answer the question.
+Never make up information, always use the context to answer the question.
+If the context does not contain enough information to answer the question, respond with "I cannot answer that based on the provided context."
+"""

haiku/rag/store/models/chunk.py

@@ -3,10 +3,12 @@ from pydantic import BaseModel
 
 class Chunk(BaseModel):
     """
-    Represents a document with an ID, content, and metadata.
+    Represents a chunk with content, metadata, and optional document information.
     """
 
     id: int | None = None
     document_id: int
     content: str
     metadata: dict = {}
+    document_uri: str | None = None
+    document_meta: dict = {}

haiku/rag/store/repositories/chunk.py

@@ -240,9 +240,10 @@ class ChunkRepository(BaseRepository[Chunk]):
         # Search for similar chunks using sqlite-vec
         cursor.execute(
             """
-            SELECT c.id, c.document_id, c.content, c.metadata, distance
+            SELECT c.id, c.document_id, c.content, c.metadata, distance, d.uri, d.metadata as document_metadata
             FROM chunk_embeddings
             JOIN chunks c ON c.id = chunk_embeddings.chunk_id
+            JOIN documents d ON c.document_id = d.id
             WHERE embedding MATCH :embedding AND k = :k
             ORDER BY distance
             """,
@@ -257,10 +258,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 1.0 / (1.0 + distance),
             )
-            for chunk_id, document_id, content, metadata_json, distance in results
+            for chunk_id, document_id, content, metadata_json, distance, document_uri, document_metadata_json in results
         ]
 
     async def search_chunks_fts(
@@ -281,9 +286,10 @@ class ChunkRepository(BaseRepository[Chunk]):
         # Search using FTS5
         cursor.execute(
             """
-            SELECT c.id, c.document_id, c.content, c.metadata, rank
+            SELECT c.id, c.document_id, c.content, c.metadata, rank, d.uri, d.metadata as document_metadata
             FROM chunks_fts
             JOIN chunks c ON c.id = chunks_fts.rowid
+            JOIN documents d ON c.document_id = d.id
             WHERE chunks_fts MATCH :query
             ORDER BY rank
             LIMIT :limit
@@ -300,10 +306,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 -rank,
             )
-            for chunk_id, document_id, content, metadata_json, rank in results
+            for chunk_id, document_id, content, metadata_json, rank, document_uri, document_metadata_json in results
             # FTS5 rank is negative BM25 score
         ]
 
@@ -325,7 +335,6 @@ class ChunkRepository(BaseRepository[Chunk]):
         words = re.findall(r"\b\w+\b", query.lower())
         # Join with OR to find chunks containing any of the keywords
         fts_query = " OR ".join(words) if words else query
-
         # Perform hybrid search using RRF (Reciprocal Rank Fusion)
         cursor.execute(
             """
@@ -369,9 +378,10 @@ class ChunkRepository(BaseRepository[Chunk]):
                 LEFT JOIN vector_search v ON a.id = v.id
                 LEFT JOIN fts_search f ON a.id = f.id
             )
-            SELECT id, document_id, content, metadata, rrf_score
-            FROM rrf_scores
-            ORDER BY rrf_score DESC
+            SELECT r.id, r.document_id, r.content, r.metadata, r.rrf_score, d.uri, d.metadata as document_metadata
+            FROM rrf_scores r
+            JOIN documents d ON r.document_id = d.id
+            ORDER BY r.rrf_score DESC
             LIMIT :limit
             """,
             {
@@ -391,10 +401,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 rrf_score,
             )
-            for chunk_id, document_id, content, metadata_json, rrf_score in results
+            for chunk_id, document_id, content, metadata_json, rrf_score, document_uri, document_metadata_json in results
         ]
 
     async def get_by_document_id(self, document_id: int) -> list[Chunk]:
@@ -405,9 +419,11 @@ class ChunkRepository(BaseRepository[Chunk]):
         cursor = self.store._connection.cursor()
         cursor.execute(
             """
-            SELECT id, document_id, content, metadata
-            FROM chunks WHERE document_id = :document_id
-            ORDER BY JSON_EXTRACT(metadata, '$.order')
+            SELECT c.id, c.document_id, c.content, c.metadata, d.uri, d.metadata as document_metadata
+            FROM chunks c
+            JOIN documents d ON c.document_id = d.id
+            WHERE c.document_id = :document_id
+            ORDER BY JSON_EXTRACT(c.metadata, '$.order')
             """,
             {"document_id": document_id},
         )
@@ -419,6 +435,10 @@ class ChunkRepository(BaseRepository[Chunk]):
                 document_id=document_id,
                 content=content,
                 metadata=json.loads(metadata_json) if metadata_json else {},
+                document_uri=document_uri,
+                document_meta=json.loads(document_metadata_json)
+                if document_metadata_json
+                else {},
             )
-            for chunk_id, document_id, content, metadata_json in rows
+            for chunk_id, document_id, content, metadata_json, document_uri, document_metadata_json in rows
         ]

haiku_rag-0.3.0.dist-info/METADATA

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: haiku.rag
+Version: 0.3.0
+Summary: Retrieval Augmented Generation (RAG) with SQLite
+Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: RAG,mcp,ml,sqlite,sqlite-vec
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows :: Windows 10
+Classifier: Operating System :: Microsoft :: Windows :: Windows 11
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: fastmcp>=2.8.1
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: markitdown[audio-transcription,docx,pdf,pptx,xlsx]>=0.1.2
+Requires-Dist: ollama>=0.5.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: rich>=14.0.0
+Requires-Dist: sqlite-vec>=0.1.6
+Requires-Dist: tiktoken>=0.9.0
+Requires-Dist: typer>=0.16.0
+Requires-Dist: watchfiles>=1.1.0
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
+Provides-Extra: voyageai
+Requires-Dist: voyageai>=0.3.2; extra == 'voyageai'
+Description-Content-Type: text/markdown
+
+# Haiku SQLite RAG
+
+Retrieval-Augmented Generation (RAG) library on SQLite.
+
+`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work on SQLite alone without the need for external vector databases. It uses [sqlite-vec](https://github.com/asg017/sqlite-vec) for storing the embeddings and performs semantic (vector) search as well as full-text search combined through Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
+
+## Features
+
+- **Local SQLite**: No external servers required
+- **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
+- **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
+- **Question answering**: Built-in QA agents on your documents
+- **File monitoring**: Auto-index files when run as server
+- **40+ file formats**: PDF, DOCX, HTML, Markdown, audio, URLs
+- **MCP server**: Expose as tools for AI assistants
+- **CLI & Python API**: Use from command line or Python
+
+## Quick Start
+
+```bash
+# Install
+uv pip install haiku.rag
+
+# Add documents
+haiku-rag add "Your content here"
+haiku-rag add-src document.pdf
+
+# Search
+haiku-rag search "query"
+
+# Ask questions
+haiku-rag ask "Who is the author of haiku.rag?"
+
+# Start server with file monitoring
+export MONITOR_DIRECTORIES="/path/to/docs"
+haiku-rag serve
+```
+
+## Python Usage
+
+```python
+from haiku.rag.client import HaikuRAG
+
+async with HaikuRAG("database.db") as client:
+    # Add document
+    doc = await client.create_document("Your content")
+
+    # Search
+    results = await client.search("query")
+    for chunk, score in results:
+        print(f"{score:.3f}: {chunk.content}")
+
+    # Ask questions
+    answer = await client.ask("Who is the author of haiku.rag?")
+    print(answer)
+```
+
+## MCP Server
+
+Use with AI assistants like Claude Desktop:
+
+```bash
+haiku-rag serve --stdio
+```
+
+Provides tools for document management and search directly in your AI assistant.
+
+## Documentation
+
+Full documentation at: https://ggozad.github.io/haiku.rag/
+
+- [Installation](https://ggozad.github.io/haiku.rag/installation/) - Provider setup
+- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
+- [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
+- [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs

haiku_rag-0.3.0.dist-info/RECORD

@@ -0,0 +1,35 @@
+haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+haiku/rag/app.py,sha256=aiytRhpyvDMbn0uVGN-yxfwpWiwGQ_vpNbtzjGBfkOg,5436
+haiku/rag/chunker.py,sha256=lSSPWgNAe7gNZL_yNLmDtqxJix4YclOiG7gbARcEpV8,1871
+haiku/rag/cli.py,sha256=rnDdC4SHUKbF02NR46F7kWVRLM2Nl-6XOU_mOYoAVCg,4456
+haiku/rag/client.py,sha256=la-8r8cD35nJjNZN5TgZfaVHVJiG1ro8Pel3ADmOCtU,10092
+haiku/rag/config.py,sha256=wXVBWqQTJ8eomSv_fRa7IX34t5jOYW9KCBz3YEkSi14,1309
+haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
+haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
+haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
+haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
+haiku/rag/utils.py,sha256=6xVM6z2OmhzB4FEDlPbMsr_ZBBmCbMQb83nP6E2UdxY,629
+haiku/rag/embeddings/__init__.py,sha256=4jUPe2FyIf8BGZ7AncWSlBdNXG3URejBbnkhQf3JiD0,1505
+haiku/rag/embeddings/base.py,sha256=PTAWKTU-Q-hXIhbRK1o6pIdpaW7DFdzJXQ0Nzc6VI-w,379
+haiku/rag/embeddings/ollama.py,sha256=hWdrTiuJwNSRYCqP0WP-z6XXA3RBGkAiknZMsPLH0qU,441
+haiku/rag/embeddings/openai.py,sha256=reh8AykG2f9f5hhRDmqSsjiuCPi9SsXfe2YEZFlxXk8,550
+haiku/rag/embeddings/voyageai.py,sha256=jc0JywdLJD3Ee1MUv1m8MhWCEo0enNnVcrIBtUvD-Ss,534
+haiku/rag/qa/__init__.py,sha256=k8mU7--BEIyGRyARxNShrDM8mFNxN9c9dxl8PAw5lpM,1013
+haiku/rag/qa/base.py,sha256=4ZTM_l5FAZ9cA0f8NeqRJiUAmjatwCTmSoclFw0gTFQ,1349
+haiku/rag/qa/ollama.py,sha256=poShrse-RgLTwa5gbVzoERNTrn5QRpovJCZKYkIpOZI,2393
+haiku/rag/qa/openai.py,sha256=yBbSjGlG4Lo5p2B2NOTa5C6JceX0OJ1jXar_ABFZYYI,3849
+haiku/rag/qa/prompts.py,sha256=dAz2HjD4eJ8tcW534Tx7EuFOs6pSv2kPr7yrHnHtS0E,535
+haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
+haiku/rag/store/engine.py,sha256=BeYZRZ08zaYeeu375ysnAL3tGz4roA3GzP7WRNwznCo,2603
+haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
+haiku/rag/store/models/chunk.py,sha256=lmbPOOTz-N4PXhrA5XCUxyRcSTZBo135fqkV1mwnGcE,309
+haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
+haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPvrDIHVhY5T1A,263
+haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
+haiku/rag/store/repositories/chunk.py,sha256=TzPbYKovC3HnTpGWkzU1zuJpphiUMoHHTKmS-4x75jk,15950
+haiku/rag/store/repositories/document.py,sha256=xpWOpjHFbhVwNJ1gpusEKNY6l_Qyibg9y_bdHCwcfpk,7133
+haiku_rag-0.3.0.dist-info/METADATA,sha256=2VTIC4nIgcS3LKPS3e32ckcLkxjwcTspdS7HVzdJCjs,3719
+haiku_rag-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.3.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.3.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.3.0.dist-info/RECORD,,

haiku_rag-0.1.0.dist-info/METADATA

@@ -1,195 +0,0 @@
-Metadata-Version: 2.4
-Name: haiku.rag
-Version: 0.1.0
-Summary: Retrieval Augmented Generation (RAG) with SQLite
-Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
-License: MIT
-License-File: LICENSE
-Classifier: Development Status :: 4 - Beta
-Classifier: Environment :: Console
-Classifier: Intended Audience :: Developers
-Classifier: Operating System :: MacOS
-Classifier: Operating System :: Microsoft :: Windows :: Windows 10
-Classifier: Operating System :: Microsoft :: Windows :: Windows 11
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Typing :: Typed
-Requires-Python: >=3.10
-Requires-Dist: fastmcp>=2.8.1
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: markitdown[audio-transcription,docx,pdf,pptx,xlsx]>=0.1.2
-Requires-Dist: ollama>=0.5.1
-Requires-Dist: pydantic>=2.11.7
-Requires-Dist: python-dotenv>=1.1.0
-Requires-Dist: rich>=14.0.0
-Requires-Dist: sqlite-vec>=0.1.6
-Requires-Dist: tiktoken>=0.9.0
-Requires-Dist: typer>=0.16.0
-Requires-Dist: watchfiles>=1.1.0
-Provides-Extra: voyageai
-Requires-Dist: voyageai>=0.3.2; extra == 'voyageai'
-Description-Content-Type: text/markdown
-
-# Haiku SQLite RAG
-
-A SQLite-based Retrieval-Augmented Generation (RAG) system built for efficient document storage, chunking, and hybrid search capabilities.
-
-## Features
-- **Local SQLite**: No need to run additional servers
-- **Support for various embedding providers**: You can use Ollama, VoyageAI or add your own
-- **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
-- **Multi-format Support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a url!
-
-## Installation
-
-```bash
-uv pip install haiku.rag
-```
-
-By default Ollama (with the `mxbai-embed-large` model) is used for the embeddings.
-For other providers use:
-
-- **VoyageAI**: `uv pip install haiku.rag --extra voyageai`
-
-## Configuration
-
-If you want to use an alternative embeddings provider (Ollama being the default) you will need to set the provider details through environment variables:
-
-By default:
-
-```bash
-EMBEDDING_PROVIDER="ollama"
-EMBEDDING_MODEL="mxbai-embed-large" # or any other model
-EMBEDDING_VECTOR_DIM=1024
-```
-
-For VoyageAI:
-```bash
-EMBEDDING_PROVIDER="voyageai"
-EMBEDDING_MODEL="voyage-3.5" # or any other model
-EMBEDDING_VECTOR_DIM=1024
-```
-
-## Command Line Interface
-
-`haiku.rag` includes a CLI application for managing documents and performing searches from the command line:
-
-### Available Commands
-
-```bash
-# List all documents
-haiku-rag list
-
-# Add document from text
-haiku-rag add "Your document content here"
-
-# Add document from file or URL
-haiku-rag add-src /path/to/document.pdf
-haiku-rag add-src https://example.com/article.html
-
-# Get and display a specific document
-haiku-rag get 1
-
-# Delete a document by ID
-haiku-rag delete 1
-
-# Search documents
-haiku-rag search "machine learning"
-
-# Search with custom options
-haiku-rag search "python programming" --limit 10 --k 100
-
-# Start MCP server (default HTTP transport)
-haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
-```
-
-All commands support the `--db` option to specify a custom database path. Run
-```bash
-haiku-rag command -h
-```
-to see additional parameters for a command.
-
-## MCP Server
-
-`haiku.rag` includes a Model Context Protocol (MCP) server that exposes RAG functionality as tools for AI assistants like Claude Desktop. The MCP server provides the following tools:
-
-- `add_document_from_file` - Add documents from local file paths
-- `add_document_from_url` - Add documents from URLs
-- `add_document_from_text` - Add documents from raw text content
-- `search_documents` - Search documents using hybrid search
-- `get_document` - Retrieve specific documents by ID
-- `list_documents` - List all documents with pagination
-- `delete_document` - Delete documents by ID
-
-You can start the server (using Streamble HTTP, stdio or SSE transports) with:
-
-```bash
-# Start with default HTTP transport
-haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
-```
-
-## Using `haiku.rag` from python
-
-### Managing documents
-
-```python
-from pathlib import Path
-from haiku.rag.client import HaikuRAG
-
-# Use as async context manager (recommended)
-async with HaikuRAG("path/to/database.db") as client:
-    # Create document from text
-    doc = await client.create_document(
-        content="Your document content here",
-        uri="doc://example",
-        metadata={"source": "manual", "topic": "example"}
-    )
-
-    # Create document from file (auto-parses content)
-    doc = await client.create_document_from_source("path/to/document.pdf")
-
-    # Create document from URL
-    doc = await client.create_document_from_source("https://example.com/article.html")
-
-    # Retrieve documents
-    doc = await client.get_document_by_id(1)
-    doc = await client.get_document_by_uri("file:///path/to/document.pdf")
-
-    # List all documents with pagination
-    docs = await client.list_documents(limit=10, offset=0)
-
-    # Update document content
-    doc.content = "Updated content"
-    await client.update_document(doc)
-
-    # Delete document
-    await client.delete_document(doc.id)
-
-    # Search documents using hybrid search (vector + full-text)
-    results = await client.search("machine learning algorithms", limit=5)
-    for chunk, score in results:
-        print(f"Score: {score:.3f}")
-        print(f"Content: {chunk.content}")
-        print(f"Document ID: {chunk.document_id}")
-        print("---")
-```
-
-## Searching documents
-
-```python
-async with HaikuRAG("database.db") as client:
-
-    results = await client.search(
-        query="machine learning",
-        limit=5,  # Maximum results to return, defaults to 5
-        k=60      # RRF parameter for reciprocal rank fusion, defaults to 60
-    )
-
-    # Process results
-    for chunk, relevance_score in results:
-        print(f"Relevance: {relevance_score:.3f}")
-        print(f"Content: {chunk.content}")
-        print(f"From document: {chunk.document_id}")
-```

haiku_rag-0.1.0.dist-info/RECORD

@@ -1,27 +0,0 @@
-haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-haiku/rag/app.py,sha256=jJb5THgH3nbh2K8uiYsMVlkqVbSkIGEyxPMISM3epMA,4546
-haiku/rag/chunker.py,sha256=lSSPWgNAe7gNZL_yNLmDtqxJix4YclOiG7gbARcEpV8,1871
-haiku/rag/cli.py,sha256=XOxl7H86La7fB4DvsEJxtNuSfZgOtwqQDmECSaxv4sY,4020
-haiku/rag/client.py,sha256=H5zE-HO8Asxo-_vEcnxFqvQixdiTFTqvNH8EkH7Xo4E,9713
-haiku/rag/config.py,sha256=GxpfUwsQmfzQcknIAPEET_Qu-0WFYtPkHrV3arvNdxM,596
-haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
-haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
-haiku/rag/utils.py,sha256=6xVM6z2OmhzB4FEDlPbMsr_ZBBmCbMQb83nP6E2UdxY,629
-haiku/rag/embeddings/__init__.py,sha256=jOamqhoeFX9J-ThwvVyHGd2s8jqJzA8B6J4sxHGZ39o,1007
-haiku/rag/embeddings/base.py,sha256=PTAWKTU-Q-hXIhbRK1o6pIdpaW7DFdzJXQ0Nzc6VI-w,379
-haiku/rag/embeddings/ollama.py,sha256=i_w7hbh-_ukysco274fLkQuFRgaFq0zIwIs8CNmRcLE,440
-haiku/rag/embeddings/voyageai.py,sha256=MPioqQ0duzjglqvnN_8ftVq11fvBrcpV03p9MMLwflM,533
-haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
-haiku/rag/store/engine.py,sha256=BeYZRZ08zaYeeu375ysnAL3tGz4roA3GzP7WRNwznCo,2603
-haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
-haiku/rag/store/models/chunk.py,sha256=D-fLHXtItXXyClj_KaE1OV-QQ-urDGS7lTE-qv2VHjw,223
-haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
-haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPvrDIHVhY5T1A,263
-haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
-haiku/rag/store/repositories/chunk.py,sha256=6zABVlb5zbMQ4s50z9qb53ieHYaiv4CjgxpbsXxs814,14639
-haiku/rag/store/repositories/document.py,sha256=xpWOpjHFbhVwNJ1gpusEKNY6l_Qyibg9y_bdHCwcfpk,7133
-haiku_rag-0.1.0.dist-info/METADATA,sha256=kDmX6IcmvyL8ss4Go30_UDaSBA4TTzpkp6unzcDOgnM,6141
-haiku_rag-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.1.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.1.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.1.0.dist-info/RECORD,,