haiku.rag 0.3.0__tar.gz → 0.3.1__tar.gz

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.3.0
+Version: 0.3.1
 Summary: Retrieval Augmented Generation (RAG) with SQLite
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
@@ -29,6 +29,8 @@ Requires-Dist: sqlite-vec>=0.1.6
 Requires-Dist: tiktoken>=0.9.0
 Requires-Dist: typer>=0.16.0
 Requires-Dist: watchfiles>=1.1.0
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.56.0; extra == 'anthropic'
 Provides-Extra: openai
 Requires-Dist: openai>=1.0.0; extra == 'openai'
 Provides-Extra: voyageai
@@ -45,6 +47,7 @@ Retrieval-Augmented Generation (RAG) library on SQLite.
 
 - **Local SQLite**: No external servers required
 - **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
+- **Multiple QA providers**: Ollama, OpenAI, Anthropic
 - **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
@@ -68,6 +71,9 @@ haiku-rag search "query"
 # Ask questions
 haiku-rag ask "Who is the author of haiku.rag?"
 
+# Rebuild database (re-chunk and re-embed all documents)
+haiku-rag rebuild
+
 # Start server with file monitoring
 export MONITOR_DIRECTORIES="/path/to/docs"
 haiku-rag serve

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/README.md

@@ -8,6 +8,7 @@ Retrieval-Augmented Generation (RAG) library on SQLite.
 
 - **Local SQLite**: No external servers required
 - **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
+- **Multiple QA providers**: Ollama, OpenAI, Anthropic
 - **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
@@ -31,6 +32,9 @@ haiku-rag search "query"
 # Ask questions
 haiku-rag ask "Who is the author of haiku.rag?"
 
+# Rebuild database (re-chunk and re-embed all documents)
+haiku-rag rebuild
+
 # Start server with file monitoring
 export MONITOR_DIRECTORIES="/path/to/docs"
 haiku-rag serve

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/docs/cli.md

@@ -35,6 +35,16 @@ haiku-rag get 1
 haiku-rag delete 1
 ```
 
+### Rebuild Database
+
+Rebuild the database by deleting all chunks & embeddings and re-indexing all documents:
+
+```bash
+haiku-rag rebuild
+```
+
+Use this when you want to change things like the embedding model or chunk size for example.
+
 ## Search
 
 Basic search:
@@ -56,6 +66,13 @@ haiku-rag ask "Who is the author of haiku.rag?"
 
 The QA agent will search your documents for relevant information and provide a comprehensive answer.
 
+## Configuration
+
+View current configuration settings:
+```bash
+haiku-rag settings
+```
+
 ## Server
 
 Start the MCP server:

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/docs/configuration.md

@@ -84,6 +84,22 @@ QA_MODEL="gpt-4o-mini"  # or gpt-4, gpt-3.5-turbo, etc.
 OPENAI_API_KEY="your-api-key"
 ```
 
+### Anthropic
+
+For Anthropic QA, you need to install haiku.rag with Anthropic extras:
+
+```bash
+uv pip install haiku.rag --extra anthropic
+```
+
+Then configure:
+
+```bash
+QA_PROVIDER="anthropic"
+QA_MODEL="claude-3-5-haiku-20241022"  # or claude-3-5-sonnet-20241022, etc.
+ANTHROPIC_API_KEY="your-api-key"
+```
+
 ## Other Settings
 
 ### Database and Storage

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/docs/index.md

@@ -8,7 +8,7 @@
 - **Local SQLite**: No need to run additional servers
 - **Support for various embedding providers**: Ollama, VoyageAI, OpenAI or add your own
 - **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
-- **Question Answering**: Built-in QA agents using Ollama or OpenAI.
+- **Question Answering**: Built-in QA agents using Ollama, OpenAI, or Anthropic.
 - **File monitoring**: Automatically index files when run as a server
 - **Extended file format support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a URL!
 - **MCP server**: Exposes functionality as MCP tools

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/docs/installation.md

@@ -24,6 +24,12 @@ uv pip install haiku.rag --extra voyageai
 uv pip install haiku.rag --extra openai
 ```
 
+### Anthropic
+
+```bash
+uv pip install haiku.rag --extra anthropic
+```
+
 ## Requirements
 
 - Python 3.10+

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/docs/python.md

@@ -67,6 +67,13 @@ await client.update_document(doc)
 await client.delete_document(doc.id)
 ```
 
+### Rebuilding the Database
+
+```python
+async for doc_id in client.rebuild_database():
+    print(f"Processed document {doc_id}")
+```
+
 ## Searching Documents
 
 Basic search:

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "haiku.rag"
-version = "0.3.0"
+version = "0.3.1"
 description = "Retrieval Augmented Generation (RAG) with SQLite"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
@@ -38,6 +38,7 @@ dependencies = [
 [project.optional-dependencies]
 voyageai = ["voyageai>=0.3.2"]
 openai = ["openai>=1.0.0"]
+anthropic = ["anthropic>=0.56.0"]
 
 [project.scripts]
 haiku-rag = "haiku.rag.cli:cli"

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/src/haiku/rag/app.py

@@ -3,6 +3,7 @@ from pathlib import Path
 
 from rich.console import Console
 from rich.markdown import Markdown
+from rich.progress import Progress
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.config import Config
@@ -72,6 +73,50 @@ class HaikuRAGApp:
             except Exception as e:
                 self.console.print(f"[red]Error: {e}[/red]")
 
+    async def rebuild(self):
+        async with HaikuRAG(db_path=self.db_path) as client:
+            try:
+                documents = await client.list_documents()
+                total_docs = len(documents)
+
+                if total_docs == 0:
+                    self.console.print(
+                        "[yellow]No documents found in database.[/yellow]"
+                    )
+                    return
+
+                self.console.print(
+                    f"[b]Rebuilding database with {total_docs} documents...[/b]"
+                )
+                with Progress() as progress:
+                    task = progress.add_task("Rebuilding...", total=total_docs)
+                    async for _ in client.rebuild_database():
+                        progress.update(task, advance=1)
+
+                self.console.print("[b]Database rebuild completed successfully.[/b]")
+            except Exception as e:
+                self.console.print(f"[red]Error rebuilding database: {e}[/red]")
+
+    def show_settings(self):
+        """Display current configuration settings."""
+        self.console.print("[bold]haiku.rag configuration[/bold]")
+        self.console.print()
+
+        # Get all config fields dynamically
+        for field_name, field_value in Config.model_dump().items():
+            # Format the display value
+            if isinstance(field_value, str) and (
+                "key" in field_name.lower()
+                or "password" in field_name.lower()
+                or "token" in field_name.lower()
+            ):
+                # Hide sensitive values but show if they're set
+                display_value = "✓ Set" if field_value else "✗ Not set"
+            else:
+                display_value = field_value
+
+            self.console.print(f"  [cyan]{field_name}[/cyan]: {display_value}")
+
     def _rich_print_document(self, doc: Document, truncate: bool = False):
         """Format a document for display."""
         if truncate:
@@ -99,6 +144,12 @@ class HaikuRAGApp:
             f"[repr.attrib_name]document_id[/repr.attrib_name]: {chunk.document_id} "
             f"[repr.attrib_name]score[/repr.attrib_name]: {score:.4f}"
         )
+        if chunk.document_uri:
+            self.console.print("[repr.attrib_name]document uri[/repr.attrib_name]:")
+            self.console.print(chunk.document_uri)
+        if chunk.document_meta:
+            self.console.print("[repr.attrib_name]document meta[/repr.attrib_name]:")
+            self.console.print(chunk.document_meta)
         self.console.print("[repr.attrib_name]content[/repr.attrib_name]:")
         self.console.print(content)
         self.console.rule()

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/src/haiku/rag/cli.py

@@ -128,6 +128,27 @@ def ask(
     event_loop.run_until_complete(app.ask(question=question))
 
 
+@cli.command("settings", help="Display current configuration settings")
+def settings():
+    app = HaikuRAGApp(db_path=Path())  # Don't need actual DB for settings
+    app.show_settings()
+
+
+@cli.command(
+    "rebuild",
+    help="Rebuild the database by deleting all chunks and re-indexing all documents",
+)
+def rebuild(
+    db: Path = typer.Option(
+        get_default_data_dir() / "haiku.rag.sqlite",
+        "--db",
+        help="Path to the SQLite database file",
+    ),
+):
+    app = HaikuRAGApp(db_path=db)
+    event_loop.run_until_complete(app.rebuild())
+
+
 @cli.command(
     "serve", help="Start the haiku.rag MCP server (by default in streamable HTTP mode)"
 )

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/src/haiku/rag/client.py

@@ -1,6 +1,7 @@
 import hashlib
 import mimetypes
 import tempfile
+from collections.abc import AsyncGenerator
 from pathlib import Path
 from typing import Literal
 from urllib.parse import urlparse
@@ -270,6 +271,29 @@ class HaikuRAG:
         qa_agent = get_qa_agent(self)
         return await qa_agent.answer(question)
 
+    async def rebuild_database(self) -> AsyncGenerator[int, None]:
+        """Rebuild the database by deleting all chunks and re-indexing all documents.
+
+        Yields:
+            int: The ID of the document currently being processed
+        """
+        documents = await self.list_documents()
+
+        if not documents:
+            return
+
+        await self.chunk_repository.delete_all()
+
+        for doc in documents:
+            if doc.id is not None:
+                await self.chunk_repository.create_chunks_for_document(
+                    doc.id, doc.content, commit=False
+                )
+                yield doc.id
+
+        if self.store._connection:
+            self.store._connection.commit()
+
     def close(self):
         """Close the underlying store connection."""
         self.store.close()

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/src/haiku/rag/config.py

@@ -30,6 +30,7 @@ class AppConfig(BaseModel):
     # Provider keys
     VOYAGE_API_KEY: str = ""
     OPENAI_API_KEY: str = ""
+    ANTHROPIC_API_KEY: str = ""
 
     @field_validator("MONITOR_DIRECTORIES", mode="before")
     @classmethod
@@ -49,3 +50,5 @@ if Config.OPENAI_API_KEY:
     os.environ["OPENAI_API_KEY"] = Config.OPENAI_API_KEY
 if Config.VOYAGE_API_KEY:
     os.environ["VOYAGE_API_KEY"] = Config.VOYAGE_API_KEY
+if Config.ANTHROPIC_API_KEY:
+    os.environ["ANTHROPIC_API_KEY"] = Config.ANTHROPIC_API_KEY

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/src/haiku/rag/qa/__init__.py

@@ -23,4 +23,17 @@ def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
             )
         return QuestionAnswerOpenAIAgent(client, model or "gpt-4o-mini")
 
+    if Config.QA_PROVIDER == "anthropic":
+        try:
+            from haiku.rag.qa.anthropic import QuestionAnswerAnthropicAgent
+        except ImportError:
+            raise ImportError(
+                "Anthropic QA agent requires the 'anthropic' package. "
+                "Please install haiku.rag with the 'anthropic' extra:"
+                "uv pip install haiku.rag --extra anthropic"
+            )
+        return QuestionAnswerAnthropicAgent(
+            client, model or "claude-3-5-haiku-20241022"
+        )
+
     raise ValueError(f"Unsupported QA provider: {Config.QA_PROVIDER}")

haiku_rag-0.3.1/src/haiku/rag/qa/anthropic.py

@@ -0,0 +1,112 @@
+from collections.abc import Sequence
+
+try:
+    from anthropic import AsyncAnthropic
+    from anthropic.types import MessageParam, TextBlock, ToolParam, ToolUseBlock
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerAnthropicAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "claude-3-5-haiku-20241022"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ToolParam] = [
+                ToolParam(
+                    name="search_documents",
+                    description="Search the knowledge base for relevant documents",
+                    input_schema={
+                        "type": "object",
+                        "properties": {
+                            "query": {
+                                "type": "string",
+                                "description": "The search query to find relevant documents",
+                            },
+                            "limit": {
+                                "type": "integer",
+                                "description": "Maximum number of results to return",
+                                "default": 3,
+                            },
+                        },
+                        "required": ["query"],
+                    },
+                )
+            ]
+
+        async def answer(self, question: str) -> str:
+            anthropic_client = AsyncAnthropic()
+
+            messages: list[MessageParam] = [{"role": "user", "content": question}]
+
+            response = await anthropic_client.messages.create(
+                model=self._model,
+                max_tokens=4096,
+                system=self._system_prompt,
+                messages=messages,
+                tools=self.tools,
+                temperature=0.0,
+            )
+
+            if response.stop_reason == "tool_use":
+                messages.append({"role": "assistant", "content": response.content})
+
+                # Process tool calls
+                tool_results = []
+                for content_block in response.content:
+                    if isinstance(content_block, ToolUseBlock):
+                        if content_block.name == "search_documents":
+                            args = content_block.input
+                            query = (
+                                args.get("query", question)
+                                if isinstance(args, dict)
+                                else question
+                            )
+                            limit = (
+                                int(args.get("limit", 3))
+                                if isinstance(args, dict)
+                                else 3
+                            )
+
+                            search_results = await self._client.search(
+                                query, limit=limit
+                            )
+
+                            context_chunks = []
+                            for chunk, score in search_results:
+                                context_chunks.append(
+                                    f"Content: {chunk.content}\nScore: {score:.4f}"
+                                )
+
+                            context = "\n\n".join(context_chunks)
+
+                            tool_results.append(
+                                {
+                                    "type": "tool_result",
+                                    "tool_use_id": content_block.id,
+                                    "content": context,
+                                }
+                            )
+
+                if tool_results:
+                    messages.append({"role": "user", "content": tool_results})
+
+                    final_response = await anthropic_client.messages.create(
+                        model=self._model,
+                        max_tokens=4096,
+                        system=self._system_prompt,
+                        messages=messages,
+                        temperature=0.0,
+                    )
+                    if final_response.content:
+                        first_content = final_response.content[0]
+                        if isinstance(first_content, TextBlock):
+                            return first_content.text
+                    return ""
+
+            if response.content:
+                first_content = response.content[0]
+                if isinstance(first_content, TextBlock):
+                    return first_content.text
+            return ""
+
+except ImportError:
+    pass

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/src/haiku/rag/store/repositories/chunk.py

@@ -208,6 +208,22 @@ class ChunkRepository(BaseRepository[Chunk]):
 
         return created_chunks
 
+    async def delete_all(self, commit: bool = True) -> bool:
+        """Delete all chunks from the database."""
+        if self.store._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self.store._connection.cursor()
+
+        cursor.execute("DELETE FROM chunks_fts")
+        cursor.execute("DELETE FROM chunk_embeddings")
+        cursor.execute("DELETE FROM chunks")
+
+        deleted = cursor.rowcount > 0
+        if commit:
+            self.store._connection.commit()
+        return deleted
+
     async def delete_by_document_id(
         self, document_id: int, commit: bool = True
     ) -> bool:

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/tests/test_qa.py

@@ -12,11 +12,19 @@ except ImportError:
     QuestionAnswerOpenAIAgent = None
     OPENAI_AVAILABLE = False
 
+try:
+    from haiku.rag.qa.anthropic import QuestionAnswerAnthropicAgent
+
+    ANTHROPIC_AVAILABLE = True
+except ImportError:
+    QuestionAnswerAnthropicAgent = None
+    ANTHROPIC_AVAILABLE = False
+
 from .llm_judge import LLMJudge
 
 
 @pytest.mark.asyncio
-async def test_qa_ollama_with_dataset_question(qa_corpus: Dataset):
+async def test_qa_ollama(qa_corpus: Dataset):
     """Test QA with actual question from the dataset using LLM judge."""
     client = HaikuRAG(":memory:")
     qa = QuestionAnswerOllamaAgent(client)
@@ -40,7 +48,7 @@ async def test_qa_ollama_with_dataset_question(qa_corpus: Dataset):
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(not OPENAI_AVAILABLE, reason="OpenAI not available")
-async def test_qa_openai_basic(qa_corpus: Dataset):
+async def test_qa_openai(qa_corpus: Dataset):
     """Test OpenAI QA basic functionality."""
     client = HaikuRAG(":memory:")
     qa = QuestionAnswerOpenAIAgent(client)  # type: ignore
@@ -60,3 +68,27 @@ async def test_qa_openai_basic(qa_corpus: Dataset):
     assert is_equivalent, (
         f"Generated answer not equivalent to expected answer.\nQuestion: {question}\nGenerated: {answer}\nExpected: {expected_answer}"
     )
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not ANTHROPIC_AVAILABLE, reason="Anthropic not available")
+async def test_qa_anthropic(qa_corpus: Dataset):
+    """Test Anthropic QA basic functionality."""
+    client = HaikuRAG(":memory:")
+    qa = QuestionAnswerAnthropicAgent(client)  # type: ignore
+    llm_judge = LLMJudge()
+
+    doc = qa_corpus[1]
+    await client.create_document(
+        content=doc["document_extracted"], uri=doc["document_id"]
+    )
+
+    question = doc["question"]
+    expected_answer = doc["answer"]
+
+    answer = await qa.answer(question)
+    is_equivalent = await llm_judge.judge_answers(question, answer, expected_answer)
+
+    assert is_equivalent, (
+        f"Generated answer not equivalent to expected answer.\nQuestion: {question}\nGenerated: {answer}\nExpected: {expected_answer}"
+    )

haiku_rag-0.3.1/tests/test_rebuild.py

@@ -0,0 +1,52 @@
+import pytest
+from datasets import Dataset
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.store.models.document import Document
+
+
+@pytest.mark.asyncio
+async def test_rebuild_database(qa_corpus: Dataset):
+    """Test rebuild functionality with existing documents."""
+    client = HaikuRAG(":memory:")
+
+    created_docs: list[Document] = []
+    for content in qa_corpus["document_extracted"][:3]:
+        doc = await client.create_document(
+            content=content,
+        )
+        created_docs.append(doc)
+
+    documents_before = await client.list_documents()
+    assert len(documents_before) == 3
+
+    chunks_before = []
+    for doc in created_docs:
+        assert doc.id is not None
+        doc_chunks = await client.chunk_repository.get_by_document_id(doc.id)
+        chunks_before.extend(doc_chunks)
+
+    assert len(chunks_before) > 0
+
+    # Perform rebuild
+    processed_doc_ids = []
+    async for doc_id in client.rebuild_database():
+        processed_doc_ids.append(doc_id)
+
+    # Verify all documents were processed
+    expected_doc_ids = [doc.id for doc in created_docs]
+    assert set(processed_doc_ids) == set(expected_doc_ids)
+
+    documents_after = await client.list_documents()
+    assert len(documents_after) == 3
+
+    # Verify chunks were recreated
+    chunks_after = []
+    for doc in documents_after:
+        if doc.id is not None:
+            doc_chunks = await client.chunk_repository.get_by_document_id(doc.id)
+            chunks_after.extend(doc_chunks)
+
+    assert len(chunks_after) > 0
+
+    client.close()

{haiku_rag-0.3.0 → haiku_rag-0.3.1}/uv.lock

@@ -133,6 +133,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" },
 ]
 
+[[package]]
+name = "anthropic"
+version = "0.56.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "distro" },
+    { name = "httpx" },
+    { name = "jiter" },
+    { name = "pydantic" },
+    { name = "sniffio" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4d/40/0c4eb5728466849803782c8a86eb315af1a6eb0efea6a751de120ab845c9/anthropic-0.56.0.tar.gz", hash = "sha256:56fa9eb61afa004a1664bc85eed071e77b96c579b77395e9cc893097e599f72e", size = 421538, upload-time = "2025-07-01T19:39:10.805Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e5/90/7f4d4084f9c35c3ea3e784646ec12f9b2c8cf8743b2bb5489252659b5bda/anthropic-0.56.0-py3-none-any.whl", hash = "sha256:91f1f74abdcf0958d3296b657304588cc244b1107b89f973ff6f511afdacfc56", size = 289603, upload-time = "2025-07-01T19:39:08.794Z" },
+]
+
 [[package]]
 name = "anyio"
 version = "4.9.0"
@@ -798,7 +816,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.3.0"
+version = "0.3.1"
 source = { editable = "." }
 dependencies = [
     { name = "fastmcp" },
@@ -815,6 +833,9 @@ dependencies = [
 ]
 
 [package.optional-dependencies]
+anthropic = [
+    { name = "anthropic" },
+]
 openai = [
     { name = "openai" },
 ]
@@ -837,6 +858,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
+    { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.56.0" },
     { name = "fastmcp", specifier = ">=2.8.1" },
     { name = "httpx", specifier = ">=0.28.1" },
     { name = "markitdown", extras = ["audio-transcription", "docx", "pdf", "pptx", "xlsx"], specifier = ">=0.1.2" },
@@ -851,7 +873,7 @@ requires-dist = [
     { name = "voyageai", marker = "extra == 'voyageai'", specifier = ">=0.3.2" },
     { name = "watchfiles", specifier = ">=1.1.0" },
 ]
-provides-extras = ["voyageai", "openai"]
+provides-extras = ["voyageai", "openai", "anthropic"]
 
 [package.metadata.requires-dev]
 dev = [