haiku.rag 0.10.1__tar.gz → 0.11.0__tar.gz

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.10.1
+Version: 0.11.0
 Summary: Agentic Retrieval Augmented Generation (RAG) with LanceDB
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
@@ -66,7 +66,8 @@ uv pip install haiku.rag
 
 # Add documents
 haiku-rag add "Your content here"
-haiku-rag add-src document.pdf
+haiku-rag add "Your content here" --meta author=alice --meta topic=notes
+haiku-rag add-src document.pdf --meta source=manual
 
 # Search
 haiku-rag search "query"
@@ -101,11 +102,12 @@ haiku-rag serve
 ```python
 from haiku.rag.client import HaikuRAG
 from haiku.rag.research import (
+    PlanNode,
     ResearchContext,
     ResearchDeps,
     ResearchState,
     build_research_graph,
-    PlanNode,
+    stream_research_graph,
 )
 
 async with HaikuRAG("database.lancedb") as client:
@@ -127,22 +129,40 @@ async with HaikuRAG("database.lancedb") as client:
 
     # Multi‑agent research pipeline (Plan → Search → Evaluate → Synthesize)
     graph = build_research_graph()
+    question = (
+        "What are the main drivers and trends of global temperature "
+        "anomalies since 1990?"
+    )
     state = ResearchState(
-        question=(
-            "What are the main drivers and trends of global temperature "
-            "anomalies since 1990?"
-        ),
-        context=ResearchContext(original_question="…"),
+        context=ResearchContext(original_question=question),
         max_iterations=2,
         confidence_threshold=0.8,
-        max_concurrency=3,
+        max_concurrency=2,
     )
     deps = ResearchDeps(client=client)
-    start = PlanNode(provider=None, model=None)
-    result = await graph.run(start, state=state, deps=deps)
-    report = result.output
-    print(report.title)
-    print(report.executive_summary)
+
+    # Blocking run (final result only)
+    result = await graph.run(
+        PlanNode(provider="openai", model="gpt-4o-mini"),
+        state=state,
+        deps=deps,
+    )
+    print(result.output.title)
+
+    # Streaming progress (log/report/error events)
+    async for event in stream_research_graph(
+        graph,
+        PlanNode(provider="openai", model="gpt-4o-mini"),
+        state,
+        deps,
+    ):
+        if event.type == "log":
+            iteration = event.state.iterations if event.state else state.iterations
+            print(f"[{iteration}] {event.message}")
+        elif event.type == "report":
+            print("\nResearch complete!\n")
+            print(event.report.title)
+            print(event.report.executive_summary)
 ```
 
 ## MCP Server

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/README.md

@@ -28,7 +28,8 @@ uv pip install haiku.rag
 
 # Add documents
 haiku-rag add "Your content here"
-haiku-rag add-src document.pdf
+haiku-rag add "Your content here" --meta author=alice --meta topic=notes
+haiku-rag add-src document.pdf --meta source=manual
 
 # Search
 haiku-rag search "query"
@@ -63,11 +64,12 @@ haiku-rag serve
 ```python
 from haiku.rag.client import HaikuRAG
 from haiku.rag.research import (
+    PlanNode,
     ResearchContext,
     ResearchDeps,
     ResearchState,
     build_research_graph,
-    PlanNode,
+    stream_research_graph,
 )
 
 async with HaikuRAG("database.lancedb") as client:
@@ -89,22 +91,40 @@ async with HaikuRAG("database.lancedb") as client:
 
     # Multi‑agent research pipeline (Plan → Search → Evaluate → Synthesize)
     graph = build_research_graph()
+    question = (
+        "What are the main drivers and trends of global temperature "
+        "anomalies since 1990?"
+    )
     state = ResearchState(
-        question=(
-            "What are the main drivers and trends of global temperature "
-            "anomalies since 1990?"
-        ),
-        context=ResearchContext(original_question="…"),
+        context=ResearchContext(original_question=question),
         max_iterations=2,
         confidence_threshold=0.8,
-        max_concurrency=3,
+        max_concurrency=2,
     )
     deps = ResearchDeps(client=client)
-    start = PlanNode(provider=None, model=None)
-    result = await graph.run(start, state=state, deps=deps)
-    report = result.output
-    print(report.title)
-    print(report.executive_summary)
+
+    # Blocking run (final result only)
+    result = await graph.run(
+        PlanNode(provider="openai", model="gpt-4o-mini"),
+        state=state,
+        deps=deps,
+    )
+    print(result.output.title)
+
+    # Streaming progress (log/report/error events)
+    async for event in stream_research_graph(
+        graph,
+        PlanNode(provider="openai", model="gpt-4o-mini"),
+        state,
+        deps,
+    ):
+        if event.type == "log":
+            iteration = event.state.iterations if event.state else state.iterations
+            print(f"[{iteration}] {event.message}")
+        elif event.type == "report":
+            print("\nResearch complete!\n")
+            print(event.report.title)
+            print(event.report.executive_summary)
 ```
 
 ## MCP Server

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/docs/agents.md

@@ -47,9 +47,10 @@ title: Research graph
 ---
 stateDiagram-v2
   PlanNode --> SearchDispatchNode
-  SearchDispatchNode --> EvaluateNode
-  EvaluateNode --> SearchDispatchNode
-  EvaluateNode --> SynthesizeNode
+  SearchDispatchNode --> AnalyzeInsightsNode
+  AnalyzeInsightsNode --> DecisionNode
+  DecisionNode --> SearchDispatchNode
+  DecisionNode --> SynthesizeNode
   SynthesizeNode --> [*]
 ```
 
@@ -57,12 +58,15 @@ Key nodes:
 
 - Plan: builds up to 3 standalone sub‑questions (uses an internal presearch tool)
 - Search (batched): answers sub‑questions using the KB with minimal, verbatim context
-- Evaluate: extracts insights, proposes new questions, and checks sufficiency/confidence
+- Analyze: aggregates fresh insights, updates gaps, and suggests new sub-questions
+- Decision: checks sufficiency/confidence thresholds and chooses whether to iterate
 - Synthesize: generates a final structured report
 
 Primary models:
 
 - `SearchAnswer` — one per sub‑question (query, answer, context, sources)
+- `InsightRecord` / `GapRecord` — structured tracking of findings and open issues
+- `InsightAnalysis` — output of the analysis stage (insights, gaps, commentary)
 - `EvaluationResult` — insights, new questions, sufficiency, confidence
 - `ResearchReport` — final report (title, executive summary, findings, conclusions, …)
 
@@ -76,30 +80,75 @@ haiku-rag research "How does haiku.rag organize and query documents?" \
   --verbose
 ```
 
-Python usage:
+Python usage (blocking result):
 
 ```python
 from haiku.rag.client import HaikuRAG
 from haiku.rag.research import (
+    PlanNode,
     ResearchContext,
     ResearchDeps,
     ResearchState,
     build_research_graph,
-    PlanNode,
 )
 
 async with HaikuRAG(path_to_db) as client:
     graph = build_research_graph()
+    question = "What are the main drivers and trends of global temperature anomalies since 1990?"
     state = ResearchState(
-        question="What are the main drivers and trends of global temperature anomalies since 1990?",
-        context=ResearchContext(original_question=... ),
+        context=ResearchContext(original_question=question),
         max_iterations=2,
         confidence_threshold=0.8,
-        max_concurrency=3,
+        max_concurrency=2,
     )
     deps = ResearchDeps(client=client)
-    result = await graph.run(PlanNode(provider=None, model=None), state=state, deps=deps)
+
+    result = await graph.run(
+        PlanNode(provider="openai", model="gpt-4o-mini"),
+        state=state,
+        deps=deps,
+    )
+
     report = result.output
     print(report.title)
     print(report.executive_summary)
 ```
+
+Python usage (streamed events):
+
+```python
+from haiku.rag.client import HaikuRAG
+from haiku.rag.research import (
+    PlanNode,
+    ResearchContext,
+    ResearchDeps,
+    ResearchState,
+    build_research_graph,
+    stream_research_graph,
+)
+
+async with HaikuRAG(path_to_db) as client:
+    graph = build_research_graph()
+    question = "What are the main drivers and trends of global temperature anomalies since 1990?"
+    state = ResearchState(
+        context=ResearchContext(original_question=question),
+        max_iterations=2,
+        confidence_threshold=0.8,
+        max_concurrency=2,
+    )
+    deps = ResearchDeps(client=client)
+
+    async for event in stream_research_graph(
+        graph,
+        PlanNode(provider="openai", model="gpt-4o-mini"),
+        state,
+        deps,
+    ):
+        if event.type == "log":
+            iteration = event.state.iterations if event.state else state.iterations
+            print(f"[{iteration}] {event.message}")
+        elif event.type == "report":
+            print("\nResearch complete!\n")
+            print(event.report.title)
+            print(event.report.executive_summary)
+```

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/docs/cli.md

@@ -27,6 +27,9 @@ haiku-rag list
 From text:
 ```bash
 haiku-rag add "Your document content here"
+
+# Attach metadata (repeat --meta for multiple entries)
+haiku-rag add "Your document content here" --meta author=alice --meta topic=notes
 ```
 
 From file or URL:
@@ -36,6 +39,10 @@ haiku-rag add-src https://example.com/article.html
 
 # Optionally set a human‑readable title stored in the DB schema
 haiku-rag add-src /mnt/data/doc1.pdf --title "Q3 Financial Report"
+
+# Optionally attach metadata (repeat --meta). Values use JSON parsing if possible:
+# numbers, booleans, null, arrays/objects; otherwise kept as strings.
+haiku-rag add-src /mnt/data/doc1.pdf --meta source=manual --meta page_count=12 --meta published=true
 ```
 
 !!! note
@@ -106,6 +113,8 @@ Flags:
 - `--max-concurrency`: number of sub-questions searched in parallel each iteration (default: 3)
 - `--verbose`: show planning, searching previews, evaluation summary, and stop reason
 
+When `--verbose` is set the CLI also consumes the internal research stream, printing every `log` event as agents progress through planning, search, evaluation, and synthesis. If you build your own integration, call `stream_research_graph` to access the same `log`, `report`, and `error` events and render them however you like while the graph is running.
+
 ## Server
 
 Start the MCP server:
@@ -126,6 +135,26 @@ haiku-rag settings
 
 ## Maintenance
 
+### Info (Read-only)
+
+Display database metadata without upgrading or modifying it:
+
+```bash
+haiku-rag info [--db /path/to/your.lancedb]
+```
+
+Shows:
+- path to the database
+- stored haiku.rag version (from settings)
+- embeddings provider/model and vector dimension
+- number of documents
+- table versions per table (documents, chunks)
+
+At the end, a separate “Versions” section lists runtime package versions:
+- haiku.rag
+- lancedb
+- docling
+
 ### Vacuum (Optimize and Cleanup)
 
 Reduce disk usage by optimizing and pruning old table versions across all tables:
@@ -143,6 +172,18 @@ when want to switch embeddings provider or model:
 haiku-rag rebuild
 ```
 
+### Download Models
+
+Download required runtime models:
+
+```bash
+haiku-rag download-models
+```
+
+This command:
+- Downloads Docling OCR/conversion models (no-op if already present).
+- Pulls Ollama models referenced in your configuration (embeddings, QA, research, rerank).
+
 ## Migration
 
 ### Migrate from SQLite to LanceDB

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/docs/index.md

@@ -43,7 +43,8 @@ async with HaikuRAG("database.lancedb") as client:
 Or use the CLI:
 ```bash
 haiku-rag add "Your document content"
-haiku-rag add-src /path/to/document.pdf --title "Q3 Financial Report"
+haiku-rag add "Your document content" --meta author=alice
+haiku-rag add-src /path/to/document.pdf --title "Q3 Financial Report" --meta source=manual
 haiku-rag search "query"
 haiku-rag ask "Who is the author of haiku.rag?"
 haiku-rag migrate old_database.sqlite  # Migrate from SQLite

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/docs/installation.md

@@ -72,3 +72,13 @@ VLLM_RERANK_BASE_URL="http://localhost:8001"
 - Python 3.10+
 - Ollama (for default embeddings)
 - vLLM server (for vLLM provider)
+
+## Pre-download Models (Optional)
+
+You can prefetch all required runtime models before first use:
+
+```bash
+haiku-rag download-models
+```
+
+This will download Docling models and pull any Ollama models referenced by your current configuration.

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag"
 description = "Agentic Retrieval Augmented Generation (RAG) with LanceDB"
-version = "0.10.1"
+version = "0.11.0"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 readme = { file = "README.md", content-type = "text/markdown" }

{haiku_rag-0.10.1 → haiku_rag-0.11.0}/src/haiku/rag/app.py

@@ -1,4 +1,6 @@
 import asyncio
+import json
+from importlib.metadata import version as pkg_version
 from pathlib import Path
 
 from rich.console import Console
@@ -16,6 +18,7 @@ from haiku.rag.research.graph import (
     ResearchState,
     build_research_graph,
 )
+from haiku.rag.research.stream import stream_research_graph
 from haiku.rag.store.models.chunk import Chunk
 from haiku.rag.store.models.document import Document
 
@@ -25,26 +28,141 @@ class HaikuRAGApp:
         self.db_path = db_path
         self.console = Console()
 
+    async def info(self):
+        """Display read-only information about the database without modifying it."""
+
+        import lancedb
+
+        # Basic: show path
+        self.console.print("[bold]haiku.rag database info[/bold]")
+        self.console.print(
+            f"  [repr.attrib_name]path[/repr.attrib_name]: {self.db_path}"
+        )
+
+        if not self.db_path.exists():
+            self.console.print("[red]Database path does not exist.[/red]")
+            return
+
+        # Connect without going through Store to avoid upgrades/validation writes
+        try:
+            db = lancedb.connect(self.db_path)
+            table_names = set(db.table_names())
+        except Exception as e:
+            self.console.print(f"[red]Failed to open database: {e}[/red]")
+            return
+
+        try:
+            ldb_version = pkg_version("lancedb")
+        except Exception:
+            ldb_version = "unknown"
+        try:
+            hr_version = pkg_version("haiku.rag")
+        except Exception:
+            hr_version = "unknown"
+        try:
+            docling_version = pkg_version("docling")
+        except Exception:
+            docling_version = "unknown"
+
+        # Read settings (if present) to find stored haiku.rag version and embedding config
+        stored_version = "unknown"
+        embed_provider: str | None = None
+        embed_model: str | None = None
+        vector_dim: int | None = None
+
+        if "settings" in table_names:
+            settings_tbl = db.open_table("settings")
+            arrow = settings_tbl.search().where("id = 'settings'").limit(1).to_arrow()
+            rows = arrow.to_pylist() if arrow is not None else []
+            if rows:
+                raw = rows[0].get("settings") or "{}"
+                data = json.loads(raw) if isinstance(raw, str) else (raw or {})
+                stored_version = str(data.get("version", stored_version))
+                embed_provider = data.get("EMBEDDINGS_PROVIDER")
+                embed_model = data.get("EMBEDDINGS_MODEL")
+                vector_dim = (
+                    int(data.get("EMBEDDINGS_VECTOR_DIM"))  # pyright: ignore[reportArgumentType]
+                    if data.get("EMBEDDINGS_VECTOR_DIM") is not None
+                    else None
+                )
+
+        num_docs = 0
+        if "documents" in table_names:
+            docs_tbl = db.open_table("documents")
+            num_docs = int(docs_tbl.count_rows())  # type: ignore[attr-defined]
+
+        # Table versions per table (direct API)
+        doc_versions = (
+            len(list(db.open_table("documents").list_versions()))
+            if "documents" in table_names
+            else 0
+        )
+        chunk_versions = (
+            len(list(db.open_table("chunks").list_versions()))
+            if "chunks" in table_names
+            else 0
+        )
+
+        self.console.print(
+            f"  [repr.attrib_name]haiku.rag version (db)[/repr.attrib_name]: {stored_version}"
+        )
+        if embed_provider or embed_model or vector_dim:
+            provider_part = embed_provider or "unknown"
+            model_part = embed_model or "unknown"
+            dim_part = f"{vector_dim}" if vector_dim is not None else "unknown"
+            self.console.print(
+                "  [repr.attrib_name]embeddings[/repr.attrib_name]: "
+                f"{provider_part}/{model_part} (dim: {dim_part})"
+            )
+        else:
+            self.console.print(
+                "  [repr.attrib_name]embeddings[/repr.attrib_name]: unknown"
+            )
+        self.console.print(
+            f"  [repr.attrib_name]documents[/repr.attrib_name]: {num_docs}"
+        )
+        self.console.print(
+            f"  [repr.attrib_name]versions (documents)[/repr.attrib_name]: {doc_versions}"
+        )
+        self.console.print(
+            f"  [repr.attrib_name]versions (chunks)[/repr.attrib_name]: {chunk_versions}"
+        )
+        self.console.rule()
+        self.console.print("[bold]Versions[/bold]")
+        self.console.print(
+            f"  [repr.attrib_name]haiku.rag[/repr.attrib_name]: {hr_version}"
+        )
+        self.console.print(
+            f"  [repr.attrib_name]lancedb[/repr.attrib_name]: {ldb_version}"
+        )
+        self.console.print(
+            f"  [repr.attrib_name]docling[/repr.attrib_name]: {docling_version}"
+        )
+
     async def list_documents(self):
         async with HaikuRAG(db_path=self.db_path) as self.client:
             documents = await self.client.list_documents()
             for doc in documents:
                 self._rich_print_document(doc, truncate=True)
 
-    async def add_document_from_text(self, text: str):
+    async def add_document_from_text(self, text: str, metadata: dict | None = None):
         async with HaikuRAG(db_path=self.db_path) as self.client:
-            doc = await self.client.create_document(text)
+            doc = await self.client.create_document(text, metadata=metadata)
             self._rich_print_document(doc, truncate=True)
             self.console.print(
-                f"[b]Document with id [cyan]{doc.id}[/cyan] added successfully.[/b]"
+                f"[bold green]Document {doc.id} added successfully.[/bold green]"
             )
 
-    async def add_document_from_source(self, source: str, title: str | None = None):
+    async def add_document_from_source(
+        self, source: str, title: str | None = None, metadata: dict | None = None
+    ):
         async with HaikuRAG(db_path=self.db_path) as self.client:
-            doc = await self.client.create_document_from_source(source, title=title)
+            doc = await self.client.create_document_from_source(
+                source, title=title, metadata=metadata
+            )
             self._rich_print_document(doc, truncate=True)
             self.console.print(
-                f"[b]Document with id [cyan]{doc.id}[/cyan] added successfully.[/b]"
+                f"[bold green]Document {doc.id} added successfully.[/bold green]"
             )
 
     async def get_document(self, doc_id: str):
@@ -59,7 +177,9 @@ class HaikuRAGApp:
         async with HaikuRAG(db_path=self.db_path) as self.client:
             deleted = await self.client.delete_document(doc_id)
             if deleted:
-                self.console.print(f"[b]Document {doc_id} deleted successfully.[/b]")
+                self.console.print(
+                    f"[bold green]Document {doc_id} deleted successfully.[/bold green]"
+                )
             else:
                 self.console.print(
                     f"[yellow]Document with id {doc_id} not found.[/yellow]"
@@ -69,7 +189,7 @@ class HaikuRAGApp:
         async with HaikuRAG(db_path=self.db_path) as self.client:
             results = await self.client.search(query, limit=limit)
             if not results:
-                self.console.print("[red]No results found.[/red]")
+                self.console.print("[yellow]No results found.[/yellow]")
                 return
             for chunk, score in results:
                 self._rich_print_search_result(chunk, score)
@@ -102,9 +222,9 @@ class HaikuRAGApp:
                     self.console.print()
 
                 graph = build_research_graph()
+                context = ResearchContext(original_question=question)
                 state = ResearchState(
-                    question=question,
-                    context=ResearchContext(original_question=question),
+                    context=context,
                     max_iterations=max_iterations,
                     confidence_threshold=confidence_threshold,
                     max_concurrency=max_concurrency,
@@ -117,22 +237,20 @@ class HaikuRAGApp:
                     provider=Config.RESEARCH_PROVIDER or Config.QA_PROVIDER,
                     model=Config.RESEARCH_MODEL or Config.QA_MODEL,
                 )
-                # Prefer graph.run; fall back to iter if unavailable
                 report = None
-                try:
-                    result = await graph.run(start, state=state, deps=deps)
-                    report = result.output
-                except Exception:
-                    from pydantic_graph import End
-
-                    async with graph.iter(start, state=state, deps=deps) as run:
-                        node = run.next_node
-                        while not isinstance(node, End):
-                            node = await run.next(node)
-                        if run.result:
-                            report = run.result.output
+                async for event in stream_research_graph(graph, start, state, deps):
+                    if event.type == "report":
+                        report = event.report
+                        break
+                    if event.type == "error":
+                        self.console.print(
+                            f"[red]Error during research: {event.message}[/red]"
+                        )
+                        return
+
                 if report is None:
-                    raise RuntimeError("Graph did not produce a report")
+                    self.console.print("[red]Research did not produce a report.[/red]")
+                    return
 
                 # Display the report
                 self.console.print("[bold green]Research Report[/bold green]")
@@ -202,14 +320,16 @@ class HaikuRAGApp:
                     return
 
                 self.console.print(
-                    f"[b]Rebuilding database with {total_docs} documents...[/b]"
+                    f"[bold cyan]Rebuilding database with {total_docs} documents...[/bold cyan]"
                 )
                 with Progress() as progress:
                     task = progress.add_task("Rebuilding...", total=total_docs)
                     async for _ in client.rebuild_database():
                         progress.update(task, advance=1)
 
-                self.console.print("[b]Database rebuild completed successfully.[/b]")
+                self.console.print(
+                    "[bold green]Database rebuild completed successfully.[/bold green]"
+                )
             except Exception as e:
                 self.console.print(f"[red]Error rebuilding database: {e}[/red]")
 
@@ -218,7 +338,9 @@ class HaikuRAGApp:
         try:
             async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client:
                 await client.vacuum()
-            self.console.print("[b]Vacuum completed successfully.[/b]")
+            self.console.print(
+                "[bold green]Vacuum completed successfully.[/bold green]"
+            )
         except Exception as e:
             self.console.print(f"[red]Error during vacuum: {e}[/red]")
 
@@ -240,7 +362,9 @@ class HaikuRAGApp:
             else:
                 display_value = field_value
 
-            self.console.print(f"  [cyan]{field_name}[/cyan]: {display_value}")
+            self.console.print(
+                f"  [repr.attrib_name]{field_name}[/repr.attrib_name]: {display_value}"
+            )
 
     def _rich_print_document(self, doc: Document, truncate: bool = False):
         """Format a document for display."""