haiku.rag 0.9.2__py3-none-any.whl → 0.14.0__py3-none-any.whl

haiku/rag/app.py

@@ -1,267 +0,0 @@
-import asyncio
-from pathlib import Path
-
-from rich.console import Console
-from rich.markdown import Markdown
-from rich.progress import Progress
-
-from haiku.rag.client import HaikuRAG
-from haiku.rag.config import Config
-from haiku.rag.mcp import create_mcp_server
-from haiku.rag.monitor import FileWatcher
-from haiku.rag.research.orchestrator import ResearchOrchestrator
-from haiku.rag.store.models.chunk import Chunk
-from haiku.rag.store.models.document import Document
-
-
-class HaikuRAGApp:
-    def __init__(self, db_path: Path):
-        self.db_path = db_path
-        self.console = Console()
-
-    async def list_documents(self):
-        async with HaikuRAG(db_path=self.db_path) as self.client:
-            documents = await self.client.list_documents()
-            for doc in documents:
-                self._rich_print_document(doc, truncate=True)
-
-    async def add_document_from_text(self, text: str):
-        async with HaikuRAG(db_path=self.db_path) as self.client:
-            doc = await self.client.create_document(text)
-            self._rich_print_document(doc, truncate=True)
-            self.console.print(
-                f"[b]Document with id [cyan]{doc.id}[/cyan] added successfully.[/b]"
-            )
-
-    async def add_document_from_source(self, source: str):
-        async with HaikuRAG(db_path=self.db_path) as self.client:
-            doc = await self.client.create_document_from_source(source)
-            self._rich_print_document(doc, truncate=True)
-            self.console.print(
-                f"[b]Document with id [cyan]{doc.id}[/cyan] added successfully.[/b]"
-            )
-
-    async def get_document(self, doc_id: str):
-        async with HaikuRAG(db_path=self.db_path) as self.client:
-            doc = await self.client.get_document_by_id(doc_id)
-            if doc is None:
-                self.console.print(f"[red]Document with id {doc_id} not found.[/red]")
-                return
-            self._rich_print_document(doc, truncate=False)
-
-    async def delete_document(self, doc_id: str):
-        async with HaikuRAG(db_path=self.db_path) as self.client:
-            deleted = await self.client.delete_document(doc_id)
-            if deleted:
-                self.console.print(f"[b]Document {doc_id} deleted successfully.[/b]")
-            else:
-                self.console.print(
-                    f"[yellow]Document with id {doc_id} not found.[/yellow]"
-                )
-
-    async def search(self, query: str, limit: int = 5):
-        async with HaikuRAG(db_path=self.db_path) as self.client:
-            results = await self.client.search(query, limit=limit)
-            if not results:
-                self.console.print("[red]No results found.[/red]")
-                return
-            for chunk, score in results:
-                self._rich_print_search_result(chunk, score)
-
-    async def ask(self, question: str, cite: bool = False):
-        async with HaikuRAG(db_path=self.db_path) as self.client:
-            try:
-                answer = await self.client.ask(question, cite=cite)
-                self.console.print(f"[bold blue]Question:[/bold blue] {question}")
-                self.console.print()
-                self.console.print("[bold green]Answer:[/bold green]")
-                self.console.print(Markdown(answer))
-            except Exception as e:
-                self.console.print(f"[red]Error: {e}[/red]")
-
-    async def research(
-        self, question: str, max_iterations: int = 3, verbose: bool = False
-    ):
-        """Run multi-agent research on a question."""
-        async with HaikuRAG(db_path=self.db_path) as client:
-            try:
-                # Create orchestrator with default config or fallback to QA
-                orchestrator = ResearchOrchestrator()
-
-                if verbose:
-                    self.console.print(
-                        f"[bold cyan]Starting research with {orchestrator.provider}:{orchestrator.model}[/bold cyan]"
-                    )
-                    self.console.print(f"[bold blue]Question:[/bold blue] {question}")
-                    self.console.print()
-
-                # Conduct research
-                report = await orchestrator.conduct_research(
-                    question=question,
-                    client=client,
-                    max_iterations=max_iterations,
-                    verbose=verbose,
-                    console=self.console if verbose else None,
-                )
-
-                # Display the report
-                self.console.print("[bold green]Research Report[/bold green]")
-                self.console.rule()
-
-                # Title and Executive Summary
-                self.console.print(f"[bold]{report.title}[/bold]")
-                self.console.print()
-                self.console.print("[bold cyan]Executive Summary:[/bold cyan]")
-                self.console.print(report.executive_summary)
-                self.console.print()
-
-                # Main Findings
-                if report.main_findings:
-                    self.console.print("[bold cyan]Main Findings:[/bold cyan]")
-                    for finding in report.main_findings:
-                        self.console.print(f"• {finding}")
-                    self.console.print()
-
-                # (Themes section removed)
-
-                # Conclusions
-                if report.conclusions:
-                    self.console.print("[bold cyan]Conclusions:[/bold cyan]")
-                    for conclusion in report.conclusions:
-                        self.console.print(f"• {conclusion}")
-                    self.console.print()
-
-                # Recommendations
-                if report.recommendations:
-                    self.console.print("[bold cyan]Recommendations:[/bold cyan]")
-                    for rec in report.recommendations:
-                        self.console.print(f"• {rec}")
-                    self.console.print()
-
-                # Limitations
-                if report.limitations:
-                    self.console.print("[bold yellow]Limitations:[/bold yellow]")
-                    for limitation in report.limitations:
-                        self.console.print(f"• {limitation}")
-                    self.console.print()
-
-                # Sources Summary
-                if report.sources_summary:
-                    self.console.print("[bold cyan]Sources:[/bold cyan]")
-                    self.console.print(report.sources_summary)
-
-            except Exception as e:
-                self.console.print(f"[red]Error during research: {e}[/red]")
-
-    async def rebuild(self):
-        async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client:
-            try:
-                documents = await client.list_documents()
-                total_docs = len(documents)
-
-                if total_docs == 0:
-                    self.console.print(
-                        "[yellow]No documents found in database.[/yellow]"
-                    )
-                    return
-
-                self.console.print(
-                    f"[b]Rebuilding database with {total_docs} documents...[/b]"
-                )
-                with Progress() as progress:
-                    task = progress.add_task("Rebuilding...", total=total_docs)
-                    async for _ in client.rebuild_database():
-                        progress.update(task, advance=1)
-
-                self.console.print("[b]Database rebuild completed successfully.[/b]")
-            except Exception as e:
-                self.console.print(f"[red]Error rebuilding database: {e}[/red]")
-
-    async def vacuum(self):
-        """Run database maintenance: optimize and cleanup table history."""
-        try:
-            async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client:
-                await client.vacuum()
-            self.console.print("[b]Vacuum completed successfully.[/b]")
-        except Exception as e:
-            self.console.print(f"[red]Error during vacuum: {e}[/red]")
-
-    def show_settings(self):
-        """Display current configuration settings."""
-        self.console.print("[bold]haiku.rag configuration[/bold]")
-        self.console.print()
-
-        # Get all config fields dynamically
-        for field_name, field_value in Config.model_dump().items():
-            # Format the display value
-            if isinstance(field_value, str) and (
-                "key" in field_name.lower()
-                or "password" in field_name.lower()
-                or "token" in field_name.lower()
-            ):
-                # Hide sensitive values but show if they're set
-                display_value = "✓ Set" if field_value else "✗ Not set"
-            else:
-                display_value = field_value
-
-            self.console.print(f"  [cyan]{field_name}[/cyan]: {display_value}")
-
-    def _rich_print_document(self, doc: Document, truncate: bool = False):
-        """Format a document for display."""
-        if truncate:
-            content = doc.content.splitlines()
-            if len(content) > 3:
-                content = content[:3] + ["\n…"]
-            content = "\n".join(content)
-            content = Markdown(content)
-        else:
-            content = Markdown(doc.content)
-        self.console.print(
-            f"[repr.attrib_name]id[/repr.attrib_name]: {doc.id} [repr.attrib_name]uri[/repr.attrib_name]: {doc.uri} [repr.attrib_name]meta[/repr.attrib_name]: {doc.metadata}"
-        )
-        self.console.print(
-            f"[repr.attrib_name]created at[/repr.attrib_name]: {doc.created_at} [repr.attrib_name]updated at[/repr.attrib_name]: {doc.updated_at}"
-        )
-        self.console.print("[repr.attrib_name]content[/repr.attrib_name]:")
-        self.console.print(content)
-        self.console.rule()
-
-    def _rich_print_search_result(self, chunk: Chunk, score: float):
-        """Format a search result chunk for display."""
-        content = Markdown(chunk.content)
-        self.console.print(
-            f"[repr.attrib_name]document_id[/repr.attrib_name]: {chunk.document_id} "
-            f"[repr.attrib_name]score[/repr.attrib_name]: {score:.4f}"
-        )
-        if chunk.document_uri:
-            self.console.print("[repr.attrib_name]document uri[/repr.attrib_name]:")
-            self.console.print(chunk.document_uri)
-        if chunk.document_meta:
-            self.console.print("[repr.attrib_name]document meta[/repr.attrib_name]:")
-            self.console.print(chunk.document_meta)
-        self.console.print("[repr.attrib_name]content[/repr.attrib_name]:")
-        self.console.print(content)
-        self.console.rule()
-
-    async def serve(self, transport: str | None = None):
-        """Start the MCP server."""
-        async with HaikuRAG(self.db_path) as client:
-            monitor = FileWatcher(paths=Config.MONITOR_DIRECTORIES, client=client)
-            monitor_task = asyncio.create_task(monitor.observe())
-            server = create_mcp_server(self.db_path)
-
-            try:
-                if transport == "stdio":
-                    await server.run_stdio_async()
-                elif transport == "sse":
-                    await server.run_sse_async()
-                else:
-                    await server.run_http_async(transport="streamable-http")
-            except KeyboardInterrupt:
-                pass
-            finally:
-                monitor_task.cancel()
-                try:
-                    await monitor_task
-                except asyncio.CancelledError:
-                    pass

haiku/rag/chunker.py

@@ -1,51 +0,0 @@
-from typing import ClassVar
-
-import tiktoken
-from docling.chunking import HybridChunker  # type: ignore
-from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
-from docling_core.types.doc.document import DoclingDocument
-
-from haiku.rag.config import Config
-
-
-class Chunker:
-    """A class that chunks text into smaller pieces for embedding and retrieval.
-
-    Uses docling's structure-aware chunking to create semantically meaningful chunks
-    that respect document boundaries.
-
-    Args:
-        chunk_size: The maximum size of a chunk in tokens.
-    """
-
-    encoder: ClassVar[tiktoken.Encoding] = tiktoken.encoding_for_model("gpt-4o")
-
-    def __init__(
-        self,
-        chunk_size: int = Config.CHUNK_SIZE,
-    ):
-        self.chunk_size = chunk_size
-        tokenizer = OpenAITokenizer(
-            tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size
-        )
-
-        self.chunker = HybridChunker(tokenizer=tokenizer)  # type: ignore
-
-    async def chunk(self, document: DoclingDocument) -> list[str]:
-        """Split the document into chunks using docling's structure-aware chunking.
-
-        Args:
-            document: The DoclingDocument to be split into chunks.
-
-        Returns:
-            A list of text chunks with semantic boundaries.
-        """
-        if document is None:
-            return []
-
-        # Chunk using docling's hybrid chunker
-        chunks = list(self.chunker.chunk(document))
-        return [self.chunker.contextualize(chunk) for chunk in chunks]
-
-
-chunker = Chunker()

haiku/rag/cli.py

@@ -1,359 +0,0 @@
-import asyncio
-import warnings
-from importlib.metadata import version
-from pathlib import Path
-
-import logfire
-import typer
-from rich.console import Console
-
-from haiku.rag.app import HaikuRAGApp
-from haiku.rag.config import Config
-from haiku.rag.logging import configure_cli_logging
-from haiku.rag.migration import migrate_sqlite_to_lancedb
-from haiku.rag.utils import is_up_to_date
-
-logfire.configure(send_to_logfire="if-token-present")
-logfire.instrument_pydantic_ai()
-
-if not Config.ENV == "development":
-    warnings.filterwarnings("ignore")
-
-cli = typer.Typer(
-    context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
-)
-
-console = Console()
-
-
-def complete_document_ids(ctx: typer.Context, incomplete: str):
-    """Autocomplete document IDs from the selected DB."""
-    db_path = ctx.params.get("db") or (Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb")
-
-    try:
-        from haiku.rag.client import HaikuRAG
-
-        async def _list_ids():
-            async with HaikuRAG(db_path) as client:
-                docs = await client.list_documents()
-                return [d.id for d in docs if d.id]
-
-        ids = asyncio.run(_list_ids())
-    except Exception:
-        return []
-
-    return [i for i in ids if i and i.startswith(incomplete)]
-
-
-def complete_local_paths(ctx: typer.Context, incomplete: str) -> list[str]:
-    """Autocomplete local filesystem paths.
-
-    Provides directory/file suggestions based on the current incomplete input.
-    Does not validate or restrict to specific extensions to keep it flexible
-    (URLs are still allowed to be typed manually).
-    """
-    try:
-        text = incomplete or ""
-
-        # Expand user home
-        from os.path import expanduser
-
-        expanded = expanduser(text)
-        p = Path(expanded)
-
-        # Choose directory to list and prefix to filter
-        if text == "" or text.endswith(("/", "\\")):
-            directory = p
-            prefix = ""
-        else:
-            directory = p.parent
-            prefix = p.name
-
-        if not directory.exists():
-            return []
-
-        suggestions: list[str] = []
-        for entry in directory.iterdir():
-            name = entry.name
-            if not prefix or name.startswith(prefix):
-                suggestion = str(directory / name)
-                if entry.is_dir():
-                    suggestion += "/"
-                suggestions.append(suggestion)
-        return suggestions
-    except Exception:
-        return []
-
-
-async def check_version():
-    """Check if haiku.rag is up to date and show warning if not."""
-    up_to_date, current_version, latest_version = await is_up_to_date()
-    if not up_to_date:
-        console.print(
-            f"[yellow]Warning: haiku.rag is outdated. Current: {current_version}, Latest: {latest_version}[/yellow]"
-        )
-        console.print("[yellow]Please update.[/yellow]")
-
-
-def version_callback(value: bool):
-    if value:
-        v = version("haiku.rag")
-        console.print(f"haiku.rag version {v}")
-        raise typer.Exit()
-
-
-@cli.callback()
-def main(
-    _version: bool = typer.Option(
-        False,
-        "-v",
-        "--version",
-        callback=version_callback,
-        help="Show version and exit",
-    ),
-):
-    """haiku.rag CLI - Vector database RAG system"""
-    # Ensure only haiku.rag logs are emitted in CLI context
-    configure_cli_logging()
-    # Run version check before any command
-    asyncio.run(check_version())
-
-
-@cli.command("list", help="List all stored documents")
-def list_documents(
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.list_documents())
-
-
-@cli.command("add", help="Add a document from text input")
-def add_document_text(
-    text: str = typer.Argument(
-        help="The text content of the document to add",
-    ),
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.add_document_from_text(text=text))
-
-
-@cli.command("add-src", help="Add a document from a file path or URL")
-def add_document_src(
-    source: str = typer.Argument(
-        help="The file path or URL of the document to add",
-        autocompletion=complete_local_paths,
-    ),
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.add_document_from_source(source=source))
-
-
-@cli.command("get", help="Get and display a document by its ID")
-def get_document(
-    doc_id: str = typer.Argument(
-        help="The ID of the document to get",
-        autocompletion=complete_document_ids,
-    ),
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.get_document(doc_id=doc_id))
-
-
-@cli.command("delete", help="Delete a document by its ID")
-def delete_document(
-    doc_id: str = typer.Argument(
-        help="The ID of the document to delete",
-        autocompletion=complete_document_ids,
-    ),
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.delete_document(doc_id=doc_id))
-
-
-# Add alias `rm` for delete
-cli.command("rm", help="Alias for delete: remove a document by its ID")(delete_document)
-
-
-@cli.command("search", help="Search for documents by a query")
-def search(
-    query: str = typer.Argument(
-        help="The search query to use",
-    ),
-    limit: int = typer.Option(
-        5,
-        "--limit",
-        "-l",
-        help="Maximum number of results to return",
-    ),
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.search(query=query, limit=limit))
-
-
-@cli.command("ask", help="Ask a question using the QA agent")
-def ask(
-    question: str = typer.Argument(
-        help="The question to ask",
-    ),
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-    cite: bool = typer.Option(
-        False,
-        "--cite",
-        help="Include citations in the response",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.ask(question=question, cite=cite))
-
-
-@cli.command("research", help="Run multi-agent research and output a concise report")
-def research(
-    question: str = typer.Argument(
-        help="The research question to investigate",
-    ),
-    max_iterations: int = typer.Option(
-        3,
-        "--max-iterations",
-        "-n",
-        help="Maximum search/analyze iterations",
-    ),
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-    verbose: bool = typer.Option(
-        False,
-        "--verbose",
-        help="Show verbose progress output",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(
-        app.research(
-            question=question,
-            max_iterations=max_iterations,
-            verbose=verbose,
-        )
-    )
-
-
-@cli.command("settings", help="Display current configuration settings")
-def settings():
-    app = HaikuRAGApp(db_path=Path())  # Don't need actual DB for settings
-    app.show_settings()
-
-
-@cli.command(
-    "rebuild",
-    help="Rebuild the database by deleting all chunks and re-indexing all documents",
-)
-def rebuild(
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.rebuild())
-
-
-@cli.command("vacuum", help="Optimize and clean up all tables to reduce disk usage")
-def vacuum(
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-):
-    app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.vacuum())
-
-
-@cli.command(
-    "serve", help="Start the haiku.rag MCP server (by default in streamable HTTP mode)"
-)
-def serve(
-    db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
-        "--db",
-        help="Path to the LanceDB database file",
-    ),
-    stdio: bool = typer.Option(
-        False,
-        "--stdio",
-        help="Run MCP server on stdio Transport",
-    ),
-    sse: bool = typer.Option(
-        False,
-        "--sse",
-        help="Run MCP server on SSE transport",
-    ),
-) -> None:
-    """Start the MCP server."""
-    if stdio and sse:
-        console.print("[red]Error: Cannot use both --stdio and --http options[/red]")
-        raise typer.Exit(1)
-
-    app = HaikuRAGApp(db_path=db)
-
-    transport = None
-    if stdio:
-        transport = "stdio"
-    elif sse:
-        transport = "sse"
-
-    asyncio.run(app.serve(transport=transport))
-
-
-@cli.command("migrate", help="Migrate an SQLite database to LanceDB")
-def migrate(
-    sqlite_path: Path = typer.Argument(
-        help="Path to the SQLite database file to migrate",
-    ),
-):
-    # Generate LanceDB path in same parent directory
-    lancedb_path = sqlite_path.parent / (sqlite_path.stem + ".lancedb")
-
-    success = asyncio.run(migrate_sqlite_to_lancedb(sqlite_path, lancedb_path))
-
-    if not success:
-        raise typer.Exit(1)
-
-
-if __name__ == "__main__":
-    cli()