haiku.rag 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

haiku/rag/a2a/__init__.py

@@ -57,12 +57,12 @@ def create_a2a_app(
     """
     base_storage = InMemoryStorage()
     storage = LRUMemoryStorage(
-        storage=base_storage, max_contexts=Config.A2A_MAX_CONTEXTS
+        storage=base_storage, max_contexts=Config.a2a.max_contexts
     )
     broker = InMemoryBroker()
 
     # Create the agent with native search tool
-    model = get_model(Config.QA_PROVIDER, Config.QA_MODEL)
+    model = get_model(Config.qa.provider, Config.qa.model)
     agent = Agent(
         model=model,
         deps_type=AgentDependencies,
@@ -120,7 +120,7 @@ def create_a2a_app(
     # Create FastA2A app with custom worker lifecycle
     @asynccontextmanager
     async def lifespan(app):
-        logger.info(f"Started A2A server (max contexts: {Config.A2A_MAX_CONTEXTS})")
+        logger.info(f"Started A2A server (max contexts: {Config.a2a.max_contexts})")
         async with app.task_manager:
             async with worker.run():
                 yield

haiku/rag/app.py

@@ -231,8 +231,8 @@ class HaikuRAGApp:
                     )
 
                     start_node = DeepQAPlanNode(
-                        provider=Config.QA_PROVIDER,
-                        model=Config.QA_MODEL,
+                        provider=Config.qa.provider,
+                        model=Config.qa.model,
                     )
 
                     result = await graph.run(
@@ -278,8 +278,8 @@ class HaikuRAGApp:
                 )
 
                 start = PlanNode(
-                    provider=Config.RESEARCH_PROVIDER or Config.QA_PROVIDER,
-                    model=Config.RESEARCH_MODEL or Config.QA_MODEL,
+                    provider=Config.research.provider or Config.qa.provider,
+                    model=Config.research.model or Config.qa.model,
                 )
                 report = None
                 async for event in stream_research_graph(graph, start, state, deps):
@@ -474,7 +474,9 @@ class HaikuRAGApp:
 
             # Start file monitor if enabled
             if enable_monitor:
-                monitor = FileWatcher(paths=Config.MONITOR_DIRECTORIES, client=client)
+                monitor = FileWatcher(
+                    paths=Config.storage.monitor_directories, client=client
+                )
                 monitor_task = asyncio.create_task(monitor.observe())
                 tasks.append(monitor_task)
 

haiku/rag/chunker.py

@@ -22,7 +22,7 @@ class Chunker:
 
     def __init__(
         self,
-        chunk_size: int = Config.CHUNK_SIZE,
+        chunk_size: int = Config.processing.chunk_size,
     ):
         self.chunk_size = chunk_size
         tokenizer = OpenAITokenizer(

haiku/rag/cli.py

@@ -42,10 +42,21 @@ def main(
         callback=version_callback,
         help="Show version and exit",
     ),
+    config: Path | None = typer.Option(
+        None,
+        "--config",
+        help="Path to YAML configuration file",
+    ),
 ):
     """haiku.rag CLI - Vector database RAG system"""
+    # Store config path in environment for config loader to use
+    if config:
+        import os
+
+        os.environ["HAIKU_RAG_CONFIG_PATH"] = str(config.absolute())
+
     # Configure logging minimally for CLI context
-    if Config.ENV == "development":
+    if Config.environment == "development":
         # Lazy import logfire only in development
         try:
             import logfire  # type: ignore
@@ -69,7 +80,7 @@ def main(
 @cli.command("list", help="List all stored documents")
 def list_documents(
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -116,7 +127,7 @@ def add_document_text(
         metavar="KEY=VALUE",
     ),
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -145,7 +156,7 @@ def add_document_src(
         metavar="KEY=VALUE",
     ),
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -167,7 +178,7 @@ def get_document(
         help="The ID of the document to get",
     ),
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -184,7 +195,7 @@ def delete_document(
         help="The ID of the document to delete",
     ),
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -211,7 +222,7 @@ def search(
         help="Maximum number of results to return",
     ),
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -228,7 +239,7 @@ def ask(
         help="The question to ask",
     ),
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -276,7 +287,7 @@ def research(
         help="Max concurrent searches per iteration (planned)",
     ),
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -308,13 +319,61 @@ def settings():
     app.show_settings()
 
 
+@cli.command("init-config", help="Generate a YAML configuration file")
+def init_config(
+    output: Path = typer.Argument(
+        Path("haiku.rag.yaml"),
+        help="Output path for the config file",
+    ),
+    from_env: bool = typer.Option(
+        False,
+        "--from-env",
+        help="Migrate settings from .env file",
+    ),
+):
+    """Generate a YAML configuration file with defaults or from .env."""
+    import yaml
+
+    from haiku.rag.config.loader import generate_default_config, load_config_from_env
+
+    if output.exists():
+        typer.echo(
+            f"Error: {output} already exists. Remove it first or choose a different path."
+        )
+        raise typer.Exit(1)
+
+    if from_env:
+        # Load from environment variables (including .env if present)
+        from dotenv import load_dotenv
+
+        load_dotenv()
+        config_data = load_config_from_env()
+        if not config_data:
+            typer.echo("Warning: No environment variables found to migrate.")
+            typer.echo("Generating default configuration instead.")
+            config_data = generate_default_config()
+    else:
+        config_data = generate_default_config()
+
+    # Write YAML with comments
+    with open(output, "w") as f:
+        f.write("# haiku.rag configuration file\n")
+        f.write(
+            "# See https://ggozad.github.io/haiku.rag/configuration/ for details\n\n"
+        )
+        yaml.dump(config_data, f, default_flow_style=False, sort_keys=False)
+
+    typer.echo(f"Configuration file created: {output}")
+    typer.echo("Edit the file to customize your settings.")
+
+
 @cli.command(
     "rebuild",
     help="Rebuild the database by deleting all chunks and re-indexing all documents",
 )
 def rebuild(
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -328,7 +387,7 @@ def rebuild(
 @cli.command("vacuum", help="Optimize and clean up all tables to reduce disk usage")
 def vacuum(
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -342,7 +401,7 @@ def vacuum(
 @cli.command("info", help="Show read-only database info (no upgrades or writes)")
 def info(
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -371,7 +430,7 @@ def download_models_cmd():
 )
 def serve(
     db: Path = typer.Option(
-        Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
         help="Path to the LanceDB database file",
     ),
@@ -442,24 +501,6 @@ def serve(
     )
 
 
-@cli.command("migrate", help="Migrate an SQLite database to LanceDB")
-def migrate(
-    sqlite_path: Path = typer.Argument(
-        help="Path to the SQLite database file to migrate",
-    ),
-):
-    # Generate LanceDB path in same parent directory
-    lancedb_path = sqlite_path.parent / (sqlite_path.stem + ".lancedb")
-
-    # Lazy import to avoid heavy deps on simple invocations
-    from haiku.rag.migration import migrate_sqlite_to_lancedb
-
-    success = asyncio.run(migrate_sqlite_to_lancedb(sqlite_path, lancedb_path))
-
-    if not success:
-        raise typer.Exit(1)
-
-
 @cli.command(
     "a2aclient", help="Run interactive client to chat with haiku.rag's A2A server"
 )

haiku/rag/client.py

@@ -8,8 +8,7 @@ from urllib.parse import urlparse
 
 import httpx
 
-from haiku.rag.config import Config
-from haiku.rag.reader import FileReader
+from haiku.rag.config import AppConfig, Config
 from haiku.rag.reranking import get_reranker
 from haiku.rag.store.engine import Store
 from haiku.rag.store.models.chunk import Chunk
@@ -17,7 +16,6 @@ from haiku.rag.store.models.document import Document
 from haiku.rag.store.repositories.chunk import ChunkRepository
 from haiku.rag.store.repositories.document import DocumentRepository
 from haiku.rag.store.repositories.settings import SettingsRepository
-from haiku.rag.utils import text_to_docling_document
 
 logger = logging.getLogger(__name__)
 
@@ -27,16 +25,23 @@ class HaikuRAG:
 
     def __init__(
         self,
-        db_path: Path = Config.DEFAULT_DATA_DIR / "haiku.rag.lancedb",
+        db_path: Path | None = None,
+        config: AppConfig = Config,
         skip_validation: bool = False,
     ):
         """Initialize the RAG client with a database path.
 
         Args:
-            db_path: Path to the database file.
+            db_path: Path to the database file. If None, uses config.storage.data_dir.
+            config: Configuration to use. Defaults to global Config.
             skip_validation: Whether to skip configuration validation on database load.
         """
-        self.store = Store(db_path, skip_validation=skip_validation)
+        self._config = config
+        if db_path is None:
+            db_path = self._config.storage.data_dir / "haiku.rag.lancedb"
+        self.store = Store(
+            db_path, config=self._config, skip_validation=skip_validation
+        )
         self.document_repository = DocumentRepository(self.store)
         self.chunk_repository = ChunkRepository(self.store)
 
@@ -91,6 +96,9 @@ class HaikuRAG:
         Returns:
             The created Document instance.
         """
+        # Lazy import to avoid loading docling
+        from haiku.rag.utils import text_to_docling_document
+
         # Convert content to DoclingDocument for processing
         docling_document = text_to_docling_document(content)
 
@@ -127,6 +135,8 @@ class HaikuRAG:
             ValueError: If the file/URL cannot be parsed or doesn't exist
             httpx.RequestError: If URL request fails
         """
+        # Lazy import to avoid loading docling
+        from haiku.rag.reader import FileReader
 
         # Normalize metadata
         metadata = metadata or {}
@@ -181,6 +191,9 @@ class HaikuRAG:
         Raises:
             ValueError: If the file cannot be parsed or doesn't exist
         """
+        # Lazy import to avoid loading docling
+        from haiku.rag.reader import FileReader
+
         metadata = metadata or {}
 
         if source_path.suffix.lower() not in FileReader.extensions:
@@ -256,6 +269,9 @@ class HaikuRAG:
             ValueError: If the content cannot be parsed
             httpx.RequestError: If URL request fails
         """
+        # Lazy import to avoid loading docling
+        from haiku.rag.reader import FileReader
+
         metadata = metadata or {}
 
         async with httpx.AsyncClient() as client:
@@ -379,6 +395,9 @@ class HaikuRAG:
 
     async def update_document(self, document: Document) -> Document:
         """Update an existing document."""
+        # Lazy import to avoid loading docling
+        from haiku.rag.utils import text_to_docling_document
+
         # Convert content to DoclingDocument
         docling_document = text_to_docling_document(document.content)
 
@@ -418,7 +437,7 @@ class HaikuRAG:
             List of (chunk, score) tuples ordered by relevance.
         """
         # Get reranker if available
-        reranker = get_reranker()
+        reranker = get_reranker(config=self._config)
 
         if reranker is None:
             # No reranking - return direct search results
@@ -440,18 +459,20 @@ class HaikuRAG:
     async def expand_context(
         self,
         search_results: list[tuple[Chunk, float]],
-        radius: int = Config.CONTEXT_CHUNK_RADIUS,
+        radius: int | None = None,
     ) -> list[tuple[Chunk, float]]:
         """Expand search results with adjacent chunks, merging overlapping chunks.
 
         Args:
             search_results: List of (chunk, score) tuples from search.
             radius: Number of adjacent chunks to include before/after each chunk.
-                   Defaults to CONTEXT_CHUNK_RADIUS config setting.
+                   If None, uses config.processing.context_chunk_radius.
 
         Returns:
             List of (chunk, score) tuples with expanded and merged context chunks.
         """
+        if radius is None:
+            radius = self._config.processing.context_chunk_radius
         if radius == 0:
             return search_results
 
@@ -581,7 +602,9 @@ class HaikuRAG:
         """
         from haiku.rag.qa import get_qa_agent
 
-        qa_agent = get_qa_agent(self, use_citations=cite, system_prompt=system_prompt)
+        qa_agent = get_qa_agent(
+            self, config=self._config, use_citations=cite, system_prompt=system_prompt
+        )
         return await qa_agent.answer(question)
 
     async def rebuild_database(self) -> AsyncGenerator[str, None]:
@@ -597,6 +620,9 @@ class HaikuRAG:
         Yields:
             int: The ID of the document currently being processed
         """
+        # Lazy import to avoid loading docling
+        from haiku.rag.utils import text_to_docling_document
+
         await self.chunk_repository.delete_all()
         self.store.recreate_embeddings_table()
 

haiku/rag/config/__init__.py

@@ -0,0 +1,50 @@
+import os
+
+from haiku.rag.config.loader import (
+    find_config_file,
+    generate_default_config,
+    load_config_from_env,
+    load_yaml_config,
+)
+from haiku.rag.config.models import (
+    A2AConfig,
+    AppConfig,
+    EmbeddingsConfig,
+    LanceDBConfig,
+    OllamaConfig,
+    ProcessingConfig,
+    ProvidersConfig,
+    QAConfig,
+    RerankingConfig,
+    ResearchConfig,
+    StorageConfig,
+    VLLMConfig,
+)
+
+__all__ = [
+    "Config",
+    "AppConfig",
+    "StorageConfig",
+    "LanceDBConfig",
+    "EmbeddingsConfig",
+    "RerankingConfig",
+    "QAConfig",
+    "ResearchConfig",
+    "ProcessingConfig",
+    "OllamaConfig",
+    "VLLMConfig",
+    "ProvidersConfig",
+    "A2AConfig",
+    "find_config_file",
+    "load_yaml_config",
+    "generate_default_config",
+    "load_config_from_env",
+]
+
+# Load config from YAML file or use defaults
+config_path = find_config_file(None)
+if config_path:
+    yaml_data = load_yaml_config(config_path)
+    Config = AppConfig.model_validate(yaml_data)
+else:
+    Config = AppConfig()

haiku/rag/config/loader.py

@@ -0,0 +1,137 @@
+import os
+from pathlib import Path
+
+import yaml
+
+
+def find_config_file(cli_path: Path | None = None) -> Path | None:
+    """Find the YAML config file using the search path.
+
+    Search order:
+    1. CLI-provided path (via HAIKU_RAG_CONFIG_PATH env var or parameter)
+    2. ./haiku.rag.yaml (current directory)
+    3. ~/.config/haiku.rag/config.yaml (user config)
+
+    Returns None if no config file is found.
+    """
+    # Check environment variable first (set by CLI --config flag)
+    if not cli_path:
+        env_path = os.getenv("HAIKU_RAG_CONFIG_PATH")
+        if env_path:
+            cli_path = Path(env_path)
+
+    if cli_path:
+        if cli_path.exists():
+            return cli_path
+        raise FileNotFoundError(f"Config file not found: {cli_path}")
+
+    cwd_config = Path.cwd() / "haiku.rag.yaml"
+    if cwd_config.exists():
+        return cwd_config
+
+    user_config_dir = Path.home() / ".config" / "haiku.rag"
+    user_config = user_config_dir / "config.yaml"
+    if user_config.exists():
+        return user_config
+
+    return None
+
+
+def load_yaml_config(path: Path) -> dict:
+    """Load and parse a YAML config file."""
+    with open(path) as f:
+        data = yaml.safe_load(f)
+    return data or {}
+
+
+def generate_default_config() -> dict:
+    """Generate a default YAML config structure with documentation."""
+    return {
+        "environment": "production",
+        "storage": {
+            "data_dir": "",
+            "monitor_directories": [],
+            "disable_autocreate": False,
+            "vacuum_retention_seconds": 60,
+        },
+        "lancedb": {"uri": "", "api_key": "", "region": ""},
+        "embeddings": {
+            "provider": "ollama",
+            "model": "qwen3-embedding",
+            "vector_dim": 4096,
+        },
+        "reranking": {"provider": "", "model": ""},
+        "qa": {"provider": "ollama", "model": "gpt-oss"},
+        "research": {"provider": "", "model": ""},
+        "processing": {
+            "chunk_size": 256,
+            "context_chunk_radius": 0,
+            "markdown_preprocessor": "",
+        },
+        "providers": {
+            "ollama": {"base_url": "http://localhost:11434"},
+            "vllm": {
+                "embeddings_base_url": "",
+                "rerank_base_url": "",
+                "qa_base_url": "",
+                "research_base_url": "",
+            },
+        },
+        "a2a": {"max_contexts": 1000},
+    }
+
+
+def load_config_from_env() -> dict:
+    """Load current config from environment variables (for migration)."""
+    result = {}
+
+    env_mappings = {
+        "ENV": "environment",
+        "DEFAULT_DATA_DIR": ("storage", "data_dir"),
+        "MONITOR_DIRECTORIES": ("storage", "monitor_directories"),
+        "DISABLE_DB_AUTOCREATE": ("storage", "disable_autocreate"),
+        "VACUUM_RETENTION_SECONDS": ("storage", "vacuum_retention_seconds"),
+        "LANCEDB_URI": ("lancedb", "uri"),
+        "LANCEDB_API_KEY": ("lancedb", "api_key"),
+        "LANCEDB_REGION": ("lancedb", "region"),
+        "EMBEDDINGS_PROVIDER": ("embeddings", "provider"),
+        "EMBEDDINGS_MODEL": ("embeddings", "model"),
+        "EMBEDDINGS_VECTOR_DIM": ("embeddings", "vector_dim"),
+        "RERANK_PROVIDER": ("reranking", "provider"),
+        "RERANK_MODEL": ("reranking", "model"),
+        "QA_PROVIDER": ("qa", "provider"),
+        "QA_MODEL": ("qa", "model"),
+        "RESEARCH_PROVIDER": ("research", "provider"),
+        "RESEARCH_MODEL": ("research", "model"),
+        "CHUNK_SIZE": ("processing", "chunk_size"),
+        "CONTEXT_CHUNK_RADIUS": ("processing", "context_chunk_radius"),
+        "MARKDOWN_PREPROCESSOR": ("processing", "markdown_preprocessor"),
+        "OLLAMA_BASE_URL": ("providers", "ollama", "base_url"),
+        "VLLM_EMBEDDINGS_BASE_URL": ("providers", "vllm", "embeddings_base_url"),
+        "VLLM_RERANK_BASE_URL": ("providers", "vllm", "rerank_base_url"),
+        "VLLM_QA_BASE_URL": ("providers", "vllm", "qa_base_url"),
+        "VLLM_RESEARCH_BASE_URL": ("providers", "vllm", "research_base_url"),
+        "A2A_MAX_CONTEXTS": ("a2a", "max_contexts"),
+    }
+
+    for env_var, path in env_mappings.items():
+        value = os.getenv(env_var)
+        if value is not None:
+            # Special handling for MONITOR_DIRECTORIES - parse comma-separated list
+            if env_var == "MONITOR_DIRECTORIES":
+                if value.strip():
+                    value = [p.strip() for p in value.split(",") if p.strip()]
+                else:
+                    value = []
+
+            if isinstance(path, tuple):
+                current = result
+                for key in path[:-1]:
+                    if key not in current:
+                        current[key] = {}
+                    current = current[key]
+                current[path[-1]] = value
+            else:
+                result[path] = value
+
+    return result

haiku/rag/config/models.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from haiku.rag.utils import get_default_data_dir
+
+
+class StorageConfig(BaseModel):
+    data_dir: Path = Field(default_factory=get_default_data_dir)
+    monitor_directories: list[Path] = []
+    disable_autocreate: bool = False
+    vacuum_retention_seconds: int = 60
+
+
+class LanceDBConfig(BaseModel):
+    uri: str = ""
+    api_key: str = ""
+    region: str = ""
+
+
+class EmbeddingsConfig(BaseModel):
+    provider: str = "ollama"
+    model: str = "qwen3-embedding"
+    vector_dim: int = 4096
+
+
+class RerankingConfig(BaseModel):
+    provider: str = ""
+    model: str = ""
+
+
+class QAConfig(BaseModel):
+    provider: str = "ollama"
+    model: str = "gpt-oss"
+
+
+class ResearchConfig(BaseModel):
+    provider: str = "ollama"
+    model: str = "gpt-oss"
+
+
+class ProcessingConfig(BaseModel):
+    chunk_size: int = 256
+    context_chunk_radius: int = 0
+    markdown_preprocessor: str = ""
+
+
+class OllamaConfig(BaseModel):
+    base_url: str = Field(
+        default_factory=lambda: __import__("os").environ.get(
+            "OLLAMA_BASE_URL", "http://localhost:11434"
+        )
+    )
+
+
+class VLLMConfig(BaseModel):
+    embeddings_base_url: str = ""
+    rerank_base_url: str = ""
+    qa_base_url: str = ""
+    research_base_url: str = ""
+
+
+class ProvidersConfig(BaseModel):
+    ollama: OllamaConfig = Field(default_factory=OllamaConfig)
+    vllm: VLLMConfig = Field(default_factory=VLLMConfig)
+
+
+class A2AConfig(BaseModel):
+    max_contexts: int = 1000
+
+
+class AppConfig(BaseModel):
+    environment: str = "production"
+    storage: StorageConfig = Field(default_factory=StorageConfig)
+    lancedb: LanceDBConfig = Field(default_factory=LanceDBConfig)
+    embeddings: EmbeddingsConfig = Field(default_factory=EmbeddingsConfig)
+    reranking: RerankingConfig = Field(default_factory=RerankingConfig)
+    qa: QAConfig = Field(default_factory=QAConfig)
+    research: ResearchConfig = Field(default_factory=ResearchConfig)
+    processing: ProcessingConfig = Field(default_factory=ProcessingConfig)
+    providers: ProvidersConfig = Field(default_factory=ProvidersConfig)
+    a2a: A2AConfig = Field(default_factory=A2AConfig)