haiku.rag-slim 0.16.0__py3-none-any.whl → 0.24.0__py3-none-any.whl

haiku/rag/config/__init__.py

@@ -8,38 +8,44 @@ from haiku.rag.config.loader import (
 from haiku.rag.config.models import (
     AGUIConfig,
     AppConfig,
+    ConversionOptions,
+    EmbeddingModelConfig,
     EmbeddingsConfig,
     LanceDBConfig,
+    ModelConfig,
     MonitorConfig,
     OllamaConfig,
     ProcessingConfig,
+    PromptsConfig,
     ProvidersConfig,
     QAConfig,
     RerankingConfig,
     ResearchConfig,
     StorageConfig,
-    VLLMConfig,
 )
 
 __all__ = [
     "Config",
     "AGUIConfig",
     "AppConfig",
-    "StorageConfig",
-    "MonitorConfig",
-    "LanceDBConfig",
+    "ConversionOptions",
+    "EmbeddingModelConfig",
     "EmbeddingsConfig",
-    "RerankingConfig",
-    "QAConfig",
-    "ResearchConfig",
-    "ProcessingConfig",
+    "LanceDBConfig",
+    "ModelConfig",
+    "MonitorConfig",
     "OllamaConfig",
-    "VLLMConfig",
+    "ProcessingConfig",
+    "PromptsConfig",
     "ProvidersConfig",
+    "QAConfig",
+    "RerankingConfig",
+    "ResearchConfig",
+    "StorageConfig",
     "find_config_file",
-    "load_yaml_config",
     "generate_default_config",
     "get_config",
+    "load_yaml_config",
     "set_config",
 ]
 

haiku/rag/config/loader.py

@@ -48,47 +48,8 @@ def load_yaml_config(path: Path) -> dict:
 
 
 def generate_default_config() -> dict:
-    """Generate a default YAML config structure with documentation."""
-    return {
-        "environment": "production",
-        "storage": {
-            "data_dir": "",
-            "vacuum_retention_seconds": 86400,
-        },
-        "monitor": {
-            "directories": [],
-            "ignore_patterns": [],
-            "include_patterns": [],
-        },
-        "lancedb": {"uri": "", "api_key": "", "region": ""},
-        "embeddings": {
-            "provider": "ollama",
-            "model": "qwen3-embedding",
-            "vector_dim": 4096,
-        },
-        "reranking": {"provider": "", "model": ""},
-        "qa": {"provider": "ollama", "model": "gpt-oss"},
-        "research": {"provider": "", "model": ""},
-        "processing": {
-            "chunk_size": 256,
-            "context_chunk_radius": 0,
-            "markdown_preprocessor": "",
-        },
-        "providers": {
-            "ollama": {"base_url": "http://localhost:11434"},
-            "vllm": {
-                "embeddings_base_url": "",
-                "rerank_base_url": "",
-                "qa_base_url": "",
-                "research_base_url": "",
-            },
-        },
-        "agui": {
-            "host": "0.0.0.0",
-            "port": 8000,
-            "cors_origins": ["*"],
-            "cors_credentials": True,
-            "cors_methods": ["GET", "POST", "OPTIONS"],
-            "cors_headers": ["*"],
-        },
-    }
+    """Generate a default YAML config structure from AppConfig defaults."""
+    from haiku.rag.config.models import AppConfig
+
+    default_config = AppConfig()
+    return default_config.model_dump(mode="json", exclude_none=False)

haiku/rag/config/models.py

@@ -1,12 +1,51 @@
 from pathlib import Path
+from typing import Literal
 
 from pydantic import BaseModel, Field
 
 from haiku.rag.utils import get_default_data_dir
 
 
+class ModelConfig(BaseModel):
+    """Configuration for a language model.
+
+    Attributes:
+        provider: Model provider (ollama, openai, anthropic, etc.)
+        name: Model name/identifier
+        base_url: Optional base URL for OpenAI-compatible servers (vLLM, LM Studio, etc.)
+        enable_thinking: Control reasoning behavior (true/false/None for default)
+        temperature: Sampling temperature (0.0 to 1.0+)
+        max_tokens: Maximum tokens to generate
+    """
+
+    provider: str = "ollama"
+    name: str = "gpt-oss"
+    base_url: str | None = None
+
+    enable_thinking: bool | None = None
+    temperature: float | None = None
+    max_tokens: int | None = None
+
+
+class EmbeddingModelConfig(BaseModel):
+    """Configuration for an embedding model.
+
+    Attributes:
+        provider: Model provider (ollama, openai, voyageai, cohere, sentence-transformers)
+        name: Model name/identifier
+        vector_dim: Vector dimensions produced by the model
+        base_url: Optional base URL for OpenAI-compatible servers (vLLM, LM Studio, etc.)
+    """
+
+    provider: str = "ollama"
+    name: str = "qwen3-embedding:4b"
+    vector_dim: int = 2560
+    base_url: str | None = None
+
+
 class StorageConfig(BaseModel):
     data_dir: Path = Field(default_factory=get_default_data_dir)
+    auto_vacuum: bool = True
     vacuum_retention_seconds: int = 86400
 
 
@@ -24,36 +63,94 @@ class LanceDBConfig(BaseModel):
 
 
 class EmbeddingsConfig(BaseModel):
-    provider: str = "ollama"
-    model: str = "qwen3-embedding"
-    vector_dim: int = 4096
+    model: EmbeddingModelConfig = Field(default_factory=EmbeddingModelConfig)
 
 
 class RerankingConfig(BaseModel):
-    provider: str = ""
-    model: str = ""
+    model: ModelConfig | None = None
 
 
 class QAConfig(BaseModel):
-    provider: str = "ollama"
-    model: str = "gpt-oss"
+    model: ModelConfig = Field(
+        default_factory=lambda: ModelConfig(
+            provider="ollama",
+            name="gpt-oss",
+            enable_thinking=False,
+        )
+    )
     max_sub_questions: int = 3
     max_iterations: int = 2
     max_concurrency: int = 1
 
 
 class ResearchConfig(BaseModel):
-    provider: str = "ollama"
-    model: str = "gpt-oss"
+    model: ModelConfig = Field(
+        default_factory=lambda: ModelConfig(
+            provider="ollama",
+            name="gpt-oss",
+            enable_thinking=False,
+        )
+    )
     max_iterations: int = 3
     confidence_threshold: float = 0.8
     max_concurrency: int = 1
 
 
+class PictureDescriptionConfig(BaseModel):
+    """Configuration for VLM-based picture description."""
+
+    enabled: bool = False
+    model: ModelConfig = Field(
+        default_factory=lambda: ModelConfig(
+            provider="ollama",
+            name="ministral-3",
+        )
+    )
+    timeout: int = 90
+    max_tokens: int = 200
+
+
+class ConversionOptions(BaseModel):
+    """Options for document conversion."""
+
+    # OCR options
+    do_ocr: bool = True
+    force_ocr: bool = False
+    ocr_lang: list[str] = []
+
+    # Table options
+    do_table_structure: bool = True
+    table_mode: Literal["fast", "accurate"] = "accurate"
+    table_cell_matching: bool = True
+
+    # Image options
+    images_scale: float = 2.0
+    generate_picture_images: bool = False
+
+    # VLM picture description
+    picture_description: PictureDescriptionConfig = Field(
+        default_factory=PictureDescriptionConfig
+    )
+
+
 class ProcessingConfig(BaseModel):
     chunk_size: int = 256
-    context_chunk_radius: int = 0
-    markdown_preprocessor: str = ""
+    converter: str = "docling-local"
+    chunker: str = "docling-local"
+    chunker_type: str = "hybrid"
+    chunking_tokenizer: str = "Qwen/Qwen3-Embedding-0.6B"
+    chunking_merge_peers: bool = True
+    chunking_use_markdown_tables: bool = False
+    conversion_options: ConversionOptions = Field(default_factory=ConversionOptions)
+
+
+class SearchConfig(BaseModel):
+    limit: int = 5
+    context_radius: int = 0
+    max_context_items: int = 10
+    max_context_chars: int = 10000
+    vector_index_metric: Literal["cosine", "l2", "dot"] = "cosine"
+    vector_refine_factor: int = 30
 
 
 class OllamaConfig(BaseModel):
@@ -64,16 +161,14 @@ class OllamaConfig(BaseModel):
     )
 
 
-class VLLMConfig(BaseModel):
-    embeddings_base_url: str = ""
-    rerank_base_url: str = ""
-    qa_base_url: str = ""
-    research_base_url: str = ""
+class DoclingServeConfig(BaseModel):
+    base_url: str = "http://localhost:5001"
+    api_key: str = ""
 
 
 class ProvidersConfig(BaseModel):
     ollama: OllamaConfig = Field(default_factory=OllamaConfig)
-    vllm: VLLMConfig = Field(default_factory=VLLMConfig)
+    docling_serve: DoclingServeConfig = Field(default_factory=DoclingServeConfig)
 
 
 class AGUIConfig(BaseModel):
@@ -85,6 +180,18 @@ class AGUIConfig(BaseModel):
     cors_headers: list[str] = ["*"]
 
 
+class PromptsConfig(BaseModel):
+    domain_preamble: str = ""
+    qa: str | None = None
+    synthesis: str | None = None
+    picture_description: str = (
+        "Describe this image for a blind user. "
+        "State the image type (screenshot, chart, photo, etc.), "
+        "what it depicts, any visible text, and key visual details. "
+        "Be concise and accurate."
+    )
+
+
 class AppConfig(BaseModel):
     environment: str = "production"
     storage: StorageConfig = Field(default_factory=StorageConfig)
@@ -95,5 +202,7 @@ class AppConfig(BaseModel):
     qa: QAConfig = Field(default_factory=QAConfig)
     research: ResearchConfig = Field(default_factory=ResearchConfig)
     processing: ProcessingConfig = Field(default_factory=ProcessingConfig)
+    search: SearchConfig = Field(default_factory=SearchConfig)
     providers: ProvidersConfig = Field(default_factory=ProvidersConfig)
     agui: AGUIConfig = Field(default_factory=AGUIConfig)
+    prompts: PromptsConfig = Field(default_factory=PromptsConfig)

haiku/rag/converters/__init__.py

@@ -0,0 +1,31 @@
+"""Document converter abstraction for haiku.rag."""
+
+from haiku.rag.config import AppConfig, Config
+from haiku.rag.converters.base import DocumentConverter
+
+__all__ = ["DocumentConverter", "get_converter"]
+
+
+def get_converter(config: AppConfig = Config) -> DocumentConverter:
+    """Get a document converter instance based on configuration.
+
+    Args:
+        config: Configuration to use. Defaults to global Config.
+
+    Returns:
+        DocumentConverter instance configured according to the config.
+
+    Raises:
+        ValueError: If the converter provider is not recognized.
+    """
+    if config.processing.converter == "docling-local":
+        from haiku.rag.converters.docling_local import DoclingLocalConverter
+
+        return DoclingLocalConverter(config)
+
+    if config.processing.converter == "docling-serve":
+        from haiku.rag.converters.docling_serve import DoclingServeConverter
+
+        return DoclingServeConverter(config)
+
+    raise ValueError(f"Unsupported converter provider: {config.processing.converter}")

haiku/rag/converters/base.py

@@ -0,0 +1,63 @@
+"""Base class for document converters."""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from docling_core.types.doc.document import DoclingDocument
+
+
+class DocumentConverter(ABC):
+    """Abstract base class for document converters.
+
+    Document converters are responsible for converting various document formats
+    (PDF, DOCX, HTML, etc.) into DoclingDocument format for further processing.
+    """
+
+    @property
+    @abstractmethod
+    def supported_extensions(self) -> list[str]:
+        """Return list of file extensions supported by this converter.
+
+        Returns:
+            List of file extensions (including the dot, e.g., [".pdf", ".docx"]).
+        """
+        pass
+
+    @abstractmethod
+    async def convert_file(self, path: Path) -> "DoclingDocument":
+        """Convert a file to DoclingDocument format.
+
+        Args:
+            path: Path to the file to convert.
+
+        Returns:
+            DoclingDocument representation of the file.
+
+        Raises:
+            ValueError: If the file cannot be converted.
+        """
+        pass
+
+    SUPPORTED_FORMATS = ("md", "html", "plain")
+
+    @abstractmethod
+    async def convert_text(
+        self, text: str, name: str = "content.md", format: str = "md"
+    ) -> "DoclingDocument":
+        """Convert text content to DoclingDocument format.
+
+        Args:
+            text: The text content to convert.
+            name: The name to use for the document (defaults to "content.md").
+            format: The format of the text content ("md", "html", or "plain").
+                Defaults to "md". Use "plain" for plain text without parsing.
+
+        Returns:
+            DoclingDocument representation of the text.
+
+        Raises:
+            ValueError: If the text cannot be converted or format is unsupported.
+        """
+        pass

haiku/rag/converters/docling_local.py

@@ -0,0 +1,193 @@
+"""Local docling converter implementation."""
+
+import asyncio
+from pathlib import Path
+from typing import TYPE_CHECKING, ClassVar, cast
+
+from haiku.rag.config import AppConfig
+from haiku.rag.converters.base import DocumentConverter
+from haiku.rag.converters.text_utils import TextFileHandler
+
+if TYPE_CHECKING:
+    from docling_core.types.doc.document import DoclingDocument
+
+    from haiku.rag.config.models import ModelConfig
+
+
+class DoclingLocalConverter(DocumentConverter):
+    """Converter that uses local docling for document conversion.
+
+    This converter runs docling locally in-process to convert documents.
+    It handles various document formats including PDF, DOCX, HTML, and plain text.
+    """
+
+    # Extensions supported by docling
+    docling_extensions: ClassVar[list[str]] = [
+        ".adoc",
+        ".asc",
+        ".asciidoc",
+        ".bmp",
+        ".csv",
+        ".docx",
+        ".html",
+        ".xhtml",
+        ".jpeg",
+        ".jpg",
+        ".md",
+        ".pdf",
+        ".png",
+        ".pptx",
+        ".tiff",
+        ".xlsx",
+        ".xml",
+        ".webp",
+    ]
+
+    def __init__(self, config: AppConfig):
+        """Initialize the converter with configuration.
+
+        Args:
+            config: Application configuration containing conversion options.
+        """
+        self.config = config
+
+    @property
+    def supported_extensions(self) -> list[str]:
+        """Return list of file extensions supported by this converter."""
+        return self.docling_extensions + TextFileHandler.text_extensions
+
+    def _get_vlm_api_url(self, model: "ModelConfig") -> str:
+        """Construct VLM API URL from model config."""
+        if model.base_url:
+            base = model.base_url.rstrip("/")
+            return f"{base}/v1/chat/completions"
+
+        if model.provider == "ollama":
+            base = self.config.providers.ollama.base_url.rstrip("/")
+            return f"{base}/v1/chat/completions"
+
+        if model.provider == "openai":
+            return "https://api.openai.com/v1/chat/completions"
+
+        raise ValueError(f"Unsupported VLM provider: {model.provider}")
+
+    def _sync_convert_docling_file(self, path: Path) -> "DoclingDocument":
+        """Synchronous conversion of docling-supported files."""
+        from docling.backend.docling_parse_backend import DoclingParseDocumentBackend
+        from docling.datamodel.base_models import InputFormat
+        from docling.datamodel.pipeline_options import (
+            OcrAutoOptions,
+            PdfPipelineOptions,
+            PictureDescriptionApiOptions,
+            TableFormerMode,
+            TableStructureOptions,
+        )
+        from docling.document_converter import (
+            DocumentConverter as DoclingDocConverter,
+        )
+        from docling.document_converter import (
+            FormatOption,
+            PdfFormatOption,
+        )
+
+        opts = self.config.processing.conversion_options
+        pic_desc = opts.picture_description
+
+        pipeline_options = PdfPipelineOptions(
+            do_ocr=opts.do_ocr,
+            do_table_structure=opts.do_table_structure,
+            images_scale=opts.images_scale,
+            generate_page_images=True,
+            generate_picture_images=opts.generate_picture_images or pic_desc.enabled,
+            table_structure_options=TableStructureOptions(
+                do_cell_matching=opts.table_cell_matching,
+                mode=(
+                    TableFormerMode.FAST
+                    if opts.table_mode == "fast"
+                    else TableFormerMode.ACCURATE
+                ),
+            ),
+            ocr_options=OcrAutoOptions(
+                force_full_page_ocr=opts.force_ocr,
+                lang=opts.ocr_lang if opts.ocr_lang else [],
+            ),
+            do_picture_description=pic_desc.enabled,
+        )
+
+        if pic_desc.enabled:
+            from pydantic import AnyUrl
+
+            prompt = self.config.prompts.picture_description
+
+            pipeline_options.enable_remote_services = True
+            pipeline_options.picture_description_options = PictureDescriptionApiOptions(
+                url=AnyUrl(self._get_vlm_api_url(pic_desc.model)),
+                params=dict(
+                    model=pic_desc.model.name,
+                    max_completion_tokens=pic_desc.max_tokens,
+                ),
+                prompt=prompt,
+                timeout=pic_desc.timeout,
+            )
+
+        format_options = cast(
+            dict[InputFormat, FormatOption],
+            {
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_options=pipeline_options,
+                    backend=DoclingParseDocumentBackend,
+                )
+            },
+        )
+
+        converter = DoclingDocConverter(format_options=format_options)
+        result = converter.convert(path)
+        return result.document
+
+    async def convert_file(self, path: Path) -> "DoclingDocument":
+        """Convert a file to DoclingDocument using local docling.
+
+        Args:
+            path: Path to the file to convert.
+
+        Returns:
+            DoclingDocument representation of the file.
+
+        Raises:
+            ValueError: If the file cannot be converted.
+        """
+        try:
+            file_extension = path.suffix.lower()
+
+            if file_extension in self.docling_extensions:
+                return await asyncio.to_thread(self._sync_convert_docling_file, path)
+            elif file_extension in TextFileHandler.text_extensions:
+                content = await asyncio.to_thread(path.read_text, encoding="utf-8")
+                prepared_content = TextFileHandler.prepare_text_content(
+                    content, file_extension
+                )
+                return await self.convert_text(prepared_content, name=f"{path.stem}.md")
+            else:
+                content = await asyncio.to_thread(path.read_text, encoding="utf-8")
+                return await self.convert_text(content, name=f"{path.stem}.md")
+        except Exception:
+            raise ValueError(f"Failed to parse file: {path}")
+
+    async def convert_text(
+        self, text: str, name: str = "content.md", format: str = "md"
+    ) -> "DoclingDocument":
+        """Convert text content to DoclingDocument using local docling.
+
+        Args:
+            text: The text content to convert.
+            name: The name to use for the document (defaults to "content.md").
+            format: The format of the text content ("md", "html", or "plain").
+                Defaults to "md". Use "plain" for plain text without parsing.
+
+        Returns:
+            DoclingDocument representation of the text.
+
+        Raises:
+            ValueError: If the text cannot be converted or format is unsupported.
+        """
+        return await TextFileHandler.text_to_docling_document(text, name, format)