kreuzberg 3.11.4__py3-none-any.whl → 3.13.0__py3-none-any.whl

kreuzberg/_config.py

@@ -1,10 +1,3 @@
-"""Configuration discovery and loading for Kreuzberg.
-
-This module provides configuration loading from both kreuzberg.toml and pyproject.toml files.
-Configuration is automatically discovered by searching up the directory tree from the current
-working directory.
-"""
-
 from __future__ import annotations
 
 import sys
@@ -16,16 +9,143 @@ if sys.version_info >= (3, 11):
 else:  # pragma: no cover
     import tomli as tomllib  # type: ignore[import-not-found]
 
-from kreuzberg._gmft import GMFTConfig
-from kreuzberg._ocr._easyocr import EasyOCRConfig
-from kreuzberg._ocr._paddleocr import PaddleOCRConfig
-from kreuzberg._ocr._tesseract import TesseractConfig
-from kreuzberg._types import ExtractionConfig, OcrBackendType
+from kreuzberg._types import (
+    EasyOCRConfig,
+    ExtractionConfig,
+    GMFTConfig,
+    HTMLToMarkdownConfig,
+    OcrBackendType,
+    PaddleOCRConfig,
+    PSMMode,
+    TesseractConfig,
+)
 from kreuzberg.exceptions import ValidationError
 
 if TYPE_CHECKING:
     from collections.abc import MutableMapping
 
+_CONFIG_FIELDS = [
+    "force_ocr",
+    "chunk_content",
+    "extract_tables",
+    "max_chars",
+    "max_overlap",
+    "ocr_backend",
+    "extract_entities",
+    "extract_keywords",
+    "auto_detect_language",
+    "enable_quality_processing",
+    "auto_detect_document_type",
+    "document_type_confidence_threshold",
+    "document_classification_mode",
+    "keyword_count",
+]
+
+_VALID_OCR_BACKENDS = {"tesseract", "easyocr", "paddleocr"}
+
+
+def _merge_file_config(config_dict: dict[str, Any], file_config: dict[str, Any]) -> None:
+    if not file_config:
+        return
+    for field in _CONFIG_FIELDS:
+        if field in file_config:
+            config_dict[field] = file_config[field]
+
+
+def _merge_cli_args(config_dict: dict[str, Any], cli_args: MutableMapping[str, Any]) -> None:
+    for field in _CONFIG_FIELDS:
+        if field in cli_args and cli_args[field] is not None:
+            config_dict[field] = cli_args[field]
+
+
+def _build_ocr_config_from_cli(
+    ocr_backend: str, cli_args: MutableMapping[str, Any]
+) -> TesseractConfig | EasyOCRConfig | PaddleOCRConfig | None:
+    config_key = f"{ocr_backend}_config"
+    if not cli_args.get(config_key):
+        return None
+
+    backend_args = cli_args[config_key]
+    try:
+        match ocr_backend:
+            case "tesseract":
+                return TesseractConfig(**backend_args)
+            case "easyocr":
+                return EasyOCRConfig(**backend_args)
+            case "paddleocr":
+                return PaddleOCRConfig(**backend_args)
+            case _:
+                return None
+    except (TypeError, ValueError) as e:
+        raise ValidationError(
+            f"Invalid {ocr_backend} configuration from CLI: {e}",
+            context={"backend": ocr_backend, "config": backend_args, "error": str(e)},
+        ) from e
+
+
+def _configure_ocr_backend(
+    config_dict: dict[str, Any],
+    file_config: dict[str, Any],
+    cli_args: MutableMapping[str, Any],
+) -> None:
+    ocr_backend = config_dict.get("ocr_backend")
+    if not ocr_backend or ocr_backend == "none":
+        return
+
+    ocr_config = _build_ocr_config_from_cli(ocr_backend, cli_args)
+    if not ocr_config and file_config:
+        ocr_config = parse_ocr_backend_config(file_config, ocr_backend)
+
+    if ocr_config:
+        config_dict["ocr_config"] = ocr_config
+
+
+def _configure_gmft(
+    config_dict: dict[str, Any],
+    file_config: dict[str, Any],
+    cli_args: MutableMapping[str, Any],
+) -> None:
+    if not config_dict.get("extract_tables"):
+        return
+
+    gmft_config = None
+    try:
+        if cli_args.get("gmft_config"):
+            gmft_config = GMFTConfig(**cli_args["gmft_config"])
+        elif "gmft" in file_config and isinstance(file_config["gmft"], dict):
+            gmft_config = GMFTConfig(**file_config["gmft"])
+    except (TypeError, ValueError) as e:
+        raise ValidationError(
+            f"Invalid GMFT configuration: {e}",
+            context={"gmft_config": cli_args.get("gmft_config") or file_config.get("gmft"), "error": str(e)},
+        ) from e
+
+    if gmft_config:
+        config_dict["gmft_config"] = gmft_config
+
+
+def _create_ocr_config(
+    backend: str, backend_config: dict[str, Any]
+) -> TesseractConfig | EasyOCRConfig | PaddleOCRConfig:
+    match backend:
+        case "tesseract":
+            processed_config = backend_config.copy()
+            if "psm" in processed_config and isinstance(processed_config["psm"], int):
+                try:
+                    processed_config["psm"] = PSMMode(processed_config["psm"])
+                except ValueError as e:
+                    raise ValidationError(
+                        f"Invalid PSM mode value: {processed_config['psm']}",
+                        context={"psm_value": processed_config["psm"], "error": str(e)},
+                    ) from e
+            return TesseractConfig(**processed_config)
+        case "easyocr":
+            return EasyOCRConfig(**backend_config)
+        case "paddleocr":
+            return PaddleOCRConfig(**backend_config)
+        case _:
+            raise ValueError(f"Unknown backend: {backend}")
+
 
 def load_config_from_file(config_path: Path) -> dict[str, Any]:
     """Load configuration from a TOML file.
@@ -47,15 +167,12 @@ def load_config_from_file(config_path: Path) -> dict[str, Any]:
     except tomllib.TOMLDecodeError as e:
         raise ValidationError(f"Invalid TOML in configuration file: {e}") from e
 
-    # Handle both kreuzberg.toml (root level) and pyproject.toml ([tool.kreuzberg])
     if config_path.name == "kreuzberg.toml":
         return data  # type: ignore[no-any-return]
 
-    # For other files, check if they have [tool.kreuzberg] section
-    if config_path.name == "pyproject.toml" or ("tool" in data and "kreuzberg" in data.get("tool", {})):
+    if config_path.name == "pyproject.toml":
         return data.get("tool", {}).get("kreuzberg", {})  # type: ignore[no-any-return]
 
-    # Otherwise assume root-level configuration
     return data  # type: ignore[no-any-return]
 
 
@@ -89,29 +206,27 @@ def parse_ocr_backend_config(
 
     Returns:
         Backend-specific configuration object or None.
+
+    Raises:
+        ValidationError: If the backend configuration is invalid.
     """
     if backend not in config_dict:
         return None
 
     backend_config = config_dict[backend]
     if not isinstance(backend_config, dict):
-        return None
-
-    match backend:
-        case "tesseract":
-            # Convert psm integer to PSMMode enum if needed
-            processed_config = backend_config.copy()
-            if "psm" in processed_config and isinstance(processed_config["psm"], int):
-                from kreuzberg._ocr._tesseract import PSMMode  # noqa: PLC0415
+        raise ValidationError(
+            f"Invalid configuration for OCR backend '{backend}': expected dict, got {type(backend_config).__name__}",
+            context={"backend": backend, "config_type": type(backend_config).__name__},
+        )
 
-                processed_config["psm"] = PSMMode(processed_config["psm"])
-            return TesseractConfig(**processed_config)
-        case "easyocr":
-            return EasyOCRConfig(**backend_config)
-        case "paddleocr":
-            return PaddleOCRConfig(**backend_config)
-        case _:
-            return None
+    try:
+        return _create_ocr_config(backend, backend_config)
+    except (TypeError, ValueError) as e:
+        raise ValidationError(
+            f"Invalid configuration for OCR backend '{backend}': {e}",
+            context={"backend": backend, "config": backend_config, "error": str(e)},
+        ) from e
 
 
 def build_extraction_config_from_dict(config_dict: dict[str, Any]) -> ExtractionConfig:
@@ -122,53 +237,87 @@ def build_extraction_config_from_dict(config_dict: dict[str, Any]) -> Extraction
 
     Returns:
         ExtractionConfig instance.
+
+    Raises:
+        ValidationError: If the configuration is invalid.
     """
-    extraction_config: dict[str, Any] = {}
-
-    # Copy basic configuration fields using dictionary comprehension
-    basic_fields = {
-        "force_ocr",
-        "chunk_content",
-        "extract_tables",
-        "max_chars",
-        "max_overlap",
-        "ocr_backend",
-        "extract_entities",
-        "extract_keywords",
-        "auto_detect_language",
-        "enable_quality_processing",
-        "auto_detect_document_type",
-        "document_type_confidence_threshold",
-        "document_classification_mode",
-        "keyword_count",
-    }
-    extraction_config = extraction_config | {
-        field: config_dict[field] for field in basic_fields if field in config_dict
-    }
-
-    # Handle OCR backend configuration
+    extraction_config: dict[str, Any] = {field: config_dict[field] for field in _CONFIG_FIELDS if field in config_dict}
+
     ocr_backend = extraction_config.get("ocr_backend")
     if ocr_backend and ocr_backend != "none":
-        # Validate OCR backend
-        valid_backends = {"tesseract", "easyocr", "paddleocr"}
-        if ocr_backend not in valid_backends:
+        if ocr_backend not in _VALID_OCR_BACKENDS:
             raise ValidationError(
-                f"Invalid OCR backend: {ocr_backend}. Must be one of: {', '.join(sorted(valid_backends))} or 'none'",
-                context={"provided": ocr_backend, "valid": sorted(valid_backends)},
+                f"Invalid OCR backend: {ocr_backend}. Must be one of: {', '.join(sorted(_VALID_OCR_BACKENDS))} or 'none'",
+                context={"provided": ocr_backend, "valid": sorted(_VALID_OCR_BACKENDS)},
             )
         ocr_config = parse_ocr_backend_config(config_dict, ocr_backend)
         if ocr_config:
             extraction_config["ocr_config"] = ocr_config
 
-    # Handle GMFT configuration for table extraction
     if extraction_config.get("extract_tables") and "gmft" in config_dict and isinstance(config_dict["gmft"], dict):
-        extraction_config["gmft_config"] = GMFTConfig(**config_dict["gmft"])
+        try:
+            extraction_config["gmft_config"] = GMFTConfig(**config_dict["gmft"])
+        except (TypeError, ValueError) as e:
+            raise ValidationError(
+                f"Invalid GMFT configuration: {e}",
+                context={"gmft_config": config_dict["gmft"], "error": str(e)},
+            ) from e
+
+    if "html_to_markdown" in config_dict and isinstance(config_dict["html_to_markdown"], dict):
+        try:
+            extraction_config["html_to_markdown_config"] = HTMLToMarkdownConfig(**config_dict["html_to_markdown"])
+        except (TypeError, ValueError) as e:
+            raise ValidationError(
+                f"Invalid HTML to Markdown configuration: {e}",
+                context={"html_to_markdown_config": config_dict["html_to_markdown"], "error": str(e)},
+            ) from e
 
-    # Convert "none" to None for ocr_backend
     if extraction_config.get("ocr_backend") == "none":
         extraction_config["ocr_backend"] = None
 
-    return ExtractionConfig(**extraction_config)
+    try:
+        return ExtractionConfig(**extraction_config)
+    except (TypeError, ValueError) as e:
+        raise ValidationError(
+            f"Invalid extraction configuration: {e}",
+            context={"config": extraction_config, "error": str(e)},
+        ) from e
+
+
+def build_extraction_config(
+    file_config: dict[str, Any],
+    cli_args: MutableMapping[str, Any],
+) -> ExtractionConfig:
+    """Build ExtractionConfig from file config and CLI arguments.
+
+    Args:
+        file_config: Configuration loaded from file.
+        cli_args: CLI arguments.
+
+    Returns:
+        ExtractionConfig instance.
+
+    Raises:
+        ValidationError: If the combined configuration is invalid.
+    """
+    config_dict: dict[str, Any] = {}
+
+    _merge_file_config(config_dict, file_config)
+    _merge_cli_args(config_dict, cli_args)
+
+    _configure_ocr_backend(config_dict, file_config, cli_args)
+    _configure_gmft(config_dict, file_config, cli_args)
+
+    if config_dict.get("ocr_backend") == "none":
+        config_dict["ocr_backend"] = None
+
+    try:
+        return ExtractionConfig(**config_dict)
+    except (TypeError, ValueError) as e:
+        raise ValidationError(
+            f"Invalid extraction configuration: {e}",
+            context={"config": config_dict, "error": str(e)},
+        ) from e
 
 
 def find_config_file(start_path: Path | None = None) -> Path | None:
@@ -183,16 +332,17 @@ def find_config_file(start_path: Path | None = None) -> Path | None:
 
     Returns:
         Path to the configuration file or None if not found.
+
+    Raises:
+        ValidationError: If a config file exists but cannot be read or has invalid TOML.
     """
     current = start_path or Path.cwd()
 
     while current != current.parent:
-        # First, look for kreuzberg.toml
         kreuzberg_toml = current / "kreuzberg.toml"
         if kreuzberg_toml.exists():
             return kreuzberg_toml
 
-        # Then, look for pyproject.toml with [tool.kreuzberg] section
         pyproject_toml = current / "pyproject.toml"
         if pyproject_toml.exists():
             try:
@@ -200,8 +350,16 @@ def find_config_file(start_path: Path | None = None) -> Path | None:
                     data = tomllib.load(f)
                 if "tool" in data and "kreuzberg" in data["tool"]:
                     return pyproject_toml
-            except Exception:  # noqa: BLE001
-                pass
+            except OSError as e:
+                raise ValidationError(
+                    f"Failed to read pyproject.toml: {e}",
+                    context={"file": str(pyproject_toml), "error": str(e)},
+                ) from e
+            except tomllib.TOMLDecodeError as e:
+                raise ValidationError(
+                    f"Invalid TOML in pyproject.toml: {e}",
+                    context={"file": str(pyproject_toml), "error": str(e)},
+                ) from e
 
         current = current.parent
     return None
@@ -215,19 +373,18 @@ def load_default_config(start_path: Path | None = None) -> ExtractionConfig | No
 
     Returns:
         ExtractionConfig instance or None if no configuration found.
+
+    Raises:
+        ValidationError: If configuration file exists but contains invalid configuration.
     """
     config_path = find_config_file(start_path)
     if not config_path:
         return None
 
-    try:
-        config_dict = load_config_from_file(config_path)
-        if not config_dict:
-            return None
-        return build_extraction_config_from_dict(config_dict)
-    except Exception:  # noqa: BLE001
-        # Silently ignore configuration errors for default loading
+    config_dict = load_config_from_file(config_path)
+    if not config_dict:
         return None
+    return build_extraction_config_from_dict(config_dict)
 
 
 def load_config_from_path(config_path: Path | str) -> ExtractionConfig:
@@ -278,143 +435,30 @@ def discover_and_load_config(start_path: Path | str | None = None) -> Extraction
     return build_extraction_config_from_dict(config_dict)
 
 
-def try_discover_config(start_path: Path | str | None = None) -> ExtractionConfig | None:
-    """Try to discover and load configuration, returning None if not found.
+def discover_config(start_path: Path | str | None = None) -> ExtractionConfig | None:
+    """Discover and load configuration, returning None if no config file found.
+
+    If a config file is found, attempts to load it. Any errors during loading will bubble up.
 
     Args:
         start_path: Directory to start searching from. Defaults to current working directory.
 
     Returns:
-        ExtractionConfig instance or None if no configuration found.
-    """
-    try:
-        return discover_and_load_config(start_path)
-    except ValidationError:
-        return None
+        ExtractionConfig instance or None if no configuration file found.
 
-
-# Legacy functions for backward compatibility with CLI
-
-# Define common configuration fields to avoid repetition
-_CONFIG_FIELDS = [
-    "force_ocr",
-    "chunk_content",
-    "extract_tables",
-    "max_chars",
-    "max_overlap",
-    "ocr_backend",
-    "extract_entities",
-    "extract_keywords",
-    "auto_detect_language",
-    "enable_quality_processing",
-    "auto_detect_document_type",
-    "document_type_confidence_threshold",
-    "document_classification_mode",
-    "keyword_count",
-]
-
-
-def _merge_file_config(config_dict: dict[str, Any], file_config: dict[str, Any]) -> None:
-    """Merge file configuration into config dictionary."""
-    if not file_config:
-        return
-
-    for field in _CONFIG_FIELDS:
-        if field in file_config:
-            config_dict[field] = file_config[field]
-
-
-def _merge_cli_args(config_dict: dict[str, Any], cli_args: MutableMapping[str, Any]) -> None:
-    """Merge CLI arguments into config dictionary."""
-    for field in _CONFIG_FIELDS:
-        if field in cli_args and cli_args[field] is not None:
-            config_dict[field] = cli_args[field]
-
-
-def _build_ocr_config_from_cli(
-    ocr_backend: str, cli_args: MutableMapping[str, Any]
-) -> TesseractConfig | EasyOCRConfig | PaddleOCRConfig | None:
-    """Build OCR config from CLI arguments."""
-    config_key = f"{ocr_backend}_config"
-    if not cli_args.get(config_key):
-        return None
-
-    backend_args = cli_args[config_key]
-    if ocr_backend == "tesseract":
-        return TesseractConfig(**backend_args)
-    if ocr_backend == "easyocr":
-        return EasyOCRConfig(**backend_args)
-    if ocr_backend == "paddleocr":
-        return PaddleOCRConfig(**backend_args)
-    return None
-
-
-def _configure_ocr_backend(
-    config_dict: dict[str, Any],
-    file_config: dict[str, Any],
-    cli_args: MutableMapping[str, Any],
-) -> None:
-    """Configure OCR backend in config dictionary."""
-    ocr_backend = config_dict.get("ocr_backend")
-    if not ocr_backend or ocr_backend == "none":
-        return
-
-    # Try CLI config first, then file config
-    ocr_config = _build_ocr_config_from_cli(ocr_backend, cli_args)
-    if not ocr_config and file_config:
-        ocr_config = parse_ocr_backend_config(file_config, ocr_backend)
-
-    if ocr_config:
-        config_dict["ocr_config"] = ocr_config
-
-
-def _configure_gmft(
-    config_dict: dict[str, Any],
-    file_config: dict[str, Any],
-    cli_args: MutableMapping[str, Any],
-) -> None:
-    """Configure GMFT in config dictionary."""
-    if not config_dict.get("extract_tables"):
-        return
-
-    gmft_config = None
-    if cli_args.get("gmft_config"):
-        gmft_config = GMFTConfig(**cli_args["gmft_config"])
-    elif "gmft" in file_config and isinstance(file_config["gmft"], dict):
-        gmft_config = GMFTConfig(**file_config["gmft"])
-
-    if gmft_config:
-        config_dict["gmft_config"] = gmft_config
-
-
-def build_extraction_config(
-    file_config: dict[str, Any],
-    cli_args: MutableMapping[str, Any],
-) -> ExtractionConfig:
-    """Build ExtractionConfig from file config and CLI arguments.
-
-    Args:
-        file_config: Configuration loaded from file.
-        cli_args: CLI arguments.
-
-    Returns:
-        ExtractionConfig instance.
+    Raises:
+        ValidationError: If a configuration file exists but is invalid.
     """
-    config_dict: dict[str, Any] = {}
-
-    # Merge configurations: file first, then CLI overrides
-    _merge_file_config(config_dict, file_config)
-    _merge_cli_args(config_dict, cli_args)
-
-    # Configure complex components
-    _configure_ocr_backend(config_dict, file_config, cli_args)
-    _configure_gmft(config_dict, file_config, cli_args)
+    search_path = Path(start_path) if start_path else None
+    config_path = find_config_file(search_path)
 
-    # Convert "none" to None for ocr_backend
-    if config_dict.get("ocr_backend") == "none":
-        config_dict["ocr_backend"] = None
+    if not config_path:
+        return None
 
-    return ExtractionConfig(**config_dict)
+    config_dict = load_config_from_file(config_path)
+    if not config_dict:
+        return None
+    return build_extraction_config_from_dict(config_dict)
 
 
 def find_default_config() -> Path | None:

kreuzberg/_document_classification.py

@@ -51,10 +51,8 @@ def _get_translated_text(result: ExtractionResult) -> str:
     Raises:
         MissingDependencyError: If the deep-translator package is not installed
     """
-    # Combine content with metadata for classification
     text_to_classify = result.content
     if result.metadata:
-        # Add metadata values to the text for classification
         metadata_text = " ".join(str(value) for value in result.metadata.values() if value)
         text_to_classify = f"{text_to_classify} {metadata_text}"
 
@@ -68,7 +66,6 @@ def _get_translated_text(result: ExtractionResult) -> str:
     try:
         return str(GoogleTranslator(source="auto", target="en").translate(text_to_classify).lower())
     except Exception:  # noqa: BLE001
-        # Fall back to original content in lowercase if translation fails
         return text_to_classify.lower()
 
 
@@ -131,13 +128,10 @@ def classify_document_from_layout(
     if not all(col in layout_df.columns for col in ["text", "top", "height"]):
         return None, None
 
-    # Use layout text for classification, not the content
     layout_text = " ".join(layout_df["text"].astype(str).tolist())
 
-    # Translate layout text directly for classification
     text_to_classify = layout_text
     if result.metadata:
-        # Add metadata values to the text for classification
         metadata_text = " ".join(str(value) for value in result.metadata.values() if value)
         text_to_classify = f"{text_to_classify} {metadata_text}"
 
@@ -146,7 +140,6 @@ def classify_document_from_layout(
 
         translated_text = str(GoogleTranslator(source="auto", target="en").translate(text_to_classify).lower())
     except Exception:  # noqa: BLE001
-        # Fall back to original content in lowercase if translation fails
         translated_text = text_to_classify.lower()
 
     layout_df["translated_text"] = translated_text
@@ -184,7 +177,6 @@ def auto_detect_document_type(
         layout_result = get_ocr_backend("tesseract").process_file_sync(file_path, **config.get_config_dict())
         result.document_type, result.document_type_confidence = classify_document_from_layout(layout_result, config)
     elif result.layout is not None and not result.layout.empty:
-        # Use layout-based classification if layout data is available
         result.document_type, result.document_type_confidence = classify_document_from_layout(result, config)
     else:
         result.document_type, result.document_type_confidence = classify_document(result, config)