kreuzberg 4.0.6__cp310-abi3-macosx_14_0_arm64.whl

kreuzberg/__main__.py

@@ -0,0 +1,160 @@
+"""Proxy entry point that forwards to the Rust-based Kreuzberg CLI.
+
+This keeps `python -m kreuzberg` and the `kreuzberg` console script working
+without shipping an additional Python CLI implementation.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+_FEATURE_SUBCOMMANDS: dict[str, str] = {"serve": "all", "mcp": "all"}
+
+
+def _iter_dev_cli_candidates(workspace_root: Path) -> list[Path]:
+    suffixes = [".exe"] if sys.platform == "win32" else [""]
+    candidate_dirs = ("target/release", "target/debug")
+    candidate_names = ("kreuzberg-cli", "kreuzberg")
+
+    candidates: list[Path] = []
+    for directory in candidate_dirs:
+        for name in candidate_names:
+            for suffix in suffixes:
+                candidate = workspace_root / directory / f"{name}{suffix}"
+                if candidate.exists():
+                    candidates.append(candidate)
+    return candidates
+
+
+def _binary_supports_subcommand(binary: Path, subcommand: str) -> bool:
+    try:
+        probe = subprocess.run(
+            [str(binary), subcommand, "--help"],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=2,
+        )
+
+        if probe.returncode == 0:
+            return True
+
+        stderr = probe.stderr.lower()
+        return subcommand not in stderr or "unrecognized subcommand" not in stderr
+    except subprocess.TimeoutExpired:
+        return False
+
+
+def _build_cli_with_features(workspace_root: Path, feature: str) -> bool:
+    """Build CLI with specified features. Returns True if successful."""
+    cargo = shutil.which("cargo")
+    if cargo is None:
+        return False
+
+    try:
+        result = subprocess.run(
+            [cargo, "build", "-p", "kreuzberg-cli", "--features", feature],
+            cwd=workspace_root,
+            check=False,
+            capture_output=True,
+            timeout=300,
+        )
+        return result.returncode == 0
+    except (subprocess.TimeoutExpired, OSError):
+        return False
+
+
+def _discover_dev_cli_binary(requested_subcommand: str | None) -> str | None:
+    """Return the path to a locally built CLI binary if available."""
+    workspace_root = Path(__file__).resolve().parents[3]
+    candidates = _iter_dev_cli_candidates(workspace_root)
+
+    if requested_subcommand is None:
+        if candidates:
+            return str(candidates[0])
+        return None
+
+    for candidate in candidates:
+        if _binary_supports_subcommand(candidate, requested_subcommand):
+            return str(candidate)
+
+    if not (workspace_root / "Cargo.toml").exists():
+        return None
+
+    feature = _FEATURE_SUBCOMMANDS.get(requested_subcommand)
+    if feature is None:
+        return None
+
+    if not _build_cli_with_features(workspace_root, feature):
+        return None
+
+    for candidate in _iter_dev_cli_candidates(workspace_root):
+        if _binary_supports_subcommand(candidate, requested_subcommand):
+            return str(candidate)
+
+    return None
+
+
+def _find_packaged_cli_binary() -> str | None:
+    """Look for the CLI binary in common installation paths before building one."""
+    package_dir = Path(__file__).parent
+    for name in ("kreuzberg-cli", "kreuzberg", "kreuzberg-cli.exe", "kreuzberg.exe"):
+        candidate = package_dir / name
+        if candidate.exists() and candidate.is_file():
+            return str(candidate)
+
+    script_dir = Path(sys.executable).parent
+    for name in ("kreuzberg-cli", "kreuzberg"):
+        candidate = script_dir / name
+        if candidate.exists():
+            try:
+                with candidate.open("rb") as f:
+                    header = f.read(2)
+                    if header == b"#!":
+                        continue
+            except OSError:
+                continue
+            return str(candidate)
+    return None
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Execute the Rust CLI with the provided arguments."""
+    args = list(argv[1:] if argv is not None else sys.argv[1:])
+
+    requested_subcommand: str | None = None
+    if args:
+        first = args[0]
+        if not first.startswith("-"):
+            requested_subcommand = first
+
+    cli_path = shutil.which("kreuzberg-cli")
+
+    if cli_path is None:
+        cli_path = _find_packaged_cli_binary()
+
+    if cli_path is None:
+        cli_path = _discover_dev_cli_binary(requested_subcommand)
+
+    if cli_path is None:
+        sys.stderr.write(
+            "The embedded Kreuzberg CLI binary could not be located. "
+            "This indicates a packaging issue with the wheel; please open an issue at "
+            "https://github.com/kreuzberg-dev/kreuzberg/issues so we can investigate.\n",
+        )
+        return 1
+
+    completed = subprocess.run([cli_path, *args], check=False)
+    return completed.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

kreuzberg/_setup_lib_path.py

@@ -0,0 +1,143 @@
+"""Set up dynamic library search paths for bundled native libraries.
+
+This module must be imported before _internal_bindings to ensure pdfium
+and other native libraries can be found at runtime without requiring users
+to manually set DYLD_LIBRARY_PATH (macOS), LD_LIBRARY_PATH (Linux), or
+PATH (Windows).
+
+Additionally, on macOS, this module fixes the library install names if needed
+using install_name_tool, ensuring @loader_path is used for relative references.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import platform
+import subprocess
+import sys
+from pathlib import Path
+
+
+def setup_library_paths() -> None:
+    """Add package directory to dynamic library search path.
+
+    This ensures bundled native libraries (pdfium, etc.) can be found
+    at runtime across all platforms.
+    """
+    package_dir = Path(__file__).parent.resolve()
+
+    system = platform.system()
+
+    if system == "Darwin":
+        _fix_macos_install_names(package_dir)
+        _setup_macos_paths(package_dir)
+    elif system == "Linux":
+        _setup_linux_paths(package_dir)
+    elif system == "Windows":
+        _setup_windows_paths(package_dir)
+
+
+def _fix_macos_install_names(package_dir: Path) -> None:
+    so_file = package_dir / "_internal_bindings.abi3.so"
+    pdfium_lib = package_dir / "libpdfium.dylib"
+
+    if not so_file.exists() or not pdfium_lib.exists():
+        return
+
+    try:
+        result = subprocess.run(
+            ["otool", "-L", str(so_file)],  # noqa: S607
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=5,
+        )
+
+        if "@loader_path/libpdfium.dylib" in result.stdout:
+            return
+
+        if "./libpdfium.dylib" in result.stdout:
+            with contextlib.suppress(subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
+                subprocess.run(
+                    [  # noqa: S607
+                        "install_name_tool",
+                        "-change",
+                        "./libpdfium.dylib",
+                        "@loader_path/libpdfium.dylib",
+                        str(so_file),
+                    ],
+                    check=True,
+                    timeout=5,
+                    capture_output=True,
+                )
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
+        pass
+
+
+def _setup_macos_paths(package_dir: Path) -> None:
+    current_path = os.environ.get("DYLD_LIBRARY_PATH", "")
+    package_str = str(package_dir)
+
+    if package_str not in current_path:
+        if current_path:
+            os.environ["DYLD_LIBRARY_PATH"] = f"{package_str}:{current_path}"
+        else:
+            os.environ["DYLD_LIBRARY_PATH"] = package_str
+
+    current_fallback = os.environ.get("DYLD_FALLBACK_LIBRARY_PATH", "")
+    if package_str not in current_fallback:
+        if current_fallback:
+            os.environ["DYLD_FALLBACK_LIBRARY_PATH"] = f"{package_str}:{current_fallback}"
+        else:
+            os.environ["DYLD_FALLBACK_LIBRARY_PATH"] = f"{package_str}:/usr/local/lib:/usr/lib"
+
+
+def _setup_linux_paths(package_dir: Path) -> None:
+    current_path = os.environ.get("LD_LIBRARY_PATH", "")
+    package_str = str(package_dir)
+
+    if package_str not in current_path:
+        if current_path:
+            os.environ["LD_LIBRARY_PATH"] = f"{package_str}:{current_path}"
+        else:
+            os.environ["LD_LIBRARY_PATH"] = package_str
+
+    try:
+        import ctypes  # noqa: PLC0415
+        import ctypes.util  # noqa: PLC0415
+
+        lib_path = package_dir / "libpdfium.so"
+        if lib_path.exists():
+            with contextlib.suppress(OSError):
+                ctypes.CDLL(str(lib_path))
+    except (ImportError, AttributeError):
+        pass
+
+
+def _setup_windows_paths(package_dir: Path) -> None:
+    package_str = str(package_dir)
+
+    current_path = os.environ.get("PATH", "")
+    if package_str not in current_path:
+        if current_path:
+            os.environ["PATH"] = f"{package_str};{current_path}"
+        else:
+            os.environ["PATH"] = package_str
+
+    if sys.version_info >= (3, 8) and hasattr(os, "add_dll_directory"):
+        with contextlib.suppress(OSError, AttributeError):
+            os.add_dll_directory(str(package_dir))
+
+    try:
+        import ctypes  # noqa: PLC0415
+
+        lib_path = package_dir / "pdfium.dll"
+        if lib_path.exists():
+            with contextlib.suppress(OSError):
+                ctypes.CDLL(str(lib_path))
+    except (ImportError, AttributeError):
+        pass
+
+
+setup_library_paths()

kreuzberg/exceptions.py

@@ -0,0 +1,254 @@
+"""Exception classes for Kreuzberg.
+
+All Kreuzberg exceptions inherit from KreuzbergError and support optional context
+for debugging information.
+"""
+
+import json
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Any
+
+
+class ErrorCode(IntEnum):
+    """Error codes from the Kreuzberg FFI layer.
+
+    Attributes:
+        SUCCESS: No error occurred
+        GENERIC_ERROR: Generic/unknown error
+        PANIC: A panic occurred in the library
+        INVALID_ARGUMENT: Invalid argument provided
+        IO_ERROR: I/O operation failed
+        PARSING_ERROR: Document parsing failed
+        OCR_ERROR: OCR processing failed
+        MISSING_DEPENDENCY: Required dependency not found
+
+    Example:
+        >>> from kreuzberg import get_last_error_code, ErrorCode
+        >>> code = get_last_error_code()
+        >>> if code == ErrorCode.PANIC:
+        ...     print("A panic occurred")
+    """
+
+    SUCCESS = 0
+    GENERIC_ERROR = 1
+    PANIC = 2
+    INVALID_ARGUMENT = 3
+    IO_ERROR = 4
+    PARSING_ERROR = 5
+    OCR_ERROR = 6
+    MISSING_DEPENDENCY = 7
+
+
+@dataclass(frozen=True, slots=True)
+class PanicContext:
+    """Structured panic context information from FFI layer.
+
+    Attributes:
+        file: Source file where panic occurred
+        line: Line number in source file
+        function: Function name where panic occurred
+        message: Panic message
+        timestamp_secs: Unix timestamp (seconds since epoch) when panic occurred
+
+    Example:
+        >>> import json
+        >>> from kreuzberg import get_last_panic_context, PanicContext
+        >>> context_json = get_last_panic_context()
+        >>> if context_json:
+        ...     data = json.loads(context_json)
+        ...     context = PanicContext(**data)
+        ...     print(f"Panic at {context.file}:{context.line}")
+    """
+
+    file: str
+    line: int
+    function: str
+    message: str
+    timestamp_secs: int
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "PanicContext":
+        """Parse panic context from JSON string.
+
+        Args:
+            json_str: JSON string with panic context (as returned by get_last_panic_context)
+
+        Returns:
+            PanicContext dataclass instance
+
+        Raises:
+            ValueError: If JSON is invalid or missing required fields
+        """
+        data = json.loads(json_str)
+        return cls(**data)
+
+
+class KreuzbergError(Exception):
+    """Base exception class for all Kreuzberg errors.
+
+    All Kreuzberg exceptions support an optional context dictionary for debugging
+    information. The context is serialized to JSON when the exception is converted
+    to a string.
+
+    Args:
+        message: Human-readable error message
+        context: Optional dictionary with debugging context (file paths, config, etc.)
+
+    Example:
+        >>> raise KreuzbergError("Failed to parse document", context={"file": "document.pdf", "page": 5})
+
+    """
+
+    def __init__(self, message: str, *, context: dict[str, Any] | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.context = context
+
+    def __str__(self) -> str:
+        """Format error with context as JSON."""
+        error_name = self.__class__.__name__
+        if self.context:
+            serialized_context = self._serialize_context(self.context)
+            context_json = json.dumps(serialized_context, sort_keys=True)
+            return f"{error_name}: {self.message}\nContext: {context_json}"
+        return f"{error_name}: {self.message}"
+
+    @staticmethod
+    def _serialize_context(context: dict[str, Any]) -> dict[str, Any]:
+        def serialize_value(value: Any) -> Any:
+            if isinstance(value, bytes):
+                return value.decode("utf-8", errors="replace")
+            if isinstance(value, Exception):
+                return {"type": type(value).__name__, "message": str(value)}
+            if isinstance(value, tuple):
+                return [serialize_value(item) for item in value]
+            if isinstance(value, list):
+                return [serialize_value(item) for item in value]
+            if isinstance(value, dict):
+                return {k: serialize_value(v) for k, v in value.items()}
+            return value
+
+        serialized: dict[str, Any] = serialize_value(context)
+        return serialized
+
+
+class ValidationError(KreuzbergError):
+    """Raised when input validation fails.
+
+    This includes configuration validation, parameter validation, and input
+    data validation errors.
+
+    Example:
+        >>> raise ValidationError("Invalid language code", context={"language": "xyz", "supported": ["en", "de"]})
+
+    """
+
+
+class ParsingError(KreuzbergError):
+    """Raised when document parsing fails.
+
+    This includes errors from extractors when they cannot parse a document
+    (corrupt files, unsupported features, etc.).
+
+    Example:
+        >>> raise ParsingError("Failed to parse PDF", context={"file": "document.pdf", "extractor": "pdf"})
+
+    """
+
+
+class OCRError(KreuzbergError):
+    """Raised when OCR processing fails.
+
+    This includes errors from OCR backends during text extraction from images.
+
+    Example:
+        >>> raise OCRError("OCR processing failed", context={"backend": "tesseract", "language": "en"})
+
+    """
+
+
+class MissingDependencyError(KreuzbergError):
+    """Raised when a required dependency is not installed.
+
+    This includes missing Python packages and missing system dependencies
+    (tesseract, pandoc, etc.).
+
+    Example:
+        >>> raise MissingDependencyError(
+        ...     "EasyOCR not installed", context={"package": "easyocr", "install_command": "pip install kreuzberg[easyocr]"}
+        ... )
+
+    """
+
+    @classmethod
+    def create_for_package(
+        cls,
+        *,
+        dependency_group: str,
+        functionality: str,
+        package_name: str,
+    ) -> "MissingDependencyError":
+        """Create a MissingDependencyError for a missing package.
+
+        This is a convenience method for creating standardized error messages
+        for missing optional dependencies.
+
+        Args:
+            dependency_group: The optional dependency group (e.g., "ocr", "api", "cli")
+            functionality: Description of what functionality requires this package
+            package_name: Name of the missing package
+
+        Returns:
+            MissingDependencyError with formatted message and context
+
+        Example:
+            >>> error = MissingDependencyError.create_for_package(
+            ...     dependency_group="easyocr", functionality="EasyOCR backend", package_name="easyocr"
+            ... )
+            >>> raise error
+
+        """
+        install_cmd = f"pip install kreuzberg[{dependency_group}]"
+        message = f"Missing required dependency '{package_name}' for {functionality}. Install with: {install_cmd}"
+        context = {
+            "package": package_name,
+            "dependency_group": dependency_group,
+            "functionality": functionality,
+            "install_command": install_cmd,
+        }
+        return cls(message, context=context)
+
+
+class CacheError(KreuzbergError):
+    """Raised when cache operations fail.
+
+    This includes errors during cache reads, writes, or invalidations.
+
+    Example:
+        >>> raise CacheError("Failed to write cache", context={"path": "/tmp/cache", "operation": "write"})
+
+    """
+
+
+class ImageProcessingError(KreuzbergError):
+    """Raised when image manipulation fails.
+
+    This includes errors during image resizing, format conversion, or other
+    image processing operations.
+
+    Example:
+        >>> raise ImageProcessingError("Failed to resize image", context={"width": 1920, "height": 1080})
+
+    """
+
+
+class PluginError(KreuzbergError):
+    """Raised when plugin operations fail.
+
+    This includes errors during plugin initialization, registration, or execution.
+
+    Example:
+        >>> raise PluginError("Plugin initialization failed", context={"plugin_name": "pdf-extractor"})
+
+    """

kreuzberg/ocr/__init__.py

@@ -0,0 +1,25 @@
+"""Python OCR backend implementations.
+
+These backends can be imported and manually registered, or they will be
+auto-registered when kreuzberg is imported (if their dependencies are installed).
+
+Each backend has a separate optional dependency group:
+- EasyOCR: pip install "kreuzberg[easyocr]"
+- PaddleOCR: pip install "kreuzberg[paddleocr]"
+"""
+
+from __future__ import annotations
+
+__all__ = ["EasyOCRBackend", "OcrBackendProtocol", "PaddleOCRBackend"]
+
+from kreuzberg.ocr.protocol import OcrBackendProtocol
+
+try:
+    from kreuzberg.ocr.easyocr import EasyOCRBackend
+except ImportError:
+    EasyOCRBackend = None  # type: ignore[assignment,misc]
+
+try:
+    from kreuzberg.ocr.paddleocr import PaddleOCRBackend
+except ImportError:
+    PaddleOCRBackend = None  # type: ignore[assignment,misc]