kreuzberg 3.17.3__tar.gz → 3.20.1__tar.gz

{kreuzberg-3.17.3 → kreuzberg-3.20.1}/PKG-INFO

@@ -1,13 +1,11 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: kreuzberg
-Version: 3.17.3
+Version: 3.20.1
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
-Project-URL: documentation, https://kreuzberg.dev
-Project-URL: homepage, https://github.com/Goldziher/kreuzberg
+Keywords: async,document-analysis,document-classification,document-intelligence,document-processing,extensible,information-extraction,mcp,metadata-extraction,model-context-protocol,ocr,pandoc,pdf-extraction,pdfium,plugin-architecture,rag,retrieval-augmented-generation,structured-data,table-extraction,tesseract,text-extraction
+Author: Na'aman Hirschfeld
 Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
 License: MIT
-License-File: LICENSE
-Keywords: async,document-analysis,document-classification,document-intelligence,document-processing,extensible,information-extraction,mcp,metadata-extraction,model-context-protocol,ocr,pandoc,pdf-extraction,pdfium,plugin-architecture,rag,retrieval-augmented-generation,structured-data,table-extraction,tesseract,text-extraction
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
@@ -27,67 +25,56 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Text Processing :: General
 Classifier: Typing :: Typed
-Requires-Python: >=3.10
-Requires-Dist: anyio>=4.10.0
+Requires-Dist: anyio>=4.11.0
 Requires-Dist: chardetng-py>=0.3.5
-Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
-Requires-Dist: html-to-markdown[lxml]>=1.14.0
+Requires-Dist: exceptiongroup>=1.2.2 ; python_full_version < '3.11'
+Requires-Dist: html-to-markdown>=2.1.0
 Requires-Dist: langcodes>=3.5.0
-Requires-Dist: mcp>=1.14.1
+Requires-Dist: mcp>=1.17.0
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: playa-pdf>=0.7.0
-Requires-Dist: polars>=1.33.1
+Requires-Dist: polars>=1.34.0
 Requires-Dist: psutil>=7.1.0
 Requires-Dist: pypdfium2==4.30.0
 Requires-Dist: python-calamine>=0.5.3
 Requires-Dist: python-pptx>=1.0.2
-Requires-Dist: typing-extensions>=4.15.0; python_version < '3.12'
+Requires-Dist: transformers>=4.55.0
+Requires-Dist: typing-extensions>=4.15.0 ; python_full_version < '3.12'
+Requires-Dist: mailparse>=1.0.15 ; extra == 'additional-extensions'
+Requires-Dist: tomli>=2.0.0 ; python_full_version < '3.11' and extra == 'additional-extensions'
+Requires-Dist: kreuzberg[additional-extensions,api,chunking,cli,crypto,document-classification,easyocr,entity-extraction,gmft,langdetect,paddleocr] ; extra == 'all'
+Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.18.0 ; extra == 'api'
+Requires-Dist: semantic-text-splitter>=0.28.0 ; extra == 'chunking'
+Requires-Dist: click>=8.3.0 ; extra == 'cli'
+Requires-Dist: rich>=14.2.0 ; extra == 'cli'
+Requires-Dist: tomli>=2.0.0 ; python_full_version < '3.11' and extra == 'cli'
+Requires-Dist: playa-pdf[crypto]>=0.7.0 ; extra == 'crypto'
+Requires-Dist: deep-translator>=1.11.4 ; extra == 'document-classification'
+Requires-Dist: easyocr>=1.7.2 ; python_full_version < '3.14' and extra == 'easyocr'
+Requires-Dist: keybert>=0.9.0 ; extra == 'entity-extraction'
+Requires-Dist: spacy>=3.8.7 ; python_full_version < '3.14' and extra == 'entity-extraction'
+Requires-Dist: gmft>=0.4.2 ; extra == 'gmft'
+Requires-Dist: transformers>=4.57.0 ; extra == 'gmft'
+Requires-Dist: fast-langdetect>=1.0.0 ; extra == 'langdetect'
+Requires-Dist: paddleocr>=3.2.0 ; python_full_version < '3.14' and extra == 'paddleocr'
+Requires-Dist: paddlepaddle>=3.2.0 ; python_full_version < '3.14' and extra == 'paddleocr'
+Requires-Dist: setuptools>=80.9.0 ; extra == 'paddleocr'
+Requires-Python: >=3.10
+Project-URL: documentation, https://kreuzberg.dev
+Project-URL: homepage, https://github.com/Goldziher/kreuzberg
 Provides-Extra: additional-extensions
-Requires-Dist: mailparse>=1.0.15; extra == 'additional-extensions'
-Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'additional-extensions'
 Provides-Extra: all
-Requires-Dist: click>=8.2.1; extra == 'all'
-Requires-Dist: deep-translator>=1.11.4; extra == 'all'
-Requires-Dist: easyocr>=1.7.2; extra == 'all'
-Requires-Dist: fast-langdetect>=1.0.0; extra == 'all'
-Requires-Dist: gmft>=0.4.2; extra == 'all'
-Requires-Dist: keybert>=0.9.0; extra == 'all'
-Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.17.0; extra == 'all'
-Requires-Dist: mailparse>=1.0.15; extra == 'all'
-Requires-Dist: paddleocr>=3.2.0; extra == 'all'
-Requires-Dist: paddlepaddle>=3.2.0; extra == 'all'
-Requires-Dist: playa-pdf[crypto]>=0.7.0; extra == 'all'
-Requires-Dist: rich>=14.1.0; extra == 'all'
-Requires-Dist: semantic-text-splitter>=0.28.0; extra == 'all'
-Requires-Dist: setuptools>=80.9.0; extra == 'all'
-Requires-Dist: spacy>=3.8.7; extra == 'all'
-Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'all'
 Provides-Extra: api
-Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.17.0; extra == 'api'
 Provides-Extra: chunking
-Requires-Dist: semantic-text-splitter>=0.28.0; extra == 'chunking'
 Provides-Extra: cli
-Requires-Dist: click>=8.2.1; extra == 'cli'
-Requires-Dist: rich>=14.1.0; extra == 'cli'
-Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'cli'
 Provides-Extra: crypto
-Requires-Dist: playa-pdf[crypto]>=0.7.0; extra == 'crypto'
 Provides-Extra: document-classification
-Requires-Dist: deep-translator>=1.11.4; extra == 'document-classification'
 Provides-Extra: easyocr
-Requires-Dist: easyocr>=1.7.2; extra == 'easyocr'
 Provides-Extra: entity-extraction
-Requires-Dist: keybert>=0.9.0; extra == 'entity-extraction'
-Requires-Dist: spacy>=3.8.7; extra == 'entity-extraction'
 Provides-Extra: gmft
-Requires-Dist: gmft>=0.4.2; extra == 'gmft'
 Provides-Extra: langdetect
-Requires-Dist: fast-langdetect>=1.0.0; extra == 'langdetect'
 Provides-Extra: paddleocr
-Requires-Dist: paddleocr>=3.2.0; extra == 'paddleocr'
-Requires-Dist: paddlepaddle>=3.2.0; extra == 'paddleocr'
-Requires-Dist: setuptools>=80.9.0; extra == 'paddleocr'
 Description-Content-Type: text/markdown
 
 # Kreuzberg

{kreuzberg-3.17.3 → kreuzberg-3.20.1}/kreuzberg/_api/main.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import base64
 import io
+import os
 import traceback
 from json import dumps
 from typing import TYPE_CHECKING, Annotated, Any, Literal
@@ -100,6 +101,35 @@ def exception_handler(request: Request[Any, Any, Any], exception: KreuzbergError
     )
 
 
+def _get_max_upload_size() -> int:
+    """Get the maximum upload size from environment variable.
+
+    Returns:
+        Maximum upload size in bytes. Defaults to 1GB if not set.
+
+    Environment Variables:
+        KREUZBERG_MAX_UPLOAD_SIZE: Maximum upload size in bytes (default: 1073741824 = 1GB)
+    """
+    default_size = 1024 * 1024 * 1024
+    try:
+        size = int(os.environ.get("KREUZBERG_MAX_UPLOAD_SIZE", default_size))
+        return size if size >= 0 else default_size
+    except ValueError:
+        return default_size
+
+
+def _is_opentelemetry_enabled() -> bool:
+    """Check if OpenTelemetry should be enabled.
+
+    Returns:
+        True if OpenTelemetry should be enabled, False otherwise.
+
+    Environment Variables:
+        KREUZBERG_ENABLE_OPENTELEMETRY: Enable OpenTelemetry tracing (true/false) (default: true)
+    """
+    return os.environ.get("KREUZBERG_ENABLE_OPENTELEMETRY", "true").lower() in ("true", "1", "yes", "on")
+
+
 def general_exception_handler(request: Request[Any, Any, Any], exception: Exception) -> Response[Any]:
     error_type = type(exception).__name__
     error_message = str(exception)
@@ -242,7 +272,7 @@ async def handle_files_upload(  # noqa: PLR0913
     - Language detection (if enabled)
 
     Supports various file formats including PDF, Office documents, images, and more.
-    Maximum file size: 1GB per file.
+    Maximum file size: Configurable via KREUZBERG_MAX_UPLOAD_SIZE environment variable (default: 1GB per file).
 
     Args:
         request: The HTTP request object
@@ -280,6 +310,9 @@ async def handle_files_upload(  # noqa: PLR0913
     """
     static_config = discover_config_cached()
 
+    if not data:
+        raise ValidationError("No files provided for extraction", context={"file_count": 0})
+
     min_dims = _create_dimension_tuple(image_ocr_min_width, image_ocr_min_height)
     max_dims = _create_dimension_tuple(image_ocr_max_width, image_ocr_max_height)
 
@@ -379,9 +412,18 @@ type_encoders = {
     Image.Image: _pil_image_encoder,
 }
 
+
+def _get_plugins() -> list[Any]:
+    """Get configured plugins based on environment variables."""
+    plugins = []
+    if _is_opentelemetry_enabled():
+        plugins.append(OpenTelemetryPlugin(OpenTelemetryConfig()))
+    return plugins
+
+
 app = Litestar(
     route_handlers=[handle_files_upload, health_check, get_configuration],
-    plugins=[OpenTelemetryPlugin(OpenTelemetryConfig())],
+    plugins=_get_plugins(),
     logging_config=StructLoggingConfig(),
     openapi_config=openapi_config,
     exception_handlers={
@@ -389,5 +431,5 @@ app = Litestar(
         Exception: general_exception_handler,
     },
     type_encoders=type_encoders,
-    request_max_body_size=1024 * 1024 * 1024,
+    request_max_body_size=_get_max_upload_size(),
 )

{kreuzberg-3.17.3 → kreuzberg-3.20.1}/kreuzberg/_entity_extraction.py

@@ -2,19 +2,77 @@ from __future__ import annotations
 
 import os
 import re
+import shutil
 import subprocess
-import sys
 from functools import lru_cache
 from itertools import chain
 from typing import TYPE_CHECKING, Any
 
+import anyio
+
 from kreuzberg._types import Entity, SpacyEntityExtractionConfig
+from kreuzberg._utils._sync import run_sync
 from kreuzberg.exceptions import KreuzbergError, MissingDependencyError
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
 
+def is_uv_available() -> bool:
+    """Check if uv is available in the environment."""
+    return shutil.which("uv") is not None
+
+
+def get_spacy_model_url(model_name: str, version: str = "3.8.0") -> str:
+    """Get the direct download URL for a spaCy model.
+
+    Args:
+        model_name: Name of the spaCy model (e.g., 'en_core_web_sm')
+        version: Model version to download (default: 3.8.0)
+
+    Returns:
+        Direct download URL for the model
+    """
+    return f"https://github.com/explosion/spacy-models/releases/download/{model_name}-{version}/{model_name}-{version}-py3-none-any.whl"
+
+
+async def install_spacy_model_with_uv(model_name: str) -> subprocess.CompletedProcess[str]:
+    """Install spaCy model using uv.
+
+    Args:
+        model_name: Name of the spaCy model to install
+
+    Returns:
+        Completed process result
+    """
+    model_url = get_spacy_model_url(model_name)
+    return await run_sync(
+        subprocess.run,
+        ["uv", "pip", "install", model_url],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+async def install_spacy_model_with_spacy(model_name: str) -> bool:
+    """Install spaCy model using spacy download function.
+
+    Args:
+        model_name: Name of the spaCy model to install
+
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        import spacy.cli.download  # noqa: PLC0415
+
+        await run_sync(spacy.cli.download, model_name)  # type: ignore[attr-defined]
+        return True
+    except (ImportError, OSError, RuntimeError):
+        return False
+
+
 def extract_entities(
     text: str,
     entity_types: Sequence[str] = ("PERSON", "ORGANIZATION", "LOCATION", "DATE", "EMAIL", "PHONE"),
@@ -46,11 +104,11 @@ def extract_entities(
             functionality="Entity Extraction",
         ) from e
 
-    model_name = _select_spacy_model(languages, spacy_config)
+    model_name = select_spacy_model(languages, spacy_config)
     if not model_name:
         return entities
 
-    nlp = _load_spacy_model(model_name, spacy_config)
+    nlp = load_spacy_model(model_name, spacy_config)
 
     if len(text) > spacy_config.max_doc_length:
         text = text[: spacy_config.max_doc_length]
@@ -74,7 +132,7 @@ def extract_entities(
 
 
 @lru_cache(maxsize=32)
-def _load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig) -> Any:
+def load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig) -> Any:
     try:
         import spacy  # noqa: PLC0415
     except ImportError:
@@ -86,22 +144,54 @@ def _load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig
     try:
         nlp = spacy.load(model_name)
     except OSError:
-        result = subprocess.run(
-            [sys.executable, "-m", "spacy", "download", model_name],
-            capture_output=True,
-            text=True,
-            check=False,
-        )
 
-        if result.returncode != 0:
+        async def install_model() -> tuple[bool, str | None]:
+            """Install model and return success status and error message."""
+            try:
+                success = await install_spacy_model_with_spacy(model_name)
+                if success:
+                    return True, None
+            except (ImportError, OSError, RuntimeError) as e:
+                spacy_error = str(e)
+            else:
+                spacy_error = "spaCy download failed"
+
+            if is_uv_available():
+                try:
+                    result = await install_spacy_model_with_uv(model_name)
+                    return result.returncode == 0, result.stderr
+                except (OSError, subprocess.SubprocessError) as e:
+                    return False, f"spaCy: {spacy_error}, uv: {e!s}"
+
+            return False, spacy_error
+
+        try:
+            success, error_details = anyio.run(install_model)
+        except SystemExit as e:
+            success, error_details = False, f"spaCy CLI exit code: {e.code}"
+
+        if not success:
+            if is_uv_available():
+                model_url = get_spacy_model_url(model_name)
+                manual_install_cmd = f"uv pip install {model_url}"
+            else:
+                manual_install_cmd = f"python -m spacy download {model_name}"
+
             error_msg = (
-                f"Failed to download spaCy model '{model_name}'. "
-                f"Please install it manually with: python -m spacy download {model_name}"
+                f"Failed to download spaCy model '{model_name}'. Please install it manually with: {manual_install_cmd}"
             )
-            if result.stderr:
-                error_msg += f"\nError details: {result.stderr}"
+
+            if error_details:
+                error_msg += f"\nError details: {error_details}"
+
             raise KreuzbergError(
-                error_msg, context={"model": model_name, "stderr": result.stderr, "return_code": result.returncode}
+                error_msg,
+                context={
+                    "model": model_name,
+                    "manual_install_cmd": manual_install_cmd,
+                    "error_details": error_details,
+                    "uv_available": is_uv_available(),
+                },
             ) from None
 
         try:
@@ -118,7 +208,7 @@ def _load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig
     return nlp
 
 
-def _select_spacy_model(languages: list[str] | None, spacy_config: SpacyEntityExtractionConfig) -> str | None:
+def select_spacy_model(languages: list[str] | None, spacy_config: SpacyEntityExtractionConfig) -> str | None:
     if not languages:
         return spacy_config.get_model_for_language("en")
 
@@ -140,7 +230,7 @@ def extract_keywords(
         kw_model = KeyBERT()
         keywords = kw_model.extract_keywords(text, top_n=keyword_count)
         return [(kw, float(score)) for kw, score in keywords]
-    except (RuntimeError, OSError, ValueError):
+    except ValueError:
         return []
     except ImportError as e:  # pragma: no cover
         raise MissingDependencyError.create_for_package(

kreuzberg-3.20.1/kreuzberg/_error_handling.py

@@ -0,0 +1,182 @@
+"""Type-safe error handling utilities for extraction pipeline."""
+
+from __future__ import annotations
+
+import traceback
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+from kreuzberg._types import ErrorContextType, ExtractionResult, Metadata, ProcessingErrorDict
+from kreuzberg.exceptions import KreuzbergError, MissingDependencyError, ValidationError
+
+
+def should_exception_bubble_up(exception: Exception, context: ErrorContextType = "unknown") -> bool:
+    """Determine if an exception should bubble up or be handled gracefully.
+
+    Args:
+        exception: The exception to classify
+        context: The context where the exception occurred (e.g., "batch_processing", "single_extraction", "optional_feature")
+
+    Returns:
+        True if the exception should bubble up, False if it should be handled gracefully
+    """
+    if isinstance(exception, (SystemExit, KeyboardInterrupt, MemoryError, OSError, RuntimeError)):
+        return True
+
+    if isinstance(exception, MissingDependencyError):
+        return True
+
+    if isinstance(exception, ValidationError):
+        if context == "batch_processing":
+            return False
+
+        return context != "optional_feature"
+
+    if isinstance(exception, KreuzbergError) and context == "optional_feature":
+        return False
+
+    if context == "batch_processing":
+        return isinstance(exception, (SystemExit, KeyboardInterrupt, MemoryError, OSError, RuntimeError))
+
+    return not (context == "optional_feature" and isinstance(exception, (IOError, ImportError)))
+
+
+class FeatureProcessingError:
+    """Type-safe processing error for extraction features."""
+
+    def __init__(self, feature: str, error: Exception) -> None:
+        self._feature = feature
+        self._error = error
+        self._traceback = traceback.format_exc()
+
+    @property
+    def feature(self) -> str:
+        return self._feature
+
+    @property
+    def error_type(self) -> str:
+        return type(self._error).__name__
+
+    @property
+    def error_message(self) -> str:
+        return str(self._error)
+
+    @property
+    def traceback(self) -> str:
+        return self._traceback
+
+    def to_dict(self) -> ProcessingErrorDict:
+        return {
+            "feature": self.feature,
+            "error_type": self.error_type,
+            "error_message": self.error_message,
+            "traceback": self.traceback,
+        }
+
+
+def safe_feature_execution(
+    feature_name: str,
+    execution_func: Callable[[], Any],
+    default_value: Any,
+    result: ExtractionResult,
+    context: ErrorContextType = "optional_feature",
+) -> Any:
+    """Safely execute a feature extraction function with proper error handling.
+
+    Args:
+        feature_name: Name of the feature being executed
+        execution_func: Function to execute that may raise exceptions
+        default_value: Default value to return if execution fails
+        result: ExtractionResult to update with error information
+        context: The context for exception handling decisions
+
+    Returns:
+        Either the successful result or the default value
+    """
+    try:
+        return execution_func()
+    except Exception as e:
+        if should_exception_bubble_up(e, context):
+            raise
+
+        _add_processing_error(result, FeatureProcessingError(feature_name, e))
+        return default_value
+
+
+def _add_processing_error(result: ExtractionResult, error: FeatureProcessingError) -> None:
+    """Add a processing error to the result metadata in a type-safe way."""
+    if result.metadata is None:
+        result.metadata = {}
+
+    if "processing_errors" not in result.metadata:
+        result.metadata["processing_errors"] = []
+
+    errors_list = result.metadata["processing_errors"]
+    if isinstance(errors_list, list):
+        errors_list.append(error.to_dict())
+    else:
+        result.metadata["processing_errors"] = [error.to_dict()]
+
+
+def preserve_result_with_errors(
+    result: ExtractionResult,
+    errors: list[FeatureProcessingError],
+) -> ExtractionResult:
+    """Preserve a successful extraction result while adding error information.
+
+    This is used when core extraction succeeds but optional features fail.
+
+    Args:
+        result: The successful extraction result
+        errors: List of errors that occurred during optional processing
+
+    Returns:
+        The result with error information added to metadata
+    """
+    for error in errors:
+        _add_processing_error(result, error)
+
+    return result
+
+
+def create_error_result(
+    content: str,
+    mime_type: str,
+    errors: list[FeatureProcessingError],
+    **metadata_kwargs: Any,
+) -> ExtractionResult:
+    """Create an error result with proper type safety.
+
+    Args:
+        content: Error content to include
+        mime_type: MIME type of the result
+        errors: List of errors that occurred
+        **metadata_kwargs: Additional metadata to include
+
+    Returns:
+        An ExtractionResult with error information
+    """
+    metadata: Metadata = {
+        "error": f"Multiple processing errors occurred: {len(errors)} errors",
+        "error_context": {
+            "error_count": len(errors),
+            "errors": [error.to_dict() for error in errors],
+            **metadata_kwargs,
+        },
+        "processing_errors": [error.to_dict() for error in errors],
+    }
+
+    return ExtractionResult(
+        content=content,
+        chunks=[],
+        mime_type=mime_type,
+        metadata=metadata,
+        entities=[],
+        keywords=[],
+        detected_languages=[],
+        tables=[],
+        images=[],
+        image_ocr_results=[],
+    )

{kreuzberg-3.17.3 → kreuzberg-3.20.1}/kreuzberg/_extractors/_base.py

@@ -230,13 +230,13 @@ class Extractor(ABC):
                 confidence_score=None,
                 processing_time=duration,
             )
-        except (OSError, ValueError) as e:  # pragma: no cover
+        except ValueError as e:  # pragma: no cover
             return ImageOCRResult(
                 image=target,
                 ocr_result=ExtractionResult(content="", mime_type="text/plain", metadata={}),
                 skipped_reason=f"OCR failed: {type(e).__name__}: {e}",
             )
-        except (RuntimeError, TypeError) as e:  # pragma: no cover
+        except TypeError as e:  # pragma: no cover
             return ImageOCRResult(
                 image=target,
                 ocr_result=ExtractionResult(content="", mime_type="text/plain", metadata={}),