kreuzberg 3.17.2__py3-none-any.whl → 3.18.0__py3-none-any.whl

kreuzberg/_api/main.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import base64
 import io
+import os
 import traceback
 from json import dumps
 from typing import TYPE_CHECKING, Annotated, Any, Literal
@@ -100,6 +101,36 @@ def exception_handler(request: Request[Any, Any, Any], exception: KreuzbergError
     )
 
 
+def _get_max_upload_size() -> int:
+    """Get the maximum upload size from environment variable.
+
+    Returns:
+        Maximum upload size in bytes. Defaults to 1GB if not set.
+
+    Environment Variables:
+        KREUZBERG_MAX_UPLOAD_SIZE: Maximum upload size in bytes (default: 1073741824 = 1GB)
+    """
+    default_size = 1024 * 1024 * 1024  # 1GB
+    try:
+        size = int(os.environ.get("KREUZBERG_MAX_UPLOAD_SIZE", default_size))
+        # Return default if negative
+        return size if size >= 0 else default_size
+    except ValueError:
+        return default_size
+
+
+def _is_opentelemetry_enabled() -> bool:
+    """Check if OpenTelemetry should be enabled.
+
+    Returns:
+        True if OpenTelemetry should be enabled, False otherwise.
+
+    Environment Variables:
+        KREUZBERG_ENABLE_OPENTELEMETRY: Enable OpenTelemetry tracing (true/false) (default: true)
+    """
+    return os.environ.get("KREUZBERG_ENABLE_OPENTELEMETRY", "true").lower() in ("true", "1", "yes", "on")
+
+
 def general_exception_handler(request: Request[Any, Any, Any], exception: Exception) -> Response[Any]:
     error_type = type(exception).__name__
     error_message = str(exception)
@@ -242,7 +273,7 @@ async def handle_files_upload(  # noqa: PLR0913
     - Language detection (if enabled)
 
     Supports various file formats including PDF, Office documents, images, and more.
-    Maximum file size: 1GB per file.
+    Maximum file size: Configurable via KREUZBERG_MAX_UPLOAD_SIZE environment variable (default: 1GB per file).
 
     Args:
         request: The HTTP request object
@@ -379,9 +410,18 @@ type_encoders = {
     Image.Image: _pil_image_encoder,
 }
 
+
+def _get_plugins() -> list[Any]:
+    """Get configured plugins based on environment variables."""
+    plugins = []
+    if _is_opentelemetry_enabled():
+        plugins.append(OpenTelemetryPlugin(OpenTelemetryConfig()))
+    return plugins
+
+
 app = Litestar(
     route_handlers=[handle_files_upload, health_check, get_configuration],
-    plugins=[OpenTelemetryPlugin(OpenTelemetryConfig())],
+    plugins=_get_plugins(),
     logging_config=StructLoggingConfig(),
     openapi_config=openapi_config,
     exception_handlers={
@@ -389,5 +429,5 @@ app = Litestar(
         Exception: general_exception_handler,
     },
     type_encoders=type_encoders,
-    request_max_body_size=1024 * 1024 * 1024,
+    request_max_body_size=_get_max_upload_size(),
 )

kreuzberg/_entity_extraction.py

@@ -2,17 +2,77 @@ from __future__ import annotations
 
 import os
 import re
+import shutil
+import subprocess
 from functools import lru_cache
 from itertools import chain
 from typing import TYPE_CHECKING, Any
 
+import anyio
+
 from kreuzberg._types import Entity, SpacyEntityExtractionConfig
-from kreuzberg.exceptions import MissingDependencyError
+from kreuzberg._utils._sync import run_sync
+from kreuzberg.exceptions import KreuzbergError, MissingDependencyError
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
 
+def is_uv_available() -> bool:
+    """Check if uv is available in the environment."""
+    return shutil.which("uv") is not None
+
+
+def get_spacy_model_url(model_name: str, version: str = "3.8.0") -> str:
+    """Get the direct download URL for a spaCy model.
+
+    Args:
+        model_name: Name of the spaCy model (e.g., 'en_core_web_sm')
+        version: Model version to download (default: 3.8.0)
+
+    Returns:
+        Direct download URL for the model
+    """
+    return f"https://github.com/explosion/spacy-models/releases/download/{model_name}-{version}/{model_name}-{version}-py3-none-any.whl"
+
+
+async def install_spacy_model_with_uv(model_name: str) -> subprocess.CompletedProcess[str]:
+    """Install spaCy model using uv.
+
+    Args:
+        model_name: Name of the spaCy model to install
+
+    Returns:
+        Completed process result
+    """
+    model_url = get_spacy_model_url(model_name)
+    return await run_sync(
+        subprocess.run,
+        ["uv", "pip", "install", model_url],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+async def install_spacy_model_with_spacy(model_name: str) -> bool:
+    """Install spaCy model using spacy download function.
+
+    Args:
+        model_name: Name of the spaCy model to install
+
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        import spacy.cli.download  # noqa: PLC0415
+
+        await run_sync(spacy.cli.download, model_name)  # type: ignore[attr-defined]
+        return True
+    except (ImportError, OSError, RuntimeError):
+        return False
+
+
 def extract_entities(
     text: str,
     entity_types: Sequence[str] = ("PERSON", "ORGANIZATION", "LOCATION", "DATE", "EMAIL", "PHONE"),
@@ -44,13 +104,11 @@ def extract_entities(
             functionality="Entity Extraction",
         ) from e
 
-    model_name = _select_spacy_model(languages, spacy_config)
+    model_name = select_spacy_model(languages, spacy_config)
     if not model_name:
         return entities
 
-    nlp = _load_spacy_model(model_name, spacy_config)
-    if not nlp:
-        return entities
+    nlp = load_spacy_model(model_name, spacy_config)
 
     if len(text) > spacy_config.max_doc_length:
         text = text[: spacy_config.max_doc_length]
@@ -74,23 +132,87 @@ def extract_entities(
 
 
 @lru_cache(maxsize=32)
-def _load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig) -> Any:
+def load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig) -> Any:
     try:
         import spacy  # noqa: PLC0415
+    except ImportError:
+        return None
 
-        if spacy_config.model_cache_dir:
-            os.environ["SPACY_DATA"] = str(spacy_config.model_cache_dir)
+    if spacy_config.model_cache_dir:
+        os.environ["SPACY_DATA"] = str(spacy_config.model_cache_dir)
 
+    try:
         nlp = spacy.load(model_name)
+    except OSError:
+        # Try to download the model automatically
+        async def install_model() -> tuple[bool, str | None]:
+            """Install model and return success status and error message."""
+            # First try spaCy's built-in download
+            try:
+                success = await install_spacy_model_with_spacy(model_name)
+                if success:
+                    return True, None
+            except (ImportError, OSError, RuntimeError) as e:
+                spacy_error = str(e)
+            else:
+                spacy_error = "spaCy download failed"
+
+            # If spaCy download failed and uv is available, try uv as fallback
+            if is_uv_available():
+                try:
+                    result = await install_spacy_model_with_uv(model_name)
+                    return result.returncode == 0, result.stderr
+                except (OSError, subprocess.SubprocessError) as e:
+                    return False, f"spaCy: {spacy_error}, uv: {e!s}"
+
+            return False, spacy_error
+
+        # Run the async installation in a sync context
+        try:
+            success, error_details = anyio.run(install_model)
+        except (OSError, RuntimeError) as e:
+            success, error_details = False, str(e)
+
+        if not success:
+            # Generate appropriate error message based on available tools
+            if is_uv_available():
+                model_url = get_spacy_model_url(model_name)
+                manual_install_cmd = f"uv pip install {model_url}"
+            else:
+                manual_install_cmd = f"python -m spacy download {model_name}"
+
+            error_msg = (
+                f"Failed to download spaCy model '{model_name}'. Please install it manually with: {manual_install_cmd}"
+            )
 
-        nlp.max_length = spacy_config.max_doc_length
+            if error_details:
+                error_msg += f"\nError details: {error_details}"
 
-        return nlp
-    except (OSError, ImportError):
-        return None
+            raise KreuzbergError(
+                error_msg,
+                context={
+                    "model": model_name,
+                    "manual_install_cmd": manual_install_cmd,
+                    "error_details": error_details,
+                    "uv_available": is_uv_available(),
+                },
+            ) from None
+
+        try:
+            nlp = spacy.load(model_name)
+        except OSError as e:
+            raise KreuzbergError(
+                f"Failed to load spaCy model '{model_name}' even after successful download. "
+                f"Please verify your spaCy installation and try reinstalling the model.",
+                context={"model": model_name, "error": str(e)},
+            ) from e
+
+    nlp.max_length = spacy_config.max_doc_length
+
+    return nlp
 
 
-def _select_spacy_model(languages: list[str] | None, spacy_config: SpacyEntityExtractionConfig) -> str | None:
+def select_spacy_model(languages: list[str] | None, spacy_config: SpacyEntityExtractionConfig) -> str | None:
     if not languages:
         return spacy_config.get_model_for_language("en")
 

kreuzberg/_language_detection.py

@@ -23,9 +23,7 @@ def detect_languages(text: str, config: LanguageDetectionConfig | None = None) -
         config = LanguageDetectionConfig()
 
     try:
-        # detect always returns a list, use k parameter for multiple languages
         k = config.top_k if config.multilingual else 1
-        # Use the model from config directly
         model = config.model
         results = detect(text, model=model, k=k)
 

kreuzberg/extraction.py

@@ -76,7 +76,6 @@ def _validate_and_post_process_helper(
             result.keywords = None
 
     if config.auto_detect_language:
-        # Use provided config or create one with the model from ExtractionConfig
         lang_config = config.language_detection_config
         if lang_config is None:
             from kreuzberg._types import LanguageDetectionConfig  # noqa: PLC0415

{kreuzberg-3.17.2.dist-info → kreuzberg-3.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.17.2
+Version: 3.18.0
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -28,12 +28,12 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Text Processing :: General
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: anyio>=4.10.0
+Requires-Dist: anyio>=4.11.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
-Requires-Dist: html-to-markdown[lxml]>=1.14.0
+Requires-Dist: html-to-markdown[lxml]>=1.16.0
 Requires-Dist: langcodes>=3.5.0
-Requires-Dist: mcp>=1.14.1
+Requires-Dist: mcp>=1.15.0
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: playa-pdf>=0.7.0

{kreuzberg-3.17.2.dist-info → kreuzberg-3.18.0.dist-info}/RECORD

@@ -4,20 +4,20 @@ kreuzberg/_chunker.py,sha256=lRXvVN60vmWaTxa1b3QzvE-jBmOqYzh5dY-3Kl6pSqI,1427
 kreuzberg/_config.py,sha256=ZYIcnJAjDnbWW_2WBy7NlOk1Ol6WpoMG5FMNMmHpqSY,13086
 kreuzberg/_constants.py,sha256=gY6SpCi9za59ghRuLX_z7xfSok6qqvPbvEnv4BLczqI,265
 kreuzberg/_document_classification.py,sha256=55aDxDIJ65qK6yEXt-fRYTn8LgALvYsWssjWSheVpR0,5697
-kreuzberg/_entity_extraction.py,sha256=YvcELIo3kV8A_WbzwNjhKn7rPhkZXjbpNMgm2UK0oJw,3621
+kreuzberg/_entity_extraction.py,sha256=zbwgvS_2M4JibmVVnclkmie0nmZQtyHtT_ucdbQc6nU,7837
 kreuzberg/_gmft.py,sha256=gfRXOsv-K9R7Y0zZ2SUa5wid3FpP2eFIlg5nepWcz1Q,20827
-kreuzberg/_language_detection.py,sha256=OwIWIddERPEz8krU_Aq0_KjRF6MHP-LpugH6Y6miwOc,1204
+kreuzberg/_language_detection.py,sha256=y48gNaexnC6OIVTh3yBjXDumMeIKMggCDuacoXa7AvU,1080
 kreuzberg/_mime_types.py,sha256=duEMDBg_qIf9A02tXAC_2znD-wgE-2BBMW9ofyYTJjE,8622
 kreuzberg/_playa.py,sha256=p4G5ymSSCbQoDeXJjH-yuVzdd4y-wKcolqDthjPtqok,11413
 kreuzberg/_registry.py,sha256=8XYT-vPhNYMAbB5RBIUKz-1Zdg48OCnBcdVZzBq6YwY,3307
 kreuzberg/_types.py,sha256=ttY61QI8mruCI70Af3owlU-O5LdvQ6gOqIZTGQ9PaVs,49129
 kreuzberg/cli.py,sha256=OoHA5MiIcRBATFJpb-FZYlZfpohxL2AbVgamyhnEMFo,14342
 kreuzberg/exceptions.py,sha256=KiGAfIX3_TkGYG1h9eTZ_E_pALsAqhZ_A3XfhwxwaS0,2909
-kreuzberg/extraction.py,sha256=jwzWdomwrl-2z1UznLoURLyqD5r0U-rFABXSBV2B2wA,19063
+kreuzberg/extraction.py,sha256=ArsmHcJDvjx9Cog3IQ0D52oS9GbaH_Yhs5mfJfGgiaM,18982
 kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_api/_config_cache.py,sha256=gX_ezGNq6SCpTn02yFkn24zMVrQwfIk8-u5XkKJiHFg,8774
-kreuzberg/_api/main.py,sha256=_tBZaRiq7qq7x4nXkVRgU5FBivLFJ_dmadAc7aT0H_k,13901
+kreuzberg/_api/main.py,sha256=5LiqgyeHJy0GLLa-ehB0bq8ftEUYfM1Pt6f0j_a0dso,15190
 kreuzberg/_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_extractors/_base.py,sha256=4MRBXdLsgdtdrTuupWb2IT9YpRSnNPpWWviS2mfeOXg,9961
 kreuzberg/_extractors/_email.py,sha256=DzNthVbmbdlajDUfs0nNwbHNvG0CAQVqJsRfsatHtf0,8799
@@ -121,8 +121,8 @@ kreuzberg/_utils/_string.py,sha256=wVyvEHByHBeu_6evmqJGv9Ml-NAwkyz60n8l-7L5Cw0,4
 kreuzberg/_utils/_sync.py,sha256=gb828WYfVtkB4wKslJrPMmrdeI1h3htWceq-gywHtO4,3184
 kreuzberg/_utils/_table.py,sha256=OVg6T2QnerMhVNb1juLTBSIjyjFiE5-OrUWr5NSCgnQ,6493
 kreuzberg/_utils/_tmp.py,sha256=mwZ0BFzhGPfYa2tt8qSjUjfcHnSYvbQT4VlPRCRc_q8,2038
-kreuzberg-3.17.2.dist-info/METADATA,sha256=JlkYHBdCVl74XDARQuuvrhEncD2CaWBkodwO_S8ddH8,12351
-kreuzberg-3.17.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kreuzberg-3.17.2.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
-kreuzberg-3.17.2.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-3.17.2.dist-info/RECORD,,
+kreuzberg-3.18.0.dist-info/METADATA,sha256=Z54em4GwMd18BmlIWmq1AHtCdFStstMV5RAXaB4x3_0,12351
+kreuzberg-3.18.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kreuzberg-3.18.0.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
+kreuzberg-3.18.0.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-3.18.0.dist-info/RECORD,,