kreuzberg 3.17.0__py3-none-any.whl → 3.17.1__py3-none-any.whl

kreuzberg/_language_detection.py

@@ -1,60 +1,37 @@
 from __future__ import annotations
 
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any
 
 from kreuzberg._types import LanguageDetectionConfig
 from kreuzberg.exceptions import MissingDependencyError
 
-if TYPE_CHECKING:
-    from fast_langdetect import LangDetectConfig as FastLangDetectConfig
-
-try:
-    from fast_langdetect import LangDetectConfig as FastLangDetectConfig
-    from fast_langdetect import detect, detect_multilingual
-
-    HAS_FAST_LANGDETECT = True
-except ImportError:  # pragma: no cover
-    HAS_FAST_LANGDETECT = False
-    detect = None
-    detect_multilingual = None
-    FastLangDetectConfig = None
-
 _CACHE_SIZE = 128
 
 
-def _create_fast_langdetect_config(config: LanguageDetectionConfig) -> FastLangDetectConfig | None:
-    if not HAS_FAST_LANGDETECT or FastLangDetectConfig is None:
-        return None
-
-    kwargs: dict[str, Any] = {
-        "allow_fallback": config.allow_fallback,
-    }
-    if config.cache_dir is not None:
-        kwargs["cache_dir"] = config.cache_dir
-
-    return FastLangDetectConfig(**kwargs)
-
-
 @lru_cache(maxsize=_CACHE_SIZE)
 def detect_languages(text: str, config: LanguageDetectionConfig | None = None) -> list[str] | None:
-    if not HAS_FAST_LANGDETECT or detect is None or detect_multilingual is None:
+    try:
+        from fast_langdetect import detect  # noqa: PLC0415
+    except ImportError as e:
         raise MissingDependencyError.create_for_package(
-            dependency_group="langdetect", functionality="language detection", package_name="fast-langdetect"
-        )
+            dependency_group="langdetect",
+            functionality="language detection",
+            package_name="fast-langdetect",
+        ) from e
 
     if config is None:
         config = LanguageDetectionConfig()
 
     try:
-        if config.multilingual:
-            results = detect_multilingual(text, low_memory=config.low_memory, k=config.top_k)
-
-            return [result["lang"].lower() for result in results if result.get("lang")]
-
-        result = detect(text, low_memory=config.low_memory)
-        if result and result.get("lang"):
-            return [result["lang"].lower()]
+        # detect always returns a list, use k parameter for multiple languages
+        k = config.top_k if config.multilingual else 1
+        # Use the model from config directly
+        model = config.model
+        results = detect(text, model=model, k=k)
+
+        if results:
+            langs = [result["lang"].lower() for result in results if result.get("lang")]
+            return langs if langs else None
         return None
     except Exception:  # noqa: BLE001
         return None

kreuzberg/_types.py

@@ -402,9 +402,12 @@ class ImageOCRConfig(ConfigDict):
 
 @dataclass(unsafe_hash=True, frozen=True, slots=True)
 class LanguageDetectionConfig(ConfigDict):
-    low_memory: bool = True
-    """If True, uses a smaller model (~200MB). If False, uses a larger, more accurate model.
-    Defaults to True for better memory efficiency."""
+    model: Literal["lite", "full", "auto"] = "auto"
+    """Language detection model to use:
+    - 'lite': Smaller, faster model with good accuracy
+    - 'full': Larger model with highest accuracy
+    - 'auto': Automatically choose based on memory availability (default)
+    """
     top_k: int = 3
     """Maximum number of languages to return for multilingual detection."""
     multilingual: bool = False
@@ -412,8 +415,8 @@ class LanguageDetectionConfig(ConfigDict):
     If False, uses single language detection."""
     cache_dir: str | None = None
     """Custom directory for model cache. If None, uses system default."""
-    allow_fallback: bool = True
-    """If True, falls back to small model if large model fails."""
+    low_memory: bool = True
+    """Deprecated. Use 'model' parameter instead. If True, uses 'lite' model."""
 
 
 @dataclass(unsafe_hash=True, frozen=True, slots=True)
@@ -983,8 +986,14 @@ class ExtractionConfig(ConfigDict):
     """Custom entity patterns as a frozenset of (entity_type, regex_pattern) tuples."""
     auto_detect_language: bool = False
     """Whether to automatically detect language and configure OCR accordingly."""
+    language_detection_model: Literal["lite", "full", "auto"] = "auto"
+    """Language detection model to use when auto_detect_language is True.
+    - 'lite': Smaller, faster model with good accuracy
+    - 'full': Larger model with highest accuracy
+    - 'auto': Automatically choose based on memory availability (default)
+    """
     language_detection_config: LanguageDetectionConfig | None = None
-    """Configuration for language detection. If None, uses default settings."""
+    """Configuration for language detection. If None, uses default settings with language_detection_model."""
     spacy_entity_extraction_config: SpacyEntityExtractionConfig | None = None
     """Configuration for spaCy entity extraction. If None, uses default settings."""
     auto_detect_document_type: bool = False

kreuzberg/extraction.py

@@ -76,9 +76,16 @@ def _validate_and_post_process_helper(
             result.keywords = None
 
     if config.auto_detect_language:
+        # Use provided config or create one with the model from ExtractionConfig
+        lang_config = config.language_detection_config
+        if lang_config is None:
+            from kreuzberg._types import LanguageDetectionConfig  # noqa: PLC0415
+
+            lang_config = LanguageDetectionConfig(model=config.language_detection_model)
+
         result.detected_languages = detect_languages(
             result.content,
-            config=config.language_detection_config,
+            config=lang_config,
         )
 
     if config.auto_detect_document_type:

{kreuzberg-3.17.0.dist-info → kreuzberg-3.17.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.17.0
+Version: 3.17.1
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -33,12 +33,12 @@ Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
 Requires-Dist: html-to-markdown[lxml]>=1.13.0
 Requires-Dist: langcodes>=3.5.0
-Requires-Dist: mcp>=1.14.0
+Requires-Dist: mcp>=1.14.1
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: playa-pdf>=0.7.0
 Requires-Dist: polars>=1.33.1
-Requires-Dist: psutil>=7.0.0
+Requires-Dist: psutil>=7.1.0
 Requires-Dist: pypdfium2==4.30.0
 Requires-Dist: python-calamine>=0.5.3
 Requires-Dist: python-pptx>=1.0.2

{kreuzberg-3.17.0.dist-info → kreuzberg-3.17.1.dist-info}/RECORD

@@ -6,14 +6,14 @@ kreuzberg/_constants.py,sha256=gY6SpCi9za59ghRuLX_z7xfSok6qqvPbvEnv4BLczqI,265
 kreuzberg/_document_classification.py,sha256=55aDxDIJ65qK6yEXt-fRYTn8LgALvYsWssjWSheVpR0,5697
 kreuzberg/_entity_extraction.py,sha256=YvcELIo3kV8A_WbzwNjhKn7rPhkZXjbpNMgm2UK0oJw,3621
 kreuzberg/_gmft.py,sha256=gfRXOsv-K9R7Y0zZ2SUa5wid3FpP2eFIlg5nepWcz1Q,20827
-kreuzberg/_language_detection.py,sha256=T9p6aimB7QFXAQiEntIMZeH_Z62E52E8fBQ43hWuyhs,1960
+kreuzberg/_language_detection.py,sha256=OwIWIddERPEz8krU_Aq0_KjRF6MHP-LpugH6Y6miwOc,1204
 kreuzberg/_mime_types.py,sha256=duEMDBg_qIf9A02tXAC_2znD-wgE-2BBMW9ofyYTJjE,8622
 kreuzberg/_playa.py,sha256=p4G5ymSSCbQoDeXJjH-yuVzdd4y-wKcolqDthjPtqok,11413
 kreuzberg/_registry.py,sha256=8XYT-vPhNYMAbB5RBIUKz-1Zdg48OCnBcdVZzBq6YwY,3307
-kreuzberg/_types.py,sha256=uULpUfQzpt_AAr8epOvIl3cdB9TkNTFrxWQssnZg_IM,48655
+kreuzberg/_types.py,sha256=ttY61QI8mruCI70Af3owlU-O5LdvQ6gOqIZTGQ9PaVs,49129
 kreuzberg/cli.py,sha256=OoHA5MiIcRBATFJpb-FZYlZfpohxL2AbVgamyhnEMFo,14342
 kreuzberg/exceptions.py,sha256=KiGAfIX3_TkGYG1h9eTZ_E_pALsAqhZ_A3XfhwxwaS0,2909
-kreuzberg/extraction.py,sha256=Z2rBVGs8oteXU1mynHCd9q1yKz9NNA5tQdWq35jP2EE,18743
+kreuzberg/extraction.py,sha256=jwzWdomwrl-2z1UznLoURLyqD5r0U-rFABXSBV2B2wA,19063
 kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_api/_config_cache.py,sha256=gX_ezGNq6SCpTn02yFkn24zMVrQwfIk8-u5XkKJiHFg,8774
@@ -121,8 +121,8 @@ kreuzberg/_utils/_string.py,sha256=wVyvEHByHBeu_6evmqJGv9Ml-NAwkyz60n8l-7L5Cw0,4
 kreuzberg/_utils/_sync.py,sha256=gb828WYfVtkB4wKslJrPMmrdeI1h3htWceq-gywHtO4,3184
 kreuzberg/_utils/_table.py,sha256=OVg6T2QnerMhVNb1juLTBSIjyjFiE5-OrUWr5NSCgnQ,6493
 kreuzberg/_utils/_tmp.py,sha256=mwZ0BFzhGPfYa2tt8qSjUjfcHnSYvbQT4VlPRCRc_q8,2038
-kreuzberg-3.17.0.dist-info/METADATA,sha256=4iVwQUo4FVNSwj8h6oEqNAT5B6zm-J-u5k3Jy3Pv3L0,12351
-kreuzberg-3.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kreuzberg-3.17.0.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
-kreuzberg-3.17.0.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-3.17.0.dist-info/RECORD,,
+kreuzberg-3.17.1.dist-info/METADATA,sha256=ttfOl3XA6b-M2BMY7v1cfASGm_Qe91HPzfRcAf_-zU8,12351
+kreuzberg-3.17.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kreuzberg-3.17.1.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
+kreuzberg-3.17.1.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-3.17.1.dist-info/RECORD,,