kreuzberg 3.17.0__tar.gz → 3.17.2__tar.gz

{kreuzberg-3.17.0 → kreuzberg-3.17.2}/.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
     hooks:
       - id: pyproject-fmt
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.0
+    rev: v0.13.1
     hooks:
       - id: ruff
         args: ["--fix", "--unsafe-fixes"]

{kreuzberg-3.17.0 → kreuzberg-3.17.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.17.0
+Version: 3.17.2
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -31,14 +31,14 @@ Requires-Python: >=3.10
 Requires-Dist: anyio>=4.10.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
-Requires-Dist: html-to-markdown[lxml]>=1.13.0
+Requires-Dist: html-to-markdown[lxml]>=1.14.0
 Requires-Dist: langcodes>=3.5.0
-Requires-Dist: mcp>=1.14.0
+Requires-Dist: mcp>=1.14.1
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: playa-pdf>=0.7.0
 Requires-Dist: polars>=1.33.1
-Requires-Dist: psutil>=7.0.0
+Requires-Dist: psutil>=7.1.0
 Requires-Dist: pypdfium2==4.30.0
 Requires-Dist: python-calamine>=0.5.3
 Requires-Dist: python-pptx>=1.0.2

kreuzberg-3.17.2/kreuzberg/_language_detection.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+from kreuzberg._types import LanguageDetectionConfig
+from kreuzberg.exceptions import MissingDependencyError
+
+_CACHE_SIZE = 128
+
+
+@lru_cache(maxsize=_CACHE_SIZE)
+def detect_languages(text: str, config: LanguageDetectionConfig | None = None) -> list[str] | None:
+    try:
+        from fast_langdetect import detect  # noqa: PLC0415
+    except ImportError as e:
+        raise MissingDependencyError.create_for_package(
+            dependency_group="langdetect",
+            functionality="language detection",
+            package_name="fast-langdetect",
+        ) from e
+
+    if config is None:
+        config = LanguageDetectionConfig()
+
+    try:
+        # detect always returns a list, use k parameter for multiple languages
+        k = config.top_k if config.multilingual else 1
+        # Use the model from config directly
+        model = config.model
+        results = detect(text, model=model, k=k)
+
+        if results:
+            langs = [result["lang"].lower() for result in results if result.get("lang")]
+            return langs if langs else None
+        return None
+    except Exception:  # noqa: BLE001
+        return None

{kreuzberg-3.17.0 → kreuzberg-3.17.2}/kreuzberg/_types.py

@@ -402,9 +402,12 @@ class ImageOCRConfig(ConfigDict):
 
 @dataclass(unsafe_hash=True, frozen=True, slots=True)
 class LanguageDetectionConfig(ConfigDict):
-    low_memory: bool = True
-    """If True, uses a smaller model (~200MB). If False, uses a larger, more accurate model.
-    Defaults to True for better memory efficiency."""
+    model: Literal["lite", "full", "auto"] = "auto"
+    """Language detection model to use:
+    - 'lite': Smaller, faster model with good accuracy
+    - 'full': Larger model with highest accuracy
+    - 'auto': Automatically choose based on memory availability (default)
+    """
     top_k: int = 3
     """Maximum number of languages to return for multilingual detection."""
     multilingual: bool = False
@@ -412,8 +415,8 @@ class LanguageDetectionConfig(ConfigDict):
     If False, uses single language detection."""
     cache_dir: str | None = None
     """Custom directory for model cache. If None, uses system default."""
-    allow_fallback: bool = True
-    """If True, falls back to small model if large model fails."""
+    low_memory: bool = True
+    """Deprecated. Use 'model' parameter instead. If True, uses 'lite' model."""
 
 
 @dataclass(unsafe_hash=True, frozen=True, slots=True)
@@ -983,8 +986,14 @@ class ExtractionConfig(ConfigDict):
     """Custom entity patterns as a frozenset of (entity_type, regex_pattern) tuples."""
     auto_detect_language: bool = False
     """Whether to automatically detect language and configure OCR accordingly."""
+    language_detection_model: Literal["lite", "full", "auto"] = "auto"
+    """Language detection model to use when auto_detect_language is True.
+    - 'lite': Smaller, faster model with good accuracy
+    - 'full': Larger model with highest accuracy
+    - 'auto': Automatically choose based on memory availability (default)
+    """
     language_detection_config: LanguageDetectionConfig | None = None
-    """Configuration for language detection. If None, uses default settings."""
+    """Configuration for language detection. If None, uses default settings with language_detection_model."""
     spacy_entity_extraction_config: SpacyEntityExtractionConfig | None = None
     """Configuration for spaCy entity extraction. If None, uses default settings."""
     auto_detect_document_type: bool = False

{kreuzberg-3.17.0 → kreuzberg-3.17.2}/kreuzberg/extraction.py

@@ -76,9 +76,16 @@ def _validate_and_post_process_helper(
             result.keywords = None
 
     if config.auto_detect_language:
+        # Use provided config or create one with the model from ExtractionConfig
+        lang_config = config.language_detection_config
+        if lang_config is None:
+            from kreuzberg._types import LanguageDetectionConfig  # noqa: PLC0415
+
+            lang_config = LanguageDetectionConfig(model=config.language_detection_model)
+
         result.detected_languages = detect_languages(
             result.content,
-            config=config.language_detection_config,
+            config=lang_config,
         )
 
     if config.auto_detect_document_type:

{kreuzberg-3.17.0 → kreuzberg-3.17.2}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [ "hatchling" ]
 
 [project]
 name = "kreuzberg"
-version = "3.17.0"
+version = "3.17.2"
 description = "Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats"
 readme = "README.md"
 keywords = [
@@ -60,14 +60,14 @@ dependencies = [
   "anyio>=4.10.0",
   "chardetng-py>=0.3.5",
   "exceptiongroup>=1.2.2; python_version<'3.11'",
-  "html-to-markdown[lxml]>=1.13.0",
+  "html-to-markdown[lxml]>=1.14.0",
   "langcodes>=3.5.0",
-  "mcp>=1.14.0",
+  "mcp>=1.14.1",
   "msgspec>=0.18.0",
   "numpy>=2.0.0",
   "playa-pdf>=0.7.0",
   "polars>=1.33.1",
-  "psutil>=7.0.0",
+  "psutil>=7.1.0",
   "pypdfium2==4.30.0",                                # pinned due to bug in 4.30.1, until v5 is stable
   "python-calamine>=0.5.3",
   "python-pptx>=1.0.2",
@@ -110,7 +110,7 @@ scripts.kreuzberg-mcp = "kreuzberg._mcp.server:main"
 [dependency-groups]
 dev = [
   "covdefaults>=2.3.0",
-  "mypy>=1.18.1",
+  "mypy>=1.18.2",
   "pre-commit>=4.3.0",
   "pytest>=8.4.2",
   "pytest-cov>=7.0.0",
@@ -118,7 +118,7 @@ dev = [
   "pytest-rerunfailures>=16.0.1",
   "pytest-timeout>=2.4.0",
   "rich>=14.1.0",
-  "ruff>=0.13.0",
+  "ruff>=0.13.1",
   "tabulate>=0.9.0",
   "trio>=0.31.0",
   "uv-bump",

kreuzberg-3.17.2/tests/features/language_detection_test.py

@@ -0,0 +1,354 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from unittest.mock import Mock, patch
+
+import pytest
+
+from kreuzberg._language_detection import detect_languages
+from kreuzberg._types import LanguageDetectionConfig
+from kreuzberg.exceptions import MissingDependencyError
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+
+@pytest.fixture(autouse=True)
+def clear_language_detection_cache() -> Generator[None, None, None]:
+    detect_languages.cache_clear()
+    yield
+    detect_languages.cache_clear()
+
+
+def test_detect_languages_when_library_missing() -> None:
+    text = "This is some English text."
+
+    # Mock the import statement inside the function
+    with patch.dict("sys.modules", {"fast_langdetect": None}):
+        with pytest.raises(MissingDependencyError) as exc_info:
+            detect_languages(text)
+
+        error = exc_info.value
+        assert "fast-langdetect" in str(error)
+        assert "language detection" in str(error)
+
+
+def test_detect_languages_single_language_success() -> None:
+    text = "This is some English text."
+    mock_detect_result = [{"lang": "EN", "score": 0.99}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text)
+
+    assert result == ["en"]
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_single_language_no_lang_key() -> None:
+    text = "This is some text."
+    mock_detect_result = [{"score": 0.50}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text)
+
+    assert result is None
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_single_language_empty_lang() -> None:
+    text = "This is some text."
+    mock_detect_result = [{"lang": "", "score": 0.50}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text)
+
+    assert result is None
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_single_language_none_result() -> None:
+    text = "This is some text."
+    mock_detect = Mock(return_value=None)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text)
+
+    assert result is None
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_multilingual_success() -> None:
+    text = "Hello world. Bonjour le monde."
+    config = LanguageDetectionConfig(multilingual=True, top_k=3)
+
+    mock_multilingual_results = [
+        {"lang": "EN", "score": 0.8},
+        {"lang": "FR", "score": 0.7},
+        {"lang": "ES", "score": 0.1},
+    ]
+
+    mock_detect = Mock(return_value=mock_multilingual_results)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result == ["en", "fr", "es"]
+    mock_detect.assert_called_once_with(text, model="auto", k=3)
+
+
+def test_detect_languages_multilingual_with_top_k() -> None:
+    text = "Hello world. Bonjour le monde."
+    config = LanguageDetectionConfig(multilingual=True, top_k=2)
+
+    mock_multilingual_results = [{"lang": "EN", "score": 0.8}, {"lang": "FR", "score": 0.7}]
+
+    mock_detect = Mock(return_value=mock_multilingual_results)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result == ["en", "fr"]
+    mock_detect.assert_called_once_with(text, model="auto", k=2)
+
+
+def test_detect_languages_multilingual_results_missing_lang() -> None:
+    text = "Mixed language text."
+    config = LanguageDetectionConfig(multilingual=True)
+
+    mock_multilingual_results = [
+        {"lang": "EN", "score": 0.8},
+        {"score": 0.6},
+        {"lang": "", "score": 0.4},
+        {"lang": "FR", "score": 0.3},
+    ]
+
+    mock_detect = Mock(return_value=mock_multilingual_results)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result == ["en", "fr"]
+
+
+def test_detect_languages_with_default_config() -> None:
+    text = "This is English text."
+    mock_detect_result = [{"lang": "EN", "score": 0.95}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config=None)
+
+    assert result == ["en"]
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_single_language_with_config() -> None:
+    text = "This is English text."
+    config = LanguageDetectionConfig(multilingual=False)
+    mock_detect_result = [{"lang": "EN", "score": 0.95}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result == ["en"]
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_exception_handling() -> None:
+    text = "This is some text."
+
+    mock_detect = Mock(side_effect=RuntimeError("Detection failed"))
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text)
+
+    assert result is None
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_multilingual_exception_handling() -> None:
+    text = "Mixed language text."
+    config = LanguageDetectionConfig(multilingual=True)
+
+    mock_detect = Mock(side_effect=ValueError("Multilingual detection failed"))
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result is None
+    mock_detect.assert_called_once_with(text, model="auto", k=3)
+
+
+def test_detect_languages_caching_behavior() -> None:
+    text = "This is English text."
+    mock_detect_result = [{"lang": "EN", "score": 0.95}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        config = LanguageDetectionConfig()
+        result1 = detect_languages(text, config)
+        result2 = detect_languages(text, config)
+
+    assert result1 == ["en"]
+    assert result2 == ["en"]
+    mock_detect.assert_called_once_with(text, model="auto", k=1)
+
+
+def test_detect_languages_cache_different_configs() -> None:
+    text = "This is English text."
+    mock_detect_result = [{"lang": "EN", "score": 0.95}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        config1 = LanguageDetectionConfig(multilingual=False)
+        config2 = LanguageDetectionConfig(multilingual=True, top_k=2)
+
+        result1 = detect_languages(text, config1)
+        result2 = detect_languages(text, config2)
+
+    assert result1 == ["en"]
+    assert result2 == ["en"]
+    assert mock_detect.call_count == 2
+    mock_detect.assert_any_call(text, model="auto", k=1)
+    mock_detect.assert_any_call(text, model="auto", k=2)
+
+
+# Real integration tests without mocks
+def test_detect_languages_real_single_language() -> None:
+    text = "This is definitely an English text with multiple sentences. It should be detected as English."
+    result = detect_languages(text)
+
+    assert result is not None
+    assert len(result) == 1
+    assert result[0] == "en"
+
+
+def test_detect_languages_real_multilingual() -> None:
+    # Text with mixed languages
+    text = "Hello world. Bonjour le monde. Hola mundo. Ciao mondo."
+    config = LanguageDetectionConfig(multilingual=True, top_k=4)
+    result = detect_languages(text, config)
+
+    assert result is not None
+    assert len(result) >= 1
+    # The exact languages detected may vary, but we should get at least one
+    assert all(isinstance(lang, str) for lang in result)
+    assert all(len(lang) == 2 for lang in result)  # Language codes should be 2 chars
+
+
+def test_detect_languages_real_empty_text() -> None:
+    text = ""
+    result = detect_languages(text)
+
+    # Empty text should return None or raise an exception (caught and returns None)
+    # Note: fast_langdetect may return a default language for empty text
+    assert result is None or (isinstance(result, list) and len(result) <= 1)
+
+
+def test_detect_languages_real_with_config() -> None:
+    text = "This is an English sentence."
+    config = LanguageDetectionConfig(multilingual=False)
+    result = detect_languages(text, config)
+
+    assert result is not None
+    assert len(result) == 1
+    assert result[0] == "en"
+
+
+def test_detect_languages_real_french_text() -> None:
+    text = "Ceci est un texte en français. Il devrait être détecté comme français."
+    result = detect_languages(text)
+
+    assert result is not None
+    assert len(result) == 1
+    # Note: Model accuracy may vary, checking that we get a valid language code
+    assert isinstance(result[0], str)
+    assert len(result[0]) == 2
+
+
+def test_detect_languages_real_german_text() -> None:
+    text = "Dies ist ein deutscher Text. Es sollte als Deutsch erkannt werden."
+    result = detect_languages(text)
+
+    assert result is not None
+    assert len(result) == 1
+    # Note: Model accuracy may vary, checking that we get a valid language code
+    assert isinstance(result[0], str)
+    assert len(result[0]) == 2
+
+
+def test_detect_languages_real_spanish_text() -> None:
+    text = "Este es un texto en español. Debería ser detectado como español."
+    result = detect_languages(text)
+
+    assert result is not None
+    assert len(result) == 1
+    # Note: Model accuracy may vary, checking that we get a valid language code
+    assert isinstance(result[0], str)
+    assert len(result[0]) == 2
+
+
+def test_detect_languages_real_mixed_languages_with_top_k() -> None:
+    # Text with multiple languages - should detect top languages
+    text = "English text. Texte français. Deutscher Text. Texto español."
+    config = LanguageDetectionConfig(multilingual=True, top_k=2)
+    result = detect_languages(text, config)
+
+    assert result is not None
+    # Should detect at least 1, up to 2 languages
+    assert 1 <= len(result) <= 2
+    assert all(isinstance(lang, str) for lang in result)
+    assert all(len(lang) == 2 for lang in result)
+
+
+def test_detect_languages_with_lite_model() -> None:
+    text = "This is English text."
+    config = LanguageDetectionConfig(model="lite")
+    mock_detect_result = [{"lang": "EN", "score": 0.95}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result == ["en"]
+    mock_detect.assert_called_once_with(text, model="lite", k=1)
+
+
+def test_detect_languages_with_full_model() -> None:
+    text = "This is English text."
+    config = LanguageDetectionConfig(model="full")
+    mock_detect_result = [{"lang": "EN", "score": 0.95}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result == ["en"]
+    mock_detect.assert_called_once_with(text, model="full", k=1)
+
+
+def test_detect_languages_with_auto_model() -> None:
+    text = "This is English text."
+    config = LanguageDetectionConfig(model="auto")
+    mock_detect_result = [{"lang": "EN", "score": 0.95}]
+
+    mock_detect = Mock(return_value=mock_detect_result)
+
+    with patch("fast_langdetect.detect", mock_detect):
+        result = detect_languages(text, config)
+
+    assert result == ["en"]
+    mock_detect.assert_called_once_with(text, model="auto", k=1)