kreuzberg 3.17.1__tar.gz → 3.17.3__tar.gz

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/.github/workflows/ci.yaml

@@ -44,15 +44,21 @@ jobs:
             uv sync --all-extras --dev
           shell: bash
 
-      - name: Load Cached Pre-Commit Dependencies
-        id: cached-pre-commit-dependencies
+      - name: Install prek
+        run: |
+          # Install prek using uv (recommended method)
+          uv tool install prek
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Load Cached Prek Dependencies
+        id: cached-prek-dependencies
         uses: actions/cache@v4
         with:
-          path: ~/.cache/pre-commit/
-          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+          path: ~/.cache/prek/
+          key: prek|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
 
-      - name: Execute Pre-Commit
-        run: uv run pre-commit run --show-diff-on-failure --color=always --all-files
+      - name: Execute Prek
+        run: prek run --show-diff-on-failure --color=always --all-files
 
   coverage:
     needs: validate
@@ -119,7 +125,7 @@ jobs:
           shell: bash
 
       - name: Upload Coverage to DeepSource
-        if: always()
+        if: always() && secrets.DEEPSOURCE_DSN != '' && needs.test-pr.result == 'success'
         env:
           DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
         run: |
@@ -214,7 +220,7 @@ jobs:
 
   coverage-pr:
     needs: test-pr
-    if: github.event_name == 'pull_request' && always()
+    if: github.event_name == 'pull_request' && needs.test-pr.result == 'success'
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -227,7 +233,7 @@ jobs:
           name: coverage-pr-${{ github.sha }}
 
       - name: Upload Coverage to DeepSource
-        if: always()
+        if: always() && secrets.DEEPSOURCE_DSN != '' && needs.test-pr.result == 'success'
         env:
           DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
         run: |

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.17.1
+Version: 3.17.3
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -31,7 +31,7 @@ Requires-Python: >=3.10
 Requires-Dist: anyio>=4.10.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
-Requires-Dist: html-to-markdown[lxml]>=1.13.0
+Requires-Dist: html-to-markdown[lxml]>=1.14.0
 Requires-Dist: langcodes>=3.5.0
 Requires-Dist: mcp>=1.14.1
 Requires-Dist: msgspec>=0.18.0

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/Taskfile.yml

@@ -9,7 +9,7 @@ tasks:
     desc: "Install dependencies with uv"
     cmds:
       - uv sync --all-extras --all-packages
-      - pre-commit install && pre-commit install -hook-type commit-msg
+      - prek install && prek install --hook-type commit-msg
 
   update:
     desc: "Update the dependencies"
@@ -17,7 +17,7 @@ tasks:
       - uv run uv-bump
       - cd benchmarks && uv run uv-bump && cd -
       - uv sync --all-extras --all-packages --upgrade
-      - pre-commit autoupdate
+      - prek autoupdate
 
   test:
     desc: "Run tests with pytest"
@@ -32,7 +32,7 @@ tasks:
   lint:
     desc: "Lint code with ruff and docs with markdownlint"
     cmds:
-      - pre-commit run --all-files
+      - prek run --all-files
 
   docs:build:
     desc: "Build documentation"

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/ai-rulez.yaml

@@ -368,9 +368,10 @@ rules:
       - Fix linting issues: `ruff check --fix`
       - Type check: `mypy`
 
-      ### Pre-commit
-      - Install hooks: `pre-commit install && pre-commit install --hook-type commit-msg`
-      - Run manually: `pre-commit run --all-files`
+      ### Prek
+      - Install prek: `uv tool install prek`
+      - Install hooks: `prek install && prek install --hook-type commit-msg`
+      - Run manually: `prek run --all-files`
 
       ### Documentation
       - Build docs: `uv run mkdocs build --clean --strict`

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/docs/contributing.md

@@ -18,10 +18,14 @@ Thank you for contributing to Kreuzberg!
     uv sync --all-extras --dev
     ```
 
-1. **Install pre-commit hooks**:
+1. **Install prek and hooks**:
 
     ```bash
-    pre-commit install && pre-commit install --hook-type commit-msg
+    # Install prek using uv (recommended)
+    uv tool install prek
+
+    # Install git hooks
+    prek install && prek install --hook-type commit-msg
     ```
 
 ## Development
@@ -42,8 +46,8 @@ uv run ruff check                  # Lint
 uv run ruff check --fix            # Auto-fix issues
 uv run mypy                        # Type check
 
-# Pre-commit
-uv run pre-commit run --all-files  # Run all checks manually
+# Prek
+prek run --all-files  # Run all checks manually
 
 # Documentation
 uv run mkdocs serve                # Serve docs locally
@@ -70,7 +74,7 @@ Use [Conventional Commits](https://www.conventionalcommits.org/):
 
 - Python 3.10-3.13 supported
 - System dependencies (optional): Tesseract, Pandoc
-- Pre-commit runs automatically on commit
+- Prek runs automatically on commit
 - Join our [Discord](https://discord.gg/pXxagNK2zN) for help
 
 ## License

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/kreuzberg/_entity_extraction.py

@@ -2,12 +2,14 @@ from __future__ import annotations
 
 import os
 import re
+import subprocess
+import sys
 from functools import lru_cache
 from itertools import chain
 from typing import TYPE_CHECKING, Any
 
 from kreuzberg._types import Entity, SpacyEntityExtractionConfig
-from kreuzberg.exceptions import MissingDependencyError
+from kreuzberg.exceptions import KreuzbergError, MissingDependencyError
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -49,8 +51,6 @@ def extract_entities(
         return entities
 
     nlp = _load_spacy_model(model_name, spacy_config)
-    if not nlp:
-        return entities
 
     if len(text) > spacy_config.max_doc_length:
         text = text[: spacy_config.max_doc_length]
@@ -77,17 +77,45 @@ def extract_entities(
 def _load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig) -> Any:
     try:
         import spacy  # noqa: PLC0415
+    except ImportError:
+        return None
 
-        if spacy_config.model_cache_dir:
-            os.environ["SPACY_DATA"] = str(spacy_config.model_cache_dir)
+    if spacy_config.model_cache_dir:
+        os.environ["SPACY_DATA"] = str(spacy_config.model_cache_dir)
 
+    try:
         nlp = spacy.load(model_name)
+    except OSError:
+        result = subprocess.run(
+            [sys.executable, "-m", "spacy", "download", model_name],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
 
-        nlp.max_length = spacy_config.max_doc_length
-
-        return nlp
-    except (OSError, ImportError):
-        return None
+        if result.returncode != 0:
+            error_msg = (
+                f"Failed to download spaCy model '{model_name}'. "
+                f"Please install it manually with: python -m spacy download {model_name}"
+            )
+            if result.stderr:
+                error_msg += f"\nError details: {result.stderr}"
+            raise KreuzbergError(
+                error_msg, context={"model": model_name, "stderr": result.stderr, "return_code": result.returncode}
+            ) from None
+
+        try:
+            nlp = spacy.load(model_name)
+        except OSError as e:
+            raise KreuzbergError(
+                f"Failed to load spaCy model '{model_name}' even after successful download. "
+                f"Please verify your spaCy installation and try reinstalling the model.",
+                context={"model": model_name, "error": str(e)},
+            ) from e
+
+    nlp.max_length = spacy_config.max_doc_length
+
+    return nlp
 
 
 def _select_spacy_model(languages: list[str] | None, spacy_config: SpacyEntityExtractionConfig) -> str | None:

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/kreuzberg/_language_detection.py

@@ -23,9 +23,7 @@ def detect_languages(text: str, config: LanguageDetectionConfig | None = None) -
         config = LanguageDetectionConfig()
 
     try:
-        # detect always returns a list, use k parameter for multiple languages
         k = config.top_k if config.multilingual else 1
-        # Use the model from config directly
         model = config.model
         results = detect(text, model=model, k=k)
 

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/kreuzberg/extraction.py

@@ -76,7 +76,6 @@ def _validate_and_post_process_helper(
             result.keywords = None
 
     if config.auto_detect_language:
-        # Use provided config or create one with the model from ExtractionConfig
         lang_config = config.language_detection_config
         if lang_config is None:
             from kreuzberg._types import LanguageDetectionConfig  # noqa: PLC0415

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [ "hatchling" ]
 
 [project]
 name = "kreuzberg"
-version = "3.17.1"
+version = "3.17.3"
 description = "Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats"
 readme = "README.md"
 keywords = [
@@ -60,7 +60,7 @@ dependencies = [
   "anyio>=4.10.0",
   "chardetng-py>=0.3.5",
   "exceptiongroup>=1.2.2; python_version<'3.11'",
-  "html-to-markdown[lxml]>=1.13.0",
+  "html-to-markdown[lxml]>=1.14.0",
   "langcodes>=3.5.0",
   "mcp>=1.14.1",
   "msgspec>=0.18.0",
@@ -111,7 +111,6 @@ scripts.kreuzberg-mcp = "kreuzberg._mcp.server:main"
 dev = [
   "covdefaults>=2.3.0",
   "mypy>=1.18.2",
-  "pre-commit>=4.3.0",
   "pytest>=8.4.2",
   "pytest-cov>=7.0.0",
   "pytest-mock>=3.15.1",

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/tests/features/entity_extraction_test.py

@@ -11,7 +11,7 @@ from kreuzberg._entity_extraction import (
     extract_keywords,
 )
 from kreuzberg._types import SpacyEntityExtractionConfig
-from kreuzberg.exceptions import MissingDependencyError
+from kreuzberg.exceptions import KreuzbergError, MissingDependencyError
 
 
 def test_extract_entities_with_custom_patterns_only() -> None:
@@ -58,11 +58,11 @@ def test_extract_entities_spacy_load_fails() -> None:
 
     with (
         patch("kreuzberg._entity_extraction._select_spacy_model", return_value="en_core_web_sm"),
-        patch("kreuzberg._entity_extraction._load_spacy_model", return_value=None),
+        patch("kreuzberg._entity_extraction._load_spacy_model") as mock_load,
     ):
-        entities = extract_entities(text)
-
-    assert entities == []
+        mock_load.side_effect = KreuzbergError("Model download failed")
+        with pytest.raises(KreuzbergError, match="Model download failed"):
+            extract_entities(text)
 
 
 def test_extract_entities_with_spacy_success() -> None:
@@ -216,26 +216,82 @@ def test_load_spacy_model_with_cache_dir() -> None:
     os.environ.update(original_env)
 
 
-def test_load_spacy_model_failure() -> None:
+def test_load_spacy_model_auto_download_success() -> None:
     config = SpacyEntityExtractionConfig()
 
     mock_spacy = Mock()
-    mock_spacy.load.side_effect = OSError("Model not found")
+    mock_nlp = Mock()
+    mock_spacy.load.side_effect = [OSError("Model not found"), mock_nlp]
 
-    with patch.dict("sys.modules", {"spacy": mock_spacy}):
+    mock_subprocess = Mock()
+    mock_subprocess.run.return_value.returncode = 0
+
+    with (
+        patch.dict("sys.modules", {"spacy": mock_spacy}),
+        patch("subprocess.run", mock_subprocess.run),
+    ):
         _load_spacy_model.cache_clear()
         result = _load_spacy_model("en_core_web_sm", config)
 
-    assert result is None
+    assert result == mock_nlp
+    assert mock_nlp.max_length == config.max_doc_length
+    assert mock_spacy.load.call_count == 2
+    mock_subprocess.run.assert_called_once()
 
+    call_args = mock_subprocess.run.call_args[0][0]
+    assert "-m" in call_args
+    assert "spacy" in call_args
+    assert "download" in call_args
+    assert "en_core_web_sm" in call_args
 
-def test_load_spacy_model_import_error() -> None:
+
+def test_load_spacy_model_download_then_load_failure() -> None:
     config = SpacyEntityExtractionConfig()
 
     mock_spacy = Mock()
-    mock_spacy.load.side_effect = ImportError("Module not found")
+    mock_spacy.load.side_effect = [OSError("Model not found"), OSError("Load failed")]
 
-    with patch.dict("sys.modules", {"spacy": mock_spacy}):
+    mock_subprocess = Mock()
+    mock_subprocess.run.return_value.returncode = 0
+
+    with (
+        patch.dict("sys.modules", {"spacy": mock_spacy}),
+        patch("subprocess.run", mock_subprocess.run),
+    ):
+        _load_spacy_model.cache_clear()
+        with pytest.raises(KreuzbergError, match="Failed to load spaCy model"):
+            _load_spacy_model("en_core_web_sm", config)
+
+    assert mock_spacy.load.call_count == 2
+    mock_subprocess.run.assert_called_once()
+
+
+def test_load_spacy_model_auto_download_failure() -> None:
+    config = SpacyEntityExtractionConfig()
+
+    mock_spacy = Mock()
+    mock_spacy.load.side_effect = OSError("Model not found")
+
+    mock_subprocess = Mock()
+    mock_subprocess.run.return_value.returncode = 1
+    mock_subprocess.run.return_value.stderr = "Download error"
+
+    with (
+        patch.dict("sys.modules", {"spacy": mock_spacy}),
+        patch("subprocess.run", mock_subprocess.run),
+    ):
+        _load_spacy_model.cache_clear()
+        with pytest.raises(KreuzbergError, match="Failed to download spaCy model"):
+            _load_spacy_model("en_core_web_sm", config)
+
+    mock_spacy.load.assert_called_once_with("en_core_web_sm")
+    mock_subprocess.run.assert_called_once()
+
+
+def test_load_spacy_model_import_error() -> None:
+    config = SpacyEntityExtractionConfig()
+
+    with patch.dict("sys.modules", {"spacy": None}):
         _load_spacy_model.cache_clear()
         result = _load_spacy_model("en_core_web_sm", config)
 

{kreuzberg-3.17.1 → kreuzberg-3.17.3}/tests/features/language_detection_test.py

@@ -23,7 +23,6 @@ def clear_language_detection_cache() -> Generator[None, None, None]:
 def test_detect_languages_when_library_missing() -> None:
     text = "This is some English text."
 
-    # Mock the import statement inside the function
     with patch.dict("sys.modules", {"fast_langdetect": None}):
         with pytest.raises(MissingDependencyError) as exc_info:
             detect_languages(text)
@@ -224,7 +223,6 @@ def test_detect_languages_cache_different_configs() -> None:
     mock_detect.assert_any_call(text, model="auto", k=2)
 
 
-# Real integration tests without mocks
 def test_detect_languages_real_single_language() -> None:
     text = "This is definitely an English text with multiple sentences. It should be detected as English."
     result = detect_languages(text)
@@ -235,24 +233,20 @@ def test_detect_languages_real_single_language() -> None:
 
 
 def test_detect_languages_real_multilingual() -> None:
-    # Text with mixed languages
     text = "Hello world. Bonjour le monde. Hola mundo. Ciao mondo."
     config = LanguageDetectionConfig(multilingual=True, top_k=4)
     result = detect_languages(text, config)
 
     assert result is not None
     assert len(result) >= 1
-    # The exact languages detected may vary, but we should get at least one
     assert all(isinstance(lang, str) for lang in result)
-    assert all(len(lang) == 2 for lang in result)  # Language codes should be 2 chars
+    assert all(len(lang) == 2 for lang in result)
 
 
 def test_detect_languages_real_empty_text() -> None:
     text = ""
     result = detect_languages(text)
 
-    # Empty text should return None or raise an exception (caught and returns None)
-    # Note: fast_langdetect may return a default language for empty text
     assert result is None or (isinstance(result, list) and len(result) <= 1)
 
 
@@ -272,7 +266,6 @@ def test_detect_languages_real_french_text() -> None:
 
     assert result is not None
     assert len(result) == 1
-    # Note: Model accuracy may vary, checking that we get a valid language code
     assert isinstance(result[0], str)
     assert len(result[0]) == 2
 
@@ -283,7 +276,6 @@ def test_detect_languages_real_german_text() -> None:
 
     assert result is not None
     assert len(result) == 1
-    # Note: Model accuracy may vary, checking that we get a valid language code
     assert isinstance(result[0], str)
     assert len(result[0]) == 2
 
@@ -294,19 +286,16 @@ def test_detect_languages_real_spanish_text() -> None:
 
     assert result is not None
     assert len(result) == 1
-    # Note: Model accuracy may vary, checking that we get a valid language code
     assert isinstance(result[0], str)
     assert len(result[0]) == 2
 
 
 def test_detect_languages_real_mixed_languages_with_top_k() -> None:
-    # Text with multiple languages - should detect top languages
     text = "English text. Texte français. Deutscher Text. Texto español."
     config = LanguageDetectionConfig(multilingual=True, top_k=2)
     result = detect_languages(text, config)
 
     assert result is not None
-    # Should detect at least 1, up to 2 languages
     assert 1 <= len(result) <= 2
     assert all(isinstance(lang, str) for lang in result)
     assert all(len(lang) == 2 for lang in result)