kreuzberg 3.17.2__tar.gz → 3.18.0__tar.gz

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/.github/workflows/ci.yaml

@@ -44,15 +44,21 @@ jobs:
             uv sync --all-extras --dev
           shell: bash
 
-      - name: Load Cached Pre-Commit Dependencies
-        id: cached-pre-commit-dependencies
+      - name: Install prek
+        run: |
+          # Install prek using uv (recommended method)
+          uv tool install prek
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Load Cached Prek Dependencies
+        id: cached-prek-dependencies
         uses: actions/cache@v4
         with:
-          path: ~/.cache/pre-commit/
-          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+          path: ~/.cache/prek/
+          key: prek|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
 
-      - name: Execute Pre-Commit
-        run: uv run pre-commit run --show-diff-on-failure --color=always --all-files
+      - name: Execute Prek
+        run: prek run --show-diff-on-failure --color=always --all-files
 
   coverage:
     needs: validate
@@ -119,7 +125,7 @@ jobs:
           shell: bash
 
       - name: Upload Coverage to DeepSource
-        if: always()
+        if: always() && secrets.DEEPSOURCE_DSN != '' && needs.test-pr.result == 'success'
         env:
           DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
         run: |
@@ -214,7 +220,7 @@ jobs:
 
   coverage-pr:
     needs: test-pr
-    if: github.event_name == 'pull_request' && always()
+    if: github.event_name == 'pull_request' && needs.test-pr.result == 'success'
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -227,7 +233,7 @@ jobs:
           name: coverage-pr-${{ github.sha }}
 
       - name: Upload Coverage to DeepSource
-        if: always()
+        if: always() && secrets.DEEPSOURCE_DSN != '' && needs.test-pr.result == 'success'
         env:
           DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
         run: |

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
     hooks:
       - id: pyproject-fmt
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.1
+    rev: v0.13.2
     hooks:
       - id: ruff
         args: ["--fix", "--unsafe-fixes"]

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.17.2
+Version: 3.18.0
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -28,12 +28,12 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Text Processing :: General
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: anyio>=4.10.0
+Requires-Dist: anyio>=4.11.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
-Requires-Dist: html-to-markdown[lxml]>=1.14.0
+Requires-Dist: html-to-markdown[lxml]>=1.16.0
 Requires-Dist: langcodes>=3.5.0
-Requires-Dist: mcp>=1.14.1
+Requires-Dist: mcp>=1.15.0
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: playa-pdf>=0.7.0

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/Taskfile.yml

@@ -9,7 +9,7 @@ tasks:
     desc: "Install dependencies with uv"
     cmds:
       - uv sync --all-extras --all-packages
-      - pre-commit install && pre-commit install -hook-type commit-msg
+      - prek install && prek install --hook-type commit-msg
 
   update:
     desc: "Update the dependencies"
@@ -17,7 +17,7 @@ tasks:
       - uv run uv-bump
       - cd benchmarks && uv run uv-bump && cd -
       - uv sync --all-extras --all-packages --upgrade
-      - pre-commit autoupdate
+      - prek autoupdate
 
   test:
     desc: "Run tests with pytest"
@@ -32,7 +32,7 @@ tasks:
   lint:
     desc: "Lint code with ruff and docs with markdownlint"
     cmds:
-      - pre-commit run --all-files
+      - prek run --all-files
 
   docs:build:
     desc: "Build documentation"

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/ai-rulez.yaml

@@ -368,9 +368,10 @@ rules:
       - Fix linting issues: `ruff check --fix`
       - Type check: `mypy`
 
-      ### Pre-commit
-      - Install hooks: `pre-commit install && pre-commit install --hook-type commit-msg`
-      - Run manually: `pre-commit run --all-files`
+      ### Prek
+      - Install prek: `uv tool install prek`
+      - Install hooks: `prek install && prek install --hook-type commit-msg`
+      - Run manually: `prek run --all-files`
 
       ### Documentation
       - Build docs: `uv run mkdocs build --clean --strict`

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/docs/contributing.md

@@ -18,10 +18,14 @@ Thank you for contributing to Kreuzberg!
     uv sync --all-extras --dev
     ```
 
-1. **Install pre-commit hooks**:
+1. **Install prek and hooks**:
 
     ```bash
-    pre-commit install && pre-commit install --hook-type commit-msg
+    # Install prek using uv (recommended)
+    uv tool install prek
+
+    # Install git hooks
+    prek install && prek install --hook-type commit-msg
     ```
 
 ## Development
@@ -42,8 +46,8 @@ uv run ruff check                  # Lint
 uv run ruff check --fix            # Auto-fix issues
 uv run mypy                        # Type check
 
-# Pre-commit
-uv run pre-commit run --all-files  # Run all checks manually
+# Prek
+prek run --all-files  # Run all checks manually
 
 # Documentation
 uv run mkdocs serve                # Serve docs locally
@@ -70,7 +74,7 @@ Use [Conventional Commits](https://www.conventionalcommits.org/):
 
 - Python 3.10-3.13 supported
 - System dependencies (optional): Tesseract, Pandoc
-- Pre-commit runs automatically on commit
+- Prek runs automatically on commit
 - Join our [Discord](https://discord.gg/pXxagNK2zN) for help
 
 ## License

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/docs/user-guide/api-server.md

@@ -62,7 +62,7 @@ Extract text from one or more files.
 - Method: `POST`
 - Content-Type: `multipart/form-data`
 - Body: One or more files with field name `data`
-- **Maximum file size: 1GB per file**
+- **Maximum file size: Configurable via `KREUZBERG_MAX_UPLOAD_SIZE` environment variable (default: 1GB per file)**
 
 **Response:**
 
@@ -463,6 +463,37 @@ The API server uses the default Kreuzberg extraction configuration:
 - PDF, image, and document extraction is supported
 - Table extraction with GMFT (if installed)
 
+### Environment Variables
+
+The API server can be configured using environment variables for production deployments:
+
+#### Server Configuration
+
+| Variable                         | Description                  | Default            | Example            |
+| -------------------------------- | ---------------------------- | ------------------ | ------------------ |
+| `KREUZBERG_MAX_UPLOAD_SIZE`      | Maximum upload size in bytes | `1073741824` (1GB) | `2147483648` (2GB) |
+| `KREUZBERG_ENABLE_OPENTELEMETRY` | Enable OpenTelemetry tracing | `true`             | `false`            |
+
+#### Usage Examples
+
+```bash
+# Set 2GB upload limit
+export KREUZBERG_MAX_UPLOAD_SIZE=2147483648
+litestar --app kreuzberg._api.main:app run
+
+# Disable telemetry
+export KREUZBERG_ENABLE_OPENTELEMETRY=false
+uvicorn kreuzberg._api.main:app --host 0.0.0.0 --port 8000
+
+# Production settings with Docker
+docker run -p 8000:8000 \
+  -e KREUZBERG_MAX_UPLOAD_SIZE=5368709120 \
+  -e KREUZBERG_ENABLE_OPENTELEMETRY=true \
+  goldziher/kreuzberg:latest
+```
+
+**Note**: Boolean environment variables accept `true`/`false`, `1`/`0`, `yes`/`no`, or `on`/`off` values.
+
 To use custom configuration, modify the extraction call in your own API wrapper:
 
 ```python

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/kreuzberg/_api/main.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import base64
 import io
+import os
 import traceback
 from json import dumps
 from typing import TYPE_CHECKING, Annotated, Any, Literal
@@ -100,6 +101,36 @@ def exception_handler(request: Request[Any, Any, Any], exception: KreuzbergError
     )
 
 
+def _get_max_upload_size() -> int:
+    """Get the maximum upload size from environment variable.
+
+    Returns:
+        Maximum upload size in bytes. Defaults to 1GB if not set.
+
+    Environment Variables:
+        KREUZBERG_MAX_UPLOAD_SIZE: Maximum upload size in bytes (default: 1073741824 = 1GB)
+    """
+    default_size = 1024 * 1024 * 1024  # 1GB
+    try:
+        size = int(os.environ.get("KREUZBERG_MAX_UPLOAD_SIZE", default_size))
+        # Return default if negative
+        return size if size >= 0 else default_size
+    except ValueError:
+        return default_size
+
+
+def _is_opentelemetry_enabled() -> bool:
+    """Check if OpenTelemetry should be enabled.
+
+    Returns:
+        True if OpenTelemetry should be enabled, False otherwise.
+
+    Environment Variables:
+        KREUZBERG_ENABLE_OPENTELEMETRY: Enable OpenTelemetry tracing (true/false) (default: true)
+    """
+    return os.environ.get("KREUZBERG_ENABLE_OPENTELEMETRY", "true").lower() in ("true", "1", "yes", "on")
+
+
 def general_exception_handler(request: Request[Any, Any, Any], exception: Exception) -> Response[Any]:
     error_type = type(exception).__name__
     error_message = str(exception)
@@ -242,7 +273,7 @@ async def handle_files_upload(  # noqa: PLR0913
     - Language detection (if enabled)
 
     Supports various file formats including PDF, Office documents, images, and more.
-    Maximum file size: 1GB per file.
+    Maximum file size: Configurable via KREUZBERG_MAX_UPLOAD_SIZE environment variable (default: 1GB per file).
 
     Args:
         request: The HTTP request object
@@ -379,9 +410,18 @@ type_encoders = {
     Image.Image: _pil_image_encoder,
 }
 
+
+def _get_plugins() -> list[Any]:
+    """Get configured plugins based on environment variables."""
+    plugins = []
+    if _is_opentelemetry_enabled():
+        plugins.append(OpenTelemetryPlugin(OpenTelemetryConfig()))
+    return plugins
+
+
 app = Litestar(
     route_handlers=[handle_files_upload, health_check, get_configuration],
-    plugins=[OpenTelemetryPlugin(OpenTelemetryConfig())],
+    plugins=_get_plugins(),
     logging_config=StructLoggingConfig(),
     openapi_config=openapi_config,
     exception_handlers={
@@ -389,5 +429,5 @@ app = Litestar(
         Exception: general_exception_handler,
     },
     type_encoders=type_encoders,
-    request_max_body_size=1024 * 1024 * 1024,
+    request_max_body_size=_get_max_upload_size(),
 )

kreuzberg-3.18.0/kreuzberg/_entity_extraction.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+from functools import lru_cache
+from itertools import chain
+from typing import TYPE_CHECKING, Any
+
+import anyio
+
+from kreuzberg._types import Entity, SpacyEntityExtractionConfig
+from kreuzberg._utils._sync import run_sync
+from kreuzberg.exceptions import KreuzbergError, MissingDependencyError
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+def is_uv_available() -> bool:
+    """Check if uv is available in the environment."""
+    return shutil.which("uv") is not None
+
+
+def get_spacy_model_url(model_name: str, version: str = "3.8.0") -> str:
+    """Get the direct download URL for a spaCy model.
+
+    Args:
+        model_name: Name of the spaCy model (e.g., 'en_core_web_sm')
+        version: Model version to download (default: 3.8.0)
+
+    Returns:
+        Direct download URL for the model
+    """
+    return f"https://github.com/explosion/spacy-models/releases/download/{model_name}-{version}/{model_name}-{version}-py3-none-any.whl"
+
+
+async def install_spacy_model_with_uv(model_name: str) -> subprocess.CompletedProcess[str]:
+    """Install spaCy model using uv.
+
+    Args:
+        model_name: Name of the spaCy model to install
+
+    Returns:
+        Completed process result
+    """
+    model_url = get_spacy_model_url(model_name)
+    return await run_sync(
+        subprocess.run,
+        ["uv", "pip", "install", model_url],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+async def install_spacy_model_with_spacy(model_name: str) -> bool:
+    """Install spaCy model using spacy download function.
+
+    Args:
+        model_name: Name of the spaCy model to install
+
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        import spacy.cli.download  # noqa: PLC0415
+
+        await run_sync(spacy.cli.download, model_name)  # type: ignore[attr-defined]
+        return True
+    except (ImportError, OSError, RuntimeError):
+        return False
+
+
+def extract_entities(
+    text: str,
+    entity_types: Sequence[str] = ("PERSON", "ORGANIZATION", "LOCATION", "DATE", "EMAIL", "PHONE"),
+    custom_patterns: frozenset[tuple[str, str]] | None = None,
+    languages: list[str] | None = None,
+    spacy_config: SpacyEntityExtractionConfig | None = None,
+) -> list[Entity]:
+    entities: list[Entity] = []
+    if custom_patterns:
+        entities.extend(
+            chain.from_iterable(
+                (
+                    Entity(type=ent_type, text=match.group(), start=match.start(), end=match.end())
+                    for match in re.finditer(pattern, text)
+                )
+                for ent_type, pattern in custom_patterns
+            )
+        )
+
+    if spacy_config is None:
+        spacy_config = SpacyEntityExtractionConfig()
+
+    try:
+        import spacy  # noqa: F401, PLC0415
+    except ImportError as e:  # pragma: no cover
+        raise MissingDependencyError.create_for_package(
+            package_name="spacy",
+            dependency_group="entity-extraction",
+            functionality="Entity Extraction",
+        ) from e
+
+    model_name = select_spacy_model(languages, spacy_config)
+    if not model_name:
+        return entities
+
+    nlp = load_spacy_model(model_name, spacy_config)
+
+    if len(text) > spacy_config.max_doc_length:
+        text = text[: spacy_config.max_doc_length]
+
+    doc = nlp(text)
+
+    entity_type_mapping = {etype.upper() for etype in entity_types}
+
+    entities.extend(
+        Entity(
+            type=ent.label_,
+            text=ent.text,
+            start=ent.start_char,
+            end=ent.end_char,
+        )
+        for ent in doc.ents
+        if ent.label_ in entity_type_mapping or ent.label_.upper() in entity_type_mapping
+    )
+
+    return entities
+
+
+@lru_cache(maxsize=32)
+def load_spacy_model(model_name: str, spacy_config: SpacyEntityExtractionConfig) -> Any:
+    try:
+        import spacy  # noqa: PLC0415
+    except ImportError:
+        return None
+
+    if spacy_config.model_cache_dir:
+        os.environ["SPACY_DATA"] = str(spacy_config.model_cache_dir)
+
+    try:
+        nlp = spacy.load(model_name)
+    except OSError:
+        # Try to download the model automatically
+        async def install_model() -> tuple[bool, str | None]:
+            """Install model and return success status and error message."""
+            # First try spaCy's built-in download
+            try:
+                success = await install_spacy_model_with_spacy(model_name)
+                if success:
+                    return True, None
+            except (ImportError, OSError, RuntimeError) as e:
+                spacy_error = str(e)
+            else:
+                spacy_error = "spaCy download failed"
+
+            # If spaCy download failed and uv is available, try uv as fallback
+            if is_uv_available():
+                try:
+                    result = await install_spacy_model_with_uv(model_name)
+                    return result.returncode == 0, result.stderr
+                except (OSError, subprocess.SubprocessError) as e:
+                    return False, f"spaCy: {spacy_error}, uv: {e!s}"
+
+            return False, spacy_error
+
+        # Run the async installation in a sync context
+        try:
+            success, error_details = anyio.run(install_model)
+        except (OSError, RuntimeError) as e:
+            success, error_details = False, str(e)
+
+        if not success:
+            # Generate appropriate error message based on available tools
+            if is_uv_available():
+                model_url = get_spacy_model_url(model_name)
+                manual_install_cmd = f"uv pip install {model_url}"
+            else:
+                manual_install_cmd = f"python -m spacy download {model_name}"
+
+            error_msg = (
+                f"Failed to download spaCy model '{model_name}'. Please install it manually with: {manual_install_cmd}"
+            )
+
+            if error_details:
+                error_msg += f"\nError details: {error_details}"
+
+            raise KreuzbergError(
+                error_msg,
+                context={
+                    "model": model_name,
+                    "manual_install_cmd": manual_install_cmd,
+                    "error_details": error_details,
+                    "uv_available": is_uv_available(),
+                },
+            ) from None
+
+        try:
+            nlp = spacy.load(model_name)
+        except OSError as e:
+            raise KreuzbergError(
+                f"Failed to load spaCy model '{model_name}' even after successful download. "
+                f"Please verify your spaCy installation and try reinstalling the model.",
+                context={"model": model_name, "error": str(e)},
+            ) from e
+
+    nlp.max_length = spacy_config.max_doc_length
+
+    return nlp
+
+
+def select_spacy_model(languages: list[str] | None, spacy_config: SpacyEntityExtractionConfig) -> str | None:
+    if not languages:
+        return spacy_config.get_model_for_language("en")
+
+    for lang in languages:
+        model_name = spacy_config.get_model_for_language(lang)
+        if model_name:
+            return model_name
+
+    return spacy_config.get_fallback_model()
+
+
+def extract_keywords(
+    text: str,
+    keyword_count: int = 10,
+) -> list[tuple[str, float]]:
+    try:
+        from keybert import KeyBERT  # noqa: PLC0415
+
+        kw_model = KeyBERT()
+        keywords = kw_model.extract_keywords(text, top_n=keyword_count)
+        return [(kw, float(score)) for kw, score in keywords]
+    except (RuntimeError, OSError, ValueError):
+        return []
+    except ImportError as e:  # pragma: no cover
+        raise MissingDependencyError.create_for_package(
+            package_name="keybert",
+            dependency_group="entity-extraction",
+            functionality="Keyword Extraction",
+        ) from e

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/kreuzberg/_language_detection.py

@@ -23,9 +23,7 @@ def detect_languages(text: str, config: LanguageDetectionConfig | None = None) -
         config = LanguageDetectionConfig()
 
     try:
-        # detect always returns a list, use k parameter for multiple languages
         k = config.top_k if config.multilingual else 1
-        # Use the model from config directly
         model = config.model
         results = detect(text, model=model, k=k)
 

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/kreuzberg/extraction.py

@@ -76,7 +76,6 @@ def _validate_and_post_process_helper(
             result.keywords = None
 
     if config.auto_detect_language:
-        # Use provided config or create one with the model from ExtractionConfig
         lang_config = config.language_detection_config
         if lang_config is None:
             from kreuzberg._types import LanguageDetectionConfig  # noqa: PLC0415

{kreuzberg-3.17.2 → kreuzberg-3.18.0}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [ "hatchling" ]
 
 [project]
 name = "kreuzberg"
-version = "3.17.2"
+version = "3.18.0"
 description = "Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats"
 readme = "README.md"
 keywords = [
@@ -57,12 +57,12 @@ classifiers = [
 ]
 
 dependencies = [
-  "anyio>=4.10.0",
+  "anyio>=4.11.0",
   "chardetng-py>=0.3.5",
   "exceptiongroup>=1.2.2; python_version<'3.11'",
-  "html-to-markdown[lxml]>=1.14.0",
+  "html-to-markdown[lxml]>=1.16.0",
   "langcodes>=3.5.0",
-  "mcp>=1.14.1",
+  "mcp>=1.15.0",
   "msgspec>=0.18.0",
   "numpy>=2.0.0",
   "playa-pdf>=0.7.0",
@@ -111,14 +111,13 @@ scripts.kreuzberg-mcp = "kreuzberg._mcp.server:main"
 dev = [
   "covdefaults>=2.3.0",
   "mypy>=1.18.2",
-  "pre-commit>=4.3.0",
   "pytest>=8.4.2",
   "pytest-cov>=7.0.0",
   "pytest-mock>=3.15.1",
   "pytest-rerunfailures>=16.0.1",
   "pytest-timeout>=2.4.0",
   "rich>=14.1.0",
-  "ruff>=0.13.1",
+  "ruff>=0.13.2",
   "tabulate>=0.9.0",
   "trio>=0.31.0",
   "uv-bump",