kreuzberg 3.13.3__tar.gz → 3.14.1__tar.gz

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/.github/workflows/ci.yaml

@@ -7,7 +7,6 @@ on:
   push:
     branches:
       - main
-      - feat/smart-multiprocessing
 
 jobs:
   validate:
@@ -38,7 +37,7 @@ jobs:
               echo "Removing existing .venv directory on Windows"
               rm -rf .venv
             fi
-            uv sync --all-packages --all-extras --dev
+            uv sync --all-extras --dev
           shell: bash
 
       - name: Load Cached Pre-Commit Dependencies
@@ -53,6 +52,7 @@ jobs:
 
   coverage:
     needs: validate
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
@@ -88,7 +88,7 @@ jobs:
           max_attempts: 3
           retry_wait_seconds: 30
           command: |
-            uv sync --all-packages --all-extras --dev
+            uv sync --all-extras --dev
           shell: bash
 
       - name: Install System Dependencies
@@ -115,7 +115,7 @@ jobs:
           shell: bash
 
       - name: Upload Coverage to DeepSource
-        if: always() && github.event_name == 'push'
+        if: always()
         env:
           DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
         run: |
@@ -134,15 +134,178 @@ jobs:
             .coverage
           retention-days: 7
 
-  test:
-    needs: coverage
+  test-pr:
+    needs: validate
+    if: github.event_name == 'pull_request' && needs.validate.result == 'success'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test-category:
+          - name: "core"
+            path: "tests/core,tests/utils"
+            system-deps: false
+            timeout: 15
+          - name: "extractors"
+            path: "tests/extractors"
+            system-deps: true
+            timeout: 20
+          - name: "integration"
+            path: "tests/integration,tests/api"
+            system-deps: true
+            timeout: 25
+          - name: "features"
+            path: "tests/features,tests/interfaces,tests/mcp,tests/multiprocessing,tests/ocr"
+            system-deps: true
+            timeout: 20
+    timeout-minutes: ${{ matrix.test-category.timeout }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - name: Cache Python Dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: python-dependencies-ubuntu-latest-3.13-${{ matrix.test-category.name }}-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            python-dependencies-ubuntu-latest-3.13-
+
+      - name: Install Dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Install System Dependencies
+        if: matrix.test-category.system-deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr tesseract-ocr-deu pandoc
+
+      - name: Run Tests - ${{ matrix.test-category.name }}
+        run: uv run pytest $(echo "${{ matrix.test-category.path }}" | tr ',' ' ') -v --reruns 1 --reruns-delay 1 --cov=kreuzberg --cov-append --cov-report=lcov:coverage-${{ matrix.test-category.name }}.lcov
+
+      - name: Upload Coverage Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.test-category.name }}-${{ github.sha }}
+          path: coverage-${{ matrix.test-category.name }}.lcov
+          retention-days: 1
+
+  coverage-pr:
+    needs: test-pr
+    if: github.event_name == 'pull_request' && always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Download Coverage Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*-${{ github.sha }}
+          merge-multiple: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - name: Install Dependencies
+        run: uv sync --dev
+
+      - name: Combine Coverage Reports
+        run: |
+          # Install lcov for combining reports
+          sudo apt-get update && sudo apt-get install -y lcov
+
+          # List available coverage files
+          echo "Available coverage files:"
+          find . -name "coverage-*.lcov" -type f || echo "No coverage files found"
+
+          # Combine all lcov files if they exist
+          coverage_files=($(find . -name "coverage-*.lcov" -type f))
+          if [ ${#coverage_files[@]} -gt 0 ]; then
+            echo "Combining ${#coverage_files[@]} coverage files..."
+            if [ ${#coverage_files[@]} -eq 1 ]; then
+              # Only one file, just copy it
+              cp "${coverage_files[0]}" coverage.lcov
+            else
+              # Multiple files, combine them
+              lcov --rc branch_coverage=1 $(printf " -a %s" "${coverage_files[@]}") -o coverage.lcov
+            fi
+          else
+            echo "No coverage files to combine, creating empty coverage.lcov"
+            echo "TN:" > coverage.lcov
+            echo "end_of_record" >> coverage.lcov
+          fi
+
+      - name: Upload Coverage to DeepSource
+        if: always()
+        env:
+          DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
+        run: |
+          # Install DeepSource CLI
+          curl -fsSL https://deepsource.io/cli | sh
+          # Upload coverage report
+          ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.lcov
+
+  test-full:
+    needs: validate
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.validate.result == 'success'
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python: ["3.10", "3.11", "3.12", "3.13"]
-    timeout-minutes: 120
+        test-category:
+          - name: "core"
+            path: "tests/core,tests/utils"
+            system-deps: false
+            timeout: 20
+          - name: "extractors"
+            path: "tests/extractors"
+            system-deps: true
+            timeout: 25
+          - name: "integration"
+            path: "tests/integration,tests/api"
+            system-deps: true
+            timeout: 30
+          - name: "features"
+            path: "tests/features,tests/interfaces,tests/mcp,tests/multiprocessing,tests/ocr"
+            system-deps: true
+            timeout: 25
+        exclude:
+          - test-category: {name: "extractors"}
+            python: "3.11"
+          - test-category: {name: "extractors"}
+            python: "3.12"
+          - test-category: {name: "integration"}
+            python: "3.11"
+          - test-category: {name: "integration"}
+            python: "3.12"
+          - test-category: {name: "features"}
+            python: "3.11"
+          - test-category: {name: "features"}
+            python: "3.12"
+    timeout-minutes: ${{ matrix.test-category.timeout }}
     steps:
       - name: Checkout
         uses: actions/checkout@v5
@@ -180,7 +343,7 @@ jobs:
               echo "Removing existing .venv directory on Windows"
               rm -rf .venv
             fi
-            uv sync --all-packages --all-extras --dev
+            uv sync --all-extras --dev
           shell: bash
 
       - name: Cache Test Artifacts
@@ -190,7 +353,7 @@ jobs:
           key: pytest-cache-${{ matrix.os }}-${{ matrix.python }}
 
       - name: Cache and Install Homebrew (macOS)
-        if: runner.os == 'macOS'
+        if: runner.os == 'macOS' && matrix.test-category.system-deps
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 10
@@ -204,7 +367,7 @@ jobs:
           shell: bash
 
       - name: Cache and Install APT Packages (Linux)
-        if: runner.os == 'Linux'
+        if: runner.os == 'Linux' && matrix.test-category.system-deps
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 5
@@ -216,7 +379,7 @@ jobs:
           shell: bash
 
       - name: Install System Dependencies (Windows)
-        if: runner.os == 'Windows'
+        if: runner.os == 'Windows' && matrix.test-category.system-deps
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 10
@@ -231,12 +394,12 @@ jobs:
             pandoc --version
           shell: pwsh
 
-      - name: Run Tests (without coverage)
+      - name: Run Tests - ${{ matrix.test-category.name }}
         uses: nick-fields/retry@v3
         with:
-          timeout_minutes: 15
-          max_attempts: 3
-          retry_wait_seconds: 10
+          timeout_minutes: 10
+          max_attempts: 2
+          retry_wait_seconds: 5
           command: |
-            uv run pytest -s -vvv --reruns 2 --reruns-delay 1
+            uv run pytest $(echo "${{ matrix.test-category.path }}" | tr ',' ' ') -v --reruns 1 --reruns-delay 1
           shell: bash

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/.github/workflows/docker-e2e-tests.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   test-docker-images:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 360
     strategy:
       matrix:
         image:

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/.github/workflows/publish-docker.yml

@@ -28,6 +28,7 @@ jobs:
     needs: test-images
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'release' }}
     runs-on: ubuntu-latest
+    timeout-minutes: 360
     permissions:
       contents: read
       packages: write

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/.pre-commit-config.yaml

@@ -5,12 +5,6 @@ repos:
       - id: commitlint
         stages: [commit-msg]
         additional_dependencies: ["@commitlint/config-conventional"]
-  # Temporarily disabled due to CI environment issues
-  # - repo: https://github.com/Goldziher/ai-rulez
-  #   rev: v2.0.1
-  #   hooks:
-  #     - id: ai-rulez-validate
-  #     - id: ai-rulez-generate
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
     hooks:
@@ -54,7 +48,7 @@ repos:
     hooks:
       - id: pyproject-fmt
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.12
+    rev: v0.13.0
     hooks:
       - id: ruff
         args: ["--fix", "--unsafe-fixes"]

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.13.3
+Version: 3.14.1
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -31,10 +31,10 @@ Requires-Python: >=3.10
 Requires-Dist: anyio>=4.10.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
-Requires-Dist: html-to-markdown[lxml]>=1.10.0
-Requires-Dist: mcp>=1.13.0
+Requires-Dist: html-to-markdown[lxml]>=1.11.0
+Requires-Dist: mcp>=1.14.0
 Requires-Dist: msgspec>=0.18.0
-Requires-Dist: numpy>=1.24.0
+Requires-Dist: numpy>=2.0.0
 Requires-Dist: playa-pdf>=0.7.0
 Requires-Dist: polars>=1.33.1
 Requires-Dist: psutil>=7.0.0

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/ai-rulez.yaml

@@ -388,6 +388,18 @@ rules:
       - When committing, always use the format specified in the CLAUDE.md
     name: Important Instructions
     priority: critical
+  - content: |
+      ### Docstrings and Comments Guidelines
+      - **NO docstrings in private functions** (functions starting with `_`)
+      - **NO docstrings in private files** (files starting with `_`)
+      - **NO docstrings in private folders** (folders starting with `_`)
+      - **NO docstrings in test files** (files in `tests/` directory)
+      - **ONLY use docstrings in public API** (what's documented in API reference)
+      - **NO redundant comments** - code should be self-documenting
+      - **Comments only when necessary** to explain complex logic or non-obvious decisions
+      - **Prefer clear variable/function names** over comments
+    name: Documentation Standards
+    priority: critical
   - content: |
       ### Core Extraction Flow
       1. **Entry Point**: `extraction.py` provides main functions (`extract_file`, `extract_bytes`, etc.)
@@ -425,14 +437,68 @@ rules:
     name: Error Handling
     priority: medium
   - content: |
-      - Test files in `tests/test_source_files/` for various formats
-      - Mock OCR responses for predictable testing
-      - Both sync and async test variants
-      - Comprehensive error case coverage
-      - OCR tests marked as `xfail` in CI environments for resilience
-      - Integration tests use timeouts and retry logic where appropriate
+      ### File Organization and Naming
+      - **Test files MUST end with `_test.py`** - No exceptions
+      - **Logical directory structure**:
+        ```
+        tests/
+        ├── core/           # Core functionality (extraction, config, types)
+        ├── features/       # Feature-specific (chunking, language detection)
+        ├── integration/    # Integration tests (API, real file processing)
+        ├── interfaces/     # User interfaces (CLI, MCP)
+        ├── extractors/     # File format extractors (PDF, image, etc.)
+        ├── ocr/           # OCR backend tests
+        ├── utils/         # Utility function tests
+        └── e2e/           # End-to-end tests
+        ```
+
+      ### Test Structure and Patterns
+      - **Only function-based tests** - No class-based test methods
+      - **Test naming**: `test_<function>_<scenario>_<expected_outcome>`
+        - ✅ `test_extract_pdf_with_ocr_returns_text()`
+        - ✅ `test_extract_file_raises_validation_error_when_file_missing()`
+        - ❌ `test_basic_extraction()` (too vague)
+      - **Async/sync variants**: Test both async and sync versions where applicable
+      - **Parameterized tests**: Use `@pytest.mark.parametrize` for multiple scenarios
+
+      ### Mocking Guidelines
+      - **NEVER mock anyio/asyncio** - Only mock external dependencies
+      - **Mock external services**: OCR engines, file system operations, network calls
+      - **Use real objects when possible**: Prefer `tmp_path` over mocking file operations
+      - **Mock sparingly**: Only when necessary for isolation or performance
+      - **Add comments for legitimate mocks**: Explain why mocking is required
+        ```python
+        # Mock OCR backend for predictable testing ~keep
+        mock_ocr = mocker.patch("kreuzberg._ocr.get_backend")
+        ```
+
+      ### Fixtures and Test Data
+      - **Session-scoped fixtures** for stateless objects (extractors, configs)
+      - **Shared fixtures** in `tests/conftest.py` for common test data
+      - **Test files** in `tests/test_source_files/` for various formats
+      - **Temporary files** using pytest's `tmp_path` fixture
+
+      ### Test Coverage and Quality
+      - **95% minimum coverage** requirement
+      - **Test all error paths** - Every exception should be tested
+      - **Edge cases**: Empty inputs, large files, malformed data
+      - **Performance considerations**: Mark slow tests appropriately
+      - **CI resilience**: OCR tests marked as `xfail` in CI environments
+
+      ### Test Helpers and Utilities
+      - **Shared assertions** in `tests/extractors/test_helpers.py`:
+        - `assert_valid_extraction_result()` - Standard result validation
+        - `assert_extraction_error()` - Error case validation
+        - `assert_ocr_result()` - OCR-specific validation
+      - **Avoid repetitive assertions** - Use helper functions
+
+      ### Integration vs Unit Tests
+      - **Unit tests**: Fast, isolated, mock external dependencies
+      - **Integration tests**: Real file processing, external services, end-to-end flows
+      - **Separation**: Integration tests in `tests/integration/` directory
+      - **Timeouts**: Integration tests use timeouts and retry logic
     name: Testing Patterns
-    priority: low
+    priority: critical
   - content: |
       ### GitHub Actions Workflows
       - **Release**: Automated PyPI publishing via GitHub releases, triggers Docker builds

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/docs/examples/extraction-examples.md

@@ -135,7 +135,7 @@ async def extract_tables_from_pdf():
         print(f"Table {i+1} on page {table['page_number']}:")
         print(table["text"])  # Markdown formatted table
 
-        # Work with the pandas DataFrame
+        # Work with the polars DataFrame
         df = table["df"]
         print(f"Table shape: {df.shape}")
 

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/docs/user-guide/api-server.md

@@ -62,6 +62,7 @@ Extract text from one or more files.
 - Method: `POST`
 - Content-Type: `multipart/form-data`
 - Body: One or more files with field name `data`
+- **Maximum file size: 1GB per file**
 
 **Response:**
 
@@ -222,6 +223,42 @@ curl -X POST "http://localhost:8000/extract?max_chars=1000" \
 
 Result: max_chars will be 500 (from header)
 
+## Interactive API Documentation
+
+Kreuzberg automatically generates comprehensive OpenAPI documentation that you can access through your web browser when the API server is running.
+
+### Accessing the Documentation
+
+Once the API server is running, you can access interactive documentation at:
+
+- **OpenAPI Schema**: `http://localhost:8000/schema/openapi.json`
+- **Swagger UI**: `http://localhost:8000/schema/swagger`
+- **ReDoc Documentation**: `http://localhost:8000/schema/redoc`
+- **Stoplight Elements**: `http://localhost:8000/schema/elements`
+- **RapiDoc**: `http://localhost:8000/schema/rapidoc`
+
+### Features
+
+The interactive documentation provides:
+
+- **Complete API Reference**: All endpoints with detailed parameter descriptions
+- **Try It Out**: Test API endpoints directly from the browser
+- **Request/Response Examples**: Sample requests and responses for each endpoint
+- **Schema Validation**: Interactive validation of request parameters
+- **Download Options**: Export the OpenAPI specification
+
+### Example Usage
+
+```bash
+# Start the API server
+litestar --app kreuzberg._api.main:app run
+
+# Open your browser to view the documentation
+open http://localhost:8000/schema/swagger
+```
+
+The documentation includes examples for all configuration options, making it easy to understand the full capabilities of the extraction API.
+
 #### Error Handling
 
 Invalid configuration returns appropriate error responses:
@@ -258,6 +295,27 @@ Error responses include:
 }
 ```
 
+### Debugging 500 Errors
+
+For detailed error information when 500 Internal Server Errors occur, set the `DEBUG` environment variable:
+
+```bash
+# Enable debug mode for detailed 500 error responses
+DEBUG=1 litestar --app kreuzberg._api.main:app run
+
+# Or with uvicorn
+DEBUG=1 uvicorn kreuzberg._api.main:app --host 0.0.0.0 --port 8000
+```
+
+When `DEBUG=1` is set, 500 errors will include:
+
+- Full stack traces
+- Detailed error context
+- Internal state information
+- Request debugging details
+
+⚠️ **Warning**: Only enable debug mode in development environments. Debug information may expose sensitive details and should never be used in production.
+
 ## Features
 
 - **Runtime Configuration**: Configure extraction via query parameters and HTTP headers
@@ -301,6 +359,7 @@ For production use, consider:
 1. **Monitoring**: Enable OpenTelemetry exporters
 1. **Rate Limiting**: Add rate limiting middleware
 1. **Authentication**: Add authentication middleware if needed
+1. **Security**: Ensure `DEBUG` environment variable is not set
 
 Example production command:
 

{kreuzberg-3.13.3 → kreuzberg-3.14.1}/docs/user-guide/extraction-configuration.md

@@ -57,6 +57,13 @@ detector_base_threshold = 0.9
 remove_null_rows = true
 enable_multi_header = true
 
+# DPI and Image Processing configuration
+target_dpi = 150                  # Target DPI for document processing
+max_image_dimension = 25000       # Maximum pixel dimension before auto-scaling
+auto_adjust_dpi = true           # Automatically adjust DPI for large documents
+min_dpi = 72                     # Minimum DPI threshold
+max_dpi = 600                    # Maximum DPI threshold
+
 # Language detection configuration
 [language_detection]
 multilingual = true
@@ -91,6 +98,13 @@ auto_detect_document_type = true
 document_classification_mode = "text"
 type_confidence_threshold = 0.5
 
+# DPI and Image Processing
+target_dpi = 150
+max_image_dimension = 25000
+auto_adjust_dpi = true
+min_dpi = 72
+max_dpi = 600
+
 [tool.kreuzberg.tesseract]
 language = "eng"
 psm = 6
@@ -536,6 +550,67 @@ python -m spacy download fr_core_news_sm   # French
 
 Available spaCy models include: `en_core_web_sm`, `de_core_news_sm`, `fr_core_news_sm`, `es_core_news_sm`, `pt_core_news_sm`, `it_core_news_sm`, `nl_core_news_sm`, `zh_core_web_sm`, `ja_core_news_sm`, `ko_core_news_sm`, `ru_core_news_sm`, and many others.
 
+### DPI and Image Processing
+
+Kreuzberg provides intelligent DPI (dots per inch) configuration to optimize document processing quality and performance. This feature automatically handles image scaling for large documents while maintaining OCR quality.
+
+```python
+from kreuzberg import extract_file, ExtractionConfig
+
+# Default DPI configuration (optimized for most documents)
+result = await extract_file("large_document.pdf")
+
+# Custom DPI configuration for high-quality documents
+config = ExtractionConfig(
+    target_dpi=200,  # Higher quality for detailed documents
+    max_image_dimension=30000,  # Allow larger images
+    auto_adjust_dpi=True,  # Automatically scale down if too large
+    min_dpi=100,  # Higher minimum for quality
+    max_dpi=400,  # Lower maximum to control processing time
+)
+result = await extract_file("technical_drawing.pdf", config=config)
+
+# Fast processing configuration for large batches
+config = ExtractionConfig(
+    target_dpi=120,  # Lower DPI for faster processing
+    max_image_dimension=15000,  # Smaller maximum size
+    auto_adjust_dpi=True,  # Still allow automatic scaling
+)
+result = await extract_file("large_batch_document.pdf", config=config)
+```
+
+#### DPI Configuration Options
+
+- **`target_dpi`** (default: 150): The desired DPI for document processing. Higher values provide better quality but slower processing.
+
+- **`max_image_dimension`** (default: 25000): Maximum pixel dimension (width or height) before automatic scaling kicks in.
+
+- **`auto_adjust_dpi`** (default: True): Automatically reduce DPI for oversized documents to stay within memory and processing limits.
+
+- **`min_dpi`** / **`max_dpi`** (defaults: 72/600): Bounds for automatic DPI adjustment to ensure quality remains within acceptable ranges.
+
+#### When to Adjust DPI Settings
+
+**Increase DPI for:**
+
+- Technical documents with small text or fine details
+- Documents that will undergo further image processing
+- High-quality archival processing
+
+**Decrease DPI for:**
+
+- Large batch processing where speed is important
+- Documents with simple layouts and large text
+- Memory-constrained environments
+
+**Use auto-adjustment for:**
+
+- Mixed document types with varying sizes
+- Unknown document dimensions
+- Production environments processing diverse content
+
+The DPI system prevents "Image too large" errors while maintaining optimal quality-performance balance.
+
 ### Batch Processing
 
 ```python