kreuzberg 3.15.0__tar.gz → 3.17.0__tar.gz

{kreuzberg-3.15.0 → kreuzberg-3.17.0}/.github/workflows/ci.yaml

@@ -8,6 +8,10 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   validate:
     runs-on: ubuntu-latest
@@ -138,27 +142,7 @@ jobs:
     needs: validate
     if: github.event_name == 'pull_request' && needs.validate.result == 'success'
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        test-category:
-          - name: "core"
-            path: "tests/core,tests/utils"
-            system-deps: false
-            timeout: 15
-          - name: "extractors"
-            path: "tests/extractors"
-            system-deps: true
-            timeout: 20
-          - name: "integration"
-            path: "tests/integration,tests/api"
-            system-deps: true
-            timeout: 25
-          - name: "features"
-            path: "tests/features,tests/interfaces,tests/mcp,tests/multiprocessing,tests/ocr"
-            system-deps: true
-            timeout: 20
-    timeout-minutes: ${{ matrix.test-category.timeout }}
+    timeout-minutes: 45
     steps:
       - name: Checkout
         uses: actions/checkout@v5
@@ -170,36 +154,62 @@ jobs:
 
       - name: Install Python
         uses: actions/setup-python@v6
+        id: setup-python
         with:
           python-version: "3.13"
 
       - name: Cache Python Dependencies
+        id: python-cache
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/uv
             .venv
-          key: python-dependencies-ubuntu-latest-3.13-${{ matrix.test-category.name }}-${{ hashFiles('uv.lock') }}
+          key: python-dependencies-ubuntu-latest-3.13-${{ hashFiles('uv.lock') }}
           restore-keys: |
             python-dependencies-ubuntu-latest-3.13-
 
       - name: Install Dependencies
-        run: uv sync --all-extras --dev
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            uv sync --all-extras --dev
+          shell: bash
 
       - name: Install System Dependencies
-        if: matrix.test-category.system-deps
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y tesseract-ocr tesseract-ocr-deu pandoc
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y tesseract-ocr tesseract-ocr-deu pandoc
+          shell: bash
 
-      - name: Run Tests - ${{ matrix.test-category.name }}
-        run: uv run pytest $(echo "${{ matrix.test-category.path }}" | tr ',' ' ') -v --reruns 1 --reruns-delay 1 --cov=kreuzberg --cov-append --cov-report=lcov:coverage-${{ matrix.test-category.name }}.lcov
+      - name: Run All Tests with Coverage
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 15
+          max_attempts: 3
+          retry_wait_seconds: 10
+          command: |
+            uv run coverage erase
+            uv run pytest -s -vvv --cov=kreuzberg --cov-report=lcov:coverage.lcov --cov-report=term --cov-config=pyproject.toml --reruns 2 --reruns-delay 1
+            uv run coverage report --precision=2
+          shell: bash
 
       - name: Upload Coverage Artifacts
+        if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-${{ matrix.test-category.name }}-${{ github.sha }}
-          path: coverage-${{ matrix.test-category.name }}.lcov
+          name: coverage-pr-${{ github.sha }}
+          path: |
+            coverage.lcov
+            .coverage
           retention-days: 1
 
   coverage-pr:
@@ -212,49 +222,9 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Download Coverage Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: coverage-*-${{ github.sha }}
-          merge-multiple: true
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-
-      - name: Install Python
-        uses: actions/setup-python@v6
+        uses: actions/download-artifact@v5
         with:
-          python-version: "3.13"
-
-      - name: Install Dependencies
-        run: uv sync --dev
-
-      - name: Combine Coverage Reports
-        run: |
-          # Install lcov for combining reports
-          sudo apt-get update && sudo apt-get install -y lcov
-
-          # List available coverage files
-          echo "Available coverage files:"
-          find . -name "coverage-*.lcov" -type f || echo "No coverage files found"
-
-          # Combine all lcov files if they exist
-          coverage_files=($(find . -name "coverage-*.lcov" -type f))
-          if [ ${#coverage_files[@]} -gt 0 ]; then
-            echo "Combining ${#coverage_files[@]} coverage files..."
-            if [ ${#coverage_files[@]} -eq 1 ]; then
-              # Only one file, just copy it
-              cp "${coverage_files[0]}" coverage.lcov
-            else
-              # Multiple files, combine them
-              lcov --rc branch_coverage=1 $(printf " -a %s" "${coverage_files[@]}") -o coverage.lcov
-            fi
-          else
-            echo "No coverage files to combine, creating empty coverage.lcov"
-            echo "TN:" > coverage.lcov
-            echo "end_of_record" >> coverage.lcov
-          fi
+          name: coverage-pr-${{ github.sha }}
 
       - name: Upload Coverage to DeepSource
         if: always()

{kreuzberg-3.15.0 → kreuzberg-3.17.0}/.github/workflows/docker-e2e-tests.yml

@@ -4,6 +4,10 @@ on:
   workflow_dispatch:
   workflow_call:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test-docker-images:
     runs-on: ubuntu-latest

{kreuzberg-3.15.0 → kreuzberg-3.17.0}/.github/workflows/docs.yml

@@ -17,7 +17,7 @@ permissions:
 
 concurrency:
   group: "pages"
-  cancel-in-progress: false
+  cancel-in-progress: true
 
 jobs:
   build:

{kreuzberg-3.15.0 → kreuzberg-3.17.0}/.github/workflows/test-docker-builds.yml

@@ -3,6 +3,10 @@ name: Test Docker Builds (No Push)
 on:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test-build-all-images:
     runs-on: ubuntu-latest

{kreuzberg-3.15.0 → kreuzberg-3.17.0}/.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
       - id: name-tests-test
         args:
           - --pytest
-        exclude: factories|test_utils|completion.py|test_data
+        exclude: factories|test_utils|completion.py|test_data|docker_e2e.py
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-toml
@@ -26,7 +26,7 @@ repos:
     hooks:
       - id: mdformat
         additional_dependencies:
-          - mdformat-mkdocs==4.0.0
+          - mdformat-mkdocs==4.1.0
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.45.0
     hooks:
@@ -36,6 +36,7 @@ repos:
     hooks:
       - id: blacken-docs
         args: ["--pyi", "--line-length", "130"]
+        exclude: tests/features/token_reduction_test.py
         additional_dependencies:
           - black==25.1.0
   - repo: https://github.com/rbubley/mirrors-prettier
@@ -59,7 +60,7 @@ repos:
       - id: codespell
         exclude: ^tests|^scripts|^kreuzberg/_tesseract|^kreuzberg/_mime_types
         additional_dependencies:
-          - tomli
+          - tomli==2.2.1
   - repo: https://github.com/jsh9/pydoclint
     rev: 0.7.3
     hooks:

kreuzberg-3.17.0/.prettierignore

@@ -0,0 +1 @@
+ATTRIBUTIONS.md

kreuzberg-3.17.0/ATTRIBUTIONS.md

@@ -0,0 +1,47 @@
+# Third-Party Attributions
+
+This file contains attributions for third-party code, data, and libraries used in Kreuzberg.
+
+## Stopwords Data
+
+The stopwords data in `kreuzberg/_token_reduction/stop_words.json` is derived from the [stopwords-iso](https://github.com/stopwords-iso/stopwords-iso) project.
+
+**Original Author:** Gene Diaz and contributors
+**License:** MIT License
+**Source:** <https://github.com/stopwords-iso/stopwords-iso>
+
+### MIT License (stopwords-iso)
+
+```text
+MIT License
+
+Copyright (c) stopwords-iso contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+
+### Changes Made
+
+The original stopwords-iso data was used as-is with no modifications to the word lists themselves. The data was packaged into Kreuzberg's `_token_reduction` module for use in the token reduction feature.
+
+______________________________________________________________________
+
+## Other Third-Party Dependencies
+
+All other third-party dependencies are listed in `pyproject.toml` with their respective licenses. This section is specifically for bundled/vendored code and data.

{kreuzberg-3.15.0 → kreuzberg-3.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.15.0
+Version: 3.17.0
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -31,7 +31,8 @@ Requires-Python: >=3.10
 Requires-Dist: anyio>=4.10.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
-Requires-Dist: html-to-markdown[lxml]>=1.11.0
+Requires-Dist: html-to-markdown[lxml]>=1.13.0
+Requires-Dist: langcodes>=3.5.0
 Requires-Dist: mcp>=1.14.0
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: numpy>=2.0.0
@@ -49,7 +50,7 @@ Provides-Extra: all
 Requires-Dist: click>=8.2.1; extra == 'all'
 Requires-Dist: deep-translator>=1.11.4; extra == 'all'
 Requires-Dist: easyocr>=1.7.2; extra == 'all'
-Requires-Dist: fast-langdetect>=0.3.2; extra == 'all'
+Requires-Dist: fast-langdetect>=1.0.0; extra == 'all'
 Requires-Dist: gmft>=0.4.2; extra == 'all'
 Requires-Dist: keybert>=0.9.0; extra == 'all'
 Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.17.0; extra == 'all'
@@ -82,7 +83,7 @@ Requires-Dist: spacy>=3.8.7; extra == 'entity-extraction'
 Provides-Extra: gmft
 Requires-Dist: gmft>=0.4.2; extra == 'gmft'
 Provides-Extra: langdetect
-Requires-Dist: fast-langdetect>=0.3.2; extra == 'langdetect'
+Requires-Dist: fast-langdetect>=1.0.0; extra == 'langdetect'
 Provides-Extra: paddleocr
 Requires-Dist: paddleocr>=3.2.0; extra == 'paddleocr'
 Requires-Dist: paddlepaddle>=3.2.0; extra == 'paddleocr'
@@ -109,7 +110,7 @@ Description-Content-Type: text/markdown
 - **Text Extraction**: High-fidelity text extraction preserving document structure and formatting
 - **Image Extraction**: Extract embedded images from PDFs, presentations, HTML, and Office documents with optional OCR
 - **Metadata Extraction**: Comprehensive metadata including author, creation date, language, and document properties
-- **Format Support**: 18 document types including PDF, Microsoft Office, images, HTML, and structured data formats
+- **Format Support**: 21 document types including PDF, Microsoft Office, images, HTML, and structured data formats
 - **OCR Integration**: Tesseract OCR with markdown output (default) and table extraction from scanned documents
 - **Document Classification**: Automatic document type detection (contracts, forms, invoices, receipts, reports)
 
@@ -227,14 +228,15 @@ claude mcp add kreuzberg uvx kreuzberg-mcp
 
 ## Supported Formats
 
-| Category          | Formats                        |
-| ----------------- | ------------------------------ |
-| **Documents**     | PDF, DOCX, DOC, RTF, TXT, EPUB |
-| **Images**        | JPG, PNG, TIFF, BMP, GIF, WEBP |
-| **Spreadsheets**  | XLSX, XLS, CSV, ODS            |
-| **Presentations** | PPTX, PPT, ODP                 |
-| **Web**           | HTML, XML, MHTML               |
-| **Archives**      | Support via extraction         |
+| Category            | Formats                        |
+| ------------------- | ------------------------------ |
+| **Documents**       | PDF, DOCX, DOC, RTF, TXT, EPUB |
+| **Images**          | JPG, PNG, TIFF, BMP, GIF, WEBP |
+| **Spreadsheets**    | XLSX, XLS, CSV, ODS            |
+| **Presentations**   | PPTX, PPT, ODP                 |
+| **Web**             | HTML, XML, MHTML               |
+| **Structured Data** | JSON, YAML, TOML               |
+| **Archives**        | Support via extraction         |
 
 ## 📊 Performance Characteristics
 

{kreuzberg-3.15.0 → kreuzberg-3.17.0}/README.md

@@ -18,7 +18,7 @@
 - **Text Extraction**: High-fidelity text extraction preserving document structure and formatting
 - **Image Extraction**: Extract embedded images from PDFs, presentations, HTML, and Office documents with optional OCR
 - **Metadata Extraction**: Comprehensive metadata including author, creation date, language, and document properties
-- **Format Support**: 18 document types including PDF, Microsoft Office, images, HTML, and structured data formats
+- **Format Support**: 21 document types including PDF, Microsoft Office, images, HTML, and structured data formats
 - **OCR Integration**: Tesseract OCR with markdown output (default) and table extraction from scanned documents
 - **Document Classification**: Automatic document type detection (contracts, forms, invoices, receipts, reports)
 
@@ -136,14 +136,15 @@ claude mcp add kreuzberg uvx kreuzberg-mcp
 
 ## Supported Formats
 
-| Category          | Formats                        |
-| ----------------- | ------------------------------ |
-| **Documents**     | PDF, DOCX, DOC, RTF, TXT, EPUB |
-| **Images**        | JPG, PNG, TIFF, BMP, GIF, WEBP |
-| **Spreadsheets**  | XLSX, XLS, CSV, ODS            |
-| **Presentations** | PPTX, PPT, ODP                 |
-| **Web**           | HTML, XML, MHTML               |
-| **Archives**      | Support via extraction         |
+| Category            | Formats                        |
+| ------------------- | ------------------------------ |
+| **Documents**       | PDF, DOCX, DOC, RTF, TXT, EPUB |
+| **Images**          | JPG, PNG, TIFF, BMP, GIF, WEBP |
+| **Spreadsheets**    | XLSX, XLS, CSV, ODS            |
+| **Presentations**   | PPTX, PPT, ODP                 |
+| **Web**             | HTML, XML, MHTML               |
+| **Structured Data** | JSON, YAML, TOML               |
+| **Archives**        | Support via extraction         |
 
 ## 📊 Performance Characteristics
 

kreuzberg-3.17.0/Taskfile.yml

@@ -0,0 +1,50 @@
+version: "3"
+
+env:
+  DOCKER_BUILDKIT: 1
+  BUILDKIT_PROGRESS: plain
+
+tasks:
+  setup:
+    desc: "Install dependencies with uv"
+    cmds:
+      - uv sync --all-extras --all-packages
+      - pre-commit install && pre-commit install -hook-type commit-msg
+
+  update:
+    desc: "Update the dependencies"
+    cmds:
+      - uv run uv-bump
+      - cd benchmarks && uv run uv-bump && cd -
+      - uv sync --all-extras --all-packages --upgrade
+      - pre-commit autoupdate
+
+  test:
+    desc: "Run tests with pytest"
+    cmds:
+      - uv run pytest
+
+  test:cov:
+    desc: "Run tests with coverage"
+    cmds:
+      - uv run pytest --cov
+
+  lint:
+    desc: "Lint code with ruff and docs with markdownlint"
+    cmds:
+      - pre-commit run --all-files
+
+  docs:build:
+    desc: "Build documentation"
+    cmds:
+      - uv run mkdocs build --clean --strict
+
+  docs:serve:
+    desc: "Serve documentation locally"
+    cmds:
+      - uv run mkdocs serve
+
+  default:
+    desc: "Show available tasks"
+    cmds:
+      - task --list