kreuzberg 3.16.0__tar.gz → 3.17.1__tar.gz

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/.github/workflows/ci.yaml

@@ -8,6 +8,10 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   validate:
     runs-on: ubuntu-latest
@@ -138,27 +142,7 @@ jobs:
     needs: validate
     if: github.event_name == 'pull_request' && needs.validate.result == 'success'
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        test-category:
-          - name: "core"
-            path: "tests/core,tests/utils"
-            system-deps: false
-            timeout: 15
-          - name: "extractors"
-            path: "tests/extractors"
-            system-deps: true
-            timeout: 20
-          - name: "integration"
-            path: "tests/integration,tests/api"
-            system-deps: true
-            timeout: 25
-          - name: "features"
-            path: "tests/features,tests/interfaces,tests/mcp,tests/multiprocessing,tests/ocr"
-            system-deps: true
-            timeout: 20
-    timeout-minutes: ${{ matrix.test-category.timeout }}
+    timeout-minutes: 45
     steps:
       - name: Checkout
         uses: actions/checkout@v5
@@ -170,36 +154,62 @@ jobs:
 
       - name: Install Python
         uses: actions/setup-python@v6
+        id: setup-python
         with:
           python-version: "3.13"
 
       - name: Cache Python Dependencies
+        id: python-cache
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/uv
             .venv
-          key: python-dependencies-ubuntu-latest-3.13-${{ matrix.test-category.name }}-${{ hashFiles('uv.lock') }}
+          key: python-dependencies-ubuntu-latest-3.13-${{ hashFiles('uv.lock') }}
           restore-keys: |
             python-dependencies-ubuntu-latest-3.13-
 
       - name: Install Dependencies
-        run: uv sync --all-extras --dev
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            uv sync --all-extras --dev
+          shell: bash
 
       - name: Install System Dependencies
-        if: matrix.test-category.system-deps
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y tesseract-ocr tesseract-ocr-deu pandoc
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y tesseract-ocr tesseract-ocr-deu pandoc
+          shell: bash
 
-      - name: Run Tests - ${{ matrix.test-category.name }}
-        run: uv run pytest $(echo "${{ matrix.test-category.path }}" | tr ',' ' ') -v --reruns 1 --reruns-delay 1 --cov=kreuzberg --cov-append --cov-report=lcov:coverage-${{ matrix.test-category.name }}.lcov
+      - name: Run All Tests with Coverage
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 15
+          max_attempts: 3
+          retry_wait_seconds: 10
+          command: |
+            uv run coverage erase
+            uv run pytest -s -vvv --cov=kreuzberg --cov-report=lcov:coverage.lcov --cov-report=term --cov-config=pyproject.toml --reruns 2 --reruns-delay 1
+            uv run coverage report --precision=2
+          shell: bash
 
       - name: Upload Coverage Artifacts
+        if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-${{ matrix.test-category.name }}-${{ github.sha }}
-          path: coverage-${{ matrix.test-category.name }}.lcov
+          name: coverage-pr-${{ github.sha }}
+          path: |
+            coverage.lcov
+            .coverage
           retention-days: 1
 
   coverage-pr:
@@ -214,47 +224,7 @@ jobs:
       - name: Download Coverage Artifacts
         uses: actions/download-artifact@v5
         with:
-          pattern: coverage-*-${{ github.sha }}
-          merge-multiple: true
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-
-      - name: Install Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: "3.13"
-
-      - name: Install Dependencies
-        run: uv sync --dev
-
-      - name: Combine Coverage Reports
-        run: |
-          # Install lcov for combining reports
-          sudo apt-get update && sudo apt-get install -y lcov
-
-          # List available coverage files
-          echo "Available coverage files:"
-          find . -name "coverage-*.lcov" -type f || echo "No coverage files found"
-
-          # Combine all lcov files if they exist
-          coverage_files=($(find . -name "coverage-*.lcov" -type f))
-          if [ ${#coverage_files[@]} -gt 0 ]; then
-            echo "Combining ${#coverage_files[@]} coverage files..."
-            if [ ${#coverage_files[@]} -eq 1 ]; then
-              # Only one file, just copy it
-              cp "${coverage_files[0]}" coverage.lcov
-            else
-              # Multiple files, combine them
-              lcov --rc branch_coverage=1 $(printf " -a %s" "${coverage_files[@]}") -o coverage.lcov
-            fi
-          else
-            echo "No coverage files to combine, creating empty coverage.lcov"
-            echo "TN:" > coverage.lcov
-            echo "end_of_record" >> coverage.lcov
-          fi
+          name: coverage-pr-${{ github.sha }}
 
       - name: Upload Coverage to DeepSource
         if: always()

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/.github/workflows/docker-e2e-tests.yml

@@ -4,6 +4,10 @@ on:
   workflow_dispatch:
   workflow_call:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test-docker-images:
     runs-on: ubuntu-latest

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/.github/workflows/docs.yml

@@ -17,7 +17,7 @@ permissions:
 
 concurrency:
   group: "pages"
-  cancel-in-progress: false
+  cancel-in-progress: true
 
 jobs:
   build:

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/.github/workflows/test-docker-builds.yml

@@ -3,6 +3,10 @@ name: Test Docker Builds (No Push)
 on:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test-build-all-images:
     runs-on: ubuntu-latest

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
     hooks:
       - id: mdformat
         additional_dependencies:
-          - mdformat-mkdocs==4.0.0
+          - mdformat-mkdocs==4.1.0
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.45.0
     hooks:
@@ -36,6 +36,7 @@ repos:
     hooks:
       - id: blacken-docs
         args: ["--pyi", "--line-length", "130"]
+        exclude: tests/features/token_reduction_test.py
         additional_dependencies:
           - black==25.1.0
   - repo: https://github.com/rbubley/mirrors-prettier
@@ -48,7 +49,7 @@ repos:
     hooks:
       - id: pyproject-fmt
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.0
+    rev: v0.13.1
     hooks:
       - id: ruff
         args: ["--fix", "--unsafe-fixes"]
@@ -59,7 +60,7 @@ repos:
       - id: codespell
         exclude: ^tests|^scripts|^kreuzberg/_tesseract|^kreuzberg/_mime_types
         additional_dependencies:
-          - tomli
+          - tomli==2.2.1
   - repo: https://github.com/jsh9/pydoclint
     rev: 0.7.3
     hooks:

kreuzberg-3.17.1/.prettierignore

@@ -0,0 +1 @@
+ATTRIBUTIONS.md

kreuzberg-3.17.1/ATTRIBUTIONS.md

@@ -0,0 +1,47 @@
+# Third-Party Attributions
+
+This file contains attributions for third-party code, data, and libraries used in Kreuzberg.
+
+## Stopwords Data
+
+The stopwords data in `kreuzberg/_token_reduction/stop_words.json` is derived from the [stopwords-iso](https://github.com/stopwords-iso/stopwords-iso) project.
+
+**Original Author:** Gene Diaz and contributors
+**License:** MIT License
+**Source:** <https://github.com/stopwords-iso/stopwords-iso>
+
+### MIT License (stopwords-iso)
+
+```text
+MIT License
+
+Copyright (c) stopwords-iso contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+
+### Changes Made
+
+The original stopwords-iso data was used as-is with no modifications to the word lists themselves. The data was packaged into Kreuzberg's `_token_reduction` module for use in the token reduction feature.
+
+______________________________________________________________________
+
+## Other Third-Party Dependencies
+
+All other third-party dependencies are listed in `pyproject.toml` with their respective licenses. This section is specifically for bundled/vendored code and data.

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.16.0
+Version: 3.17.1
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -32,12 +32,13 @@ Requires-Dist: anyio>=4.10.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
 Requires-Dist: html-to-markdown[lxml]>=1.13.0
-Requires-Dist: mcp>=1.14.0
+Requires-Dist: langcodes>=3.5.0
+Requires-Dist: mcp>=1.14.1
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: playa-pdf>=0.7.0
 Requires-Dist: polars>=1.33.1
-Requires-Dist: psutil>=7.0.0
+Requires-Dist: psutil>=7.1.0
 Requires-Dist: pypdfium2==4.30.0
 Requires-Dist: python-calamine>=0.5.3
 Requires-Dist: python-pptx>=1.0.2
@@ -49,7 +50,7 @@ Provides-Extra: all
 Requires-Dist: click>=8.2.1; extra == 'all'
 Requires-Dist: deep-translator>=1.11.4; extra == 'all'
 Requires-Dist: easyocr>=1.7.2; extra == 'all'
-Requires-Dist: fast-langdetect>=0.3.2; extra == 'all'
+Requires-Dist: fast-langdetect>=1.0.0; extra == 'all'
 Requires-Dist: gmft>=0.4.2; extra == 'all'
 Requires-Dist: keybert>=0.9.0; extra == 'all'
 Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.17.0; extra == 'all'
@@ -82,7 +83,7 @@ Requires-Dist: spacy>=3.8.7; extra == 'entity-extraction'
 Provides-Extra: gmft
 Requires-Dist: gmft>=0.4.2; extra == 'gmft'
 Provides-Extra: langdetect
-Requires-Dist: fast-langdetect>=0.3.2; extra == 'langdetect'
+Requires-Dist: fast-langdetect>=1.0.0; extra == 'langdetect'
 Provides-Extra: paddleocr
 Requires-Dist: paddleocr>=3.2.0; extra == 'paddleocr'
 Requires-Dist: paddlepaddle>=3.2.0; extra == 'paddleocr'

kreuzberg-3.17.1/benchmarks/token_reduction_compression_benchmark.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import json
+import statistics
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+from kreuzberg import extract_bytes_sync
+from kreuzberg._token_reduction import get_reduction_stats, reduce_tokens
+from kreuzberg._types import ExtractionConfig, TokenReductionConfig
+
+
+@dataclass
+class CompressionResult:
+    text_type: str
+    mode: str
+    original_length: int
+    reduced_length: int
+    original_tokens: int
+    reduced_tokens: int
+    character_reduction_ratio: float
+    token_reduction_ratio: float
+    processing_time_ms: float
+
+    @property
+    def character_compression_percent(self) -> float:
+        return self.character_reduction_ratio * 100
+
+    @property
+    def token_compression_percent(self) -> float:
+        return self.token_reduction_ratio * 100
+
+
+@dataclass
+class CompressionBenchmarkSuite:
+    results: list[CompressionResult] = field(default_factory=list)
+    total_tests: int = 0
+    total_time_ms: float = 0.0
+
+    def add_result(self, result: CompressionResult) -> None:
+        self.results.append(result)
+        self.total_tests += 1
+        self.total_time_ms += result.processing_time_ms
+
+    def get_summary(self) -> dict[str, Any]:
+        if not self.results:
+            return {}
+
+        by_mode: dict[str, Any] = {}
+        for result in self.results:
+            if result.mode not in by_mode:
+                by_mode[result.mode] = []
+            by_mode[result.mode].append(result)
+
+        mode_stats = {}
+        for mode, results in by_mode.items():
+            char_ratios = [r.character_compression_percent for r in results]
+            token_ratios = [r.token_compression_percent for r in results]
+            times = [r.processing_time_ms for r in results]
+
+            mode_stats[mode] = {
+                "tests": len(results),
+                "character_compression": {
+                    "mean": statistics.mean(char_ratios),
+                    "median": statistics.median(char_ratios),
+                    "stdev": statistics.stdev(char_ratios) if len(char_ratios) > 1 else 0.0,
+                    "min": min(char_ratios),
+                    "max": max(char_ratios),
+                },
+                "token_compression": {
+                    "mean": statistics.mean(token_ratios),
+                    "median": statistics.median(token_ratios),
+                    "stdev": statistics.stdev(token_ratios) if len(token_ratios) > 1 else 0.0,
+                    "min": min(token_ratios),
+                    "max": max(token_ratios),
+                },
+                "performance": {
+                    "avg_time_ms": statistics.mean(times),
+                    "total_time_ms": sum(times),
+                },
+            }
+
+        return {
+            "summary": {
+                "total_tests": self.total_tests,
+                "total_time_ms": self.total_time_ms,
+                "avg_time_per_test_ms": self.total_time_ms / self.total_tests if self.total_tests > 0 else 0.0,
+            },
+            "by_mode": mode_stats,
+            "detailed_results": [
+                {
+                    "text_type": r.text_type,
+                    "mode": r.mode,
+                    "original_length": r.original_length,
+                    "reduced_length": r.reduced_length,
+                    "original_tokens": r.original_tokens,
+                    "reduced_tokens": r.reduced_tokens,
+                    "character_compression_percent": r.character_compression_percent,
+                    "token_compression_percent": r.token_compression_percent,
+                    "processing_time_ms": r.processing_time_ms,
+                }
+                for r in self.results
+            ],
+        }
+
+
+class TokenReductionCompressionBenchmark:
+    def __init__(self) -> None:
+        self.test_texts = self._create_test_texts()
+        self.modes: list[Literal["light", "moderate"]] = ["light", "moderate"]
+
+    def _create_test_texts(self) -> dict[str, str]:
+        return {
+            "formal_document": """
+                The quarterly financial report demonstrates significant improvements in operational efficiency and market positioning.
+                Our comprehensive analysis reveals that the implementation of strategic initiatives has resulted in measurable outcomes
+                across multiple key performance indicators. The organization's commitment to excellence and continuous improvement
+                is evident in these results. Furthermore, the systematic approach to risk management and quality assurance has
+                contributed to enhanced stakeholder confidence and sustainable growth trajectory. The board of directors acknowledges
+                the exceptional efforts of the management team and all employees in achieving these remarkable results.
+            """.strip(),
+            "casual_conversation": """
+                Hey there! I was just thinking about that amazing movie we watched last weekend. It was really incredible, wasn't it?
+                The way they told the story was so compelling and the characters were just wonderful. I think it's one of the best
+                films I've seen this year. What did you think about it? I'd love to hear your thoughts and maybe we could discuss
+                some of the themes that really stood out to me. There were so many interesting elements that I'm still thinking about.
+            """.strip(),
+            "technical_manual": """
+                Configure the system parameters by accessing the administrative interface through the main configuration panel.
+                Navigate to Settings > Advanced > Network Configuration and verify that all connection parameters are correctly
+                initialized. The TCP/IP stack must be properly configured with appropriate DNS resolution settings and gateway
+                routing tables. Execute the diagnostic utilities to validate network connectivity and ensure that all protocols
+                are functioning within acceptable performance thresholds. Document any configuration changes in the system log
+                for future reference and troubleshooting procedures.
+            """.strip(),
+            "news_article": """
+                Local authorities announced today that the new public transportation system will begin operations next month,
+                connecting several major districts across the metropolitan area. The project, which has been in development for
+                over three years, represents a significant investment in sustainable urban infrastructure. City officials expect
+                the system to reduce traffic congestion and provide affordable transportation options for thousands of daily
+                commuters. Environmental impact studies indicate that the implementation will contribute to reduced carbon emissions
+                and improved air quality throughout the region.
+            """.strip(),
+            "literature_excerpt": """
+                The old lighthouse stood majestically against the stormy horizon, its weathered stones bearing witness to countless
+                tempests and countless ships that had sought its guiding light. Sarah approached the ancient structure with a sense
+                of reverence, knowing that within its walls lay the stories of generations of lighthouse keepers who had dedicated
+                their lives to the safety of maritime travelers. The wind howled through the nearby cliffs, carrying with it the
+                salt spray of crashing waves and the whispered secrets of the sea itself.
+            """.strip(),
+            "scientific_abstract": """
+                This study investigates the relationship between cognitive load and working memory performance in multilingual
+                individuals under various experimental conditions. Participants (n=127) completed a series of standardized
+                assessments while neural activity was monitored using electroencephalography. Results indicate significant
+                correlations between language switching frequency and executive control efficiency (p<0.001). The findings suggest
+                that bilingual advantages in cognitive flexibility extend to domain-general executive functions, with implications
+                for educational policy and cognitive training interventions.
+            """.strip(),
+            "stopword_heavy": """
+                And so it was that he went to the store, and then he bought some things that he needed for the house. But when
+                he got back to the place where he lived, he realized that he had forgotten to get the most important thing that
+                he had originally planned to purchase. So he had to go back to the store again, and this time he made sure to
+                get everything that he needed. It was a bit frustrating, but in the end, everything worked out just fine.
+            """.strip(),
+            "technical_jargon": """
+                The microservices architecture implements a distributed system pattern utilizing containerized deployments
+                orchestrated through Kubernetes clusters. API gateways facilitate service discovery and load balancing across
+                multiple availability zones. The event-driven messaging infrastructure leverages Apache Kafka for asynchronous
+                communication between bounded contexts. Monitoring and observability are achieved through OpenTelemetry
+                instrumentation with Prometheus metrics collection and Grafana visualization dashboards.
+            """.strip(),
+            "minimal_stopwords": """
+                Python programming language offers powerful features. Machine learning algorithms require extensive datasets.
+                Neural networks demonstrate remarkable performance capabilities. Developers utilize frameworks like TensorFlow.
+                Data preprocessing involves cleaning, transformation, validation procedures. Model training requires computational
+                resources, optimization techniques. Evaluation metrics include accuracy, precision, recall measurements.
+                Production deployment considerations encompass scalability, monitoring, maintenance requirements.
+            """.strip(),
+        }
+
+    def test_compression_effectiveness(
+        self, text: str, text_type: str, mode: Literal["light", "moderate"]
+    ) -> CompressionResult:
+        config = TokenReductionConfig(mode=mode, preserve_markdown=False)
+
+        start_time = time.perf_counter()
+        reduced_text = reduce_tokens(text, config=config, language="en")
+        processing_time = (time.perf_counter() - start_time) * 1000
+
+        stats = get_reduction_stats(text, reduced_text)
+
+        return CompressionResult(
+            text_type=text_type,
+            mode=mode,
+            original_length=len(text),
+            reduced_length=len(reduced_text),
+            original_tokens=stats["original_tokens"],
+            reduced_tokens=stats["reduced_tokens"],
+            character_reduction_ratio=stats["character_reduction_ratio"],
+            token_reduction_ratio=stats["token_reduction_ratio"],
+            processing_time_ms=processing_time,
+        )
+
+    def run_comprehensive_benchmark(self) -> CompressionBenchmarkSuite:
+        suite = CompressionBenchmarkSuite()
+
+        for text_type, text in self.test_texts.items():
+            for mode in self.modes:
+                result = self.test_compression_effectiveness(text, text_type, mode)
+                suite.add_result(result)
+
+        return suite
+
+    def run_pipeline_integration_test(self) -> dict[str, Any]:
+        pipeline_results: dict[str, Any] = {}
+
+        for mode in self.modes:
+            config = ExtractionConfig(token_reduction=TokenReductionConfig(mode=mode))
+
+            test_text = self.test_texts["formal_document"]
+
+            start_time = time.perf_counter()
+            result = extract_bytes_sync(test_text.encode("utf-8"), "text/plain", config)
+            processing_time = (time.perf_counter() - start_time) * 1000
+
+            reduction_stats = result.metadata.get("token_reduction", {})
+
+            pipeline_results[mode] = {
+                "original_length": len(test_text),
+                "reduced_length": len(result.content),
+                "processing_time_ms": processing_time,
+                "reduction_stats": reduction_stats,
+                "metadata_present": "token_reduction" in result.metadata,
+            }
+
+        return pipeline_results
+
+
+def main() -> None:
+    benchmark = TokenReductionCompressionBenchmark()
+
+    suite = benchmark.run_comprehensive_benchmark()
+
+    pipeline_results = benchmark.run_pipeline_integration_test()
+
+    summary = suite.get_summary()
+
+    for _stats in summary["by_mode"].values():
+        pass
+
+    output_dir = Path("benchmarks/results")
+    output_dir.mkdir(exist_ok=True)
+
+    full_results = {
+        "compression_benchmark": summary,
+        "pipeline_integration": pipeline_results,
+        "timestamp": time.time(),
+    }
+
+    output_file = output_dir / "token_reduction_compression.json"
+    with output_file.open("w") as f:
+        json.dump(full_results, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/docs/api-reference/types.md

@@ -84,6 +84,12 @@ Configuration options for converting HTML content to Markdown:
 
 ::: kreuzberg.HTMLToMarkdownConfig
 
+## Token Reduction Configuration
+
+Configuration options for token reduction and text optimization:
+
+::: kreuzberg.TokenReductionConfig
+
 ## PSMMode (Page Segmentation Mode)
 
 ::: kreuzberg.PSMMode

{kreuzberg-3.16.0 → kreuzberg-3.17.1}/docs/user-guide/index.md

@@ -8,6 +8,7 @@ This guide provides comprehensive documentation for the Kreuzberg document intel
 - [Extraction Configuration](extraction-configuration.md) - Configure the extraction process ([API](../api-reference/types.md#extractionconfig))
 - [Metadata Extraction](metadata-extraction.md) - Document metadata extraction ([API](../api-reference/types.md#metadata))
 - [Content Chunking](chunking.md) - Split documents into manageable chunks
+- [Token Reduction](token-reduction.md) - Optimize text for LLMs and storage ([API](../api-reference/types.md#tokenreductionconfig))
 - [Document Classification](document-classification.md) - Automatic document type detection
 - [OCR Configuration](ocr-configuration.md) - Configure OCR settings ([API](../api-reference/ocr-configuration.md))
 - [OCR Backends](ocr-backends.md) - Choose and configure different OCR engines