kreuzberg 3.8.1__tar.gz → 3.9.0__tar.gz

kreuzberg-3.9.0/.deepsource.toml

@@ -0,0 +1,54 @@
+version = 1
+
+test_patterns = ["tests/**"]
+
+exclude_patterns = [
+    # Virtual environments
+    ".venv/**",
+    "venv/**",
+
+    # Build and distribution artifacts
+    "dist/**",
+    "build/**",
+    "*.egg-info/**",
+
+    # Documentation
+    "docs/**",
+    "site/**",
+
+    # Cache directories
+    "**/__pycache__/**",
+    ".pytest_cache/**",
+    ".mypy_cache/**",
+    ".ruff_cache/**",
+    ".coverage",
+    "htmlcov/**",
+
+    # Benchmarks and performance tests
+    "benchmarks/**",
+
+    # IDE and editor files
+    ".idea/**",
+    ".vscode/**",
+
+    # Version control
+    ".git/**",
+
+    # Temporary and generated files
+    "*.pyc",
+    ".DS_Store",
+    "*.swp",
+    "*.swo",
+]
+
+[[analyzers]]
+name = "test-coverage"
+
+[[analyzers]]
+name = "python"
+
+[analyzers.meta]
+runtime_version = "3.x.x"
+
+[[transformers]]
+name = "ruff"

kreuzberg-3.9.0/.github/workflows/ci.yaml

@@ -0,0 +1,197 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+      - feat/smart-multiprocessing
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install Dependencies
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            if [[ "${{ runner.os }}" == "Windows" ]] && [[ -d ".venv" ]]; then
+              echo "Removing existing .venv directory on Windows"
+              rm -rf .venv
+            fi
+            uv sync --all-packages --all-extras --dev
+          shell: bash
+
+      - name: Load Cached Pre-Commit Dependencies
+        id: cached-pre-commit-dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit/
+          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - name: Execute Pre-Commit
+        run: uv run pre-commit run --show-diff-on-failure --color=always --all-files
+
+  test:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macOS-latest, windows-latest ]
+        python: ${{ github.event_name == 'pull_request' && fromJSON('["3.13"]') || fromJSON('["3.10", "3.11", "3.12", "3.13"]') }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        id: setup-python
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Cache Python Dependencies
+        id: python-cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: python-dependencies-${{ matrix.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            python-dependencies-${{ matrix.os }}-${{ matrix.python }}-
+
+      - name: Install Dependencies
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            if [[ "${{ runner.os }}" == "Windows" ]] && [[ -d ".venv" ]]; then
+              echo "Removing existing .venv directory on Windows"
+              rm -rf .venv
+            fi
+            uv sync --all-packages --all-extras --dev
+          shell: bash
+
+      - name: Cache Test Artifacts
+        uses: actions/cache@v4
+        with:
+          path: .pytest_cache/
+          key: pytest-cache-${{ matrix.os }}-${{ matrix.python }}
+
+      - name: Cache and Install Homebrew (macOS)
+        if: runner.os == 'macOS'
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            # Using the underlying homebrew commands instead of the action
+            brew update || true
+            brew install tesseract tesseract-lang pandoc || brew upgrade tesseract tesseract-lang pandoc || true
+            brew list tesseract tesseract-lang pandoc
+          shell: bash
+
+      - name: Cache and Install APT Packages (Linux)
+        if: runner.os == 'Linux'
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y tesseract-ocr tesseract-ocr-deu pandoc
+          shell: bash
+
+      - name: Install System Dependencies (Windows)
+        if: runner.os == 'Windows'
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          retry_wait_seconds: 30
+          command: |
+            choco install -y tesseract pandoc --no-progress
+            Write-Output "C:\Program Files\Tesseract-OCR" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+            Write-Output "C:\Program Files\Pandoc" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+            $env:PATH = "C:\Program Files\Tesseract-OCR;C:\Program Files\Pandoc;" + $env:PATH
+            tesseract --version
+            pandoc --version
+          shell: pwsh
+
+      - name: Clean Coverage Data
+        run: |
+          rm -f .coverage .coverage.* coverage.lcov htmlcov/* || true
+        shell: bash
+
+      - name: Run Tests with Coverage
+        run: |
+          uv run coverage erase
+          uv run pytest -s -vvv --cov=kreuzberg --cov-report=lcov:coverage.lcov --cov-report=term --cov-config=pyproject.toml
+
+      - name: Upload Coverage Artifacts
+        if: matrix.os == 'ubuntu-latest' && matrix.python == '3.13'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: coverage.lcov
+          retention-days: 1
+
+  upload-coverage:
+    needs: test
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Download Coverage Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage-report
+          path: .
+
+      - name: Install DeepSource CLI
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 3
+          max_attempts: 3
+          retry_wait_seconds: 10
+          command: |
+            curl -fsSL https://deepsource.io/cli | sh
+          shell: bash
+
+      - name: Upload Coverage to DeepSource
+        env:
+          DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
+        run: |
+          ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.lcov

{kreuzberg-3.8.1 → kreuzberg-3.9.0}/.gitignore

@@ -1,5 +1,6 @@
 *$py.class
 *.Cache
+.clause/
 *.cscfg
 *.egg-info/
 *.log
@@ -9,6 +10,8 @@
 *temp/
 .coverage
 .coverage*
+coverage.lcov
+htmlcov/
 .cursorrules
 .dist/
 .DS_store

{kreuzberg-3.8.1 → kreuzberg-3.9.0}/.pre-commit-config.yaml

@@ -53,7 +53,7 @@ repos:
     hooks:
       - id: pyproject-fmt
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.2
+    rev: v0.12.3
     hooks:
       - id: ruff
         args: ["--fix", "--unsafe-fixes"]

kreuzberg-3.9.0/PKG-INFO

@@ -0,0 +1,269 @@
+Metadata-Version: 2.4
+Name: kreuzberg
+Version: 3.9.0
+Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
+Project-URL: documentation, https://kreuzberg.dev
+Project-URL: homepage, https://github.com/Goldziher/kreuzberg
+Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: async,document-analysis,document-classification,document-intelligence,document-processing,extensible,information-extraction,mcp,metadata-extraction,model-context-protocol,ocr,pandoc,pdf-extraction,pdfium,plugin-architecture,rag,retrieval-augmented-generation,structured-data,table-extraction,tesseract,text-extraction
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Multimedia :: Graphics :: Capture :: Scanners
+Classifier: Topic :: Office/Business :: Office Suites
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: General
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: anyio>=4.9.0
+Requires-Dist: chardetng-py>=0.3.4
+Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
+Requires-Dist: html-to-markdown[lxml]>=1.8.0
+Requires-Dist: mcp>=1.11.0
+Requires-Dist: msgspec>=0.18.0
+Requires-Dist: playa-pdf>=0.6.1
+Requires-Dist: psutil>=7.0.0
+Requires-Dist: pypdfium2==4.30.0
+Requires-Dist: python-calamine>=0.3.2
+Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: typing-extensions>=4.14.0; python_version < '3.12'
+Provides-Extra: additional-extensions
+Requires-Dist: mailparse>=1.0.15; extra == 'additional-extensions'
+Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'additional-extensions'
+Provides-Extra: all
+Requires-Dist: click>=8.2.1; extra == 'all'
+Requires-Dist: easyocr>=1.7.2; extra == 'all'
+Requires-Dist: fast-langdetect>=0.3.2; extra == 'all'
+Requires-Dist: gmft>=0.4.2; extra == 'all'
+Requires-Dist: keybert>=0.9.0; extra == 'all'
+Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.16.0; extra == 'all'
+Requires-Dist: mailparse>=1.0.15; extra == 'all'
+Requires-Dist: paddleocr>=3.1.0; extra == 'all'
+Requires-Dist: paddlepaddle>=3.1.0; extra == 'all'
+Requires-Dist: rich>=14.0.0; extra == 'all'
+Requires-Dist: semantic-text-splitter>=0.27.0; extra == 'all'
+Requires-Dist: setuptools>=80.9.0; extra == 'all'
+Requires-Dist: spacy>=3.8.7; extra == 'all'
+Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'all'
+Provides-Extra: api
+Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.16.0; extra == 'api'
+Provides-Extra: auto-classify-document-type
+Requires-Dist: deep-translator>=1.11.4; extra == 'auto-classify-document-type'
+Requires-Dist: pandas>=2.3.1; extra == 'auto-classify-document-type'
+Provides-Extra: chunking
+Requires-Dist: semantic-text-splitter>=0.27.0; extra == 'chunking'
+Provides-Extra: cli
+Requires-Dist: click>=8.2.1; extra == 'cli'
+Requires-Dist: rich>=14.0.0; extra == 'cli'
+Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'cli'
+Provides-Extra: easyocr
+Requires-Dist: easyocr>=1.7.2; extra == 'easyocr'
+Provides-Extra: entity-extraction
+Requires-Dist: keybert>=0.9.0; extra == 'entity-extraction'
+Requires-Dist: spacy>=3.8.7; extra == 'entity-extraction'
+Provides-Extra: gmft
+Requires-Dist: gmft>=0.4.2; extra == 'gmft'
+Provides-Extra: langdetect
+Requires-Dist: fast-langdetect>=0.3.2; extra == 'langdetect'
+Provides-Extra: paddleocr
+Requires-Dist: paddleocr>=3.1.0; extra == 'paddleocr'
+Requires-Dist: paddlepaddle>=3.1.0; extra == 'paddleocr'
+Requires-Dist: setuptools>=80.9.0; extra == 'paddleocr'
+Description-Content-Type: text/markdown
+
+# Kreuzberg
+
+[![Discord](https://img.shields.io/badge/Discord-Join%20our%20community-7289da)](https://discord.gg/pXxagNK2zN)
+[![PyPI version](https://badge.fury.io/py/kreuzberg.svg)](https://badge.fury.io/py/kreuzberg)
+[![Documentation](https://img.shields.io/badge/docs-kreuzberg.dev-blue)](https://kreuzberg.dev/)
+[![Benchmarks](https://img.shields.io/badge/benchmarks-fastest%20CPU-orange)](https://benchmarks.kreuzberg.dev/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![DeepSource](https://app.deepsource.com/gh/Goldziher/kreuzberg.svg/?label=code+coverage&show_trend=true&token=U8AW1VWWSLwVhrbtL8LmLBDN)](https://app.deepsource.com/gh/Goldziher/kreuzberg/)
+
+**A document intelligence framework for Python.** Extract text, metadata, and structured information from diverse document formats through a unified, extensible API. Built on established open source foundations including Pandoc, PDFium, and Tesseract.
+
+📖 **[Complete Documentation](https://kreuzberg.dev/)**
+
+## Framework Overview
+
+### Document Intelligence Capabilities
+
+- **Text Extraction**: High-fidelity text extraction preserving document structure and formatting
+- **Metadata Extraction**: Comprehensive metadata including author, creation date, language, and document properties
+- **Format Support**: 18 document types including PDF, Microsoft Office, images, HTML, and structured data formats
+- **OCR Integration**: Multiple OCR engines (Tesseract, EasyOCR, PaddleOCR) with automatic fallback
+- **Table Detection**: Structured table extraction with cell-level precision via GMFT integration
+- **Document Classification**: Automatic document type detection (contracts, forms, invoices, receipts, reports)
+
+### Technical Architecture
+
+- **Performance**: Highest throughput among Python document processing frameworks (30+ docs/second)
+- **Resource Efficiency**: 71MB installation, ~360MB runtime memory footprint
+- **Extensibility**: Plugin architecture for custom extractors via the Extractor base class
+- **API Design**: Synchronous and asynchronous APIs with consistent interfaces
+- **Type Safety**: Complete type annotations throughout the codebase
+
+### Open Source Foundation
+
+Kreuzberg leverages established open source technologies:
+
+- **Pandoc**: Universal document converter for robust format support
+- **PDFium**: Google's PDF rendering engine for accurate PDF processing
+- **Tesseract**: Google's OCR engine for text recognition
+- **Python-docx/pptx**: Native Microsoft Office format support
+
+## Quick Start
+
+### Extract Text with CLI
+
+```bash
+# Extract text from any file to markdown
+uvx kreuzberg extract document.pdf > output.md
+
+# With all features (OCR, table extraction, etc.)
+uvx --from "kreuzberg[all]" kreuzberg extract invoice.pdf --ocr --format markdown
+
+# Extract with rich metadata
+uvx kreuzberg extract report.pdf --show-metadata --format json
+```
+
+### Python Usage
+
+**Async (recommended for web apps):**
+
+```python
+from kreuzberg import extract_file
+
+# In your async function
+result = await extract_file("presentation.pptx")
+print(result.content)
+
+# Rich metadata extraction
+print(f"Title: {result.metadata.title}")
+print(f"Author: {result.metadata.author}")
+print(f"Page count: {result.metadata.page_count}")
+print(f"Created: {result.metadata.created_at}")
+```
+
+**Sync (for scripts and CLI tools):**
+
+```python
+from kreuzberg import extract_file_sync
+
+result = extract_file_sync("report.docx")
+print(result.content)
+
+# Access rich metadata
+print(f"Language: {result.metadata.language}")
+print(f"Word count: {result.metadata.word_count}")
+print(f"Keywords: {result.metadata.keywords}")
+```
+
+### Docker
+
+```bash
+# Run the REST API
+docker run -p 8000:8000 goldziher/kreuzberg
+
+# Extract via API
+curl -X POST -F "file=@document.pdf" http://localhost:8000/extract
+```
+
+📖 **[Installation Guide](https://kreuzberg.dev/getting-started/installation/)** • **[CLI Documentation](https://kreuzberg.dev/cli/)** • **[API Reference](https://kreuzberg.dev/api-reference/)**
+
+## Deployment Options
+
+### 🤖 MCP Server (AI Integration)
+
+**Add to Claude Desktop with one command:**
+
+```bash
+claude mcp add kreuzberg uvx -- --from "kreuzberg[all]" kreuzberg-mcp
+```
+
+**Or configure manually in `claude_desktop_config.json`:**
+
+```json
+{
+  "mcpServers": {
+    "kreuzberg": {
+      "command": "uvx",
+      "args": ["--from", "kreuzberg[all]", "kreuzberg-mcp"]
+    }
+  }
+}
+```
+
+**MCP capabilities:**
+
+- Extract text from PDFs, images, Office docs, and more
+- Full OCR support with multiple engines
+- Table extraction and metadata parsing
+
+📖 **[MCP Documentation](https://kreuzberg.dev/user-guide/mcp-server/)**
+
+## Supported Formats
+
+| Category          | Formats                        |
+| ----------------- | ------------------------------ |
+| **Documents**     | PDF, DOCX, DOC, RTF, TXT, EPUB |
+| **Images**        | JPG, PNG, TIFF, BMP, GIF, WEBP |
+| **Spreadsheets**  | XLSX, XLS, CSV, ODS            |
+| **Presentations** | PPTX, PPT, ODP                 |
+| **Web**           | HTML, XML, MHTML               |
+| **Archives**      | Support via extraction         |
+
+## 📊 Performance Characteristics
+
+[View comprehensive benchmarks](https://benchmarks.kreuzberg.dev/) • [Benchmark methodology](https://github.com/Goldziher/python-text-extraction-libs-benchmarks) • [**Detailed Analysis**](https://kreuzberg.dev/performance-analysis/)
+
+### Technical Specifications
+
+| Metric                       | Kreuzberg Sync | Kreuzberg Async | Benchmarked        |
+| ---------------------------- | -------------- | --------------- | ------------------ |
+| **Throughput (tiny files)**  | 31.78 files/s  | 23.94 files/s   | Highest throughput |
+| **Throughput (small files)** | 8.91 files/s   | 9.31 files/s    | Highest throughput |
+| **Memory footprint**         | 359.8 MB       | 395.2 MB        | Lowest usage       |
+| **Installation size**        | 71 MB          | 71 MB           | Smallest size      |
+| **Success rate**             | 100%           | 100%            | Perfect            |
+| **Supported formats**        | 18             | 18              | Comprehensive      |
+
+### Architecture Advantages
+
+- **Native C extensions**: Built on PDFium and Tesseract for maximum performance
+- **Async/await support**: True asynchronous processing with intelligent task scheduling
+- **Memory efficiency**: Streaming architecture minimizes memory allocation
+- **Process pooling**: Automatic multiprocessing for CPU-intensive operations
+- **Optimized data flow**: Efficient data handling with minimal transformations
+
+> **Benchmark details**: Tests include PDFs, Word docs, HTML, images, and spreadsheets in multiple languages (English, Hebrew, German, Chinese, Japanese, Korean) on standardized hardware.
+
+## Documentation
+
+### Quick Links
+
+- [Installation Guide](https://kreuzberg.dev/getting-started/installation/) - Setup and dependencies
+- [User Guide](https://kreuzberg.dev/user-guide/) - Comprehensive usage guide
+- [Performance Analysis](https://kreuzberg.dev/performance-analysis/) - Detailed benchmark results
+- [API Reference](https://kreuzberg.dev/api-reference/) - Complete API documentation
+- [Docker Guide](https://kreuzberg.dev/user-guide/docker/) - Container deployment
+- [REST API](https://kreuzberg.dev/user-guide/api-server/) - HTTP endpoints
+- [CLI Guide](https://kreuzberg.dev/cli/) - Command-line usage
+- [OCR Configuration](https://kreuzberg.dev/user-guide/ocr-configuration/) - OCR engine setup
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.