kreuzberg 3.11.3__tar.gz → 3.13.0__tar.gz

{kreuzberg-3.11.3 → kreuzberg-3.13.0}/.docker/Dockerfile

@@ -13,7 +13,15 @@ COPY kreuzberg kreuzberg
 
 # Install dependencies with optimizations
 RUN --mount=type=cache,target=/tmp/uv-cache \
-    uv sync --extra api${EXTRAS:+ --extra ${EXTRAS}} --no-editable --no-dev --compile-bytecode && \
+    if [ -z "$EXTRAS" ]; then \
+        uv sync --extra api --no-editable --no-dev --compile-bytecode; \
+    else \
+        extras_args="--extra api"; \
+        for extra in $EXTRAS; do \
+            extras_args="$extras_args --extra $extra"; \
+        done; \
+        uv sync $extras_args --no-editable --no-dev --compile-bytecode; \
+    fi && \
     rm -rf /app/.venv/lib/python*/site-packages/**/__pycache__ && \
     find /app/.venv -type f -name "*.pyc" -delete && \
     find /app/.venv -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true && \
@@ -28,11 +36,24 @@ ENV PYTHONUNBUFFERED=1
 ENV PATH="/app/.venv/bin:$PATH"
 
 # Install runtime dependencies
+# Languages included: English (default), Spanish, French, German, Italian, Portuguese,
+# Chinese (simplified & traditional), Japanese, Arabic, Russian, Hindi ~keep
 RUN apt-get update && apt-get install -y --no-install-recommends \
     pandoc \
     tesseract-ocr \
     tesseract-ocr-eng \
     tesseract-ocr-osd \
+    tesseract-ocr-spa \
+    tesseract-ocr-fra \
+    tesseract-ocr-deu \
+    tesseract-ocr-ita \
+    tesseract-ocr-por \
+    tesseract-ocr-chi-sim \
+    tesseract-ocr-chi-tra \
+    tesseract-ocr-jpn \
+    tesseract-ocr-ara \
+    tesseract-ocr-rus \
+    tesseract-ocr-hin \
     libglib2.0-0 \
     libsm6 \
     libxext6 \
@@ -46,9 +67,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY --from=builder /app/.venv /app/.venv
 COPY --from=builder /app/kreuzberg /app/kreuzberg
 
-# Create non-root user
+# Create non-root user and cache directory
 RUN groupadd -r appuser && useradd -r -g appuser -d /app -s /sbin/nologin appuser && \
+    mkdir -p /app/.kreuzberg && \
     chown -R appuser:appuser /app
 
+# Set default cache directory to prevent permission issues
+ENV KREUZBERG_CACHE_DIR=/app/.kreuzberg
+
 USER appuser
 CMD ["litestar", "--app", "kreuzberg._api.main:app", "run", "--host", "0.0.0.0"]

kreuzberg-3.13.0/.docker/README.md

@@ -0,0 +1,190 @@
+# Kreuzberg Docker Images
+
+[![GitHub](https://img.shields.io/badge/GitHub-Goldziher%2Fkreuzberg-blue)](https://github.com/Goldziher/kreuzberg)
+[![PyPI](https://badge.fury.io/py/kreuzberg.svg)](https://badge.fury.io/py/kreuzberg)
+[![Documentation](https://img.shields.io/badge/docs-kreuzberg.dev-blue)](https://kreuzberg.dev/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/Goldziher/kreuzberg/blob/main/LICENSE)
+
+High-performance Python library for text extraction from documents, available as optimized Docker images.
+
+**Source Code**: [github.com/Goldziher/kreuzberg](https://github.com/Goldziher/kreuzberg)
+
+## Quick Start
+
+```bash
+docker run -p 8000:8000 goldziher/kreuzberg:latest
+```
+
+## Available Images
+
+### Base Image (`latest`)
+
+- **Image**: `goldziher/kreuzberg:latest`
+- **Size**: ~550MB compressed
+- **Includes**: REST API server, CLI tools, Tesseract OCR with 12 business languages
+- **Languages**: English, Spanish, French, German, Italian, Portuguese, Chinese (Simplified & Traditional), Japanese, Arabic, Russian, Hindi
+- **Use cases**: Basic document processing, simple API deployments, cost-conscious workflows
+
+### Core Image (`core`)
+
+- **Image**: `goldziher/kreuzberg-core:latest`
+- **Size**: ~700MB compressed
+- **Includes**: Everything from base plus:
+    - Text chunking (semantic-text-splitter)
+    - Encrypted PDF support (crypto)
+    - Document classification
+    - Language detection
+    - Email parsing (.eml, .msg)
+    - Additional format extensions
+- **Use cases**: RAG applications, document intelligence, enterprise workflows, multi-language processing
+
+## Usage
+
+### Extract Files via API
+
+```bash
+# Single file with base image
+curl -X POST http://localhost:8000/extract \
+  -F "data=@document.pdf"
+
+# With core image - chunking for RAG
+docker run -p 8000:8000 goldziher/kreuzberg-core:latest
+curl -X POST http://localhost:8000/extract \
+  -F "data=@document.pdf" \
+  -F "chunk_content=true" \
+  -F "max_chars=1000"
+
+# Language detection
+curl -X POST http://localhost:8000/extract \
+  -F "data=@document.pdf" \
+  -F "auto_detect_language=true"
+
+# Encrypted PDF
+curl -X POST http://localhost:8000/extract \
+  -F "data=@encrypted.pdf" \
+  -F "password=secretpassword"
+```
+
+### Docker Compose
+
+```yaml
+version: '3.8'
+
+services:
+  kreuzberg:
+    image: goldziher/kreuzberg-core:latest
+    ports:
+      - "8000:8000"
+    volumes:
+      - kreuzberg-cache:/app/.kreuzberg
+    environment:
+      - PYTHONUNBUFFERED=1
+      - KREUZBERG_CACHE_DIR=/app/.kreuzberg
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+volumes:
+  kreuzberg-cache:
+```
+
+## Custom Images
+
+Create tailored images for your specific needs:
+
+### Example: RAG-Optimized
+
+```dockerfile
+FROM goldziher/kreuzberg:latest
+
+USER root
+
+# Add chunking and language detection
+RUN pip install --upgrade "kreuzberg[chunking,langdetect]"
+
+USER appuser
+
+# RAG-optimized defaults
+ENV KREUZBERG_CHUNK_CONTENT=true
+ENV KREUZBERG_MAX_CHARS=1000
+ENV KREUZBERG_AUTO_DETECT_LANGUAGE=true
+```
+
+### Example: Crypto Support
+
+```dockerfile
+FROM goldziher/kreuzberg:latest
+
+USER root
+
+# Add encrypted PDF support
+RUN pip install --upgrade "kreuzberg[crypto]"
+
+USER appuser
+```
+
+## Configuration
+
+### Environment Variables
+
+- `KREUZBERG_CACHE_DIR`: Cache directory (default: `/app/.kreuzberg`)
+- `KREUZBERG_CHUNK_CONTENT`: Enable chunking (`true`/`false`)
+- `KREUZBERG_AUTO_DETECT_LANGUAGE`: Enable language detection (`true`/`false`)
+- `KREUZBERG_OCR_BACKEND`: OCR backend (`tesseract` or `none`)
+
+### Configuration File
+
+Mount `kreuzberg.toml`:
+
+```toml
+chunk_content = true
+auto_detect_language = true
+max_chars = 1000
+ocr_backend = "tesseract"
+
+[tesseract]
+language = "eng+spa+fra+deu"
+psm = 6
+```
+
+```bash
+docker run -p 8000:8000 \
+  -v "$(pwd)/kreuzberg.toml:/app/kreuzberg.toml:ro" \
+  goldziher/kreuzberg-core:latest
+```
+
+## Features
+
+- **🚀 High Performance**: Optimized for speed and efficiency
+- **📄 Multiple Formats**: PDF, DOCX, images, HTML, and more
+- **🔍 OCR Support**: Built-in Tesseract with 12 business languages
+- **🔒 Secure**: Runs as non-root user, no external API calls
+- **📦 Ready to Use**: Pre-configured API server
+
+## Documentation
+
+- **[GitHub Repository](https://github.com/Goldziher/kreuzberg)** - Source code and issue tracking
+- **[Full Documentation](https://kreuzberg.dev/)** - Complete user guide and API reference
+- **[API Documentation](https://kreuzberg.dev/user-guide/api-server/)** - REST API endpoints and usage
+- **[Docker Guide](https://kreuzberg.dev/user-guide/docker/)** - Detailed Docker usage guide
+
+## Support
+
+- **Issues**: [github.com/Goldziher/kreuzberg/issues](https://github.com/Goldziher/kreuzberg/issues)
+- **Discussions**: [github.com/Goldziher/kreuzberg/discussions](https://github.com/Goldziher/kreuzberg/discussions)
+- **Discord**: [Join our community](https://discord.gg/pXxagNK2zN)
+
+## Contributing
+
+Contributions are welcome! See our [Contributing Guide](https://github.com/Goldziher/kreuzberg/blob/main/docs/contributing.md).
+
+## License
+
+MIT License - see [LICENSE](https://github.com/Goldziher/kreuzberg/blob/main/LICENSE) for details.
+
+______________________________________________________________________
+
+Made with ❤️ by the [Kreuzberg contributors](https://github.com/Goldziher/kreuzberg/graphs/contributors)

{kreuzberg-3.11.3 → kreuzberg-3.13.0}/.github/workflows/ci.yaml

@@ -51,7 +51,6 @@ jobs:
       - name: Execute Pre-Commit
         run: uv run pre-commit run --show-diff-on-failure --color=always --all-files
 
-  # Coverage job runs first, only on Python 3.13 Ubuntu
   coverage:
     needs: validate
     runs-on: ubuntu-latest
@@ -135,7 +134,6 @@ jobs:
             .coverage
           retention-days: 7
 
-  # Full test matrix runs only after coverage succeeds
   test:
     needs: coverage
     runs-on: ${{ matrix.os }}
@@ -144,10 +142,6 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python: ["3.10", "3.11", "3.12", "3.13"]
-        exclude:
-          # Skip Python 3.13 on macOS for now due to compatibility issues
-          - os: macos-latest
-            python: "3.13"
     timeout-minutes: 30
     steps:
       - name: Checkout

{kreuzberg-3.11.3 → kreuzberg-3.13.0}/.github/workflows/docker-e2e-tests.yml

@@ -11,10 +11,8 @@ jobs:
     strategy:
       matrix:
         image:
-          - { name: "core", extras: "" }
-          - { name: "easyocr", extras: "easyocr" }
-          - { name: "paddle", extras: "paddleocr" }
-          - { name: "gmft", extras: "gmft" }
+          - { name: "base", extras: "cli" }
+          - { name: "core", extras: "cli chunking crypto document-classification langdetect additional-extensions" }
       fail-fast: false
 
     steps:
@@ -92,9 +90,7 @@ jobs:
           docker build -f .docker/Dockerfile \
             --build-arg EXTRAS="${{ matrix.image.extras }}" \
             -t kreuzberg:${{ matrix.image.name }} \
-            --cache-from type=gha \
-            --cache-to type=gha,mode=max \
-            --load \
+            --no-cache \
             .
 
           echo "Built image:"
@@ -104,7 +100,7 @@ jobs:
         run: |
           mkdir -p tests/e2e/logs
           echo "Running E2E tests for ${{ matrix.image.name }}..."
-          python3 tests/e2e/docker_images_test.py --image ${{ matrix.image.name }}
+          python3 tests/e2e/docker_e2e_test.py --image ${{ matrix.image.name }}
 
       - name: Generate test report - ${{ matrix.image.name }}
         if: always()

{kreuzberg-3.11.3 → kreuzberg-3.13.0}/.github/workflows/docs.yml

@@ -50,7 +50,7 @@ jobs:
           uv run mkdocs build --clean --strict
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v4
         with:
           path: ./site
 

{kreuzberg-3.11.3 → kreuzberg-3.13.0}/.github/workflows/publish-docker.yml

@@ -2,19 +2,32 @@ name: Publish Docker Images
 
 on:
   workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to build (leave empty to use latest git tag)'
+        required: false
+        type: string
+      build_base:
+        description: 'Build base image'
+        required: true
+        type: boolean
+        default: true
+      build_core:
+        description: 'Build core image'
+        required: true
+        type: boolean
+        default: true
   release:
     types: [published]
 
 jobs:
-  # Run E2E tests first
   test-images:
     uses: ./.github/workflows/docker-e2e-tests.yml
 
-  # Build and publish images after tests pass
   build-and-publish:
     needs: test-images
-    runs-on: ubuntu-latest
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'release' }}
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
@@ -22,34 +35,49 @@ jobs:
       version: ${{ steps.get_version.outputs.VERSION }}
 
     strategy:
-      max-parallel: 2
+      max-parallel: 1  # Build one at a time to save disk space ~keep
       matrix:
         include:
+          - name: base
+            extras: "cli"
+            image_name: "goldziher/kreuzberg"
+            should_build: ${{ github.event_name == 'release' || inputs.build_base == true }}
           - name: core
-            extras: ""
-            tag_suffix: ""
-          - name: easyocr
-            extras: "easyocr"
-            tag_suffix: "-easyocr"
-          - name: paddle
-            extras: "paddleocr"
-            tag_suffix: "-paddle"
-          - name: gmft
-            extras: "gmft"
-            tag_suffix: "-gmft"
-          - name: all
-            extras: "all"
-            tag_suffix: "-all"
+            extras: "cli chunking crypto document-classification langdetect additional-extensions"
+            image_name: "goldziher/kreuzberg-core"
+            should_build: ${{ github.event_name == 'release' || inputs.build_core == true }}
 
     steps:
       - name: Free up disk space
         run: |
+          echo "Initial disk space:"
+          df -h /
+
+          # Remove unnecessary large directories (saves ~30GB) ~keep
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/ghc
           sudo rm -rf /opt/hostedtoolcache/CodeQL
-          sudo docker system prune -af
-          df -h
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/local/lib/node_modules
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache
+
+          # Clean apt
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+
+          # Remove swap to free up space
+          sudo swapoff -a
+          sudo rm -f /swapfile
+
+          # Clean Docker completely
+          docker system prune -af --volumes || true
+          docker builder prune -af || true
+
+          echo "Available disk space after cleanup:"
+          df -h /
 
       - name: Checkout repository
         uses: actions/checkout@v5
@@ -61,6 +89,8 @@ jobs:
         run: |
           if [ "${{ github.event_name }}" = "release" ]; then
             VERSION="${{ github.event.release.tag_name }}"
+          elif [ -n "${{ inputs.version }}" ]; then
+            VERSION="${{ inputs.version }}"
           else
             git fetch --tags
             VERSION=$(git tag --sort=-version:refname | head -n1)
@@ -81,15 +111,17 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
+        if: ${{ matrix.should_build }}
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: goldziher/kreuzberg
+          images: ${{ matrix.image_name }}
           tags: |
-            type=raw,value=${{ steps.get_version.outputs.VERSION }}${{ matrix.tag_suffix }}
-            type=raw,value=latest${{ matrix.tag_suffix }}
+            type=raw,value=${{ steps.get_version.outputs.VERSION }}
+            type=raw,value=latest
 
       - name: Build and push Docker image to Docker Hub
+        if: ${{ matrix.should_build }}
         uses: docker/build-push-action@v6
         with:
           context: .
@@ -100,15 +132,32 @@ jobs:
             EXTRAS=${{ matrix.extras }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          no-cache: true
+
+      - name: Clean up after build
+        if: always()
+        run: |
+          # Remove all Docker images and containers
+          docker stop $(docker ps -aq) || true
+          docker rm $(docker ps -aq) || true
+          docker rmi $(docker images -q) || true
+
+          # Clean all Docker data
+          docker system prune -af --volumes || true
+          docker builder prune -af || true
+
+          # Clear buildkit cache
+          docker buildx prune -af || true
+
+          echo "Disk space after cleanup:"
+          df -h /
 
       - name: Update Docker Hub README
+        if: ${{ matrix.should_build }}
         uses: peter-evans/dockerhub-description@v4
-        if: matrix.name == 'core'
         continue-on-error: true
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-          repository: goldziher/kreuzberg
+          repository: ${{ matrix.image_name }}
           readme-filepath: ./.docker/README.md

kreuzberg-3.13.0/.github/workflows/test-docker-builds.yml

@@ -0,0 +1,97 @@
+name: Test Docker Builds (No Push)
+
+on:
+  workflow_dispatch:
+
+jobs:
+  test-build-all-images:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1  # Build one at a time to save disk space ~keep
+      matrix:
+        include:
+          - name: base
+            extras: "cli"
+            image_name: "goldziher/kreuzberg"
+          - name: core
+            extras: "cli,chunking,crypto,document-classification,langdetect,additional-extensions"
+            image_name: "goldziher/kreuzberg-core"
+
+    steps:
+      - name: Free up disk space
+        run: |
+          echo "Initial disk space:"
+          df -h /
+
+          # Remove unnecessary large directories (saves ~30GB)
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/local/lib/node_modules
+          sudo rm -rf /opt/microsoft
+          sudo rm -rf /usr/local/.ghcup
+          sudo rm -rf /opt/hostedtoolcache
+
+          # Clean apt
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+
+          # Remove swap to free up space
+          sudo swapoff -a
+          sudo rm -f /swapfile
+
+          # Clean Docker completely
+          docker system prune -af --volumes || true
+          docker builder prune -af || true
+
+          echo "Available disk space after cleanup:"
+          df -h /
+
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image - ${{ matrix.name }}
+        run: |
+          echo "===================="
+          echo "Building ${{ matrix.name }} image"
+          echo "===================="
+
+          docker buildx build \
+            --file ./.docker/Dockerfile \
+            --build-arg EXTRAS="${{ matrix.extras }}" \
+            --tag ${{ matrix.image_name }}:test \
+            --platform linux/amd64 \
+            --no-cache \
+            --load \
+            .
+
+          echo "Image built successfully:"
+          docker images | grep "${{ matrix.image_name }}:test" || true
+
+          echo "Current disk usage:"
+          df -h /
+
+      - name: Clean up after build
+        if: always()
+        run: |
+          echo "Cleaning up Docker resources..."
+
+          # Remove all Docker images and containers
+          docker stop $(docker ps -aq) || true
+          docker rm $(docker ps -aq) || true
+          docker rmi $(docker images -q) || true
+
+          # Clean all Docker data
+          docker system prune -af --volumes || true
+          docker builder prune -af || true
+
+          # Clear buildkit cache
+          docker buildx prune -af || true
+
+          echo "Disk space after cleanup:"
+          df -h /

{kreuzberg-3.11.3 → kreuzberg-3.13.0}/.gitignore

@@ -39,3 +39,30 @@ requirements.txt
 site/
 .cache/
 dist/
+build/
+.task/
+tests/e2e/test_report.json
+tests/e2e/logs/
+
+# Additional build artifacts
+*.whl
+*.tar.gz
+.tox/
+.nox/
+wheels/
+share/python-wheels/
+
+# Documentation builds
+docs/_build/
+docs/build/
+
+# Node.js (if any frontend tools are used)
+node_modules/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Temporary files
+*.tmp
+*.temp
+.tmp/

{kreuzberg-3.11.3 → kreuzberg-3.13.0}/.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
         stages: [commit-msg]
         additional_dependencies: ["@commitlint/config-conventional"]
   - repo: https://github.com/Goldziher/ai-rulez
-    rev: v1.6.0
+    rev: v1.6.1
     hooks:
       - id: ai-rulez-validate
       - id: ai-rulez-generate
@@ -53,7 +53,7 @@ repos:
     hooks:
       - id: pyproject-fmt
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.10
+    rev: v0.12.11
     hooks:
       - id: ruff
         args: ["--fix", "--unsafe-fixes"]
@@ -66,7 +66,7 @@ repos:
         additional_dependencies:
           - tomli
   - repo: https://github.com/jsh9/pydoclint
-    rev: 0.6.10
+    rev: 0.7.3
     hooks:
       - id: pydoclint
         args: