kreuzberg 3.11.2__tar.gz → 3.11.4__tar.gz

kreuzberg-3.11.4/.docker/Dockerfile

@@ -0,0 +1,58 @@
+# Build stage
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm AS builder
+ARG EXTRAS=""
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV UV_LINK_MODE=copy
+ENV UV_CACHE_DIR=/tmp/uv-cache
+
+# Copy dependency files
+COPY pyproject.toml uv.lock README.md ./
+COPY kreuzberg kreuzberg
+
+# Install dependencies with optimizations
+RUN --mount=type=cache,target=/tmp/uv-cache \
+    uv sync --extra api${EXTRAS:+ --extra ${EXTRAS}} --no-editable --no-dev --compile-bytecode && \
+    rm -rf /app/.venv/lib/python*/site-packages/**/__pycache__ && \
+    find /app/.venv -type f -name "*.pyc" -delete && \
+    find /app/.venv -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true && \
+    find /app/.venv -type d -name "test" -exec rm -rf {} + 2>/dev/null || true
+
+# Runtime stage
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS runtime
+ARG EXTRAS=""
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pandoc \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    tesseract-ocr-osd \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    libgomp1 \
+    libgl1 \
+    libglib2.0-0 \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Copy virtual environment from builder
+COPY --from=builder /app/.venv /app/.venv
+COPY --from=builder /app/kreuzberg /app/kreuzberg
+
+# Create non-root user and cache directory
+RUN groupadd -r appuser && useradd -r -g appuser -d /app -s /sbin/nologin appuser && \
+    mkdir -p /app/.kreuzberg && \
+    chown -R appuser:appuser /app
+
+# Set default cache directory to prevent permission issues
+ENV KREUZBERG_CACHE_DIR=/app/.kreuzberg
+
+USER appuser
+CMD ["litestar", "--app", "kreuzberg._api.main:app", "run", "--host", "0.0.0.0"]

kreuzberg-3.11.4/.github/workflows/docker-e2e-tests.yml

@@ -0,0 +1,152 @@
+name: Docker E2E Tests
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  test-docker-images:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        image:
+          - { name: "core", extras: "" }
+          - { name: "easyocr", extras: "easyocr" }
+          - { name: "paddle", extras: "paddleocr" }
+          - { name: "gmft", extras: "gmft" }
+          - { name: "all", extras: "all" }
+      fail-fast: false
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install test dependencies
+        run: |
+          uv pip install --system asyncio
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc tesseract-ocr jq bc
+
+      - name: Free up disk space
+        run: |
+          echo "Initial disk usage:"
+          df -h
+
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+          sudo apt-get clean
+          sudo apt-get autoremove -y
+
+          docker system prune -af --volumes
+
+          sudo swapoff -a
+          sudo rm -f /swapfile
+
+          echo "Available disk space after cleanup:"
+          df -h
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:latest
+            network=host
+
+      - name: Configure Docker
+        run: |
+          sudo tee /etc/docker/daemon.json > /dev/null <<EOF
+          {
+            "max-concurrent-downloads": 10,
+            "max-concurrent-uploads": 10,
+            "storage-driver": "overlay2"
+          }
+          EOF
+
+          sudo systemctl restart docker
+          docker info
+
+      - name: Build Docker image - ${{ matrix.image.name }}
+        run: |
+          export DOCKER_BUILDKIT=1
+          export BUILDKIT_PROGRESS=plain
+
+          echo "Building ${{ matrix.image.name }} image..."
+          docker build -f .docker/Dockerfile \
+            --build-arg EXTRAS="${{ matrix.image.extras }}" \
+            -t kreuzberg:${{ matrix.image.name }} \
+            --cache-from type=gha \
+            --cache-to type=gha,mode=max \
+            --load \
+            .
+
+          echo "Built image:"
+          docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" | grep kreuzberg:${{ matrix.image.name }} || true
+
+      - name: Run E2E tests - ${{ matrix.image.name }}
+        run: |
+          mkdir -p tests/e2e/logs
+          echo "Running E2E tests for ${{ matrix.image.name }}..."
+          python3 tests/e2e/docker_images_test.py --image ${{ matrix.image.name }}
+
+      - name: Generate test report - ${{ matrix.image.name }}
+        if: always()
+        run: |
+          if [ -f "tests/e2e/test_report.json" ]; then
+            echo "## Test Report Summary for ${{ matrix.image.name }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`json" >> $GITHUB_STEP_SUMMARY
+            jq '.["${{ matrix.image.name }}"]' tests/e2e/test_report.json >> $GITHUB_STEP_SUMMARY || echo "No results for ${{ matrix.image.name }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Collect Docker logs on failure
+        if: failure()
+        run: |
+          mkdir -p tests/e2e/docker-logs
+
+          for container in $(docker ps -a --filter "name=kreuzberg-test" --format "{{.Names}}"); do
+            docker logs "$container" > "tests/e2e/docker-logs/${container}.log" 2>&1 || true
+          done
+
+          docker info > tests/e2e/docker-logs/docker-info.txt 2>&1 || true
+          docker version > tests/e2e/docker-logs/docker-version.txt 2>&1 || true
+
+      - name: Upload test results - ${{ matrix.image.name }}
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-e2e-test-results-${{ matrix.image.name }}
+          path: |
+            tests/e2e/logs/
+            tests/e2e/*.log
+            tests/e2e/*.json
+            tests/e2e/docker-logs/
+          retention-days: 7
+
+      - name: Clean up Docker resources
+        if: always()
+        run: |
+          docker ps -aq --filter "name=kreuzberg-test" | xargs -r docker rm -f || true
+          docker rmi kreuzberg:${{ matrix.image.name }} || true
+          docker system prune -af --volumes || true
+
+          echo "Final disk usage after ${{ matrix.image.name }}:"
+          df -h

{kreuzberg-3.11.2 → kreuzberg-3.11.4}/.github/workflows/publish-docker.yml

@@ -1,4 +1,3 @@
-
 name: Publish Docker Images
 
 on:
@@ -7,12 +6,20 @@ on:
     types: [published]
 
 jobs:
-  build-and-push:
+  # Run E2E tests first
+  test-images:
+    uses: ./.github/workflows/docker-e2e-tests.yml
+
+  # Build and publish images after tests pass
+  build-and-publish:
+    needs: test-images
     runs-on: ubuntu-latest
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'release' }}
     permissions:
       contents: read
       packages: write
+    outputs:
+      version: ${{ steps.get_version.outputs.VERSION }}
 
     strategy:
       max-parallel: 2
@@ -37,7 +44,6 @@ jobs:
     steps:
       - name: Free up disk space
         run: |
-          # Remove large unnecessary packages to free up space
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/ghc
@@ -54,10 +60,8 @@ jobs:
         id: get_version
         run: |
           if [ "${{ github.event_name }}" = "release" ]; then
-            # For release events, use the release tag
             VERSION="${{ github.event.release.tag_name }}"
           else
-            # For workflow_dispatch, get the latest tag
             git fetch --tags
             VERSION=$(git tag --sort=-version:refname | head -n1)
           fi
@@ -82,12 +86,10 @@ jobs:
         with:
           images: goldziher/kreuzberg
           tags: |
-            # Release version tag (e.g., v3.0.0-easyocr)
             type=raw,value=${{ steps.get_version.outputs.VERSION }}${{ matrix.tag_suffix }}
-            # Latest tag for each variant (e.g., latest-easyocr)
             type=raw,value=latest${{ matrix.tag_suffix }}
 
-      - name: Build and push Docker image
+      - name: Build and push Docker image to Docker Hub
         uses: docker/build-push-action@v6
         with:
           context: .

{kreuzberg-3.11.2 → kreuzberg-3.11.4}/.gitignore

@@ -12,6 +12,7 @@
 .coverage*
 coverage.lcov
 htmlcov/
+.claude/
 .cursorrules
 .dist/
 .DS_store
@@ -38,3 +39,6 @@ requirements.txt
 site/
 .cache/
 dist/
+.task/
+tests/e2e/test_report.json
+tests/e2e/logs/

{kreuzberg-3.11.2 → kreuzberg-3.11.4}/.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
         stages: [commit-msg]
         additional_dependencies: ["@commitlint/config-conventional"]
   - repo: https://github.com/Goldziher/ai-rulez
-    rev: v1.5.1
+    rev: v1.6.0
     hooks:
       - id: ai-rulez-validate
       - id: ai-rulez-generate
@@ -53,7 +53,7 @@ repos:
     hooks:
       - id: pyproject-fmt
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.9
+    rev: v0.12.10
     hooks:
       - id: ruff
         args: ["--fix", "--unsafe-fixes"]

{kreuzberg-3.11.2 → kreuzberg-3.11.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.11.2
+Version: 3.11.4
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -52,9 +52,9 @@ Requires-Dist: gmft>=0.4.2; extra == 'all'
 Requires-Dist: keybert>=0.9.0; extra == 'all'
 Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.17.0; extra == 'all'
 Requires-Dist: mailparse>=1.0.15; extra == 'all'
-Requires-Dist: paddleocr>=3.1.1; extra == 'all'
-Requires-Dist: paddlepaddle>=3.1.0; extra == 'all'
-Requires-Dist: pandas>=2.3.1; extra == 'all'
+Requires-Dist: paddleocr>=3.2.0; extra == 'all'
+Requires-Dist: paddlepaddle>=3.1.1; extra == 'all'
+Requires-Dist: pandas>=2.3.2; extra == 'all'
 Requires-Dist: playa-pdf[crypto]>=0.7.0; extra == 'all'
 Requires-Dist: rich>=14.1.0; extra == 'all'
 Requires-Dist: semantic-text-splitter>=0.27.0; extra == 'all'
@@ -73,7 +73,7 @@ Provides-Extra: crypto
 Requires-Dist: playa-pdf[crypto]>=0.7.0; extra == 'crypto'
 Provides-Extra: document-classification
 Requires-Dist: deep-translator>=1.11.4; extra == 'document-classification'
-Requires-Dist: pandas>=2.3.1; extra == 'document-classification'
+Requires-Dist: pandas>=2.3.2; extra == 'document-classification'
 Provides-Extra: easyocr
 Requires-Dist: easyocr>=1.7.2; extra == 'easyocr'
 Provides-Extra: entity-extraction
@@ -84,8 +84,8 @@ Requires-Dist: gmft>=0.4.2; extra == 'gmft'
 Provides-Extra: langdetect
 Requires-Dist: fast-langdetect>=0.3.2; extra == 'langdetect'
 Provides-Extra: paddleocr
-Requires-Dist: paddleocr>=3.1.1; extra == 'paddleocr'
-Requires-Dist: paddlepaddle>=3.1.0; extra == 'paddleocr'
+Requires-Dist: paddleocr>=3.2.0; extra == 'paddleocr'
+Requires-Dist: paddlepaddle>=3.1.1; extra == 'paddleocr'
 Requires-Dist: setuptools>=80.9.0; extra == 'paddleocr'
 Description-Content-Type: text/markdown
 

kreuzberg-3.11.4/Taskfile.yml

@@ -0,0 +1,160 @@
+version: "3"
+
+env:
+  DOCKER_BUILDKIT: 1
+  BUILDKIT_PROGRESS: plain
+
+vars:
+  DOCKER_DIR: .docker
+  DOCKERFILE: "{{.DOCKER_DIR}}/Dockerfile"
+  TEST_DIR: tests/e2e
+  LOG_DIR: "{{.TEST_DIR}}/logs"
+
+tasks:
+  # Main task for Docker E2E testing
+  docker:e2e:
+    desc: "Build Docker images and run E2E tests"
+    deps:
+      - docker:build
+    cmds:
+      - uv run python {{.TEST_DIR}}/docker_images_test.py
+
+  # Docker build tasks
+  docker:build:
+    desc: "Build all Docker images for testing"
+    deps:
+      - docker:build:core
+      - docker:build:easyocr
+      - docker:build:paddle
+      - docker:build:gmft
+    cmds:
+      - docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" | grep kreuzberg || true
+
+  docker:build:core:
+    desc: "Build core Docker image"
+    cmds:
+      - docker build -f {{.DOCKERFILE}} --build-arg EXTRAS="" -t kreuzberg:core .
+    sources:
+      - "{{.DOCKERFILE}}"
+      - kreuzberg/**/*.py
+      - pyproject.toml
+
+  docker:build:easyocr:
+    desc: "Build EasyOCR Docker image"
+    cmds:
+      - docker build -f {{.DOCKERFILE}} --build-arg EXTRAS="easyocr" -t kreuzberg:easyocr .
+    sources:
+      - "{{.DOCKERFILE}}"
+      - kreuzberg/**/*.py
+      - pyproject.toml
+
+  docker:build:paddle:
+    desc: "Build PaddleOCR Docker image"
+    cmds:
+      - docker build -f {{.DOCKERFILE}} --build-arg EXTRAS="paddleocr" -t kreuzberg:paddle .
+    sources:
+      - "{{.DOCKERFILE}}"
+      - kreuzberg/**/*.py
+      - pyproject.toml
+
+  docker:build:gmft:
+    desc: "Build GMFT Docker image"
+    cmds:
+      - docker build -f {{.DOCKERFILE}} --build-arg EXTRAS="gmft" -t kreuzberg:gmft .
+    sources:
+      - "{{.DOCKERFILE}}"
+      - kreuzberg/**/*.py
+      - pyproject.toml
+
+  # Test runner variants
+  docker:test:
+    desc: "Run Docker E2E tests (images must be built)"
+    cmds:
+      - uv run python {{.TEST_DIR}}/docker_images_test.py
+
+  # Utility tasks
+  docker:clean:
+    desc: "Clean up Docker test images and containers"
+    cmds:
+      - docker ps -aq --filter "name=kreuzberg-test" | xargs -r docker rm -f 2>/dev/null || true
+      - docker rmi kreuzberg:core kreuzberg:easyocr kreuzberg:paddle kreuzberg:gmft 2>/dev/null || true
+      - docker system prune -f
+
+  docker:logs:
+    desc: "Show logs from test containers"
+    cmds:
+      - docker ps -a --filter "name=kreuzberg-test" --format "table {{.Names}}\t{{.Status}}"
+      - for container in $(docker ps -a --filter "name=kreuzberg-test" --format "{{.Names}}"); do echo "=== $container ==="; docker logs --tail 50 "$container" 2>&1 || true; done
+
+  # Development tasks
+  install:
+    desc: "Install dependencies with uv"
+    cmds:
+      - uv sync
+
+  install:all:
+    desc: "Install all optional dependencies with uv"
+    cmds:
+      - uv sync --all-extras
+
+  test:
+    desc: "Run tests with pytest"
+    cmds:
+      - uv run pytest
+
+  test:cov:
+    desc: "Run tests with coverage"
+    cmds:
+      - uv run pytest --cov
+
+  format:
+    desc: "Format code with ruff"
+    cmds:
+      - uv run ruff format
+
+  lint:
+    desc: "Lint code with ruff"
+    cmds:
+      - uv run ruff check
+
+  lint:fix:
+    desc: "Fix linting issues with ruff"
+    cmds:
+      - uv run ruff check --fix
+
+  typecheck:
+    desc: "Type check with mypy"
+    cmds:
+      - uv run mypy
+
+  # Documentation tasks
+  docs:build:
+    desc: "Build documentation"
+    cmds:
+      - uv run mkdocs build --clean --strict
+
+  docs:serve:
+    desc: "Serve documentation locally"
+    cmds:
+      - uv run mkdocs serve
+
+  # CI/CD tasks
+  ci:
+    desc: "Run full CI pipeline locally"
+    deps:
+      - format
+      - lint
+      - typecheck
+      - test:cov
+
+  ci:docker:
+    desc: "Run Docker CI pipeline"
+    deps:
+      - docker:build
+      - docker:test
+
+  # Default task
+  default:
+    desc: "Show available tasks"
+    cmds:
+      - task --list