ocr-foundry 0.2.0__tar.gz

ocr_foundry-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,165 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  python:
+    name: "python ${{ matrix.python-version }} (rust: ${{ matrix.rust-accel }})"
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13", "3.14"]
+        rust-accel: [true, false]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install system dependencies (fonts, RAQM, FriBidi, FreeType)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libraqm0 libraqm-dev libfribidi0 libfribidi-dev libfreetype6-dev fonts-noto-core fonts-noto-cjk fonts-dejavu-core fonts-khmeros fonts-khmeros-core || sudo apt-get install -y libraqm0 libraqm-dev libfribidi0 libfribidi-dev libfreetype6-dev fonts-noto-core fonts-noto-cjk fonts-dejavu-core fonts-khmeros
+
+      - name: Set up font directories and compatibility symlinks
+        run: |
+          sudo ln -s /usr/share/fonts/truetype/noto /usr/share/fonts/noto || true
+          sudo ln -s /usr/share/fonts/opentype/noto /usr/share/fonts/noto-cjk || true
+          mkdir -p fonts/english fonts/khmer
+          ln -sf /usr/share/fonts/truetype/noto/NotoSans-Regular.ttf fonts/english/times.ttf
+          if [ -f /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf ]; then
+            ln -sf /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf fonts/english/ARIAL.TTF
+          else
+            ln -sf /usr/share/fonts/truetype/noto/NotoSans-Regular.ttf fonts/english/ARIAL.TTF
+          fi
+          ln -sf /usr/share/fonts/truetype/noto/NotoSans-Bold.ttf fonts/english/ARIALBD.TTF
+          ln -sf /usr/share/fonts/truetype/noto/NotoSans-Italic.ttf fonts/english/ARIALI.TTF
+          ln -sf /usr/share/fonts/truetype/noto/NotoSans-BoldItalic.ttf fonts/english/ARIALBI.TTF
+          ln -sf /usr/share/fonts/truetype/noto/NotoSansKhmer-Regular.ttf fonts/khmer/NotoSansKhmer-Regular.ttf
+          ln -sf /usr/share/fonts/truetype/noto/NotoSansKhmer-Bold.ttf fonts/khmer/NotoSansKhmer-Bold.ttf
+          if [ -f /usr/share/fonts/truetype/ttf-khmeros-core/KhmerOS.ttf ]; then
+            ln -sf /usr/share/fonts/truetype/ttf-khmeros-core/KhmerOS.ttf fonts/khmer/KhmerOS.ttf
+          elif [ -f /usr/share/fonts/truetype/khmeros/KhmerOS.ttf ]; then
+            ln -sf /usr/share/fonts/truetype/khmeros/KhmerOS.ttf fonts/khmer/KhmerOS.ttf
+          fi
+
+      - name: Install Rust toolchain
+        if: matrix.rust-accel
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "1.99.0"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.13.7"
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Sync dependencies with native Rust extension
+        if: matrix.rust-accel
+        run: uv sync --extra dev --extra rust --python ${{ matrix.python-version }}
+
+      - name: Sync dependencies pure-Python (no Rust extension)
+        if: ${{ !matrix.rust-accel }}
+        run: uv sync --extra dev --python ${{ matrix.python-version }}
+
+      - name: Assert native Rust acceleration is active
+        if: matrix.rust-accel
+        run: |
+          uv run python -c "from khocr_gen._rust_accel import HAS_RUST_ACCEL, _native; assert HAS_RUST_ACCEL and _native is not None, 'Native acceleration must be active'"
+
+      - name: Assert native extension is absent and pure-Python fallback is active
+        if: ${{ !matrix.rust-accel }}
+        run: |
+          uv run python -c "from khocr_gen._rust_accel import HAS_RUST_ACCEL, _native; assert not HAS_RUST_ACCEL and _native is None, 'Pure-Python fallback must be active without native extension'"
+
+      - name: Lint (ruff)
+        run: uv run ruff check .
+
+      - name: Type check (ty)
+        run: uv run ty check
+
+      - name: Test (pytest with coverage)
+        run: uv run pytest --cov=khocr_gen --cov-report=term-missing
+
+      - name: CLI rendering smoke test (words, tables, transforms, vertical, fallback)
+        run: |
+          mkdir -p /tmp/smoke_fonts /tmp/smoke_in /tmp/smoke_out
+          ln -sf /usr/share/fonts/truetype/noto/NotoSans-Regular.ttf /tmp/smoke_fonts/NotoSans-Regular.ttf || ln -sf /usr/share/fonts/noto/NotoSans-Regular.ttf /tmp/smoke_fonts/NotoSans-Regular.ttf
+          ln -sf /usr/share/fonts/truetype/noto/NotoSansKhmer-Regular.ttf /tmp/smoke_fonts/NotoSansKhmer-Regular.ttf || ln -sf /usr/share/fonts/noto/NotoSansKhmer-Regular.ttf /tmp/smoke_fonts/NotoSansKhmer-Regular.ttf
+          ln -sf /usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc /tmp/smoke_fonts/NotoSerifCJK-Regular.ttc || ln -sf /usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc /tmp/smoke_fonts/NotoSerifCJK-Regular.ttc
+
+          cat << 'EOF' > /tmp/smoke_in/corpus_horizontal.txt
+          Hello world 123
+          សួស្តី Hello 123
+          Testing line layout and table cells
+          EOF
+
+          uv run ocr-foundry layout \
+            --corpus /tmp/smoke_in/corpus_horizontal.txt \
+            --output /tmp/smoke_out/horizontal \
+            --fonts /tmp/smoke_fonts \
+            --multilingual \
+            --cluster-font-fallback \
+            --include-words \
+            --enable-tables --table-rows 2 --table-cols 2 \
+            --writing-mode horizontal-tb \
+            --page-width 500 --page-height 400 --margin 20 \
+            --page-scale 1.1 --page-rotation 5 --page-flip both \
+            --page-elastic-amplitude 2 --page-perspective 0 2 -2 0 0 -2 2 0
+
+          test -f /tmp/smoke_out/horizontal/annotations.jsonl
+          test -d /tmp/smoke_out/horizontal/pages
+
+          cat << 'EOF' > /tmp/smoke_in/corpus_vertical.txt
+          日本 語文
+          縦書 文字
+          EOF
+
+          uv run ocr-foundry layout \
+            --corpus /tmp/smoke_in/corpus_vertical.txt \
+            --output /tmp/smoke_out/vertical \
+            --fonts /tmp/smoke_fonts \
+            --multilingual \
+            --include-words \
+            --enable-tables --table-rows 1 --table-cols 2 \
+            --writing-mode vertical-rl \
+            --page-width 500 --page-height 400 --margin 20
+
+          test -f /tmp/smoke_out/vertical/annotations.jsonl
+          test -d /tmp/smoke_out/vertical/pages
+
+  rust:
+    name: rust (fmt, clippy, build)
+    runs-on: ubuntu-24.04
+    defaults:
+      run:
+        working-directory: rust
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "1.99.0"
+          components: rustfmt, clippy
+
+      - name: Cache cargo
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+
+      - name: Format check
+        run: cargo fmt --check
+
+      - name: Clippy
+        run: cargo clippy --all-targets -- -D warnings
+
+      - name: Build (release)
+        run: cargo build --release

ocr_foundry-0.2.0/.github/workflows/publish-deprecated-khocr-gen.yml

@@ -0,0 +1,31 @@
+name: Publish deprecated khocr-gen
+
+on:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Build sdist and wheel
+        working-directory: packaging/khocr-gen
+        run: uv build
+
+      - name: Publish to PyPI
+        working-directory: packaging/khocr-gen
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          if [ -z "${PYPI_TOKEN}" ]; then
+            echo "PYPI_TOKEN is empty; refusing to publish." >&2
+            exit 1
+          fi
+          uv publish --token "$PYPI_TOKEN"

ocr_foundry-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags: ["*"]
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Publish to PyPI
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          if [ -z "${PYPI_TOKEN}" ]; then
+            echo "PYPI_TOKEN is empty; refusing to publish." >&2
+            exit 1
+          fi
+          uv publish --token "$PYPI_TOKEN"

ocr_foundry-0.2.0/.gitignore

@@ -0,0 +1,18 @@
+/.pytest_cache/
+/.ruff_cache/
+/.venv/
+/.superpowers/
+/.claude/
+
+/rust/target/
+/dist/
+/corpus/
+/fonts*/
+/data/
+/docs/superpowers/
+
+__pycache__/
+uv.lock
+.coverage
+.coverage.*
+htmlcov/

ocr_foundry-0.2.0/AGENTS.md

@@ -0,0 +1,71 @@
+# AGENTS.md
+
+`khocr-gen` generates synthetic OCR training data for mixed Khmer/English text.
+
+Python 3.12+ owns the pipeline. `rust/` provides optional PyO3 acceleration. Keep the Python path working when the Rust extension is unavailable.
+
+## Read first
+
+- `src/khocr_gen/config.py` for the generation option surface.
+- `docs/` for reference behavior.
+- `wiki/` for user-facing documentation.
+
+Code and tests are the source of truth when documentation disagrees.
+
+## Layout
+
+- `src/khocr_gen/` — CLI, rendering, normalization, augmentation, generation, storage.
+- `rust/` — optional `khocr-gen-core` acceleration.
+- `configs/` — example generation and combine configs.
+- `tests/` — pytest suite.
+- `docs/` — detailed reference docs.
+- `wiki/` — source for the GitHub wiki.
+
+## Invariants
+
+- OCR labels must match the text represented by the rendered image.
+- Augmentations may degrade appearance, not silently change label semantics.
+- Keep Khmer normalization behavior explicit and tested.
+- CLI flags override YAML values; YAML overrides built-in defaults.
+- Seeded generation should stay reproducible where the pipeline promises determinism.
+- Dataset splits must remain disjoint according to their documented semantics.
+- Variable-height rendering must not clip glyphs.
+- Rust acceleration is optional. Preserve a compatible Python fallback.
+
+## Working rules
+
+Inspect the relevant implementation and tests before editing.
+
+Keep changes scoped. Don't mix unrelated cleanup into behavioral work.
+
+When adding or changing an option, update its parser/config path, tests, example config where useful, and affected docs.
+
+When touching Rust acceleration, check the Python fallback and boundary in `src/khocr_gen/_rust_accel.py`.
+
+Don't add generated datasets, fonts, build output, or local environments to the repository.
+
+## Validation
+
+For Python changes:
+
+```bash
+uv sync --extra dev
+uv run ruff check .
+uv run pytest --cov=khocr_gen --cov-report=term-missing
+uv run ty check
+```
+
+`ty` is currently informational in CI.
+
+For Rust changes:
+
+```bash
+cd rust
+cargo fmt --check
+cargo clippy --all-targets -- -D warnings
+cargo build --release
+```
+
+Existing Rust format or clippy debt may be non-blocking in CI. Don't introduce new failures.
+
+Run focused tests while iterating, then the relevant full checks before claiming completion.