ocr-foundry 0.2.0__py3-none-any.whl

khocr_gen/errors.py

@@ -0,0 +1,33 @@
+"""Application-level error types for khocr-gen."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(slots=True)
+class ErrorContext:
+    """Machine-readable context attached to domain-level exceptions."""
+
+    code: str
+    detail: str
+
+
+class KhocrGenError(Exception):
+    """Base application exception."""
+
+    def __init__(self, message: str, *, code: str = "internal_error") -> None:
+        super().__init__(message)
+        self.context = ErrorContext(code=code, detail=message)
+
+
+class InputValidationError(KhocrGenError):
+    """Raised when user input is invalid or incomplete."""
+
+
+class GenerationError(KhocrGenError):
+    """Raised when dataset generation fails."""
+
+
+class FontLoadError(KhocrGenError):
+    """Raised when a font file cannot be loaded."""

khocr_gen/evaluate_cmd.py

@@ -0,0 +1,50 @@
+"""Offline scoring CLI; deliberately independent of rendering/model dependencies."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import argparse
+
+from .evaluation import evaluate_files
+
+
+def add_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--reference",
+        required=True,
+        help="Reference UTF-8 JSONL with id/text and optional metadata.language",
+    )
+    parser.add_argument(
+        "--predictions", required=True, help="Prediction UTF-8 JSONL with matching id/text"
+    )
+    parser.add_argument(
+        "--normalization",
+        choices=["none", "NFC"],
+        default="none",
+        help="Always report raw scores; NFC additionally reports Unicode NFC scores",
+    )
+    parser.add_argument(
+        "--word-tokenization",
+        choices=["whitespace"],
+        default=None,
+        help="Opt in to whitespace WER; not linguistic segmentation for unspaced scripts",
+    )
+
+
+def run(args: argparse.Namespace) -> int:
+    try:
+        report = evaluate_files(
+            args.reference,
+            args.predictions,
+            normalization=args.normalization,
+            word_tokenization=args.word_tokenization,
+        )
+    except (ValueError, OSError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    print(json.dumps(report, ensure_ascii=False, allow_nan=False, indent=2))
+    return 0

khocr_gen/evaluation.py

@@ -0,0 +1,259 @@
+"""Offline, stdlib-only scoring of explicitly paired OCR JSONL transcripts.
+
+See docs/EVALUATION.md for the input schema, metric semantics, and heldout protocol.
+This module neither runs an OCR model nor loads synthetic generation dependencies.
+"""
+
+from __future__ import annotations
+
+import json
+import unicodedata
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+    from os import PathLike
+
+
+@dataclass(frozen=True)
+class _Sample:
+    text: str
+    language: str
+
+
+def _string(value: object, field: str, context: str, *, nonempty: bool = False) -> str:
+    if not isinstance(value, str) or (nonempty and not value.strip()):
+        qualifier = "nonempty " if nonempty else ""
+        raise ValueError(f"{context}: {field} must be a {qualifier}string")
+    try:
+        value.encode("utf-8")
+    except UnicodeEncodeError as exc:
+        raise ValueError(f"{context}: {field} contains an unpaired Unicode surrogate") from exc
+    return value
+
+
+def _invalid_constant(value: str) -> None:
+    raise ValueError(f"invalid JSON constant {value}")
+
+
+def _read_samples(path: str | PathLike[str], *, reference: bool) -> dict[str, _Sample]:
+    samples: dict[str, _Sample] = {}
+    role = "reference" if reference else "prediction"
+    try:
+        with Path(path).open(encoding="utf-8") as stream:
+            for line_number, line in enumerate(stream, 1):
+                context = f"{role} {path}:{line_number}"
+                try:
+                    record = json.loads(line, parse_constant=_invalid_constant)
+                except ValueError as exc:
+                    raise ValueError(f"{context}: invalid JSON: {exc}") from exc
+                if not isinstance(record, dict):
+                    raise ValueError(f"{context}: each record must be a JSON object")
+                sample_id = _string(record.get("id"), "id", context, nonempty=True)
+                if sample_id in samples:
+                    raise ValueError(f"{context}: duplicate id {sample_id!r}")
+                text = _string(record.get("text"), "text", context)
+                language = "und"
+                if reference:
+                    metadata = record.get("metadata", {})
+                    if not isinstance(metadata, dict):
+                        raise ValueError(f"{context}: metadata must be a JSON object")
+                    language = _string(
+                        metadata.get("language", "und"),
+                        "metadata.language",
+                        context,
+                        nonempty=True,
+                    )
+                samples[sample_id] = _Sample(text, language)
+    except UnicodeDecodeError as exc:
+        raise ValueError(f"{role} {path}: invalid UTF-8: {exc}") from exc
+    if not samples:
+        raise ValueError(f"{role} {path}: empty corpus")
+    return samples
+
+
+def _edit_distance(reference: Sequence[str], prediction: Sequence[str]) -> int:
+    """Unit-cost Levenshtein distance, with memory linear in the shorter sequence."""
+    if len(reference) < len(prediction):
+        reference, prediction = prediction, reference
+    previous = list(range(len(prediction) + 1))
+    for row, expected in enumerate(reference, 1):
+        current = [row]
+        for column, actual in enumerate(prediction, 1):
+            current.append(
+                min(
+                    previous[column] + 1,
+                    current[column - 1] + 1,
+                    previous[column - 1] + (expected != actual),
+                )
+            )
+        previous = current
+    return previous[-1]
+
+
+@dataclass
+class _Counts:
+    samples: int = 0
+    exact_lines: int = 0
+    empty_reference_lines: int = 0
+    character_edits: int = 0
+    reference_characters: int = 0
+    word_edits: int = 0
+    reference_words: int = 0
+
+    def metrics(self, *, words: bool) -> dict[str, Any]:
+        result = {
+            "samples": self.samples,
+            "exact_lines": self.exact_lines,
+            "line_accuracy": self.exact_lines / self.samples,
+            "empty_reference_lines": self.empty_reference_lines,
+            "character_edits": self.character_edits,
+            "reference_characters": self.reference_characters,
+            "cer": (
+                self.character_edits / self.reference_characters
+                if self.reference_characters
+                else None
+            ),
+        }
+        if words:
+            result.update(
+                word_edits=self.word_edits,
+                reference_words=self.reference_words,
+                wer=self.word_edits / self.reference_words if self.reference_words else None,
+            )
+        return result
+
+
+def _score(
+    references: dict[str, _Sample],
+    predictions: dict[str, _Sample],
+    *,
+    normalization: str,
+    words: bool,
+) -> dict[str, Any]:
+    overall = _Counts()
+    by_language: dict[str, _Counts] = {}
+    for sample_id, reference in references.items():
+        expected = reference.text
+        actual = predictions[sample_id].text
+        if normalization == "NFC":
+            expected = unicodedata.normalize("NFC", expected)
+            actual = unicodedata.normalize("NFC", actual)
+        character_edits = _edit_distance(expected, actual)
+        expected_words = expected.split() if words else []
+        word_edits = _edit_distance(expected_words, actual.split()) if words else 0
+        group = by_language.setdefault(reference.language, _Counts())
+        for counts in (overall, group):
+            counts.samples += 1
+            counts.exact_lines += expected == actual
+            counts.empty_reference_lines += not expected
+            counts.character_edits += character_edits
+            counts.reference_characters += len(expected)
+            counts.word_edits += word_edits
+            counts.reference_words += len(expected_words)
+    return {
+        "overall": overall.metrics(words=words),
+        "by_language": {
+            language: counts.metrics(words=words)
+            for language, counts in sorted(by_language.items())
+        },
+    }
+
+
+def score_records(
+    references: Mapping[str, tuple[str, str]],
+    predictions: Mapping[str, str],
+    *,
+    normalization: str = "none",
+    word_tokenization: str | None = None,
+) -> dict[str, Any]:
+    """Score in-memory records with the same semantics as ``evaluate_files``.
+
+    ``references`` maps each ID to ``(text, group label)``; ``predictions`` maps
+    each ID to predicted text. The ID sets must match exactly and be nonempty.
+    Returns the ``evaluate_files`` report; ``by_language`` is keyed by the supplied
+    group labels, so callers may group by any reference attribute.
+    """
+    _check_options(normalization, word_tokenization)
+    if not references:
+        raise ValueError("reference records: empty corpus")
+    missing = sorted(references.keys() - predictions.keys())
+    extra = sorted(predictions.keys() - references.keys())
+    if missing or extra:
+        raise ValueError(
+            f"sample ID mismatch: missing prediction IDs {missing!r}; "
+            f"extra prediction IDs {extra!r}"
+        )
+    return _report(
+        {key: _Sample(text, group) for key, (text, group) in references.items()},
+        {key: _Sample(text, "und") for key, text in predictions.items()},
+        normalization=normalization,
+        word_tokenization=word_tokenization,
+    )
+
+
+def _check_options(normalization: str, word_tokenization: str | None) -> None:
+    if normalization not in ("none", "NFC"):
+        raise ValueError("normalization must be 'none' or 'NFC'")
+    if word_tokenization not in (None, "whitespace"):
+        raise ValueError("word_tokenization must be None or 'whitespace'")
+
+
+def _report(
+    references: dict[str, _Sample],
+    predictions: dict[str, _Sample],
+    *,
+    normalization: str,
+    word_tokenization: str | None,
+) -> dict[str, Any]:
+    words = word_tokenization == "whitespace"
+    result: dict[str, Any] = {
+        "schema_version": 1,
+        "normalization": normalization,
+        "word_tokenization": word_tokenization,
+        "unicode_version": unicodedata.unidata_version,
+        "raw": _score(references, predictions, normalization="none", words=words),
+    }
+    if normalization == "NFC":
+        result["normalized"] = _score(references, predictions, normalization="NFC", words=words)
+    return result
+
+
+def evaluate_files(
+    reference_path: str | PathLike[str],
+    prediction_path: str | PathLike[str],
+    *,
+    normalization: str = "none",
+    word_tokenization: str | None = None,
+) -> dict[str, Any]:
+    """Score UTF-8 JSONL files with identical explicit string IDs.
+
+    Records require ``id`` and ``text`` strings. Reference ``metadata.language``
+    supplies grouping (default ``und``); prediction metadata never supplies truth.
+    ``normalization`` accepts ``none`` or ``NFC``. Raw metrics are always returned;
+    NFC additionally returns a separate ``normalized`` report. WER is omitted
+    unless ``word_tokenization='whitespace'`` is explicitly selected.
+
+    CER/WER are corpus edit totals divided by reference totals, or None for a
+    zero denominator. Empty reference lines still contribute insertion errors.
+    A corpus with no records is an error. No text is stripped or case-folded.
+
+    Return only JSON-serializable values, without writing files or printing.
+    Raise ValueError for invalid options, encoding, records, or ID mismatches;
+    raise OSError for file access failures. CLI adapters should catch both.
+    """
+    _check_options(normalization, word_tokenization)
+    references = _read_samples(reference_path, reference=True)
+    predictions = _read_samples(prediction_path, reference=False)
+    missing = sorted(references.keys() - predictions.keys())
+    extra = sorted(predictions.keys() - references.keys())
+    if missing or extra:
+        raise ValueError(
+            f"sample ID mismatch: missing prediction IDs {missing!r}; "
+            f"extra prediction IDs {extra!r}"
+        )
+    return _report(
+        references, predictions, normalization=normalization, word_tokenization=word_tokenization
+    )

khocr_gen/fonts.py

@@ -0,0 +1,316 @@
+"""Font management and script-aware font selection."""
+
+from __future__ import annotations
+
+import random
+from collections import OrderedDict
+from pathlib import Path
+from typing import Any, ClassVar
+
+from PIL import ImageFont
+
+from .provenance import get_font_info, get_font_sha256
+from .text_support import (
+    check_cluster_fallback_coverage,
+    find_fallback_fonts_for_clusters,
+    get_bidi_levels,
+    resolve_bidi_runs,
+    split_grapheme_clusters,
+)
+
+__all__ = [
+    "FontManager",
+    "check_cluster_fallback_coverage",
+    "find_fallback_fonts_for_clusters",
+    "get_bidi_levels",
+    "resolve_bidi_runs",
+    "split_grapheme_clusters",
+]
+
+try:
+    from PIL import Image  # noqa: F401
+
+    HAS_PIL = True
+except ImportError:
+    HAS_PIL = False
+
+
+class FontManager:
+    """Load and select fonts for Khmer and English text.
+
+    Font files are expected under `<fonts_dir>/khmer/` and `<fonts_dir>/english/` subdirectories.
+    Fonts placed directly in *fonts_dir* are added to both pools as a fallback.
+
+    Supported formats: ttf, otf, ttc, woff, woff2.
+    """
+
+    _FONT_EXTENSIONS: ClassVar[set[str]] = {".ttf", ".otf", ".ttc", ".otc", ".woff", ".woff2"}
+
+    # Sizes each font file is made available at (see `_load_fonts_from`).
+    # Registration is metadata-only (path + size, no FreeType face); actual
+    # faces are loaded lazily by `get_font_by_path_and_size`.
+    _STANDARD_SIZES: ClassVar[list[int]] = [28, 32, 36, 40, 44, 48]
+
+    # Bound on the font-face cache (see `get_font_by_path_and_size`).
+    # Every (font_path, size) pair -- proportional sizing's near-continuous
+    # scale as well as the standard sizes above -- is loaded lazily on first
+    # use; without a cap, a long run (or `font_mode=all`, which touches every
+    # font at every standard size) would pin every FreeType face it has ever
+    # seen in memory at once, in every process.
+    _DYNAMIC_FONT_CACHE_SIZE: ClassVar[int] = 1024
+
+    def __init__(
+        self,
+        language: str = "mixed",
+        fonts_dir: str = "fonts",
+        verbose: bool = True,
+        dynamic_font_cache_size: int = _DYNAMIC_FONT_CACHE_SIZE,
+        *,
+        multilingual: bool = False,
+        font_pool: str | None = None,
+    ) -> None:
+        if not HAS_PIL:
+            raise ImportError("Pillow is required. Install with: pip install Pillow")
+
+        self.language = language
+        self.multilingual = multilingual
+        self.font_pool = font_pool
+        self.pools: dict[str, list[tuple[str, int, Any]]] = {}
+        self.fonts_dir = Path(fonts_dir)
+        self.verbose = verbose
+        self.khmer_fonts: list[tuple[str, int, Any]] = []
+        self.english_fonts: list[tuple[str, int, Any]] = []
+        self.all_fonts: list[tuple[str, int, Any]] = []
+        self._dynamic_font_cache_size = max(0, dynamic_font_cache_size)
+        self._dynamic_font_cache: OrderedDict[tuple[str, int], Any] = OrderedDict()
+        self._font_styles: dict[str, set[str]] = {}
+        self._text_has_khmer_cache: dict[str, bool] = {}
+        self._load_fonts()
+        if font_pool is not None and font_pool not in self.pools:
+            raise ValueError(
+                f"Unknown font pool {font_pool!r}; available: {', '.join(sorted(self.pools)) or '(none)'}"
+            )
+
+    @classmethod
+    def _collect_font_files(cls, directory: Path) -> list[Path]:
+        """Recursively collect font files under *directory*."""
+        files: list[Path] = []
+        if not directory.exists():
+            return files
+        for path in sorted(directory.rglob("*")):
+            if path.is_file() and path.suffix.lower() in cls._FONT_EXTENSIONS:
+                files.append(path)
+        return files
+
+    def _load_fonts_from(self, directory: Path, target_list: list) -> int:
+        """Register every font file under *directory* into *target_list*.
+
+        Registration only records `(path, size, None)` entries; the actual
+        FreeType face is loaded on first use (see `get_font_by_path_and_size`).
+        """
+        font_paths = self._collect_font_files(directory)
+        added = 0
+        for font_path in font_paths:
+            for size in self._STANDARD_SIZES:
+                entry = (str(font_path), size, None)
+                target_list.append(entry)
+                self.all_fonts.append(entry)
+                added += 1
+        return added
+
+    def _load_fonts(self) -> None:
+        """Load fonts from khmer/ and english/ subdirectories."""
+        if self.verbose:
+            print(f"\nLoading fonts from: {self.fonts_dir.absolute()}")
+
+        if not self.fonts_dir.exists():
+            print(f"  Warning: Fonts directory not found: {self.fonts_dir}")
+            print("  Creating directory structure...")
+            (self.fonts_dir / "khmer").mkdir(parents=True, exist_ok=True)
+            (self.fonts_dir / "english").mkdir(parents=True, exist_ok=True)
+            print("\n  No fonts found!")
+            print(f"    Khmer   → {(self.fonts_dir / 'khmer').absolute()}")
+            print(f"    English → {(self.fonts_dir / 'english').absolute()}")
+            return
+
+        khmer_dir = self.fonts_dir / "khmer"
+        english_dir = self.fonts_dir / "english"
+
+        self.pools = {"khmer": self.khmer_fonts, "english": self.english_fonts}
+        for directory in sorted(self.fonts_dir.iterdir()):
+            if directory.is_dir():
+                pool = self.pools.setdefault(directory.name, [])
+                self._load_fonts_from(directory, pool)
+
+        # Fallback: fonts placed directly in fonts_dir go to both pools
+        root_font_paths = [
+            p for p in self._collect_font_files(self.fonts_dir) if p.parent == self.fonts_dir
+        ]
+        n_root = 0
+        self.pools["root"] = []
+        if root_font_paths:
+            for font_path in root_font_paths:
+                for size in self._STANDARD_SIZES:
+                    entry = (str(font_path), size, None)
+                    self.pools["root"].append(entry)
+                    self.khmer_fonts.append(entry)
+                    self.english_fonts.append(entry)
+                    self.all_fonts.append(entry)
+                    n_root += 1
+
+        if not self.all_fonts:
+            print("\n  No font files found.")
+            print(
+                f"  Please add fonts to:\n    Khmer   -> {khmer_dir.absolute()}\n    English -> {english_dir.absolute()}"
+            )
+            return
+
+        if self.verbose:
+            print("\n  Font Summary:")
+            print(f"    Total entries  : {len(self.all_fonts)} (across all sizes)")
+            print(f"    Khmer entries  : {len(self.khmer_fonts)}")
+            print(f"    English entries: {len(self.english_fonts)}")
+
+    def _text_has_khmer(self, text: str) -> bool:
+        """Check if *text* contains any Khmer Unicode characters."""
+        cached = self._text_has_khmer_cache.get(text)
+        if cached is not None:
+            return cached
+        result = any("ក" <= c <= "៿" for c in text)
+        if len(self._text_has_khmer_cache) >= 100_000:
+            self._text_has_khmer_cache.clear()
+        self._text_has_khmer_cache[text] = result
+        return result
+
+    def _pool_for(self, text: str) -> list:
+        """Return the script-appropriate font-entry pool for *text*."""
+        if self.font_pool is not None:
+            return self.pools[self.font_pool]
+        if self.multilingual:
+            return self.all_fonts
+        if self.language in self.pools and self.language != "mixed":
+            return self.pools[self.language]
+        has_khmer = self._text_has_khmer(text)
+        if has_khmer and self.khmer_fonts:
+            return self.khmer_fonts
+        if not has_khmer and self.english_fonts:
+            return self.english_fonts
+        return self.all_fonts
+
+    @staticmethod
+    def _detect_font_style(font_path: str) -> set[str]:
+        """Detect bold/italic style tags for a font file from its name + filename."""
+        tags: set[str] = set()
+        stem = Path(font_path).stem.lower()
+        if "bold" in stem:
+            tags.add("bold")
+        if "italic" in stem or "oblique" in stem:
+            tags.add("italic")
+        try:
+            style_name = ImageFont.truetype(font_path, 28).getname()[1]
+            style = style_name.lower() if style_name else ""
+            if "bold" in style:
+                tags.add("bold")
+            if "italic" in style or "oblique" in style:
+                tags.add("italic")
+        except Exception:
+            pass
+        return tags
+
+    def _style_tags(self, font_path: str) -> set[str]:
+        """Cached style tags for a font path (see `_detect_font_style`)."""
+        tags = self._font_styles.get(font_path)
+        if tags is None:
+            tags = self._detect_font_style(font_path)
+            self._font_styles[font_path] = tags
+        return tags
+
+    def random_font_path_with_style(self, text: str, styles: set[str]) -> str | None:
+        """Random font *path* from the script-appropriate pool whose style matches *styles*.
+
+        - ``{"bold"}`` -> any font tagged bold (a bold-italic font qualifies).
+        - ``{"italic"}`` -> any font tagged italic/oblique.
+        - ``{"bold", "italic"}`` -> prefer a font tagged both, else a bold font.
+        Returns ``None`` when no match exists.
+        """
+        if not styles:
+            return None
+        pool = self._pool_for(text)
+
+        def _paths(required: set[str]) -> list[str]:
+            return sorted({entry[0] for entry in pool if required <= self._style_tags(entry[0])})
+
+        if styles == {"bold", "italic"}:
+            both = _paths({"bold", "italic"})
+            if both:
+                return random.choice(both)
+            bold = _paths({"bold"})
+            if bold:
+                return random.choice(bold)
+            return None
+
+        matches = _paths(styles)
+        return random.choice(matches) if matches else None
+
+    def get_random_font(self, text: str) -> Any:
+        """Get a random font appropriate for *text*."""
+        pool = self._pool_for(text)
+        if not pool:
+            return None
+        path, size, _ = random.choice(pool)
+        return self.get_font_by_path_and_size(path, size)
+
+    def get_random_font_for_script(self, script: str) -> tuple[Any, int] | tuple[None, None]:
+        """Get a random font for a specific script ('khmer' or 'english')."""
+        if script == "khmer" and self.khmer_fonts:
+            path, size, _ = random.choice(self.khmer_fonts)
+        elif script == "english" and self.english_fonts:
+            path, size, _ = random.choice(self.english_fonts)
+        elif self.all_fonts:
+            path, size, _ = random.choice(self.all_fonts)
+        else:
+            return None, None
+        return self.get_font_by_path_and_size(path, size), size
+
+    def get_font_by_ref(self, font_ref: Any) -> Any:
+        """Resolve a `(path, size)` reference back to a font, loading lazily."""
+        if (
+            isinstance(font_ref, tuple)
+            and len(font_ref) >= 2
+            and font_ref[0] is not None
+            and font_ref[1] is not None
+        ):
+            return self.get_font_by_path_and_size(str(font_ref[0]), int(font_ref[1]))
+        return font_ref
+
+    def get_font_by_path_and_size(self, font_path: str, size: int) -> Any:
+        """Get font by path and size, loading and caching lazily.
+
+        Every (font_path, size) pair -- including the standard sizes each
+        font is registered at by `_load_fonts` -- is loaded on first use and
+        served from a bounded LRU cache, so neither a long run nor
+        `font_mode=all` (which touches every font at every standard size)
+        can pin every FreeType face in memory at once.
+        """
+        key = (str(font_path), int(size))
+        if key in self._dynamic_font_cache:
+            self._dynamic_font_cache.move_to_end(key)
+            return self._dynamic_font_cache[key]
+        try:
+            font = ImageFont.truetype(str(font_path), int(size), index=0)
+        except Exception:
+            return None
+        if self._dynamic_font_cache_size > 0:
+            self._dynamic_font_cache[key] = font
+            self._dynamic_font_cache.move_to_end(key)
+            while len(self._dynamic_font_cache) > self._dynamic_font_cache_size:
+                self._dynamic_font_cache.popitem(last=False)
+        return font
+
+    def get_font_sha256(self, font_path: Path | str) -> str:
+        """Return the process-cached font digest used by execution metadata."""
+        return get_font_sha256(font_path)
+
+    def get_available_fonts(self) -> list[dict[str, Any]]:
+        """Inventory registered font binaries once, independently of size entries."""
+        return [get_font_info(path) for path in sorted({entry[0] for entry in self.all_fonts})]