ocr-foundry 0.2.0__py3-none-any.whl

khocr_gen/benchmark_cmd.py

@@ -0,0 +1,103 @@
+"""Frozen-benchmark CLI (freeze/verify/compare); independent of rendering dependencies."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import argparse
+
+from .benchmark import (
+    DEFAULT_LOCK,
+    SPLITS,
+    compare,
+    format_comparison_markdown,
+    freeze,
+    verify,
+)
+
+
+def _expectations(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--expect-dataset-fingerprint", help="Fail unless the dataset matches")
+    parser.add_argument("--expect-suite-fingerprint", help="Fail unless dataset+runs match")
+
+
+def add_args(parser: argparse.ArgumentParser) -> None:
+    sub = parser.add_subparsers(dest="benchmark_command", metavar="ACTION", required=True)
+    p = sub.add_parser("freeze", help="Validate a manifest and write a new lock beside it")
+    p.add_argument("--manifest", required=True, help="Suite manifest JSON; its directory is root")
+    p.add_argument("--lock", default=DEFAULT_LOCK, help="New lock path relative to the root")
+    p = sub.add_parser("verify", help="Verify a lock against its files and leakage rules")
+    p.add_argument("--lock", required=True)
+    p.add_argument(
+        "--check-near-duplicates",
+        action="store_true",
+        default=False,
+        help="Opt-in cross-split near-duplicate line leakage detection via character n-gram Jaccard similarity",
+    )
+    p.add_argument(
+        "--near-duplicate-threshold",
+        type=float,
+        default=0.8,
+        help="Jaccard similarity threshold for near-duplicate detection (default: 0.8)",
+    )
+    _expectations(p)
+    p = sub.add_parser("compare", help="Verify, then score all named runs on frozen references")
+    p.add_argument("--lock", required=True)
+    p.add_argument(
+        "--split",
+        action="append",
+        choices=SPLITS,
+        help="Split to score (repeatable); default: test",
+    )
+    p.add_argument("--baseline", help="Baseline run name (default: the only baseline-role run)")
+    p.add_argument("--normalization", choices=["none", "NFC"], default="none")
+    p.add_argument("--word-tokenization", choices=["whitespace"], default=None)
+    p.add_argument(
+        "--format",
+        choices=["text", "json", "markdown"],
+        default="text",
+        help="Output format (default: text)",
+    )
+    _expectations(p)
+
+
+def run(args: argparse.Namespace) -> int:
+    expected = {
+        "expect_dataset_fingerprint": getattr(args, "expect_dataset_fingerprint", None),
+        "expect_suite_fingerprint": getattr(args, "expect_suite_fingerprint", None),
+    }
+    try:
+        if args.benchmark_command == "freeze":
+            report = freeze(args.manifest, args.lock)
+        elif args.benchmark_command == "verify":
+            report = verify(
+                args.lock,
+                check_near_duplicates=getattr(args, "check_near_duplicates", False),
+                near_duplicate_threshold=float(getattr(args, "near_duplicate_threshold", 0.8)),
+                **expected,
+            )
+        else:
+            report = compare(
+                args.lock,
+                splits=args.split or ["test"],
+                baseline=args.baseline,
+                normalization=args.normalization,
+                word_tokenization=args.word_tokenization,
+                **expected,
+            )
+        fmt = getattr(args, "format", "json")
+        if fmt == "markdown":
+            output = format_comparison_markdown(report)
+        elif fmt == "text":
+            # Default comparison text output: formatted JSON (same as prior runs)
+            output = json.dumps(report, ensure_ascii=False, allow_nan=False, indent=2)
+        else:  # json
+            output = json.dumps(report, ensure_ascii=False, allow_nan=False, indent=2)
+    except (ValueError, OSError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    print(output)
+    return 0

khocr_gen/cli.py

@@ -0,0 +1,246 @@
+"""OCR Foundry - CLI entrypoint shared by `ocr-foundry` and `khocr-gen`.
+
+Supports generate, combine, verify, view, and offline evaluate.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from . import __version__
+from .config_loader import (
+    apply_config_defaults,
+    load_yaml_config,
+    resolve_config_path,
+    validate_config_keys,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+CommandModule = Any  # duck-typed: has add_args(parser) and run(args) -> int
+
+
+class _SubParser(argparse.ArgumentParser):
+    """Subcommand parser that tracks whether its args have been lazily configured."""
+
+    _lazy_loaded: bool = False
+
+
+def _load_generate() -> CommandModule:
+    from . import generate
+
+    return generate
+
+
+def _load_combine() -> CommandModule:
+    from . import combine_cmd
+
+    return combine_cmd
+
+
+def _load_verify() -> CommandModule:
+    from . import verify
+
+    return verify
+
+
+def _load_view() -> CommandModule:
+    from . import viewer
+
+    return viewer
+
+
+def _load_evaluate() -> CommandModule:
+    from . import evaluate_cmd
+
+    return evaluate_cmd
+
+
+def _load_benchmark() -> CommandModule:
+    from . import benchmark_cmd
+
+    return benchmark_cmd
+
+
+def _load_layout() -> CommandModule:
+    from . import layout_cmd
+
+    return layout_cmd
+
+
+_COMMAND_LOADERS: dict[str, Callable[[], CommandModule]] = {
+    "benchmark": _load_benchmark,
+    "evaluate": _load_evaluate,
+    "generate": _load_generate,
+    "combine": _load_combine,
+    "layout": _load_layout,
+    "verify": _load_verify,
+    "view": _load_view,
+}
+
+_LEAF_HELP: dict[str, str] = {
+    "benchmark": "Freeze, verify and compare OCR benchmark suites from supplied predictions",
+    "evaluate": "Score supplied OCR predictions against reference JSONL (offline CER/line accuracy)",
+    "generate": "Generate synthetic training images from a text corpus",
+    "combine": "Combine multiple generated datasets into one merged LMDB dataset",
+    "layout": "Compose corpus lines into single-column page images with line annotations",
+    "verify": "Visual verification of augmentation methods at fixed MIN/MAX intensity (settable via --min/--max)",
+    "view": "Preview and extract images from an LMDB (.mdb) database",
+}
+
+# Commands that support -c / --config YAML loading
+_CONFIG_COMMANDS: frozenset[str] = frozenset({"generate", "combine", "layout"})
+
+
+def _first_positional(tokens: list[str]) -> tuple[str | None, bool]:
+    """Return (first non-flag token, whether a help/version flag preceded it)."""
+    help_seen = False
+    for token in tokens:
+        if token == "--":
+            return None, help_seen
+        if token in {"-h", "--help", "-v", "--version"}:
+            help_seen = True
+            continue
+        if token.startswith("-"):
+            continue
+        return token, help_seen
+    return None, help_seen
+
+
+def _extract_command(argv: list[str]) -> str | None:
+    command, _ = _first_positional(argv)
+    if command is None or command not in _COMMAND_LOADERS:
+        return None
+    return command
+
+
+def _configure_subcommand(
+    command: str,
+    subparser: _SubParser,
+) -> None:
+    if subparser._lazy_loaded:
+        return
+
+    module = _COMMAND_LOADERS[command]()
+    module.add_args(subparser)
+    subparser.set_defaults(func=module.run)
+    subparser._lazy_loaded = True
+
+
+def _pre_config(argv: list[str]) -> str | None:
+    """Extract the -c/--config value from *argv* without a full parse."""
+    i = 0
+    while i < len(argv):
+        token = argv[i]
+        if token == "--":
+            break
+        if token in ("-c", "--config") and i + 1 < len(argv):
+            return argv[i + 1]
+        if token.startswith("--config="):
+            return token[len("--config=") :]
+        if token.startswith("-c="):
+            return token[3:]
+        i += 1
+    return None
+
+
+def _apply_config_defaults(
+    command: str,
+    subparser: argparse.ArgumentParser,
+    argv: list[str],
+) -> None:
+    """Load YAML config (if any) and push values as argparse defaults."""
+    if command not in _CONFIG_COMMANDS:
+        return
+
+    explicit_config = _pre_config(argv)
+    # Layout historically has no automatic recipe; explicit YAML is opt-in.
+    if command == "layout" and explicit_config is None:
+        return
+    config_path = resolve_config_path(explicit_config, command)
+    if config_path is None:
+        return
+
+    try:
+        cfg = load_yaml_config(config_path)
+    except FileNotFoundError as exc:
+        if explicit_config is not None:
+            print(f"error: {exc}")
+            sys.exit(2)
+        return
+    except Exception as exc:
+        print(f"error: failed to load config {config_path}: {exc}")
+        sys.exit(2)
+
+    cfg.pop("config", None)
+
+    if cfg:
+        validate_config_keys(subparser, cfg, command)
+        apply_config_defaults(subparser, cfg)
+        print(f"[config] loaded: {config_path}")
+
+
+def _make_parser() -> tuple[argparse.ArgumentParser, dict[str, _SubParser]]:
+    # Preserve the invoked alias in usage/errors, including Windows console scripts.
+    executable = Path(sys.argv[0]).name.removesuffix(".exe")
+    prog = "khocr-gen" if executable == "khocr-gen" else "ocr-foundry"
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        description=(
+            "OCR Foundry - synthetic OCR line generation with legacy Khmer/English "
+            "and opt-in multilingual RAQM rendering, isolated augmentation, "
+            "and offline evaluation."
+        ),
+        epilog="ocr-foundry and khocr-gen run the same CLI; install the ocr-foundry distribution (with khocr-gen alias).",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    # Existing scripts may parse the legacy command's exact version output.
+    version = (
+        f"khocr-gen {__version__}"
+        if prog == "khocr-gen"
+        else f"OCR Foundry {__version__} (ocr-foundry {__version__})"
+    )
+    parser.add_argument("-v", "--version", action="version", version=version)
+
+    subparsers = parser.add_subparsers(
+        dest="command", metavar="COMMAND", title="Commands", parser_class=_SubParser
+    )
+    subparsers.required = True
+
+    command_parsers: dict[str, _SubParser] = {}
+    for cmd_name in _COMMAND_LOADERS:
+        command_parsers[cmd_name] = subparsers.add_parser(
+            cmd_name,
+            help=_LEAF_HELP[cmd_name],
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        )
+
+    return parser, command_parsers
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = list(sys.argv[1:] if argv is None else argv)
+
+    parser, command_parsers = _make_parser()
+
+    extracted = _extract_command(argv)
+    if extracted is not None:
+        command = extracted
+        subparser = command_parsers[command]
+        _configure_subcommand(command, subparser)
+        _apply_config_defaults(command, subparser, argv)
+
+    args = parser.parse_args(argv)
+    if not hasattr(args, "func"):
+        parser.error(f"Command '{getattr(args, 'command', None)}' was not configured.")
+    sys.exit(args.func(args))
+
+
+if __name__ == "__main__":
+    main()

khocr_gen/combine.py

@@ -0,0 +1,214 @@
+"""Combine multiple generated datasets into one merged dataset.
+
+Supports n-way merge: `data_1 + data_2 + ... + data_n`.
+
+Each input dataset is a directory containing `train/`, `val/`, and/or `test/` subdirectories,
+where each split is either "raw" (`labels.txt` + `images/`) or "lmdb" (`lmdb/` subdirectory).
+Inputs may use different formats independently per split; the merged output is always LMDB.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from tqdm import tqdm
+
+from .lmdb_pack import pack_lmdb
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+SPLITS: tuple[str, ...] = ("train", "val", "test")
+
+
+def _write_vocab(chars: set[str], output_path: Path) -> None:
+    """Write a vocab.json covering *chars*, with "<unk>" reserved at index 0."""
+    vocab: dict[str, int] = {"<unk>": 0}
+    for idx, ch in enumerate(sorted(chars), start=1):
+        if ch != "<unk>":
+            vocab[ch] = idx
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with output_path.open("w", encoding="utf-8") as fh:
+        json.dump(vocab, fh, indent=2, ensure_ascii=False)
+
+
+def _detect_split_format(split_dir: Path) -> str | None:
+    """Return "lmdb", "raw", or None (split absent/unrecognized)."""
+    if (split_dir / "lmdb" / "data.mdb").exists():
+        return "lmdb"
+    if (split_dir / "labels.txt").exists():
+        return "raw"
+    return None
+
+
+def _iter_raw_split_samples(split_dir: Path) -> Iterator[tuple[bytes, str, str]]:
+    """Yield (image_bytes, file_extension, text) for a raw labels.txt + images/ split."""
+    labels_path = split_dir / "labels.txt"
+    images_dir = split_dir / "images"
+
+    with labels_path.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.rstrip("\n")
+            if not line:
+                continue
+            parts = line.split("\t", 1)
+            if len(parts) < 2:
+                continue
+            img_name, text = parts[0].strip(), parts[1]
+
+            img_path = images_dir / img_name
+            if not img_path.exists():
+                img_path = split_dir / img_name
+            if not img_path.exists():
+                continue
+
+            ext = img_path.suffix or ".jpg"
+            yield img_path.read_bytes(), ext, text
+
+
+def _iter_lmdb_split_samples(split_dir: Path) -> Iterator[tuple[bytes, str, str]]:
+    """Yield (image_bytes, file_extension, text) for an lmdb/ split.
+
+    Image bytes are the exact bytes stored in the LMDB (already JPEG-encoded).
+    """
+    import lmdb  # type: ignore[import-untyped]
+
+    lmdb_dir = split_dir / "lmdb"
+    env = lmdb.open(str(lmdb_dir), readonly=True, lock=False, readahead=False, meminit=False)
+    try:
+        with env.begin(write=False, buffers=True) as txn:
+            raw = txn.get(b"num-samples")
+            if raw is None:
+                return
+            num_samples = int(bytes(raw).decode())
+            for i in range(1, num_samples + 1):
+                img_bytes = txn.get(f"image-{i:09d}".encode())
+                lbl_bytes = txn.get(f"label-{i:09d}".encode())
+                if img_bytes is None or lbl_bytes is None:
+                    continue
+                yield bytes(img_bytes), ".jpg", bytes(lbl_bytes).decode("utf-8")
+    finally:
+        env.close()
+
+
+def _iter_split_samples(split_dir: Path, fmt: str) -> Iterator[tuple[bytes, str, str]]:
+    if fmt == "lmdb":
+        yield from _iter_lmdb_split_samples(split_dir)
+    else:
+        yield from _iter_raw_split_samples(split_dir)
+
+
+def combine_datasets(
+    input_dirs: list[str | Path],
+    output_dir: str | Path,
+    *,
+    keep_raw: bool = False,
+    jpeg_quality: int = 90,
+    map_size_gb: int = 256,
+    build_vocab: bool = True,
+    vocab_path: str | Path | None = None,
+    verbose: bool = False,
+) -> dict[str, int]:
+    """Merge multiple datasets into *output_dir*, per split, as LMDB.
+
+    Each dataset root may have any of `train/`, `val/`, and/or `test/`;
+    a split missing from an input is simply skipped.
+    A split absent from *all* inputs is skipped entirely.
+
+    Args:
+        input_dirs: List of dataset root directories to merge.
+        output_dir: Where to write the merged dataset.
+        keep_raw: If True, keep the intermediate images/ directory after LMDB pack.
+        jpeg_quality: JPEG quality for stored images (1-100).
+        map_size_gb: LMDB map size in GiB.
+        build_vocab: If True, write a merged vocab.json covering every character
+            across all merged splits.
+        vocab_path: Where to write the merged vocab.json; defaults to
+            `output_dir/vocab.json`.
+        verbose: Print progress messages.
+
+    Returns:
+        A dict of split -> merged sample count.
+    """
+    roots = [Path(d).expanduser().resolve() for d in input_dirs]
+    out_root = Path(output_dir).expanduser().resolve()
+
+    for root in roots:
+        if not root.exists():
+            raise FileNotFoundError(f"Dataset not found: {root}")
+
+    counts: dict[str, int] = {}
+    vocab_chars: set[str] = set()
+
+    for split in SPLITS:
+        # Collect all input splits that exist
+        split_inputs: list[tuple[Path, str]] = []
+        for root in roots:
+            split_dir = root / split
+            fmt = _detect_split_format(split_dir)
+            if fmt is not None:
+                split_inputs.append((split_dir, fmt))
+
+        if not split_inputs:
+            if verbose:
+                print(f"  No '{split}' split found in any input, skipping")
+            continue
+
+        out_split = out_root / split
+        merged_images = out_split / "images"
+        merged_labels = out_split / "labels.txt"
+        merged_images.mkdir(parents=True, exist_ok=True)
+
+        n = 0
+        with merged_labels.open("w", encoding="utf-8") as lf:
+            for split_dir, fmt in split_inputs:
+                if verbose:
+                    print(f"  Merging {split_dir} ({fmt}) into {split}...")
+                samples = tqdm(
+                    _iter_split_samples(split_dir, fmt),
+                    desc=f"  Merging {split}",
+                    unit="img",
+                )
+                for img_bytes, ext, text in samples:
+                    n += 1
+                    img_name = f"img_{n:09d}{ext}"
+                    (merged_images / img_name).write_bytes(img_bytes)
+                    lf.write(f"{img_name}\t{text}\n")
+                    if build_vocab:
+                        vocab_chars.update(text)
+
+        if n == 0:
+            merged_labels.unlink(missing_ok=True)
+            shutil.rmtree(merged_images, ignore_errors=True)
+            continue
+
+        counts[split] = n
+
+        # Pack to LMDB
+        lmdb_out = out_split / "lmdb"
+        lmdb_out.mkdir(parents=True, exist_ok=True)
+        if verbose:
+            print(f"  Packing LMDB for {split} ({n} samples)...")
+        pack_lmdb(
+            labels_file=str(merged_labels),
+            images_dir=str(merged_images),
+            out_dir=str(lmdb_out),
+            jpeg_quality=jpeg_quality,
+            map_size_gb=map_size_gb,
+            verbose=verbose,
+        )
+
+        if not keep_raw:
+            shutil.rmtree(merged_images)
+
+    if build_vocab and vocab_chars:
+        vp = Path(vocab_path).expanduser().resolve() if vocab_path else out_root / "vocab.json"
+        if verbose:
+            print(f"  Writing merged vocab ({len(vocab_chars)} unique characters) -> {vp}")
+        _write_vocab(vocab_chars, vp)
+
+    return counts

khocr_gen/combine_cmd.py

@@ -0,0 +1,125 @@
+"""`khocr-gen combine` - merge multiple generated datasets into one."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import argparse
+
+
+def run(args: argparse.Namespace) -> int:
+    """Entry point called by cli.py."""
+    from .combine import combine_datasets
+
+    input_dirs: list[Path] = [Path(d).expanduser().resolve() for d in args.datasets]
+    output_dir = Path(args.output).expanduser().resolve()
+
+    for d in input_dirs:
+        if not d.exists():
+            print(f"Dataset not found: {d}", file=sys.stderr)
+            return 2
+
+    if output_dir.exists() and any(output_dir.iterdir()):
+        if not args.overwrite:
+            print(f"Output directory already exists. Re-run with --overwrite: {output_dir}")
+            return 1
+        import shutil
+
+        shutil.rmtree(output_dir)
+
+    print(f"\nCombining {len(input_dirs)} datasets:")
+    for d in input_dirs:
+        print(f"   {d}")
+    print(f"   -> {output_dir}")
+
+    try:
+        counts = combine_datasets(
+            input_dirs=[str(d) for d in input_dirs],
+            output_dir=output_dir,
+            keep_raw=getattr(args, "keep_raw", False),
+            jpeg_quality=args.jpeg_quality,
+            map_size_gb=args.map_size_gb,
+            build_vocab=not getattr(args, "skip_vocab", False),
+            vocab_path=getattr(args, "vocab", None) or None,
+            verbose=args.verbose,
+        )
+    except FileNotFoundError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+
+    if not counts:
+        print("Nothing to combine, no dataset has a train, val, or test split.")
+        return 1
+
+    for split, n in counts.items():
+        print(f"   {split}: wrote {n} samples to {output_dir / split / 'lmdb'}")
+
+    if not getattr(args, "skip_vocab", False):
+        vocab_out = (
+            Path(args.vocab).expanduser().resolve() if args.vocab else output_dir / "vocab.json"
+        )
+        if vocab_out.exists():
+            print(f"   vocab: wrote merged vocab.json to {vocab_out}")
+
+    print(f"\nCombine complete. Dataset -> {output_dir}")
+    return 0
+
+
+def add_args(parser: argparse.ArgumentParser) -> None:
+    """Register all combine sub-command arguments onto *parser*."""
+
+    parser.add_argument(
+        "-c",
+        "--config",
+        default=None,
+        metavar="FILE",
+        help="YAML config file. Values override argparse defaults; CLI flags override YAML.",
+    )
+    parser.add_argument(
+        "datasets",
+        nargs="+",
+        metavar="DATASET",
+        help="One or more dataset directories to merge (data_1 data_2 ... data_n)",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        default="data_combined",
+        metavar="DIR",
+        help="Output directory for the merged dataset",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action="store_true",
+        help="Overwrite existing output directory without prompting",
+    )
+    parser.add_argument(
+        "--keep-raw", action="store_true", help="Keep raw image files on disk after LMDB packing"
+    )
+    parser.add_argument(
+        "--jpeg-quality",
+        type=int,
+        default=90,
+        metavar="N",
+        help="JPEG compression quality for LMDB images",
+    )
+    parser.add_argument(
+        "--map-size-gb", type=int, default=256, metavar="N", help="LMDB map size in GiB"
+    )
+    parser.add_argument(
+        "--vocab",
+        default=None,
+        metavar="FILE",
+        help="Path to write merged vocab.json; defaults to OUTPUT/vocab.json",
+    )
+    parser.add_argument(
+        "--skip-vocab",
+        action="store_true",
+        help="Do not build a merged vocab.json",
+    )
+    parser.add_argument(
+        "--verbose", action="store_true", help="Print merge and LMDB packing progress"
+    )