ocr-bench 0.0.2__tar.gz → 0.0.4__tar.gz

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocr-bench
-Version: 0.0.2
+Version: 0.0.4
 Summary: OCR model evaluation toolkit — VLM-as-judge with per-dataset leaderboards
 Author: Daniel van Strien
 Author-email: Daniel van Strien <davanstrien@gmail.com>

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ocr-bench"
-version = "0.0.2"
+version = "0.0.4"
 description = "OCR model evaluation toolkit — VLM-as-judge with per-dataset leaderboards"
 readme = "README.md"
 authors = [
@@ -44,6 +44,7 @@ select = ["E", "F", "I", "UP"]
 
 [dependency-groups]
 dev = [
+    "hypothesis>=6.156.1",
     "pytest>=9.0.2",
     "ruff>=0.11",
     "ty>=0.0.17",

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/src/ocr_bench/backends.py

@@ -5,19 +5,57 @@ from __future__ import annotations
 import abc
 from collections import Counter
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from itertools import zip_longest
 from typing import Any
 
 import stamina
 import structlog
 from huggingface_hub import InferenceClient
-from openai import OpenAI
+from openai import APIConnectionError, OpenAI
 
 from ocr_bench.judge import JUDGE_SCHEMA, Comparison, parse_judge_output
 
 logger = structlog.get_logger()
 
-# Retry on these exception types with exponential backoff + jitter.
-_RETRYABLE = (Exception,)
+# Number of attempts for a transient judge call (connection/timeout/429/5xx).
+_RETRY_ATTEMPTS = 5
+
+# Transport-level failures worth retrying — no HTTP status (connection reset,
+# DNS failure, read timeout, server hang-up mid-response). `APIConnectionError`
+# also covers the openai client's `APITimeoutError` (a subclass). HTTP errors are
+# handled by status code below.
+_TRANSPORT_ERRORS: tuple[type[BaseException], ...] = (APIConnectionError,)
+try:  # requests is always present via huggingface_hub; guard defensively
+    from requests.exceptions import ConnectionError as _ReqConnectionError
+    from requests.exceptions import Timeout as _ReqTimeout
+
+    _TRANSPORT_ERRORS += (_ReqConnectionError, _ReqTimeout)
+except Exception:  # pragma: no cover
+    pass
+try:  # httpx backs huggingface_hub's InferenceClient — the default judge path.
+    # Providers (e.g. Novita) disconnect on long runs, raising httpx
+    # TransportError subclasses (RemoteProtocolError, ConnectError, timeouts).
+    from httpx import TransportError as _HttpxTransportError
+
+    _TRANSPORT_ERRORS += (_HttpxTransportError,)
+except Exception:  # pragma: no cover
+    pass
+
+
+def _is_retryable(exc: Exception) -> bool:
+    """Retry only *transient* judge failures.
+
+    Retries connection/timeout errors and HTTP 429 (rate limit) or 5xx
+    (server) responses. Fatal errors — a bad/expired token, a typo'd model
+    id, or any other 4xx — are NOT retried, so they surface immediately
+    instead of after several pointless backoffs.
+    """
+    status = getattr(exc, "status_code", None)
+    if status is None:
+        status = getattr(getattr(exc, "response", None), "status_code", None)
+    if status is not None:
+        return status == 429 or status >= 500
+    return isinstance(exc, _TRANSPORT_ERRORS)
 
 
 class JudgeBackend(abc.ABC):
@@ -38,7 +76,14 @@ class JudgeBackend(abc.ABC):
         or an empty dict on failure.
         """
         if self.concurrency <= 1 or len(comparisons) <= 1:
-            return [self._call_single(comp) for comp in comparisons]
+            results = []
+            for i, comp in enumerate(comparisons):
+                try:
+                    results.append(self._call_single(comp))
+                except Exception as exc:
+                    logger.warning("judge_call_failed", idx=i, error=str(exc))
+                    results.append({})
+            return results
 
         # Concurrent execution preserving order
         results: list[dict[str, str]] = [{}] * len(comparisons)
@@ -69,11 +114,11 @@ class InferenceProviderJudge(JudgeBackend):
         self.name = f"{provider + ':' if provider else ''}{model}"
         self.model = model
         self.max_tokens = max_tokens
-        self.client = InferenceClient(model=model, provider=provider)  # type: ignore[invalid-argument-type]
+        self.client = InferenceClient(model=model, provider=provider)  # ty: ignore[invalid-argument-type]
 
-    @stamina.retry(on=_RETRYABLE, attempts=6)
+    @stamina.retry(on=_is_retryable, attempts=_RETRY_ATTEMPTS)
     def _call_single(self, comp: Comparison) -> dict[str, str]:
-        response = self.client.chat_completion(  # type: ignore[no-matching-overload]
+        response = self.client.chat_completion(  # ty: ignore[no-matching-overload]
             messages=comp.messages,
             max_tokens=self.max_tokens,
             temperature=0.0,
@@ -108,11 +153,11 @@ class OpenAICompatibleJudge(JudgeBackend):
         self.concurrency = concurrency
         self.client = OpenAI(base_url=base_url, api_key=api_key)
 
-    @stamina.retry(on=_RETRYABLE, attempts=3)
+    @stamina.retry(on=_is_retryable, attempts=_RETRY_ATTEMPTS)
     def _call_single(self, comp: Comparison) -> dict[str, str]:
         response = self.client.chat.completions.create(
             model=self.model,
-            messages=comp.messages,  # type: ignore[invalid-argument-type]
+            messages=comp.messages,  # ty: ignore[invalid-argument-type]
             max_tokens=self.max_tokens,
             temperature=self.temperature,
             extra_body=self.extra_body,
@@ -165,7 +210,7 @@ def parse_judge_spec(
                 model=model_name,
                 api_key=token,
                 max_tokens=max_tokens,
-                temperature=0.7,
+                temperature=0.0,
                 extra_body={"chat_template_kwargs": {"enable_thinking": False}},
                 concurrency=concurrency,
             )
@@ -207,26 +252,31 @@ def aggregate_jury_votes(
     if not all_results:
         return []
 
-    n_comparisons = len(all_results[0])
-    n_judges = len(all_results)
     aggregated: list[dict[str, Any]] = []
 
-    for i in range(n_comparisons):
+    # Transpose judge-major results to comparison-major; a judge that
+    # returned a short list pads out with failures.
+    for judge_results in zip_longest(*all_results, fillvalue={}):
         votes: list[str] = []
         reasons: list[str] = []
-        for j in range(n_judges):
-            result = all_results[j][i] if i < len(all_results[j]) else {}
+        for name, result in zip(judge_names, judge_results):
             winner = result.get("winner", "")
             if winner:
                 votes.append(winner)
-                reasons.append(f"{judge_names[j]}: {result.get('reason', '')}")
+                reasons.append(f"{name}: {result.get('reason', '')}")
 
         if not votes:
             aggregated.append({"winner": "tie", "reason": "no valid votes", "agreement": "0/0"})
             continue
 
         counter = Counter(votes)
-        majority_winner, majority_count = counter.most_common(1)[0]
+        top = counter.most_common(2)
+        majority_winner, majority_count = top[0]
+        if len(top) > 1 and top[1][1] == majority_count:
+            # No strict majority (1-1, 2-2, three-way split, ...): record a
+            # tie rather than letting Counter's insertion order side with
+            # whichever judge happens to be listed first.
+            majority_winner = "tie"
         agreement = f"{majority_count}/{len(votes)}"
 
         aggregated.append({

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/src/ocr_bench/cli.py

@@ -6,6 +6,7 @@ import argparse
 import sys
 
 import structlog
+from openai import OpenAIError
 from rich.console import Console
 from rich.table import Table
 
@@ -87,6 +88,15 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Don't publish results (default: publish to {dataset}-results)",
     )
+    judge.add_argument(
+        "--license",
+        default=None,
+        help=(
+            "License tag for the published results dataset card, e.g. cc0-1.0 "
+            "(default: none — the results embed source-derived text, so only "
+            "the publisher knows the right license)"
+        ),
+    )
     judge.add_argument(
         "--full-rejudge",
         action="store_true",
@@ -190,7 +200,9 @@ def _convert_results(
     """Convert judged comparisons + aggregated outputs into ComparisonResult list."""
     results: list[ComparisonResult] = []
     for comp, result in zip(comparisons, aggregated):
-        if not result:
+        # Skip failures: empty dict (single judge failed) and 0/0 "ties"
+        # (every judge in a jury failed) — neither is a real verdict.
+        if not result or result.get("agreement") == "0/0":
             continue
         results.append(
             ComparisonResult(
@@ -433,6 +445,7 @@ def cmd_judge(args: argparse.Namespace) -> None:
                     board,
                     metadata,
                     existing_metadata=existing_meta_rows,
+                    license_id=args.license,
                 )
                 console.print(f"\nResults published to [bold]{results_repo}[/bold]")
             return
@@ -464,7 +477,13 @@ def cmd_judge(args: argparse.Namespace) -> None:
             valid_comparisons=len(new_results),
             from_prs=from_prs,
         )
-        publish_results(results_repo, board, metadata, existing_metadata=existing_meta_rows)
+        publish_results(
+            results_repo,
+            board,
+            metadata,
+            existing_metadata=existing_meta_rows,
+            license_id=args.license,
+        )
         console.print(f"\nResults published to [bold]{results_repo}[/bold]")
 
 
@@ -489,7 +508,8 @@ def cmd_run(args: argparse.Namespace) -> None:
         for slug in sorted(MODEL_REGISTRY):
             cfg = MODEL_REGISTRY[slug]
             default = " (default)" if slug in DEFAULT_MODELS else ""
-            table.add_row(slug + default, cfg.model_id, cfg.size, cfg.default_flavor)
+            gpu = cfg.default_flavor + (" (image-mode)" if cfg.image else "")
+            table.add_row(slug + default, cfg.model_id, cfg.size, gpu)
 
         console.print(table)
         console.print(f"\nDefault set: {', '.join(DEFAULT_MODELS)}")
@@ -528,6 +548,10 @@ def cmd_run(args: argparse.Namespace) -> None:
             console.print(f"[cyan]{slug}[/cyan] ({cfg.model_id})")
             console.print(f"  Flavor:  {flavor}")
             console.print(f"  Timeout: {args.timeout}")
+            if cfg.image:
+                console.print(f"  Image:   {cfg.image}")
+                console.print(f"  Python:  {cfg.python}")
+                console.print(f"  Env:     {cfg.env}")
             console.print(f"  Script:  {cfg.script}")
             console.print(f"  Args:    {' '.join(script_args)}")
             console.print()
@@ -587,7 +611,7 @@ SPACE_TEMPLATE = "davanstrien/ocr-bench-space-template"
 
 def cmd_publish(args: argparse.Namespace) -> None:
     """Deploy results viewer as a Hugging Face Space."""
-    from huggingface_hub import HfApi
+    from huggingface_hub import HfApi, SpaceHardware
 
     api = HfApi()
     results = args.results
@@ -599,13 +623,26 @@ def cmd_publish(args: argparse.Namespace) -> None:
         from_id=SPACE_TEMPLATE,
         to_id=space_id,
         private=args.private if args.private else None,
-        hardware="cpu-basic",
+        hardware=SpaceHardware.CPU_BASIC,
         exist_ok=True,
         variables=[{"key": "REPOS", "value": results}],
     )
 
     api.add_space_variable(repo_id=space_id, key="REPOS", value=results)
 
+    # Update Space metadata to link to results dataset
+    try:
+        from huggingface_hub import metadata_update
+
+        metadata_update(
+            space_id,
+            {"datasets": [results], "tags": ["ocr-bench"]},
+            repo_type="space",
+            overwrite=True,
+        )
+    except Exception as exc:
+        logger.warning("space_metadata_update_failed", error=str(exc))
+
     url = f"https://huggingface.co/spaces/{space_id}"
     console.print(f"[green]Space published![/green] {url}")
 
@@ -630,3 +667,10 @@ def main() -> None:
     except DatasetError as exc:
         console.print(f"[red]Error:[/red] {exc}")
         sys.exit(1)
+    except (OpenAIError, OSError) as exc:
+        # Judge or Hub request failed (bad/expired token, unknown model id,
+        # rate limit, provider/network outage) — every requests/HfHubHTTPError
+        # subclasses OSError — or another OS-level error. Fail with a clean
+        # message instead of dumping a traceback on the user.
+        console.print(f"[red]Error:[/red] {exc}")
+        sys.exit(1)

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/src/ocr_bench/dataset.py

@@ -211,6 +211,37 @@ def load_config_dataset(
     if unified is None:
         raise DatasetError("No configs loaded successfully")
 
+    # Disambiguate configs that resolve to the same model_id (mirrors the flat
+    # discover_ocr_columns path). Downstream the ELO keys models by these values,
+    # so two configs of the same model run with different settings (e.g.
+    # `nuextract3` vs `nuextract3-rep`) would silently collapse into one
+    # leaderboard row. On collision, label by config name; keep the bare
+    # model_id when unique.
+    model_counts: dict[str, int] = {}
+    for model_id in ocr_columns.values():
+        model_counts[model_id] = model_counts.get(model_id, 0) + 1
+    duplicates = sorted(mid for mid, n in model_counts.items() if n > 1)
+    if duplicates:
+        # Capture the colliding config → model_id mapping before relabeling so
+        # the warning is actionable (which configs/model_ids collided, in which
+        # repo) when running multiple datasets/config sweeps.
+        collided = {
+            config: model_id
+            for config, model_id in ocr_columns.items()
+            if model_counts[model_id] > 1
+        }
+        for config, model_id in list(ocr_columns.items()):
+            if model_counts[model_id] > 1:
+                short = model_id.split("/")[-1] if "/" in model_id else model_id
+                ocr_columns[config] = f"{short} ({config})"
+        logger.warning(
+            "duplicate_model_ids",
+            repo_id=repo_id,
+            model_ids=duplicates,
+            collided_configs=collided,
+            note="configs sharing a model_id were labelled by config name to keep them distinct",
+        )
+
     return unified, ocr_columns
 
 

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/src/ocr_bench/judge.py

@@ -275,15 +275,27 @@ def parse_judge_output(text: str) -> dict[str, str]:
     """
     text = text.strip()
     if text.startswith("```"):
-        text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
+        # A truncated response can be a bare opening fence with no body
+        parts = text.split("\n", 1)
+        text = parts[1].rsplit("```", 1)[0].strip() if len(parts) == 2 else ""
     try:
         result = json.loads(text)
-        winner = result.get("winner", "tie").upper().strip()
-        if winner == "TIE":
-            winner = "tie"
-        if winner not in ("A", "B", "tie"):
-            winner = "tie"
-        return {"winner": winner, "reason": result.get("reason", "")}
     except json.JSONDecodeError:
         logger.warning("Failed to parse judge output: %s", text[:200])
         return {}
+    if not isinstance(result, dict):
+        logger.warning("Judge output is not a JSON object: %s", text[:200])
+        return {}
+    winner = result.get("winner", "tie")
+    if not isinstance(winner, str):
+        logger.warning("Judge output has non-string winner: %s", text[:200])
+        return {}
+    winner = winner.upper().strip()
+    if winner == "TIE":
+        winner = "tie"
+    if winner not in ("A", "B", "tie"):
+        winner = "tie"
+    reason = result.get("reason", "")
+    if not isinstance(reason, str):
+        reason = json.dumps(reason)
+    return {"winner": winner, "reason": reason}

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/src/ocr_bench/publish.py

@@ -127,6 +127,7 @@ def publish_results(
     board: Leaderboard,
     metadata: EvalMetadata,
     existing_metadata: list[dict] | None = None,
+    license_id: str | None = None,
 ) -> None:
     """Push evaluation results to Hub as a dataset with multiple configs.
 
@@ -159,7 +160,7 @@ def publish_results(
     logger.info("published_metadata", repo=repo_id, n=len(all_meta))
 
     # README — auto-generated dataset card with leaderboard
-    readme = _build_readme(repo_id, rows, board, metadata)
+    readme = _build_readme(repo_id, rows, board, metadata, license_id=license_id)
     api = HfApi()
     api.upload_file(
         path_or_fileobj=readme.encode(),
@@ -175,6 +176,7 @@ def _build_readme(
     rows: list[dict],
     board: Leaderboard,
     metadata: EvalMetadata,
+    license_id: str | None = None,
 ) -> str:
     """Build a dataset card README with the leaderboard table."""
     has_ci = bool(board.elo_ci)
@@ -187,12 +189,19 @@ def _build_readme(
     judge_str = ", ".join(j.split("/")[-1] for j in judges) if judges else "N/A"
     n_comparisons = len(board.comparison_log)
 
-    lines = [
-        "---",
-        "license: mit",
+    # The card license describes the published results DATA (which embeds
+    # OCR text derived from the source dataset), not this tool — so there is
+    # no correct default; it's declared per-run via --license or set on the
+    # Hub repo by the publisher.
+    lines = ["---"]
+    if license_id:
+        lines.append(f"license: {license_id}")
+    lines += [
         "tags:",
         "  - ocr-bench",
         "  - leaderboard",
+        "source_datasets:",
+        f"  - {metadata.source_dataset}",
         "configs:",
         "  - config_name: default",
         "    data_files:",
@@ -230,7 +239,8 @@ def _build_readme(
         lines.append("|------|-------|--------|-----|------|--------|------|------|")
 
     for rank, row in enumerate(rows, 1):
-        model = row["model"]
+        # Escape pipes so arbitrary model names can't break the table
+        model = str(row["model"]).replace("|", "\\|")
         elo = row["elo"]
         params = row.get("params", "")
         if has_ci and "elo_low" in row:

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/src/ocr_bench/run.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import time
 from dataclasses import dataclass, field
+from typing import Any
 
 import structlog
 from huggingface_hub import HfApi, get_token
@@ -13,13 +14,33 @@ logger = structlog.get_logger()
 
 @dataclass
 class ModelConfig:
-    """Configuration for a single OCR model."""
+    """Configuration for a single OCR model.
+
+    ``image`` / ``python`` / ``env`` are only needed by "image-mode" models —
+    ones whose CUDA kernels (e.g. flashinfer for Qwen3.5) must come from a
+    prebuilt Docker image because the default uv-script image lacks ``nvcc``.
+    They are passed straight through to ``run_uv_job`` and left ``None`` for
+    every standard model, which keeps the launch call identical to before.
+    """
 
     script: str
     model_id: str
     size: str
     default_flavor: str = "l4x1"
     default_args: list[str] = field(default_factory=list)
+    image: str | None = None
+    python: str | None = None
+    env: dict[str, str] | None = None
+
+
+# Image-mode invocation for models needing prebuilt CUDA kernels (Qwen3.5 /
+# flashinfer). The default uv-script image has no ``nvcc`` so flashinfer's JIT
+# compile fails at vLLM warmup; the vllm/vllm-openai image ships them prebuilt.
+# ``python`` points at that image's interpreter and ``env`` puts its site-packages
+# on the path so ``uv run`` reuses them instead of rebuilding.
+_VLLM_OPENAI_IMAGE = "vllm/vllm-openai:latest"
+_VLLM_OPENAI_PYTHON = "/usr/bin/python3"
+_VLLM_OPENAI_ENV = {"PYTHONPATH": "/usr/local/lib/python3.12/dist-packages"}
 
 
 MODEL_REGISTRY: dict[str, ModelConfig] = {
@@ -54,6 +75,37 @@ MODEL_REGISTRY: dict[str, ModelConfig] = {
         size="2.1B",
         default_flavor="l4x1",
     ),
+    "qianfan-ocr": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/qianfan-ocr.py",
+        model_id="baidu/Qianfan-OCR",
+        size="4.7B",
+        default_flavor="l4x1",
+    ),
+    "dots-mocr": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/dots-mocr.py",
+        model_id="rednote-hilab/dots.mocr",
+        size="3B",
+        default_flavor="l4x1",
+    ),
+    # Image-mode models (Qwen3.5 / flashinfer) — need the vllm/vllm-openai image.
+    "nuextract3": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/nuextract3.py",
+        model_id="numind/NuExtract3",
+        size="4B",
+        default_flavor="a100-large",
+        image=_VLLM_OPENAI_IMAGE,
+        python=_VLLM_OPENAI_PYTHON,
+        env=_VLLM_OPENAI_ENV,
+    ),
+    "paddleocr-vl-1.6": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/paddleocr-vl-1.6.py",
+        model_id="PaddlePaddle/PaddleOCR-VL-1.6",
+        size="0.9B",
+        default_flavor="a100-large",
+        image=_VLLM_OPENAI_IMAGE,
+        python=_VLLM_OPENAI_PYTHON,
+        env=_VLLM_OPENAI_ENV,
+    ),
 }
 
 DEFAULT_MODELS = ["glm-ocr", "deepseek-ocr", "lighton-ocr-2", "dots-ocr", "firered-ocr"]
@@ -145,13 +197,30 @@ def launch_ocr_jobs(
             extra_args=config.default_args or None,
         )
 
-        logger.info("launching_job", model=slug, flavor=flavor, script=config.script)
+        # Only image-mode models set image/python/env; standard models keep the
+        # exact same run_uv_job call as before (no extra kwargs).
+        extra_kwargs: dict[str, Any] = {}
+        if config.image:
+            extra_kwargs["image"] = config.image
+        if config.python:
+            extra_kwargs["python"] = config.python
+        if config.env:
+            extra_kwargs["env"] = config.env
+
+        logger.info(
+            "launching_job",
+            model=slug,
+            flavor=flavor,
+            script=config.script,
+            image=config.image,
+        )
         job = api.run_uv_job(
             script=config.script,
             script_args=script_args,
             flavor=flavor,
             secrets={"HF_TOKEN": token},
             timeout=timeout,
+            **extra_kwargs,
         )
         jobs.append(JobRun(model_slug=slug, job_id=job.id, job_url=job.url))
         logger.info("job_launched", model=slug, job_id=job.id, url=job.url)

{ocr_bench-0.0.2 → ocr_bench-0.0.4}/src/ocr_bench/web.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import html
 import io
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
@@ -67,8 +68,10 @@ def _build_pair_summary_html(comparisons: list[dict[str, Any]]) -> str:
 
     rows = []
     for (ma, mb), counts in sorted(pair_counts.items()):
-        short_a = _short_model(ma)
-        short_b = _short_model(mb)
+        # Model names come from dataset columns — escape before embedding in
+        # HTML that the template renders with `| safe`.
+        short_a = html.escape(_short_model(ma))
+        short_b = html.escape(_short_model(mb))
         wins, losses, ties = counts["W"], counts["L"], counts["T"]
         rows.append(
             f"<tr><td>{short_a}</td><td>{short_b}</td>"