pink-eval 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pink/__init__.py ADDED
@@ -0,0 +1,28 @@
1
+ """PINK: Penalized INK-based score for faithful handwritten-math OCR evaluation.
2
+
3
+ Seong et al., "When VLMs 'Fix' Students: Identifying and Penalizing Over-Correction in
4
+ the Evaluation of Multi-line Handwritten Math OCR", EMNLP 2026.
5
+ """
6
+
7
+ __version__ = "1.0.0"
8
+
9
+ from .judge import JUDGE_PRESETS, JudgeConfig, grade, judge_from_preset
10
+ from .rubric import GRADING_INSTRUCTIONS, RUBRIC_ITEMS, SCORE_COLUMNS
11
+ from .score import DEFAULT_THRESHOLD, PINKResult, compute_pink, load_grades, penalize, pink_table
12
+
13
+ __all__ = [
14
+ "__version__",
15
+ "DEFAULT_THRESHOLD",
16
+ "GRADING_INSTRUCTIONS",
17
+ "JUDGE_PRESETS",
18
+ "JudgeConfig",
19
+ "PINKResult",
20
+ "RUBRIC_ITEMS",
21
+ "SCORE_COLUMNS",
22
+ "compute_pink",
23
+ "grade",
24
+ "judge_from_preset",
25
+ "load_grades",
26
+ "penalize",
27
+ "pink_table",
28
+ ]
pink/_runner.py ADDED
@@ -0,0 +1,102 @@
1
+ """Parallel, resumable execution of per-sample API calls with incremental output."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import csv
6
+ import json
7
+ import threading
8
+ from concurrent.futures import ThreadPoolExecutor, as_completed
9
+ from pathlib import Path
10
+ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence
11
+
12
+ import pandas as pd
13
+
14
+
15
+ def read_jsonl(path) -> List[Dict[str, Any]]:
16
+ with open(path, encoding="utf-8") as f:
17
+ return [json.loads(line) for line in f if line.strip()]
18
+
19
+
20
+ def write_jsonl(path, rows: Iterable[Dict[str, Any]]) -> None:
21
+ Path(path).parent.mkdir(parents=True, exist_ok=True)
22
+ with open(path, "w", encoding="utf-8") as f:
23
+ for row in rows:
24
+ f.write(json.dumps(row, ensure_ascii=False) + "\n")
25
+
26
+
27
+ def _read_rows(path: Path) -> List[Dict[str, Any]]:
28
+ if not path.exists() or path.stat().st_size == 0:
29
+ return []
30
+ if path.suffix == ".csv":
31
+ return pd.read_csv(path, dtype={"sample_id": str}, keep_default_na=False).to_dict("records")
32
+ return read_jsonl(path)
33
+
34
+
35
+ def _is_success(row: Dict[str, Any]) -> bool:
36
+ return str(row.get("success", "")).lower() in ("true", "1")
37
+
38
+
39
+ def _compact(path: Path, columns: Optional[Sequence[str]]) -> None:
40
+ """Keep one row per sample (the last successful one, else the last one)."""
41
+ best: Dict[str, Dict[str, Any]] = {}
42
+ for row in _read_rows(path):
43
+ sid = str(row["sample_id"])
44
+ if sid not in best or _is_success(row) or not _is_success(best[sid]):
45
+ best[sid] = row
46
+ rows = list(best.values())
47
+ if path.suffix == ".csv":
48
+ pd.DataFrame(rows, columns=list(columns) if columns else None).to_csv(path, index=False)
49
+ else:
50
+ write_jsonl(path, rows)
51
+
52
+
53
+ def run_jobs(items: Sequence[Dict[str, Any]], fn: Callable[[Dict[str, Any]], Dict[str, Any]], out_path,
54
+ columns: Optional[Sequence[str]] = None, workers: int = 8, resume: bool = True,
55
+ desc: str = "") -> Dict[str, int]:
56
+ """Call ``fn(item)`` for every item not yet done and append each result row to ``out_path``.
57
+
58
+ ``out_path`` is CSV (``columns`` required) or JSONL. Re-running the same command
59
+ skips samples that already succeeded, so interrupted runs can simply be restarted.
60
+ """
61
+ out_path = Path(out_path)
62
+ out_path.parent.mkdir(parents=True, exist_ok=True)
63
+ if not resume and out_path.exists():
64
+ out_path.unlink()
65
+ done = {str(r["sample_id"]) for r in _read_rows(out_path) if _is_success(r)}
66
+ todo = [it for it in items if str(it["sample_id"]) not in done]
67
+
68
+ lock = threading.Lock()
69
+ is_csv = out_path.suffix == ".csv"
70
+ new_file = not out_path.exists() or out_path.stat().st_size == 0
71
+ fh = open(out_path, "a", encoding="utf-8", newline="")
72
+ writer = csv.DictWriter(fh, fieldnames=list(columns), extrasaction="ignore") if is_csv else None
73
+ if is_csv and new_file:
74
+ writer.writeheader()
75
+
76
+ try:
77
+ from tqdm import tqdm
78
+ bar = tqdm(total=len(todo), desc=desc or out_path.name)
79
+ except ImportError: # pragma: no cover
80
+ bar = None
81
+
82
+ n_ok = 0
83
+ try:
84
+ with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
85
+ futures = [pool.submit(fn, it) for it in todo]
86
+ for fut in as_completed(futures):
87
+ row = fut.result()
88
+ with lock:
89
+ if is_csv:
90
+ writer.writerow(row)
91
+ else:
92
+ fh.write(json.dumps(row, ensure_ascii=False) + "\n")
93
+ fh.flush()
94
+ n_ok += _is_success(row)
95
+ if bar is not None:
96
+ bar.update(1)
97
+ finally:
98
+ fh.close()
99
+ if bar is not None:
100
+ bar.close()
101
+ _compact(out_path, columns)
102
+ return {"skipped": len(done), "attempted": len(todo), "succeeded": n_ok, "failed": len(todo) - n_ok}
pink/cli.py ADDED
@@ -0,0 +1,162 @@
1
+ """Command-line interface: ``pink fermat | ocr | grade | score``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+ from functools import partial
9
+ from pathlib import Path
10
+ from typing import List, Optional
11
+
12
+ from . import __version__
13
+
14
+
15
+ def _limit(items: list, limit: Optional[int]) -> list:
16
+ return items if limit is None else items[:limit]
17
+
18
+
19
+ def _check_endpoint(client, base_url: Optional[str], model: str) -> None:
20
+ """Fail fast if a self-hosted endpoint is down or does not serve ``model``."""
21
+ if not base_url or "api.openai.com" in base_url:
22
+ return
23
+ try:
24
+ served = [m.id for m in client.models.list().data]
25
+ except Exception as e: # noqa: BLE001
26
+ sys.exit(f"cannot reach {base_url} ({type(e).__name__}). Is the server up? "
27
+ "Check with: curl {}/models".format(base_url.rstrip("/")))
28
+ if model not in served:
29
+ sys.exit(f"{base_url} does not serve {model!r}; it serves {served}. "
30
+ "Pass --model, or start the server with --served-model-name.")
31
+
32
+
33
+ def cmd_fermat(args: argparse.Namespace) -> int:
34
+ from .data import export_fermat
35
+
36
+ manifest = export_fermat(args.out_dir, limit=args.limit, cache_dir=args.cache_dir, seed=args.seed)
37
+ n = sum(1 for _ in open(manifest, encoding="utf-8"))
38
+ print(f"wrote {n} samples -> {manifest}")
39
+ print(f"next: grade the oracle once per judge, e.g.\n pink grade --input {manifest} --out oracle_grades.csv")
40
+ return 0
41
+
42
+
43
+ def cmd_ocr(args: argparse.Namespace) -> int:
44
+ from ._runner import read_jsonl, run_jobs
45
+ from .ocr import make_client, transcribe_row
46
+
47
+ items = _limit(read_jsonl(args.manifest), args.limit)
48
+ image_dir = args.image_dir or Path(args.manifest).parent
49
+ client = make_client(args.base_url, args.api_key)
50
+ _check_endpoint(client, args.base_url, args.model)
51
+ fn = partial(transcribe_row, client, args.model, image_dir=image_dir, prompt=args.prompt,
52
+ max_tokens=args.max_tokens, temperature=args.temperature)
53
+ stats = run_jobs(items, fn, args.out, workers=args.workers, resume=not args.overwrite, desc=f"ocr:{args.model}")
54
+ print(json.dumps(stats))
55
+ return 0 if stats["failed"] == 0 else 1
56
+
57
+
58
+ def cmd_grade(args: argparse.Namespace) -> int:
59
+ from ._runner import read_jsonl, run_jobs
60
+ from .judge import JudgeConfig, grade, judge_from_preset, make_client
61
+ from .rubric import JUSTIFICATION_COLUMNS, SCORE_COLUMNS
62
+
63
+ if args.judge:
64
+ cfg = judge_from_preset(args.judge, model=args.model, base_url=args.base_url, api_key=args.api_key,
65
+ temperature=args.temperature, max_tokens=args.max_tokens,
66
+ reasoning_effort=args.reasoning_effort)
67
+ elif args.model:
68
+ cfg = JudgeConfig(model=args.model, base_url=args.base_url, api_key=args.api_key,
69
+ temperature=args.temperature, max_tokens=args.max_tokens,
70
+ reasoning_effort=args.reasoning_effort)
71
+ else:
72
+ sys.exit("pass --judge {gpt-5,qwen3-next-80b} or --model (with --base-url for a local server)")
73
+
74
+ items = [it for it in read_jsonl(args.input) if "solution" in it]
75
+ items = _limit(items, args.limit)
76
+ client = make_client(cfg)
77
+ _check_endpoint(client, cfg.base_url, cfg.model)
78
+
79
+ def fn(item):
80
+ return grade(client, cfg, item.get("problem", ""), item.get("solution", ""), str(item["sample_id"]))
81
+
82
+ columns = ["sample_id", "success", "total_score", *SCORE_COLUMNS, *JUSTIFICATION_COLUMNS, "attempts", "error"]
83
+ out = Path(args.out)
84
+ stats = run_jobs(items, fn, out, columns=columns if out.suffix == ".csv" else None,
85
+ workers=args.workers, resume=not args.overwrite, desc=f"grade:{cfg.model}")
86
+ print(json.dumps(stats))
87
+ return 0 if stats["failed"] == 0 else 1
88
+
89
+
90
+ def cmd_score(args: argparse.Namespace) -> int:
91
+ from .score import compute_pink, load_grades, pink_table
92
+
93
+ model, oracle = load_grades(args.model), load_grades(args.oracle)
94
+ result = compute_pink(model, oracle, threshold=args.threshold, missing=args.missing)
95
+ if args.per_sample:
96
+ pink_table(model, oracle, threshold=args.threshold, missing=args.missing).to_csv(args.per_sample)
97
+ print(json.dumps(result.to_dict(), indent=2) if args.json else result)
98
+ return 0
99
+
100
+
101
+ def build_parser() -> argparse.ArgumentParser:
102
+ p = argparse.ArgumentParser(prog="pink", description="PINK: penalized, over-correction-aware evaluation "
103
+ "of handwritten math OCR (EMNLP 2026).")
104
+ p.add_argument("--version", action="version", version=f"pink {__version__}")
105
+ sub = p.add_subparsers(dest="command", required=True)
106
+
107
+ f = sub.add_parser("fermat", help="download FERMAT and write images/ + manifest.jsonl (oracle transcriptions)")
108
+ f.add_argument("--out-dir", required=True)
109
+ f.add_argument("--limit", type=int, help="export a random subset of N samples (for a quick try)")
110
+ f.add_argument("--seed", type=int, default=0, help="seed for the --limit subset")
111
+ f.add_argument("--cache-dir", help="Hugging Face datasets cache directory")
112
+ f.set_defaults(func=cmd_fermat)
113
+
114
+ o = sub.add_parser("ocr", help="transcribe images with a VLM behind an OpenAI-compatible API")
115
+ o.add_argument("--manifest", required=True, help="manifest.jsonl from `pink fermat` (or your own)")
116
+ o.add_argument("--image-dir", help="directory the manifest's image paths are relative to (default: its folder)")
117
+ o.add_argument("--model", required=True, help="model name as served by the endpoint")
118
+ o.add_argument("--base-url", help="e.g. http://localhost:8000/v1 for `vllm serve` (default: OpenAI API)")
119
+ o.add_argument("--api-key")
120
+ o.add_argument("--prompt", choices=["default", "faithful"], default="default")
121
+ o.add_argument("--max-tokens", type=int, default=4096)
122
+ o.add_argument("--temperature", type=float, default=0.0)
123
+ o.add_argument("--workers", type=int, default=8)
124
+ o.add_argument("--limit", type=int)
125
+ o.add_argument("--overwrite", action="store_true", help="start from scratch instead of resuming")
126
+ o.add_argument("--out", required=True, help="output .jsonl (input for `pink grade`)")
127
+ o.set_defaults(func=cmd_ocr)
128
+
129
+ g = sub.add_parser("grade", help="grade solutions with the rubric-based LLM judge")
130
+ g.add_argument("--input", required=True, help="jsonl with sample_id, problem, solution")
131
+ g.add_argument("--out", required=True, help="output grades (.csv or .jsonl)")
132
+ g.add_argument("--judge", choices=["gpt-5", "qwen3-next-80b"], help="paper judge preset")
133
+ g.add_argument("--model", help="judge model name (overrides the preset)")
134
+ g.add_argument("--base-url", help="OpenAI-compatible endpoint of the judge")
135
+ g.add_argument("--api-key")
136
+ g.add_argument("--temperature", type=float)
137
+ g.add_argument("--max-tokens", type=int)
138
+ g.add_argument("--reasoning-effort", choices=["minimal", "low", "medium", "high"])
139
+ g.add_argument("--workers", type=int, default=8)
140
+ g.add_argument("--limit", type=int)
141
+ g.add_argument("--overwrite", action="store_true", help="start from scratch instead of resuming")
142
+ g.set_defaults(func=cmd_grade)
143
+
144
+ s = sub.add_parser("score", help="compute PINK from model and oracle grades")
145
+ s.add_argument("--model", required=True, help="grades of the VLM transcriptions")
146
+ s.add_argument("--oracle", required=True, help="grades of the oracle transcriptions (same judge!)")
147
+ s.add_argument("--threshold", type=float, default=10.0, help="minor/major boundary T (default 10)")
148
+ s.add_argument("--missing", choices=["drop", "zero"], default="drop",
149
+ help="oracle samples without a model grade: drop them (paper) or score them 0")
150
+ s.add_argument("--per-sample", metavar="CSV", help="also write per-sample penalized scores")
151
+ s.add_argument("--json", action="store_true")
152
+ s.set_defaults(func=cmd_score)
153
+ return p
154
+
155
+
156
+ def main(argv: Optional[List[str]] = None) -> int:
157
+ args = build_parser().parse_args(argv)
158
+ return args.func(args)
159
+
160
+
161
+ if __name__ == "__main__": # pragma: no cover
162
+ sys.exit(main())
pink/data.py ADDED
@@ -0,0 +1,101 @@
1
+ """FERMAT (Nath et al., ACL 2025) as an OCR-faithfulness benchmark.
2
+
3
+ The dataset is downloaded from the Hugging Face Hub (``ai4bharat/FERMAT``, CC-BY-4.0).
4
+ It is gated: accept the terms on its page and log in (``huggingface-cli login``) first.
5
+
6
+ Each sample is a handwritten image of a math question and a student solution that
7
+ contains a (possibly superficial) error. The *oracle transcription* is the LaTeX of what
8
+ the student actually wrote (FERMAT's ``pert_a``), with FERMAT's ``\\textcolor{red}{...}``
9
+ markers around the inserted error removed so the judge cannot see where the error is.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from pathlib import Path
15
+ from typing import Any, Dict, List, Optional
16
+
17
+ FERMAT_HF_ID = "ai4bharat/FERMAT"
18
+
19
+ _RED = "\\textcolor{red}{"
20
+
21
+
22
+ def _unwrap_red(text: str) -> str:
23
+ """Replace every ``\\textcolor{red}{X}`` by ``X`` (brace-balanced, so nested LaTeX survives)."""
24
+ out, i = [], 0
25
+ while True:
26
+ j = text.find(_RED, i)
27
+ if j < 0:
28
+ out.append(text[i:])
29
+ return "".join(out)
30
+ out.append(text[i:j])
31
+ k, depth = j + len(_RED), 1
32
+ while k < len(text) and depth:
33
+ depth += {"{": 1, "}": -1}.get(text[k], 0)
34
+ k += 1
35
+ out.append(text[j + len(_RED):k - 1] if depth == 0 else text[j + len(_RED):])
36
+ i = k
37
+
38
+
39
+ def oracle_transcription(pert_a: str) -> str:
40
+ """FERMAT ``pert_a`` -> oracle transcription used by PINK (red error markers removed)."""
41
+ return _unwrap_red(str(pert_a)).replace("\r\n", "\n")
42
+
43
+
44
+ def _image_ext(data: bytes, path: Optional[str]) -> str:
45
+ if data[:8] == b"\x89PNG\r\n\x1a\n":
46
+ return ".png"
47
+ if data[:3] == b"\xff\xd8\xff":
48
+ return ".jpg"
49
+ if path and Path(path).suffix:
50
+ return Path(path).suffix.lower()
51
+ return ".png"
52
+
53
+
54
+ def load_fermat(split: str = "train", cache_dir: Optional[str] = None):
55
+ """Load FERMAT as a ``datasets.Dataset`` with undecoded images (``{"bytes", "path"}``)."""
56
+ try:
57
+ from datasets import Image, load_dataset
58
+ except ImportError as e: # pragma: no cover
59
+ raise ImportError("FERMAT support needs `pip install pink-eval[fermat]`") from e
60
+ ds = load_dataset(FERMAT_HF_ID, split=split, cache_dir=cache_dir)
61
+ return ds.cast_column("image", Image(decode=False))
62
+
63
+
64
+ def export_fermat(out_dir, limit: Optional[int] = None, ids: Optional[List[str]] = None,
65
+ cache_dir: Optional[str] = None, seed: int = 0) -> Path:
66
+ """Write ``images/`` and ``manifest.jsonl`` to ``out_dir``.
67
+
68
+ Each manifest line holds ``sample_id``, ``image`` (relative path), ``problem`` (the
69
+ question text given to the judge) and ``solution`` (the oracle transcription), plus a
70
+ few FERMAT metadata fields. The manifest itself is the oracle input for ``pink grade``.
71
+ With ``limit``, a random subset (fixed ``seed``) is exported instead of the full set.
72
+ """
73
+ from ._runner import write_jsonl
74
+
75
+ out_dir = Path(out_dir)
76
+ (out_dir / "images").mkdir(parents=True, exist_ok=True)
77
+ ds = load_fermat(cache_dir=cache_dir)
78
+ if ids:
79
+ wanted = set(ids)
80
+ ds = ds.filter(lambda r: r["new_custom_id"] in wanted)
81
+ if limit is not None:
82
+ ds = ds.shuffle(seed=seed).select(range(min(limit, len(ds))))
83
+
84
+ rows: List[Dict[str, Any]] = []
85
+ for r in ds:
86
+ sid = r["new_custom_id"]
87
+ img = r["image"]
88
+ rel = f"images/{sid}{_image_ext(img['bytes'], img.get('path'))}"
89
+ (out_dir / rel).write_bytes(img["bytes"])
90
+ rows.append({
91
+ "sample_id": sid,
92
+ "image": rel,
93
+ "problem": str(r["orig_q"]).replace("\r\n", "\n"),
94
+ "solution": oracle_transcription(r["pert_a"]),
95
+ "domain": r.get("domain_code"),
96
+ "grade": r.get("grade"),
97
+ "has_error": r.get("has_error"),
98
+ })
99
+ manifest = out_dir / "manifest.jsonl"
100
+ write_jsonl(manifest, rows)
101
+ return manifest
pink/judge.py ADDED
@@ -0,0 +1,202 @@
1
+ """LLM judge: rubric-based semantic grading (paper Sec. 3.1).
2
+
3
+ Any OpenAI-compatible chat-completions endpoint works: the OpenAI API (GPT-5, the
4
+ paper's main judge) or a self-hosted model served with vLLM (e.g.
5
+ Qwen3-Next-80B-A3B-Thinking, the paper's open-source judge, see App. E).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import os
12
+ import re
13
+ import time
14
+ from dataclasses import dataclass, field, replace
15
+ from typing import Any, Dict, List, Optional, Tuple
16
+
17
+ from .rubric import (
18
+ GRADING_INSTRUCTIONS,
19
+ JUSTIFICATION_COLUMNS,
20
+ POINTS_PER_ITEM,
21
+ RUBRIC_ITEMS,
22
+ SCORE_COLUMNS,
23
+ build_user_message,
24
+ canonical_item,
25
+ )
26
+
27
+
28
+ @dataclass
29
+ class JudgeConfig:
30
+ """How to reach the judge and which sampling parameters to send.
31
+
32
+ Parameters left as ``None`` are not sent, so the endpoint's defaults apply.
33
+ """
34
+
35
+ model: str
36
+ base_url: Optional[str] = None #: ``None`` = OpenAI API
37
+ api_key: Optional[str] = None #: default: $OPENAI_API_KEY, or "EMPTY" for local servers
38
+ temperature: Optional[float] = None
39
+ top_p: Optional[float] = None
40
+ max_tokens: Optional[int] = None
41
+ reasoning_effort: Optional[str] = None #: OpenAI reasoning models only
42
+ extra_body: Dict[str, Any] = field(default_factory=dict)
43
+ max_attempts: int = 5 #: retries when the response cannot be parsed
44
+ timeout: float = 600.0
45
+
46
+
47
+ #: Judges used in the paper. ``gpt-5`` is the main judge; ``qwen3-next-80b`` is the
48
+ #: open-source judge (r=0.99 / Kendall tau=0.90 with GPT-5 at the VLM level, App. E.1).
49
+ JUDGE_PRESETS: Dict[str, JudgeConfig] = {
50
+ "gpt-5": JudgeConfig(model="gpt-5-2025-08-07", reasoning_effort="medium"),
51
+ "qwen3-next-80b": JudgeConfig(
52
+ model="Qwen/Qwen3-Next-80B-A3B-Thinking",
53
+ base_url="http://localhost:8000/v1",
54
+ temperature=0.6,
55
+ top_p=0.95,
56
+ ),
57
+ }
58
+
59
+
60
+ def judge_from_preset(name: str, **overrides: Any) -> JudgeConfig:
61
+ if name not in JUDGE_PRESETS:
62
+ raise KeyError(f"unknown judge preset {name!r}; choose from {sorted(JUDGE_PRESETS)}")
63
+ overrides = {k: v for k, v in overrides.items() if v is not None}
64
+ return replace(JUDGE_PRESETS[name], **overrides)
65
+
66
+
67
+ def _is_openai(cfg: JudgeConfig) -> bool:
68
+ return cfg.base_url is None or "api.openai.com" in cfg.base_url
69
+
70
+
71
+ def make_client(cfg: JudgeConfig):
72
+ from openai import OpenAI
73
+
74
+ key = cfg.api_key or os.environ.get("OPENAI_API_KEY") or ("EMPTY" if not _is_openai(cfg) else None)
75
+ return OpenAI(api_key=key, base_url=cfg.base_url, timeout=cfg.timeout)
76
+
77
+
78
+ # --------------------------------------------------------------------------- parsing
79
+
80
+ def parse_judge_response(text: str) -> Dict[str, Any]:
81
+ """Extract the JSON grading object from a (possibly noisy) judge response."""
82
+ if text is None:
83
+ raise ValueError("empty response")
84
+ text = text.strip()
85
+ if "</think>" in text: # reasoning trace without an opening tag
86
+ text = text.rsplit("</think>", 1)[1].strip()
87
+ text = re.sub(r"<(think|thought|reasoning)>[\s\S]*?</\1>", "", text).strip()
88
+
89
+ fences = re.findall(r"```(?:json)?\s*([\s\S]*?)\s*```", text)
90
+ for block in reversed(fences):
91
+ try:
92
+ return json.loads(block)
93
+ except json.JSONDecodeError:
94
+ continue
95
+ try:
96
+ return json.loads(text)
97
+ except json.JSONDecodeError:
98
+ pass
99
+
100
+ candidates: List[str] = []
101
+ depth, start = 0, -1
102
+ for i, ch in enumerate(text):
103
+ if ch == "{":
104
+ if depth == 0:
105
+ start = i
106
+ depth += 1
107
+ elif ch == "}" and depth:
108
+ depth -= 1
109
+ if depth == 0:
110
+ candidates.append(text[start:i + 1])
111
+ pool = [c for c in candidates if '"scores"' in c] or candidates
112
+ for cand in reversed(pool):
113
+ for attempt in (cand, re.sub(r",\s*([}\]])", r"\1", cand)):
114
+ try:
115
+ return json.loads(attempt)
116
+ except json.JSONDecodeError:
117
+ continue
118
+ raise ValueError(f"could not parse JSON from judge response: {text[:200]!r}")
119
+
120
+
121
+ def _to_score(value: Any) -> int:
122
+ if isinstance(value, str) and "/" in value:
123
+ value = value.split("/")[0]
124
+ score = int(round(float(value)))
125
+ return max(0, min(POINTS_PER_ITEM, score))
126
+
127
+
128
+ def extract_scores(parsed: Dict[str, Any]) -> Tuple[Dict[str, int], Dict[str, str]]:
129
+ """Return ``(scores, justifications)`` keyed by rubric item; raises if an item is missing."""
130
+ entries = parsed.get("scores") if isinstance(parsed, dict) else None
131
+ if not isinstance(entries, list):
132
+ raise ValueError("judge response has no 'scores' list")
133
+ entries = [e for e in entries if isinstance(e, dict)]
134
+
135
+ scores: Dict[str, int] = {}
136
+ justifs: Dict[str, str] = {}
137
+ for e in entries:
138
+ item = canonical_item(str(e.get("item", "")))
139
+ if item is not None and item not in scores:
140
+ scores[item] = _to_score(e.get("score", 0))
141
+ justifs[item] = str(e.get("justification", ""))
142
+ if len(scores) < len(RUBRIC_ITEMS) and len(entries) == len(RUBRIC_ITEMS):
143
+ # item names unrecognisable but the list has the right shape: use the rubric order
144
+ scores = {item: _to_score(e.get("score", 0)) for item, e in zip(RUBRIC_ITEMS, entries)}
145
+ justifs = {item: str(e.get("justification", "")) for item, e in zip(RUBRIC_ITEMS, entries)}
146
+ missing = [i for i in RUBRIC_ITEMS if i not in scores]
147
+ if missing:
148
+ raise ValueError(f"judge response is missing rubric items: {missing}")
149
+ return scores, justifs
150
+
151
+
152
+ # --------------------------------------------------------------------------- grading
153
+
154
+ def _request_kwargs(cfg: JudgeConfig) -> Dict[str, Any]:
155
+ kw: Dict[str, Any] = {}
156
+ if cfg.temperature is not None:
157
+ kw["temperature"] = cfg.temperature
158
+ if cfg.top_p is not None:
159
+ kw["top_p"] = cfg.top_p
160
+ if cfg.max_tokens is not None:
161
+ kw["max_completion_tokens" if _is_openai(cfg) else "max_tokens"] = cfg.max_tokens
162
+ if cfg.reasoning_effort is not None:
163
+ kw["reasoning_effort"] = cfg.reasoning_effort
164
+ if cfg.extra_body:
165
+ kw["extra_body"] = dict(cfg.extra_body)
166
+ return kw
167
+
168
+
169
+ def grade(client, cfg: JudgeConfig, problem: str, solution: str, sample_id: str = "",
170
+ instructions: str = GRADING_INSTRUCTIONS) -> Dict[str, Any]:
171
+ """Grade one solution. Returns a grade-file row (see :data:`SCORE_COLUMNS`)."""
172
+ row: Dict[str, Any] = {"sample_id": sample_id, "success": False, "total_score": None}
173
+ if not str(solution or "").strip():
174
+ row["error"] = "empty solution"
175
+ return row
176
+ messages = [
177
+ {"role": "system", "content": instructions},
178
+ {"role": "user", "content": build_user_message(problem, solution)},
179
+ ]
180
+ error = ""
181
+ for attempt in range(cfg.max_attempts):
182
+ finish = None
183
+ try:
184
+ resp = client.chat.completions.create(model=cfg.model, messages=messages, **_request_kwargs(cfg))
185
+ finish = getattr(resp.choices[0], "finish_reason", None)
186
+ content = resp.choices[0].message.content or ""
187
+ scores, justifs = extract_scores(parse_judge_response(content))
188
+ except Exception as e: # noqa: BLE001 - keep the run going, record the failure
189
+ error = f"{type(e).__name__}: {e}"
190
+ if finish == "length":
191
+ error = ("response truncated at the token limit (finish_reason=length): give the judge more room, "
192
+ "e.g. a larger --max-model-len for vLLM or a larger --max-tokens. " + error)
193
+ if attempt + 1 < cfg.max_attempts:
194
+ time.sleep(min(2 ** attempt, 30))
195
+ continue
196
+ row.update({"success": True, "total_score": sum(scores.values()), "attempts": attempt + 1})
197
+ for item, s_col, j_col in zip(RUBRIC_ITEMS, SCORE_COLUMNS, JUSTIFICATION_COLUMNS):
198
+ row[s_col] = scores[item]
199
+ row[j_col] = justifs[item]
200
+ return row
201
+ row.update({"error": error, "attempts": cfg.max_attempts})
202
+ return row
pink/ocr.py ADDED
@@ -0,0 +1,133 @@
1
+ """VLM transcription of handwritten math images (paper Sec. 4, prompt in Fig. 18).
2
+
3
+ Works with any OpenAI-compatible vision endpoint (OpenAI, Gemini's OpenAI-compatible
4
+ API, or an open model served with ``vllm serve``). You can also transcribe with your own
5
+ stack and skip this module entirely; ``pink grade`` only needs the resulting text.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import base64
11
+ import mimetypes
12
+ import os
13
+ import re
14
+ from pathlib import Path
15
+ from typing import Any, Dict, Optional, Tuple
16
+
17
+ SYSTEM_PROMPT = (
18
+ "You are a math assistant specializing in extracting mathematical question and answer content from "
19
+ "handwritten images of math problems by middle or high school students. Your task is to analyze the given "
20
+ "Image, which contains the handwritten math Question-Answer pair, and convert it into LaTeX format. Follow "
21
+ "the specific instructions provided carefully and output your response strictly in the requested format. "
22
+ "Ensure that your evaluation is precise, accurate, and based solely on the information visible in the image."
23
+ )
24
+
25
+ USER_PROMPT = """The image provided contains a handwritten math problem with both a Question and an Answer at a middle or high school level. Your task is to explicitly perform OCR on the handwritten text and extract the content in LaTeX format. Return only the extracted content, formatted in LaTeX, exactly as it appears in the image, in the **Question:** and **Answer:** fields.
26
+
27
+ Ensure that no extra information is added that is not in the image.
28
+
29
+ Please return the LaTeX output as follows:
30
+ **Question:**<Extracted Question text in LaTeX>
31
+ **Answer:**<Extracted Answer text in LaTeX>"""
32
+
33
+ # Prompt with explicit faithfulness instructions (paper Fig. 19, App. F). In the paper it
34
+ # reduced over-correction only slightly and did not improve PINK on average.
35
+ FAITHFUL_SYSTEM_PROMPT = (
36
+ "You are a math assistant specializing in extracting mathematical question and answer content from "
37
+ "handwritten images of math problems by middle or high school students. Your task is to analyze the given "
38
+ "Image, which contains the handwritten math Question-Answer pair, and convert it into LaTeX format. Follow "
39
+ "the specific instructions provided carefully and output your response strictly in the requested format. "
40
+ "Ensure that your transcription is precise, accurate, and based solely on the information visible in the "
41
+ "image. IMPORTANT: You must transcribe EXACTLY what is written, including any errors or mistakes made by "
42
+ "the student. Do NOT correct or fix any mathematical errors."
43
+ )
44
+
45
+ FAITHFUL_USER_PROMPT = """The image provided contains a handwritten math problem with both a Question and an Answer at a middle or high school level. Your task is to explicitly perform OCR on the handwritten text and extract the content in LaTeX format.
46
+
47
+ CRITICAL INSTRUCTIONS FOR FAITHFUL TRANSCRIPTION:
48
+ - Transcribe EXACTLY what is written in the image, including any mathematical errors or mistakes.
49
+ - Do NOT correct, fix, or improve any errors in the student's work.
50
+ - If the student wrote "2 + 2 = 5", you must transcribe it as "2 + 2 = 5", NOT "2 + 2 = 4".
51
+ - Your goal is FAITHFUL TRANSCRIPTION, not mathematical correction.
52
+ - Preserve all calculation errors, wrong answers, and mistakes exactly as they appear.
53
+
54
+ Return only the extracted content, formatted in LaTeX, exactly as it appears in the image, in the **Question:** and **Answer:** fields.
55
+
56
+ Ensure that no extra information is added that is not in the image.
57
+
58
+ Please return the LaTeX output as follows:
59
+ **Question:**<Extracted Question text in LaTeX>
60
+ **Answer:**<Extracted Answer text in LaTeX>"""
61
+
62
+ PROMPTS: Dict[str, Tuple[str, str]] = {
63
+ "default": (SYSTEM_PROMPT, USER_PROMPT),
64
+ "faithful": (FAITHFUL_SYSTEM_PROMPT, FAITHFUL_USER_PROMPT),
65
+ }
66
+
67
+
68
+ def clean_transcription(text: Any) -> str:
69
+ """Strip markdown fences and the **Question:** / **Answer:** labels from a raw VLM output."""
70
+ text = "" if text is None else str(text)
71
+ text = re.sub(r"^\s*```[a-zA-Z]*\s*\n?", "", text)
72
+ text = re.sub(r"\n?```\s*$", "", text)
73
+ text = re.sub(r"\*\*(Question|Answer):\*\*[ \t]*", "", text)
74
+ text = re.sub(r"\\textbf\{(Question|Answer)\s*:?\s*\}:?[ \t]*", "", text)
75
+ text = re.sub(r"^[ \t]*(Question|Answer):[ \t]*", "", text, flags=re.MULTILINE)
76
+ return text.strip()
77
+
78
+
79
+ def image_to_data_url(image_path) -> str:
80
+ mime = mimetypes.guess_type(str(image_path))[0] or "image/png"
81
+ with open(image_path, "rb") as f:
82
+ return f"data:{mime};base64,{base64.b64encode(f.read()).decode()}"
83
+
84
+
85
+ def make_client(base_url: Optional[str] = None, api_key: Optional[str] = None, timeout: float = 600.0):
86
+ from openai import OpenAI
87
+
88
+ key = api_key or os.environ.get("OPENAI_API_KEY") or ("EMPTY" if base_url else None)
89
+ return OpenAI(api_key=key, base_url=base_url, timeout=timeout)
90
+
91
+
92
+ def transcribe(client, model: str, image_path, prompt: str = "default", max_tokens: Optional[int] = 4096,
93
+ temperature: Optional[float] = 0.0) -> str:
94
+ """Transcribe one image and return the raw model output."""
95
+ system, user = PROMPTS[prompt]
96
+ kwargs: Dict[str, Any] = {}
97
+ if max_tokens is not None:
98
+ kwargs["max_tokens"] = max_tokens
99
+ if temperature is not None:
100
+ kwargs["temperature"] = temperature
101
+ resp = client.chat.completions.create(
102
+ model=model,
103
+ messages=[
104
+ {"role": "system", "content": system},
105
+ {"role": "user", "content": [
106
+ {"type": "text", "text": user},
107
+ {"type": "image_url", "image_url": {"url": image_to_data_url(image_path)}},
108
+ ]},
109
+ ],
110
+ **kwargs,
111
+ )
112
+ return resp.choices[0].message.content or ""
113
+
114
+
115
+ def transcribe_row(client, model: str, item: Dict[str, Any], image_dir, attempts: int = 3,
116
+ **kwargs: Any) -> Dict[str, Any]:
117
+ """Transcribe the image of one manifest item; returns a row ready for ``pink grade``."""
118
+ row = {"sample_id": item["sample_id"], "problem": item.get("problem", ""), "solution": "",
119
+ "raw_output": "", "success": False}
120
+ image_path = Path(image_dir) / item["image"]
121
+ error = ""
122
+ for _ in range(attempts):
123
+ try:
124
+ raw = transcribe(client, model, image_path, **kwargs)
125
+ except Exception as e: # noqa: BLE001
126
+ error = f"{type(e).__name__}: {e}"
127
+ continue
128
+ row.update(raw_output=raw, solution=clean_transcription(raw), success=bool(raw.strip()))
129
+ if row["success"]:
130
+ return row
131
+ error = "empty output"
132
+ row["error"] = error
133
+ return row
pink/rubric.py ADDED
@@ -0,0 +1,113 @@
1
+ """The 5-component, 100-point grading rubric used by PINK (paper Sec. 3.1, App. B.1, Fig. 16)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from typing import Dict, List
7
+
8
+ #: Rubric components, in the order the judge is asked to report them.
9
+ RUBRIC_ITEMS: List[str] = [
10
+ "Core Formula/Principle Identification",
11
+ "Condition/Boundary Application",
12
+ "Calculation Accuracy (Early-Mid)",
13
+ "Calculation Accuracy (Late)",
14
+ "Final Value Verification",
15
+ ]
16
+
17
+ #: Maximum score of every rubric component.
18
+ POINTS_PER_ITEM: int = 20
19
+
20
+
21
+ def _slug(item: str) -> str:
22
+ return re.sub(r"[^A-Za-z0-9-]+", "_", item).strip("_")
23
+
24
+
25
+ #: Column names used in grade files: ``score_<item>`` and ``justif_<item>``.
26
+ SCORE_COLUMNS: List[str] = [f"score_{_slug(i)}" for i in RUBRIC_ITEMS]
27
+ JUSTIFICATION_COLUMNS: List[str] = [f"justif_{_slug(i)}" for i in RUBRIC_ITEMS]
28
+
29
+ #: System prompt for the LLM judge (paper Fig. 16).
30
+ GRADING_INSTRUCTIONS: str = """# Math Exam Grading Instructions
31
+
32
+ ## Role
33
+ You are a math exam grader. Evaluate the student solution using the provided rubric and grading guidelines.
34
+
35
+ ## Rubric
36
+ {
37
+ "rubric": [
38
+ {
39
+ "item": "Core Formula/Principle Identification",
40
+ "points": 20,
41
+ "criterion": "Correctly identified and stated the necessary formulas for solving the problem"
42
+ },
43
+ {
44
+ "item": "Condition/Boundary Application",
45
+ "points": 20,
46
+ "criterion": "Accurately applied given conditions (boundaries, constraints, units, etc.)"
47
+ },
48
+ {
49
+ "item": "Calculation Accuracy (Early-Mid)",
50
+ "points": 20,
51
+ "criterion": "Intermediate calculations were performed without omissions or contradictions"
52
+ },
53
+ {
54
+ "item": "Calculation Accuracy (Late)",
55
+ "points": 20,
56
+ "criterion": "Final transformations and simplifications were logical and error-free"
57
+ },
58
+ {
59
+ "item": "Final Value Verification",
60
+ "points": 20,
61
+ "criterion": "Final answer correctly incorporates all conditions and calculations"
62
+ }
63
+ ],
64
+ "notes": "Points may be deducted for unit, rounding, or notation errors"
65
+ }
66
+
67
+ ## Grading Guidelines
68
+ 1. **Analyze each expression character by character** to identify any errors in formulas, calculations, or transformations, and provide a detailed justification explaining exactly where and why points were awarded or deducted.
69
+ 2. Include specific numbers, equations, or character positions in justification when necessary (e.g., "In line 3, the expression 2x+3 should be 2x-3, where the + sign at position 3 is incorrect").
70
+ 3. Total score must be an integer between 0-100.
71
+ 4. Output only the JSON format below. No additional text.
72
+
73
+ ## Output Format
74
+ {
75
+ "scores": [
76
+ {"item": "Core Formula/Principle Identification", "score": 0~20, "justification": "..."},
77
+ {"item": "Condition/Boundary Application", "score": 0~20, "justification": "..."},
78
+ {"item": "Calculation Accuracy (Early-Mid)", "score": 0~20, "justification": "..."},
79
+ {"item": "Calculation Accuracy (Late)", "score": 0~20, "justification": "..."},
80
+ {"item": "Final Value Verification", "score": 0~20, "justification": "..."}
81
+ ],
82
+ "total": <integer>
83
+ }"""
84
+
85
+ #: User message for the LLM judge. ``solution`` is the text being graded
86
+ #: (an OCR transcription, or the oracle transcription of the student's work).
87
+ USER_TEMPLATE: str = (
88
+ "**Problem:**\n{problem}\n\n"
89
+ "**Student Solution:**\n{solution}\n\n"
90
+ "Evaluate this solution according to the rubric and output the JSON result."
91
+ )
92
+
93
+
94
+ def build_user_message(problem: str, solution: str) -> str:
95
+ return USER_TEMPLATE.format(problem=problem, solution=solution)
96
+
97
+
98
+ def _norm(name: str) -> str:
99
+ return re.sub(r"[^a-z0-9]", "", name.lower())
100
+
101
+
102
+ _CANONICAL: Dict[str, str] = {_norm(i): i for i in RUBRIC_ITEMS}
103
+
104
+
105
+ def canonical_item(name: str) -> str | None:
106
+ """Map a (possibly slightly reworded) item name returned by a judge to a rubric item."""
107
+ key = _norm(name)
108
+ if not key:
109
+ return None
110
+ if key in _CANONICAL:
111
+ return _CANONICAL[key]
112
+ hits = [canon for canon_key, canon in _CANONICAL.items() if canon_key in key or key in canon_key]
113
+ return hits[0] if len(hits) == 1 else None
pink/score.py ADDED
@@ -0,0 +1,147 @@
1
+ """PINK score: over-correction penalty (paper Eq. 2-3) and aggregation (Eq. 4).
2
+
3
+ For every rubric component r of a sample, with dS_r = S_model,r - S_oracle,r :
4
+
5
+ S'_r = S_model,r if dS_r <= 0 (no over-correction)
6
+ S_oracle,r if 0 < dS_r <= T (minor: revert to the oracle score)
7
+ 0 if dS_r > T (major: nullify)
8
+
9
+ S_penalized = sum_r S'_r
10
+ PINK = sum_n S_penalized,n / sum_n S_oracle,n
11
+
12
+ with T = 10 (half of a 20-point component).
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ from dataclasses import asdict, dataclass, field
19
+ from pathlib import Path
20
+ from typing import Dict, Union
21
+
22
+ import numpy as np
23
+ import pandas as pd
24
+
25
+ from .rubric import RUBRIC_ITEMS, SCORE_COLUMNS
26
+
27
+ DEFAULT_THRESHOLD: float = 10.0
28
+
29
+ PathLike = Union[str, Path]
30
+
31
+
32
+ def penalize(model_scores, oracle_scores, threshold: float = DEFAULT_THRESHOLD) -> np.ndarray:
33
+ """Apply the two-tier over-correction penalty element-wise (Eq. 2)."""
34
+ m = np.asarray(model_scores, dtype=float)
35
+ o = np.asarray(oracle_scores, dtype=float)
36
+ delta = m - o
37
+ return np.where(delta <= 0, m, np.where(delta <= threshold, o, 0.0))
38
+
39
+
40
+ def load_grades(path: PathLike) -> pd.DataFrame:
41
+ """Load a grade file written by ``pink grade`` (CSV or JSONL).
42
+
43
+ Returns successfully graded rows, indexed by ``sample_id``, with one column per
44
+ rubric component (see :data:`pink.rubric.SCORE_COLUMNS`).
45
+ """
46
+ path = Path(path)
47
+ if path.suffix == ".jsonl":
48
+ df = pd.read_json(path, lines=True, dtype={"sample_id": str})
49
+ else:
50
+ df = pd.read_csv(path, dtype={"sample_id": str})
51
+ missing = [c for c in ["sample_id", *SCORE_COLUMNS] if c not in df.columns]
52
+ if missing:
53
+ raise ValueError(f"{path}: missing columns {missing}")
54
+ if "success" in df.columns:
55
+ ok = df["success"].astype(str).str.lower().isin(["true", "1"])
56
+ df = df[ok]
57
+ df = df.dropna(subset=SCORE_COLUMNS)
58
+ df = df.drop_duplicates("sample_id", keep="last").set_index("sample_id")
59
+ return df[SCORE_COLUMNS].astype(float)
60
+
61
+
62
+ def pink_table(model: pd.DataFrame, oracle: pd.DataFrame, threshold: float = DEFAULT_THRESHOLD,
63
+ missing: str = "drop") -> pd.DataFrame:
64
+ """Per-sample table with model, oracle and penalized totals.
65
+
66
+ ``model`` / ``oracle`` are DataFrames indexed by sample id with :data:`SCORE_COLUMNS`
67
+ (e.g. from :func:`load_grades`). Only samples graded for the oracle are used.
68
+ ``missing`` controls oracle samples without a model grade: ``"drop"`` excludes them
69
+ (as in the paper), ``"zero"`` counts them as a 0-score transcription.
70
+ """
71
+ if missing not in ("drop", "zero"):
72
+ raise ValueError("missing must be 'drop' or 'zero'")
73
+ if missing == "drop":
74
+ ids = oracle.index.intersection(model.index)
75
+ m = model.loc[ids, SCORE_COLUMNS]
76
+ else:
77
+ ids = oracle.index
78
+ m = model.reindex(ids)[SCORE_COLUMNS].fillna(0.0)
79
+ o = oracle.loc[ids, SCORE_COLUMNS]
80
+
81
+ out = pd.DataFrame(index=ids)
82
+ out.index.name = "sample_id"
83
+ pen = penalize(m.to_numpy(), o.to_numpy(), threshold)
84
+ over = m.to_numpy() > o.to_numpy()
85
+ for j in range(len(RUBRIC_ITEMS)):
86
+ out[f"over_{SCORE_COLUMNS[j][len('score_'):]}"] = over[:, j]
87
+ out["model_total"] = m.sum(axis=1).to_numpy()
88
+ out["oracle_total"] = o.sum(axis=1).to_numpy()
89
+ out["penalized_total"] = pen.sum(axis=1)
90
+ out["over_corrected"] = over.any(axis=1)
91
+ return out
92
+
93
+
94
+ @dataclass
95
+ class PINKResult:
96
+ pink: float #: PINK score (Eq. 4)
97
+ raw_ratio: float #: sum(S_model) / sum(S_oracle), i.e. without the penalty
98
+ over_correction_rate: float #: share of samples with >= 1 over-corrected component
99
+ n: int #: samples used
100
+ n_oracle: int #: samples with an oracle grade
101
+ n_model: int #: samples with a model grade
102
+ threshold: float = DEFAULT_THRESHOLD
103
+ over_corrections_per_item: Dict[str, int] = field(default_factory=dict)
104
+
105
+ def to_dict(self) -> dict:
106
+ return asdict(self)
107
+
108
+ def __str__(self) -> str:
109
+ lines = [
110
+ f"PINK = {self.pink:.4f}",
111
+ f"raw (no penalty) = {self.raw_ratio:.4f}",
112
+ f"over-correction rate = {self.over_correction_rate * 100:.1f}%",
113
+ f"samples used = {self.n} (oracle graded: {self.n_oracle}, model graded: {self.n_model})",
114
+ ]
115
+ return "\n".join(lines)
116
+
117
+
118
+ def compute_pink(model: Union[pd.DataFrame, PathLike], oracle: Union[pd.DataFrame, PathLike],
119
+ threshold: float = DEFAULT_THRESHOLD, missing: str = "drop") -> PINKResult:
120
+ """Compute PINK from model grades and oracle grades (DataFrames or grade-file paths)."""
121
+ model_df = model if isinstance(model, pd.DataFrame) else load_grades(model)
122
+ oracle_df = oracle if isinstance(oracle, pd.DataFrame) else load_grades(oracle)
123
+ table = pink_table(model_df, oracle_df, threshold=threshold, missing=missing)
124
+
125
+ oracle_sum = table["oracle_total"].sum()
126
+ if len(table) == 0 or oracle_sum <= 0:
127
+ pink = raw = float("nan")
128
+ else:
129
+ pink = float(table["penalized_total"].sum() / oracle_sum)
130
+ raw = float(table["model_total"].sum() / oracle_sum)
131
+ per_item = {
132
+ item: int(table[f"over_{col[len('score_'):]}"].sum()) for item, col in zip(RUBRIC_ITEMS, SCORE_COLUMNS)
133
+ }
134
+ return PINKResult(
135
+ pink=pink,
136
+ raw_ratio=raw,
137
+ over_correction_rate=float(table["over_corrected"].mean()) if len(table) else float("nan"),
138
+ n=len(table),
139
+ n_oracle=len(oracle_df),
140
+ n_model=len(model_df),
141
+ threshold=threshold,
142
+ over_corrections_per_item=per_item,
143
+ )
144
+
145
+
146
+ def result_to_json(result: PINKResult) -> str:
147
+ return json.dumps(result.to_dict(), indent=2)
@@ -0,0 +1,199 @@
1
+ Metadata-Version: 2.4
2
+ Name: pink-eval
3
+ Version: 1.0.0
4
+ Summary: PINK: over-correction-aware evaluation of handwritten math OCR (EMNLP 2026)
5
+ Author: Jin Seong, Wencke Liermann, Minho Kim, Jong-hun Shin, Soojong Lim
6
+ License-Expression: BSD-3-Clause
7
+ Project-URL: Paper, https://aclanthology.org/2026.emnlp-main.476/
8
+ Project-URL: Code, https://github.com/9115jin/PINK
9
+ Keywords: OCR,handwritten math,vision-language models,evaluation,LLM-as-a-judge,over-correction
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
12
+ Requires-Python: >=3.10
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: numpy>=1.21
16
+ Requires-Dist: pandas>=1.3
17
+ Requires-Dist: openai>=1.55
18
+ Requires-Dist: tqdm>=4.0
19
+ Provides-Extra: fermat
20
+ Requires-Dist: datasets>=2.18; extra == "fermat"
21
+ Provides-Extra: dev
22
+ Requires-Dist: pytest>=7; extra == "dev"
23
+ Dynamic: license-file
24
+
25
+ # PINK: Penalized INK-based Score
26
+
27
+ **When VLMs ‘Fix’ Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math OCR**
28
+ <br>Jin Seong\*, Wencke Liermann\*, Minho Kim, Jong-hun Shin, Soojong Lim · *EMNLP 2026 (Main, Oral)*
29
+
30
+ [![Paper](https://img.shields.io/badge/Paper-ACL%20Anthology-red)](https://aclanthology.org/2026.emnlp-main.476/)
31
+ [![PyPI](https://img.shields.io/pypi/v/pink-eval)](https://pypi.org/project/pink-eval/)
32
+ [![Tests](https://github.com/9115jin/PINK/actions/workflows/tests.yml/badge.svg)](https://github.com/9115jin/PINK/actions/workflows/tests.yml)
33
+ [![License: BSD-3-Clause](https://img.shields.io/badge/License-BSD--3--Clause-blue.svg)](https://github.com/9115jin/PINK/blob/main/LICENSE)
34
+
35
+ Vision-language models transcribing a student's handwritten solution often **“fix” the student's mistakes** instead of copying them. The transcription looks good, and string metrics like BLEU barely notice it. But the error a teacher or tutoring system needed to see is gone. We call this **over-correction**. Across 15 VLMs, 42–66% of transcriptions contain at least one over-correction.
36
+
37
+ **PINK** is an OCR metric built to catch it:
38
+
39
+ 1. An LLM judge grades a transcription with a 5-component, 100-point rubric. This measures *semantic* fidelity, so `\frac{1}{2}` and `0.5` count as the same.
40
+ 2. The judge also grades the *oracle* transcription, i.e. what the student actually wrote, errors included.
41
+ 3. Any rubric component on which the transcription scores **higher than the oracle** is an over-correction. It is reverted to the oracle score if the gain is ≤ 10 points and set to 0 if the gain is larger.
42
+
43
+ ![PINK overview](https://raw.githubusercontent.com/9115jin/PINK/main/assets/pink_overview.png)
44
+
45
+ ## Installation
46
+
47
+ ```bash
48
+ pip install "pink-eval[fermat]"
49
+ # latest development version: pip install "pink-eval[fermat] @ git+https://github.com/9115jin/PINK.git"
50
+ ```
51
+
52
+ Python ≥ 3.10. `[fermat]` adds the Hugging Face `datasets` package, which you need only to download FERMAT.
53
+
54
+ ## Quickstart (offline, no API key)
55
+
56
+ ```bash
57
+ git clone https://github.com/9115jin/PINK.git && cd PINK && pip install -e .
58
+ python examples/quickstart.py
59
+ ```
60
+
61
+ This scores Qwen2.5-VL-7B on 30 FERMAT samples whose transcriptions were already graded:
62
+
63
+ ```
64
+ PINK = 0.7400
65
+ raw (no penalty) = 0.9657
66
+ over-correction rate = 63.3%
67
+ ```
68
+
69
+ The raw ratio is the judge score of the transcription divided by the judge score of the oracle, with no penalty. Here it is close to 1, so the transcription looks almost as good as the student's original. PINK shows that much of that score comes from fixed student errors.
70
+
71
+ ## Evaluate your VLM on FERMAT
72
+
73
+ FERMAT ([Nath et al., 2025](https://aclanthology.org/2025.acl-long.720/)) contains 2,244 handwritten multi-line solutions with known student errors. It is hosted on Hugging Face under CC-BY-4.0. The dataset is gated, so first accept its terms at [huggingface.co/datasets/ai4bharat/FERMAT](https://huggingface.co/datasets/ai4bharat/FERMAT), then run `huggingface-cli login`.
74
+
75
+ ```bash
76
+ # 1) Download FERMAT. Writes images/ and manifest.jsonl (problem + oracle transcription per sample).
77
+ pink fermat --out-dir data/fermat # add --limit 20 for a quick try
78
+
79
+ # 2) Grade the oracle transcriptions. Do this once per judge and reuse the result for every VLM.
80
+ export OPENAI_API_KEY=...
81
+ pink grade --judge gpt-5 --input data/fermat/manifest.jsonl --out runs/oracle_grades.csv
82
+
83
+ # 3) Transcribe the images with your VLM. Any OpenAI-compatible endpoint works, e.g. `vllm serve`.
84
+ pink ocr --manifest data/fermat/manifest.jsonl --model Qwen/Qwen2.5-VL-7B-Instruct \
85
+ --base-url http://localhost:8000/v1 --out runs/qwen7b/ocr.jsonl
86
+
87
+ # 4) Grade the transcriptions with the same judge, then score.
88
+ pink grade --judge gpt-5 --input runs/qwen7b/ocr.jsonl --out runs/qwen7b/grades.csv
89
+ pink score --model runs/qwen7b/grades.csv --oracle runs/oracle_grades.csv
90
+ ```
91
+
92
+ All commands run requests in parallel (`--workers`), retry failed calls, and resume where they stopped if you re-run them. Already-finished samples are skipped. You can also transcribe with your own pipeline: `pink grade` only needs a JSONL file with `sample_id`, `problem` and `solution` (your transcription).
93
+
94
+ > **Always grade the model and the oracle with the same judge and prompt.** PINK compares the two scores component by component.
95
+
96
+ ### Judges
97
+
98
+ | `--judge` | Model | Notes |
99
+ |---|---|---|
100
+ | `gpt-5` | `gpt-5-2025-08-07`, reasoning effort `medium` | Main judge in the paper. Uses the OpenAI API. |
101
+ | `qwen3-next-80b` | `Qwen/Qwen3-Next-80B-A3B-Thinking` (temperature 0.6, top-p 0.95) | Open-source judge. VLM-level PINK agrees with GPT-5 at r = 0.99 and Kendall τ = 0.90 (paper App. E). Serve it with [`scripts/serve_qwen3_judge.sh`](https://github.com/9115jin/PINK/blob/main/scripts/serve_qwen3_judge.sh). |
102
+
103
+ Any other chat model also works, e.g. `--model <name> --base-url <url>`. PINK values are comparable only when they come from the same judge.
104
+
105
+ To run the open-source judge yourself:
106
+
107
+ ```bash
108
+ bash scripts/serve_qwen3_judge.sh # 8x 80GB GPUs (BF16)
109
+ MODEL=Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 TP=4 bash scripts/serve_qwen3_judge.sh # 4x 48GB GPUs (FP8)
110
+ pink grade --judge qwen3-next-80b --input data/fermat/manifest.jsonl --out runs/oracle_grades_qwen.csv
111
+ ```
112
+
113
+ The judge reasons for several thousand tokens per sample. If grades fail with `finish_reason=length`, give the server a longer context (`MAX_LEN`, default 16384).
114
+
115
+ ## Use PINK on your own data
116
+
117
+ PINK is not tied to FERMAT. For each sample you need the **problem**, the **oracle transcription** (what is actually written) and the **model transcription**:
118
+
119
+ ```jsonl
120
+ {"sample_id": "q1", "problem": "Factorise 4y^2 - 12y + 9.", "solution": "<oracle transcription>"}
121
+ ```
122
+
123
+ Write one such file for the oracle and one for the model (same `sample_id`s), run `pink grade` on each, then run `pink score`. From Python:
124
+
125
+ ```python
126
+ from pink import compute_pink, penalize
127
+
128
+ result = compute_pink("runs/model_grades.csv", "runs/oracle_grades.csv")
129
+ print(result.pink, result.over_correction_rate)
130
+
131
+ penalize(model_scores=[20, 20, 12], oracle_scores=[20, 5, 10]) # -> [20, 0, 10]
132
+ ```
133
+
134
+ ## FERMAT results (GPT-5 judge, paper Table 1)
135
+
136
+ | # | Model | PINK ↑ | Over-correction rate |
137
+ |---:|---|:---:|:---:|
138
+ | 1 | Gemini 2.5 Flash | 0.935 | 59.3% |
139
+ | 2 | Qwen2.5-VL 72B | 0.920 | 65.7% |
140
+ | 3 | Qwen2.5-VL 32B | 0.919 | 66.2% |
141
+ | 4 | Ovis2 34B | 0.916 | 62.5% |
142
+ | 5 | InternVL3 78B | 0.911 | 58.5% |
143
+ | 6 | GPT-4o | 0.907 | 62.3% |
144
+ | 7 | InternVL3 38B | 0.905 | 58.8% |
145
+ | 8 | Ovis2 8B | 0.887 | 53.6% |
146
+ | 9 | InternVL3 8B | 0.873 | 50.8% |
147
+ | 10 | Pixtral Large (124B) | 0.850 | 56.1% |
148
+ | 11 | Qwen2.5-VL 7B | 0.832 | 54.6% |
149
+ | 12 | Llama 3.2 90B Vision | 0.814 | 50.1% |
150
+ | 13 | GPT-4o mini | 0.787 | 49.6% |
151
+ | 14 | Pixtral 12B | 0.722 | 43.9% |
152
+ | 15 | Llama 3.2 11B Vision | 0.676 | 42.1% |
153
+
154
+ Over-correction rate = share of transcriptions with at least one over-corrected rubric component (paper Fig. 14).
155
+ The printed Table 1 in the paper has the two Pixtral PINK values swapped. The values above are the correct ones, consistent with Figures 3, 6 and 15 of the paper.
156
+
157
+ ## Notes
158
+
159
+ - **Prompts.** The judge prompt (`pink.rubric.GRADING_INSTRUCTIONS`) is the one shown in paper Fig. 16. The OCR prompt (`pink.ocr.PROMPTS["default"]`) is Fig. 18. `--prompt faithful` selects the faithfulness-instructed variant from Fig. 19 (App. F).
160
+ - **Oracle transcription.** We use FERMAT's `pert_a` (the student's erroneous solution) and remove its `\textcolor{red}{…}` error markers. Otherwise the judge would see where the error is.
161
+ - **Formula.** PINK = Σ<sub>n</sub> S<sub>penalized,n</sub> / Σ<sub>n</sub> S<sub>oracle,n</sub> (paper Eq. 4). Here S<sub>oracle,n</sub> is the sum of the oracle's component scores. A transcription that scores the same as the oracle gets exactly 1.0. Samples that the judge could not grade are left out (`--missing zero` counts them as 0 instead).
162
+
163
+ ## Citation
164
+
165
+ ```bibtex
166
+ @inproceedings{seong-etal-2026-vlms,
167
+ title = "When {VLM}s `Fix' Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math {OCR}",
168
+ author = "Seong, Jin and Liermann, Wencke and Kim, Minho and Shin, Jong-hun and Lim, Soojong",
169
+ booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
170
+ month = oct,
171
+ year = "2026",
172
+ address = "Budapest, Hungary",
173
+ publisher = "Association for Computational Linguistics",
174
+ url = "https://aclanthology.org/2026.emnlp-main.476/",
175
+ doi = "10.18653/2026.emnlp-main.476",
176
+ pages = "10932--10953"
177
+ }
178
+ ```
179
+
180
+ If you evaluate on FERMAT, please also cite the dataset:
181
+
182
+ ```bibtex
183
+ @inproceedings{nath-etal-2025-vision,
184
+ title = "Can Vision-Language Models Evaluate Handwritten Math?",
185
+ author = "Nath, Oikantik and Bathina, Hanani and Khan, Mohammed Safi Ur Rahman and Khapra, Mitesh M",
186
+ booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
187
+ year = "2025",
188
+ address = "Vienna, Austria",
189
+ publisher = "Association for Computational Linguistics",
190
+ url = "https://aclanthology.org/2025.acl-long.720/",
191
+ doi = "10.18653/v1/2025.acl-long.720",
192
+ pages = "14784--14814"
193
+ }
194
+ ```
195
+
196
+ ## License
197
+
198
+ Code: [BSD-3-Clause](https://github.com/9115jin/PINK/blob/main/LICENSE), © 2026 Electronics and Telecommunications Research Institute (ETRI).
199
+ The example grade files in `examples/sample/` contain judge justifications that quote FERMAT samples; FERMAT is © its authors and licensed under CC-BY-4.0.
@@ -0,0 +1,14 @@
1
+ pink/__init__.py,sha256=i8jsZ33GPJ-SbTwgN1ksNjXL2sJgeCPvBWRFCufJnZY,798
2
+ pink/_runner.py,sha256=39XvOj5tVyls73bbwpM04pCIaUhFJDd-RpOlqq-vxcs,3879
3
+ pink/cli.py,sha256=WE9mKjqMB763LWMqNrzMj3alUQRXJRDsZYK5GtoIyZc,8123
4
+ pink/data.py,sha256=I8bPju3WWNLXoHYBtLISQ2uMdCRAKl5yE4KFxN_0wN0,3927
5
+ pink/judge.py,sha256=oImIaP8Ku1HHa4JTUZv_MRgAhohc5khJZy-2lPgBdt0,8030
6
+ pink/ocr.py,sha256=hcVKOGemCvYjaeiHVASNaw7wmILC7iChac77f2jChV4,6616
7
+ pink/rubric.py,sha256=HaLUfFiZGFGeWtCtcGqqkA1oFICvaH4GoyZAJnXfRC0,4149
8
+ pink/score.py,sha256=yMYGz1FGH1yf2Swmg03BixzwZ8nPiUfT2nVZrAWcS1Y,5859
9
+ pink_eval-1.0.0.dist-info/licenses/LICENSE,sha256=pOg8L_qDHmpKO-gUc79fjIgQPZKemvCyxQBpEjPfk58,1547
10
+ pink_eval-1.0.0.dist-info/METADATA,sha256=QIU-8Hc8bTiX4ojDpFGKCId6VVkUAkCSgi9laS87ZUo,10599
11
+ pink_eval-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
12
+ pink_eval-1.0.0.dist-info/entry_points.txt,sha256=G1ZfYBepYQgaD_0JAv27MqipuXugIn0eskEQDKEjF0s,39
13
+ pink_eval-1.0.0.dist-info/top_level.txt,sha256=J0e5Cs7fv25M2HbtIHzDnamL9HOdHAll6Uue20lRzy8,5
14
+ pink_eval-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ pink = pink.cli:main
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, Electronics and Telecommunications Research Institute (ETRI)
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1 @@
1
+ pink