pink-eval 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pink/__init__.py +28 -0
- pink/_runner.py +102 -0
- pink/cli.py +162 -0
- pink/data.py +101 -0
- pink/judge.py +202 -0
- pink/ocr.py +133 -0
- pink/rubric.py +113 -0
- pink/score.py +147 -0
- pink_eval-1.0.0.dist-info/METADATA +199 -0
- pink_eval-1.0.0.dist-info/RECORD +14 -0
- pink_eval-1.0.0.dist-info/WHEEL +5 -0
- pink_eval-1.0.0.dist-info/entry_points.txt +2 -0
- pink_eval-1.0.0.dist-info/licenses/LICENSE +28 -0
- pink_eval-1.0.0.dist-info/top_level.txt +1 -0
pink/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""PINK: Penalized INK-based score for faithful handwritten-math OCR evaluation.
|
|
2
|
+
|
|
3
|
+
Seong et al., "When VLMs 'Fix' Students: Identifying and Penalizing Over-Correction in
|
|
4
|
+
the Evaluation of Multi-line Handwritten Math OCR", EMNLP 2026.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__version__ = "1.0.0"
|
|
8
|
+
|
|
9
|
+
from .judge import JUDGE_PRESETS, JudgeConfig, grade, judge_from_preset
|
|
10
|
+
from .rubric import GRADING_INSTRUCTIONS, RUBRIC_ITEMS, SCORE_COLUMNS
|
|
11
|
+
from .score import DEFAULT_THRESHOLD, PINKResult, compute_pink, load_grades, penalize, pink_table
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"__version__",
|
|
15
|
+
"DEFAULT_THRESHOLD",
|
|
16
|
+
"GRADING_INSTRUCTIONS",
|
|
17
|
+
"JUDGE_PRESETS",
|
|
18
|
+
"JudgeConfig",
|
|
19
|
+
"PINKResult",
|
|
20
|
+
"RUBRIC_ITEMS",
|
|
21
|
+
"SCORE_COLUMNS",
|
|
22
|
+
"compute_pink",
|
|
23
|
+
"grade",
|
|
24
|
+
"judge_from_preset",
|
|
25
|
+
"load_grades",
|
|
26
|
+
"penalize",
|
|
27
|
+
"pink_table",
|
|
28
|
+
]
|
pink/_runner.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Parallel, resumable execution of per-sample API calls with incremental output."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
import threading
|
|
8
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence
|
|
11
|
+
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def read_jsonl(path) -> List[Dict[str, Any]]:
|
|
16
|
+
with open(path, encoding="utf-8") as f:
|
|
17
|
+
return [json.loads(line) for line in f if line.strip()]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def write_jsonl(path, rows: Iterable[Dict[str, Any]]) -> None:
|
|
21
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
22
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
23
|
+
for row in rows:
|
|
24
|
+
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _read_rows(path: Path) -> List[Dict[str, Any]]:
|
|
28
|
+
if not path.exists() or path.stat().st_size == 0:
|
|
29
|
+
return []
|
|
30
|
+
if path.suffix == ".csv":
|
|
31
|
+
return pd.read_csv(path, dtype={"sample_id": str}, keep_default_na=False).to_dict("records")
|
|
32
|
+
return read_jsonl(path)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _is_success(row: Dict[str, Any]) -> bool:
|
|
36
|
+
return str(row.get("success", "")).lower() in ("true", "1")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _compact(path: Path, columns: Optional[Sequence[str]]) -> None:
|
|
40
|
+
"""Keep one row per sample (the last successful one, else the last one)."""
|
|
41
|
+
best: Dict[str, Dict[str, Any]] = {}
|
|
42
|
+
for row in _read_rows(path):
|
|
43
|
+
sid = str(row["sample_id"])
|
|
44
|
+
if sid not in best or _is_success(row) or not _is_success(best[sid]):
|
|
45
|
+
best[sid] = row
|
|
46
|
+
rows = list(best.values())
|
|
47
|
+
if path.suffix == ".csv":
|
|
48
|
+
pd.DataFrame(rows, columns=list(columns) if columns else None).to_csv(path, index=False)
|
|
49
|
+
else:
|
|
50
|
+
write_jsonl(path, rows)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def run_jobs(items: Sequence[Dict[str, Any]], fn: Callable[[Dict[str, Any]], Dict[str, Any]], out_path,
|
|
54
|
+
columns: Optional[Sequence[str]] = None, workers: int = 8, resume: bool = True,
|
|
55
|
+
desc: str = "") -> Dict[str, int]:
|
|
56
|
+
"""Call ``fn(item)`` for every item not yet done and append each result row to ``out_path``.
|
|
57
|
+
|
|
58
|
+
``out_path`` is CSV (``columns`` required) or JSONL. Re-running the same command
|
|
59
|
+
skips samples that already succeeded, so interrupted runs can simply be restarted.
|
|
60
|
+
"""
|
|
61
|
+
out_path = Path(out_path)
|
|
62
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
if not resume and out_path.exists():
|
|
64
|
+
out_path.unlink()
|
|
65
|
+
done = {str(r["sample_id"]) for r in _read_rows(out_path) if _is_success(r)}
|
|
66
|
+
todo = [it for it in items if str(it["sample_id"]) not in done]
|
|
67
|
+
|
|
68
|
+
lock = threading.Lock()
|
|
69
|
+
is_csv = out_path.suffix == ".csv"
|
|
70
|
+
new_file = not out_path.exists() or out_path.stat().st_size == 0
|
|
71
|
+
fh = open(out_path, "a", encoding="utf-8", newline="")
|
|
72
|
+
writer = csv.DictWriter(fh, fieldnames=list(columns), extrasaction="ignore") if is_csv else None
|
|
73
|
+
if is_csv and new_file:
|
|
74
|
+
writer.writeheader()
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
from tqdm import tqdm
|
|
78
|
+
bar = tqdm(total=len(todo), desc=desc or out_path.name)
|
|
79
|
+
except ImportError: # pragma: no cover
|
|
80
|
+
bar = None
|
|
81
|
+
|
|
82
|
+
n_ok = 0
|
|
83
|
+
try:
|
|
84
|
+
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
|
85
|
+
futures = [pool.submit(fn, it) for it in todo]
|
|
86
|
+
for fut in as_completed(futures):
|
|
87
|
+
row = fut.result()
|
|
88
|
+
with lock:
|
|
89
|
+
if is_csv:
|
|
90
|
+
writer.writerow(row)
|
|
91
|
+
else:
|
|
92
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
93
|
+
fh.flush()
|
|
94
|
+
n_ok += _is_success(row)
|
|
95
|
+
if bar is not None:
|
|
96
|
+
bar.update(1)
|
|
97
|
+
finally:
|
|
98
|
+
fh.close()
|
|
99
|
+
if bar is not None:
|
|
100
|
+
bar.close()
|
|
101
|
+
_compact(out_path, columns)
|
|
102
|
+
return {"skipped": len(done), "attempted": len(todo), "succeeded": n_ok, "failed": len(todo) - n_ok}
|
pink/cli.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Command-line interface: ``pink fermat | ocr | grade | score``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from functools import partial
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import List, Optional
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _limit(items: list, limit: Optional[int]) -> list:
|
|
16
|
+
return items if limit is None else items[:limit]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _check_endpoint(client, base_url: Optional[str], model: str) -> None:
|
|
20
|
+
"""Fail fast if a self-hosted endpoint is down or does not serve ``model``."""
|
|
21
|
+
if not base_url or "api.openai.com" in base_url:
|
|
22
|
+
return
|
|
23
|
+
try:
|
|
24
|
+
served = [m.id for m in client.models.list().data]
|
|
25
|
+
except Exception as e: # noqa: BLE001
|
|
26
|
+
sys.exit(f"cannot reach {base_url} ({type(e).__name__}). Is the server up? "
|
|
27
|
+
"Check with: curl {}/models".format(base_url.rstrip("/")))
|
|
28
|
+
if model not in served:
|
|
29
|
+
sys.exit(f"{base_url} does not serve {model!r}; it serves {served}. "
|
|
30
|
+
"Pass --model, or start the server with --served-model-name.")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def cmd_fermat(args: argparse.Namespace) -> int:
|
|
34
|
+
from .data import export_fermat
|
|
35
|
+
|
|
36
|
+
manifest = export_fermat(args.out_dir, limit=args.limit, cache_dir=args.cache_dir, seed=args.seed)
|
|
37
|
+
n = sum(1 for _ in open(manifest, encoding="utf-8"))
|
|
38
|
+
print(f"wrote {n} samples -> {manifest}")
|
|
39
|
+
print(f"next: grade the oracle once per judge, e.g.\n pink grade --input {manifest} --out oracle_grades.csv")
|
|
40
|
+
return 0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def cmd_ocr(args: argparse.Namespace) -> int:
|
|
44
|
+
from ._runner import read_jsonl, run_jobs
|
|
45
|
+
from .ocr import make_client, transcribe_row
|
|
46
|
+
|
|
47
|
+
items = _limit(read_jsonl(args.manifest), args.limit)
|
|
48
|
+
image_dir = args.image_dir or Path(args.manifest).parent
|
|
49
|
+
client = make_client(args.base_url, args.api_key)
|
|
50
|
+
_check_endpoint(client, args.base_url, args.model)
|
|
51
|
+
fn = partial(transcribe_row, client, args.model, image_dir=image_dir, prompt=args.prompt,
|
|
52
|
+
max_tokens=args.max_tokens, temperature=args.temperature)
|
|
53
|
+
stats = run_jobs(items, fn, args.out, workers=args.workers, resume=not args.overwrite, desc=f"ocr:{args.model}")
|
|
54
|
+
print(json.dumps(stats))
|
|
55
|
+
return 0 if stats["failed"] == 0 else 1
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def cmd_grade(args: argparse.Namespace) -> int:
|
|
59
|
+
from ._runner import read_jsonl, run_jobs
|
|
60
|
+
from .judge import JudgeConfig, grade, judge_from_preset, make_client
|
|
61
|
+
from .rubric import JUSTIFICATION_COLUMNS, SCORE_COLUMNS
|
|
62
|
+
|
|
63
|
+
if args.judge:
|
|
64
|
+
cfg = judge_from_preset(args.judge, model=args.model, base_url=args.base_url, api_key=args.api_key,
|
|
65
|
+
temperature=args.temperature, max_tokens=args.max_tokens,
|
|
66
|
+
reasoning_effort=args.reasoning_effort)
|
|
67
|
+
elif args.model:
|
|
68
|
+
cfg = JudgeConfig(model=args.model, base_url=args.base_url, api_key=args.api_key,
|
|
69
|
+
temperature=args.temperature, max_tokens=args.max_tokens,
|
|
70
|
+
reasoning_effort=args.reasoning_effort)
|
|
71
|
+
else:
|
|
72
|
+
sys.exit("pass --judge {gpt-5,qwen3-next-80b} or --model (with --base-url for a local server)")
|
|
73
|
+
|
|
74
|
+
items = [it for it in read_jsonl(args.input) if "solution" in it]
|
|
75
|
+
items = _limit(items, args.limit)
|
|
76
|
+
client = make_client(cfg)
|
|
77
|
+
_check_endpoint(client, cfg.base_url, cfg.model)
|
|
78
|
+
|
|
79
|
+
def fn(item):
|
|
80
|
+
return grade(client, cfg, item.get("problem", ""), item.get("solution", ""), str(item["sample_id"]))
|
|
81
|
+
|
|
82
|
+
columns = ["sample_id", "success", "total_score", *SCORE_COLUMNS, *JUSTIFICATION_COLUMNS, "attempts", "error"]
|
|
83
|
+
out = Path(args.out)
|
|
84
|
+
stats = run_jobs(items, fn, out, columns=columns if out.suffix == ".csv" else None,
|
|
85
|
+
workers=args.workers, resume=not args.overwrite, desc=f"grade:{cfg.model}")
|
|
86
|
+
print(json.dumps(stats))
|
|
87
|
+
return 0 if stats["failed"] == 0 else 1
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def cmd_score(args: argparse.Namespace) -> int:
|
|
91
|
+
from .score import compute_pink, load_grades, pink_table
|
|
92
|
+
|
|
93
|
+
model, oracle = load_grades(args.model), load_grades(args.oracle)
|
|
94
|
+
result = compute_pink(model, oracle, threshold=args.threshold, missing=args.missing)
|
|
95
|
+
if args.per_sample:
|
|
96
|
+
pink_table(model, oracle, threshold=args.threshold, missing=args.missing).to_csv(args.per_sample)
|
|
97
|
+
print(json.dumps(result.to_dict(), indent=2) if args.json else result)
|
|
98
|
+
return 0
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
102
|
+
p = argparse.ArgumentParser(prog="pink", description="PINK: penalized, over-correction-aware evaluation "
|
|
103
|
+
"of handwritten math OCR (EMNLP 2026).")
|
|
104
|
+
p.add_argument("--version", action="version", version=f"pink {__version__}")
|
|
105
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
106
|
+
|
|
107
|
+
f = sub.add_parser("fermat", help="download FERMAT and write images/ + manifest.jsonl (oracle transcriptions)")
|
|
108
|
+
f.add_argument("--out-dir", required=True)
|
|
109
|
+
f.add_argument("--limit", type=int, help="export a random subset of N samples (for a quick try)")
|
|
110
|
+
f.add_argument("--seed", type=int, default=0, help="seed for the --limit subset")
|
|
111
|
+
f.add_argument("--cache-dir", help="Hugging Face datasets cache directory")
|
|
112
|
+
f.set_defaults(func=cmd_fermat)
|
|
113
|
+
|
|
114
|
+
o = sub.add_parser("ocr", help="transcribe images with a VLM behind an OpenAI-compatible API")
|
|
115
|
+
o.add_argument("--manifest", required=True, help="manifest.jsonl from `pink fermat` (or your own)")
|
|
116
|
+
o.add_argument("--image-dir", help="directory the manifest's image paths are relative to (default: its folder)")
|
|
117
|
+
o.add_argument("--model", required=True, help="model name as served by the endpoint")
|
|
118
|
+
o.add_argument("--base-url", help="e.g. http://localhost:8000/v1 for `vllm serve` (default: OpenAI API)")
|
|
119
|
+
o.add_argument("--api-key")
|
|
120
|
+
o.add_argument("--prompt", choices=["default", "faithful"], default="default")
|
|
121
|
+
o.add_argument("--max-tokens", type=int, default=4096)
|
|
122
|
+
o.add_argument("--temperature", type=float, default=0.0)
|
|
123
|
+
o.add_argument("--workers", type=int, default=8)
|
|
124
|
+
o.add_argument("--limit", type=int)
|
|
125
|
+
o.add_argument("--overwrite", action="store_true", help="start from scratch instead of resuming")
|
|
126
|
+
o.add_argument("--out", required=True, help="output .jsonl (input for `pink grade`)")
|
|
127
|
+
o.set_defaults(func=cmd_ocr)
|
|
128
|
+
|
|
129
|
+
g = sub.add_parser("grade", help="grade solutions with the rubric-based LLM judge")
|
|
130
|
+
g.add_argument("--input", required=True, help="jsonl with sample_id, problem, solution")
|
|
131
|
+
g.add_argument("--out", required=True, help="output grades (.csv or .jsonl)")
|
|
132
|
+
g.add_argument("--judge", choices=["gpt-5", "qwen3-next-80b"], help="paper judge preset")
|
|
133
|
+
g.add_argument("--model", help="judge model name (overrides the preset)")
|
|
134
|
+
g.add_argument("--base-url", help="OpenAI-compatible endpoint of the judge")
|
|
135
|
+
g.add_argument("--api-key")
|
|
136
|
+
g.add_argument("--temperature", type=float)
|
|
137
|
+
g.add_argument("--max-tokens", type=int)
|
|
138
|
+
g.add_argument("--reasoning-effort", choices=["minimal", "low", "medium", "high"])
|
|
139
|
+
g.add_argument("--workers", type=int, default=8)
|
|
140
|
+
g.add_argument("--limit", type=int)
|
|
141
|
+
g.add_argument("--overwrite", action="store_true", help="start from scratch instead of resuming")
|
|
142
|
+
g.set_defaults(func=cmd_grade)
|
|
143
|
+
|
|
144
|
+
s = sub.add_parser("score", help="compute PINK from model and oracle grades")
|
|
145
|
+
s.add_argument("--model", required=True, help="grades of the VLM transcriptions")
|
|
146
|
+
s.add_argument("--oracle", required=True, help="grades of the oracle transcriptions (same judge!)")
|
|
147
|
+
s.add_argument("--threshold", type=float, default=10.0, help="minor/major boundary T (default 10)")
|
|
148
|
+
s.add_argument("--missing", choices=["drop", "zero"], default="drop",
|
|
149
|
+
help="oracle samples without a model grade: drop them (paper) or score them 0")
|
|
150
|
+
s.add_argument("--per-sample", metavar="CSV", help="also write per-sample penalized scores")
|
|
151
|
+
s.add_argument("--json", action="store_true")
|
|
152
|
+
s.set_defaults(func=cmd_score)
|
|
153
|
+
return p
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
157
|
+
args = build_parser().parse_args(argv)
|
|
158
|
+
return args.func(args)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
if __name__ == "__main__": # pragma: no cover
|
|
162
|
+
sys.exit(main())
|
pink/data.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""FERMAT (Nath et al., ACL 2025) as an OCR-faithfulness benchmark.
|
|
2
|
+
|
|
3
|
+
The dataset is downloaded from the Hugging Face Hub (``ai4bharat/FERMAT``, CC-BY-4.0).
|
|
4
|
+
It is gated: accept the terms on its page and log in (``huggingface-cli login``) first.
|
|
5
|
+
|
|
6
|
+
Each sample is a handwritten image of a math question and a student solution that
|
|
7
|
+
contains a (possibly superficial) error. The *oracle transcription* is the LaTeX of what
|
|
8
|
+
the student actually wrote (FERMAT's ``pert_a``), with FERMAT's ``\\textcolor{red}{...}``
|
|
9
|
+
markers around the inserted error removed so the judge cannot see where the error is.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Dict, List, Optional
|
|
16
|
+
|
|
17
|
+
FERMAT_HF_ID = "ai4bharat/FERMAT"
|
|
18
|
+
|
|
19
|
+
_RED = "\\textcolor{red}{"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _unwrap_red(text: str) -> str:
|
|
23
|
+
"""Replace every ``\\textcolor{red}{X}`` by ``X`` (brace-balanced, so nested LaTeX survives)."""
|
|
24
|
+
out, i = [], 0
|
|
25
|
+
while True:
|
|
26
|
+
j = text.find(_RED, i)
|
|
27
|
+
if j < 0:
|
|
28
|
+
out.append(text[i:])
|
|
29
|
+
return "".join(out)
|
|
30
|
+
out.append(text[i:j])
|
|
31
|
+
k, depth = j + len(_RED), 1
|
|
32
|
+
while k < len(text) and depth:
|
|
33
|
+
depth += {"{": 1, "}": -1}.get(text[k], 0)
|
|
34
|
+
k += 1
|
|
35
|
+
out.append(text[j + len(_RED):k - 1] if depth == 0 else text[j + len(_RED):])
|
|
36
|
+
i = k
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def oracle_transcription(pert_a: str) -> str:
|
|
40
|
+
"""FERMAT ``pert_a`` -> oracle transcription used by PINK (red error markers removed)."""
|
|
41
|
+
return _unwrap_red(str(pert_a)).replace("\r\n", "\n")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _image_ext(data: bytes, path: Optional[str]) -> str:
|
|
45
|
+
if data[:8] == b"\x89PNG\r\n\x1a\n":
|
|
46
|
+
return ".png"
|
|
47
|
+
if data[:3] == b"\xff\xd8\xff":
|
|
48
|
+
return ".jpg"
|
|
49
|
+
if path and Path(path).suffix:
|
|
50
|
+
return Path(path).suffix.lower()
|
|
51
|
+
return ".png"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def load_fermat(split: str = "train", cache_dir: Optional[str] = None):
|
|
55
|
+
"""Load FERMAT as a ``datasets.Dataset`` with undecoded images (``{"bytes", "path"}``)."""
|
|
56
|
+
try:
|
|
57
|
+
from datasets import Image, load_dataset
|
|
58
|
+
except ImportError as e: # pragma: no cover
|
|
59
|
+
raise ImportError("FERMAT support needs `pip install pink-eval[fermat]`") from e
|
|
60
|
+
ds = load_dataset(FERMAT_HF_ID, split=split, cache_dir=cache_dir)
|
|
61
|
+
return ds.cast_column("image", Image(decode=False))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def export_fermat(out_dir, limit: Optional[int] = None, ids: Optional[List[str]] = None,
|
|
65
|
+
cache_dir: Optional[str] = None, seed: int = 0) -> Path:
|
|
66
|
+
"""Write ``images/`` and ``manifest.jsonl`` to ``out_dir``.
|
|
67
|
+
|
|
68
|
+
Each manifest line holds ``sample_id``, ``image`` (relative path), ``problem`` (the
|
|
69
|
+
question text given to the judge) and ``solution`` (the oracle transcription), plus a
|
|
70
|
+
few FERMAT metadata fields. The manifest itself is the oracle input for ``pink grade``.
|
|
71
|
+
With ``limit``, a random subset (fixed ``seed``) is exported instead of the full set.
|
|
72
|
+
"""
|
|
73
|
+
from ._runner import write_jsonl
|
|
74
|
+
|
|
75
|
+
out_dir = Path(out_dir)
|
|
76
|
+
(out_dir / "images").mkdir(parents=True, exist_ok=True)
|
|
77
|
+
ds = load_fermat(cache_dir=cache_dir)
|
|
78
|
+
if ids:
|
|
79
|
+
wanted = set(ids)
|
|
80
|
+
ds = ds.filter(lambda r: r["new_custom_id"] in wanted)
|
|
81
|
+
if limit is not None:
|
|
82
|
+
ds = ds.shuffle(seed=seed).select(range(min(limit, len(ds))))
|
|
83
|
+
|
|
84
|
+
rows: List[Dict[str, Any]] = []
|
|
85
|
+
for r in ds:
|
|
86
|
+
sid = r["new_custom_id"]
|
|
87
|
+
img = r["image"]
|
|
88
|
+
rel = f"images/{sid}{_image_ext(img['bytes'], img.get('path'))}"
|
|
89
|
+
(out_dir / rel).write_bytes(img["bytes"])
|
|
90
|
+
rows.append({
|
|
91
|
+
"sample_id": sid,
|
|
92
|
+
"image": rel,
|
|
93
|
+
"problem": str(r["orig_q"]).replace("\r\n", "\n"),
|
|
94
|
+
"solution": oracle_transcription(r["pert_a"]),
|
|
95
|
+
"domain": r.get("domain_code"),
|
|
96
|
+
"grade": r.get("grade"),
|
|
97
|
+
"has_error": r.get("has_error"),
|
|
98
|
+
})
|
|
99
|
+
manifest = out_dir / "manifest.jsonl"
|
|
100
|
+
write_jsonl(manifest, rows)
|
|
101
|
+
return manifest
|
pink/judge.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""LLM judge: rubric-based semantic grading (paper Sec. 3.1).
|
|
2
|
+
|
|
3
|
+
Any OpenAI-compatible chat-completions endpoint works: the OpenAI API (GPT-5, the
|
|
4
|
+
paper's main judge) or a self-hosted model served with vLLM (e.g.
|
|
5
|
+
Qwen3-Next-80B-A3B-Thinking, the paper's open-source judge, see App. E).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import time
|
|
14
|
+
from dataclasses import dataclass, field, replace
|
|
15
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
16
|
+
|
|
17
|
+
from .rubric import (
|
|
18
|
+
GRADING_INSTRUCTIONS,
|
|
19
|
+
JUSTIFICATION_COLUMNS,
|
|
20
|
+
POINTS_PER_ITEM,
|
|
21
|
+
RUBRIC_ITEMS,
|
|
22
|
+
SCORE_COLUMNS,
|
|
23
|
+
build_user_message,
|
|
24
|
+
canonical_item,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class JudgeConfig:
|
|
30
|
+
"""How to reach the judge and which sampling parameters to send.
|
|
31
|
+
|
|
32
|
+
Parameters left as ``None`` are not sent, so the endpoint's defaults apply.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
model: str
|
|
36
|
+
base_url: Optional[str] = None #: ``None`` = OpenAI API
|
|
37
|
+
api_key: Optional[str] = None #: default: $OPENAI_API_KEY, or "EMPTY" for local servers
|
|
38
|
+
temperature: Optional[float] = None
|
|
39
|
+
top_p: Optional[float] = None
|
|
40
|
+
max_tokens: Optional[int] = None
|
|
41
|
+
reasoning_effort: Optional[str] = None #: OpenAI reasoning models only
|
|
42
|
+
extra_body: Dict[str, Any] = field(default_factory=dict)
|
|
43
|
+
max_attempts: int = 5 #: retries when the response cannot be parsed
|
|
44
|
+
timeout: float = 600.0
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
#: Judges used in the paper. ``gpt-5`` is the main judge; ``qwen3-next-80b`` is the
|
|
48
|
+
#: open-source judge (r=0.99 / Kendall tau=0.90 with GPT-5 at the VLM level, App. E.1).
|
|
49
|
+
JUDGE_PRESETS: Dict[str, JudgeConfig] = {
|
|
50
|
+
"gpt-5": JudgeConfig(model="gpt-5-2025-08-07", reasoning_effort="medium"),
|
|
51
|
+
"qwen3-next-80b": JudgeConfig(
|
|
52
|
+
model="Qwen/Qwen3-Next-80B-A3B-Thinking",
|
|
53
|
+
base_url="http://localhost:8000/v1",
|
|
54
|
+
temperature=0.6,
|
|
55
|
+
top_p=0.95,
|
|
56
|
+
),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def judge_from_preset(name: str, **overrides: Any) -> JudgeConfig:
|
|
61
|
+
if name not in JUDGE_PRESETS:
|
|
62
|
+
raise KeyError(f"unknown judge preset {name!r}; choose from {sorted(JUDGE_PRESETS)}")
|
|
63
|
+
overrides = {k: v for k, v in overrides.items() if v is not None}
|
|
64
|
+
return replace(JUDGE_PRESETS[name], **overrides)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_openai(cfg: JudgeConfig) -> bool:
|
|
68
|
+
return cfg.base_url is None or "api.openai.com" in cfg.base_url
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def make_client(cfg: JudgeConfig):
|
|
72
|
+
from openai import OpenAI
|
|
73
|
+
|
|
74
|
+
key = cfg.api_key or os.environ.get("OPENAI_API_KEY") or ("EMPTY" if not _is_openai(cfg) else None)
|
|
75
|
+
return OpenAI(api_key=key, base_url=cfg.base_url, timeout=cfg.timeout)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# --------------------------------------------------------------------------- parsing
|
|
79
|
+
|
|
80
|
+
def parse_judge_response(text: str) -> Dict[str, Any]:
|
|
81
|
+
"""Extract the JSON grading object from a (possibly noisy) judge response."""
|
|
82
|
+
if text is None:
|
|
83
|
+
raise ValueError("empty response")
|
|
84
|
+
text = text.strip()
|
|
85
|
+
if "</think>" in text: # reasoning trace without an opening tag
|
|
86
|
+
text = text.rsplit("</think>", 1)[1].strip()
|
|
87
|
+
text = re.sub(r"<(think|thought|reasoning)>[\s\S]*?</\1>", "", text).strip()
|
|
88
|
+
|
|
89
|
+
fences = re.findall(r"```(?:json)?\s*([\s\S]*?)\s*```", text)
|
|
90
|
+
for block in reversed(fences):
|
|
91
|
+
try:
|
|
92
|
+
return json.loads(block)
|
|
93
|
+
except json.JSONDecodeError:
|
|
94
|
+
continue
|
|
95
|
+
try:
|
|
96
|
+
return json.loads(text)
|
|
97
|
+
except json.JSONDecodeError:
|
|
98
|
+
pass
|
|
99
|
+
|
|
100
|
+
candidates: List[str] = []
|
|
101
|
+
depth, start = 0, -1
|
|
102
|
+
for i, ch in enumerate(text):
|
|
103
|
+
if ch == "{":
|
|
104
|
+
if depth == 0:
|
|
105
|
+
start = i
|
|
106
|
+
depth += 1
|
|
107
|
+
elif ch == "}" and depth:
|
|
108
|
+
depth -= 1
|
|
109
|
+
if depth == 0:
|
|
110
|
+
candidates.append(text[start:i + 1])
|
|
111
|
+
pool = [c for c in candidates if '"scores"' in c] or candidates
|
|
112
|
+
for cand in reversed(pool):
|
|
113
|
+
for attempt in (cand, re.sub(r",\s*([}\]])", r"\1", cand)):
|
|
114
|
+
try:
|
|
115
|
+
return json.loads(attempt)
|
|
116
|
+
except json.JSONDecodeError:
|
|
117
|
+
continue
|
|
118
|
+
raise ValueError(f"could not parse JSON from judge response: {text[:200]!r}")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _to_score(value: Any) -> int:
|
|
122
|
+
if isinstance(value, str) and "/" in value:
|
|
123
|
+
value = value.split("/")[0]
|
|
124
|
+
score = int(round(float(value)))
|
|
125
|
+
return max(0, min(POINTS_PER_ITEM, score))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def extract_scores(parsed: Dict[str, Any]) -> Tuple[Dict[str, int], Dict[str, str]]:
|
|
129
|
+
"""Return ``(scores, justifications)`` keyed by rubric item; raises if an item is missing."""
|
|
130
|
+
entries = parsed.get("scores") if isinstance(parsed, dict) else None
|
|
131
|
+
if not isinstance(entries, list):
|
|
132
|
+
raise ValueError("judge response has no 'scores' list")
|
|
133
|
+
entries = [e for e in entries if isinstance(e, dict)]
|
|
134
|
+
|
|
135
|
+
scores: Dict[str, int] = {}
|
|
136
|
+
justifs: Dict[str, str] = {}
|
|
137
|
+
for e in entries:
|
|
138
|
+
item = canonical_item(str(e.get("item", "")))
|
|
139
|
+
if item is not None and item not in scores:
|
|
140
|
+
scores[item] = _to_score(e.get("score", 0))
|
|
141
|
+
justifs[item] = str(e.get("justification", ""))
|
|
142
|
+
if len(scores) < len(RUBRIC_ITEMS) and len(entries) == len(RUBRIC_ITEMS):
|
|
143
|
+
# item names unrecognisable but the list has the right shape: use the rubric order
|
|
144
|
+
scores = {item: _to_score(e.get("score", 0)) for item, e in zip(RUBRIC_ITEMS, entries)}
|
|
145
|
+
justifs = {item: str(e.get("justification", "")) for item, e in zip(RUBRIC_ITEMS, entries)}
|
|
146
|
+
missing = [i for i in RUBRIC_ITEMS if i not in scores]
|
|
147
|
+
if missing:
|
|
148
|
+
raise ValueError(f"judge response is missing rubric items: {missing}")
|
|
149
|
+
return scores, justifs
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# --------------------------------------------------------------------------- grading
|
|
153
|
+
|
|
154
|
+
def _request_kwargs(cfg: JudgeConfig) -> Dict[str, Any]:
|
|
155
|
+
kw: Dict[str, Any] = {}
|
|
156
|
+
if cfg.temperature is not None:
|
|
157
|
+
kw["temperature"] = cfg.temperature
|
|
158
|
+
if cfg.top_p is not None:
|
|
159
|
+
kw["top_p"] = cfg.top_p
|
|
160
|
+
if cfg.max_tokens is not None:
|
|
161
|
+
kw["max_completion_tokens" if _is_openai(cfg) else "max_tokens"] = cfg.max_tokens
|
|
162
|
+
if cfg.reasoning_effort is not None:
|
|
163
|
+
kw["reasoning_effort"] = cfg.reasoning_effort
|
|
164
|
+
if cfg.extra_body:
|
|
165
|
+
kw["extra_body"] = dict(cfg.extra_body)
|
|
166
|
+
return kw
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def grade(client, cfg: JudgeConfig, problem: str, solution: str, sample_id: str = "",
|
|
170
|
+
instructions: str = GRADING_INSTRUCTIONS) -> Dict[str, Any]:
|
|
171
|
+
"""Grade one solution. Returns a grade-file row (see :data:`SCORE_COLUMNS`)."""
|
|
172
|
+
row: Dict[str, Any] = {"sample_id": sample_id, "success": False, "total_score": None}
|
|
173
|
+
if not str(solution or "").strip():
|
|
174
|
+
row["error"] = "empty solution"
|
|
175
|
+
return row
|
|
176
|
+
messages = [
|
|
177
|
+
{"role": "system", "content": instructions},
|
|
178
|
+
{"role": "user", "content": build_user_message(problem, solution)},
|
|
179
|
+
]
|
|
180
|
+
error = ""
|
|
181
|
+
for attempt in range(cfg.max_attempts):
|
|
182
|
+
finish = None
|
|
183
|
+
try:
|
|
184
|
+
resp = client.chat.completions.create(model=cfg.model, messages=messages, **_request_kwargs(cfg))
|
|
185
|
+
finish = getattr(resp.choices[0], "finish_reason", None)
|
|
186
|
+
content = resp.choices[0].message.content or ""
|
|
187
|
+
scores, justifs = extract_scores(parse_judge_response(content))
|
|
188
|
+
except Exception as e: # noqa: BLE001 - keep the run going, record the failure
|
|
189
|
+
error = f"{type(e).__name__}: {e}"
|
|
190
|
+
if finish == "length":
|
|
191
|
+
error = ("response truncated at the token limit (finish_reason=length): give the judge more room, "
|
|
192
|
+
"e.g. a larger --max-model-len for vLLM or a larger --max-tokens. " + error)
|
|
193
|
+
if attempt + 1 < cfg.max_attempts:
|
|
194
|
+
time.sleep(min(2 ** attempt, 30))
|
|
195
|
+
continue
|
|
196
|
+
row.update({"success": True, "total_score": sum(scores.values()), "attempts": attempt + 1})
|
|
197
|
+
for item, s_col, j_col in zip(RUBRIC_ITEMS, SCORE_COLUMNS, JUSTIFICATION_COLUMNS):
|
|
198
|
+
row[s_col] = scores[item]
|
|
199
|
+
row[j_col] = justifs[item]
|
|
200
|
+
return row
|
|
201
|
+
row.update({"error": error, "attempts": cfg.max_attempts})
|
|
202
|
+
return row
|
pink/ocr.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""VLM transcription of handwritten math images (paper Sec. 4, prompt in Fig. 18).
|
|
2
|
+
|
|
3
|
+
Works with any OpenAI-compatible vision endpoint (OpenAI, Gemini's OpenAI-compatible
|
|
4
|
+
API, or an open model served with ``vllm serve``). You can also transcribe with your own
|
|
5
|
+
stack and skip this module entirely; ``pink grade`` only needs the resulting text.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import base64
|
|
11
|
+
import mimetypes
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Dict, Optional, Tuple
|
|
16
|
+
|
|
17
|
+
SYSTEM_PROMPT = (
|
|
18
|
+
"You are a math assistant specializing in extracting mathematical question and answer content from "
|
|
19
|
+
"handwritten images of math problems by middle or high school students. Your task is to analyze the given "
|
|
20
|
+
"Image, which contains the handwritten math Question-Answer pair, and convert it into LaTeX format. Follow "
|
|
21
|
+
"the specific instructions provided carefully and output your response strictly in the requested format. "
|
|
22
|
+
"Ensure that your evaluation is precise, accurate, and based solely on the information visible in the image."
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
USER_PROMPT = """The image provided contains a handwritten math problem with both a Question and an Answer at a middle or high school level. Your task is to explicitly perform OCR on the handwritten text and extract the content in LaTeX format. Return only the extracted content, formatted in LaTeX, exactly as it appears in the image, in the **Question:** and **Answer:** fields.
|
|
26
|
+
|
|
27
|
+
Ensure that no extra information is added that is not in the image.
|
|
28
|
+
|
|
29
|
+
Please return the LaTeX output as follows:
|
|
30
|
+
**Question:**<Extracted Question text in LaTeX>
|
|
31
|
+
**Answer:**<Extracted Answer text in LaTeX>"""
|
|
32
|
+
|
|
33
|
+
# Prompt with explicit faithfulness instructions (paper Fig. 19, App. F). In the paper it
|
|
34
|
+
# reduced over-correction only slightly and did not improve PINK on average.
|
|
35
|
+
FAITHFUL_SYSTEM_PROMPT = (
|
|
36
|
+
"You are a math assistant specializing in extracting mathematical question and answer content from "
|
|
37
|
+
"handwritten images of math problems by middle or high school students. Your task is to analyze the given "
|
|
38
|
+
"Image, which contains the handwritten math Question-Answer pair, and convert it into LaTeX format. Follow "
|
|
39
|
+
"the specific instructions provided carefully and output your response strictly in the requested format. "
|
|
40
|
+
"Ensure that your transcription is precise, accurate, and based solely on the information visible in the "
|
|
41
|
+
"image. IMPORTANT: You must transcribe EXACTLY what is written, including any errors or mistakes made by "
|
|
42
|
+
"the student. Do NOT correct or fix any mathematical errors."
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
FAITHFUL_USER_PROMPT = """The image provided contains a handwritten math problem with both a Question and an Answer at a middle or high school level. Your task is to explicitly perform OCR on the handwritten text and extract the content in LaTeX format.
|
|
46
|
+
|
|
47
|
+
CRITICAL INSTRUCTIONS FOR FAITHFUL TRANSCRIPTION:
|
|
48
|
+
- Transcribe EXACTLY what is written in the image, including any mathematical errors or mistakes.
|
|
49
|
+
- Do NOT correct, fix, or improve any errors in the student's work.
|
|
50
|
+
- If the student wrote "2 + 2 = 5", you must transcribe it as "2 + 2 = 5", NOT "2 + 2 = 4".
|
|
51
|
+
- Your goal is FAITHFUL TRANSCRIPTION, not mathematical correction.
|
|
52
|
+
- Preserve all calculation errors, wrong answers, and mistakes exactly as they appear.
|
|
53
|
+
|
|
54
|
+
Return only the extracted content, formatted in LaTeX, exactly as it appears in the image, in the **Question:** and **Answer:** fields.
|
|
55
|
+
|
|
56
|
+
Ensure that no extra information is added that is not in the image.
|
|
57
|
+
|
|
58
|
+
Please return the LaTeX output as follows:
|
|
59
|
+
**Question:**<Extracted Question text in LaTeX>
|
|
60
|
+
**Answer:**<Extracted Answer text in LaTeX>"""
|
|
61
|
+
|
|
62
|
+
PROMPTS: Dict[str, Tuple[str, str]] = {
|
|
63
|
+
"default": (SYSTEM_PROMPT, USER_PROMPT),
|
|
64
|
+
"faithful": (FAITHFUL_SYSTEM_PROMPT, FAITHFUL_USER_PROMPT),
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def clean_transcription(text: Any) -> str:
|
|
69
|
+
"""Strip markdown fences and the **Question:** / **Answer:** labels from a raw VLM output."""
|
|
70
|
+
text = "" if text is None else str(text)
|
|
71
|
+
text = re.sub(r"^\s*```[a-zA-Z]*\s*\n?", "", text)
|
|
72
|
+
text = re.sub(r"\n?```\s*$", "", text)
|
|
73
|
+
text = re.sub(r"\*\*(Question|Answer):\*\*[ \t]*", "", text)
|
|
74
|
+
text = re.sub(r"\\textbf\{(Question|Answer)\s*:?\s*\}:?[ \t]*", "", text)
|
|
75
|
+
text = re.sub(r"^[ \t]*(Question|Answer):[ \t]*", "", text, flags=re.MULTILINE)
|
|
76
|
+
return text.strip()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def image_to_data_url(image_path) -> str:
|
|
80
|
+
mime = mimetypes.guess_type(str(image_path))[0] or "image/png"
|
|
81
|
+
with open(image_path, "rb") as f:
|
|
82
|
+
return f"data:{mime};base64,{base64.b64encode(f.read()).decode()}"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def make_client(base_url: Optional[str] = None, api_key: Optional[str] = None, timeout: float = 600.0):
|
|
86
|
+
from openai import OpenAI
|
|
87
|
+
|
|
88
|
+
key = api_key or os.environ.get("OPENAI_API_KEY") or ("EMPTY" if base_url else None)
|
|
89
|
+
return OpenAI(api_key=key, base_url=base_url, timeout=timeout)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def transcribe(client, model: str, image_path, prompt: str = "default", max_tokens: Optional[int] = 4096,
|
|
93
|
+
temperature: Optional[float] = 0.0) -> str:
|
|
94
|
+
"""Transcribe one image and return the raw model output."""
|
|
95
|
+
system, user = PROMPTS[prompt]
|
|
96
|
+
kwargs: Dict[str, Any] = {}
|
|
97
|
+
if max_tokens is not None:
|
|
98
|
+
kwargs["max_tokens"] = max_tokens
|
|
99
|
+
if temperature is not None:
|
|
100
|
+
kwargs["temperature"] = temperature
|
|
101
|
+
resp = client.chat.completions.create(
|
|
102
|
+
model=model,
|
|
103
|
+
messages=[
|
|
104
|
+
{"role": "system", "content": system},
|
|
105
|
+
{"role": "user", "content": [
|
|
106
|
+
{"type": "text", "text": user},
|
|
107
|
+
{"type": "image_url", "image_url": {"url": image_to_data_url(image_path)}},
|
|
108
|
+
]},
|
|
109
|
+
],
|
|
110
|
+
**kwargs,
|
|
111
|
+
)
|
|
112
|
+
return resp.choices[0].message.content or ""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def transcribe_row(client, model: str, item: Dict[str, Any], image_dir, attempts: int = 3,
|
|
116
|
+
**kwargs: Any) -> Dict[str, Any]:
|
|
117
|
+
"""Transcribe the image of one manifest item; returns a row ready for ``pink grade``."""
|
|
118
|
+
row = {"sample_id": item["sample_id"], "problem": item.get("problem", ""), "solution": "",
|
|
119
|
+
"raw_output": "", "success": False}
|
|
120
|
+
image_path = Path(image_dir) / item["image"]
|
|
121
|
+
error = ""
|
|
122
|
+
for _ in range(attempts):
|
|
123
|
+
try:
|
|
124
|
+
raw = transcribe(client, model, image_path, **kwargs)
|
|
125
|
+
except Exception as e: # noqa: BLE001
|
|
126
|
+
error = f"{type(e).__name__}: {e}"
|
|
127
|
+
continue
|
|
128
|
+
row.update(raw_output=raw, solution=clean_transcription(raw), success=bool(raw.strip()))
|
|
129
|
+
if row["success"]:
|
|
130
|
+
return row
|
|
131
|
+
error = "empty output"
|
|
132
|
+
row["error"] = error
|
|
133
|
+
return row
|
pink/rubric.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""The 5-component, 100-point grading rubric used by PINK (paper Sec. 3.1, App. B.1, Fig. 16)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from typing import Dict, List
|
|
7
|
+
|
|
8
|
+
#: Rubric components, in the order the judge is asked to report them.
|
|
9
|
+
RUBRIC_ITEMS: List[str] = [
|
|
10
|
+
"Core Formula/Principle Identification",
|
|
11
|
+
"Condition/Boundary Application",
|
|
12
|
+
"Calculation Accuracy (Early-Mid)",
|
|
13
|
+
"Calculation Accuracy (Late)",
|
|
14
|
+
"Final Value Verification",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
#: Maximum score of every rubric component.
|
|
18
|
+
POINTS_PER_ITEM: int = 20
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _slug(item: str) -> str:
|
|
22
|
+
return re.sub(r"[^A-Za-z0-9-]+", "_", item).strip("_")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
#: Column names used in grade files: ``score_<item>`` and ``justif_<item>``.
|
|
26
|
+
SCORE_COLUMNS: List[str] = [f"score_{_slug(i)}" for i in RUBRIC_ITEMS]
|
|
27
|
+
JUSTIFICATION_COLUMNS: List[str] = [f"justif_{_slug(i)}" for i in RUBRIC_ITEMS]
|
|
28
|
+
|
|
29
|
+
#: System prompt for the LLM judge (paper Fig. 16).
|
|
30
|
+
GRADING_INSTRUCTIONS: str = """# Math Exam Grading Instructions
|
|
31
|
+
|
|
32
|
+
## Role
|
|
33
|
+
You are a math exam grader. Evaluate the student solution using the provided rubric and grading guidelines.
|
|
34
|
+
|
|
35
|
+
## Rubric
|
|
36
|
+
{
|
|
37
|
+
"rubric": [
|
|
38
|
+
{
|
|
39
|
+
"item": "Core Formula/Principle Identification",
|
|
40
|
+
"points": 20,
|
|
41
|
+
"criterion": "Correctly identified and stated the necessary formulas for solving the problem"
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"item": "Condition/Boundary Application",
|
|
45
|
+
"points": 20,
|
|
46
|
+
"criterion": "Accurately applied given conditions (boundaries, constraints, units, etc.)"
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"item": "Calculation Accuracy (Early-Mid)",
|
|
50
|
+
"points": 20,
|
|
51
|
+
"criterion": "Intermediate calculations were performed without omissions or contradictions"
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"item": "Calculation Accuracy (Late)",
|
|
55
|
+
"points": 20,
|
|
56
|
+
"criterion": "Final transformations and simplifications were logical and error-free"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"item": "Final Value Verification",
|
|
60
|
+
"points": 20,
|
|
61
|
+
"criterion": "Final answer correctly incorporates all conditions and calculations"
|
|
62
|
+
}
|
|
63
|
+
],
|
|
64
|
+
"notes": "Points may be deducted for unit, rounding, or notation errors"
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
## Grading Guidelines
|
|
68
|
+
1. **Analyze each expression character by character** to identify any errors in formulas, calculations, or transformations, and provide a detailed justification explaining exactly where and why points were awarded or deducted.
|
|
69
|
+
2. Include specific numbers, equations, or character positions in justification when necessary (e.g., "In line 3, the expression 2x+3 should be 2x-3, where the + sign at position 3 is incorrect").
|
|
70
|
+
3. Total score must be an integer between 0-100.
|
|
71
|
+
4. Output only the JSON format below. No additional text.
|
|
72
|
+
|
|
73
|
+
## Output Format
|
|
74
|
+
{
|
|
75
|
+
"scores": [
|
|
76
|
+
{"item": "Core Formula/Principle Identification", "score": 0~20, "justification": "..."},
|
|
77
|
+
{"item": "Condition/Boundary Application", "score": 0~20, "justification": "..."},
|
|
78
|
+
{"item": "Calculation Accuracy (Early-Mid)", "score": 0~20, "justification": "..."},
|
|
79
|
+
{"item": "Calculation Accuracy (Late)", "score": 0~20, "justification": "..."},
|
|
80
|
+
{"item": "Final Value Verification", "score": 0~20, "justification": "..."}
|
|
81
|
+
],
|
|
82
|
+
"total": <integer>
|
|
83
|
+
}"""
|
|
84
|
+
|
|
85
|
+
#: User message for the LLM judge. ``solution`` is the text being graded
|
|
86
|
+
#: (an OCR transcription, or the oracle transcription of the student's work).
|
|
87
|
+
USER_TEMPLATE: str = (
|
|
88
|
+
"**Problem:**\n{problem}\n\n"
|
|
89
|
+
"**Student Solution:**\n{solution}\n\n"
|
|
90
|
+
"Evaluate this solution according to the rubric and output the JSON result."
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def build_user_message(problem: str, solution: str) -> str:
|
|
95
|
+
return USER_TEMPLATE.format(problem=problem, solution=solution)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _norm(name: str) -> str:
|
|
99
|
+
return re.sub(r"[^a-z0-9]", "", name.lower())
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
_CANONICAL: Dict[str, str] = {_norm(i): i for i in RUBRIC_ITEMS}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def canonical_item(name: str) -> str | None:
|
|
106
|
+
"""Map a (possibly slightly reworded) item name returned by a judge to a rubric item."""
|
|
107
|
+
key = _norm(name)
|
|
108
|
+
if not key:
|
|
109
|
+
return None
|
|
110
|
+
if key in _CANONICAL:
|
|
111
|
+
return _CANONICAL[key]
|
|
112
|
+
hits = [canon for canon_key, canon in _CANONICAL.items() if canon_key in key or key in canon_key]
|
|
113
|
+
return hits[0] if len(hits) == 1 else None
|
pink/score.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""PINK score: over-correction penalty (paper Eq. 2-3) and aggregation (Eq. 4).
|
|
2
|
+
|
|
3
|
+
For every rubric component r of a sample, with dS_r = S_model,r - S_oracle,r :
|
|
4
|
+
|
|
5
|
+
S'_r = S_model,r if dS_r <= 0 (no over-correction)
|
|
6
|
+
S_oracle,r if 0 < dS_r <= T (minor: revert to the oracle score)
|
|
7
|
+
0 if dS_r > T (major: nullify)
|
|
8
|
+
|
|
9
|
+
S_penalized = sum_r S'_r
|
|
10
|
+
PINK = sum_n S_penalized,n / sum_n S_oracle,n
|
|
11
|
+
|
|
12
|
+
with T = 10 (half of a 20-point component).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
from dataclasses import asdict, dataclass, field
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Dict, Union
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
import pandas as pd
|
|
24
|
+
|
|
25
|
+
from .rubric import RUBRIC_ITEMS, SCORE_COLUMNS
|
|
26
|
+
|
|
27
|
+
DEFAULT_THRESHOLD: float = 10.0
|
|
28
|
+
|
|
29
|
+
PathLike = Union[str, Path]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def penalize(model_scores, oracle_scores, threshold: float = DEFAULT_THRESHOLD) -> np.ndarray:
|
|
33
|
+
"""Apply the two-tier over-correction penalty element-wise (Eq. 2)."""
|
|
34
|
+
m = np.asarray(model_scores, dtype=float)
|
|
35
|
+
o = np.asarray(oracle_scores, dtype=float)
|
|
36
|
+
delta = m - o
|
|
37
|
+
return np.where(delta <= 0, m, np.where(delta <= threshold, o, 0.0))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_grades(path: PathLike) -> pd.DataFrame:
|
|
41
|
+
"""Load a grade file written by ``pink grade`` (CSV or JSONL).
|
|
42
|
+
|
|
43
|
+
Returns successfully graded rows, indexed by ``sample_id``, with one column per
|
|
44
|
+
rubric component (see :data:`pink.rubric.SCORE_COLUMNS`).
|
|
45
|
+
"""
|
|
46
|
+
path = Path(path)
|
|
47
|
+
if path.suffix == ".jsonl":
|
|
48
|
+
df = pd.read_json(path, lines=True, dtype={"sample_id": str})
|
|
49
|
+
else:
|
|
50
|
+
df = pd.read_csv(path, dtype={"sample_id": str})
|
|
51
|
+
missing = [c for c in ["sample_id", *SCORE_COLUMNS] if c not in df.columns]
|
|
52
|
+
if missing:
|
|
53
|
+
raise ValueError(f"{path}: missing columns {missing}")
|
|
54
|
+
if "success" in df.columns:
|
|
55
|
+
ok = df["success"].astype(str).str.lower().isin(["true", "1"])
|
|
56
|
+
df = df[ok]
|
|
57
|
+
df = df.dropna(subset=SCORE_COLUMNS)
|
|
58
|
+
df = df.drop_duplicates("sample_id", keep="last").set_index("sample_id")
|
|
59
|
+
return df[SCORE_COLUMNS].astype(float)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def pink_table(model: pd.DataFrame, oracle: pd.DataFrame, threshold: float = DEFAULT_THRESHOLD,
|
|
63
|
+
missing: str = "drop") -> pd.DataFrame:
|
|
64
|
+
"""Per-sample table with model, oracle and penalized totals.
|
|
65
|
+
|
|
66
|
+
``model`` / ``oracle`` are DataFrames indexed by sample id with :data:`SCORE_COLUMNS`
|
|
67
|
+
(e.g. from :func:`load_grades`). Only samples graded for the oracle are used.
|
|
68
|
+
``missing`` controls oracle samples without a model grade: ``"drop"`` excludes them
|
|
69
|
+
(as in the paper), ``"zero"`` counts them as a 0-score transcription.
|
|
70
|
+
"""
|
|
71
|
+
if missing not in ("drop", "zero"):
|
|
72
|
+
raise ValueError("missing must be 'drop' or 'zero'")
|
|
73
|
+
if missing == "drop":
|
|
74
|
+
ids = oracle.index.intersection(model.index)
|
|
75
|
+
m = model.loc[ids, SCORE_COLUMNS]
|
|
76
|
+
else:
|
|
77
|
+
ids = oracle.index
|
|
78
|
+
m = model.reindex(ids)[SCORE_COLUMNS].fillna(0.0)
|
|
79
|
+
o = oracle.loc[ids, SCORE_COLUMNS]
|
|
80
|
+
|
|
81
|
+
out = pd.DataFrame(index=ids)
|
|
82
|
+
out.index.name = "sample_id"
|
|
83
|
+
pen = penalize(m.to_numpy(), o.to_numpy(), threshold)
|
|
84
|
+
over = m.to_numpy() > o.to_numpy()
|
|
85
|
+
for j in range(len(RUBRIC_ITEMS)):
|
|
86
|
+
out[f"over_{SCORE_COLUMNS[j][len('score_'):]}"] = over[:, j]
|
|
87
|
+
out["model_total"] = m.sum(axis=1).to_numpy()
|
|
88
|
+
out["oracle_total"] = o.sum(axis=1).to_numpy()
|
|
89
|
+
out["penalized_total"] = pen.sum(axis=1)
|
|
90
|
+
out["over_corrected"] = over.any(axis=1)
|
|
91
|
+
return out
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass
|
|
95
|
+
class PINKResult:
|
|
96
|
+
pink: float #: PINK score (Eq. 4)
|
|
97
|
+
raw_ratio: float #: sum(S_model) / sum(S_oracle), i.e. without the penalty
|
|
98
|
+
over_correction_rate: float #: share of samples with >= 1 over-corrected component
|
|
99
|
+
n: int #: samples used
|
|
100
|
+
n_oracle: int #: samples with an oracle grade
|
|
101
|
+
n_model: int #: samples with a model grade
|
|
102
|
+
threshold: float = DEFAULT_THRESHOLD
|
|
103
|
+
over_corrections_per_item: Dict[str, int] = field(default_factory=dict)
|
|
104
|
+
|
|
105
|
+
def to_dict(self) -> dict:
|
|
106
|
+
return asdict(self)
|
|
107
|
+
|
|
108
|
+
def __str__(self) -> str:
|
|
109
|
+
lines = [
|
|
110
|
+
f"PINK = {self.pink:.4f}",
|
|
111
|
+
f"raw (no penalty) = {self.raw_ratio:.4f}",
|
|
112
|
+
f"over-correction rate = {self.over_correction_rate * 100:.1f}%",
|
|
113
|
+
f"samples used = {self.n} (oracle graded: {self.n_oracle}, model graded: {self.n_model})",
|
|
114
|
+
]
|
|
115
|
+
return "\n".join(lines)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def compute_pink(model: Union[pd.DataFrame, PathLike], oracle: Union[pd.DataFrame, PathLike],
|
|
119
|
+
threshold: float = DEFAULT_THRESHOLD, missing: str = "drop") -> PINKResult:
|
|
120
|
+
"""Compute PINK from model grades and oracle grades (DataFrames or grade-file paths)."""
|
|
121
|
+
model_df = model if isinstance(model, pd.DataFrame) else load_grades(model)
|
|
122
|
+
oracle_df = oracle if isinstance(oracle, pd.DataFrame) else load_grades(oracle)
|
|
123
|
+
table = pink_table(model_df, oracle_df, threshold=threshold, missing=missing)
|
|
124
|
+
|
|
125
|
+
oracle_sum = table["oracle_total"].sum()
|
|
126
|
+
if len(table) == 0 or oracle_sum <= 0:
|
|
127
|
+
pink = raw = float("nan")
|
|
128
|
+
else:
|
|
129
|
+
pink = float(table["penalized_total"].sum() / oracle_sum)
|
|
130
|
+
raw = float(table["model_total"].sum() / oracle_sum)
|
|
131
|
+
per_item = {
|
|
132
|
+
item: int(table[f"over_{col[len('score_'):]}"].sum()) for item, col in zip(RUBRIC_ITEMS, SCORE_COLUMNS)
|
|
133
|
+
}
|
|
134
|
+
return PINKResult(
|
|
135
|
+
pink=pink,
|
|
136
|
+
raw_ratio=raw,
|
|
137
|
+
over_correction_rate=float(table["over_corrected"].mean()) if len(table) else float("nan"),
|
|
138
|
+
n=len(table),
|
|
139
|
+
n_oracle=len(oracle_df),
|
|
140
|
+
n_model=len(model_df),
|
|
141
|
+
threshold=threshold,
|
|
142
|
+
over_corrections_per_item=per_item,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def result_to_json(result: PINKResult) -> str:
|
|
147
|
+
return json.dumps(result.to_dict(), indent=2)
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pink-eval
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: PINK: over-correction-aware evaluation of handwritten math OCR (EMNLP 2026)
|
|
5
|
+
Author: Jin Seong, Wencke Liermann, Minho Kim, Jong-hun Shin, Soojong Lim
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Paper, https://aclanthology.org/2026.emnlp-main.476/
|
|
8
|
+
Project-URL: Code, https://github.com/9115jin/PINK
|
|
9
|
+
Keywords: OCR,handwritten math,vision-language models,evaluation,LLM-as-a-judge,over-correction
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: numpy>=1.21
|
|
16
|
+
Requires-Dist: pandas>=1.3
|
|
17
|
+
Requires-Dist: openai>=1.55
|
|
18
|
+
Requires-Dist: tqdm>=4.0
|
|
19
|
+
Provides-Extra: fermat
|
|
20
|
+
Requires-Dist: datasets>=2.18; extra == "fermat"
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
# PINK: Penalized INK-based Score
|
|
26
|
+
|
|
27
|
+
**When VLMs ‘Fix’ Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math OCR**
|
|
28
|
+
<br>Jin Seong\*, Wencke Liermann\*, Minho Kim, Jong-hun Shin, Soojong Lim · *EMNLP 2026 (Main, Oral)*
|
|
29
|
+
|
|
30
|
+
[](https://aclanthology.org/2026.emnlp-main.476/)
|
|
31
|
+
[](https://pypi.org/project/pink-eval/)
|
|
32
|
+
[](https://github.com/9115jin/PINK/actions/workflows/tests.yml)
|
|
33
|
+
[](https://github.com/9115jin/PINK/blob/main/LICENSE)
|
|
34
|
+
|
|
35
|
+
Vision-language models transcribing a student's handwritten solution often **“fix” the student's mistakes** instead of copying them. The transcription looks good, and string metrics like BLEU barely notice it. But the error a teacher or tutoring system needed to see is gone. We call this **over-correction**. Across 15 VLMs, 42–66% of transcriptions contain at least one over-correction.
|
|
36
|
+
|
|
37
|
+
**PINK** is an OCR metric built to catch it:
|
|
38
|
+
|
|
39
|
+
1. An LLM judge grades a transcription with a 5-component, 100-point rubric. This measures *semantic* fidelity, so `\frac{1}{2}` and `0.5` count as the same.
|
|
40
|
+
2. The judge also grades the *oracle* transcription, i.e. what the student actually wrote, errors included.
|
|
41
|
+
3. Any rubric component on which the transcription scores **higher than the oracle** is an over-correction. It is reverted to the oracle score if the gain is ≤ 10 points and set to 0 if the gain is larger.
|
|
42
|
+
|
|
43
|
+

|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install "pink-eval[fermat]"
|
|
49
|
+
# latest development version: pip install "pink-eval[fermat] @ git+https://github.com/9115jin/PINK.git"
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Python ≥ 3.10. `[fermat]` adds the Hugging Face `datasets` package, which you need only to download FERMAT.
|
|
53
|
+
|
|
54
|
+
## Quickstart (offline, no API key)
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
git clone https://github.com/9115jin/PINK.git && cd PINK && pip install -e .
|
|
58
|
+
python examples/quickstart.py
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
This scores Qwen2.5-VL-7B on 30 FERMAT samples whose transcriptions were already graded:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
PINK = 0.7400
|
|
65
|
+
raw (no penalty) = 0.9657
|
|
66
|
+
over-correction rate = 63.3%
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The raw ratio is the judge score of the transcription divided by the judge score of the oracle, with no penalty. Here it is close to 1, so the transcription looks almost as good as the student's original. PINK shows that much of that score comes from fixed student errors.
|
|
70
|
+
|
|
71
|
+
## Evaluate your VLM on FERMAT
|
|
72
|
+
|
|
73
|
+
FERMAT ([Nath et al., 2025](https://aclanthology.org/2025.acl-long.720/)) contains 2,244 handwritten multi-line solutions with known student errors. It is hosted on Hugging Face under CC-BY-4.0. The dataset is gated, so first accept its terms at [huggingface.co/datasets/ai4bharat/FERMAT](https://huggingface.co/datasets/ai4bharat/FERMAT), then run `huggingface-cli login`.
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# 1) Download FERMAT. Writes images/ and manifest.jsonl (problem + oracle transcription per sample).
|
|
77
|
+
pink fermat --out-dir data/fermat # add --limit 20 for a quick try
|
|
78
|
+
|
|
79
|
+
# 2) Grade the oracle transcriptions. Do this once per judge and reuse the result for every VLM.
|
|
80
|
+
export OPENAI_API_KEY=...
|
|
81
|
+
pink grade --judge gpt-5 --input data/fermat/manifest.jsonl --out runs/oracle_grades.csv
|
|
82
|
+
|
|
83
|
+
# 3) Transcribe the images with your VLM. Any OpenAI-compatible endpoint works, e.g. `vllm serve`.
|
|
84
|
+
pink ocr --manifest data/fermat/manifest.jsonl --model Qwen/Qwen2.5-VL-7B-Instruct \
|
|
85
|
+
--base-url http://localhost:8000/v1 --out runs/qwen7b/ocr.jsonl
|
|
86
|
+
|
|
87
|
+
# 4) Grade the transcriptions with the same judge, then score.
|
|
88
|
+
pink grade --judge gpt-5 --input runs/qwen7b/ocr.jsonl --out runs/qwen7b/grades.csv
|
|
89
|
+
pink score --model runs/qwen7b/grades.csv --oracle runs/oracle_grades.csv
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
All commands run requests in parallel (`--workers`), retry failed calls, and resume where they stopped if you re-run them. Already-finished samples are skipped. You can also transcribe with your own pipeline: `pink grade` only needs a JSONL file with `sample_id`, `problem` and `solution` (your transcription).
|
|
93
|
+
|
|
94
|
+
> **Always grade the model and the oracle with the same judge and prompt.** PINK compares the two scores component by component.
|
|
95
|
+
|
|
96
|
+
### Judges
|
|
97
|
+
|
|
98
|
+
| `--judge` | Model | Notes |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `gpt-5` | `gpt-5-2025-08-07`, reasoning effort `medium` | Main judge in the paper. Uses the OpenAI API. |
|
|
101
|
+
| `qwen3-next-80b` | `Qwen/Qwen3-Next-80B-A3B-Thinking` (temperature 0.6, top-p 0.95) | Open-source judge. VLM-level PINK agrees with GPT-5 at r = 0.99 and Kendall τ = 0.90 (paper App. E). Serve it with [`scripts/serve_qwen3_judge.sh`](https://github.com/9115jin/PINK/blob/main/scripts/serve_qwen3_judge.sh). |
|
|
102
|
+
|
|
103
|
+
Any other chat model also works, e.g. `--model <name> --base-url <url>`. PINK values are comparable only when they come from the same judge.
|
|
104
|
+
|
|
105
|
+
To run the open-source judge yourself:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
bash scripts/serve_qwen3_judge.sh # 8x 80GB GPUs (BF16)
|
|
109
|
+
MODEL=Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 TP=4 bash scripts/serve_qwen3_judge.sh # 4x 48GB GPUs (FP8)
|
|
110
|
+
pink grade --judge qwen3-next-80b --input data/fermat/manifest.jsonl --out runs/oracle_grades_qwen.csv
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The judge reasons for several thousand tokens per sample. If grades fail with `finish_reason=length`, give the server a longer context (`MAX_LEN`, default 16384).
|
|
114
|
+
|
|
115
|
+
## Use PINK on your own data
|
|
116
|
+
|
|
117
|
+
PINK is not tied to FERMAT. For each sample you need the **problem**, the **oracle transcription** (what is actually written) and the **model transcription**:
|
|
118
|
+
|
|
119
|
+
```jsonl
|
|
120
|
+
{"sample_id": "q1", "problem": "Factorise 4y^2 - 12y + 9.", "solution": "<oracle transcription>"}
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Write one such file for the oracle and one for the model (same `sample_id`s), run `pink grade` on each, then run `pink score`. From Python:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from pink import compute_pink, penalize
|
|
127
|
+
|
|
128
|
+
result = compute_pink("runs/model_grades.csv", "runs/oracle_grades.csv")
|
|
129
|
+
print(result.pink, result.over_correction_rate)
|
|
130
|
+
|
|
131
|
+
penalize(model_scores=[20, 20, 12], oracle_scores=[20, 5, 10]) # -> [20, 0, 10]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## FERMAT results (GPT-5 judge, paper Table 1)
|
|
135
|
+
|
|
136
|
+
| # | Model | PINK ↑ | Over-correction rate |
|
|
137
|
+
|---:|---|:---:|:---:|
|
|
138
|
+
| 1 | Gemini 2.5 Flash | 0.935 | 59.3% |
|
|
139
|
+
| 2 | Qwen2.5-VL 72B | 0.920 | 65.7% |
|
|
140
|
+
| 3 | Qwen2.5-VL 32B | 0.919 | 66.2% |
|
|
141
|
+
| 4 | Ovis2 34B | 0.916 | 62.5% |
|
|
142
|
+
| 5 | InternVL3 78B | 0.911 | 58.5% |
|
|
143
|
+
| 6 | GPT-4o | 0.907 | 62.3% |
|
|
144
|
+
| 7 | InternVL3 38B | 0.905 | 58.8% |
|
|
145
|
+
| 8 | Ovis2 8B | 0.887 | 53.6% |
|
|
146
|
+
| 9 | InternVL3 8B | 0.873 | 50.8% |
|
|
147
|
+
| 10 | Pixtral Large (124B) | 0.850 | 56.1% |
|
|
148
|
+
| 11 | Qwen2.5-VL 7B | 0.832 | 54.6% |
|
|
149
|
+
| 12 | Llama 3.2 90B Vision | 0.814 | 50.1% |
|
|
150
|
+
| 13 | GPT-4o mini | 0.787 | 49.6% |
|
|
151
|
+
| 14 | Pixtral 12B | 0.722 | 43.9% |
|
|
152
|
+
| 15 | Llama 3.2 11B Vision | 0.676 | 42.1% |
|
|
153
|
+
|
|
154
|
+
Over-correction rate = share of transcriptions with at least one over-corrected rubric component (paper Fig. 14).
|
|
155
|
+
The printed Table 1 in the paper has the two Pixtral PINK values swapped. The values above are the correct ones, consistent with Figures 3, 6 and 15 of the paper.
|
|
156
|
+
|
|
157
|
+
## Notes
|
|
158
|
+
|
|
159
|
+
- **Prompts.** The judge prompt (`pink.rubric.GRADING_INSTRUCTIONS`) is the one shown in paper Fig. 16. The OCR prompt (`pink.ocr.PROMPTS["default"]`) is Fig. 18. `--prompt faithful` selects the faithfulness-instructed variant from Fig. 19 (App. F).
|
|
160
|
+
- **Oracle transcription.** We use FERMAT's `pert_a` (the student's erroneous solution) and remove its `\textcolor{red}{…}` error markers. Otherwise the judge would see where the error is.
|
|
161
|
+
- **Formula.** PINK = Σ<sub>n</sub> S<sub>penalized,n</sub> / Σ<sub>n</sub> S<sub>oracle,n</sub> (paper Eq. 4). Here S<sub>oracle,n</sub> is the sum of the oracle's component scores. A transcription that scores the same as the oracle gets exactly 1.0. Samples that the judge could not grade are left out (`--missing zero` counts them as 0 instead).
|
|
162
|
+
|
|
163
|
+
## Citation
|
|
164
|
+
|
|
165
|
+
```bibtex
|
|
166
|
+
@inproceedings{seong-etal-2026-vlms,
|
|
167
|
+
title = "When {VLM}s `Fix' Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math {OCR}",
|
|
168
|
+
author = "Seong, Jin and Liermann, Wencke and Kim, Minho and Shin, Jong-hun and Lim, Soojong",
|
|
169
|
+
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
|
|
170
|
+
month = oct,
|
|
171
|
+
year = "2026",
|
|
172
|
+
address = "Budapest, Hungary",
|
|
173
|
+
publisher = "Association for Computational Linguistics",
|
|
174
|
+
url = "https://aclanthology.org/2026.emnlp-main.476/",
|
|
175
|
+
doi = "10.18653/2026.emnlp-main.476",
|
|
176
|
+
pages = "10932--10953"
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
If you evaluate on FERMAT, please also cite the dataset:
|
|
181
|
+
|
|
182
|
+
```bibtex
|
|
183
|
+
@inproceedings{nath-etal-2025-vision,
|
|
184
|
+
title = "Can Vision-Language Models Evaluate Handwritten Math?",
|
|
185
|
+
author = "Nath, Oikantik and Bathina, Hanani and Khan, Mohammed Safi Ur Rahman and Khapra, Mitesh M",
|
|
186
|
+
booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
|
|
187
|
+
year = "2025",
|
|
188
|
+
address = "Vienna, Austria",
|
|
189
|
+
publisher = "Association for Computational Linguistics",
|
|
190
|
+
url = "https://aclanthology.org/2025.acl-long.720/",
|
|
191
|
+
doi = "10.18653/v1/2025.acl-long.720",
|
|
192
|
+
pages = "14784--14814"
|
|
193
|
+
}
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
## License
|
|
197
|
+
|
|
198
|
+
Code: [BSD-3-Clause](https://github.com/9115jin/PINK/blob/main/LICENSE), © 2026 Electronics and Telecommunications Research Institute (ETRI).
|
|
199
|
+
The example grade files in `examples/sample/` contain judge justifications that quote FERMAT samples; FERMAT is © its authors and licensed under CC-BY-4.0.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
pink/__init__.py,sha256=i8jsZ33GPJ-SbTwgN1ksNjXL2sJgeCPvBWRFCufJnZY,798
|
|
2
|
+
pink/_runner.py,sha256=39XvOj5tVyls73bbwpM04pCIaUhFJDd-RpOlqq-vxcs,3879
|
|
3
|
+
pink/cli.py,sha256=WE9mKjqMB763LWMqNrzMj3alUQRXJRDsZYK5GtoIyZc,8123
|
|
4
|
+
pink/data.py,sha256=I8bPju3WWNLXoHYBtLISQ2uMdCRAKl5yE4KFxN_0wN0,3927
|
|
5
|
+
pink/judge.py,sha256=oImIaP8Ku1HHa4JTUZv_MRgAhohc5khJZy-2lPgBdt0,8030
|
|
6
|
+
pink/ocr.py,sha256=hcVKOGemCvYjaeiHVASNaw7wmILC7iChac77f2jChV4,6616
|
|
7
|
+
pink/rubric.py,sha256=HaLUfFiZGFGeWtCtcGqqkA1oFICvaH4GoyZAJnXfRC0,4149
|
|
8
|
+
pink/score.py,sha256=yMYGz1FGH1yf2Swmg03BixzwZ8nPiUfT2nVZrAWcS1Y,5859
|
|
9
|
+
pink_eval-1.0.0.dist-info/licenses/LICENSE,sha256=pOg8L_qDHmpKO-gUc79fjIgQPZKemvCyxQBpEjPfk58,1547
|
|
10
|
+
pink_eval-1.0.0.dist-info/METADATA,sha256=QIU-8Hc8bTiX4ojDpFGKCId6VVkUAkCSgi9laS87ZUo,10599
|
|
11
|
+
pink_eval-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
12
|
+
pink_eval-1.0.0.dist-info/entry_points.txt,sha256=G1ZfYBepYQgaD_0JAv27MqipuXugIn0eskEQDKEjF0s,39
|
|
13
|
+
pink_eval-1.0.0.dist-info/top_level.txt,sha256=J0e5Cs7fv25M2HbtIHzDnamL9HOdHAll6Uue20lRzy8,5
|
|
14
|
+
pink_eval-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Electronics and Telecommunications Research Institute (ETRI)
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pink
|