pink-eval 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pink_eval-1.0.0/LICENSE +28 -0
- pink_eval-1.0.0/PKG-INFO +199 -0
- pink_eval-1.0.0/README.md +175 -0
- pink_eval-1.0.0/pyproject.toml +42 -0
- pink_eval-1.0.0/setup.cfg +4 -0
- pink_eval-1.0.0/src/pink/__init__.py +28 -0
- pink_eval-1.0.0/src/pink/_runner.py +102 -0
- pink_eval-1.0.0/src/pink/cli.py +162 -0
- pink_eval-1.0.0/src/pink/data.py +101 -0
- pink_eval-1.0.0/src/pink/judge.py +202 -0
- pink_eval-1.0.0/src/pink/ocr.py +133 -0
- pink_eval-1.0.0/src/pink/rubric.py +113 -0
- pink_eval-1.0.0/src/pink/score.py +147 -0
- pink_eval-1.0.0/src/pink_eval.egg-info/PKG-INFO +199 -0
- pink_eval-1.0.0/src/pink_eval.egg-info/SOURCES.txt +21 -0
- pink_eval-1.0.0/src/pink_eval.egg-info/dependency_links.txt +1 -0
- pink_eval-1.0.0/src/pink_eval.egg-info/entry_points.txt +2 -0
- pink_eval-1.0.0/src/pink_eval.egg-info/requires.txt +10 -0
- pink_eval-1.0.0/src/pink_eval.egg-info/top_level.txt +1 -0
- pink_eval-1.0.0/tests/test_cli.py +15 -0
- pink_eval-1.0.0/tests/test_data_ocr.py +19 -0
- pink_eval-1.0.0/tests/test_judge.py +106 -0
- pink_eval-1.0.0/tests/test_score.py +66 -0
pink_eval-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Electronics and Telecommunications Research Institute (ETRI)
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
pink_eval-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pink-eval
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: PINK: over-correction-aware evaluation of handwritten math OCR (EMNLP 2026)
|
|
5
|
+
Author: Jin Seong, Wencke Liermann, Minho Kim, Jong-hun Shin, Soojong Lim
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Paper, https://aclanthology.org/2026.emnlp-main.476/
|
|
8
|
+
Project-URL: Code, https://github.com/9115jin/PINK
|
|
9
|
+
Keywords: OCR,handwritten math,vision-language models,evaluation,LLM-as-a-judge,over-correction
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: numpy>=1.21
|
|
16
|
+
Requires-Dist: pandas>=1.3
|
|
17
|
+
Requires-Dist: openai>=1.55
|
|
18
|
+
Requires-Dist: tqdm>=4.0
|
|
19
|
+
Provides-Extra: fermat
|
|
20
|
+
Requires-Dist: datasets>=2.18; extra == "fermat"
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
# PINK: Penalized INK-based Score
|
|
26
|
+
|
|
27
|
+
**When VLMs ‘Fix’ Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math OCR**
|
|
28
|
+
<br>Jin Seong\*, Wencke Liermann\*, Minho Kim, Jong-hun Shin, Soojong Lim · *EMNLP 2026 (Main, Oral)*
|
|
29
|
+
|
|
30
|
+
[](https://aclanthology.org/2026.emnlp-main.476/)
|
|
31
|
+
[](https://pypi.org/project/pink-eval/)
|
|
32
|
+
[](https://github.com/9115jin/PINK/actions/workflows/tests.yml)
|
|
33
|
+
[](https://github.com/9115jin/PINK/blob/main/LICENSE)
|
|
34
|
+
|
|
35
|
+
Vision-language models transcribing a student's handwritten solution often **“fix” the student's mistakes** instead of copying them. The transcription looks good, and string metrics like BLEU barely notice it. But the error a teacher or tutoring system needed to see is gone. We call this **over-correction**. Across 15 VLMs, 42–66% of transcriptions contain at least one over-correction.
|
|
36
|
+
|
|
37
|
+
**PINK** is an OCR metric built to catch it:
|
|
38
|
+
|
|
39
|
+
1. An LLM judge grades a transcription with a 5-component, 100-point rubric. This measures *semantic* fidelity, so `\frac{1}{2}` and `0.5` count as the same.
|
|
40
|
+
2. The judge also grades the *oracle* transcription, i.e. what the student actually wrote, errors included.
|
|
41
|
+
3. Any rubric component on which the transcription scores **higher than the oracle** is an over-correction. It is reverted to the oracle score if the gain is ≤ 10 points and set to 0 if the gain is larger.
|
|
42
|
+
|
|
43
|
+

|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install "pink-eval[fermat]"
|
|
49
|
+
# latest development version: pip install "pink-eval[fermat] @ git+https://github.com/9115jin/PINK.git"
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Python ≥ 3.10. `[fermat]` adds the Hugging Face `datasets` package, which you need only to download FERMAT.
|
|
53
|
+
|
|
54
|
+
## Quickstart (offline, no API key)
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
git clone https://github.com/9115jin/PINK.git && cd PINK && pip install -e .
|
|
58
|
+
python examples/quickstart.py
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
This scores Qwen2.5-VL-7B on 30 FERMAT samples whose transcriptions were already graded:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
PINK = 0.7400
|
|
65
|
+
raw (no penalty) = 0.9657
|
|
66
|
+
over-correction rate = 63.3%
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The raw ratio is the judge score of the transcription divided by the judge score of the oracle, with no penalty. Here it is close to 1, so the transcription looks almost as good as the student's original. PINK shows that much of that score comes from fixed student errors.
|
|
70
|
+
|
|
71
|
+
## Evaluate your VLM on FERMAT
|
|
72
|
+
|
|
73
|
+
FERMAT ([Nath et al., 2025](https://aclanthology.org/2025.acl-long.720/)) contains 2,244 handwritten multi-line solutions with known student errors. It is hosted on Hugging Face under CC-BY-4.0. The dataset is gated, so first accept its terms at [huggingface.co/datasets/ai4bharat/FERMAT](https://huggingface.co/datasets/ai4bharat/FERMAT), then run `huggingface-cli login`.
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# 1) Download FERMAT. Writes images/ and manifest.jsonl (problem + oracle transcription per sample).
|
|
77
|
+
pink fermat --out-dir data/fermat # add --limit 20 for a quick try
|
|
78
|
+
|
|
79
|
+
# 2) Grade the oracle transcriptions. Do this once per judge and reuse the result for every VLM.
|
|
80
|
+
export OPENAI_API_KEY=...
|
|
81
|
+
pink grade --judge gpt-5 --input data/fermat/manifest.jsonl --out runs/oracle_grades.csv
|
|
82
|
+
|
|
83
|
+
# 3) Transcribe the images with your VLM. Any OpenAI-compatible endpoint works, e.g. `vllm serve`.
|
|
84
|
+
pink ocr --manifest data/fermat/manifest.jsonl --model Qwen/Qwen2.5-VL-7B-Instruct \
|
|
85
|
+
--base-url http://localhost:8000/v1 --out runs/qwen7b/ocr.jsonl
|
|
86
|
+
|
|
87
|
+
# 4) Grade the transcriptions with the same judge, then score.
|
|
88
|
+
pink grade --judge gpt-5 --input runs/qwen7b/ocr.jsonl --out runs/qwen7b/grades.csv
|
|
89
|
+
pink score --model runs/qwen7b/grades.csv --oracle runs/oracle_grades.csv
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
All commands run requests in parallel (`--workers`), retry failed calls, and resume where they stopped if you re-run them. Already-finished samples are skipped. You can also transcribe with your own pipeline: `pink grade` only needs a JSONL file with `sample_id`, `problem` and `solution` (your transcription).
|
|
93
|
+
|
|
94
|
+
> **Always grade the model and the oracle with the same judge and prompt.** PINK compares the two scores component by component.
|
|
95
|
+
|
|
96
|
+
### Judges
|
|
97
|
+
|
|
98
|
+
| `--judge` | Model | Notes |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `gpt-5` | `gpt-5-2025-08-07`, reasoning effort `medium` | Main judge in the paper. Uses the OpenAI API. |
|
|
101
|
+
| `qwen3-next-80b` | `Qwen/Qwen3-Next-80B-A3B-Thinking` (temperature 0.6, top-p 0.95) | Open-source judge. VLM-level PINK agrees with GPT-5 at r = 0.99 and Kendall τ = 0.90 (paper App. E). Serve it with [`scripts/serve_qwen3_judge.sh`](https://github.com/9115jin/PINK/blob/main/scripts/serve_qwen3_judge.sh). |
|
|
102
|
+
|
|
103
|
+
Any other chat model also works, e.g. `--model <name> --base-url <url>`. PINK values are comparable only when they come from the same judge.
|
|
104
|
+
|
|
105
|
+
To run the open-source judge yourself:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
bash scripts/serve_qwen3_judge.sh # 8x 80GB GPUs (BF16)
|
|
109
|
+
MODEL=Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 TP=4 bash scripts/serve_qwen3_judge.sh # 4x 48GB GPUs (FP8)
|
|
110
|
+
pink grade --judge qwen3-next-80b --input data/fermat/manifest.jsonl --out runs/oracle_grades_qwen.csv
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The judge reasons for several thousand tokens per sample. If grades fail with `finish_reason=length`, give the server a longer context (`MAX_LEN`, default 16384).
|
|
114
|
+
|
|
115
|
+
## Use PINK on your own data
|
|
116
|
+
|
|
117
|
+
PINK is not tied to FERMAT. For each sample you need the **problem**, the **oracle transcription** (what is actually written) and the **model transcription**:
|
|
118
|
+
|
|
119
|
+
```jsonl
|
|
120
|
+
{"sample_id": "q1", "problem": "Factorise 4y^2 - 12y + 9.", "solution": "<oracle transcription>"}
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Write one such file for the oracle and one for the model (same `sample_id`s), run `pink grade` on each, then run `pink score`. From Python:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from pink import compute_pink, penalize
|
|
127
|
+
|
|
128
|
+
result = compute_pink("runs/model_grades.csv", "runs/oracle_grades.csv")
|
|
129
|
+
print(result.pink, result.over_correction_rate)
|
|
130
|
+
|
|
131
|
+
penalize(model_scores=[20, 20, 12], oracle_scores=[20, 5, 10]) # -> [20, 0, 10]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## FERMAT results (GPT-5 judge, paper Table 1)
|
|
135
|
+
|
|
136
|
+
| # | Model | PINK ↑ | Over-correction rate |
|
|
137
|
+
|---:|---|:---:|:---:|
|
|
138
|
+
| 1 | Gemini 2.5 Flash | 0.935 | 59.3% |
|
|
139
|
+
| 2 | Qwen2.5-VL 72B | 0.920 | 65.7% |
|
|
140
|
+
| 3 | Qwen2.5-VL 32B | 0.919 | 66.2% |
|
|
141
|
+
| 4 | Ovis2 34B | 0.916 | 62.5% |
|
|
142
|
+
| 5 | InternVL3 78B | 0.911 | 58.5% |
|
|
143
|
+
| 6 | GPT-4o | 0.907 | 62.3% |
|
|
144
|
+
| 7 | InternVL3 38B | 0.905 | 58.8% |
|
|
145
|
+
| 8 | Ovis2 8B | 0.887 | 53.6% |
|
|
146
|
+
| 9 | InternVL3 8B | 0.873 | 50.8% |
|
|
147
|
+
| 10 | Pixtral Large (124B) | 0.850 | 56.1% |
|
|
148
|
+
| 11 | Qwen2.5-VL 7B | 0.832 | 54.6% |
|
|
149
|
+
| 12 | Llama 3.2 90B Vision | 0.814 | 50.1% |
|
|
150
|
+
| 13 | GPT-4o mini | 0.787 | 49.6% |
|
|
151
|
+
| 14 | Pixtral 12B | 0.722 | 43.9% |
|
|
152
|
+
| 15 | Llama 3.2 11B Vision | 0.676 | 42.1% |
|
|
153
|
+
|
|
154
|
+
Over-correction rate = share of transcriptions with at least one over-corrected rubric component (paper Fig. 14).
|
|
155
|
+
The printed Table 1 in the paper has the two Pixtral PINK values swapped. The values above are the correct ones, consistent with Figures 3, 6 and 15 of the paper.
|
|
156
|
+
|
|
157
|
+
## Notes
|
|
158
|
+
|
|
159
|
+
- **Prompts.** The judge prompt (`pink.rubric.GRADING_INSTRUCTIONS`) is the one shown in paper Fig. 16. The OCR prompt (`pink.ocr.PROMPTS["default"]`) is Fig. 18. `--prompt faithful` selects the faithfulness-instructed variant from Fig. 19 (App. F).
|
|
160
|
+
- **Oracle transcription.** We use FERMAT's `pert_a` (the student's erroneous solution) and remove its `\textcolor{red}{…}` error markers. Otherwise the judge would see where the error is.
|
|
161
|
+
- **Formula.** PINK = Σ<sub>n</sub> S<sub>penalized,n</sub> / Σ<sub>n</sub> S<sub>oracle,n</sub> (paper Eq. 4). Here S<sub>oracle,n</sub> is the sum of the oracle's component scores. A transcription that scores the same as the oracle gets exactly 1.0. Samples that the judge could not grade are left out (`--missing zero` counts them as 0 instead).
|
|
162
|
+
|
|
163
|
+
## Citation
|
|
164
|
+
|
|
165
|
+
```bibtex
|
|
166
|
+
@inproceedings{seong-etal-2026-vlms,
|
|
167
|
+
title = "When {VLM}s `Fix' Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math {OCR}",
|
|
168
|
+
author = "Seong, Jin and Liermann, Wencke and Kim, Minho and Shin, Jong-hun and Lim, Soojong",
|
|
169
|
+
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
|
|
170
|
+
month = oct,
|
|
171
|
+
year = "2026",
|
|
172
|
+
address = "Budapest, Hungary",
|
|
173
|
+
publisher = "Association for Computational Linguistics",
|
|
174
|
+
url = "https://aclanthology.org/2026.emnlp-main.476/",
|
|
175
|
+
doi = "10.18653/2026.emnlp-main.476",
|
|
176
|
+
pages = "10932--10953"
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
If you evaluate on FERMAT, please also cite the dataset:
|
|
181
|
+
|
|
182
|
+
```bibtex
|
|
183
|
+
@inproceedings{nath-etal-2025-vision,
|
|
184
|
+
title = "Can Vision-Language Models Evaluate Handwritten Math?",
|
|
185
|
+
author = "Nath, Oikantik and Bathina, Hanani and Khan, Mohammed Safi Ur Rahman and Khapra, Mitesh M",
|
|
186
|
+
booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
|
|
187
|
+
year = "2025",
|
|
188
|
+
address = "Vienna, Austria",
|
|
189
|
+
publisher = "Association for Computational Linguistics",
|
|
190
|
+
url = "https://aclanthology.org/2025.acl-long.720/",
|
|
191
|
+
doi = "10.18653/v1/2025.acl-long.720",
|
|
192
|
+
pages = "14784--14814"
|
|
193
|
+
}
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
## License
|
|
197
|
+
|
|
198
|
+
Code: [BSD-3-Clause](https://github.com/9115jin/PINK/blob/main/LICENSE), © 2026 Electronics and Telecommunications Research Institute (ETRI).
|
|
199
|
+
The example grade files in `examples/sample/` contain judge justifications that quote FERMAT samples; FERMAT is © its authors and licensed under CC-BY-4.0.
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# PINK: Penalized INK-based Score
|
|
2
|
+
|
|
3
|
+
**When VLMs ‘Fix’ Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math OCR**
|
|
4
|
+
<br>Jin Seong\*, Wencke Liermann\*, Minho Kim, Jong-hun Shin, Soojong Lim · *EMNLP 2026 (Main, Oral)*
|
|
5
|
+
|
|
6
|
+
[](https://aclanthology.org/2026.emnlp-main.476/)
|
|
7
|
+
[](https://pypi.org/project/pink-eval/)
|
|
8
|
+
[](https://github.com/9115jin/PINK/actions/workflows/tests.yml)
|
|
9
|
+
[](https://github.com/9115jin/PINK/blob/main/LICENSE)
|
|
10
|
+
|
|
11
|
+
Vision-language models transcribing a student's handwritten solution often **“fix” the student's mistakes** instead of copying them. The transcription looks good, and string metrics like BLEU barely notice it. But the error a teacher or tutoring system needed to see is gone. We call this **over-correction**. Across 15 VLMs, 42–66% of transcriptions contain at least one over-correction.
|
|
12
|
+
|
|
13
|
+
**PINK** is an OCR metric built to catch it:
|
|
14
|
+
|
|
15
|
+
1. An LLM judge grades a transcription with a 5-component, 100-point rubric. This measures *semantic* fidelity, so `\frac{1}{2}` and `0.5` count as the same.
|
|
16
|
+
2. The judge also grades the *oracle* transcription, i.e. what the student actually wrote, errors included.
|
|
17
|
+
3. Any rubric component on which the transcription scores **higher than the oracle** is an over-correction. It is reverted to the oracle score if the gain is ≤ 10 points and set to 0 if the gain is larger.
|
|
18
|
+
|
|
19
|
+

|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install "pink-eval[fermat]"
|
|
25
|
+
# latest development version: pip install "pink-eval[fermat] @ git+https://github.com/9115jin/PINK.git"
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Python ≥ 3.10. `[fermat]` adds the Hugging Face `datasets` package, which you need only to download FERMAT.
|
|
29
|
+
|
|
30
|
+
## Quickstart (offline, no API key)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
git clone https://github.com/9115jin/PINK.git && cd PINK && pip install -e .
|
|
34
|
+
python examples/quickstart.py
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
This scores Qwen2.5-VL-7B on 30 FERMAT samples whose transcriptions were already graded:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
PINK = 0.7400
|
|
41
|
+
raw (no penalty) = 0.9657
|
|
42
|
+
over-correction rate = 63.3%
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The raw ratio is the judge score of the transcription divided by the judge score of the oracle, with no penalty. Here it is close to 1, so the transcription looks almost as good as the student's original. PINK shows that much of that score comes from fixed student errors.
|
|
46
|
+
|
|
47
|
+
## Evaluate your VLM on FERMAT
|
|
48
|
+
|
|
49
|
+
FERMAT ([Nath et al., 2025](https://aclanthology.org/2025.acl-long.720/)) contains 2,244 handwritten multi-line solutions with known student errors. It is hosted on Hugging Face under CC-BY-4.0. The dataset is gated, so first accept its terms at [huggingface.co/datasets/ai4bharat/FERMAT](https://huggingface.co/datasets/ai4bharat/FERMAT), then run `huggingface-cli login`.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# 1) Download FERMAT. Writes images/ and manifest.jsonl (problem + oracle transcription per sample).
|
|
53
|
+
pink fermat --out-dir data/fermat # add --limit 20 for a quick try
|
|
54
|
+
|
|
55
|
+
# 2) Grade the oracle transcriptions. Do this once per judge and reuse the result for every VLM.
|
|
56
|
+
export OPENAI_API_KEY=...
|
|
57
|
+
pink grade --judge gpt-5 --input data/fermat/manifest.jsonl --out runs/oracle_grades.csv
|
|
58
|
+
|
|
59
|
+
# 3) Transcribe the images with your VLM. Any OpenAI-compatible endpoint works, e.g. `vllm serve`.
|
|
60
|
+
pink ocr --manifest data/fermat/manifest.jsonl --model Qwen/Qwen2.5-VL-7B-Instruct \
|
|
61
|
+
--base-url http://localhost:8000/v1 --out runs/qwen7b/ocr.jsonl
|
|
62
|
+
|
|
63
|
+
# 4) Grade the transcriptions with the same judge, then score.
|
|
64
|
+
pink grade --judge gpt-5 --input runs/qwen7b/ocr.jsonl --out runs/qwen7b/grades.csv
|
|
65
|
+
pink score --model runs/qwen7b/grades.csv --oracle runs/oracle_grades.csv
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
All commands run requests in parallel (`--workers`), retry failed calls, and resume where they stopped if you re-run them. Already-finished samples are skipped. You can also transcribe with your own pipeline: `pink grade` only needs a JSONL file with `sample_id`, `problem` and `solution` (your transcription).
|
|
69
|
+
|
|
70
|
+
> **Always grade the model and the oracle with the same judge and prompt.** PINK compares the two scores component by component.
|
|
71
|
+
|
|
72
|
+
### Judges
|
|
73
|
+
|
|
74
|
+
| `--judge` | Model | Notes |
|
|
75
|
+
|---|---|---|
|
|
76
|
+
| `gpt-5` | `gpt-5-2025-08-07`, reasoning effort `medium` | Main judge in the paper. Uses the OpenAI API. |
|
|
77
|
+
| `qwen3-next-80b` | `Qwen/Qwen3-Next-80B-A3B-Thinking` (temperature 0.6, top-p 0.95) | Open-source judge. VLM-level PINK agrees with GPT-5 at r = 0.99 and Kendall τ = 0.90 (paper App. E). Serve it with [`scripts/serve_qwen3_judge.sh`](https://github.com/9115jin/PINK/blob/main/scripts/serve_qwen3_judge.sh). |
|
|
78
|
+
|
|
79
|
+
Any other chat model also works, e.g. `--model <name> --base-url <url>`. PINK values are comparable only when they come from the same judge.
|
|
80
|
+
|
|
81
|
+
To run the open-source judge yourself:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
bash scripts/serve_qwen3_judge.sh # 8x 80GB GPUs (BF16)
|
|
85
|
+
MODEL=Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 TP=4 bash scripts/serve_qwen3_judge.sh # 4x 48GB GPUs (FP8)
|
|
86
|
+
pink grade --judge qwen3-next-80b --input data/fermat/manifest.jsonl --out runs/oracle_grades_qwen.csv
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The judge reasons for several thousand tokens per sample. If grades fail with `finish_reason=length`, give the server a longer context (`MAX_LEN`, default 16384).
|
|
90
|
+
|
|
91
|
+
## Use PINK on your own data
|
|
92
|
+
|
|
93
|
+
PINK is not tied to FERMAT. For each sample you need the **problem**, the **oracle transcription** (what is actually written) and the **model transcription**:
|
|
94
|
+
|
|
95
|
+
```jsonl
|
|
96
|
+
{"sample_id": "q1", "problem": "Factorise 4y^2 - 12y + 9.", "solution": "<oracle transcription>"}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Write one such file for the oracle and one for the model (same `sample_id`s), run `pink grade` on each, then run `pink score`. From Python:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from pink import compute_pink, penalize
|
|
103
|
+
|
|
104
|
+
result = compute_pink("runs/model_grades.csv", "runs/oracle_grades.csv")
|
|
105
|
+
print(result.pink, result.over_correction_rate)
|
|
106
|
+
|
|
107
|
+
penalize(model_scores=[20, 20, 12], oracle_scores=[20, 5, 10]) # -> [20, 0, 10]
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## FERMAT results (GPT-5 judge, paper Table 1)
|
|
111
|
+
|
|
112
|
+
| # | Model | PINK ↑ | Over-correction rate |
|
|
113
|
+
|---:|---|:---:|:---:|
|
|
114
|
+
| 1 | Gemini 2.5 Flash | 0.935 | 59.3% |
|
|
115
|
+
| 2 | Qwen2.5-VL 72B | 0.920 | 65.7% |
|
|
116
|
+
| 3 | Qwen2.5-VL 32B | 0.919 | 66.2% |
|
|
117
|
+
| 4 | Ovis2 34B | 0.916 | 62.5% |
|
|
118
|
+
| 5 | InternVL3 78B | 0.911 | 58.5% |
|
|
119
|
+
| 6 | GPT-4o | 0.907 | 62.3% |
|
|
120
|
+
| 7 | InternVL3 38B | 0.905 | 58.8% |
|
|
121
|
+
| 8 | Ovis2 8B | 0.887 | 53.6% |
|
|
122
|
+
| 9 | InternVL3 8B | 0.873 | 50.8% |
|
|
123
|
+
| 10 | Pixtral Large (124B) | 0.850 | 56.1% |
|
|
124
|
+
| 11 | Qwen2.5-VL 7B | 0.832 | 54.6% |
|
|
125
|
+
| 12 | Llama 3.2 90B Vision | 0.814 | 50.1% |
|
|
126
|
+
| 13 | GPT-4o mini | 0.787 | 49.6% |
|
|
127
|
+
| 14 | Pixtral 12B | 0.722 | 43.9% |
|
|
128
|
+
| 15 | Llama 3.2 11B Vision | 0.676 | 42.1% |
|
|
129
|
+
|
|
130
|
+
Over-correction rate = share of transcriptions with at least one over-corrected rubric component (paper Fig. 14).
|
|
131
|
+
The printed Table 1 in the paper has the two Pixtral PINK values swapped. The values above are the correct ones, consistent with Figures 3, 6 and 15 of the paper.
|
|
132
|
+
|
|
133
|
+
## Notes
|
|
134
|
+
|
|
135
|
+
- **Prompts.** The judge prompt (`pink.rubric.GRADING_INSTRUCTIONS`) is the one shown in paper Fig. 16. The OCR prompt (`pink.ocr.PROMPTS["default"]`) is Fig. 18. `--prompt faithful` selects the faithfulness-instructed variant from Fig. 19 (App. F).
|
|
136
|
+
- **Oracle transcription.** We use FERMAT's `pert_a` (the student's erroneous solution) and remove its `\textcolor{red}{…}` error markers. Otherwise the judge would see where the error is.
|
|
137
|
+
- **Formula.** PINK = Σ<sub>n</sub> S<sub>penalized,n</sub> / Σ<sub>n</sub> S<sub>oracle,n</sub> (paper Eq. 4). Here S<sub>oracle,n</sub> is the sum of the oracle's component scores. A transcription that scores the same as the oracle gets exactly 1.0. Samples that the judge could not grade are left out (`--missing zero` counts them as 0 instead).
|
|
138
|
+
|
|
139
|
+
## Citation
|
|
140
|
+
|
|
141
|
+
```bibtex
|
|
142
|
+
@inproceedings{seong-etal-2026-vlms,
|
|
143
|
+
title = "When {VLM}s `Fix' Students: Identifying and Penalizing Over-Correction in the Evaluation of Multi-line Handwritten Math {OCR}",
|
|
144
|
+
author = "Seong, Jin and Liermann, Wencke and Kim, Minho and Shin, Jong-hun and Lim, Soojong",
|
|
145
|
+
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
|
|
146
|
+
month = oct,
|
|
147
|
+
year = "2026",
|
|
148
|
+
address = "Budapest, Hungary",
|
|
149
|
+
publisher = "Association for Computational Linguistics",
|
|
150
|
+
url = "https://aclanthology.org/2026.emnlp-main.476/",
|
|
151
|
+
doi = "10.18653/2026.emnlp-main.476",
|
|
152
|
+
pages = "10932--10953"
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
If you evaluate on FERMAT, please also cite the dataset:
|
|
157
|
+
|
|
158
|
+
```bibtex
|
|
159
|
+
@inproceedings{nath-etal-2025-vision,
|
|
160
|
+
title = "Can Vision-Language Models Evaluate Handwritten Math?",
|
|
161
|
+
author = "Nath, Oikantik and Bathina, Hanani and Khan, Mohammed Safi Ur Rahman and Khapra, Mitesh M",
|
|
162
|
+
booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
|
|
163
|
+
year = "2025",
|
|
164
|
+
address = "Vienna, Austria",
|
|
165
|
+
publisher = "Association for Computational Linguistics",
|
|
166
|
+
url = "https://aclanthology.org/2025.acl-long.720/",
|
|
167
|
+
doi = "10.18653/v1/2025.acl-long.720",
|
|
168
|
+
pages = "14784--14814"
|
|
169
|
+
}
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## License
|
|
173
|
+
|
|
174
|
+
Code: [BSD-3-Clause](https://github.com/9115jin/PINK/blob/main/LICENSE), © 2026 Electronics and Telecommunications Research Institute (ETRI).
|
|
175
|
+
The example grade files in `examples/sample/` contain judge justifications that quote FERMAT samples; FERMAT is © its authors and licensed under CC-BY-4.0.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pink-eval"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "PINK: over-correction-aware evaluation of handwritten math OCR (EMNLP 2026)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "BSD-3-Clause"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Jin Seong" },
|
|
15
|
+
{ name = "Wencke Liermann" },
|
|
16
|
+
{ name = "Minho Kim" },
|
|
17
|
+
{ name = "Jong-hun Shin" },
|
|
18
|
+
{ name = "Soojong Lim" },
|
|
19
|
+
]
|
|
20
|
+
keywords = ["OCR", "handwritten math", "vision-language models", "evaluation", "LLM-as-a-judge", "over-correction"]
|
|
21
|
+
classifiers = [
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
24
|
+
]
|
|
25
|
+
dependencies = ["numpy>=1.21", "pandas>=1.3", "openai>=1.55", "tqdm>=4.0"]
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
fermat = ["datasets>=2.18"]
|
|
29
|
+
dev = ["pytest>=7"]
|
|
30
|
+
|
|
31
|
+
[project.scripts]
|
|
32
|
+
pink = "pink.cli:main"
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Paper = "https://aclanthology.org/2026.emnlp-main.476/"
|
|
36
|
+
Code = "https://github.com/9115jin/PINK"
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.dynamic]
|
|
39
|
+
version = { attr = "pink.__version__" }
|
|
40
|
+
|
|
41
|
+
[tool.setuptools.packages.find]
|
|
42
|
+
where = ["src"]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""PINK: Penalized INK-based score for faithful handwritten-math OCR evaluation.
|
|
2
|
+
|
|
3
|
+
Seong et al., "When VLMs 'Fix' Students: Identifying and Penalizing Over-Correction in
|
|
4
|
+
the Evaluation of Multi-line Handwritten Math OCR", EMNLP 2026.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__version__ = "1.0.0"
|
|
8
|
+
|
|
9
|
+
from .judge import JUDGE_PRESETS, JudgeConfig, grade, judge_from_preset
|
|
10
|
+
from .rubric import GRADING_INSTRUCTIONS, RUBRIC_ITEMS, SCORE_COLUMNS
|
|
11
|
+
from .score import DEFAULT_THRESHOLD, PINKResult, compute_pink, load_grades, penalize, pink_table
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"__version__",
|
|
15
|
+
"DEFAULT_THRESHOLD",
|
|
16
|
+
"GRADING_INSTRUCTIONS",
|
|
17
|
+
"JUDGE_PRESETS",
|
|
18
|
+
"JudgeConfig",
|
|
19
|
+
"PINKResult",
|
|
20
|
+
"RUBRIC_ITEMS",
|
|
21
|
+
"SCORE_COLUMNS",
|
|
22
|
+
"compute_pink",
|
|
23
|
+
"grade",
|
|
24
|
+
"judge_from_preset",
|
|
25
|
+
"load_grades",
|
|
26
|
+
"penalize",
|
|
27
|
+
"pink_table",
|
|
28
|
+
]
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Parallel, resumable execution of per-sample API calls with incremental output."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
import threading
|
|
8
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence
|
|
11
|
+
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def read_jsonl(path) -> List[Dict[str, Any]]:
|
|
16
|
+
with open(path, encoding="utf-8") as f:
|
|
17
|
+
return [json.loads(line) for line in f if line.strip()]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def write_jsonl(path, rows: Iterable[Dict[str, Any]]) -> None:
|
|
21
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
22
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
23
|
+
for row in rows:
|
|
24
|
+
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _read_rows(path: Path) -> List[Dict[str, Any]]:
|
|
28
|
+
if not path.exists() or path.stat().st_size == 0:
|
|
29
|
+
return []
|
|
30
|
+
if path.suffix == ".csv":
|
|
31
|
+
return pd.read_csv(path, dtype={"sample_id": str}, keep_default_na=False).to_dict("records")
|
|
32
|
+
return read_jsonl(path)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _is_success(row: Dict[str, Any]) -> bool:
|
|
36
|
+
return str(row.get("success", "")).lower() in ("true", "1")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _compact(path: Path, columns: Optional[Sequence[str]]) -> None:
|
|
40
|
+
"""Keep one row per sample (the last successful one, else the last one)."""
|
|
41
|
+
best: Dict[str, Dict[str, Any]] = {}
|
|
42
|
+
for row in _read_rows(path):
|
|
43
|
+
sid = str(row["sample_id"])
|
|
44
|
+
if sid not in best or _is_success(row) or not _is_success(best[sid]):
|
|
45
|
+
best[sid] = row
|
|
46
|
+
rows = list(best.values())
|
|
47
|
+
if path.suffix == ".csv":
|
|
48
|
+
pd.DataFrame(rows, columns=list(columns) if columns else None).to_csv(path, index=False)
|
|
49
|
+
else:
|
|
50
|
+
write_jsonl(path, rows)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def run_jobs(items: Sequence[Dict[str, Any]], fn: Callable[[Dict[str, Any]], Dict[str, Any]], out_path,
|
|
54
|
+
columns: Optional[Sequence[str]] = None, workers: int = 8, resume: bool = True,
|
|
55
|
+
desc: str = "") -> Dict[str, int]:
|
|
56
|
+
"""Call ``fn(item)`` for every item not yet done and append each result row to ``out_path``.
|
|
57
|
+
|
|
58
|
+
``out_path`` is CSV (``columns`` required) or JSONL. Re-running the same command
|
|
59
|
+
skips samples that already succeeded, so interrupted runs can simply be restarted.
|
|
60
|
+
"""
|
|
61
|
+
out_path = Path(out_path)
|
|
62
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
if not resume and out_path.exists():
|
|
64
|
+
out_path.unlink()
|
|
65
|
+
done = {str(r["sample_id"]) for r in _read_rows(out_path) if _is_success(r)}
|
|
66
|
+
todo = [it for it in items if str(it["sample_id"]) not in done]
|
|
67
|
+
|
|
68
|
+
lock = threading.Lock()
|
|
69
|
+
is_csv = out_path.suffix == ".csv"
|
|
70
|
+
new_file = not out_path.exists() or out_path.stat().st_size == 0
|
|
71
|
+
fh = open(out_path, "a", encoding="utf-8", newline="")
|
|
72
|
+
writer = csv.DictWriter(fh, fieldnames=list(columns), extrasaction="ignore") if is_csv else None
|
|
73
|
+
if is_csv and new_file:
|
|
74
|
+
writer.writeheader()
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
from tqdm import tqdm
|
|
78
|
+
bar = tqdm(total=len(todo), desc=desc or out_path.name)
|
|
79
|
+
except ImportError: # pragma: no cover
|
|
80
|
+
bar = None
|
|
81
|
+
|
|
82
|
+
n_ok = 0
|
|
83
|
+
try:
|
|
84
|
+
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
|
85
|
+
futures = [pool.submit(fn, it) for it in todo]
|
|
86
|
+
for fut in as_completed(futures):
|
|
87
|
+
row = fut.result()
|
|
88
|
+
with lock:
|
|
89
|
+
if is_csv:
|
|
90
|
+
writer.writerow(row)
|
|
91
|
+
else:
|
|
92
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
93
|
+
fh.flush()
|
|
94
|
+
n_ok += _is_success(row)
|
|
95
|
+
if bar is not None:
|
|
96
|
+
bar.update(1)
|
|
97
|
+
finally:
|
|
98
|
+
fh.close()
|
|
99
|
+
if bar is not None:
|
|
100
|
+
bar.close()
|
|
101
|
+
_compact(out_path, columns)
|
|
102
|
+
return {"skipped": len(done), "attempted": len(todo), "succeeded": n_ok, "failed": len(todo) - n_ok}
|