teaching-over-solving 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- teaching_over_solving-0.1.0.dist-info/METADATA +190 -0
- teaching_over_solving-0.1.0.dist-info/RECORD +14 -0
- teaching_over_solving-0.1.0.dist-info/WHEEL +5 -0
- teaching_over_solving-0.1.0.dist-info/entry_points.txt +3 -0
- teaching_over_solving-0.1.0.dist-info/licenses/LICENSE +21 -0
- teaching_over_solving-0.1.0.dist-info/top_level.txt +1 -0
- tos_diagnostic/__init__.py +6 -0
- tos_diagnostic/__main__.py +3 -0
- tos_diagnostic/cli.py +127 -0
- tos_diagnostic/data/mathtutorbench_leaderboard.csv +9 -0
- tos_diagnostic/data/mathtutorbench_leaderboard.json +6 -0
- tos_diagnostic/diagnostic.py +242 -0
- tos_diagnostic/report.py +189 -0
- tos_diagnostic/rubrics.py +100 -0
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: teaching-over-solving
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Teaching-over-solving diagnostic: separate how well an LLM tutor solves from how well it teaches.
|
|
5
|
+
Author: Junyi Yao, Zihao Zheng, Baichuan Li
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/Garyouki/teaching-over-solving
|
|
8
|
+
Project-URL: Paper, https://openreview.net/forum?id=aZ1KTkxECP
|
|
9
|
+
Keywords: llm-tutors,education,evaluation,pedagogy,benchmark
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Intended Audience :: Education
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Topic :: Education
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# Teaching-over-Solving Diagnostic
|
|
28
|
+
|
|
29
|
+
A small tool for one question: **is an LLM tutor as good at teaching as it is
|
|
30
|
+
at solving?**
|
|
31
|
+
|
|
32
|
+
A model that solves math problems well can still be a weak tutor. It may give
|
|
33
|
+
away the answer, skip scaffolding, or ignore what the student already knows.
|
|
34
|
+
If a benchmark folds everything into one "helpfulness" score, that difference
|
|
35
|
+
disappears.
|
|
36
|
+
|
|
37
|
+
This diagnostic keeps the two apart. From per-task benchmark scores it computes:
|
|
38
|
+
|
|
39
|
+
- **Solve:** the mean of solving-oriented tasks;
|
|
40
|
+
- **Teach:** the mean of pedagogy-oriented tasks;
|
|
41
|
+
- **Gap = Teach − Solve:** positive means relatively stronger at teaching.
|
|
42
|
+
|
|
43
|
+
Across a set of models it also reports how far each model's rank moves between
|
|
44
|
+
the two views, the Pearson and Spearman correlations between Solve and Teach,
|
|
45
|
+
and a leave-one-out range that shows how fragile those correlations are.
|
|
46
|
+
|
|
47
|
+
This is the companion tool for *Beyond Helpfulness: A Teaching-over-Solving
|
|
48
|
+
Diagnostic for Measuring Educational Impact in LLM Tutors* (5th Workshop on NLP
|
|
49
|
+
for Positive Impact, 2026, [OpenReview](https://openreview.net/forum?id=aZ1KTkxECP)).
|
|
50
|
+
|
|
51
|
+
> **Scope.** This is a descriptive diagnostic on benchmark scores. It does not
|
|
52
|
+
> measure student learning. With few models, correlations are fragile, so read
|
|
53
|
+
> them together with the leave-one-out range.
|
|
54
|
+
|
|
55
|
+

|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
Python 3.10 or newer. No dependencies.
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m pip install teaching-over-solving
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
This installs the `tos-diagnostic` command (`teaching-over-solving` also works)
|
|
66
|
+
and the `tos_diagnostic` Python package. To work from source:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
git clone https://github.com/Garyouki/teaching-over-solving.git
|
|
70
|
+
cd teaching-over-solving
|
|
71
|
+
python -m pip install -e ".[dev]"
|
|
72
|
+
python -m pytest
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Reproduce the paper
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
tos-diagnostic paper --report report.md --svg scatter.svg
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
This uses the eight public MathTutorBench leaderboard models bundled with the
|
|
82
|
+
package. It reproduces the paper's Table 1 composites, the Spearman correlation
|
|
83
|
+
(0.690), the leave-one-out Pearson range (0.251–0.887), and the two rank shifts
|
|
84
|
+
the paper describes:
|
|
85
|
+
|
|
86
|
+
- LLaMA3.1-8B-Instruct rises from rank 7 on solving to rank 4 on teaching
|
|
87
|
+
(gap +0.077).
|
|
88
|
+
- Qwen2.5-Math-7B-Instruct falls from rank 4 to rank 8 (gap −0.452).
|
|
89
|
+
|
|
90
|
+
The exact Pearson correlation is 0.4217. The paper prints it as 0.421.
|
|
91
|
+
|
|
92
|
+
[examples/paper_report.md](examples/paper_report.md) shows the full report.
|
|
93
|
+
|
|
94
|
+
## Diagnose your own tutor
|
|
95
|
+
|
|
96
|
+
### MathTutorBench scores
|
|
97
|
+
|
|
98
|
+
Put one row per model in a CSV with the nine MathTutorBench task scores. The
|
|
99
|
+
leaderboard's own column headers (for example `Problem Solving` or
|
|
100
|
+
`Scaffolding (Hard)`) are recognized.
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
tos-diagnostic score my_models.csv --reference --report report.md --svg scatter.svg
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`--reference` places your models next to the eight public models. Scores can
|
|
107
|
+
be fractions (`0.64`) or percentages (`64%`).
|
|
108
|
+
|
|
109
|
+
| Group | MathTutorBench tasks |
|
|
110
|
+
|---|---|
|
|
111
|
+
| Solve | problem solving, solution correctness, mistake location, mistake correction |
|
|
112
|
+
| Teach | Socratic questioning, scaffolding win rate, pedagogy instruction-following win rate, scaffolding (hard), pedagogy instruction-following (hard) |
|
|
113
|
+
|
|
114
|
+
The paper treats mistake location and mistake correction as solving tasks,
|
|
115
|
+
because they test whether the model can find or repair the mathematical state
|
|
116
|
+
of the problem.
|
|
117
|
+
|
|
118
|
+
### Any other benchmark
|
|
119
|
+
|
|
120
|
+
Name the columns for each side:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
tos-diagnostic score scores.csv --solve accuracy,final_answer \
|
|
124
|
+
--teach scaffolding,guiding_questions,rigor
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Use this for any benchmark that reports separate solving and teaching scores
|
|
128
|
+
per model, including your own internal evaluations.
|
|
129
|
+
|
|
130
|
+
Missing scores stop the run by default. `--allow-missing` averages the scores
|
|
131
|
+
that are present and marks those models in the report.
|
|
132
|
+
|
|
133
|
+
### Python
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from tos_diagnostic import diagnose, load_scores, with_reference
|
|
137
|
+
|
|
138
|
+
result = diagnose(with_reference(load_scores("my_models.csv")))
|
|
139
|
+
for m in result["models"]:
|
|
140
|
+
print(m["model"], round(m["gap"], 3), m["solve_rank"], "->", m["teach_rank"])
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
## Rubric agency density
|
|
144
|
+
|
|
145
|
+
The paper's second study counts how often tutoring-benchmark rubrics reward
|
|
146
|
+
agency-preserving behavior, such as asking guiding questions or giving hints
|
|
147
|
+
without revealing the next step. `tos-diagnostic rubrics` computes these
|
|
148
|
+
densities from rubric items you have tagged:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
tos-diagnostic rubrics examples/example_rubrics.jsonl
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
The keyword rules are transparent and replaceable. Reading TutorBench's raw
|
|
155
|
+
rubric format, and matching the paper's Table 2 exactly, is not yet
|
|
156
|
+
re-validated in this package. See [docs/rubric_rules.md](docs/rubric_rules.md).
|
|
157
|
+
|
|
158
|
+
## Limitations
|
|
159
|
+
|
|
160
|
+
- Benchmark scores are not learning outcomes. A positive gap does not show that
|
|
161
|
+
a tutor helps students learn more.
|
|
162
|
+
- The public leaderboard has only eight models. Removing one model moves the
|
|
163
|
+
Pearson correlation between 0.251 and 0.887.
|
|
164
|
+
- Other task groupings are possible. The default grouping is a transparent
|
|
165
|
+
choice, not a definitive taxonomy.
|
|
166
|
+
- The diagnostic works on aggregate scores. It does not check answer disclosure
|
|
167
|
+
turn by turn.
|
|
168
|
+
|
|
169
|
+
## Data
|
|
170
|
+
|
|
171
|
+
`src/tos_diagnostic/data/mathtutorbench_leaderboard.csv` contains the public
|
|
172
|
+
MathTutorBench leaderboard scores for eight models
|
|
173
|
+
([Macina et al., EMNLP 2025](https://eth-lre.github.io/mathtutorbench/)). Please
|
|
174
|
+
cite MathTutorBench when you use them.
|
|
175
|
+
|
|
176
|
+
## Citation
|
|
177
|
+
|
|
178
|
+
```bibtex
|
|
179
|
+
@inproceedings{yao2026beyond,
|
|
180
|
+
title = {Beyond Helpfulness: A Teaching-over-Solving Diagnostic for Measuring Educational Impact in {LLM} Tutors},
|
|
181
|
+
author = {Junyi Yao and Zihao Zheng and Baichuan Li},
|
|
182
|
+
booktitle = {5th Workshop on NLP for Positive Impact},
|
|
183
|
+
year = {2026},
|
|
184
|
+
url = {https://openreview.net/forum?id=aZ1KTkxECP}
|
|
185
|
+
}
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
## License
|
|
189
|
+
|
|
190
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
teaching_over_solving-0.1.0.dist-info/licenses/LICENSE,sha256=oof5D1sFcGaIIzWNZrag3s-lqk_oyGqcwzMOGWo3mKQ,1092
|
|
2
|
+
tos_diagnostic/__init__.py,sha256=Oud9zKuZOsaDlk9th5gPw2FoKIbqm6QVgakBOxpXjn4,256
|
|
3
|
+
tos_diagnostic/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
|
|
4
|
+
tos_diagnostic/cli.py,sha256=zLPIpzZ_69EQtA-t8esCiOMjWS92n6t2DfCbM3qZ8hY,5388
|
|
5
|
+
tos_diagnostic/diagnostic.py,sha256=dgAPCfIpuSBsK8zWEt2aU6uMe7162EkVowLIz1tdNjM,8820
|
|
6
|
+
tos_diagnostic/report.py,sha256=VF7mv3Eyv9Mc-0rRV0gD259eaqPz4i1YkfaNT7ISR3o,8473
|
|
7
|
+
tos_diagnostic/rubrics.py,sha256=SKUhjUcuWsuOP3GxUNXba-exRSA2Zy09Us8bfORGPwI,3521
|
|
8
|
+
tos_diagnostic/data/mathtutorbench_leaderboard.csv,sha256=lTXidbxWXLwHvvhVnqSN7a6rwCCGXDRXpz-ViRwx0NI,686
|
|
9
|
+
tos_diagnostic/data/mathtutorbench_leaderboard.json,sha256=aVgWkHFW3HkaeKsFfV8Bek-qsjq_Z01fs4t3fXEEVlU,436
|
|
10
|
+
teaching_over_solving-0.1.0.dist-info/METADATA,sha256=206ViNHdJ1nUtB0W--pOouf4lgNhF84NGwz15NGxL6I,6940
|
|
11
|
+
teaching_over_solving-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
12
|
+
teaching_over_solving-0.1.0.dist-info/entry_points.txt,sha256=qC-FGShBB-uknysndVp51ibuTTkJMwcNr-AGPiNJTqM,107
|
|
13
|
+
teaching_over_solving-0.1.0.dist-info/top_level.txt,sha256=rd2-lLuLKx3MyvxkWUqmcosPlRoEZi-VQCp3NGwbfkw,15
|
|
14
|
+
teaching_over_solving-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Junyi Yao, Zihao Zheng, Baichuan Li
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tos_diagnostic
|
tos_diagnostic/cli.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Command-line interface: ``tos-diagnostic <command>``."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from . import __version__
|
|
10
|
+
from .diagnostic import (MTB_SOLVE, MTB_TEACH, DiagnosticError, diagnose, load_scores,
|
|
11
|
+
reference_scores, reference_source, with_reference)
|
|
12
|
+
from .report import render, scatter_svg
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _cols(text):
|
|
16
|
+
return tuple(c.strip() for c in text.split(",") if c.strip())
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _print_table(result):
|
|
20
|
+
print(f"{'model':32s} {'solve':>6s} {'teach':>6s} {'gap':>7s} rank S->T")
|
|
21
|
+
for m in sorted(result["models"], key=lambda m: m["gap"], reverse=True):
|
|
22
|
+
tag = " (ref)" if m["reference"] else ""
|
|
23
|
+
print(f"{(m['model'] + tag)[:32]:32s} {m['solve']:6.3f} {m['teach']:6.3f} "
|
|
24
|
+
f"{m['gap']:+7.3f} {m['solve_rank']}->{m['teach_rank']}")
|
|
25
|
+
print(f"pearson {result['pearson']:.4f} spearman {result['spearman']:.4f}")
|
|
26
|
+
if result["loo_pearson_range"]:
|
|
27
|
+
lo, hi = result["loo_pearson_range"]
|
|
28
|
+
print(f"leave-one-out pearson {lo:.3f} to {hi:.3f}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _write_outputs(result, args, title):
|
|
32
|
+
svg_name = None
|
|
33
|
+
if args.svg:
|
|
34
|
+
Path(args.svg).write_text(scatter_svg(result), encoding="utf-8")
|
|
35
|
+
svg_name = Path(args.svg).name
|
|
36
|
+
if args.report:
|
|
37
|
+
Path(args.report).write_text(render(result, title, svg_name), encoding="utf-8")
|
|
38
|
+
if args.json:
|
|
39
|
+
Path(args.json).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
|
40
|
+
for label, path in (("report", args.report), ("figure", args.svg), ("json", args.json)):
|
|
41
|
+
if path:
|
|
42
|
+
print(f"{label} written to {path}")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def cmd_paper(args):
|
|
46
|
+
src = reference_source()
|
|
47
|
+
print(f"Source: {src['source']}")
|
|
48
|
+
result = diagnose(reference_scores())
|
|
49
|
+
_print_table(result)
|
|
50
|
+
_write_outputs(result, args, "Teaching-over-solving diagnostic: public MathTutorBench "
|
|
51
|
+
"leaderboard")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cmd_score(args):
|
|
55
|
+
rows = load_scores(args.scores, model_col=args.model_col,
|
|
56
|
+
canonicalize=not (args.solve or args.teach))
|
|
57
|
+
custom = bool(args.solve or args.teach)
|
|
58
|
+
if custom and not (args.solve and args.teach):
|
|
59
|
+
raise DiagnosticError("pass both --solve and --teach, or neither")
|
|
60
|
+
solve = _cols(args.solve) if custom else MTB_SOLVE
|
|
61
|
+
teach = _cols(args.teach) if custom else MTB_TEACH
|
|
62
|
+
if args.reference:
|
|
63
|
+
if custom:
|
|
64
|
+
raise DiagnosticError("--reference only works with the default MathTutorBench columns")
|
|
65
|
+
rows = with_reference(rows)
|
|
66
|
+
result = diagnose(rows, solve, teach, allow_missing=args.allow_missing)
|
|
67
|
+
_print_table(result)
|
|
68
|
+
_write_outputs(result, args, args.title or "Teaching-over-solving diagnostic")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def cmd_rubrics(args):
|
|
72
|
+
from .rubrics import density, load_items, load_rules
|
|
73
|
+
res = density(load_items(args.items), load_rules(args.rules))
|
|
74
|
+
print(f"{'use case':28s} {'rubrics':>7s} {'agency dim.':>12s} {'agency skill':>13s}")
|
|
75
|
+
for name, r in res.items():
|
|
76
|
+
print(f"{name[:28]:28s} {r['rubrics']:7d} {r['agency_dimension']:12.3f} "
|
|
77
|
+
f"{r['agency_skill']:13.3f}")
|
|
78
|
+
if args.json:
|
|
79
|
+
Path(args.json).write_text(json.dumps(res, indent=2), encoding="utf-8")
|
|
80
|
+
print(f"json written to {args.json}")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def main(argv=None):
|
|
84
|
+
ap = argparse.ArgumentParser(
|
|
85
|
+
prog="tos-diagnostic",
|
|
86
|
+
description="Separate how well an LLM tutor solves from how well it teaches.")
|
|
87
|
+
ap.add_argument("--version", action="version", version=__version__)
|
|
88
|
+
sub = ap.add_subparsers(dest="command", required=True)
|
|
89
|
+
|
|
90
|
+
def outputs(p):
|
|
91
|
+
p.add_argument("--report", help="write a Markdown report here")
|
|
92
|
+
p.add_argument("--svg", help="write a Solve-vs-Teach scatter plot (SVG) here")
|
|
93
|
+
p.add_argument("--json", help="write full results as JSON here")
|
|
94
|
+
|
|
95
|
+
p = sub.add_parser("paper", help="reproduce the paper's Study 1 on the public leaderboard")
|
|
96
|
+
outputs(p)
|
|
97
|
+
p.set_defaults(func=cmd_paper)
|
|
98
|
+
|
|
99
|
+
p = sub.add_parser("score", help="score your own models from a CSV of task scores")
|
|
100
|
+
p.add_argument("scores", help="CSV: one row per model, one column per task score")
|
|
101
|
+
p.add_argument("--model-col", default="model", help="name of the model column")
|
|
102
|
+
p.add_argument("--solve", help="comma-separated solving columns (custom benchmark)")
|
|
103
|
+
p.add_argument("--teach", help="comma-separated teaching columns (custom benchmark)")
|
|
104
|
+
p.add_argument("--reference", action="store_true",
|
|
105
|
+
help="compare against the eight public MathTutorBench models")
|
|
106
|
+
p.add_argument("--allow-missing", action="store_true",
|
|
107
|
+
help="average available scores when some task scores are missing")
|
|
108
|
+
p.add_argument("--title", help="report title")
|
|
109
|
+
outputs(p)
|
|
110
|
+
p.set_defaults(func=cmd_score)
|
|
111
|
+
|
|
112
|
+
p = sub.add_parser("rubrics", help="agency-oriented density of tagged rubric items")
|
|
113
|
+
p.add_argument("items", help="JSONL or CSV with use_case and tags per rubric item")
|
|
114
|
+
p.add_argument("--rules", help="JSON file with agency_dimension and agency_skill keywords")
|
|
115
|
+
p.add_argument("--json", help="write results as JSON here")
|
|
116
|
+
p.set_defaults(func=cmd_rubrics)
|
|
117
|
+
|
|
118
|
+
args = ap.parse_args(argv)
|
|
119
|
+
try:
|
|
120
|
+
args.func(args)
|
|
121
|
+
except DiagnosticError as e:
|
|
122
|
+
print(f"error: {e}", file=sys.stderr)
|
|
123
|
+
sys.exit(2)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
main()
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
model,problem_solving,socratic_questioning,solution_correctness,mistake_location,mistake_correction,scaffolding_win_rate,pedagogy_if_win_rate,scaffolding_hard,pedagogy_if_hard
|
|
2
|
+
LearnLM-1.5-Pro,0.94,0.32,0.75,0.57,0.74,0.64,0.68,0.66,0.67
|
|
3
|
+
GPT-4o,0.9,0.48,0.67,0.37,0.84,0.5,0.82,0.46,0.7
|
|
4
|
+
LLaMA3.1-70B-Instruct,0.91,0.29,0.71,0.56,0.19,0.63,0.7,0.49,0.49
|
|
5
|
+
LLaMA3.1-8B-Instruct,0.7,0.29,0.63,0.29,0.09,0.61,0.67,0.46,0.49
|
|
6
|
+
LLaMA3.2-3B-Instruct,0.6,0.29,0.67,0.41,0.13,0.64,0.63,0.45,0.4
|
|
7
|
+
Llemma-7B-ScienceTutor,0.62,0.29,0.66,0.29,0.16,0.37,0.48,0.38,0.42
|
|
8
|
+
Qwen2.5-7B-SocraticLM,0.73,0.32,0.05,0.39,0.23,0.39,0.39,0.28,0.28
|
|
9
|
+
Qwen2.5-Math-7B-Instruct,0.88,0.35,0.43,0.47,0.49,0.06,0.07,0.05,0.05
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{
|
|
2
|
+
"source": "Public MathTutorBench leaderboard (Macina et al., EMNLP 2025), https://eth-lre.github.io/mathtutorbench/",
|
|
3
|
+
"retrieved": "2026-11-20, via the benchmarklist.com mirror of the leaderboard (sampled 2026-05-27)",
|
|
4
|
+
"note": "Eight publicly reported models and nine task scores, as used in the Teaching-over-Solving paper. Values reproduce the paper's Table 1 composites exactly.",
|
|
5
|
+
"csv": "mathtutorbench_leaderboard.csv"
|
|
6
|
+
}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Teaching-over-solving gap diagnostic (Study 1 of the paper).
|
|
2
|
+
|
|
3
|
+
For each model m:
|
|
4
|
+
|
|
5
|
+
Solve(m) = mean of the solving-oriented task scores
|
|
6
|
+
Teach(m) = mean of the pedagogy-oriented task scores
|
|
7
|
+
Gap(m) = Teach(m) - Solve(m)
|
|
8
|
+
|
|
9
|
+
A positive gap means the model is relatively stronger at teaching than at
|
|
10
|
+
solving. A negative gap means the reverse. Across a set of models the
|
|
11
|
+
diagnostic also reports how far each model's rank moves between the two
|
|
12
|
+
views, and the Pearson and Spearman correlations between the composites,
|
|
13
|
+
with a leave-one-out range for Pearson.
|
|
14
|
+
|
|
15
|
+
The default task grouping is the one used in the paper for MathTutorBench.
|
|
16
|
+
Any benchmark with per-model task scores can be used by passing its own
|
|
17
|
+
solving and teaching columns.
|
|
18
|
+
|
|
19
|
+
Pure Python, no third-party dependencies.
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import csv
|
|
24
|
+
import json
|
|
25
|
+
import math
|
|
26
|
+
from importlib import resources
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
# MathTutorBench grouping used in the paper.
|
|
30
|
+
MTB_SOLVE = ("problem_solving", "solution_correctness", "mistake_location",
|
|
31
|
+
"mistake_correction")
|
|
32
|
+
MTB_TEACH = ("socratic_questioning", "scaffolding_win_rate", "pedagogy_if_win_rate",
|
|
33
|
+
"scaffolding_hard", "pedagogy_if_hard")
|
|
34
|
+
MTB_COLUMNS = MTB_SOLVE + MTB_TEACH
|
|
35
|
+
|
|
36
|
+
# Column names people commonly use for the same MathTutorBench tasks.
|
|
37
|
+
_ALIASES = {
|
|
38
|
+
"problemsolving": "problem_solving",
|
|
39
|
+
"socraticquestioning": "socratic_questioning",
|
|
40
|
+
"solutioncorrectness": "solution_correctness",
|
|
41
|
+
"studentsolutioncorrectness": "solution_correctness",
|
|
42
|
+
"solutionverification": "solution_correctness",
|
|
43
|
+
"mistakelocation": "mistake_location",
|
|
44
|
+
"mistakecorrection": "mistake_correction",
|
|
45
|
+
"scaffoldingwinrate": "scaffolding_win_rate",
|
|
46
|
+
"scaffoldinggeneration": "scaffolding_win_rate",
|
|
47
|
+
"scaffolding": "scaffolding_win_rate",
|
|
48
|
+
"pedagogyifwinrate": "pedagogy_if_win_rate",
|
|
49
|
+
"pedagogyfollowing": "pedagogy_if_win_rate",
|
|
50
|
+
"pedagogyif": "pedagogy_if_win_rate",
|
|
51
|
+
"scaffoldinghard": "scaffolding_hard",
|
|
52
|
+
"scaffoldinggenerationhard": "scaffolding_hard",
|
|
53
|
+
"pedagogyifhard": "pedagogy_if_hard",
|
|
54
|
+
"pedagogyfollowinghard": "pedagogy_if_hard",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class DiagnosticError(ValueError):
|
|
59
|
+
"""Raised for inputs the diagnostic cannot score."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------- loading
|
|
63
|
+
|
|
64
|
+
def _canon(name: str) -> str:
|
|
65
|
+
key = "".join(ch for ch in name.lower() if ch.isalnum())
|
|
66
|
+
return _ALIASES.get(key, name.strip())
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def load_scores(path, model_col: str = "model", canonicalize: bool = True) -> list[dict]:
|
|
70
|
+
"""Read a CSV with one row per model and one column per task score."""
|
|
71
|
+
with open(path, newline="", encoding="utf-8-sig") as f:
|
|
72
|
+
reader = csv.DictReader(f)
|
|
73
|
+
if not reader.fieldnames or model_col not in reader.fieldnames:
|
|
74
|
+
raise DiagnosticError(f"{path}: needs a '{model_col}' column")
|
|
75
|
+
rows = []
|
|
76
|
+
for raw in reader:
|
|
77
|
+
row = {"model": raw[model_col].strip()}
|
|
78
|
+
for k, v in raw.items():
|
|
79
|
+
if k == model_col or k is None:
|
|
80
|
+
continue
|
|
81
|
+
row[_canon(k) if canonicalize else k.strip()] = v
|
|
82
|
+
rows.append(row)
|
|
83
|
+
return rows
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def reference_scores() -> list[dict]:
|
|
87
|
+
"""The eight public MathTutorBench leaderboard models used in the paper."""
|
|
88
|
+
path = resources.files("tos_diagnostic") / "data" / "mathtutorbench_leaderboard.csv"
|
|
89
|
+
with resources.as_file(path) as p:
|
|
90
|
+
return load_scores(p)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def reference_source() -> dict:
|
|
94
|
+
path = resources.files("tos_diagnostic") / "data" / "mathtutorbench_leaderboard.json"
|
|
95
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _number(value, model, col):
|
|
99
|
+
if value is None or str(value).strip() == "":
|
|
100
|
+
return None
|
|
101
|
+
try:
|
|
102
|
+
x = float(str(value).strip().rstrip("%"))
|
|
103
|
+
except ValueError:
|
|
104
|
+
raise DiagnosticError(f"{model}: '{col}' is not a number: {value!r}") from None
|
|
105
|
+
if str(value).strip().endswith("%"):
|
|
106
|
+
x /= 100.0
|
|
107
|
+
if math.isnan(x):
|
|
108
|
+
return None
|
|
109
|
+
return x
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ---------------------------------------------------------------- statistics
|
|
113
|
+
|
|
114
|
+
def mean(xs):
|
|
115
|
+
xs = list(xs)
|
|
116
|
+
return sum(xs) / len(xs) if xs else float("nan")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def pearson(x, y):
|
|
120
|
+
n = len(x)
|
|
121
|
+
if n < 3:
|
|
122
|
+
return float("nan")
|
|
123
|
+
mx, my = mean(x), mean(y)
|
|
124
|
+
sxy = sum((a - mx) * (b - my) for a, b in zip(x, y))
|
|
125
|
+
sxx = sum((a - mx) ** 2 for a in x)
|
|
126
|
+
syy = sum((b - my) ** 2 for b in y)
|
|
127
|
+
if sxx == 0 or syy == 0:
|
|
128
|
+
return float("nan")
|
|
129
|
+
return sxy / math.sqrt(sxx * syy)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def average_ranks(values):
|
|
133
|
+
"""Ranks starting at 1 for the smallest value; ties share their mean rank."""
|
|
134
|
+
order = sorted(range(len(values)), key=lambda i: values[i])
|
|
135
|
+
ranks = [0.0] * len(values)
|
|
136
|
+
i = 0
|
|
137
|
+
while i < len(order):
|
|
138
|
+
j = i
|
|
139
|
+
while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]:
|
|
140
|
+
j += 1
|
|
141
|
+
for k in range(i, j + 1):
|
|
142
|
+
ranks[order[k]] = (i + j) / 2 + 1
|
|
143
|
+
i = j + 1
|
|
144
|
+
return ranks
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def spearman(x, y):
|
|
148
|
+
return pearson(average_ranks(x), average_ranks(y))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def leaderboard_ranks(values):
|
|
152
|
+
"""1 = highest score. Ties share the best rank (1, 2, 2, 4)."""
|
|
153
|
+
return [1 + sum(1 for w in values if w > v) for v in values]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# ---------------------------------------------------------------- diagnostic
|
|
157
|
+
|
|
158
|
+
def composites(rows, solve_cols=MTB_SOLVE, teach_cols=MTB_TEACH, allow_missing=False):
|
|
159
|
+
"""Compute Solve, Teach, and Gap for each model row."""
|
|
160
|
+
solve_cols, teach_cols = tuple(solve_cols), tuple(teach_cols)
|
|
161
|
+
if not solve_cols or not teach_cols:
|
|
162
|
+
raise DiagnosticError("need at least one solving column and one teaching column")
|
|
163
|
+
overlap = set(solve_cols) & set(teach_cols)
|
|
164
|
+
if overlap:
|
|
165
|
+
raise DiagnosticError(f"columns cannot be in both groups: {sorted(overlap)}")
|
|
166
|
+
seen, out = set(), []
|
|
167
|
+
for row in rows:
|
|
168
|
+
model = row["model"]
|
|
169
|
+
if not model:
|
|
170
|
+
raise DiagnosticError("a row has an empty model name")
|
|
171
|
+
if model in seen:
|
|
172
|
+
raise DiagnosticError(f"duplicate model: {model}")
|
|
173
|
+
seen.add(model)
|
|
174
|
+
parts = {}
|
|
175
|
+
for group, cols in (("solve", solve_cols), ("teach", teach_cols)):
|
|
176
|
+
vals, missing = [], []
|
|
177
|
+
for c in cols:
|
|
178
|
+
if c not in row:
|
|
179
|
+
missing.append(c)
|
|
180
|
+
continue
|
|
181
|
+
v = _number(row[c], model, c)
|
|
182
|
+
if v is None:
|
|
183
|
+
missing.append(c)
|
|
184
|
+
else:
|
|
185
|
+
vals.append(v)
|
|
186
|
+
if missing and not allow_missing:
|
|
187
|
+
raise DiagnosticError(
|
|
188
|
+
f"{model}: missing {group} scores for {missing}. "
|
|
189
|
+
"Pass allow_missing=True (CLI: --allow-missing) to average the "
|
|
190
|
+
"available scores instead.")
|
|
191
|
+
if not vals:
|
|
192
|
+
raise DiagnosticError(f"{model}: no {group} scores at all")
|
|
193
|
+
parts[group] = (mean(vals), missing)
|
|
194
|
+
solve, teach = parts["solve"][0], parts["teach"][0]
|
|
195
|
+
out.append({
|
|
196
|
+
"model": model,
|
|
197
|
+
"solve": solve,
|
|
198
|
+
"teach": teach,
|
|
199
|
+
"gap": teach - solve,
|
|
200
|
+
"missing": parts["solve"][1] + parts["teach"][1],
|
|
201
|
+
"reference": bool(row.get("_reference")),
|
|
202
|
+
})
|
|
203
|
+
return out
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def diagnose(rows, solve_cols=MTB_SOLVE, teach_cols=MTB_TEACH, allow_missing=False) -> dict:
|
|
207
|
+
"""Full diagnostic over a set of models."""
|
|
208
|
+
models = composites(rows, solve_cols, teach_cols, allow_missing)
|
|
209
|
+
s = [m["solve"] for m in models]
|
|
210
|
+
t = [m["teach"] for m in models]
|
|
211
|
+
rs, rt = leaderboard_ranks(s), leaderboard_ranks(t)
|
|
212
|
+
for m, a, b in zip(models, rs, rt):
|
|
213
|
+
m["solve_rank"], m["teach_rank"] = a, b
|
|
214
|
+
m["rank_shift"] = a - b # positive: rises when evaluation moves to teaching
|
|
215
|
+
n = len(models)
|
|
216
|
+
loo = []
|
|
217
|
+
if n >= 4:
|
|
218
|
+
for i in range(n):
|
|
219
|
+
loo.append((models[i]["model"],
|
|
220
|
+
pearson([v for j, v in enumerate(s) if j != i],
|
|
221
|
+
[v for j, v in enumerate(t) if j != i])))
|
|
222
|
+
return {
|
|
223
|
+
"n_models": n,
|
|
224
|
+
"solve_columns": list(solve_cols),
|
|
225
|
+
"teach_columns": list(teach_cols),
|
|
226
|
+
"models": models,
|
|
227
|
+
"pearson": pearson(s, t),
|
|
228
|
+
"spearman": spearman(s, t),
|
|
229
|
+
"loo_pearson": loo,
|
|
230
|
+
"loo_pearson_range": ((min(v for _, v in loo), max(v for _, v in loo))
|
|
231
|
+
if loo else None),
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def with_reference(rows):
|
|
236
|
+
"""Append the public MathTutorBench models (marked as reference) to user rows.
|
|
237
|
+
|
|
238
|
+
User models that share a name with a reference model replace it.
|
|
239
|
+
"""
|
|
240
|
+
names = {r["model"] for r in rows}
|
|
241
|
+
ref = [dict(r, _reference=True) for r in reference_scores() if r["model"] not in names]
|
|
242
|
+
return list(rows) + ref
|
tos_diagnostic/report.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Markdown report and SVG scatter plot for the gap diagnostic."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import datetime
|
|
5
|
+
import math
|
|
6
|
+
from xml.sax.saxutils import escape
|
|
7
|
+
|
|
8
|
+
from . import __version__
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _f(x, d=3, sign=False):
|
|
12
|
+
if x is None or (isinstance(x, float) and math.isnan(x)):
|
|
13
|
+
return "n/a"
|
|
14
|
+
return f"{x:+.{d}f}" if sign else f"{x:.{d}f}"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def render(result: dict, title: str = "Teaching-over-solving diagnostic",
|
|
18
|
+
svg_name: str | None = None) -> str:
|
|
19
|
+
models = sorted(result["models"], key=lambda m: m["gap"], reverse=True)
|
|
20
|
+
has_ref = any(m["reference"] for m in models)
|
|
21
|
+
lines = []
|
|
22
|
+
w = lines.append
|
|
23
|
+
w(f"# {title}")
|
|
24
|
+
w("")
|
|
25
|
+
w(f"Generated {datetime.date.today().isoformat()} with tos-diagnostic {__version__}. "
|
|
26
|
+
f"{result['n_models']} models.")
|
|
27
|
+
w("")
|
|
28
|
+
w("- **Solve** = mean of: " + ", ".join(f"`{c}`" for c in result["solve_columns"]))
|
|
29
|
+
w("- **Teach** = mean of: " + ", ".join(f"`{c}`" for c in result["teach_columns"]))
|
|
30
|
+
w("- **Gap** = Teach − Solve. Positive means relatively stronger at teaching than at "
|
|
31
|
+
"solving.")
|
|
32
|
+
w("")
|
|
33
|
+
w("> **Scope.** This is a descriptive diagnostic on benchmark scores. It does not "
|
|
34
|
+
"measure student learning, and a small number of models makes the correlations "
|
|
35
|
+
"fragile.")
|
|
36
|
+
w("")
|
|
37
|
+
|
|
38
|
+
w("## Summary")
|
|
39
|
+
w("")
|
|
40
|
+
w(f"- Pearson correlation between Solve and Teach: {_f(result['pearson'])}. "
|
|
41
|
+
f"Spearman: {_f(result['spearman'])}.")
|
|
42
|
+
if result["loo_pearson_range"]:
|
|
43
|
+
lo, hi = result["loo_pearson_range"]
|
|
44
|
+
w(f"- Leave-one-out Pearson ranges from {_f(lo)} to {_f(hi)}.")
|
|
45
|
+
drop, val = max(result["loo_pearson"], key=lambda kv: abs(kv[1] - result["pearson"]))
|
|
46
|
+
w(f"- Removing **{drop}** changes Pearson the most (to {_f(val)}).")
|
|
47
|
+
movers = [m for m in models if abs(m["rank_shift"]) >= 2]
|
|
48
|
+
for m in sorted(movers, key=lambda m: -abs(m["rank_shift"])):
|
|
49
|
+
direction = "rises" if m["rank_shift"] > 0 else "falls"
|
|
50
|
+
w(f"- **{m['model']}** {direction} from rank {m['solve_rank']} on solving to rank "
|
|
51
|
+
f"{m['teach_rank']} on teaching (gap {_f(m['gap'], sign=True)}).")
|
|
52
|
+
if not movers:
|
|
53
|
+
w("- No model moves two or more ranks between the two views.")
|
|
54
|
+
w("")
|
|
55
|
+
|
|
56
|
+
w("## Models")
|
|
57
|
+
w("")
|
|
58
|
+
head = "| Model | Solve | Teach | Gap | Solve rank | Teach rank | Shift |"
|
|
59
|
+
sep = "|---|---:|---:|---:|---:|---:|---:|"
|
|
60
|
+
w(head)
|
|
61
|
+
w(sep)
|
|
62
|
+
for m in models:
|
|
63
|
+
name = m["model"] + (" *(reference)*" if m["reference"] else "")
|
|
64
|
+
if m["missing"]:
|
|
65
|
+
name += " †"
|
|
66
|
+
w(f"| {name} | {_f(m['solve'])} | {_f(m['teach'])} | {_f(m['gap'], sign=True)} | "
|
|
67
|
+
f"{m['solve_rank']} | {m['teach_rank']} | {m['rank_shift']:+d} |".replace("| +0 |", "| 0 |"))
|
|
68
|
+
w("")
|
|
69
|
+
if has_ref:
|
|
70
|
+
w("*Reference* rows are the eight public MathTutorBench leaderboard models used in "
|
|
71
|
+
"the paper.")
|
|
72
|
+
if any(m["missing"] for m in models):
|
|
73
|
+
w("† Averaged over the available scores only; some task scores were missing.")
|
|
74
|
+
w("")
|
|
75
|
+
if svg_name:
|
|
76
|
+
w(f"")
|
|
77
|
+
w("")
|
|
78
|
+
w("Points below the diagonal are relatively stronger at solving than at teaching.")
|
|
79
|
+
w("")
|
|
80
|
+
|
|
81
|
+
w("## How to read this")
|
|
82
|
+
w("")
|
|
83
|
+
w("A strong solver is not automatically a strong tutor. A large negative gap is a "
|
|
84
|
+
"flag to check whether the model gives away answers or skips scaffolding when a "
|
|
85
|
+
"learner needs support. Report Solve and Teach separately rather than as one "
|
|
86
|
+
"helpfulness score.")
|
|
87
|
+
w("")
|
|
88
|
+
w("## Citation")
|
|
89
|
+
w("")
|
|
90
|
+
w("Yao, Zheng, and Li. *Beyond Helpfulness: A Teaching-over-Solving Diagnostic for "
|
|
91
|
+
"Measuring Educational Impact in LLM Tutors.* 5th Workshop on NLP for Positive "
|
|
92
|
+
"Impact, 2026.")
|
|
93
|
+
w("")
|
|
94
|
+
return "\n".join(lines)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def scatter_svg(result: dict, width: int = 640, height: int = 520) -> str:
|
|
98
|
+
"""Solve (x) versus Teach (y) with the diagonal. No dependencies."""
|
|
99
|
+
pad_l, pad_r, pad_t, pad_b = 64, 24, 24, 56
|
|
100
|
+
pw, ph = width - pad_l - pad_r, height - pad_t - pad_b
|
|
101
|
+
vals = [v for m in result["models"] for v in (m["solve"], m["teach"])]
|
|
102
|
+
lo = max(0.0, math.floor(min(vals) * 10) / 10 - 0.05)
|
|
103
|
+
hi = min(1.0, math.ceil(max(vals) * 10) / 10 + 0.05)
|
|
104
|
+
if hi - lo < 0.2:
|
|
105
|
+
hi = lo + 0.2
|
|
106
|
+
|
|
107
|
+
def sx(v):
|
|
108
|
+
return pad_l + (v - lo) / (hi - lo) * pw
|
|
109
|
+
|
|
110
|
+
def sy(v):
|
|
111
|
+
return pad_t + (1 - (v - lo) / (hi - lo)) * ph
|
|
112
|
+
|
|
113
|
+
o = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
|
|
114
|
+
f'viewBox="0 0 {width} {height}" font-family="Helvetica, Arial, sans-serif" '
|
|
115
|
+
f'font-size="12">',
|
|
116
|
+
f'<rect width="{width}" height="{height}" fill="#ffffff"/>']
|
|
117
|
+
tick = math.ceil(lo * 10) / 10
|
|
118
|
+
while tick <= hi + 1e-9:
|
|
119
|
+
x, y = sx(tick), sy(tick)
|
|
120
|
+
o.append(f'<line x1="{x:.1f}" y1="{pad_t}" x2="{x:.1f}" y2="{pad_t + ph}" '
|
|
121
|
+
f'stroke="#eeeeee"/>')
|
|
122
|
+
o.append(f'<line x1="{pad_l}" y1="{y:.1f}" x2="{pad_l + pw}" y2="{y:.1f}" '
|
|
123
|
+
f'stroke="#eeeeee"/>')
|
|
124
|
+
o.append(f'<text x="{x:.1f}" y="{pad_t + ph + 18}" text-anchor="middle" '
|
|
125
|
+
f'fill="#555555">{tick:.1f}</text>')
|
|
126
|
+
o.append(f'<text x="{pad_l - 8}" y="{y + 4:.1f}" text-anchor="end" '
|
|
127
|
+
f'fill="#555555">{tick:.1f}</text>')
|
|
128
|
+
tick = round(tick + 0.1, 10)
|
|
129
|
+
o.append(f'<rect x="{pad_l}" y="{pad_t}" width="{pw}" height="{ph}" fill="none" '
|
|
130
|
+
f'stroke="#999999"/>')
|
|
131
|
+
o.append(f'<line x1="{sx(lo):.1f}" y1="{sy(lo):.1f}" x2="{sx(hi):.1f}" y2="{sy(hi):.1f}" '
|
|
132
|
+
f'stroke="#999999" stroke-dasharray="4 4"/>')
|
|
133
|
+
o.append(f'<text x="{pad_l + pw / 2}" y="{height - 16}" text-anchor="middle" '
|
|
134
|
+
f'fill="#222222">Solve composite</text>')
|
|
135
|
+
o.append(f'<text transform="translate(18 {pad_t + ph / 2}) rotate(-90)" '
|
|
136
|
+
f'text-anchor="middle" fill="#222222">Teach composite</text>')
|
|
137
|
+
points = [(sx(m["solve"]), sy(m["teach"]), m) for m in result["models"]]
|
|
138
|
+
for x, y, m in points:
|
|
139
|
+
color = "#9aa5b1" if m["reference"] else "#2563eb"
|
|
140
|
+
o.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="5" fill="{color}"/>')
|
|
141
|
+
|
|
142
|
+
# Greedy label placement: try four spots around each point and keep the
|
|
143
|
+
# first one that does not overlap a placed label or a point.
|
|
144
|
+
placed = []
|
|
145
|
+
|
|
146
|
+
def crowding(box):
|
|
147
|
+
"""None if the label leaves the canvas, else how many things it overlaps."""
|
|
148
|
+
x0, y0, x1, y1 = box
|
|
149
|
+
if x0 < 4 or x1 > width - 4 or y0 < pad_t or y1 > pad_t + ph:
|
|
150
|
+
return None
|
|
151
|
+
hits = sum(1 for b in placed
|
|
152
|
+
if not (x1 < b[0] or x0 > b[2] or y1 < b[1] or y0 > b[3]))
|
|
153
|
+
hits += sum(1 for px, py, _ in points
|
|
154
|
+
if x0 - 9 < px < x1 + 9 and y0 - 9 < py < y1 + 9)
|
|
155
|
+
return hits
|
|
156
|
+
|
|
157
|
+
for x, y, m in sorted(points, key=lambda p: (p[1], p[0])):
|
|
158
|
+
label = m["model"]
|
|
159
|
+
tw, th = 6.6 * len(label), 12
|
|
160
|
+
spots = [("start", x + 8, y - 6), ("start", x + 8, y + 16),
|
|
161
|
+
("end", x - 8, y - 6), ("end", x - 8, y + 16),
|
|
162
|
+
("start", x + 8, y - 20), ("end", x - 8, y - 20),
|
|
163
|
+
("start", x + 8, y + 30), ("end", x - 8, y + 30)]
|
|
164
|
+
best = None
|
|
165
|
+
for order, (anchor, tx, ty) in enumerate(spots):
|
|
166
|
+
x0 = tx if anchor == "start" else tx - tw
|
|
167
|
+
box = (x0, ty - th, x0 + tw, ty + 2)
|
|
168
|
+
score = crowding(box)
|
|
169
|
+
if score is None:
|
|
170
|
+
continue
|
|
171
|
+
if best is None or (score, order) < best[0]:
|
|
172
|
+
best = ((score, order), (anchor, tx, ty, box))
|
|
173
|
+
if score == 0:
|
|
174
|
+
break
|
|
175
|
+
if best is None: # nothing fits on the canvas: place left of the point
|
|
176
|
+
anchor, tx, ty = "end", x - 8, y - 6
|
|
177
|
+
best = (None, (anchor, tx, ty, (tx - tw, ty - th, tx, ty + 2)))
|
|
178
|
+
chosen = best[1]
|
|
179
|
+
anchor, tx, ty, box = chosen
|
|
180
|
+
placed.append(box)
|
|
181
|
+
# Leader line from the point to the nearest corner of its label.
|
|
182
|
+
lx = box[0] if anchor == "start" else box[2]
|
|
183
|
+
ly = box[3] if ty < y else box[1]
|
|
184
|
+
o.append(f'<line x1="{x:.1f}" y1="{y:.1f}" x2="{lx:.1f}" y2="{ly:.1f}" '
|
|
185
|
+
f'stroke="#bbbbbb" stroke-width="0.8"/>')
|
|
186
|
+
o.append(f'<text x="{tx:.1f}" y="{ty:.1f}" text-anchor="{anchor}" '
|
|
187
|
+
f'fill="#222222">{escape(label)}</text>')
|
|
188
|
+
o.append("</svg>")
|
|
189
|
+
return "\n".join(o)
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Agency-oriented rubric density (Study 2 of the paper).
|
|
2
|
+
|
|
3
|
+
Given rubric items that carry metadata tags, count how many items reward
|
|
4
|
+
agency-preserving tutoring:
|
|
5
|
+
|
|
6
|
+
- agency-dimension density: share of items with at least one tag about
|
|
7
|
+
instruction following or calibration to the student's level;
|
|
8
|
+
- agency-skill density: share of items with at least one explicit tutoring-skill
|
|
9
|
+
tag such as asking guiding questions, giving hints, eliciting reflection, or
|
|
10
|
+
diagnosing the learner's state.
|
|
11
|
+
|
|
12
|
+
The mapping is a transparent keyword rule on tags, not a human coding study.
|
|
13
|
+
Rules can be replaced with a JSON file.
|
|
14
|
+
|
|
15
|
+
Input is one rubric item per line (JSONL) or per row (CSV):
|
|
16
|
+
|
|
17
|
+
{"use_case": "active_learning", "tags": ["instruction following", "ask guiding questions"]}
|
|
18
|
+
|
|
19
|
+
Status: the paper's Table 2 was computed on the public TutorBench sample.
|
|
20
|
+
Converting that dataset's raw rubric format into these tags has not yet been
|
|
21
|
+
re-validated against Table 2 in this package. See docs/rubric_rules.md.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import csv
|
|
26
|
+
import json
|
|
27
|
+
from collections import OrderedDict
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
DEFAULT_RULES = {
|
|
31
|
+
"agency_dimension": [
|
|
32
|
+
"instruction following",
|
|
33
|
+
"instruction-following",
|
|
34
|
+
"calibration to student",
|
|
35
|
+
"student level",
|
|
36
|
+
"student-level",
|
|
37
|
+
"calibrat",
|
|
38
|
+
],
|
|
39
|
+
"agency_skill": [
|
|
40
|
+
"guiding question",
|
|
41
|
+
"hint",
|
|
42
|
+
"elicit",
|
|
43
|
+
"reflection",
|
|
44
|
+
"misconception",
|
|
45
|
+
"learner state",
|
|
46
|
+
"student state",
|
|
47
|
+
"diagnos",
|
|
48
|
+
],
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def load_rules(path=None) -> dict:
|
|
53
|
+
if not path:
|
|
54
|
+
return {k: list(v) for k, v in DEFAULT_RULES.items()}
|
|
55
|
+
rules = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
56
|
+
for key in ("agency_dimension", "agency_skill"):
|
|
57
|
+
if key not in rules or not isinstance(rules[key], list) or not rules[key]:
|
|
58
|
+
raise ValueError(f"rules file needs a non-empty list '{key}'")
|
|
59
|
+
return rules
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def load_items(path) -> list[dict]:
|
|
63
|
+
path = Path(path)
|
|
64
|
+
if path.suffix.lower() == ".csv":
|
|
65
|
+
with open(path, newline="", encoding="utf-8-sig") as f:
|
|
66
|
+
rows = list(csv.DictReader(f))
|
|
67
|
+
items = [{"use_case": r.get("use_case", "").strip(),
|
|
68
|
+
"tags": [t.strip() for t in (r.get("tags") or "").split(";") if t.strip()]}
|
|
69
|
+
for r in rows]
|
|
70
|
+
else:
|
|
71
|
+
with open(path, encoding="utf-8") as f:
|
|
72
|
+
items = [json.loads(line) for line in f if line.strip()]
|
|
73
|
+
for i, it in enumerate(items):
|
|
74
|
+
if not it.get("use_case"):
|
|
75
|
+
raise ValueError(f"item {i}: missing use_case")
|
|
76
|
+
if not isinstance(it.get("tags"), list):
|
|
77
|
+
raise ValueError(f"item {i}: tags must be a list")
|
|
78
|
+
return items
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _matches(tags, keywords):
|
|
82
|
+
low = [str(t).lower() for t in tags]
|
|
83
|
+
return any(k.lower() in t for t in low for k in keywords)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def density(items, rules=None) -> OrderedDict:
|
|
87
|
+
rules = rules or DEFAULT_RULES
|
|
88
|
+
groups = OrderedDict()
|
|
89
|
+
for it in items:
|
|
90
|
+
groups.setdefault(it["use_case"], []).append(it)
|
|
91
|
+
groups["all"] = list(items)
|
|
92
|
+
out = OrderedDict()
|
|
93
|
+
for name, its in groups.items():
|
|
94
|
+
n = len(its)
|
|
95
|
+
dim = sum(_matches(it["tags"], rules["agency_dimension"]) for it in its)
|
|
96
|
+
skill = sum(_matches(it["tags"], rules["agency_skill"]) for it in its)
|
|
97
|
+
out[name] = {"rubrics": n,
|
|
98
|
+
"agency_dimension": dim / n if n else float("nan"),
|
|
99
|
+
"agency_skill": skill / n if n else float("nan")}
|
|
100
|
+
return out
|