teaching-over-solving 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. teaching_over_solving-0.1.0/LICENSE +21 -0
  2. teaching_over_solving-0.1.0/PKG-INFO +190 -0
  3. teaching_over_solving-0.1.0/README.md +164 -0
  4. teaching_over_solving-0.1.0/pyproject.toml +51 -0
  5. teaching_over_solving-0.1.0/setup.cfg +4 -0
  6. teaching_over_solving-0.1.0/src/teaching_over_solving.egg-info/PKG-INFO +190 -0
  7. teaching_over_solving-0.1.0/src/teaching_over_solving.egg-info/SOURCES.txt +20 -0
  8. teaching_over_solving-0.1.0/src/teaching_over_solving.egg-info/dependency_links.txt +1 -0
  9. teaching_over_solving-0.1.0/src/teaching_over_solving.egg-info/entry_points.txt +3 -0
  10. teaching_over_solving-0.1.0/src/teaching_over_solving.egg-info/requires.txt +3 -0
  11. teaching_over_solving-0.1.0/src/teaching_over_solving.egg-info/top_level.txt +1 -0
  12. teaching_over_solving-0.1.0/src/tos_diagnostic/__init__.py +6 -0
  13. teaching_over_solving-0.1.0/src/tos_diagnostic/__main__.py +3 -0
  14. teaching_over_solving-0.1.0/src/tos_diagnostic/cli.py +127 -0
  15. teaching_over_solving-0.1.0/src/tos_diagnostic/data/mathtutorbench_leaderboard.csv +9 -0
  16. teaching_over_solving-0.1.0/src/tos_diagnostic/data/mathtutorbench_leaderboard.json +6 -0
  17. teaching_over_solving-0.1.0/src/tos_diagnostic/diagnostic.py +242 -0
  18. teaching_over_solving-0.1.0/src/tos_diagnostic/report.py +189 -0
  19. teaching_over_solving-0.1.0/src/tos_diagnostic/rubrics.py +100 -0
  20. teaching_over_solving-0.1.0/tests/test_diagnostic.py +75 -0
  21. teaching_over_solving-0.1.0/tests/test_paper.py +44 -0
  22. teaching_over_solving-0.1.0/tests/test_rubrics_and_cli.py +60 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Junyi Yao, Zihao Zheng, Baichuan Li
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,190 @@
1
+ Metadata-Version: 2.4
2
+ Name: teaching-over-solving
3
+ Version: 0.1.0
4
+ Summary: Teaching-over-solving diagnostic: separate how well an LLM tutor solves from how well it teaches.
5
+ Author: Junyi Yao, Zihao Zheng, Baichuan Li
6
+ License-Expression: MIT
7
+ Project-URL: Repository, https://github.com/Garyouki/teaching-over-solving
8
+ Project-URL: Paper, https://openreview.net/forum?id=aZ1KTkxECP
9
+ Keywords: llm-tutors,education,evaluation,pedagogy,benchmark
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Intended Audience :: Education
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Education
20
+ Requires-Python: >=3.10
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Provides-Extra: dev
24
+ Requires-Dist: pytest>=8; extra == "dev"
25
+ Dynamic: license-file
26
+
27
+ # Teaching-over-Solving Diagnostic
28
+
29
+ A small tool for one question: **is an LLM tutor as good at teaching as it is
30
+ at solving?**
31
+
32
+ A model that solves math problems well can still be a weak tutor. It may give
33
+ away the answer, skip scaffolding, or ignore what the student already knows.
34
+ If a benchmark folds everything into one "helpfulness" score, that difference
35
+ disappears.
36
+
37
+ This diagnostic keeps the two apart. From per-task benchmark scores it computes:
38
+
39
+ - **Solve:** the mean of solving-oriented tasks;
40
+ - **Teach:** the mean of pedagogy-oriented tasks;
41
+ - **Gap = Teach − Solve:** positive means relatively stronger at teaching.
42
+
43
+ Across a set of models it also reports how far each model's rank moves between
44
+ the two views, the Pearson and Spearman correlations between Solve and Teach,
45
+ and a leave-one-out range that shows how fragile those correlations are.
46
+
47
+ This is the companion tool for *Beyond Helpfulness: A Teaching-over-Solving
48
+ Diagnostic for Measuring Educational Impact in LLM Tutors* (5th Workshop on NLP
49
+ for Positive Impact, 2026, [OpenReview](https://openreview.net/forum?id=aZ1KTkxECP)).
50
+
51
+ > **Scope.** This is a descriptive diagnostic on benchmark scores. It does not
52
+ > measure student learning. With few models, correlations are fragile, so read
53
+ > them together with the leave-one-out range.
54
+
55
+ ![Solve versus Teach on the public MathTutorBench leaderboard](examples/paper_scatter.svg)
56
+
57
+ ## Install
58
+
59
+ Python 3.10 or newer. No dependencies.
60
+
61
+ ```bash
62
+ python -m pip install teaching-over-solving
63
+ ```
64
+
65
+ This installs the `tos-diagnostic` command (`teaching-over-solving` also works)
66
+ and the `tos_diagnostic` Python package. To work from source:
67
+
68
+ ```bash
69
+ git clone https://github.com/Garyouki/teaching-over-solving.git
70
+ cd teaching-over-solving
71
+ python -m pip install -e ".[dev]"
72
+ python -m pytest
73
+ ```
74
+
75
+ ## Reproduce the paper
76
+
77
+ ```bash
78
+ tos-diagnostic paper --report report.md --svg scatter.svg
79
+ ```
80
+
81
+ This uses the eight public MathTutorBench leaderboard models bundled with the
82
+ package. It reproduces the paper's Table 1 composites, the Spearman correlation
83
+ (0.690), the leave-one-out Pearson range (0.251–0.887), and the two rank shifts
84
+ the paper describes:
85
+
86
+ - LLaMA3.1-8B-Instruct rises from rank 7 on solving to rank 4 on teaching
87
+ (gap +0.077).
88
+ - Qwen2.5-Math-7B-Instruct falls from rank 4 to rank 8 (gap −0.452).
89
+
90
+ The exact Pearson correlation is 0.4217. The paper prints it as 0.421.
91
+
92
+ [examples/paper_report.md](examples/paper_report.md) shows the full report.
93
+
94
+ ## Diagnose your own tutor
95
+
96
+ ### MathTutorBench scores
97
+
98
+ Put one row per model in a CSV with the nine MathTutorBench task scores. The
99
+ leaderboard's own column headers (for example `Problem Solving` or
100
+ `Scaffolding (Hard)`) are recognized.
101
+
102
+ ```bash
103
+ tos-diagnostic score my_models.csv --reference --report report.md --svg scatter.svg
104
+ ```
105
+
106
+ `--reference` places your models next to the eight public models. Scores can
107
+ be fractions (`0.64`) or percentages (`64%`).
108
+
109
+ | Group | MathTutorBench tasks |
110
+ |---|---|
111
+ | Solve | problem solving, solution correctness, mistake location, mistake correction |
112
+ | Teach | Socratic questioning, scaffolding win rate, pedagogy instruction-following win rate, scaffolding (hard), pedagogy instruction-following (hard) |
113
+
114
+ The paper treats mistake location and mistake correction as solving tasks,
115
+ because they test whether the model can find or repair the mathematical state
116
+ of the problem.
117
+
118
+ ### Any other benchmark
119
+
120
+ Name the columns for each side:
121
+
122
+ ```bash
123
+ tos-diagnostic score scores.csv --solve accuracy,final_answer \
124
+ --teach scaffolding,guiding_questions,rigor
125
+ ```
126
+
127
+ Use this for any benchmark that reports separate solving and teaching scores
128
+ per model, including your own internal evaluations.
129
+
130
+ Missing scores stop the run by default. `--allow-missing` averages the scores
131
+ that are present and marks those models in the report.
132
+
133
+ ### Python
134
+
135
+ ```python
136
+ from tos_diagnostic import diagnose, load_scores, with_reference
137
+
138
+ result = diagnose(with_reference(load_scores("my_models.csv")))
139
+ for m in result["models"]:
140
+ print(m["model"], round(m["gap"], 3), m["solve_rank"], "->", m["teach_rank"])
141
+ ```
142
+
143
+ ## Rubric agency density
144
+
145
+ The paper's second study counts how often tutoring-benchmark rubrics reward
146
+ agency-preserving behavior, such as asking guiding questions or giving hints
147
+ without revealing the next step. `tos-diagnostic rubrics` computes these
148
+ densities from rubric items you have tagged:
149
+
150
+ ```bash
151
+ tos-diagnostic rubrics examples/example_rubrics.jsonl
152
+ ```
153
+
154
+ The keyword rules are transparent and replaceable. Reading TutorBench's raw
155
+ rubric format, and matching the paper's Table 2 exactly, is not yet
156
+ re-validated in this package. See [docs/rubric_rules.md](docs/rubric_rules.md).
157
+
158
+ ## Limitations
159
+
160
+ - Benchmark scores are not learning outcomes. A positive gap does not show that
161
+ a tutor helps students learn more.
162
+ - The public leaderboard has only eight models. Removing one model moves the
163
+ Pearson correlation between 0.251 and 0.887.
164
+ - Other task groupings are possible. The default grouping is a transparent
165
+ choice, not a definitive taxonomy.
166
+ - The diagnostic works on aggregate scores. It does not check answer disclosure
167
+ turn by turn.
168
+
169
+ ## Data
170
+
171
+ `src/tos_diagnostic/data/mathtutorbench_leaderboard.csv` contains the public
172
+ MathTutorBench leaderboard scores for eight models
173
+ ([Macina et al., EMNLP 2025](https://eth-lre.github.io/mathtutorbench/)). Please
174
+ cite MathTutorBench when you use them.
175
+
176
+ ## Citation
177
+
178
+ ```bibtex
179
+ @inproceedings{yao2026beyond,
180
+ title = {Beyond Helpfulness: A Teaching-over-Solving Diagnostic for Measuring Educational Impact in {LLM} Tutors},
181
+ author = {Junyi Yao and Zihao Zheng and Baichuan Li},
182
+ booktitle = {5th Workshop on NLP for Positive Impact},
183
+ year = {2026},
184
+ url = {https://openreview.net/forum?id=aZ1KTkxECP}
185
+ }
186
+ ```
187
+
188
+ ## License
189
+
190
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,164 @@
1
+ # Teaching-over-Solving Diagnostic
2
+
3
+ A small tool for one question: **is an LLM tutor as good at teaching as it is
4
+ at solving?**
5
+
6
+ A model that solves math problems well can still be a weak tutor. It may give
7
+ away the answer, skip scaffolding, or ignore what the student already knows.
8
+ If a benchmark folds everything into one "helpfulness" score, that difference
9
+ disappears.
10
+
11
+ This diagnostic keeps the two apart. From per-task benchmark scores it computes:
12
+
13
+ - **Solve:** the mean of solving-oriented tasks;
14
+ - **Teach:** the mean of pedagogy-oriented tasks;
15
+ - **Gap = Teach − Solve:** positive means relatively stronger at teaching.
16
+
17
+ Across a set of models it also reports how far each model's rank moves between
18
+ the two views, the Pearson and Spearman correlations between Solve and Teach,
19
+ and a leave-one-out range that shows how fragile those correlations are.
20
+
21
+ This is the companion tool for *Beyond Helpfulness: A Teaching-over-Solving
22
+ Diagnostic for Measuring Educational Impact in LLM Tutors* (5th Workshop on NLP
23
+ for Positive Impact, 2026, [OpenReview](https://openreview.net/forum?id=aZ1KTkxECP)).
24
+
25
+ > **Scope.** This is a descriptive diagnostic on benchmark scores. It does not
26
+ > measure student learning. With few models, correlations are fragile, so read
27
+ > them together with the leave-one-out range.
28
+
29
+ ![Solve versus Teach on the public MathTutorBench leaderboard](examples/paper_scatter.svg)
30
+
31
+ ## Install
32
+
33
+ Python 3.10 or newer. No dependencies.
34
+
35
+ ```bash
36
+ python -m pip install teaching-over-solving
37
+ ```
38
+
39
+ This installs the `tos-diagnostic` command (`teaching-over-solving` also works)
40
+ and the `tos_diagnostic` Python package. To work from source:
41
+
42
+ ```bash
43
+ git clone https://github.com/Garyouki/teaching-over-solving.git
44
+ cd teaching-over-solving
45
+ python -m pip install -e ".[dev]"
46
+ python -m pytest
47
+ ```
48
+
49
+ ## Reproduce the paper
50
+
51
+ ```bash
52
+ tos-diagnostic paper --report report.md --svg scatter.svg
53
+ ```
54
+
55
+ This uses the eight public MathTutorBench leaderboard models bundled with the
56
+ package. It reproduces the paper's Table 1 composites, the Spearman correlation
57
+ (0.690), the leave-one-out Pearson range (0.251–0.887), and the two rank shifts
58
+ the paper describes:
59
+
60
+ - LLaMA3.1-8B-Instruct rises from rank 7 on solving to rank 4 on teaching
61
+ (gap +0.077).
62
+ - Qwen2.5-Math-7B-Instruct falls from rank 4 to rank 8 (gap −0.452).
63
+
64
+ The exact Pearson correlation is 0.4217. The paper prints it as 0.421.
65
+
66
+ [examples/paper_report.md](examples/paper_report.md) shows the full report.
67
+
68
+ ## Diagnose your own tutor
69
+
70
+ ### MathTutorBench scores
71
+
72
+ Put one row per model in a CSV with the nine MathTutorBench task scores. The
73
+ leaderboard's own column headers (for example `Problem Solving` or
74
+ `Scaffolding (Hard)`) are recognized.
75
+
76
+ ```bash
77
+ tos-diagnostic score my_models.csv --reference --report report.md --svg scatter.svg
78
+ ```
79
+
80
+ `--reference` places your models next to the eight public models. Scores can
81
+ be fractions (`0.64`) or percentages (`64%`).
82
+
83
+ | Group | MathTutorBench tasks |
84
+ |---|---|
85
+ | Solve | problem solving, solution correctness, mistake location, mistake correction |
86
+ | Teach | Socratic questioning, scaffolding win rate, pedagogy instruction-following win rate, scaffolding (hard), pedagogy instruction-following (hard) |
87
+
88
+ The paper treats mistake location and mistake correction as solving tasks,
89
+ because they test whether the model can find or repair the mathematical state
90
+ of the problem.
91
+
92
+ ### Any other benchmark
93
+
94
+ Name the columns for each side:
95
+
96
+ ```bash
97
+ tos-diagnostic score scores.csv --solve accuracy,final_answer \
98
+ --teach scaffolding,guiding_questions,rigor
99
+ ```
100
+
101
+ Use this for any benchmark that reports separate solving and teaching scores
102
+ per model, including your own internal evaluations.
103
+
104
+ Missing scores stop the run by default. `--allow-missing` averages the scores
105
+ that are present and marks those models in the report.
106
+
107
+ ### Python
108
+
109
+ ```python
110
+ from tos_diagnostic import diagnose, load_scores, with_reference
111
+
112
+ result = diagnose(with_reference(load_scores("my_models.csv")))
113
+ for m in result["models"]:
114
+ print(m["model"], round(m["gap"], 3), m["solve_rank"], "->", m["teach_rank"])
115
+ ```
116
+
117
+ ## Rubric agency density
118
+
119
+ The paper's second study counts how often tutoring-benchmark rubrics reward
120
+ agency-preserving behavior, such as asking guiding questions or giving hints
121
+ without revealing the next step. `tos-diagnostic rubrics` computes these
122
+ densities from rubric items you have tagged:
123
+
124
+ ```bash
125
+ tos-diagnostic rubrics examples/example_rubrics.jsonl
126
+ ```
127
+
128
+ The keyword rules are transparent and replaceable. Reading TutorBench's raw
129
+ rubric format, and matching the paper's Table 2 exactly, is not yet
130
+ re-validated in this package. See [docs/rubric_rules.md](docs/rubric_rules.md).
131
+
132
+ ## Limitations
133
+
134
+ - Benchmark scores are not learning outcomes. A positive gap does not show that
135
+ a tutor helps students learn more.
136
+ - The public leaderboard has only eight models. Removing one model moves the
137
+ Pearson correlation between 0.251 and 0.887.
138
+ - Other task groupings are possible. The default grouping is a transparent
139
+ choice, not a definitive taxonomy.
140
+ - The diagnostic works on aggregate scores. It does not check answer disclosure
141
+ turn by turn.
142
+
143
+ ## Data
144
+
145
+ `src/tos_diagnostic/data/mathtutorbench_leaderboard.csv` contains the public
146
+ MathTutorBench leaderboard scores for eight models
147
+ ([Macina et al., EMNLP 2025](https://eth-lre.github.io/mathtutorbench/)). Please
148
+ cite MathTutorBench when you use them.
149
+
150
+ ## Citation
151
+
152
+ ```bibtex
153
+ @inproceedings{yao2026beyond,
154
+ title = {Beyond Helpfulness: A Teaching-over-Solving Diagnostic for Measuring Educational Impact in {LLM} Tutors},
155
+ author = {Junyi Yao and Zihao Zheng and Baichuan Li},
156
+ booktitle = {5th Workshop on NLP for Positive Impact},
157
+ year = {2026},
158
+ url = {https://openreview.net/forum?id=aZ1KTkxECP}
159
+ }
160
+ ```
161
+
162
+ ## License
163
+
164
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,51 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "teaching-over-solving"
7
+ version = "0.1.0"
8
+ description = "Teaching-over-solving diagnostic: separate how well an LLM tutor solves from how well it teaches."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [
14
+ { name = "Junyi Yao" },
15
+ { name = "Zihao Zheng" },
16
+ { name = "Baichuan Li" },
17
+ ]
18
+ keywords = ["llm-tutors", "education", "evaluation", "pedagogy", "benchmark"]
19
+ classifiers = [
20
+ "Development Status :: 3 - Alpha",
21
+ "Intended Audience :: Science/Research",
22
+ "Intended Audience :: Education",
23
+ "Programming Language :: Python :: 3",
24
+ "Programming Language :: Python :: 3.10",
25
+ "Programming Language :: Python :: 3.11",
26
+ "Programming Language :: Python :: 3.12",
27
+ "Programming Language :: Python :: 3.13",
28
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
29
+ "Topic :: Education",
30
+ ]
31
+ dependencies = []
32
+
33
+ [project.optional-dependencies]
34
+ dev = ["pytest>=8"]
35
+
36
+ [project.scripts]
37
+ teaching-over-solving = "tos_diagnostic.cli:main"
38
+ tos-diagnostic = "tos_diagnostic.cli:main"
39
+
40
+ [project.urls]
41
+ Repository = "https://github.com/Garyouki/teaching-over-solving"
42
+ Paper = "https://openreview.net/forum?id=aZ1KTkxECP"
43
+
44
+ [tool.setuptools.packages.find]
45
+ where = ["src"]
46
+
47
+ [tool.setuptools.package-data]
48
+ tos_diagnostic = ["data/*.csv", "data/*.json"]
49
+
50
+ [tool.pytest.ini_options]
51
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,190 @@
1
+ Metadata-Version: 2.4
2
+ Name: teaching-over-solving
3
+ Version: 0.1.0
4
+ Summary: Teaching-over-solving diagnostic: separate how well an LLM tutor solves from how well it teaches.
5
+ Author: Junyi Yao, Zihao Zheng, Baichuan Li
6
+ License-Expression: MIT
7
+ Project-URL: Repository, https://github.com/Garyouki/teaching-over-solving
8
+ Project-URL: Paper, https://openreview.net/forum?id=aZ1KTkxECP
9
+ Keywords: llm-tutors,education,evaluation,pedagogy,benchmark
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Intended Audience :: Education
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Education
20
+ Requires-Python: >=3.10
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Provides-Extra: dev
24
+ Requires-Dist: pytest>=8; extra == "dev"
25
+ Dynamic: license-file
26
+
27
+ # Teaching-over-Solving Diagnostic
28
+
29
+ A small tool for one question: **is an LLM tutor as good at teaching as it is
30
+ at solving?**
31
+
32
+ A model that solves math problems well can still be a weak tutor. It may give
33
+ away the answer, skip scaffolding, or ignore what the student already knows.
34
+ If a benchmark folds everything into one "helpfulness" score, that difference
35
+ disappears.
36
+
37
+ This diagnostic keeps the two apart. From per-task benchmark scores it computes:
38
+
39
+ - **Solve:** the mean of solving-oriented tasks;
40
+ - **Teach:** the mean of pedagogy-oriented tasks;
41
+ - **Gap = Teach − Solve:** positive means relatively stronger at teaching.
42
+
43
+ Across a set of models it also reports how far each model's rank moves between
44
+ the two views, the Pearson and Spearman correlations between Solve and Teach,
45
+ and a leave-one-out range that shows how fragile those correlations are.
46
+
47
+ This is the companion tool for *Beyond Helpfulness: A Teaching-over-Solving
48
+ Diagnostic for Measuring Educational Impact in LLM Tutors* (5th Workshop on NLP
49
+ for Positive Impact, 2026, [OpenReview](https://openreview.net/forum?id=aZ1KTkxECP)).
50
+
51
+ > **Scope.** This is a descriptive diagnostic on benchmark scores. It does not
52
+ > measure student learning. With few models, correlations are fragile, so read
53
+ > them together with the leave-one-out range.
54
+
55
+ ![Solve versus Teach on the public MathTutorBench leaderboard](examples/paper_scatter.svg)
56
+
57
+ ## Install
58
+
59
+ Python 3.10 or newer. No dependencies.
60
+
61
+ ```bash
62
+ python -m pip install teaching-over-solving
63
+ ```
64
+
65
+ This installs the `tos-diagnostic` command (`teaching-over-solving` also works)
66
+ and the `tos_diagnostic` Python package. To work from source:
67
+
68
+ ```bash
69
+ git clone https://github.com/Garyouki/teaching-over-solving.git
70
+ cd teaching-over-solving
71
+ python -m pip install -e ".[dev]"
72
+ python -m pytest
73
+ ```
74
+
75
+ ## Reproduce the paper
76
+
77
+ ```bash
78
+ tos-diagnostic paper --report report.md --svg scatter.svg
79
+ ```
80
+
81
+ This uses the eight public MathTutorBench leaderboard models bundled with the
82
+ package. It reproduces the paper's Table 1 composites, the Spearman correlation
83
+ (0.690), the leave-one-out Pearson range (0.251–0.887), and the two rank shifts
84
+ the paper describes:
85
+
86
+ - LLaMA3.1-8B-Instruct rises from rank 7 on solving to rank 4 on teaching
87
+ (gap +0.077).
88
+ - Qwen2.5-Math-7B-Instruct falls from rank 4 to rank 8 (gap −0.452).
89
+
90
+ The exact Pearson correlation is 0.4217. The paper prints it as 0.421.
91
+
92
+ [examples/paper_report.md](examples/paper_report.md) shows the full report.
93
+
94
+ ## Diagnose your own tutor
95
+
96
+ ### MathTutorBench scores
97
+
98
+ Put one row per model in a CSV with the nine MathTutorBench task scores. The
99
+ leaderboard's own column headers (for example `Problem Solving` or
100
+ `Scaffolding (Hard)`) are recognized.
101
+
102
+ ```bash
103
+ tos-diagnostic score my_models.csv --reference --report report.md --svg scatter.svg
104
+ ```
105
+
106
+ `--reference` places your models next to the eight public models. Scores can
107
+ be fractions (`0.64`) or percentages (`64%`).
108
+
109
+ | Group | MathTutorBench tasks |
110
+ |---|---|
111
+ | Solve | problem solving, solution correctness, mistake location, mistake correction |
112
+ | Teach | Socratic questioning, scaffolding win rate, pedagogy instruction-following win rate, scaffolding (hard), pedagogy instruction-following (hard) |
113
+
114
+ The paper treats mistake location and mistake correction as solving tasks,
115
+ because they test whether the model can find or repair the mathematical state
116
+ of the problem.
117
+
118
+ ### Any other benchmark
119
+
120
+ Name the columns for each side:
121
+
122
+ ```bash
123
+ tos-diagnostic score scores.csv --solve accuracy,final_answer \
124
+ --teach scaffolding,guiding_questions,rigor
125
+ ```
126
+
127
+ Use this for any benchmark that reports separate solving and teaching scores
128
+ per model, including your own internal evaluations.
129
+
130
+ Missing scores stop the run by default. `--allow-missing` averages the scores
131
+ that are present and marks those models in the report.
132
+
133
+ ### Python
134
+
135
+ ```python
136
+ from tos_diagnostic import diagnose, load_scores, with_reference
137
+
138
+ result = diagnose(with_reference(load_scores("my_models.csv")))
139
+ for m in result["models"]:
140
+ print(m["model"], round(m["gap"], 3), m["solve_rank"], "->", m["teach_rank"])
141
+ ```
142
+
143
+ ## Rubric agency density
144
+
145
+ The paper's second study counts how often tutoring-benchmark rubrics reward
146
+ agency-preserving behavior, such as asking guiding questions or giving hints
147
+ without revealing the next step. `tos-diagnostic rubrics` computes these
148
+ densities from rubric items you have tagged:
149
+
150
+ ```bash
151
+ tos-diagnostic rubrics examples/example_rubrics.jsonl
152
+ ```
153
+
154
+ The keyword rules are transparent and replaceable. Reading TutorBench's raw
155
+ rubric format, and matching the paper's Table 2 exactly, is not yet
156
+ re-validated in this package. See [docs/rubric_rules.md](docs/rubric_rules.md).
157
+
158
+ ## Limitations
159
+
160
+ - Benchmark scores are not learning outcomes. A positive gap does not show that
161
+ a tutor helps students learn more.
162
+ - The public leaderboard has only eight models. Removing one model moves the
163
+ Pearson correlation between 0.251 and 0.887.
164
+ - Other task groupings are possible. The default grouping is a transparent
165
+ choice, not a definitive taxonomy.
166
+ - The diagnostic works on aggregate scores. It does not check answer disclosure
167
+ turn by turn.
168
+
169
+ ## Data
170
+
171
+ `src/tos_diagnostic/data/mathtutorbench_leaderboard.csv` contains the public
172
+ MathTutorBench leaderboard scores for eight models
173
+ ([Macina et al., EMNLP 2025](https://eth-lre.github.io/mathtutorbench/)). Please
174
+ cite MathTutorBench when you use them.
175
+
176
+ ## Citation
177
+
178
+ ```bibtex
179
+ @inproceedings{yao2026beyond,
180
+ title = {Beyond Helpfulness: A Teaching-over-Solving Diagnostic for Measuring Educational Impact in {LLM} Tutors},
181
+ author = {Junyi Yao and Zihao Zheng and Baichuan Li},
182
+ booktitle = {5th Workshop on NLP for Positive Impact},
183
+ year = {2026},
184
+ url = {https://openreview.net/forum?id=aZ1KTkxECP}
185
+ }
186
+ ```
187
+
188
+ ## License
189
+
190
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,20 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ src/teaching_over_solving.egg-info/PKG-INFO
5
+ src/teaching_over_solving.egg-info/SOURCES.txt
6
+ src/teaching_over_solving.egg-info/dependency_links.txt
7
+ src/teaching_over_solving.egg-info/entry_points.txt
8
+ src/teaching_over_solving.egg-info/requires.txt
9
+ src/teaching_over_solving.egg-info/top_level.txt
10
+ src/tos_diagnostic/__init__.py
11
+ src/tos_diagnostic/__main__.py
12
+ src/tos_diagnostic/cli.py
13
+ src/tos_diagnostic/diagnostic.py
14
+ src/tos_diagnostic/report.py
15
+ src/tos_diagnostic/rubrics.py
16
+ src/tos_diagnostic/data/mathtutorbench_leaderboard.csv
17
+ src/tos_diagnostic/data/mathtutorbench_leaderboard.json
18
+ tests/test_diagnostic.py
19
+ tests/test_paper.py
20
+ tests/test_rubrics_and_cli.py
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ teaching-over-solving = tos_diagnostic.cli:main
3
+ tos-diagnostic = tos_diagnostic.cli:main
@@ -0,0 +1,6 @@
1
+ """Teaching-over-solving diagnostic for LLM tutors."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ from .diagnostic import (MTB_SOLVE, MTB_TEACH, DiagnosticError, composites, # noqa: E402,F401
6
+ diagnose, load_scores, reference_scores, with_reference)
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ main()