sabermath 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sabermath/__init__.py +30 -0
- sabermath/__main__.py +66 -0
- sabermath/__version__.py +1 -0
- sabermath/benchmark.py +343 -0
- sabermath/data.py +11 -0
- sabermath/instructions.py +42 -0
- sabermath/metrics.py +48 -0
- sabermath/processors/__init__.py +6 -0
- sabermath/processors/base.py +23 -0
- sabermath/processors/embedding_processor.py +134 -0
- sabermath/processors/openrouter_processor.py +97 -0
- sabermath/processors/st_processor.py +128 -0
- sabermath/processors/unknown_processor.py +26 -0
- sabermath/processors/vllm_processor.py +283 -0
- sabermath/schemas.py +46 -0
- sabermath-1.0.0.dist-info/METADATA +200 -0
- sabermath-1.0.0.dist-info/RECORD +21 -0
- sabermath-1.0.0.dist-info/WHEEL +5 -0
- sabermath-1.0.0.dist-info/entry_points.txt +2 -0
- sabermath-1.0.0.dist-info/licenses/LICENCE +427 -0
- sabermath-1.0.0.dist-info/top_level.txt +1 -0
sabermath/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from .schemas import Branch, BranchResult, Report, Task, TaskResult
|
|
2
|
+
|
|
3
|
+
_LAZY = {
|
|
4
|
+
"evaluate": ".benchmark",
|
|
5
|
+
"EmbeddingProcessor": ".processors",
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"evaluate",
|
|
10
|
+
"Task",
|
|
11
|
+
"Report",
|
|
12
|
+
"TaskResult",
|
|
13
|
+
"Branch",
|
|
14
|
+
"BranchResult",
|
|
15
|
+
"EmbeddingProcessor",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def __getattr__(name: str):
|
|
20
|
+
if name not in _LAZY:
|
|
21
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
22
|
+
import importlib
|
|
23
|
+
|
|
24
|
+
value = getattr(importlib.import_module(_LAZY[name], __name__), name)
|
|
25
|
+
globals()[name] = value
|
|
26
|
+
return value
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def __dir__():
|
|
30
|
+
return sorted(__all__)
|
sabermath/__main__.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Command-line interface: `sabermath evaluate | download | help`."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from typing import get_args
|
|
6
|
+
|
|
7
|
+
from sabermath.schemas import Task
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _evaluate(args):
|
|
11
|
+
if args.vllm and args.device:
|
|
12
|
+
sys.exit("error: --device is not supported with --vllm")
|
|
13
|
+
|
|
14
|
+
from sabermath import evaluate
|
|
15
|
+
|
|
16
|
+
report = evaluate(
|
|
17
|
+
args.model,
|
|
18
|
+
tasks=args.task,
|
|
19
|
+
use_vllm=args.vllm,
|
|
20
|
+
init_kwargs={"device": args.device} if args.device else None,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
print(f"\n{report.model} ({report.processor}), nDCG@{report.k}")
|
|
24
|
+
for task in report.tasks:
|
|
25
|
+
print(f"\n{task.task:24}{task.ndcg_at_k:.4f}")
|
|
26
|
+
for branch in task.branches:
|
|
27
|
+
print(f" {branch.branch:22}{branch.ndcg_at_k:.4f}")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _download(args):
|
|
31
|
+
from sabermath.data import load_data
|
|
32
|
+
|
|
33
|
+
queries, documents = load_data()
|
|
34
|
+
print(f"Downloaded {len(queries)} queries and {len(documents)} documents.")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main(argv=None):
|
|
38
|
+
parser = argparse.ArgumentParser(
|
|
39
|
+
prog="sabermath", description="SABER-Math: a reranking benchmark for math retrieval."
|
|
40
|
+
)
|
|
41
|
+
sub = parser.add_subparsers(dest="command", metavar="<command>")
|
|
42
|
+
|
|
43
|
+
p = sub.add_parser("evaluate", help="Evaluate a model on the benchmark.")
|
|
44
|
+
p.add_argument("model", help="Hugging Face model id.")
|
|
45
|
+
p.add_argument("--task", action="append", choices=get_args(Task),
|
|
46
|
+
help="Task to run; repeat for several (default: all three).")
|
|
47
|
+
p.add_argument("--device", help="Device for Sentence Transformers, e.g. cuda or cpu.")
|
|
48
|
+
p.add_argument("--vllm", action="store_true", help="Load the model with vLLM.")
|
|
49
|
+
p.set_defaults(func=_evaluate)
|
|
50
|
+
|
|
51
|
+
p = sub.add_parser("download", help="Download the benchmark data into the cache.")
|
|
52
|
+
p.set_defaults(func=_download)
|
|
53
|
+
|
|
54
|
+
p = sub.add_parser("help", help="Show help for sabermath or one of its commands.")
|
|
55
|
+
p.add_argument("topic", nargs="?", choices=["evaluate", "download", "help"])
|
|
56
|
+
|
|
57
|
+
args = parser.parse_args(argv)
|
|
58
|
+
if args.command in (None, "help"):
|
|
59
|
+
topic = getattr(args, "topic", None)
|
|
60
|
+
(sub.choices[topic] if topic else parser).print_help()
|
|
61
|
+
return
|
|
62
|
+
args.func(args)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
if __name__ == "__main__":
|
|
66
|
+
main()
|
sabermath/__version__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.0"
|
sabermath/benchmark.py
ADDED
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Callable, Literal, get_args
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
from tqdm import tqdm
|
|
6
|
+
from datasets import Dataset
|
|
7
|
+
|
|
8
|
+
from sabermath.data import load_data
|
|
9
|
+
from sabermath.instructions import (
|
|
10
|
+
DEFAULT_INSTRUCTION_TEMPLATE,
|
|
11
|
+
format_instructed_query,
|
|
12
|
+
)
|
|
13
|
+
from sabermath.metrics import compute_ndcg_at_k
|
|
14
|
+
from sabermath.schemas import (
|
|
15
|
+
Task,
|
|
16
|
+
Report,
|
|
17
|
+
TaskResult,
|
|
18
|
+
Branch,
|
|
19
|
+
BranchResult,
|
|
20
|
+
DCGVariant,
|
|
21
|
+
)
|
|
22
|
+
from sabermath.processors import (
|
|
23
|
+
ModelProcessor,
|
|
24
|
+
SentenceTransformersProcessor as STProcessor,
|
|
25
|
+
VLLMProcessor,
|
|
26
|
+
UnknownProcessor,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
ALL_TASKS = get_args(Task)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _naive_type_check(obj, module: str, name: str) -> bool:
|
|
34
|
+
t = type(obj)
|
|
35
|
+
# Match the top-level package: the classes live in submodules
|
|
36
|
+
# (e.g. sentence_transformers.sentence_transformer.model, vllm.entrypoints.llm).
|
|
37
|
+
return t.__module__.split(".")[0] == module and t.__name__ == name
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _make_processor(model, use_vllm: bool, init_kwargs: dict):
|
|
41
|
+
if isinstance(model, str):
|
|
42
|
+
if use_vllm:
|
|
43
|
+
return VLLMProcessor.from_huggingface(model, **init_kwargs)
|
|
44
|
+
return STProcessor.from_huggingface(model, **init_kwargs)
|
|
45
|
+
|
|
46
|
+
if use_vllm:
|
|
47
|
+
raise RuntimeError(
|
|
48
|
+
"Flag 'use_vllm' is only valid when loading a "
|
|
49
|
+
"model from HuggingFace path"
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
if isinstance(model, ModelProcessor):
|
|
53
|
+
if init_kwargs:
|
|
54
|
+
raise RuntimeError(
|
|
55
|
+
"Processor has already been initialized, so "
|
|
56
|
+
"'init_kwargs' is invalid."
|
|
57
|
+
)
|
|
58
|
+
return model
|
|
59
|
+
|
|
60
|
+
if _naive_type_check(model, "vllm", "LLM"):
|
|
61
|
+
return VLLMProcessor(model, **init_kwargs)
|
|
62
|
+
|
|
63
|
+
if _naive_type_check(model, "sentence_transformers", "SentenceTransformer"):
|
|
64
|
+
return STProcessor(model, **init_kwargs)
|
|
65
|
+
|
|
66
|
+
return UnknownProcessor(model, **init_kwargs)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _get_branch_row_idxs(
|
|
70
|
+
targets: Dataset,
|
|
71
|
+
branch: Branch,
|
|
72
|
+
) -> list[int]:
|
|
73
|
+
branches_all = targets["domains"]
|
|
74
|
+
return [i for i, branches in enumerate(branches_all) if branch in branches]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class TaskSettings:
|
|
79
|
+
queries_ds: Dataset
|
|
80
|
+
documents_ds: Dataset
|
|
81
|
+
processor: ModelProcessor
|
|
82
|
+
k: int
|
|
83
|
+
dcg_variant: Literal["linear", "exponent"] | None
|
|
84
|
+
show_progress_bar: bool
|
|
85
|
+
scores_kwargs: dict
|
|
86
|
+
query_instruction: str | None = None
|
|
87
|
+
instruction_template: str = DEFAULT_INSTRUCTION_TEMPLATE
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def evaluate_task(
|
|
91
|
+
name: Task,
|
|
92
|
+
settings: TaskSettings,
|
|
93
|
+
return_ndcgs: bool = False,
|
|
94
|
+
on_progress: Callable[[], None] | None = None,
|
|
95
|
+
) -> TaskResult | tuple[TaskResult, list[float | None]]:
|
|
96
|
+
queries_ds = settings.queries_ds
|
|
97
|
+
documents_ds = settings.documents_ds
|
|
98
|
+
processor = settings.processor
|
|
99
|
+
k = settings.k
|
|
100
|
+
dcg_variant = settings.dcg_variant
|
|
101
|
+
show_progress_bar = settings.show_progress_bar
|
|
102
|
+
scores_kwargs = settings.scores_kwargs
|
|
103
|
+
|
|
104
|
+
query_version = "full" if name == "full-full" else "statement"
|
|
105
|
+
document_version = "statement" if name == "statement-statement" else "full"
|
|
106
|
+
|
|
107
|
+
queries = transform(queries_ds, query_version)
|
|
108
|
+
|
|
109
|
+
if settings.query_instruction is not None:
|
|
110
|
+
queries = [
|
|
111
|
+
format_instructed_query(
|
|
112
|
+
settings.query_instruction, q, settings.instruction_template
|
|
113
|
+
)
|
|
114
|
+
for q in queries
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
all_ndcgs_by_idx: dict[int, float | None] = {}
|
|
118
|
+
ndcg_by_query_idx: dict[int, float] = {}
|
|
119
|
+
|
|
120
|
+
for i, query in tqdm(
|
|
121
|
+
enumerate(queries),
|
|
122
|
+
total=len(queries),
|
|
123
|
+
disable=not show_progress_bar,
|
|
124
|
+
):
|
|
125
|
+
doc_ids = list(queries_ds[i]["candidates"])
|
|
126
|
+
relevance_scores = list(queries_ds[i]["relevance_scores"])
|
|
127
|
+
|
|
128
|
+
documents = transform(documents_ds, document_version, doc_ids)
|
|
129
|
+
model_scores = processor.get_scores(
|
|
130
|
+
query,
|
|
131
|
+
documents,
|
|
132
|
+
show_progress_bar=False,
|
|
133
|
+
**scores_kwargs,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
if model_scores is None:
|
|
137
|
+
all_ndcgs_by_idx[i] = None
|
|
138
|
+
else:
|
|
139
|
+
model_ranked_local_idxs = np.argsort(-np.asarray(model_scores))
|
|
140
|
+
|
|
141
|
+
model_ranked_relevance_scores = np.asarray(relevance_scores)[
|
|
142
|
+
model_ranked_local_idxs
|
|
143
|
+
]
|
|
144
|
+
ndcg = compute_ndcg_at_k(
|
|
145
|
+
model_ranked_relevance_scores, k=k, variant=dcg_variant
|
|
146
|
+
)
|
|
147
|
+
all_ndcgs_by_idx[i] = ndcg
|
|
148
|
+
ndcg_by_query_idx[i] = ndcg
|
|
149
|
+
|
|
150
|
+
if on_progress is not None:
|
|
151
|
+
on_progress()
|
|
152
|
+
|
|
153
|
+
all_ndcgs = [all_ndcgs_by_idx.get(i) for i in range(len(queries))]
|
|
154
|
+
ndcgs = np.asarray(list(ndcg_by_query_idx.values()))
|
|
155
|
+
ndcg_at_k = float(ndcgs.mean()) if ndcgs.size > 0 else 0.0
|
|
156
|
+
|
|
157
|
+
BRANCHES = get_args(Branch)
|
|
158
|
+
branch_results: list[BranchResult] = []
|
|
159
|
+
|
|
160
|
+
for branch in BRANCHES:
|
|
161
|
+
idxs = _get_branch_row_idxs(queries_ds, branch)
|
|
162
|
+
branch_ndcgs = [
|
|
163
|
+
ndcg_by_query_idx[idx] for idx in idxs if idx in ndcg_by_query_idx
|
|
164
|
+
]
|
|
165
|
+
branch_ndcg_at_k = (
|
|
166
|
+
float(np.mean(branch_ndcgs)) if len(branch_ndcgs) > 0 else 0.0
|
|
167
|
+
)
|
|
168
|
+
br = BranchResult(branch, branch_ndcg_at_k)
|
|
169
|
+
branch_results.append(br)
|
|
170
|
+
|
|
171
|
+
report = TaskResult(
|
|
172
|
+
name,
|
|
173
|
+
ndcg_at_k,
|
|
174
|
+
branch_results,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
if return_ndcgs:
|
|
178
|
+
return report, all_ndcgs
|
|
179
|
+
else:
|
|
180
|
+
return report
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def transform(
|
|
184
|
+
ds: Dataset,
|
|
185
|
+
version: Literal["statement", "full"],
|
|
186
|
+
idxs: list[int] | None = None,
|
|
187
|
+
) -> list[str]:
|
|
188
|
+
if idxs is not None:
|
|
189
|
+
ds = ds.select(idxs)
|
|
190
|
+
|
|
191
|
+
if version == "full":
|
|
192
|
+
statements = list(ds["problem"])
|
|
193
|
+
solutions = list(ds["solution"])
|
|
194
|
+
|
|
195
|
+
return [
|
|
196
|
+
f"Problem: {statement}\n\nSolution: {solution}"
|
|
197
|
+
for statement, solution in zip(statements, solutions)
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
return list(ds["problem"])
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def evaluate(
|
|
204
|
+
model,
|
|
205
|
+
tasks: list[Task] | None = None,
|
|
206
|
+
k: int = 10,
|
|
207
|
+
*,
|
|
208
|
+
dcg_variant: DCGVariant = "exponent",
|
|
209
|
+
use_vllm: bool = False,
|
|
210
|
+
init_kwargs: dict | None = None,
|
|
211
|
+
scores_kwargs: dict | None = None,
|
|
212
|
+
on_progress: Callable[[], None] | None = None,
|
|
213
|
+
instruction: str | None = None,
|
|
214
|
+
instruction_template: str = DEFAULT_INSTRUCTION_TEMPLATE,
|
|
215
|
+
verbose: bool = True,
|
|
216
|
+
show_progress_bars: bool = True,
|
|
217
|
+
# Below here: development arguments, not part of the published protocol.
|
|
218
|
+
queries: Dataset | None = None,
|
|
219
|
+
documents: Dataset | None = None,
|
|
220
|
+
return_ndcgs: bool = False,
|
|
221
|
+
) -> Report | tuple[Report, dict[Task, list[float | None]]]:
|
|
222
|
+
if tasks is None:
|
|
223
|
+
tasks = ALL_TASKS
|
|
224
|
+
|
|
225
|
+
for task in tasks:
|
|
226
|
+
if task not in ALL_TASKS:
|
|
227
|
+
raise ValueError(f"Invalid task: {task}")
|
|
228
|
+
|
|
229
|
+
if dcg_variant not in get_args(DCGVariant):
|
|
230
|
+
raise ValueError(f"Invalid DCG variant: {dcg_variant}")
|
|
231
|
+
|
|
232
|
+
def vprint(text: str) -> None:
|
|
233
|
+
if verbose:
|
|
234
|
+
print(text)
|
|
235
|
+
|
|
236
|
+
init_kwargs = init_kwargs or {}
|
|
237
|
+
scores_kwargs = scores_kwargs or {}
|
|
238
|
+
|
|
239
|
+
vprint("[~] Loading model...")
|
|
240
|
+
|
|
241
|
+
processor = _make_processor(model, use_vllm, init_kwargs)
|
|
242
|
+
|
|
243
|
+
vprint("[+] Model loaded.")
|
|
244
|
+
|
|
245
|
+
task_results = []
|
|
246
|
+
|
|
247
|
+
tasks = set(tasks) & set(ALL_TASKS)
|
|
248
|
+
|
|
249
|
+
if queries is None or documents is None:
|
|
250
|
+
vprint(
|
|
251
|
+
f"[~] Loading data for task{'' if len(tasks) == 1 else 's'} "
|
|
252
|
+
f"\"{','.join(tasks)}\"..."
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
queries, documents = load_data()
|
|
256
|
+
|
|
257
|
+
vprint(f"[+] Loaded {len(queries)} queries.")
|
|
258
|
+
|
|
259
|
+
query_instruction = None
|
|
260
|
+
|
|
261
|
+
if instruction is not None:
|
|
262
|
+
vprint(
|
|
263
|
+
f"[~] Setting instruction ({instruction_template} template): "
|
|
264
|
+
f"{instruction}"
|
|
265
|
+
)
|
|
266
|
+
# The instruction wraps the QUERY TEXT at scoring time (see
|
|
267
|
+
# TaskSettings.query_instruction), never the dataset's problem column:
|
|
268
|
+
# the column route mutated the input before the task transform, so the
|
|
269
|
+
# same prompt produced different strings per task.
|
|
270
|
+
query_instruction = instruction
|
|
271
|
+
vprint("[+] Instruction set.")
|
|
272
|
+
|
|
273
|
+
settings = TaskSettings(
|
|
274
|
+
queries_ds=queries,
|
|
275
|
+
documents_ds=documents,
|
|
276
|
+
processor=processor,
|
|
277
|
+
k=k,
|
|
278
|
+
dcg_variant=dcg_variant,
|
|
279
|
+
show_progress_bar=show_progress_bars,
|
|
280
|
+
scores_kwargs=scores_kwargs,
|
|
281
|
+
query_instruction=query_instruction,
|
|
282
|
+
instruction_template=instruction_template,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
ndcgs = {}
|
|
286
|
+
|
|
287
|
+
if "statement-statement" in tasks:
|
|
288
|
+
vprint('[~] Evaluating on task "statement vs. statement"...')
|
|
289
|
+
st_st_res, st_st_ndcgs = evaluate_task(
|
|
290
|
+
"statement-statement",
|
|
291
|
+
settings,
|
|
292
|
+
True,
|
|
293
|
+
on_progress=on_progress,
|
|
294
|
+
)
|
|
295
|
+
ndcgs["statement-statement"] = st_st_ndcgs
|
|
296
|
+
ndcg_at_k = st_st_res.ndcg_at_k
|
|
297
|
+
vprint(f"[+] Statement-statement nDCG@{k} ({dcg_variant}): {ndcg_at_k}")
|
|
298
|
+
task_results.append(st_st_res)
|
|
299
|
+
|
|
300
|
+
if "statement-full" in tasks:
|
|
301
|
+
vprint('[~] Evaluating on task "statement vs. full statement + solution"...')
|
|
302
|
+
st_fl_res, st_fl_ndcgs = evaluate_task(
|
|
303
|
+
"statement-full",
|
|
304
|
+
settings,
|
|
305
|
+
True,
|
|
306
|
+
on_progress=on_progress,
|
|
307
|
+
)
|
|
308
|
+
ndcgs["statement-full"] = st_fl_ndcgs
|
|
309
|
+
ndcg_at_k = st_fl_res.ndcg_at_k
|
|
310
|
+
vprint(f"[+] Statement-full nDCG@{k} ({dcg_variant}): {ndcg_at_k}")
|
|
311
|
+
task_results.append(st_fl_res)
|
|
312
|
+
|
|
313
|
+
if "full-full" in tasks:
|
|
314
|
+
vprint(
|
|
315
|
+
'[~] Evaluating on task "full problem + solution vs. '
|
|
316
|
+
'full problem + solution"...'
|
|
317
|
+
)
|
|
318
|
+
fl_fl_res, fl_fl_ndcgs = evaluate_task(
|
|
319
|
+
"full-full",
|
|
320
|
+
settings,
|
|
321
|
+
True,
|
|
322
|
+
on_progress=on_progress,
|
|
323
|
+
)
|
|
324
|
+
ndcgs["full-full"] = fl_fl_ndcgs
|
|
325
|
+
ndcg_at_k = fl_fl_res.ndcg_at_k
|
|
326
|
+
vprint(f"[+] Full-full nDCG@{k} ({dcg_variant}): {ndcg_at_k}")
|
|
327
|
+
task_results.append(fl_fl_res)
|
|
328
|
+
|
|
329
|
+
del queries
|
|
330
|
+
del documents
|
|
331
|
+
|
|
332
|
+
report = Report(
|
|
333
|
+
model=processor.model or "unknown",
|
|
334
|
+
processor=processor.processor or "unknown",
|
|
335
|
+
dcg_variant=dcg_variant,
|
|
336
|
+
k=k,
|
|
337
|
+
tasks=task_results,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
if return_ndcgs:
|
|
341
|
+
return report, ndcgs
|
|
342
|
+
else:
|
|
343
|
+
return report
|
sabermath/data.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from datasets import load_dataset, Dataset
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
TARGETS_DATASET = "INSAIT-Institute/SaberMath-queries"
|
|
5
|
+
CANDIDATES_DATASET = "INSAIT-Institute/SaberMath-documents"
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_data() -> tuple[Dataset, Dataset]:
|
|
9
|
+
targets = load_dataset(TARGETS_DATASET, split="train")
|
|
10
|
+
candidates = load_dataset(CANDIDATES_DATASET, split="train")
|
|
11
|
+
return targets, candidates
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
INSTRUCTIONS: dict[str, str | None] = {
|
|
2
|
+
"p0": None,
|
|
3
|
+
"p1": (
|
|
4
|
+
"You are an expert mathematical retrieval engine. Your job is to "
|
|
5
|
+
"retrieve sample problems that are *conceptually close* to a target "
|
|
6
|
+
"problem, judged by the *mathematical ideas and solution techniques "
|
|
7
|
+
"actually used* (not surface wording)."
|
|
8
|
+
),
|
|
9
|
+
"p2": (
|
|
10
|
+
"Given a mathematical query, retrieve documents that share identical "
|
|
11
|
+
"problem-solving structure and logical steps, even if they use "
|
|
12
|
+
"entirely different variables, symbols, or natural language phrasing."
|
|
13
|
+
),
|
|
14
|
+
"p3": "Find the most relevant document based on the query.",
|
|
15
|
+
"pm": (
|
|
16
|
+
"Given a math problem query, retrieve documents that are "
|
|
17
|
+
"mathematically relevant to the query"
|
|
18
|
+
),
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
INSTRUCTION_KEYS = tuple(INSTRUCTIONS)
|
|
22
|
+
|
|
23
|
+
INSTRUCTION_TEMPLATES: dict[str, str] = {
|
|
24
|
+
"canonical": "Instruct: {instruction}\nQuery: {query}",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
DEFAULT_INSTRUCTION_TEMPLATE = "canonical"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def format_instructed_query(
|
|
31
|
+
instruction: str,
|
|
32
|
+
problem: str,
|
|
33
|
+
template: str = DEFAULT_INSTRUCTION_TEMPLATE,
|
|
34
|
+
) -> str:
|
|
35
|
+
try:
|
|
36
|
+
pattern = INSTRUCTION_TEMPLATES[template]
|
|
37
|
+
except KeyError:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
f"Unknown instruction template '{template}' - valid: "
|
|
40
|
+
f"{', '.join(INSTRUCTION_TEMPLATES)}"
|
|
41
|
+
) from None
|
|
42
|
+
return pattern.format(instruction=instruction, query=problem)
|
sabermath/metrics.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from typing import Literal, Sequence
|
|
2
|
+
import numpy as np
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def cosine_similarity(query_emb: np.ndarray, docs_embs: np.ndarray) -> np.ndarray:
|
|
6
|
+
q_norm = np.linalg.norm(query_emb)
|
|
7
|
+
d_norms = np.linalg.norm(docs_embs, axis=1, keepdims=True)
|
|
8
|
+
|
|
9
|
+
if q_norm == 0:
|
|
10
|
+
raise ValueError("Query embedding has zero norm")
|
|
11
|
+
|
|
12
|
+
if np.any(d_norms == 0):
|
|
13
|
+
raise ValueError("At least one document embedding has zero norm")
|
|
14
|
+
|
|
15
|
+
query_emb = query_emb / q_norm
|
|
16
|
+
docs_embs = docs_embs / d_norms
|
|
17
|
+
return docs_embs @ query_emb
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def dcg_at_k(
|
|
21
|
+
relevances: Sequence[float],
|
|
22
|
+
k: int = 10,
|
|
23
|
+
variant: Literal["linear", "exponent"] = "linear",
|
|
24
|
+
) -> float:
|
|
25
|
+
relevances = np.asarray(relevances[:k], dtype=float)
|
|
26
|
+
|
|
27
|
+
if relevances.size == 0:
|
|
28
|
+
return 0.0
|
|
29
|
+
discounts = 1.0 / np.log2(np.arange(2, relevances.size + 2))
|
|
30
|
+
|
|
31
|
+
if variant == "linear":
|
|
32
|
+
gains = relevances
|
|
33
|
+
elif variant == "exponent":
|
|
34
|
+
gains = np.power(2, relevances) - 1
|
|
35
|
+
else:
|
|
36
|
+
raise ValueError("variant must be either 'linear' or 'exponent'")
|
|
37
|
+
|
|
38
|
+
return float(np.sum(gains * discounts))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def compute_ndcg_at_k(
|
|
42
|
+
relevances: Sequence[float],
|
|
43
|
+
k: int = 10,
|
|
44
|
+
variant: Literal["linear", "exponent"] = "linear",
|
|
45
|
+
) -> float:
|
|
46
|
+
dcg = dcg_at_k(relevances, k, variant)
|
|
47
|
+
idcg = dcg_at_k(sorted(relevances, reverse=True), k, variant)
|
|
48
|
+
return 0.0 if idcg == 0 else dcg / idcg
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from .base import ModelProcessor
|
|
2
|
+
from .embedding_processor import EmbeddingProcessor
|
|
3
|
+
from .st_processor import SentenceTransformersProcessor
|
|
4
|
+
from .vllm_processor import VLLMProcessor
|
|
5
|
+
from .openrouter_processor import OpenRouterEmbeddingProcessor
|
|
6
|
+
from .unknown_processor import UnknownProcessor
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import ClassVar
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ModelProcessor(ABC):
|
|
8
|
+
processor: ClassVar[str | None]
|
|
9
|
+
|
|
10
|
+
@property
|
|
11
|
+
def model(self) -> str | None:
|
|
12
|
+
return None
|
|
13
|
+
|
|
14
|
+
@abstractmethod
|
|
15
|
+
def get_scores(
|
|
16
|
+
self,
|
|
17
|
+
query: str,
|
|
18
|
+
documents: list[str],
|
|
19
|
+
*,
|
|
20
|
+
show_progress_bar: bool = True,
|
|
21
|
+
**kwargs,
|
|
22
|
+
) -> list[float] | np.ndarray | None:
|
|
23
|
+
pass
|