sabermath 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sabermath/__init__.py ADDED
@@ -0,0 +1,30 @@
1
+ from .schemas import Branch, BranchResult, Report, Task, TaskResult
2
+
3
+ _LAZY = {
4
+ "evaluate": ".benchmark",
5
+ "EmbeddingProcessor": ".processors",
6
+ }
7
+
8
+ __all__ = [
9
+ "evaluate",
10
+ "Task",
11
+ "Report",
12
+ "TaskResult",
13
+ "Branch",
14
+ "BranchResult",
15
+ "EmbeddingProcessor",
16
+ ]
17
+
18
+
19
+ def __getattr__(name: str):
20
+ if name not in _LAZY:
21
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
22
+ import importlib
23
+
24
+ value = getattr(importlib.import_module(_LAZY[name], __name__), name)
25
+ globals()[name] = value
26
+ return value
27
+
28
+
29
+ def __dir__():
30
+ return sorted(__all__)
sabermath/__main__.py ADDED
@@ -0,0 +1,66 @@
1
+ """Command-line interface: `sabermath evaluate | download | help`."""
2
+
3
+ import argparse
4
+ import sys
5
+ from typing import get_args
6
+
7
+ from sabermath.schemas import Task
8
+
9
+
10
+ def _evaluate(args):
11
+ if args.vllm and args.device:
12
+ sys.exit("error: --device is not supported with --vllm")
13
+
14
+ from sabermath import evaluate
15
+
16
+ report = evaluate(
17
+ args.model,
18
+ tasks=args.task,
19
+ use_vllm=args.vllm,
20
+ init_kwargs={"device": args.device} if args.device else None,
21
+ )
22
+
23
+ print(f"\n{report.model} ({report.processor}), nDCG@{report.k}")
24
+ for task in report.tasks:
25
+ print(f"\n{task.task:24}{task.ndcg_at_k:.4f}")
26
+ for branch in task.branches:
27
+ print(f" {branch.branch:22}{branch.ndcg_at_k:.4f}")
28
+
29
+
30
+ def _download(args):
31
+ from sabermath.data import load_data
32
+
33
+ queries, documents = load_data()
34
+ print(f"Downloaded {len(queries)} queries and {len(documents)} documents.")
35
+
36
+
37
+ def main(argv=None):
38
+ parser = argparse.ArgumentParser(
39
+ prog="sabermath", description="SABER-Math: a reranking benchmark for math retrieval."
40
+ )
41
+ sub = parser.add_subparsers(dest="command", metavar="<command>")
42
+
43
+ p = sub.add_parser("evaluate", help="Evaluate a model on the benchmark.")
44
+ p.add_argument("model", help="Hugging Face model id.")
45
+ p.add_argument("--task", action="append", choices=get_args(Task),
46
+ help="Task to run; repeat for several (default: all three).")
47
+ p.add_argument("--device", help="Device for Sentence Transformers, e.g. cuda or cpu.")
48
+ p.add_argument("--vllm", action="store_true", help="Load the model with vLLM.")
49
+ p.set_defaults(func=_evaluate)
50
+
51
+ p = sub.add_parser("download", help="Download the benchmark data into the cache.")
52
+ p.set_defaults(func=_download)
53
+
54
+ p = sub.add_parser("help", help="Show help for sabermath or one of its commands.")
55
+ p.add_argument("topic", nargs="?", choices=["evaluate", "download", "help"])
56
+
57
+ args = parser.parse_args(argv)
58
+ if args.command in (None, "help"):
59
+ topic = getattr(args, "topic", None)
60
+ (sub.choices[topic] if topic else parser).print_help()
61
+ return
62
+ args.func(args)
63
+
64
+
65
+ if __name__ == "__main__":
66
+ main()
@@ -0,0 +1 @@
1
+ __version__ = "1.0.0"
sabermath/benchmark.py ADDED
@@ -0,0 +1,343 @@
1
+ from dataclasses import dataclass
2
+ from typing import Callable, Literal, get_args
3
+
4
+ import numpy as np
5
+ from tqdm import tqdm
6
+ from datasets import Dataset
7
+
8
+ from sabermath.data import load_data
9
+ from sabermath.instructions import (
10
+ DEFAULT_INSTRUCTION_TEMPLATE,
11
+ format_instructed_query,
12
+ )
13
+ from sabermath.metrics import compute_ndcg_at_k
14
+ from sabermath.schemas import (
15
+ Task,
16
+ Report,
17
+ TaskResult,
18
+ Branch,
19
+ BranchResult,
20
+ DCGVariant,
21
+ )
22
+ from sabermath.processors import (
23
+ ModelProcessor,
24
+ SentenceTransformersProcessor as STProcessor,
25
+ VLLMProcessor,
26
+ UnknownProcessor,
27
+ )
28
+
29
+
30
+ ALL_TASKS = get_args(Task)
31
+
32
+
33
+ def _naive_type_check(obj, module: str, name: str) -> bool:
34
+ t = type(obj)
35
+ # Match the top-level package: the classes live in submodules
36
+ # (e.g. sentence_transformers.sentence_transformer.model, vllm.entrypoints.llm).
37
+ return t.__module__.split(".")[0] == module and t.__name__ == name
38
+
39
+
40
+ def _make_processor(model, use_vllm: bool, init_kwargs: dict):
41
+ if isinstance(model, str):
42
+ if use_vllm:
43
+ return VLLMProcessor.from_huggingface(model, **init_kwargs)
44
+ return STProcessor.from_huggingface(model, **init_kwargs)
45
+
46
+ if use_vllm:
47
+ raise RuntimeError(
48
+ "Flag 'use_vllm' is only valid when loading a "
49
+ "model from HuggingFace path"
50
+ )
51
+
52
+ if isinstance(model, ModelProcessor):
53
+ if init_kwargs:
54
+ raise RuntimeError(
55
+ "Processor has already been initialized, so "
56
+ "'init_kwargs' is invalid."
57
+ )
58
+ return model
59
+
60
+ if _naive_type_check(model, "vllm", "LLM"):
61
+ return VLLMProcessor(model, **init_kwargs)
62
+
63
+ if _naive_type_check(model, "sentence_transformers", "SentenceTransformer"):
64
+ return STProcessor(model, **init_kwargs)
65
+
66
+ return UnknownProcessor(model, **init_kwargs)
67
+
68
+
69
+ def _get_branch_row_idxs(
70
+ targets: Dataset,
71
+ branch: Branch,
72
+ ) -> list[int]:
73
+ branches_all = targets["domains"]
74
+ return [i for i, branches in enumerate(branches_all) if branch in branches]
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class TaskSettings:
79
+ queries_ds: Dataset
80
+ documents_ds: Dataset
81
+ processor: ModelProcessor
82
+ k: int
83
+ dcg_variant: Literal["linear", "exponent"] | None
84
+ show_progress_bar: bool
85
+ scores_kwargs: dict
86
+ query_instruction: str | None = None
87
+ instruction_template: str = DEFAULT_INSTRUCTION_TEMPLATE
88
+
89
+
90
+ def evaluate_task(
91
+ name: Task,
92
+ settings: TaskSettings,
93
+ return_ndcgs: bool = False,
94
+ on_progress: Callable[[], None] | None = None,
95
+ ) -> TaskResult | tuple[TaskResult, list[float | None]]:
96
+ queries_ds = settings.queries_ds
97
+ documents_ds = settings.documents_ds
98
+ processor = settings.processor
99
+ k = settings.k
100
+ dcg_variant = settings.dcg_variant
101
+ show_progress_bar = settings.show_progress_bar
102
+ scores_kwargs = settings.scores_kwargs
103
+
104
+ query_version = "full" if name == "full-full" else "statement"
105
+ document_version = "statement" if name == "statement-statement" else "full"
106
+
107
+ queries = transform(queries_ds, query_version)
108
+
109
+ if settings.query_instruction is not None:
110
+ queries = [
111
+ format_instructed_query(
112
+ settings.query_instruction, q, settings.instruction_template
113
+ )
114
+ for q in queries
115
+ ]
116
+
117
+ all_ndcgs_by_idx: dict[int, float | None] = {}
118
+ ndcg_by_query_idx: dict[int, float] = {}
119
+
120
+ for i, query in tqdm(
121
+ enumerate(queries),
122
+ total=len(queries),
123
+ disable=not show_progress_bar,
124
+ ):
125
+ doc_ids = list(queries_ds[i]["candidates"])
126
+ relevance_scores = list(queries_ds[i]["relevance_scores"])
127
+
128
+ documents = transform(documents_ds, document_version, doc_ids)
129
+ model_scores = processor.get_scores(
130
+ query,
131
+ documents,
132
+ show_progress_bar=False,
133
+ **scores_kwargs,
134
+ )
135
+
136
+ if model_scores is None:
137
+ all_ndcgs_by_idx[i] = None
138
+ else:
139
+ model_ranked_local_idxs = np.argsort(-np.asarray(model_scores))
140
+
141
+ model_ranked_relevance_scores = np.asarray(relevance_scores)[
142
+ model_ranked_local_idxs
143
+ ]
144
+ ndcg = compute_ndcg_at_k(
145
+ model_ranked_relevance_scores, k=k, variant=dcg_variant
146
+ )
147
+ all_ndcgs_by_idx[i] = ndcg
148
+ ndcg_by_query_idx[i] = ndcg
149
+
150
+ if on_progress is not None:
151
+ on_progress()
152
+
153
+ all_ndcgs = [all_ndcgs_by_idx.get(i) for i in range(len(queries))]
154
+ ndcgs = np.asarray(list(ndcg_by_query_idx.values()))
155
+ ndcg_at_k = float(ndcgs.mean()) if ndcgs.size > 0 else 0.0
156
+
157
+ BRANCHES = get_args(Branch)
158
+ branch_results: list[BranchResult] = []
159
+
160
+ for branch in BRANCHES:
161
+ idxs = _get_branch_row_idxs(queries_ds, branch)
162
+ branch_ndcgs = [
163
+ ndcg_by_query_idx[idx] for idx in idxs if idx in ndcg_by_query_idx
164
+ ]
165
+ branch_ndcg_at_k = (
166
+ float(np.mean(branch_ndcgs)) if len(branch_ndcgs) > 0 else 0.0
167
+ )
168
+ br = BranchResult(branch, branch_ndcg_at_k)
169
+ branch_results.append(br)
170
+
171
+ report = TaskResult(
172
+ name,
173
+ ndcg_at_k,
174
+ branch_results,
175
+ )
176
+
177
+ if return_ndcgs:
178
+ return report, all_ndcgs
179
+ else:
180
+ return report
181
+
182
+
183
+ def transform(
184
+ ds: Dataset,
185
+ version: Literal["statement", "full"],
186
+ idxs: list[int] | None = None,
187
+ ) -> list[str]:
188
+ if idxs is not None:
189
+ ds = ds.select(idxs)
190
+
191
+ if version == "full":
192
+ statements = list(ds["problem"])
193
+ solutions = list(ds["solution"])
194
+
195
+ return [
196
+ f"Problem: {statement}\n\nSolution: {solution}"
197
+ for statement, solution in zip(statements, solutions)
198
+ ]
199
+
200
+ return list(ds["problem"])
201
+
202
+
203
+ def evaluate(
204
+ model,
205
+ tasks: list[Task] | None = None,
206
+ k: int = 10,
207
+ *,
208
+ dcg_variant: DCGVariant = "exponent",
209
+ use_vllm: bool = False,
210
+ init_kwargs: dict | None = None,
211
+ scores_kwargs: dict | None = None,
212
+ on_progress: Callable[[], None] | None = None,
213
+ instruction: str | None = None,
214
+ instruction_template: str = DEFAULT_INSTRUCTION_TEMPLATE,
215
+ verbose: bool = True,
216
+ show_progress_bars: bool = True,
217
+ # Below here: development arguments, not part of the published protocol.
218
+ queries: Dataset | None = None,
219
+ documents: Dataset | None = None,
220
+ return_ndcgs: bool = False,
221
+ ) -> Report | tuple[Report, dict[Task, list[float | None]]]:
222
+ if tasks is None:
223
+ tasks = ALL_TASKS
224
+
225
+ for task in tasks:
226
+ if task not in ALL_TASKS:
227
+ raise ValueError(f"Invalid task: {task}")
228
+
229
+ if dcg_variant not in get_args(DCGVariant):
230
+ raise ValueError(f"Invalid DCG variant: {dcg_variant}")
231
+
232
+ def vprint(text: str) -> None:
233
+ if verbose:
234
+ print(text)
235
+
236
+ init_kwargs = init_kwargs or {}
237
+ scores_kwargs = scores_kwargs or {}
238
+
239
+ vprint("[~] Loading model...")
240
+
241
+ processor = _make_processor(model, use_vllm, init_kwargs)
242
+
243
+ vprint("[+] Model loaded.")
244
+
245
+ task_results = []
246
+
247
+ tasks = set(tasks) & set(ALL_TASKS)
248
+
249
+ if queries is None or documents is None:
250
+ vprint(
251
+ f"[~] Loading data for task{'' if len(tasks) == 1 else 's'} "
252
+ f"\"{','.join(tasks)}\"..."
253
+ )
254
+
255
+ queries, documents = load_data()
256
+
257
+ vprint(f"[+] Loaded {len(queries)} queries.")
258
+
259
+ query_instruction = None
260
+
261
+ if instruction is not None:
262
+ vprint(
263
+ f"[~] Setting instruction ({instruction_template} template): "
264
+ f"{instruction}"
265
+ )
266
+ # The instruction wraps the QUERY TEXT at scoring time (see
267
+ # TaskSettings.query_instruction), never the dataset's problem column:
268
+ # the column route mutated the input before the task transform, so the
269
+ # same prompt produced different strings per task.
270
+ query_instruction = instruction
271
+ vprint("[+] Instruction set.")
272
+
273
+ settings = TaskSettings(
274
+ queries_ds=queries,
275
+ documents_ds=documents,
276
+ processor=processor,
277
+ k=k,
278
+ dcg_variant=dcg_variant,
279
+ show_progress_bar=show_progress_bars,
280
+ scores_kwargs=scores_kwargs,
281
+ query_instruction=query_instruction,
282
+ instruction_template=instruction_template,
283
+ )
284
+
285
+ ndcgs = {}
286
+
287
+ if "statement-statement" in tasks:
288
+ vprint('[~] Evaluating on task "statement vs. statement"...')
289
+ st_st_res, st_st_ndcgs = evaluate_task(
290
+ "statement-statement",
291
+ settings,
292
+ True,
293
+ on_progress=on_progress,
294
+ )
295
+ ndcgs["statement-statement"] = st_st_ndcgs
296
+ ndcg_at_k = st_st_res.ndcg_at_k
297
+ vprint(f"[+] Statement-statement nDCG@{k} ({dcg_variant}): {ndcg_at_k}")
298
+ task_results.append(st_st_res)
299
+
300
+ if "statement-full" in tasks:
301
+ vprint('[~] Evaluating on task "statement vs. full statement + solution"...')
302
+ st_fl_res, st_fl_ndcgs = evaluate_task(
303
+ "statement-full",
304
+ settings,
305
+ True,
306
+ on_progress=on_progress,
307
+ )
308
+ ndcgs["statement-full"] = st_fl_ndcgs
309
+ ndcg_at_k = st_fl_res.ndcg_at_k
310
+ vprint(f"[+] Statement-full nDCG@{k} ({dcg_variant}): {ndcg_at_k}")
311
+ task_results.append(st_fl_res)
312
+
313
+ if "full-full" in tasks:
314
+ vprint(
315
+ '[~] Evaluating on task "full problem + solution vs. '
316
+ 'full problem + solution"...'
317
+ )
318
+ fl_fl_res, fl_fl_ndcgs = evaluate_task(
319
+ "full-full",
320
+ settings,
321
+ True,
322
+ on_progress=on_progress,
323
+ )
324
+ ndcgs["full-full"] = fl_fl_ndcgs
325
+ ndcg_at_k = fl_fl_res.ndcg_at_k
326
+ vprint(f"[+] Full-full nDCG@{k} ({dcg_variant}): {ndcg_at_k}")
327
+ task_results.append(fl_fl_res)
328
+
329
+ del queries
330
+ del documents
331
+
332
+ report = Report(
333
+ model=processor.model or "unknown",
334
+ processor=processor.processor or "unknown",
335
+ dcg_variant=dcg_variant,
336
+ k=k,
337
+ tasks=task_results,
338
+ )
339
+
340
+ if return_ndcgs:
341
+ return report, ndcgs
342
+ else:
343
+ return report
sabermath/data.py ADDED
@@ -0,0 +1,11 @@
1
+ from datasets import load_dataset, Dataset
2
+
3
+
4
+ TARGETS_DATASET = "INSAIT-Institute/SaberMath-queries"
5
+ CANDIDATES_DATASET = "INSAIT-Institute/SaberMath-documents"
6
+
7
+
8
+ def load_data() -> tuple[Dataset, Dataset]:
9
+ targets = load_dataset(TARGETS_DATASET, split="train")
10
+ candidates = load_dataset(CANDIDATES_DATASET, split="train")
11
+ return targets, candidates
@@ -0,0 +1,42 @@
1
+ INSTRUCTIONS: dict[str, str | None] = {
2
+ "p0": None,
3
+ "p1": (
4
+ "You are an expert mathematical retrieval engine. Your job is to "
5
+ "retrieve sample problems that are *conceptually close* to a target "
6
+ "problem, judged by the *mathematical ideas and solution techniques "
7
+ "actually used* (not surface wording)."
8
+ ),
9
+ "p2": (
10
+ "Given a mathematical query, retrieve documents that share identical "
11
+ "problem-solving structure and logical steps, even if they use "
12
+ "entirely different variables, symbols, or natural language phrasing."
13
+ ),
14
+ "p3": "Find the most relevant document based on the query.",
15
+ "pm": (
16
+ "Given a math problem query, retrieve documents that are "
17
+ "mathematically relevant to the query"
18
+ ),
19
+ }
20
+
21
+ INSTRUCTION_KEYS = tuple(INSTRUCTIONS)
22
+
23
+ INSTRUCTION_TEMPLATES: dict[str, str] = {
24
+ "canonical": "Instruct: {instruction}\nQuery: {query}",
25
+ }
26
+
27
+ DEFAULT_INSTRUCTION_TEMPLATE = "canonical"
28
+
29
+
30
+ def format_instructed_query(
31
+ instruction: str,
32
+ problem: str,
33
+ template: str = DEFAULT_INSTRUCTION_TEMPLATE,
34
+ ) -> str:
35
+ try:
36
+ pattern = INSTRUCTION_TEMPLATES[template]
37
+ except KeyError:
38
+ raise ValueError(
39
+ f"Unknown instruction template '{template}' - valid: "
40
+ f"{', '.join(INSTRUCTION_TEMPLATES)}"
41
+ ) from None
42
+ return pattern.format(instruction=instruction, query=problem)
sabermath/metrics.py ADDED
@@ -0,0 +1,48 @@
1
+ from typing import Literal, Sequence
2
+ import numpy as np
3
+
4
+
5
+ def cosine_similarity(query_emb: np.ndarray, docs_embs: np.ndarray) -> np.ndarray:
6
+ q_norm = np.linalg.norm(query_emb)
7
+ d_norms = np.linalg.norm(docs_embs, axis=1, keepdims=True)
8
+
9
+ if q_norm == 0:
10
+ raise ValueError("Query embedding has zero norm")
11
+
12
+ if np.any(d_norms == 0):
13
+ raise ValueError("At least one document embedding has zero norm")
14
+
15
+ query_emb = query_emb / q_norm
16
+ docs_embs = docs_embs / d_norms
17
+ return docs_embs @ query_emb
18
+
19
+
20
+ def dcg_at_k(
21
+ relevances: Sequence[float],
22
+ k: int = 10,
23
+ variant: Literal["linear", "exponent"] = "linear",
24
+ ) -> float:
25
+ relevances = np.asarray(relevances[:k], dtype=float)
26
+
27
+ if relevances.size == 0:
28
+ return 0.0
29
+ discounts = 1.0 / np.log2(np.arange(2, relevances.size + 2))
30
+
31
+ if variant == "linear":
32
+ gains = relevances
33
+ elif variant == "exponent":
34
+ gains = np.power(2, relevances) - 1
35
+ else:
36
+ raise ValueError("variant must be either 'linear' or 'exponent'")
37
+
38
+ return float(np.sum(gains * discounts))
39
+
40
+
41
+ def compute_ndcg_at_k(
42
+ relevances: Sequence[float],
43
+ k: int = 10,
44
+ variant: Literal["linear", "exponent"] = "linear",
45
+ ) -> float:
46
+ dcg = dcg_at_k(relevances, k, variant)
47
+ idcg = dcg_at_k(sorted(relevances, reverse=True), k, variant)
48
+ return 0.0 if idcg == 0 else dcg / idcg
@@ -0,0 +1,6 @@
1
+ from .base import ModelProcessor
2
+ from .embedding_processor import EmbeddingProcessor
3
+ from .st_processor import SentenceTransformersProcessor
4
+ from .vllm_processor import VLLMProcessor
5
+ from .openrouter_processor import OpenRouterEmbeddingProcessor
6
+ from .unknown_processor import UnknownProcessor
@@ -0,0 +1,23 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import ClassVar
3
+
4
+ import numpy as np
5
+
6
+
7
+ class ModelProcessor(ABC):
8
+ processor: ClassVar[str | None]
9
+
10
+ @property
11
+ def model(self) -> str | None:
12
+ return None
13
+
14
+ @abstractmethod
15
+ def get_scores(
16
+ self,
17
+ query: str,
18
+ documents: list[str],
19
+ *,
20
+ show_progress_bar: bool = True,
21
+ **kwargs,
22
+ ) -> list[float] | np.ndarray | None:
23
+ pass