pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace_lab/graders.py ADDED
@@ -0,0 +1,626 @@
1
+ """pace_lab.graders: answer extraction and grading for multiple choice, math and hidden-profile items (standard library; sympy optional).
2
+
3
+ THE RULES (also the contract the unit tests check)
4
+
5
+ 1. Final-answer rule. An "explicit final answer" is (a) the contents of a brace-balanced \\boxed{...} or \\fbox{...}, or (b) the payload of an
6
+ answer marker: "Answer:", "Final answer:", "the answer is", "the correct answer/option/choice is", "= " is NOT a marker. When a response contains
7
+ several explicit final answers, the one that appears LAST in the text wins (a model that corrects itself is graded on its correction). The grade
8
+ records `conflict=True` when earlier explicit answers differ from the winner, so conflicts can be counted and hand-checked. Nothing is guessed:
9
+ when there is no explicit answer the extractor falls back only where stated below, otherwise it returns None and the item is incorrect.
10
+ 2. Multiple choice. The letter (A-J, limited to the number of options) is read from the winning candidate: "C", "(C)", "**C**", "C.", "Option C",
11
+ "\\text{C}", "\\boxed{C}". If there is no marker or box, the last non-empty line is accepted only if it is a bare letter ("C", "(C)", "**C.**",
12
+ "Option C"). The words "A" and "I" followed by a lowercase word ("Answer: A student ...", "Answer: I think") are not letters. Lowercase letters
13
+ count only when the whole payload is that single letter.
14
+ 3. Math. The winning candidate is normalised (see `normalize_math`) and compared with the gold answer by `math_equal`: identical normalised
15
+ strings; else value equivalence of parsed expressions (exact rationals, floats with rel. tolerance 1e-9, and for expressions with variables
16
+ six seeded random points; with sympy installed, `Expr.equals` may additionally veto a numeric match); sequences / tuples / intervals element by
17
+ element in order with identical brackets; equations side by side. Decimal/fraction equality is by value: 0.5 = 1/2 = \\frac{1}{2}, 5.0 = 5.
18
+ A percent sign is dropped ("50%" = 50, but 50% is NOT 0.5). A thousands separator is removed only in a whole-number pattern ("1,000" = 1000, while
19
+ "(1,000)" stays a pair). Unicode minus signs map to "-", a leading word "minus"/"negative" maps to "-". A mixed number "2\\frac{1}{2}" means 2 + 1/2.
20
+ Spelled-out numbers ("three") are NOT converted. Optional `allow_last_number_fallback` (off by default) takes the last number in the response.
21
+ 4. Hidden profile. The candidate is the one name (Avery, Blake, Casey, Drew) in the winning explicit-answer payload, ignoring names that follow
22
+ "not", "rather than", "instead of" or "unlike"; two different remaining names make the answer ambiguous (None). With no explicit answer the last
23
+ non-empty line is used if it contains exactly one candidate name; otherwise None.
24
+
25
+ Backend: this module never needs sympy. `SYMPY_AVAILABLE` says whether the optional veto is active (the studio venv does not have sympy at the time of
26
+ writing; the pure-Python evaluator below is the documented fallback and is what the tests exercise).
27
+ """
28
+ from __future__ import annotations
29
+
30
+ import math
31
+ import random
32
+ import re
33
+ from dataclasses import dataclass
34
+ from fractions import Fraction
35
+ from typing import Dict, List, Optional, Sequence, Tuple
36
+
37
+ try: # optional
38
+ import sympy as _sp
39
+ SYMPY_AVAILABLE = True
40
+ except Exception: # pragma: no cover - depends on the environment
41
+ _sp = None
42
+ SYMPY_AVAILABLE = False
43
+
44
+ NAMES = ("Avery", "Blake", "Casey", "Drew")
45
+ LETTERS = "ABCDEFGHIJ"
46
+
47
+
48
+ @dataclass
49
+ class Grade:
50
+ correct: bool
51
+ extracted: Optional[str] # raw payload of the winning explicit answer (None if none)
52
+ normalized: Optional[str] = None
53
+ method: str = "none" # boxed | marker | last_line | last_number | none
54
+ conflict: bool = False # earlier explicit answers differ from the winner
55
+ n_candidates: int = 0
56
+
57
+
58
+ # ----------------------------------------------------------------------------- shared extraction
59
+ def balanced_after(text: str, open_idx: int) -> Optional[str]:
60
+ """Contents of the brace group whose '{' is at open_idx (text[open_idx] == '{'); None if unbalanced (e.g. truncated output)."""
61
+ depth, j = 0, open_idx
62
+ while j < len(text):
63
+ c = text[j]
64
+ if c == "\\" and j + 1 < len(text): # an escaped brace (\{ or \}) does not count
65
+ j += 2
66
+ continue
67
+ if c == "{":
68
+ depth += 1
69
+ elif c == "}":
70
+ depth -= 1
71
+ if depth == 0:
72
+ return text[open_idx + 1: j]
73
+ j += 1
74
+ return None
75
+
76
+
77
+ _BOX = re.compile(r"\\(?:boxed|fbox)\s*\{")
78
+ _MARKER_STRICT = re.compile(r"(?i)\b(?:final\s+answer|correct\s+(?:answer|option|choice)|answer)\b\s*(?:\*\*|__)?\s*(?:(?:is|are|would\s+be|should\s+be)\b\s*:?|:)\s*(?:\*\*|__)?\s*")
79
+
80
+
81
+ def find_candidates(text: str) -> List[Tuple[int, str, str]]:
82
+ """All explicit final answers as (position, kind, payload), in text order. kind is 'boxed' or 'marker'."""
83
+ text = text or ""
84
+ found: List[Tuple[int, str, str]] = []
85
+ for m in _BOX.finditer(text):
86
+ inner = balanced_after(text, m.end() - 1)
87
+ if inner is not None:
88
+ found.append((m.start(), "boxed", inner.strip()))
89
+ for m in _MARKER_STRICT.finditer(text):
90
+ rest = text[m.end():]
91
+ rest = rest.lstrip(" \t")
92
+ if rest.startswith("\n"): # "Answer:\nC"
93
+ rest = rest.lstrip()
94
+ line = rest.split("\n", 1)[0]
95
+ if _BOX.search(line): # the box is its own candidate
96
+ continue
97
+ payload = line.strip()
98
+ if payload:
99
+ found.append((m.start(), "marker", payload))
100
+ found.sort(key=lambda x: x[0])
101
+ return found
102
+
103
+
104
+ def _strip_md(s: str) -> str:
105
+ s = s.strip()
106
+ s = re.sub(r"^[\s*_`>]+", "", s)
107
+ s = re.sub(r"[\s*_`]+$", "", s)
108
+ return s
109
+
110
+
111
+ # ----------------------------------------------------------------------------- multiple choice
112
+ def _letter_from(payload: str, n_options: int) -> Optional[str]:
113
+ s = payload.strip()
114
+ s = re.sub(r"^\\text(?:bf)?\s*\{(.*)\}$", r"\1", s).strip()
115
+ bold = s.startswith(("*", "_", "("))
116
+ s = re.sub(r"^[\s*_`$(\[]+", "", s)
117
+ s = re.sub(r"(?i)^(?:option|choice|answer)\s*[:\-]?\s*", "", s)
118
+ s = re.sub(r"^[\s*_`$(\[]+", "", s)
119
+ s = re.sub(r"^\\text(?:bf)?\s*\{([^}]*)\}", r"\1", s)
120
+ m = re.match(r"([A-Za-z])(?![A-Za-z0-9])", s)
121
+ if not m:
122
+ return None
123
+ ch = m.group(1)
124
+ after = s[m.end():]
125
+ if ch.islower():
126
+ return ch.upper() if re.fullmatch(r"[\s.)*_`$\]}:]*", after) and ch.upper() in LETTERS[:n_options] else None
127
+ if ch not in LETTERS[:n_options]:
128
+ return None
129
+ if ch in "AI" and not bold:
130
+ nxt = re.match(r"\s+([A-Za-z]+)", after)
131
+ if nxt and nxt.group(1)[0].islower() and nxt.group(1).lower() not in ("because", "since", "as", "is", "and", "or", "but"):
132
+ return None
133
+ return ch
134
+
135
+
136
+ def extract_mc_letter(text: str, n_options: int = 10) -> Grade:
137
+ """Letter of the last explicit answer (rule 2). Returns a Grade-like record with correct=False; use grade_mc to compare with gold."""
138
+ cands = find_candidates(text)
139
+ letters = [(pos, kind, pay, _letter_from(pay, n_options)) for pos, kind, pay in cands]
140
+ letters = [x for x in letters if x[3]]
141
+ if letters:
142
+ pos, kind, pay, letter = letters[-1]
143
+ conflict = any(x[3] != letter for x in letters[:-1])
144
+ return Grade(False, pay, letter, kind, conflict, len(letters))
145
+ lines = [ln for ln in (text or "").splitlines() if ln.strip()]
146
+ if lines:
147
+ last = _strip_md(lines[-1])
148
+ m = re.fullmatch(r"(?i:option\s+)?\(?([A-J])\)?[.):]?", last)
149
+ if m and m.group(1) in LETTERS[:n_options]:
150
+ return Grade(False, lines[-1].strip(), m.group(1), "last_line", False, 1)
151
+ return Grade(False, None, None, "none", False, 0)
152
+
153
+
154
+ def grade_mc(text: str, gold: str, n_options: int = 10) -> Grade:
155
+ g = extract_mc_letter(text, n_options)
156
+ g.correct = g.normalized is not None and g.normalized == str(gold).strip().upper()
157
+ return g
158
+
159
+
160
+ # ----------------------------------------------------------------------------- math: normalisation
161
+ _UNICODE_MINUS = "\u2212\u2012\u2796\ufe63\uff0d"
162
+ _UNITS = ("degrees", "degree", "deg", "cm", "mm", "km", "m", "inches", "inch", "feet", "foot", "ft", "units", "unit", "square units", "sq units", "meters", "meter",
163
+ "dollars", "dollar", "cents", "cent", "hours", "hour", "minutes", "minute", "seconds", "second", "mph", "days", "day", "years", "year", "percent",
164
+ "pounds", "ounces", "gallons", "liters", "miles", "mile", "yards", "yard", "sq ft", "cubic units", "radians", "radian")
165
+ _NUM_RE = r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)"
166
+
167
+
168
+ def _unwrap_commands(s: str) -> str:
169
+ """\\text{x}, \\textbf{x}, \\mathrm{x}, \\mbox{x}, \\boxed{x} -> x (brace-balanced)."""
170
+ pat = re.compile(r"\\(?:text|textbf|textit|mathrm|mathbf|mbox|boxed|fbox|operatorname)\s*\{")
171
+ while True:
172
+ m = pat.search(s)
173
+ if not m:
174
+ return s
175
+ inner = balanced_after(s, m.end() - 1)
176
+ if inner is None:
177
+ return s[: m.start()] + s[m.end():]
178
+ end = m.end() + len(inner) + 1
179
+ s = s[: m.start()] + inner + s[end:]
180
+
181
+
182
+ def normalize_math(s: str) -> str:
183
+ """Canonical string for comparing math answers (rule 3). Total: never raises, returns '' for empty input."""
184
+ if s is None:
185
+ return ""
186
+ s = str(s).strip()
187
+ # drop a trailing unit written as \text{ cm} / \mbox{ units} after a number, before text is unwrapped
188
+ s = re.sub(r"(" + _NUM_RE + r")\s*\\(?:text|mbox|mathrm)\s*\{\s*[A-Za-z .^0-9]*\}\s*(?:\^\s*\{?\d\}?)?\s*$", r"\1", s)
189
+ s = _unwrap_commands(s)
190
+ for ch in _UNICODE_MINUS:
191
+ s = s.replace(ch, "-")
192
+ s = re.sub(r"^\s*[\u2013\u2014]\s*(?=[\d.\\(])", "-", s) # en/em dash used as a leading minus
193
+ s = s.replace("\\(", "").replace("\\)", "").replace("\\[", "").replace("\\]", "")
194
+ s = s.replace("\\$", "").replace("$", "")
195
+ s = s.replace("{,}", ",").replace("\\displaystyle", "")
196
+ for tok in ("\\left", "\\right", "\\!", "\\,", "\\;", "\\:", "\\quad", "\\qquad", "\\ ", "\\;"):
197
+ s = s.replace(tok, "")
198
+ s = s.replace("dfrac", "frac").replace("tfrac", "frac")
199
+ s = s.replace("\\%", "").replace("%", "").replace("^\\circ", "").replace("^{\\circ}", "").replace("\\circ", "").replace("\u00b0", "")
200
+ s = s.strip()
201
+ s = re.sub(r"(?i)^(?:minus|negative)\s+", "-", s)
202
+ low = s.lower()
203
+ for u in sorted(_UNITS, key=len, reverse=True): # "5 degrees" -> "5" (only after a number)
204
+ if re.fullmatch(_NUM_RE + r"\s*" + re.escape(u) + r"\.?", low):
205
+ s = re.sub(r"\s*[A-Za-z ]+\.?$", "", s)
206
+ break
207
+ s = re.sub(r"\s+", "", s)
208
+ s = re.sub(r"^[a-zA-Z]=(?=[^=]+$)", "", s) # "x=5" -> "5"
209
+ s = re.sub(r"^(-?)(\d+)\\frac\{(\d+)\}\{(\d+)\}$", r"\1(\2+\\frac{\3}{\4})", s) # mixed number
210
+ s = re.sub(r"\\frac(\d)(\d)", r"\\frac{\1}{\2}", s)
211
+ s = re.sub(r"\\sqrt(\d+)", r"\\sqrt{\1}", s)
212
+ s = re.sub(r"\^(\d)(?=\d)", r"^{\1}", s)
213
+ s = re.sub(r"[.\u3002]+$", "", s) if re.search(r"[\w)}]\.$", s) else s # trailing period(s), not a lone "."
214
+ if re.fullmatch(r"[-+]?\d{1,3}(?:,\d{3})+(?:\.\d+)?", s): # thousands separators, whole-number pattern only
215
+ s = s.replace(",", "")
216
+ m = re.fullmatch(r"\(([A-Za-z])\)", s)
217
+ if m:
218
+ s = m.group(1)
219
+ if re.fullmatch(r"[A-Za-z]+", s):
220
+ s = s.lower()
221
+ return s
222
+
223
+
224
+ # ----------------------------------------------------------------------------- math: tiny expression engine
225
+ _TOKEN = re.compile(r"\d+(?:\.\d+)?|\.\d+|\\[A-Za-z]+|[A-Za-z]|[-+*/^()={}\[\],]")
226
+ _FUNCS = {"sin": math.sin, "cos": math.cos, "tan": math.tan, "ln": math.log, "log": math.log10, "exp": math.exp}
227
+
228
+
229
+ class _ParseFail(Exception):
230
+ pass
231
+
232
+
233
+ class _Parser:
234
+ def __init__(self, s: str):
235
+ pos, toks = 0, []
236
+ while pos < len(s):
237
+ m = _TOKEN.match(s, pos)
238
+ if not m:
239
+ raise _ParseFail("bad char %r" % s[pos])
240
+ toks.append(m.group(0))
241
+ pos = m.end()
242
+ self.t, self.i = toks, 0
243
+
244
+ def peek(self):
245
+ return self.t[self.i] if self.i < len(self.t) else None
246
+
247
+ def take(self, expect=None):
248
+ tok = self.peek()
249
+ if tok is None or (expect is not None and tok != expect):
250
+ raise _ParseFail("expected %s got %s" % (expect, tok))
251
+ self.i += 1
252
+ return tok
253
+
254
+ def parse(self):
255
+ node = self.expr()
256
+ if self.peek() is not None:
257
+ raise _ParseFail("trailing " + self.peek())
258
+ return node
259
+
260
+ def expr(self):
261
+ node = self.term()
262
+ while self.peek() in ("+", "-"):
263
+ op = self.take()
264
+ node = ("add" if op == "+" else "sub", node, self.term())
265
+ return node
266
+
267
+ def _starts_atom(self, tok):
268
+ if tok is None:
269
+ return False
270
+ return bool(re.fullmatch(r"\d+(?:\.\d+)?|\.\d+|[A-Za-z]|\(|\{", tok)) or tok in ("\\frac", "\\sqrt", "\\pi") or (tok.startswith("\\") and tok[1:] in _FUNCS)
271
+
272
+ def term(self):
273
+ node = self.unary()
274
+ while True:
275
+ tok = self.peek()
276
+ if tok in ("*", "\\cdot", "\\times"):
277
+ self.take()
278
+ node = ("mul", node, self.unary())
279
+ elif tok in ("/", "\\div"):
280
+ self.take()
281
+ node = ("div", node, self.unary())
282
+ elif self._starts_atom(tok):
283
+ node = ("mul", node, self.unary()) # implicit multiplication
284
+ else:
285
+ return node
286
+
287
+ def unary(self):
288
+ if self.peek() == "-":
289
+ self.take()
290
+ return ("neg", self.unary())
291
+ if self.peek() == "+":
292
+ self.take()
293
+ return self.unary()
294
+ return self.power()
295
+
296
+ def power(self):
297
+ base = self.atom()
298
+ if self.peek() == "^":
299
+ self.take()
300
+ if self.peek() == "-":
301
+ self.take()
302
+ exp = ("neg", self.atom())
303
+ else:
304
+ exp = self.atom()
305
+ return ("pow", base, exp)
306
+ return base
307
+
308
+ def group(self):
309
+ if self.peek() == "{":
310
+ self.take()
311
+ node = self.expr()
312
+ self.take("}")
313
+ return node
314
+ return self.atom()
315
+
316
+ def atom(self):
317
+ tok = self.peek()
318
+ if tok is None:
319
+ raise _ParseFail("unexpected end")
320
+ if re.fullmatch(r"\d+(?:\.\d+)?|\.\d+", tok):
321
+ self.take()
322
+ return ("num", Fraction(tok))
323
+ if tok == "\\pi":
324
+ self.take()
325
+ return ("pi",)
326
+ if tok == "\\frac":
327
+ self.take()
328
+ a = self.group()
329
+ b = self.group()
330
+ return ("div", a, b)
331
+ if tok == "\\sqrt":
332
+ self.take()
333
+ n = None
334
+ if self.peek() == "[":
335
+ self.take()
336
+ n = self.expr()
337
+ self.take("]")
338
+ return ("sqrt", self.group(), n)
339
+ if tok.startswith("\\") and tok[1:] in _FUNCS:
340
+ self.take()
341
+ arg = self.group() if self.peek() in ("{",) else (self.parenthesised() if self.peek() == "(" else self.atom())
342
+ return ("fn", tok[1:], arg)
343
+ if tok in ("(", "{"):
344
+ return self.parenthesised()
345
+ if re.fullmatch(r"[A-Za-z]", tok):
346
+ self.take()
347
+ return ("var", tok)
348
+ raise _ParseFail("unexpected " + tok)
349
+
350
+ def parenthesised(self):
351
+ open_ = self.take()
352
+ node = self.expr()
353
+ self.take(")" if open_ == "(" else "}")
354
+ return node
355
+
356
+
357
+ def parse_expr(s: str):
358
+ """AST for a normalised latex-ish expression, or None when it is outside the supported grammar."""
359
+ try:
360
+ return _Parser(s).parse()
361
+ except (_ParseFail, RecursionError):
362
+ return None
363
+
364
+
365
+ def _free_vars(node, acc=None):
366
+ acc = set() if acc is None else acc
367
+ if node[0] == "var":
368
+ acc.add(node[1])
369
+ elif node[0] in ("add", "sub", "mul", "div", "pow"):
370
+ _free_vars(node[1], acc)
371
+ _free_vars(node[2], acc)
372
+ elif node[0] in ("neg",):
373
+ _free_vars(node[1], acc)
374
+ elif node[0] == "sqrt":
375
+ _free_vars(node[1], acc)
376
+ if node[2] is not None:
377
+ _free_vars(node[2], acc)
378
+ elif node[0] == "fn":
379
+ _free_vars(node[2], acc)
380
+ return acc
381
+
382
+
383
+ def _pow(a, b):
384
+ if isinstance(a, Fraction) and isinstance(b, Fraction) and b.denominator == 1 and abs(b.numerator) <= 4000:
385
+ if a == 0 and b < 0:
386
+ raise ZeroDivisionError
387
+ return a ** int(b)
388
+ r = float(a) ** float(b)
389
+ if isinstance(r, complex):
390
+ raise ValueError("complex")
391
+ return r
392
+
393
+
394
+ def _root(a, n):
395
+ if isinstance(a, Fraction) and isinstance(n, Fraction) and n.denominator == 1 and n > 0 and a >= 0:
396
+ k = int(n)
397
+ num, den = round(a.numerator ** (1.0 / k)), round(a.denominator ** (1.0 / k))
398
+ if num ** k == a.numerator and den ** k == a.denominator:
399
+ return Fraction(num, den)
400
+ if float(a) < 0 and not (float(n) % 2 == 1):
401
+ raise ValueError("even root of negative")
402
+ f = abs(float(a)) ** (1.0 / float(n))
403
+ return -f if float(a) < 0 else f
404
+
405
+
406
+ def _ev(node, env: Dict[str, Fraction]):
407
+ k = node[0]
408
+ if k == "num":
409
+ return node[1]
410
+ if k == "var":
411
+ return env[node[1]]
412
+ if k == "pi":
413
+ return math.pi
414
+ if k == "neg":
415
+ return -_ev(node[1], env)
416
+ if k == "add":
417
+ return _ev(node[1], env) + _ev(node[2], env)
418
+ if k == "sub":
419
+ return _ev(node[1], env) - _ev(node[2], env)
420
+ if k == "mul":
421
+ return _ev(node[1], env) * _ev(node[2], env)
422
+ if k == "div":
423
+ d = _ev(node[2], env)
424
+ if d == 0:
425
+ raise ZeroDivisionError
426
+ return _ev(node[1], env) / d
427
+ if k == "pow":
428
+ return _pow(_ev(node[1], env), _ev(node[2], env))
429
+ if k == "sqrt":
430
+ return _root(_ev(node[1], env), Fraction(2) if node[2] is None else _ev(node[2], env))
431
+ if k == "fn":
432
+ x = float(_ev(node[2], env))
433
+ return _FUNCS[node[1]](x)
434
+ raise ValueError(k)
435
+
436
+
437
+ def _close(a, b) -> bool:
438
+ if isinstance(a, Fraction) and isinstance(b, Fraction):
439
+ return a == b
440
+ return math.isclose(float(a), float(b), rel_tol=1e-9, abs_tol=1e-12)
441
+
442
+
443
+ def _to_sympy(node): # pragma: no cover - only with sympy installed
444
+ k = node[0]
445
+ if k == "num":
446
+ return _sp.Rational(node[1].numerator, node[1].denominator)
447
+ if k == "var":
448
+ return _sp.Symbol(node[1])
449
+ if k == "pi":
450
+ return _sp.pi
451
+ if k == "neg":
452
+ return -_to_sympy(node[1])
453
+ if k in ("add", "sub", "mul", "div", "pow"):
454
+ a, b = _to_sympy(node[1]), _to_sympy(node[2])
455
+ return {"add": a + b, "sub": a - b, "mul": a * b, "div": a / b, "pow": a ** b}[k]
456
+ if k == "sqrt":
457
+ n = _sp.Integer(2) if node[2] is None else _to_sympy(node[2])
458
+ return _to_sympy(node[1]) ** (1 / n)
459
+ if k == "fn":
460
+ return {"sin": _sp.sin, "cos": _sp.cos, "tan": _sp.tan, "ln": _sp.log, "exp": _sp.exp, "log": lambda x: _sp.log(x, 10)}[node[1]](_to_sympy(node[2]))
461
+ raise ValueError(k)
462
+
463
+
464
+ def expr_equal(a, b) -> bool:
465
+ """Equivalence of two ASTs: exact/float value without variables, six seeded points with variables; sympy may veto a match (Expr.equals is False)."""
466
+ names = sorted(_free_vars(a) | _free_vars(b))
467
+ rng = random.Random(2026)
468
+ ok_points = 0
469
+ for _ in range(12 if names else 1):
470
+ env = {v: Fraction(rng.randint(2, 19), rng.randint(1, 5)) for v in names}
471
+ try:
472
+ va, vb = _ev(a, env), _ev(b, env)
473
+ except (ZeroDivisionError, ValueError, OverflowError):
474
+ continue
475
+ if not _close(va, vb):
476
+ return False
477
+ ok_points += 1
478
+ if ok_points >= 6:
479
+ break
480
+ if ok_points < (1 if not names else 3):
481
+ return False
482
+ if SYMPY_AVAILABLE: # pragma: no cover
483
+ try:
484
+ verdict = (_to_sympy(a) - _to_sympy(b)).equals(0)
485
+ if verdict is False:
486
+ return False
487
+ except Exception:
488
+ pass
489
+ return True
490
+
491
+
492
+ def _split_top(s: str, sep: str = ",") -> List[str]:
493
+ out, depth, cur = [], 0, []
494
+ for ch in s:
495
+ if ch in "([{":
496
+ depth += 1
497
+ elif ch in ")]}":
498
+ depth -= 1
499
+ if ch == sep and depth == 0:
500
+ out.append("".join(cur))
501
+ cur = []
502
+ else:
503
+ cur.append(ch)
504
+ out.append("".join(cur))
505
+ return out
506
+
507
+
508
+ def _as_sequence(s: str) -> Optional[Tuple[str, str, List[str]]]:
509
+ if len(s) >= 2 and s[0] in "([" and s[-1] in ")]":
510
+ depth = 0
511
+ for i, ch in enumerate(s):
512
+ depth += ch in "([{"
513
+ depth -= ch in ")]}"
514
+ if depth == 0 and i < len(s) - 1:
515
+ break
516
+ else:
517
+ items = _split_top(s[1:-1])
518
+ if len(items) > 1:
519
+ return s[0], s[-1], items
520
+ return None
521
+ items = _split_top(s)
522
+ return ("", "", items) if len(items) > 1 else None
523
+
524
+
525
+ def _equal_norm(a: str, b: str) -> bool:
526
+ if a == b:
527
+ return True
528
+ if not a or not b:
529
+ return False
530
+ sa, sb = _as_sequence(a), _as_sequence(b)
531
+ if sa or sb:
532
+ return bool(sa and sb and sa[0] == sb[0] and sa[1] == sb[1] and len(sa[2]) == len(sb[2]) and all(_equal_norm(x, y) for x, y in zip(sa[2], sb[2])))
533
+ if a.count("=") == 1 and b.count("=") == 1:
534
+ (la, ra), (lb, rb) = a.split("="), b.split("=")
535
+ return _equal_norm(la, lb) and _equal_norm(ra, rb)
536
+ ea, eb = parse_expr(a), parse_expr(b)
537
+ if ea is None or eb is None:
538
+ return False
539
+ return expr_equal(ea, eb)
540
+
541
+
542
+ def math_equal(pred: Optional[str], gold: Optional[str]) -> bool:
543
+ """Is the predicted answer equivalent to the gold answer (rule 3)? Both sides pass through normalize_math."""
544
+ if pred is None or gold is None:
545
+ return False
546
+ return _equal_norm(normalize_math(pred), normalize_math(gold))
547
+
548
+
549
+ # ----------------------------------------------------------------------------- math: extraction and grading
550
+ _LAST_NUMBER = re.compile(r"[-\u2212]?\d[\d,]*(?:\.\d+)?")
551
+
552
+
553
+ def extract_math_answer(text: str, allow_last_number_fallback: bool = False) -> Grade:
554
+ cands = find_candidates(text)
555
+ cands = [c for c in cands if c[2]]
556
+ if cands:
557
+ pos, kind, payload = cands[-1]
558
+ norm = normalize_math(_strip_md(payload) if kind == "marker" else payload)
559
+ conflict = any(normalize_math(_strip_md(p) if k == "marker" else p) != norm for _, k, p in cands[:-1])
560
+ return Grade(False, payload, norm, kind, conflict, len(cands))
561
+ if allow_last_number_fallback:
562
+ nums = _LAST_NUMBER.findall(text or "")
563
+ if nums:
564
+ return Grade(False, nums[-1], normalize_math(nums[-1]), "last_number", False, 0)
565
+ return Grade(False, None, None, "none", False, 0)
566
+
567
+
568
+ def grade_math(text: str, gold: str, allow_last_number_fallback: bool = False) -> Grade:
569
+ g = extract_math_answer(text, allow_last_number_fallback)
570
+ g.correct = g.normalized is not None and gold is not None and _equal_norm(g.normalized, normalize_math(gold))
571
+ return g
572
+
573
+
574
+ # ----------------------------------------------------------------------------- hidden profile
575
+ _NEG = re.compile(r"(?i)\b(?:not|rather\s+than|instead\s+of|unlike|except|over)\s+(?:\*\*)?(avery|blake|casey|drew)\b")
576
+
577
+
578
+ def _names_in(payload: str, names: Sequence[str] = NAMES) -> List[str]:
579
+ s = _NEG.sub(" ", payload)
580
+ found = []
581
+ for m in re.finditer(r"(?i)\b(" + "|".join(map(re.escape, names)) + r")\b", s):
582
+ nm = next(n for n in names if n.lower() == m.group(1).lower())
583
+ if nm not in found:
584
+ found.append(nm)
585
+ return found
586
+
587
+
588
+ def extract_candidate(text: str, names: Sequence[str] = NAMES) -> Grade:
589
+ """Candidate name chosen by the response (rule 4). `normalized` holds the canonical-cased name or None."""
590
+ cands = find_candidates(text)
591
+ scored = []
592
+ for pos, kind, payload in cands:
593
+ got = _names_in(payload, names)
594
+ if got:
595
+ scored.append((pos, kind, payload, got))
596
+ if scored:
597
+ pos, kind, payload, got = scored[-1]
598
+ conflict = any(x[3] != got for x in scored[:-1])
599
+ if len(got) == 1:
600
+ return Grade(False, payload, got[0], kind, conflict, len(scored))
601
+ return Grade(False, payload, None, kind, True, len(scored)) # two names in one final answer: ambiguous
602
+ lines = [ln for ln in (text or "").splitlines() if ln.strip()]
603
+ if lines:
604
+ got = _names_in(lines[-1], names)
605
+ if len(got) == 1:
606
+ return Grade(False, lines[-1].strip(), got[0], "last_line", False, 1)
607
+ return Grade(False, None, None, "none", False, 0)
608
+
609
+
610
+ def grade_hidden_profile(text: str, gold: str, names: Sequence[str] = NAMES) -> Grade:
611
+ g = extract_candidate(text, names)
612
+ g.correct = g.normalized is not None and g.normalized.lower() == str(gold).strip().lower()
613
+ return g
614
+
615
+
616
+ # ----------------------------------------------------------------------------- dispatcher
617
+ def grade_item(item: dict, text: str) -> Grade:
618
+ """Grade a response against an item of the pace_lab.data schema ('kind' is 'mc', 'num' or 'name')."""
619
+ kind = item["kind"]
620
+ if kind == "mc":
621
+ return grade_mc(text, item["gold"], int(item.get("meta", {}).get("n_options", 10)))
622
+ if kind == "num":
623
+ return grade_math(text, item["gold"])
624
+ if kind == "name":
625
+ return grade_hidden_profile(text, item["gold"])
626
+ raise ValueError("no grader for kind %r" % kind)