pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace_lab/graders.py
ADDED
|
@@ -0,0 +1,626 @@
|
|
|
1
|
+
"""pace_lab.graders: answer extraction and grading for multiple choice, math and hidden-profile items (standard library; sympy optional).
|
|
2
|
+
|
|
3
|
+
THE RULES (also the contract the unit tests check)
|
|
4
|
+
|
|
5
|
+
1. Final-answer rule. An "explicit final answer" is (a) the contents of a brace-balanced \\boxed{...} or \\fbox{...}, or (b) the payload of an
|
|
6
|
+
answer marker: "Answer:", "Final answer:", "the answer is", "the correct answer/option/choice is", "= " is NOT a marker. When a response contains
|
|
7
|
+
several explicit final answers, the one that appears LAST in the text wins (a model that corrects itself is graded on its correction). The grade
|
|
8
|
+
records `conflict=True` when earlier explicit answers differ from the winner, so conflicts can be counted and hand-checked. Nothing is guessed:
|
|
9
|
+
when there is no explicit answer the extractor falls back only where stated below, otherwise it returns None and the item is incorrect.
|
|
10
|
+
2. Multiple choice. The letter (A-J, limited to the number of options) is read from the winning candidate: "C", "(C)", "**C**", "C.", "Option C",
|
|
11
|
+
"\\text{C}", "\\boxed{C}". If there is no marker or box, the last non-empty line is accepted only if it is a bare letter ("C", "(C)", "**C.**",
|
|
12
|
+
"Option C"). The words "A" and "I" followed by a lowercase word ("Answer: A student ...", "Answer: I think") are not letters. Lowercase letters
|
|
13
|
+
count only when the whole payload is that single letter.
|
|
14
|
+
3. Math. The winning candidate is normalised (see `normalize_math`) and compared with the gold answer by `math_equal`: identical normalised
|
|
15
|
+
strings; else value equivalence of parsed expressions (exact rationals, floats with rel. tolerance 1e-9, and for expressions with variables
|
|
16
|
+
six seeded random points; with sympy installed, `Expr.equals` may additionally veto a numeric match); sequences / tuples / intervals element by
|
|
17
|
+
element in order with identical brackets; equations side by side. Decimal/fraction equality is by value: 0.5 = 1/2 = \\frac{1}{2}, 5.0 = 5.
|
|
18
|
+
A percent sign is dropped ("50%" = 50, but 50% is NOT 0.5). A thousands separator is removed only in a whole-number pattern ("1,000" = 1000, while
|
|
19
|
+
"(1,000)" stays a pair). Unicode minus signs map to "-", a leading word "minus"/"negative" maps to "-". A mixed number "2\\frac{1}{2}" means 2 + 1/2.
|
|
20
|
+
Spelled-out numbers ("three") are NOT converted. Optional `allow_last_number_fallback` (off by default) takes the last number in the response.
|
|
21
|
+
4. Hidden profile. The candidate is the one name (Avery, Blake, Casey, Drew) in the winning explicit-answer payload, ignoring names that follow
|
|
22
|
+
"not", "rather than", "instead of" or "unlike"; two different remaining names make the answer ambiguous (None). With no explicit answer the last
|
|
23
|
+
non-empty line is used if it contains exactly one candidate name; otherwise None.
|
|
24
|
+
|
|
25
|
+
Backend: this module never needs sympy. `SYMPY_AVAILABLE` says whether the optional veto is active (the studio venv does not have sympy at the time of
|
|
26
|
+
writing; the pure-Python evaluator below is the documented fallback and is what the tests exercise).
|
|
27
|
+
"""
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import math
|
|
31
|
+
import random
|
|
32
|
+
import re
|
|
33
|
+
from dataclasses import dataclass
|
|
34
|
+
from fractions import Fraction
|
|
35
|
+
from typing import Dict, List, Optional, Sequence, Tuple
|
|
36
|
+
|
|
37
|
+
try: # optional
|
|
38
|
+
import sympy as _sp
|
|
39
|
+
SYMPY_AVAILABLE = True
|
|
40
|
+
except Exception: # pragma: no cover - depends on the environment
|
|
41
|
+
_sp = None
|
|
42
|
+
SYMPY_AVAILABLE = False
|
|
43
|
+
|
|
44
|
+
NAMES = ("Avery", "Blake", "Casey", "Drew")
|
|
45
|
+
LETTERS = "ABCDEFGHIJ"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class Grade:
|
|
50
|
+
correct: bool
|
|
51
|
+
extracted: Optional[str] # raw payload of the winning explicit answer (None if none)
|
|
52
|
+
normalized: Optional[str] = None
|
|
53
|
+
method: str = "none" # boxed | marker | last_line | last_number | none
|
|
54
|
+
conflict: bool = False # earlier explicit answers differ from the winner
|
|
55
|
+
n_candidates: int = 0
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# ----------------------------------------------------------------------------- shared extraction
|
|
59
|
+
def balanced_after(text: str, open_idx: int) -> Optional[str]:
|
|
60
|
+
"""Contents of the brace group whose '{' is at open_idx (text[open_idx] == '{'); None if unbalanced (e.g. truncated output)."""
|
|
61
|
+
depth, j = 0, open_idx
|
|
62
|
+
while j < len(text):
|
|
63
|
+
c = text[j]
|
|
64
|
+
if c == "\\" and j + 1 < len(text): # an escaped brace (\{ or \}) does not count
|
|
65
|
+
j += 2
|
|
66
|
+
continue
|
|
67
|
+
if c == "{":
|
|
68
|
+
depth += 1
|
|
69
|
+
elif c == "}":
|
|
70
|
+
depth -= 1
|
|
71
|
+
if depth == 0:
|
|
72
|
+
return text[open_idx + 1: j]
|
|
73
|
+
j += 1
|
|
74
|
+
return None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
_BOX = re.compile(r"\\(?:boxed|fbox)\s*\{")
|
|
78
|
+
_MARKER_STRICT = re.compile(r"(?i)\b(?:final\s+answer|correct\s+(?:answer|option|choice)|answer)\b\s*(?:\*\*|__)?\s*(?:(?:is|are|would\s+be|should\s+be)\b\s*:?|:)\s*(?:\*\*|__)?\s*")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def find_candidates(text: str) -> List[Tuple[int, str, str]]:
|
|
82
|
+
"""All explicit final answers as (position, kind, payload), in text order. kind is 'boxed' or 'marker'."""
|
|
83
|
+
text = text or ""
|
|
84
|
+
found: List[Tuple[int, str, str]] = []
|
|
85
|
+
for m in _BOX.finditer(text):
|
|
86
|
+
inner = balanced_after(text, m.end() - 1)
|
|
87
|
+
if inner is not None:
|
|
88
|
+
found.append((m.start(), "boxed", inner.strip()))
|
|
89
|
+
for m in _MARKER_STRICT.finditer(text):
|
|
90
|
+
rest = text[m.end():]
|
|
91
|
+
rest = rest.lstrip(" \t")
|
|
92
|
+
if rest.startswith("\n"): # "Answer:\nC"
|
|
93
|
+
rest = rest.lstrip()
|
|
94
|
+
line = rest.split("\n", 1)[0]
|
|
95
|
+
if _BOX.search(line): # the box is its own candidate
|
|
96
|
+
continue
|
|
97
|
+
payload = line.strip()
|
|
98
|
+
if payload:
|
|
99
|
+
found.append((m.start(), "marker", payload))
|
|
100
|
+
found.sort(key=lambda x: x[0])
|
|
101
|
+
return found
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _strip_md(s: str) -> str:
|
|
105
|
+
s = s.strip()
|
|
106
|
+
s = re.sub(r"^[\s*_`>]+", "", s)
|
|
107
|
+
s = re.sub(r"[\s*_`]+$", "", s)
|
|
108
|
+
return s
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ----------------------------------------------------------------------------- multiple choice
|
|
112
|
+
def _letter_from(payload: str, n_options: int) -> Optional[str]:
|
|
113
|
+
s = payload.strip()
|
|
114
|
+
s = re.sub(r"^\\text(?:bf)?\s*\{(.*)\}$", r"\1", s).strip()
|
|
115
|
+
bold = s.startswith(("*", "_", "("))
|
|
116
|
+
s = re.sub(r"^[\s*_`$(\[]+", "", s)
|
|
117
|
+
s = re.sub(r"(?i)^(?:option|choice|answer)\s*[:\-]?\s*", "", s)
|
|
118
|
+
s = re.sub(r"^[\s*_`$(\[]+", "", s)
|
|
119
|
+
s = re.sub(r"^\\text(?:bf)?\s*\{([^}]*)\}", r"\1", s)
|
|
120
|
+
m = re.match(r"([A-Za-z])(?![A-Za-z0-9])", s)
|
|
121
|
+
if not m:
|
|
122
|
+
return None
|
|
123
|
+
ch = m.group(1)
|
|
124
|
+
after = s[m.end():]
|
|
125
|
+
if ch.islower():
|
|
126
|
+
return ch.upper() if re.fullmatch(r"[\s.)*_`$\]}:]*", after) and ch.upper() in LETTERS[:n_options] else None
|
|
127
|
+
if ch not in LETTERS[:n_options]:
|
|
128
|
+
return None
|
|
129
|
+
if ch in "AI" and not bold:
|
|
130
|
+
nxt = re.match(r"\s+([A-Za-z]+)", after)
|
|
131
|
+
if nxt and nxt.group(1)[0].islower() and nxt.group(1).lower() not in ("because", "since", "as", "is", "and", "or", "but"):
|
|
132
|
+
return None
|
|
133
|
+
return ch
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def extract_mc_letter(text: str, n_options: int = 10) -> Grade:
|
|
137
|
+
"""Letter of the last explicit answer (rule 2). Returns a Grade-like record with correct=False; use grade_mc to compare with gold."""
|
|
138
|
+
cands = find_candidates(text)
|
|
139
|
+
letters = [(pos, kind, pay, _letter_from(pay, n_options)) for pos, kind, pay in cands]
|
|
140
|
+
letters = [x for x in letters if x[3]]
|
|
141
|
+
if letters:
|
|
142
|
+
pos, kind, pay, letter = letters[-1]
|
|
143
|
+
conflict = any(x[3] != letter for x in letters[:-1])
|
|
144
|
+
return Grade(False, pay, letter, kind, conflict, len(letters))
|
|
145
|
+
lines = [ln for ln in (text or "").splitlines() if ln.strip()]
|
|
146
|
+
if lines:
|
|
147
|
+
last = _strip_md(lines[-1])
|
|
148
|
+
m = re.fullmatch(r"(?i:option\s+)?\(?([A-J])\)?[.):]?", last)
|
|
149
|
+
if m and m.group(1) in LETTERS[:n_options]:
|
|
150
|
+
return Grade(False, lines[-1].strip(), m.group(1), "last_line", False, 1)
|
|
151
|
+
return Grade(False, None, None, "none", False, 0)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def grade_mc(text: str, gold: str, n_options: int = 10) -> Grade:
|
|
155
|
+
g = extract_mc_letter(text, n_options)
|
|
156
|
+
g.correct = g.normalized is not None and g.normalized == str(gold).strip().upper()
|
|
157
|
+
return g
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
# ----------------------------------------------------------------------------- math: normalisation
|
|
161
|
+
_UNICODE_MINUS = "\u2212\u2012\u2796\ufe63\uff0d"
|
|
162
|
+
_UNITS = ("degrees", "degree", "deg", "cm", "mm", "km", "m", "inches", "inch", "feet", "foot", "ft", "units", "unit", "square units", "sq units", "meters", "meter",
|
|
163
|
+
"dollars", "dollar", "cents", "cent", "hours", "hour", "minutes", "minute", "seconds", "second", "mph", "days", "day", "years", "year", "percent",
|
|
164
|
+
"pounds", "ounces", "gallons", "liters", "miles", "mile", "yards", "yard", "sq ft", "cubic units", "radians", "radian")
|
|
165
|
+
_NUM_RE = r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)"
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _unwrap_commands(s: str) -> str:
|
|
169
|
+
"""\\text{x}, \\textbf{x}, \\mathrm{x}, \\mbox{x}, \\boxed{x} -> x (brace-balanced)."""
|
|
170
|
+
pat = re.compile(r"\\(?:text|textbf|textit|mathrm|mathbf|mbox|boxed|fbox|operatorname)\s*\{")
|
|
171
|
+
while True:
|
|
172
|
+
m = pat.search(s)
|
|
173
|
+
if not m:
|
|
174
|
+
return s
|
|
175
|
+
inner = balanced_after(s, m.end() - 1)
|
|
176
|
+
if inner is None:
|
|
177
|
+
return s[: m.start()] + s[m.end():]
|
|
178
|
+
end = m.end() + len(inner) + 1
|
|
179
|
+
s = s[: m.start()] + inner + s[end:]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def normalize_math(s: str) -> str:
|
|
183
|
+
"""Canonical string for comparing math answers (rule 3). Total: never raises, returns '' for empty input."""
|
|
184
|
+
if s is None:
|
|
185
|
+
return ""
|
|
186
|
+
s = str(s).strip()
|
|
187
|
+
# drop a trailing unit written as \text{ cm} / \mbox{ units} after a number, before text is unwrapped
|
|
188
|
+
s = re.sub(r"(" + _NUM_RE + r")\s*\\(?:text|mbox|mathrm)\s*\{\s*[A-Za-z .^0-9]*\}\s*(?:\^\s*\{?\d\}?)?\s*$", r"\1", s)
|
|
189
|
+
s = _unwrap_commands(s)
|
|
190
|
+
for ch in _UNICODE_MINUS:
|
|
191
|
+
s = s.replace(ch, "-")
|
|
192
|
+
s = re.sub(r"^\s*[\u2013\u2014]\s*(?=[\d.\\(])", "-", s) # en/em dash used as a leading minus
|
|
193
|
+
s = s.replace("\\(", "").replace("\\)", "").replace("\\[", "").replace("\\]", "")
|
|
194
|
+
s = s.replace("\\$", "").replace("$", "")
|
|
195
|
+
s = s.replace("{,}", ",").replace("\\displaystyle", "")
|
|
196
|
+
for tok in ("\\left", "\\right", "\\!", "\\,", "\\;", "\\:", "\\quad", "\\qquad", "\\ ", "\\;"):
|
|
197
|
+
s = s.replace(tok, "")
|
|
198
|
+
s = s.replace("dfrac", "frac").replace("tfrac", "frac")
|
|
199
|
+
s = s.replace("\\%", "").replace("%", "").replace("^\\circ", "").replace("^{\\circ}", "").replace("\\circ", "").replace("\u00b0", "")
|
|
200
|
+
s = s.strip()
|
|
201
|
+
s = re.sub(r"(?i)^(?:minus|negative)\s+", "-", s)
|
|
202
|
+
low = s.lower()
|
|
203
|
+
for u in sorted(_UNITS, key=len, reverse=True): # "5 degrees" -> "5" (only after a number)
|
|
204
|
+
if re.fullmatch(_NUM_RE + r"\s*" + re.escape(u) + r"\.?", low):
|
|
205
|
+
s = re.sub(r"\s*[A-Za-z ]+\.?$", "", s)
|
|
206
|
+
break
|
|
207
|
+
s = re.sub(r"\s+", "", s)
|
|
208
|
+
s = re.sub(r"^[a-zA-Z]=(?=[^=]+$)", "", s) # "x=5" -> "5"
|
|
209
|
+
s = re.sub(r"^(-?)(\d+)\\frac\{(\d+)\}\{(\d+)\}$", r"\1(\2+\\frac{\3}{\4})", s) # mixed number
|
|
210
|
+
s = re.sub(r"\\frac(\d)(\d)", r"\\frac{\1}{\2}", s)
|
|
211
|
+
s = re.sub(r"\\sqrt(\d+)", r"\\sqrt{\1}", s)
|
|
212
|
+
s = re.sub(r"\^(\d)(?=\d)", r"^{\1}", s)
|
|
213
|
+
s = re.sub(r"[.\u3002]+$", "", s) if re.search(r"[\w)}]\.$", s) else s # trailing period(s), not a lone "."
|
|
214
|
+
if re.fullmatch(r"[-+]?\d{1,3}(?:,\d{3})+(?:\.\d+)?", s): # thousands separators, whole-number pattern only
|
|
215
|
+
s = s.replace(",", "")
|
|
216
|
+
m = re.fullmatch(r"\(([A-Za-z])\)", s)
|
|
217
|
+
if m:
|
|
218
|
+
s = m.group(1)
|
|
219
|
+
if re.fullmatch(r"[A-Za-z]+", s):
|
|
220
|
+
s = s.lower()
|
|
221
|
+
return s
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
# ----------------------------------------------------------------------------- math: tiny expression engine
|
|
225
|
+
_TOKEN = re.compile(r"\d+(?:\.\d+)?|\.\d+|\\[A-Za-z]+|[A-Za-z]|[-+*/^()={}\[\],]")
|
|
226
|
+
_FUNCS = {"sin": math.sin, "cos": math.cos, "tan": math.tan, "ln": math.log, "log": math.log10, "exp": math.exp}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class _ParseFail(Exception):
|
|
230
|
+
pass
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
class _Parser:
|
|
234
|
+
def __init__(self, s: str):
|
|
235
|
+
pos, toks = 0, []
|
|
236
|
+
while pos < len(s):
|
|
237
|
+
m = _TOKEN.match(s, pos)
|
|
238
|
+
if not m:
|
|
239
|
+
raise _ParseFail("bad char %r" % s[pos])
|
|
240
|
+
toks.append(m.group(0))
|
|
241
|
+
pos = m.end()
|
|
242
|
+
self.t, self.i = toks, 0
|
|
243
|
+
|
|
244
|
+
def peek(self):
|
|
245
|
+
return self.t[self.i] if self.i < len(self.t) else None
|
|
246
|
+
|
|
247
|
+
def take(self, expect=None):
|
|
248
|
+
tok = self.peek()
|
|
249
|
+
if tok is None or (expect is not None and tok != expect):
|
|
250
|
+
raise _ParseFail("expected %s got %s" % (expect, tok))
|
|
251
|
+
self.i += 1
|
|
252
|
+
return tok
|
|
253
|
+
|
|
254
|
+
def parse(self):
|
|
255
|
+
node = self.expr()
|
|
256
|
+
if self.peek() is not None:
|
|
257
|
+
raise _ParseFail("trailing " + self.peek())
|
|
258
|
+
return node
|
|
259
|
+
|
|
260
|
+
def expr(self):
|
|
261
|
+
node = self.term()
|
|
262
|
+
while self.peek() in ("+", "-"):
|
|
263
|
+
op = self.take()
|
|
264
|
+
node = ("add" if op == "+" else "sub", node, self.term())
|
|
265
|
+
return node
|
|
266
|
+
|
|
267
|
+
def _starts_atom(self, tok):
|
|
268
|
+
if tok is None:
|
|
269
|
+
return False
|
|
270
|
+
return bool(re.fullmatch(r"\d+(?:\.\d+)?|\.\d+|[A-Za-z]|\(|\{", tok)) or tok in ("\\frac", "\\sqrt", "\\pi") or (tok.startswith("\\") and tok[1:] in _FUNCS)
|
|
271
|
+
|
|
272
|
+
def term(self):
|
|
273
|
+
node = self.unary()
|
|
274
|
+
while True:
|
|
275
|
+
tok = self.peek()
|
|
276
|
+
if tok in ("*", "\\cdot", "\\times"):
|
|
277
|
+
self.take()
|
|
278
|
+
node = ("mul", node, self.unary())
|
|
279
|
+
elif tok in ("/", "\\div"):
|
|
280
|
+
self.take()
|
|
281
|
+
node = ("div", node, self.unary())
|
|
282
|
+
elif self._starts_atom(tok):
|
|
283
|
+
node = ("mul", node, self.unary()) # implicit multiplication
|
|
284
|
+
else:
|
|
285
|
+
return node
|
|
286
|
+
|
|
287
|
+
def unary(self):
|
|
288
|
+
if self.peek() == "-":
|
|
289
|
+
self.take()
|
|
290
|
+
return ("neg", self.unary())
|
|
291
|
+
if self.peek() == "+":
|
|
292
|
+
self.take()
|
|
293
|
+
return self.unary()
|
|
294
|
+
return self.power()
|
|
295
|
+
|
|
296
|
+
def power(self):
|
|
297
|
+
base = self.atom()
|
|
298
|
+
if self.peek() == "^":
|
|
299
|
+
self.take()
|
|
300
|
+
if self.peek() == "-":
|
|
301
|
+
self.take()
|
|
302
|
+
exp = ("neg", self.atom())
|
|
303
|
+
else:
|
|
304
|
+
exp = self.atom()
|
|
305
|
+
return ("pow", base, exp)
|
|
306
|
+
return base
|
|
307
|
+
|
|
308
|
+
def group(self):
|
|
309
|
+
if self.peek() == "{":
|
|
310
|
+
self.take()
|
|
311
|
+
node = self.expr()
|
|
312
|
+
self.take("}")
|
|
313
|
+
return node
|
|
314
|
+
return self.atom()
|
|
315
|
+
|
|
316
|
+
def atom(self):
|
|
317
|
+
tok = self.peek()
|
|
318
|
+
if tok is None:
|
|
319
|
+
raise _ParseFail("unexpected end")
|
|
320
|
+
if re.fullmatch(r"\d+(?:\.\d+)?|\.\d+", tok):
|
|
321
|
+
self.take()
|
|
322
|
+
return ("num", Fraction(tok))
|
|
323
|
+
if tok == "\\pi":
|
|
324
|
+
self.take()
|
|
325
|
+
return ("pi",)
|
|
326
|
+
if tok == "\\frac":
|
|
327
|
+
self.take()
|
|
328
|
+
a = self.group()
|
|
329
|
+
b = self.group()
|
|
330
|
+
return ("div", a, b)
|
|
331
|
+
if tok == "\\sqrt":
|
|
332
|
+
self.take()
|
|
333
|
+
n = None
|
|
334
|
+
if self.peek() == "[":
|
|
335
|
+
self.take()
|
|
336
|
+
n = self.expr()
|
|
337
|
+
self.take("]")
|
|
338
|
+
return ("sqrt", self.group(), n)
|
|
339
|
+
if tok.startswith("\\") and tok[1:] in _FUNCS:
|
|
340
|
+
self.take()
|
|
341
|
+
arg = self.group() if self.peek() in ("{",) else (self.parenthesised() if self.peek() == "(" else self.atom())
|
|
342
|
+
return ("fn", tok[1:], arg)
|
|
343
|
+
if tok in ("(", "{"):
|
|
344
|
+
return self.parenthesised()
|
|
345
|
+
if re.fullmatch(r"[A-Za-z]", tok):
|
|
346
|
+
self.take()
|
|
347
|
+
return ("var", tok)
|
|
348
|
+
raise _ParseFail("unexpected " + tok)
|
|
349
|
+
|
|
350
|
+
def parenthesised(self):
|
|
351
|
+
open_ = self.take()
|
|
352
|
+
node = self.expr()
|
|
353
|
+
self.take(")" if open_ == "(" else "}")
|
|
354
|
+
return node
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def parse_expr(s: str):
|
|
358
|
+
"""AST for a normalised latex-ish expression, or None when it is outside the supported grammar."""
|
|
359
|
+
try:
|
|
360
|
+
return _Parser(s).parse()
|
|
361
|
+
except (_ParseFail, RecursionError):
|
|
362
|
+
return None
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _free_vars(node, acc=None):
|
|
366
|
+
acc = set() if acc is None else acc
|
|
367
|
+
if node[0] == "var":
|
|
368
|
+
acc.add(node[1])
|
|
369
|
+
elif node[0] in ("add", "sub", "mul", "div", "pow"):
|
|
370
|
+
_free_vars(node[1], acc)
|
|
371
|
+
_free_vars(node[2], acc)
|
|
372
|
+
elif node[0] in ("neg",):
|
|
373
|
+
_free_vars(node[1], acc)
|
|
374
|
+
elif node[0] == "sqrt":
|
|
375
|
+
_free_vars(node[1], acc)
|
|
376
|
+
if node[2] is not None:
|
|
377
|
+
_free_vars(node[2], acc)
|
|
378
|
+
elif node[0] == "fn":
|
|
379
|
+
_free_vars(node[2], acc)
|
|
380
|
+
return acc
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _pow(a, b):
|
|
384
|
+
if isinstance(a, Fraction) and isinstance(b, Fraction) and b.denominator == 1 and abs(b.numerator) <= 4000:
|
|
385
|
+
if a == 0 and b < 0:
|
|
386
|
+
raise ZeroDivisionError
|
|
387
|
+
return a ** int(b)
|
|
388
|
+
r = float(a) ** float(b)
|
|
389
|
+
if isinstance(r, complex):
|
|
390
|
+
raise ValueError("complex")
|
|
391
|
+
return r
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _root(a, n):
|
|
395
|
+
if isinstance(a, Fraction) and isinstance(n, Fraction) and n.denominator == 1 and n > 0 and a >= 0:
|
|
396
|
+
k = int(n)
|
|
397
|
+
num, den = round(a.numerator ** (1.0 / k)), round(a.denominator ** (1.0 / k))
|
|
398
|
+
if num ** k == a.numerator and den ** k == a.denominator:
|
|
399
|
+
return Fraction(num, den)
|
|
400
|
+
if float(a) < 0 and not (float(n) % 2 == 1):
|
|
401
|
+
raise ValueError("even root of negative")
|
|
402
|
+
f = abs(float(a)) ** (1.0 / float(n))
|
|
403
|
+
return -f if float(a) < 0 else f
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _ev(node, env: Dict[str, Fraction]):
|
|
407
|
+
k = node[0]
|
|
408
|
+
if k == "num":
|
|
409
|
+
return node[1]
|
|
410
|
+
if k == "var":
|
|
411
|
+
return env[node[1]]
|
|
412
|
+
if k == "pi":
|
|
413
|
+
return math.pi
|
|
414
|
+
if k == "neg":
|
|
415
|
+
return -_ev(node[1], env)
|
|
416
|
+
if k == "add":
|
|
417
|
+
return _ev(node[1], env) + _ev(node[2], env)
|
|
418
|
+
if k == "sub":
|
|
419
|
+
return _ev(node[1], env) - _ev(node[2], env)
|
|
420
|
+
if k == "mul":
|
|
421
|
+
return _ev(node[1], env) * _ev(node[2], env)
|
|
422
|
+
if k == "div":
|
|
423
|
+
d = _ev(node[2], env)
|
|
424
|
+
if d == 0:
|
|
425
|
+
raise ZeroDivisionError
|
|
426
|
+
return _ev(node[1], env) / d
|
|
427
|
+
if k == "pow":
|
|
428
|
+
return _pow(_ev(node[1], env), _ev(node[2], env))
|
|
429
|
+
if k == "sqrt":
|
|
430
|
+
return _root(_ev(node[1], env), Fraction(2) if node[2] is None else _ev(node[2], env))
|
|
431
|
+
if k == "fn":
|
|
432
|
+
x = float(_ev(node[2], env))
|
|
433
|
+
return _FUNCS[node[1]](x)
|
|
434
|
+
raise ValueError(k)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _close(a, b) -> bool:
|
|
438
|
+
if isinstance(a, Fraction) and isinstance(b, Fraction):
|
|
439
|
+
return a == b
|
|
440
|
+
return math.isclose(float(a), float(b), rel_tol=1e-9, abs_tol=1e-12)
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _to_sympy(node): # pragma: no cover - only with sympy installed
|
|
444
|
+
k = node[0]
|
|
445
|
+
if k == "num":
|
|
446
|
+
return _sp.Rational(node[1].numerator, node[1].denominator)
|
|
447
|
+
if k == "var":
|
|
448
|
+
return _sp.Symbol(node[1])
|
|
449
|
+
if k == "pi":
|
|
450
|
+
return _sp.pi
|
|
451
|
+
if k == "neg":
|
|
452
|
+
return -_to_sympy(node[1])
|
|
453
|
+
if k in ("add", "sub", "mul", "div", "pow"):
|
|
454
|
+
a, b = _to_sympy(node[1]), _to_sympy(node[2])
|
|
455
|
+
return {"add": a + b, "sub": a - b, "mul": a * b, "div": a / b, "pow": a ** b}[k]
|
|
456
|
+
if k == "sqrt":
|
|
457
|
+
n = _sp.Integer(2) if node[2] is None else _to_sympy(node[2])
|
|
458
|
+
return _to_sympy(node[1]) ** (1 / n)
|
|
459
|
+
if k == "fn":
|
|
460
|
+
return {"sin": _sp.sin, "cos": _sp.cos, "tan": _sp.tan, "ln": _sp.log, "exp": _sp.exp, "log": lambda x: _sp.log(x, 10)}[node[1]](_to_sympy(node[2]))
|
|
461
|
+
raise ValueError(k)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def expr_equal(a, b) -> bool:
|
|
465
|
+
"""Equivalence of two ASTs: exact/float value without variables, six seeded points with variables; sympy may veto a match (Expr.equals is False)."""
|
|
466
|
+
names = sorted(_free_vars(a) | _free_vars(b))
|
|
467
|
+
rng = random.Random(2026)
|
|
468
|
+
ok_points = 0
|
|
469
|
+
for _ in range(12 if names else 1):
|
|
470
|
+
env = {v: Fraction(rng.randint(2, 19), rng.randint(1, 5)) for v in names}
|
|
471
|
+
try:
|
|
472
|
+
va, vb = _ev(a, env), _ev(b, env)
|
|
473
|
+
except (ZeroDivisionError, ValueError, OverflowError):
|
|
474
|
+
continue
|
|
475
|
+
if not _close(va, vb):
|
|
476
|
+
return False
|
|
477
|
+
ok_points += 1
|
|
478
|
+
if ok_points >= 6:
|
|
479
|
+
break
|
|
480
|
+
if ok_points < (1 if not names else 3):
|
|
481
|
+
return False
|
|
482
|
+
if SYMPY_AVAILABLE: # pragma: no cover
|
|
483
|
+
try:
|
|
484
|
+
verdict = (_to_sympy(a) - _to_sympy(b)).equals(0)
|
|
485
|
+
if verdict is False:
|
|
486
|
+
return False
|
|
487
|
+
except Exception:
|
|
488
|
+
pass
|
|
489
|
+
return True
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _split_top(s: str, sep: str = ",") -> List[str]:
|
|
493
|
+
out, depth, cur = [], 0, []
|
|
494
|
+
for ch in s:
|
|
495
|
+
if ch in "([{":
|
|
496
|
+
depth += 1
|
|
497
|
+
elif ch in ")]}":
|
|
498
|
+
depth -= 1
|
|
499
|
+
if ch == sep and depth == 0:
|
|
500
|
+
out.append("".join(cur))
|
|
501
|
+
cur = []
|
|
502
|
+
else:
|
|
503
|
+
cur.append(ch)
|
|
504
|
+
out.append("".join(cur))
|
|
505
|
+
return out
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _as_sequence(s: str) -> Optional[Tuple[str, str, List[str]]]:
|
|
509
|
+
if len(s) >= 2 and s[0] in "([" and s[-1] in ")]":
|
|
510
|
+
depth = 0
|
|
511
|
+
for i, ch in enumerate(s):
|
|
512
|
+
depth += ch in "([{"
|
|
513
|
+
depth -= ch in ")]}"
|
|
514
|
+
if depth == 0 and i < len(s) - 1:
|
|
515
|
+
break
|
|
516
|
+
else:
|
|
517
|
+
items = _split_top(s[1:-1])
|
|
518
|
+
if len(items) > 1:
|
|
519
|
+
return s[0], s[-1], items
|
|
520
|
+
return None
|
|
521
|
+
items = _split_top(s)
|
|
522
|
+
return ("", "", items) if len(items) > 1 else None
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _equal_norm(a: str, b: str) -> bool:
|
|
526
|
+
if a == b:
|
|
527
|
+
return True
|
|
528
|
+
if not a or not b:
|
|
529
|
+
return False
|
|
530
|
+
sa, sb = _as_sequence(a), _as_sequence(b)
|
|
531
|
+
if sa or sb:
|
|
532
|
+
return bool(sa and sb and sa[0] == sb[0] and sa[1] == sb[1] and len(sa[2]) == len(sb[2]) and all(_equal_norm(x, y) for x, y in zip(sa[2], sb[2])))
|
|
533
|
+
if a.count("=") == 1 and b.count("=") == 1:
|
|
534
|
+
(la, ra), (lb, rb) = a.split("="), b.split("=")
|
|
535
|
+
return _equal_norm(la, lb) and _equal_norm(ra, rb)
|
|
536
|
+
ea, eb = parse_expr(a), parse_expr(b)
|
|
537
|
+
if ea is None or eb is None:
|
|
538
|
+
return False
|
|
539
|
+
return expr_equal(ea, eb)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def math_equal(pred: Optional[str], gold: Optional[str]) -> bool:
|
|
543
|
+
"""Is the predicted answer equivalent to the gold answer (rule 3)? Both sides pass through normalize_math."""
|
|
544
|
+
if pred is None or gold is None:
|
|
545
|
+
return False
|
|
546
|
+
return _equal_norm(normalize_math(pred), normalize_math(gold))
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
# ----------------------------------------------------------------------------- math: extraction and grading
|
|
550
|
+
_LAST_NUMBER = re.compile(r"[-\u2212]?\d[\d,]*(?:\.\d+)?")
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def extract_math_answer(text: str, allow_last_number_fallback: bool = False) -> Grade:
|
|
554
|
+
cands = find_candidates(text)
|
|
555
|
+
cands = [c for c in cands if c[2]]
|
|
556
|
+
if cands:
|
|
557
|
+
pos, kind, payload = cands[-1]
|
|
558
|
+
norm = normalize_math(_strip_md(payload) if kind == "marker" else payload)
|
|
559
|
+
conflict = any(normalize_math(_strip_md(p) if k == "marker" else p) != norm for _, k, p in cands[:-1])
|
|
560
|
+
return Grade(False, payload, norm, kind, conflict, len(cands))
|
|
561
|
+
if allow_last_number_fallback:
|
|
562
|
+
nums = _LAST_NUMBER.findall(text or "")
|
|
563
|
+
if nums:
|
|
564
|
+
return Grade(False, nums[-1], normalize_math(nums[-1]), "last_number", False, 0)
|
|
565
|
+
return Grade(False, None, None, "none", False, 0)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def grade_math(text: str, gold: str, allow_last_number_fallback: bool = False) -> Grade:
|
|
569
|
+
g = extract_math_answer(text, allow_last_number_fallback)
|
|
570
|
+
g.correct = g.normalized is not None and gold is not None and _equal_norm(g.normalized, normalize_math(gold))
|
|
571
|
+
return g
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
# ----------------------------------------------------------------------------- hidden profile
|
|
575
|
+
_NEG = re.compile(r"(?i)\b(?:not|rather\s+than|instead\s+of|unlike|except|over)\s+(?:\*\*)?(avery|blake|casey|drew)\b")
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _names_in(payload: str, names: Sequence[str] = NAMES) -> List[str]:
|
|
579
|
+
s = _NEG.sub(" ", payload)
|
|
580
|
+
found = []
|
|
581
|
+
for m in re.finditer(r"(?i)\b(" + "|".join(map(re.escape, names)) + r")\b", s):
|
|
582
|
+
nm = next(n for n in names if n.lower() == m.group(1).lower())
|
|
583
|
+
if nm not in found:
|
|
584
|
+
found.append(nm)
|
|
585
|
+
return found
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def extract_candidate(text: str, names: Sequence[str] = NAMES) -> Grade:
|
|
589
|
+
"""Candidate name chosen by the response (rule 4). `normalized` holds the canonical-cased name or None."""
|
|
590
|
+
cands = find_candidates(text)
|
|
591
|
+
scored = []
|
|
592
|
+
for pos, kind, payload in cands:
|
|
593
|
+
got = _names_in(payload, names)
|
|
594
|
+
if got:
|
|
595
|
+
scored.append((pos, kind, payload, got))
|
|
596
|
+
if scored:
|
|
597
|
+
pos, kind, payload, got = scored[-1]
|
|
598
|
+
conflict = any(x[3] != got for x in scored[:-1])
|
|
599
|
+
if len(got) == 1:
|
|
600
|
+
return Grade(False, payload, got[0], kind, conflict, len(scored))
|
|
601
|
+
return Grade(False, payload, None, kind, True, len(scored)) # two names in one final answer: ambiguous
|
|
602
|
+
lines = [ln for ln in (text or "").splitlines() if ln.strip()]
|
|
603
|
+
if lines:
|
|
604
|
+
got = _names_in(lines[-1], names)
|
|
605
|
+
if len(got) == 1:
|
|
606
|
+
return Grade(False, lines[-1].strip(), got[0], "last_line", False, 1)
|
|
607
|
+
return Grade(False, None, None, "none", False, 0)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def grade_hidden_profile(text: str, gold: str, names: Sequence[str] = NAMES) -> Grade:
|
|
611
|
+
g = extract_candidate(text, names)
|
|
612
|
+
g.correct = g.normalized is not None and g.normalized.lower() == str(gold).strip().lower()
|
|
613
|
+
return g
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
# ----------------------------------------------------------------------------- dispatcher
|
|
617
|
+
def grade_item(item: dict, text: str) -> Grade:
|
|
618
|
+
"""Grade a response against an item of the pace_lab.data schema ('kind' is 'mc', 'num' or 'name')."""
|
|
619
|
+
kind = item["kind"]
|
|
620
|
+
if kind == "mc":
|
|
621
|
+
return grade_mc(text, item["gold"], int(item.get("meta", {}).get("n_options", 10)))
|
|
622
|
+
if kind == "num":
|
|
623
|
+
return grade_math(text, item["gold"])
|
|
624
|
+
if kind == "name":
|
|
625
|
+
return grade_hidden_profile(text, item["gold"])
|
|
626
|
+
raise ValueError("no grader for kind %r" % kind)
|