commonground-engine 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- commonground/__init__.py +0 -0
- commonground/__main__.py +499 -0
- commonground/abduce.py +1332 -0
- commonground/argmap.py +207 -0
- commonground/bench.py +547 -0
- commonground/crosscamp.py +408 -0
- commonground/extract.py +239 -0
- commonground/fol.py +496 -0
- commonground/folio.py +139 -0
- commonground/formalize.py +249 -0
- commonground/ground.py +245 -0
- commonground/lean.py +246 -0
- commonground/load.py +52 -0
- commonground/model.py +94 -0
- commonground/smt.py +137 -0
- commonground/verdict.py +118 -0
- commonground_engine-0.2.0.dist-info/METADATA +59 -0
- commonground_engine-0.2.0.dist-info/RECORD +20 -0
- commonground_engine-0.2.0.dist-info/WHEEL +5 -0
- commonground_engine-0.2.0.dist-info/top_level.txt +1 -0
commonground/bench.py
ADDED
|
@@ -0,0 +1,547 @@
|
|
|
1
|
+
"""Benchmark runs on FOLIO. Every number in the repository's README comes from ``reference``."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import csv
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import platform
|
|
9
|
+
import time
|
|
10
|
+
from collections import Counter
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from . import folio, lean
|
|
14
|
+
from .fol import And, BINARY, Formula, Not, Or, ParseError, QUANT, parse, to_text
|
|
15
|
+
from .ground import epr_bound
|
|
16
|
+
from .verdict import decide
|
|
17
|
+
|
|
18
|
+
LABELS = ("True", "False", "Uncertain")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def engine_hash() -> str:
|
|
22
|
+
"""Hash of the engine's source files: pins a results file to the code that produced it."""
|
|
23
|
+
digest = hashlib.sha256()
|
|
24
|
+
for path in sorted(Path(__file__).parent.glob("*.py")):
|
|
25
|
+
digest.update(path.name.encode() + b"\0" + path.read_bytes() + b"\0")
|
|
26
|
+
return digest.hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def environment(lean_bin: str | None) -> dict:
|
|
30
|
+
import pysat
|
|
31
|
+
import z3
|
|
32
|
+
return {
|
|
33
|
+
"python": platform.python_version(),
|
|
34
|
+
"z3": z3.get_version_string(),
|
|
35
|
+
"pysat": pysat.__version__,
|
|
36
|
+
"lean": lean.TOOLCHAIN if lean_bin else None,
|
|
37
|
+
"engine_sha256": engine_hash(),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _flatten(f: Formula):
|
|
42
|
+
"""The formula with nested ∧ (and nested ∨) flattened, so regrouping alone does not count as a change."""
|
|
43
|
+
if isinstance(f, (And, Or)):
|
|
44
|
+
parts = []
|
|
45
|
+
for side in (f.left, f.right):
|
|
46
|
+
g = _flatten(side)
|
|
47
|
+
parts.extend(g[1] if isinstance(g, tuple) and g[0] == type(f).__name__ else [g])
|
|
48
|
+
return (type(f).__name__, parts)
|
|
49
|
+
if isinstance(f, Not):
|
|
50
|
+
return ("Not", _flatten(f.body))
|
|
51
|
+
if isinstance(f, BINARY):
|
|
52
|
+
return (type(f).__name__, _flatten(f.left), _flatten(f.right))
|
|
53
|
+
if isinstance(f, QUANT):
|
|
54
|
+
return (type(f).__name__, f.var, _flatten(f.body))
|
|
55
|
+
return f
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def bracketing_ambiguous(text: str) -> bool:
|
|
59
|
+
"""Does the formula mix ∧ and ∨ without brackets, so that the two reading conventions differ?"""
|
|
60
|
+
try:
|
|
61
|
+
return _flatten(parse(text)) != _flatten(parse(text, connectives="flat-right"))
|
|
62
|
+
except ParseError:
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _verdicts(examples, scope: str, connectives: str, timeout_ms: int, engine: str = "auto"):
|
|
67
|
+
rows = []
|
|
68
|
+
for ex in examples:
|
|
69
|
+
row = {"index": ex.index, "gold": ex.label}
|
|
70
|
+
try:
|
|
71
|
+
parsed = folio.parse_example(ex, scope=scope, connectives=connectives)
|
|
72
|
+
except ParseError as exc:
|
|
73
|
+
rows.append({**row, "verdict": "ParseError", "note": str(exc), "parsed": None})
|
|
74
|
+
continue
|
|
75
|
+
verdict = decide(parsed.premises, parsed.conclusion, timeout_ms, engine)
|
|
76
|
+
rows.append({**row, "verdict": verdict.label, "parsed": parsed, "detail": verdict})
|
|
77
|
+
return rows
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _agreement(rows) -> int:
|
|
81
|
+
return sum(r["verdict"] == r["gold"] for r in rows)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def reference(data_file: Path, out_dir: Path, lean_bin: str | None, timeout_ms: int = 5000) -> dict:
|
|
85
|
+
"""Decide every FOLIO validation example from the dataset's own formulas and audit the result."""
|
|
86
|
+
examples = folio.load(data_file)
|
|
87
|
+
started = time.time()
|
|
88
|
+
rows = _verdicts(examples, "auto", "standard", timeout_ms)
|
|
89
|
+
solver_seconds = time.time() - started
|
|
90
|
+
|
|
91
|
+
# Second engine: grounding to SAT. Where it gives a definite answer it must agree with the first.
|
|
92
|
+
cross = {"compared": 0, "agree": 0, "grounding_definite": 0, "in_decidable_class": 0}
|
|
93
|
+
for r in rows:
|
|
94
|
+
if r["parsed"] is None:
|
|
95
|
+
continue
|
|
96
|
+
p = r["parsed"]
|
|
97
|
+
r["decidable_class"] = epr_bound(p.premises + [p.conclusion]) is not None and \
|
|
98
|
+
epr_bound(p.premises + [Not(p.conclusion)]) is not None
|
|
99
|
+
cross["in_decidable_class"] += r["decidable_class"]
|
|
100
|
+
other = decide(p.premises, p.conclusion, timeout_ms, engine="grounding").label
|
|
101
|
+
r["grounding"] = other
|
|
102
|
+
if other != "Undecided":
|
|
103
|
+
cross["grounding_definite"] += 1
|
|
104
|
+
if other != "Undecided" and r["verdict"] != "Undecided":
|
|
105
|
+
cross["compared"] += 1
|
|
106
|
+
cross["agree"] += other == r["verdict"]
|
|
107
|
+
|
|
108
|
+
# Lean: certify every entailment verdict; offer every false statement as a soundness probe.
|
|
109
|
+
lean_stats = None
|
|
110
|
+
if lean_bin:
|
|
111
|
+
proofs, probes = [], []
|
|
112
|
+
for r in rows:
|
|
113
|
+
if r["parsed"] is None:
|
|
114
|
+
continue
|
|
115
|
+
p = r["parsed"]
|
|
116
|
+
if r["verdict"] == "True":
|
|
117
|
+
proofs.append((r, p.premises, p.conclusion))
|
|
118
|
+
probes.append((p.premises, Not(p.conclusion)))
|
|
119
|
+
elif r["verdict"] == "False":
|
|
120
|
+
proofs.append((r, p.premises, Not(p.conclusion)))
|
|
121
|
+
probes.append((p.premises, p.conclusion))
|
|
122
|
+
elif r["verdict"] == "Uncertain":
|
|
123
|
+
probes += [(p.premises, p.conclusion), (p.premises, Not(p.conclusion))]
|
|
124
|
+
elif r["verdict"] == "Inconsistent":
|
|
125
|
+
proofs.append((r, p.premises, None))
|
|
126
|
+
started = time.time()
|
|
127
|
+
certificates = lean.certify([(pr, goal) for _, pr, goal in proofs], lean_bin)
|
|
128
|
+
proof_seconds = time.time() - started
|
|
129
|
+
for (r, _, _), certificate in zip(proofs, certificates):
|
|
130
|
+
r["lean"] = "certified" if certificate.accepted else f"not certified: {certificate.reason}"
|
|
131
|
+
started = time.time()
|
|
132
|
+
wrongly = sum(c.accepted for c in lean.certify(probes, lean_bin))
|
|
133
|
+
lean_stats = {
|
|
134
|
+
"entailment_verdicts": len(proofs),
|
|
135
|
+
"certified_by_kernel": sum(c.accepted for c in certificates),
|
|
136
|
+
"axioms_used": sorted({a for c in certificates for a in c.axioms}),
|
|
137
|
+
"seconds": round(proof_seconds, 1),
|
|
138
|
+
"false_statements_offered": len(probes),
|
|
139
|
+
"false_statements_accepted": wrongly,
|
|
140
|
+
"probe_seconds": round(time.time() - started, 1),
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
# How much do the results depend on reading conventions?
|
|
144
|
+
conventions = {}
|
|
145
|
+
for scope in ("auto", "tight", "wide"):
|
|
146
|
+
for connectives in ("standard", "flat-right"):
|
|
147
|
+
if (scope, connectives) == ("auto", "standard"):
|
|
148
|
+
conventions["auto/standard"] = _agreement(rows)
|
|
149
|
+
else:
|
|
150
|
+
conventions[f"{scope}/{connectives}"] = _agreement(_verdicts(examples, scope, connectives, timeout_ms))
|
|
151
|
+
alt = {r["index"]: r["verdict"] for r in _verdicts(examples, "auto", "flat-right", timeout_ms)}
|
|
152
|
+
for r in rows:
|
|
153
|
+
r["verdict_flat_right"] = alt[r["index"]]
|
|
154
|
+
|
|
155
|
+
parsed_rows = [r for r in rows if r["parsed"] is not None]
|
|
156
|
+
confusion = Counter((r["gold"], r["verdict"]) for r in rows)
|
|
157
|
+
summary = {
|
|
158
|
+
"dataset": {"name": "FOLIO validation", "examples": len(examples),
|
|
159
|
+
"revision": folio.revision(data_file.parents[2]), "licence": "CC BY-SA 4.0"},
|
|
160
|
+
"environment": environment(lean_bin),
|
|
161
|
+
"parsing": {
|
|
162
|
+
"examples_parsed": len(parsed_rows),
|
|
163
|
+
"examples_with_a_malformed_formula": len(rows) - len(parsed_rows),
|
|
164
|
+
"examples_needing_bracket_repair": sum(bool(r["parsed"].repaired_brackets) for r in parsed_rows),
|
|
165
|
+
"examples_with_unbound_variables_closed": sum(bool(r["parsed"].closed_variables) for r in parsed_rows),
|
|
166
|
+
"examples_with_ambiguous_bracketing": sum(
|
|
167
|
+
any(bracketing_ambiguous(folio.repair_brackets(t)[0]) for t in ex.premises_fol + [ex.conclusion_fol])
|
|
168
|
+
for ex in examples),
|
|
169
|
+
"examples_with_unequal_sentence_and_formula_counts": sum(
|
|
170
|
+
len(ex.premises_text) != len(ex.premises_fol) for ex in examples),
|
|
171
|
+
},
|
|
172
|
+
"agreement_with_labels": {"agree": _agreement(rows), "of": len(rows),
|
|
173
|
+
"rate": round(_agreement(rows) / len(rows), 4)},
|
|
174
|
+
"confusion_gold_then_verdict": {f"{g} -> {v}": n for (g, v), n in sorted(confusion.items())},
|
|
175
|
+
"verdict_counts": dict(Counter(r["verdict"] for r in rows)),
|
|
176
|
+
"second_engine": cross,
|
|
177
|
+
"lean": lean_stats,
|
|
178
|
+
"agreement_by_reading_convention": conventions,
|
|
179
|
+
"solver_seconds": round(solver_seconds, 1),
|
|
180
|
+
"disagreements": [r["index"] for r in rows if r["verdict"] != r["gold"]],
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
184
|
+
(out_dir / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
185
|
+
with open(out_dir / "per_example.csv", "w", newline="", encoding="utf-8") as fh:
|
|
186
|
+
writer = csv.writer(fh)
|
|
187
|
+
writer.writerow(["index", "gold_label", "verdict", "agrees", "verdict_if_and_or_grouped_right",
|
|
188
|
+
"second_engine_verdict", "in_decidable_class", "lean", "countermodel_sizes", "note"])
|
|
189
|
+
for r in rows:
|
|
190
|
+
detail = r.get("detail")
|
|
191
|
+
sizes = ""
|
|
192
|
+
if detail is not None:
|
|
193
|
+
sizes = "/".join(str(m.size) for m in (detail.model_true, detail.model_false) if m is not None)
|
|
194
|
+
writer.writerow([r["index"], r["gold"], r["verdict"], r["verdict"] == r["gold"], r["verdict_flat_right"],
|
|
195
|
+
r.get("grounding", ""), r.get("decidable_class", ""), r.get("lean", ""), sizes,
|
|
196
|
+
r.get("note", "")])
|
|
197
|
+
return summary
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def abduction(data_file: Path, out_dir: Path, limit: int | None = None, timeout_ms: int = 5000,
|
|
201
|
+
lean_bin: str | None = None, lam: float = 0.3, max_checks: int = 12000) -> dict:
|
|
202
|
+
"""Hide-premise abduction benchmark on FOLIO validation (CG8-1).
|
|
203
|
+
|
|
204
|
+
Per example the conclusion is the camp's only endorsed claim and the pool is the
|
|
205
|
+
example's own premises, offered as unscored candidates (p = 0.5 default). Recovery
|
|
206
|
+
is graded cost-independently: a certified axiom system made only of premises exists
|
|
207
|
+
(read from the certified enumeration; ``search_complete`` is required for a definite
|
|
208
|
+
negative). The benchmark grades sufficiency recovery, never truth. An LLM-proposed-
|
|
209
|
+
candidate run needs a live endpoint and is recorded as "not run" without one.
|
|
210
|
+
"""
|
|
211
|
+
from . import abduce
|
|
212
|
+
from .argmap import TooMany
|
|
213
|
+
examples = folio.load(data_file)[:limit]
|
|
214
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
215
|
+
config = {"mode": "abduction", "lam": lam, "max_checks": max_checks,
|
|
216
|
+
"timeout_ms": timeout_ms, "lean": bool(lean_bin)}
|
|
217
|
+
config_file = out_dir / "config.json"
|
|
218
|
+
if config_file.exists():
|
|
219
|
+
previous = json.loads(config_file.read_text(encoding="utf-8"))
|
|
220
|
+
if previous != config:
|
|
221
|
+
raise ConfigurationMismatch(
|
|
222
|
+
f"{out_dir} holds a run made with {previous}; use an empty directory for {config}")
|
|
223
|
+
else:
|
|
224
|
+
config_file.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
|
|
225
|
+
|
|
226
|
+
rows: list[dict] = []
|
|
227
|
+
with (out_dir / "runs.jsonl").open("a", encoding="utf-8") as fh:
|
|
228
|
+
for ex in examples:
|
|
229
|
+
row: dict = {"index": ex.index, "gold": ex.label}
|
|
230
|
+
started = time.time()
|
|
231
|
+
try:
|
|
232
|
+
parsed = folio.parse_example(ex)
|
|
233
|
+
except (ParseError, folio.SignatureError) as exc:
|
|
234
|
+
row.update(parsed=False, error=str(exc))
|
|
235
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
236
|
+
rows.append(row)
|
|
237
|
+
continue
|
|
238
|
+
row["parsed"] = True
|
|
239
|
+
texts = list(ex.premises_text) + [ex.conclusion_text]
|
|
240
|
+
doc = "\n".join(texts)
|
|
241
|
+
spans, off = [], 0
|
|
242
|
+
for t in texts:
|
|
243
|
+
spans.append((off, off + len(t)))
|
|
244
|
+
off += len(t) + 1
|
|
245
|
+
record_ = abduce.Record(
|
|
246
|
+
f"ex-{ex.index}", [abduce.Document("doc", doc)],
|
|
247
|
+
[abduce.RecordStatement("conclusion", "endorsed", to_text(parsed.conclusion),
|
|
248
|
+
"doc", ex.conclusion_text, spans[-1])])
|
|
249
|
+
# One candidate per premise formula, each citing a verbatim sentence. In some FOLIO
|
|
250
|
+
# examples the sentence and formula counts differ (results/folio-validation/AUDIT.md):
|
|
251
|
+
# the pairing is then positional only, a formula beyond the last sentence cites the
|
|
252
|
+
# last one, and the row says so. Recovery is graded on the formulas, not the quotes.
|
|
253
|
+
last = max(len(ex.premises_text) - 1, 0)
|
|
254
|
+
candidates = [abduce.Candidate(to_text(p), [("doc", texts[min(i, last)], spans[min(i, last)])])
|
|
255
|
+
for i, p in enumerate(parsed.premises)]
|
|
256
|
+
if len(parsed.premises) != len(ex.premises_text):
|
|
257
|
+
row["unequal_counts"] = True
|
|
258
|
+
try:
|
|
259
|
+
report = abduce.report_camp(record_, abduce.ScriptedSource(candidates), {}, lam=lam,
|
|
260
|
+
timeout_ms=timeout_ms, lean_bin=lean_bin,
|
|
261
|
+
max_checks=max_checks)
|
|
262
|
+
except TooMany as exc:
|
|
263
|
+
row.update(pool_too_large=True, error=str(exc), seconds=round(time.time() - started, 2))
|
|
264
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
265
|
+
rows.append(row)
|
|
266
|
+
continue
|
|
267
|
+
premise_texts = {to_text(p) for p in parsed.premises}
|
|
268
|
+
recovered_sets = [s_ for s_ in report["selection"]["all_certified"]
|
|
269
|
+
if set(s_) <= premise_texts]
|
|
270
|
+
row["search_complete"] = report["search_complete"]
|
|
271
|
+
row["existence"] = report["existence"]["status"]
|
|
272
|
+
row["recovered"] = bool(recovered_sets) if report["search_complete"] else None
|
|
273
|
+
row["min_recovered_size"] = min((len(s_) for s_ in recovered_sets), default=None)
|
|
274
|
+
row["reference_premises"] = len(parsed.premises)
|
|
275
|
+
row["badge"] = report["systems"][0]["badge"] if report["systems"] else None
|
|
276
|
+
row["seconds"] = round(time.time() - started, 2)
|
|
277
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
278
|
+
rows.append(row)
|
|
279
|
+
|
|
280
|
+
parsed_rows = [r for r in rows if r.get("parsed")]
|
|
281
|
+
definite = [r for r in parsed_rows if r.get("recovered") is not None]
|
|
282
|
+
recovered = [r for r in definite if r["recovered"]]
|
|
283
|
+
sizes = [r["min_recovered_size"] for r in recovered]
|
|
284
|
+
refs = [r["reference_premises"] for r in parsed_rows if "reference_premises" in r]
|
|
285
|
+
summary = {
|
|
286
|
+
**config,
|
|
287
|
+
"dataset": {"name": "FOLIO validation", "examples": len(rows),
|
|
288
|
+
"revision": folio.revision(data_file.parents[2]), "licence": "CC BY-SA 4.0"},
|
|
289
|
+
"examples": len(rows),
|
|
290
|
+
"parsed": len(parsed_rows),
|
|
291
|
+
"pool_too_large": sum(1 for r in rows if r.get("pool_too_large")),
|
|
292
|
+
"unequal_counts": sum(1 for r in rows if r.get("unequal_counts")),
|
|
293
|
+
"search_incomplete": sum(1 for r in parsed_rows if not r.get("search_complete", True)),
|
|
294
|
+
"definite": len(definite),
|
|
295
|
+
"recovered": len(recovered),
|
|
296
|
+
"recovery_rate": round(len(recovered) / len(definite), 4) if definite else None,
|
|
297
|
+
"recovery_interval_95": _interval(len(recovered), len(definite)),
|
|
298
|
+
"min_recovered_size_mean": round(sum(sizes) / len(sizes), 3) if sizes else None,
|
|
299
|
+
"reference_premises_mean": round(sum(refs) / len(refs), 3) if refs else None,
|
|
300
|
+
"badges": dict(Counter(r["badge"] for r in parsed_rows if r.get("badge"))),
|
|
301
|
+
"environment": environment(lean_bin),
|
|
302
|
+
}
|
|
303
|
+
(out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
304
|
+
return summary
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def wilson_interval(successes: int, n: int, z: float = 1.959964) -> tuple[float, float] | None:
|
|
308
|
+
"""Wilson score interval for a proportion (95 % by default). None when there is no observation."""
|
|
309
|
+
if n == 0:
|
|
310
|
+
return None
|
|
311
|
+
p = successes / n
|
|
312
|
+
centre = (p + z * z / (2 * n)) / (1 + z * z / n)
|
|
313
|
+
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
|
|
314
|
+
return max(0.0, centre - half), min(1.0, centre + half)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def mcnemar_exact(only_first: int, only_second: int) -> float:
|
|
318
|
+
"""Two-sided exact McNemar p-value from the two discordant counts of a paired comparison."""
|
|
319
|
+
n = only_first + only_second
|
|
320
|
+
if n == 0:
|
|
321
|
+
return 1.0
|
|
322
|
+
tail = sum(math.comb(n, i) for i in range(min(only_first, only_second) + 1)) / 2 ** n
|
|
323
|
+
return min(1.0, 2 * tail)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _interval(successes: int, n: int) -> list[float] | None:
|
|
327
|
+
interval = wilson_interval(successes, n)
|
|
328
|
+
return [round(interval[0], 4), round(interval[1], 4)] if interval else None
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
REFUSALS_IN_A_ROW = 3
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
class ConfigurationMismatch(ValueError):
|
|
335
|
+
"""The output directory already holds a run made with another mode, model or provider."""
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def with_model(data_file: Path, out_dir: Path, client, mode: str, limit: int | None = None,
|
|
339
|
+
timeout_ms: int = 5000) -> dict:
|
|
340
|
+
"""Run a language model on FOLIO validation: ``mode`` is "llm" (formalize, then solve) or "direct".
|
|
341
|
+
|
|
342
|
+
Results are appended to ``runs.jsonl`` as they arrive, so an interrupted run can be resumed.
|
|
343
|
+
A provider failure stops the run instead of being scored as a wrong answer. A request the
|
|
344
|
+
provider refuses as such (HTTP 400 or 422) is recorded as ``ProviderRefused``, unless
|
|
345
|
+
``REFUSALS_IN_A_ROW`` of them come one after the other: that is a setting the provider does
|
|
346
|
+
not accept, not a property of the examples, so the run stops and none of them is recorded.
|
|
347
|
+
"""
|
|
348
|
+
from . import formalize # imported here: the reference benchmark must not depend on it
|
|
349
|
+
if mode not in ("llm", "direct"):
|
|
350
|
+
raise ValueError('mode must be "llm" or "direct"')
|
|
351
|
+
examples = folio.load(data_file)[:limit]
|
|
352
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
353
|
+
|
|
354
|
+
describe = getattr(client, "describe", None)
|
|
355
|
+
config = {"mode": mode, "client": describe() if callable(describe) else {"model": type(client).__name__}}
|
|
356
|
+
config_file = out_dir / "config.json"
|
|
357
|
+
if config_file.exists():
|
|
358
|
+
previous = json.loads(config_file.read_text(encoding="utf-8"))
|
|
359
|
+
if previous != config:
|
|
360
|
+
raise ConfigurationMismatch(
|
|
361
|
+
f"{out_dir} holds a run made with {previous}; use an empty directory for {config}")
|
|
362
|
+
else:
|
|
363
|
+
config_file.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
|
|
364
|
+
|
|
365
|
+
log = out_dir / "runs.jsonl"
|
|
366
|
+
done = {}
|
|
367
|
+
if log.exists():
|
|
368
|
+
for line in log.read_text(encoding="utf-8").splitlines():
|
|
369
|
+
row = json.loads(line)
|
|
370
|
+
done[row["index"]] = row
|
|
371
|
+
refused: list[dict] = [] # refusals not yet written: they may turn out to be a configuration problem
|
|
372
|
+
|
|
373
|
+
def record(fh, row: dict) -> None:
|
|
374
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
375
|
+
fh.flush()
|
|
376
|
+
done[row["index"]] = row
|
|
377
|
+
|
|
378
|
+
with open(log, "a", encoding="utf-8") as fh:
|
|
379
|
+
for ex in examples:
|
|
380
|
+
if ex.index in done:
|
|
381
|
+
continue
|
|
382
|
+
row = {"index": ex.index, "gold": ex.label, "mode": mode}
|
|
383
|
+
started = time.time()
|
|
384
|
+
try:
|
|
385
|
+
if mode == "direct":
|
|
386
|
+
row["verdict"], usage = formalize.direct(client, ex.premises_text, ex.conclusion_text)
|
|
387
|
+
row["usage"] = [usage]
|
|
388
|
+
else:
|
|
389
|
+
verdict, result = formalize.answer(client, ex.premises_text, ex.conclusion_text,
|
|
390
|
+
timeout_ms=timeout_ms)
|
|
391
|
+
row.update(verdict=verdict.label, attempts=result.attempts, usage=result.usage,
|
|
392
|
+
formulas_by_premise=result.by_sentence, conclusion_formula=result.conclusion_text)
|
|
393
|
+
except formalize.FormalizationError as exc:
|
|
394
|
+
row.update(verdict="NoFormalization", attempts=len(exc.history))
|
|
395
|
+
except formalize.ProviderError as exc:
|
|
396
|
+
if exc.status not in (400, 422):
|
|
397
|
+
raise
|
|
398
|
+
row.update(verdict="ProviderRefused", note=str(exc)[:300], seconds=round(time.time() - started, 2))
|
|
399
|
+
refused.append(row)
|
|
400
|
+
if len(refused) >= REFUSALS_IN_A_ROW:
|
|
401
|
+
raise formalize.ProviderError(
|
|
402
|
+
f"{len(refused)} requests in a row were refused, the last with: {exc}. "
|
|
403
|
+
"That is a setting the provider does not accept (the model name, response_format, "
|
|
404
|
+
"temperature, max_tokens), not the examples; nothing was recorded for them.",
|
|
405
|
+
exc.status) from None
|
|
406
|
+
continue
|
|
407
|
+
row["seconds"] = round(time.time() - started, 2)
|
|
408
|
+
for earlier in refused: # an answer followed: those refusals were about their own examples
|
|
409
|
+
record(fh, earlier)
|
|
410
|
+
refused.clear()
|
|
411
|
+
record(fh, row)
|
|
412
|
+
for earlier in refused:
|
|
413
|
+
record(fh, earlier)
|
|
414
|
+
rows = [done[ex.index] for ex in examples]
|
|
415
|
+
answered = [r for r in rows if r["verdict"] in LABELS]
|
|
416
|
+
correct = sum(r["verdict"] == r["gold"] for r in rows)
|
|
417
|
+
correct_answered = sum(r["verdict"] == r["gold"] for r in answered)
|
|
418
|
+
tokens = Counter()
|
|
419
|
+
for r in rows:
|
|
420
|
+
for used in r.get("usage", []):
|
|
421
|
+
for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
|
|
422
|
+
if isinstance(used.get(key), int):
|
|
423
|
+
tokens[key] += used[key]
|
|
424
|
+
confusion = Counter((r["gold"], r["verdict"]) for r in rows)
|
|
425
|
+
summary = {
|
|
426
|
+
**config,
|
|
427
|
+
"dataset": {"name": "FOLIO validation", "examples": len(rows),
|
|
428
|
+
"revision": folio.revision(data_file.parents[2]), "licence": "CC BY-SA 4.0"},
|
|
429
|
+
"examples": len(rows),
|
|
430
|
+
"correct": correct,
|
|
431
|
+
"accuracy": round(correct / len(rows), 4) if rows else None,
|
|
432
|
+
"accuracy_interval_95": _interval(correct, len(rows)),
|
|
433
|
+
"answered": len(answered),
|
|
434
|
+
"accuracy_when_answered": round(correct_answered / len(answered), 4) if answered else None,
|
|
435
|
+
"accuracy_when_answered_interval_95": _interval(correct_answered, len(answered)),
|
|
436
|
+
"verdict_counts": dict(Counter(r["verdict"] for r in rows)),
|
|
437
|
+
"confusion_gold_then_verdict": {f"{g} -> {v}": n for (g, v), n in sorted(confusion.items())},
|
|
438
|
+
"tokens": dict(tokens),
|
|
439
|
+
"environment": environment(None),
|
|
440
|
+
}
|
|
441
|
+
(out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
442
|
+
return summary
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _runs(directory: Path) -> dict[int, dict]:
|
|
446
|
+
rows = {}
|
|
447
|
+
for line in (directory / "runs.jsonl").read_text(encoding="utf-8").splitlines():
|
|
448
|
+
row = json.loads(line)
|
|
449
|
+
rows[row["index"]] = row
|
|
450
|
+
return rows
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def compare(first_dir: Path, second_dir: Path) -> dict:
|
|
454
|
+
"""Paired comparison of two model runs on the examples both of them answered or failed.
|
|
455
|
+
|
|
456
|
+
The two runs see the same examples, so the test is McNemar's: only the examples on
|
|
457
|
+
which exactly one run is right carry information about which run is better.
|
|
458
|
+
"""
|
|
459
|
+
first, second = _runs(first_dir), _runs(second_dir)
|
|
460
|
+
shared = sorted(set(first) & set(second))
|
|
461
|
+
if not shared:
|
|
462
|
+
raise ValueError("the two runs have no example in common")
|
|
463
|
+
for index in shared:
|
|
464
|
+
if first[index]["gold"] != second[index]["gold"]:
|
|
465
|
+
raise ValueError(f"example {index} has different labels in the two runs: not the same dataset")
|
|
466
|
+
right_first = {i for i in shared if first[i]["verdict"] == first[i]["gold"]}
|
|
467
|
+
right_second = {i for i in shared if second[i]["verdict"] == second[i]["gold"]}
|
|
468
|
+
only_first, only_second = len(right_first - right_second), len(right_second - right_first)
|
|
469
|
+
|
|
470
|
+
def side(directory: Path, right: set[int]) -> dict:
|
|
471
|
+
config = json.loads((directory / "config.json").read_text(encoding="utf-8")) \
|
|
472
|
+
if (directory / "config.json").exists() else {}
|
|
473
|
+
return {"directory": str(directory), **config, "correct": len(right),
|
|
474
|
+
"accuracy": round(len(right) / len(shared), 4), "accuracy_interval_95": _interval(len(right), len(shared))}
|
|
475
|
+
|
|
476
|
+
return {
|
|
477
|
+
"examples_compared": len(shared),
|
|
478
|
+
"first": side(first_dir, right_first),
|
|
479
|
+
"second": side(second_dir, right_second),
|
|
480
|
+
"both_right": len(right_first & right_second),
|
|
481
|
+
"only_first_right": only_first,
|
|
482
|
+
"only_second_right": only_second,
|
|
483
|
+
"neither_right": len(shared) - len(right_first | right_second),
|
|
484
|
+
# Four significant figures, not six decimals: a p of 2e-8 must not be reported as 0.
|
|
485
|
+
"mcnemar_exact_p": float(f"{mcnemar_exact(only_first, only_second):.4g}"),
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
RUN_NAMES = {"direct": "The model answers itself", "llm": "The model writes formulas, the solver answers"}
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _percent(value: float | None) -> str:
|
|
493
|
+
# Two decimals: the summaries store proportions to four places, so nothing is rounded twice.
|
|
494
|
+
return "n/a" if value is None else f"{100 * value:.2f} %"
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _reference_on(indices: list[int], published: Path) -> tuple[int, int] | None:
|
|
498
|
+
"""How many of these examples the reference run gets right, from its per-example file."""
|
|
499
|
+
try:
|
|
500
|
+
with open(published / "per_example.csv", newline="", encoding="utf-8") as fh:
|
|
501
|
+
agrees = {int(row["index"]): row["agrees"] == "True" for row in csv.DictReader(fh)}
|
|
502
|
+
except (OSError, KeyError, ValueError):
|
|
503
|
+
return None
|
|
504
|
+
if not all(i in agrees for i in indices):
|
|
505
|
+
return None
|
|
506
|
+
return sum(agrees[i] for i in indices), len(indices)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def report(first_dir: Path, second_dir: Path, published: Path | None = None) -> str:
|
|
510
|
+
"""Markdown summary of two model runs: each run's accuracy with its interval, then the paired comparison."""
|
|
511
|
+
comparison = compare(first_dir, second_dir)
|
|
512
|
+
summaries = [json.loads((d / "summary.json").read_text(encoding="utf-8")) for d in (first_dir, second_dir)]
|
|
513
|
+
client = summaries[0].get("client", {})
|
|
514
|
+
lines = [
|
|
515
|
+
f"## FOLIO validation with `{client.get('model', 'an unnamed model')}` ({client.get('host', 'unknown host')})",
|
|
516
|
+
"",
|
|
517
|
+
"| Run | Examples | Correct | Accuracy | 95 % interval | Answered | Accuracy when answered | Tokens |",
|
|
518
|
+
"|---|---|---|---|---|---|---|---|",
|
|
519
|
+
]
|
|
520
|
+
for summary in summaries:
|
|
521
|
+
interval = summary.get("accuracy_interval_95")
|
|
522
|
+
lines.append("| {name} | {n} | {correct} | {accuracy} | {interval} | {answered} | {when} | {tokens} |".format(
|
|
523
|
+
name=RUN_NAMES.get(summary.get("mode"), summary.get("mode", "?")),
|
|
524
|
+
n=summary["examples"], correct=summary["correct"], accuracy=_percent(summary["accuracy"]),
|
|
525
|
+
interval=f"{_percent(interval[0])} to {_percent(interval[1])}" if interval else "n/a",
|
|
526
|
+
answered=summary["answered"], when=_percent(summary["accuracy_when_answered"]),
|
|
527
|
+
tokens=summary.get("tokens", {}).get("total_tokens")
|
|
528
|
+
or sum(summary.get("tokens", {}).get(k, 0) for k in ("prompt_tokens", "completion_tokens")) or "not reported"))
|
|
529
|
+
lines += [
|
|
530
|
+
"",
|
|
531
|
+
f"On the {comparison['examples_compared']} examples both runs cover: both right on "
|
|
532
|
+
f"{comparison['both_right']}, only the first on {comparison['only_first_right']}, only the second on "
|
|
533
|
+
f"{comparison['only_second_right']}, neither on {comparison['neither_right']}. "
|
|
534
|
+
f"Exact McNemar test on the two middle counts: p = {comparison['mcnemar_exact_p']:.4g}.",
|
|
535
|
+
]
|
|
536
|
+
published = published or Path(__file__).resolve().parents[2] / "results" / "folio-validation"
|
|
537
|
+
shared = sorted(set(_runs(first_dir)) & set(_runs(second_dir)))
|
|
538
|
+
reference_run = _reference_on(shared, published)
|
|
539
|
+
if reference_run:
|
|
540
|
+
lines += ["", f"For scale, the dataset's own formulas through the same solver agree with the label on "
|
|
541
|
+
f"{reference_run[0]} of these {reference_run[1]} examples."]
|
|
542
|
+
for summary in summaries:
|
|
543
|
+
counts = ", ".join(f"{verdict}: {n}" for verdict, n in sorted(summary["verdict_counts"].items()))
|
|
544
|
+
lines += ["", f"{RUN_NAMES.get(summary.get('mode'), summary.get('mode', '?'))}, verdicts given: {counts}."]
|
|
545
|
+
lines += ["", "An answer that is not one of True, False, Uncertain (no usable formalization, an inconsistent "
|
|
546
|
+
"one, a refused request) counts as wrong in \"Accuracy\" and is left out of \"Answered\"."]
|
|
547
|
+
return "\n".join(lines) + "\n"
|