commonground-engine 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
commonground/bench.py ADDED
@@ -0,0 +1,547 @@
1
+ """Benchmark runs on FOLIO. Every number in the repository's README comes from ``reference``."""
2
+ from __future__ import annotations
3
+
4
+ import csv
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import platform
9
+ import time
10
+ from collections import Counter
11
+ from pathlib import Path
12
+
13
+ from . import folio, lean
14
+ from .fol import And, BINARY, Formula, Not, Or, ParseError, QUANT, parse, to_text
15
+ from .ground import epr_bound
16
+ from .verdict import decide
17
+
18
+ LABELS = ("True", "False", "Uncertain")
19
+
20
+
21
+ def engine_hash() -> str:
22
+ """Hash of the engine's source files: pins a results file to the code that produced it."""
23
+ digest = hashlib.sha256()
24
+ for path in sorted(Path(__file__).parent.glob("*.py")):
25
+ digest.update(path.name.encode() + b"\0" + path.read_bytes() + b"\0")
26
+ return digest.hexdigest()
27
+
28
+
29
+ def environment(lean_bin: str | None) -> dict:
30
+ import pysat
31
+ import z3
32
+ return {
33
+ "python": platform.python_version(),
34
+ "z3": z3.get_version_string(),
35
+ "pysat": pysat.__version__,
36
+ "lean": lean.TOOLCHAIN if lean_bin else None,
37
+ "engine_sha256": engine_hash(),
38
+ }
39
+
40
+
41
+ def _flatten(f: Formula):
42
+ """The formula with nested ∧ (and nested ∨) flattened, so regrouping alone does not count as a change."""
43
+ if isinstance(f, (And, Or)):
44
+ parts = []
45
+ for side in (f.left, f.right):
46
+ g = _flatten(side)
47
+ parts.extend(g[1] if isinstance(g, tuple) and g[0] == type(f).__name__ else [g])
48
+ return (type(f).__name__, parts)
49
+ if isinstance(f, Not):
50
+ return ("Not", _flatten(f.body))
51
+ if isinstance(f, BINARY):
52
+ return (type(f).__name__, _flatten(f.left), _flatten(f.right))
53
+ if isinstance(f, QUANT):
54
+ return (type(f).__name__, f.var, _flatten(f.body))
55
+ return f
56
+
57
+
58
+ def bracketing_ambiguous(text: str) -> bool:
59
+ """Does the formula mix ∧ and ∨ without brackets, so that the two reading conventions differ?"""
60
+ try:
61
+ return _flatten(parse(text)) != _flatten(parse(text, connectives="flat-right"))
62
+ except ParseError:
63
+ return False
64
+
65
+
66
+ def _verdicts(examples, scope: str, connectives: str, timeout_ms: int, engine: str = "auto"):
67
+ rows = []
68
+ for ex in examples:
69
+ row = {"index": ex.index, "gold": ex.label}
70
+ try:
71
+ parsed = folio.parse_example(ex, scope=scope, connectives=connectives)
72
+ except ParseError as exc:
73
+ rows.append({**row, "verdict": "ParseError", "note": str(exc), "parsed": None})
74
+ continue
75
+ verdict = decide(parsed.premises, parsed.conclusion, timeout_ms, engine)
76
+ rows.append({**row, "verdict": verdict.label, "parsed": parsed, "detail": verdict})
77
+ return rows
78
+
79
+
80
+ def _agreement(rows) -> int:
81
+ return sum(r["verdict"] == r["gold"] for r in rows)
82
+
83
+
84
+ def reference(data_file: Path, out_dir: Path, lean_bin: str | None, timeout_ms: int = 5000) -> dict:
85
+ """Decide every FOLIO validation example from the dataset's own formulas and audit the result."""
86
+ examples = folio.load(data_file)
87
+ started = time.time()
88
+ rows = _verdicts(examples, "auto", "standard", timeout_ms)
89
+ solver_seconds = time.time() - started
90
+
91
+ # Second engine: grounding to SAT. Where it gives a definite answer it must agree with the first.
92
+ cross = {"compared": 0, "agree": 0, "grounding_definite": 0, "in_decidable_class": 0}
93
+ for r in rows:
94
+ if r["parsed"] is None:
95
+ continue
96
+ p = r["parsed"]
97
+ r["decidable_class"] = epr_bound(p.premises + [p.conclusion]) is not None and \
98
+ epr_bound(p.premises + [Not(p.conclusion)]) is not None
99
+ cross["in_decidable_class"] += r["decidable_class"]
100
+ other = decide(p.premises, p.conclusion, timeout_ms, engine="grounding").label
101
+ r["grounding"] = other
102
+ if other != "Undecided":
103
+ cross["grounding_definite"] += 1
104
+ if other != "Undecided" and r["verdict"] != "Undecided":
105
+ cross["compared"] += 1
106
+ cross["agree"] += other == r["verdict"]
107
+
108
+ # Lean: certify every entailment verdict; offer every false statement as a soundness probe.
109
+ lean_stats = None
110
+ if lean_bin:
111
+ proofs, probes = [], []
112
+ for r in rows:
113
+ if r["parsed"] is None:
114
+ continue
115
+ p = r["parsed"]
116
+ if r["verdict"] == "True":
117
+ proofs.append((r, p.premises, p.conclusion))
118
+ probes.append((p.premises, Not(p.conclusion)))
119
+ elif r["verdict"] == "False":
120
+ proofs.append((r, p.premises, Not(p.conclusion)))
121
+ probes.append((p.premises, p.conclusion))
122
+ elif r["verdict"] == "Uncertain":
123
+ probes += [(p.premises, p.conclusion), (p.premises, Not(p.conclusion))]
124
+ elif r["verdict"] == "Inconsistent":
125
+ proofs.append((r, p.premises, None))
126
+ started = time.time()
127
+ certificates = lean.certify([(pr, goal) for _, pr, goal in proofs], lean_bin)
128
+ proof_seconds = time.time() - started
129
+ for (r, _, _), certificate in zip(proofs, certificates):
130
+ r["lean"] = "certified" if certificate.accepted else f"not certified: {certificate.reason}"
131
+ started = time.time()
132
+ wrongly = sum(c.accepted for c in lean.certify(probes, lean_bin))
133
+ lean_stats = {
134
+ "entailment_verdicts": len(proofs),
135
+ "certified_by_kernel": sum(c.accepted for c in certificates),
136
+ "axioms_used": sorted({a for c in certificates for a in c.axioms}),
137
+ "seconds": round(proof_seconds, 1),
138
+ "false_statements_offered": len(probes),
139
+ "false_statements_accepted": wrongly,
140
+ "probe_seconds": round(time.time() - started, 1),
141
+ }
142
+
143
+ # How much do the results depend on reading conventions?
144
+ conventions = {}
145
+ for scope in ("auto", "tight", "wide"):
146
+ for connectives in ("standard", "flat-right"):
147
+ if (scope, connectives) == ("auto", "standard"):
148
+ conventions["auto/standard"] = _agreement(rows)
149
+ else:
150
+ conventions[f"{scope}/{connectives}"] = _agreement(_verdicts(examples, scope, connectives, timeout_ms))
151
+ alt = {r["index"]: r["verdict"] for r in _verdicts(examples, "auto", "flat-right", timeout_ms)}
152
+ for r in rows:
153
+ r["verdict_flat_right"] = alt[r["index"]]
154
+
155
+ parsed_rows = [r for r in rows if r["parsed"] is not None]
156
+ confusion = Counter((r["gold"], r["verdict"]) for r in rows)
157
+ summary = {
158
+ "dataset": {"name": "FOLIO validation", "examples": len(examples),
159
+ "revision": folio.revision(data_file.parents[2]), "licence": "CC BY-SA 4.0"},
160
+ "environment": environment(lean_bin),
161
+ "parsing": {
162
+ "examples_parsed": len(parsed_rows),
163
+ "examples_with_a_malformed_formula": len(rows) - len(parsed_rows),
164
+ "examples_needing_bracket_repair": sum(bool(r["parsed"].repaired_brackets) for r in parsed_rows),
165
+ "examples_with_unbound_variables_closed": sum(bool(r["parsed"].closed_variables) for r in parsed_rows),
166
+ "examples_with_ambiguous_bracketing": sum(
167
+ any(bracketing_ambiguous(folio.repair_brackets(t)[0]) for t in ex.premises_fol + [ex.conclusion_fol])
168
+ for ex in examples),
169
+ "examples_with_unequal_sentence_and_formula_counts": sum(
170
+ len(ex.premises_text) != len(ex.premises_fol) for ex in examples),
171
+ },
172
+ "agreement_with_labels": {"agree": _agreement(rows), "of": len(rows),
173
+ "rate": round(_agreement(rows) / len(rows), 4)},
174
+ "confusion_gold_then_verdict": {f"{g} -> {v}": n for (g, v), n in sorted(confusion.items())},
175
+ "verdict_counts": dict(Counter(r["verdict"] for r in rows)),
176
+ "second_engine": cross,
177
+ "lean": lean_stats,
178
+ "agreement_by_reading_convention": conventions,
179
+ "solver_seconds": round(solver_seconds, 1),
180
+ "disagreements": [r["index"] for r in rows if r["verdict"] != r["gold"]],
181
+ }
182
+
183
+ out_dir.mkdir(parents=True, exist_ok=True)
184
+ (out_dir / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
185
+ with open(out_dir / "per_example.csv", "w", newline="", encoding="utf-8") as fh:
186
+ writer = csv.writer(fh)
187
+ writer.writerow(["index", "gold_label", "verdict", "agrees", "verdict_if_and_or_grouped_right",
188
+ "second_engine_verdict", "in_decidable_class", "lean", "countermodel_sizes", "note"])
189
+ for r in rows:
190
+ detail = r.get("detail")
191
+ sizes = ""
192
+ if detail is not None:
193
+ sizes = "/".join(str(m.size) for m in (detail.model_true, detail.model_false) if m is not None)
194
+ writer.writerow([r["index"], r["gold"], r["verdict"], r["verdict"] == r["gold"], r["verdict_flat_right"],
195
+ r.get("grounding", ""), r.get("decidable_class", ""), r.get("lean", ""), sizes,
196
+ r.get("note", "")])
197
+ return summary
198
+
199
+
200
+ def abduction(data_file: Path, out_dir: Path, limit: int | None = None, timeout_ms: int = 5000,
201
+ lean_bin: str | None = None, lam: float = 0.3, max_checks: int = 12000) -> dict:
202
+ """Hide-premise abduction benchmark on FOLIO validation (CG8-1).
203
+
204
+ Per example the conclusion is the camp's only endorsed claim and the pool is the
205
+ example's own premises, offered as unscored candidates (p = 0.5 default). Recovery
206
+ is graded cost-independently: a certified axiom system made only of premises exists
207
+ (read from the certified enumeration; ``search_complete`` is required for a definite
208
+ negative). The benchmark grades sufficiency recovery, never truth. An LLM-proposed-
209
+ candidate run needs a live endpoint and is recorded as "not run" without one.
210
+ """
211
+ from . import abduce
212
+ from .argmap import TooMany
213
+ examples = folio.load(data_file)[:limit]
214
+ out_dir.mkdir(parents=True, exist_ok=True)
215
+ config = {"mode": "abduction", "lam": lam, "max_checks": max_checks,
216
+ "timeout_ms": timeout_ms, "lean": bool(lean_bin)}
217
+ config_file = out_dir / "config.json"
218
+ if config_file.exists():
219
+ previous = json.loads(config_file.read_text(encoding="utf-8"))
220
+ if previous != config:
221
+ raise ConfigurationMismatch(
222
+ f"{out_dir} holds a run made with {previous}; use an empty directory for {config}")
223
+ else:
224
+ config_file.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
225
+
226
+ rows: list[dict] = []
227
+ with (out_dir / "runs.jsonl").open("a", encoding="utf-8") as fh:
228
+ for ex in examples:
229
+ row: dict = {"index": ex.index, "gold": ex.label}
230
+ started = time.time()
231
+ try:
232
+ parsed = folio.parse_example(ex)
233
+ except (ParseError, folio.SignatureError) as exc:
234
+ row.update(parsed=False, error=str(exc))
235
+ fh.write(json.dumps(row, ensure_ascii=False) + "\n")
236
+ rows.append(row)
237
+ continue
238
+ row["parsed"] = True
239
+ texts = list(ex.premises_text) + [ex.conclusion_text]
240
+ doc = "\n".join(texts)
241
+ spans, off = [], 0
242
+ for t in texts:
243
+ spans.append((off, off + len(t)))
244
+ off += len(t) + 1
245
+ record_ = abduce.Record(
246
+ f"ex-{ex.index}", [abduce.Document("doc", doc)],
247
+ [abduce.RecordStatement("conclusion", "endorsed", to_text(parsed.conclusion),
248
+ "doc", ex.conclusion_text, spans[-1])])
249
+ # One candidate per premise formula, each citing a verbatim sentence. In some FOLIO
250
+ # examples the sentence and formula counts differ (results/folio-validation/AUDIT.md):
251
+ # the pairing is then positional only, a formula beyond the last sentence cites the
252
+ # last one, and the row says so. Recovery is graded on the formulas, not the quotes.
253
+ last = max(len(ex.premises_text) - 1, 0)
254
+ candidates = [abduce.Candidate(to_text(p), [("doc", texts[min(i, last)], spans[min(i, last)])])
255
+ for i, p in enumerate(parsed.premises)]
256
+ if len(parsed.premises) != len(ex.premises_text):
257
+ row["unequal_counts"] = True
258
+ try:
259
+ report = abduce.report_camp(record_, abduce.ScriptedSource(candidates), {}, lam=lam,
260
+ timeout_ms=timeout_ms, lean_bin=lean_bin,
261
+ max_checks=max_checks)
262
+ except TooMany as exc:
263
+ row.update(pool_too_large=True, error=str(exc), seconds=round(time.time() - started, 2))
264
+ fh.write(json.dumps(row, ensure_ascii=False) + "\n")
265
+ rows.append(row)
266
+ continue
267
+ premise_texts = {to_text(p) for p in parsed.premises}
268
+ recovered_sets = [s_ for s_ in report["selection"]["all_certified"]
269
+ if set(s_) <= premise_texts]
270
+ row["search_complete"] = report["search_complete"]
271
+ row["existence"] = report["existence"]["status"]
272
+ row["recovered"] = bool(recovered_sets) if report["search_complete"] else None
273
+ row["min_recovered_size"] = min((len(s_) for s_ in recovered_sets), default=None)
274
+ row["reference_premises"] = len(parsed.premises)
275
+ row["badge"] = report["systems"][0]["badge"] if report["systems"] else None
276
+ row["seconds"] = round(time.time() - started, 2)
277
+ fh.write(json.dumps(row, ensure_ascii=False) + "\n")
278
+ rows.append(row)
279
+
280
+ parsed_rows = [r for r in rows if r.get("parsed")]
281
+ definite = [r for r in parsed_rows if r.get("recovered") is not None]
282
+ recovered = [r for r in definite if r["recovered"]]
283
+ sizes = [r["min_recovered_size"] for r in recovered]
284
+ refs = [r["reference_premises"] for r in parsed_rows if "reference_premises" in r]
285
+ summary = {
286
+ **config,
287
+ "dataset": {"name": "FOLIO validation", "examples": len(rows),
288
+ "revision": folio.revision(data_file.parents[2]), "licence": "CC BY-SA 4.0"},
289
+ "examples": len(rows),
290
+ "parsed": len(parsed_rows),
291
+ "pool_too_large": sum(1 for r in rows if r.get("pool_too_large")),
292
+ "unequal_counts": sum(1 for r in rows if r.get("unequal_counts")),
293
+ "search_incomplete": sum(1 for r in parsed_rows if not r.get("search_complete", True)),
294
+ "definite": len(definite),
295
+ "recovered": len(recovered),
296
+ "recovery_rate": round(len(recovered) / len(definite), 4) if definite else None,
297
+ "recovery_interval_95": _interval(len(recovered), len(definite)),
298
+ "min_recovered_size_mean": round(sum(sizes) / len(sizes), 3) if sizes else None,
299
+ "reference_premises_mean": round(sum(refs) / len(refs), 3) if refs else None,
300
+ "badges": dict(Counter(r["badge"] for r in parsed_rows if r.get("badge"))),
301
+ "environment": environment(lean_bin),
302
+ }
303
+ (out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
304
+ return summary
305
+
306
+
307
+ def wilson_interval(successes: int, n: int, z: float = 1.959964) -> tuple[float, float] | None:
308
+ """Wilson score interval for a proportion (95 % by default). None when there is no observation."""
309
+ if n == 0:
310
+ return None
311
+ p = successes / n
312
+ centre = (p + z * z / (2 * n)) / (1 + z * z / n)
313
+ half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
314
+ return max(0.0, centre - half), min(1.0, centre + half)
315
+
316
+
317
+ def mcnemar_exact(only_first: int, only_second: int) -> float:
318
+ """Two-sided exact McNemar p-value from the two discordant counts of a paired comparison."""
319
+ n = only_first + only_second
320
+ if n == 0:
321
+ return 1.0
322
+ tail = sum(math.comb(n, i) for i in range(min(only_first, only_second) + 1)) / 2 ** n
323
+ return min(1.0, 2 * tail)
324
+
325
+
326
+ def _interval(successes: int, n: int) -> list[float] | None:
327
+ interval = wilson_interval(successes, n)
328
+ return [round(interval[0], 4), round(interval[1], 4)] if interval else None
329
+
330
+
331
+ REFUSALS_IN_A_ROW = 3
332
+
333
+
334
+ class ConfigurationMismatch(ValueError):
335
+ """The output directory already holds a run made with another mode, model or provider."""
336
+
337
+
338
+ def with_model(data_file: Path, out_dir: Path, client, mode: str, limit: int | None = None,
339
+ timeout_ms: int = 5000) -> dict:
340
+ """Run a language model on FOLIO validation: ``mode`` is "llm" (formalize, then solve) or "direct".
341
+
342
+ Results are appended to ``runs.jsonl`` as they arrive, so an interrupted run can be resumed.
343
+ A provider failure stops the run instead of being scored as a wrong answer. A request the
344
+ provider refuses as such (HTTP 400 or 422) is recorded as ``ProviderRefused``, unless
345
+ ``REFUSALS_IN_A_ROW`` of them come one after the other: that is a setting the provider does
346
+ not accept, not a property of the examples, so the run stops and none of them is recorded.
347
+ """
348
+ from . import formalize # imported here: the reference benchmark must not depend on it
349
+ if mode not in ("llm", "direct"):
350
+ raise ValueError('mode must be "llm" or "direct"')
351
+ examples = folio.load(data_file)[:limit]
352
+ out_dir.mkdir(parents=True, exist_ok=True)
353
+
354
+ describe = getattr(client, "describe", None)
355
+ config = {"mode": mode, "client": describe() if callable(describe) else {"model": type(client).__name__}}
356
+ config_file = out_dir / "config.json"
357
+ if config_file.exists():
358
+ previous = json.loads(config_file.read_text(encoding="utf-8"))
359
+ if previous != config:
360
+ raise ConfigurationMismatch(
361
+ f"{out_dir} holds a run made with {previous}; use an empty directory for {config}")
362
+ else:
363
+ config_file.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
364
+
365
+ log = out_dir / "runs.jsonl"
366
+ done = {}
367
+ if log.exists():
368
+ for line in log.read_text(encoding="utf-8").splitlines():
369
+ row = json.loads(line)
370
+ done[row["index"]] = row
371
+ refused: list[dict] = [] # refusals not yet written: they may turn out to be a configuration problem
372
+
373
+ def record(fh, row: dict) -> None:
374
+ fh.write(json.dumps(row, ensure_ascii=False) + "\n")
375
+ fh.flush()
376
+ done[row["index"]] = row
377
+
378
+ with open(log, "a", encoding="utf-8") as fh:
379
+ for ex in examples:
380
+ if ex.index in done:
381
+ continue
382
+ row = {"index": ex.index, "gold": ex.label, "mode": mode}
383
+ started = time.time()
384
+ try:
385
+ if mode == "direct":
386
+ row["verdict"], usage = formalize.direct(client, ex.premises_text, ex.conclusion_text)
387
+ row["usage"] = [usage]
388
+ else:
389
+ verdict, result = formalize.answer(client, ex.premises_text, ex.conclusion_text,
390
+ timeout_ms=timeout_ms)
391
+ row.update(verdict=verdict.label, attempts=result.attempts, usage=result.usage,
392
+ formulas_by_premise=result.by_sentence, conclusion_formula=result.conclusion_text)
393
+ except formalize.FormalizationError as exc:
394
+ row.update(verdict="NoFormalization", attempts=len(exc.history))
395
+ except formalize.ProviderError as exc:
396
+ if exc.status not in (400, 422):
397
+ raise
398
+ row.update(verdict="ProviderRefused", note=str(exc)[:300], seconds=round(time.time() - started, 2))
399
+ refused.append(row)
400
+ if len(refused) >= REFUSALS_IN_A_ROW:
401
+ raise formalize.ProviderError(
402
+ f"{len(refused)} requests in a row were refused, the last with: {exc}. "
403
+ "That is a setting the provider does not accept (the model name, response_format, "
404
+ "temperature, max_tokens), not the examples; nothing was recorded for them.",
405
+ exc.status) from None
406
+ continue
407
+ row["seconds"] = round(time.time() - started, 2)
408
+ for earlier in refused: # an answer followed: those refusals were about their own examples
409
+ record(fh, earlier)
410
+ refused.clear()
411
+ record(fh, row)
412
+ for earlier in refused:
413
+ record(fh, earlier)
414
+ rows = [done[ex.index] for ex in examples]
415
+ answered = [r for r in rows if r["verdict"] in LABELS]
416
+ correct = sum(r["verdict"] == r["gold"] for r in rows)
417
+ correct_answered = sum(r["verdict"] == r["gold"] for r in answered)
418
+ tokens = Counter()
419
+ for r in rows:
420
+ for used in r.get("usage", []):
421
+ for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
422
+ if isinstance(used.get(key), int):
423
+ tokens[key] += used[key]
424
+ confusion = Counter((r["gold"], r["verdict"]) for r in rows)
425
+ summary = {
426
+ **config,
427
+ "dataset": {"name": "FOLIO validation", "examples": len(rows),
428
+ "revision": folio.revision(data_file.parents[2]), "licence": "CC BY-SA 4.0"},
429
+ "examples": len(rows),
430
+ "correct": correct,
431
+ "accuracy": round(correct / len(rows), 4) if rows else None,
432
+ "accuracy_interval_95": _interval(correct, len(rows)),
433
+ "answered": len(answered),
434
+ "accuracy_when_answered": round(correct_answered / len(answered), 4) if answered else None,
435
+ "accuracy_when_answered_interval_95": _interval(correct_answered, len(answered)),
436
+ "verdict_counts": dict(Counter(r["verdict"] for r in rows)),
437
+ "confusion_gold_then_verdict": {f"{g} -> {v}": n for (g, v), n in sorted(confusion.items())},
438
+ "tokens": dict(tokens),
439
+ "environment": environment(None),
440
+ }
441
+ (out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
442
+ return summary
443
+
444
+
445
+ def _runs(directory: Path) -> dict[int, dict]:
446
+ rows = {}
447
+ for line in (directory / "runs.jsonl").read_text(encoding="utf-8").splitlines():
448
+ row = json.loads(line)
449
+ rows[row["index"]] = row
450
+ return rows
451
+
452
+
453
+ def compare(first_dir: Path, second_dir: Path) -> dict:
454
+ """Paired comparison of two model runs on the examples both of them answered or failed.
455
+
456
+ The two runs see the same examples, so the test is McNemar's: only the examples on
457
+ which exactly one run is right carry information about which run is better.
458
+ """
459
+ first, second = _runs(first_dir), _runs(second_dir)
460
+ shared = sorted(set(first) & set(second))
461
+ if not shared:
462
+ raise ValueError("the two runs have no example in common")
463
+ for index in shared:
464
+ if first[index]["gold"] != second[index]["gold"]:
465
+ raise ValueError(f"example {index} has different labels in the two runs: not the same dataset")
466
+ right_first = {i for i in shared if first[i]["verdict"] == first[i]["gold"]}
467
+ right_second = {i for i in shared if second[i]["verdict"] == second[i]["gold"]}
468
+ only_first, only_second = len(right_first - right_second), len(right_second - right_first)
469
+
470
+ def side(directory: Path, right: set[int]) -> dict:
471
+ config = json.loads((directory / "config.json").read_text(encoding="utf-8")) \
472
+ if (directory / "config.json").exists() else {}
473
+ return {"directory": str(directory), **config, "correct": len(right),
474
+ "accuracy": round(len(right) / len(shared), 4), "accuracy_interval_95": _interval(len(right), len(shared))}
475
+
476
+ return {
477
+ "examples_compared": len(shared),
478
+ "first": side(first_dir, right_first),
479
+ "second": side(second_dir, right_second),
480
+ "both_right": len(right_first & right_second),
481
+ "only_first_right": only_first,
482
+ "only_second_right": only_second,
483
+ "neither_right": len(shared) - len(right_first | right_second),
484
+ # Four significant figures, not six decimals: a p of 2e-8 must not be reported as 0.
485
+ "mcnemar_exact_p": float(f"{mcnemar_exact(only_first, only_second):.4g}"),
486
+ }
487
+
488
+
489
+ RUN_NAMES = {"direct": "The model answers itself", "llm": "The model writes formulas, the solver answers"}
490
+
491
+
492
+ def _percent(value: float | None) -> str:
493
+ # Two decimals: the summaries store proportions to four places, so nothing is rounded twice.
494
+ return "n/a" if value is None else f"{100 * value:.2f} %"
495
+
496
+
497
+ def _reference_on(indices: list[int], published: Path) -> tuple[int, int] | None:
498
+ """How many of these examples the reference run gets right, from its per-example file."""
499
+ try:
500
+ with open(published / "per_example.csv", newline="", encoding="utf-8") as fh:
501
+ agrees = {int(row["index"]): row["agrees"] == "True" for row in csv.DictReader(fh)}
502
+ except (OSError, KeyError, ValueError):
503
+ return None
504
+ if not all(i in agrees for i in indices):
505
+ return None
506
+ return sum(agrees[i] for i in indices), len(indices)
507
+
508
+
509
+ def report(first_dir: Path, second_dir: Path, published: Path | None = None) -> str:
510
+ """Markdown summary of two model runs: each run's accuracy with its interval, then the paired comparison."""
511
+ comparison = compare(first_dir, second_dir)
512
+ summaries = [json.loads((d / "summary.json").read_text(encoding="utf-8")) for d in (first_dir, second_dir)]
513
+ client = summaries[0].get("client", {})
514
+ lines = [
515
+ f"## FOLIO validation with `{client.get('model', 'an unnamed model')}` ({client.get('host', 'unknown host')})",
516
+ "",
517
+ "| Run | Examples | Correct | Accuracy | 95 % interval | Answered | Accuracy when answered | Tokens |",
518
+ "|---|---|---|---|---|---|---|---|",
519
+ ]
520
+ for summary in summaries:
521
+ interval = summary.get("accuracy_interval_95")
522
+ lines.append("| {name} | {n} | {correct} | {accuracy} | {interval} | {answered} | {when} | {tokens} |".format(
523
+ name=RUN_NAMES.get(summary.get("mode"), summary.get("mode", "?")),
524
+ n=summary["examples"], correct=summary["correct"], accuracy=_percent(summary["accuracy"]),
525
+ interval=f"{_percent(interval[0])} to {_percent(interval[1])}" if interval else "n/a",
526
+ answered=summary["answered"], when=_percent(summary["accuracy_when_answered"]),
527
+ tokens=summary.get("tokens", {}).get("total_tokens")
528
+ or sum(summary.get("tokens", {}).get(k, 0) for k in ("prompt_tokens", "completion_tokens")) or "not reported"))
529
+ lines += [
530
+ "",
531
+ f"On the {comparison['examples_compared']} examples both runs cover: both right on "
532
+ f"{comparison['both_right']}, only the first on {comparison['only_first_right']}, only the second on "
533
+ f"{comparison['only_second_right']}, neither on {comparison['neither_right']}. "
534
+ f"Exact McNemar test on the two middle counts: p = {comparison['mcnemar_exact_p']:.4g}.",
535
+ ]
536
+ published = published or Path(__file__).resolve().parents[2] / "results" / "folio-validation"
537
+ shared = sorted(set(_runs(first_dir)) & set(_runs(second_dir)))
538
+ reference_run = _reference_on(shared, published)
539
+ if reference_run:
540
+ lines += ["", f"For scale, the dataset's own formulas through the same solver agree with the label on "
541
+ f"{reference_run[0]} of these {reference_run[1]} examples."]
542
+ for summary in summaries:
543
+ counts = ", ".join(f"{verdict}: {n}" for verdict, n in sorted(summary["verdict_counts"].items()))
544
+ lines += ["", f"{RUN_NAMES.get(summary.get('mode'), summary.get('mode', '?'))}, verdicts given: {counts}."]
545
+ lines += ["", "An answer that is not one of True, False, Uncertain (no usable formalization, an inconsistent "
546
+ "one, a refused request) counts as wrong in \"Accuracy\" and is left out of \"Answered\"."]
547
+ return "\n".join(lines) + "\n"