pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
cme/rollout_card.py ADDED
@@ -0,0 +1,137 @@
1
+ """Export a CME compute block in a form that sits inside a rollout card (Masters, Liu and Albrecht 2026, arXiv:2605.12131).
2
+
3
+ What the rollout-card paper specifies (its Appendix E): a card is a logical directory with manifest.json, five append-only JSONL streams (events, nodes, edges,
4
+ annotations, mutations) and optional blobs; a card lists the reporting rules behind every reported number, each with a name, an implementation or version,
5
+ a configuration, input streams, an output target and a drops manifest; and new metadata goes into annotations.jsonl under a namespace. The paper has no dedicated
6
+ compute section.
7
+
8
+ What this module does: it maps the CME resource vector R and the frontier gap onto those two extension points, without claiming more.
9
+ * `annotations`: rows in the annotations-stream schema (target_type, target_id, namespace, sequence, payload, created_at) with namespace "cme.compute",
10
+ one row for the MAS resource vector, one for the SAS vector and ratios (if a baseline is present), one for the frontier-gap result;
11
+ * `reporting_rules`: one entry for the frontier-gap rule with its configuration and a drops manifest (fields read, structures collapsed, loss classes).
12
+ Limits, repeated in every export: this is a compute block, not a complete rollout card (no events, nodes or edges are written); the manifest key names are not
13
+ listed in the rollout-card paper, so none are invented here; and the output has not been checked with the reference validator (ERGON), which is not
14
+ installed here. Nothing is estimated: a dimension that the log does not report stays null.
15
+ Formats: JSON, YAML (written by a small emitter, no dependency) and a one-page LaTeX block.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import datetime
20
+ import json
21
+ from typing import Any, Dict, List, Optional
22
+
23
+ SPEC = "arXiv:2605.12131"
24
+ NAMESPACE = "cme.compute"
25
+ DIMENSIONS = ("r_reason", "r_token", "r_cost", "r_lat", "r_sample", "r_sys")
26
+ DIM_LABEL = {"r_reason": "thinking tokens", "r_token": "total tokens", "r_cost": "cost (USD)", "r_lat": "latency (s)", "r_sample": "independent samples", "r_sys": "tool calls"}
27
+ LIMITS = ("A compute block, not a complete rollout card: no events, nodes or edges are included. Manifest key names are not specified in the rollout-card paper and "
28
+ "are not invented here. Not checked with the reference validator.")
29
+
30
+
31
+ def build_block(rep: dict, run_id: str = "run", created_at: Optional[str] = None, version: str = "0.1") -> Dict[str, Any]:
32
+ """Build the compute block from a `cme.build_report` result. `created_at` defaults to the current UTC time (ISO 8601)."""
33
+ when = created_at or datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()
34
+ rc = rep.get("resource_card") or {}
35
+ g = rep["frontier_gap"]
36
+ rows: List[dict] = []
37
+
38
+ def add(payload: dict) -> None:
39
+ rows.append({"target_type": "run", "target_id": run_id, "namespace": NAMESPACE, "sequence": len(rows) + 1, "payload": payload, "created_at": when})
40
+ add({"kind": "resource_vector", "system": rep.get("system", "mas"), "unit": "per item (mean)", "dimensions": {d: (rc.get("mas_per_item") or {}).get(d) for d in DIMENSIONS}})
41
+ if rc.get("sas_per_item") is not None:
42
+ add({"kind": "resource_vector", "system": "sas", "unit": "per item (mean)", "dimensions": {d: rc["sas_per_item"].get(d) for d in DIMENSIONS}})
43
+ add({"kind": "ratios_mas_over_sas", "dimensions": {d: rc["ratios_mas_over_sas"].get(d) for d in DIMENSIONS},
44
+ "control_states": rc.get("control_states"), "note": "null: not reported by a side, or 0/0"})
45
+ add({"kind": "frontier_gap", "estimate": g["gap"], "ci95": [g["ci_low"], g["ci_high"]], "interval": "paired bootstrap over items", "n_items": g["n_items"],
46
+ "system_accuracy": g["system_accuracy"], "frontier_accuracy_at_system_tokens": g["frontier_at_system_tokens"], "system_mean_tokens": g["system_mean_tokens"],
47
+ "range_status": g["range_status"], "greedy_accuracy": rep.get("greedy_accuracy"), "pool_k": rep["pool_k"]})
48
+ from . import __version__
49
+ rule = {"name": "cme.frontier_gap", "implementation": "cme %s (cme.frontier.frontier_gap)" % __version__,
50
+ "configuration": {"n_boot": g["n_boot"], "pool_k": rep["pool_k"], "tokens": "prompt plus completion tokens of every call of the item",
51
+ "frontier": "isotonic (pool-adjacent-violators) self-consistency accuracy at the system's mean tokens"},
52
+ "input_streams": ["per-call usage log (cme run log)", "greedy and sampled answer pool (cme baseline)"],
53
+ "output_target": "annotations.jsonl, namespace %s" % NAMESPACE,
54
+ "drops_manifest": {"fields_read": ["item id", "correct", "prompt_tokens", "completion_tokens", "latency"],
55
+ "collapsed": ["per-call records summed to per-item totals", "agents and rounds pooled"],
56
+ "loss_classes": ["message content not read", "agent identity erased", "round structure erased", "items without a complete pool dropped (%d)" % rep.get("pool_items_dropped_incomplete", 0),
57
+ "calls without a usage block not counted (%d)" % rep.get("system_calls_missing_usage", 0)]}}
58
+ return {"format": "cme-compute-block", "version": version, "targets": "rollout card (%s), annotations namespace and reporting-rule registry" % SPEC, "limits": LIMITS,
59
+ "annotations": rows, "reporting_rules": [rule]}
60
+
61
+
62
+ def _scalar(v: Any) -> str:
63
+ if v is None:
64
+ return "null"
65
+ if isinstance(v, bool):
66
+ return "true" if v else "false"
67
+ if isinstance(v, (int, float)):
68
+ return repr(v)
69
+ s = str(v)
70
+ return json.dumps(s) if (not s or s != s.strip() or any(c in s for c in ':#{}[],&*!|>\'"%@`\n') or s.lower() in ("null", "true", "false", "yes", "no")
71
+ or s[0] in "-?0123456789") else s
72
+
73
+
74
+ def to_yaml(obj: Any, indent: int = 0) -> str:
75
+ """Minimal YAML emitter for nested dicts, lists and scalars (block style)."""
76
+ pad = " " * indent
77
+ if isinstance(obj, dict):
78
+ if not obj:
79
+ return pad + "{}\n"
80
+ out = ""
81
+ for k, v in obj.items():
82
+ if isinstance(v, (dict, list)) and v:
83
+ out += "%s%s:\n%s" % (pad, k, to_yaml(v, indent + 1))
84
+ else:
85
+ out += "%s%s: %s\n" % (pad, k, "{}" if isinstance(v, dict) else "[]" if isinstance(v, list) else _scalar(v))
86
+ return out
87
+ if isinstance(obj, list):
88
+ out = ""
89
+ for v in obj:
90
+ if isinstance(v, (dict, list)) and v:
91
+ body = to_yaml(v, indent + 1)
92
+ out += "%s- %s" % (pad, body[len(pad) + 2:])
93
+ else:
94
+ out += "%s- %s\n" % (pad, _scalar(v))
95
+ return out
96
+ return pad + _scalar(obj) + "\n"
97
+
98
+
99
+ def _tex(s: str) -> str:
100
+ bs = chr(92)
101
+ return s.replace(bs, "").replace("_", bs + "_").replace("%", bs + "%").replace("&", bs + "&").replace("#", bs + "#")
102
+
103
+
104
+ def to_latex(block: dict, title: str = "CME compute block") -> str:
105
+ """One-page LaTeX block: resource vector (MAS, SAS, ratio) and the frontier-gap line. Null cells print n/a."""
106
+ bs = chr(92)
107
+ ann = {a["payload"]["kind"] + ":" + str(a["payload"].get("system", "")): a["payload"] for a in block["annotations"]}
108
+ mas = next(p for p in (a["payload"] for a in block["annotations"]) if p["kind"] == "resource_vector")
109
+ sas = next((a["payload"] for a in block["annotations"] if a["payload"]["kind"] == "resource_vector" and a["payload"]["system"] == "sas"), None)
110
+ rat = next((a["payload"] for a in block["annotations"] if a["payload"]["kind"] == "ratios_mas_over_sas"), None)
111
+ gap = next(a["payload"] for a in block["annotations"] if a["payload"]["kind"] == "frontier_gap")
112
+
113
+ def f(v, nd=3):
114
+ return "n/a" if v is None else ("inf" if v == float("inf") else ("%.0f" % v if abs(v) >= 100 else "%.*g" % (nd, v)))
115
+ lines = [bs + "begin{table}[t]", bs + "centering", bs + "small",
116
+ bs + "caption{" + _tex(title) + " (namespace " + bs + "texttt{" + _tex(NAMESPACE) + "}). Per-item means from the run log; n/a: not reported by the log. "
117
+ "Compute block for a rollout card (" + SPEC.replace("arXiv:", "arXiv:") + "), not a complete card.}",
118
+ bs + "label{tab:cme_compute_block}", bs + "begin{tabular}{lrrr}", bs + "toprule", "Dimension & " + _tex(mas["system"].upper()) + " & SAS & Ratio " + bs + bs, bs + "midrule"]
119
+ for d in DIMENSIONS:
120
+ lines.append("%s (%s) & %s & %s & %s %s" % (_tex(d), _tex(DIM_LABEL[d]), f(mas["dimensions"][d]), f(sas["dimensions"][d]) if sas else "n/a", f(rat["dimensions"][d]) if rat else "n/a", bs + bs))
121
+ lines += [bs + "midrule",
122
+ bs + "multicolumn{4}{l}{Frontier gap $%+.3f$, 95%% paired bootstrap interval [$%+.3f$, $%+.3f$] over %d items; frontier at the system's mean tokens %.3f (%s).} %s" % (
123
+ gap["estimate"], gap["ci95"][0], gap["ci95"][1], gap["n_items"], gap["frontier_accuracy_at_system_tokens"], _tex(gap["range_status"]), bs + bs),
124
+ bs + "bottomrule", bs + "end{tabular}", bs + "end{table}"]
125
+ return "\n".join(lines).replace("95%", "95" + bs + "%") + "\n"
126
+
127
+
128
+ def write_block(rep: dict, prefix: str, run_id: str = "run") -> List[str]:
129
+ """Write PREFIX.json, PREFIX.yaml and PREFIX.tex; return the paths."""
130
+ block = build_block(rep, run_id)
131
+ paths = []
132
+ for ext, text in (("json", json.dumps(block, indent=2) + "\n"), ("yaml", to_yaml(block)), ("tex", to_latex(block))):
133
+ p = "%s.%s" % (prefix, ext)
134
+ with open(p, "w", encoding="utf-8") as fh:
135
+ fh.write(text)
136
+ paths.append(p)
137
+ return paths
cme/tasks.py ADDED
@@ -0,0 +1,266 @@
1
+ """Task loaders and scorers for the controlled experiment (Phase 2 pilots).
2
+
3
+ Data come from the public Hugging Face datasets-server JSON API (no parquet reader needed) and are cached under
4
+ data/experiments/<task>.jsonl with a provenance header. Subsets are drawn with a fixed seed (42) from the filtered pool so a run is
5
+ reproducible. Items share one schema:
6
+ {"id", "task", "prompt", "gold", "kind": "mc" | "num" | "text" | "code", "meta", "views": optional list of agent-specific prompts}
7
+ `views` is used only by the split-evidence positive control, where each agent sees part of the evidence.
8
+
9
+ Substitutions to disclose: plain MBPP (the `test_list` asserts, 500 test items) stands in for MBPP+; the positive-control items are
10
+ synthetic (generated here, seed 42), they are not a published benchmark.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import os
16
+ import random
17
+ import re
18
+ import subprocess
19
+ import sys
20
+ import tempfile
21
+ import time
22
+ import urllib.error
23
+ import urllib.parse
24
+ import urllib.request
25
+ from typing import Dict, List, Optional
26
+
27
+ ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
28
+ DATA = os.path.join(ROOT, "data", "experiments")
29
+ API = "https://datasets-server.huggingface.co/rows"
30
+ LETTERS = "ABCDEFGHIJ"
31
+
32
+ SOURCES = {
33
+ "mmlu_pro": ("TIGER-Lab/MMLU-Pro", "default", "test"),
34
+ "math500_l45": ("HuggingFaceH4/MATH-500", "default", "test"),
35
+ "musique_4hop": ("bdsaglam/musique", "answerable", "validation"),
36
+ "mbpp": ("google-research-datasets/mbpp", "full", "test"),
37
+ "gsm8k": ("openai/gsm8k", "main", "test"),
38
+ }
39
+
40
+
41
+ def _get(dataset: str, config: str, split: str, offset: int, length: int) -> List[dict]:
42
+ q = urllib.parse.urlencode({"dataset": dataset, "config": config, "split": split, "offset": offset, "length": length})
43
+ for attempt in range(8):
44
+ try:
45
+ with urllib.request.urlopen(API + "?" + q, timeout=60) as r:
46
+ return [x["row"] for x in json.loads(r.read())["rows"]]
47
+ except urllib.error.HTTPError as e:
48
+ time.sleep(min(120, float(e.headers.get("Retry-After", 0) or 0) or 20 * (attempt + 1))) # 429 rate limit: back off
49
+ except Exception:
50
+ time.sleep(2 * (attempt + 1))
51
+ raise RuntimeError("datasets-server unavailable for %s offset %d" % (dataset, offset))
52
+
53
+
54
+ def _fetch_all(dataset: str, config: str, split: str, total: int) -> List[dict]:
55
+ rows: List[dict] = []
56
+ for off in range(0, total, 100):
57
+ rows += _get(dataset, config, split, off, min(100, total - off))
58
+ return rows
59
+
60
+
61
+ def _mc_prompt(q: str, opts: List[str]) -> str:
62
+ return q.strip() + "\n\nOptions:\n" + "\n".join("%s. %s" % (LETTERS[i], o) for i, o in enumerate(opts)) + "\n\nGive the letter of the correct option."
63
+
64
+
65
+ def load_task(task: str, n: int, seed: int = 42, refresh: bool = False, stratify: bool = False) -> List[dict]:
66
+ """Return n items for a task (deterministic subset). Fetches and caches the filtered pool on first use."""
67
+ if task == "hidden_profile":
68
+ return hidden_profile(n, seed)
69
+ os.makedirs(DATA, exist_ok=True)
70
+ path = os.path.join(DATA, task + ".jsonl")
71
+ if refresh or not os.path.exists(path):
72
+ ds, cfg, sp = SOURCES[task]
73
+ if task == "mmlu_pro":
74
+ starts = sorted(random.Random(seed).sample(range(0, 12000, 100), 40)) # forty seeded 100-row blocks over the whole split (about 4,000 rows); the split is ordered by category, so one block is one category and 40 blocks cover all of them
75
+ rows = []
76
+ for st in starts:
77
+ rows += _get(ds, cfg, sp, st, 100)
78
+ time.sleep(0.5) # stay under the datasets-server rate limit
79
+ items = [{"id": "mmlu%05d" % r["question_id"], "task": task, "prompt": _mc_prompt(r["question"], r["options"]), "gold": r["answer"],
80
+ "kind": "mc", "meta": {"category": r["category"], "n_options": len(r["options"])}} for r in rows]
81
+ elif task == "math500_l45":
82
+ rows = [r for r in _fetch_all(ds, cfg, sp, 500) if r["level"] in (4, 5)]
83
+ items = [{"id": r["unique_id"], "task": task, "prompt": r["problem"].strip(), "gold": r["answer"], "kind": "num",
84
+ "meta": {"level": r["level"], "subject": r["subject"]}} for r in rows]
85
+ elif task == "musique_4hop":
86
+ rows = [r for r in _fetch_all(ds, cfg, sp, 2417) if str(r["id"]).startswith("4hop")]
87
+ items = []
88
+ for r in rows:
89
+ ctx = "\n\n".join("[%s] %s" % (p["title"], p["paragraph_text"]) for p in r["paragraphs"])
90
+ items.append({"id": r["id"], "task": task, "prompt": "Paragraphs:\n" + ctx + "\n\nQuestion: " + r["question"] + "\nAnswer with a short phrase.",
91
+ "gold": [r["answer"]] + list(r.get("answer_aliases") or []), "kind": "text", "meta": {}})
92
+ elif task == "gsm8k":
93
+ rows = _fetch_all(ds, cfg, sp, 1319)
94
+ items = [{"id": "gsm%04d" % i, "task": task, "prompt": r["question"].strip(), "gold": r["answer"].split("####")[-1].strip().replace(",", ""), "kind": "num", "meta": {}}
95
+ for i, r in enumerate(rows)]
96
+ elif task == "mbpp":
97
+ rows = _fetch_all(ds, cfg, sp, 500)
98
+ items = [{"id": "mbpp%d" % r["task_id"], "task": task,
99
+ "prompt": r["text"].strip() + "\nYour code must pass this test:\n" + r["test_list"][0] + "\nReturn only the Python function in a ```python block.",
100
+ "gold": {"tests": r["test_list"], "setup": r["test_setup_code"]}, "kind": "code", "meta": {}} for r in rows]
101
+ else:
102
+ raise ValueError("unknown task " + task)
103
+ with open(path, "w", encoding="utf-8") as fh:
104
+ fh.write(json.dumps({"_provenance": {"source": SOURCES[task], "api": API, "fetched": time.strftime("%Y-%m-%d"), "pool_size": len(items), "seed": seed}}) + "\n")
105
+ for it in items:
106
+ fh.write(json.dumps(it) + "\n")
107
+ items = [json.loads(l) for l in open(path, encoding="utf-8") if l.strip() and not l.startswith('{"_provenance"')]
108
+ random.Random(seed).shuffle(items)
109
+ if stratify: # round-robin over categories (MMLU-Pro category, MATH subject) in the seeded order, so each stratum contributes about equally
110
+ key = lambda it: str(it["meta"].get("category") or it["meta"].get("subject") or "")
111
+ groups = {}
112
+ for it in items:
113
+ groups.setdefault(key(it), []).append(it)
114
+ picked, i = [], 0
115
+ while len(picked) < n and any(i < len(g) for g in groups.values()):
116
+ picked += [g[i] for _, g in sorted(groups.items()) if i < len(g)]
117
+ i += 1
118
+ return picked[:n]
119
+ return items[:n]
120
+
121
+
122
+ # ----------------------------------------------------------------------------- scoring
123
+ def _norm_num(s: str) -> str:
124
+ s = str(s).strip().strip("$").replace("\\!", "").replace("\\,", "").replace("\\left", "").replace("\\right", "").replace("dfrac", "frac").replace("tfrac", "frac")
125
+ s = re.sub(r"\\text\{([^}]*)\}", r"\1", s)
126
+ s = re.sub(r"\\boxed\{(.*)\}", r"\1", s)
127
+ s = s.replace("\\$", "").replace("$", "").replace(" ", "").rstrip(".").lower()
128
+ s = re.sub(r"^(-?\d+)/(\d+)$", r"\\frac{\1}{\2}", s)
129
+ s = re.sub(r"^(-?\d+)\.0+$", r"\1", s)
130
+ s = re.sub(r"(degrees?|units?)$", "", s)
131
+ s = s.replace("^{\\circ}", "").replace("^\\circ", "").replace("\\%", "").replace("%", "")
132
+ return s.replace(",", "") if re.fullmatch(r"-?[\d,]+(\.\d+)?", s) else s
133
+
134
+
135
+ def _norm_text(s: str) -> str:
136
+ s = re.sub(r"\b(a|an|the)\b", " ", str(s).lower())
137
+ return " ".join(re.sub(r"[^\w\s]", "", s).split())
138
+
139
+
140
+ _ANS = re.compile(r"answer\s*[:=]\s*(.+)", re.I)
141
+ _CONF = re.compile(r"confidence\s*[:=]\s*(\d+(?:\.\d+)?)", re.I)
142
+
143
+
144
+ BOXED = "\\boxed{"
145
+
146
+
147
+ def last_boxed(text: str) -> str:
148
+ """Contents of the last boxed{...} expression (brace-balanced), or ''."""
149
+ text = text or ""
150
+ i = text.rfind(BOXED)
151
+ if i < 0:
152
+ return ""
153
+ start = i + len(BOXED)
154
+ j, depth = start, 1
155
+ while j < len(text) and depth:
156
+ depth += {"{": 1, "}": -1}.get(text[j], 0)
157
+ j += 1
158
+ return text[start: j - 1] if depth == 0 else ""
159
+
160
+
161
+ _IS = re.compile(r"answer is:?\s*\$?\(?([^\n$]+)", re.I)
162
+
163
+
164
+ def parse_answer(text: str) -> str:
165
+ """Final 'Answer:' payload (first line of it); else the last boxed content; else the payload of 'the answer is ...'; else ''."""
166
+ m = _ANS.findall(text or "")
167
+ if m:
168
+ got = m[-1].strip().split("\n")[0].strip()
169
+ return last_boxed(got) or got
170
+ return last_boxed(text) or ((_IS.findall(text or "") or [""])[-1].strip().rstrip(".)"))
171
+
172
+
173
+ def parse_confidence(text: str) -> Optional[float]:
174
+ m = _CONF.findall(text or "")
175
+ return min(100.0, max(0.0, float(m[-1]))) / 100.0 if m else None
176
+
177
+
178
+ def extract_code(text: str) -> str:
179
+ m = re.findall(r"```(?:python)?\n(.*?)```", text or "", re.S)
180
+ return m[-1] if m else (text or "")
181
+
182
+
183
+ def run_tests(code: str, tests: List[str], setup: str = "", timeout: float = 10.0) -> bool:
184
+ """Execute generated code plus the asserts in a separate Python process with a timeout (no sandbox beyond that; run on a trusted dev machine)."""
185
+ src = (setup or "") + "\n" + code + "\n" + "\n".join(tests) + "\n"
186
+ with tempfile.TemporaryDirectory() as d:
187
+ p = os.path.join(d, "t.py")
188
+ open(p, "w", encoding="utf-8").write(src)
189
+ try:
190
+ return subprocess.run([sys.executable, "-I", p], cwd=d, capture_output=True, timeout=timeout).returncode == 0
191
+ except subprocess.TimeoutExpired:
192
+ return False
193
+
194
+
195
+ def normalize_answer(item: dict, ans: str) -> str:
196
+ k = item["kind"]
197
+ if k == "mc":
198
+ m = re.search(r"\b([A-J])\b", (ans or "").upper())
199
+ return m.group(1) if m else ""
200
+ if k == "num":
201
+ return _norm_num(ans)
202
+ if k == "text":
203
+ return _norm_text(ans)
204
+ if k == "name":
205
+ m = re.search(r"\b(avery|blake|casey|drew)\b", (ans or "").lower())
206
+ return m.group(1) if m else ""
207
+ return (ans or "").strip()
208
+
209
+
210
+ def score(item: dict, ans_text: str) -> bool:
211
+ """Is a response correct? For code, ans_text is the full response (code is extracted and run)."""
212
+ if item["kind"] == "code":
213
+ return run_tests(extract_code(ans_text), item["gold"]["tests"], item["gold"].get("setup", ""))
214
+ a = normalize_answer(item, parse_answer(ans_text))
215
+ g = item["gold"]
216
+ if item["kind"] == "text":
217
+ return a != "" and any(a == _norm_text(x) for x in g)
218
+ if item["kind"] == "num":
219
+ return a != "" and a == _norm_num(g)
220
+ if item["kind"] == "name":
221
+ return a == str(g).lower()
222
+ return a == str(g).strip().upper()
223
+
224
+
225
+ # ----------------------------------------------------------------------------- positive control
226
+ def hidden_profile(n: int, seed: int = 42, n_agents: int = 5) -> List[dict]:
227
+ """Synthetic split-evidence task (hidden-profile design, Stasser and Titus 1985).
228
+
229
+ Four candidates are scored 1-5 on 10 criteria with equal weights; the correct choice has the highest total. Criteria 1-4 are shared
230
+ (every agent sees them) and favour a decoy; criteria 5-10 are split so that each agent sees only two of the six and the pooled
231
+ total favours the target. A single agent shown all evidence (`prompt`) can compute the answer; one agent's `views[i]` cannot,
232
+ and independent voting over views is expected to favour the decoy. Generated, not published: report it as synthetic."""
233
+ rng = random.Random(seed)
234
+ names = ["Avery", "Blake", "Casey", "Drew"]
235
+ out = []
236
+ while len(out) < n:
237
+ target, decoy = rng.sample(range(4), 2)
238
+ sc = [[rng.randint(2, 4) for _ in range(10)] for _ in range(4)]
239
+ for j in range(4): # shared criteria: the decoy leads clearly
240
+ for c in range(4):
241
+ sc[c][j] = min(sc[c][j], 3)
242
+ sc[decoy][j] = 5
243
+ sc[target][j] = 2
244
+ for j in range(4, 10): # split criteria: the target leads
245
+ for c in range(4):
246
+ sc[c][j] = min(sc[c][j], 3)
247
+ sc[target][j] = 5
248
+ sc[decoy][j] = 2
249
+ tot = [sum(r) for r in sc]
250
+ if sorted(tot)[-1] == sorted(tot)[-2] or tot.index(max(tot)) != target:
251
+ continue
252
+ crit = ["criterion %d" % (j + 1) for j in range(10)]
253
+ facts = lambda js: "\n".join("%s on %s: %d" % (names[c], crit[j], sc[c][j]) for j in js for c in range(4))
254
+ head = "A committee must pick one of four candidates (Avery, Blake, Casey, Drew) using 10 equally weighted criteria, each scored 1 to 5. The best candidate has the highest total score.\n\n"
255
+ tail = "\n\nWhich candidate has the highest total over all 10 criteria? Give the name."
256
+ full = head + "All scores:\n" + facts(range(10)) + tail
257
+ order = list(range(4, 10))
258
+ rng.shuffle(order)
259
+ views = []
260
+ for a in range(n_agents):
261
+ mine = sorted([order[(2 * a) % 6], order[(2 * a + 1) % 6]])
262
+ views.append(head + "You see only part of the evidence. Scores you can see:\n" + facts(list(range(4)) + mine) +
263
+ "\n\nOther committee members see the other criteria. Which candidate has the highest total over all 10 criteria? Give the name.")
264
+ out.append({"id": "hp%04d" % len(out), "task": "hidden_profile", "prompt": full, "gold": names[target], "kind": "name", "meta": {"synthetic": True, "totals": tot},
265
+ "views": views})
266
+ return out
cme/verified_debate.py ADDED
@@ -0,0 +1,127 @@
1
+ """Route II harness and debate-trace export.
2
+
3
+ s12_verified_debate (strategy 12): numeric tasks only. Every response must end with 'Expression: ...', 'Answer: ...' and 'Confidence: ...'. Two debates start from the same round-1 responses:
4
+ S12u ungated: agents update freely (the same prompts as S12);
5
+ S12 gated: an agent's changed answer is accepted only if cme.verifier accepts that response (the expression evaluates to the stated answer); otherwise the agent keeps its
6
+ previous accepted response. The gate is deterministic and runs no model-written code.
7
+ The only difference between S12u and S12 is the gate, so the change in transition rates between them (right-to-wrong in particular) is attributable to the verifier. The verifier checks
8
+ arithmetic, not the reading of the problem (see cme.verifier), so it is partial.
9
+ Results carry round_answers and round_tokens per agent and round, which export_trace writes as the JSONL that `cme dynamics` reads.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import os
15
+ from typing import List, Optional
16
+
17
+ VSOLVE = ("{q}\n\nThink step by step. Finish with three final lines: 'Expression: <one arithmetic expression using only numbers and + - * / ( ) that computes the answer from the "
18
+ "quantities in the problem>', 'Answer: <answer>' and 'Confidence: <0-100>'.")
19
+ VDEBATE = ("{q}\n\nYour previous response:\n{own}\n\nOther agents responded:\n{peers}\n\nUse their reasoning as additional information, but verify it yourself. Give your updated solution. "
20
+ "Finish with three final lines: 'Expression: <one arithmetic expression using only numbers and + - * / ( )>', 'Answer: <answer>' and 'Confidence: <0-100>'.")
21
+
22
+
23
+ def _round1(ctx, item):
24
+ return [ctx.call("V1|%s|%s|%d" % (ctx.model, item["id"], i), ctx.model, VSOLVE.format(q=item["prompt"]), ctx.temperature, think=ctx.think_default) for i in range(ctx.agents)]
25
+
26
+
27
+ def _debate(ctx, item, name, r1, gated):
28
+ from . import experiment as E
29
+ from .verifier import verify
30
+ state = list(r1)
31
+ answers = [[E.ans_of(item, r) for r in state]]
32
+ tokens = [[[r.get("prompt_tokens"), r.get("completion_tokens")] for r in state]]
33
+ rejected = changed = 0
34
+ for rd in range(2, ctx.rounds + 1):
35
+ texts = [r["text"] for r in state]
36
+ nxt, tok = [], []
37
+ for j in range(len(state)):
38
+ peers = E._peers(ctx, texts, j, False, None)
39
+ new = ctx.call("%s|%s|%s|r%d|a%d" % (name, ctx.model, item["id"], rd, j), ctx.model, VDEBATE.format(q=item["prompt"], own=E._tail(texts[j]), peers=peers), ctx.temperature,
40
+ think=ctx.think_default, comm_chars=len(peers))
41
+ tok.append([new.get("prompt_tokens"), new.get("completion_tokens")])
42
+ old_a, new_a = E.ans_of(item, state[j]), E.ans_of(item, new)
43
+ if new_a != old_a:
44
+ if gated and not verify(new["text"])[0]:
45
+ rejected += 1
46
+ nxt.append(state[j])
47
+ continue
48
+ changed += 1
49
+ nxt.append(new)
50
+ state = nxt
51
+ answers.append([E.ans_of(item, r) for r in state])
52
+ tokens.append(tok)
53
+ leaders, cr = E.vote(answers[-1], E._is_gold(item))
54
+ return leaders, cr, {"round_answers": answers, "round_tokens": tokens, "rejected": rejected, "changed": changed}
55
+
56
+
57
+ def s12_verified_debate(ctx, item):
58
+ from . import experiment as E
59
+ if item["kind"] != "num":
60
+ raise ValueError("strategy 12 needs a numeric task")
61
+ r1 = _round1(ctx, item)
62
+ base = list(ctx.used)
63
+ lu, cu, eu = _debate(ctx, item, "S12u", r1, False)
64
+ res_u = E._result(ctx, "S12u", item, cu, "|".join(lu), eu)
65
+ ctx.used = list(base)
66
+ lg, cg, eg = _debate(ctx, item, "S12", r1, True)
67
+ res_g = E._result(ctx, "S12", item, cg, "|".join(lg), eg)
68
+ return [res_u, res_g]
69
+
70
+
71
+ def export_trace(out_dir: str, items: List[dict], strategy: str, model: str, agents: int, path: str) -> int:
72
+ """Write one JSONL record per agent answer and round for every finished item of `strategy`: item, agent, round, correct, prompt_tokens, completion_tokens, strategy.
73
+ Strategies S12 and S12u use the accepted answers stored in their results; S4, S4v, S5 and S7 are reconstructed from the cached calls (round 1 is the pool sample or the agent's view)."""
74
+ from . import experiment as E
75
+ by_id = {it["id"]: it for it in items}
76
+ store = E.Store(os.path.join(out_dir, "calls.jsonl"))
77
+ n = 0
78
+ with open(path, "w", encoding="utf-8") as fh:
79
+ for line in open(os.path.join(out_dir, "results.jsonl"), encoding="utf-8"):
80
+ if not line.strip():
81
+ continue
82
+ r = json.loads(line)
83
+ if r["strategy"] != strategy or r["item"] not in by_id:
84
+ continue
85
+ item, gold = by_id[r["item"]], E._is_gold(by_id[r["item"]])
86
+ rows = []
87
+ if "round_answers" in r:
88
+ for rd, (ans, tok) in enumerate(zip(r["round_answers"], r["round_tokens"]), 1):
89
+ for a, (an, (pt, ct)) in enumerate(zip(ans, tok)):
90
+ rows.append((a, rd, bool(gold(an)), pt, ct))
91
+ else:
92
+ rounds = max(int(k.split("|")[3][1:]) for k in r["keys"] if "|r" in k) if any("|r" in k for k in r["keys"]) else 1
93
+ for a in range(agents):
94
+ for rd in range(1, rounds + 1):
95
+ if rd == 1:
96
+ key = ("r1v|%s|%s|%d" if item.get("views") else "pool|%s|%s|%d") % (model, item["id"], a)
97
+ else:
98
+ key = "%s|%s|%s|r%d|a%d" % (strategy, "+".join([model] * agents), item["id"], rd, a) # the debate key names the model of every agent
99
+ rec = store.d.get(key)
100
+ if rec is None:
101
+ raise KeyError("call %s is not in the cache" % key)
102
+ rows.append((a, rd, bool(gold(E.ans_of(item, rec))), rec.get("prompt_tokens"), rec.get("completion_tokens")))
103
+ for a, rd, ok, pt, ct in rows:
104
+ rec = {"item": r["item"], "agent": a, "round": rd, "correct": ok, "strategy": strategy}
105
+ if pt is not None and ct is not None:
106
+ rec["prompt_tokens"], rec["completion_tokens"] = pt, ct
107
+ fh.write(json.dumps(rec) + "\n")
108
+ n += 1
109
+ return n
110
+
111
+
112
+ def _shuffle_sentences(body: str, q: str, j: int) -> str:
113
+ """Peer message with its sentences in a deterministic random order (same words, same length, the reasoning order destroyed)."""
114
+ import hashlib
115
+ import random
116
+ import re
117
+ parts = [x for x in re.split(r"(?<=[.!?])\s+", body.strip()) if x]
118
+ random.Random(int(hashlib.sha256(("%s|%d" % (body[:40], j)).encode()).hexdigest()[:8], 16)).shuffle(parts)
119
+ return " ".join(parts)
120
+
121
+
122
+ def s13_shuffled_debate(ctx, item):
123
+ """Debate whose peer messages have their sentences shuffled: the words and the length are kept, the argument order is not. Round 1 is the first `agents` pool samples."""
124
+ from . import experiment as E
125
+ leaders, cr, ex = E._debate(ctx, item, "S13", [ctx.model] * ctx.agents, E.round1(ctx, item, ctx.agents), False, False, peer_transform=_shuffle_sentences)
126
+ return E._result(ctx, "S13", item, cr, "|".join(leaders), ex)
127
+
cme/verifier.py ADDED
@@ -0,0 +1,92 @@
1
+ """Deterministic arithmetic verifier for Route II (an external check in the system's tool budget, R_sys > 0).
2
+
3
+ A response passes if it contains a line `Expression: <arithmetic expression>` and a line `Answer: <number>` and the expression, evaluated by a restricted parser, equals the answer.
4
+ The parser accepts numbers, the operators + - * / // % ** and parentheses, and nothing else: there are no names, calls, attributes or strings, and no code is executed (ast is used
5
+ only to parse). It checks that the calculation is consistent with the stated answer. It does not check that the expression models the problem, so a wrong plan with correct arithmetic
6
+ passes: the verifier is partial, with a false-positive rate for wrong answers above zero.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import ast
11
+ import operator
12
+ import re
13
+ from fractions import Fraction
14
+ from typing import Optional, Tuple
15
+
16
+ _BIN = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod, ast.Pow: operator.pow}
17
+ _UN = {ast.UAdd: operator.pos, ast.USub: operator.neg}
18
+ MAX_NODES, MAX_EXP, MAX_ABS = 200, 12, 10 ** 15
19
+
20
+
21
+ def _clean(expr: str) -> str:
22
+ e = expr.strip().strip("`$ ").rstrip(".")
23
+ e = e.replace("\u00d7", "*").replace("\u00f7", "/").replace("\u2212", "-").replace("^", "**").replace("\\times", "*").replace("\\cdot", "*").replace("\\div", "/")
24
+ e = re.sub(r"(?<=\d),(?=\d{3}\b)", "", e) # thousands separators
25
+ e = re.sub(r"\$|\\left|\\right", "", e)
26
+ e = re.sub(r"(\d+(?:\.\d+)?)\s*%(?=\s*(?:$|[-+*/)]))", r"(\1/100)", e) # 12% -> (12/100) when % is a postfix; 7%4 stays a modulo
27
+ return e
28
+
29
+
30
+ def safe_eval(expr: str) -> Fraction:
31
+ """Evaluate an arithmetic expression exactly (Fraction); raises ValueError for anything outside the grammar or for oversized values."""
32
+ tree = ast.parse(_clean(expr), mode="eval")
33
+ if sum(1 for _ in ast.walk(tree)) > MAX_NODES:
34
+ raise ValueError("expression too long")
35
+
36
+ def ev(n):
37
+ if isinstance(n, ast.Expression):
38
+ return ev(n.body)
39
+ if isinstance(n, ast.Constant) and isinstance(n.value, (int, float)) and not isinstance(n.value, bool):
40
+ return Fraction(str(n.value))
41
+ if isinstance(n, ast.UnaryOp) and type(n.op) in _UN:
42
+ return _UN[type(n.op)](ev(n.operand))
43
+ if isinstance(n, ast.BinOp) and type(n.op) in _BIN:
44
+ a, b = ev(n.left), ev(n.right)
45
+ if isinstance(n.op, ast.Pow):
46
+ if b.denominator != 1 or abs(b) > MAX_EXP:
47
+ raise ValueError("exponent not allowed")
48
+ v = a ** int(b)
49
+ elif isinstance(n.op, (ast.Div, ast.FloorDiv, ast.Mod)) and b == 0:
50
+ raise ValueError("division by zero")
51
+ else:
52
+ v = _BIN[type(n.op)](a, b)
53
+ if abs(v) > MAX_ABS:
54
+ raise ValueError("value too large")
55
+ return Fraction(v)
56
+ raise ValueError("not an arithmetic expression")
57
+ return ev(tree)
58
+
59
+
60
+ def _number(text: str) -> Optional[Fraction]:
61
+ t = text.strip().strip("*$ ").rstrip(".").replace(",", "")
62
+ t = re.sub(r"\\boxed\{(.*)\}", r"\1", t)
63
+ t = re.sub(r"[A-Za-z%\\ ]+$", "", t).strip() # trailing units
64
+ try:
65
+ if re.fullmatch(r"-?\d+/\d+", t):
66
+ return Fraction(t)
67
+ return Fraction(t)
68
+ except (ValueError, ZeroDivisionError):
69
+ return None
70
+
71
+
72
+ def parse_lines(text: str) -> Tuple[Optional[str], Optional[str]]:
73
+ ex = re.findall(r"(?im)^\s*\**expression\**\s*:\s*(.+)$", text or "")
74
+ an = re.findall(r"(?im)^\s*\**answer\**\s*:\s*(.+)$", text or "")
75
+ return (ex[-1].strip() if ex else None, an[-1].strip() if an else None)
76
+
77
+
78
+ def verify(text: str) -> Tuple[bool, str]:
79
+ """(passed, reason). Reasons: missing line, not arithmetic, answer not a number, mismatch, ok."""
80
+ expr, ans = parse_lines(text)
81
+ if expr is None or ans is None:
82
+ return False, "missing expression or answer line"
83
+ try:
84
+ val = safe_eval(expr)
85
+ except (SyntaxError, ValueError, ZeroDivisionError, RecursionError, OverflowError):
86
+ return False, "not an arithmetic expression"
87
+ a = _number(ans)
88
+ if a is None:
89
+ return False, "answer is not a number"
90
+ d = len(re.sub(r"[^0-9.]", "", ans).partition(".")[2]) # an answer rounded to d decimals matches the exact value rounded to d decimals
91
+ ok = val == a or (d > 0 and round(val, d) == a)
92
+ return ok, "ok" if ok else "expression does not equal the answer"
mae_eval/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """mae-eval: compute-accounting and audit utilities for multi-agent LLM evaluations (prototype).
2
+
3
+ See docs/MAEO_SPECIFICATION.md. Prototype status: no provider price table is bundled; register prices explicitly.
4
+ """
5
+ from .auditor import ComputeParityError, EvaluationAuditor, Flag
6
+ from .tracker import CallRecord, PricingUnavailable, ResourceTracker, Session
7
+ from .vector import ControlState, ResourceVector, ResourceVectorError
8
+
9
+ __version__ = "0.1.0"
10
+ __all__ = ["ResourceTracker", "EvaluationAuditor", "ResourceVector", "ControlState", "ResourceVectorError",
11
+ "ComputeParityError", "PricingUnavailable", "Flag", "Session", "CallRecord"]
mae_eval/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())