pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
cme/frontier.py ADDED
@@ -0,0 +1,143 @@
1
+ """Self-consistency frontier and frontier gap with a paired bootstrap.
2
+
3
+ A sample pool holds, for each item, up to K independent sampled answers with token counts. For k = 1..K the expected accuracy of
4
+ plurality voting over k samples is estimated over random k-subsets (ties share credit equally). The cost of k samples is k times the
5
+ mean tokens per sample. The frontier is the nondecreasing (isotonic) fit of accuracy against tokens, interpolated linearly.
6
+
7
+ frontier gap = accuracy of a system - frontier(mean tokens the system used). A positive gap means the system sits above what
8
+ sampling alone would reach at that measured token cost. The bootstrap resamples items and recomputes everything (pool curve,
9
+ isotonic fit, interpolation, system accuracy), so the interval accounts for sampling noise in both sides.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from collections import Counter
15
+ from collections.abc import Sequence
16
+
17
+ import numpy as np
18
+
19
+
20
+ def plurality_credit(answers: Sequence[str], gold: str) -> float:
21
+ """1 if the plurality answer equals gold; 1/m if gold is one of m tied leaders; 0 otherwise."""
22
+ c = Counter(answer for answer in answers if answer != "")
23
+ if not c:
24
+ return 0.0
25
+ top = max(c.values())
26
+ leaders = [a for a, v in c.items() if v == top]
27
+ return (1.0 / len(leaders)) if gold in leaders else 0.0
28
+
29
+
30
+ def item_vote_curve(
31
+ answers: Sequence[str], gold: str, K: int, n_perm: int, rng: np.random.Generator
32
+ ) -> np.ndarray:
33
+ """Expected plurality accuracy of k = 1..K samples for one item. Each random permutation gives nested prefixes; the value at
34
+ k equal to the number of samples is exact (order does not matter)."""
35
+ n = len(answers)
36
+ if K > n:
37
+ raise ValueError("K exceeds the number of samples for an item")
38
+ out = np.zeros(K)
39
+ for _ in range(n_perm):
40
+ c: Counter = Counter()
41
+ for k, i in enumerate(rng.permutation(n)[:K], 1):
42
+ if answers[i] != "":
43
+ c[answers[i]] += 1
44
+ if not c:
45
+ continue
46
+ top = max(c.values())
47
+ leaders = [a for a, v in c.items() if v == top]
48
+ out[k - 1] += (1.0 / len(leaders)) if gold in leaders else 0.0
49
+ return out / n_perm
50
+
51
+
52
+ def pav_increasing(y: np.ndarray, w: np.ndarray | None = None) -> np.ndarray:
53
+ """Weighted pool-adjacent-violators: the nondecreasing least-squares fit to y."""
54
+ y = np.asarray(y, dtype=float)
55
+ w = np.ones_like(y) if w is None else np.asarray(w, dtype=float)
56
+ blocks: list[list[float]] = [] # [value, weight, count]
57
+ for yi, wi in zip(y, w):
58
+ blocks.append([yi, wi, 1])
59
+ while len(blocks) > 1 and blocks[-2][0] > blocks[-1][0]:
60
+ v2, w2, c2 = blocks.pop()
61
+ v1, w1, c1 = blocks.pop()
62
+ blocks.append([(v1 * w1 + v2 * w2) / (w1 + w2), w1 + w2, c1 + c2])
63
+ return np.concatenate([[b[0]] * int(b[2]) for b in blocks])
64
+
65
+
66
+ def frontier_at(tokens_k: np.ndarray, acc_k: np.ndarray, t: float) -> tuple[float, str]:
67
+ """Isotonic frontier interpolated at t. Returns (value, status) with status in {'inside', 'below range', 'beyond range'}."""
68
+ fit = pav_increasing(acc_k)
69
+ if t < tokens_k[0]:
70
+ return float(fit[0]), "below range"
71
+ if t > tokens_k[-1]:
72
+ return float(fit[-1]), "beyond range"
73
+ return float(np.interp(t, tokens_k, fit)), "inside"
74
+
75
+
76
+ class Pool:
77
+ """Per-item sampled answers and token counts. items: id -> dict(gold, answers[list[str]], tokens[list[float]])."""
78
+
79
+ def __init__(self, items: dict[str, dict], K: int | None = None):
80
+ self.items = items
81
+ self.K = K or min(len(v["answers"]) for v in items.values())
82
+ if self.K < 1:
83
+ raise ValueError("pool has an item with no samples")
84
+
85
+ def curves(
86
+ self, n_perm: int = 100, seed: int = 0
87
+ ) -> tuple[list[str], np.ndarray, np.ndarray]:
88
+ """Return (item ids, acc[item, k], tokens_per_sample[item]) with k = 1..K."""
89
+ rng = np.random.default_rng(seed)
90
+ ids = sorted(self.items)
91
+ acc = np.zeros((len(ids), self.K))
92
+ tok = np.zeros(len(ids))
93
+ for i, it in enumerate(ids):
94
+ v = self.items[it]
95
+ acc[i] = item_vote_curve(
96
+ v["answers"][: len(v["answers"])], v["gold"], self.K, n_perm, rng
97
+ )
98
+ tok[i] = float(np.mean(v["tokens"]))
99
+ return ids, acc, tok
100
+
101
+
102
+ def frontier_curve(acc: np.ndarray, tok: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
103
+ """Population-level points: tokens_k = k * mean tokens per sample, acc_k = mean over items."""
104
+ K = acc.shape[1]
105
+ return np.arange(1, K + 1) * float(tok.mean()), acc.mean(axis=0)
106
+
107
+
108
+ def frontier_gap(
109
+ acc: np.ndarray,
110
+ tok: np.ndarray,
111
+ sys_correct: np.ndarray,
112
+ sys_tokens: np.ndarray,
113
+ n_boot: int = 2000,
114
+ seed: int = 0,
115
+ alpha: float = 0.05,
116
+ ) -> dict:
117
+ """Paired bootstrap of (system accuracy - frontier at the system's mean tokens). Arrays are aligned by item."""
118
+ n = acc.shape[0]
119
+ if not (len(sys_correct) == len(sys_tokens) == n):
120
+ raise ValueError("pool and system arrays must cover the same items")
121
+ tk, ak = frontier_curve(acc, tok)
122
+ f0, status = frontier_at(tk, ak, float(sys_tokens.mean()))
123
+ est = float(sys_correct.mean() - f0)
124
+ rng = np.random.default_rng(seed)
125
+ gaps = np.empty(n_boot)
126
+ for b in range(n_boot):
127
+ ix = rng.integers(0, n, n)
128
+ tkb, akb = frontier_curve(acc[ix], tok[ix])
129
+ fb, _ = frontier_at(tkb, akb, float(sys_tokens[ix].mean()))
130
+ gaps[b] = sys_correct[ix].mean() - fb
131
+ lo, hi = np.quantile(gaps, [alpha / 2, 1 - alpha / 2])
132
+ return {
133
+ "gap": est,
134
+ "ci_low": float(lo),
135
+ "ci_high": float(hi),
136
+ "p_gap_le_0": float((np.sum(gaps <= 0) + 1) / (n_boot + 1)),
137
+ "frontier_at_system_tokens": f0,
138
+ "range_status": status,
139
+ "system_accuracy": float(sys_correct.mean()),
140
+ "system_mean_tokens": float(sys_tokens.mean()),
141
+ "n_items": int(n),
142
+ "n_boot": n_boot,
143
+ }
cme/hidden_profile.py ADDED
@@ -0,0 +1,135 @@
1
+ """cme hidden-profile: a configurable generator for the synthetic split-evidence (hidden-profile) task, with a built-in validity checker.
2
+
3
+ Design (Stasser and Titus 1985): four candidates are scored on `criteria` equally weighted criteria. `criteria - split` criteria are shared by all agents and favour a
4
+ decoy candidate; the `split` other criteria favour the target and are divided among `agents` agents, each of whom sees the shared criteria and `ceil(split / agents)`
5
+ of the split ones. Every single view then favours the decoy, and only the pooled evidence favours the target. Scores run from `low = 2` to `low + margin`.
6
+
7
+ With the default knobs (agents 5, criteria 10, split 6, margin 3, text format) the prompts, views and gold answers are identical to `cme.tasks.hidden_profile` (same seed, same random stream; `meta` has extra keys),
8
+ which is the task behind the released 500-item file.
9
+
10
+ Validity checks, computed from the generated scores (not from the construction): in every individual view the decoy has the strictly highest total of the four
11
+ candidates; on the pooled evidence the target has the strictly highest total; every split criterion appears in at least one view. Infeasible knobs are rejected
12
+ before generation: the shared criteria must outnumber the split criteria in one view (`criteria - split > ceil(split / agents)`) and the split criteria must outnumber
13
+ the shared ones (`split > criteria - split`), and `margin` must be at least 2.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import math
18
+ import random
19
+ from typing import Dict, List, Optional
20
+
21
+ NAMES = ["Avery", "Blake", "Casey", "Drew"]
22
+ LOW = 2
23
+
24
+
25
+ def check_knobs(agents: int, criteria: int, split: int, margin: int) -> None:
26
+ if agents < 2:
27
+ raise ValueError("agents must be at least 2")
28
+ if margin < 2:
29
+ raise ValueError("margin must be at least 2")
30
+ if not 1 <= split < criteria:
31
+ raise ValueError("split must satisfy 1 <= split < criteria")
32
+ shared, view = criteria - split, math.ceil(split / agents)
33
+ if shared <= view:
34
+ raise ValueError("infeasible: the decoy cannot lead a single view; need criteria - split (%d) > ceil(split / agents) (%d)" % (shared, view))
35
+ if split <= shared:
36
+ raise ValueError("infeasible: the target cannot lead the pooled evidence; need split (%d) > criteria - split (%d)" % (split, shared))
37
+
38
+
39
+ def _scores_text(sc, js):
40
+ return "\n".join("%s on %s: %d" % (NAMES[c], "criterion %d" % (j + 1), sc[c][j]) for j in js for c in range(4))
41
+
42
+
43
+ def _scores_numeric(sc, js):
44
+ rows = ["criterion," + ",".join(NAMES)]
45
+ rows += ["%d," % (j + 1) + ",".join(str(sc[c][j]) for c in range(4)) for j in js]
46
+ return "\n".join(rows)
47
+
48
+
49
+ def generate(n: int, agents: int = 5, criteria: int = 10, split: int = 6, margin: int = 3, seed: int = 42, fmt: str = "text") -> List[dict]:
50
+ """Return n items {id, task, prompt, gold, kind, meta, views}. See the module docstring."""
51
+ if fmt not in ("text", "numeric"):
52
+ raise ValueError("fmt must be 'text' or 'numeric'")
53
+ check_knobs(agents, criteria, split, margin)
54
+ shared, view = criteria - split, math.ceil(split / agents)
55
+ high, cap = LOW + margin, min(LOW + margin - 2, 4)
56
+ rng = random.Random(seed)
57
+ out: List[dict] = []
58
+ while len(out) < n:
59
+ target, decoy = rng.sample(range(4), 2)
60
+ sc = [[rng.randint(2, 4) for _ in range(criteria)] for _ in range(4)]
61
+ for j in range(shared): # shared criteria: the decoy leads clearly
62
+ for c in range(4):
63
+ sc[c][j] = min(sc[c][j], cap)
64
+ sc[decoy][j] = high
65
+ sc[target][j] = LOW
66
+ for j in range(shared, criteria): # split criteria: the target leads
67
+ for c in range(4):
68
+ sc[c][j] = min(sc[c][j], cap)
69
+ sc[target][j] = high
70
+ sc[decoy][j] = LOW
71
+ tot = [sum(r) for r in sc]
72
+ if sorted(tot)[-1] == sorted(tot)[-2] or tot.index(max(tot)) != target:
73
+ continue
74
+ shown = _scores_text if fmt == "text" else _scores_numeric
75
+ head = ("A committee must pick one of four candidates (%s) using %d equally weighted criteria, each scored %d to %d. The best candidate has the highest total score.\n\n"
76
+ % (", ".join(NAMES), criteria, 1, high))
77
+ tail = "\n\nWhich candidate has the highest total over all %d criteria? Give the name." % criteria
78
+ full = head + "All scores:\n" + shown(sc, range(criteria)) + tail
79
+ order = list(range(shared, criteria))
80
+ rng.shuffle(order)
81
+ views = []
82
+ for a in range(agents):
83
+ mine = sorted(order[(view * a + k) % split] for k in range(view))
84
+ views.append(head + "You see only part of the evidence. Scores you can see:\n" + shown(sc, list(range(shared)) + mine) +
85
+ "\n\nOther committee members see the other criteria. Which candidate has the highest total over all %d criteria? Give the name." % criteria)
86
+ out.append({"id": "hp%04d" % len(out), "task": "hidden_profile", "prompt": full, "gold": NAMES[target], "kind": "name",
87
+ "meta": {"synthetic": True, "totals": tot, "decoy": NAMES[decoy], "knobs": dict(agents=agents, criteria=criteria, split=split, margin=margin, seed=seed, fmt=fmt)},
88
+ "views": views})
89
+ return out
90
+
91
+
92
+ def _parse_view(text: str, fmt: str, criteria: int):
93
+ """Recover {criterion_index: {name: score}} from a prompt or view, in either format."""
94
+ got: Dict[int, Dict[str, int]] = {}
95
+ if fmt == "text":
96
+ import re
97
+ for name, j, s in re.findall(r"(\w+) on criterion (\d+): (\d+)", text):
98
+ got.setdefault(int(j) - 1, {})[name] = int(s)
99
+ else:
100
+ for line in text.splitlines():
101
+ parts = line.split(",")
102
+ if len(parts) == 5 and parts[0].isdigit():
103
+ got[int(parts[0]) - 1] = dict(zip(NAMES, map(int, parts[1:])))
104
+ return got
105
+
106
+
107
+ def validate(items: List[dict]) -> Dict[str, int]:
108
+ """Check each item from its prompt text: decoy strictly leads every view, target strictly leads the pooled evidence, every split criterion is seen. Returns counts."""
109
+ bad = dict(items=len(items), decoy_not_leading_a_view=0, target_not_leading_pooled=0, uncovered_split_criteria=0)
110
+ for it in items:
111
+ k = it["meta"]["knobs"]
112
+ fmt = k["fmt"] if "fmt" in k else "text"
113
+ full = _parse_view(it["prompt"], fmt, k["criteria"])
114
+ decoy, target = it["meta"]["decoy"], it["gold"]
115
+ tot = {nm: sum(full[j][nm] for j in full) for nm in NAMES}
116
+ best = sorted(tot.values())[-1]
117
+ if not (tot[target] == best and list(tot.values()).count(best) == 1):
118
+ bad["target_not_leading_pooled"] += 1
119
+ seen = set()
120
+ shared = k["criteria"] - k["split"]
121
+ view_bad = False
122
+ for v in it["views"]:
123
+ part = _parse_view(v, fmt, k["criteria"])
124
+ seen |= {j for j in part if j >= shared}
125
+ vt = {nm: sum(part[j][nm] for j in part) for nm in NAMES}
126
+ vb = sorted(vt.values())[-1]
127
+ if not (vt[decoy] == vb and list(vt.values()).count(vb) == 1):
128
+ view_bad = True
129
+ bad["decoy_not_leading_a_view"] += view_bad
130
+ bad["uncovered_split_criteria"] += len(set(range(shared, k["criteria"])) - seen) > 0
131
+ return bad
132
+
133
+
134
+ def valid(counts: Dict[str, int]) -> bool:
135
+ return not any(v for k, v in counts.items() if k != "items")
cme/proxy.py ADDED
@@ -0,0 +1,106 @@
1
+ """cme proxy: an OpenAI-compatible logging proxy (standard library only).
2
+
3
+ Point any agent framework's base URL at http://localhost:<port>/v1. Requests are forwarded unchanged to the upstream server and every
4
+ call is appended to a JSONL log in the mae_eval trace format (so `mae-eval audit` and `cme report` read it), with latency and cost.
5
+
6
+ Optional request headers set the labels (they are removed before forwarding):
7
+ X-CME-System mas | sas (default mas) X-CME-Agent agent id (default agent)
8
+ X-CME-Round integer round (default 0) X-CME-Item item id (default empty, or the first line of --item-file when the caller cannot set headers)
9
+ Non-streaming requests only: a request with "stream": true is rejected with HTTP 400 and a clear message (v0.1 limitation).
10
+ Prices are never bundled; pass --price-in/--price-out (USD per million tokens) or cost stays null.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import threading
16
+ import time
17
+ import urllib.error
18
+ import urllib.request
19
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
20
+ from typing import Optional
21
+
22
+ _LABELS = ("x-cme-system", "x-cme-agent", "x-cme-round", "x-cme-item")
23
+
24
+
25
+ def _item_from_file(path: Optional[str]) -> str:
26
+ """Item id for callers that cannot send the X-CME-Item header (third-party code): the first line of `path`, re-read at every call. Serial runs only."""
27
+ if not path:
28
+ return ""
29
+ try:
30
+ with open(path, encoding="utf-8") as fh:
31
+ return fh.readline().strip()
32
+ except OSError:
33
+ return ""
34
+
35
+
36
+ def make_handler(upstream: str, log_path: str, price_in: Optional[float], price_out: Optional[float], api_key: Optional[str], item_file: Optional[str] = None, default_system: str = "mas"):
37
+ lock = threading.Lock()
38
+ up = upstream.rstrip("/")
39
+
40
+ class H(BaseHTTPRequestHandler):
41
+ def _send(self, code: int, payload: bytes, ctype: str = "application/json"):
42
+ self.send_response(code)
43
+ self.send_header("Content-Type", ctype)
44
+ self.send_header("Content-Length", str(len(payload)))
45
+ self.end_headers()
46
+ self.wfile.write(payload)
47
+
48
+ def _forward(self, method: str, body: Optional[bytes]):
49
+ headers = {k: v for k, v in self.headers.items() if k.lower() not in _LABELS + ("host", "content-length", "connection")}
50
+ if api_key:
51
+ headers["Authorization"] = "Bearer " + api_key
52
+ req = urllib.request.Request(up + self.path[len("/v1"):] if self.path.startswith("/v1") else up + self.path,
53
+ data=body, headers=headers, method=method)
54
+ t0 = time.time()
55
+ try:
56
+ with urllib.request.urlopen(req, timeout=900) as r:
57
+ return r.status, r.read(), t0, time.time()
58
+ except urllib.error.HTTPError as e:
59
+ return e.code, e.read(), t0, time.time()
60
+
61
+ def do_GET(self):
62
+ code, data, _, _ = self._forward("GET", None)
63
+ self._send(code, data)
64
+
65
+ def do_POST(self):
66
+ n = int(self.headers.get("Content-Length", 0))
67
+ raw = self.rfile.read(n)
68
+ try:
69
+ req = json.loads(raw)
70
+ except json.JSONDecodeError:
71
+ return self._send(400, b'{"error": "invalid JSON"}')
72
+ if req.get("stream"):
73
+ return self._send(400, b'{"error": "cme proxy v0.1 does not support streaming; set stream=false"}')
74
+ code, data, t0, t1 = self._forward("POST", raw)
75
+ if code == 200:
76
+ try:
77
+ resp = json.loads(data)
78
+ u = resp.get("usage") or {}
79
+ det = u.get("completion_tokens_details") or {}
80
+ pt, ct = u.get("prompt_tokens"), u.get("completion_tokens")
81
+ cost = None
82
+ if None not in (price_in, price_out, pt, ct):
83
+ cost = (pt * price_in + ct * price_out) / 1e6
84
+ rec = {"system": self.headers.get("X-CME-System", default_system), "agent_id": self.headers.get("X-CME-Agent", "agent"),
85
+ "round": int(self.headers.get("X-CME-Round", 0)), "item": self.headers.get("X-CME-Item") or _item_from_file(item_file),
86
+ "model": resp.get("model", req.get("model")), "prompt_tokens": pt, "completion_tokens": ct,
87
+ "reasoning_tokens": det.get("reasoning_tokens"), "n_completions": len(resp.get("choices", [])) or 1,
88
+ "start": t0, "end": t1, "latency": t1 - t0, "cost": cost}
89
+ if pt is None or ct is None:
90
+ rec["usage_missing"] = True # logged, never guessed
91
+ with lock, open(log_path, "a", encoding="utf-8") as fh:
92
+ fh.write(json.dumps(rec) + "\n")
93
+ except (ValueError, TypeError):
94
+ pass
95
+ self._send(code, data)
96
+
97
+ def log_message(self, *a): # quiet
98
+ pass
99
+
100
+ return H
101
+
102
+
103
+ def serve(upstream: str, log_path: str, port: int = 8089, host: str = "127.0.0.1", price_in: Optional[float] = None,
104
+ price_out: Optional[float] = None, api_key: Optional[str] = None, item_file: Optional[str] = None, default_system: str = "mas") -> ThreadingHTTPServer:
105
+ srv = ThreadingHTTPServer((host, port), make_handler(upstream, log_path, price_in, price_out, api_key, item_file, default_system))
106
+ return srv
cme/rebuttal.py ADDED
@@ -0,0 +1,76 @@
1
+ """Nine-column rebuttal table: Study | Benchmark | Model | MA configuration | Baseline rung | MA acc | Baseline acc | Raw gap | Compute-normalized gap.
2
+
3
+ Two sources:
4
+ * the literature contrasts (outputs/baseline_tier_contrasts.csv: study, category = rung A/B/C, baseline, key 'Study|bench|model|arm',
5
+ p_base, p_mas): one row per (configuration, baseline). Raw gap = MA acc - baseline acc. Compute-normalized gap = MA acc minus the
6
+ strongest single-agent baseline reported for the same configuration (the highest baseline accuracy over its rungs), so it equals the raw
7
+ gap when the row already uses the strongest baseline and is lower when a stronger one exists. It is a within-study comparison, not a
8
+ frontier estimate.
9
+ * a `cme report` run: one row; rung from the baseline classifier (the greedy arm is rung A), raw gap against greedy, and the
10
+ compute-normalized gap is the frontier gap at the system's measured tokens.
11
+ Unreported cells print as n/a.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import csv
16
+ import re
17
+ from typing import Dict, List, Optional
18
+
19
+ COLUMNS = ["Study", "Benchmark", "Model", "MA configuration", "Baseline rung", "MA acc", "Baseline acc", "Raw gap", "Compute-normalized gap"]
20
+
21
+
22
+ def _f(x: Optional[float], signed: bool = False) -> str:
23
+ if x is None:
24
+ return "n/a"
25
+ return ("%+.3f" if signed else "%.3f") % x
26
+
27
+
28
+ def rows_from_contrasts(path: str, study: Optional[str] = None) -> List[Dict[str, str]]:
29
+ raw = list(csv.DictReader(open(path, encoding="utf-8")))
30
+ best: Dict[str, float] = {}
31
+ for r in raw:
32
+ best[r["key"]] = max(best.get(r["key"], -1.0), float(r["p_base"]))
33
+ out = []
34
+ for r in raw:
35
+ if study and r["study"] != study:
36
+ continue
37
+ _, bench, model, arm = r["key"].split("|", 3)
38
+ pb, pm = float(r["p_base"]), float(r["p_mas"])
39
+ out.append({"Study": r["study"], "Benchmark": bench, "Model": model, "MA configuration": arm, "Baseline rung": r["category"],
40
+ "MA acc": _f(pm), "Baseline acc": _f(pb), "Raw gap": _f(pm - pb, True), "Compute-normalized gap": _f(pm - best[r["key"]], True)})
41
+ return out
42
+
43
+
44
+ def row_from_report(rep: dict, benchmark: str = "n/a", model: str = "n/a", config: Optional[dict] = None) -> Dict[str, str]:
45
+ """One row for a `cme report` result. `config` describes the baseline (default: the greedy arm, rung A)."""
46
+ from mae_eval.classifier import classify_baseline_rung
47
+ g = rep["frontier_gap"]
48
+ rung = classify_baseline_rung(config or {"n_samples": 1})
49
+ ga = rep.get("greedy_accuracy")
50
+ return {"Study": "this run", "Benchmark": benchmark, "Model": model, "MA configuration": rep.get("system", "n/a"), "Baseline rung": rung,
51
+ "MA acc": _f(g["system_accuracy"]), "Baseline acc": _f(ga), "Raw gap": _f(None if ga is None else g["system_accuracy"] - ga, True),
52
+ "Compute-normalized gap": _f(g["gap"], True) + " [%s, %s]" % (_f(g["ci_low"], True), _f(g["ci_high"], True))}
53
+
54
+
55
+ def to_markdown(rows: List[Dict[str, str]]) -> str:
56
+ lines = ["| " + " | ".join(COLUMNS) + " |", "|" + "---|" * len(COLUMNS)]
57
+ lines += ["| " + " | ".join(r[c] for c in COLUMNS) + " |" for r in rows]
58
+ return "\n".join(lines) + "\n"
59
+
60
+
61
+ def _tex(s: str) -> str:
62
+ for a, b in (("&", "\\&"), ("%", "\\%"), ("_", "\\_"), ("#", "\\#"), ("$", "\\$")):
63
+ s = s.replace(a, b)
64
+ return s
65
+
66
+
67
+ def _signed(s: str) -> str:
68
+ return re.sub(r"([+-])(\d)", r"$\1$\2", s)
69
+
70
+
71
+ def to_latex(rows: List[Dict[str, str]], caption: str = "Contrasts by baseline rung.", label: str = "tab:rebuttal") -> str:
72
+ head = " & ".join(c for c in COLUMNS) + r" \\"
73
+ body = [" & ".join(_signed(_tex(r[c])) if c in ("Raw gap", "Compute-normalized gap") else _tex(r[c]) for c in COLUMNS) + r" \\"
74
+ for r in rows]
75
+ return "\n".join([r"\begin{table}[t]", r"\centering", r"\scriptsize", r"\caption{%s}" % caption, r"\label{%s}" % label, r"\resizebox{\textwidth}{!}{%",
76
+ r"\begin{tabular}{lllllrrrr}", r"\toprule", head, r"\midrule"] + body + [r"\bottomrule", r"\end{tabular}}", r"\end{table}"]) + "\n"
cme/report.py ADDED
@@ -0,0 +1,197 @@
1
+ """cme report: frontier gap, per-item resource card, LaTeX table and vector figure from a baseline pool and a system log.
2
+
3
+ Inputs
4
+ --pool baseline cache written by `cme baseline` (greedy + sampled completions per item)
5
+ --logs JSONL of the system under test: call records (proxy format: system, item, prompt_tokens, completion_tokens, optional
6
+ reasoning_tokens, start, end, cost, tool_calls) and result records {"item", "system", "correct": true|false}
7
+ (a record that has "correct" and no token fields is a result).
8
+ The resource card reports per-item means for the system (MAS) and the greedy baseline (SAS) and passes them to mae_eval's auditor.
9
+ A ratio is n/a when both sides are 0 or when either side is unreported.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ from typing import Dict, List, Optional
15
+
16
+ import numpy as np
17
+
18
+ from mae_eval.auditor import EvaluationAuditor
19
+ from mae_eval.vector import DIMENSIONS, ResourceVector
20
+
21
+ from .baseline import load_pool
22
+ from .frontier import frontier_curve, frontier_gap, frontier_at, pav_increasing
23
+
24
+ OKABE_ITO = {"blue": "#0072B2", "orange": "#E69F00", "green": "#009E73", "verm": "#D55E00", "grey": "#666666"}
25
+
26
+
27
+ def read_run(path: str, system: str):
28
+ calls: Dict[str, List[dict]] = {}
29
+ results: Dict[str, bool] = {}
30
+ skipped = 0
31
+ for l in open(path, encoding="utf-8"):
32
+ if not l.strip():
33
+ continue
34
+ r = json.loads(l)
35
+ if r.get("system", system) != system:
36
+ continue
37
+ if "correct" in r and "prompt_tokens" not in r:
38
+ results[str(r["item"])] = bool(r["correct"])
39
+ elif r.get("prompt_tokens") is None or r.get("completion_tokens") is None:
40
+ skipped += 1 # usage missing: counted, never guessed
41
+ else:
42
+ calls.setdefault(str(r.get("item", "")), []).append(r)
43
+ return calls, results, skipped
44
+
45
+
46
+ def _mean(xs: List[Optional[float]]) -> Optional[float]:
47
+ return None if (not xs or any(x is None for x in xs)) else float(sum(xs) / len(xs))
48
+
49
+
50
+ def _vector(per_item: List[List[dict]], price_in: Optional[float], price_out: Optional[float]) -> ResourceVector:
51
+ """Per-item mean resource vector from a list (one entry per item) of call-record lists."""
52
+ tok, reas, cost, lat, samp, sysc = [], [], [], [], [], []
53
+ for calls in per_item:
54
+ tok.append(sum(c["prompt_tokens"] + c["completion_tokens"] for c in calls))
55
+ reas.append(None if any(c.get("reasoning_tokens") is None for c in calls) else sum(c["reasoning_tokens"] for c in calls))
56
+ if price_in is not None and price_out is not None:
57
+ cost.append(sum((c["prompt_tokens"] * price_in + c["completion_tokens"] * price_out) / 1e6 for c in calls))
58
+ elif all(c.get("cost") is not None for c in calls):
59
+ cost.append(sum(c["cost"] for c in calls))
60
+ else:
61
+ cost.append(None)
62
+ if all(c.get("start") is not None and c.get("end") is not None for c in calls):
63
+ lat.append(max(c["end"] for c in calls) - min(c["start"] for c in calls)) # makespan, concurrency-aware
64
+ else:
65
+ lat.append(None if any(c.get("latency") is None for c in calls) else sum(c["latency"] for c in calls))
66
+ samp.append(sum(int(c.get("n_completions", 1)) for c in calls))
67
+ sysc.append(sum(int(c.get("tool_calls", 0)) for c in calls))
68
+ return ResourceVector(r_reason=_mean(reas), r_token=_mean(tok), r_cost=_mean(cost), r_lat=_mean(lat), r_sample=_mean(samp), r_sys=_mean(sysc))
69
+
70
+
71
+ def build_report(pool_path: str, log_path: str, system: str = "mas", price_in: Optional[float] = None, price_out: Optional[float] = None,
72
+ n_boot: int = 2000, n_perm: int = 100, seed: int = 0, k: Optional[int] = None) -> dict:
73
+ pool, greedy, dropped = load_pool(pool_path, k)
74
+ ids, acc, tok = pool.curves(n_perm, seed)
75
+ calls, results, skipped = read_run(log_path, system)
76
+ common = [i for i in ids if i in results and i in calls]
77
+ if not common:
78
+ raise ValueError("no item has both a pool and a system result; check item ids")
79
+ pos = {i: n for n, i in enumerate(ids)}
80
+ sel = [pos[i] for i in common]
81
+ sys_correct = np.array([float(results[i]) for i in common])
82
+ sys_tokens = np.array([float(sum(c["prompt_tokens"] + c["completion_tokens"] for c in calls[i])) for i in common])
83
+ gap = frontier_gap(acc[sel], tok[sel], sys_correct, sys_tokens, n_boot=n_boot, seed=seed)
84
+ tk, ak = frontier_curve(acc[sel], tok[sel])
85
+ fit = pav_increasing(ak)
86
+ sas_items = [[{"prompt_tokens": greedy[i]["prompt_tokens"], "completion_tokens": greedy[i]["completion_tokens"],
87
+ "reasoning_tokens": greedy[i].get("reasoning_tokens"), "latency": greedy[i]["latency"]}] for i in common if i in greedy]
88
+ card: dict = {}
89
+ if len(sas_items) == len(common) and all(c[0]["prompt_tokens"] is not None for c in sas_items):
90
+ sas_v = _vector(sas_items, price_in, price_out)
91
+ mas_v = _vector([calls[i] for i in common], price_in, price_out)
92
+ aud = EvaluationAuditor(mas_v, sas_v)
93
+ card = {"mas_per_item": mas_v.to_dict(), "sas_per_item": sas_v.to_dict(), "ratios_mas_over_sas": aud.ratios(),
94
+ "control_states": {d: s.value for d, s in aud.control_states().items()}}
95
+ greedy_acc = float(np.mean([greedy[i]["correct"] for i in common if i in greedy])) if any(i in greedy for i in common) else None
96
+ return {"n_items": len(common), "pool_k": pool.K, "pool_items_dropped_incomplete": dropped, "system_calls_missing_usage": skipped,
97
+ "frontier": {"tokens": tk.tolist(), "sc_accuracy": ak.tolist(), "isotonic": fit.tolist()}, "greedy_accuracy": greedy_acc,
98
+ "frontier_gap": gap, "resource_card": card, "system": system}
99
+
100
+
101
+ def to_latex(rep: dict, caption: str = "Frontier gap and per-item resource card.", label: str = "tab:cme_report") -> str:
102
+ g = rep["frontier_gap"]
103
+ rows = [r"\begin{table}[h]", r"\centering", r"\small", r"\caption{%s}" % caption, r"\label{%s}" % label, r"\begin{tabular}{lrr}", r"\toprule",
104
+ r"Quantity & System & Greedy baseline \\", r"\midrule"]
105
+ rc = rep.get("resource_card") or {}
106
+ for d in DIMENSIONS:
107
+ if rc:
108
+ m, s = rc["mas_per_item"][d], rc["sas_per_item"][d]
109
+ fmt = lambda x: "n/a" if x is None else ("%.4g" % x)
110
+ rows.append(r"%s & %s & %s \\" % (d.replace("_", r"\_"), fmt(m), fmt(s)))
111
+ rows += [r"\midrule", r"Accuracy & %.3f & %s \\" % (g["system_accuracy"], "n/a" if rep["greedy_accuracy"] is None else "%.3f" % rep["greedy_accuracy"]),
112
+ r"Self-consistency frontier at system tokens & \multicolumn{2}{r}{%.3f (%s)} \\" % (g["frontier_at_system_tokens"], g["range_status"]),
113
+ r"Frontier gap (95\%% bootstrap CI) & \multicolumn{2}{r}{$%+.3f$ [$%+.3f$, $%+.3f$]} \\" % (g["gap"], g["ci_low"], g["ci_high"]),
114
+ r"\bottomrule", r"\end{tabular}", r"\end{table}"]
115
+ return "\n".join(rows) + "\n"
116
+
117
+
118
+ def plot(rep: dict, out_path: str) -> None:
119
+ import matplotlib
120
+ matplotlib.use("Agg")
121
+ import matplotlib.pyplot as plt
122
+ import matplotlib.ticker
123
+ matplotlib.rcParams["pdf.fonttype"] = 42
124
+ f, g = rep["frontier"], rep["frontier_gap"]
125
+ fig, ax = plt.subplots(figsize=(4.6, 3.2))
126
+ ax.plot(f["tokens"], f["sc_accuracy"], "o", color=OKABE_ITO["grey"], ms=3.5, label="Self-consistency, k = 1 to %d" % rep["pool_k"])
127
+ ax.plot(f["tokens"], f["isotonic"], "-", color=OKABE_ITO["blue"], lw=1.6, label="Isotonic frontier")
128
+ if rep["greedy_accuracy"] is not None:
129
+ ax.plot([f["tokens"][0]], [rep["greedy_accuracy"]], "s", color=OKABE_ITO["green"], label="Greedy")
130
+ ax.plot([g["system_mean_tokens"]], [g["system_accuracy"]], "D", color=OKABE_ITO["verm"], label="System (%s)" % rep["system"])
131
+ ax.vlines(g["system_mean_tokens"], g["frontier_at_system_tokens"], g["system_accuracy"], colors=OKABE_ITO["verm"], lw=1.2, linestyles=":")
132
+ ax.set_xscale("log")
133
+ ax.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter())
134
+ ax.set_xlabel("Mean tokens per item (prompt + completion)")
135
+ ax.set_ylabel("Accuracy")
136
+ ax.set_title("gap %+.3f [%+.3f, %+.3f]" % (g["gap"], g["ci_low"], g["ci_high"]), fontsize=9)
137
+ ax.legend(fontsize=7, frameon=False)
138
+ ax.spines[["top", "right"]].set_visible(False)
139
+ fig.tight_layout()
140
+ fig.savefig(out_path, bbox_inches="tight")
141
+ plt.close(fig)
142
+
143
+
144
+ OI_CYCLE = ["#D55E00", "#009E73", "#E69F00", "#CC79A7", "#56B4E9", "#F0E442", "#000000"] # Okabe-Ito, skipping the frontier blue
145
+
146
+
147
+ def frontier_report(pool_path: str, k: Optional[int] = None, n_perm: int = 100, seed: int = 0) -> dict:
148
+ """Self-consistency frontier of a sample pool: points for k = 1..K, isotonic fit, greedy accuracy. No system needed."""
149
+ pool, greedy, dropped = load_pool(pool_path, k)
150
+ ids, acc, tok = pool.curves(n_perm, seed)
151
+ tk, ak = frontier_curve(acc, tok)
152
+ gacc = float(np.mean([greedy[i]["correct"] for i in ids if i in greedy])) if any(i in greedy for i in ids) else None
153
+ return {"n_items": len(ids), "pool_k": pool.K, "pool_items_dropped_incomplete": dropped, "tokens": tk.tolist(), "sc_accuracy": ak.tolist(),
154
+ "isotonic": pav_increasing(ak).tolist(), "greedy_accuracy": gacc}
155
+
156
+
157
+ def system_points(log_path: str) -> List[dict]:
158
+ """Accuracy and mean tokens per item of every system in a run log (systems are the values of the 'system' field)."""
159
+ names = []
160
+ for l in open(log_path, encoding="utf-8"):
161
+ if l.strip():
162
+ s = json.loads(l).get("system")
163
+ if s and s not in names:
164
+ names.append(s)
165
+ out = []
166
+ for s in names:
167
+ calls, results, _ = read_run(log_path, s)
168
+ common = [i for i in results if i in calls]
169
+ if common:
170
+ out.append({"system": s, "accuracy": float(np.mean([results[i] for i in common])), "n_items": len(common),
171
+ "mean_tokens": float(np.mean([sum(c["prompt_tokens"] + c["completion_tokens"] for c in calls[i]) for i in common]))})
172
+ return out
173
+
174
+
175
+ def plot_frontier(fr: dict, points: List[dict], out_path: str) -> None:
176
+ """Frontier curve with system points (Okabe-Ito palette); .pdf is vector, .png is raster."""
177
+ import matplotlib
178
+ matplotlib.use("Agg")
179
+ import matplotlib.pyplot as plt
180
+ import matplotlib.ticker
181
+ matplotlib.rcParams["pdf.fonttype"] = 42
182
+ fig, ax = plt.subplots(figsize=(5.4, 3.5))
183
+ ax.plot(fr["tokens"], fr["sc_accuracy"], "o", color="#666666", ms=3.5, label="self-consistency, k = 1 to %d" % fr["pool_k"])
184
+ ax.plot(fr["tokens"], fr["isotonic"], "-", color="#0072B2", lw=1.6, label="isotonic frontier")
185
+ if fr.get("greedy_accuracy") is not None:
186
+ ax.plot([fr["tokens"][0]], [fr["greedy_accuracy"]], "s", color="#56B4E9", mec="white", label="greedy")
187
+ for i, p in enumerate(points):
188
+ ax.plot([p["mean_tokens"]], [p["accuracy"]], "D", color=OI_CYCLE[i % len(OI_CYCLE)], mec="white", ms=7, label=p["system"])
189
+ ax.set_xscale("log")
190
+ ax.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter())
191
+ ax.set_xlabel("Mean tokens per item (prompt + completion)")
192
+ ax.set_ylabel("Accuracy")
193
+ ax.legend(fontsize=7, frameon=False, loc="lower right")
194
+ ax.spines[["top", "right"]].set_visible(False)
195
+ fig.tight_layout()
196
+ fig.savefig(out_path, bbox_inches="tight", dpi=200)
197
+ plt.close(fig)