pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/analysis.py ADDED
@@ -0,0 +1,129 @@
1
+ """Load a confirmatory cell (one model x one task) and estimate frontier gaps per strategy.
2
+
3
+ A cell directory holds calls.jsonl (every call) and results.jsonl (one row per strategy x item). The pool frontier is rebuilt from the
4
+ cached pool calls with the same scorer used at run time, so re-scoring never needs a model. Correctness of a pool sample is mapped to
5
+ the label 'G' (wrong answers keep their normalised text) so plurality voting uses the scorer's notion of correct.
6
+
7
+ Estimand (docs/PREREGISTRATION.md section 5): gap(s) = accuracy(s) - F(mean tokens of s), F the isotonic self-consistency frontier,
8
+ with a paired bootstrap over items. `pool="views"` builds the frontier from the independent split-evidence agents (S2v) instead.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import os
14
+ from typing import Dict, List, Optional
15
+
16
+ import numpy as np
17
+
18
+ from . import experiment as E
19
+ from . import tasks as T
20
+ from .frontier import Pool, frontier_curve, frontier_gap, pav_increasing
21
+
22
+
23
+ def read_results(cell_dir: str) -> Dict[str, Dict[str, dict]]:
24
+ out: Dict[str, Dict[str, dict]] = {}
25
+ p = os.path.join(cell_dir, "results.jsonl")
26
+ if os.path.exists(p):
27
+ for l in open(p, encoding="utf-8"):
28
+ if l.strip():
29
+ r = json.loads(l)
30
+ out.setdefault(r["strategy"], {})[r["item"]] = r
31
+ return out
32
+
33
+
34
+ def build_pool(store: E.Store, model: str, items: List[dict], pool: str = "pool", k: int = 16) -> Pool:
35
+ """Pool of per-item sampled answers and token counts. pool='pool' uses the 16 full-evidence samples; 'views' the 5 split-evidence agents."""
36
+ pitems = {}
37
+ for it in items:
38
+ if it["kind"] == "code":
39
+ continue
40
+ keys = ["pool|%s|%s|%d" % (model, it["id"], i) for i in range(k)] if pool == "pool" else ["r1v|%s|%s|%d" % (model, it["id"], i) for i in range(k)]
41
+ recs = [store.d.get(x) for x in keys]
42
+ if any(r is None or r.get("prompt_tokens") is None or r.get("completion_tokens") is None for r in recs):
43
+ continue
44
+ pitems[it["id"]] = {"gold": "G", "answers": ["G" if T.score(it, r["text"]) else (E.ans_of(it, r) or "?") + "_wrong" for r in recs],
45
+ "tokens": [r["prompt_tokens"] + r["completion_tokens"] for r in recs]}
46
+ return Pool(pitems, k) if pitems else None
47
+
48
+
49
+ def cell_gaps(cell_dir: str, model: str, items: List[dict], strategies: Optional[List[str]] = None, pool: str = "pool", k: Optional[int] = None,
50
+ n_boot: int = 2000, n_perm: int = 100, seed: int = 0, tokens: str = "total") -> dict:
51
+ """Frontier, per-strategy accuracy, mean tokens and gap with a bootstrap interval. tokens='completion' is the sensitivity analysis."""
52
+ store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
53
+ k = k or (16 if pool == "pool" else 5)
54
+ P = build_pool(store, model, items, pool, k)
55
+ if P is None:
56
+ return {"error": "no complete pool in %s" % cell_dir}
57
+ if tokens == "completion": # recompute pool tokens with completion tokens only
58
+ for it in P.items:
59
+ keys = ["pool|%s|%s|%d" % (model, it, i) if pool == "pool" else "r1v|%s|%s|%d" % (model, it, i) for i in range(k)]
60
+ P.items[it]["tokens"] = [store.d[x]["completion_tokens"] for x in keys]
61
+ ids, acc, tok = P.curves(n_perm, seed)
62
+ res = read_results(cell_dir)
63
+ out = {"model": model, "pool": pool, "k": k, "n_pool_items": len(ids), "tokens": tokens, "strategies": {}}
64
+ tk, ak = frontier_curve(acc, tok)
65
+ out["frontier"] = {"tokens": tk.tolist(), "sc_accuracy": ak.tolist(), "isotonic": pav_increasing(ak).tolist()}
66
+ for s, rows in sorted(res.items()):
67
+ if strategies and s not in strategies:
68
+ continue
69
+ common = [i for i in ids if i in rows]
70
+ if len(common) < 5:
71
+ continue
72
+ pos = {i: n for n, i in enumerate(ids)}
73
+ sel = [pos[i] for i in common]
74
+ use = (lambda r: r["completion_tokens"]) if tokens == "completion" else (lambda r: r["prompt_tokens"] + r["completion_tokens"])
75
+ sc = np.array([float(rows[i]["correct"]) for i in common])
76
+ st = np.array([float(use(rows[i])) for i in common])
77
+ g = frontier_gap(acc[sel], tok[sel], sc, st, n_boot=n_boot, seed=seed)
78
+ g["n_items"] = len(common)
79
+ out["strategies"][s] = g
80
+ return out
81
+
82
+
83
+ def difference_of_gaps(cell_dir: str, model: str, items: List[dict], a: str, b: str, n_boot: int = 2000, seed: int = 0, n_perm: int = 100) -> Optional[dict]:
84
+ """Paired bootstrap of gap(a) - gap(b) on the items both strategies finished (used for H3: S4 versus S3)."""
85
+ store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
86
+ P = build_pool(store, model, items, "pool", 16)
87
+ res = read_results(cell_dir)
88
+ if P is None or a not in res or b not in res:
89
+ return None
90
+ ids, acc, tok = P.curves(n_perm, seed)
91
+ common = [i for i in ids if i in res[a] and i in res[b]]
92
+ if len(common) < 5:
93
+ return None
94
+ pos = {i: n for n, i in enumerate(ids)}
95
+ sel = np.array([pos[i] for i in common])
96
+ sa = np.array([float(res[a][i]["correct"]) for i in common]); ta = np.array([float(res[a][i]["prompt_tokens"] + res[a][i]["completion_tokens"]) for i in common])
97
+ sb = np.array([float(res[b][i]["correct"]) for i in common]); tb = np.array([float(res[b][i]["prompt_tokens"] + res[b][i]["completion_tokens"]) for i in common])
98
+ from .frontier import frontier_at
99
+ rng = np.random.default_rng(seed)
100
+ n = len(common)
101
+ A, T_ = acc[sel], tok[sel]
102
+
103
+ def one(ix):
104
+ tk, ak = frontier_curve(A[ix], T_[ix])
105
+ ga = sa[ix].mean() - frontier_at(tk, ak, float(ta[ix].mean()))[0]
106
+ gb = sb[ix].mean() - frontier_at(tk, ak, float(tb[ix].mean()))[0]
107
+ return ga - gb
108
+ est = one(np.arange(n))
109
+ boots = np.array([one(rng.integers(0, n, n)) for _ in range(n_boot)])
110
+ lo, hi = np.quantile(boots, [0.025, 0.975])
111
+ p_two = float(min(1.0, 2 * (min(np.sum(boots <= 0), np.sum(boots >= 0)) + 1) / (n_boot + 1))) # bootstrap two-sided p for H0: difference = 0
112
+ return {"difference": float(est), "ci_low": float(lo), "ci_high": float(hi), "p_two_sided": p_two, "n_items": n, "a": a, "b": b}
113
+
114
+
115
+ def holm(pvals: Dict[str, float], alpha: float = 0.05) -> Dict[str, dict]:
116
+ """Holm step-down: returns adjusted p-values and rejection flags."""
117
+ order = sorted(pvals, key=pvals.get)
118
+ m = len(order)
119
+ out, running = {}, 0.0
120
+ for i, name in enumerate(order):
121
+ adj = min(1.0, (m - i) * pvals[name])
122
+ running = max(running, adj)
123
+ out[name] = {"p": pvals[name], "p_holm": running, "reject": running < alpha}
124
+ return out
125
+
126
+
127
+ def one_sided_p(boots_gap: np.ndarray) -> float:
128
+ """Bootstrap p-value for H0: gap <= 0 (share of resamples with gap <= 0)."""
129
+ return float((np.sum(boots_gap <= 0) + 1) / (len(boots_gap) + 1))
pace/audit.py ADDED
@@ -0,0 +1,145 @@
1
+ """pace audit: a one-screen diagnostic card for a multi-agent evaluation (standard library plus mae_eval).
2
+
3
+ pace audit run_trace.jsonl # a run log with "mas" and "sas" calls: measured MAS/SAS resource ratios and parity
4
+ pace audit run_trace.jsonl --baseline-config sas.json # plus the rung of the single-agent baseline (keys: code/mae_eval/classifier.py)
5
+ pace audit sas.json # a baseline configuration only: its rung and whether a matched resource is declared
6
+
7
+ Badge rules (stated on the card):
8
+ PASS: COMPUTE-MATCHED a log was given, token parity holds within the tolerance (r_token ratio within 1 +/- tolerance), and, if a baseline configuration
9
+ was given, its rung is C or D; or only a configuration was given and its rung is C (rule R2) or D.
10
+ WARNING: BASELINE NOT COMPUTE-MATCHED anything else: parity violated or not measurable, or the baseline is rung A or B, or rung C with the matching unverified (R3).
11
+ Rung A or B does not make a baseline wrong; it means the comparison does not separate coordination from extra computation. Nothing here estimates tokens.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import math
17
+ from typing import Dict, Optional
18
+
19
+ BIBTEX = """@unpublished{cme2026,
20
+ title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent {LLMs}},
21
+ author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
22
+ year = {2026},
23
+ note = {Manuscript in preparation}
24
+ }"""
25
+
26
+
27
+ def _fmt(r: Optional[float]) -> str:
28
+ if r is None:
29
+ return "n/a"
30
+ return "inf" if math.isinf(r) else "%.2f\u00d7" % r
31
+
32
+
33
+ def load_input(path: str):
34
+ """Return ('config', dict) for a single JSON object without a 'system' key, else ('log', {system: [records]})."""
35
+ text = open(path, encoding="utf-8").read()
36
+ try:
37
+ obj = json.loads(text)
38
+ if isinstance(obj, dict) and "system" not in obj:
39
+ return "config", obj
40
+ except json.JSONDecodeError:
41
+ pass
42
+ from mae_eval.cli import TraceError, load_trace
43
+ recs = []
44
+ for no, line in enumerate(text.splitlines(), 1):
45
+ if line.strip():
46
+ r = json.loads(line)
47
+ if "correct" in r and "prompt_tokens" not in r: # a per-item result line written by pace.log_result or by hand
48
+ continue
49
+ recs.append({k: v for k, v in r.items() if v is not None}) # null = unreported (proxy and wrap logs): treat as absent, never as 0 by guess
50
+ import tempfile, os
51
+ fd, tmp = tempfile.mkstemp(suffix=".jsonl")
52
+ try:
53
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
54
+ fh.write("\n".join(json.dumps(r) for r in recs) + "\n")
55
+ try:
56
+ return "log", load_trace(tmp)
57
+ except TraceError as e:
58
+ raise ValueError(str(e))
59
+ finally:
60
+ os.unlink(tmp)
61
+
62
+
63
+ def audit(path: str, baseline_config: Optional[str] = None, tolerance: float = 0.05, price_in: Optional[float] = None, price_out: Optional[float] = None) -> Dict:
64
+ from mae_eval.auditor import EvaluationAuditor
65
+ from mae_eval.classifier import classify_with_reason
66
+ from mae_eval.cli import session_from_records
67
+ from mae_eval.tracker import ResourceTracker
68
+ kind, data = load_input(path)
69
+ card: Dict = {"input": kind, "tolerance": tolerance, "rung": None, "rule": None, "rung_reason": None, "ratios": None, "parity": None, "notes": []}
70
+ cfg = None
71
+ if kind == "config":
72
+ cfg = data
73
+ elif baseline_config:
74
+ cfg = json.load(open(baseline_config, encoding="utf-8"))
75
+ if cfg is not None:
76
+ card["rung"], card["rule"], card["rung_reason"] = classify_with_reason(cfg)
77
+ if kind == "log":
78
+ if "mas" not in data or "sas" not in data:
79
+ card["notes"].append("the log needs both 'mas' and 'sas' calls to compute ratios; systems found: %s" % ", ".join(sorted(data)))
80
+ else:
81
+ priced = price_in is not None and price_out is not None
82
+ tracker = ResourceTracker("model", "cme-audit")
83
+ if priced:
84
+ tracker.register_pricing(price_in, price_out)
85
+ vec = {k: session_from_records(v, tracker).get_resource_vector(allow_missing_pricing=not priced) for k, v in data.items() if k in ("mas", "sas")}
86
+ aud = EvaluationAuditor(vec["mas"], vec["sas"])
87
+ r = aud.ratios()
88
+ card["ratios"] = {"r_token": r["r_token"], "r_lat": r["r_lat"], "r_cost": r["r_cost"]}
89
+ tok = r["r_token"]
90
+ card["parity"] = bool(tok is not None and not math.isinf(tok) and abs(tok - 1.0) <= tolerance)
91
+ if not priced:
92
+ card["notes"].append("no prices given: cost ratio not computed")
93
+ elif cfg is not None:
94
+ card["notes"].append("a configuration alone has no measured tokens: no resource ratios")
95
+ if card["rung"] is None:
96
+ card["notes"].append("baseline rung not classified: pass --baseline-config or a configuration file")
97
+ rung, rule = card["rung"], card["rule"]
98
+ if kind == "log":
99
+ ok = bool(card["parity"]) and (rung in (None, "C", "D"))
100
+ if rung is None:
101
+ ok = False # parity alone with an unknown baseline is not enough to call the comparison matched
102
+ else:
103
+ ok = rung == "D" or (rung == "C" and rule == "R2")
104
+ card["badge"] = "PASS: COMPUTE-MATCHED" if ok else "WARNING: BASELINE NOT COMPUTE-MATCHED"
105
+ card["ok"] = ok
106
+ return card
107
+
108
+
109
+ def render(card: Dict, ascii_only: bool = False) -> str:
110
+ w = 66
111
+ line = "\u2500" * w
112
+ out = ["\u250c" + line + "\u2510"]
113
+
114
+ def row(s=""):
115
+ import textwrap
116
+ for part in textwrap.wrap(s, w - 3, subsequent_indent=" ", drop_whitespace=False)[:] or [""]:
117
+ out.append("\u2502 " + part.ljust(w - 1) + "\u2502")
118
+ row("PACE audit")
119
+ out.append("\u251c" + line + "\u2524")
120
+ if card["rung"]:
121
+ cat = {"A": "Category A (single greedy pass)", "B": "Category B (sampled, unmatched tokens)", "C": "Category C (resource-matched or test-time scaling)",
122
+ "D": "Category D (budget sweep)"}[card["rung"]]
123
+ row("Baseline rung : %s [%s]" % (cat, card["rule"]))
124
+ row(" %s" % card["rung_reason"])
125
+ else:
126
+ row("Baseline rung : not classified")
127
+ r = card["ratios"]
128
+ if r:
129
+ row("MAS / SAS : R_token %s R_lat %s R_cost %s" % (_fmt(r["r_token"]), _fmt(r["r_lat"]), _fmt(r["r_cost"])))
130
+ row("Token parity : %s (tolerance %.0f%%)" % ("holds" if card["parity"] else "violated", 100 * card["tolerance"]))
131
+ else:
132
+ row("MAS / SAS : not available")
133
+ row()
134
+ row("[%s]" % card["badge"])
135
+ for n in card["notes"]:
136
+ row("note: " + n)
137
+ out.append("\u2514" + line + "\u2518")
138
+ out.append("")
139
+ out.append("To cite (BibTeX; the manuscript is in preparation):")
140
+ out.append(BIBTEX)
141
+ text = "\n".join(out)
142
+ if ascii_only:
143
+ for u, c in (("\u250c", "+"), ("\u2510", "+"), ("\u2514", "+"), ("\u2518", "+"), ("\u251c", "+"), ("\u2524", "+"), ("\u2500", "-"), ("\u2502", "|"), ("\u00d7", "x")):
144
+ text = text.replace(u, c)
145
+ return text
pace/baseline.py ADDED
@@ -0,0 +1,162 @@
1
+ """pace baseline: a cached, resumable greedy run plus a pool of independent sampled completions.
2
+
3
+ Every call is appended to a JSONL cache as soon as it finishes, so an interrupted run resumes where it stopped (a record is keyed by
4
+ item id, arm and index). Sampled arms use temperature > 0 and a distinct seed per (item, index); the greedy arm uses temperature 0.
5
+ Token counts come from the endpoint's usage field. Reasoning tokens are recorded only when the endpoint reports them; otherwise the
6
+ field is null and nothing is inferred.
7
+
8
+ Benchmarks: a JSONL file with one object per line {"id", "question", "answer"}; or the built-in `toy` set (synthetic arithmetic, for
9
+ smoke tests only; it says nothing about any model).
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import hashlib
14
+ import json
15
+ import os
16
+ import re
17
+ import time
18
+ import urllib.request
19
+ from typing import Callable, Dict, Iterable, List, Optional
20
+
21
+ PROMPT = ("Solve the problem. Think step by step, then finish with a final line of the form 'Answer: <answer>'.\n\nProblem: {q}")
22
+
23
+
24
+ def toy_bench(n: int = 20) -> List[dict]:
25
+ """Synthetic arithmetic items. Not a benchmark: used only to exercise the pipeline."""
26
+ return [{"id": "toy%03d" % i, "question": "What is %d + %d * %d?" % (i + 3, i + 2, 4), "answer": str(i + 3 + (i + 2) * 4)} for i in range(n)]
27
+
28
+
29
+ def load_bench(spec: str, limit: Optional[int] = None) -> List[dict]:
30
+ if spec == "toy":
31
+ items = toy_bench(limit or 20)
32
+ else:
33
+ items = [json.loads(l) for l in open(spec, encoding="utf-8") if l.strip()]
34
+ for it in items:
35
+ if not {"id", "question", "answer"} <= set(it):
36
+ raise ValueError("bench items need id, question and answer")
37
+ return items[:limit] if limit else items
38
+
39
+
40
+ _ANS = re.compile(r"answer\s*[:=]\s*(.+)", re.I)
41
+
42
+
43
+ def extract_answer(text: str) -> str:
44
+ """Last 'Answer: x' line, else the last number or standalone letter A-J, normalised (trimmed, lower-cased, no $ , or trailing period)."""
45
+ m = _ANS.findall(text or "")
46
+ cand = m[-1] if m else ((re.findall(r"-?\d[\d,]*\.?\d*", text or "") or [""])[-1])
47
+ cand = cand.strip().strip("*$ ").rstrip(".").replace(",", "").lower()
48
+ cand = re.sub(r"^\\boxed\{(.*)\}$", r"\1", cand)
49
+ return cand
50
+
51
+
52
+ def norm_gold(a: str) -> str:
53
+ return str(a).strip().rstrip(".").replace(",", "").lower()
54
+
55
+
56
+ class OpenAICompatClient:
57
+ """Minimal client for /v1/chat/completions (Ollama, vLLM, llama.cpp server, OpenAI-compatible hosts). Standard library only."""
58
+
59
+ def __init__(self, base_url: str = "http://localhost:11434/v1", api_key: str = "none", timeout: float = 600.0):
60
+ self.base, self.key, self.timeout = base_url.rstrip("/"), api_key, timeout
61
+
62
+ def chat(self, model: str, prompt: str, temperature: float, seed: int, max_tokens: int = 2048) -> dict:
63
+ body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}], "temperature": temperature, "seed": seed,
64
+ "max_tokens": max_tokens, "stream": False}).encode()
65
+ req = urllib.request.Request(self.base + "/chat/completions", data=body,
66
+ headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.key})
67
+ t0 = time.time()
68
+ with urllib.request.urlopen(req, timeout=self.timeout) as r:
69
+ data = json.loads(r.read())
70
+ lat = time.time() - t0
71
+ msg = data["choices"][0]["message"]
72
+ usage = data.get("usage", {})
73
+ det = usage.get("completion_tokens_details") or {}
74
+ return {"text": msg.get("content") or "", "prompt_tokens": usage.get("prompt_tokens"), "completion_tokens": usage.get("completion_tokens"),
75
+ "reasoning_tokens": det.get("reasoning_tokens"), "latency": lat}
76
+
77
+
78
+ def ollama_digest(model: str, host: str = "http://localhost:11434") -> Optional[str]:
79
+ """Best-effort model digest from Ollama /api/tags; None when unavailable."""
80
+ try:
81
+ with urllib.request.urlopen(host + "/api/tags", timeout=5) as r:
82
+ for m in json.loads(r.read()).get("models", []):
83
+ if m.get("name") == model or m.get("model") == model:
84
+ return m.get("digest")
85
+ except Exception:
86
+ return None
87
+ return None
88
+
89
+
90
+ def read_cache(path: str) -> Dict[tuple, dict]:
91
+ done: Dict[tuple, dict] = {}
92
+ if os.path.exists(path):
93
+ for l in open(path, encoding="utf-8"):
94
+ if l.strip():
95
+ r = json.loads(l)
96
+ done[(r["item"], r["arm"], r["idx"])] = r
97
+ return done
98
+
99
+
100
+ def sample_seed(base_seed: int, item: str, idx: int) -> int:
101
+ """Distinct, reproducible seed per (item, sample index)."""
102
+ return int(hashlib.sha256(("%d|%s|%d" % (base_seed, item, idx)).encode()).hexdigest()[:8], 16)
103
+
104
+
105
+ def run_baseline(client, model: str, items: Iterable[dict], out_path: str, k: int = 16, temperature: float = 0.7, seed: int = 0,
106
+ max_tokens: int = 2048, digest: Optional[str] = None, progress: Optional[Callable[[str], None]] = None) -> dict:
107
+ """Run greedy + k sampled completions per item, appending to out_path. Returns counts: new, cached, errors."""
108
+ if temperature <= 0:
109
+ raise ValueError("sampled arms need temperature > 0")
110
+ done = read_cache(out_path)
111
+ new = cached = errors = 0
112
+ with open(out_path, "a", encoding="utf-8") as fh:
113
+ for it in items:
114
+ gold = norm_gold(it["answer"])
115
+ jobs = [("greedy", 0, 0.0)] + [("sample", j, temperature) for j in range(k)]
116
+ for arm, j, temp in jobs:
117
+ if (it["id"], arm, j) in done:
118
+ cached += 1
119
+ continue
120
+ s = sample_seed(seed, it["id"], j) if arm == "sample" else seed
121
+ try:
122
+ r = client.chat(model, PROMPT.format(q=it["question"]), temp, s, max_tokens)
123
+ except Exception as e: # network or server error: skip and report, do not cache
124
+ errors += 1
125
+ if progress:
126
+ progress("error %s %s %d: %s" % (it["id"], arm, j, e))
127
+ continue
128
+ ans = extract_answer(r["text"])
129
+ rec = {"item": it["id"], "arm": arm, "idx": j, "seed": s, "temperature": temp, "model": model, "model_digest": digest,
130
+ "answer": ans, "gold": gold, "correct": ans == gold, "prompt_tokens": r["prompt_tokens"],
131
+ "completion_tokens": r["completion_tokens"], "reasoning_tokens": r["reasoning_tokens"], "latency": r["latency"]}
132
+ fh.write(json.dumps(rec) + "\n")
133
+ fh.flush()
134
+ new += 1
135
+ if progress:
136
+ progress("item %s done (new %d, cached %d, errors %d)" % (it["id"], new, cached, errors))
137
+ return {"new": new, "cached": cached, "errors": errors}
138
+
139
+
140
+ def load_pool(path: str, k: Optional[int] = None):
141
+ """Build a frontier.Pool and the greedy records from a baseline cache. Items lacking k samples are dropped (reported)."""
142
+ from .frontier import Pool
143
+ rows = list(read_cache(path).values())
144
+ by: Dict[str, dict] = {}
145
+ greedy: Dict[str, dict] = {}
146
+ for r in rows:
147
+ if r["arm"] == "greedy":
148
+ greedy[r["item"]] = r
149
+ else:
150
+ d = by.setdefault(r["item"], {"gold": r["gold"], "samples": {}})
151
+ d["samples"][r["idx"]] = r
152
+ need = k or max((len(d["samples"]) for d in by.values()), default=0)
153
+ items = {}
154
+ dropped = 0
155
+ for it, d in by.items():
156
+ ss = [d["samples"][i] for i in sorted(d["samples"])]
157
+ if len(ss) < need or any(s["prompt_tokens"] is None or s["completion_tokens"] is None for s in ss):
158
+ dropped += 1
159
+ continue
160
+ items[it] = {"gold": d["gold"], "answers": [s["answer"] for s in ss[:need]],
161
+ "tokens": [s["prompt_tokens"] + s["completion_tokens"] for s in ss[:need]]}
162
+ return Pool(items, need), greedy, dropped
pace/card.py ADDED
@@ -0,0 +1,205 @@
1
+ """Standalone PACE resource vectors and card exports."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections import defaultdict
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+
11
+ def load_log(path: str) -> list[dict[str, Any]]:
12
+ rows = [
13
+ json.loads(line)
14
+ for line in Path(path).read_text(encoding="utf-8").splitlines()
15
+ if line.strip()
16
+ ]
17
+ if not rows:
18
+ raise ValueError("empty PACE log")
19
+ required = {"item", "agent", "round", "correct"}
20
+ if not all(required <= set(row) for row in rows):
21
+ raise ValueError("PACE logs require item, agent, round, and correct")
22
+ return rows
23
+
24
+
25
+ def resource_vector(rows: list[dict[str, Any]]) -> dict[str, float | None]:
26
+ items = {str(row["item"]) for row in rows}
27
+ n_items = max(1, len(items))
28
+ calls = len(rows)
29
+ rounds = {int(row["round"]) for row in rows}
30
+ # Independent trajectories per item = calls made in the item's first round, before any communication (an agent that answers again in a later round is the same trajectory, not a new sample)
31
+ first_round, later_calls = {}, 0
32
+ for row in rows:
33
+ first_round.setdefault(str(row["item"]), int(row["round"]))
34
+ first_round[str(row["item"])] = min(first_round[str(row["item"])], int(row["round"]))
35
+ trajectories = {}
36
+ for row in rows:
37
+ key = str(row["item"])
38
+ if int(row["round"]) == first_round[key]:
39
+ trajectories[key] = trajectories.get(key, 0) + 1
40
+ else:
41
+ later_calls += 1
42
+ token_values = [
43
+ float(row["prompt_tokens"]) + float(row["completion_tokens"])
44
+ for row in rows
45
+ if row.get("prompt_tokens") is not None
46
+ and row.get("completion_tokens") is not None
47
+ ]
48
+ reason_values = [
49
+ float(row["reasoning_tokens"])
50
+ for row in rows
51
+ if row.get("reasoning_tokens") is not None
52
+ ]
53
+ cost_values = [
54
+ float(row["cost_usd"])
55
+ for row in rows
56
+ if row.get("cost_usd") is not None
57
+ ]
58
+ lat_values = [
59
+ float(row["latency_s"])
60
+ for row in rows
61
+ if row.get("latency_s") is not None
62
+ ]
63
+ return {
64
+ "r_reason": (sum(reason_values) / n_items) if reason_values else None,
65
+ "r_token": (sum(token_values) / n_items) if token_values else None,
66
+ "r_cost": (sum(cost_values) / n_items) if cost_values else None,
67
+ "r_lat": (sum(lat_values) / n_items) if lat_values else None,
68
+ "r_sample": sum(trajectories.values()) / n_items,
69
+ "r_sys": float(
70
+ sum(1 for row in rows if row.get("tool_calls")) / n_items
71
+ ),
72
+ "r_call": calls / n_items,
73
+ "r_round": float(max(rounds) - min(rounds) + 1),
74
+ "r_context": later_calls / n_items,
75
+ "r_scaff": float(
76
+ sum(1 for row in rows if row.get("scaffold") or row.get("tool_calls"))
77
+ ),
78
+ }
79
+
80
+
81
+ def classify_baseline(vector: dict[str, float | None]) -> str:
82
+ """Rung (A, B or C) that this log would occupy as a single-agent baseline, following Table 7 of the paper; "n/a" if the log is a multi-agent system.
83
+
84
+ A: one greedy-style pass (one trajectory, one round, no later-round context). B: several independent trajectories in one round (sampled; no named
85
+ resource is known to be matched). C: one trajectory refined over several rounds (a named resource, the iteration count, may be matched; confirm with
86
+ `pace classify`). Rung D (the Pareto frontier) is a property of a family of runs and cannot be read from one log.
87
+ """
88
+ return classify_baseline_detail(vector)[0]
89
+
90
+
91
+ def classify_baseline_detail(vector: dict[str, float | None]) -> tuple[str, str]:
92
+ r_round = float(vector.get("r_round") or 0.0)
93
+ r_sample = float(vector.get("r_sample") or 0.0)
94
+ r_context = float(vector.get("r_context") or 0.0)
95
+ if r_round <= 1 and r_sample <= 1 and r_context <= 0:
96
+ return "A", "one trajectory, one round"
97
+ if r_round <= 1 and r_context <= 0:
98
+ return "B", "%.3g independent trajectories in one round, no resource known to be matched" % r_sample
99
+ if r_sample <= 1:
100
+ return "C", "one trajectory over %.3g rounds; C only if the iteration ceiling is matched" % r_round
101
+ return "n/a", "several trajectories with later-round communication: a multi-agent system, not a single-agent baseline"
102
+
103
+
104
+ def feasibility(vector: dict[str, float | None]) -> dict[str, Any]:
105
+ if vector.get("r_token") is None:
106
+ return {
107
+ "status": "insufficient_data",
108
+ "active_axes": [k for k, v in vector.items() if v is not None and v > 0],
109
+ "direction": "missing token logs",
110
+ }
111
+ positive = [key for key, value in vector.items() if value is not None and value > 0]
112
+ return {
113
+ "status": "measured_only" if positive else "insufficient_data",
114
+ "active_axes": positive,
115
+ "direction": "requires baseline comparison for directional feasibility",
116
+ }
117
+
118
+
119
+ def transition_matrix(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
120
+ grouped = defaultdict(dict)
121
+ for row in rows:
122
+ grouped[(str(row["item"]), str(row["agent"]))][int(row["round"])] = bool(
123
+ row["correct"]
124
+ )
125
+ counts = defaultdict(lambda: [0, 0])
126
+ by_item = defaultdict(dict)
127
+ for (item, agent), states in grouped.items():
128
+ by_item[item][agent] = states
129
+ for agents in by_item.values():
130
+ rounds = sorted({round_no for states in agents.values() for round_no in states})
131
+ for current, following in zip(rounds, rounds[1:]):
132
+ present = [
133
+ states
134
+ for states in agents.values()
135
+ if current in states and following in states
136
+ ]
137
+ for states in present:
138
+ peers_right = sum(int(other[current]) for other in present) - int(
139
+ states[current]
140
+ )
141
+ key = "own=%d,peers_right=%d" % (int(states[current]), peers_right)
142
+ counts[key][0] += int(states[following])
143
+ counts[key][1] += 1
144
+ return {
145
+ key: {"p_next": good / total, "n": total}
146
+ for key, (good, total) in counts.items()
147
+ }
148
+
149
+
150
+ def build_card(rows: list[dict[str, Any]]) -> dict[str, Any]:
151
+ vector = resource_vector(rows)
152
+ return {
153
+ "schema": "pace-card/v1",
154
+ "n_records": len(rows),
155
+ "n_items": len({str(row["item"]) for row in rows}),
156
+ "resource_vector": vector,
157
+ "baseline": classify_baseline(vector),
158
+ "baseline_reason": classify_baseline_detail(vector)[1],
159
+ "feasibility": feasibility(vector),
160
+ "transition_matrix": transition_matrix(rows),
161
+ }
162
+
163
+
164
+ def to_markdown(card: dict[str, Any]) -> str:
165
+ lines = [
166
+ "# PACE Card",
167
+ "",
168
+ "| Field | Value |",
169
+ "|---|---:|",
170
+ "| Baseline | %s |" % card["baseline"],
171
+ "| Items | %s |" % card["n_items"],
172
+ ]
173
+ lines += [
174
+ "| %s | %s |" % (key, ("%.3f" % value if value is not None else "n/a"))
175
+ for key, value in card["resource_vector"].items()
176
+ ]
177
+ lines += [
178
+ "",
179
+ "## Transition matrix",
180
+ "",
181
+ "| State | P(correct at t+1) | n |",
182
+ "|---|---:|---:|",
183
+ ]
184
+ lines += [
185
+ "| %s | %.3f | %d |" % (key, value["p_next"], value["n"])
186
+ for key, value in card["transition_matrix"].items()
187
+ ]
188
+ return "\n".join(lines) + "\n"
189
+
190
+
191
+ def to_latex(card: dict[str, Any]) -> str:
192
+ rows = [
193
+ "\\begin{tcolorbox}[title={PACE Card}]",
194
+ "\\textbf{Baseline:} %s\\par" % card["baseline"],
195
+ "\\begin{tabular}{lr}",
196
+ "\\toprule",
197
+ "Axis & Value\\\\",
198
+ "\\midrule",
199
+ ]
200
+ rows += [
201
+ "%s & %s\\\\" % (key.replace("_", "\\_"), ("%.3f" % value if value is not None else "n/a"))
202
+ for key, value in card["resource_vector"].items()
203
+ ]
204
+ rows += ["\\bottomrule", "\\end{tabular}", "\\end{tcolorbox}"]
205
+ return "\n".join(rows) + "\n"