pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
mae_eval/auditor.py ADDED
@@ -0,0 +1,112 @@
1
+ """EvaluationAuditor: compares a multi-agent resource vector with its single-agent baseline and flags confounders."""
2
+ from __future__ import annotations
3
+
4
+ import math
5
+ from dataclasses import dataclass
6
+ from typing import Dict, List, Optional, Sequence, Union
7
+
8
+ from .tracker import Session
9
+ from .vector import DIMENSIONS, ControlState, ResourceVector
10
+
11
+ # Default dimensions that must be matched for a parity claim, by the question being asked.
12
+ PARITY_DIMS_DEFAULT = ("r_token",)
13
+
14
+ _MESSAGES = {
15
+ "r_sample": "sample counts differ: gains may reflect ensembling (more draws), not coordination",
16
+ "r_reason": "reasoning-token budgets differ: thinking compute is not matched",
17
+ "r_token": "total tokens differ: scaffold, history echoes, or extra calls consume more budget",
18
+ "r_cost": "monetary cost differs: comparison is not cost-matched",
19
+ "r_lat": "wall-clock latency differs (serialization or concurrency)",
20
+ "r_sys": "tool or search invocations differ",
21
+ }
22
+
23
+
24
+ class ComputeParityError(AssertionError):
25
+ """Raised by assert_compute_parity when a dimension is outside tolerance."""
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class Flag:
30
+ dimension: str
31
+ ratio: Optional[float]
32
+ state: ControlState
33
+ message: str
34
+
35
+
36
+ def _as_vector(x: Union[ResourceVector, Session]) -> ResourceVector:
37
+ return x.get_resource_vector(allow_missing_pricing=True) if isinstance(x, Session) else x
38
+
39
+
40
+ class EvaluationAuditor:
41
+ """mas / sas may be ResourceVector or a recorded Session.
42
+
43
+ approx_tolerance: relative deviation up to which a dimension counts as 'Approximately Matched' (default 0.25).
44
+ """
45
+
46
+ def __init__(self, mas: Union[ResourceVector, Session], sas: Union[ResourceVector, Session, None] = None,
47
+ approx_tolerance: float = 0.25):
48
+ self.mas = _as_vector(mas)
49
+ self.sas = _as_vector(sas) if sas is not None else None
50
+ self.approx_tolerance = approx_tolerance
51
+
52
+ def _need_sas(self) -> ResourceVector:
53
+ if self.sas is None:
54
+ raise ValueError("a single-agent baseline is required for this operation")
55
+ return self.sas
56
+
57
+ def ratios(self) -> Dict[str, Optional[float]]:
58
+ """MAS / SAS per dimension. None if either side is unreported or both are 0 (0/0 is undefined, printed as n/a); inf if the baseline is 0 and MAS is not."""
59
+ sas = self._need_sas()
60
+ out: Dict[str, Optional[float]] = {}
61
+ for d in DIMENSIONS:
62
+ m, s = getattr(self.mas, d), getattr(sas, d)
63
+ if m is None or s is None:
64
+ out[d] = None
65
+ elif s == 0:
66
+ out[d] = None if m == 0 else math.inf
67
+ else:
68
+ out[d] = m / s
69
+ return out
70
+
71
+ def control_states(self, tolerance: float = 0.05) -> Dict[str, ControlState]:
72
+ states: Dict[str, ControlState] = {}
73
+ sas = self._need_sas()
74
+ for d, r in self.ratios().items():
75
+ if r is None and getattr(self.mas, d) == 0 and getattr(sas, d) == 0:
76
+ states[d] = ControlState.STRICT # both systems consumed none: equal, but the ratio is undefined
77
+ elif r is None:
78
+ states[d] = ControlState.UNREPORTED
79
+ else:
80
+ dev = abs(r - 1.0)
81
+ states[d] = (ControlState.STRICT if dev <= tolerance
82
+ else ControlState.APPROXIMATE if dev <= self.approx_tolerance
83
+ else ControlState.UNMATCHED)
84
+ return states
85
+
86
+ def flag_confounders(self, tolerance: float = 0.05) -> List[Flag]:
87
+ """One Flag per dimension that is not strictly controlled, with an explanation."""
88
+ ratios = self.ratios()
89
+ flags = []
90
+ for d, st in self.control_states(tolerance).items():
91
+ if st is ControlState.STRICT:
92
+ continue
93
+ msg = _MESSAGES[d] if st is not ControlState.UNREPORTED else "not reported by one or both systems"
94
+ flags.append(Flag(d, ratios[d], st, msg))
95
+ return flags
96
+
97
+ def assert_compute_parity(self, tolerance: float = 0.05, dims: Sequence[str] = PARITY_DIMS_DEFAULT) -> None:
98
+ """Raise ComputeParityError unless every dimension in `dims` is reported and within `tolerance` of ratio 1."""
99
+ bad = []
100
+ ratios = self.ratios()
101
+ for d in dims:
102
+ if d not in DIMENSIONS:
103
+ raise ValueError("unknown dimension %r" % d)
104
+ r = ratios[d]
105
+ if r is None and getattr(self.mas, d) == 0 and getattr(self._need_sas(), d) == 0:
106
+ continue # 0/0: both systems consumed none of this resource
107
+ if r is None:
108
+ bad.append("%s: unreported" % d)
109
+ elif abs(r - 1.0) > tolerance:
110
+ bad.append("%s: MAS/SAS = %.3f (tolerance %.3f)" % (d, r, tolerance))
111
+ if bad:
112
+ raise ComputeParityError("compute parity violated: " + "; ".join(bad))
mae_eval/classifier.py ADDED
@@ -0,0 +1,62 @@
1
+ """Classify a single-agent baseline into a rung of the baseline ladder (paper Section 3, Figure 1).
2
+
3
+ Rungs: A greedy single pass; B sampled baseline (k independent samples, no communication, tokens not matched); C a named resource held
4
+ equal, or declared single-agent test-time scaling; D a budget sweep or Pareto frontier. The rung describes the baseline a multi-agent
5
+ system is compared against, so a configuration that communicates between agents is not a baseline and is rejected.
6
+
7
+ Five rules, applied in this order (first match wins):
8
+ R1 budget sweep or Pareto frontier (n_budgets >= 3, or pareto true) -> D
9
+ R2 a named matched resource (thinking tokens, total tokens, cost, latency,
10
+ iterations, generations) with matching declared -> C
11
+ R3 extended thinking or search (MCTS, tree search, best-of-n with a verifier)
12
+ without a named matched resource -> C (matching unverified; see `reason`)
13
+ R4 more than one independent sample combined by vote or selection -> B
14
+ R5 otherwise (one sample, no test-time scaling) -> A
15
+
16
+ Note on the wording "Rung D: combined multi-agent plus sampling or search architectures": that describes a system under test, not a
17
+ baseline, and the paper's Section 3 defines D as a Pareto sweep, so this module follows the paper. Evaluate such combined systems
18
+ as the multi-agent side of a contrast against the rung chosen here.
19
+
20
+ config keys (all optional): n_samples (int), aggregation (e.g. "majority_vote", "best_of_n"), extended_thinking (bool),
21
+ thinking_budget (int tokens), search ("mcts", "tree", ...), matched_resource (str), matching_declared (bool, default True when a
22
+ resource is named), n_budgets (int), pareto (bool), communication (bool; must be false or absent).
23
+ """
24
+ from __future__ import annotations
25
+
26
+ from typing import Dict, Literal, Tuple
27
+
28
+ Rung = Literal["A", "B", "C", "D"]
29
+ MATCHABLE = {"thinking_tokens", "total_tokens", "tokens", "cost", "latency", "iterations", "generations", "calls", "flops"}
30
+ SEARCH = {"mcts", "tree", "tree_search", "beam", "best_first"}
31
+
32
+
33
+ def classify_with_reason(config: Dict) -> Tuple[Rung, str, str]:
34
+ """Return (rung, rule id, reason). Raises ValueError for a communicating configuration or invalid values."""
35
+ if config.get("communication"):
36
+ raise ValueError("a configuration with inter-agent communication is not a single-agent baseline")
37
+ n_samples = int(config.get("n_samples", 1))
38
+ if n_samples < 1:
39
+ raise ValueError("n_samples must be at least 1")
40
+ n_budgets = int(config.get("n_budgets", 0) or 0)
41
+ if config.get("pareto") or n_budgets >= 3:
42
+ return "D", "R1", "budget sweep or Pareto frontier"
43
+ res = str(config.get("matched_resource", "") or "").lower().replace(" ", "_")
44
+ if res:
45
+ if res not in MATCHABLE:
46
+ raise ValueError("unknown matched_resource %r; use one of %s" % (res, sorted(MATCHABLE)))
47
+ if config.get("matching_declared", True):
48
+ return "C", "R2", "resource held equal: %s" % res
49
+ search = str(config.get("search", "") or "").lower()
50
+ aggregation = str(config.get("aggregation", "") or "").lower()
51
+ if config.get("extended_thinking") or config.get("thinking_budget") or search in SEARCH or (aggregation == "best_of_n" and config.get("verifier")):
52
+ return "C", "R3", "extended thinking or search without a named matched resource (matching unverified)"
53
+ if n_samples > 1:
54
+ if aggregation not in ("majority_vote", "vote", "plurality", "best_of_n", "self_consistency", "selection", ""):
55
+ raise ValueError("unknown aggregation %r" % aggregation)
56
+ return "B", "R4", "%d independent samples combined without communication" % n_samples
57
+ return "A", "R5", "single sample, no test-time scaling"
58
+
59
+
60
+ def classify_baseline_rung(config_dict: Dict) -> Rung:
61
+ """Rung (A, B, C or D) of a single-agent baseline configuration."""
62
+ return classify_with_reason(config_dict)[0]
mae_eval/cli.py ADDED
@@ -0,0 +1,210 @@
1
+ """Command-line interface: ``mae-eval audit --log trace.jsonl``.
2
+
3
+ Trace format: JSON Lines, one generation or tool call per line. Fields:
4
+ system "mas" or "sas" (required)
5
+ agent_id string (default "agent")
6
+ prompt_tokens, completion_tokens integers (required unless "chunks" is given for streaming records)
7
+ chunks optional list of per-chunk token counts for a streamed completion (replaces completion_tokens)
8
+ reasoning_tokens, scaffold_tokens integers, default 0 (subsets of completion / prompt tokens)
9
+ tool_calls integer, default 0; n_completions integer, default 1 (0 for a pure tool call)
10
+ start, end seconds on a common clock, default 0 (concurrent calls overlap)
11
+ Note: code/examples/sample_trace.jsonl is a hand-written synthetic example, not a trace of a model run.
12
+ A MAS/SAS ratio prints as n/a when both values are 0 (0/0 is undefined).
13
+ Exit status: 0 success, 1 input error, 2 parity violated (only with --assert-parity).
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import json
19
+ import sys
20
+ from typing import Dict, List, Optional, Sequence
21
+
22
+ from .auditor import ComputeParityError, EvaluationAuditor
23
+ from .tracker import ResourceTracker, Session
24
+ from .vector import DIMENSIONS, ResourceVector
25
+
26
+
27
+ class TraceError(ValueError):
28
+ pass
29
+
30
+
31
+ def load_trace(path: str) -> Dict[str, List[dict]]:
32
+ """Read a JSONL trace and group records by system. Raises TraceError with the line number on bad input."""
33
+ groups: Dict[str, List[dict]] = {}
34
+ with open(path, "r", encoding="utf-8") as fh:
35
+ for no, line in enumerate(fh, 1):
36
+ line = line.strip()
37
+ if not line:
38
+ continue
39
+ try:
40
+ rec = json.loads(line)
41
+ except json.JSONDecodeError as e:
42
+ raise TraceError("line %d: invalid JSON (%s)" % (no, e.msg))
43
+ if not isinstance(rec, dict):
44
+ raise TraceError("line %d: expected a JSON object" % no)
45
+ system = rec.get("system")
46
+ if system not in ("mas", "sas"):
47
+ raise TraceError("line %d: 'system' must be 'mas' or 'sas'" % no)
48
+ if "chunks" not in rec and ("prompt_tokens" not in rec or "completion_tokens" not in rec):
49
+ raise TraceError("line %d: need prompt_tokens and completion_tokens (or chunks)" % no)
50
+ rec["_line"] = no
51
+ groups.setdefault(system, []).append(rec)
52
+ if not groups:
53
+ raise TraceError("trace is empty")
54
+ return groups
55
+
56
+
57
+ def session_from_records(records: List[dict], tracker: ResourceTracker) -> Session:
58
+ with tracker.trace() as s:
59
+ for r in records:
60
+ try:
61
+ common = dict(reasoning_tokens=int(r.get("reasoning_tokens", 0)), scaffold_tokens=int(r.get("scaffold_tokens", 0)),
62
+ tool_calls=int(r.get("tool_calls", 0)), start=float(r.get("start", 0.0)))
63
+ agent = str(r.get("agent_id", "agent"))
64
+ if "chunks" in r:
65
+ s.record_stream(agent, int(r.get("prompt_tokens", 0)), r["chunks"], **common)
66
+ else:
67
+ s.record_call(agent, int(r["prompt_tokens"]), int(r["completion_tokens"]), n_completions=int(r.get("n_completions", 1)),
68
+ end=float(r.get("end", common["start"])), **common)
69
+ except (ValueError, TypeError) as e:
70
+ raise TraceError("line %d: %s" % (r["_line"], e))
71
+ return s
72
+
73
+
74
+ def _vec_dict(v: ResourceVector) -> dict:
75
+ d = v.to_dict(include_scaffold=True)
76
+ d["scaffold_pct"] = v.scaffold_pct
77
+ return d
78
+
79
+
80
+ def run_audit(args: argparse.Namespace, out=None) -> int:
81
+ out = out or sys.stdout
82
+ try:
83
+ groups = load_trace(args.log)
84
+ except (OSError, TraceError) as e:
85
+ print("error: %s" % e, file=sys.stderr)
86
+ return 1
87
+ tracker = ResourceTracker(args.model, "cli")
88
+ priced = args.price_in is not None and args.price_out is not None
89
+ if priced:
90
+ tracker.register_pricing(args.price_in, args.price_out)
91
+ vectors: Dict[str, ResourceVector] = {}
92
+ try:
93
+ for name, recs in groups.items():
94
+ vectors[name] = session_from_records(recs, tracker).get_resource_vector(allow_missing_pricing=not priced)
95
+ except (TraceError, ValueError) as e:
96
+ print("error: %s" % e, file=sys.stderr)
97
+ return 1
98
+ report: dict = {"vectors": {k: _vec_dict(v) for k, v in vectors.items()}}
99
+ status = 0
100
+ if "mas" in vectors and "sas" in vectors:
101
+ aud = EvaluationAuditor(vectors["mas"], vectors["sas"])
102
+ dims = tuple(d.strip() for d in args.dims.split(",") if d.strip())
103
+ report["ratios_mas_over_sas"] = aud.ratios()
104
+ report["control_states"] = {k: v.value for k, v in aud.control_states(args.tolerance).items()}
105
+ report["flags"] = [{"dimension": f.dimension, "ratio": f.ratio, "state": f.state.value, "message": f.message}
106
+ for f in aud.flag_confounders(args.tolerance)]
107
+ try:
108
+ aud.assert_compute_parity(args.tolerance, dims)
109
+ report["parity"] = {"dims": list(dims), "tolerance": args.tolerance, "ok": True}
110
+ except ComputeParityError as e:
111
+ report["parity"] = {"dims": list(dims), "tolerance": args.tolerance, "ok": False, "detail": str(e)}
112
+ if args.assert_parity:
113
+ status = 2
114
+ else:
115
+ report["note"] = "only one system present; no comparison made"
116
+ if args.json:
117
+ json.dump(report, out, indent=2, default=lambda x: None if x != x else str(x))
118
+ out.write("\n")
119
+ else:
120
+ for name, v in vectors.items():
121
+ out.write("%s: " % name.upper() + ", ".join("%s=%s" % (d, getattr(v, d)) for d in DIMENSIONS))
122
+ if v.scaffold_pct is not None:
123
+ out.write(", scaffold=%.1f%%" % v.scaffold_pct)
124
+ out.write("\n")
125
+ if "ratios_mas_over_sas" in report:
126
+ out.write("MAS/SAS ratios: " + ", ".join("%s=%s" % (k, "n/a" if r is None else "%.3g" % r) for k, r in report["ratios_mas_over_sas"].items()) + "\n")
127
+ for f in report["flags"]:
128
+ out.write("FLAG %s [%s]: %s\n" % (f["dimension"], f["state"], f["message"]))
129
+ out.write("Parity on %s within %.3g: %s\n" % (",".join(report["parity"]["dims"]), args.tolerance, "OK" if report["parity"]["ok"] else "VIOLATED"))
130
+ return status
131
+
132
+
133
+ def run_checklist(args: argparse.Namespace, out=None) -> int:
134
+ out = out or sys.stdout
135
+ if args.format == "latex":
136
+ latex_str = r"""\begin{table}[h]
137
+ \centering
138
+ \small
139
+ \caption{\textbf{Compute-Matched Evaluation (CME) Reviewer Checklist Compliance.}}
140
+ \label{tab:cme_checklist}
141
+ \begin{tabular}{lp{8.5cm}c}
142
+ \toprule
143
+ Item & Evaluation Criterion & Response \\
144
+ \midrule
145
+ 1. Resource Vector $\mathbf{R}$ & Full 6D vector reported ($R_{\text{reason}}, R_{\text{token}}, R_{\text{cost}}, R_{\text{lat}}, R_{\text{sample}}, R_{\text{sys}}$) & Yes \\
146
+ 2. Equal Compute Baselines & Compared against CoT-SC or budget-matched tokens, not solely greedy & Yes \\
147
+ 3. Scaffold Accounting & Prompts, roles, and echoed history deducted from reasoning budget & Yes \\
148
+ 4. Dispersion and Sample Rigor & Benchmark test sizes ($N_{\text{items}}$) and paired statistics reported & Yes \\
149
+ 5. Task Decomposability & Separated communicative debate from external tool/verifier access & Yes \\
150
+ 6. Architecture Search & Disclosed cumulative search costs and negative configurations & Yes \\
151
+ \bottomrule
152
+ \end{tabular}
153
+ \end{table}
154
+ """
155
+ out.write(latex_str)
156
+ elif args.format == "json":
157
+ checklist_data = {
158
+ "version": "1.0.0",
159
+ "standard": "CME-Reviewer-Checklist",
160
+ "items": [
161
+ {"id": 1, "name": "Resource Vector R", "status": "Yes"},
162
+ {"id": 2, "name": "Equal Compute Baseline", "status": "Yes"},
163
+ {"id": 3, "name": "Scaffold Accounting", "status": "Yes"},
164
+ {"id": 4, "name": "Dispersion and Sample Rigor", "status": "Yes"},
165
+ {"id": 5, "name": "Task Decomposability and Tools", "status": "Yes"},
166
+ {"id": 6, "name": "Architecture Search Disclosure", "status": "Yes"}
167
+ ]
168
+ }
169
+ json.dump(checklist_data, out, indent=2)
170
+ out.write("\n")
171
+ else: # markdown
172
+ md_str = """# CME Reviewer Checklist
173
+ 1. [x] **Resource Vector R:** Reported 6D vector (thinking, total, cost, latency, samples, sys).
174
+ 2. [x] **Equal Compute Baseline:** Compared against CoT-SC or budget-matched tokens.
175
+ 3. [x] **Scaffold Accounting:** Deducted prompt and history overhead from effective reasoning.
176
+ 4. [x] **Dispersion and Sample Rigor:** Reported benchmark item counts and paired variance.
177
+ 5. [x] **Task Decomposability:** Isolated communicative debate from external tool feedback.
178
+ 6. [x] **Architecture Search:** Disclosed search compute and negative topologies.
179
+ """
180
+ out.write(md_str)
181
+ return 0
182
+
183
+
184
+ def build_parser() -> argparse.ArgumentParser:
185
+ p = argparse.ArgumentParser(prog="mae-eval", description="Compute-accounting audits for multi-agent LLM evaluations.")
186
+ sub = p.add_subparsers(dest="cmd", required=True)
187
+ a = sub.add_parser("audit", help="audit a JSONL evaluation trace")
188
+ a.add_argument("--log", required=True, help="path to a JSONL trace (see module docstring for the schema)")
189
+ a.add_argument("--tolerance", type=float, default=0.05, help="relative tolerance for 'strictly controlled' (default 0.05)")
190
+ a.add_argument("--dims", default="r_token", help="comma-separated dimensions that must match for parity (default r_token)")
191
+ a.add_argument("--model", default="model", help="model name label for pricing")
192
+ a.add_argument("--price-in", type=float, default=None, help="USD per million input tokens (you must supply it; none is bundled)")
193
+ a.add_argument("--price-out", type=float, default=None, help="USD per million output tokens")
194
+ a.add_argument("--assert-parity", action="store_true", help="exit with status 2 if parity is violated")
195
+ a.add_argument("--json", action="store_true", help="print a JSON report")
196
+ a.set_defaults(func=run_audit)
197
+
198
+ c = sub.add_parser("checklist", help="generate a CME compliance checklist template (LaTeX, Markdown, or JSON)")
199
+ c.add_argument("--format", choices=["markdown", "latex", "json"], default="markdown", help="output format (default: markdown)")
200
+ c.set_defaults(func=run_checklist)
201
+ return p
202
+
203
+
204
+ def main(argv: Optional[Sequence[str]] = None) -> int:
205
+ args = build_parser().parse_args(argv)
206
+ return args.func(args)
207
+
208
+
209
+ if __name__ == "__main__":
210
+ sys.exit(main())
mae_eval/tracker.py ADDED
@@ -0,0 +1,162 @@
1
+ """ResourceTracker: records model-generation calls and turns them into a ResourceVector.
2
+
3
+ Token accounting convention (matches vector.py):
4
+ prompt_tokens all input tokens of a call (includes scaffold_tokens)
5
+ completion_tokens all output tokens of a call (includes reasoning_tokens)
6
+ scaffold_tokens the part of prompt_tokens that is system prompt, role declaration, or echoed message history
7
+ reasoning_tokens the part of completion_tokens that is thinking / scratchpad
8
+ Cost is computed from prices YOU register. This library ships no price table: provider prices change and cannot be verified
9
+ offline, so a stale default would silently corrupt the r_cost axis.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import time
14
+ from contextlib import contextmanager
15
+ from dataclasses import dataclass
16
+ from typing import Callable, Dict, Iterator, List, Optional, Tuple
17
+
18
+ from .vector import ResourceVector
19
+
20
+
21
+ class PricingUnavailable(KeyError):
22
+ """No price registered for (model, tier)."""
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class CallRecord:
27
+ agent_id: str
28
+ prompt_tokens: int
29
+ completion_tokens: int
30
+ reasoning_tokens: int = 0
31
+ scaffold_tokens: int = 0
32
+ tool_calls: int = 0
33
+ n_completions: int = 1
34
+ start: float = 0.0
35
+ end: float = 0.0
36
+
37
+ def __post_init__(self) -> None:
38
+ if min(self.prompt_tokens, self.completion_tokens, self.reasoning_tokens, self.scaffold_tokens, self.tool_calls) < 0:
39
+ raise ValueError("token and tool counts must be >= 0")
40
+ if self.reasoning_tokens > self.completion_tokens:
41
+ raise ValueError("reasoning_tokens cannot exceed completion_tokens")
42
+ if self.scaffold_tokens > self.prompt_tokens:
43
+ raise ValueError("scaffold_tokens cannot exceed prompt_tokens")
44
+ if self.n_completions < 0:
45
+ raise ValueError("n_completions must be >= 0 (0 for pure tool calls)")
46
+ if self.end < self.start:
47
+ raise ValueError("end before start")
48
+
49
+
50
+ class Session:
51
+ """Holds the calls recorded inside one tracker.trace() block."""
52
+
53
+ def __init__(self, tracker: "ResourceTracker", clock: Callable[[], float]):
54
+ self._tracker = tracker
55
+ self._clock = clock
56
+ self.calls: List[CallRecord] = []
57
+
58
+ def record_call(self, agent_id: str, prompt_tokens: int, completion_tokens: int, *, reasoning_tokens: int = 0,
59
+ scaffold_tokens: int = 0, tool_calls: int = 0, n_completions: int = 1,
60
+ start: Optional[float] = None, end: Optional[float] = None) -> CallRecord:
61
+ """Record one generation call. Pass explicit start/end (seconds) to describe concurrent calls; otherwise
62
+ both default to the tracker clock (zero duration)."""
63
+ now = self._clock()
64
+ rec = CallRecord(agent_id, int(prompt_tokens), int(completion_tokens), int(reasoning_tokens), int(scaffold_tokens),
65
+ int(tool_calls), int(n_completions), now if start is None else float(start), now if end is None else float(end))
66
+ self.calls.append(rec)
67
+ return rec
68
+
69
+ def record_stream(self, agent_id: str, prompt_tokens: int, chunks, *, reasoning_tokens: int = 0, scaffold_tokens: int = 0,
70
+ tool_calls: int = 0, start: Optional[float] = None, chunk_times=None) -> CallRecord:
71
+ """Record one streamed completion. `chunks` is an iterable of token counts per streamed chunk; the completion token
72
+ count is their sum. `chunk_times` (optional, same length) gives each chunk's arrival time; the call ends at the last one."""
73
+ chunks = [int(c) for c in chunks]
74
+ if not chunks:
75
+ raise ValueError("a streamed completion needs at least one chunk")
76
+ if any(c < 0 for c in chunks):
77
+ raise ValueError("chunk token counts must be >= 0")
78
+ t0 = self._clock() if start is None else float(start)
79
+ end = t0
80
+ if chunk_times is not None:
81
+ chunk_times = [float(t) for t in chunk_times]
82
+ if len(chunk_times) != len(chunks):
83
+ raise ValueError("chunk_times must match chunks")
84
+ end = max(chunk_times)
85
+ return self.record_call(agent_id, prompt_tokens, sum(chunks), reasoning_tokens=reasoning_tokens, scaffold_tokens=scaffold_tokens,
86
+ tool_calls=tool_calls, start=t0, end=end)
87
+
88
+ def record_tool_call(self, agent_id: str, *, attempts: int = 1, start: Optional[float] = None, end: Optional[float] = None) -> CallRecord:
89
+ """Record a tool invocation. Every attempt, including retries, counts toward r_sys; the interval from `start` to `end`
90
+ (spanning all attempts and back-off waits) counts toward wall-clock latency. No model tokens or completions are added."""
91
+ if attempts < 1:
92
+ raise ValueError("attempts must be >= 1")
93
+ return self.record_call(agent_id, 0, 0, tool_calls=attempts, n_completions=0, start=start, end=end)
94
+
95
+ # ---- aggregates ----
96
+ @property
97
+ def serial_latency(self) -> float:
98
+ return sum(c.end - c.start for c in self.calls)
99
+
100
+ @property
101
+ def makespan(self) -> float:
102
+ """Wall-clock time from the first call start to the last call end (concurrent calls overlap)."""
103
+ if not self.calls:
104
+ return 0.0
105
+ return max(c.end for c in self.calls) - min(c.start for c in self.calls)
106
+
107
+ def get_resource_vector(self, allow_missing_pricing: bool = False) -> ResourceVector:
108
+ if not self.calls:
109
+ raise ValueError("no calls recorded")
110
+ prompt = sum(c.prompt_tokens for c in self.calls)
111
+ completion = sum(c.completion_tokens for c in self.calls)
112
+ try:
113
+ cost: Optional[float] = self._tracker.price(prompt, completion)
114
+ except PricingUnavailable:
115
+ if not allow_missing_pricing:
116
+ raise
117
+ cost = None
118
+ vec = ResourceVector(
119
+ r_reason=float(sum(c.reasoning_tokens for c in self.calls)),
120
+ r_token=float(prompt + completion),
121
+ r_cost=cost,
122
+ r_lat=self.makespan,
123
+ r_sample=int(sum(c.n_completions for c in self.calls)),
124
+ r_sys=int(sum(c.tool_calls for c in self.calls)),
125
+ r_scaffold=float(sum(c.scaffold_tokens for c in self.calls)),
126
+ )
127
+ return vec.validate(strict=cost is not None)
128
+
129
+ def scaffold_by_agent(self) -> Dict[str, float]:
130
+ out: Dict[str, float] = {}
131
+ for c in self.calls:
132
+ out[c.agent_id] = out.get(c.agent_id, 0.0) + c.scaffold_tokens
133
+ return out
134
+
135
+
136
+ class ResourceTracker:
137
+ """Usage: tracker = ResourceTracker(model, pricing_tier); tracker.register_pricing(...); with tracker.trace() as s: ..."""
138
+
139
+ _PRICES: Dict[Tuple[str, str], Tuple[float, float]] = {}
140
+
141
+ def __init__(self, model: str, pricing_tier: str = "standard", clock: Callable[[], float] = time.monotonic):
142
+ self.model = model
143
+ self.pricing_tier = pricing_tier
144
+ self._clock = clock
145
+ self._prices: Dict[Tuple[str, str], Tuple[float, float]] = {}
146
+
147
+ def register_pricing(self, input_usd_per_mtok: float, output_usd_per_mtok: float,
148
+ model: Optional[str] = None, tier: Optional[str] = None) -> None:
149
+ if input_usd_per_mtok < 0 or output_usd_per_mtok < 0:
150
+ raise ValueError("prices must be >= 0")
151
+ self._prices[(model or self.model, tier or self.pricing_tier)] = (float(input_usd_per_mtok), float(output_usd_per_mtok))
152
+
153
+ def price(self, prompt_tokens: int, completion_tokens: int) -> float:
154
+ key = (self.model, self.pricing_tier)
155
+ if key not in self._prices:
156
+ raise PricingUnavailable("no price registered for %s/%s; call register_pricing()" % key)
157
+ pin, pout = self._prices[key]
158
+ return (prompt_tokens * pin + completion_tokens * pout) / 1e6
159
+
160
+ @contextmanager
161
+ def trace(self) -> Iterator[Session]:
162
+ yield Session(self, self._clock)
mae_eval/vector.py ADDED
@@ -0,0 +1,91 @@
1
+ """ResourceVector: the 6-dimensional resource-accounting vector R of the MAEO specification (docs/MAEO_SPECIFICATION.md, section 2).
2
+
3
+ R = <r_reason, r_token, r_cost, r_lat, r_sample, r_sys>. A field set to None means "unreported" (the fourth control state).
4
+ Scaffold tokens (system prompts, role declarations, echoed message history) are tracked separately as r_scaffold and are
5
+ counted INSIDE r_token; reasoning tokens are counted inside completion tokens, and r_token = prompt + completion.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import math
10
+ from dataclasses import dataclass, fields
11
+ from enum import Enum
12
+ from typing import Any, Dict, Optional
13
+
14
+ DIMENSIONS = ("r_reason", "r_token", "r_cost", "r_lat", "r_sample", "r_sys")
15
+ _INT_DIMS = {"r_sample", "r_sys"}
16
+
17
+
18
+ class ControlState(str, Enum):
19
+ """How a resource dimension is controlled between a multi-agent system and its single-agent baseline."""
20
+ STRICT = "Strictly Controlled"
21
+ APPROXIMATE = "Approximately Matched"
22
+ UNMATCHED = "Unmatched (Measured)"
23
+ UNREPORTED = "Unreported (Omitted)"
24
+
25
+
26
+ class ResourceVectorError(ValueError):
27
+ """Raised when a ResourceVector violates the specification."""
28
+
29
+
30
+ @dataclass
31
+ class ResourceVector:
32
+ r_reason: Optional[float] = None # total reasoning / scratchpad tokens
33
+ r_token: Optional[float] = None # total prompt + completion tokens (includes scaffold)
34
+ r_cost: Optional[float] = None # total monetary inference cost, USD
35
+ r_lat: Optional[float] = None # wall-clock latency in seconds (makespan, concurrency-aware)
36
+ r_sample: Optional[int] = None # total model completions drawn
37
+ r_sys: Optional[int] = None # tool invocations / search queries
38
+ r_scaffold: Optional[float] = None # scaffold tokens (subset of r_token); not part of the 6-vector
39
+
40
+ # ---- derived ----
41
+ @property
42
+ def scaffold_pct(self) -> Optional[float]:
43
+ """Scaffold tokens as a percentage of r_token; None if either is unreported or r_token is 0."""
44
+ if self.r_scaffold is None or not self.r_token:
45
+ return None
46
+ return 100.0 * self.r_scaffold / self.r_token
47
+
48
+ # ---- serialization ----
49
+ def to_dict(self, include_scaffold: bool = False) -> Dict[str, Any]:
50
+ d = {k: getattr(self, k) for k in DIMENSIONS}
51
+ if include_scaffold:
52
+ d["r_scaffold"] = self.r_scaffold
53
+ return d
54
+
55
+ @classmethod
56
+ def from_dict(cls, d: Dict[str, Any]) -> "ResourceVector":
57
+ known = {f.name for f in fields(cls)}
58
+ extra = set(d) - known
59
+ if extra:
60
+ raise ResourceVectorError("unknown resource fields: %s" % sorted(extra))
61
+ return cls(**{k: d.get(k) for k in known if k in d})
62
+
63
+ # ---- validation against the MAEO submission schema ----
64
+ def validate(self, strict: bool = True) -> "ResourceVector":
65
+ """Check the vector against docs/MAEO_SPECIFICATION.md section 2.
66
+
67
+ strict=True (submission mode): every dimension must be present (not None).
68
+ strict=False (audit mode): None is allowed and means 'unreported'.
69
+ Always: numbers must be finite and non-negative; r_sample and r_sys must be integers; r_sample >= 1 when given;
70
+ r_reason <= r_token and r_scaffold <= r_token when those are given.
71
+ """
72
+ for name in DIMENSIONS:
73
+ v = getattr(self, name)
74
+ if v is None:
75
+ if strict:
76
+ raise ResourceVectorError("%s is required for a submission (use None only for audit mode)" % name)
77
+ continue
78
+ if isinstance(v, bool) or not isinstance(v, (int, float)):
79
+ raise ResourceVectorError("%s must be a number, got %r" % (name, type(v).__name__))
80
+ if not math.isfinite(v) or v < 0:
81
+ raise ResourceVectorError("%s must be finite and >= 0, got %r" % (name, v))
82
+ if name in _INT_DIMS and float(v) != int(v):
83
+ raise ResourceVectorError("%s must be an integer, got %r" % (name, v))
84
+ if self.r_sample is not None and self.r_sample < 1:
85
+ raise ResourceVectorError("r_sample must be >= 1")
86
+ if self.r_token is not None:
87
+ for sub in ("r_reason", "r_scaffold"):
88
+ val = getattr(self, sub)
89
+ if val is not None and val > self.r_token:
90
+ raise ResourceVectorError("%s (%s) cannot exceed r_token (%s)" % (sub, val, self.r_token))
91
+ return self
pace/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """cme: measure a multi-agent system against the self-consistency frontier at its measured token cost.
2
+
3
+ Commands: `pace proxy` (logging proxy), `pace baseline` (greedy + sample pool), `pace report` (frontier gap, resource card, table, figure).
4
+ Built on mae_eval (resource vector and parity auditor). Python API: pace.frontier_gap, pace.build_report, pace.resource_card, pace.wrap (log a client's usage without a proxy), pace.track (label items and rounds).
5
+ """
6
+ from .frontier import Pool, frontier_at, frontier_curve, frontier_gap, pav_increasing
7
+ from .report import build_report, to_latex
8
+ from .client import wrap, track, log_result
9
+
10
+ __version__ = "0.1.0"
11
+ __all__ = ["Pool", "frontier_at", "frontier_curve", "frontier_gap", "pav_increasing", "build_report", "to_latex", "wrap", "track", "log_result"]
pace/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())