pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
mae_eval/auditor.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""EvaluationAuditor: compares a multi-agent resource vector with its single-agent baseline and flags confounders."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Dict, List, Optional, Sequence, Union
|
|
7
|
+
|
|
8
|
+
from .tracker import Session
|
|
9
|
+
from .vector import DIMENSIONS, ControlState, ResourceVector
|
|
10
|
+
|
|
11
|
+
# Default dimensions that must be matched for a parity claim, by the question being asked.
|
|
12
|
+
PARITY_DIMS_DEFAULT = ("r_token",)
|
|
13
|
+
|
|
14
|
+
_MESSAGES = {
|
|
15
|
+
"r_sample": "sample counts differ: gains may reflect ensembling (more draws), not coordination",
|
|
16
|
+
"r_reason": "reasoning-token budgets differ: thinking compute is not matched",
|
|
17
|
+
"r_token": "total tokens differ: scaffold, history echoes, or extra calls consume more budget",
|
|
18
|
+
"r_cost": "monetary cost differs: comparison is not cost-matched",
|
|
19
|
+
"r_lat": "wall-clock latency differs (serialization or concurrency)",
|
|
20
|
+
"r_sys": "tool or search invocations differ",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ComputeParityError(AssertionError):
|
|
25
|
+
"""Raised by assert_compute_parity when a dimension is outside tolerance."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Flag:
|
|
30
|
+
dimension: str
|
|
31
|
+
ratio: Optional[float]
|
|
32
|
+
state: ControlState
|
|
33
|
+
message: str
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _as_vector(x: Union[ResourceVector, Session]) -> ResourceVector:
|
|
37
|
+
return x.get_resource_vector(allow_missing_pricing=True) if isinstance(x, Session) else x
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class EvaluationAuditor:
|
|
41
|
+
"""mas / sas may be ResourceVector or a recorded Session.
|
|
42
|
+
|
|
43
|
+
approx_tolerance: relative deviation up to which a dimension counts as 'Approximately Matched' (default 0.25).
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(self, mas: Union[ResourceVector, Session], sas: Union[ResourceVector, Session, None] = None,
|
|
47
|
+
approx_tolerance: float = 0.25):
|
|
48
|
+
self.mas = _as_vector(mas)
|
|
49
|
+
self.sas = _as_vector(sas) if sas is not None else None
|
|
50
|
+
self.approx_tolerance = approx_tolerance
|
|
51
|
+
|
|
52
|
+
def _need_sas(self) -> ResourceVector:
|
|
53
|
+
if self.sas is None:
|
|
54
|
+
raise ValueError("a single-agent baseline is required for this operation")
|
|
55
|
+
return self.sas
|
|
56
|
+
|
|
57
|
+
def ratios(self) -> Dict[str, Optional[float]]:
|
|
58
|
+
"""MAS / SAS per dimension. None if either side is unreported or both are 0 (0/0 is undefined, printed as n/a); inf if the baseline is 0 and MAS is not."""
|
|
59
|
+
sas = self._need_sas()
|
|
60
|
+
out: Dict[str, Optional[float]] = {}
|
|
61
|
+
for d in DIMENSIONS:
|
|
62
|
+
m, s = getattr(self.mas, d), getattr(sas, d)
|
|
63
|
+
if m is None or s is None:
|
|
64
|
+
out[d] = None
|
|
65
|
+
elif s == 0:
|
|
66
|
+
out[d] = None if m == 0 else math.inf
|
|
67
|
+
else:
|
|
68
|
+
out[d] = m / s
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
def control_states(self, tolerance: float = 0.05) -> Dict[str, ControlState]:
|
|
72
|
+
states: Dict[str, ControlState] = {}
|
|
73
|
+
sas = self._need_sas()
|
|
74
|
+
for d, r in self.ratios().items():
|
|
75
|
+
if r is None and getattr(self.mas, d) == 0 and getattr(sas, d) == 0:
|
|
76
|
+
states[d] = ControlState.STRICT # both systems consumed none: equal, but the ratio is undefined
|
|
77
|
+
elif r is None:
|
|
78
|
+
states[d] = ControlState.UNREPORTED
|
|
79
|
+
else:
|
|
80
|
+
dev = abs(r - 1.0)
|
|
81
|
+
states[d] = (ControlState.STRICT if dev <= tolerance
|
|
82
|
+
else ControlState.APPROXIMATE if dev <= self.approx_tolerance
|
|
83
|
+
else ControlState.UNMATCHED)
|
|
84
|
+
return states
|
|
85
|
+
|
|
86
|
+
def flag_confounders(self, tolerance: float = 0.05) -> List[Flag]:
|
|
87
|
+
"""One Flag per dimension that is not strictly controlled, with an explanation."""
|
|
88
|
+
ratios = self.ratios()
|
|
89
|
+
flags = []
|
|
90
|
+
for d, st in self.control_states(tolerance).items():
|
|
91
|
+
if st is ControlState.STRICT:
|
|
92
|
+
continue
|
|
93
|
+
msg = _MESSAGES[d] if st is not ControlState.UNREPORTED else "not reported by one or both systems"
|
|
94
|
+
flags.append(Flag(d, ratios[d], st, msg))
|
|
95
|
+
return flags
|
|
96
|
+
|
|
97
|
+
def assert_compute_parity(self, tolerance: float = 0.05, dims: Sequence[str] = PARITY_DIMS_DEFAULT) -> None:
|
|
98
|
+
"""Raise ComputeParityError unless every dimension in `dims` is reported and within `tolerance` of ratio 1."""
|
|
99
|
+
bad = []
|
|
100
|
+
ratios = self.ratios()
|
|
101
|
+
for d in dims:
|
|
102
|
+
if d not in DIMENSIONS:
|
|
103
|
+
raise ValueError("unknown dimension %r" % d)
|
|
104
|
+
r = ratios[d]
|
|
105
|
+
if r is None and getattr(self.mas, d) == 0 and getattr(self._need_sas(), d) == 0:
|
|
106
|
+
continue # 0/0: both systems consumed none of this resource
|
|
107
|
+
if r is None:
|
|
108
|
+
bad.append("%s: unreported" % d)
|
|
109
|
+
elif abs(r - 1.0) > tolerance:
|
|
110
|
+
bad.append("%s: MAS/SAS = %.3f (tolerance %.3f)" % (d, r, tolerance))
|
|
111
|
+
if bad:
|
|
112
|
+
raise ComputeParityError("compute parity violated: " + "; ".join(bad))
|
mae_eval/classifier.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Classify a single-agent baseline into a rung of the baseline ladder (paper Section 3, Figure 1).
|
|
2
|
+
|
|
3
|
+
Rungs: A greedy single pass; B sampled baseline (k independent samples, no communication, tokens not matched); C a named resource held
|
|
4
|
+
equal, or declared single-agent test-time scaling; D a budget sweep or Pareto frontier. The rung describes the baseline a multi-agent
|
|
5
|
+
system is compared against, so a configuration that communicates between agents is not a baseline and is rejected.
|
|
6
|
+
|
|
7
|
+
Five rules, applied in this order (first match wins):
|
|
8
|
+
R1 budget sweep or Pareto frontier (n_budgets >= 3, or pareto true) -> D
|
|
9
|
+
R2 a named matched resource (thinking tokens, total tokens, cost, latency,
|
|
10
|
+
iterations, generations) with matching declared -> C
|
|
11
|
+
R3 extended thinking or search (MCTS, tree search, best-of-n with a verifier)
|
|
12
|
+
without a named matched resource -> C (matching unverified; see `reason`)
|
|
13
|
+
R4 more than one independent sample combined by vote or selection -> B
|
|
14
|
+
R5 otherwise (one sample, no test-time scaling) -> A
|
|
15
|
+
|
|
16
|
+
Note on the wording "Rung D: combined multi-agent plus sampling or search architectures": that describes a system under test, not a
|
|
17
|
+
baseline, and the paper's Section 3 defines D as a Pareto sweep, so this module follows the paper. Evaluate such combined systems
|
|
18
|
+
as the multi-agent side of a contrast against the rung chosen here.
|
|
19
|
+
|
|
20
|
+
config keys (all optional): n_samples (int), aggregation (e.g. "majority_vote", "best_of_n"), extended_thinking (bool),
|
|
21
|
+
thinking_budget (int tokens), search ("mcts", "tree", ...), matched_resource (str), matching_declared (bool, default True when a
|
|
22
|
+
resource is named), n_budgets (int), pareto (bool), communication (bool; must be false or absent).
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from typing import Dict, Literal, Tuple
|
|
27
|
+
|
|
28
|
+
Rung = Literal["A", "B", "C", "D"]
|
|
29
|
+
MATCHABLE = {"thinking_tokens", "total_tokens", "tokens", "cost", "latency", "iterations", "generations", "calls", "flops"}
|
|
30
|
+
SEARCH = {"mcts", "tree", "tree_search", "beam", "best_first"}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def classify_with_reason(config: Dict) -> Tuple[Rung, str, str]:
|
|
34
|
+
"""Return (rung, rule id, reason). Raises ValueError for a communicating configuration or invalid values."""
|
|
35
|
+
if config.get("communication"):
|
|
36
|
+
raise ValueError("a configuration with inter-agent communication is not a single-agent baseline")
|
|
37
|
+
n_samples = int(config.get("n_samples", 1))
|
|
38
|
+
if n_samples < 1:
|
|
39
|
+
raise ValueError("n_samples must be at least 1")
|
|
40
|
+
n_budgets = int(config.get("n_budgets", 0) or 0)
|
|
41
|
+
if config.get("pareto") or n_budgets >= 3:
|
|
42
|
+
return "D", "R1", "budget sweep or Pareto frontier"
|
|
43
|
+
res = str(config.get("matched_resource", "") or "").lower().replace(" ", "_")
|
|
44
|
+
if res:
|
|
45
|
+
if res not in MATCHABLE:
|
|
46
|
+
raise ValueError("unknown matched_resource %r; use one of %s" % (res, sorted(MATCHABLE)))
|
|
47
|
+
if config.get("matching_declared", True):
|
|
48
|
+
return "C", "R2", "resource held equal: %s" % res
|
|
49
|
+
search = str(config.get("search", "") or "").lower()
|
|
50
|
+
aggregation = str(config.get("aggregation", "") or "").lower()
|
|
51
|
+
if config.get("extended_thinking") or config.get("thinking_budget") or search in SEARCH or (aggregation == "best_of_n" and config.get("verifier")):
|
|
52
|
+
return "C", "R3", "extended thinking or search without a named matched resource (matching unverified)"
|
|
53
|
+
if n_samples > 1:
|
|
54
|
+
if aggregation not in ("majority_vote", "vote", "plurality", "best_of_n", "self_consistency", "selection", ""):
|
|
55
|
+
raise ValueError("unknown aggregation %r" % aggregation)
|
|
56
|
+
return "B", "R4", "%d independent samples combined without communication" % n_samples
|
|
57
|
+
return "A", "R5", "single sample, no test-time scaling"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def classify_baseline_rung(config_dict: Dict) -> Rung:
|
|
61
|
+
"""Rung (A, B, C or D) of a single-agent baseline configuration."""
|
|
62
|
+
return classify_with_reason(config_dict)[0]
|
mae_eval/cli.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Command-line interface: ``mae-eval audit --log trace.jsonl``.
|
|
2
|
+
|
|
3
|
+
Trace format: JSON Lines, one generation or tool call per line. Fields:
|
|
4
|
+
system "mas" or "sas" (required)
|
|
5
|
+
agent_id string (default "agent")
|
|
6
|
+
prompt_tokens, completion_tokens integers (required unless "chunks" is given for streaming records)
|
|
7
|
+
chunks optional list of per-chunk token counts for a streamed completion (replaces completion_tokens)
|
|
8
|
+
reasoning_tokens, scaffold_tokens integers, default 0 (subsets of completion / prompt tokens)
|
|
9
|
+
tool_calls integer, default 0; n_completions integer, default 1 (0 for a pure tool call)
|
|
10
|
+
start, end seconds on a common clock, default 0 (concurrent calls overlap)
|
|
11
|
+
Note: code/examples/sample_trace.jsonl is a hand-written synthetic example, not a trace of a model run.
|
|
12
|
+
A MAS/SAS ratio prints as n/a when both values are 0 (0/0 is undefined).
|
|
13
|
+
Exit status: 0 success, 1 input error, 2 parity violated (only with --assert-parity).
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import json
|
|
19
|
+
import sys
|
|
20
|
+
from typing import Dict, List, Optional, Sequence
|
|
21
|
+
|
|
22
|
+
from .auditor import ComputeParityError, EvaluationAuditor
|
|
23
|
+
from .tracker import ResourceTracker, Session
|
|
24
|
+
from .vector import DIMENSIONS, ResourceVector
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class TraceError(ValueError):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def load_trace(path: str) -> Dict[str, List[dict]]:
|
|
32
|
+
"""Read a JSONL trace and group records by system. Raises TraceError with the line number on bad input."""
|
|
33
|
+
groups: Dict[str, List[dict]] = {}
|
|
34
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
35
|
+
for no, line in enumerate(fh, 1):
|
|
36
|
+
line = line.strip()
|
|
37
|
+
if not line:
|
|
38
|
+
continue
|
|
39
|
+
try:
|
|
40
|
+
rec = json.loads(line)
|
|
41
|
+
except json.JSONDecodeError as e:
|
|
42
|
+
raise TraceError("line %d: invalid JSON (%s)" % (no, e.msg))
|
|
43
|
+
if not isinstance(rec, dict):
|
|
44
|
+
raise TraceError("line %d: expected a JSON object" % no)
|
|
45
|
+
system = rec.get("system")
|
|
46
|
+
if system not in ("mas", "sas"):
|
|
47
|
+
raise TraceError("line %d: 'system' must be 'mas' or 'sas'" % no)
|
|
48
|
+
if "chunks" not in rec and ("prompt_tokens" not in rec or "completion_tokens" not in rec):
|
|
49
|
+
raise TraceError("line %d: need prompt_tokens and completion_tokens (or chunks)" % no)
|
|
50
|
+
rec["_line"] = no
|
|
51
|
+
groups.setdefault(system, []).append(rec)
|
|
52
|
+
if not groups:
|
|
53
|
+
raise TraceError("trace is empty")
|
|
54
|
+
return groups
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def session_from_records(records: List[dict], tracker: ResourceTracker) -> Session:
|
|
58
|
+
with tracker.trace() as s:
|
|
59
|
+
for r in records:
|
|
60
|
+
try:
|
|
61
|
+
common = dict(reasoning_tokens=int(r.get("reasoning_tokens", 0)), scaffold_tokens=int(r.get("scaffold_tokens", 0)),
|
|
62
|
+
tool_calls=int(r.get("tool_calls", 0)), start=float(r.get("start", 0.0)))
|
|
63
|
+
agent = str(r.get("agent_id", "agent"))
|
|
64
|
+
if "chunks" in r:
|
|
65
|
+
s.record_stream(agent, int(r.get("prompt_tokens", 0)), r["chunks"], **common)
|
|
66
|
+
else:
|
|
67
|
+
s.record_call(agent, int(r["prompt_tokens"]), int(r["completion_tokens"]), n_completions=int(r.get("n_completions", 1)),
|
|
68
|
+
end=float(r.get("end", common["start"])), **common)
|
|
69
|
+
except (ValueError, TypeError) as e:
|
|
70
|
+
raise TraceError("line %d: %s" % (r["_line"], e))
|
|
71
|
+
return s
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _vec_dict(v: ResourceVector) -> dict:
|
|
75
|
+
d = v.to_dict(include_scaffold=True)
|
|
76
|
+
d["scaffold_pct"] = v.scaffold_pct
|
|
77
|
+
return d
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def run_audit(args: argparse.Namespace, out=None) -> int:
|
|
81
|
+
out = out or sys.stdout
|
|
82
|
+
try:
|
|
83
|
+
groups = load_trace(args.log)
|
|
84
|
+
except (OSError, TraceError) as e:
|
|
85
|
+
print("error: %s" % e, file=sys.stderr)
|
|
86
|
+
return 1
|
|
87
|
+
tracker = ResourceTracker(args.model, "cli")
|
|
88
|
+
priced = args.price_in is not None and args.price_out is not None
|
|
89
|
+
if priced:
|
|
90
|
+
tracker.register_pricing(args.price_in, args.price_out)
|
|
91
|
+
vectors: Dict[str, ResourceVector] = {}
|
|
92
|
+
try:
|
|
93
|
+
for name, recs in groups.items():
|
|
94
|
+
vectors[name] = session_from_records(recs, tracker).get_resource_vector(allow_missing_pricing=not priced)
|
|
95
|
+
except (TraceError, ValueError) as e:
|
|
96
|
+
print("error: %s" % e, file=sys.stderr)
|
|
97
|
+
return 1
|
|
98
|
+
report: dict = {"vectors": {k: _vec_dict(v) for k, v in vectors.items()}}
|
|
99
|
+
status = 0
|
|
100
|
+
if "mas" in vectors and "sas" in vectors:
|
|
101
|
+
aud = EvaluationAuditor(vectors["mas"], vectors["sas"])
|
|
102
|
+
dims = tuple(d.strip() for d in args.dims.split(",") if d.strip())
|
|
103
|
+
report["ratios_mas_over_sas"] = aud.ratios()
|
|
104
|
+
report["control_states"] = {k: v.value for k, v in aud.control_states(args.tolerance).items()}
|
|
105
|
+
report["flags"] = [{"dimension": f.dimension, "ratio": f.ratio, "state": f.state.value, "message": f.message}
|
|
106
|
+
for f in aud.flag_confounders(args.tolerance)]
|
|
107
|
+
try:
|
|
108
|
+
aud.assert_compute_parity(args.tolerance, dims)
|
|
109
|
+
report["parity"] = {"dims": list(dims), "tolerance": args.tolerance, "ok": True}
|
|
110
|
+
except ComputeParityError as e:
|
|
111
|
+
report["parity"] = {"dims": list(dims), "tolerance": args.tolerance, "ok": False, "detail": str(e)}
|
|
112
|
+
if args.assert_parity:
|
|
113
|
+
status = 2
|
|
114
|
+
else:
|
|
115
|
+
report["note"] = "only one system present; no comparison made"
|
|
116
|
+
if args.json:
|
|
117
|
+
json.dump(report, out, indent=2, default=lambda x: None if x != x else str(x))
|
|
118
|
+
out.write("\n")
|
|
119
|
+
else:
|
|
120
|
+
for name, v in vectors.items():
|
|
121
|
+
out.write("%s: " % name.upper() + ", ".join("%s=%s" % (d, getattr(v, d)) for d in DIMENSIONS))
|
|
122
|
+
if v.scaffold_pct is not None:
|
|
123
|
+
out.write(", scaffold=%.1f%%" % v.scaffold_pct)
|
|
124
|
+
out.write("\n")
|
|
125
|
+
if "ratios_mas_over_sas" in report:
|
|
126
|
+
out.write("MAS/SAS ratios: " + ", ".join("%s=%s" % (k, "n/a" if r is None else "%.3g" % r) for k, r in report["ratios_mas_over_sas"].items()) + "\n")
|
|
127
|
+
for f in report["flags"]:
|
|
128
|
+
out.write("FLAG %s [%s]: %s\n" % (f["dimension"], f["state"], f["message"]))
|
|
129
|
+
out.write("Parity on %s within %.3g: %s\n" % (",".join(report["parity"]["dims"]), args.tolerance, "OK" if report["parity"]["ok"] else "VIOLATED"))
|
|
130
|
+
return status
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def run_checklist(args: argparse.Namespace, out=None) -> int:
|
|
134
|
+
out = out or sys.stdout
|
|
135
|
+
if args.format == "latex":
|
|
136
|
+
latex_str = r"""\begin{table}[h]
|
|
137
|
+
\centering
|
|
138
|
+
\small
|
|
139
|
+
\caption{\textbf{Compute-Matched Evaluation (CME) Reviewer Checklist Compliance.}}
|
|
140
|
+
\label{tab:cme_checklist}
|
|
141
|
+
\begin{tabular}{lp{8.5cm}c}
|
|
142
|
+
\toprule
|
|
143
|
+
Item & Evaluation Criterion & Response \\
|
|
144
|
+
\midrule
|
|
145
|
+
1. Resource Vector $\mathbf{R}$ & Full 6D vector reported ($R_{\text{reason}}, R_{\text{token}}, R_{\text{cost}}, R_{\text{lat}}, R_{\text{sample}}, R_{\text{sys}}$) & Yes \\
|
|
146
|
+
2. Equal Compute Baselines & Compared against CoT-SC or budget-matched tokens, not solely greedy & Yes \\
|
|
147
|
+
3. Scaffold Accounting & Prompts, roles, and echoed history deducted from reasoning budget & Yes \\
|
|
148
|
+
4. Dispersion and Sample Rigor & Benchmark test sizes ($N_{\text{items}}$) and paired statistics reported & Yes \\
|
|
149
|
+
5. Task Decomposability & Separated communicative debate from external tool/verifier access & Yes \\
|
|
150
|
+
6. Architecture Search & Disclosed cumulative search costs and negative configurations & Yes \\
|
|
151
|
+
\bottomrule
|
|
152
|
+
\end{tabular}
|
|
153
|
+
\end{table}
|
|
154
|
+
"""
|
|
155
|
+
out.write(latex_str)
|
|
156
|
+
elif args.format == "json":
|
|
157
|
+
checklist_data = {
|
|
158
|
+
"version": "1.0.0",
|
|
159
|
+
"standard": "CME-Reviewer-Checklist",
|
|
160
|
+
"items": [
|
|
161
|
+
{"id": 1, "name": "Resource Vector R", "status": "Yes"},
|
|
162
|
+
{"id": 2, "name": "Equal Compute Baseline", "status": "Yes"},
|
|
163
|
+
{"id": 3, "name": "Scaffold Accounting", "status": "Yes"},
|
|
164
|
+
{"id": 4, "name": "Dispersion and Sample Rigor", "status": "Yes"},
|
|
165
|
+
{"id": 5, "name": "Task Decomposability and Tools", "status": "Yes"},
|
|
166
|
+
{"id": 6, "name": "Architecture Search Disclosure", "status": "Yes"}
|
|
167
|
+
]
|
|
168
|
+
}
|
|
169
|
+
json.dump(checklist_data, out, indent=2)
|
|
170
|
+
out.write("\n")
|
|
171
|
+
else: # markdown
|
|
172
|
+
md_str = """# CME Reviewer Checklist
|
|
173
|
+
1. [x] **Resource Vector R:** Reported 6D vector (thinking, total, cost, latency, samples, sys).
|
|
174
|
+
2. [x] **Equal Compute Baseline:** Compared against CoT-SC or budget-matched tokens.
|
|
175
|
+
3. [x] **Scaffold Accounting:** Deducted prompt and history overhead from effective reasoning.
|
|
176
|
+
4. [x] **Dispersion and Sample Rigor:** Reported benchmark item counts and paired variance.
|
|
177
|
+
5. [x] **Task Decomposability:** Isolated communicative debate from external tool feedback.
|
|
178
|
+
6. [x] **Architecture Search:** Disclosed search compute and negative topologies.
|
|
179
|
+
"""
|
|
180
|
+
out.write(md_str)
|
|
181
|
+
return 0
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
185
|
+
p = argparse.ArgumentParser(prog="mae-eval", description="Compute-accounting audits for multi-agent LLM evaluations.")
|
|
186
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
187
|
+
a = sub.add_parser("audit", help="audit a JSONL evaluation trace")
|
|
188
|
+
a.add_argument("--log", required=True, help="path to a JSONL trace (see module docstring for the schema)")
|
|
189
|
+
a.add_argument("--tolerance", type=float, default=0.05, help="relative tolerance for 'strictly controlled' (default 0.05)")
|
|
190
|
+
a.add_argument("--dims", default="r_token", help="comma-separated dimensions that must match for parity (default r_token)")
|
|
191
|
+
a.add_argument("--model", default="model", help="model name label for pricing")
|
|
192
|
+
a.add_argument("--price-in", type=float, default=None, help="USD per million input tokens (you must supply it; none is bundled)")
|
|
193
|
+
a.add_argument("--price-out", type=float, default=None, help="USD per million output tokens")
|
|
194
|
+
a.add_argument("--assert-parity", action="store_true", help="exit with status 2 if parity is violated")
|
|
195
|
+
a.add_argument("--json", action="store_true", help="print a JSON report")
|
|
196
|
+
a.set_defaults(func=run_audit)
|
|
197
|
+
|
|
198
|
+
c = sub.add_parser("checklist", help="generate a CME compliance checklist template (LaTeX, Markdown, or JSON)")
|
|
199
|
+
c.add_argument("--format", choices=["markdown", "latex", "json"], default="markdown", help="output format (default: markdown)")
|
|
200
|
+
c.set_defaults(func=run_checklist)
|
|
201
|
+
return p
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
205
|
+
args = build_parser().parse_args(argv)
|
|
206
|
+
return args.func(args)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
if __name__ == "__main__":
|
|
210
|
+
sys.exit(main())
|
mae_eval/tracker.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""ResourceTracker: records model-generation calls and turns them into a ResourceVector.
|
|
2
|
+
|
|
3
|
+
Token accounting convention (matches vector.py):
|
|
4
|
+
prompt_tokens all input tokens of a call (includes scaffold_tokens)
|
|
5
|
+
completion_tokens all output tokens of a call (includes reasoning_tokens)
|
|
6
|
+
scaffold_tokens the part of prompt_tokens that is system prompt, role declaration, or echoed message history
|
|
7
|
+
reasoning_tokens the part of completion_tokens that is thinking / scratchpad
|
|
8
|
+
Cost is computed from prices YOU register. This library ships no price table: provider prices change and cannot be verified
|
|
9
|
+
offline, so a stale default would silently corrupt the r_cost axis.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import time
|
|
14
|
+
from contextlib import contextmanager
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Callable, Dict, Iterator, List, Optional, Tuple
|
|
17
|
+
|
|
18
|
+
from .vector import ResourceVector
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PricingUnavailable(KeyError):
|
|
22
|
+
"""No price registered for (model, tier)."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class CallRecord:
|
|
27
|
+
agent_id: str
|
|
28
|
+
prompt_tokens: int
|
|
29
|
+
completion_tokens: int
|
|
30
|
+
reasoning_tokens: int = 0
|
|
31
|
+
scaffold_tokens: int = 0
|
|
32
|
+
tool_calls: int = 0
|
|
33
|
+
n_completions: int = 1
|
|
34
|
+
start: float = 0.0
|
|
35
|
+
end: float = 0.0
|
|
36
|
+
|
|
37
|
+
def __post_init__(self) -> None:
|
|
38
|
+
if min(self.prompt_tokens, self.completion_tokens, self.reasoning_tokens, self.scaffold_tokens, self.tool_calls) < 0:
|
|
39
|
+
raise ValueError("token and tool counts must be >= 0")
|
|
40
|
+
if self.reasoning_tokens > self.completion_tokens:
|
|
41
|
+
raise ValueError("reasoning_tokens cannot exceed completion_tokens")
|
|
42
|
+
if self.scaffold_tokens > self.prompt_tokens:
|
|
43
|
+
raise ValueError("scaffold_tokens cannot exceed prompt_tokens")
|
|
44
|
+
if self.n_completions < 0:
|
|
45
|
+
raise ValueError("n_completions must be >= 0 (0 for pure tool calls)")
|
|
46
|
+
if self.end < self.start:
|
|
47
|
+
raise ValueError("end before start")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Session:
|
|
51
|
+
"""Holds the calls recorded inside one tracker.trace() block."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, tracker: "ResourceTracker", clock: Callable[[], float]):
|
|
54
|
+
self._tracker = tracker
|
|
55
|
+
self._clock = clock
|
|
56
|
+
self.calls: List[CallRecord] = []
|
|
57
|
+
|
|
58
|
+
def record_call(self, agent_id: str, prompt_tokens: int, completion_tokens: int, *, reasoning_tokens: int = 0,
|
|
59
|
+
scaffold_tokens: int = 0, tool_calls: int = 0, n_completions: int = 1,
|
|
60
|
+
start: Optional[float] = None, end: Optional[float] = None) -> CallRecord:
|
|
61
|
+
"""Record one generation call. Pass explicit start/end (seconds) to describe concurrent calls; otherwise
|
|
62
|
+
both default to the tracker clock (zero duration)."""
|
|
63
|
+
now = self._clock()
|
|
64
|
+
rec = CallRecord(agent_id, int(prompt_tokens), int(completion_tokens), int(reasoning_tokens), int(scaffold_tokens),
|
|
65
|
+
int(tool_calls), int(n_completions), now if start is None else float(start), now if end is None else float(end))
|
|
66
|
+
self.calls.append(rec)
|
|
67
|
+
return rec
|
|
68
|
+
|
|
69
|
+
def record_stream(self, agent_id: str, prompt_tokens: int, chunks, *, reasoning_tokens: int = 0, scaffold_tokens: int = 0,
|
|
70
|
+
tool_calls: int = 0, start: Optional[float] = None, chunk_times=None) -> CallRecord:
|
|
71
|
+
"""Record one streamed completion. `chunks` is an iterable of token counts per streamed chunk; the completion token
|
|
72
|
+
count is their sum. `chunk_times` (optional, same length) gives each chunk's arrival time; the call ends at the last one."""
|
|
73
|
+
chunks = [int(c) for c in chunks]
|
|
74
|
+
if not chunks:
|
|
75
|
+
raise ValueError("a streamed completion needs at least one chunk")
|
|
76
|
+
if any(c < 0 for c in chunks):
|
|
77
|
+
raise ValueError("chunk token counts must be >= 0")
|
|
78
|
+
t0 = self._clock() if start is None else float(start)
|
|
79
|
+
end = t0
|
|
80
|
+
if chunk_times is not None:
|
|
81
|
+
chunk_times = [float(t) for t in chunk_times]
|
|
82
|
+
if len(chunk_times) != len(chunks):
|
|
83
|
+
raise ValueError("chunk_times must match chunks")
|
|
84
|
+
end = max(chunk_times)
|
|
85
|
+
return self.record_call(agent_id, prompt_tokens, sum(chunks), reasoning_tokens=reasoning_tokens, scaffold_tokens=scaffold_tokens,
|
|
86
|
+
tool_calls=tool_calls, start=t0, end=end)
|
|
87
|
+
|
|
88
|
+
def record_tool_call(self, agent_id: str, *, attempts: int = 1, start: Optional[float] = None, end: Optional[float] = None) -> CallRecord:
|
|
89
|
+
"""Record a tool invocation. Every attempt, including retries, counts toward r_sys; the interval from `start` to `end`
|
|
90
|
+
(spanning all attempts and back-off waits) counts toward wall-clock latency. No model tokens or completions are added."""
|
|
91
|
+
if attempts < 1:
|
|
92
|
+
raise ValueError("attempts must be >= 1")
|
|
93
|
+
return self.record_call(agent_id, 0, 0, tool_calls=attempts, n_completions=0, start=start, end=end)
|
|
94
|
+
|
|
95
|
+
# ---- aggregates ----
|
|
96
|
+
@property
|
|
97
|
+
def serial_latency(self) -> float:
|
|
98
|
+
return sum(c.end - c.start for c in self.calls)
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def makespan(self) -> float:
|
|
102
|
+
"""Wall-clock time from the first call start to the last call end (concurrent calls overlap)."""
|
|
103
|
+
if not self.calls:
|
|
104
|
+
return 0.0
|
|
105
|
+
return max(c.end for c in self.calls) - min(c.start for c in self.calls)
|
|
106
|
+
|
|
107
|
+
def get_resource_vector(self, allow_missing_pricing: bool = False) -> ResourceVector:
|
|
108
|
+
if not self.calls:
|
|
109
|
+
raise ValueError("no calls recorded")
|
|
110
|
+
prompt = sum(c.prompt_tokens for c in self.calls)
|
|
111
|
+
completion = sum(c.completion_tokens for c in self.calls)
|
|
112
|
+
try:
|
|
113
|
+
cost: Optional[float] = self._tracker.price(prompt, completion)
|
|
114
|
+
except PricingUnavailable:
|
|
115
|
+
if not allow_missing_pricing:
|
|
116
|
+
raise
|
|
117
|
+
cost = None
|
|
118
|
+
vec = ResourceVector(
|
|
119
|
+
r_reason=float(sum(c.reasoning_tokens for c in self.calls)),
|
|
120
|
+
r_token=float(prompt + completion),
|
|
121
|
+
r_cost=cost,
|
|
122
|
+
r_lat=self.makespan,
|
|
123
|
+
r_sample=int(sum(c.n_completions for c in self.calls)),
|
|
124
|
+
r_sys=int(sum(c.tool_calls for c in self.calls)),
|
|
125
|
+
r_scaffold=float(sum(c.scaffold_tokens for c in self.calls)),
|
|
126
|
+
)
|
|
127
|
+
return vec.validate(strict=cost is not None)
|
|
128
|
+
|
|
129
|
+
def scaffold_by_agent(self) -> Dict[str, float]:
|
|
130
|
+
out: Dict[str, float] = {}
|
|
131
|
+
for c in self.calls:
|
|
132
|
+
out[c.agent_id] = out.get(c.agent_id, 0.0) + c.scaffold_tokens
|
|
133
|
+
return out
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class ResourceTracker:
|
|
137
|
+
"""Usage: tracker = ResourceTracker(model, pricing_tier); tracker.register_pricing(...); with tracker.trace() as s: ..."""
|
|
138
|
+
|
|
139
|
+
_PRICES: Dict[Tuple[str, str], Tuple[float, float]] = {}
|
|
140
|
+
|
|
141
|
+
def __init__(self, model: str, pricing_tier: str = "standard", clock: Callable[[], float] = time.monotonic):
|
|
142
|
+
self.model = model
|
|
143
|
+
self.pricing_tier = pricing_tier
|
|
144
|
+
self._clock = clock
|
|
145
|
+
self._prices: Dict[Tuple[str, str], Tuple[float, float]] = {}
|
|
146
|
+
|
|
147
|
+
def register_pricing(self, input_usd_per_mtok: float, output_usd_per_mtok: float,
|
|
148
|
+
model: Optional[str] = None, tier: Optional[str] = None) -> None:
|
|
149
|
+
if input_usd_per_mtok < 0 or output_usd_per_mtok < 0:
|
|
150
|
+
raise ValueError("prices must be >= 0")
|
|
151
|
+
self._prices[(model or self.model, tier or self.pricing_tier)] = (float(input_usd_per_mtok), float(output_usd_per_mtok))
|
|
152
|
+
|
|
153
|
+
def price(self, prompt_tokens: int, completion_tokens: int) -> float:
|
|
154
|
+
key = (self.model, self.pricing_tier)
|
|
155
|
+
if key not in self._prices:
|
|
156
|
+
raise PricingUnavailable("no price registered for %s/%s; call register_pricing()" % key)
|
|
157
|
+
pin, pout = self._prices[key]
|
|
158
|
+
return (prompt_tokens * pin + completion_tokens * pout) / 1e6
|
|
159
|
+
|
|
160
|
+
@contextmanager
|
|
161
|
+
def trace(self) -> Iterator[Session]:
|
|
162
|
+
yield Session(self, self._clock)
|
mae_eval/vector.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""ResourceVector: the 6-dimensional resource-accounting vector R of the MAEO specification (docs/MAEO_SPECIFICATION.md, section 2).
|
|
2
|
+
|
|
3
|
+
R = <r_reason, r_token, r_cost, r_lat, r_sample, r_sys>. A field set to None means "unreported" (the fourth control state).
|
|
4
|
+
Scaffold tokens (system prompts, role declarations, echoed message history) are tracked separately as r_scaffold and are
|
|
5
|
+
counted INSIDE r_token; reasoning tokens are counted inside completion tokens, and r_token = prompt + completion.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from dataclasses import dataclass, fields
|
|
11
|
+
from enum import Enum
|
|
12
|
+
from typing import Any, Dict, Optional
|
|
13
|
+
|
|
14
|
+
DIMENSIONS = ("r_reason", "r_token", "r_cost", "r_lat", "r_sample", "r_sys")
|
|
15
|
+
_INT_DIMS = {"r_sample", "r_sys"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ControlState(str, Enum):
|
|
19
|
+
"""How a resource dimension is controlled between a multi-agent system and its single-agent baseline."""
|
|
20
|
+
STRICT = "Strictly Controlled"
|
|
21
|
+
APPROXIMATE = "Approximately Matched"
|
|
22
|
+
UNMATCHED = "Unmatched (Measured)"
|
|
23
|
+
UNREPORTED = "Unreported (Omitted)"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ResourceVectorError(ValueError):
|
|
27
|
+
"""Raised when a ResourceVector violates the specification."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class ResourceVector:
|
|
32
|
+
r_reason: Optional[float] = None # total reasoning / scratchpad tokens
|
|
33
|
+
r_token: Optional[float] = None # total prompt + completion tokens (includes scaffold)
|
|
34
|
+
r_cost: Optional[float] = None # total monetary inference cost, USD
|
|
35
|
+
r_lat: Optional[float] = None # wall-clock latency in seconds (makespan, concurrency-aware)
|
|
36
|
+
r_sample: Optional[int] = None # total model completions drawn
|
|
37
|
+
r_sys: Optional[int] = None # tool invocations / search queries
|
|
38
|
+
r_scaffold: Optional[float] = None # scaffold tokens (subset of r_token); not part of the 6-vector
|
|
39
|
+
|
|
40
|
+
# ---- derived ----
|
|
41
|
+
@property
|
|
42
|
+
def scaffold_pct(self) -> Optional[float]:
|
|
43
|
+
"""Scaffold tokens as a percentage of r_token; None if either is unreported or r_token is 0."""
|
|
44
|
+
if self.r_scaffold is None or not self.r_token:
|
|
45
|
+
return None
|
|
46
|
+
return 100.0 * self.r_scaffold / self.r_token
|
|
47
|
+
|
|
48
|
+
# ---- serialization ----
|
|
49
|
+
def to_dict(self, include_scaffold: bool = False) -> Dict[str, Any]:
|
|
50
|
+
d = {k: getattr(self, k) for k in DIMENSIONS}
|
|
51
|
+
if include_scaffold:
|
|
52
|
+
d["r_scaffold"] = self.r_scaffold
|
|
53
|
+
return d
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def from_dict(cls, d: Dict[str, Any]) -> "ResourceVector":
|
|
57
|
+
known = {f.name for f in fields(cls)}
|
|
58
|
+
extra = set(d) - known
|
|
59
|
+
if extra:
|
|
60
|
+
raise ResourceVectorError("unknown resource fields: %s" % sorted(extra))
|
|
61
|
+
return cls(**{k: d.get(k) for k in known if k in d})
|
|
62
|
+
|
|
63
|
+
# ---- validation against the MAEO submission schema ----
|
|
64
|
+
def validate(self, strict: bool = True) -> "ResourceVector":
|
|
65
|
+
"""Check the vector against docs/MAEO_SPECIFICATION.md section 2.
|
|
66
|
+
|
|
67
|
+
strict=True (submission mode): every dimension must be present (not None).
|
|
68
|
+
strict=False (audit mode): None is allowed and means 'unreported'.
|
|
69
|
+
Always: numbers must be finite and non-negative; r_sample and r_sys must be integers; r_sample >= 1 when given;
|
|
70
|
+
r_reason <= r_token and r_scaffold <= r_token when those are given.
|
|
71
|
+
"""
|
|
72
|
+
for name in DIMENSIONS:
|
|
73
|
+
v = getattr(self, name)
|
|
74
|
+
if v is None:
|
|
75
|
+
if strict:
|
|
76
|
+
raise ResourceVectorError("%s is required for a submission (use None only for audit mode)" % name)
|
|
77
|
+
continue
|
|
78
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
79
|
+
raise ResourceVectorError("%s must be a number, got %r" % (name, type(v).__name__))
|
|
80
|
+
if not math.isfinite(v) or v < 0:
|
|
81
|
+
raise ResourceVectorError("%s must be finite and >= 0, got %r" % (name, v))
|
|
82
|
+
if name in _INT_DIMS and float(v) != int(v):
|
|
83
|
+
raise ResourceVectorError("%s must be an integer, got %r" % (name, v))
|
|
84
|
+
if self.r_sample is not None and self.r_sample < 1:
|
|
85
|
+
raise ResourceVectorError("r_sample must be >= 1")
|
|
86
|
+
if self.r_token is not None:
|
|
87
|
+
for sub in ("r_reason", "r_scaffold"):
|
|
88
|
+
val = getattr(self, sub)
|
|
89
|
+
if val is not None and val > self.r_token:
|
|
90
|
+
raise ResourceVectorError("%s (%s) cannot exceed r_token (%s)" % (sub, val, self.r_token))
|
|
91
|
+
return self
|
pace/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""cme: measure a multi-agent system against the self-consistency frontier at its measured token cost.
|
|
2
|
+
|
|
3
|
+
Commands: `pace proxy` (logging proxy), `pace baseline` (greedy + sample pool), `pace report` (frontier gap, resource card, table, figure).
|
|
4
|
+
Built on mae_eval (resource vector and parity auditor). Python API: pace.frontier_gap, pace.build_report, pace.resource_card, pace.wrap (log a client's usage without a proxy), pace.track (label items and rounds).
|
|
5
|
+
"""
|
|
6
|
+
from .frontier import Pool, frontier_at, frontier_curve, frontier_gap, pav_increasing
|
|
7
|
+
from .report import build_report, to_latex
|
|
8
|
+
from .client import wrap, track, log_result
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
__all__ = ["Pool", "frontier_at", "frontier_curve", "frontier_gap", "pav_increasing", "build_report", "to_latex", "wrap", "track", "log_result"]
|