pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/simulator.py ADDED
@@ -0,0 +1,199 @@
1
+ """Deterministic Monte Carlo generators for PACE theory diagnostics."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import numpy as np
6
+
7
+
8
+ def _validate_probability(value: float, name: str) -> None:
9
+ if not 0 <= value <= 1:
10
+ raise ValueError("%s must be in [0, 1]" % name)
11
+
12
+
13
+ def _majority(values: np.ndarray, n_choices: int) -> int:
14
+ counts = np.bincount(values.astype(int), minlength=n_choices)
15
+ leaders = np.flatnonzero(counts == counts.max())
16
+ return int(leaders[0])
17
+
18
+
19
+ def simulate_correlated_voting(
20
+ n_items: int,
21
+ k_agents: int,
22
+ p_mean: float,
23
+ rho_peer: float,
24
+ n_choices: int = 2,
25
+ seed: int = 0,
26
+ ) -> dict:
27
+ """Simulate item difficulty as a beta latent variable and correlated answers."""
28
+ if n_items < 1 or k_agents < 1 or n_choices < 2:
29
+ raise ValueError(
30
+ "n_items, k_agents, and n_choices must be positive; n_choices >= 2"
31
+ )
32
+ _validate_probability(p_mean, "p_mean")
33
+ _validate_probability(rho_peer, "rho_peer")
34
+ rng = np.random.default_rng(seed)
35
+ concentration = max(2.0, 1.0 / max(rho_peer, 1e-9) - 1.0) if rho_peer else 1e9
36
+ alpha, beta = (
37
+ max(1e-6, p_mean * concentration),
38
+ max(1e-6, (1 - p_mean) * concentration),
39
+ )
40
+ truth = rng.integers(0, n_choices, n_items)
41
+ answers = np.empty((n_items, k_agents), dtype=int)
42
+ for i in range(n_items):
43
+ difficulty = float(rng.beta(alpha, beta))
44
+ for agent in range(k_agents):
45
+ answers[i, agent] = (
46
+ truth[i]
47
+ if rng.random() < difficulty
48
+ else int(
49
+ rng.integers(0, n_choices - 1)
50
+ + (rng.integers(0, n_choices) >= truth[i])
51
+ )
52
+ % n_choices
53
+ )
54
+ correct = answers == truth[:, None]
55
+ p1 = float(correct.mean())
56
+ votes = np.array([_majority(row, n_choices) for row in answers])
57
+ p_vote = float(np.mean(votes == truth))
58
+ correlations = []
59
+ if k_agents > 1:
60
+ correlations = np.corrcoef(correct.T)[np.triu_indices(k_agents, 1)].tolist()
61
+ rho_total = float(np.nanmean(correlations)) if correlations else 0.0
62
+ k_eff = k_agents / (1 + (k_agents - 1) * rho_peer)
63
+ hoeffding = (
64
+ max(0.0, 1.0 - np.exp(-2 * k_eff * max(p_mean - 0.5, 0) ** 2))
65
+ if n_choices == 2
66
+ else max(0.0, 1.0 - np.exp(-2 * k_eff * max(p_mean - 1 / n_choices, 0) ** 2))
67
+ )
68
+ return {
69
+ "p1": p1,
70
+ "p_vote": p_vote,
71
+ "rho_total": rho_total,
72
+ "rho_peer": rho_peer,
73
+ "k_eff": k_eff,
74
+ "hoeffding_lower_bound": hoeffding,
75
+ "answers": answers.tolist(),
76
+ "truth": truth.tolist(),
77
+ }
78
+
79
+
80
+ def simulate_debate_dynamics(
81
+ n_items: int,
82
+ k_agents: int,
83
+ t_rounds: int,
84
+ p1: float,
85
+ regime: str = "constant",
86
+ params: dict | None = None,
87
+ n_replicates: int = 1,
88
+ verifier_fp: float | None = None,
89
+ verifier_fn: float | None = None,
90
+ seed: int = 0,
91
+ ) -> list[dict]:
92
+ """Generate scored, round-indexed transcripts under a peer-influence regime."""
93
+ if min(n_items, k_agents, t_rounds, n_replicates) < 1:
94
+ raise ValueError("counts must be positive")
95
+ _validate_probability(p1, "p1")
96
+ if regime not in {"constant", "linear_martingale", "convex_amplification"}:
97
+ raise ValueError("unknown regime: %s" % regime)
98
+ params = params or {"a": 0.25, "b": 0.15}
99
+ a, b = float(params.get("a", 0.25)), float(params.get("b", 0.15))
100
+ if min(a, b) < 0 or max(a, b) > 1:
101
+ raise ValueError("regime rates must be in [0, 1]")
102
+ if verifier_fp is not None or verifier_fn is not None:
103
+ _validate_probability(float(verifier_fp or 0), "verifier_fp")
104
+ _validate_probability(float(verifier_fn or 0), "verifier_fn")
105
+ rng = np.random.default_rng(seed)
106
+ rows = []
107
+ for item_index in range(n_items):
108
+ for replicate in range(n_replicates):
109
+ states = rng.random(k_agents) < p1
110
+ item = (
111
+ "item_%03d#rep%d" % (item_index, replicate)
112
+ if n_replicates > 1
113
+ else "item_%03d" % item_index
114
+ )
115
+ for round_num in range(1, t_rounds + 1):
116
+ for agent in range(k_agents):
117
+ prompt = 100 + (round_num - 1) * k_agents * 20
118
+ rows.append(
119
+ {
120
+ "item": item,
121
+ "replicate": replicate,
122
+ "agent": "agent_%d" % agent,
123
+ "round": round_num,
124
+ "correct": bool(states[agent]),
125
+ "answer": "A" if states[agent] else "B",
126
+ "prompt_tokens": prompt,
127
+ "completion_tokens": 20,
128
+ }
129
+ )
130
+ if round_num == t_rounds:
131
+ continue
132
+ previous = states.copy()
133
+ correct_peers = int(previous.sum())
134
+ for agent in range(k_agents):
135
+ x = (
136
+ (correct_peers - int(previous[agent])) / max(1, k_agents - 1)
137
+ if previous[agent]
138
+ else (k_agents - 1 - correct_peers) / max(1, k_agents - 1)
139
+ )
140
+ power = (
141
+ 0
142
+ if regime == "constant"
143
+ else 1
144
+ if regime == "linear_martingale"
145
+ else 2
146
+ )
147
+ rate = (a if not previous[agent] else b) * (
148
+ x**power if power else 1
149
+ )
150
+ if rng.random() < rate:
151
+ is_improvement = not previous[agent]
152
+ if (
153
+ verifier_fn is not None
154
+ and is_improvement
155
+ and rng.random() < verifier_fn
156
+ ):
157
+ continue
158
+ if (
159
+ verifier_fp is not None
160
+ and not is_improvement
161
+ and rng.random() < verifier_fp
162
+ ):
163
+ continue
164
+ states[agent] = not states[agent]
165
+ return rows
166
+
167
+
168
+ def simulate_scaffolding_scaling_curve(
169
+ budgets: list[float],
170
+ k_agents: int,
171
+ t_rounds: int,
172
+ c0: float,
173
+ l_msg: float,
174
+ b_half: float = 500.0,
175
+ p_min: float = 0.25,
176
+ p_max: float = 0.85,
177
+ ) -> list[dict]:
178
+ """Map total budgets to crowded MAS and SAS effective-thinking curves."""
179
+ if min(k_agents, t_rounds) < 1 or b_half <= 0 or p_max < p_min:
180
+ raise ValueError("invalid scaling parameters")
181
+ out = []
182
+ cost = (
183
+ k_agents * t_rounds * c0 + 0.5 * k_agents**2 * t_rounds * (t_rounds - 1) * l_msg
184
+ )
185
+ for budget in budgets:
186
+ b_eff = max(0.0, (float(budget) - cost) / (k_agents * t_rounds))
187
+ sas = max(0.0, float(budget) - c0)
188
+ hill = lambda b: p_min + (p_max - p_min) * b / (b + b_half)
189
+ out.append(
190
+ {
191
+ "budget": float(budget),
192
+ "c_total": cost,
193
+ "b_eff": b_eff,
194
+ "b_eff_sas": sas,
195
+ "p_mas": hill(b_eff),
196
+ "p_sas": hill(sas),
197
+ }
198
+ )
199
+ return out