pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
@@ -0,0 +1,162 @@
1
+ """Synthetic benchmark generators for the three PACE escape routes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import itertools
6
+ import math
7
+
8
+ import numpy as np
9
+
10
+
11
+ def generate_clue_grid_items(
12
+ n_items: int = 50, k_agents: int = 3, seed: int = 42
13
+ ) -> list[dict]:
14
+ if n_items < 1 or k_agents < 2:
15
+ raise ValueError("n_items must be positive and k_agents >= 2")
16
+ rng = np.random.default_rng(seed)
17
+ out = []
18
+ suspects = ["Avery", "Blake", "Casey", "Drew"]
19
+ for index in range(n_items):
20
+ culprit = suspects[int(rng.integers(0, len(suspects)))]
21
+ private = [
22
+ [
23
+ "Clue %d excludes %s." % (j + 1, suspect)
24
+ for j, suspect in enumerate(suspects)
25
+ if suspect != culprit and j % k_agents == agent
26
+ ]
27
+ for agent in range(k_agents)
28
+ ]
29
+ shared = ["Decoy clue: the most visible suspect is Blake."]
30
+ out.append(
31
+ {
32
+ "id": "grid_%03d" % index,
33
+ "route": "information_partitioning",
34
+ "gold": culprit,
35
+ "split": [
36
+ {
37
+ "agent": i,
38
+ "prompt": "Find the culprit. " + " ".join(shared + clues),
39
+ }
40
+ for i, clues in enumerate(private)
41
+ ],
42
+ "shared": "Find the culprit. "
43
+ + " ".join(shared + [clue for clues in private for clue in clues]),
44
+ "private_clues": private,
45
+ }
46
+ )
47
+ return out
48
+
49
+
50
+ def generate_distributed_code_items(
51
+ n_items: int = 30, k_agents: int = 3, seed: int = 42
52
+ ) -> list[dict]:
53
+ if n_items < 1 or k_agents != 3:
54
+ raise ValueError("distributed code benchmark requires exactly three agents")
55
+ return [
56
+ {
57
+ "id": "code_%03d" % i,
58
+ "route": "information_partitioning",
59
+ "gold": "rename payload to payload_json",
60
+ "split": [
61
+ {"agent": 0, "prompt": "Caller module sends payload."},
62
+ {"agent": 1, "prompt": "API schema requires payload_json."},
63
+ {"agent": 2, "prompt": "Callee reads payload_json."},
64
+ ],
65
+ "shared": "Caller sends payload; schema and callee require payload_json.",
66
+ "synthetic": True, # template item: every item is identical (one question), see docs/REVISION notes; not a benchmark
67
+ }
68
+ for i in range(n_items)
69
+ ]
70
+
71
+
72
+ def evaluate_with_verifier_threshold(
73
+ p1: float,
74
+ tau_ic: float,
75
+ tau_ci: float,
76
+ epsilon_fp: float,
77
+ epsilon_fn: float,
78
+ t_rounds: int = 3,
79
+ ) -> dict:
80
+ if (
81
+ not 0 <= p1 <= 1
82
+ or min(tau_ic, tau_ci, epsilon_fp, epsilon_fn) < 0
83
+ or max(epsilon_fp, epsilon_fn) > 1
84
+ or t_rounds < 1
85
+ ):
86
+ raise ValueError("invalid verifier parameters")
87
+ kappa = tau_ic / tau_ci if tau_ci else float("inf")
88
+ p_star = kappa / (1 + kappa) if math.isfinite(kappa) else 1.0
89
+ kappa_v = kappa * (1 - epsilon_fn) / max(epsilon_fp, 1e-12)
90
+ p_star_v = kappa_v / (1 + kappa_v) if math.isfinite(kappa_v) else 1.0
91
+ critical = kappa * ((1 - p1) / p1) * (1 - epsilon_fn) if p1 else float("inf")
92
+ trajectory = [p1]
93
+ for _ in range(1, t_rounds):
94
+ current = trajectory[-1]
95
+ trajectory.append(
96
+ current * (1 - tau_ci * epsilon_fp)
97
+ + (1 - current) * tau_ic * (1 - epsilon_fn)
98
+ )
99
+ return {
100
+ "kappa": kappa,
101
+ "p_star": p_star,
102
+ "kappa_v": kappa_v,
103
+ "p_star_v": p_star_v,
104
+ "epsilon_fp_crit": critical,
105
+ "trajectory": trajectory,
106
+ "strictly_improves": epsilon_fp < critical,
107
+ }
108
+
109
+
110
+ def select_heterogeneous_panel(
111
+ model_item_matrix: dict[str, list[int]], k_panel: int = 3
112
+ ) -> dict:
113
+ if not model_item_matrix or k_panel < 1 or k_panel > len(model_item_matrix):
114
+ raise ValueError("invalid model panel")
115
+ names = list(model_item_matrix)
116
+ length = len(model_item_matrix[names[0]])
117
+ if any(len(model_item_matrix[name]) != length for name in names):
118
+ raise ValueError("all model vectors must have equal length")
119
+ accuracy = {name: float(np.mean(model_item_matrix[name])) for name in names}
120
+ correlations = {}
121
+ for left, right in itertools.combinations(names, 2):
122
+ e1, e2 = (
123
+ 1 - np.asarray(model_item_matrix[left]),
124
+ 1 - np.asarray(model_item_matrix[right]),
125
+ )
126
+ correlations[(left, right)] = (
127
+ float(np.corrcoef(e1, e2)[0, 1]) if np.std(e1) and np.std(e2) else 0.0
128
+ )
129
+ panels = []
130
+ for panel in itertools.combinations(names, k_panel):
131
+ pairs = [
132
+ correlations[tuple(sorted(pair))]
133
+ for pair in itertools.combinations(panel, 2)
134
+ ]
135
+ rho = float(np.mean(pairs)) if pairs else 0.0
136
+ k_eff = k_panel / (1 + (k_panel - 1) * max(0.0, rho))
137
+ vote = np.mean(
138
+ np.sum([model_item_matrix[name] for name in panel], axis=0)
139
+ >= math.ceil(k_panel / 2)
140
+ )
141
+ panels.append(
142
+ {
143
+ "models": list(panel),
144
+ "rho_bar": rho,
145
+ "k_eff": float(k_eff),
146
+ "vote_accuracy": float(vote),
147
+ }
148
+ )
149
+ best = max(panels, key=lambda row: (row["k_eff"], row["vote_accuracy"]))
150
+ best_single = max(accuracy, key=accuracy.get)
151
+ return {
152
+ "accuracy": accuracy,
153
+ "pairwise_error_correlation": {
154
+ "%s|%s" % key: value for key, value in correlations.items()
155
+ },
156
+ "panels": panels,
157
+ "optimal_panel": best,
158
+ "homogeneous_baseline": {
159
+ "models": [best_single] * k_panel,
160
+ "accuracy": accuracy[best_single],
161
+ },
162
+ }
@@ -0,0 +1,16 @@
1
+ """Instrument CrewAI, which sends its model calls through litellm: wrap litellm's `completion` and label the kickoff. Needs `pip install crewai litellm`.
2
+ Written from the public CrewAI and litellm APIs and not run in this repository's test environment (crewai is not installed there). CrewAI may run
3
+ tasks in worker threads; if calls appear without labels, wrap the kickoff inside each worker or check that your version propagates context variables."""
4
+ import pace
5
+ import litellm
6
+ from crewai import Agent, Crew, Task
7
+
8
+ litellm.completion = pace.wrap(litellm, log_path="pace_trace.jsonl").completion # CrewAI calls litellm.completion at call time
9
+
10
+ llm = "ollama/llama3.1:8b"
11
+ a = Agent(role="solver", goal="Answer arithmetic questions", backstory="A careful solver.", llm=llm)
12
+ b = Agent(role="checker", goal="Check an answer", backstory="A careful checker.", llm=llm)
13
+ tasks = [Task(description="Compute 17 * 24.", expected_output="A number", agent=a), Task(description="Check the previous answer.", expected_output="A number", agent=b)]
14
+ with pace.track(item_id="item_001", system="mas"):
15
+ result = Crew(agents=[a, b], tasks=tasks).kickoff()
16
+ pace.log_result("pace_trace.jsonl", "item_001", correct="408" in str(result), system="mas")