trust-engine-mcp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trust_engine_mcp/__init__.py +3 -0
- trust_engine_mcp/__main__.py +3 -0
- trust_engine_mcp/data/aider-model-diet.json +193 -0
- trust_engine_mcp/data/retirements.json +998 -0
- trust_engine_mcp/data/tasks_template.csv +4 -0
- trust_engine_mcp/data.py +106 -0
- trust_engine_mcp/models.py +61 -0
- trust_engine_mcp/pilot.py +181 -0
- trust_engine_mcp/server.py +142 -0
- trust_engine_mcp/tools.py +442 -0
- trust_engine_mcp/usage.py +341 -0
- trust_engine_mcp-0.1.0.dist-info/METADATA +135 -0
- trust_engine_mcp-0.1.0.dist-info/RECORD +17 -0
- trust_engine_mcp-0.1.0.dist-info/WHEEL +5 -0
- trust_engine_mcp-0.1.0.dist-info/entry_points.txt +2 -0
- trust_engine_mcp-0.1.0.dist-info/licenses/LICENSE +21 -0
- trust_engine_mcp-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Model diet for a coding agent (Aider): Opus vs Sonnet vs Haiku 5.5",
|
|
3
|
+
"publisher": {
|
|
4
|
+
"name": "Trust Engine AI",
|
|
5
|
+
"url": "https://agenttrustapi.com/",
|
|
6
|
+
"email": "hello@agenttrustapi.com"
|
|
7
|
+
},
|
|
8
|
+
"date": "2026-11-11",
|
|
9
|
+
"report_url": "https://agenttrustapi.com/reports/model-diet-aider/",
|
|
10
|
+
"method_url": "https://agenttrustapi.com/reports/model-diet-aider/protocol.md",
|
|
11
|
+
"raw_data_url": "https://agenttrustapi.com/reports/model-diet-aider/results.csv",
|
|
12
|
+
"agent": "Aider 0.86.2, default settings per model",
|
|
13
|
+
"exam": "Aider polyglot benchmark, Python part; hidden tests; one attempt plus one retry with the test output",
|
|
14
|
+
"exercises_in_analysis": 27,
|
|
15
|
+
"exercises_planned": 34,
|
|
16
|
+
"solved_means": "all hidden tests pass after at most one retry (pass@2)",
|
|
17
|
+
"cost_method": "tokens x list price, read from the agent's own output; not yet reconciled with the invoice",
|
|
18
|
+
"models": [
|
|
19
|
+
{
|
|
20
|
+
"model": "claude-haiku-5-5",
|
|
21
|
+
"name": "Claude Haiku 5.5",
|
|
22
|
+
"exercises": 27,
|
|
23
|
+
"solved_first_try": 15,
|
|
24
|
+
"solved": 22,
|
|
25
|
+
"total_cost_usd": 0.2023,
|
|
26
|
+
"cost_per_exercise_usd": 0.0075,
|
|
27
|
+
"cost_per_solved_usd": 0.0092,
|
|
28
|
+
"median_seconds": 30.2,
|
|
29
|
+
"failed": [
|
|
30
|
+
"dot-dsl",
|
|
31
|
+
"forth",
|
|
32
|
+
"hangman",
|
|
33
|
+
"paasio",
|
|
34
|
+
"rest-api"
|
|
35
|
+
],
|
|
36
|
+
"list_price_usd_per_mtok": {
|
|
37
|
+
"input": 0.1,
|
|
38
|
+
"output": 0.5
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"model": "claude-sonnet-5-5",
|
|
43
|
+
"name": "Claude Sonnet 5.5",
|
|
44
|
+
"exercises": 27,
|
|
45
|
+
"solved_first_try": 25,
|
|
46
|
+
"solved": 26,
|
|
47
|
+
"total_cost_usd": 3.9499,
|
|
48
|
+
"cost_per_exercise_usd": 0.1463,
|
|
49
|
+
"cost_per_solved_usd": 0.1519,
|
|
50
|
+
"median_seconds": 27.4,
|
|
51
|
+
"failed": [
|
|
52
|
+
"paasio"
|
|
53
|
+
],
|
|
54
|
+
"list_price_usd_per_mtok": {
|
|
55
|
+
"input": 2,
|
|
56
|
+
"output": 10
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"model": "claude-opus-5-5",
|
|
61
|
+
"name": "Claude Opus 5.5",
|
|
62
|
+
"exercises": 27,
|
|
63
|
+
"solved_first_try": 26,
|
|
64
|
+
"solved": 26,
|
|
65
|
+
"total_cost_usd": 7.7078,
|
|
66
|
+
"cost_per_exercise_usd": 0.2855,
|
|
67
|
+
"cost_per_solved_usd": 0.2965,
|
|
68
|
+
"median_seconds": 33.1,
|
|
69
|
+
"failed": [
|
|
70
|
+
"paasio"
|
|
71
|
+
],
|
|
72
|
+
"list_price_usd_per_mtok": {
|
|
73
|
+
"input": 4,
|
|
74
|
+
"output": 20
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
],
|
|
78
|
+
"pairs": [
|
|
79
|
+
{
|
|
80
|
+
"from": "claude-haiku-5-5",
|
|
81
|
+
"to": "claude-sonnet-5-5",
|
|
82
|
+
"cost_per_solved_ratio": 16.5109,
|
|
83
|
+
"change_pct": 1551.1,
|
|
84
|
+
"solved_from": 22,
|
|
85
|
+
"solved_to": 26,
|
|
86
|
+
"lost_tasks": [],
|
|
87
|
+
"gained_tasks": [
|
|
88
|
+
"dot-dsl",
|
|
89
|
+
"forth",
|
|
90
|
+
"hangman",
|
|
91
|
+
"rest-api"
|
|
92
|
+
]
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"from": "claude-haiku-5-5",
|
|
96
|
+
"to": "claude-opus-5-5",
|
|
97
|
+
"cost_per_solved_ratio": 32.2283,
|
|
98
|
+
"change_pct": 3122.8,
|
|
99
|
+
"solved_from": 22,
|
|
100
|
+
"solved_to": 26,
|
|
101
|
+
"lost_tasks": [],
|
|
102
|
+
"gained_tasks": [
|
|
103
|
+
"dot-dsl",
|
|
104
|
+
"forth",
|
|
105
|
+
"hangman",
|
|
106
|
+
"rest-api"
|
|
107
|
+
]
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"from": "claude-sonnet-5-5",
|
|
111
|
+
"to": "claude-haiku-5-5",
|
|
112
|
+
"cost_per_solved_ratio": 0.0606,
|
|
113
|
+
"change_pct": -93.9,
|
|
114
|
+
"solved_from": 26,
|
|
115
|
+
"solved_to": 22,
|
|
116
|
+
"lost_tasks": [
|
|
117
|
+
"dot-dsl",
|
|
118
|
+
"forth",
|
|
119
|
+
"hangman",
|
|
120
|
+
"rest-api"
|
|
121
|
+
],
|
|
122
|
+
"gained_tasks": []
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
"from": "claude-sonnet-5-5",
|
|
126
|
+
"to": "claude-opus-5-5",
|
|
127
|
+
"cost_per_solved_ratio": 1.9519,
|
|
128
|
+
"change_pct": 95.2,
|
|
129
|
+
"solved_from": 26,
|
|
130
|
+
"solved_to": 26,
|
|
131
|
+
"lost_tasks": [],
|
|
132
|
+
"gained_tasks": []
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"from": "claude-opus-5-5",
|
|
136
|
+
"to": "claude-haiku-5-5",
|
|
137
|
+
"cost_per_solved_ratio": 0.031,
|
|
138
|
+
"change_pct": -96.9,
|
|
139
|
+
"solved_from": 26,
|
|
140
|
+
"solved_to": 22,
|
|
141
|
+
"lost_tasks": [
|
|
142
|
+
"dot-dsl",
|
|
143
|
+
"forth",
|
|
144
|
+
"hangman",
|
|
145
|
+
"rest-api"
|
|
146
|
+
],
|
|
147
|
+
"gained_tasks": []
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
"from": "claude-opus-5-5",
|
|
151
|
+
"to": "claude-sonnet-5-5",
|
|
152
|
+
"cost_per_solved_ratio": 0.5123,
|
|
153
|
+
"change_pct": -48.8,
|
|
154
|
+
"solved_from": 26,
|
|
155
|
+
"solved_to": 26,
|
|
156
|
+
"lost_tasks": [],
|
|
157
|
+
"gained_tasks": []
|
|
158
|
+
}
|
|
159
|
+
],
|
|
160
|
+
"haiku_losses": [
|
|
161
|
+
{
|
|
162
|
+
"exercise": "forth",
|
|
163
|
+
"what": "a small language interpreter: arithmetic and division-by-zero handling",
|
|
164
|
+
"failing_tests": 11,
|
|
165
|
+
"tests": 54
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
"exercise": "rest-api",
|
|
169
|
+
"what": "a small REST API: balances between users",
|
|
170
|
+
"failing_tests": 8,
|
|
171
|
+
"tests": 9
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
"exercise": "dot-dsl",
|
|
175
|
+
"what": "a graph parser: errors for malformed input",
|
|
176
|
+
"failing_tests": 3,
|
|
177
|
+
"tests": 12
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
"exercise": "hangman",
|
|
181
|
+
"what": "a word game: the same correct letter twice counts as a miss",
|
|
182
|
+
"failing_tests": 1,
|
|
183
|
+
"tests": 7
|
|
184
|
+
}
|
|
185
|
+
],
|
|
186
|
+
"limits": [
|
|
187
|
+
"One agent, one language, 27 of 34 exercises (API credits ran out mid-run; every exercise that finished for all three models is reported).",
|
|
188
|
+
"Public benchmark: models may have seen it. Comparing models is fair; absolute rates are not a forecast.",
|
|
189
|
+
"The agent fetched some web pages named in the exercises, which the protocol did not allow.",
|
|
190
|
+
"Costs are estimates from token counts.",
|
|
191
|
+
"One public benchmark, not your tasks: a pilot on 20-30 of your own tasks measures what matters to you."
|
|
192
|
+
]
|
|
193
|
+
}
|