pi-free 2.2.5 → 2.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/README.md +1 -1
  3. package/banner.svg +1 -1
  4. package/config.ts +803 -779
  5. package/constants.ts +0 -3
  6. package/index.ts +30 -24
  7. package/lib/built-in-toggle.ts +426 -427
  8. package/lib/provider-cache.ts +22 -0
  9. package/package.json +1 -1
  10. package/provider-failover/benchmark-lookup.ts +4 -1
  11. package/provider-failover/benchmarks-chunk-0.ts +570 -570
  12. package/provider-failover/benchmarks-chunk-1.ts +676 -676
  13. package/provider-failover/benchmarks-chunk-2.ts +673 -673
  14. package/provider-failover/benchmarks-chunk-3.ts +680 -680
  15. package/provider-failover/benchmarks-chunk-4.ts +683 -683
  16. package/provider-failover/benchmarks-chunk-5.ts +816 -474
  17. package/provider-helper.ts +64 -0
  18. package/providers/bai/bai.ts +8 -2
  19. package/providers/crofai/crofai.ts +8 -2
  20. package/providers/deepinfra/deepinfra.ts +8 -2
  21. package/providers/dynamic-built-in/index.ts +36 -26
  22. package/providers/kilo/kilo.ts +41 -22
  23. package/providers/novita/novita.ts +8 -2
  24. package/providers/openmodel/openmodel.ts +8 -2
  25. package/providers/routeway/routeway.ts +8 -2
  26. package/providers/sambanova/sambanova.ts +12 -6
  27. package/providers/together/together.ts +8 -2
  28. package/providers/tokenrouter/tokenrouter.ts +8 -2
  29. package/providers/zenmux/zenmux.ts +8 -2
  30. package/providers/codestral/codestral.ts +0 -128
package/provider-failover/benchmarks-chunk-3.ts CHANGED
@@ -1,20 +1,20 @@
1
1
  // Auto-generated benchmark data chunk 3
2
- // Models: llama-3.1-instruct-70b .. magistral-medium-1 (90 entries)
3
- // Last updated: 2026-06-01
2
+ // Models: gpt-5-medium .. mistral-large-2-jul-24 (90 entries)
3
+ // Last updated: 2026-07-01
4
4
  // DO NOT EDIT MANUALLY — generated by scripts/update-benchmarks.ts
5
5
 
6
6
  import type { HardcodedBenchmark } from "./hardcoded-benchmarks.ts";
7
7
 
8
8
  export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
9
- "llama-3.1-instruct-70b": {
9
+ "gpt-5-medium": {
10
10
  // AA specific benchmarks
11
- codingIndex: 10.9,
12
- mathIndex: 4,
11
+ codingIndex: undefined,
12
+ mathIndex: 91.7,
13
13
 
14
14
  // Academic benchmarks
15
- mmluPro: 0.676,
16
- gpqa: 0.409,
17
- hle: 0.046,
15
+ mmluPro: 0.867,
16
+ gpqa: 0.842,
17
+ hle: 0.235,
18
18
 
19
19
  // Capabilities
20
20
  contextWindow: 8192,
@@ -22,18 +22,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
22
22
  supportsVision: false,
23
23
 
24
24
  // Metadata
25
- lastUpdated: "2026-06-01",
26
- originalModel: "Llama 3.1 Instruct 70B",
25
+ lastUpdated: "2026-07-01",
26
+ originalModel: "GPT-5 (medium)",
27
27
  },
28
- "llama-3.1-instruct-8b": {
28
+ "gpt-5.5-instant-may-2026": {
29
29
  // AA specific benchmarks
30
- codingIndex: 4.9,
31
- mathIndex: 4.3,
30
+ codingIndex: undefined,
31
+ mathIndex: undefined,
32
32
 
33
33
  // Academic benchmarks
34
- mmluPro: 0.476,
35
- gpqa: 0.259,
36
- hle: 0.051,
34
+ mmluPro: undefined,
35
+ gpqa: 0.846,
36
+ hle: 0.203,
37
37
 
38
38
  // Capabilities
39
39
  contextWindow: 8192,
@@ -41,18 +41,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
41
41
  supportsVision: false,
42
42
 
43
43
  // Metadata
44
- lastUpdated: "2026-06-01",
45
- originalModel: "Llama 3.1 Instruct 8B",
44
+ lastUpdated: "2026-07-01",
45
+ originalModel: "GPT-5.5 Instant (May 2026)",
46
46
  },
47
- "llama-3.2-instruct-3b": {
47
+ "gpt-5-codex-high": {
48
48
  // AA specific benchmarks
49
49
  codingIndex: undefined,
50
- mathIndex: 3.3,
50
+ mathIndex: 98.7,
51
51
 
52
52
  // Academic benchmarks
53
- mmluPro: 0.347,
54
- gpqa: 0.255,
55
- hle: 0.052,
53
+ mmluPro: 0.865,
54
+ gpqa: 0.837,
55
+ hle: 0.256,
56
56
 
57
57
  // Capabilities
58
58
  contextWindow: 8192,
@@ -60,18 +60,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
60
60
  supportsVision: false,
61
61
 
62
62
  // Metadata
63
- lastUpdated: "2026-06-01",
64
- originalModel: "Llama 3.2 Instruct 3B",
63
+ lastUpdated: "2026-07-01",
64
+ originalModel: "GPT-5 Codex (high)",
65
65
  },
66
- "llama-3-instruct-70b": {
66
+ "gpt-4.5-preview": {
67
67
  // AA specific benchmarks
68
- codingIndex: 6.8,
68
+ codingIndex: undefined,
69
69
  mathIndex: undefined,
70
70
 
71
71
  // Academic benchmarks
72
- mmluPro: 0.574,
73
- gpqa: 0.379,
74
- hle: 0.044,
72
+ mmluPro: undefined,
73
+ gpqa: undefined,
74
+ hle: undefined,
75
75
 
76
76
  // Capabilities
77
77
  contextWindow: 8192,
@@ -79,18 +79,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
79
79
  supportsVision: false,
80
80
 
81
81
  // Metadata
82
- lastUpdated: "2026-06-01",
83
- originalModel: "Llama 3 Instruct 70B",
82
+ lastUpdated: "2026-07-01",
83
+ originalModel: "GPT-4.5 (Preview)",
84
84
  },
85
- "llama-3-instruct-8b": {
85
+ "o1-pro": {
86
86
  // AA specific benchmarks
87
- codingIndex: 4,
87
+ codingIndex: undefined,
88
88
  mathIndex: undefined,
89
89
 
90
90
  // Academic benchmarks
91
- mmluPro: 0.405,
92
- gpqa: 0.296,
93
- hle: 0.051,
91
+ mmluPro: undefined,
92
+ gpqa: undefined,
93
+ hle: undefined,
94
94
 
95
95
  // Capabilities
96
96
  contextWindow: 8192,
@@ -98,18 +98,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
98
98
  supportsVision: false,
99
99
 
100
100
  // Metadata
101
- lastUpdated: "2026-06-01",
102
- originalModel: "Llama 3 Instruct 8B",
101
+ lastUpdated: "2026-07-01",
102
+ originalModel: "o1-pro",
103
103
  },
104
- "llama-3.2-instruct-1b": {
104
+ "gpt-5-chatgpt": {
105
105
  // AA specific benchmarks
106
- codingIndex: 0.6,
107
- mathIndex: 0,
106
+ codingIndex: undefined,
107
+ mathIndex: 48.3,
108
108
 
109
109
  // Academic benchmarks
110
- mmluPro: 0.2,
111
- gpqa: 0.196,
112
- hle: 0.053,
110
+ mmluPro: 0.82,
111
+ gpqa: 0.686,
112
+ hle: 0.058,
113
113
 
114
114
  // Capabilities
115
115
  contextWindow: 8192,
@@ -117,17 +117,17 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
117
117
  supportsVision: false,
118
118
 
119
119
  // Metadata
120
- lastUpdated: "2026-06-01",
121
- originalModel: "Llama 3.2 Instruct 1B",
120
+ lastUpdated: "2026-07-01",
121
+ originalModel: "GPT-5 (ChatGPT)",
122
122
  },
123
- "llama-2-chat-70b": {
123
+ "gpt-4o-march-2025-chatgpt-4o-latest": {
124
124
  // AA specific benchmarks
125
125
  codingIndex: undefined,
126
- mathIndex: undefined,
126
+ mathIndex: 25.7,
127
127
 
128
128
  // Academic benchmarks
129
- mmluPro: 0.406,
130
- gpqa: 0.327,
129
+ mmluPro: 0.803,
130
+ gpqa: 0.655,
131
131
  hle: 0.05,
132
132
 
133
133
  // Capabilities
@@ -136,18 +136,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
136
136
  supportsVision: false,
137
137
 
138
138
  // Metadata
139
- lastUpdated: "2026-06-01",
140
- originalModel: "Llama 2 Chat 70B",
139
+ lastUpdated: "2026-07-01",
140
+ originalModel: "GPT-4o (March 2025, chatgpt-4o-latest)",
141
141
  },
142
- "llama-2-chat-7b": {
142
+ "gpt-5.4-low": {
143
143
  // AA specific benchmarks
144
144
  codingIndex: undefined,
145
145
  mathIndex: undefined,
146
146
 
147
147
  // Academic benchmarks
148
- mmluPro: 0.164,
149
- gpqa: 0.227,
150
- hle: 0.058,
148
+ mmluPro: undefined,
149
+ gpqa: 0.871,
150
+ hle: 0.289,
151
151
 
152
152
  // Capabilities
153
153
  contextWindow: 8192,
@@ -155,18 +155,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
155
155
  supportsVision: false,
156
156
 
157
157
  // Metadata
158
- lastUpdated: "2026-06-01",
159
- originalModel: "Llama 2 Chat 7B",
158
+ lastUpdated: "2026-07-01",
159
+ originalModel: "GPT-5.4 (low)",
160
160
  },
161
- "llama-2-chat-13b": {
161
+ "gpt-5.1-high": {
162
162
  // AA specific benchmarks
163
163
  codingIndex: undefined,
164
- mathIndex: undefined,
164
+ mathIndex: 94,
165
165
 
166
166
  // Academic benchmarks
167
- mmluPro: 0.406,
168
- gpqa: 0.321,
169
- hle: 0.047,
167
+ mmluPro: 0.87,
168
+ gpqa: 0.873,
169
+ hle: 0.265,
170
170
 
171
171
  // Capabilities
172
172
  contextWindow: 8192,
@@ -174,18 +174,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
174
174
  supportsVision: false,
175
175
 
176
176
  // Metadata
177
- lastUpdated: "2026-06-01",
178
- originalModel: "Llama 2 Chat 13B",
177
+ lastUpdated: "2026-07-01",
178
+ originalModel: "GPT-5.1 (high)",
179
179
  },
180
- "gemini-2.0-pro-experimental-feb-25": {
180
+ "gpt-5.2-xhigh": {
181
181
  // AA specific benchmarks
182
- codingIndex: 25.5,
183
- mathIndex: undefined,
182
+ codingIndex: undefined,
183
+ mathIndex: 99,
184
184
 
185
185
  // Academic benchmarks
186
- mmluPro: 0.805,
187
- gpqa: 0.622,
188
- hle: 0.068,
186
+ mmluPro: 0.874,
187
+ gpqa: 0.903,
188
+ hle: 0.354,
189
189
 
190
190
  // Capabilities
191
191
  contextWindow: 8192,
@@ -193,18 +193,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
193
193
  supportsVision: false,
194
194
 
195
195
  // Metadata
196
- lastUpdated: "2026-06-01",
197
- originalModel: "Gemini 2.0 Pro Experimental (Feb '25)",
196
+ lastUpdated: "2026-07-01",
197
+ originalModel: "GPT-5.2 (xhigh)",
198
198
  },
199
- "gemini-2.0-flash-experimental": {
199
+ "gpt-5.4-non-reasoning": {
200
200
  // AA specific benchmarks
201
201
  codingIndex: undefined,
202
202
  mathIndex: undefined,
203
203
 
204
204
  // Academic benchmarks
205
- mmluPro: 0.782,
206
- gpqa: 0.636,
207
- hle: 0.047,
205
+ mmluPro: undefined,
206
+ gpqa: 0.748,
207
+ hle: 0.106,
208
208
 
209
209
  // Capabilities
210
210
  contextWindow: 8192,
@@ -212,18 +212,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
212
212
  supportsVision: false,
213
213
 
214
214
  // Metadata
215
- lastUpdated: "2026-06-01",
216
- originalModel: "Gemini 2.0 Flash (experimental)",
215
+ lastUpdated: "2026-07-01",
216
+ originalModel: "GPT-5.4 (Non-reasoning)",
217
217
  },
218
- "gemini-1.5-pro-sep-24": {
218
+ "gpt-5-high": {
219
219
  // AA specific benchmarks
220
- codingIndex: 23.6,
221
- mathIndex: undefined,
220
+ codingIndex: undefined,
221
+ mathIndex: 94.3,
222
222
 
223
223
  // Academic benchmarks
224
- mmluPro: 0.75,
225
- gpqa: 0.589,
226
- hle: 0.049,
224
+ mmluPro: 0.871,
225
+ gpqa: 0.854,
226
+ hle: 0.265,
227
227
 
228
228
  // Capabilities
229
229
  contextWindow: 8192,
@@ -231,18 +231,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
231
231
  supportsVision: false,
232
232
 
233
233
  // Metadata
234
- lastUpdated: "2026-06-01",
235
- originalModel: "Gemini 1.5 Pro (Sep '24)",
234
+ lastUpdated: "2026-07-01",
235
+ originalModel: "GPT-5 (high)",
236
236
  },
237
- "gemini-2.0-flash-lite-preview": {
237
+ "gpt-4.1": {
238
238
  // AA specific benchmarks
239
239
  codingIndex: undefined,
240
- mathIndex: undefined,
240
+ mathIndex: 34.7,
241
241
 
242
242
  // Academic benchmarks
243
- mmluPro: undefined,
244
- gpqa: 0.542,
245
- hle: 0.044,
243
+ mmluPro: 0.806,
244
+ gpqa: 0.666,
245
+ hle: 0.046,
246
246
 
247
247
  // Capabilities
248
248
  contextWindow: 8192,
@@ -250,18 +250,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
250
250
  supportsVision: false,
251
251
 
252
252
  // Metadata
253
- lastUpdated: "2026-06-01",
254
- originalModel: "Gemini 2.0 Flash-Lite (Preview)",
253
+ lastUpdated: "2026-07-01",
254
+ originalModel: "GPT-4.1",
255
255
  },
256
- "gemini-2.0-flash-feb-25": {
256
+ "o3-mini": {
257
257
  // AA specific benchmarks
258
- codingIndex: 13.6,
259
- mathIndex: 21.7,
258
+ codingIndex: undefined,
259
+ mathIndex: undefined,
260
260
 
261
261
  // Academic benchmarks
262
- mmluPro: 0.779,
263
- gpqa: 0.623,
264
- hle: 0.053,
262
+ mmluPro: 0.791,
263
+ gpqa: 0.748,
264
+ hle: 0.087,
265
265
 
266
266
  // Capabilities
267
267
  contextWindow: 8192,
@@ -269,18 +269,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
269
269
  supportsVision: false,
270
270
 
271
271
  // Metadata
272
- lastUpdated: "2026-06-01",
273
- originalModel: "Gemini 2.0 Flash (Feb '25)",
272
+ lastUpdated: "2026-07-01",
273
+ originalModel: "o3-mini",
274
274
  },
275
- "gemini-1.5-flash-sep-24": {
275
+ "o3-mini-high": {
276
276
  // AA specific benchmarks
277
- codingIndex: undefined,
277
+ codingIndex: 42.1,
278
278
  mathIndex: undefined,
279
279
 
280
280
  // Academic benchmarks
281
- mmluPro: 0.68,
282
- gpqa: 0.463,
283
- hle: 0.035,
281
+ mmluPro: 0.802,
282
+ gpqa: 0.773,
283
+ hle: 0.123,
284
284
 
285
285
  // Capabilities
286
286
  contextWindow: 8192,
@@ -288,18 +288,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
288
288
  supportsVision: false,
289
289
 
290
290
  // Metadata
291
- lastUpdated: "2026-06-01",
292
- originalModel: "Gemini 1.5 Flash (Sep '24)",
291
+ lastUpdated: "2026-07-01",
292
+ originalModel: "o3-mini (high)",
293
293
  },
294
- "gemini-1.5-flash-8b": {
294
+ "gpt-5-nano-minimal": {
295
295
  // AA specific benchmarks
296
296
  codingIndex: undefined,
297
- mathIndex: undefined,
297
+ mathIndex: 27.3,
298
298
 
299
299
  // Academic benchmarks
300
- mmluPro: 0.569,
301
- gpqa: 0.359,
302
- hle: 0.045,
300
+ mmluPro: 0.556,
301
+ gpqa: 0.428,
302
+ hle: 0.041,
303
303
 
304
304
  // Capabilities
305
305
  contextWindow: 8192,
@@ -307,18 +307,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
307
307
  supportsVision: false,
308
308
 
309
309
  // Metadata
310
- lastUpdated: "2026-06-01",
311
- originalModel: "Gemini 1.5 Flash-8B",
310
+ lastUpdated: "2026-07-01",
311
+ originalModel: "GPT-5 nano (minimal)",
312
312
  },
313
- "gemma-3-1b-instruct": {
313
+ "gpt-5.1-codex-high": {
314
314
  // AA specific benchmarks
315
- codingIndex: 0.2,
316
- mathIndex: 3.3,
315
+ codingIndex: undefined,
316
+ mathIndex: 95.7,
317
317
 
318
318
  // Academic benchmarks
319
- mmluPro: 0.135,
320
- gpqa: 0.237,
321
- hle: 0.052,
319
+ mmluPro: 0.86,
320
+ gpqa: 0.86,
321
+ hle: 0.234,
322
322
 
323
323
  // Capabilities
324
324
  contextWindow: 8192,
@@ -326,18 +326,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
326
326
  supportsVision: false,
327
327
 
328
328
  // Metadata
329
- lastUpdated: "2026-06-01",
330
- originalModel: "Gemma 3 1B Instruct",
329
+ lastUpdated: "2026-07-01",
330
+ originalModel: "GPT-5.1 Codex (high)",
331
331
  },
332
- "gemini-2.5-flash-lite-non-reasoning": {
332
+ "gpt-5.2-codex-xhigh": {
333
333
  // AA specific benchmarks
334
- codingIndex: 7.4,
335
- mathIndex: 35.3,
334
+ codingIndex: undefined,
335
+ mathIndex: undefined,
336
336
 
337
337
  // Academic benchmarks
338
- mmluPro: 0.724,
339
- gpqa: 0.474,
340
- hle: 0.037,
338
+ mmluPro: undefined,
339
+ gpqa: 0.899,
340
+ hle: 0.335,
341
341
 
342
342
  // Capabilities
343
343
  contextWindow: 8192,
@@ -345,18 +345,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
345
345
  supportsVision: false,
346
346
 
347
347
  // Metadata
348
- lastUpdated: "2026-06-01",
349
- originalModel: "Gemini 2.5 Flash-Lite (Non-reasoning)",
348
+ lastUpdated: "2026-07-01",
349
+ originalModel: "GPT-5.2 Codex (xhigh)",
350
350
  },
351
- "gemini-3-pro-preview-high": {
351
+ "llama-3.1-instruct-70b": {
352
352
  // AA specific benchmarks
353
- codingIndex: 46.5,
354
- mathIndex: 95.7,
353
+ codingIndex: undefined,
354
+ mathIndex: 4,
355
355
 
356
356
  // Academic benchmarks
357
- mmluPro: 0.898,
358
- gpqa: 0.908,
359
- hle: 0.372,
357
+ mmluPro: 0.676,
358
+ gpqa: 0.409,
359
+ hle: 0.046,
360
360
 
361
361
  // Capabilities
362
362
  contextWindow: 8192,
@@ -364,18 +364,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
364
364
  supportsVision: false,
365
365
 
366
366
  // Metadata
367
- lastUpdated: "2026-06-01",
368
- originalModel: "Gemini 3 Pro Preview (high)",
367
+ lastUpdated: "2026-07-01",
368
+ originalModel: "Llama 3.1 Instruct 70B",
369
369
  },
370
- "gemma-3n-e2b-instruct": {
370
+ "llama-3.1-instruct-8b": {
371
371
  // AA specific benchmarks
372
- codingIndex: 2.2,
373
- mathIndex: 10.3,
372
+ codingIndex: 5.4,
373
+ mathIndex: 4.3,
374
374
 
375
375
  // Academic benchmarks
376
- mmluPro: 0.378,
377
- gpqa: 0.229,
378
- hle: 0.04,
376
+ mmluPro: 0.476,
377
+ gpqa: 0.259,
378
+ hle: 0.051,
379
379
 
380
380
  // Capabilities
381
381
  contextWindow: 8192,
@@ -383,18 +383,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
383
383
  supportsVision: false,
384
384
 
385
385
  // Metadata
386
- lastUpdated: "2026-06-01",
387
- originalModel: "Gemma 3n E2B Instruct",
386
+ lastUpdated: "2026-07-01",
387
+ originalModel: "Llama 3.1 Instruct 8B",
388
388
  },
389
- "gemma-3-12b-instruct": {
389
+ "llama-3.2-instruct-3b": {
390
390
  // AA specific benchmarks
391
- codingIndex: 6.3,
392
- mathIndex: 18.3,
391
+ codingIndex: undefined,
392
+ mathIndex: 3.3,
393
393
 
394
394
  // Academic benchmarks
395
- mmluPro: 0.595,
396
- gpqa: 0.349,
397
- hle: 0.048,
395
+ mmluPro: 0.347,
396
+ gpqa: 0.255,
397
+ hle: 0.052,
398
398
 
399
399
  // Capabilities
400
400
  contextWindow: 8192,
@@ -402,18 +402,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
402
402
  supportsVision: false,
403
403
 
404
404
  // Metadata
405
- lastUpdated: "2026-06-01",
406
- originalModel: "Gemma 3 12B Instruct",
405
+ lastUpdated: "2026-07-01",
406
+ originalModel: "Llama 3.2 Instruct 3B",
407
407
  },
408
- "gemma-3-27b-instruct": {
408
+ "llama-3-instruct-70b": {
409
409
  // AA specific benchmarks
410
- codingIndex: 9.6,
411
- mathIndex: 20.7,
410
+ codingIndex: undefined,
411
+ mathIndex: undefined,
412
412
 
413
413
  // Academic benchmarks
414
- mmluPro: 0.669,
415
- gpqa: 0.428,
416
- hle: 0.047,
414
+ mmluPro: 0.574,
415
+ gpqa: 0.379,
416
+ hle: 0.044,
417
417
 
418
418
  // Capabilities
419
419
  contextWindow: 8192,
@@ -421,18 +421,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
421
421
  supportsVision: false,
422
422
 
423
423
  // Metadata
424
- lastUpdated: "2026-06-01",
425
- originalModel: "Gemma 3 27B Instruct",
424
+ lastUpdated: "2026-07-01",
425
+ originalModel: "Llama 3 Instruct 70B",
426
426
  },
427
- "gemini-2.5-flash-preview-sep-25-reasoning": {
427
+ "llama-3-instruct-8b": {
428
428
  // AA specific benchmarks
429
- codingIndex: 24.6,
430
- mathIndex: 78.3,
429
+ codingIndex: undefined,
430
+ mathIndex: undefined,
431
431
 
432
432
  // Academic benchmarks
433
- mmluPro: 0.842,
434
- gpqa: 0.793,
435
- hle: 0.127,
433
+ mmluPro: 0.405,
434
+ gpqa: 0.296,
435
+ hle: 0.051,
436
436
 
437
437
  // Capabilities
438
438
  contextWindow: 8192,
@@ -440,18 +440,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
440
440
  supportsVision: false,
441
441
 
442
442
  // Metadata
443
- lastUpdated: "2026-06-01",
444
- originalModel: "Gemini 2.5 Flash Preview (Sep '25) (Reasoning)",
443
+ lastUpdated: "2026-07-01",
444
+ originalModel: "Llama 3 Instruct 8B",
445
445
  },
446
- "gemini-1.5-pro-may-24": {
446
+ "llama-3.2-instruct-1b": {
447
447
  // AA specific benchmarks
448
- codingIndex: 19.8,
449
- mathIndex: undefined,
448
+ codingIndex: undefined,
449
+ mathIndex: 0,
450
450
 
451
451
  // Academic benchmarks
452
- mmluPro: 0.657,
453
- gpqa: 0.371,
454
- hle: 0.039,
452
+ mmluPro: 0.2,
453
+ gpqa: 0.196,
454
+ hle: 0.053,
455
455
 
456
456
  // Capabilities
457
457
  contextWindow: 8192,
@@ -459,18 +459,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
459
459
  supportsVision: false,
460
460
 
461
461
  // Metadata
462
- lastUpdated: "2026-06-01",
463
- originalModel: "Gemini 1.5 Pro (May '24)",
462
+ lastUpdated: "2026-07-01",
463
+ originalModel: "Llama 3.2 Instruct 1B",
464
464
  },
465
- "gemma-3-4b-instruct": {
465
+ "llama-2-chat-70b": {
466
466
  // AA specific benchmarks
467
- codingIndex: 2.9,
468
- mathIndex: 12.7,
467
+ codingIndex: undefined,
468
+ mathIndex: undefined,
469
469
 
470
470
  // Academic benchmarks
471
- mmluPro: 0.417,
472
- gpqa: 0.291,
473
- hle: 0.052,
471
+ mmluPro: 0.406,
472
+ gpqa: 0.327,
473
+ hle: 0.05,
474
474
 
475
475
  // Capabilities
476
476
  contextWindow: 8192,
@@ -478,18 +478,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
478
478
  supportsVision: false,
479
479
 
480
480
  // Metadata
481
- lastUpdated: "2026-06-01",
482
- originalModel: "Gemma 3 4B Instruct",
481
+ lastUpdated: "2026-07-01",
482
+ originalModel: "Llama 2 Chat 70B",
483
483
  },
484
- "gemini-2.5-pro-preview-may-25": {
484
+ "llama-2-chat-7b": {
485
485
  // AA specific benchmarks
486
486
  codingIndex: undefined,
487
487
  mathIndex: undefined,
488
488
 
489
489
  // Academic benchmarks
490
- mmluPro: 0.837,
491
- gpqa: 0.822,
492
- hle: 0.154,
490
+ mmluPro: 0.164,
491
+ gpqa: 0.227,
492
+ hle: 0.058,
493
493
 
494
494
  // Capabilities
495
495
  contextWindow: 8192,
@@ -497,18 +497,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
497
497
  supportsVision: false,
498
498
 
499
499
  // Metadata
500
- lastUpdated: "2026-06-01",
501
- originalModel: "Gemini 2.5 Pro Preview (May' 25)",
500
+ lastUpdated: "2026-07-01",
501
+ originalModel: "Llama 2 Chat 7B",
502
502
  },
503
- "gemini-2.0-flash-thinking-experimental-jan-25": {
503
+ "llama-2-chat-13b": {
504
504
  // AA specific benchmarks
505
- codingIndex: 24.1,
505
+ codingIndex: undefined,
506
506
  mathIndex: undefined,
507
507
 
508
508
  // Academic benchmarks
509
- mmluPro: 0.798,
510
- gpqa: 0.701,
511
- hle: 0.071,
509
+ mmluPro: 0.406,
510
+ gpqa: 0.321,
511
+ hle: 0.047,
512
512
 
513
513
  // Capabilities
514
514
  contextWindow: 8192,
@@ -516,37 +516,37 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
516
516
  supportsVision: false,
517
517
 
518
518
  // Metadata
519
- lastUpdated: "2026-06-01",
520
- originalModel: "Gemini 2.0 Flash Thinking Experimental (Jan '25)",
519
+ lastUpdated: "2026-07-01",
520
+ originalModel: "Llama 2 Chat 13B",
521
521
  },
522
- "gemma-3n-e4b-instruct-preview-may-25": {
522
+ "gemini-2.0-pro-experimental-feb-25": {
523
523
  // AA specific benchmarks
524
- codingIndex: undefined,
524
+ codingIndex: 25.5,
525
525
  mathIndex: undefined,
526
526
 
527
527
  // Academic benchmarks
528
- mmluPro: 0.483,
529
- gpqa: 0.278,
530
- hle: 0.049,
531
-
528
+ mmluPro: 0.805,
529
+ gpqa: 0.622,
530
+ hle: 0.068,
531
+
532
532
  // Capabilities
533
533
  contextWindow: 8192,
534
534
  supportsReasoning: false,
535
535
  supportsVision: false,
536
536
 
537
537
  // Metadata
538
- lastUpdated: "2026-06-01",
539
- originalModel: "Gemma 3n E4B Instruct Preview (May '25)",
538
+ lastUpdated: "2026-07-01",
539
+ originalModel: "Gemini 2.0 Pro Experimental (Feb '25)",
540
540
  },
541
- "gemini-1.0-ultra": {
541
+ "gemini-2.0-flash-experimental": {
542
542
  // AA specific benchmarks
543
- codingIndex: 17.6,
543
+ codingIndex: undefined,
544
544
  mathIndex: undefined,
545
545
 
546
546
  // Academic benchmarks
547
- mmluPro: undefined,
548
- gpqa: undefined,
549
- hle: undefined,
547
+ mmluPro: 0.782,
548
+ gpqa: 0.636,
549
+ hle: 0.047,
550
550
 
551
551
  // Capabilities
552
552
  contextWindow: 8192,
@@ -554,18 +554,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
554
554
  supportsVision: false,
555
555
 
556
556
  // Metadata
557
- lastUpdated: "2026-06-01",
558
- originalModel: "Gemini 1.0 Ultra",
557
+ lastUpdated: "2026-07-01",
558
+ originalModel: "Gemini 2.0 Flash (experimental)",
559
559
  },
560
- "gemma-3n-e4b-instruct": {
560
+ "gemini-1.5-pro-sep-24": {
561
561
  // AA specific benchmarks
562
- codingIndex: 4.2,
563
- mathIndex: 14.3,
562
+ codingIndex: 23.6,
563
+ mathIndex: undefined,
564
564
 
565
565
  // Academic benchmarks
566
- mmluPro: 0.488,
567
- gpqa: 0.296,
568
- hle: 0.044,
566
+ mmluPro: 0.75,
567
+ gpqa: 0.589,
568
+ hle: 0.049,
569
569
 
570
570
  // Capabilities
571
571
  contextWindow: 8192,
@@ -573,18 +573,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
573
573
  supportsVision: false,
574
574
 
575
575
  // Metadata
576
- lastUpdated: "2026-06-01",
577
- originalModel: "Gemma 3n E4B Instruct",
576
+ lastUpdated: "2026-07-01",
577
+ originalModel: "Gemini 1.5 Pro (Sep '24)",
578
578
  },
579
- "gemini-2.5-flash-lite-reasoning": {
579
+ "gemini-2.0-flash-lite-preview": {
580
580
  // AA specific benchmarks
581
- codingIndex: 9.5,
582
- mathIndex: 53.3,
581
+ codingIndex: undefined,
582
+ mathIndex: undefined,
583
583
 
584
584
  // Academic benchmarks
585
- mmluPro: 0.759,
586
- gpqa: 0.625,
587
- hle: 0.064,
585
+ mmluPro: undefined,
586
+ gpqa: 0.542,
587
+ hle: 0.044,
588
588
 
589
589
  // Capabilities
590
590
  contextWindow: 8192,
@@ -592,18 +592,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
592
592
  supportsVision: false,
593
593
 
594
594
  // Metadata
595
- lastUpdated: "2026-06-01",
596
- originalModel: "Gemini 2.5 Flash-Lite (Reasoning)",
595
+ lastUpdated: "2026-07-01",
596
+ originalModel: "Gemini 2.0 Flash-Lite (Preview)",
597
597
  },
598
- "gemini-2.0-flash-thinking-experimental-dec-24": {
598
+ "gemini-2.0-flash-feb-25": {
599
599
  // AA specific benchmarks
600
600
  codingIndex: undefined,
601
- mathIndex: undefined,
601
+ mathIndex: 21.7,
602
602
 
603
603
  // Academic benchmarks
604
- mmluPro: undefined,
605
- gpqa: undefined,
606
- hle: undefined,
604
+ mmluPro: 0.779,
605
+ gpqa: 0.623,
606
+ hle: 0.053,
607
607
 
608
608
  // Capabilities
609
609
  contextWindow: 8192,
@@ -611,18 +611,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
611
611
  supportsVision: false,
612
612
 
613
613
  // Metadata
614
- lastUpdated: "2026-06-01",
615
- originalModel: "Gemini 2.0 Flash Thinking Experimental (Dec '24)",
614
+ lastUpdated: "2026-07-01",
615
+ originalModel: "Gemini 2.0 Flash (Feb '25)",
616
616
  },
617
- "gemini-2.5-flash-reasoning": {
617
+ "gemini-1.5-flash-sep-24": {
618
618
  // AA specific benchmarks
619
- codingIndex: 22.2,
620
- mathIndex: 73.3,
619
+ codingIndex: undefined,
620
+ mathIndex: undefined,
621
621
 
622
622
  // Academic benchmarks
623
- mmluPro: 0.832,
624
- gpqa: 0.79,
625
- hle: 0.111,
623
+ mmluPro: 0.68,
624
+ gpqa: 0.463,
625
+ hle: 0.035,
626
626
 
627
627
  // Capabilities
628
628
  contextWindow: 8192,
@@ -630,18 +630,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
630
630
  supportsVision: false,
631
631
 
632
632
  // Metadata
633
- lastUpdated: "2026-06-01",
634
- originalModel: "Gemini 2.5 Flash (Reasoning)",
633
+ lastUpdated: "2026-07-01",
634
+ originalModel: "Gemini 1.5 Flash (Sep '24)",
635
635
  },
636
- "gemini-1.5-flash-may-24": {
636
+ "gemini-1.5-flash-8b": {
637
637
  // AA specific benchmarks
638
638
  codingIndex: undefined,
639
639
  mathIndex: undefined,
640
640
 
641
641
  // Academic benchmarks
642
- mmluPro: 0.574,
643
- gpqa: 0.324,
644
- hle: 0.042,
642
+ mmluPro: 0.569,
643
+ gpqa: 0.359,
644
+ hle: 0.045,
645
645
 
646
646
  // Capabilities
647
647
  contextWindow: 8192,
@@ -649,18 +649,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
649
649
  supportsVision: false,
650
650
 
651
651
  // Metadata
652
- lastUpdated: "2026-06-01",
653
- originalModel: "Gemini 1.5 Flash (May '24)",
652
+ lastUpdated: "2026-07-01",
653
+ originalModel: "Gemini 1.5 Flash-8B",
654
654
  },
655
- "gemini-2.5-flash-preview-reasoning": {
655
+ "gemma-3-27b-instruct": {
656
656
  // AA specific benchmarks
657
- codingIndex: undefined,
658
- mathIndex: undefined,
657
+ codingIndex: 10.1,
658
+ mathIndex: 20.7,
659
659
 
660
660
  // Academic benchmarks
661
- mmluPro: 0.8,
662
- gpqa: 0.698,
663
- hle: 0.116,
661
+ mmluPro: 0.669,
662
+ gpqa: 0.428,
663
+ hle: 0.047,
664
664
 
665
665
  // Capabilities
666
666
  contextWindow: 8192,
@@ -668,18 +668,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
668
668
  supportsVision: false,
669
669
 
670
670
  // Metadata
671
- lastUpdated: "2026-06-01",
672
- originalModel: "Gemini 2.5 Flash Preview (Reasoning)",
671
+ lastUpdated: "2026-07-01",
672
+ originalModel: "Gemma 3 27B Instruct",
673
673
  },
674
- "gemini-2.5-flash-lite-preview-sep-25-reasoning": {
674
+ "palm-2": {
675
675
  // AA specific benchmarks
676
- codingIndex: 18.2,
677
- mathIndex: 68.7,
676
+ codingIndex: 4.6,
677
+ mathIndex: undefined,
678
678
 
679
679
  // Academic benchmarks
680
- mmluPro: 0.808,
681
- gpqa: 0.709,
682
- hle: 0.066,
680
+ mmluPro: undefined,
681
+ gpqa: undefined,
682
+ hle: undefined,
683
683
 
684
684
  // Capabilities
685
685
  contextWindow: 8192,
@@ -687,18 +687,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
687
687
  supportsVision: false,
688
688
 
689
689
  // Metadata
690
- lastUpdated: "2026-06-01",
691
- originalModel: "Gemini 2.5 Flash-Lite Preview (Sep '25) (Reasoning)",
690
+ lastUpdated: "2026-07-01",
691
+ originalModel: "PALM-2",
692
692
  },
693
- "gemini-2.5-flash-preview-sep-25-non-reasoning": {
693
+ "gemini-3-pro-preview-low": {
694
694
  // AA specific benchmarks
695
- codingIndex: 22.1,
696
- mathIndex: 56.7,
695
+ codingIndex: undefined,
696
+ mathIndex: 86.7,
697
697
 
698
698
  // Academic benchmarks
699
- mmluPro: 0.836,
700
- gpqa: 0.766,
701
- hle: 0.078,
699
+ mmluPro: 0.895,
700
+ gpqa: 0.887,
701
+ hle: 0.276,
702
702
 
703
703
  // Capabilities
704
704
  contextWindow: 8192,
@@ -706,18 +706,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
706
706
  supportsVision: false,
707
707
 
708
708
  // Metadata
709
- lastUpdated: "2026-06-01",
710
- originalModel: "Gemini 2.5 Flash Preview (Sep '25) (Non-reasoning)",
709
+ lastUpdated: "2026-07-01",
710
+ originalModel: "Gemini 3 Pro Preview (low)",
711
711
  },
712
- "gemini-2.5-flash-preview-non-reasoning": {
712
+ "gemini-2.0-flash-thinking-experimental-dec-24": {
713
713
  // AA specific benchmarks
714
714
  codingIndex: undefined,
715
715
  mathIndex: undefined,
716
716
 
717
717
  // Academic benchmarks
718
- mmluPro: 0.783,
719
- gpqa: 0.594,
720
- hle: 0.05,
718
+ mmluPro: undefined,
719
+ gpqa: undefined,
720
+ hle: undefined,
721
721
 
722
722
  // Capabilities
723
723
  contextWindow: 8192,
@@ -725,18 +725,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
725
725
  supportsVision: false,
726
726
 
727
727
  // Metadata
728
- lastUpdated: "2026-06-01",
729
- originalModel: "Gemini 2.5 Flash Preview (Non-reasoning)",
728
+ lastUpdated: "2026-07-01",
729
+ originalModel: "Gemini 2.0 Flash Thinking Experimental (Dec '24)",
730
730
  },
731
- "gemini-2.5-pro-preview-mar-25": {
731
+ "gemini-2.5-flash-preview-non-reasoning": {
732
732
  // AA specific benchmarks
733
- codingIndex: 46.7,
733
+ codingIndex: undefined,
734
734
  mathIndex: undefined,
735
735
 
736
736
  // Academic benchmarks
737
- mmluPro: 0.858,
738
- gpqa: 0.836,
739
- hle: 0.171,
737
+ mmluPro: 0.783,
738
+ gpqa: 0.594,
739
+ hle: 0.05,
740
740
 
741
741
  // Capabilities
742
742
  contextWindow: 8192,
@@ -744,18 +744,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
744
744
  supportsVision: false,
745
745
 
746
746
  // Metadata
747
- lastUpdated: "2026-06-01",
748
- originalModel: "Gemini 2.5 Pro Preview (Mar' 25)",
747
+ lastUpdated: "2026-07-01",
748
+ originalModel: "Gemini 2.5 Flash Preview (Non-reasoning)",
749
749
  },
750
- "gemini-1.0-pro": {
750
+ "gemma-3-4b-instruct": {
751
751
  // AA specific benchmarks
752
752
  codingIndex: undefined,
753
- mathIndex: undefined,
753
+ mathIndex: 12.7,
754
754
 
755
755
  // Academic benchmarks
756
- mmluPro: 0.431,
757
- gpqa: 0.277,
758
- hle: 0.046,
756
+ mmluPro: 0.417,
757
+ gpqa: 0.291,
758
+ hle: 0.052,
759
759
 
760
760
  // Capabilities
761
761
  contextWindow: 8192,
@@ -763,18 +763,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
763
763
  supportsVision: false,
764
764
 
765
765
  // Metadata
766
- lastUpdated: "2026-06-01",
767
- originalModel: "Gemini 1.0 Pro",
766
+ lastUpdated: "2026-07-01",
767
+ originalModel: "Gemma 3 4B Instruct",
768
768
  },
769
- "gemini-2.5-flash-non-reasoning": {
769
+ "gemini-2.5-flash-preview-reasoning": {
770
770
  // AA specific benchmarks
771
- codingIndex: 17.8,
772
- mathIndex: 60.3,
771
+ codingIndex: undefined,
772
+ mathIndex: undefined,
773
773
 
774
774
  // Academic benchmarks
775
- mmluPro: 0.809,
776
- gpqa: 0.683,
777
- hle: 0.051,
775
+ mmluPro: 0.8,
776
+ gpqa: 0.698,
777
+ hle: 0.116,
778
778
 
779
779
  // Capabilities
780
780
  contextWindow: 8192,
@@ -782,18 +782,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
782
782
  supportsVision: false,
783
783
 
784
784
  // Metadata
785
- lastUpdated: "2026-06-01",
786
- originalModel: "Gemini 2.5 Flash (Non-reasoning)",
785
+ lastUpdated: "2026-07-01",
786
+ originalModel: "Gemini 2.5 Flash Preview (Reasoning)",
787
787
  },
788
- "gemini-2.5-flash-lite-preview-sep-25-non-reasoning": {
788
+ "gemma-3-1b-instruct": {
789
789
  // AA specific benchmarks
790
- codingIndex: 14.5,
791
- mathIndex: 46.7,
790
+ codingIndex: undefined,
791
+ mathIndex: 3.3,
792
792
 
793
793
  // Academic benchmarks
794
- mmluPro: 0.796,
795
- gpqa: 0.651,
796
- hle: 0.046,
794
+ mmluPro: 0.135,
795
+ gpqa: 0.237,
796
+ hle: 0.052,
797
797
 
798
798
  // Capabilities
799
799
  contextWindow: 8192,
@@ -801,18 +801,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
801
801
  supportsVision: false,
802
802
 
803
803
  // Metadata
804
- lastUpdated: "2026-06-01",
805
- originalModel: "Gemini 2.5 Flash-Lite Preview (Sep '25) (Non-reasoning)",
804
+ lastUpdated: "2026-07-01",
805
+ originalModel: "Gemma 3 1B Instruct",
806
806
  },
807
- "gemini-3-flash-preview-non-reasoning": {
807
+ "gemini-1.0-pro": {
808
808
  // AA specific benchmarks
809
- codingIndex: 37.8,
810
- mathIndex: 55.7,
809
+ codingIndex: undefined,
810
+ mathIndex: undefined,
811
811
 
812
812
  // Academic benchmarks
813
- mmluPro: 0.882,
814
- gpqa: 0.812,
815
- hle: 0.141,
813
+ mmluPro: 0.431,
814
+ gpqa: 0.277,
815
+ hle: 0.046,
816
816
 
817
817
  // Capabilities
818
818
  contextWindow: 8192,
@@ -820,18 +820,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
820
820
  supportsVision: false,
821
821
 
822
822
  // Metadata
823
- lastUpdated: "2026-06-01",
824
- originalModel: "Gemini 3 Flash Preview (Non-reasoning)",
823
+ lastUpdated: "2026-07-01",
824
+ originalModel: "Gemini 1.0 Pro",
825
825
  },
826
- "gemini-2.0-flash-lite-feb-25": {
826
+ "gemma-3-12b-instruct": {
827
827
  // AA specific benchmarks
828
- codingIndex: undefined,
829
- mathIndex: undefined,
828
+ codingIndex: 5.8,
829
+ mathIndex: 18.3,
830
830
 
831
831
  // Academic benchmarks
832
- mmluPro: 0.724,
833
- gpqa: 0.535,
834
- hle: 0.036,
832
+ mmluPro: 0.595,
833
+ gpqa: 0.349,
834
+ hle: 0.048,
835
835
 
836
836
  // Capabilities
837
837
  contextWindow: 8192,
@@ -839,18 +839,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
839
839
  supportsVision: false,
840
840
 
841
841
  // Metadata
842
- lastUpdated: "2026-06-01",
843
- originalModel: "Gemini 2.0 Flash-Lite (Feb '25)",
842
+ lastUpdated: "2026-07-01",
843
+ originalModel: "Gemma 3 12B Instruct",
844
844
  },
845
- "gemini-3-flash-preview-reasoning": {
845
+ "gemini-3-pro-preview-high": {
846
846
  // AA specific benchmarks
847
- codingIndex: 42.6,
848
- mathIndex: 97,
847
+ codingIndex: undefined,
848
+ mathIndex: 95.7,
849
849
 
850
850
  // Academic benchmarks
851
- mmluPro: 0.89,
852
- gpqa: 0.898,
853
- hle: 0.347,
851
+ mmluPro: 0.898,
852
+ gpqa: 0.908,
853
+ hle: 0.372,
854
854
 
855
855
  // Capabilities
856
856
  contextWindow: 8192,
@@ -858,18 +858,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
858
858
  supportsVision: false,
859
859
 
860
860
  // Metadata
861
- lastUpdated: "2026-06-01",
862
- originalModel: "Gemini 3 Flash Preview (Reasoning)",
861
+ lastUpdated: "2026-07-01",
862
+ originalModel: "Gemini 3 Pro Preview (high)",
863
863
  },
864
- "gemini-3-pro-preview-low": {
864
+ "gemini-2.5-flash-preview-sep-25-reasoning": {
865
865
  // AA specific benchmarks
866
- codingIndex: 39.4,
867
- mathIndex: 86.7,
866
+ codingIndex: undefined,
867
+ mathIndex: 78.3,
868
868
 
869
869
  // Academic benchmarks
870
- mmluPro: 0.895,
871
- gpqa: 0.887,
872
- hle: 0.276,
870
+ mmluPro: 0.842,
871
+ gpqa: 0.793,
872
+ hle: 0.127,
873
873
 
874
874
  // Capabilities
875
875
  contextWindow: 8192,
@@ -877,18 +877,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
877
877
  supportsVision: false,
878
878
 
879
879
  // Metadata
880
- lastUpdated: "2026-06-01",
881
- originalModel: "Gemini 3 Pro Preview (low)",
880
+ lastUpdated: "2026-07-01",
881
+ originalModel: "Gemini 2.5 Flash Preview (Sep '25) (Reasoning)",
882
882
  },
883
- "palm-2": {
883
+ "gemma-3n-e4b-instruct-preview-may-25": {
884
884
  // AA specific benchmarks
885
- codingIndex: 4.6,
885
+ codingIndex: undefined,
886
886
  mathIndex: undefined,
887
887
 
888
888
  // Academic benchmarks
889
- mmluPro: undefined,
890
- gpqa: undefined,
891
- hle: undefined,
889
+ mmluPro: 0.483,
890
+ gpqa: 0.278,
891
+ hle: 0.049,
892
892
 
893
893
  // Capabilities
894
894
  contextWindow: 8192,
@@ -896,18 +896,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
896
896
  supportsVision: false,
897
897
 
898
898
  // Metadata
899
- lastUpdated: "2026-06-01",
900
- originalModel: "PALM-2",
899
+ lastUpdated: "2026-07-01",
900
+ originalModel: "Gemma 3n E4B Instruct Preview (May '25)",
901
901
  },
902
- "claude-3.5-sonnet-oct-24": {
902
+ "gemini-1.0-ultra": {
903
903
  // AA specific benchmarks
904
- codingIndex: 30.2,
904
+ codingIndex: 17.6,
905
905
  mathIndex: undefined,
906
906
 
907
907
  // Academic benchmarks
908
- mmluPro: 0.772,
909
- gpqa: 0.599,
910
- hle: 0.039,
908
+ mmluPro: undefined,
909
+ gpqa: undefined,
910
+ hle: undefined,
911
911
 
912
912
  // Capabilities
913
913
  contextWindow: 8192,
@@ -915,18 +915,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
915
915
  supportsVision: false,
916
916
 
917
917
  // Metadata
918
- lastUpdated: "2026-06-01",
919
- originalModel: "Claude 3.5 Sonnet (Oct '24)",
918
+ lastUpdated: "2026-07-01",
919
+ originalModel: "Gemini 1.0 Ultra",
920
920
  },
921
- "claude-3.5-sonnet-june-24": {
921
+ "gemini-2.0-flash-thinking-experimental-jan-25": {
922
922
  // AA specific benchmarks
923
- codingIndex: 26,
923
+ codingIndex: 24.1,
924
924
  mathIndex: undefined,
925
925
 
926
926
  // Academic benchmarks
927
- mmluPro: 0.751,
928
- gpqa: 0.56,
929
- hle: 0.037,
927
+ mmluPro: 0.798,
928
+ gpqa: 0.701,
929
+ hle: 0.071,
930
930
 
931
931
  // Capabilities
932
932
  contextWindow: 8192,
@@ -934,18 +934,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
934
934
  supportsVision: false,
935
935
 
936
936
  // Metadata
937
- lastUpdated: "2026-06-01",
938
- originalModel: "Claude 3.5 Sonnet (June '24)",
937
+ lastUpdated: "2026-07-01",
938
+ originalModel: "Gemini 2.0 Flash Thinking Experimental (Jan '25)",
939
939
  },
940
- "claude-3-opus": {
940
+ "gemma-3n-e4b-instruct": {
941
941
  // AA specific benchmarks
942
- codingIndex: 19.5,
943
- mathIndex: undefined,
942
+ codingIndex: 3.2,
943
+ mathIndex: 14.3,
944
944
 
945
945
  // Academic benchmarks
946
- mmluPro: 0.696,
947
- gpqa: 0.489,
948
- hle: 0.031,
946
+ mmluPro: 0.488,
947
+ gpqa: 0.296,
948
+ hle: 0.044,
949
949
 
950
950
  // Capabilities
951
951
  contextWindow: 8192,
@@ -953,18 +953,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
953
953
  supportsVision: false,
954
954
 
955
955
  // Metadata
956
- lastUpdated: "2026-06-01",
957
- originalModel: "Claude 3 Opus",
956
+ lastUpdated: "2026-07-01",
957
+ originalModel: "Gemma 3n E4B Instruct",
958
958
  },
959
- "claude-3.5-haiku": {
959
+ "gemma-3n-e2b-instruct": {
960
960
  // AA specific benchmarks
961
- codingIndex: 10.7,
962
- mathIndex: undefined,
961
+ codingIndex: undefined,
962
+ mathIndex: 10.3,
963
963
 
964
964
  // Academic benchmarks
965
- mmluPro: 0.634,
966
- gpqa: 0.408,
967
- hle: 0.035,
965
+ mmluPro: 0.378,
966
+ gpqa: 0.229,
967
+ hle: 0.04,
968
968
 
969
969
  // Capabilities
970
970
  contextWindow: 8192,
@@ -972,18 +972,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
972
972
  supportsVision: false,
973
973
 
974
974
  // Metadata
975
- lastUpdated: "2026-06-01",
976
- originalModel: "Claude 3.5 Haiku",
975
+ lastUpdated: "2026-07-01",
976
+ originalModel: "Gemma 3n E2B Instruct",
977
977
  },
978
- "claude-3-sonnet": {
978
+ "gemini-1.5-flash-may-24": {
979
979
  // AA specific benchmarks
980
980
  codingIndex: undefined,
981
981
  mathIndex: undefined,
982
982
 
983
983
  // Academic benchmarks
984
- mmluPro: 0.579,
985
- gpqa: 0.4,
986
- hle: 0.038,
984
+ mmluPro: 0.574,
985
+ gpqa: 0.324,
986
+ hle: 0.042,
987
987
 
988
988
  // Capabilities
989
989
  contextWindow: 8192,
@@ -991,18 +991,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
991
991
  supportsVision: false,
992
992
 
993
993
  // Metadata
994
- lastUpdated: "2026-06-01",
995
- originalModel: "Claude 3 Sonnet",
994
+ lastUpdated: "2026-07-01",
995
+ originalModel: "Gemini 1.5 Flash (May '24)",
996
996
  },
997
- "claude-3-haiku": {
997
+ "gemini-2.5-pro-preview-may-25": {
998
998
  // AA specific benchmarks
999
- codingIndex: 6.7,
999
+ codingIndex: undefined,
1000
1000
  mathIndex: undefined,
1001
1001
 
1002
1002
  // Academic benchmarks
1003
- mmluPro: undefined,
1004
- gpqa: 0.374,
1005
- hle: 0.039,
1003
+ mmluPro: 0.837,
1004
+ gpqa: 0.822,
1005
+ hle: 0.154,
1006
1006
 
1007
1007
  // Capabilities
1008
1008
  contextWindow: 8192,
@@ -1010,18 +1010,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1010
1010
  supportsVision: false,
1011
1011
 
1012
1012
  // Metadata
1013
- lastUpdated: "2026-06-01",
1014
- originalModel: "Claude 3 Haiku",
1013
+ lastUpdated: "2026-07-01",
1014
+ originalModel: "Gemini 2.5 Pro Preview (May' 25)",
1015
1015
  },
1016
- "claude-instant": {
1016
+ "gemini-2.5-flash-non-reasoning": {
1017
1017
  // AA specific benchmarks
1018
- codingIndex: 7.8,
1019
- mathIndex: undefined,
1018
+ codingIndex: undefined,
1019
+ mathIndex: 60.3,
1020
1020
 
1021
1021
  // Academic benchmarks
1022
- mmluPro: 0.434,
1023
- gpqa: 0.33,
1024
- hle: 0.038,
1022
+ mmluPro: 0.809,
1023
+ gpqa: 0.683,
1024
+ hle: 0.051,
1025
1025
 
1026
1026
  // Capabilities
1027
1027
  contextWindow: 8192,
@@ -1029,18 +1029,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1029
1029
  supportsVision: false,
1030
1030
 
1031
1031
  // Metadata
1032
- lastUpdated: "2026-06-01",
1033
- originalModel: "Claude Instant",
1032
+ lastUpdated: "2026-07-01",
1033
+ originalModel: "Gemini 2.5 Flash (Non-reasoning)",
1034
1034
  },
1035
- "claude-4.5-sonnet-reasoning": {
1035
+ "gemini-3-flash-preview-non-reasoning": {
1036
1036
  // AA specific benchmarks
1037
- codingIndex: 38.6,
1038
- mathIndex: 88,
1037
+ codingIndex: undefined,
1038
+ mathIndex: 55.7,
1039
1039
 
1040
1040
  // Academic benchmarks
1041
- mmluPro: 0.875,
1042
- gpqa: 0.834,
1043
- hle: 0.173,
1041
+ mmluPro: 0.882,
1042
+ gpqa: 0.812,
1043
+ hle: 0.141,
1044
1044
 
1045
1045
  // Capabilities
1046
1046
  contextWindow: 8192,
@@ -1048,18 +1048,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1048
1048
  supportsVision: false,
1049
1049
 
1050
1050
  // Metadata
1051
- lastUpdated: "2026-06-01",
1052
- originalModel: "Claude 4.5 Sonnet (Reasoning)",
1051
+ lastUpdated: "2026-07-01",
1052
+ originalModel: "Gemini 3 Flash Preview (Non-reasoning)",
1053
1053
  },
1054
- "claude-4-sonnet-non-reasoning": {
1054
+ "gemini-2.0-flash-lite-feb-25": {
1055
1055
  // AA specific benchmarks
1056
- codingIndex: 30.6,
1057
- mathIndex: 38,
1056
+ codingIndex: undefined,
1057
+ mathIndex: undefined,
1058
1058
 
1059
1059
  // Academic benchmarks
1060
- mmluPro: 0.837,
1061
- gpqa: 0.683,
1062
- hle: 0.04,
1060
+ mmluPro: 0.724,
1061
+ gpqa: 0.535,
1062
+ hle: 0.036,
1063
1063
 
1064
1064
  // Capabilities
1065
1065
  contextWindow: 8192,
@@ -1067,18 +1067,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1067
1067
  supportsVision: false,
1068
1068
 
1069
1069
  // Metadata
1070
- lastUpdated: "2026-06-01",
1071
- originalModel: "Claude 4 Sonnet (Non-reasoning)",
1070
+ lastUpdated: "2026-07-01",
1071
+ originalModel: "Gemini 2.0 Flash-Lite (Feb '25)",
1072
1072
  },
1073
- "claude-3.7-sonnet-non-reasoning": {
1073
+ "gemini-3-flash-preview-reasoning": {
1074
1074
  // AA specific benchmarks
1075
- codingIndex: 26.7,
1076
- mathIndex: 21,
1075
+ codingIndex: undefined,
1076
+ mathIndex: 97,
1077
1077
 
1078
1078
  // Academic benchmarks
1079
- mmluPro: 0.803,
1080
- gpqa: 0.656,
1081
- hle: 0.048,
1079
+ mmluPro: 0.89,
1080
+ gpqa: 0.898,
1081
+ hle: 0.347,
1082
1082
 
1083
1083
  // Capabilities
1084
1084
  contextWindow: 8192,
@@ -1086,18 +1086,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1086
1086
  supportsVision: false,
1087
1087
 
1088
1088
  // Metadata
1089
- lastUpdated: "2026-06-01",
1090
- originalModel: "Claude 3.7 Sonnet (Non-reasoning)",
1089
+ lastUpdated: "2026-07-01",
1090
+ originalModel: "Gemini 3 Flash Preview (Reasoning)",
1091
1091
  },
1092
- "claude-2.0": {
1092
+ "gemini-1.5-pro-may-24": {
1093
1093
  // AA specific benchmarks
1094
- codingIndex: 12.9,
1094
+ codingIndex: 19.8,
1095
1095
  mathIndex: undefined,
1096
1096
 
1097
1097
  // Academic benchmarks
1098
- mmluPro: 0.486,
1099
- gpqa: 0.344,
1100
- hle: undefined,
1098
+ mmluPro: 0.657,
1099
+ gpqa: 0.371,
1100
+ hle: 0.039,
1101
1101
 
1102
1102
  // Capabilities
1103
1103
  contextWindow: 8192,
@@ -1105,18 +1105,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1105
1105
  supportsVision: false,
1106
1106
 
1107
1107
  // Metadata
1108
- lastUpdated: "2026-06-01",
1109
- originalModel: "Claude 2.0",
1108
+ lastUpdated: "2026-07-01",
1109
+ originalModel: "Gemini 1.5 Pro (May '24)",
1110
1110
  },
1111
- "claude-4.1-opus-non-reasoning": {
1111
+ "gemini-2.5-pro-preview-mar-25": {
1112
1112
  // AA specific benchmarks
1113
- codingIndex: undefined,
1113
+ codingIndex: 46.7,
1114
1114
  mathIndex: undefined,
1115
1115
 
1116
1116
  // Academic benchmarks
1117
- mmluPro: undefined,
1118
- gpqa: undefined,
1119
- hle: undefined,
1117
+ mmluPro: 0.858,
1118
+ gpqa: 0.836,
1119
+ hle: 0.171,
1120
1120
 
1121
1121
  // Capabilities
1122
1122
  contextWindow: 8192,
@@ -1124,18 +1124,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1124
1124
  supportsVision: false,
1125
1125
 
1126
1126
  // Metadata
1127
- lastUpdated: "2026-06-01",
1128
- originalModel: "Claude 4.1 Opus (Non-reasoning)",
1127
+ lastUpdated: "2026-07-01",
1128
+ originalModel: "Gemini 2.5 Pro Preview (Mar' 25)",
1129
1129
  },
1130
- "claude-4.1-opus-reasoning": {
1130
+ "gemini-2.5-flash-lite-non-reasoning": {
1131
1131
  // AA specific benchmarks
1132
- codingIndex: 36.5,
1133
- mathIndex: 80.3,
1132
+ codingIndex: undefined,
1133
+ mathIndex: 35.3,
1134
1134
 
1135
1135
  // Academic benchmarks
1136
- mmluPro: 0.88,
1137
- gpqa: 0.809,
1138
- hle: 0.119,
1136
+ mmluPro: 0.724,
1137
+ gpqa: 0.474,
1138
+ hle: 0.037,
1139
1139
 
1140
1140
  // Capabilities
1141
1141
  contextWindow: 8192,
@@ -1143,18 +1143,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1143
1143
  supportsVision: false,
1144
1144
 
1145
1145
  // Metadata
1146
- lastUpdated: "2026-06-01",
1147
- originalModel: "Claude 4.1 Opus (Reasoning)",
1146
+ lastUpdated: "2026-07-01",
1147
+ originalModel: "Gemini 2.5 Flash-Lite (Non-reasoning)",
1148
1148
  },
1149
- "claude-4.5-sonnet-non-reasoning": {
1149
+ "gemini-2.5-flash-reasoning": {
1150
1150
  // AA specific benchmarks
1151
- codingIndex: 33.5,
1152
- mathIndex: 37,
1151
+ codingIndex: undefined,
1152
+ mathIndex: 73.3,
1153
1153
 
1154
1154
  // Academic benchmarks
1155
- mmluPro: 0.86,
1156
- gpqa: 0.727,
1157
- hle: 0.071,
1155
+ mmluPro: 0.832,
1156
+ gpqa: 0.79,
1157
+ hle: 0.111,
1158
1158
 
1159
1159
  // Capabilities
1160
1160
  contextWindow: 8192,
@@ -1162,18 +1162,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1162
1162
  supportsVision: false,
1163
1163
 
1164
1164
  // Metadata
1165
- lastUpdated: "2026-06-01",
1166
- originalModel: "Claude 4.5 Sonnet (Non-reasoning)",
1165
+ lastUpdated: "2026-07-01",
1166
+ originalModel: "Gemini 2.5 Flash (Reasoning)",
1167
1167
  },
1168
- "claude-opus-4.6-adaptive-reasoning-max-effort": {
1168
+ "gemini-2.5-flash-lite-preview-sep-25-reasoning": {
1169
1169
  // AA specific benchmarks
1170
- codingIndex: 48.1,
1171
- mathIndex: undefined,
1170
+ codingIndex: undefined,
1171
+ mathIndex: 68.7,
1172
1172
 
1173
1173
  // Academic benchmarks
1174
- mmluPro: undefined,
1175
- gpqa: 0.896,
1176
- hle: 0.367,
1174
+ mmluPro: 0.808,
1175
+ gpqa: 0.709,
1176
+ hle: 0.066,
1177
1177
 
1178
1178
  // Capabilities
1179
1179
  contextWindow: 8192,
@@ -1181,18 +1181,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1181
1181
  supportsVision: false,
1182
1182
 
1183
1183
  // Metadata
1184
- lastUpdated: "2026-06-01",
1185
- originalModel: "Claude Opus 4.6 (Adaptive Reasoning, Max Effort)",
1184
+ lastUpdated: "2026-07-01",
1185
+ originalModel: "Gemini 2.5 Flash-Lite Preview (Sep '25) (Reasoning)",
1186
1186
  },
1187
- "claude-opus-4.5-reasoning": {
1187
+ "gemini-2.5-flash-preview-sep-25-non-reasoning": {
1188
1188
  // AA specific benchmarks
1189
- codingIndex: 47.8,
1190
- mathIndex: 91.3,
1189
+ codingIndex: undefined,
1190
+ mathIndex: 56.7,
1191
1191
 
1192
1192
  // Academic benchmarks
1193
- mmluPro: 0.895,
1194
- gpqa: 0.866,
1195
- hle: 0.284,
1193
+ mmluPro: 0.836,
1194
+ gpqa: 0.766,
1195
+ hle: 0.078,
1196
1196
 
1197
1197
  // Capabilities
1198
1198
  contextWindow: 8192,
@@ -1200,18 +1200,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1200
1200
  supportsVision: false,
1201
1201
 
1202
1202
  // Metadata
1203
- lastUpdated: "2026-06-01",
1204
- originalModel: "Claude Opus 4.5 (Reasoning)",
1203
+ lastUpdated: "2026-07-01",
1204
+ originalModel: "Gemini 2.5 Flash Preview (Sep '25) (Non-reasoning)",
1205
1205
  },
1206
- "claude-3.7-sonnet-reasoning": {
1206
+ "gemini-2.5-flash-lite-reasoning": {
1207
1207
  // AA specific benchmarks
1208
- codingIndex: 27.6,
1209
- mathIndex: 56.3,
1208
+ codingIndex: undefined,
1209
+ mathIndex: 53.3,
1210
1210
 
1211
1211
  // Academic benchmarks
1212
- mmluPro: 0.837,
1213
- gpqa: 0.772,
1214
- hle: 0.103,
1212
+ mmluPro: 0.759,
1213
+ gpqa: 0.625,
1214
+ hle: 0.064,
1215
1215
 
1216
1216
  // Capabilities
1217
1217
  contextWindow: 8192,
@@ -1219,18 +1219,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1219
1219
  supportsVision: false,
1220
1220
 
1221
1221
  // Metadata
1222
- lastUpdated: "2026-06-01",
1223
- originalModel: "Claude 3.7 Sonnet (Reasoning)",
1222
+ lastUpdated: "2026-07-01",
1223
+ originalModel: "Gemini 2.5 Flash-Lite (Reasoning)",
1224
1224
  },
1225
- "claude-opus-4.6-non-reasoning-high-effort": {
1225
+ "gemini-2.5-flash-lite-preview-sep-25-non-reasoning": {
1226
1226
  // AA specific benchmarks
1227
- codingIndex: 47.6,
1228
- mathIndex: undefined,
1227
+ codingIndex: undefined,
1228
+ mathIndex: 46.7,
1229
1229
 
1230
1230
  // Academic benchmarks
1231
- mmluPro: undefined,
1232
- gpqa: 0.84,
1233
- hle: 0.186,
1231
+ mmluPro: 0.796,
1232
+ gpqa: 0.651,
1233
+ hle: 0.046,
1234
1234
 
1235
1235
  // Capabilities
1236
1236
  contextWindow: 8192,
@@ -1238,18 +1238,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1238
1238
  supportsVision: false,
1239
1239
 
1240
1240
  // Metadata
1241
- lastUpdated: "2026-06-01",
1242
- originalModel: "Claude Opus 4.6 (Non-reasoning, High Effort)",
1241
+ lastUpdated: "2026-07-01",
1242
+ originalModel: "Gemini 2.5 Flash-Lite Preview (Sep '25) (Non-reasoning)",
1243
1243
  },
1244
- "claude-4-opus-non-reasoning": {
1244
+ "claude-3.5-sonnet-oct-24": {
1245
1245
  // AA specific benchmarks
1246
- codingIndex: undefined,
1247
- mathIndex: 36.3,
1246
+ codingIndex: 30.2,
1247
+ mathIndex: undefined,
1248
1248
 
1249
1249
  // Academic benchmarks
1250
- mmluPro: 0.86,
1251
- gpqa: 0.701,
1252
- hle: 0.059,
1250
+ mmluPro: 0.772,
1251
+ gpqa: 0.599,
1252
+ hle: 0.039,
1253
1253
 
1254
1254
  // Capabilities
1255
1255
  contextWindow: 8192,
@@ -1257,18 +1257,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1257
1257
  supportsVision: false,
1258
1258
 
1259
1259
  // Metadata
1260
- lastUpdated: "2026-06-01",
1261
- originalModel: "Claude 4 Opus (Non-reasoning)",
1260
+ lastUpdated: "2026-07-01",
1261
+ originalModel: "Claude 3.5 Sonnet (Oct '24)",
1262
1262
  },
1263
- "claude-opus-4.5-non-reasoning": {
1263
+ "claude-3.5-sonnet-june-24": {
1264
1264
  // AA specific benchmarks
1265
- codingIndex: 42.9,
1266
- mathIndex: 62.7,
1265
+ codingIndex: 26,
1266
+ mathIndex: undefined,
1267
1267
 
1268
1268
  // Academic benchmarks
1269
- mmluPro: 0.889,
1270
- gpqa: 0.81,
1271
- hle: 0.129,
1269
+ mmluPro: 0.751,
1270
+ gpqa: 0.56,
1271
+ hle: 0.037,
1272
1272
 
1273
1273
  // Capabilities
1274
1274
  contextWindow: 8192,
@@ -1276,18 +1276,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1276
1276
  supportsVision: false,
1277
1277
 
1278
1278
  // Metadata
1279
- lastUpdated: "2026-06-01",
1280
- originalModel: "Claude Opus 4.5 (Non-reasoning)",
1279
+ lastUpdated: "2026-07-01",
1280
+ originalModel: "Claude 3.5 Sonnet (June '24)",
1281
1281
  },
1282
- "claude-4-sonnet-reasoning": {
1282
+ "claude-3-opus": {
1283
1283
  // AA specific benchmarks
1284
- codingIndex: 34.1,
1285
- mathIndex: 74.3,
1284
+ codingIndex: 19.5,
1285
+ mathIndex: undefined,
1286
1286
 
1287
1287
  // Academic benchmarks
1288
- mmluPro: 0.842,
1289
- gpqa: 0.777,
1290
- hle: 0.096,
1288
+ mmluPro: 0.696,
1289
+ gpqa: 0.489,
1290
+ hle: 0.031,
1291
1291
 
1292
1292
  // Capabilities
1293
1293
  contextWindow: 8192,
@@ -1295,18 +1295,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1295
1295
  supportsVision: false,
1296
1296
 
1297
1297
  // Metadata
1298
- lastUpdated: "2026-06-01",
1299
- originalModel: "Claude 4 Sonnet (Reasoning)",
1298
+ lastUpdated: "2026-07-01",
1299
+ originalModel: "Claude 3 Opus",
1300
1300
  },
1301
- "claude-4-opus-reasoning": {
1301
+ "claude-3.5-haiku": {
1302
1302
  // AA specific benchmarks
1303
- codingIndex: 34,
1304
- mathIndex: 73.3,
1303
+ codingIndex: 15.9,
1304
+ mathIndex: undefined,
1305
1305
 
1306
1306
  // Academic benchmarks
1307
- mmluPro: 0.873,
1308
- gpqa: 0.796,
1309
- hle: 0.117,
1307
+ mmluPro: 0.634,
1308
+ gpqa: 0.408,
1309
+ hle: 0.035,
1310
1310
 
1311
1311
  // Capabilities
1312
1312
  contextWindow: 8192,
@@ -1314,18 +1314,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1314
1314
  supportsVision: false,
1315
1315
 
1316
1316
  // Metadata
1317
- lastUpdated: "2026-06-01",
1318
- originalModel: "Claude 4 Opus (Reasoning)",
1317
+ lastUpdated: "2026-07-01",
1318
+ originalModel: "Claude 3.5 Haiku",
1319
1319
  },
1320
- "claude-2.1": {
1320
+ "claude-3-sonnet": {
1321
1321
  // AA specific benchmarks
1322
- codingIndex: 14,
1322
+ codingIndex: undefined,
1323
1323
  mathIndex: undefined,
1324
1324
 
1325
1325
  // Academic benchmarks
1326
- mmluPro: 0.495,
1327
- gpqa: 0.319,
1328
- hle: 0.042,
1326
+ mmluPro: 0.579,
1327
+ gpqa: 0.4,
1328
+ hle: 0.038,
1329
1329
 
1330
1330
  // Capabilities
1331
1331
  contextWindow: 8192,
@@ -1333,18 +1333,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1333
1333
  supportsVision: false,
1334
1334
 
1335
1335
  // Metadata
1336
- lastUpdated: "2026-06-01",
1337
- originalModel: "Claude 2.1",
1336
+ lastUpdated: "2026-07-01",
1337
+ originalModel: "Claude 3 Sonnet",
1338
1338
  },
1339
- "mistral-large-2-nov-24": {
1339
+ "claude-3-haiku": {
1340
1340
  // AA specific benchmarks
1341
- codingIndex: 13.8,
1342
- mathIndex: 14,
1341
+ codingIndex: undefined,
1342
+ mathIndex: undefined,
1343
1343
 
1344
1344
  // Academic benchmarks
1345
- mmluPro: 0.697,
1346
- gpqa: 0.486,
1347
- hle: 0.04,
1345
+ mmluPro: undefined,
1346
+ gpqa: 0.374,
1347
+ hle: 0.039,
1348
1348
 
1349
1349
  // Capabilities
1350
1350
  contextWindow: 8192,
@@ -1352,18 +1352,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1352
1352
  supportsVision: false,
1353
1353
 
1354
1354
  // Metadata
1355
- lastUpdated: "2026-06-01",
1356
- originalModel: "Mistral Large 2 (Nov '24)",
1355
+ lastUpdated: "2026-07-01",
1356
+ originalModel: "Claude 3 Haiku",
1357
1357
  },
1358
- "mistral-large-2-jul-24": {
1358
+ "claude-instant": {
1359
1359
  // AA specific benchmarks
1360
- codingIndex: undefined,
1361
- mathIndex: 0,
1360
+ codingIndex: 7.8,
1361
+ mathIndex: undefined,
1362
1362
 
1363
1363
  // Academic benchmarks
1364
- mmluPro: 0.683,
1365
- gpqa: 0.472,
1366
- hle: 0.032,
1364
+ mmluPro: 0.434,
1365
+ gpqa: 0.33,
1366
+ hle: 0.038,
1367
1367
 
1368
1368
  // Capabilities
1369
1369
  contextWindow: 8192,
@@ -1371,18 +1371,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1371
1371
  supportsVision: false,
1372
1372
 
1373
1373
  // Metadata
1374
- lastUpdated: "2026-06-01",
1375
- originalModel: "Mistral Large 2 (Jul '24)",
1374
+ lastUpdated: "2026-07-01",
1375
+ originalModel: "Claude Instant",
1376
1376
  },
1377
- "pixtral-large": {
1377
+ "claude-opus-4.6-adaptive-reasoning-max-effort": {
1378
1378
  // AA specific benchmarks
1379
1379
  codingIndex: undefined,
1380
- mathIndex: 2.3,
1380
+ mathIndex: undefined,
1381
1381
 
1382
1382
  // Academic benchmarks
1383
- mmluPro: 0.701,
1384
- gpqa: 0.505,
1385
- hle: 0.036,
1383
+ mmluPro: undefined,
1384
+ gpqa: 0.896,
1385
+ hle: 0.367,
1386
1386
 
1387
1387
  // Capabilities
1388
1388
  contextWindow: 8192,
@@ -1390,18 +1390,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1390
1390
  supportsVision: false,
1391
1391
 
1392
1392
  // Metadata
1393
- lastUpdated: "2026-06-01",
1394
- originalModel: "Pixtral Large",
1393
+ lastUpdated: "2026-07-01",
1394
+ originalModel: "Claude Opus 4.6 (Adaptive Reasoning, Max Effort)",
1395
1395
  },
1396
- "mistral-small-3": {
1396
+ "claude-opus-4.5-non-reasoning": {
1397
1397
  // AA specific benchmarks
1398
1398
  codingIndex: undefined,
1399
- mathIndex: 4.3,
1399
+ mathIndex: 62.7,
1400
1400
 
1401
1401
  // Academic benchmarks
1402
- mmluPro: 0.652,
1403
- gpqa: 0.462,
1404
- hle: 0.041,
1402
+ mmluPro: 0.889,
1403
+ gpqa: 0.81,
1404
+ hle: 0.129,
1405
1405
 
1406
1406
  // Capabilities
1407
1407
  contextWindow: 8192,
@@ -1409,18 +1409,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1409
1409
  supportsVision: false,
1410
1410
 
1411
1411
  // Metadata
1412
- lastUpdated: "2026-06-01",
1413
- originalModel: "Mistral Small 3",
1412
+ lastUpdated: "2026-07-01",
1413
+ originalModel: "Claude Opus 4.5 (Non-reasoning)",
1414
1414
  },
1415
- "mistral-small-sep-24": {
1415
+ "claude-4.1-opus-non-reasoning": {
1416
1416
  // AA specific benchmarks
1417
1417
  codingIndex: undefined,
1418
1418
  mathIndex: undefined,
1419
1419
 
1420
1420
  // Academic benchmarks
1421
- mmluPro: 0.529,
1422
- gpqa: 0.381,
1423
- hle: 0.043,
1421
+ mmluPro: undefined,
1422
+ gpqa: undefined,
1423
+ hle: undefined,
1424
1424
 
1425
1425
  // Capabilities
1426
1426
  contextWindow: 8192,
@@ -1428,18 +1428,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1428
1428
  supportsVision: false,
1429
1429
 
1430
1430
  // Metadata
1431
- lastUpdated: "2026-06-01",
1432
- originalModel: "Mistral Small (Sep '24)",
1431
+ lastUpdated: "2026-07-01",
1432
+ originalModel: "Claude 4.1 Opus (Non-reasoning)",
1433
1433
  },
1434
- "mixtral-8x22b-instruct": {
1434
+ "claude-3.7-sonnet-non-reasoning": {
1435
1435
  // AA specific benchmarks
1436
1436
  codingIndex: undefined,
1437
- mathIndex: undefined,
1437
+ mathIndex: 21,
1438
1438
 
1439
1439
  // Academic benchmarks
1440
- mmluPro: 0.537,
1441
- gpqa: 0.332,
1442
- hle: 0.041,
1440
+ mmluPro: 0.803,
1441
+ gpqa: 0.656,
1442
+ hle: 0.048,
1443
1443
 
1444
1444
  // Capabilities
1445
1445
  contextWindow: 8192,
@@ -1447,18 +1447,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1447
1447
  supportsVision: false,
1448
1448
 
1449
1449
  // Metadata
1450
- lastUpdated: "2026-06-01",
1451
- originalModel: "Mixtral 8x22B Instruct",
1450
+ lastUpdated: "2026-07-01",
1451
+ originalModel: "Claude 3.7 Sonnet (Non-reasoning)",
1452
1452
  },
1453
- "mistral-small-feb-24": {
1453
+ "claude-4-sonnet-non-reasoning": {
1454
1454
  // AA specific benchmarks
1455
1455
  codingIndex: undefined,
1456
- mathIndex: undefined,
1456
+ mathIndex: 38,
1457
1457
 
1458
1458
  // Academic benchmarks
1459
- mmluPro: 0.419,
1460
- gpqa: 0.302,
1461
- hle: 0.044,
1459
+ mmluPro: 0.837,
1460
+ gpqa: 0.683,
1461
+ hle: 0.04,
1462
1462
 
1463
1463
  // Capabilities
1464
1464
  contextWindow: 8192,
@@ -1466,18 +1466,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1466
1466
  supportsVision: false,
1467
1467
 
1468
1468
  // Metadata
1469
- lastUpdated: "2026-06-01",
1470
- originalModel: "Mistral Small (Feb '24)",
1469
+ lastUpdated: "2026-07-01",
1470
+ originalModel: "Claude 4 Sonnet (Non-reasoning)",
1471
1471
  },
1472
- "mistral-large-feb-24": {
1472
+ "claude-4.1-opus-reasoning": {
1473
1473
  // AA specific benchmarks
1474
1474
  codingIndex: undefined,
1475
- mathIndex: undefined,
1475
+ mathIndex: 80.3,
1476
1476
 
1477
1477
  // Academic benchmarks
1478
- mmluPro: 0.515,
1479
- gpqa: 0.351,
1480
- hle: 0.034,
1478
+ mmluPro: 0.88,
1479
+ gpqa: 0.809,
1480
+ hle: 0.119,
1481
1481
 
1482
1482
  // Capabilities
1483
1483
  contextWindow: 8192,
@@ -1485,18 +1485,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1485
1485
  supportsVision: false,
1486
1486
 
1487
1487
  // Metadata
1488
- lastUpdated: "2026-06-01",
1489
- originalModel: "Mistral Large (Feb '24)",
1488
+ lastUpdated: "2026-07-01",
1489
+ originalModel: "Claude 4.1 Opus (Reasoning)",
1490
1490
  },
1491
- "mixtral-8x7b-instruct": {
1491
+ "claude-4-sonnet-reasoning": {
1492
1492
  // AA specific benchmarks
1493
- codingIndex: undefined,
1494
- mathIndex: undefined,
1493
+ codingIndex: 37.6,
1494
+ mathIndex: 74.3,
1495
1495
 
1496
1496
  // Academic benchmarks
1497
- mmluPro: 0.387,
1498
- gpqa: 0.292,
1499
- hle: 0.045,
1497
+ mmluPro: 0.842,
1498
+ gpqa: 0.777,
1499
+ hle: 0.096,
1500
1500
 
1501
1501
  // Capabilities
1502
1502
  contextWindow: 8192,
@@ -1504,18 +1504,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1504
1504
  supportsVision: false,
1505
1505
 
1506
1506
  // Metadata
1507
- lastUpdated: "2026-06-01",
1508
- originalModel: "Mixtral 8x7B Instruct",
1507
+ lastUpdated: "2026-07-01",
1508
+ originalModel: "Claude 4 Sonnet (Reasoning)",
1509
1509
  },
1510
- "mistral-7b-instruct": {
1510
+ "claude-2.1": {
1511
1511
  // AA specific benchmarks
1512
- codingIndex: undefined,
1512
+ codingIndex: 14,
1513
1513
  mathIndex: undefined,
1514
1514
 
1515
1515
  // Academic benchmarks
1516
- mmluPro: 0.245,
1517
- gpqa: 0.177,
1518
- hle: 0.043,
1516
+ mmluPro: 0.495,
1517
+ gpqa: 0.319,
1518
+ hle: 0.042,
1519
1519
 
1520
1520
  // Capabilities
1521
1521
  contextWindow: 8192,
@@ -1523,18 +1523,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1523
1523
  supportsVision: false,
1524
1524
 
1525
1525
  // Metadata
1526
- lastUpdated: "2026-06-01",
1527
- originalModel: "Mistral 7B Instruct",
1526
+ lastUpdated: "2026-07-01",
1527
+ originalModel: "Claude 2.1",
1528
1528
  },
1529
- "mistral-saba": {
1529
+ "claude-4.5-sonnet-non-reasoning": {
1530
1530
  // AA specific benchmarks
1531
1531
  codingIndex: undefined,
1532
- mathIndex: undefined,
1532
+ mathIndex: 37,
1533
1533
 
1534
1534
  // Academic benchmarks
1535
- mmluPro: 0.611,
1536
- gpqa: 0.424,
1537
- hle: 0.041,
1535
+ mmluPro: 0.86,
1536
+ gpqa: 0.727,
1537
+ hle: 0.071,
1538
1538
 
1539
1539
  // Capabilities
1540
1540
  contextWindow: 8192,
@@ -1542,18 +1542,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1542
1542
  supportsVision: false,
1543
1543
 
1544
1544
  // Metadata
1545
- lastUpdated: "2026-06-01",
1546
- originalModel: "Mistral Saba",
1545
+ lastUpdated: "2026-07-01",
1546
+ originalModel: "Claude 4.5 Sonnet (Non-reasoning)",
1547
1547
  },
1548
- "mistral-small-3.2": {
1548
+ "claude-4.5-sonnet-reasoning": {
1549
1549
  // AA specific benchmarks
1550
- codingIndex: 13.3,
1551
- mathIndex: 27,
1550
+ codingIndex: undefined,
1551
+ mathIndex: 88,
1552
1552
 
1553
1553
  // Academic benchmarks
1554
- mmluPro: 0.681,
1555
- gpqa: 0.505,
1556
- hle: 0.043,
1554
+ mmluPro: 0.875,
1555
+ gpqa: 0.834,
1556
+ hle: 0.173,
1557
1557
 
1558
1558
  // Capabilities
1559
1559
  contextWindow: 8192,
@@ -1561,18 +1561,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1561
1561
  supportsVision: false,
1562
1562
 
1563
1563
  // Metadata
1564
- lastUpdated: "2026-06-01",
1565
- originalModel: "Mistral Small 3.2",
1564
+ lastUpdated: "2026-07-01",
1565
+ originalModel: "Claude 4.5 Sonnet (Reasoning)",
1566
1566
  },
1567
- "mistral-small-3.1": {
1567
+ "claude-4-opus-reasoning": {
1568
1568
  // AA specific benchmarks
1569
- codingIndex: 13.9,
1570
- mathIndex: 3.7,
1569
+ codingIndex: undefined,
1570
+ mathIndex: 73.3,
1571
1571
 
1572
1572
  // Academic benchmarks
1573
- mmluPro: 0.659,
1574
- gpqa: 0.454,
1575
- hle: 0.048,
1573
+ mmluPro: 0.873,
1574
+ gpqa: 0.796,
1575
+ hle: 0.117,
1576
1576
 
1577
1577
  // Capabilities
1578
1578
  contextWindow: 8192,
@@ -1580,18 +1580,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1580
1580
  supportsVision: false,
1581
1581
 
1582
1582
  // Metadata
1583
- lastUpdated: "2026-06-01",
1584
- originalModel: "Mistral Small 3.1",
1583
+ lastUpdated: "2026-07-01",
1584
+ originalModel: "Claude 4 Opus (Reasoning)",
1585
1585
  },
1586
- "mistral-medium-3": {
1586
+ "claude-opus-4.6-non-reasoning-high-effort": {
1587
1587
  // AA specific benchmarks
1588
- codingIndex: 13.6,
1589
- mathIndex: 30.3,
1588
+ codingIndex: undefined,
1589
+ mathIndex: undefined,
1590
1590
 
1591
1591
  // Academic benchmarks
1592
- mmluPro: 0.76,
1593
- gpqa: 0.578,
1594
- hle: 0.043,
1592
+ mmluPro: undefined,
1593
+ gpqa: 0.84,
1594
+ hle: 0.186,
1595
1595
 
1596
1596
  // Capabilities
1597
1597
  contextWindow: 8192,
@@ -1599,18 +1599,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1599
1599
  supportsVision: false,
1600
1600
 
1601
1601
  // Metadata
1602
- lastUpdated: "2026-06-01",
1603
- originalModel: "Mistral Medium 3",
1602
+ lastUpdated: "2026-07-01",
1603
+ originalModel: "Claude Opus 4.6 (Non-reasoning, High Effort)",
1604
1604
  },
1605
- "magistral-small-1": {
1605
+ "claude-4-opus-non-reasoning": {
1606
1606
  // AA specific benchmarks
1607
- codingIndex: 11.1,
1608
- mathIndex: 41.3,
1607
+ codingIndex: undefined,
1608
+ mathIndex: 36.3,
1609
1609
 
1610
1610
  // Academic benchmarks
1611
- mmluPro: 0.746,
1612
- gpqa: 0.641,
1613
- hle: 0.072,
1611
+ mmluPro: 0.86,
1612
+ gpqa: 0.701,
1613
+ hle: 0.059,
1614
1614
 
1615
1615
  // Capabilities
1616
1616
  contextWindow: 8192,
@@ -1618,18 +1618,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1618
1618
  supportsVision: false,
1619
1619
 
1620
1620
  // Metadata
1621
- lastUpdated: "2026-06-01",
1622
- originalModel: "Magistral Small 1",
1621
+ lastUpdated: "2026-07-01",
1622
+ originalModel: "Claude 4 Opus (Non-reasoning)",
1623
1623
  },
1624
- "devstral-small-may-25": {
1624
+ "claude-opus-4.5-reasoning": {
1625
1625
  // AA specific benchmarks
1626
- codingIndex: 12.2,
1627
- mathIndex: undefined,
1626
+ codingIndex: undefined,
1627
+ mathIndex: 91.3,
1628
1628
 
1629
1629
  // Academic benchmarks
1630
- mmluPro: 0.632,
1631
- gpqa: 0.434,
1632
- hle: 0.04,
1630
+ mmluPro: 0.895,
1631
+ gpqa: 0.866,
1632
+ hle: 0.284,
1633
1633
 
1634
1634
  // Capabilities
1635
1635
  contextWindow: 8192,
@@ -1637,18 +1637,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1637
1637
  supportsVision: false,
1638
1638
 
1639
1639
  // Metadata
1640
- lastUpdated: "2026-06-01",
1641
- originalModel: "Devstral Small (May '25)",
1640
+ lastUpdated: "2026-07-01",
1641
+ originalModel: "Claude Opus 4.5 (Reasoning)",
1642
1642
  },
1643
- "mistral-medium": {
1643
+ "claude-3.7-sonnet-reasoning": {
1644
1644
  // AA specific benchmarks
1645
- codingIndex: undefined,
1646
- mathIndex: undefined,
1645
+ codingIndex: 36.4,
1646
+ mathIndex: 56.3,
1647
1647
 
1648
1648
  // Academic benchmarks
1649
- mmluPro: 0.491,
1650
- gpqa: 0.349,
1651
- hle: 0.034,
1649
+ mmluPro: 0.837,
1650
+ gpqa: 0.772,
1651
+ hle: 0.103,
1652
1652
 
1653
1653
  // Capabilities
1654
1654
  contextWindow: 8192,
@@ -1656,18 +1656,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1656
1656
  supportsVision: false,
1657
1657
 
1658
1658
  // Metadata
1659
- lastUpdated: "2026-06-01",
1660
- originalModel: "Mistral Medium",
1659
+ lastUpdated: "2026-07-01",
1660
+ originalModel: "Claude 3.7 Sonnet (Reasoning)",
1661
1661
  },
1662
- "devstral-small-jul-25": {
1662
+ "claude-2.0": {
1663
1663
  // AA specific benchmarks
1664
- codingIndex: 12.1,
1665
- mathIndex: 29.3,
1664
+ codingIndex: 12.9,
1665
+ mathIndex: undefined,
1666
1666
 
1667
1667
  // Academic benchmarks
1668
- mmluPro: 0.622,
1669
- gpqa: 0.414,
1670
- hle: 0.037,
1668
+ mmluPro: 0.486,
1669
+ gpqa: 0.344,
1670
+ hle: undefined,
1671
1671
 
1672
1672
  // Capabilities
1673
1673
  contextWindow: 8192,
@@ -1675,18 +1675,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1675
1675
  supportsVision: false,
1676
1676
 
1677
1677
  // Metadata
1678
- lastUpdated: "2026-06-01",
1679
- originalModel: "Devstral Small (Jul '25)",
1678
+ lastUpdated: "2026-07-01",
1679
+ originalModel: "Claude 2.0",
1680
1680
  },
1681
- "devstral-medium": {
1681
+ "mistral-large-2-nov-24": {
1682
1682
  // AA specific benchmarks
1683
- codingIndex: 15.9,
1684
- mathIndex: 4.7,
1683
+ codingIndex: undefined,
1684
+ mathIndex: 14,
1685
1685
 
1686
1686
  // Academic benchmarks
1687
- mmluPro: 0.708,
1688
- gpqa: 0.492,
1689
- hle: 0.038,
1687
+ mmluPro: 0.697,
1688
+ gpqa: 0.486,
1689
+ hle: 0.04,
1690
1690
 
1691
1691
  // Capabilities
1692
1692
  contextWindow: 8192,
@@ -1694,18 +1694,18 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1694
1694
  supportsVision: false,
1695
1695
 
1696
1696
  // Metadata
1697
- lastUpdated: "2026-06-01",
1698
- originalModel: "Devstral Medium",
1697
+ lastUpdated: "2026-07-01",
1698
+ originalModel: "Mistral Large 2 (Nov '24)",
1699
1699
  },
1700
- "magistral-medium-1": {
1700
+ "mistral-large-2-jul-24": {
1701
1701
  // AA specific benchmarks
1702
- codingIndex: 16,
1703
- mathIndex: 40.3,
1702
+ codingIndex: undefined,
1703
+ mathIndex: 0,
1704
1704
 
1705
1705
  // Academic benchmarks
1706
- mmluPro: 0.753,
1707
- gpqa: 0.679,
1708
- hle: 0.095,
1706
+ mmluPro: 0.683,
1707
+ gpqa: 0.472,
1708
+ hle: 0.032,
1709
1709
 
1710
1710
  // Capabilities
1711
1711
  contextWindow: 8192,
@@ -1713,7 +1713,7 @@ export const BENCHMARKS_CHUNK_3: Record<string, HardcodedBenchmark> = {
1713
1713
  supportsVision: false,
1714
1714
 
1715
1715
  // Metadata
1716
- lastUpdated: "2026-06-01",
1717
- originalModel: "Magistral Medium 1",
1716
+ lastUpdated: "2026-07-01",
1717
+ originalModel: "Mistral Large 2 (Jul '24)",
1718
1718
  },
1719
1719
  };