pi-free 2.2.5 → 2.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +3 -2
  3. package/banner.svg +1 -1
  4. package/config.ts +821 -779
  5. package/constants.ts +3 -3
  6. package/index.ts +29 -24
  7. package/lib/built-in-toggle.ts +426 -427
  8. package/lib/provider-cache.ts +22 -0
  9. package/package.json +74 -74
  10. package/provider-failover/benchmark-lookup.ts +4 -1
  11. package/provider-failover/benchmarks-chunk-0.ts +570 -570
  12. package/provider-failover/benchmarks-chunk-1.ts +676 -676
  13. package/provider-failover/benchmarks-chunk-2.ts +673 -673
  14. package/provider-failover/benchmarks-chunk-3.ts +680 -680
  15. package/provider-failover/benchmarks-chunk-4.ts +683 -683
  16. package/provider-failover/benchmarks-chunk-5.ts +816 -474
  17. package/provider-helper.ts +64 -0
  18. package/providers/anyapi/anyapi.ts +270 -0
  19. package/providers/bai/bai.ts +8 -2
  20. package/providers/crofai/crofai.ts +8 -2
  21. package/providers/deepinfra/deepinfra.ts +8 -2
  22. package/providers/dynamic-built-in/index.ts +36 -26
  23. package/providers/kilo/kilo.ts +42 -28
  24. package/providers/novita/novita.ts +8 -2
  25. package/providers/opencode-session.ts +69 -18
  26. package/providers/openmodel/openmodel.ts +8 -2
  27. package/providers/routeway/routeway.ts +8 -2
  28. package/providers/sambanova/sambanova.ts +12 -6
  29. package/providers/together/together.ts +8 -2
  30. package/providers/tokenrouter/tokenrouter.ts +12 -7
  31. package/providers/zenmux/zenmux.ts +8 -2
  32. package/providers/codestral/codestral.ts +0 -128
package/provider-failover/benchmarks-chunk-4.ts CHANGED
@@ -1,39 +1,20 @@
1
1
  // Auto-generated benchmark data chunk 4
2
- // Models: mistral-medium-3.1 .. glm-4.5v-reasoning (90 entries)
3
- // Last updated: 2026-06-01
2
+ // Models: pixtral-large .. olmo-3-32b-think (90 entries)
3
+ // Last updated: 2026-07-01
4
4
  // DO NOT EDIT MANUALLY — generated by scripts/update-benchmarks.ts
5
5
 
6
6
  import type { HardcodedBenchmark } from "./hardcoded-benchmarks.ts";
7
7
 
8
8
  export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
9
- "mistral-medium-3.1": {
10
- // AA specific benchmarks
11
- codingIndex: 18.3,
12
- mathIndex: 38.3,
13
-
14
- // Academic benchmarks
15
- mmluPro: 0.683,
16
- gpqa: 0.588,
17
- hle: 0.044,
18
-
19
- // Capabilities
20
- contextWindow: 8192,
21
- supportsReasoning: false,
22
- supportsVision: false,
23
-
24
- // Metadata
25
- lastUpdated: "2026-06-01",
26
- originalModel: "Mistral Medium 3.1",
27
- },
28
- "deepseek-r1-distill-llama-70b": {
9
+ "pixtral-large": {
29
10
  // AA specific benchmarks
30
- codingIndex: 11.4,
31
- mathIndex: 53.7,
11
+ codingIndex: undefined,
12
+ mathIndex: 2.3,
32
13
 
33
14
  // Academic benchmarks
34
- mmluPro: 0.795,
35
- gpqa: 0.402,
36
- hle: 0.061,
15
+ mmluPro: 0.701,
16
+ gpqa: 0.505,
17
+ hle: 0.036,
37
18
 
38
19
  // Capabilities
39
20
  contextWindow: 8192,
@@ -41,18 +22,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
41
22
  supportsVision: false,
42
23
 
43
24
  // Metadata
44
- lastUpdated: "2026-06-01",
45
- originalModel: "DeepSeek R1 Distill Llama 70B",
25
+ lastUpdated: "2026-07-01",
26
+ originalModel: "Pixtral Large",
46
27
  },
47
- "deepseek-r1-distill-qwen-32b": {
28
+ "mistral-small-3": {
48
29
  // AA specific benchmarks
49
30
  codingIndex: undefined,
50
- mathIndex: 63,
31
+ mathIndex: 4.3,
51
32
 
52
33
  // Academic benchmarks
53
- mmluPro: 0.739,
54
- gpqa: 0.615,
55
- hle: 0.055,
34
+ mmluPro: 0.652,
35
+ gpqa: 0.462,
36
+ hle: 0.041,
56
37
 
57
38
  // Capabilities
58
39
  contextWindow: 8192,
@@ -60,18 +41,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
60
41
  supportsVision: false,
61
42
 
62
43
  // Metadata
63
- lastUpdated: "2026-06-01",
64
- originalModel: "DeepSeek R1 Distill Qwen 32B",
44
+ lastUpdated: "2026-07-01",
45
+ originalModel: "Mistral Small 3",
65
46
  },
66
- "deepseek-v3-dec-24": {
47
+ "mistral-small-sep-24": {
67
48
  // AA specific benchmarks
68
- codingIndex: 16.4,
69
- mathIndex: 26,
49
+ codingIndex: undefined,
50
+ mathIndex: undefined,
70
51
 
71
52
  // Academic benchmarks
72
- mmluPro: 0.752,
73
- gpqa: 0.557,
74
- hle: 0.036,
53
+ mmluPro: 0.529,
54
+ gpqa: 0.381,
55
+ hle: 0.043,
75
56
 
76
57
  // Capabilities
77
58
  contextWindow: 8192,
@@ -79,18 +60,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
79
60
  supportsVision: false,
80
61
 
81
62
  // Metadata
82
- lastUpdated: "2026-06-01",
83
- originalModel: "DeepSeek V3 (Dec '24)",
63
+ lastUpdated: "2026-07-01",
64
+ originalModel: "Mistral Small (Sep '24)",
84
65
  },
85
- "deepseek-r1-distill-qwen-14b": {
66
+ "mixtral-8x22b-instruct": {
86
67
  // AA specific benchmarks
87
68
  codingIndex: undefined,
88
- mathIndex: 55.7,
69
+ mathIndex: undefined,
89
70
 
90
71
  // Academic benchmarks
91
- mmluPro: 0.74,
92
- gpqa: 0.484,
93
- hle: 0.044,
72
+ mmluPro: 0.537,
73
+ gpqa: 0.332,
74
+ hle: 0.041,
94
75
 
95
76
  // Capabilities
96
77
  contextWindow: 8192,
@@ -98,18 +79,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
98
79
  supportsVision: false,
99
80
 
100
81
  // Metadata
101
- lastUpdated: "2026-06-01",
102
- originalModel: "DeepSeek R1 Distill Qwen 14B",
82
+ lastUpdated: "2026-07-01",
83
+ originalModel: "Mixtral 8x22B Instruct",
103
84
  },
104
- "deepseek-v2.5-dec-24": {
85
+ "mistral-small-feb-24": {
105
86
  // AA specific benchmarks
106
87
  codingIndex: undefined,
107
88
  mathIndex: undefined,
108
89
 
109
90
  // Academic benchmarks
110
- mmluPro: undefined,
111
- gpqa: undefined,
112
- hle: undefined,
91
+ mmluPro: 0.419,
92
+ gpqa: 0.302,
93
+ hle: 0.044,
113
94
 
114
95
  // Capabilities
115
96
  contextWindow: 8192,
@@ -117,18 +98,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
117
98
  supportsVision: false,
118
99
 
119
100
  // Metadata
120
- lastUpdated: "2026-06-01",
121
- originalModel: "DeepSeek-V2.5 (Dec '24)",
101
+ lastUpdated: "2026-07-01",
102
+ originalModel: "Mistral Small (Feb '24)",
122
103
  },
123
- "deepseek-coder-v2": {
104
+ "mistral-large-feb-24": {
124
105
  // AA specific benchmarks
125
106
  codingIndex: undefined,
126
107
  mathIndex: undefined,
127
108
 
128
109
  // Academic benchmarks
129
- mmluPro: undefined,
130
- gpqa: undefined,
131
- hle: undefined,
110
+ mmluPro: 0.515,
111
+ gpqa: 0.351,
112
+ hle: 0.034,
132
113
 
133
114
  // Capabilities
134
115
  contextWindow: 8192,
@@ -136,18 +117,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
136
117
  supportsVision: false,
137
118
 
138
119
  // Metadata
139
- lastUpdated: "2026-06-01",
140
- originalModel: "DeepSeek-Coder-V2",
120
+ lastUpdated: "2026-07-01",
121
+ originalModel: "Mistral Large (Feb '24)",
141
122
  },
142
- "deepseek-r1-distill-llama-8b": {
123
+ "mixtral-8x7b-instruct": {
143
124
  // AA specific benchmarks
144
125
  codingIndex: undefined,
145
- mathIndex: 41.3,
126
+ mathIndex: undefined,
146
127
 
147
128
  // Academic benchmarks
148
- mmluPro: 0.543,
149
- gpqa: 0.302,
150
- hle: 0.042,
129
+ mmluPro: 0.387,
130
+ gpqa: 0.292,
131
+ hle: 0.045,
151
132
 
152
133
  // Capabilities
153
134
  contextWindow: 8192,
@@ -155,18 +136,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
155
136
  supportsVision: false,
156
137
 
157
138
  // Metadata
158
- lastUpdated: "2026-06-01",
159
- originalModel: "DeepSeek R1 Distill Llama 8B",
139
+ lastUpdated: "2026-07-01",
140
+ originalModel: "Mixtral 8x7B Instruct",
160
141
  },
161
- "deepseek-llm-67b-chat-v1": {
142
+ "mistral-7b-instruct": {
162
143
  // AA specific benchmarks
163
144
  codingIndex: undefined,
164
145
  mathIndex: undefined,
165
146
 
166
147
  // Academic benchmarks
167
- mmluPro: undefined,
168
- gpqa: undefined,
169
- hle: undefined,
148
+ mmluPro: 0.245,
149
+ gpqa: 0.177,
150
+ hle: 0.043,
170
151
 
171
152
  // Capabilities
172
153
  contextWindow: 8192,
@@ -174,18 +155,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
174
155
  supportsVision: false,
175
156
 
176
157
  // Metadata
177
- lastUpdated: "2026-06-01",
178
- originalModel: "DeepSeek LLM 67B Chat (V1)",
158
+ lastUpdated: "2026-07-01",
159
+ originalModel: "Mistral 7B Instruct",
179
160
  },
180
- "deepseek-r1-distill-qwen-1.5b": {
161
+ "mistral-medium-3.1": {
181
162
  // AA specific benchmarks
182
- codingIndex: undefined,
183
- mathIndex: 22,
163
+ codingIndex: 20.5,
164
+ mathIndex: 38.3,
184
165
 
185
166
  // Academic benchmarks
186
- mmluPro: 0.269,
187
- gpqa: 0.098,
188
- hle: 0.033,
167
+ mmluPro: 0.683,
168
+ gpqa: 0.588,
169
+ hle: 0.044,
189
170
 
190
171
  // Capabilities
191
172
  contextWindow: 8192,
@@ -193,18 +174,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
193
174
  supportsVision: false,
194
175
 
195
176
  // Metadata
196
- lastUpdated: "2026-06-01",
197
- originalModel: "DeepSeek R1 Distill Qwen 1.5B",
177
+ lastUpdated: "2026-07-01",
178
+ originalModel: "Mistral Medium 3.1",
198
179
  },
199
- "deepseek-v3-0324": {
180
+ "mistral-saba": {
200
181
  // AA specific benchmarks
201
- codingIndex: 22,
202
- mathIndex: 41,
182
+ codingIndex: undefined,
183
+ mathIndex: undefined,
203
184
 
204
185
  // Academic benchmarks
205
- mmluPro: 0.819,
206
- gpqa: 0.655,
207
- hle: 0.052,
186
+ mmluPro: 0.611,
187
+ gpqa: 0.424,
188
+ hle: 0.041,
208
189
 
209
190
  // Capabilities
210
191
  contextWindow: 8192,
@@ -212,18 +193,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
212
193
  supportsVision: false,
213
194
 
214
195
  // Metadata
215
- lastUpdated: "2026-06-01",
216
- originalModel: "DeepSeek V3 0324",
196
+ lastUpdated: "2026-07-01",
197
+ originalModel: "Mistral Saba",
217
198
  },
218
- "deepseek-v3.2-reasoning": {
199
+ "devstral-small-jul-25": {
219
200
  // AA specific benchmarks
220
- codingIndex: 36.7,
221
- mathIndex: 92,
201
+ codingIndex: undefined,
202
+ mathIndex: 29.3,
222
203
 
223
204
  // Academic benchmarks
224
- mmluPro: 0.862,
225
- gpqa: 0.84,
226
- hle: 0.222,
205
+ mmluPro: 0.622,
206
+ gpqa: 0.414,
207
+ hle: 0.037,
227
208
 
228
209
  // Capabilities
229
210
  contextWindow: 8192,
@@ -231,18 +212,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
231
212
  supportsVision: false,
232
213
 
233
214
  // Metadata
234
- lastUpdated: "2026-06-01",
235
- originalModel: "DeepSeek V3.2 (Reasoning)",
215
+ lastUpdated: "2026-07-01",
216
+ originalModel: "Devstral Small (Jul '25)",
236
217
  },
237
- "deepseek-v3.2-non-reasoning": {
218
+ "mistral-small-3.1": {
238
219
  // AA specific benchmarks
239
- codingIndex: 34.6,
240
- mathIndex: 59,
220
+ codingIndex: 26.3,
221
+ mathIndex: 3.7,
241
222
 
242
223
  // Academic benchmarks
243
- mmluPro: 0.837,
244
- gpqa: 0.751,
245
- hle: 0.105,
224
+ mmluPro: 0.659,
225
+ gpqa: 0.454,
226
+ hle: 0.048,
246
227
 
247
228
  // Capabilities
248
229
  contextWindow: 8192,
@@ -250,18 +231,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
250
231
  supportsVision: false,
251
232
 
252
233
  // Metadata
253
- lastUpdated: "2026-06-01",
254
- originalModel: "DeepSeek V3.2 (Non-reasoning)",
234
+ lastUpdated: "2026-07-01",
235
+ originalModel: "Mistral Small 3.1",
255
236
  },
256
- "deepseek-r1-jan-25": {
237
+ "mistral-medium": {
257
238
  // AA specific benchmarks
258
- codingIndex: 15.9,
259
- mathIndex: 68,
239
+ codingIndex: undefined,
240
+ mathIndex: undefined,
260
241
 
261
242
  // Academic benchmarks
262
- mmluPro: 0.844,
263
- gpqa: 0.708,
264
- hle: 0.093,
243
+ mmluPro: 0.491,
244
+ gpqa: 0.349,
245
+ hle: 0.034,
265
246
 
266
247
  // Capabilities
267
248
  contextWindow: 8192,
@@ -269,18 +250,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
269
250
  supportsVision: false,
270
251
 
271
252
  // Metadata
272
- lastUpdated: "2026-06-01",
273
- originalModel: "DeepSeek R1 (Jan '25)",
253
+ lastUpdated: "2026-07-01",
254
+ originalModel: "Mistral Medium",
274
255
  },
275
- "deepseek-v3.1-non-reasoning": {
256
+ "magistral-small-1": {
276
257
  // AA specific benchmarks
277
- codingIndex: 28.4,
278
- mathIndex: 49.7,
258
+ codingIndex: undefined,
259
+ mathIndex: 41.3,
279
260
 
280
261
  // Academic benchmarks
281
- mmluPro: 0.833,
282
- gpqa: 0.735,
283
- hle: 0.063,
262
+ mmluPro: 0.746,
263
+ gpqa: 0.641,
264
+ hle: 0.072,
284
265
 
285
266
  // Capabilities
286
267
  contextWindow: 8192,
@@ -288,18 +269,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
288
269
  supportsVision: false,
289
270
 
290
271
  // Metadata
291
- lastUpdated: "2026-06-01",
292
- originalModel: "DeepSeek V3.1 (Non-reasoning)",
272
+ lastUpdated: "2026-07-01",
273
+ originalModel: "Magistral Small 1",
293
274
  },
294
- "deepseek-r1-0528-may-25": {
275
+ "mistral-small-3.2": {
295
276
  // AA specific benchmarks
296
- codingIndex: 24,
297
- mathIndex: 76,
277
+ codingIndex: 12.5,
278
+ mathIndex: 27,
298
279
 
299
280
  // Academic benchmarks
300
- mmluPro: 0.849,
301
- gpqa: 0.813,
302
- hle: 0.149,
281
+ mmluPro: 0.681,
282
+ gpqa: 0.505,
283
+ hle: 0.043,
303
284
 
304
285
  // Capabilities
305
286
  contextWindow: 8192,
@@ -307,18 +288,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
307
288
  supportsVision: false,
308
289
 
309
290
  // Metadata
310
- lastUpdated: "2026-06-01",
311
- originalModel: "DeepSeek R1 0528 (May '25)",
291
+ lastUpdated: "2026-07-01",
292
+ originalModel: "Mistral Small 3.2",
312
293
  },
313
- "deepseek-v3.1-terminus-non-reasoning": {
294
+ "devstral-small-may-25": {
314
295
  // AA specific benchmarks
315
- codingIndex: 31.9,
316
- mathIndex: 53.7,
296
+ codingIndex: undefined,
297
+ mathIndex: undefined,
317
298
 
318
299
  // Academic benchmarks
319
- mmluPro: 0.836,
320
- gpqa: 0.751,
321
- hle: 0.084,
300
+ mmluPro: 0.632,
301
+ gpqa: 0.434,
302
+ hle: 0.04,
322
303
 
323
304
  // Capabilities
324
305
  contextWindow: 8192,
@@ -326,18 +307,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
326
307
  supportsVision: false,
327
308
 
328
309
  // Metadata
329
- lastUpdated: "2026-06-01",
330
- originalModel: "DeepSeek V3.1 Terminus (Non-reasoning)",
310
+ lastUpdated: "2026-07-01",
311
+ originalModel: "Devstral Small (May '25)",
331
312
  },
332
- "deepseek-v3.1-reasoning": {
313
+ "devstral-medium": {
333
314
  // AA specific benchmarks
334
- codingIndex: 29.7,
335
- mathIndex: 89.7,
315
+ codingIndex: undefined,
316
+ mathIndex: 4.7,
336
317
 
337
318
  // Academic benchmarks
338
- mmluPro: 0.851,
339
- gpqa: 0.779,
340
- hle: 0.13,
319
+ mmluPro: 0.708,
320
+ gpqa: 0.492,
321
+ hle: 0.038,
341
322
 
342
323
  // Capabilities
343
324
  contextWindow: 8192,
@@ -345,18 +326,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
345
326
  supportsVision: false,
346
327
 
347
328
  // Metadata
348
- lastUpdated: "2026-06-01",
349
- originalModel: "DeepSeek V3.1 (Reasoning)",
329
+ lastUpdated: "2026-07-01",
330
+ originalModel: "Devstral Medium",
350
331
  },
351
- "deepseek-v3.1-terminus-reasoning": {
332
+ "magistral-medium-1": {
352
333
  // AA specific benchmarks
353
- codingIndex: 33.7,
354
- mathIndex: 89.7,
334
+ codingIndex: undefined,
335
+ mathIndex: 40.3,
355
336
 
356
337
  // Academic benchmarks
357
- mmluPro: 0.851,
358
- gpqa: 0.792,
359
- hle: 0.152,
338
+ mmluPro: 0.753,
339
+ gpqa: 0.679,
340
+ hle: 0.095,
360
341
 
361
342
  // Capabilities
362
343
  contextWindow: 8192,
@@ -364,18 +345,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
364
345
  supportsVision: false,
365
346
 
366
347
  // Metadata
367
- lastUpdated: "2026-06-01",
368
- originalModel: "DeepSeek V3.1 Terminus (Reasoning)",
348
+ lastUpdated: "2026-07-01",
349
+ originalModel: "Magistral Medium 1",
369
350
  },
370
- "deepseek-v3.2-exp-non-reasoning": {
351
+ "mistral-medium-3": {
371
352
  // AA specific benchmarks
372
- codingIndex: 30,
373
- mathIndex: 57.7,
353
+ codingIndex: undefined,
354
+ mathIndex: 30.3,
374
355
 
375
356
  // Academic benchmarks
376
- mmluPro: 0.836,
377
- gpqa: 0.738,
378
- hle: 0.086,
357
+ mmluPro: 0.76,
358
+ gpqa: 0.578,
359
+ hle: 0.043,
379
360
 
380
361
  // Capabilities
381
362
  contextWindow: 8192,
@@ -383,18 +364,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
383
364
  supportsVision: false,
384
365
 
385
366
  // Metadata
386
- lastUpdated: "2026-06-01",
387
- originalModel: "DeepSeek V3.2 Exp (Non-reasoning)",
367
+ lastUpdated: "2026-07-01",
368
+ originalModel: "Mistral Medium 3",
388
369
  },
389
- "deepseek-v2.5": {
370
+ "deepseek-r1-distill-llama-70b": {
390
371
  // AA specific benchmarks
391
372
  codingIndex: undefined,
392
- mathIndex: undefined,
373
+ mathIndex: 53.7,
393
374
 
394
375
  // Academic benchmarks
395
- mmluPro: undefined,
396
- gpqa: undefined,
397
- hle: undefined,
376
+ mmluPro: 0.795,
377
+ gpqa: 0.402,
378
+ hle: 0.061,
398
379
 
399
380
  // Capabilities
400
381
  contextWindow: 8192,
@@ -402,18 +383,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
402
383
  supportsVision: false,
403
384
 
404
385
  // Metadata
405
- lastUpdated: "2026-06-01",
406
- originalModel: "DeepSeek-V2.5",
386
+ lastUpdated: "2026-07-01",
387
+ originalModel: "DeepSeek R1 Distill Llama 70B",
407
388
  },
408
- "deepseek-coder-v2-lite-instruct": {
389
+ "deepseek-r1-distill-qwen-32b": {
409
390
  // AA specific benchmarks
410
391
  codingIndex: undefined,
411
- mathIndex: undefined,
392
+ mathIndex: 63,
412
393
 
413
394
  // Academic benchmarks
414
- mmluPro: 0.429,
415
- gpqa: 0.319,
416
- hle: 0.053,
395
+ mmluPro: 0.739,
396
+ gpqa: 0.615,
397
+ hle: 0.055,
417
398
 
418
399
  // Capabilities
419
400
  contextWindow: 8192,
@@ -421,18 +402,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
421
402
  supportsVision: false,
422
403
 
423
404
  // Metadata
424
- lastUpdated: "2026-06-01",
425
- originalModel: "DeepSeek Coder V2 Lite Instruct",
405
+ lastUpdated: "2026-07-01",
406
+ originalModel: "DeepSeek R1 Distill Qwen 32B",
426
407
  },
427
- "deepseek-r1-0528-qwen3-8b": {
408
+ "deepseek-v3-dec-24": {
428
409
  // AA specific benchmarks
429
- codingIndex: 7.8,
430
- mathIndex: 63.7,
410
+ codingIndex: 23,
411
+ mathIndex: 26,
431
412
 
432
413
  // Academic benchmarks
433
- mmluPro: 0.739,
434
- gpqa: 0.612,
435
- hle: 0.056,
414
+ mmluPro: 0.752,
415
+ gpqa: 0.557,
416
+ hle: 0.036,
436
417
 
437
418
  // Capabilities
438
419
  contextWindow: 8192,
@@ -440,18 +421,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
440
421
  supportsVision: false,
441
422
 
442
423
  // Metadata
443
- lastUpdated: "2026-06-01",
444
- originalModel: "DeepSeek R1 0528 Qwen3 8B",
424
+ lastUpdated: "2026-07-01",
425
+ originalModel: "DeepSeek V3 (Dec '24)",
445
426
  },
446
- "deepseek-v3.2-exp-reasoning": {
427
+ "deepseek-r1-distill-qwen-14b": {
447
428
  // AA specific benchmarks
448
- codingIndex: 33.3,
449
- mathIndex: 87.7,
429
+ codingIndex: undefined,
430
+ mathIndex: 55.7,
450
431
 
451
432
  // Academic benchmarks
452
- mmluPro: 0.85,
453
- gpqa: 0.797,
454
- hle: 0.138,
433
+ mmluPro: 0.74,
434
+ gpqa: 0.484,
435
+ hle: 0.044,
455
436
 
456
437
  // Capabilities
457
438
  contextWindow: 8192,
@@ -459,18 +440,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
459
440
  supportsVision: false,
460
441
 
461
442
  // Metadata
462
- lastUpdated: "2026-06-01",
463
- originalModel: "DeepSeek V3.2 Exp (Reasoning)",
443
+ lastUpdated: "2026-07-01",
444
+ originalModel: "DeepSeek R1 Distill Qwen 14B",
464
445
  },
465
- "deepseek-v3.2-speciale": {
446
+ "deepseek-v2.5-dec-24": {
466
447
  // AA specific benchmarks
467
- codingIndex: 37.9,
468
- mathIndex: 96.7,
448
+ codingIndex: undefined,
449
+ mathIndex: undefined,
469
450
 
470
451
  // Academic benchmarks
471
- mmluPro: 0.863,
472
- gpqa: 0.871,
473
- hle: 0.261,
452
+ mmluPro: undefined,
453
+ gpqa: undefined,
454
+ hle: undefined,
474
455
 
475
456
  // Capabilities
476
457
  contextWindow: 8192,
@@ -478,10 +459,10 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
478
459
  supportsVision: false,
479
460
 
480
461
  // Metadata
481
- lastUpdated: "2026-06-01",
482
- originalModel: "DeepSeek V3.2 Speciale",
462
+ lastUpdated: "2026-07-01",
463
+ originalModel: "DeepSeek-V2.5 (Dec '24)",
483
464
  },
484
- "deepseek-v2-chat": {
465
+ "deepseek-coder-v2": {
485
466
  // AA specific benchmarks
486
467
  codingIndex: undefined,
487
468
  mathIndex: undefined,
@@ -497,18 +478,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
497
478
  supportsVision: false,
498
479
 
499
480
  // Metadata
500
- lastUpdated: "2026-06-01",
501
- originalModel: "DeepSeek-V2-Chat",
481
+ lastUpdated: "2026-07-01",
482
+ originalModel: "DeepSeek-Coder-V2",
502
483
  },
503
- "sonar-pro": {
484
+ "deepseek-r1-distill-llama-8b": {
504
485
  // AA specific benchmarks
505
486
  codingIndex: undefined,
506
- mathIndex: undefined,
487
+ mathIndex: 41.3,
507
488
 
508
489
  // Academic benchmarks
509
- mmluPro: 0.755,
510
- gpqa: 0.578,
511
- hle: 0.079,
490
+ mmluPro: 0.543,
491
+ gpqa: 0.302,
492
+ hle: 0.042,
512
493
 
513
494
  // Capabilities
514
495
  contextWindow: 8192,
@@ -516,10 +497,10 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
516
497
  supportsVision: false,
517
498
 
518
499
  // Metadata
519
- lastUpdated: "2026-06-01",
520
- originalModel: "Sonar Pro",
500
+ lastUpdated: "2026-07-01",
501
+ originalModel: "DeepSeek R1 Distill Llama 8B",
521
502
  },
522
- "sonar-reasoning-pro": {
503
+ "deepseek-llm-67b-chat-v1": {
523
504
  // AA specific benchmarks
524
505
  codingIndex: undefined,
525
506
  mathIndex: undefined,
@@ -535,37 +516,37 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
535
516
  supportsVision: false,
536
517
 
537
518
  // Metadata
538
- lastUpdated: "2026-06-01",
539
- originalModel: "Sonar Reasoning Pro",
519
+ lastUpdated: "2026-07-01",
520
+ originalModel: "DeepSeek LLM 67B Chat (V1)",
540
521
  },
541
- "sonar-reasoning": {
522
+ "deepseek-r1-distill-qwen-1.5b": {
542
523
  // AA specific benchmarks
543
524
  codingIndex: undefined,
544
- mathIndex: undefined,
525
+ mathIndex: 22,
545
526
 
546
527
  // Academic benchmarks
547
- mmluPro: undefined,
548
- gpqa: 0.623,
549
- hle: undefined,
550
-
528
+ mmluPro: 0.269,
529
+ gpqa: 0.098,
530
+ hle: 0.033,
531
+
551
532
  // Capabilities
552
533
  contextWindow: 8192,
553
534
  supportsReasoning: false,
554
535
  supportsVision: false,
555
536
 
556
537
  // Metadata
557
- lastUpdated: "2026-06-01",
558
- originalModel: "Sonar Reasoning",
538
+ lastUpdated: "2026-07-01",
539
+ originalModel: "DeepSeek R1 Distill Qwen 1.5B",
559
540
  },
560
- "sonar": {
541
+ "deepseek-v3.2-speciale": {
561
542
  // AA specific benchmarks
562
543
  codingIndex: undefined,
563
- mathIndex: undefined,
544
+ mathIndex: 96.7,
564
545
 
565
546
  // Academic benchmarks
566
- mmluPro: 0.689,
567
- gpqa: 0.471,
568
- hle: 0.073,
547
+ mmluPro: 0.863,
548
+ gpqa: 0.871,
549
+ hle: 0.261,
569
550
 
570
551
  // Capabilities
571
552
  contextWindow: 8192,
@@ -573,18 +554,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
573
554
  supportsVision: false,
574
555
 
575
556
  // Metadata
576
- lastUpdated: "2026-06-01",
577
- originalModel: "Sonar",
557
+ lastUpdated: "2026-07-01",
558
+ originalModel: "DeepSeek V3.2 Speciale",
578
559
  },
579
- "grok-beta": {
560
+ "deepseek-r1-jan-25": {
580
561
  // AA specific benchmarks
581
- codingIndex: undefined,
582
- mathIndex: undefined,
562
+ codingIndex: 24.6,
563
+ mathIndex: 68,
583
564
 
584
565
  // Academic benchmarks
585
- mmluPro: 0.703,
586
- gpqa: 0.471,
587
- hle: 0.047,
566
+ mmluPro: 0.844,
567
+ gpqa: 0.708,
568
+ hle: 0.093,
588
569
 
589
570
  // Capabilities
590
571
  contextWindow: 8192,
@@ -592,18 +573,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
592
573
  supportsVision: false,
593
574
 
594
575
  // Metadata
595
- lastUpdated: "2026-06-01",
596
- originalModel: "Grok Beta",
576
+ lastUpdated: "2026-07-01",
577
+ originalModel: "DeepSeek R1 (Jan '25)",
597
578
  },
598
- "grok-3": {
579
+ "deepseek-v3-0324": {
599
580
  // AA specific benchmarks
600
- codingIndex: 19.8,
601
- mathIndex: 58,
581
+ codingIndex: 21.2,
582
+ mathIndex: 41,
602
583
 
603
584
  // Academic benchmarks
604
- mmluPro: 0.799,
605
- gpqa: 0.693,
606
- hle: 0.051,
585
+ mmluPro: 0.819,
586
+ gpqa: 0.655,
587
+ hle: 0.052,
607
588
 
608
589
  // Capabilities
609
590
  contextWindow: 8192,
@@ -611,18 +592,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
611
592
  supportsVision: false,
612
593
 
613
594
  // Metadata
614
- lastUpdated: "2026-06-01",
615
- originalModel: "Grok 3",
595
+ lastUpdated: "2026-07-01",
596
+ originalModel: "DeepSeek V3 0324",
616
597
  },
617
- "grok-3-mini-reasoning-high": {
598
+ "deepseek-r1-0528-may-25": {
618
599
  // AA specific benchmarks
619
- codingIndex: 25.2,
620
- mathIndex: 84.7,
600
+ codingIndex: undefined,
601
+ mathIndex: 76,
621
602
 
622
603
  // Academic benchmarks
623
- mmluPro: 0.828,
624
- gpqa: 0.791,
625
- hle: 0.111,
604
+ mmluPro: 0.849,
605
+ gpqa: 0.813,
606
+ hle: 0.149,
626
607
 
627
608
  // Capabilities
628
609
  contextWindow: 8192,
@@ -630,18 +611,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
630
611
  supportsVision: false,
631
612
 
632
613
  // Metadata
633
- lastUpdated: "2026-06-01",
634
- originalModel: "Grok 3 mini Reasoning (high)",
614
+ lastUpdated: "2026-07-01",
615
+ originalModel: "DeepSeek R1 0528 (May '25)",
635
616
  },
636
- "grok-4.20-0309-v2-reasoning": {
617
+ "deepseek-v3.2-exp-non-reasoning": {
637
618
  // AA specific benchmarks
638
- codingIndex: 40.5,
639
- mathIndex: undefined,
619
+ codingIndex: undefined,
620
+ mathIndex: 57.7,
640
621
 
641
622
  // Academic benchmarks
642
- mmluPro: undefined,
643
- gpqa: 0.911,
644
- hle: 0.322,
623
+ mmluPro: 0.836,
624
+ gpqa: 0.738,
625
+ hle: 0.086,
645
626
 
646
627
  // Capabilities
647
628
  contextWindow: 8192,
@@ -649,18 +630,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
649
630
  supportsVision: false,
650
631
 
651
632
  // Metadata
652
- lastUpdated: "2026-06-01",
653
- originalModel: "Grok 4.20 0309 v2 (Reasoning)",
633
+ lastUpdated: "2026-07-01",
634
+ originalModel: "DeepSeek V3.2 Exp (Non-reasoning)",
654
635
  },
655
- "grok-4": {
636
+ "deepseek-v3.1-terminus-reasoning": {
656
637
  // AA specific benchmarks
657
- codingIndex: 40.5,
658
- mathIndex: 92.7,
638
+ codingIndex: undefined,
639
+ mathIndex: 89.7,
659
640
 
660
641
  // Academic benchmarks
661
- mmluPro: 0.866,
662
- gpqa: 0.877,
663
- hle: 0.239,
642
+ mmluPro: 0.851,
643
+ gpqa: 0.792,
644
+ hle: 0.152,
664
645
 
665
646
  // Capabilities
666
647
  contextWindow: 8192,
@@ -668,18 +649,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
668
649
  supportsVision: false,
669
650
 
670
651
  // Metadata
671
- lastUpdated: "2026-06-01",
672
- originalModel: "Grok 4",
652
+ lastUpdated: "2026-07-01",
653
+ originalModel: "DeepSeek V3.1 Terminus (Reasoning)",
673
654
  },
674
- "grok-4-fast-non-reasoning": {
655
+ "deepseek-v3.1-reasoning": {
675
656
  // AA specific benchmarks
676
- codingIndex: 19,
677
- mathIndex: 41.3,
657
+ codingIndex: undefined,
658
+ mathIndex: 89.7,
678
659
 
679
660
  // Academic benchmarks
680
- mmluPro: 0.73,
681
- gpqa: 0.606,
682
- hle: 0.05,
661
+ mmluPro: 0.851,
662
+ gpqa: 0.779,
663
+ hle: 0.13,
683
664
 
684
665
  // Capabilities
685
666
  contextWindow: 8192,
@@ -687,18 +668,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
687
668
  supportsVision: false,
688
669
 
689
670
  // Metadata
690
- lastUpdated: "2026-06-01",
691
- originalModel: "Grok 4 Fast (Non-reasoning)",
671
+ lastUpdated: "2026-07-01",
672
+ originalModel: "DeepSeek V3.1 (Reasoning)",
692
673
  },
693
- "grok-4-fast-reasoning": {
674
+ "deepseek-v2-chat": {
694
675
  // AA specific benchmarks
695
- codingIndex: 27.4,
696
- mathIndex: 89.7,
676
+ codingIndex: undefined,
677
+ mathIndex: undefined,
697
678
 
698
679
  // Academic benchmarks
699
- mmluPro: 0.85,
700
- gpqa: 0.847,
701
- hle: 0.17,
680
+ mmluPro: undefined,
681
+ gpqa: undefined,
682
+ hle: undefined,
702
683
 
703
684
  // Capabilities
704
685
  contextWindow: 8192,
@@ -706,18 +687,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
706
687
  supportsVision: false,
707
688
 
708
689
  // Metadata
709
- lastUpdated: "2026-06-01",
710
- originalModel: "Grok 4 Fast (Reasoning)",
690
+ lastUpdated: "2026-07-01",
691
+ originalModel: "DeepSeek-V2-Chat",
711
692
  },
712
- "grok-code-fast-1": {
693
+ "deepseek-v3.2-non-reasoning": {
713
694
  // AA specific benchmarks
714
- codingIndex: 23.7,
715
- mathIndex: 43.3,
695
+ codingIndex: undefined,
696
+ mathIndex: 59,
716
697
 
717
698
  // Academic benchmarks
718
- mmluPro: 0.793,
719
- gpqa: 0.727,
720
- hle: 0.075,
699
+ mmluPro: 0.837,
700
+ gpqa: 0.751,
701
+ hle: 0.105,
721
702
 
722
703
  // Capabilities
723
704
  contextWindow: 8192,
@@ -725,10 +706,10 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
725
706
  supportsVision: false,
726
707
 
727
708
  // Metadata
728
- lastUpdated: "2026-06-01",
729
- originalModel: "Grok Code Fast 1",
709
+ lastUpdated: "2026-07-01",
710
+ originalModel: "DeepSeek V3.2 (Non-reasoning)",
730
711
  },
731
- "grok-3-reasoning-beta": {
712
+ "deepseek-v2.5": {
732
713
  // AA specific benchmarks
733
714
  codingIndex: undefined,
734
715
  mathIndex: undefined,
@@ -744,18 +725,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
744
725
  supportsVision: false,
745
726
 
746
727
  // Metadata
747
- lastUpdated: "2026-06-01",
748
- originalModel: "Grok 3 Reasoning Beta",
728
+ lastUpdated: "2026-07-01",
729
+ originalModel: "DeepSeek-V2.5",
749
730
  },
750
- "grok-4.20-0309-reasoning": {
731
+ "deepseek-v3.2-exp-reasoning": {
751
732
  // AA specific benchmarks
752
- codingIndex: 42.2,
753
- mathIndex: undefined,
733
+ codingIndex: undefined,
734
+ mathIndex: 87.7,
754
735
 
755
736
  // Academic benchmarks
756
- mmluPro: undefined,
757
- gpqa: 0.885,
758
- hle: 0.3,
737
+ mmluPro: 0.85,
738
+ gpqa: 0.797,
739
+ hle: 0.138,
759
740
 
760
741
  // Capabilities
761
742
  contextWindow: 8192,
@@ -763,18 +744,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
763
744
  supportsVision: false,
764
745
 
765
746
  // Metadata
766
- lastUpdated: "2026-06-01",
767
- originalModel: "Grok 4.20 0309 (Reasoning)",
747
+ lastUpdated: "2026-07-01",
748
+ originalModel: "DeepSeek V3.2 Exp (Reasoning)",
768
749
  },
769
- "grok-4.1-fast-reasoning": {
750
+ "deepseek-v3.2-reasoning": {
770
751
  // AA specific benchmarks
771
- codingIndex: 30.9,
772
- mathIndex: 89.3,
752
+ codingIndex: undefined,
753
+ mathIndex: 92,
773
754
 
774
755
  // Academic benchmarks
775
- mmluPro: 0.854,
776
- gpqa: 0.853,
777
- hle: 0.176,
756
+ mmluPro: 0.862,
757
+ gpqa: 0.84,
758
+ hle: 0.222,
778
759
 
779
760
  // Capabilities
780
761
  contextWindow: 8192,
@@ -782,18 +763,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
782
763
  supportsVision: false,
783
764
 
784
765
  // Metadata
785
- lastUpdated: "2026-06-01",
786
- originalModel: "Grok 4.1 Fast (Reasoning)",
766
+ lastUpdated: "2026-07-01",
767
+ originalModel: "DeepSeek V3.2 (Reasoning)",
787
768
  },
788
- "grok-2-dec-24": {
769
+ "deepseek-v3.1-terminus-non-reasoning": {
789
770
  // AA specific benchmarks
790
771
  codingIndex: undefined,
791
- mathIndex: undefined,
772
+ mathIndex: 53.7,
792
773
 
793
774
  // Academic benchmarks
794
- mmluPro: 0.709,
795
- gpqa: 0.51,
796
- hle: 0.038,
775
+ mmluPro: 0.836,
776
+ gpqa: 0.751,
777
+ hle: 0.084,
797
778
 
798
779
  // Capabilities
799
780
  contextWindow: 8192,
@@ -801,18 +782,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
801
782
  supportsVision: false,
802
783
 
803
784
  // Metadata
804
- lastUpdated: "2026-06-01",
805
- originalModel: "Grok 2 (Dec '24)",
785
+ lastUpdated: "2026-07-01",
786
+ originalModel: "DeepSeek V3.1 Terminus (Non-reasoning)",
806
787
  },
807
- "grok-4.1-fast-non-reasoning": {
788
+ "deepseek-v3.1-non-reasoning": {
808
789
  // AA specific benchmarks
809
- codingIndex: 19.5,
810
- mathIndex: 34.3,
790
+ codingIndex: undefined,
791
+ mathIndex: 49.7,
811
792
 
812
793
  // Academic benchmarks
813
- mmluPro: 0.743,
814
- gpqa: 0.637,
815
- hle: 0.05,
794
+ mmluPro: 0.833,
795
+ gpqa: 0.735,
796
+ hle: 0.063,
816
797
 
817
798
  // Capabilities
818
799
  contextWindow: 8192,
@@ -820,18 +801,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
820
801
  supportsVision: false,
821
802
 
822
803
  // Metadata
823
- lastUpdated: "2026-06-01",
824
- originalModel: "Grok 4.1 Fast (Non-reasoning)",
804
+ lastUpdated: "2026-07-01",
805
+ originalModel: "DeepSeek V3.1 (Non-reasoning)",
825
806
  },
826
- "grok-4.20-0309-non-reasoning": {
807
+ "deepseek-r1-0528-qwen3-8b": {
827
808
  // AA specific benchmarks
828
- codingIndex: 25.4,
829
- mathIndex: undefined,
809
+ codingIndex: undefined,
810
+ mathIndex: 63.7,
830
811
 
831
812
  // Academic benchmarks
832
- mmluPro: undefined,
833
- gpqa: 0.785,
834
- hle: 0.225,
813
+ mmluPro: 0.739,
814
+ gpqa: 0.612,
815
+ hle: 0.056,
835
816
 
836
817
  // Capabilities
837
818
  contextWindow: 8192,
@@ -839,18 +820,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
839
820
  supportsVision: false,
840
821
 
841
822
  // Metadata
842
- lastUpdated: "2026-06-01",
843
- originalModel: "Grok 4.20 0309 (Non-reasoning)",
823
+ lastUpdated: "2026-07-01",
824
+ originalModel: "DeepSeek R1 0528 Qwen3 8B",
844
825
  },
845
- "grok-4.20-0309-v2-non-reasoning": {
826
+ "deepseek-coder-v2-lite-instruct": {
846
827
  // AA specific benchmarks
847
- codingIndex: 22,
828
+ codingIndex: undefined,
848
829
  mathIndex: undefined,
849
830
 
850
831
  // Academic benchmarks
851
- mmluPro: undefined,
852
- gpqa: 0.776,
853
- hle: 0.242,
832
+ mmluPro: 0.429,
833
+ gpqa: 0.319,
834
+ hle: 0.053,
854
835
 
855
836
  // Capabilities
856
837
  contextWindow: 8192,
@@ -858,18 +839,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
858
839
  supportsVision: false,
859
840
 
860
841
  // Metadata
861
- lastUpdated: "2026-06-01",
862
- originalModel: "Grok 4.20 0309 v2 (Non-reasoning)",
842
+ lastUpdated: "2026-07-01",
843
+ originalModel: "DeepSeek Coder V2 Lite Instruct",
863
844
  },
864
- "openchat-3.5-1210": {
845
+ "sonar-pro": {
865
846
  // AA specific benchmarks
866
847
  codingIndex: undefined,
867
848
  mathIndex: undefined,
868
849
 
869
850
  // Academic benchmarks
870
- mmluPro: 0.31,
871
- gpqa: 0.23,
872
- hle: 0.048,
851
+ mmluPro: 0.755,
852
+ gpqa: 0.578,
853
+ hle: 0.079,
873
854
 
874
855
  // Capabilities
875
856
  contextWindow: 8192,
@@ -877,18 +858,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
877
858
  supportsVision: false,
878
859
 
879
860
  // Metadata
880
- lastUpdated: "2026-06-01",
881
- originalModel: "OpenChat 3.5 (1210)",
861
+ lastUpdated: "2026-07-01",
862
+ originalModel: "Sonar Pro",
882
863
  },
883
- "nova-pro": {
864
+ "sonar-reasoning": {
884
865
  // AA specific benchmarks
885
- codingIndex: 11,
886
- mathIndex: 7,
866
+ codingIndex: undefined,
867
+ mathIndex: undefined,
887
868
 
888
869
  // Academic benchmarks
889
- mmluPro: 0.691,
890
- gpqa: 0.499,
891
- hle: 0.034,
870
+ mmluPro: undefined,
871
+ gpqa: 0.623,
872
+ hle: undefined,
892
873
 
893
874
  // Capabilities
894
875
  contextWindow: 8192,
@@ -896,18 +877,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
896
877
  supportsVision: false,
897
878
 
898
879
  // Metadata
899
- lastUpdated: "2026-06-01",
900
- originalModel: "Nova Pro",
880
+ lastUpdated: "2026-07-01",
881
+ originalModel: "Sonar Reasoning",
901
882
  },
902
- "nova-lite": {
883
+ "sonar": {
903
884
  // AA specific benchmarks
904
- codingIndex: 5.1,
905
- mathIndex: 7,
885
+ codingIndex: undefined,
886
+ mathIndex: undefined,
906
887
 
907
888
  // Academic benchmarks
908
- mmluPro: 0.59,
909
- gpqa: 0.433,
910
- hle: 0.046,
889
+ mmluPro: 0.689,
890
+ gpqa: 0.471,
891
+ hle: 0.073,
911
892
 
912
893
  // Capabilities
913
894
  contextWindow: 8192,
@@ -915,18 +896,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
915
896
  supportsVision: false,
916
897
 
917
898
  // Metadata
918
- lastUpdated: "2026-06-01",
919
- originalModel: "Nova Lite",
899
+ lastUpdated: "2026-07-01",
900
+ originalModel: "Sonar",
920
901
  },
921
- "phi-3-mini-instruct-3.8b": {
902
+ "sonar-reasoning-pro": {
922
903
  // AA specific benchmarks
923
- codingIndex: 3,
924
- mathIndex: 0.3,
904
+ codingIndex: undefined,
905
+ mathIndex: undefined,
925
906
 
926
907
  // Academic benchmarks
927
- mmluPro: 0.435,
928
- gpqa: 0.319,
929
- hle: 0.044,
908
+ mmluPro: undefined,
909
+ gpqa: undefined,
910
+ hle: undefined,
930
911
 
931
912
  // Capabilities
932
913
  contextWindow: 8192,
@@ -934,18 +915,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
934
915
  supportsVision: false,
935
916
 
936
917
  // Metadata
937
- lastUpdated: "2026-06-01",
938
- originalModel: "Phi-3 Mini Instruct 3.8B",
918
+ lastUpdated: "2026-07-01",
919
+ originalModel: "Sonar Reasoning Pro",
939
920
  },
940
- "lfm-40b": {
921
+ "grok-beta": {
941
922
  // AA specific benchmarks
942
923
  codingIndex: undefined,
943
924
  mathIndex: undefined,
944
925
 
945
926
  // Academic benchmarks
946
- mmluPro: 0.425,
947
- gpqa: 0.327,
948
- hle: 0.049,
927
+ mmluPro: 0.703,
928
+ gpqa: 0.471,
929
+ hle: 0.047,
949
930
 
950
931
  // Capabilities
951
932
  contextWindow: 8192,
@@ -953,18 +934,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
953
934
  supportsVision: false,
954
935
 
955
936
  // Metadata
956
- lastUpdated: "2026-06-01",
957
- originalModel: "LFM 40B",
937
+ lastUpdated: "2026-07-01",
938
+ originalModel: "Grok Beta",
958
939
  },
959
- "lfm2-1.2b": {
940
+ "grok-3": {
960
941
  // AA specific benchmarks
961
- codingIndex: 0.8,
962
- mathIndex: 3.3,
942
+ codingIndex: undefined,
943
+ mathIndex: 58,
963
944
 
964
945
  // Academic benchmarks
965
- mmluPro: 0.257,
966
- gpqa: 0.228,
967
- hle: 0.057,
946
+ mmluPro: 0.799,
947
+ gpqa: 0.693,
948
+ hle: 0.051,
968
949
 
969
950
  // Capabilities
970
951
  contextWindow: 8192,
@@ -972,18 +953,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
972
953
  supportsVision: false,
973
954
 
974
955
  // Metadata
975
- lastUpdated: "2026-06-01",
976
- originalModel: "LFM2 1.2B",
956
+ lastUpdated: "2026-07-01",
957
+ originalModel: "Grok 3",
977
958
  },
978
- "solar-mini": {
959
+ "grok-3-mini-reasoning-high": {
979
960
  // AA specific benchmarks
980
961
  codingIndex: undefined,
981
- mathIndex: undefined,
962
+ mathIndex: 84.7,
982
963
 
983
964
  // Academic benchmarks
984
- mmluPro: undefined,
985
- gpqa: undefined,
986
- hle: undefined,
965
+ mmluPro: 0.828,
966
+ gpqa: 0.791,
967
+ hle: 0.111,
987
968
 
988
969
  // Capabilities
989
970
  contextWindow: 8192,
@@ -991,18 +972,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
991
972
  supportsVision: false,
992
973
 
993
974
  // Metadata
994
- lastUpdated: "2026-06-01",
995
- originalModel: "Solar Mini",
975
+ lastUpdated: "2026-07-01",
976
+ originalModel: "Grok 3 mini Reasoning (high)",
996
977
  },
997
- "solar-pro-2-preview-reasoning": {
978
+ "grok-4.1-fast-reasoning": {
998
979
  // AA specific benchmarks
999
980
  codingIndex: undefined,
1000
- mathIndex: undefined,
981
+ mathIndex: 89.3,
1001
982
 
1002
983
  // Academic benchmarks
1003
- mmluPro: 0.768,
1004
- gpqa: 0.578,
1005
- hle: 0.057,
984
+ mmluPro: 0.854,
985
+ gpqa: 0.853,
986
+ hle: 0.176,
1006
987
 
1007
988
  // Capabilities
1008
989
  contextWindow: 8192,
@@ -1010,18 +991,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1010
991
  supportsVision: false,
1011
992
 
1012
993
  // Metadata
1013
- lastUpdated: "2026-06-01",
1014
- originalModel: "Solar Pro 2 (Preview) (Reasoning)",
994
+ lastUpdated: "2026-07-01",
995
+ originalModel: "Grok 4.1 Fast (Reasoning)",
1015
996
  },
1016
- "solar-pro-2-preview-non-reasoning": {
997
+ "grok-4.20-0309-v2-reasoning": {
1017
998
  // AA specific benchmarks
1018
999
  codingIndex: undefined,
1019
1000
  mathIndex: undefined,
1020
1001
 
1021
1002
  // Academic benchmarks
1022
- mmluPro: 0.725,
1023
- gpqa: 0.544,
1024
- hle: 0.038,
1003
+ mmluPro: undefined,
1004
+ gpqa: 0.911,
1005
+ hle: 0.322,
1025
1006
 
1026
1007
  // Capabilities
1027
1008
  contextWindow: 8192,
@@ -1029,18 +1010,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1029
1010
  supportsVision: false,
1030
1011
 
1031
1012
  // Metadata
1032
- lastUpdated: "2026-06-01",
1033
- originalModel: "Solar Pro 2 (Preview) (Non-reasoning)",
1013
+ lastUpdated: "2026-07-01",
1014
+ originalModel: "Grok 4.20 0309 v2 (Reasoning)",
1034
1015
  },
1035
- "dbrx-instruct": {
1016
+ "grok-code-fast-1": {
1036
1017
  // AA specific benchmarks
1037
1018
  codingIndex: undefined,
1038
- mathIndex: undefined,
1019
+ mathIndex: 43.3,
1039
1020
 
1040
1021
  // Academic benchmarks
1041
- mmluPro: 0.397,
1042
- gpqa: 0.331,
1043
- hle: 0.066,
1022
+ mmluPro: 0.793,
1023
+ gpqa: 0.727,
1024
+ hle: 0.075,
1044
1025
 
1045
1026
  // Capabilities
1046
1027
  contextWindow: 8192,
@@ -1048,18 +1029,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1048
1029
  supportsVision: false,
1049
1030
 
1050
1031
  // Metadata
1051
- lastUpdated: "2026-06-01",
1052
- originalModel: "DBRX Instruct",
1032
+ lastUpdated: "2026-07-01",
1033
+ originalModel: "Grok Code Fast 1",
1053
1034
  },
1054
- "minimax-m2.1": {
1035
+ "grok-4.1-fast-non-reasoning": {
1055
1036
  // AA specific benchmarks
1056
- codingIndex: 32.8,
1057
- mathIndex: 82.7,
1037
+ codingIndex: undefined,
1038
+ mathIndex: 34.3,
1058
1039
 
1059
1040
  // Academic benchmarks
1060
- mmluPro: 0.875,
1061
- gpqa: 0.83,
1062
- hle: 0.222,
1041
+ mmluPro: 0.743,
1042
+ gpqa: 0.637,
1043
+ hle: 0.05,
1063
1044
 
1064
1045
  // Capabilities
1065
1046
  contextWindow: 8192,
@@ -1067,18 +1048,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1067
1048
  supportsVision: false,
1068
1049
 
1069
1050
  // Metadata
1070
- lastUpdated: "2026-06-01",
1071
- originalModel: "MiniMax-M2.1",
1051
+ lastUpdated: "2026-07-01",
1052
+ originalModel: "Grok 4.1 Fast (Non-reasoning)",
1072
1053
  },
1073
- "minimax-m2.5": {
1054
+ "grok-4": {
1074
1055
  // AA specific benchmarks
1075
- codingIndex: 37.4,
1076
- mathIndex: undefined,
1056
+ codingIndex: undefined,
1057
+ mathIndex: 92.7,
1077
1058
 
1078
1059
  // Academic benchmarks
1079
- mmluPro: undefined,
1080
- gpqa: 0.848,
1081
- hle: 0.191,
1060
+ mmluPro: 0.866,
1061
+ gpqa: 0.877,
1062
+ hle: 0.239,
1082
1063
 
1083
1064
  // Capabilities
1084
1065
  contextWindow: 8192,
@@ -1086,18 +1067,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1086
1067
  supportsVision: false,
1087
1068
 
1088
1069
  // Metadata
1089
- lastUpdated: "2026-06-01",
1090
- originalModel: "MiniMax-M2.5",
1070
+ lastUpdated: "2026-07-01",
1071
+ originalModel: "Grok 4",
1091
1072
  },
1092
- "minimax-m2": {
1073
+ "grok-4-fast-reasoning": {
1093
1074
  // AA specific benchmarks
1094
- codingIndex: 29.2,
1095
- mathIndex: 78.3,
1075
+ codingIndex: undefined,
1076
+ mathIndex: 89.7,
1096
1077
 
1097
1078
  // Academic benchmarks
1098
- mmluPro: 0.82,
1099
- gpqa: 0.777,
1100
- hle: 0.125,
1079
+ mmluPro: 0.85,
1080
+ gpqa: 0.847,
1081
+ hle: 0.17,
1101
1082
 
1102
1083
  // Capabilities
1103
1084
  contextWindow: 8192,
@@ -1105,18 +1086,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1105
1086
  supportsVision: false,
1106
1087
 
1107
1088
  // Metadata
1108
- lastUpdated: "2026-06-01",
1109
- originalModel: "MiniMax-M2",
1089
+ lastUpdated: "2026-07-01",
1090
+ originalModel: "Grok 4 Fast (Reasoning)",
1110
1091
  },
1111
- "minimax-m1-80k": {
1092
+ "grok-4-fast-non-reasoning": {
1112
1093
  // AA specific benchmarks
1113
- codingIndex: 14.5,
1114
- mathIndex: 61,
1094
+ codingIndex: undefined,
1095
+ mathIndex: 41.3,
1115
1096
 
1116
1097
  // Academic benchmarks
1117
- mmluPro: 0.816,
1118
- gpqa: 0.697,
1119
- hle: 0.082,
1098
+ mmluPro: 0.73,
1099
+ gpqa: 0.606,
1100
+ hle: 0.05,
1120
1101
 
1121
1102
  // Capabilities
1122
1103
  contextWindow: 8192,
@@ -1124,18 +1105,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1124
1105
  supportsVision: false,
1125
1106
 
1126
1107
  // Metadata
1127
- lastUpdated: "2026-06-01",
1128
- originalModel: "MiniMax M1 80k",
1108
+ lastUpdated: "2026-07-01",
1109
+ originalModel: "Grok 4 Fast (Non-reasoning)",
1129
1110
  },
1130
- "minimax-m1-40k": {
1111
+ "grok-4.20-0309-reasoning": {
1131
1112
  // AA specific benchmarks
1132
- codingIndex: 14.1,
1133
- mathIndex: 13.7,
1113
+ codingIndex: undefined,
1114
+ mathIndex: undefined,
1134
1115
 
1135
1116
  // Academic benchmarks
1136
- mmluPro: 0.808,
1137
- gpqa: 0.682,
1138
- hle: 0.075,
1117
+ mmluPro: undefined,
1118
+ gpqa: 0.885,
1119
+ hle: 0.3,
1139
1120
 
1140
1121
  // Capabilities
1141
1122
  contextWindow: 8192,
@@ -1143,18 +1124,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1143
1124
  supportsVision: false,
1144
1125
 
1145
1126
  // Metadata
1146
- lastUpdated: "2026-06-01",
1147
- originalModel: "MiniMax M1 40k",
1127
+ lastUpdated: "2026-07-01",
1128
+ originalModel: "Grok 4.20 0309 (Reasoning)",
1148
1129
  },
1149
- "llama-3.3-nemotron-super-49b-v1-reasoning": {
1130
+ "grok-4.20-0309-non-reasoning": {
1150
1131
  // AA specific benchmarks
1151
- codingIndex: 9.4,
1152
- mathIndex: 54.7,
1132
+ codingIndex: undefined,
1133
+ mathIndex: undefined,
1153
1134
 
1154
1135
  // Academic benchmarks
1155
- mmluPro: 0.785,
1156
- gpqa: 0.643,
1157
- hle: 0.065,
1136
+ mmluPro: undefined,
1137
+ gpqa: 0.785,
1138
+ hle: 0.225,
1158
1139
 
1159
1140
  // Capabilities
1160
1141
  contextWindow: 8192,
@@ -1162,18 +1143,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1162
1143
  supportsVision: false,
1163
1144
 
1164
1145
  // Metadata
1165
- lastUpdated: "2026-06-01",
1166
- originalModel: "Llama 3.3 Nemotron Super 49B v1 (Reasoning)",
1146
+ lastUpdated: "2026-07-01",
1147
+ originalModel: "Grok 4.20 0309 (Non-reasoning)",
1167
1148
  },
1168
- "llama-3.3-nemotron-super-49b-v1-non-reasoning": {
1149
+ "grok-2-dec-24": {
1169
1150
  // AA specific benchmarks
1170
- codingIndex: 7.6,
1171
- mathIndex: 7.7,
1151
+ codingIndex: undefined,
1152
+ mathIndex: undefined,
1172
1153
 
1173
1154
  // Academic benchmarks
1174
- mmluPro: 0.698,
1175
- gpqa: 0.517,
1176
- hle: 0.035,
1155
+ mmluPro: 0.709,
1156
+ gpqa: 0.51,
1157
+ hle: 0.038,
1177
1158
 
1178
1159
  // Capabilities
1179
1160
  contextWindow: 8192,
@@ -1181,18 +1162,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1181
1162
  supportsVision: false,
1182
1163
 
1183
1164
  // Metadata
1184
- lastUpdated: "2026-06-01",
1185
- originalModel: "Llama 3.3 Nemotron Super 49B v1 (Non-reasoning)",
1165
+ lastUpdated: "2026-07-01",
1166
+ originalModel: "Grok 2 (Dec '24)",
1186
1167
  },
1187
- "llama-3.1-nemotron-nano-4b-v1.1-reasoning": {
1168
+ "grok-3-reasoning-beta": {
1188
1169
  // AA specific benchmarks
1189
1170
  codingIndex: undefined,
1190
- mathIndex: 50,
1171
+ mathIndex: undefined,
1191
1172
 
1192
1173
  // Academic benchmarks
1193
- mmluPro: 0.556,
1194
- gpqa: 0.408,
1195
- hle: 0.051,
1174
+ mmluPro: undefined,
1175
+ gpqa: undefined,
1176
+ hle: undefined,
1196
1177
 
1197
1178
  // Capabilities
1198
1179
  contextWindow: 8192,
@@ -1200,18 +1181,37 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1200
1181
  supportsVision: false,
1201
1182
 
1202
1183
  // Metadata
1203
- lastUpdated: "2026-06-01",
1204
- originalModel: "Llama 3.1 Nemotron Nano 4B v1.1 (Reasoning)",
1184
+ lastUpdated: "2026-07-01",
1185
+ originalModel: "Grok 3 Reasoning Beta",
1186
+ },
1187
+ "grok-4.20-0309-v2-non-reasoning": {
1188
+ // AA specific benchmarks
1189
+ codingIndex: undefined,
1190
+ mathIndex: undefined,
1191
+
1192
+ // Academic benchmarks
1193
+ mmluPro: undefined,
1194
+ gpqa: 0.776,
1195
+ hle: 0.242,
1196
+
1197
+ // Capabilities
1198
+ contextWindow: 8192,
1199
+ supportsReasoning: false,
1200
+ supportsVision: false,
1201
+
1202
+ // Metadata
1203
+ lastUpdated: "2026-07-01",
1204
+ originalModel: "Grok 4.20 0309 v2 (Non-reasoning)",
1205
1205
  },
1206
- "kimi-k2.5-reasoning": {
1206
+ "openchat-3.5-1210": {
1207
1207
  // AA specific benchmarks
1208
- codingIndex: 39.6,
1208
+ codingIndex: undefined,
1209
1209
  mathIndex: undefined,
1210
1210
 
1211
1211
  // Academic benchmarks
1212
- mmluPro: undefined,
1213
- gpqa: 0.879,
1214
- hle: 0.294,
1212
+ mmluPro: 0.31,
1213
+ gpqa: 0.23,
1214
+ hle: 0.048,
1215
1215
 
1216
1216
  // Capabilities
1217
1217
  contextWindow: 8192,
@@ -1219,18 +1219,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1219
1219
  supportsVision: false,
1220
1220
 
1221
1221
  // Metadata
1222
- lastUpdated: "2026-06-01",
1223
- originalModel: "Kimi K2.5 (Reasoning)",
1222
+ lastUpdated: "2026-07-01",
1223
+ originalModel: "OpenChat 3.5 (1210)",
1224
1224
  },
1225
- "kimi-k2-0905": {
1225
+ "nova-pro": {
1226
1226
  // AA specific benchmarks
1227
- codingIndex: 25.9,
1228
- mathIndex: 57.3,
1227
+ codingIndex: undefined,
1228
+ mathIndex: 7,
1229
1229
 
1230
1230
  // Academic benchmarks
1231
- mmluPro: 0.819,
1232
- gpqa: 0.767,
1233
- hle: 0.063,
1231
+ mmluPro: 0.691,
1232
+ gpqa: 0.499,
1233
+ hle: 0.034,
1234
1234
 
1235
1235
  // Capabilities
1236
1236
  contextWindow: 8192,
@@ -1238,18 +1238,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1238
1238
  supportsVision: false,
1239
1239
 
1240
1240
  // Metadata
1241
- lastUpdated: "2026-06-01",
1242
- originalModel: "Kimi K2 0905",
1241
+ lastUpdated: "2026-07-01",
1242
+ originalModel: "Nova Pro",
1243
1243
  },
1244
- "kimi-k2": {
1244
+ "nova-lite": {
1245
1245
  // AA specific benchmarks
1246
- codingIndex: 22.1,
1247
- mathIndex: 57,
1246
+ codingIndex: undefined,
1247
+ mathIndex: 7,
1248
1248
 
1249
1249
  // Academic benchmarks
1250
- mmluPro: 0.824,
1251
- gpqa: 0.766,
1252
- hle: 0.07,
1250
+ mmluPro: 0.59,
1251
+ gpqa: 0.433,
1252
+ hle: 0.046,
1253
1253
 
1254
1254
  // Capabilities
1255
1255
  contextWindow: 8192,
@@ -1257,18 +1257,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1257
1257
  supportsVision: false,
1258
1258
 
1259
1259
  // Metadata
1260
- lastUpdated: "2026-06-01",
1261
- originalModel: "Kimi K2",
1260
+ lastUpdated: "2026-07-01",
1261
+ originalModel: "Nova Lite",
1262
1262
  },
1263
- "kimi-k2-thinking": {
1263
+ "phi-3-mini-instruct-3.8b": {
1264
1264
  // AA specific benchmarks
1265
- codingIndex: 34.8,
1266
- mathIndex: 94.7,
1265
+ codingIndex: undefined,
1266
+ mathIndex: 0.3,
1267
1267
 
1268
1268
  // Academic benchmarks
1269
- mmluPro: 0.848,
1270
- gpqa: 0.838,
1271
- hle: 0.223,
1269
+ mmluPro: 0.435,
1270
+ gpqa: 0.319,
1271
+ hle: 0.044,
1272
1272
 
1273
1273
  // Capabilities
1274
1274
  contextWindow: 8192,
@@ -1276,18 +1276,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1276
1276
  supportsVision: false,
1277
1277
 
1278
1278
  // Metadata
1279
- lastUpdated: "2026-06-01",
1280
- originalModel: "Kimi K2 Thinking",
1279
+ lastUpdated: "2026-07-01",
1280
+ originalModel: "Phi-3 Mini Instruct 3.8B",
1281
1281
  },
1282
- "kimi-k2.5-non-reasoning": {
1282
+ "lfm-40b": {
1283
1283
  // AA specific benchmarks
1284
- codingIndex: 25.8,
1284
+ codingIndex: undefined,
1285
1285
  mathIndex: undefined,
1286
1286
 
1287
1287
  // Academic benchmarks
1288
- mmluPro: undefined,
1289
- gpqa: 0.789,
1290
- hle: 0.123,
1288
+ mmluPro: 0.425,
1289
+ gpqa: 0.327,
1290
+ hle: 0.049,
1291
1291
 
1292
1292
  // Capabilities
1293
1293
  contextWindow: 8192,
@@ -1295,18 +1295,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1295
1295
  supportsVision: false,
1296
1296
 
1297
1297
  // Metadata
1298
- lastUpdated: "2026-06-01",
1299
- originalModel: "Kimi K2.5 (Non-reasoning)",
1298
+ lastUpdated: "2026-07-01",
1299
+ originalModel: "LFM 40B",
1300
1300
  },
1301
- "step-3.5-flash": {
1301
+ "lfm2-1.2b": {
1302
1302
  // AA specific benchmarks
1303
- codingIndex: 31.6,
1304
- mathIndex: undefined,
1303
+ codingIndex: undefined,
1304
+ mathIndex: 3.3,
1305
1305
 
1306
1306
  // Academic benchmarks
1307
- mmluPro: undefined,
1308
- gpqa: 0.831,
1309
- hle: 0.191,
1307
+ mmluPro: 0.257,
1308
+ gpqa: 0.228,
1309
+ hle: 0.057,
1310
1310
 
1311
1311
  // Capabilities
1312
1312
  contextWindow: 8192,
@@ -1314,18 +1314,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1314
1314
  supportsVision: false,
1315
1315
 
1316
1316
  // Metadata
1317
- lastUpdated: "2026-06-01",
1318
- originalModel: "Step 3.5 Flash",
1317
+ lastUpdated: "2026-07-01",
1318
+ originalModel: "LFM2 1.2B",
1319
1319
  },
1320
- "llama-3.1-tulu3-405b": {
1320
+ "solar-mini": {
1321
1321
  // AA specific benchmarks
1322
1322
  codingIndex: undefined,
1323
1323
  mathIndex: undefined,
1324
1324
 
1325
1325
  // Academic benchmarks
1326
- mmluPro: 0.716,
1327
- gpqa: 0.516,
1328
- hle: 0.035,
1326
+ mmluPro: undefined,
1327
+ gpqa: undefined,
1328
+ hle: undefined,
1329
1329
 
1330
1330
  // Capabilities
1331
1331
  contextWindow: 8192,
@@ -1333,18 +1333,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1333
1333
  supportsVision: false,
1334
1334
 
1335
1335
  // Metadata
1336
- lastUpdated: "2026-06-01",
1337
- originalModel: "Llama 3.1 Tulu3 405B",
1336
+ lastUpdated: "2026-07-01",
1337
+ originalModel: "Solar Mini",
1338
1338
  },
1339
- "olmo-2-7b": {
1339
+ "solar-pro-2-preview-non-reasoning": {
1340
1340
  // AA specific benchmarks
1341
- codingIndex: 1.2,
1342
- mathIndex: 0.7,
1341
+ codingIndex: undefined,
1342
+ mathIndex: undefined,
1343
1343
 
1344
1344
  // Academic benchmarks
1345
- mmluPro: 0.282,
1346
- gpqa: 0.288,
1347
- hle: 0.055,
1345
+ mmluPro: 0.725,
1346
+ gpqa: 0.544,
1347
+ hle: 0.038,
1348
1348
 
1349
1349
  // Capabilities
1350
1350
  contextWindow: 8192,
@@ -1352,18 +1352,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1352
1352
  supportsVision: false,
1353
1353
 
1354
1354
  // Metadata
1355
- lastUpdated: "2026-06-01",
1356
- originalModel: "OLMo 2 7B",
1355
+ lastUpdated: "2026-07-01",
1356
+ originalModel: "Solar Pro 2 (Preview) (Non-reasoning)",
1357
1357
  },
1358
- "olmo-3-32b-think": {
1358
+ "solar-pro-2-preview-reasoning": {
1359
1359
  // AA specific benchmarks
1360
- codingIndex: 10.5,
1361
- mathIndex: 73.7,
1360
+ codingIndex: undefined,
1361
+ mathIndex: undefined,
1362
1362
 
1363
1363
  // Academic benchmarks
1364
- mmluPro: 0.759,
1365
- gpqa: 0.61,
1366
- hle: 0.059,
1364
+ mmluPro: 0.768,
1365
+ gpqa: 0.578,
1366
+ hle: 0.057,
1367
1367
 
1368
1368
  // Capabilities
1369
1369
  contextWindow: 8192,
@@ -1371,18 +1371,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1371
1371
  supportsVision: false,
1372
1372
 
1373
1373
  // Metadata
1374
- lastUpdated: "2026-06-01",
1375
- originalModel: "Olmo 3 32B Think",
1374
+ lastUpdated: "2026-07-01",
1375
+ originalModel: "Solar Pro 2 (Preview) (Reasoning)",
1376
1376
  },
1377
- "olmo-2-32b": {
1377
+ "dbrx-instruct": {
1378
1378
  // AA specific benchmarks
1379
- codingIndex: 2.7,
1380
- mathIndex: 3.3,
1379
+ codingIndex: undefined,
1380
+ mathIndex: undefined,
1381
1381
 
1382
1382
  // Academic benchmarks
1383
- mmluPro: 0.511,
1384
- gpqa: 0.328,
1385
- hle: 0.037,
1383
+ mmluPro: 0.397,
1384
+ gpqa: 0.331,
1385
+ hle: 0.066,
1386
1386
 
1387
1387
  // Capabilities
1388
1388
  contextWindow: 8192,
@@ -1390,18 +1390,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1390
1390
  supportsVision: false,
1391
1391
 
1392
1392
  // Metadata
1393
- lastUpdated: "2026-06-01",
1394
- originalModel: "OLMo 2 32B",
1393
+ lastUpdated: "2026-07-01",
1394
+ originalModel: "DBRX Instruct",
1395
1395
  },
1396
- "granite-3.3-8b-non-reasoning": {
1396
+ "minimax-m2.5": {
1397
1397
  // AA specific benchmarks
1398
- codingIndex: 3.4,
1399
- mathIndex: 6.7,
1398
+ codingIndex: undefined,
1399
+ mathIndex: undefined,
1400
1400
 
1401
1401
  // Academic benchmarks
1402
- mmluPro: 0.468,
1403
- gpqa: 0.338,
1404
- hle: 0.042,
1402
+ mmluPro: undefined,
1403
+ gpqa: 0.848,
1404
+ hle: 0.191,
1405
1405
 
1406
1406
  // Capabilities
1407
1407
  contextWindow: 8192,
@@ -1409,18 +1409,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1409
1409
  supportsVision: false,
1410
1410
 
1411
1411
  // Metadata
1412
- lastUpdated: "2026-06-01",
1413
- originalModel: "Granite 3.3 8B (Non-reasoning)",
1412
+ lastUpdated: "2026-07-01",
1413
+ originalModel: "MiniMax-M2.5",
1414
1414
  },
1415
- "reka-flash-sep-24": {
1415
+ "minimax-m2": {
1416
1416
  // AA specific benchmarks
1417
1417
  codingIndex: undefined,
1418
- mathIndex: undefined,
1418
+ mathIndex: 78.3,
1419
1419
 
1420
1420
  // Academic benchmarks
1421
- mmluPro: undefined,
1422
- gpqa: undefined,
1423
- hle: undefined,
1421
+ mmluPro: 0.82,
1422
+ gpqa: 0.777,
1423
+ hle: 0.125,
1424
1424
 
1425
1425
  // Capabilities
1426
1426
  contextWindow: 8192,
@@ -1428,18 +1428,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1428
1428
  supportsVision: false,
1429
1429
 
1430
1430
  // Metadata
1431
- lastUpdated: "2026-06-01",
1432
- originalModel: "Reka Flash (Sep '24)",
1431
+ lastUpdated: "2026-07-01",
1432
+ originalModel: "MiniMax-M2",
1433
1433
  },
1434
- "hermes-3---llama-3.1-70b": {
1434
+ "minimax-m1-80k": {
1435
1435
  // AA specific benchmarks
1436
1436
  codingIndex: undefined,
1437
- mathIndex: undefined,
1437
+ mathIndex: 61,
1438
1438
 
1439
1439
  // Academic benchmarks
1440
- mmluPro: 0.571,
1441
- gpqa: 0.401,
1442
- hle: 0.041,
1440
+ mmluPro: 0.816,
1441
+ gpqa: 0.697,
1442
+ hle: 0.082,
1443
1443
 
1444
1444
  // Capabilities
1445
1445
  contextWindow: 8192,
@@ -1447,18 +1447,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1447
1447
  supportsVision: false,
1448
1448
 
1449
1449
  // Metadata
1450
- lastUpdated: "2026-06-01",
1451
- originalModel: "Hermes 3 - Llama-3.1 70B",
1450
+ lastUpdated: "2026-07-01",
1451
+ originalModel: "MiniMax M1 80k",
1452
1452
  },
1453
- "mimo-v2-flash-reasoning": {
1453
+ "minimax-m2.1": {
1454
1454
  // AA specific benchmarks
1455
- codingIndex: 31.8,
1456
- mathIndex: 96.3,
1455
+ codingIndex: undefined,
1456
+ mathIndex: 82.7,
1457
1457
 
1458
1458
  // Academic benchmarks
1459
- mmluPro: 0.843,
1460
- gpqa: 0.846,
1461
- hle: 0.211,
1459
+ mmluPro: 0.875,
1460
+ gpqa: 0.83,
1461
+ hle: 0.222,
1462
1462
 
1463
1463
  // Capabilities
1464
1464
  contextWindow: 8192,
@@ -1466,18 +1466,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1466
1466
  supportsVision: false,
1467
1467
 
1468
1468
  // Metadata
1469
- lastUpdated: "2026-06-01",
1470
- originalModel: "MiMo-V2-Flash (Reasoning)",
1469
+ lastUpdated: "2026-07-01",
1470
+ originalModel: "MiniMax-M2.1",
1471
1471
  },
1472
- "mimo-v2-pro": {
1472
+ "minimax-m1-40k": {
1473
1473
  // AA specific benchmarks
1474
- codingIndex: 41.4,
1475
- mathIndex: undefined,
1474
+ codingIndex: undefined,
1475
+ mathIndex: 13.7,
1476
1476
 
1477
1477
  // Academic benchmarks
1478
- mmluPro: undefined,
1479
- gpqa: 0.87,
1480
- hle: 0.283,
1478
+ mmluPro: 0.808,
1479
+ gpqa: 0.682,
1480
+ hle: 0.075,
1481
1481
 
1482
1482
  // Capabilities
1483
1483
  contextWindow: 8192,
@@ -1485,18 +1485,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1485
1485
  supportsVision: false,
1486
1486
 
1487
1487
  // Metadata
1488
- lastUpdated: "2026-06-01",
1489
- originalModel: "MiMo-V2-Pro",
1488
+ lastUpdated: "2026-07-01",
1489
+ originalModel: "MiniMax M1 40k",
1490
1490
  },
1491
- "sarvam-m-reasoning": {
1491
+ "llama-3.3-nemotron-super-49b-v1-reasoning": {
1492
1492
  // AA specific benchmarks
1493
- codingIndex: 7.5,
1494
- mathIndex: undefined,
1493
+ codingIndex: undefined,
1494
+ mathIndex: 54.7,
1495
1495
 
1496
1496
  // Academic benchmarks
1497
- mmluPro: 0.696,
1498
- gpqa: 0.416,
1499
- hle: 0.033,
1497
+ mmluPro: 0.785,
1498
+ gpqa: 0.643,
1499
+ hle: 0.065,
1500
1500
 
1501
1501
  // Capabilities
1502
1502
  contextWindow: 8192,
@@ -1504,18 +1504,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1504
1504
  supportsVision: false,
1505
1505
 
1506
1506
  // Metadata
1507
- lastUpdated: "2026-06-01",
1508
- originalModel: "Sarvam M (Reasoning)",
1507
+ lastUpdated: "2026-07-01",
1508
+ originalModel: "Llama 3.3 Nemotron Super 49B v1 (Reasoning)",
1509
1509
  },
1510
- "glm-4.6-non-reasoning": {
1510
+ "llama-3.1-nemotron-nano-4b-v1.1-reasoning": {
1511
1511
  // AA specific benchmarks
1512
- codingIndex: 30.2,
1513
- mathIndex: 44.3,
1512
+ codingIndex: undefined,
1513
+ mathIndex: 50,
1514
1514
 
1515
1515
  // Academic benchmarks
1516
- mmluPro: 0.784,
1517
- gpqa: 0.632,
1518
- hle: 0.052,
1516
+ mmluPro: 0.556,
1517
+ gpqa: 0.408,
1518
+ hle: 0.051,
1519
1519
 
1520
1520
  // Capabilities
1521
1521
  contextWindow: 8192,
@@ -1523,18 +1523,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1523
1523
  supportsVision: false,
1524
1524
 
1525
1525
  // Metadata
1526
- lastUpdated: "2026-06-01",
1527
- originalModel: "GLM-4.6 (Non-reasoning)",
1526
+ lastUpdated: "2026-07-01",
1527
+ originalModel: "Llama 3.1 Nemotron Nano 4B v1.1 (Reasoning)",
1528
1528
  },
1529
- "glm-4.7-reasoning": {
1529
+ "llama-3.3-nemotron-super-49b-v1-non-reasoning": {
1530
1530
  // AA specific benchmarks
1531
- codingIndex: 36.3,
1532
- mathIndex: 95,
1531
+ codingIndex: undefined,
1532
+ mathIndex: 7.7,
1533
1533
 
1534
1534
  // Academic benchmarks
1535
- mmluPro: 0.856,
1536
- gpqa: 0.859,
1537
- hle: 0.251,
1535
+ mmluPro: 0.698,
1536
+ gpqa: 0.517,
1537
+ hle: 0.035,
1538
1538
 
1539
1539
  // Capabilities
1540
1540
  contextWindow: 8192,
@@ -1542,18 +1542,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1542
1542
  supportsVision: false,
1543
1543
 
1544
1544
  // Metadata
1545
- lastUpdated: "2026-06-01",
1546
- originalModel: "GLM-4.7 (Reasoning)",
1545
+ lastUpdated: "2026-07-01",
1546
+ originalModel: "Llama 3.3 Nemotron Super 49B v1 (Non-reasoning)",
1547
1547
  },
1548
- "glm-5-reasoning": {
1548
+ "kimi-k2-thinking": {
1549
1549
  // AA specific benchmarks
1550
- codingIndex: 44.2,
1551
- mathIndex: undefined,
1550
+ codingIndex: undefined,
1551
+ mathIndex: 94.7,
1552
1552
 
1553
1553
  // Academic benchmarks
1554
- mmluPro: undefined,
1555
- gpqa: 0.82,
1556
- hle: 0.272,
1554
+ mmluPro: 0.848,
1555
+ gpqa: 0.838,
1556
+ hle: 0.223,
1557
1557
 
1558
1558
  // Capabilities
1559
1559
  contextWindow: 8192,
@@ -1561,18 +1561,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1561
1561
  supportsVision: false,
1562
1562
 
1563
1563
  // Metadata
1564
- lastUpdated: "2026-06-01",
1565
- originalModel: "GLM-5 (Reasoning)",
1564
+ lastUpdated: "2026-07-01",
1565
+ originalModel: "Kimi K2 Thinking",
1566
1566
  },
1567
- "glm-4.6-reasoning": {
1567
+ "kimi-k2.5-non-reasoning": {
1568
1568
  // AA specific benchmarks
1569
- codingIndex: 29.5,
1570
- mathIndex: 86,
1569
+ codingIndex: undefined,
1570
+ mathIndex: undefined,
1571
1571
 
1572
1572
  // Academic benchmarks
1573
- mmluPro: 0.829,
1574
- gpqa: 0.78,
1575
- hle: 0.133,
1573
+ mmluPro: undefined,
1574
+ gpqa: 0.789,
1575
+ hle: 0.123,
1576
1576
 
1577
1577
  // Capabilities
1578
1578
  contextWindow: 8192,
@@ -1580,18 +1580,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1580
1580
  supportsVision: false,
1581
1581
 
1582
1582
  // Metadata
1583
- lastUpdated: "2026-06-01",
1584
- originalModel: "GLM-4.6 (Reasoning)",
1583
+ lastUpdated: "2026-07-01",
1584
+ originalModel: "Kimi K2.5 (Non-reasoning)",
1585
1585
  },
1586
- "glm-4.7-non-reasoning": {
1586
+ "kimi-k2.5-reasoning": {
1587
1587
  // AA specific benchmarks
1588
- codingIndex: 32,
1589
- mathIndex: 48,
1588
+ codingIndex: undefined,
1589
+ mathIndex: undefined,
1590
1590
 
1591
1591
  // Academic benchmarks
1592
- mmluPro: 0.794,
1593
- gpqa: 0.664,
1594
- hle: 0.061,
1592
+ mmluPro: undefined,
1593
+ gpqa: 0.879,
1594
+ hle: 0.294,
1595
1595
 
1596
1596
  // Capabilities
1597
1597
  contextWindow: 8192,
@@ -1599,18 +1599,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1599
1599
  supportsVision: false,
1600
1600
 
1601
1601
  // Metadata
1602
- lastUpdated: "2026-06-01",
1603
- originalModel: "GLM-4.7 (Non-reasoning)",
1602
+ lastUpdated: "2026-07-01",
1603
+ originalModel: "Kimi K2.5 (Reasoning)",
1604
1604
  },
1605
- "glm-4.5-reasoning": {
1605
+ "kimi-k2": {
1606
1606
  // AA specific benchmarks
1607
- codingIndex: 26.3,
1608
- mathIndex: 73.7,
1607
+ codingIndex: undefined,
1608
+ mathIndex: 57,
1609
1609
 
1610
1610
  // Academic benchmarks
1611
- mmluPro: 0.835,
1612
- gpqa: 0.782,
1613
- hle: 0.122,
1611
+ mmluPro: 0.824,
1612
+ gpqa: 0.766,
1613
+ hle: 0.07,
1614
1614
 
1615
1615
  // Capabilities
1616
1616
  contextWindow: 8192,
@@ -1618,18 +1618,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1618
1618
  supportsVision: false,
1619
1619
 
1620
1620
  // Metadata
1621
- lastUpdated: "2026-06-01",
1622
- originalModel: "GLM-4.5 (Reasoning)",
1621
+ lastUpdated: "2026-07-01",
1622
+ originalModel: "Kimi K2",
1623
1623
  },
1624
- "glm-4.7-flash-reasoning": {
1624
+ "kimi-k2-0905": {
1625
1625
  // AA specific benchmarks
1626
- codingIndex: 25.9,
1627
- mathIndex: undefined,
1626
+ codingIndex: undefined,
1627
+ mathIndex: 57.3,
1628
1628
 
1629
1629
  // Academic benchmarks
1630
- mmluPro: undefined,
1631
- gpqa: 0.581,
1632
- hle: 0.071,
1630
+ mmluPro: 0.819,
1631
+ gpqa: 0.767,
1632
+ hle: 0.063,
1633
1633
 
1634
1634
  // Capabilities
1635
1635
  contextWindow: 8192,
@@ -1637,18 +1637,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1637
1637
  supportsVision: false,
1638
1638
 
1639
1639
  // Metadata
1640
- lastUpdated: "2026-06-01",
1641
- originalModel: "GLM-4.7-Flash (Reasoning)",
1640
+ lastUpdated: "2026-07-01",
1641
+ originalModel: "Kimi K2 0905",
1642
1642
  },
1643
- "glm-5-non-reasoning": {
1643
+ "step-3.5-flash": {
1644
1644
  // AA specific benchmarks
1645
- codingIndex: 39,
1645
+ codingIndex: undefined,
1646
1646
  mathIndex: undefined,
1647
1647
 
1648
1648
  // Academic benchmarks
1649
1649
  mmluPro: undefined,
1650
- gpqa: 0.666,
1651
- hle: 0.072,
1650
+ gpqa: 0.831,
1651
+ hle: 0.191,
1652
1652
 
1653
1653
  // Capabilities
1654
1654
  contextWindow: 8192,
@@ -1656,18 +1656,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1656
1656
  supportsVision: false,
1657
1657
 
1658
1658
  // Metadata
1659
- lastUpdated: "2026-06-01",
1660
- originalModel: "GLM-5 (Non-reasoning)",
1659
+ lastUpdated: "2026-07-01",
1660
+ originalModel: "Step 3.5 Flash",
1661
1661
  },
1662
- "glm-4.7-flash-non-reasoning": {
1662
+ "llama-3.1-tulu3-405b": {
1663
1663
  // AA specific benchmarks
1664
- codingIndex: 11,
1664
+ codingIndex: undefined,
1665
1665
  mathIndex: undefined,
1666
1666
 
1667
1667
  // Academic benchmarks
1668
- mmluPro: undefined,
1669
- gpqa: 0.452,
1670
- hle: 0.049,
1668
+ mmluPro: 0.716,
1669
+ gpqa: 0.516,
1670
+ hle: 0.035,
1671
1671
 
1672
1672
  // Capabilities
1673
1673
  contextWindow: 8192,
@@ -1675,18 +1675,18 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1675
1675
  supportsVision: false,
1676
1676
 
1677
1677
  // Metadata
1678
- lastUpdated: "2026-06-01",
1679
- originalModel: "GLM-4.7-Flash (Non-reasoning)",
1678
+ lastUpdated: "2026-07-01",
1679
+ originalModel: "Llama 3.1 Tulu3 405B",
1680
1680
  },
1681
- "glm-4.6v-non-reasoning": {
1681
+ "olmo-2-7b": {
1682
1682
  // AA specific benchmarks
1683
- codingIndex: 11.1,
1684
- mathIndex: 26.3,
1683
+ codingIndex: undefined,
1684
+ mathIndex: 0.7,
1685
1685
 
1686
1686
  // Academic benchmarks
1687
- mmluPro: 0.752,
1688
- gpqa: 0.566,
1689
- hle: 0.037,
1687
+ mmluPro: 0.282,
1688
+ gpqa: 0.288,
1689
+ hle: 0.055,
1690
1690
 
1691
1691
  // Capabilities
1692
1692
  contextWindow: 8192,
@@ -1694,17 +1694,17 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1694
1694
  supportsVision: false,
1695
1695
 
1696
1696
  // Metadata
1697
- lastUpdated: "2026-06-01",
1698
- originalModel: "GLM-4.6V (Non-reasoning)",
1697
+ lastUpdated: "2026-07-01",
1698
+ originalModel: "OLMo 2 7B",
1699
1699
  },
1700
- "glm-4.5v-reasoning": {
1700
+ "olmo-3-32b-think": {
1701
1701
  // AA specific benchmarks
1702
- codingIndex: 10.9,
1703
- mathIndex: 73,
1702
+ codingIndex: undefined,
1703
+ mathIndex: 73.7,
1704
1704
 
1705
1705
  // Academic benchmarks
1706
- mmluPro: 0.788,
1707
- gpqa: 0.684,
1706
+ mmluPro: 0.759,
1707
+ gpqa: 0.61,
1708
1708
  hle: 0.059,
1709
1709
 
1710
1710
  // Capabilities
@@ -1713,7 +1713,7 @@ export const BENCHMARKS_CHUNK_4: Record<string, HardcodedBenchmark> = {
1713
1713
  supportsVision: false,
1714
1714
 
1715
1715
  // Metadata
1716
- lastUpdated: "2026-06-01",
1717
- originalModel: "GLM-4.5V (Reasoning)",
1716
+ lastUpdated: "2026-07-01",
1717
+ originalModel: "Olmo 3 32B Think",
1718
1718
  },
1719
1719
  };