rtr-bench 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,497 @@
1
+ {
2
+ "checkedOn": "2026-11-14",
3
+ "basis": "Public OpenRouter metadata only; no inference calls. Availability and prices may change.",
4
+ "modelsSource": "https://openrouter.ai/api/v1/models",
5
+ "selected": [
6
+ {
7
+ "model": "z-ai/glm-5.3",
8
+ "source": "https://openrouter.ai/api/v1/models/z-ai/glm-5.3/endpoints",
9
+ "selected": {
10
+ "tag": "wafer/us",
11
+ "status": 0,
12
+ "quantization": "fp4",
13
+ "context_length": 1048576,
14
+ "max_completion_tokens": 943718,
15
+ "supported_parameters": [
16
+ "reasoning",
17
+ "include_reasoning",
18
+ "temperature",
19
+ "top_p",
20
+ "top_k",
21
+ "min_p",
22
+ "frequency_penalty",
23
+ "presence_penalty",
24
+ "repetition_penalty",
25
+ "stop",
26
+ "seed",
27
+ "max_tokens",
28
+ "logit_bias",
29
+ "response_format",
30
+ "structured_outputs",
31
+ "tools",
32
+ "tool_choice",
33
+ "logprobs",
34
+ "top_logprobs",
35
+ "reasoning_effort"
36
+ ],
37
+ "pricing": {
38
+ "prompt": "0.0000000699",
39
+ "completion": "0.0000014",
40
+ "input_cache_read": "0.0000000689",
41
+ "discount": 0
42
+ }
43
+ },
44
+ "reasoning": {
45
+ "mandatory": true,
46
+ "default_enabled": true,
47
+ "supported_efforts": [
48
+ "max",
49
+ "high",
50
+ "low"
51
+ ],
52
+ "default_effort": "max"
53
+ }
54
+ },
55
+ {
56
+ "model": "minimax/minimax-m3.1-flash",
57
+ "source": "https://openrouter.ai/api/v1/models/minimax/minimax-m3.1-flash/endpoints",
58
+ "selected": {
59
+ "tag": "inferact-vllm/nvfp4",
60
+ "status": 0,
61
+ "quantization": "nvfp4",
62
+ "context_length": 1048576,
63
+ "max_completion_tokens": 131072,
64
+ "supported_parameters": [
65
+ "reasoning",
66
+ "include_reasoning",
67
+ "frequency_penalty",
68
+ "max_tokens",
69
+ "presence_penalty",
70
+ "repetition_penalty",
71
+ "seed",
72
+ "stop",
73
+ "temperature",
74
+ "top_k",
75
+ "top_p",
76
+ "tools",
77
+ "tool_choice",
78
+ "response_format",
79
+ "structured_outputs",
80
+ "logprobs",
81
+ "top_logprobs",
82
+ "reasoning_effort"
83
+ ],
84
+ "pricing": {
85
+ "prompt": "0.000000075",
86
+ "completion": "0.0000003",
87
+ "input_cache_read": "0.0000000075",
88
+ "discount": 0
89
+ }
90
+ },
91
+ "reasoning": {
92
+ "mandatory": true,
93
+ "supported_efforts": [
94
+ "max",
95
+ "xhigh",
96
+ "high",
97
+ "medium",
98
+ "low"
99
+ ],
100
+ "default_effort": "max"
101
+ }
102
+ },
103
+ {
104
+ "model": "xiaomi/mimo-v2.6-flash",
105
+ "source": "https://openrouter.ai/api/v1/models/xiaomi/mimo-v2.6-flash/endpoints",
106
+ "selected": {
107
+ "tag": "gmicloud/bf16",
108
+ "status": -2,
109
+ "quantization": "bf16",
110
+ "context_length": 1050000,
111
+ "max_completion_tokens": 945000,
112
+ "supported_parameters": [
113
+ "reasoning",
114
+ "include_reasoning",
115
+ "max_tokens",
116
+ "temperature",
117
+ "top_p",
118
+ "seed",
119
+ "response_format",
120
+ "structured_outputs",
121
+ "tools",
122
+ "tool_choice"
123
+ ],
124
+ "pricing": {
125
+ "prompt": "0.000000112",
126
+ "completion": "0.000000224",
127
+ "input_cache_read": "0.0000000024",
128
+ "discount": 0.2
129
+ }
130
+ },
131
+ "reasoning": {
132
+ "mandatory": false
133
+ },
134
+ "candidateReplacement": {
135
+ "tag": "xiaomi/fp8",
136
+ "status": 0,
137
+ "quantization": "fp8",
138
+ "context_length": 1048576,
139
+ "max_completion_tokens": 131072,
140
+ "supported_parameters": [
141
+ "reasoning",
142
+ "include_reasoning",
143
+ "max_tokens",
144
+ "temperature",
145
+ "top_p",
146
+ "stop",
147
+ "response_format",
148
+ "tools",
149
+ "tool_choice",
150
+ "frequency_penalty",
151
+ "presence_penalty",
152
+ "structured_outputs"
153
+ ],
154
+ "pricing": {
155
+ "prompt": "0.00000014",
156
+ "completion": "0.00000028",
157
+ "input_cache_read": "0.0000000028",
158
+ "discount": 0
159
+ }
160
+ }
161
+ },
162
+ {
163
+ "model": "deepseek/deepseek-v4.1-flash-1017",
164
+ "source": "https://openrouter.ai/api/v1/models/deepseek/deepseek-v4.1-flash-1017/endpoints",
165
+ "selected": {
166
+ "tag": "sail-research/fp4/us",
167
+ "status": 0,
168
+ "quantization": "fp4",
169
+ "context_length": 1048576,
170
+ "max_completion_tokens": 384000,
171
+ "supported_parameters": [
172
+ "reasoning",
173
+ "include_reasoning",
174
+ "frequency_penalty",
175
+ "max_tokens",
176
+ "presence_penalty",
177
+ "seed",
178
+ "stop",
179
+ "temperature",
180
+ "top_k",
181
+ "top_p",
182
+ "response_format",
183
+ "structured_outputs",
184
+ "tools",
185
+ "tool_choice",
186
+ "logprobs",
187
+ "top_logprobs",
188
+ "reasoning_effort"
189
+ ],
190
+ "pricing": {
191
+ "prompt": "0.00000007272",
192
+ "completion": "0.000000108",
193
+ "input_cache_read": "0.000000007",
194
+ "discount": 0
195
+ }
196
+ },
197
+ "reasoning": {
198
+ "mandatory": false,
199
+ "default_enabled": true,
200
+ "supported_efforts": [
201
+ "max",
202
+ "high",
203
+ "low",
204
+ "none"
205
+ ],
206
+ "default_effort": "high"
207
+ }
208
+ },
209
+ {
210
+ "model": "tencent/hy3",
211
+ "source": "https://openrouter.ai/api/v1/models/tencent/hy3/endpoints",
212
+ "selected": {
213
+ "tag": "gmicloud/bf16",
214
+ "status": 0,
215
+ "quantization": "bf16",
216
+ "context_length": 262144,
217
+ "max_completion_tokens": 235929,
218
+ "supported_parameters": [
219
+ "reasoning",
220
+ "include_reasoning",
221
+ "max_tokens",
222
+ "temperature",
223
+ "top_p",
224
+ "seed",
225
+ "response_format",
226
+ "tools",
227
+ "tool_choice",
228
+ "reasoning_effort"
229
+ ],
230
+ "pricing": {
231
+ "prompt": "0.00000014",
232
+ "completion": "0.00000058",
233
+ "input_cache_read": "0.000000035",
234
+ "discount": 0
235
+ }
236
+ },
237
+ "reasoning": {
238
+ "mandatory": false,
239
+ "default_enabled": true,
240
+ "supported_efforts": [
241
+ "high",
242
+ "low",
243
+ "none"
244
+ ],
245
+ "default_effort": "high"
246
+ }
247
+ },
248
+ {
249
+ "model": "google/gemini-3.8-flash",
250
+ "source": "https://openrouter.ai/api/v1/models/google/gemini-3.8-flash/endpoints",
251
+ "selected": {
252
+ "tag": "google-ai-studio",
253
+ "status": 0,
254
+ "quantization": "unknown",
255
+ "context_length": 1048576,
256
+ "max_completion_tokens": 65536,
257
+ "supported_parameters": [
258
+ "reasoning",
259
+ "include_reasoning",
260
+ "max_tokens",
261
+ "temperature",
262
+ "top_p",
263
+ "seed",
264
+ "response_format",
265
+ "structured_outputs",
266
+ "tool_choice",
267
+ "tools",
268
+ "reasoning_effort"
269
+ ],
270
+ "pricing": {
271
+ "prompt": "0.00000075",
272
+ "completion": "0.00000375",
273
+ "image": "0.00000075",
274
+ "audio": "0.00000075",
275
+ "input_audio_cache": "0.000000075",
276
+ "web_search": "0.014",
277
+ "internal_reasoning": "0.00000375",
278
+ "input_cache_read": "0.000000075",
279
+ "input_cache_write": "0.0000000416666666666667",
280
+ "discount": 0.5
281
+ }
282
+ },
283
+ "reasoning": {
284
+ "mandatory": true,
285
+ "default_enabled": true,
286
+ "supported_efforts": [
287
+ "high",
288
+ "medium",
289
+ "low"
290
+ ],
291
+ "default_effort": "medium"
292
+ }
293
+ },
294
+ {
295
+ "model": "openai/gpt-6-luna",
296
+ "source": "https://openrouter.ai/api/v1/models/openai/gpt-6-luna/endpoints",
297
+ "selected": {
298
+ "tag": "openai",
299
+ "status": 0,
300
+ "quantization": "unknown",
301
+ "context_length": 1050000,
302
+ "max_completion_tokens": 128000,
303
+ "supported_parameters": [
304
+ "reasoning",
305
+ "include_reasoning",
306
+ "seed",
307
+ "max_tokens",
308
+ "response_format",
309
+ "structured_outputs",
310
+ "tools",
311
+ "tool_choice",
312
+ "verbosity",
313
+ "reasoning_effort"
314
+ ],
315
+ "pricing": {
316
+ "prompt": "0.0000001",
317
+ "completion": "0.0000005",
318
+ "web_search": "0.01",
319
+ "input_cache_read": "0.00000001",
320
+ "input_cache_write": "0.000000125",
321
+ "discount": 0,
322
+ "overrides": [
323
+ {
324
+ "min_prompt_tokens": 272000,
325
+ "prompt": "0.0000002",
326
+ "completion": "0.00000075",
327
+ "input_cache_read": "0.00000002",
328
+ "input_cache_write": "0.00000025"
329
+ }
330
+ ]
331
+ }
332
+ },
333
+ "reasoning": {
334
+ "mandatory": false,
335
+ "default_enabled": true,
336
+ "supported_efforts": [
337
+ "max",
338
+ "xhigh",
339
+ "high",
340
+ "medium",
341
+ "low",
342
+ "none"
343
+ ],
344
+ "default_effort": "medium"
345
+ }
346
+ },
347
+ {
348
+ "model": "anthropic/claude-haiku-5.5",
349
+ "source": "https://openrouter.ai/api/v1/models/anthropic/claude-haiku-5.5/endpoints",
350
+ "selected": {
351
+ "tag": "anthropic",
352
+ "status": 0,
353
+ "quantization": "unknown",
354
+ "context_length": 1000000,
355
+ "max_completion_tokens": 128000,
356
+ "supported_parameters": [
357
+ "reasoning",
358
+ "include_reasoning",
359
+ "max_tokens",
360
+ "stop",
361
+ "tools",
362
+ "tool_choice",
363
+ "structured_outputs",
364
+ "response_format",
365
+ "verbosity",
366
+ "reasoning_effort"
367
+ ],
368
+ "pricing": {
369
+ "prompt": "0.0000001",
370
+ "completion": "0.0000005",
371
+ "web_search": "0.01",
372
+ "input_cache_read": "0.00000001",
373
+ "input_cache_write": "0.000000125",
374
+ "input_cache_write_1h": "0.0000002",
375
+ "discount": 0,
376
+ "overrides": [
377
+ {
378
+ "min_prompt_tokens": 100000,
379
+ "prompt": "0.0000005",
380
+ "completion": "0.0000025",
381
+ "input_cache_read": "0.00000005",
382
+ "input_cache_write": "0.000000625",
383
+ "input_cache_write_1h": "0.000001"
384
+ }
385
+ ]
386
+ }
387
+ },
388
+ "reasoning": {
389
+ "mandatory": false,
390
+ "supported_efforts": [
391
+ "max",
392
+ "xhigh",
393
+ "high",
394
+ "medium",
395
+ "low",
396
+ "none"
397
+ ],
398
+ "default_effort": "medium"
399
+ }
400
+ },
401
+ {
402
+ "model": "thinkingmachines/inkling-small",
403
+ "source": "https://openrouter.ai/api/v1/models/thinkingmachines/inkling-small/endpoints",
404
+ "selected": {
405
+ "tag": "deepinfra/fp4",
406
+ "status": 0,
407
+ "quantization": "fp4",
408
+ "context_length": 524288,
409
+ "max_completion_tokens": 262144,
410
+ "supported_parameters": [
411
+ "reasoning",
412
+ "include_reasoning",
413
+ "max_tokens",
414
+ "temperature",
415
+ "top_p",
416
+ "stop",
417
+ "frequency_penalty",
418
+ "presence_penalty",
419
+ "repetition_penalty",
420
+ "top_k",
421
+ "seed",
422
+ "min_p",
423
+ "tools",
424
+ "tool_choice",
425
+ "logit_bias",
426
+ "reasoning_effort"
427
+ ],
428
+ "pricing": {
429
+ "prompt": "0.00000045",
430
+ "completion": "0.0000012",
431
+ "input_cache_read": "0.0000001",
432
+ "discount": 0
433
+ }
434
+ },
435
+ "reasoning": {
436
+ "mandatory": false,
437
+ "default_enabled": true,
438
+ "supported_efforts": [
439
+ "max",
440
+ "high",
441
+ "medium",
442
+ "low",
443
+ "minimal",
444
+ "none"
445
+ ],
446
+ "default_effort": "high"
447
+ }
448
+ },
449
+ {
450
+ "model": "openai/gpt-oss-120b",
451
+ "source": "https://openrouter.ai/api/v1/models/openai/gpt-oss-120b/endpoints",
452
+ "selected": {
453
+ "tag": "primeintellect/fp4",
454
+ "status": 0,
455
+ "quantization": "fp4",
456
+ "context_length": 131072,
457
+ "max_completion_tokens": 65536,
458
+ "supported_parameters": [
459
+ "reasoning",
460
+ "include_reasoning",
461
+ "frequency_penalty",
462
+ "logit_bias",
463
+ "max_tokens",
464
+ "min_p",
465
+ "presence_penalty",
466
+ "repetition_penalty",
467
+ "seed",
468
+ "stop",
469
+ "temperature",
470
+ "top_a",
471
+ "top_k",
472
+ "top_p",
473
+ "tools",
474
+ "response_format",
475
+ "structured_outputs",
476
+ "tool_choice",
477
+ "reasoning_effort"
478
+ ],
479
+ "pricing": {
480
+ "prompt": "0.000000035",
481
+ "completion": "0.000000075",
482
+ "input_cache_read": "0.000000031",
483
+ "discount": 0
484
+ }
485
+ },
486
+ "reasoning": {
487
+ "mandatory": true,
488
+ "supported_efforts": [
489
+ "high",
490
+ "medium",
491
+ "low"
492
+ ],
493
+ "default_effort": "medium"
494
+ }
495
+ }
496
+ ]
497
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rtr-bench",
3
- "version": "0.2.0",
3
+ "version": "0.2.1",
4
4
  "description": "Can a language model predict how a crowd reads a word? A cultural fluency benchmark built from Mixed Messages.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -28,12 +28,17 @@
28
28
  "docs/distribution.md",
29
29
  "docs/learn-room.md",
30
30
  "docs/npm-release.md",
31
+ "docs/preflight-audit.md",
32
+ "docs/preflight-providers-2026-11-14.json",
31
33
  "tests/fixtures/distribution-pilot.json",
32
34
  "tests/fixtures/learn-room-pilot.json",
33
35
  "tests/fixtures/learn-room-four-strategies.json",
34
36
  "results/leaderboard-scatter.png",
35
37
  "README.md",
36
- "LICENSE"
38
+ "LICENSE",
39
+ "docs/development-pilot-2026-11-14.json",
40
+ "docs/final-run-plan-8000.md",
41
+ "docs/final-run-plan-8000.json"
37
42
  ],
38
43
  "scripts": {
39
44
  "test": "node --test tests/*.test.ts tests/distribution/*.test.ts tests/learn-room/*.test.ts",