rtr-bench 0.1.0 → 0.1.2

package/README.md

@@ -43,21 +43,32 @@ echo 'OPENROUTER_API_KEY=sk-or-your-key' > .env
 node src/cli.ts catalog --limit 20
 
 # run a model, then score everything on disk
-node src/cli.ts run --models <model-id> --concurrency 8
+node src/cli.ts run --models z-ai/glm-5.3-flash --concurrency 8
 node src/cli.ts report
 ```
 
-Once the package is published, the same run is one command with no clone:
+Published on npm, the same run is one command with no clone:
 
 ```bash
-npx rtr-bench <model-id>
+npx rtr-bench z-ai/glm-5.3-flash
 ```
 
 Rebuilding the dataset needs the project's database connection, so `npm run export` is included for inspection only.
 
 ## Results
 
-<!-- populated after the multi-model run. results/leaderboard.md is generated by `rtr-bench report`. -->
+| Model | Provider | Score | Cost |
+|---|---|---|---|
+| z-ai/glm-5.3-flash | TBD | TBD | TBD |
+| z-ai/glm-5.3 | TBD | TBD | TBD |
+| minimax/minimax-m3.1-flash | TBD | TBD | TBD |
+| deepseek/deepseek-v4.1-flash-1017 | TBD | TBD | TBD |
+| tencent/hy4 | TBD | TBD | TBD |
+| google/gemini-3.8-flash | TBD | TBD | TBD |
+| google/gemini-4-carbon | TBD | TBD | TBD |
+| moonshotai/kimi-k3.1-pro | TBD | TBD | TBD |
+| x-ai/grok-4.7 | TBD | TBD | TBD |
+| qwen/qwen3.8-max-0902 | TBD | TBD | TBD |
 
 ## License
 

package/dist/cli.js

@@ -119,7 +119,8 @@ function readModel(dir) {
         day: rec.day,
         order: rec.order,
         task,
-        parsed: rec.parsed ?? null
+        parsed: rec.parsed ?? null,
+        cost: rec.usage?.cost ?? rec.usage?.total_cost ?? 0
       });
     }
   }
@@ -245,7 +246,8 @@ function scoreModel(model, windows) {
     crowdDistance: mean(dists),
     gap: readAgree - predictAcc,
     instability: pred.length ? pred.filter((a) => a.unstable).length / pred.length : NaN,
-    parseFailures: calls.filter((c) => !c.parsed).length
+    parseFailures: calls.filter((c) => !c.parsed).length,
+    costUsd: calls.reduce((a, c) => a + c.cost, 0)
   };
 }
 function majorityBaseline(windows) {
@@ -531,12 +533,12 @@ function writeReport() {
   lines.push("## Leaderboard");
   lines.push("");
   lines.push(
-    "| model | score /100 | n | 95% CI | axis x | axis y | Brier | own reading agrees | crowd distance | gap | unstable |"
+    "| model | score /100 | n | cost | 95% CI | axis x | axis y | Brier | own reading agrees | crowd distance | gap | unstable |"
   );
-  lines.push("|---|---|---|---|---|---|---|---|---|---|---|");
+  lines.push("|---|---|---|---|---|---|---|---|---|---|---|---|");
   for (const s of ranked) {
     lines.push(
-      `| \`${s.model}\` | **${(100 * s.predict.acc).toFixed(1)}** | ${s.windows} | ${ci(s.predict.ci)} | ${pct(s.axes.x)} | ${pct(s.axes.y)} | ${round(s.brier)} | ${pct(s.read.agree)} | ${round(s.crowdDistance)} | ${s.gap >= 0 ? "+" : ""}${pct(s.gap)} | ${pct(s.instability)} |`
+      `| \`${s.model}\` | **${(100 * s.predict.acc).toFixed(1)}** | ${s.windows} | $${s.costUsd.toFixed(4)} | ${ci(s.predict.ci)} | ${pct(s.axes.x)} | ${pct(s.axes.y)} | ${round(s.brier)} | ${pct(s.read.agree)} | ${round(s.crowdDistance)} | ${s.gap >= 0 ? "+" : ""}${pct(s.gap)} | ${pct(s.instability)} |`
     );
   }
   lines.push("");
@@ -609,6 +611,16 @@ function writeReport() {
   console.log("wrote results/leaderboard.md and results/leaderboard.svg");
   return data;
 }
+function scoreLine(s) {
+  const [lo, hi] = s.predict.ci;
+  return [
+    s.model,
+    `${(100 * s.predict.acc).toFixed(1)} / 100`,
+    `${s.windows} windows`,
+    `95% CI ${(100 * lo).toFixed(1)} to ${(100 * hi).toFixed(1)}`,
+    `cost $${s.costUsd.toFixed(4)}`
+  ].join("   ");
+}
 function svg(scores, baselines, windowCount) {
   const rowH = 38;
   const top = 84;
@@ -710,6 +722,10 @@ if (cmd === "catalog") {
   for (const model of models) {
     await runModel(windows, { model, tasks: [...tasks], concurrency, limit });
   }
+  const data = writeReport();
+  for (const s of data.scores) {
+    if (models.includes(s.model)) console.log(scoreLine(s));
+  }
 } else if (cmd === "score" || cmd === "report" || cmd === "all") {
   const data = writeReport();
   console.log(
@@ -730,7 +746,9 @@ if (cmd === "catalog") {
     concurrency: Number(flags.concurrency ?? 6),
     limit: flags.limit ? Number(flags.limit) : void 0
   });
-  writeReport();
+  const data = writeReport();
+  const s = data.scores.find((x) => x.model === cmd);
+  if (s) console.log(scoreLine(s));
 } else {
   const here = path4.dirname(fileURLToPath2(import.meta.url));
   console.log(fs5.readFileSync(path4.join(here, "..", "README.md"), "utf8").slice(0, 1200));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rtr-bench",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Can a language model predict how a crowd reads a word? A cultural fluency benchmark built from Mixed Messages.",
   "type": "module",
   "license": "MIT",