rtr-bench 0.1.5 → 0.2.1

package/docs/development-pilot-2026-11-14.json

@@ -0,0 +1,262 @@
+{
+  "createdAt": "2026-11-14T22:47:51.731Z",
+  "purpose": "Development compatibility pilot; not a model ranking or final evaluation",
+  "protocol": {
+    "partition": "development",
+    "targets": [
+      100,
+      101,
+      102
+    ],
+    "strategy": "all",
+    "budget": 3,
+    "seed": 7,
+    "maxModelCalls": 8,
+    "maxActions": 8,
+    "maxTokens": 4000,
+    "agentPolicy": "transfer-check-v1"
+  },
+  "models": [
+    {
+      "model": "z-ai/glm-5.3",
+      "provider": "wafer/us",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/z-ai/glm-5.3/endpoints",
+      "estimateUsd": 0.024543114233333333,
+      "reservationUsd": 0.07889546239999999,
+      "capUsd": 0.2,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.2,
+        "providerTag": "wafer/us",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 6.99e-08,
+        "outputUsdPerToken": 1.4e-06
+      }
+    },
+    {
+      "model": "minimax/minimax-m3.1-flash",
+      "provider": "inferact-vllm/nvfp4",
+      "quantization": "nvfp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/minimax/minimax-m3.1-flash/endpoints",
+      "estimateUsd": 0.006733225,
+      "reservationUsd": 0.0798432,
+      "capUsd": 0.15,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.15,
+        "providerTag": "inferact-vllm/nvfp4",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 7.5e-08,
+        "outputUsdPerToken": 3e-07
+      }
+    },
+    {
+      "model": "xiaomi/mimo-v2.6-flash",
+      "provider": "xiaomi/fp8",
+      "quantization": "fp8",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/xiaomi/mimo-v2.6-flash/endpoints",
+      "estimateUsd": 0.008003380000000001,
+      "reservationUsd": 0.14792064000000002,
+      "capUsd": 0.2,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.2,
+        "providerTag": "xiaomi/fp8",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 1.4e-07,
+        "outputUsdPerToken": 2.8e-07
+      }
+    },
+    {
+      "model": "deepseek/deepseek-v4.1-flash-1017",
+      "provider": "sail-research/fp4/us",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/deepseek/deepseek-v4.1-flash-1017/endpoints",
+      "estimateUsd": 0.0035467375200000004,
+      "reservationUsd": 0.07668444672000001,
+      "capUsd": 0.1,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.1,
+        "providerTag": "sail-research/fp4/us",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 7.272e-08,
+        "outputUsdPerToken": 1.08e-07
+      }
+    },
+    {
+      "model": "tencent/hy3",
+      "provider": "gmicloud/bf16",
+      "quantization": "bf16",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/tencent/hy3/endpoints",
+      "estimateUsd": 0.01289478,
+      "reservationUsd": 0.039020160000000005,
+      "capUsd": 0.1,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.1,
+        "providerTag": "gmicloud/bf16",
+        "contextTokens": 262144,
+        "inputUsdPerToken": 1.4e-07,
+        "outputUsdPerToken": 5.8e-07
+      }
+    },
+    {
+      "model": "google/gemini-3.8-flash",
+      "provider": "google-ai-studio",
+      "quantization": "unknown",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/google/gemini-3.8-flash/endpoints",
+      "estimateUsd": 0.07956075,
+      "reservationUsd": 0.801432,
+      "capUsd": 1.05,
+      "spending": {
+        "version": "reserved-text-v2",
+        "limitUsd": 1.05,
+        "providerTag": "google-ai-studio",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 7.5e-07,
+        "outputUsdPerToken": 3.75e-06,
+        "model": "google/gemini-3.8-flash",
+        "temperatureSupported": true,
+        "maxCompletionTokens": 65536,
+        "pricing": {
+          "audio": "0.00000075",
+          "completion": "0.00000375",
+          "discount": 0.5,
+          "image": "0.00000075",
+          "input_audio_cache": "0.000000075",
+          "input_cache_read": "0.000000075",
+          "input_cache_write": "0.0000000416666666666667",
+          "internal_reasoning": "0.00000375",
+          "prompt": "0.00000075",
+          "web_search": "0.014"
+        }
+      }
+    },
+    {
+      "model": "openai/gpt-6-luna",
+      "provider": "openai",
+      "quantization": "unknown",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/openai/gpt-6-luna/endpoints",
+      "estimateUsd": 0.010608099999999999,
+      "reservationUsd": 0.13275,
+      "capUsd": 0.2,
+      "spending": {
+        "version": "reserved-text-v2",
+        "limitUsd": 0.2,
+        "providerTag": "openai/flex",
+        "contextTokens": 1050000,
+        "inputUsdPerToken": 1.25e-07,
+        "outputUsdPerToken": 3.75e-07,
+        "model": "openai/gpt-6-luna",
+        "temperatureSupported": false,
+        "maxCompletionTokens": 128000,
+        "pricing": {
+          "completion": "0.00000025",
+          "discount": 0,
+          "input_cache_read": "0.000000005",
+          "input_cache_write": "0.0000000625",
+          "overrides": [
+            {
+              "completion": "0.000000375",
+              "input_cache_read": "0.00000001",
+              "input_cache_write": "0.000000125",
+              "min_prompt_tokens": 272000,
+              "prompt": "0.0000001"
+            }
+          ],
+          "prompt": "0.00000005",
+          "web_search": "0.01"
+        }
+      }
+    },
+    {
+      "model": "anthropic/claude-haiku-5.5",
+      "provider": "anthropic",
+      "quantization": "unknown",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/anthropic/claude-haiku-5.5/endpoints",
+      "estimateUsd": 0.010608099999999999,
+      "reservationUsd": 1.01,
+      "capUsd": 1.05,
+      "spending": {
+        "version": "reserved-text-v2",
+        "limitUsd": 1.05,
+        "providerTag": "anthropic",
+        "contextTokens": 1000000,
+        "inputUsdPerToken": 1e-06,
+        "outputUsdPerToken": 2.5e-06,
+        "model": "anthropic/claude-haiku-5.5",
+        "temperatureSupported": false,
+        "maxCompletionTokens": 128000,
+        "pricing": {
+          "completion": "0.0000005",
+          "discount": 0,
+          "input_cache_read": "0.00000001",
+          "input_cache_write": "0.000000125",
+          "input_cache_write_1h": "0.0000002",
+          "overrides": [
+            {
+              "completion": "0.0000025",
+              "input_cache_read": "0.00000005",
+              "input_cache_write": "0.000000625",
+              "input_cache_write_1h": "0.000001",
+              "min_prompt_tokens": 100000,
+              "prompt": "0.0000005"
+            }
+          ],
+          "prompt": "0.0000001",
+          "web_search": "0.01"
+        }
+      }
+    },
+    {
+      "model": "thinkingmachines/inkling-small",
+      "provider": "deepinfra/fp4",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/thinkingmachines/inkling-small/endpoints",
+      "estimateUsd": 0.03061655,
+      "reservationUsd": 0.2407296,
+      "capUsd": 0.35,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.35,
+        "providerTag": "deepinfra/fp4",
+        "contextTokens": 524288,
+        "inputUsdPerToken": 4.5e-07,
+        "outputUsdPerToken": 1.2e-06
+      }
+    },
+    {
+      "model": "openai/gpt-oss-120b",
+      "provider": "primeintellect/fp4",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/openai/gpt-oss-120b/endpoints",
+      "estimateUsd": 0.0021967023333333336,
+      "reservationUsd": 0.005165664,
+      "capUsd": 0.05,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.05,
+        "providerTag": "primeintellect/fp4",
+        "contextTokens": 131072,
+        "inputUsdPerToken": 3.7e-08,
+        "outputUsdPerToken": 7.9e-08
+      }
+    }
+  ],
+  "estimateUsd": 0.18931143908666667,
+  "capUsd": 3.45,
+  "output": "results/development-preflight-2026-11-14",
+  "execution": "Sequential; identical commands resume saved records. Stop each model at its cap; never automatically raise caps, delete attempts, or launch final evaluation."
+}

package/docs/distribution.md

@@ -0,0 +1,102 @@
+# Distribution prediction protocol v1
+
+For the public Predict track, use `node src/cli.ts predict <model>`; see the [Quickstart](../README.md#quickstart). This uses the shared Learn runner’s no-example condition with an 8,000-token default in version 0.2.1 and support for `--max-usd`. `read-the-room <model>` runs Predict and Learn together. The legacy `distribution` command and its fixture format remain available unchanged.
+
+This is a separate RTR-bench track. The original winning-quadrant benchmark retains its prompts, scoring, and result paths. [Learn the Room](learn-room.md) reuses this protocol's probability validation and scorer for its interactive environment and initial baseline comparisons.
+
+## Dataset and frozen split
+
+`data/splits/v1.json` pins the exact `data/windows.json` bytes with SHA-256. The loader checks counts, complete coverage, disjoint groups, and chronology before a run or report.
+
+| Partition | Days | Windows | Purpose |
+|---|---|---:|---|
+| Reference | 1–99, included dataset windows only | 90 | Estimate the constant baseline; future evidence pool |
+| Development | 100–129 | 30 | Prompt and implementation development |
+| Evaluation | 130–174 | 45 | Final comparison after settings are fixed |
+
+The cutoffs use chronology, not crowd labels. Both Elon Musk windows (days 30 and 99) remain in reference. Prompt grouping trims, lowercases, and collapses whitespace, with an explicit alias map for reviewed same-entity variants. The current alias map is empty after reviewing the prompt list. Related concepts are not collapsed merely because they are thematically similar.
+
+Every reference window predates every target. Development outcomes never enter the reference baseline or later evidence pool. This public dataset has already been used for the original benchmark: these partitions control evidence access and tuning in the new experiment, but do not establish unseen training data. The chronological split may also expose changes in the audience over time.
+
+No separate training partition or RL experiment is defined in v1. A limited pilot selects the first N windows of its declared partition and records those IDs before execution.
+
+## Output and scoring
+
+Return a JSON object containing exactly four numeric probabilities:
+
+```json
+{
+  "performed+uniting": 0.20,
+  "honest+uniting": 0.30,
+  "performed+dividing": 0.40,
+  "honest+dividing": 0.10
+}
+```
+
+Values must be finite numbers in [0, 1]. No string coercion, prose, code fences, extra keys, or percentage values are accepted. A sum within 0.001 of one, inclusive, is normalized to one; the original sum and raw response are retained. Larger deviations fail validation.
+
+For each target, derive the observed shares from integer quadrant counts divided by submissions. Sum the four squared differences between predicted and observed shares. Average errors equally across the declared targets, then report `100 * (1 - mean_error / 2)`. A score of 100 is an exact match; 0 is the global worst case. This is not percentage accuracy, and 50 is not a chance baseline.
+
+Final invalid, missing, or API-failed predictions receive penalty 2 (zero points) and stay in the denominator. The report distinguishes an assigned failure penalty from a measured distribution error. Pending targets make a run incomplete; an incomplete run has no aggregate headline score. Reports separately show valid-response error, invalid-output rate, overall failure rate, and valid/invalid/missing/pending counts.
+
+Secondary winning-quadrant accuracy accepts every crowd co-winner. When a prediction has K exactly tied maxima, its credit is the fraction of those K quadrants that are crowd co-winners. This is the expected accuracy of a uniform tie-break, without introducing randomness. A uniform prediction against one crowd winner earns 0.25 credit.
+
+Always report both baselines on the exact same targets:
+
+- Uniform: 0.25 per quadrant.
+- Constant reference: the arithmetic mean of the 90 reference-window distributions. Each reference window has equal weight, irrespective of submission count.
+
+Report gains over baselines in score points. Confidence intervals use 2,000 bootstrap samples of entity groups with seed 7. Paired gains resample matching target differences together. These intervals describe variation across targets, not uncertainty about independent people in this self-selected audience. Five-target pilots are setup checks, not model rankings.
+
+## Calls, failures, and reproducibility
+
+The v1 prompt is frozen in `src/tracks/distribution/prompt.ts`. Only the target word, audience description, quadrant definitions, and output instructions enter the API messages. The runner sends neither reference labels nor target labels for direct distribution prediction.
+
+API runs use temperature 0, a 2,000-token output cap, and one prediction per window. There are no option rotations or automatic prompt-repair attempts. A requested provider is pinned with fallback disabled. Unpinned runs record actual serving providers. Unsupported settings fail explicitly rather than silently changing the request.
+
+Each target has at most three attempts. Only transport failures and HTTP 429/500/502/503/504 are retried, with 1- and 2-second backoffs and a 60-second timeout per attempt. Successful API responses with malformed or absent model content are terminal failures. An interrupted attempt consumes an attempt slot, since its API outcome and charge may be unknown.
+
+Each request and every attempt are saved, including raw responses, errors, provider, timestamps, and available usage. Reports sum known cost and tokens over all attempts and count attempts with unknown cost or tokens separately. Fixture runs make zero API calls and are visibly labeled synthetic.
+
+A deterministic run ID hashes dataset and split fingerprints, target/reference IDs, model and requested provider, prompt and scorer versions, generation/retry settings, and fixture contents when applicable. Identical commands resume terminal records, including failures, without rerolling them. Changing a target set or configuration produces a separate run. Increment protocol/runner versions when changing behavior.
+
+Writes use a temporary file and rename. An exclusive `run.lock` prevents simultaneous writers. After an unclean process exit, first confirm the recorded PID is no longer running, then remove that run's stale lock before resuming. Saved API responses can be recovered without another call. These controls reproduce configuration and scoring, not necessarily identical outputs from nondeterministic model services.
+
+## Commands
+
+Source commands require Node.js 22.18 or newer. The bundled CLI can also run these commands after `npm run build`.
+
+Run the five-target, zero-cost fixture pilot:
+
+```bash
+node src/cli.ts distribution --fixture tests/fixtures/distribution-pilot.json --limit 5
+```
+
+The fixture intentionally includes three valid outputs, one invalid sum, and one empty response. It is a pipeline demonstration, not a model result.
+
+For a future paid API pilot, after selecting a model/provider:
+
+```bash
+node src/cli.ts distribution <model-id> --split development --limit 5 --provider <provider-name>
+```
+
+Omit `--limit` to cover the full partition. Use `--split evaluation` only after development decisions are frozen. `--output <directory>` changes the output root, which defaults to `results/distribution/`. Each run directory contains `manifest.json`, per-target `records/`, and JSON/Markdown/HTML reports. The HTML report supports prompt filtering and inspection of every request and attempt.
+
+Regenerate a report without network access:
+
+```bash
+node src/cli.ts distribution-report <run-directory>
+```
+
+Verify locally without API spending:
+
+```bash
+npm test
+npm run build
+```
+
+## Evaluation status
+
+Both tracks have completed development validation and a [first final evaluation](../results/learn-room/z-ai-glm-5-3-flash-e9d0327447ef1856/README.md) on all 45 frozen evaluation targets. GLM 5.3 Flash through Relace scored **87.93/100** for direct distribution prediction, using the identical no-example condition in Learn the Room without duplicate calls. All 45 direct predictions were valid. Uniform and constant reference-distribution baselines scored 85.39 and 85.56, respectively; these scores are not percentage accuracy.
+
+The same run evaluated all three evidence conditions, for 180 valid predictions total. The transfer-check agent scored 85.57; see the linked report for paired comparisons, uncertainty, usage, and the frozen protocol. The original RTR score remains separate. Future models should use the same declared settings; any policy development informed by these final outcomes must disclose that this holdout has been used.