rtr-bench 0.2.0 → 0.2.1

package/README.md

@@ -201,7 +201,9 @@ Predict and Learn currently execute requests sequentially. Each Learn episode co
 
 On 45 targets, Predict and Learn together typically use about 270 model calls. Adding `--controls` brings that to about 360: 45 for Predict, 45 for each automatic control, and approximately five calls per Learn episode. Early stopping can reduce that number; additional agent turns and transport retries can increase it. The first final GLM run used 347 calls.
 
-The public commands default to 4,000 output tokens per response. Learn uses the published `transfer-check-v1` policy, up to three inspected examples, and eight model turns. Advanced runs can override `--max-tokens` and, for Learn, `--agent-policy`.
+In version 0.2.1, the public commands default to 8,000 output tokens per response. Learn uses the published `transfer-check-v1` policy, up to three inspected examples, and eight model turns. Advanced runs can override `--max-tokens` and, for Learn, `--agent-policy`.
+
+To resume a run created with the earlier 4,000-token default, repeat its original command with `--max-tokens 4000` explicitly. Changing the allowance creates a separate run. The [final evaluation plan](docs/final-run-plan-8000.md) declares 8,000 tokens for all eleven models, including a new GLM result under that same setting.
 
 The current client uses OpenRouter. Direct connections to other hosted or local endpoints are not yet implemented.
 

package/dist/cli.js

@@ -1122,6 +1122,19 @@ function meanCI(values, groups, reps = 2e3) {
 var API_URL = "https://openrouter.ai/api/v1/chat/completions";
 var MAX_ATTEMPTS = 3;
 var TIMEOUT_MS = 6e4;
+function errorCode(response, httpStatus) {
+  const code = Number(response?.error?.code);
+  return Number.isInteger(code) && code >= 400 && code <= 599 ? code : httpStatus;
+}
+function isSetupFailure(attempt) {
+  if (attempt.status !== "error") return false;
+  const type = attempt.response?.error?.metadata?.error_type;
+  if (["refusal", "content_policy_violation", "max_tokens_exceeded", "rate_limit_exceeded", "provider_overloaded", "provider_unavailable", "server", "timeout"].includes(type)) return false;
+  if (["authentication", "permission_denied", "payment_required", "invalid_request", "invalid_prompt", "not_found", "precondition_failed", "payload_too_large", "unprocessable", "context_length_exceeded", "string_too_long", "token_limit_exceeded"].includes(type)) return true;
+  return [400, 401, 402, 404, 413, 422].includes(errorCode(attempt.response, attempt.httpStatus) ?? 0);
+}
+var RunSetupError = class extends Error {
+};
 function retryDelayMs(attempt, previousAttempts) {
   const metadata = attempt?.response?.error?.metadata;
   const seconds = Number(metadata?.retry_after_seconds);
@@ -1147,6 +1160,8 @@ async function requestCompletion(body, apiKey2, fetcher = fetch) {
     const error = !res.ok || response?.error ? `HTTP ${res.status}: ${JSON.stringify(response?.error ?? response).slice(0, 500)}` : null;
     const retryHeader = res.headers.get("retry-after");
     const retryAfterMs = retryHeader ? Number.isFinite(Number(retryHeader)) ? Math.max(0, Number(retryHeader) * 1e3) : Math.max(0, Date.parse(retryHeader) - Date.now()) : 0;
+    const errorType = response?.error?.metadata?.error_type;
+    const retryableError = errorType ? ["rate_limit_exceeded", "provider_overloaded", "provider_unavailable", "server", "timeout"].includes(errorType) : [429, 500, 502, 503, 504].includes(errorCode(response, res.status) ?? 0);
     return {
       status: error ? "error" : "response",
       finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
@@ -1156,7 +1171,7 @@ async function requestCompletion(body, apiKey2, fetcher = fetch) {
       provider: typeof response?.provider === "string" ? response.provider : null,
       usage: response?.usage && typeof response.usage === "object" ? response.usage : null,
       error,
-      retriable: [429, 500, 502, 503, 504].includes(res.status),
+      retriable: !!error && retryableError,
       ...Number.isFinite(retryAfterMs) && retryAfterMs > 0 ? { retryAfterMs } : {}
     };
   } catch (e) {
@@ -1621,8 +1636,8 @@ async function discoverSpendingPolicy(model, provider, limitUsd, fetcher = fetch
   if (!suitable.length) throw new Error("No suitable text endpoint supports the spending guard");
   return suitable[0];
 }
-function spendingProvider(policy) {
-  return { order: [policy.providerTag], allow_fallbacks: false, max_price: { prompt: policy.inputUsdPerToken * 1e6, completion: policy.outputUsdPerToken * 1e6, request: 0 } };
+function spendingProvider(policy, requireParameters = false) {
+  return { order: [policy.providerTag], allow_fallbacks: false, max_price: { prompt: policy.inputUsdPerToken * 1e6, completion: policy.outputUsdPerToken * 1e6, request: 0 }, ...requireParameters ? { require_parameters: true } : {} };
 }
 function openSpendingLedger(file2, policy) {
   if (!["reserved-context-v1", "reserved-text-v2"].includes(policy.version) || !Number.isFinite(policy.limitUsd) || policy.limitUsd <= 0 || !Number.isSafeInteger(policy.contextTokens) || policy.contextTokens < 1 || !policy.providerTag || [policy.inputUsdPerToken, policy.outputUsdPerToken].some((p) => !Number.isFinite(p) || p <= 0)) throw new Error("Invalid spending policy");
@@ -1632,12 +1647,14 @@ function openSpendingLedger(file2, policy) {
   }
   const ledger = fs10.existsSync(file2) ? readJson(file2) : { policy, entries: [] };
   if (JSON.stringify(ledger.policy) !== JSON.stringify(policy)) throw new Error("Spending ledger policy mismatch");
+  if (!Array.isArray(ledger.entries) || ledger.entries.some((e) => !Number.isSafeInteger(e.reservedUnits) || e.reservedUnits < 0 || e.chargedUnits !== null && (!Number.isSafeInteger(e.chargedUnits) || e.chargedUnits < 0))) throw new Error("Invalid spending ledger entries");
   const accounted = () => ledger.entries.reduce((s, e) => s + (e.chargedUnits ?? e.reservedUnits), 0);
   const save = () => writeJson(file2, ledger);
   save();
   return {
-    reserve(request) {
-      if (JSON.stringify(request.provider) !== JSON.stringify(spendingProvider(policy))) throw new Error("Request must use the budget's pinned provider and price ceiling");
+    reserve(request, requestKey) {
+      if (ledger.entries.some((e) => e.chargedUnits !== null && e.chargedUnits > e.reservedUnits)) throw new SpendingLimitError("A saved provider charge exceeded its reservation; stopping for review");
+      if (JSON.stringify(request.provider) !== JSON.stringify(spendingProvider(policy, request.provider?.require_parameters === true))) throw new Error("Request must use the budget's pinned provider and price ceiling");
       if (!Number.isSafeInteger(request.max_tokens) || request.max_tokens <= 0) throw new Error("Budgeted requests require a finite output limit");
       if (policy.version === "reserved-text-v2") {
         const allowed = ["model", "messages", "temperature", "max_tokens", "usage", "provider"];
@@ -1647,18 +1664,23 @@ function openSpendingLedger(file2, policy) {
       }
       if (Buffer.byteLength(JSON.stringify(request.messages), "utf8") + 1024 * request.messages.length > policy.contextTokens) throw new SpendingLimitError("Request is too large for the conservative spending guard");
       const reservedUnits = units(policy.contextTokens * policy.inputUsdPerToken + request.max_tokens * policy.outputUsdPerToken);
+      if (!Number.isSafeInteger(reservedUnits)) throw new SpendingLimitError("Reservation exceeds safe accounting precision");
       if (accounted() + reservedUnits > Math.floor(policy.limitUsd * 1e9)) throw new SpendingLimitError("Spending limit reached; next request was not sent");
-      ledger.entries.push({ reservedUnits, chargedUnits: null, at: (/* @__PURE__ */ new Date()).toISOString() });
+      if (requestKey && ledger.entries.some((e) => e.requestKey === requestKey)) throw new SpendingLimitError("Saved reservation has no matching attempt; stopping for review");
+      ledger.entries.push({ reservedUnits, chargedUnits: null, at: (/* @__PURE__ */ new Date()).toISOString(), ...requestKey ? { requestKey } : {} });
       save();
       return ledger.entries.length - 1;
     },
-    settle(index, usage, attempt) {
+    settle(index, usage, attempt, requestKey) {
+      const entry = ledger.entries[index];
+      if (!Number.isSafeInteger(index) || !entry || requestKey !== void 0 && entry.requestKey !== requestKey) throw new Error("Spending reservation does not match saved attempt");
       const response = attempt?.response;
       const rejected = attempt?.httpStatus === 429 && response?.error?.code === 429 && response?.error?.metadata?.provider_error_code === "rate_limit_exceeded" && !response?.id && !response?.choices;
       const cost = usage?.cost ?? usage?.total_cost ?? (rejected ? 0 : null);
       if (typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
-        const entry = ledger.entries[index];
-        entry.chargedUnits = units(cost);
+        const chargedUnits = units(cost);
+        if (!Number.isSafeInteger(chargedUnits) || entry.chargedUnits !== null && entry.chargedUnits !== chargedUnits) throw new Error("Conflicting or invalid saved charge");
+        entry.chargedUnits = chargedUnits;
         entry.costBasis = typeof usage?.cost === "number" || typeof usage?.total_cost === "number" ? "reported-cost" : "unbilled-rate-limit: https://openrouter.ai/pricing";
         save();
         if (entry.chargedUnits > entry.reservedUnits) throw new SpendingLimitError("Provider cost exceeded its reservation; stopping for review");
@@ -1698,6 +1720,7 @@ function agentRequest(config, messages) {
   const request = { model: config.model, messages: structuredClone(messages), ...config.settings.temperature === null ? {} : { temperature: config.settings.temperature }, max_tokens: config.settings.maxTokens, usage: { include: true } };
   if (config.provider) request.provider = { order: [config.provider], allow_fallbacks: false };
   if (config.spending) request.provider = spendingProvider(config.spending);
+  if (config.version === "learn-room-runner-v4" && request.provider) request.provider.require_parameters = true;
   return request;
 }
 function callFinished(attempts) {
@@ -1821,6 +1844,7 @@ function learnRoomMarkdown(r) {
     "",
     `${r.manifest.config.partition}; ${r.scope}; ${r.complete ? "complete" : "incomplete"}. Evidence budget: ${r.manifest.config.budget}; seed: ${r.manifest.config.seed}.${r.manifest.config.agentPolicy ? ` Agent policy: ${r.manifest.config.agentPolicy}.` : ""}`,
     "",
+    ...r.manifest.stopReason ? [`Stopped: ${r.manifest.stopReason.replace(/\s+/g, " ")}`, ""] : [],
     ...temperatureNote(r) ? [temperatureNote(r), ""] : [],
     "| Strategy | Score / 100 | Valid / invalid / missing / pending | Inspections | Model turns | API calls | Known tokens | Unknown-token attempts | Known cost USD | Unknown-cost attempts |",
     "|---|---:|---|---:|---:|---:|---:|---:|---:|---:|",
@@ -1843,7 +1867,7 @@ function learnRoomHtml(r) {
   const e = escapeHtml;
   return `<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Learn the Room</title><style>body{font:16px system-ui;max-width:1100px;margin:40px auto;padding:0 20px;background:#f7f8fa;color:#20242b}article{background:white;padding:20px;margin:20px 0;border:1px solid #ccd3dd;border-radius:8px}table{border-collapse:collapse;width:100%}td,th{text-align:left;padding:9px;border-bottom:1px solid #ddd}pre{white-space:pre-wrap;overflow-wrap:anywhere;font-size:12px}summary{cursor:pointer}input,select{padding:8px;margin:8px}.badge{color:#843600;font-weight:bold}</style>
 <h1>Learn the Room</h1><p class="badge">${r.manifest.config.mode === "fixture" ? "SYNTHETIC FIXTURE \u2014 not model performance" : e(r.manifest.config.model)}</p><p>${e(r.manifest.config.partition)} \xB7 ${r.scope} \xB7 ${r.complete ? "Complete" : "Incomplete"} \xB7 Inspection budget ${r.manifest.config.budget} \xB7 Seed ${r.manifest.config.seed}${r.manifest.config.agentPolicy ? ` \xB7 Agent policy ${e(r.manifest.config.agentPolicy)}` : ""}</p>
-${temperatureNote(r) ? `<p>${temperatureNote(r)}</p>` : ""}<table><tr><th>Strategy</th><th>Score / 100</th><th>95% interval</th><th>Inspections</th><th>Model turns</th><th>API calls</th><th>Known tokens</th><th>Unknown-token attempts</th><th>Known USD</th><th>Unknown-cost attempts</th></tr>${r.summaries.map((s) => `<tr><td>${s.strategy}</td><td>${fmt2(s.score)}</td><td>${s.scoreCI ? s.scoreCI.map(fmt2).join("\u2013") : "\u2014"}</td><td>${s.inspections}</td><td>${s.modelCalls}</td><td>${s.apiCalls}</td><td>${s.tokens.known}</td><td>${s.tokens.unknownAttempts}</td><td>${s.costUsd.known.toFixed(6)}</td><td>${s.costUsd.unknownAttempts}</td></tr>`).join("")}</table><p>Uniform: ${fmt2(r.baselines.uniformScore)}. Constant reference: ${fmt2(r.baselines.constantScore)}.</p><table><tr><th>Comparison</th><th>Gain (points)</th><th>Paired 95% interval</th></tr>${r.comparisons.map((p) => `<tr><td>${p.strategy} vs ${p.control}</td><td>${fmt2(p.scoreGain)}</td><td>${p.scoreGainCI.map(fmt2).join("\u2013")}</td></tr>`).join("")}</table><p>Scores are transformed distribution errors, not percentage accuracy. Invalid and missing predictions earn zero points. Usage is reported separately. Five-target synthetic pilots demonstrate the pipeline only.</p>
+${r.manifest.stopReason ? `<p>Stopped: ${e(r.manifest.stopReason)}</p>` : ""}${temperatureNote(r) ? `<p>${temperatureNote(r)}</p>` : ""}<table><tr><th>Strategy</th><th>Score / 100</th><th>95% interval</th><th>Inspections</th><th>Model turns</th><th>API calls</th><th>Known tokens</th><th>Unknown-token attempts</th><th>Known USD</th><th>Unknown-cost attempts</th></tr>${r.summaries.map((s) => `<tr><td>${s.strategy}</td><td>${fmt2(s.score)}</td><td>${s.scoreCI ? s.scoreCI.map(fmt2).join("\u2013") : "\u2014"}</td><td>${s.inspections}</td><td>${s.modelCalls}</td><td>${s.apiCalls}</td><td>${s.tokens.known}</td><td>${s.tokens.unknownAttempts}</td><td>${s.costUsd.known.toFixed(6)}</td><td>${s.costUsd.unknownAttempts}</td></tr>`).join("")}</table><p>Uniform: ${fmt2(r.baselines.uniformScore)}. Constant reference: ${fmt2(r.baselines.constantScore)}.</p><table><tr><th>Comparison</th><th>Gain (points)</th><th>Paired 95% interval</th></tr>${r.comparisons.map((p) => `<tr><td>${p.strategy} vs ${p.control}</td><td>${fmt2(p.scoreGain)}</td><td>${p.scoreGainCI.map(fmt2).join("\u2013")}</td></tr>`).join("")}</table><p>Scores are transformed distribution errors, not percentage accuracy. Invalid and missing predictions earn zero points. Usage is reported separately. Five-target synthetic pilots demonstrate the pipeline only.</p>
 <label>Prompt <input id="query" type="search"></label><label>Strategy <select id="strategy"><option value="">All</option>${r.manifest.config.strategies.map((s) => `<option>${s}</option>`).join("")}</select></label>
 ${r.rows.map((row) => `<article data-search="${e(`${row.day} ${row.word}`.toLowerCase())}" data-strategy="${row.strategy}"><h2>${row.day}: ${e(row.word)} \xB7 ${row.strategy}</h2><p>${row.n} submissions \xB7 ${row.outcome ? `${row.outcome.status} \xB7 ${fmt2(row.outcome.score)} / 100 \xB7 ${row.outcome.inspections} inspections` : "Pending"}</p>${row.outcome?.failure ? `<p>${e(row.outcome.failure)}</p>` : ""}<table><tr><th>Quadrant</th><th>Predicted share</th><th>Observed share</th></tr>${QUADRANTS2.map((q) => `<tr><td>${q}</td><td>${row.outcome?.prediction ? fmt2(row.outcome.prediction[q]) : "\u2014"}</td><td>${fmt2(row.observed[q])}</td></tr>`).join("")}</table><details><summary>Actions and observations</summary><pre>${e(JSON.stringify(row.trace, null, 2))}</pre></details><details><summary>Model request and all attempts</summary><pre>${e(JSON.stringify({ request: row.request, attempts: row.attempts, agentCalls: row.agentCalls }, null, 2))}</pre></details></article>`).join("\n")}
 <script>function filter(){const q=document.getElementById('query').value.toLowerCase(),s=document.getElementById('strategy').value;document.querySelectorAll('article').forEach(a=>a.hidden=!a.dataset.search.includes(q)||(s&&a.dataset.strategy!==s));}document.getElementById('query').addEventListener('input',filter);document.getElementById('strategy').addEventListener('change',filter);</script></html>`;
@@ -1857,6 +1881,31 @@ function writeLearnRoomReport(dir) {
 }
 
 // src/tracks/learn-room/run.ts
+function savedBudgetRun(root, config, limitUsd) {
+  const identity = (c) => {
+    const { spending, ...rest2 } = structuredClone(c);
+    rest2.settings.temperature = 0;
+    if (rest2.version === "learn-room-runner-v3") rest2.version = "learn-room-runner-v4";
+    return JSON.stringify(rest2);
+  };
+  const prefix = config.model.replace(/[^a-zA-Z0-9]+/g, "-").toLowerCase() + "-";
+  const matches = [];
+  for (const entry of fs12.existsSync(root) ? fs12.readdirSync(root, { withFileTypes: true }) : []) {
+    if (!entry.isDirectory() || !entry.name.startsWith(prefix)) continue;
+    const file2 = path10.join(root, entry.name, "manifest.json");
+    if (!fs12.existsSync(file2)) continue;
+    const saved = readJson(file2);
+    if (saved.config.spending?.limitUsd !== limitUsd || identity(saved.config) !== identity(config)) continue;
+    if (saved.id !== digest(JSON.stringify(saved.config)) || entry.name !== prefix + saved.id.slice(0, 16)) throw new Error("Saved run fingerprint mismatch");
+    if (saved.config.version !== config.version && saved.status !== "complete") throw new Error("An incomplete run from an older runner exists; review it before creating or resuming a paid run");
+    matches.push(saved);
+  }
+  if (matches.length > 1) throw new Error("Multiple saved runs match this command; select a separate output directory after reviewing existing spending");
+  return matches[0];
+}
+function stopForSetup(attempt) {
+  if (isSetupFailure(attempt)) throw new RunSetupError(`API setup failed; run remains incomplete: ${attempt.error ?? "Inspect the saved API error"}`);
+}
 function prepareEpisode(config, strategy, day, data = loadDistributionData()) {
   const target = data.windows.find((w) => w.day === day);
   if (!target || !config.targets.includes(day) || !config.strategies.includes(strategy)) throw new Error("Episode is outside the declared run");
@@ -1866,6 +1915,7 @@ function prepareEpisode(config, strategy, day, data = loadDistributionData()) {
   const request = { model: config.model, messages, ...config.settings.temperature === null ? {} : { temperature: config.settings.temperature }, max_tokens: config.settings.maxTokens, usage: { include: true } };
   if (config.provider) request.provider = { order: [config.provider], allow_fallbacks: false };
   if (config.spending) request.provider = spendingProvider(config.spending);
+  if (config.version === "learn-room-runner-v4" && request.provider) request.provider.require_parameters = true;
   return { episode, request };
 }
 function finishEpisode(episode, attempts) {
@@ -1904,15 +1954,24 @@ async function runLearnRoom(options, client = requestCompletion) {
   const model = fixture ? "fixture/synthetic" : options.model;
   if (!model) throw new Error("Specify a model or --fixture <file>");
   if (fixture && (options.maxUsd !== void 0 || options.spending)) throw new Error("Fixture runs do not need a spending policy");
-  const spending = options.spending ?? (options.maxUsd !== void 0 ? await discoverSpendingPolicy(model, options.provider ?? "", options.maxUsd) : void 0);
-  const config = { track: "learn-room", version: "learn-room-runner-v2", scorerVersion: DISTRIBUTION_VERSION, environmentVersion: ENVIRONMENT_VERSION, strategyVersion: STRATEGY_VERSION, similarityVersion: SIMILARITY_VERSION, agentVersion: AGENT_VERSION, agentPromptHash: policy.promptHash, maxModelCalls, promptHash: PROMPT_HASH2, evidenceInstruction: EVIDENCE_INSTRUCTION, datasetHash: data.datasetHash, splitHash: data.splitHash, partition, targets, reference: data.split.reference, strategies, budget, maxActions: MAX_ACTIONS, seed, model, provider: options.provider ?? null, mode: fixture ? "fixture" : "api", fixtureHash: fixtureRaw === null ? null : digest(fixtureRaw), settings: { temperature: 0, maxTokens, maxAttempts: MAX_ATTEMPTS, timeoutMs: TIMEOUT_MS, apiUrl: API_URL } };
-  if (spending) config.spending = spending;
-  if (spending?.version === "reserved-text-v2" && !spending.temperatureSupported) config.settings.temperature = null;
-  config.version = "learn-room-runner-v3";
+  if (options.maxUsd !== void 0 && (!Number.isFinite(options.maxUsd) || options.maxUsd <= 0 || !options.provider)) throw new Error("--max-usd must be positive and requires --provider");
+  if (options.maxUsd !== void 0 && options.spending && options.spending.limitUsd !== options.maxUsd) throw new Error("Spending policy differs from requested budget");
+  let config = { track: "learn-room", version: "learn-room-runner-v4", scorerVersion: DISTRIBUTION_VERSION, environmentVersion: ENVIRONMENT_VERSION, strategyVersion: STRATEGY_VERSION, similarityVersion: SIMILARITY_VERSION, agentVersion: AGENT_VERSION, agentPromptHash: policy.promptHash, maxModelCalls, promptHash: PROMPT_HASH2, evidenceInstruction: EVIDENCE_INSTRUCTION, datasetHash: data.datasetHash, splitHash: data.splitHash, partition, targets, reference: data.split.reference, strategies, budget, maxActions: MAX_ACTIONS, seed, model, provider: options.provider ?? null, mode: fixture ? "fixture" : "api", fixtureHash: fixtureRaw === null ? null : digest(fixtureRaw), settings: { temperature: 0, maxTokens, maxAttempts: MAX_ATTEMPTS, timeoutMs: TIMEOUT_MS, apiUrl: API_URL } };
   config.settings.retryPolicy = "retry-after-v1";
   if (options.replicate !== void 0) config.replicate = options.replicate;
   if (policy.name !== "default") config.agentPolicy = policy.name;
-  const id = digest(JSON.stringify(config)), dir = path10.join(options.output ?? path10.join("results", "learn-room"), `${model.replace(/[^a-zA-Z0-9]+/g, "-").toLowerCase()}-${id.slice(0, 16)}`);
+  const root = options.output ?? path10.join("results", "learn-room");
+  const saved = !options.spending && options.maxUsd !== void 0 ? savedBudgetRun(root, config, options.maxUsd) : void 0;
+  let spending = options.spending;
+  if (saved) {
+    config = saved.config;
+    spending = config.spending;
+  } else {
+    spending ??= options.maxUsd !== void 0 ? await discoverSpendingPolicy(model, options.provider ?? "", options.maxUsd) : void 0;
+    if (spending) config.spending = spending;
+    if (spending?.version === "reserved-text-v2" && !spending.temperatureSupported) config.settings.temperature = null;
+  }
+  const id = digest(JSON.stringify(config)), dir = path10.join(root, `${model.replace(/[^a-zA-Z0-9]+/g, "-").toLowerCase()}-${id.slice(0, 16)}`);
   fs12.mkdirSync(dir, { recursive: true });
   const lock = path10.join(dir, "run.lock");
   let fd;
@@ -1927,8 +1986,25 @@ async function runLearnRoom(options, client = requestCompletion) {
     const manifestPath = path10.join(dir, "manifest.json");
     const manifest = fs12.existsSync(manifestPath) ? readJson(manifestPath) : { id, createdAt: (/* @__PURE__ */ new Date()).toISOString(), config, status: "incomplete" };
     if (manifest.id !== id || JSON.stringify(manifest.config) !== JSON.stringify(config)) throw new Error("Run manifest mismatch");
+    if (spending && !fs12.existsSync(path10.join(dir, "spending.json")) && fs12.existsSync(path10.join(dir, "episodes"))) throw new Error("Spending ledger is missing for a saved run; refusing to reset its budget");
+    if (fs12.existsSync(manifestPath)) {
+      const prior = buildLearnRoomReport(dir);
+      if (manifest.status === "complete" && prior.rows.some((row) => !fs12.existsSync(path10.join(dir, "episodes", row.strategy, `${row.day}.json`)))) throw new Error("Completed run has missing episodes; refusing to rerun saved evaluation targets");
+    }
     writeJson(manifestPath, manifest);
     const ledger = spending ? openSpendingLedger(path10.join(dir, "spending.json"), spending) : null;
+    if (ledger && config.version === "learn-room-runner-v4") {
+      for (const s of strategies) for (const day of targets) {
+        const file2 = path10.join(dir, "episodes", s, `${day}.json`);
+        if (!fs12.existsSync(file2)) continue;
+        const rec = readJson(file2);
+        const calls = s === "agent" ? rec.agentCalls ?? [] : [{ attempts: rec.attempts }];
+        for (const [turn, call2] of calls.entries()) for (const a of call2.attempts) {
+          if (a.spendingReservation === void 0) throw new Error("Saved attempt has no spending reservation");
+          ledger.settle(a.spendingReservation, a.usage, a, `${s}/${day}/${turn}/${a.index}`);
+        }
+      }
+    }
     for (const s of strategies) for (const day of targets) {
       const { episode, request } = prepareEpisode(config, s, day, data);
       const file2 = path10.join(dir, "episodes", s, `${day}.json`);
@@ -1950,8 +2026,10 @@ async function runLearnRoom(options, client = requestCompletion) {
           while (!callFinished(call2.attempts)) {
             if (!fixture && !process.env.OPENROUTER_API_KEY) throw new Error("OPENROUTER_API_KEY is not set");
             if (call2.attempts.length && !fixture) await new Promise((resolve) => setTimeout(resolve, retryDelayMs(call2.attempts.at(-1), call2.attempts.length)));
-            const reservation = ledger?.reserve(call2.request);
+            const requestKey = `${s}/${day}/${rec.agentCalls.length - 1}/${call2.attempts.length + 1}`;
+            const reservation = ledger?.reserve(call2.request, requestKey);
             const attempt = { index: call2.attempts.length + 1, startedAt: (/* @__PURE__ */ new Date()).toISOString(), status: "started", raw: "", response: null, provider: null, usage: null, error: null, retriable: true };
+            if (reservation !== void 0) attempt.spendingReservation = reservation;
             call2.attempts.push(attempt);
             rec.attempts = rec.agentCalls.flatMap((c) => c.attempts);
             writeJson(file2, rec);
@@ -1959,9 +2037,12 @@ async function runLearnRoom(options, client = requestCompletion) {
               const raw = fixture[s][day][rec.agentCalls.length - 1] ?? "";
               Object.assign(attempt, { status: "response", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), raw, response: { fixture: true, content: raw }, usage: { total_tokens: 0, cost: 0 }, retriable: false });
             } else Object.assign(attempt, await client(call2.request, process.env.OPENROUTER_API_KEY));
+            if (isSetupFailure(attempt)) attempt.retriable = true;
             writeJson(file2, rec);
-            if (reservation !== void 0) ledger.settle(reservation, attempt.usage, attempt);
+            if (reservation !== void 0) ledger.settle(reservation, attempt.usage, attempt, requestKey);
+            stopForSetup(attempt);
           }
+          if (call2.attempts.length) stopForSetup(call2.attempts.at(-1));
           applyAgentCall(episode, messages, call2);
           call2.applied = true;
           enforceAgentLimit(episode, rec.agentCalls, config);
@@ -1977,27 +2058,38 @@ async function runLearnRoom(options, client = requestCompletion) {
       while (rec.attempts.length < MAX_ATTEMPTS && !(last && (last.status === "response" || last.status === "error" && !last.retriable))) {
         if (!fixture && !process.env.OPENROUTER_API_KEY) throw new Error("OPENROUTER_API_KEY is not set");
         if (rec.attempts.length && !fixture) await new Promise((resolve) => setTimeout(resolve, retryDelayMs(rec.attempts.at(-1), rec.attempts.length)));
-        const reservation = ledger?.reserve(request);
+        const requestKey = `${s}/${day}/0/${rec.attempts.length + 1}`;
+        const reservation = ledger?.reserve(request, requestKey);
         const attempt = { index: rec.attempts.length + 1, startedAt: (/* @__PURE__ */ new Date()).toISOString(), status: "started", raw: "", response: null, provider: null, usage: null, error: null, retriable: true };
+        if (reservation !== void 0) attempt.spendingReservation = reservation;
         rec.attempts.push(attempt);
         writeJson(file2, rec);
         if (fixture) Object.assign(attempt, { status: "response", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), raw: fixture[s][day], response: { fixture: true, content: fixture[s][day] }, usage: { total_tokens: 0, cost: 0 }, retriable: false });
         else Object.assign(attempt, await client(request, process.env.OPENROUTER_API_KEY));
+        if (isSetupFailure(attempt)) attempt.retriable = true;
         writeJson(file2, rec);
         last = attempt;
-        if (reservation !== void 0) ledger.settle(reservation, attempt.usage, attempt);
+        if (reservation !== void 0) ledger.settle(reservation, attempt.usage, attempt, requestKey);
+        stopForSetup(attempt);
       }
+      if (last) stopForSetup(last);
       Object.assign(rec, finishEpisode(episode, rec.attempts), { terminal: true });
       writeJson(file2, rec);
       console.log(`learn-room: ${s} day ${day} ${rec.outcome.status}, ${rec.outcome.inspections} inspections`);
     }
     manifest.status = "complete";
+    delete manifest.stopReason;
     manifest.completedAt ??= (/* @__PURE__ */ new Date()).toISOString();
     writeJson(manifestPath, manifest);
     if (ledger) console.log(`Spending: ${JSON.stringify(ledger.summary())}`);
     return { dir, report: writeLearnRoomReport(dir) };
   } catch (error) {
-    if (error instanceof SpendingLimitError) {
+    if (error instanceof SpendingLimitError || error instanceof RunSetupError) {
+      const manifest = readJson(path10.join(dir, "manifest.json"));
+      manifest.status = "incomplete";
+      manifest.stopReason = error.message;
+      delete manifest.completedAt;
+      writeJson(path10.join(dir, "manifest.json"), manifest);
       writeLearnRoomReport(dir);
       console.error(`${error.message}. Incomplete report: ${dir}`);
     }
@@ -2049,7 +2141,7 @@ if (["read-the-room", "predict", "learn", "learn-room"].includes(cmd)) {
     partition: flags.split ?? "development",
     strategy,
     maxModelCalls: flags["max-model-calls"] === void 0 ? void 0 : Number(flags["max-model-calls"]),
-    maxTokens: flags["max-tokens"] === void 0 ? legacy ? void 0 : 4e3 : Number(flags["max-tokens"]),
+    maxTokens: flags["max-tokens"] === void 0 ? legacy ? void 0 : 8e3 : Number(flags["max-tokens"]),
     replicate: flags.replicate,
     agentPolicy: flags["agent-policy"] ?? (!legacy && cmd !== "predict" ? "transfer-check-v1" : void 0),
     maxUsd: flags["max-usd"] === void 0 ? void 0 : Number(flags["max-usd"]),

package/docs/development-pilot-2026-11-14.json

@@ -0,0 +1,262 @@
+{
+  "createdAt": "2026-11-14T22:47:51.731Z",
+  "purpose": "Development compatibility pilot; not a model ranking or final evaluation",
+  "protocol": {
+    "partition": "development",
+    "targets": [
+      100,
+      101,
+      102
+    ],
+    "strategy": "all",
+    "budget": 3,
+    "seed": 7,
+    "maxModelCalls": 8,
+    "maxActions": 8,
+    "maxTokens": 4000,
+    "agentPolicy": "transfer-check-v1"
+  },
+  "models": [
+    {
+      "model": "z-ai/glm-5.3",
+      "provider": "wafer/us",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/z-ai/glm-5.3/endpoints",
+      "estimateUsd": 0.024543114233333333,
+      "reservationUsd": 0.07889546239999999,
+      "capUsd": 0.2,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.2,
+        "providerTag": "wafer/us",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 6.99e-08,
+        "outputUsdPerToken": 1.4e-06
+      }
+    },
+    {
+      "model": "minimax/minimax-m3.1-flash",
+      "provider": "inferact-vllm/nvfp4",
+      "quantization": "nvfp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/minimax/minimax-m3.1-flash/endpoints",
+      "estimateUsd": 0.006733225,
+      "reservationUsd": 0.0798432,
+      "capUsd": 0.15,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.15,
+        "providerTag": "inferact-vllm/nvfp4",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 7.5e-08,
+        "outputUsdPerToken": 3e-07
+      }
+    },
+    {
+      "model": "xiaomi/mimo-v2.6-flash",
+      "provider": "xiaomi/fp8",
+      "quantization": "fp8",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/xiaomi/mimo-v2.6-flash/endpoints",
+      "estimateUsd": 0.008003380000000001,
+      "reservationUsd": 0.14792064000000002,
+      "capUsd": 0.2,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.2,
+        "providerTag": "xiaomi/fp8",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 1.4e-07,
+        "outputUsdPerToken": 2.8e-07
+      }
+    },
+    {
+      "model": "deepseek/deepseek-v4.1-flash-1017",
+      "provider": "sail-research/fp4/us",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/deepseek/deepseek-v4.1-flash-1017/endpoints",
+      "estimateUsd": 0.0035467375200000004,
+      "reservationUsd": 0.07668444672000001,
+      "capUsd": 0.1,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.1,
+        "providerTag": "sail-research/fp4/us",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 7.272e-08,
+        "outputUsdPerToken": 1.08e-07
+      }
+    },
+    {
+      "model": "tencent/hy3",
+      "provider": "gmicloud/bf16",
+      "quantization": "bf16",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/tencent/hy3/endpoints",
+      "estimateUsd": 0.01289478,
+      "reservationUsd": 0.039020160000000005,
+      "capUsd": 0.1,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.1,
+        "providerTag": "gmicloud/bf16",
+        "contextTokens": 262144,
+        "inputUsdPerToken": 1.4e-07,
+        "outputUsdPerToken": 5.8e-07
+      }
+    },
+    {
+      "model": "google/gemini-3.8-flash",
+      "provider": "google-ai-studio",
+      "quantization": "unknown",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/google/gemini-3.8-flash/endpoints",
+      "estimateUsd": 0.07956075,
+      "reservationUsd": 0.801432,
+      "capUsd": 1.05,
+      "spending": {
+        "version": "reserved-text-v2",
+        "limitUsd": 1.05,
+        "providerTag": "google-ai-studio",
+        "contextTokens": 1048576,
+        "inputUsdPerToken": 7.5e-07,
+        "outputUsdPerToken": 3.75e-06,
+        "model": "google/gemini-3.8-flash",
+        "temperatureSupported": true,
+        "maxCompletionTokens": 65536,
+        "pricing": {
+          "audio": "0.00000075",
+          "completion": "0.00000375",
+          "discount": 0.5,
+          "image": "0.00000075",
+          "input_audio_cache": "0.000000075",
+          "input_cache_read": "0.000000075",
+          "input_cache_write": "0.0000000416666666666667",
+          "internal_reasoning": "0.00000375",
+          "prompt": "0.00000075",
+          "web_search": "0.014"
+        }
+      }
+    },
+    {
+      "model": "openai/gpt-6-luna",
+      "provider": "openai",
+      "quantization": "unknown",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/openai/gpt-6-luna/endpoints",
+      "estimateUsd": 0.010608099999999999,
+      "reservationUsd": 0.13275,
+      "capUsd": 0.2,
+      "spending": {
+        "version": "reserved-text-v2",
+        "limitUsd": 0.2,
+        "providerTag": "openai/flex",
+        "contextTokens": 1050000,
+        "inputUsdPerToken": 1.25e-07,
+        "outputUsdPerToken": 3.75e-07,
+        "model": "openai/gpt-6-luna",
+        "temperatureSupported": false,
+        "maxCompletionTokens": 128000,
+        "pricing": {
+          "completion": "0.00000025",
+          "discount": 0,
+          "input_cache_read": "0.000000005",
+          "input_cache_write": "0.0000000625",
+          "overrides": [
+            {
+              "completion": "0.000000375",
+              "input_cache_read": "0.00000001",
+              "input_cache_write": "0.000000125",
+              "min_prompt_tokens": 272000,
+              "prompt": "0.0000001"
+            }
+          ],
+          "prompt": "0.00000005",
+          "web_search": "0.01"
+        }
+      }
+    },
+    {
+      "model": "anthropic/claude-haiku-5.5",
+      "provider": "anthropic",
+      "quantization": "unknown",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/anthropic/claude-haiku-5.5/endpoints",
+      "estimateUsd": 0.010608099999999999,
+      "reservationUsd": 1.01,
+      "capUsd": 1.05,
+      "spending": {
+        "version": "reserved-text-v2",
+        "limitUsd": 1.05,
+        "providerTag": "anthropic",
+        "contextTokens": 1000000,
+        "inputUsdPerToken": 1e-06,
+        "outputUsdPerToken": 2.5e-06,
+        "model": "anthropic/claude-haiku-5.5",
+        "temperatureSupported": false,
+        "maxCompletionTokens": 128000,
+        "pricing": {
+          "completion": "0.0000005",
+          "discount": 0,
+          "input_cache_read": "0.00000001",
+          "input_cache_write": "0.000000125",
+          "input_cache_write_1h": "0.0000002",
+          "overrides": [
+            {
+              "completion": "0.0000025",
+              "input_cache_read": "0.00000005",
+              "input_cache_write": "0.000000625",
+              "input_cache_write_1h": "0.000001",
+              "min_prompt_tokens": 100000,
+              "prompt": "0.0000005"
+            }
+          ],
+          "prompt": "0.0000001",
+          "web_search": "0.01"
+        }
+      }
+    },
+    {
+      "model": "thinkingmachines/inkling-small",
+      "provider": "deepinfra/fp4",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/thinkingmachines/inkling-small/endpoints",
+      "estimateUsd": 0.03061655,
+      "reservationUsd": 0.2407296,
+      "capUsd": 0.35,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.35,
+        "providerTag": "deepinfra/fp4",
+        "contextTokens": 524288,
+        "inputUsdPerToken": 4.5e-07,
+        "outputUsdPerToken": 1.2e-06
+      }
+    },
+    {
+      "model": "openai/gpt-oss-120b",
+      "provider": "primeintellect/fp4",
+      "quantization": "fp4",
+      "catalogStatus": 0,
+      "source": "https://openrouter.ai/api/v1/models/openai/gpt-oss-120b/endpoints",
+      "estimateUsd": 0.0021967023333333336,
+      "reservationUsd": 0.005165664,
+      "capUsd": 0.05,
+      "spending": {
+        "version": "reserved-context-v1",
+        "limitUsd": 0.05,
+        "providerTag": "primeintellect/fp4",
+        "contextTokens": 131072,
+        "inputUsdPerToken": 3.7e-08,
+        "outputUsdPerToken": 7.9e-08
+      }
+    }
+  ],
+  "estimateUsd": 0.18931143908666667,
+  "capUsd": 3.45,
+  "output": "results/development-preflight-2026-11-14",
+  "execution": "Sequential; identical commands resume saved records. Stop each model at its cap; never automatically raise caps, delete attempts, or launch final evaluation."
+}

package/docs/distribution.md

@@ -1,6 +1,6 @@
 # Distribution prediction protocol v1
 
-For the public Predict track, use `node src/cli.ts predict <model>`; see the [Quickstart](../README.md#quickstart). This uses the shared Learn runner’s no-example condition with a 4,000-token default and support for `--max-usd`. `read-the-room <model>` runs Predict and Learn together. The legacy `distribution` command and its fixture format remain available unchanged.
+For the public Predict track, use `node src/cli.ts predict <model>`; see the [Quickstart](../README.md#quickstart). This uses the shared Learn runner’s no-example condition with an 8,000-token default in version 0.2.1 and support for `--max-usd`. `read-the-room <model>` runs Predict and Learn together. The legacy `distribution` command and its fixture format remain available unchanged.
 
 This is a separate RTR-bench track. The original winning-quadrant benchmark retains its prompts, scoring, and result paths. [Learn the Room](learn-room.md) reuses this protocol's probability validation and scorer for its interactive environment and initial baseline comparisons.