tokentracker-cli 0.95.2 → 0.96.0

package/src/lib/codex-model-attribution.js

@@ -0,0 +1,128 @@
+"use strict";
+
+// Codex token_count events do not carry a model id. Every usage event is
+// therefore attributed to the model named by the most recent turn_context,
+// which Codex emits at the start of each turn and which tracks mid-session
+// model switches exactly. That per-turn path is what actually runs today.
+//
+// The model/rerouted upgrade below is forward-looking: it promotes usage to
+// the effective model when the app server reports a server-side reroute.
+//
+// Deliberately NOT used for attribution: event_msg/thread_settings_applied.
+// It carries thread_settings.model, but it fires when the user picks a model
+// in the UI - seconds to minutes before the switch takes effect - so usage
+// still streaming from the in-flight turn belongs to the previous model.
+// turn_context is the only signal that marks where a turn actually begins.
+//
+// session_meta is a defensive fallback, not an observed source. Codex has
+// never written a model into it - 0 of 10310 session_meta rows across 5849
+// local rollouts carry the field (codex-cli 0.151.0); the payload records
+// model_provider ("openai"), which is provenance rather than a model. The
+// fallback only fills selectedModel when nothing else has, so a future format
+// that does carry one attributes instead of falling through to "unknown". It
+// never overrides a turn_context model.
+
+function cleanString(value) {
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function normalizeEventName(value) {
+  return cleanString(value)?.toLowerCase().replaceAll("_", "/") || "";
+}
+
+function eventEnvelope(obj) {
+  const candidates = [
+    { name: obj?.method, data: obj?.params },
+    { name: obj?.type, data: obj?.payload },
+    { name: obj?.payload?.method, data: obj?.payload?.params },
+    { name: obj?.payload?.type, data: obj?.payload },
+    { name: obj?.payload?.msg?.method, data: obj?.payload?.msg?.params },
+    { name: obj?.payload?.msg?.type, data: obj?.payload?.msg },
+    { name: obj?.msg?.method, data: obj?.msg?.params },
+    { name: obj?.msg?.type, data: obj?.msg },
+  ];
+  return candidates.find(({ name }) => normalizeEventName(name) === "model/rerouted") || null;
+}
+
+// Forward-looking: no model/rerouted event has been observed in any real
+// rollout (verified against 5832 local files, codex-cli 0.151.0), so the
+// envelope shapes and the fromModel/toModel field names below are inferred
+// from the app-server protocol rather than confirmed against logged data.
+// Re-verify against a real sample before relying on rerouted attribution.
+function extractModelReroute(obj) {
+  const envelope = eventEnvelope(obj);
+  if (!envelope) return null;
+  const data = envelope.data && typeof envelope.data === "object" ? envelope.data : {};
+  const fromModel = cleanString(data.fromModel ?? data.from_model);
+  const toModel = cleanString(data.toModel ?? data.to_model);
+  if (!toModel) return null;
+  return {
+    fromModel,
+    toModel,
+    reason: cleanString(data.reason),
+    threadId: cleanString(data.threadId ?? data.thread_id),
+    turnId: cleanString(data.turnId ?? data.turn_id),
+  };
+}
+
+function createCodexModelAttributionState(value = {}) {
+  const selectedModel = cleanString(value.selectedModel ?? value.selected_model ?? value.model);
+  const effectiveModel = cleanString(value.effectiveModel ?? value.effective_model) || selectedModel;
+  return {
+    selectedModel,
+    effectiveModel,
+    turnId: cleanString(value.turnId ?? value.turn_id),
+    rerouted: Boolean(value.rerouted),
+    rerouteReason: cleanString(value.rerouteReason ?? value.reroute_reason),
+  };
+}
+
+function applyCodexModelEvent(state, obj) {
+  if (!state || typeof state !== "object") return null;
+  if (obj?.type === "turn_context" && obj.payload && typeof obj.payload === "object") {
+    const selectedModel = cleanString(obj.payload.model);
+    if (selectedModel) {
+      state.selectedModel = selectedModel;
+    }
+    state.effectiveModel = state.selectedModel || state.effectiveModel;
+    state.turnId = cleanString(obj.payload.turn_id ?? obj.payload.turnId) || state.turnId;
+    state.rerouted = false;
+    state.rerouteReason = null;
+    return { type: "turn_context", selectedModel };
+  }
+
+  // Strictly weaker than turn_context, and never allowed to overwrite it: a
+  // forked rollout replays the parent's session_meta rows after its own.
+  if (obj?.type === "session_meta" && obj.payload && typeof obj.payload === "object") {
+    const metaModel = cleanString(obj.payload.model);
+    if (!metaModel || state.selectedModel) return null;
+    state.selectedModel = metaModel;
+    state.effectiveModel = state.effectiveModel || metaModel;
+    return { type: "session_meta", selectedModel: metaModel };
+  }
+
+  const reroute = extractModelReroute(obj);
+  if (!reroute) return null;
+  state.selectedModel = reroute.fromModel || state.selectedModel || state.effectiveModel;
+  state.effectiveModel = reroute.toModel;
+  state.turnId = reroute.turnId || state.turnId;
+  state.rerouted = true;
+  state.rerouteReason = reroute.reason;
+  return { type: "model_rerouted", ...reroute };
+}
+
+function currentCodexModel(state) {
+  return cleanString(state?.effectiveModel) || cleanString(state?.selectedModel);
+}
+
+function snapshotCodexModelAttributionState(state) {
+  return createCodexModelAttributionState(state);
+}
+
+module.exports = {
+  applyCodexModelEvent,
+  createCodexModelAttributionState,
+  currentCodexModel,
+  extractModelReroute,
+  snapshotCodexModelAttributionState,
+};

package/src/lib/codex-rollout-parser.js

@@ -24,12 +24,21 @@ const {
   getExecutableName,
   buildExecStatsEntry,
 } = require("./categorizer-utils");
+const {
+  applyCodexModelEvent,
+  createCodexModelAttributionState,
+  currentCodexModel,
+  snapshotCodexModelAttributionState,
+} = require("./codex-model-attribution");
 
 const DISCOVERY_FULL_AUDIT_INTERVAL_MS = 60 * 1000;
 const MAX_DISCOVERY_INVENTORIES = 32;
 const DISCOVERY_INVENTORIES = new Map();
 const ZONED_DAY_FORMATTERS = new Map();
 const CODEX_RESUME_INVALIDATED = "TOKENTRACKER_CODEX_RESUME_INVALIDATED";
+// See src/lib/pricing/index.js: no observed Codex request reaches this
+// threshold yet (context window 258400), so long_context_* stays at zero.
+const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272_000;
 
 // ---------------------------------------------------------------------------
 // Timezone helpers
@@ -520,6 +529,17 @@ function finalizeExecRows(map) {
     .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
 }
 
+function finalizeModelUsageRows(map) {
+  return Array.from(map.values())
+    .map((row) => ({
+      ...row,
+      selected_models: Array.from(row.selected_models).sort(),
+      reroute_reasons: Array.from(row.reroute_reasons).sort(),
+      model_attribution: row.rerouted_usage_events > 0 ? "effective" : "selected",
+    }))
+    .sort((a, b) => b.total_tokens - a.total_tokens || a.model.localeCompare(b.model));
+}
+
 // ---------------------------------------------------------------------------
 // Main parser
 // ---------------------------------------------------------------------------
@@ -537,6 +557,9 @@ async function parseCodexRolloutFile(filePath, {
   captureContentHash = false,
   contentHashState = null,
   sourceHandle = null,
+  collectBreakdowns = true,
+  collectModelUsage = false,
+  onObject = null,
 } = {}) {
   const filePaths = (Array.isArray(filePath) ? filePath : [filePath]).filter(Boolean);
   const primaryFilePath = filePaths[0] || String(filePath || "");
@@ -566,6 +589,9 @@ async function parseCodexRolloutFile(filePath, {
   let sessionId = isResuming ? resumeState.sessionId || null : null;
   let cwd = isResuming ? resumeState.cwd || null : null;
   let model = isResuming ? resumeState.model || null : null;
+  const modelAttributionState = createCodexModelAttributionState(
+    isResuming ? resumeState.modelAttributionState || { model } : {},
+  );
   let provider = isResuming ? resumeState.provider || null : null;
   let cliVersion = isResuming ? resumeState.cliVersion || null : null;
   let forkedFromId = isResuming ? resumeState.forkedFromId || null : null;
@@ -607,9 +633,55 @@ async function parseCodexRolloutFile(filePath, {
   const byExecCommand = new Map(); // sanitized executable + subcommand -> stats
   const byExecDuration = new Map(); // duration bucket -> stats
   const byExecOutput = new Map(); // output size bucket -> stats
+  const byModel = new Map(); // effective model -> observed token usage
 
   let turnCount = 0;
 
+  function recordModelUsage(delta, rawRequestUsage) {
+    if (!collectModelUsage || !delta || delta.total_tokens <= 0) return;
+    const effectiveModel = currentCodexModel(modelAttributionState) || "unknown";
+    let row = byModel.get(effectiveModel);
+    if (!row) {
+      row = {
+        model: effectiveModel,
+        ...emptyTotals(),
+        long_context_input_tokens: 0,
+        long_context_cached_input_tokens: 0,
+        long_context_cache_creation_input_tokens: 0,
+        long_context_output_tokens: 0,
+        long_context_reasoning_output_tokens: 0,
+        usage_events: 0,
+        rerouted_usage_events: 0,
+        long_context_usage_events: 0,
+        selected_models: new Set(),
+        reroute_reasons: new Set(),
+      };
+      byModel.set(effectiveModel, row);
+    }
+    addInto(row, delta);
+    row.usage_events += 1;
+    if (modelAttributionState.selectedModel) {
+      row.selected_models.add(modelAttributionState.selectedModel);
+    }
+    if (modelAttributionState.rerouted) {
+      row.rerouted_usage_events += 1;
+      if (modelAttributionState.rerouteReason) {
+        row.reroute_reasons.add(modelAttributionState.rerouteReason);
+      }
+    }
+    // OpenAI applies the long-context tier to the whole request when its raw
+    // (cache-inclusive) input exceeds 272K tokens. Preserve the exact subset
+    // so pricing can be recomputed later without rescanning private logs.
+    if (Number(rawRequestUsage?.input_tokens || 0) > OPENAI_LONG_CONTEXT_INPUT_THRESHOLD) {
+      row.long_context_usage_events += 1;
+      row.long_context_input_tokens += delta.input_tokens;
+      row.long_context_cached_input_tokens += delta.cached_input_tokens;
+      row.long_context_cache_creation_input_tokens += delta.cache_creation_input_tokens;
+      row.long_context_output_tokens += delta.output_tokens;
+      row.long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
+    }
+  }
+
   function ensureTool(name) {
     if (!byTool.has(name)) {
       byTool.set(name, { name, ...buildToolStatsEntry() });
@@ -686,6 +758,14 @@ async function parseCodexRolloutFile(filePath, {
     }
     turnCount += 1;
 
+    if (!collectBreakdowns) {
+      addInto(totals, delta);
+      pendingToolNames = [];
+      pendingSkills = [];
+      pendingExecDetails = [];
+      return;
+    }
+
     const unique = [...new Set(pendingToolNames.filter(Boolean))];
     const tools = unique.length > 0 ? unique : ["text_response"];
     const share = 1 / tools.length;
@@ -822,6 +902,16 @@ async function parseCodexRolloutFile(filePath, {
     contentHasher,
     sourceHandle,
   })) {
+    const modelEvent = applyCodexModelEvent(modelAttributionState, obj);
+    const attributedModel = currentCodexModel(modelAttributionState);
+    model = attributedModel || model;
+    // onObject runs AFTER the attribution state machine, and is handed the
+    // model it just decided. A consumer sharing this pass (the delivery-signal
+    // collector, which attributes edit turns) therefore sees exactly the model
+    // recordModelUsage() bills the turn's tokens to. Letting it re-read
+    // turn_context.payload.model itself would make codex-model-attribution.js
+    // one of two authorities on the same question.
+    if (typeof onObject === "function") onObject(obj, attributedModel);
     const ts = typeof obj?.timestamp === "string" ? obj.timestamp : null;
     if (!ts) continue;
     const inRequestedRange = isTimestampInRequestedDayRange(ts, { from, to, timeZoneContext });
@@ -849,18 +939,19 @@ async function parseCodexRolloutFile(filePath, {
     if (obj.type === "turn_context") {
       const p = obj.payload || {};
       if (typeof p.cwd === "string") cwd = p.cwd;
-      if (typeof p.model === "string") model = p.model;
       continue;
     }
 
-    if (obj.type === "response_item" && obj.payload?.type === "function_call") {
+    if (modelEvent?.type === "model_rerouted") continue;
+
+    if (collectBreakdowns && obj.type === "response_item" && obj.payload?.type === "function_call") {
       pendingToolNames.push(normalizeToolName(obj.payload));
       const skill = extractSkillNameFromFunctionCall(obj.payload, diagnostics);
       if (skill) pendingSkills.push(skill);
       continue;
     }
 
-    if (obj.type === "event_msg" && obj.payload?.type === "exec_command_end") {
+    if (collectBreakdowns && obj.type === "event_msg" && obj.payload?.type === "exec_command_end") {
       const details = getExecKeys(obj.payload);
       if (details) pendingExecDetails.push(details);
       continue;
@@ -881,6 +972,7 @@ async function parseCodexRolloutFile(filePath, {
           attributeTurn(null);
         } else {
           if (seenTokenEvents && delta?.total_tokens > 0) seenTokenEvents.add(eventKey);
+          recordModelUsage(delta, lastUsage || rawDelta);
           attributeTurn(delta);
         }
       } else {
@@ -895,7 +987,7 @@ async function parseCodexRolloutFile(filePath, {
   const result = {
     sessionId: sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath,
     cwd,
-    model: model || provider,
+    model: currentCodexModel(modelAttributionState) || model || provider,
     provider,
     version: cliVersion,
     forkedFromId,
@@ -906,6 +998,7 @@ async function parseCodexRolloutFile(filePath, {
     filePath: primaryFilePath,
     turnCount,
     totals,
+    modelUsage: finalizeModelUsageRows(byModel),
     toolBreakdown: {
       tool_rows: finalizeToolRows(byTool),
     },
@@ -926,6 +1019,7 @@ async function parseCodexRolloutFile(filePath, {
       sessionId,
       cwd,
       model,
+      modelAttributionState: snapshotCodexModelAttributionState(modelAttributionState),
       provider,
       cliVersion,
       forkedFromId,

package/src/lib/pricing/curated-overrides.json

@@ -48,11 +48,11 @@
       "note": "Canonical undated alias emitted by Cursor. Pin to the same rates as claude-haiku-4-5-20251001 so local cost does not depend on which dated LiteLLM aliases are present in the current cache."
     },
     "gpt-5.6-sol": {
-      "input": 5,
-      "output": 30,
-      "cache_read": 0.5,
-      "cache_write": 6.25,
-      "note": "GPT-5.6 family (public 2026-07-09). Flagship tier. Not yet in LiteLLM. developers.openai.com/api/docs/pricing short-context: $5/$30 per MTok, cached input 0.1x, cache write 1.25x. Codex emits gpt-5.6-sol (+ reasoning-effort variants like gpt-5.6-solhigh, caught by fuzzy). Remove once LiteLLM carries it."
+      "input": 4,
+      "output": 20,
+      "cache_read": 0.4,
+      "cache_write": 5,
+      "note": "Official GPT-5.6 Sol standard pricing: $4 input, $0.40 cached input, $5 cache write, and $20 output per MTok. Requests above 272K input are repriced from their preserved per-request long-context subset. Codex emits gpt-5.6-sol plus reasoning-effort variants caught by fuzzy matching."
     },
     "gpt-5.6-terra": {
       "input": 2,
@@ -428,7 +428,7 @@
     },
     {
       "match": "gpt-5.6",
-      "ref": "gpt-5.6-terra"
+      "ref": "gpt-5.6-sol"
     },
     {
       "match": "kiro",

package/src/lib/pricing/index.js

@@ -25,6 +25,11 @@ const PI_SUBSCRIPTION_SOURCES = new Set([
 // server (including LM Link). Secure Cloud usage has a separate billing path
 // and is not present in these logs.
 const LOCAL_INFERENCE_SOURCES = new Set(["lmstudio"]);
+// Forward-looking: unreachable for Codex today. Every observed token_count
+// reports model_context_window = 258400, below this threshold, so no request
+// can exceed it (max raw input seen locally: 238853). The long-context
+// repricing stays in place for when OpenAI ships a larger Codex window.
+const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272_000;
 const SOURCES_WITH_AUTHORITATIVE_COST = new Set(["grok"]);
 const SEED_SNAPSHOT_PATH = path.resolve(__dirname, "seed-snapshot.json");
 const DEEPSEEK_TIME_PRICED_MODELS = [
@@ -192,7 +197,7 @@ function computeRowCost(row) {
   const reasoningCost = reasoningIncludedInOutput
     ? 0
     : (row.reasoning_output_tokens || 0) * (pricing.output || 0);
-  return (
+  const baseCost = (
     ((row.input_tokens || 0) * (pricing.input || 0) +
       (row.output_tokens || 0) * (pricing.output || 0) +
       (row.cached_input_tokens || 0) * (pricing.cache_read || 0) +
@@ -200,6 +205,37 @@ function computeRowCost(row) {
       reasoningCost) /
     1_000_000
   );
+
+  const model = String(row?.model || "").toLowerCase();
+  const usesGpt56SolLongContextTier =
+    model === "gpt-5.6" || model.includes("gpt-5.6-sol");
+  if (!usesGpt56SolLongContextTier) return baseCost;
+
+  const bounded = (value, total) => Math.min(
+    Math.max(0, Number(value) || 0),
+    Math.max(0, Number(total) || 0),
+  );
+  // The parser records only usage from requests whose cache-inclusive input
+  // exceeded 272K. OpenAI prices the whole such request at 2x input and 1.5x
+  // output, so add the premium over the standard-rate base exactly once.
+  const longInput = bounded(row.long_context_input_tokens, row.input_tokens);
+  const longCached = bounded(row.long_context_cached_input_tokens, row.cached_input_tokens);
+  const longCacheWrite = bounded(
+    row.long_context_cache_creation_input_tokens,
+    row.cache_creation_input_tokens,
+  );
+  const longOutput = bounded(row.long_context_output_tokens, row.output_tokens);
+  const longReasoning = reasoningIncludedInOutput
+    ? 0
+    : bounded(row.long_context_reasoning_output_tokens, row.reasoning_output_tokens);
+  const longContextPremium = (
+    longInput * (pricing.input || 0) +
+    longCached * (pricing.cache_read || 0) +
+    longCacheWrite * (pricing.cache_write || 0) +
+    0.5 * longOutput * (pricing.output || 0) +
+    0.5 * longReasoning * (pricing.output || 0)
+  ) / 1_000_000;
+  return baseCost + longContextPremium;
 }
 
 // Backwards-compatible MODEL_PRICING export. Test at
@@ -219,6 +255,7 @@ module.exports = {
   resetPricingForTests,
   MODEL_PRICING,
   ZERO_PRICING,
+  OPENAI_LONG_CONTEXT_INPUT_THRESHOLD,
   // Internal hooks for tests.
   __getStateForTests: () => state,
 };