tledger 0.2.1 → 0.3.1

package/lib/token-ledger-importer.mjs

@@ -11,22 +11,18 @@
  * Node 22.13 or newer is required for node:sqlite.
  */
 
-import { createHash, randomUUID } from "node:crypto";
+import { createHash } from "node:crypto";
 import { createReadStream } from "node:fs";
 import {
   access,
-  chmod,
-  mkdir,
+  mkdtemp,
   readdir,
-  rename,
   rm,
   stat,
-  writeFile,
 } from "node:fs/promises";
-import { homedir } from "node:os";
+import { homedir, tmpdir } from "node:os";
 import {
   basename,
-  dirname,
   extname,
   resolve,
 } from "node:path";
@@ -34,7 +30,13 @@ import { pathToFileURL } from "node:url";
 import { createInterface } from "node:readline";
 import { DatabaseSync } from "node:sqlite";
 
-const SCHEMA_VERSION = 1;
+import { writePrivateSnapshot } from "./token-ledger-snapshot.mjs";
+import {
+  buildUsageBuckets,
+  SNAPSHOT_SCHEMA_VERSION,
+  usageBucketStats,
+} from "./token-ledger-usage.mjs";
+
 const RATE_CARD_AS_OF = "2026-08-17";
 const FAST_MODE_MULTIPLIER = 1.5;
 const RATE_CARD_URL = "https://help.openai.com/en/articles/20001106";
@@ -42,6 +44,262 @@ const WEEK_MINUTES = 10_080;
 const UUID_AT_END =
   /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
 
+function spoolText(value, maximumLength = 2_000) {
+  try {
+    return String(value ?? "").slice(0, maximumLength);
+  } catch {
+    return "";
+  }
+}
+
+function spoolSource(value) {
+  const descriptor = sourceDescriptor(value);
+  if (descriptor.subagent) return '{"subagent":true}';
+  return descriptor.label || "";
+}
+
+async function createUsageSpool() {
+  const directory = await mkdtemp(resolve(tmpdir(), "token-ledger-import-"));
+  const path = resolve(directory, "usage.sqlite");
+  const database = new DatabaseSync(path);
+  database.exec(`
+    PRAGMA journal_mode = OFF;
+    PRAGMA synchronous = OFF;
+    PRAGMA temp_store = FILE;
+    CREATE TABLE token_events (
+      event_key TEXT PRIMARY KEY,
+      turn_id TEXT NOT NULL,
+      input_tokens REAL NOT NULL,
+      cached_input_tokens REAL NOT NULL,
+      cache_write_input_tokens REAL NOT NULL,
+      output_tokens REAL NOT NULL,
+      reasoning_tokens REAL NOT NULL,
+      total_tokens REAL NOT NULL,
+      timestamp TEXT NOT NULL,
+      thread_id TEXT NOT NULL,
+      model TEXT NOT NULL,
+      effort TEXT NOT NULL,
+      cwd TEXT NOT NULL,
+      git_origin TEXT,
+      raw_source TEXT,
+      service_tier TEXT,
+      original_likely INTEGER NOT NULL
+    ) WITHOUT ROWID;
+    CREATE TABLE tool_calls (
+      call_key TEXT PRIMARY KEY,
+      turn_id TEXT NOT NULL,
+      thread_id TEXT NOT NULL,
+      original_likely INTEGER NOT NULL
+    ) WITHOUT ROWID;
+    CREATE TABLE turn_origins (
+      turn_id TEXT PRIMARY KEY,
+      thread_id TEXT NOT NULL,
+      timestamp TEXT NOT NULL,
+      delta_ms REAL NOT NULL,
+      model TEXT NOT NULL,
+      effort TEXT NOT NULL,
+      cwd TEXT NOT NULL,
+      git_origin TEXT,
+      raw_source TEXT,
+      service_tier TEXT
+    ) WITHOUT ROWID;
+    BEGIN IMMEDIATE;
+  `);
+  const insertToken = database.prepare(`
+    INSERT OR IGNORE INTO token_events (
+      event_key, turn_id, input_tokens, cached_input_tokens,
+      cache_write_input_tokens, output_tokens, reasoning_tokens,
+      total_tokens, timestamp, thread_id, model, effort, cwd,
+      git_origin, raw_source, service_tier, original_likely
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  const promoteToken = database.prepare(`
+    UPDATE token_events
+       SET turn_id = ?, timestamp = ?, thread_id = ?, model = ?, effort = ?,
+           cwd = ?, git_origin = ?, raw_source = ?, service_tier = ?,
+           original_likely = 1
+     WHERE event_key = ? AND original_likely = 0
+  `);
+  const insertCall = database.prepare(`
+    INSERT OR IGNORE INTO tool_calls (
+      call_key, turn_id, thread_id, original_likely
+    ) VALUES (?, ?, ?, ?)
+  `);
+  const promoteCall = database.prepare(`
+    UPDATE tool_calls
+       SET turn_id = ?, thread_id = ?, original_likely = 1
+     WHERE call_key = ? AND original_likely = 0
+  `);
+  const insertOrigin = database.prepare(`
+    INSERT INTO turn_origins (
+      turn_id, thread_id, timestamp, delta_ms, model, effort, cwd,
+      git_origin, raw_source, service_tier
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(turn_id) DO UPDATE SET
+      thread_id = excluded.thread_id,
+      timestamp = excluded.timestamp,
+      delta_ms = excluded.delta_ms,
+      model = excluded.model,
+      effort = excluded.effort,
+      cwd = excluded.cwd,
+      git_origin = excluded.git_origin,
+      raw_source = excluded.raw_source,
+      service_tier = excluded.service_tier
+    WHERE excluded.delta_ms < turn_origins.delta_ms
+  `);
+  const updateOrigin = database.prepare(`
+    UPDATE turn_origins
+       SET thread_id = ?, timestamp = ?, model = ?, effort = ?, cwd = ?,
+           git_origin = ?, raw_source = ?, service_tier = ?
+     WHERE turn_id = ?
+  `);
+  let writing = true;
+
+  function originValues(candidate) {
+    return [
+      candidate.turnId,
+      candidate.threadId,
+      candidate.timestamp,
+      Number.isFinite(candidate.deltaMs)
+        ? candidate.deltaMs
+        : Number.MAX_SAFE_INTEGER,
+      spoolText(candidate.model, 200),
+      spoolText(candidate.effort, 80),
+      spoolText(candidate.cwd),
+      spoolText(candidate.gitOrigin),
+      spoolSource(candidate.rawSource),
+      candidate.serviceTier == null
+        ? null
+        : spoolText(candidate.serviceTier, 80),
+    ];
+  }
+
+  return {
+    directory,
+    insertOrigin(candidate) {
+      return insertOrigin.run(...originValues(candidate)).changes > 0;
+    },
+    updateOrigin(candidate) {
+      const values = originValues(candidate);
+      updateOrigin.run(
+        values[1],
+        values[2],
+        values[4],
+        values[5],
+        values[6],
+        values[7],
+        values[8],
+        values[9],
+        values[0],
+      );
+    },
+    insertToken(eventKey, turnId, usage, occurrence, originalLikely) {
+      const values = [
+        eventKey,
+        turnId,
+        usage.inputTokens,
+        usage.cachedInputTokens,
+        usage.cacheWriteInputTokens,
+        usage.outputTokens,
+        usage.reasoningTokens,
+        usage.totalTokens,
+        occurrence.timestamp,
+        occurrence.threadId,
+        spoolText(occurrence.model, 200),
+        spoolText(occurrence.effort, 80),
+        spoolText(occurrence.cwd),
+        spoolText(occurrence.gitOrigin),
+        spoolSource(occurrence.rawSource),
+        occurrence.serviceTier == null
+          ? null
+          : spoolText(occurrence.serviceTier, 80),
+        originalLikely ? 1 : 0,
+      ];
+      const inserted = insertToken.run(...values).changes > 0;
+      if (!inserted && originalLikely) {
+        promoteToken.run(
+          turnId,
+          occurrence.timestamp,
+          occurrence.threadId,
+          spoolText(occurrence.model, 200),
+          spoolText(occurrence.effort, 80),
+          spoolText(occurrence.cwd),
+          spoolText(occurrence.gitOrigin),
+          spoolSource(occurrence.rawSource),
+          occurrence.serviceTier == null
+            ? null
+            : spoolText(occurrence.serviceTier, 80),
+          eventKey,
+        );
+      }
+      return inserted;
+    },
+    insertCall(callKey, turnId, threadId, originalLikely) {
+      const inserted = insertCall.run(
+        callKey,
+        turnId,
+        threadId,
+        originalLikely ? 1 : 0,
+      ).changes > 0;
+      if (!inserted && originalLikely) {
+        promoteCall.run(turnId, threadId, callKey);
+      }
+    },
+    finishWrites() {
+      if (!writing) return;
+      database.exec("COMMIT");
+      writing = false;
+    },
+    tokenRows() {
+      return database.prepare(`
+        SELECT token.event_key AS eventKey, token.turn_id AS turnId,
+               token.input_tokens AS inputTokens,
+               token.cached_input_tokens AS cachedInputTokens,
+               token.cache_write_input_tokens AS cacheWriteInputTokens,
+               token.output_tokens AS outputTokens,
+               token.reasoning_tokens AS reasoningTokens,
+               token.total_tokens AS totalTokens,
+               token.timestamp, token.thread_id AS threadId,
+               token.model, token.effort, token.cwd,
+               token.git_origin AS gitOrigin,
+               token.raw_source AS rawSource,
+               token.service_tier AS serviceTier,
+               token.original_likely AS originalLikely,
+               origin.thread_id AS originThreadId,
+               origin.timestamp AS originTimestamp,
+               origin.model AS originModel,
+               origin.effort AS originEffort,
+               origin.cwd AS originCwd,
+               origin.git_origin AS originGitOrigin,
+               origin.raw_source AS originRawSource,
+               origin.service_tier AS originServiceTier
+          FROM token_events AS token
+          LEFT JOIN turn_origins AS origin ON origin.turn_id = token.turn_id
+         ORDER BY token.timestamp, token.event_key
+      `).iterate();
+    },
+    callRows() {
+      return database.prepare(`
+        SELECT call.turn_id AS turnId,
+               COALESCE(origin.thread_id, call.thread_id) AS threadId
+          FROM tool_calls AS call
+          LEFT JOIN turn_origins AS origin ON origin.turn_id = call.turn_id
+      `).iterate();
+    },
+    close() {
+      if (writing) {
+        try {
+          database.exec("ROLLBACK");
+        } catch {
+          // The spool is disposable; cleanup below remains authoritative.
+        }
+        writing = false;
+      }
+      database.close();
+    },
+  };
+}
+
 const RATE_CARD = {
   "gpt-5.6-sol": { input: 125, cached: 12.5, output: 750 },
   "gpt-5.6-terra": { input: 50, cached: 5, output: 300 },
@@ -61,7 +319,7 @@ Usage:
   node token-ledger-importer.mjs [options]
 
 Options:
-  --output <file>       Snapshot destination (default: token-ledger-snapshot.json)
+  --output <file>       Snapshot destination (default: token-ledger-snapshot-v2.json.gz)
   --codex-home <dir>    Codex data root (default: CODEX_HOME or ~/.codex)
   --since <ISO date>    Ignore model calls before this timestamp
   --no-archived         Skip archived_sessions
@@ -75,7 +333,7 @@ text.`;
 
 function parseArgs(argv) {
   const options = {
-    output: resolve("token-ledger-snapshot.json"),
+    output: resolve("token-ledger-snapshot-v2.json.gz"),
     codexHome: resolve(process.env.CODEX_HOME || `${homedir()}/.codex`),
     includeArchived: true,
     since: null,
@@ -131,24 +389,39 @@ function safeIso(value, fallback = null) {
 }
 
 function tokenTuple(value) {
-  const usage = value && typeof value === "object" ? value : {};
+  // token_count payloads arrive from untrusted JSONL. Read the named usage
+  // fields once and coerce each to a finite count so malformed records
+  // contribute zeros instead of leaking odd shapes downstream.
+  const {
+    input_tokens: inputTokens,
+    cached_input_tokens: cachedInputTokens,
+    cache_write_input_tokens: cacheWriteInputTokens,
+    output_tokens: outputTokens,
+    reasoning_output_tokens: reasoningOutputTokens,
+    total_tokens: totalTokens,
+  } = value ?? {};
   return [
-    asFiniteNumber(usage.input_tokens),
-    asFiniteNumber(usage.cached_input_tokens),
-    asFiniteNumber(usage.cache_write_input_tokens),
-    asFiniteNumber(usage.output_tokens),
-    asFiniteNumber(usage.reasoning_output_tokens),
-    asFiniteNumber(usage.total_tokens),
+    asFiniteNumber(inputTokens),
+    asFiniteNumber(cachedInputTokens),
+    asFiniteNumber(cacheWriteInputTokens),
+    asFiniteNumber(outputTokens),
+    asFiniteNumber(reasoningOutputTokens),
+    asFiniteNumber(totalTokens),
   ];
 }
 
 function usageFromTuple(tuple) {
+  // Cached input is a subset of input and reasoning is a subset of output.
+  // Clamp at export so one out-of-range source record cannot skew subset
+  // math downstream.
+  const inputTokens = tuple[0];
+  const outputTokens = tuple[3];
   return {
-    inputTokens: tuple[0],
-    cachedInputTokens: tuple[1],
+    inputTokens,
+    cachedInputTokens: Math.min(inputTokens, tuple[1]),
     cacheWriteInputTokens: tuple[2],
-    outputTokens: tuple[3],
-    reasoningTokens: tuple[4],
+    outputTokens,
+    reasoningTokens: Math.min(outputTokens, tuple[4]),
     totalTokens: tuple[5],
   };
 }
@@ -162,10 +435,14 @@ function hasDetailedBreakdown(usage) {
 }
 
 function normalizeModel(model) {
+  // Collapse underscore and whitespace separators to dashes so variants like
+  // "gpt-5.4 mini" resolve to their own rate-card entry instead of falling
+  // back to the base model's higher rate. Keep in lockstep with
+  // normalizeModel in bin/token-ledger-rates.mjs.
   const value = String(model || "unknown")
     .trim()
     .toLowerCase()
-    .replaceAll("_", "-");
+    .replace(/[\s_]+/g, "-");
   if (RATE_CARD[value]) return value;
   if (value.startsWith("gpt-5.6-sol")) return "gpt-5.6-sol";
   if (value.startsWith("gpt-5.6-terra")) return "gpt-5.6-terra";
@@ -193,25 +470,40 @@ function creditsForUsage(model, usage) {
   );
 }
 
-function parseStructuredSource(value) {
-  if (!value) return null;
-  if (typeof value === "object") return value;
-  if (typeof value !== "string") return null;
-  const trimmed = value.trim();
-  if (!trimmed.startsWith("{")) return trimmed;
+function primitiveString(value) {
   try {
-    return JSON.parse(trimmed);
+    const text = String.prototype.valueOf.call(value);
+    return text === value ? text : null;
   } catch {
-    return trimmed;
+    return null;
+  }
+}
+
+function sourceDescriptor(value) {
+  // Thread sources reach us in two representations: rollout session_meta
+  // lines carry parsed JSON (a label string or an object with subagent
+  // details) while the sqlite source column carries the same value as text.
+  // Parse both here, once, so callers only branch on the named fields.
+  if (value == null) return { label: null, subagent: null };
+  const sourceText = primitiveString(value);
+  if (sourceText === null) {
+    return { label: null, subagent: value.subagent || null };
+  }
+  const text = sourceText.trim();
+  if (text.startsWith("{")) {
+    try {
+      const parsed = JSON.parse(text);
+      return { label: null, subagent: parsed.subagent || null };
+    } catch {
+      // Not JSON after all; fall through to the plain-label case.
+    }
   }
+  return { label: text || null, subagent: null };
 }
 
 function sourceLabels(threadSource, rawSource) {
-  const source = parseStructuredSource(rawSource);
-  if (
-    threadSource === "subagent" ||
-    (source && typeof source === "object" && source.subagent)
-  ) {
+  const source = sourceDescriptor(rawSource);
+  if (threadSource === "subagent" || source.subagent) {
     return { source: "subagent", useType: "subagent" };
   }
   if (threadSource === "automation") {
@@ -220,10 +512,15 @@ function sourceLabels(threadSource, rawSource) {
   if (threadSource === "realtime_voice") {
     return { source: "voice", useType: "voice" };
   }
-  if (source === "exec") return { source: "cli", useType: "cli" };
-  if (source === "vscode") return { source: "desktop", useType: "interactive" };
-  if (typeof source === "string" && source) {
-    return { source: source.slice(0, 40), useType: threadSource || "interactive" };
+  if (source.label === "exec") return { source: "cli", useType: "cli" };
+  if (source.label === "vscode") {
+    return { source: "desktop", useType: "interactive" };
+  }
+  if (source.label) {
+    return {
+      source: source.label.slice(0, 40),
+      useType: threadSource || "interactive",
+    };
   }
   return { source: "unknown", useType: threadSource || "unknown" };
 }
@@ -291,28 +588,6 @@ async function pathExists(path) {
   }
 }
 
-export async function writePrivateSnapshot(output, snapshot) {
-  const destination = resolve(output);
-  const directory = dirname(destination);
-  const temporary = resolve(
-    directory,
-    `.token-ledger-${process.pid}-${randomUUID()}.tmp`,
-  );
-  await mkdir(directory, { recursive: true });
-  try {
-    await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
-      encoding: "utf8",
-      flag: "wx",
-      mode: 0o600,
-    });
-    await chmod(temporary, 0o600);
-    await rename(temporary, destination);
-    await chmod(destination, 0o600);
-  } finally {
-    await rm(temporary, { force: true });
-  }
-}
-
 export async function listJsonlFiles(root) {
   if (!(await pathExists(root))) return [];
   const found = [];
@@ -440,14 +715,22 @@ function taskStartCandidate(record, threadId, stateRow, fileContext) {
 }
 
 function rememberQuota(quotaMap, rateLimits, occurrence) {
-  if (!rateLimits || typeof rateLimits !== "object") return;
-  const buckets = [rateLimits.primary, rateLimits.secondary].filter(Boolean);
+  // rate_limits payloads come straight from JSONL. Read the named fields
+  // once; records without usable buckets drop out via the filter below.
+  const {
+    primary,
+    secondary,
+    limit_id: limitId,
+    limit_name: limitName,
+    plan_type: planType,
+  } = rateLimits ?? {};
+  const buckets = [primary, secondary].filter(Boolean);
   for (const bucket of buckets) {
     const windowMinutes = asFiniteNumber(bucket.window_minutes);
     const usedPercent = asFiniteNumber(bucket.used_percent);
     const resetsAt = asFiniteNumber(bucket.resets_at);
     if (!windowMinutes || !resetsAt) continue;
-    const limitKey = String(rateLimits.limit_id || rateLimits.limit_name || "anonymous");
+    const limitKey = String(limitId || limitName || "anonymous");
     const key = [
       hash(limitKey, 16),
       windowMinutes,
@@ -456,15 +739,16 @@ function rememberQuota(quotaMap, rateLimits, occurrence) {
     ].join("|");
     const candidate = {
       id: `quota-${hash(key)}`,
+      // Keep the first and last occurrence of an unchanged reading without
+      // storing every repeated provider sample in the snapshot.
       timestamp: occurrence.timestamp,
+      lastSeenAt: occurrence.timestamp,
       usedPercent,
       windowMinutes,
       resetsAt,
-      planType: String(rateLimits.plan_type || "unknown"),
+      planType: String(planType || "unknown"),
       limitKey: hash(limitKey, 16),
-      limitName: rateLimits.limit_name
-        ? String(rateLimits.limit_name).slice(0, 80)
-        : null,
+      limitName: limitName ? String(limitName).slice(0, 80) : null,
       source: "log",
       turnId: occurrence.turnId || null,
       originalLikely: occurrence.originalLikely,
@@ -472,11 +756,21 @@ function rememberQuota(quotaMap, rateLimits, occurrence) {
     const current = quotaMap.get(key);
     if (
       !current ||
-      (!current.originalLikely && candidate.originalLikely) ||
-      (current.originalLikely === candidate.originalLikely &&
-        candidate.timestamp < current.timestamp)
+      (!current.originalLikely && candidate.originalLikely)
     ) {
       quotaMap.set(key, candidate);
+    } else if (current.originalLikely === candidate.originalLikely) {
+      quotaMap.set(key, {
+        ...current,
+        timestamp:
+          candidate.timestamp < current.timestamp
+            ? candidate.timestamp
+            : current.timestamp,
+        lastSeenAt:
+          candidate.timestamp > current.lastSeenAt
+            ? candidate.timestamp
+            : current.lastSeenAt,
+      });
     }
   }
 }
@@ -514,6 +808,7 @@ async function scanRollout(path, context) {
   const callOrdinals = new Map();
   let currentTurnId = "";
   let currentCandidate = null;
+  let currentCandidateSelected = false;
   let previousCumulative = null;
 
   const input = createReadStream(path, { encoding: "utf8" });
@@ -557,10 +852,7 @@ async function scanRollout(path, context) {
           stateRow,
           fileContext,
         );
-        const currentBest = context.origins.get(currentTurnId);
-        if (!currentBest || currentCandidate.deltaMs < currentBest.deltaMs) {
-          context.origins.set(currentTurnId, currentCandidate);
-        }
+        currentCandidateSelected = context.spool.insertOrigin(currentCandidate);
       }
       continue;
     }
@@ -574,6 +866,9 @@ async function scanRollout(path, context) {
         currentCandidate.model = fileContext.model;
         currentCandidate.effort = fileContext.effort;
         currentCandidate.cwd = fileContext.cwd;
+        if (currentCandidateSelected) {
+          context.spool.updateOrigin(currentCandidate);
+        }
       }
       continue;
     }
@@ -617,14 +912,12 @@ async function scanRollout(path, context) {
       const callKey = call.stableId
         ? `id|${call.stableId}`
         : `ordinal|${ordinalBase}|${ordinal}`;
-      const existing = context.calls.get(callKey);
-      if (!existing || (!existing.originalLikely && originalLikely)) {
-        context.calls.set(callKey, {
-          turnId: currentTurnId,
-          threadId,
-          originalLikely,
-        });
-      }
+      context.spool.insertCall(
+        callKey,
+        currentTurnId,
+        threadId,
+        originalLikely,
+      );
       continue;
     }
 
@@ -657,24 +950,16 @@ async function scanRollout(path, context) {
     }
     previousCumulative = totalTuple[5];
 
-    const existing = context.tokens.get(eventKey);
-    if (existing) {
-      context.duplicateEventsSkipped += 1;
-      if (!existing.originalLikely && originalLikely) {
-        existing.occurrence = occurrence;
-        existing.originalLikely = true;
-      }
-      continue;
-    }
-
-    context.tokens.set(eventKey, {
-      key: eventKey,
-      turnId: currentTurnId,
-      usage: usageFromTuple(lastTuple),
+    const inserted = context.spool.insertToken(
+      eventKey,
+      currentTurnId,
+      usageFromTuple(lastTuple),
       occurrence,
       originalLikely,
-      dedupeQuality: currentTurnId ? "turn-exact" : "legacy-heuristic",
-    });
+    );
+    if (!inserted) {
+      context.duplicateEventsSkipped += 1;
+    }
   }
 }
 
@@ -708,144 +993,183 @@ function threadMetadata(threadId, stateRows, titles, parents, fallback = {}) {
   };
 }
 
+function resolvedOccurrence(token) {
+  const origin = token.originThreadId == null
+    ? null
+    : {
+        threadId: token.originThreadId,
+        timestamp: token.originTimestamp,
+        model: token.originModel,
+        effort: token.originEffort,
+        cwd: token.originCwd,
+        gitOrigin: token.originGitOrigin,
+        rawSource: token.originRawSource,
+        serviceTier: token.originServiceTier,
+      };
+  const occurrence = token.originalLikely || !origin ? token : origin;
+  return {
+    origin,
+    occurrence,
+    threadId: origin?.threadId || occurrence.threadId,
+    timestamp: token.timestamp || origin?.timestamp,
+  };
+}
+
+function newThreadAggregate(metadata, timestamp) {
+  return {
+    metadata,
+    firstActiveAt: timestamp,
+    lastActiveAt: timestamp,
+    inputTokens: 0,
+    cachedInputTokens: 0,
+    outputTokens: 0,
+    reasoningTokens: 0,
+    totalTokens: 0,
+    toolCalls: 0,
+    detailedTokens: 0,
+    unknownBreakdownTokens: 0,
+    rateCardCredits: 0,
+    ratedTokens: 0,
+    eventCount: 0,
+  };
+}
+
+function addToThreadAggregate(aggregate, event) {
+  aggregate.lastActiveAt = event.timestamp;
+  aggregate.inputTokens += event.inputTokens;
+  aggregate.cachedInputTokens += event.cachedInputTokens;
+  aggregate.outputTokens += event.outputTokens;
+  aggregate.reasoningTokens += event.reasoningTokens;
+  aggregate.totalTokens += event.totalTokens;
+  aggregate.toolCalls += event.toolCalls;
+  aggregate.eventCount += 1;
+  if (event.breakdownAvailable) {
+    aggregate.detailedTokens += event.totalTokens;
+  } else {
+    aggregate.unknownBreakdownTokens += event.totalTokens;
+  }
+  if (event.rateCardCredits !== null) {
+    aggregate.rateCardCredits += event.rateCardCredits;
+    aggregate.ratedTokens += event.totalTokens;
+  }
+}
+
 function buildSnapshot(context, options, titles) {
-  const events = [];
-  for (const token of context.tokens.values()) {
-    const origin = token.turnId ? context.origins.get(token.turnId) : null;
-    const occurrence =
-      token.originalLikely || !origin
-        ? token.occurrence
-        : {
-            ...token.occurrence,
-            ...origin,
-            timestamp: token.occurrence.timestamp,
-          };
-    const threadId = origin?.threadId || occurrence.threadId;
-    const metadata = threadMetadata(
-      threadId,
-      context.stateRows,
-      titles,
-      context.parents,
-      origin || occurrence,
-    );
-    const timestamp = occurrence.timestamp || origin?.timestamp;
-    if (
-      options.since &&
-      new Date(timestamp).getTime() < options.since.getTime()
-    ) {
-      continue;
-    }
-    const breakdownAvailable = hasDetailedBreakdown(token.usage);
-    const serviceTier = occurrence.serviceTier || null;
-    const baseCredits = creditsForUsage(metadata.model, token.usage);
-    events.push({
-      ...token.usage,
-      id: `evt-${hash(token.key)}`,
-      timestamp,
-      threadId,
-      threadTitle: metadata.title,
-      project: metadata.project,
-      model: metadata.model,
-      effort: metadata.effort,
-      source: metadata.source,
-      useType: metadata.useType,
-      turnId: token.turnId || "",
-      toolCalls: 0,
-      serviceTier,
-      rateCardCredits:
-        baseCredits === null
-          ? null
-          : serviceTier === "priority"
-            ? baseCredits * FAST_MODE_MULTIPLIER
-            : baseCredits,
-      breakdownAvailable,
-      dedupeQuality: token.dedupeQuality,
-    });
+  context.spool.finishWrites();
+  const sinceMs = options.since ? options.since.getTime() : -Infinity;
+  const lastEventKeyByTurn = new Map();
+  let earliestEventAt = null;
+  let latestEventAt = null;
+
+  // The spool is ordered on disk. This lightweight pass finds the event that
+  // receives each turn's tool calls without retaining the usage rows.
+  for (const token of context.spool.tokenRows()) {
+    const { threadId, timestamp } = resolvedOccurrence(token);
+    if (Date.parse(timestamp) < sinceMs) continue;
+    earliestEventAt ||= timestamp;
+    latestEventAt = timestamp;
+    const key = token.turnId || `thread:${threadId}`;
+    lastEventKeyByTurn.set(key, token.eventKey);
   }
-  events.sort(
-    (left, right) =>
-      new Date(left.timestamp).getTime() - new Date(right.timestamp).getTime(),
-  );
 
   const toolCounts = new Map();
-  for (const call of context.calls.values()) {
-    const origin = call.turnId ? context.origins.get(call.turnId) : null;
-    const threadId = origin?.threadId || call.threadId;
-    const key = call.turnId || `thread:${threadId}`;
+  for (const call of context.spool.callRows()) {
+    const key = call.turnId || `thread:${call.threadId}`;
     toolCounts.set(key, (toolCounts.get(key) || 0) + 1);
   }
-  const lastEventByTurn = new Map();
-  for (const event of events) {
-    const key = event.turnId || `thread:${event.threadId}`;
-    lastEventByTurn.set(key, event);
-  }
-  for (const [key, count] of toolCounts) {
-    const event = lastEventByTurn.get(key);
-    if (event) event.toolCalls = count;
-  }
 
-  const grouped = new Map();
-  for (const event of events) {
-    const rows = grouped.get(event.threadId) || [];
-    rows.push(event);
-    grouped.set(event.threadId, rows);
+  const threadAggregates = new Map();
+  let observedTokens = 0;
+  let detailedTokens = 0;
+  let legacyHeuristicEvents = 0;
+  let observedModelCalls = 0;
+
+  function* compactableEvents() {
+    for (const token of context.spool.tokenRows()) {
+      const { origin, occurrence, threadId, timestamp } = resolvedOccurrence(token);
+      if (Date.parse(timestamp) < sinceMs) continue;
+      const metadata = threadMetadata(
+        threadId,
+        context.stateRows,
+        titles,
+        context.parents,
+        origin || occurrence,
+      );
+      const usage = {
+        inputTokens: Number(token.inputTokens),
+        cachedInputTokens: Number(token.cachedInputTokens),
+        cacheWriteInputTokens: Number(token.cacheWriteInputTokens),
+        outputTokens: Number(token.outputTokens),
+        reasoningTokens: Number(token.reasoningTokens),
+        totalTokens: Number(token.totalTokens),
+      };
+      const breakdownAvailable = hasDetailedBreakdown(usage);
+      const serviceTier = occurrence.serviceTier || null;
+      const baseCredits = creditsForUsage(metadata.model, usage);
+      const turnKey = token.turnId || `thread:${threadId}`;
+      const event = {
+        ...usage,
+        timestamp,
+        threadId,
+        project: metadata.project,
+        model: metadata.model,
+        effort: metadata.effort,
+        source: metadata.source,
+        useType: metadata.useType,
+        toolCalls:
+          lastEventKeyByTurn.get(turnKey) === token.eventKey
+            ? toolCounts.get(turnKey) || 0
+            : 0,
+        serviceTier,
+        rateCardCredits:
+          baseCredits === null
+            ? null
+            : serviceTier === "priority"
+              ? baseCredits * FAST_MODE_MULTIPLIER
+              : baseCredits,
+        breakdownAvailable,
+      };
+
+      let aggregate = threadAggregates.get(threadId);
+      if (!aggregate) {
+        aggregate = newThreadAggregate(metadata, timestamp);
+        threadAggregates.set(threadId, aggregate);
+      }
+      addToThreadAggregate(aggregate, event);
+      observedTokens += event.totalTokens;
+      if (breakdownAvailable) detailedTokens += event.totalTokens;
+      if (!token.turnId) legacyHeuristicEvents += 1;
+      observedModelCalls += 1;
+      yield event;
+    }
   }
 
-  const allThreadIds = new Set([
-    ...context.stateRows.keys(),
-    ...grouped.keys(),
-  ]);
+  const usageBuckets = buildUsageBuckets(compactableEvents(), {
+    latestTimestampMs: latestEventAt ? Date.parse(latestEventAt) : 0,
+  });
+  const usageStats = usageBucketStats(usageBuckets);
+  toolCounts.clear();
+  lastEventKeyByTurn.clear();
+
+  const allThreadIds = new Set(context.stateRows.keys());
+  for (const threadId of threadAggregates.keys()) allThreadIds.add(threadId);
   const threads = [];
   for (const threadId of allThreadIds) {
-    const rows = grouped.get(threadId) || [];
-    const first = rows[0];
-    const last = rows.at(-1);
-    const origin = first
-      ? context.origins.get(first.turnId)
-      : null;
-    const metadata = threadMetadata(
+    const aggregate = threadAggregates.get(threadId);
+    const metadata = aggregate?.metadata || threadMetadata(
       threadId,
       context.stateRows,
       titles,
       context.parents,
-      origin || first || {},
-    );
-    const totals = rows.reduce(
-      (sum, event) => {
-        sum.inputTokens += event.inputTokens;
-        sum.cachedInputTokens += event.cachedInputTokens;
-        sum.outputTokens += event.outputTokens;
-        sum.reasoningTokens += event.reasoningTokens;
-        sum.totalTokens += event.totalTokens;
-        sum.toolCalls += event.toolCalls;
-        if (event.breakdownAvailable) sum.detailedTokens += event.totalTokens;
-        else sum.unknownBreakdownTokens += event.totalTokens;
-        if (event.rateCardCredits !== null) {
-          sum.rateCardCredits += event.rateCardCredits;
-          sum.ratedTokens += event.totalTokens;
-        }
-        return sum;
-      },
-      {
-        inputTokens: 0,
-        cachedInputTokens: 0,
-        outputTokens: 0,
-        reasoningTokens: 0,
-        totalTokens: 0,
-        toolCalls: 0,
-        detailedTokens: 0,
-        unknownBreakdownTokens: 0,
-        rateCardCredits: 0,
-        ratedTokens: 0,
-      },
     );
-    if (rows.length === 0 && !(metadata.reportedCumulativeTokens > 0)) continue;
-    const coverage =
-      rows.length === 0
+    if (!aggregate && !(metadata.reportedCumulativeTokens > 0)) continue;
+    const eventCount = aggregate?.eventCount || 0;
+    const threadCoverage =
+      eventCount === 0
         ? "unresolved"
-        : totals.detailedTokens === totals.totalTokens
+        : aggregate.detailedTokens === aggregate.totalTokens
           ? "complete"
-          : totals.detailedTokens > 0
+          : aggregate.detailedTokens > 0
             ? "partial"
             : "total-only";
     threads.push({
@@ -857,27 +1181,32 @@ function buildSnapshot(context, options, titles) {
       source: metadata.source,
       useType: metadata.useType,
       parentThreadId: metadata.parentThreadId,
-      firstActiveAt: first?.timestamp || metadata.createdAt,
-      lastActiveAt: last?.timestamp || metadata.updatedAt,
-      totalTokens: totals.totalTokens,
-      detailedTokens: totals.detailedTokens,
-      unknownBreakdownTokens: totals.unknownBreakdownTokens,
+      firstActiveAt: aggregate?.firstActiveAt || metadata.createdAt,
+      lastActiveAt: aggregate?.lastActiveAt || metadata.updatedAt,
+      totalTokens: aggregate?.totalTokens || 0,
+      detailedTokens: aggregate?.detailedTokens || 0,
+      unknownBreakdownTokens: aggregate?.unknownBreakdownTokens || 0,
       reportedCumulativeTokens: metadata.reportedCumulativeTokens,
-      inputTokens: totals.inputTokens,
-      cachedInputTokens: totals.cachedInputTokens,
-      outputTokens: totals.outputTokens,
-      reasoningTokens: totals.reasoningTokens,
+      inputTokens: aggregate?.inputTokens || 0,
+      cachedInputTokens: aggregate?.cachedInputTokens || 0,
+      outputTokens: aggregate?.outputTokens || 0,
+      reasoningTokens: aggregate?.reasoningTokens || 0,
       rateCardCredits:
-        totals.totalTokens > 0 && totals.ratedTokens === totals.totalTokens
-          ? totals.rateCardCredits
+        aggregate?.totalTokens > 0 &&
+        aggregate.ratedTokens === aggregate.totalTokens
+          ? aggregate.rateCardCredits
           : null,
-      ratedTokens: totals.ratedTokens,
-      toolCalls: totals.toolCalls,
-      eventCount: rows.length,
-      coverage,
+      ratedTokens: aggregate?.ratedTokens || 0,
+      toolCalls: aggregate?.toolCalls || 0,
+      eventCount,
+      coverage: threadCoverage,
     });
   }
   threads.sort((left, right) => right.totalTokens - left.totalTokens);
+  threadAggregates.clear();
+  allThreadIds.clear();
+  context.stateRows.clear();
+  context.parents.clear();
 
   const quotas = [...context.quotas.values()]
     .map((quota) => {
@@ -888,21 +1217,12 @@ function buildSnapshot(context, options, titles) {
     })
     .filter((quota) => {
       if (!options.since) return true;
-      return new Date(quota.timestamp).getTime() >= options.since.getTime();
+      return Date.parse(quota.lastSeenAt) >= sinceMs;
     })
     .sort(
-      (left, right) =>
-        new Date(left.timestamp).getTime() -
-        new Date(right.timestamp).getTime(),
+      (left, right) => Date.parse(left.timestamp) - Date.parse(right.timestamp),
     );
 
-  const observedTokens = events.reduce(
-    (sum, event) => sum + event.totalTokens,
-    0,
-  );
-  const detailedTokens = events
-    .filter((event) => event.breakdownAvailable)
-    .reduce((sum, event) => sum + event.totalTokens, 0);
   const unknownBreakdownTokens = observedTokens - detailedTokens;
   const stateCounterSumNonAdditive = threads.reduce(
     (sum, thread) => sum + (thread.reportedCumulativeTokens || 0),
@@ -913,8 +1233,6 @@ function buildSnapshot(context, options, titles) {
       thread.coverage === "unresolved" &&
       (thread.reportedCumulativeTokens || 0) > 0,
   ).length;
-  const earliestEventAt = events[0]?.timestamp || null;
-  const latestEventAt = events.at(-1)?.timestamp || null;
   const weeklyCandidates = quotas.filter(
     (quota) => quota.windowMinutes === WEEK_MINUTES,
   );
@@ -925,9 +1243,7 @@ function buildSnapshot(context, options, titles) {
     ...(accountWideWeekly.length ? accountWideWeekly : weeklyCandidates),
   ]
     .sort(
-      (left, right) =>
-        new Date(right.timestamp).getTime() -
-        new Date(left.timestamp).getTime(),
+      (left, right) => Date.parse(right.lastSeenAt) - Date.parse(left.lastSeenAt),
     )[0];
   const weeklyStart = weekly
     ? (weekly.resetsAt - weekly.windowMinutes * 60) * 1_000
@@ -935,19 +1251,20 @@ function buildSnapshot(context, options, titles) {
   const completeSinceWindowStart = Boolean(
     weeklyStart &&
       earliestEventAt &&
-      new Date(earliestEventAt).getTime() <= weeklyStart &&
+      Date.parse(earliestEventAt) <= weeklyStart &&
       context.parseErrors === 0,
   );
 
   const notes = [
     "Observed totals sum globally de-duplicated last_token_usage model-call events.",
+    "The snapshot keeps exact recent calls and compacts older usage into time buckets; token, cache, model, project, tool-call, and thread totals remain additive.",
     "Codex thread counters are retained only as non-additive reference values because forks and subagents inherit cumulative history.",
     "Historical rollout files can be pruned; a state-only counter cannot reveal that thread's unique token contribution.",
     "Legacy events without turn IDs use a high-specificity usage-signature heuristic and are labeled in the ledger.",
   ];
 
   return {
-    schemaVersion: SCHEMA_VERSION,
+    schemaVersion: SNAPSHOT_SCHEMA_VERSION,
     generatedAt: new Date().toISOString(),
     label: "Local Codex snapshot",
     provenance: {
@@ -968,9 +1285,10 @@ function buildSnapshot(context, options, titles) {
       unknownBreakdownTokens,
       stateCounterSumNonAdditive,
       unresolvedThreadCounters,
-      legacyHeuristicEvents: events.filter(
-        (event) => event.dedupeQuality === "legacy-heuristic",
-      ).length,
+      legacyHeuristicEvents,
+      observedModelCalls,
+      usageBucketCount: usageStats.bucketCount,
+      maximumUsageResolutionSeconds: usageStats.maximumResolutionSeconds,
       detailedPercent:
         observedTokens > 0 ? (detailedTokens / observedTokens) * 100 : 100,
       earliestEventAt,
@@ -980,7 +1298,7 @@ function buildSnapshot(context, options, titles) {
     },
     quotaObservations: quotas,
     threads,
-    events,
+    events: usageBuckets,
   };
 }
 
@@ -999,14 +1317,13 @@ export async function collectUsage(options, onProgress = () => {}) {
     .flat()
     .sort();
   const sizes = await Promise.all(files.map((path) => stat(path)));
+  const spool = await createUsageSpool();
 
   const context = {
     stateRows: state.rows,
     parents: state.parents,
-    origins: new Map(),
-    tokens: new Map(),
     quotas: new Map(),
-    calls: new Map(),
+    spool,
     filesScanned: 0,
     bytesScanned: 0,
     parseErrors: 0,
@@ -1014,24 +1331,28 @@ export async function collectUsage(options, onProgress = () => {}) {
     correctionIntervals: 0,
   };
 
-  for (let index = 0; index < files.length; index += 1) {
-    await scanRollout(files[index], context);
-    context.filesScanned += 1;
-    context.bytesScanned += sizes[index].size;
-    if (
-      index === 0 ||
-      index === files.length - 1 ||
-      (index + 1) % 10 === 0
-    ) {
-      onProgress({
-        current: index + 1,
-        total: files.length,
-        path: files[index],
-      });
+  try {
+    for (let index = 0; index < files.length; index += 1) {
+      await scanRollout(files[index], context);
+      context.filesScanned += 1;
+      context.bytesScanned += sizes[index].size;
+      if (
+        index === 0 ||
+        index === files.length - 1 ||
+        (index + 1) % 10 === 0
+      ) {
+        onProgress({
+          current: index + 1,
+          total: files.length,
+          path: files[index],
+        });
+      }
     }
+    return buildSnapshot(context, options, titles);
+  } finally {
+    spool.close();
+    await rm(spool.directory, { recursive: true, force: true });
   }
-
-  return buildSnapshot(context, options, titles);
 }
 
 async function main() {
@@ -1058,16 +1379,18 @@ async function main() {
     );
   });
   process.stdout.write("\nToken Ledger: writing privacy-reduced snapshot…\n");
-  await writePrivateSnapshot(options.output, snapshot);
-  const outputSize = (await stat(options.output)).size;
+  const writeResult = await writePrivateSnapshot(options.output, snapshot);
+  const storedSnapshot = writeResult.snapshot;
   process.stdout.write(
     [
       `Snapshot: ${options.output}`,
-      `Observed model-call tokens: ${snapshot.coverage.observedTokens.toLocaleString()}`,
-      `Unique events: ${snapshot.events.length.toLocaleString()}`,
-      `Duplicate/copied events skipped: ${snapshot.coverage.duplicateEventsSkipped.toLocaleString()}`,
-      `Threads with unresolved state-only counters: ${snapshot.coverage.unresolvedThreadCounters.toLocaleString()}`,
-      `Snapshot size: ${(outputSize / 1_000_000).toFixed(1)} MB`,
+      `Observed model-call tokens: ${storedSnapshot.coverage.observedTokens.toLocaleString()}`,
+      `Unique model calls: ${storedSnapshot.coverage.observedModelCalls.toLocaleString()}`,
+      `Stored usage buckets: ${storedSnapshot.events.length.toLocaleString()}`,
+      `Duplicate/copied events skipped: ${storedSnapshot.coverage.duplicateEventsSkipped.toLocaleString()}`,
+      `Threads with unresolved state-only counters: ${storedSnapshot.coverage.unresolvedThreadCounters.toLocaleString()}`,
+      `Snapshot size: ${(writeResult.bytesWritten / 1_000_000).toFixed(1)} MB (${writeResult.encoding}; ${(writeResult.jsonBytes / 1_000_000).toFixed(1)} MB JSON before encoding)`,
+      `Snapshot safety limit: ${(writeResult.maxBytes / 1_000_000).toFixed(1)} MB`,
     ].join("\n") + "\n",
   );
 }