tokentracker-cli 0.18.1 → 0.20.0

package/src/lib/rollout.js

@@ -3005,18 +3005,61 @@ async function parseKiroIncremental({ dbPath, jsonlPath, cursors, queuePath, onP
 // Hermes Agent — SQLite-based (sessions table in ~/.hermes/state.db)
 // ─────────────────────────────────────────────────────────────────────────────
 
-function resolveHermesDbPath() {
+function resolveHermesPath(env = process.env) {
+  const override = env.TOKENTRACKER_HERMES_HOME;
+  if (typeof override === "string" && override.trim().length > 0) {
+    return override.trim();
+  }
   const home = require("node:os").homedir();
-  return path.join(home, ".hermes", "state.db");
+  return path.join(home, ".hermes");
+}
+
+function resolveHermesDbPath(env = process.env) {
+  return path.join(resolveHermesPath(env), "state.db");
+}
+
+function resolveAllHermesDBPaths({ hermesPath, dbPath } = {}) {
+  const hermesDir = hermesPath ?? (dbPath ? path.dirname(dbPath) : resolveHermesPath());
+  const defaultDbPath = dbPath ?? path.join(hermesDir, "state.db");
+  const profilePaths = {};
+  try {
+    const profilesDir = path.join(hermesDir, "profiles");
+    const profiles = fssync.readdirSync(profilesDir, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .sort((a, b) => a.name.localeCompare(b.name));
+
+    for (const entry of profiles) {
+      const dbPath = path.join(profilesDir, entry.name, "state.db");
+      if (fssync.existsSync(dbPath)) {
+        profilePaths[entry.name] = dbPath;
+      }
+    }
+  } catch (_e) { }
+
+  return {
+    default: fssync.existsSync(defaultDbPath) ? defaultDbPath : null,
+    profiles: profilePaths,
+  }
+}
+
+function sqliteStringLiteral(value) {
+  return `'${String(value).replace(/'/g, "''")}'`;
 }
 
-function readHermesSessions(dbPath, lastCompletedEpoch) {
+function readHermesSessions(dbPath, lastCompletedEpoch, unfinishedSessionIds = []) {
   if (!dbPath || !fssync.existsSync(dbPath)) return [];
   const since = Number.isFinite(lastCompletedEpoch) && lastCompletedEpoch > 0 ? lastCompletedEpoch : 0;
-  // Fetch sessions that started after the cursor, OR sessions that are still
-  // in-progress (ended_at IS NULL).  Hermes updates token counts in real-time,
-  // so an active session keeps growing and must be re-read on every sync.
-  const sql = `SELECT id, model, started_at, ended_at, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, message_count FROM sessions WHERE (started_at > ${since} OR ended_at IS NULL) AND (input_tokens > 0 OR output_tokens > 0 OR cache_read_tokens > 0 OR reasoning_tokens > 0) ORDER BY started_at ASC`;
+  const forceIds = Array.isArray(unfinishedSessionIds)
+    ? [...new Set(unfinishedSessionIds.filter((id) => typeof id === "string" && id.length > 0))]
+    : [];
+  const forceIncludeSql = forceIds.length > 0
+    ? ` OR id IN (${forceIds.map(sqliteStringLiteral).join(",")})`
+    : "";
+  // Fetch sessions that started at/after the cursor, sessions that are still
+  // in-progress (ended_at IS NULL), OR sessions that were previously observed
+  // unfinished.  Hermes updates token counts in real-time, including a final
+  // delta when an active session later gets ended_at set.
+  const sql = `SELECT id, model, started_at, ended_at, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, message_count FROM sessions WHERE (started_at >= ${since} OR ended_at IS NULL${forceIncludeSql}) AND (input_tokens > 0 OR output_tokens > 0 OR cache_read_tokens > 0 OR reasoning_tokens > 0) ORDER BY started_at ASC`;
   let raw;
   try {
     raw = cp.execFileSync("sqlite3", ["-json", dbPath, sql], {
@@ -3037,122 +3080,173 @@ function readHermesSessions(dbPath, lastCompletedEpoch) {
   return Array.isArray(rows) ? rows : [];
 }
 
-async function parseHermesIncremental({ dbPath, cursors, queuePath, onProgress }) {
+function hasLegacyHermesDefaultState(hermesState) {
+  return (
+    typeof hermesState.lastStartedAt === "number" ||
+    typeof hermesState.lastCompletedStartedAt === "number" ||
+    (hermesState.snapshots && typeof hermesState.snapshots === "object")
+  );
+}
+
+async function parseHermesIncremental({ hermesPath, dbPath, cursors, queuePath, onProgress }) {
   await ensureDir(path.dirname(queuePath));
   const hermesState = cursors.hermes && typeof cursors.hermes === "object" ? cursors.hermes : {};
 
-  // Only advance past sessions that have fully ended.  Active sessions
-  // (ended_at IS NULL) must be re-read every sync because Hermes updates
-  // their token counts in real-time after each turn.
-  const lastCompletedStartedAt =
-    typeof hermesState.lastCompletedStartedAt === "number" ? hermesState.lastCompletedStartedAt : 0;
-
-  // Per-session snapshot from the previous sync: { [sessionId]: { in, out, cacheRead, cacheWrite, reasoning } }
-  const prevSnapshots = (hermesState.snapshots && typeof hermesState.snapshots === "object")
-    ? hermesState.snapshots : {};
-
-  const resolvedDbPath = dbPath || resolveHermesDbPath();
-  if (!fssync.existsSync(resolvedDbPath)) {
-    return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
-  }
-
-  const rows = readHermesSessions(resolvedDbPath, lastCompletedStartedAt);
-  if (rows.length === 0) {
-    cursors.hermes = { ...hermesState, lastCompletedStartedAt, updatedAt: new Date().toISOString() };
+  const dbPaths = resolveAllHermesDBPaths({ hermesPath, dbPath });
+  if (dbPaths.default === null && Object.keys(dbPaths.profiles).length === 0) {
+    // No state in any profile
     return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
   }
 
   const hourlyState = normalizeHourlyState(cursors?.hourly);
-  const touchedBuckets = new Set();
   const cb = typeof onProgress === "function" ? onProgress : null;
+  const updatedAt = new Date().toISOString();
+  let recordsProcessed = 0;
   let eventsAggregated = 0;
-  let maxCompletedStartedAt = lastCompletedStartedAt;
-  const nextSnapshots = {};
+  const touchedBuckets = new Set();
 
-  for (let i = 0; i < rows.length; i++) {
-    const row = rows[i];
-    const inputTokens = toNonNegativeInt(row.input_tokens);
-    const outputTokens = toNonNegativeInt(row.output_tokens);
-    const cacheRead = toNonNegativeInt(row.cache_read_tokens);
-    const cacheWrite = toNonNegativeInt(row.cache_write_tokens);
-    const reasoning = toNonNegativeInt(row.reasoning_tokens);
-    if (inputTokens === 0 && outputTokens === 0 && cacheRead === 0 && reasoning === 0) continue;
-
-    // Save current snapshot for next sync
-    nextSnapshots[row.id] = { in: inputTokens, out: outputTokens, cacheRead, cacheWrite, reasoning };
-
-    // Compute delta from previous snapshot (if any) so that we only count
-    // new tokens since the last sync.  First time we see a session the
-    // previous snapshot is absent, so the full amount is the delta.
-    const prev = prevSnapshots[row.id];
-    let dInput = inputTokens;
-    let dOutput = outputTokens;
-    let dCacheRead = cacheRead;
-    let dCacheWrite = cacheWrite;
-    let dReasoning = reasoning;
-    if (prev) {
-      dInput = Math.max(0, inputTokens - (prev.in || 0));
-      dOutput = Math.max(0, outputTokens - (prev.out || 0));
-      dCacheRead = Math.max(0, cacheRead - (prev.cacheRead || 0));
-      dCacheWrite = Math.max(0, cacheWrite - (prev.cacheWrite || 0));
-      dReasoning = Math.max(0, reasoning - (prev.reasoning || 0));
-    }
-    // Skip if delta is zero (session unchanged since last sync)
-    if (dInput === 0 && dOutput === 0 && dCacheRead === 0 && dCacheWrite === 0 && dReasoning === 0) continue;
-
-    // Prefer ended_at for bucket placement; fall back to started_at
-    const epochSec = row.ended_at || row.started_at;
-    if (!epochSec || !Number.isFinite(epochSec)) continue;
-    const tsIso = new Date(epochSec * 1000).toISOString();
-    const bucketStart = toUtcHalfHourStart(tsIso);
-    if (!bucketStart) continue;
+  function ingestProfile(dbPath, dbState) {
+    const trackedUnfinishedSessionIds = Array.isArray(dbState.unfinishedSessionIds)
+      ? dbState.unfinishedSessionIds
+      : [];
+    const rows = readHermesSessions(dbPath, dbState.lastCompletedStartedAt, trackedUnfinishedSessionIds);
+    recordsProcessed += rows.length;
+    if (rows.length === 0) {
+      dbState.updatedAt = updatedAt;
+      return;
+    }
+
+    // Per-session snapshot from the previous sync: { [sessionId]: { in, out, cacheRead, cacheWrite, reasoning } }
+    const prevSnapshots = (dbState.snapshots && typeof dbState.snapshots === "object")
+      ? dbState.snapshots : {};
+
+    // Only advance past sessions that have fully ended.  Active sessions
+    // (ended_at IS NULL) must be re-read every sync because Hermes updates
+    // their token counts in real-time after each turn.
+    const lastCompletedStartedAt =
+      typeof dbState.lastCompletedStartedAt === "number" ? dbState.lastCompletedStartedAt : 0;
+
+    let maxCompletedStartedAt = lastCompletedStartedAt;
+    let oldestUnfinishedStartedAt = Infinity;
+    const nextUnfinishedSessionIds = new Set();
+    const nextSnapshots = {};
+
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i];
+      const inputTokens = toNonNegativeInt(row.input_tokens);
+      const outputTokens = toNonNegativeInt(row.output_tokens);
+      const cacheRead = toNonNegativeInt(row.cache_read_tokens);
+      const cacheWrite = toNonNegativeInt(row.cache_write_tokens);
+      const reasoning = toNonNegativeInt(row.reasoning_tokens);
+      const messageCount = toNonNegativeInt(row.message_count);
+      if (inputTokens === 0 && outputTokens === 0 && cacheRead === 0 && reasoning === 0) continue;
+
+      // Save current snapshot for next sync
+      nextSnapshots[row.id] = { in: inputTokens, out: outputTokens, cacheRead, cacheWrite, reasoning, message_count: messageCount };
+
+      const startedAt = Number(row.started_at);
+      const endedAt = row.ended_at == null ? null : Number(row.ended_at);
+      if (endedAt == null) {
+        if (row.id && Number.isFinite(startedAt)) {
+          nextUnfinishedSessionIds.add(row.id);
+          oldestUnfinishedStartedAt = Math.min(oldestUnfinishedStartedAt, startedAt);
+        }
+      } else if (Number.isFinite(startedAt) && startedAt > maxCompletedStartedAt) {
+        maxCompletedStartedAt = startedAt;
+      }
 
-    const model = normalizeModelInput(row.model) || "hermes-agent";
+      // Compute delta from previous snapshot (if any) so that we only count
+      // new usage since the last sync.  First time we see a session the
+      // previous snapshot is absent, so the full amount is the delta.
+      const prev = prevSnapshots[row.id];
+      let dInput = inputTokens;
+      let dOutput = outputTokens;
+      let dCacheRead = cacheRead;
+      let dCacheWrite = cacheWrite;
+      let dReasoning = reasoning;
+      let dMessageCount = messageCount;
+      if (prev) {
+        dInput = Math.max(0, inputTokens - (prev.in || 0));
+        dOutput = Math.max(0, outputTokens - (prev.out || 0));
+        dCacheRead = Math.max(0, cacheRead - (prev.cacheRead || 0));
+        dCacheWrite = Math.max(0, cacheWrite - (prev.cacheWrite || 0));
+        dReasoning = Math.max(0, reasoning - (prev.reasoning || 0));
+        dMessageCount = Math.max(0, messageCount - (prev.message_count || 0));
+      }
+      // Skip if delta is zero (session unchanged since last sync)
+      if (dInput === 0 && dOutput === 0 && dCacheRead === 0 && dCacheWrite === 0 && dReasoning === 0) continue;
 
-    const delta = {
-      input_tokens: dInput,
-      cached_input_tokens: dCacheRead,
-      cache_creation_input_tokens: dCacheWrite,
-      output_tokens: dOutput,
-      reasoning_output_tokens: dReasoning,
-      total_tokens: dInput + dOutput + dCacheRead + dCacheWrite + dReasoning,
-      conversation_count: toNonNegativeInt(row.message_count) || 1,
-    };
+      // Prefer ended_at for bucket placement; fall back to started_at
+      const epochSec = endedAt ?? startedAt;
+      if (!epochSec || !Number.isFinite(epochSec)) continue;
+      const tsIso = new Date(epochSec * 1000).toISOString();
+      const bucketStart = toUtcHalfHourStart(tsIso);
+      if (!bucketStart) continue;
 
-    const bucket = getHourlyBucket(hourlyState, "hermes", model, bucketStart);
-    addTotals(bucket.totals, delta);
-    touchedBuckets.add(bucketKey("hermes", model, bucketStart));
-    eventsAggregated++;
+      const model = normalizeModelInput(row.model) || "hermes-agent";
 
-    // Only advance cursor past sessions that have ended
-    if (row.ended_at && row.started_at > maxCompletedStartedAt) {
-      maxCompletedStartedAt = row.started_at;
-    }
+      const delta = {
+        input_tokens: dInput,
+        cached_input_tokens: dCacheRead,
+        cache_creation_input_tokens: dCacheWrite,
+        output_tokens: dOutput,
+        reasoning_output_tokens: dReasoning,
+        total_tokens: dInput + dOutput + dCacheRead + dCacheWrite + dReasoning,
+        conversation_count: dMessageCount,
+      };
 
-    if (cb) {
-      cb({
-        index: i + 1,
-        total: rows.length,
-        recordsProcessed: i + 1,
-        eventsAggregated,
-        bucketsQueued: touchedBuckets.size,
-      });
+      const bucket = getHourlyBucket(hourlyState, "hermes", model, bucketStart);
+      addTotals(bucket.totals, delta);
+      touchedBuckets.add(bucketKey("hermes", model, bucketStart));
+      eventsAggregated++;
+
+      if (cb) {
+        cb({
+          index: i + 1,
+          total: rows.length,
+          recordsProcessed: i + 1,
+          eventsAggregated,
+          bucketsQueued: touchedBuckets.size,
+        });
+      }
     }
+
+    const nextLastCompletedStartedAt = Number.isFinite(oldestUnfinishedStartedAt)
+      ? Math.min(maxCompletedStartedAt, oldestUnfinishedStartedAt)
+      : maxCompletedStartedAt;
+
+    Object.assign(dbState, {
+      lastStartedAt: nextLastCompletedStartedAt,
+      lastCompletedStartedAt: nextLastCompletedStartedAt,
+      unfinishedSessionIds: Array.from(nextUnfinishedSessionIds),
+      snapshots: nextSnapshots,
+      updatedAt,
+    });
+  }
+
+  if (dbPaths.default) {
+    ingestProfile(dbPaths.default, hermesState);
+  }
+
+  hermesState.profiles = hermesState.profiles && typeof hermesState.profiles === "object" ? hermesState.profiles : {};
+
+  for (const [profileName, dbPath] of Object.entries(dbPaths.profiles)) {
+    const profileState = hermesState.profiles[profileName] && typeof hermesState.profiles[profileName] === "object"
+      ? hermesState.profiles[profileName]
+      : {};
+    hermesState.profiles[profileName] = profileState;
+    ingestProfile(dbPath, profileState);
   }
 
   const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
-  const updatedAt = new Date().toISOString();
   hourlyState.updatedAt = updatedAt;
   cursors.hourly = hourlyState;
   cursors.hermes = {
     ...hermesState,
-    lastStartedAt: maxCompletedStartedAt, // keep for backward compat
-    lastCompletedStartedAt: maxCompletedStartedAt,
-    snapshots: nextSnapshots,
-    updatedAt,
+    updatedAt, // Update the overall profile state timestamp even if the DB doesn't exist for the fast-path check
   };
 
-  return { recordsProcessed: rows.length, eventsAggregated, bucketsQueued };
+  return { recordsProcessed, eventsAggregated, bucketsQueued };
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -5893,15 +5987,15 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
-// Grok Build (xAI) — passive reader for ~/.grok/sessions/**/signals.json + summary.json
+// Grok Build (xAI) — passive reader for ~/.grok/sessions/**/updates.jsonl + signals.json
 // Triggered either by full scan in sync or by the SessionEnd hook writing a signal.
-// Grok exposes contextTokensUsed, which appears to be a snapshot rather than
-// per-call telemetry, so these rows are estimates and only enqueue observed
-// increases for a session.
+// updates.jsonl exposes cumulative totalTokens metadata. Grok still does not
+// expose a stable prompt/output/cache split locally, so these rows keep the
+// estimated input/output split while using better local telemetry for totals.
 // ─────────────────────────────────────────────────────────────────────────────
 
 const GROK_ESTIMATED_INPUT_RATIO = 0.8;
-const GROK_CURSOR_VERSION = 2;
+const GROK_CURSOR_VERSION = 3;
 
 function resolveGrokBuildHome(env = process.env) {
   return (
@@ -5936,9 +6030,11 @@ function resolveGrokBuildSessions(env = process.env) {
     for (const sid of sessionIds) {
       const sessionDir = path.join(cwdPath, sid);
       const signalsPath = path.join(sessionDir, "signals.json");
-      if (fssync.existsSync(signalsPath)) {
+      const updatesPath = path.join(sessionDir, "updates.jsonl");
+      if (fssync.existsSync(signalsPath) || fssync.existsSync(updatesPath)) {
         results.push({
           sessionDir,
+          updatesPath,
           signalsPath,
           summaryPath: path.join(sessionDir, "summary.json"),
           sessionId: sid,
@@ -5961,7 +6057,11 @@ function normalizeGrokSessionSnapshots(grokState) {
         totalTokens,
         messageCount: normalizeNonNegativeNumber(snapshot.messageCount),
         model: normalizeModelInput(snapshot.model) || null,
+        source: normalizeModelInput(snapshot.source) || null,
+        lastEventId: normalizeModelInput(snapshot.lastEventId) || null,
+        lastEventTimestamp: normalizeModelInput(snapshot.lastEventTimestamp) || null,
         updatedAt: normalizeModelInput(snapshot.updatedAt) || null,
+        legacySeen: snapshot.legacySeen === true,
       };
     }
   }
@@ -5971,7 +6071,7 @@ function normalizeGrokSessionSnapshots(grokState) {
       const safeSessionId = normalizeModelInput(sessionId);
       if (!safeSessionId || snapshots[safeSessionId]) continue;
       snapshots[safeSessionId] = {
-        totalTokens: Number.MAX_SAFE_INTEGER,
+        totalTokens: 0,
         messageCount: 0,
         model: null,
         updatedAt: normalizeModelInput(grokState.updatedAt) || null,
@@ -5998,6 +6098,14 @@ function readGrokJsonFile(filePath) {
   }
 }
 
+function grokUpdatesPathForSession(sess) {
+  if (typeof sess?.updatesPath === "string" && sess.updatesPath.trim()) return sess.updatesPath;
+  if (typeof sess?.sessionDir === "string" && sess.sessionDir.trim()) {
+    return path.join(sess.sessionDir, "updates.jsonl");
+  }
+  return null;
+}
+
 function grokSessionIdFor(sess) {
   return (
     normalizeModelInput(sess?.sessionId) ||
@@ -6025,11 +6133,113 @@ function grokLastActiveFromSignals(signals, summary) {
   );
 }
 
-function estimateGrokTokenDelta(totalTokens, conversationCount) {
+function grokMessageCountFromSignals(signals) {
+  return normalizeNonNegativeNumber(
+    signals?.assistantMessageCount ??
+      signals?.turnCount ??
+      signals?.num_chat_messages ??
+      signals?.messageCount,
+  );
+}
+
+function grokEffectiveTotalFromSignals(signals) {
+  if (!signals || typeof signals !== "object") return 0;
+  const beforeCompaction = normalizeNonNegativeNumber(signals.totalTokensBeforeCompaction);
+  const totalTokens = normalizeNonNegativeNumber(signals.totalTokens);
+  if (signals.contextTokensUsed == null) {
+    return beforeCompaction + totalTokens;
+  }
+  return Math.max(
+    totalTokens,
+    beforeCompaction + normalizeNonNegativeNumber(signals.contextTokensUsed),
+  );
+}
+
+function grokTimestampToIso(value) {
+  if (value == null) return null;
+  if (typeof value === "number") {
+    if (!Number.isFinite(value) || value <= 0) return null;
+    const millis = value < 10_000_000_000 ? value * 1000 : value;
+    const dt = new Date(millis);
+    return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
+  }
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (!trimmed) return null;
+    if (/^[0-9]+(?:\.[0-9]+)?$/.test(trimmed)) {
+      return grokTimestampToIso(Number(trimmed));
+    }
+    const dt = new Date(trimmed);
+    return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
+  }
+  return null;
+}
+
+function grokTimestampFromUpdate(meta, record, fallback) {
+  return (
+    grokTimestampToIso(meta?.agentTimestampMs) ||
+    grokTimestampToIso(meta?.timestampMs) ||
+    grokTimestampToIso(record?.timestamp_ms) ||
+    grokTimestampToIso(record?.timestamp) ||
+    grokTimestampToIso(record?.time) ||
+    fallback ||
+    null
+  );
+}
+
+function grokEventId(value, fallback) {
+  if (typeof value === "string" && value.trim()) return value.trim();
+  if (typeof value === "number" && Number.isFinite(value)) return String(value);
+  return fallback;
+}
+
+async function readGrokUpdateTokenEvents(updatesPath, fallbackTimestamp) {
+  if (!updatesPath) return { events: [], recordsProcessed: 0 };
+  try {
+    const stat = fssync.statSync(updatesPath);
+    if (!stat.isFile()) return { events: [], recordsProcessed: 0 };
+  } catch {
+    return { events: [], recordsProcessed: 0 };
+  }
+
+  const events = [];
+  let lineIndex = 0;
+  const input = fssync.createReadStream(updatesPath, { encoding: "utf8" });
+  const rl = readline.createInterface({ input, crlfDelay: Infinity });
+  try {
+    for await (const line of rl) {
+      lineIndex++;
+      if (!line || !line.trim()) continue;
+      let record;
+      try {
+        record = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      const meta = record?.params?._meta || record?._meta;
+      if (!meta || typeof meta !== "object") continue;
+      const totalTokens = normalizeNonNegativeNumber(meta.totalTokens);
+      if (totalTokens <= 0) continue;
+      const timestamp = grokTimestampFromUpdate(meta, record, fallbackTimestamp);
+      events.push({
+        totalTokens,
+        timestamp,
+        eventId: grokEventId(meta.eventId ?? record?.eventId ?? record?.id, String(lineIndex)),
+      });
+    }
+  } catch {
+    return { events, recordsProcessed: events.length };
+  }
+
+  return { events, recordsProcessed: events.length };
+}
+
+function estimateGrokTokenDelta(totalTokens, conversationCount, options = {}) {
   const total = Math.trunc(normalizeNonNegativeNumber(totalTokens));
   const inputTokens = Math.round(total * GROK_ESTIMATED_INPUT_RATIO);
   const outputTokens = Math.max(0, total - inputTokens);
-  const conversations = Math.max(1, Math.trunc(normalizeNonNegativeNumber(conversationCount)));
+  const rawConversations = Math.trunc(normalizeNonNegativeNumber(conversationCount));
+  const conversations = options.allowZeroConversationCount ? rawConversations : Math.max(1, rawConversations);
 
   return {
     input_tokens: inputTokens,
@@ -6062,51 +6272,103 @@ async function parseGrokBuildIncremental({
 
   let eventsAggregated = 0;
 
-  for (const sess of sessionList) {
+  for (let index = 0; index < sessionList.length; index++) {
+    const sess = sessionList[index];
     const sessionId = grokSessionIdFor(sess);
-    if (!sessionId) continue;
+    if (!sessionId) {
+      if (onProgress) onProgress({ index: index + 1, total: sessionList.length, bucketsQueued: touchedBuckets.size });
+      continue;
+    }
 
     const signals = sess?.signals && typeof sess.signals === "object"
       ? sess.signals
       : readGrokJsonFile(sess?.signalsPath);
-    if (!signals || typeof signals !== "object") continue;
+    const safeSignals = signals && typeof signals === "object" ? signals : {};
 
     const summary = sess?.summary && typeof sess.summary === "object"
       ? sess.summary
       : readGrokJsonFile(sess?.summaryPath) || {};
-    const totalTokens = normalizeNonNegativeNumber(signals.contextTokensUsed ?? signals.totalTokens);
-    if (totalTokens <= 0) {
-      continue;
-    }
-
     const previous = sessionSnapshots[sessionId] || {};
     const previousTotal = normalizeNonNegativeNumber(previous.totalTokens);
-    const deltaTokens = totalTokens > previousTotal ? totalTokens - previousTotal : 0;
-    if (deltaTokens <= 0) continue;
-
-    const messageCount = normalizeNonNegativeNumber(
-      signals.assistantMessageCount ?? signals.num_chat_messages ?? signals.messageCount,
-    );
     const previousMessageCount = normalizeNonNegativeNumber(previous.messageCount);
-    const deltaMessageCount =
-      messageCount > previousMessageCount ? messageCount - previousMessageCount : 1;
-    const model = grokModelFromSignals(signals);
-    const lastActive = grokLastActiveFromSignals(signals, summary);
-    const hourStartStr = toUtcHalfHourStart(lastActive) || toUtcHalfHourStart(Date.now());
-    if (!hourStartStr) continue;
-
-    const delta = estimateGrokTokenDelta(deltaTokens, deltaMessageCount);
-    const bucket = getHourlyBucket(hourlyState, "grok", model, hourStartStr);
-    addTotals(bucket.totals, delta);
-    touchedBuckets.add(bucketKey("grok", model, hourStartStr));
-
-    eventsAggregated++;
-    sessionSnapshots[sessionId] = {
-      totalTokens: Math.max(previousTotal, totalTokens),
-      messageCount: Math.max(previousMessageCount, messageCount),
-      model,
-      updatedAt: new Date().toISOString(),
+    const messageCount = grokMessageCountFromSignals(safeSignals);
+    const model = grokModelFromSignals(safeSignals);
+    const lastActive = grokLastActiveFromSignals(safeSignals, summary);
+
+    let highWatermark = previousTotal;
+    let observedTotal = previousTotal;
+    let tokenDeltaForSession = 0;
+    let finalTouchedHourStart = null;
+    let source = previous.source || null;
+    let lastEventId = previous.lastEventId || null;
+    let lastEventTimestamp = previous.lastEventTimestamp || null;
+    const pendingTokenDeltas = [];
+
+    const recordTokenDelta = (deltaTokens, timestamp, deltaSource) => {
+      const hourStartStr = toUtcHalfHourStart(timestamp) || toUtcHalfHourStart(Date.now());
+      if (!hourStartStr) return false;
+      pendingTokenDeltas.push({ deltaTokens, hourStartStr });
+      tokenDeltaForSession += deltaTokens;
+      finalTouchedHourStart = hourStartStr;
+      source = deltaSource;
+      lastEventTimestamp = timestamp || lastEventTimestamp;
+      return true;
     };
+
+    const updates = await readGrokUpdateTokenEvents(grokUpdatesPathForSession(sess), lastActive);
+    for (const event of updates.events) {
+      observedTotal = Math.max(observedTotal, event.totalTokens);
+      lastEventId = event.eventId || lastEventId;
+      lastEventTimestamp = event.timestamp || lastEventTimestamp;
+      if (event.totalTokens <= highWatermark) continue;
+      const deltaTokens = event.totalTokens - highWatermark;
+      highWatermark = event.totalTokens;
+      recordTokenDelta(deltaTokens, event.timestamp || lastActive, "updates");
+    }
+
+    const effectiveSignalTotal = grokEffectiveTotalFromSignals(safeSignals);
+    observedTotal = Math.max(observedTotal, effectiveSignalTotal);
+    if (effectiveSignalTotal > highWatermark) {
+      const deltaTokens = effectiveSignalTotal - highWatermark;
+      highWatermark = effectiveSignalTotal;
+      recordTokenDelta(deltaTokens, lastActive, "signals");
+    }
+
+    const finalTotal = Math.max(previousTotal, highWatermark, observedTotal);
+    const legacyBaselineOnly = previous.legacySeen && previousTotal === 0 && finalTotal > 0;
+    if (!legacyBaselineOnly) {
+      for (const pending of pendingTokenDeltas) {
+        const delta = estimateGrokTokenDelta(pending.deltaTokens, 0, { allowZeroConversationCount: true });
+        const bucket = getHourlyBucket(hourlyState, "grok", model, pending.hourStartStr);
+        addTotals(bucket.totals, delta);
+        touchedBuckets.add(bucketKey("grok", model, pending.hourStartStr));
+        eventsAggregated++;
+      }
+    }
+
+    if (!legacyBaselineOnly && tokenDeltaForSession > 0 && finalTouchedHourStart) {
+      const deltaMessageCount =
+        messageCount > previousMessageCount ? messageCount - previousMessageCount : 1;
+      const bucket = getHourlyBucket(hourlyState, "grok", model, finalTouchedHourStart);
+      addTotals(bucket.totals, { conversation_count: deltaMessageCount });
+      touchedBuckets.add(bucketKey("grok", model, finalTouchedHourStart));
+    }
+
+    if (finalTotal > 0 && (tokenDeltaForSession > 0 || previousTotal > 0 || legacyBaselineOnly)) {
+      sessionSnapshots[sessionId] = {
+        totalTokens: finalTotal,
+        messageCount: Math.max(previousMessageCount, messageCount),
+        model,
+        source: source || previous.source || null,
+        lastEventId,
+        lastEventTimestamp,
+        updatedAt: new Date().toISOString(),
+      };
+    }
+
+    if (onProgress) {
+      onProgress({ index: index + 1, total: sessionList.length, bucketsQueued: touchedBuckets.size });
+    }
   }
 
   const bucketsQueued = queuePath
@@ -6229,6 +6491,7 @@ async function parseAntigravityIncremental({
     const sameFile = prev && prev.inode === inode;
     const lastLine = sameFile ? Number(prev.lastLine || 0) : 0;
     const initialContextTokens = sameFile ? Number(prev.contextTokens || 0) : 0;
+    const initialPrevContext = sameFile ? Number(prev.previousContextTokens || 0) : 0;
     const initialModel = sameFile && typeof prev.currentModel === "string" ? prev.currentModel : null;
 
     const projectContext = projectEnabled
@@ -6247,6 +6510,7 @@ async function parseAntigravityIncremental({
       filePath,
       lastLine,
       initialContextTokens,
+      initialPrevContext,
       initialModel,
       hourlyState,
       touchedBuckets,
@@ -6263,6 +6527,7 @@ async function parseAntigravityIncremental({
       mtimeMs,
       lastLine: result.lastLine,
       contextTokens: result.contextTokens,
+      previousContextTokens: result.previousContextTokens,
       currentModel: result.currentModel,
       updatedAt: new Date().toISOString(),
     };
@@ -6300,6 +6565,7 @@ async function parseAntigravityFile({
   filePath,
   lastLine,
   initialContextTokens,
+  initialPrevContext,
   initialModel,
   hourlyState,
   touchedBuckets,
@@ -6311,7 +6577,13 @@ async function parseAntigravityFile({
 }) {
   const raw = await fs.readFile(filePath, "utf8").catch(() => "");
   if (!raw.trim()) {
-    return { lastLine: 0, eventsAggregated: 0, contextTokens: 0, currentModel: null };
+    return {
+      lastLine: 0,
+      eventsAggregated: 0,
+      contextTokens: 0,
+      previousContextTokens: 0,
+      currentModel: null,
+    };
   }
 
   const lines = raw
@@ -6325,11 +6597,16 @@ async function parseAntigravityFile({
   const canResume =
     Number.isFinite(lastLine) && lastLine > 0 && lastLine <= lines.length;
   const cachedTokens = Number.isFinite(initialContextTokens) ? initialContextTokens : 0;
+  const cachedPrev = Number.isFinite(initialPrevContext) ? initialPrevContext : 0;
   const cachedModel = typeof initialModel === "string" ? initialModel : null;
   const resumed = canResume && (cachedTokens > 0 || cachedModel !== null);
   const scanStart = resumed ? lastLine : 0;
   let currentModel = resumed ? cachedModel : null;
   let contextTokens = resumed ? cachedTokens : 0;
+  // Snapshot of contextTokens at the last PLANNER_RESPONSE we billed for. Only
+  // tokens accumulated AFTER that point count as new input on the next planner
+  // call — prevents O(N²) double-counting of the full history every turn.
+  let previousContextTokens = resumed ? cachedPrev : 0;
   let lastCompletedLine = Math.min(Number.isFinite(lastLine) ? lastLine : 0, lines.length);
 
   for (let i = scanStart; i < lines.length; i++) {
@@ -6375,22 +6652,28 @@ async function parseAntigravityFile({
 
     let model = currentModel || "antigravity-unknown";
     let delta = initTotals();
+    let billedPlanner = false;
 
     if (parsed.type === "PLANNER_RESPONSE") {
       const content = typeof parsed.content === "string" ? parsed.content : "";
       const thinking = typeof parsed.thinking === "string" ? parsed.thinking : "";
 
-      delta.input_tokens = contextTokens;
-      delta.output_tokens =
+      const inputDelta = Math.max(0, contextTokens - previousContextTokens);
+      const outputTokens =
         antigravityValueTokens(content) + antigravityValueTokens(parsed.tool_calls);
-      delta.reasoning_output_tokens = antigravityValueTokens(thinking);
-      delta.total_tokens =
-        delta.input_tokens + delta.output_tokens + delta.reasoning_output_tokens;
-      delta.billable_total_tokens = delta.total_tokens;
+      const reasoningTokens = antigravityValueTokens(thinking);
+
+      delta.input_tokens = inputDelta;
+      delta.output_tokens = outputTokens;
+      delta.reasoning_output_tokens = reasoningTokens;
+      // Match the mainstream convention (Codebuddy / Kilocode / OMP / Hermes):
+      // total_tokens = sum of every token column. No cache columns here.
+      delta.total_tokens = inputDelta + outputTokens + reasoningTokens;
       delta.conversation_count = 1;
+      billedPlanner = delta.total_tokens > 0;
     }
 
-    if (delta.total_tokens === 0) {
+    if (!billedPlanner) {
       contextTokens += eventContextTokens;
       lastCompletedLine = i + 1;
       continue;
@@ -6412,6 +6695,11 @@ async function parseAntigravityFile({
       projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
     }
     eventsAggregated += 1;
+    // Snapshot the pre-planner context first. The planner's own content+tool_calls
+    // (eventContextTokens, added below) become part of the next turn's history,
+    // so they MUST be billed as input on the next planner — don't fold them into
+    // previousContextTokens or that history vanishes from the totals.
+    previousContextTokens = contextTokens;
     contextTokens += eventContextTokens;
     lastCompletedLine = i + 1;
   }
@@ -6420,6 +6708,7 @@ async function parseAntigravityFile({
     lastLine: lastCompletedLine,
     eventsAggregated,
     contextTokens,
+    previousContextTokens,
     currentModel,
   };
 }
@@ -6505,6 +6794,7 @@ module.exports = {
   readOpencodeDbMessages,
   resolveKiroDbPath,
   resolveKiroJsonlPath,
+  resolveHermesPath,
   resolveHermesDbPath,
   resolveCopilotOtelPaths,
   parseRolloutIncremental,
@@ -6562,7 +6852,7 @@ module.exports = {
   canonicalizeProjectRef,
   deriveProjectKeyFromRef,
 
-  // Grok Build (xAI) — SessionEnd hook + passive signals.json reader
+  // Grok Build (xAI) — SessionEnd hook + passive updates.jsonl/signals.json reader
   resolveGrokBuildHome,
   resolveGrokBuildSessions,
   parseGrokBuildIncremental,