tokentracker-cli 1.4.3 → 1.4.6

package/src/lib/async-queue-reader.js

@@ -1,9 +1,9 @@
 const fs = require("node:fs/promises");
+const { createHash } = require("node:crypto");
 const { StringDecoder } = require("node:string_decoder");
 const { setImmediate: yieldToRequests } = require("node:timers/promises");
 
 const CHUNK_BYTES = 64 * 1024;
-const PROBE_BYTES = 256;
 
 function signature(stat) {
   return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
@@ -17,10 +17,18 @@ function createAsyncQueueReader({ parseRow, label, maxEntries = 4 }) {
   const cache = new Map();
   const pending = new Map();
 
-  async function probe(file, position, length) {
-    const buffer = Buffer.alloc(length);
-    const { bytesRead } = await file.read(buffer, 0, length, position);
-    return buffer.subarray(0, bytesRead);
+  async function hashPrefix(file, length) {
+    const hash = createHash("sha256");
+    const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
+    let position = 0;
+    while (position < length) {
+      const { bytesRead } = await file.read(buffer, 0, Math.min(CHUNK_BYTES, length - position), position);
+      if (!bytesRead) throw new Error("Queue truncated during validation");
+      hash.update(buffer.subarray(0, bytesRead));
+      position += bytesRead;
+      await yieldToRequests();
+    }
+    return hash;
   }
 
   async function load(queuePath) {
@@ -32,19 +40,28 @@ function createAsyncQueueReader({ parseRow, label, maxEntries = 4 }) {
       // between stat and open. Its signature always describes the bytes we read.
       const stat = await file.stat({ bigint: true });
       const nextSignature = signature(stat);
-      if (cached?.signature === nextSignature) return cached.rows;
       const identity = `${stat.dev}:${stat.ino}`;
       const size = Number(stat.size);
+      if (cached?.signature === nextSignature) {
+        // Windows can reuse mtime/ctime for same-size writes anywhere in the
+        // file. Validate every byte before reusing the parsed snapshot.
+        const hash = await hashPrefix(file, size);
+        if (hash.digest("hex") === cached.digest) return cached.rows;
+      }
       let offset = 0;
       let seen = new Map();
+      let hash = createHash("sha256");
       if (cached && cached.identity === identity && size > cached.size && cached.consumedBytes > 0) {
-        const head = await probe(file, 0, cached.head.length);
-        const anchor = await probe(file, cached.consumedBytes - cached.anchor.length, cached.anchor.length);
-        if (head.equals(cached.head) && anchor.equals(cached.anchor)) {
+        // Growth can include an interior rewrite, so verify the entire old
+        // committed prefix before parsing only the new tail.
+        const prefixHash = await hashPrefix(file, cached.consumedBytes);
+        if (prefixHash.copy().digest("hex") === cached.committedDigest) {
           offset = cached.consumedBytes;
           seen = cached.seen;
+          hash = prefixHash;
         }
       }
+      let committedHash = hash.copy();
 
       const changes = new Map();
       let malformed = 0;
@@ -66,9 +83,14 @@ function createAsyncQueueReader({ parseRow, label, maxEntries = 4 }) {
         const { bytesRead } = await file.read(buffer, 0, length, position);
         if (!bytesRead) throw new Error("Queue truncated during read");
         const newline = buffer.lastIndexOf(0x0a, bytesRead - 1);
-        if (newline >= 0) consumedBytes = position + newline + 1;
+        const chunk = buffer.subarray(0, bytesRead);
+        if (newline >= 0) {
+          consumedBytes = position + newline + 1;
+          committedHash = hash.copy().update(chunk.subarray(0, newline + 1));
+        }
+        hash.update(chunk);
         position += bytesRead;
-        remainder += decoder.write(buffer.subarray(0, bytesRead));
+        remainder += decoder.write(chunk);
         let start = 0;
         let end;
         while ((end = remainder.indexOf("\n", start)) !== -1) {
@@ -79,10 +101,6 @@ function createAsyncQueueReader({ parseRow, label, maxEntries = 4 }) {
         await yieldToRequests();
       }
       remainder += decoder.end();
-      // Capture probes before committing any state: a failed read must not mutate
-      // a snapshot still being used by another endpoint.
-      const head = await probe(file, 0, Math.min(PROBE_BYTES, consumedBytes));
-      const anchor = await probe(file, Math.max(0, consumedBytes - PROBE_BYTES), Math.min(PROBE_BYTES, consumedBytes));
       for (const [key, row] of changes) seen.set(key, row);
       const tail = new Map();
       parse(remainder, tail);
@@ -90,7 +108,10 @@ function createAsyncQueueReader({ parseRow, label, maxEntries = 4 }) {
       // key never enters the committed map (completion may change that key).
       const rows = tail.size ? Array.from(new Map([...seen, ...tail]).values()) : Array.from(seen.values());
       cache.delete(queuePath);
-      cache.set(queuePath, { signature: nextSignature, identity, size, consumedBytes, head, anchor, seen, rows });
+      cache.set(queuePath, {
+        signature: nextSignature, identity, size, consumedBytes,
+        digest: hash.digest("hex"), committedDigest: committedHash.digest("hex"), seen, rows,
+      });
       while (cache.size > maxEntries) cache.delete(cache.keys().next().value);
       if (malformed) console.error(`[LocalAPI] ${label}: skipped ${malformed} malformed line(s)`);
       return rows;

package/src/lib/codex-context-breakdown.js

@@ -35,6 +35,7 @@ const {
   finalizeExecRows,
   buildSkillStatsEntry,
   isCodexResumeInvalidated,
+  isRecoverableGroupedReadError,
 } = require("./codex-rollout-parser");
 
 // ---------------------------------------------------------------------------
@@ -473,6 +474,7 @@ async function computeCodexContextBreakdown({
       PARSED_GROUP_CACHE.delete(parsedCacheKey);
       PARSED_GROUP_CACHE.set(parsedCacheKey, cachedGroup);
     } else {
+      try {
       let captured = null;
       const singleEntry = entries.length === 1 ? entries[0] : null;
       const appendHashEligible = Boolean(
@@ -538,7 +540,7 @@ async function computeCodexContextBreakdown({
             diagnostics.incremental_parse_fallbacks += 1;
           }
         } catch (error) {
-          if (!isCodexResumeInvalidated(error)) throw error;
+          if (!isCodexResumeInvalidated(error) && !isRecoverableGroupedReadError(error)) throw error;
           diagnostics.incremental_parse_fallbacks += 1;
         } finally {
           await resumeHandle.close();
@@ -587,6 +589,11 @@ async function computeCodexContextBreakdown({
           stat: singleEntry?.stat || null,
         });
       }
+      } catch (error) {
+        // One unreadable archive/session must not fail the whole breakdown.
+        if (!isRecoverableGroupedReadError(error)) throw error;
+        continue;
+      }
     }
     if (!parsed || !parsed.totals || !parsed.totals.total_tokens) continue;
     sessions.push(parsed);

package/src/lib/codex-rollout-parser.js

@@ -42,6 +42,28 @@ const {
 } = require("./codex-usage-record");
 
 const DISCOVERY_FULL_AUDIT_INTERVAL_MS = 60 * 1000;
+const CODEX_ROLLOUT_TEXT_MAX_BYTES = 256 * 1024 * 1024;
+
+async function readCodexRolloutText(filePath, { maxOutputBytes = CODEX_ROLLOUT_TEXT_MAX_BYTES } = {}) {
+  if (typeof filePath !== "string" || !/\.jsonl\.zstd?$/i.test(filePath)) {
+    return fs.readFile(filePath, "utf8");
+  }
+  // The snapshot decoder lives in rollout.js (reused by the sync parser);
+  // require lazily so this module stays loadable without pulling the whole
+  // rollout graph at import time.
+  const rollout = require("./rollout");
+  const snapshot = await rollout.readCodexRolloutSnapshot(filePath, { maxOutputBytes });
+  if (!snapshot) {
+    // Treat undecodable / vanished archives as recoverable I/O so grouped
+    // session reads (live + archive) can still use a readable sibling copy.
+    const error = new Error(`unreadable Codex archive ${filePath}`);
+    error.code = "EIO";
+    throw error;
+  }
+  // Snapshot lines keep their "\n" terminators; the tail element may be an
+  // unterminated final line — join preserves exactly that.
+  return snapshot.lines.join("\n");
+}
 const MAX_DISCOVERY_INVENTORIES = 32;
 const DISCOVERY_INVENTORIES = new Map();
 const ZONED_DAY_FORMATTERS = new Map();
@@ -143,6 +165,11 @@ function safeJsonParse(str, diagnostics = null) {
   }
 }
 
+function isRolloutCandidateFileName(name) {
+  if (typeof name !== "string") return false;
+  return name.endsWith(".jsonl") || /^rollout-.*\.jsonl\.zstd?$/.test(name);
+}
+
 function listJsonlFiles(rootDir) {
   const out = [];
   const stack = [rootDir];
@@ -160,7 +187,7 @@ function listJsonlFiles(rootDir) {
       const filePath = path.join(dir, entry.name);
       if (entry.isDirectory()) {
         stack.push(filePath);
-      } else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
+      } else if (entry.isFile() && isRolloutCandidateFileName(entry.name)) {
         out.push(filePath);
       }
     }
@@ -331,7 +358,7 @@ function discoverCodexSessionFiles(rootDirs, options = {}) {
         stack.push(filePath);
         continue;
       }
-      if (!child.isFile() || !child.name.endsWith(".jsonl") || visited.has(filePath)) continue;
+      if (!child.isFile() || !isRolloutCandidateFileName(child.name) || visited.has(filePath)) continue;
       visited.add(filePath);
       if (diagnostics) diagnostics.discovered_files += 1;
       const cached = inventory.files.get(filePath) || null;
@@ -1175,7 +1202,13 @@ async function parseCodexRolloutFile(filePath, {
       compactionResponseIds: Array.from(compactionResponseIds),
     };
     result.endOffset = readProgress.endOffset;
-    result.appendable = Boolean(contentHasher) && (
+    // Compressed snapshots are not append-only logs: endOffset stays at the
+    // pre-decode cursor (often 0) and must never be marked appendable, or the
+    // next growth would "resume" by parsing compressed bytes as JSONL.
+    const hasCompressedArchive = filePaths.some(
+      (candidate) => typeof candidate === "string" && /\.jsonl\.zstd?$/i.test(candidate),
+    );
+    result.appendable = Boolean(contentHasher) && !hasCompressedArchive && (
       readProgress.endOffset === 0 || readProgress.lastByte === 0x0a
     );
     result.contentHashState = contentHasher;
@@ -1309,6 +1342,27 @@ async function* readCodexObjectsFull(filePath, diagnostics, fileIndex, {
   readProgress = null,
   contentHasher = null,
 } = {}) {
+  // Compressed archives are immutable snapshots, not append-only logs: byte
+  // offsets are meaningless inside them, so decode the whole text and split
+  // into lines. `readProgress` stays null so callers never persist a fake
+  // byte cursor for them.
+  if (typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath)) {
+    const text = await readCodexRolloutText(filePath);
+    let lineIndex = 0;
+    const lines = text.split("\n");
+    for (let index = 0; index < lines.length; index += 1) {
+      const isLast = index + 1 >= lines.length;
+      if (isLast && lines[index] === "") continue;
+      const lineBuffer = Buffer.from(lines[index], "utf8");
+      const obj = parseCodexLine(lineBuffer, diagnostics);
+      if (obj) {
+        yield { obj, fileIndex, lineIndex };
+        lineIndex += 1;
+      }
+    }
+    if (diagnostics) diagnostics.parsed_files += 1;
+    return;
+  }
   yield* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
     startOffset,
     endOffset,
@@ -1331,6 +1385,7 @@ function isRecoverableGroupedReadError(error) {
 
 async function* iterateCodexObjects(filePaths, diagnostics, options = {}) {
   if (filePaths.length <= 1) {
+    if (!filePaths[0]) return;
     for await (const record of readCodexObjects(filePaths[0], diagnostics, 0, options)) {
       yield record.obj;
     }
@@ -1355,7 +1410,11 @@ async function* iterateCodexObjects(filePaths, diagnostics, options = {}) {
       if (!isRecoverableGroupedReadError(error)) throw error;
     }
   }
-  if (!scannedFiles) throw new Error("all grouped Codex rollout files failed during read");
+  if (!scannedFiles) {
+    const error = new Error("all grouped Codex rollout files failed during read");
+    error.code = "EIO";
+    throw error;
+  }
   records.sort((a, b) => {
     const aTimestamp = typeof a.obj?.timestamp === "string" ? a.obj.timestamp : "";
     const bTimestamp = typeof b.obj?.timestamp === "string" ? b.obj.timestamp : "";
@@ -1373,7 +1432,7 @@ async function* iterateCodexObjects(filePaths, diagnostics, options = {}) {
 
 function rolloutSessionIdFromPath(filePath) {
   const match = path.basename(String(filePath || "")).match(
-    /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i,
+    /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl(?:\.zstd?)?$/i,
   );
   return match ? match[1] : null;
 }
@@ -1404,6 +1463,7 @@ module.exports = {
   isTimestampInRequestedDayRange,
   rolloutSessionIdFromPath,
   isCodexResumeInvalidated,
+  isRecoverableGroupedReadError,
   finalizeToolRows,
   finalizeSkillRows,
   finalizeExecRows,

package/src/lib/grok-limits.js

@@ -529,6 +529,59 @@ function deriveGrokPlanLabel(rawTier) {
   return GROK_PLAN_TIERS.get(key) ?? null;
 }
 
+// Billing product ids → the name grok.com shows in the usage legend.
+// Unknown ids are humanized below so a new product still gets a row.
+const GROK_PRODUCT_LABELS = new Map([
+  ["GrokBuild", "Grok Build"],
+  ["GrokBot", "Grok Bot"],
+  ["GrokChat", "Chat"],
+  ["GrokImagine", "Imagine"],
+  ["Cursor", "Cursor"],
+]);
+
+function normalizeGrokProductId(value) {
+  if (typeof value !== "string") return null;
+  const id = value.trim();
+  if (!/^[A-Za-z][A-Za-z0-9]{0,31}$/.test(id)) return null;
+  return id;
+}
+
+function humanizeGrokProductId(productId) {
+  const spaced = productId
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+    .replace(/[_-]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+  return spaced.slice(0, 40) || null;
+}
+
+/**
+ * Per-product shares of the same pool as creditUsagePercent.
+ * A 12% Grok Bot row is 12 points of the weekly cap, not its own 100% quota.
+ * Zero and non-finite rows are omitted (grok.com hides unused products).
+ * API order is preserved; a repeated product id keeps the first finite row.
+ */
+function normalizeGrokProductUsage(productUsage, resetAt) {
+  if (!Array.isArray(productUsage)) return [];
+  const seen = new Set();
+  const products = [];
+  for (const entry of productUsage) {
+    if (!entry || typeof entry !== "object") continue;
+    const product = normalizeGrokProductId(entry.product);
+    if (!product || seen.has(product)) continue;
+    const raw = Number(entry.usagePercent);
+    if (!Number.isFinite(raw) || raw <= 0) continue;
+    const window = buildWindow({ usedPercent: raw, resetAt });
+    if (!window || !(window.used_percent > 0)) continue;
+    const label = GROK_PRODUCT_LABELS.get(product) || humanizeGrokProductId(product);
+    if (!label) continue;
+    seen.add(product);
+    products.push({ product, label, window });
+  }
+  return products;
+}
+
 /**
  * Parse either:
  *   - Unified billing (`?format=credits`): weekly/monthly period + creditUsagePercent
@@ -614,6 +667,8 @@ function normalizeGrokBillingResponse(body) {
     billing_period_start: periodStart,
     primary_window: primaryWindow,
     secondary_window: secondaryWindow,
+    // Shares of primary_window. Do not add these on top of credit_usage_percent.
+    products: normalizeGrokProductUsage(config.productUsage, resetAt),
   };
 }
 
@@ -803,6 +858,7 @@ module.exports = {
   inferGrokPeriodTypeFromDates,
   sumProductUsagePercent,
   deriveGrokPlanLabel,
+  normalizeGrokProductUsage,
   normalizeGrokBillingResponse,
   fetchGrokBilling,
   fetchGrokLimits,