tokentracker-cli 0.80.0 → 0.80.2

package/src/lib/codex-config.js

@@ -55,12 +55,12 @@ async function restoreNotify({ configPath, notifyOriginalPath, expectedNotify })
   const originalNotify = Array.isArray(original?.notify) ? original.notify : null;
   const currentNotify = extractNotify(text);
 
-  if (expectedNotify && !arraysEqual(currentNotify, expectedNotify)) {
-    return { restored: false, skippedReason: "current-not-managed" };
+  if (!originalNotify && expectedNotify && currentNotify == null) {
+    return { restored: false, skippedReason: "no-backup-not-installed" };
   }
 
-  if (!originalNotify && expectedNotify && !arraysEqual(currentNotify, expectedNotify)) {
-    return { restored: false, skippedReason: "no-backup-not-installed" };
+  if (expectedNotify && !arraysEqual(currentNotify, expectedNotify)) {
+    return { restored: false, skippedReason: "current-not-managed" };
   }
 
   const updated = originalNotify ? setNotify(text, originalNotify) : removeNotify(text);
@@ -161,15 +161,16 @@ function extractNotify(text) {
   const lines = text.split(/\r?\n/);
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
+    if (isTomlTableHeader(line)) break;
     const m = line.match(/^\s*notify\s*=\s*(.*)\s*$/);
     if (!m) continue;
 
     const rhs = (m[1] || "").trim();
     const literal = readTomlArrayLiteral(lines, i, rhs);
-    if (!literal) continue;
+    if (!literal) return null;
 
     const parsed = parseTomlStringArray(literal);
-    if (parsed) return parsed;
+    return parsed;
   }
   return null;
 }
@@ -182,6 +183,14 @@ function setNotify(text, notifyCmd) {
   let replaced = false;
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
+    if (isTomlTableHeader(line)) {
+      if (!replaced) {
+        out.push(notifyLine);
+        replaced = true;
+      }
+      out.push(...lines.slice(i));
+      break;
+    }
     const m = line.match(/^\s*notify\s*=\s*(.*)\s*$/);
     if (m) {
       if (!replaced) {
@@ -190,7 +199,7 @@ function setNotify(text, notifyCmd) {
       }
 
       const rhs = (m[1] || "").trim();
-      i = findTomlArrayBlockEnd(lines, i, rhs);
+      i = findNotifyBlockEnd(lines, i, rhs);
       continue;
     }
     out.push(line);
@@ -211,10 +220,14 @@ function removeNotify(text) {
   const out = [];
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
+    if (isTomlTableHeader(line)) {
+      out.push(...lines.slice(i));
+      break;
+    }
     const m = line.match(/^\s*notify\s*=\s*(.*)\s*$/);
     if (m) {
       const rhs = (m[1] || "").trim();
-      i = findTomlArrayBlockEnd(lines, i, rhs);
+      i = findNotifyBlockEnd(lines, i, rhs);
       continue;
     }
     out.push(line);
@@ -225,45 +238,78 @@ function removeNotify(text) {
 function parseTomlStringArray(rhs) {
   // Minimal parser for TOML basic/literal string arrays.
   if (!rhs.startsWith("[") || !rhs.endsWith("]")) return null;
-  const inner = rhs.slice(1, -1).trim();
-  if (!inner) return [];
 
   const parts = [];
-  let current = "";
-  let inString = false;
-  let quote = null;
-  for (let i = 0; i < inner.length; i++) {
-    const ch = inner[i];
-    if (!inString) {
-      if (ch === '"' || ch === "'") {
-        inString = true;
-        quote = ch;
-        current = "";
+  let i = 1;
+  let expectValue = true;
+
+  function skipSpaceAndComments() {
+    while (i < rhs.length) {
+      const ch = rhs[i];
+      if (/\s/.test(ch)) {
+        i += 1;
+        continue;
       }
-      continue;
+      if (ch === "#") {
+        while (i < rhs.length && rhs[i] !== "\n") i += 1;
+        continue;
+      }
+      break;
     }
-    if (ch === quote) {
-      parts.push(current);
-      inString = false;
-      quote = null;
-      continue;
+  }
+
+  function parseString() {
+    const quote = rhs[i];
+    if (quote !== '"' && quote !== "'") return null;
+    i += 1;
+    let value = "";
+    while (i < rhs.length) {
+      const ch = rhs[i];
+      if (ch === quote) {
+        i += 1;
+        return value;
+      }
+      if (ch === "\n" || ch === "\r") return null;
+      if (quote === '"' && ch === "\\") {
+        if (i + 1 >= rhs.length) return null;
+        const next = rhs[i + 1];
+        if (next === "b") value += "\b";
+        else if (next === "t") value += "\t";
+        else if (next === "n") value += "\n";
+        else if (next === "f") value += "\f";
+        else if (next === "r") value += "\r";
+        else if (next === '"' || next === "\\" || next === "/") value += next;
+        else return null;
+        i += 2;
+        continue;
+      }
+      value += ch;
+      i += 1;
+    }
+    return null;
+  }
+
+  while (i < rhs.length) {
+    skipSpaceAndComments();
+    if (rhs[i] === "]") {
+      i += 1;
+      skipSpaceAndComments();
+      return i === rhs.length ? parts : null;
     }
-    if (quote === '"' && ch === "\\") {
-      if (i + 1 >= inner.length) return null;
-      const next = inner[++i];
-      if (next === "b") current += "\b";
-      else if (next === "t") current += "\t";
-      else if (next === "n") current += "\n";
-      else if (next === "f") current += "\f";
-      else if (next === "r") current += "\r";
-      else if (next === '"' || next === "\\" || next === "/") current += next;
-      else return null;
+    if (!expectValue) {
+      if (rhs[i] !== ",") return null;
+      i += 1;
+      expectValue = true;
       continue;
     }
-    current += ch;
+
+    const value = parseString();
+    if (value == null) return null;
+    parts.push(value);
+    expectValue = false;
   }
 
-  return parts.length > 0 ? parts : null;
+  return null;
 }
 
 function formatTomlStringArray(arr) {
@@ -278,6 +324,7 @@ function readTomlArrayLiteral(lines, startIndex, rhs) {
   let quote = null;
   let depth = 0;
   let sawOpen = false;
+  let invalid = false;
 
   function scanChunk(chunk) {
     for (let i = 0; i < chunk.length; i++) {
@@ -295,7 +342,10 @@ function readTomlArrayLiteral(lines, startIndex, rhs) {
         }
         if (ch === "]") {
           depth -= 1;
-          if (sawOpen && depth === 0) return i;
+          if (sawOpen && depth === 0) {
+            if (!isTomlArrayTrailer(chunk.slice(i + 1))) invalid = true;
+            return i;
+          }
         }
         continue;
       }
@@ -305,6 +355,10 @@ function readTomlArrayLiteral(lines, startIndex, rhs) {
         continue;
       }
       if (quote === '"' && ch === "\\") {
+        if (i + 1 >= chunk.length) {
+          invalid = true;
+          return -1;
+        }
         i += 1;
       }
     }
@@ -313,11 +367,13 @@ function readTomlArrayLiteral(lines, startIndex, rhs) {
 
   const parts = [first];
   let endPos = scanChunk(first);
+  if (invalid) return null;
   if (endPos !== -1) return first.slice(0, endPos + 1).trim();
 
   for (let j = startIndex + 1; j < lines.length; j++) {
     const line = lines[j];
     endPos = scanChunk(line);
+    if (invalid) return null;
     if (endPos !== -1) {
       parts.push(line.slice(0, endPos + 1));
       return parts.join("\n").trim();
@@ -336,6 +392,7 @@ function findTomlArrayBlockEnd(lines, startIndex, rhs) {
   let quote = null;
   let depth = 0;
   let sawOpen = false;
+  let invalid = false;
 
   function scanChunk(chunk) {
     for (let i = 0; i < chunk.length; i++) {
@@ -353,7 +410,10 @@ function findTomlArrayBlockEnd(lines, startIndex, rhs) {
         }
         if (ch === "]") {
           depth -= 1;
-          if (sawOpen && depth === 0) return true;
+          if (sawOpen && depth === 0) {
+            if (!isTomlArrayTrailer(chunk.slice(i + 1))) invalid = true;
+            return true;
+          }
         }
         continue;
       }
@@ -363,6 +423,10 @@ function findTomlArrayBlockEnd(lines, startIndex, rhs) {
         continue;
       }
       if (quote === '"' && ch === "\\") {
+        if (i + 1 >= chunk.length) {
+          invalid = true;
+          return false;
+        }
         i += 1;
       }
     }
@@ -370,12 +434,43 @@ function findTomlArrayBlockEnd(lines, startIndex, rhs) {
   }
 
   if (scanChunk(first)) return startIndex;
+  if (invalid) return startIndex;
   for (let j = startIndex + 1; j < lines.length; j++) {
     if (scanChunk(lines[j])) return j;
+    if (invalid) return startIndex;
   }
   return startIndex;
 }
 
+function findNotifyBlockEnd(lines, startIndex, rhs) {
+  const endIndex = findTomlArrayBlockEnd(lines, startIndex, rhs);
+  const literal = readTomlArrayLiteral(lines, startIndex, rhs);
+  if (
+    endIndex !== startIndex ||
+    !rhs.trim().startsWith("[") ||
+    (literal && parseTomlStringArray(literal))
+  ) {
+    return endIndex;
+  }
+
+  for (let j = startIndex + 1; j < lines.length; j++) {
+    if (isTopLevelTomlBoundary(lines[j])) return j - 1;
+  }
+  return lines.length - 1;
+}
+
+function isTomlTableHeader(line) {
+  return /^\s*\[\[?[^\]]+\]\]?\s*(?:#.*)?$/.test(line);
+}
+
+function isTopLevelTomlBoundary(line) {
+  return /^\s*(?:\[\[?[^\]]+\]\]?|(?:[A-Za-z0-9_-]+|"[^"]+"|'[^']+')(?:\s*\.\s*(?:[A-Za-z0-9_-]+|"[^"]+"|'[^']+'))*\s*=)/.test(line);
+}
+
+function isTomlArrayTrailer(text) {
+  return /^\s*(?:#.*)?$/.test(text);
+}
+
 function arraysEqual(a, b) {
   if (!Array.isArray(a) || !Array.isArray(b)) return false;
   if (a.length !== b.length) return false;

package/src/lib/codex-context-breakdown.js

@@ -10,7 +10,7 @@
 
 "use strict";
 
-const fs = require("node:fs");
+const crypto = require("node:crypto");
 const os = require("node:os");
 const path = require("node:path");
 
@@ -25,8 +25,8 @@ const {
 
 const {
   parseCodexRolloutFile,
-  listCodexSessionFiles,
-  dayKeyToIsoBounds,
+  discoverCodexSessionFiles,
+  rolloutSessionIdFromPath,
   finalizeToolRows,
   finalizeSkillRows,
   finalizeExecRows,
@@ -120,25 +120,55 @@ function buildDateRange({ period, date }) {
 // ---------------------------------------------------------------------------
 
 const CACHE = new Map();
+const PARSED_GROUP_CACHE = new Map();
 const CACHE_TTL_MS = 60_000;
-const CACHE_SCHEMA_VERSION = "codex-context-v2";
-
-function maxMtimeMs(files) {
-  let max = 0;
-  for (const filePath of files) {
-    try {
-      const st = fs.statSync(filePath);
-      if (st.mtimeMs > max) max = st.mtimeMs;
-    } catch {}
-  }
-  return max;
-}
+const CACHE_SCHEMA_VERSION = "codex-context-v4";
+const MAX_PARSED_GROUP_CACHE_ENTRIES = 4096;
 
 function cacheTimeZoneKey(timeZoneContext) {
   if (!timeZoneContext) return "";
   return `${timeZoneContext.timeZone || ""}|${Number.isFinite(timeZoneContext.offsetMinutes) ? timeZoneContext.offsetMinutes : ""}`;
 }
 
+function statIdentity(filePath, stat) {
+  return JSON.stringify([filePath, stat.size, stat.mtimeMs, stat.ctimeMs, stat.ino]);
+}
+
+function inventorySignature(entries) {
+  const hash = crypto.createHash("sha256");
+  for (const { filePath, stat, candidate } of entries) {
+    hash.update(statIdentity(filePath, stat));
+    hash.update(candidate ? "1" : "0");
+    hash.update("\0");
+  }
+  return hash.digest("hex");
+}
+
+function parsedGroupCacheKey(entries, { from, to, timeZoneContext }) {
+  const hash = crypto.createHash("sha256");
+  hash.update(CACHE_SCHEMA_VERSION);
+  hash.update(from || "");
+  hash.update("\0");
+  hash.update(to || "");
+  hash.update("\0");
+  hash.update(cacheTimeZoneKey(timeZoneContext));
+  for (const { filePath, stat } of entries) {
+    hash.update(statIdentity(filePath, stat));
+    hash.update("\0");
+  }
+  return hash.digest("hex");
+}
+
+function rememberParsedGroup(key, value) {
+  PARSED_GROUP_CACHE.delete(key);
+  PARSED_GROUP_CACHE.set(key, value);
+  while (PARSED_GROUP_CACHE.size > MAX_PARSED_GROUP_CACHE_ENTRIES) {
+    const oldest = PARSED_GROUP_CACHE.keys().next().value;
+    if (!oldest) break;
+    PARSED_GROUP_CACHE.delete(oldest);
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Main entry
 // ---------------------------------------------------------------------------
@@ -151,6 +181,9 @@ async function computeCodexContextBreakdown({
   codexDir = null,
   top = 20,
   timeZoneContext = null,
+  exhaustive = false,
+  nowMs = Date.now(),
+  includeDiagnostics = false,
 } = {}) {
   let fromKey = from;
   let toKey = to;
@@ -160,10 +193,33 @@ async function computeCodexContextBreakdown({
     toKey = range?.to || null;
   }
 
-  const { fromIso, toIso } = dayKeyToIsoBounds(fromKey, toKey);
-  const baseDir = codexDir || path.join(os.homedir(), ".codex", "sessions");
-
-  const files = await listCodexSessionFiles(baseDir);
+  const codexHome = process.env.CODEX_HOME || path.join(os.homedir(), ".codex");
+  const roots = codexDir
+    ? [codexDir]
+    : [path.join(codexHome, "sessions"), path.join(codexHome, "archived_sessions")];
+  const baseDir = roots.join(path.delimiter);
+  const diagnostics = {
+    cache_hit: false,
+    full_metadata_audit: false,
+    discovered_files: 0,
+    candidate_files: 0,
+    stat_calls: 0,
+    metadata_cache_hits: 0,
+    opened_files: 0,
+    parsed_files: 0,
+    parse_cache_hits: 0,
+    json_parse_calls: 0,
+  };
+  const discovered = discoverCodexSessionFiles(roots, {
+    from: fromKey,
+    to: toKey,
+    timeZoneContext,
+    exhaustive,
+    nowMs,
+    diagnostics,
+  });
+  const candidates = discovered.filter((entry) => entry.candidate);
+  const fileSignature = inventorySignature(discovered);
   const cacheKey = [
     CACHE_SCHEMA_VERSION,
     baseDir,
@@ -171,24 +227,48 @@ async function computeCodexContextBreakdown({
     toKey || "",
     cacheTimeZoneKey(timeZoneContext),
     Number.isFinite(top) ? top : 20,
-    files.length,
-    maxMtimeMs(files),
+    exhaustive ? "exhaustive" : "bounded",
+    fileSignature,
   ].join("|");
   const cached = CACHE.get(cacheKey);
   if (cached && Date.now() - cached.at < CACHE_TTL_MS) {
-    return cached.value;
+    diagnostics.cache_hit = true;
+    return includeDiagnostics ? { ...cached.value, diagnostics } : cached.value;
   }
 
   const sessions = [];
+  const seenTokenEvents = new Set();
+
+  const candidateGroups = new Map();
+  for (const entry of candidates) {
+    const sessionKey = rolloutSessionIdFromPath(entry.filePath) || entry.filePath;
+    if (!candidateGroups.has(sessionKey)) candidateGroups.set(sessionKey, []);
+    candidateGroups.get(sessionKey).push(entry);
+  }
 
-  for (const filePath of files) {
-    const parsed = await parseCodexRolloutFile(filePath, {
-      fromIso,
-      toIso,
-      from: fromKey,
-      to: toKey,
-      timeZoneContext,
-    });
+  for (const entries of candidateGroups.values()) {
+    const filePaths = entries.map((entry) => entry.filePath);
+    const cacheable = Boolean(rolloutSessionIdFromPath(filePaths[0]));
+    const parsedCacheKey = cacheable
+      ? parsedGroupCacheKey(entries, { from: fromKey, to: toKey, timeZoneContext })
+      : null;
+    let parsed = parsedCacheKey ? PARSED_GROUP_CACHE.get(parsedCacheKey) : null;
+    if (parsed) {
+      // Cacheable groups are partitioned by rollout UUID, so their token event
+      // keys cannot overlap and cache hits need not backfill seenTokenEvents.
+      diagnostics.parse_cache_hits += 1;
+      PARSED_GROUP_CACHE.delete(parsedCacheKey);
+      PARSED_GROUP_CACHE.set(parsedCacheKey, parsed);
+    } else {
+      parsed = await parseCodexRolloutFile(filePaths.length === 1 ? filePaths[0] : filePaths, {
+        from: fromKey,
+        to: toKey,
+        timeZoneContext,
+        diagnostics,
+        seenTokenEvents,
+      });
+      if (parsedCacheKey) rememberParsedGroup(parsedCacheKey, parsed);
+    }
     if (!parsed || !parsed.totals || !parsed.totals.total_tokens) continue;
     sessions.push(parsed);
   }
@@ -261,26 +341,25 @@ async function computeCodexContextBreakdown({
     }))
     .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
 
-  const nonTextToolTotal = toolRows.reduce((sum, row) => {
-    if ((row.raw_name || row.name) === "text_response") return sum;
-    return sum + Number(row.totals?.total_tokens || 0);
-  }, 0);
-  const displayedMessageTotal = Math.max(
-    0,
-    Number(grand.total_tokens || 0) - Number(grand.reasoning_output_tokens || 0) - nonTextToolTotal,
-  );
   const textResponse = toolRows.find((row) => (row.raw_name || row.name) === "text_response");
   const textResponseTotals = textResponse?.totals || emptyTotals();
   const textResponseHistoryWeight = Math.max(
     0,
     Number(textResponseTotals.cached_input_tokens || 0) + Number(textResponseTotals.cache_creation_input_tokens || 0),
   );
+  const textResponseInputWeight = Math.max(0, Number(textResponseTotals.input_tokens || 0));
+  const textResponseOutputWeight = Math.max(
+    0,
+    Number(textResponseTotals.output_tokens || 0) - Number(textResponseTotals.reasoning_output_tokens || 0),
+  );
+  const displayedMessageTotal =
+    textResponseInputWeight + textResponseHistoryWeight + textResponseOutputWeight;
   const messageAlloc = allocateByLargestRemainder(
     displayedMessageTotal,
     {
-      user_input: Math.max(0, Number(textResponseTotals.input_tokens || 0)),
+      user_input: textResponseInputWeight,
       conversation_history: textResponseHistoryWeight,
-      assistant_response: Math.max(0, Number(textResponseTotals.output_tokens || 0)),
+      assistant_response: textResponseOutputWeight,
     },
     ["user_input", "conversation_history", "assistant_response"],
   );
@@ -350,7 +429,7 @@ async function computeCodexContextBreakdown({
     scope: "supported",
     breakdown_status: "ok",
     totals: grand,
-    session_count: sessions.length,
+    session_count: new Set(sessions.map((session) => session.sessionId || session.filePath)).size,
     message_count: sessions.reduce((a, s) => a + (s.turnCount || 0), 0),
     message_breakdown: {
       categories: messageBreakdown,
@@ -387,12 +466,13 @@ async function computeCodexContextBreakdown({
     },
   };
 
-  CACHE.set(cacheKey, { at: Date.now(), value: result });
+  const cacheValue = { ...result };
+  CACHE.set(cacheKey, { at: Date.now(), value: cacheValue });
   while (CACHE.size > 32) {
     const oldest = [...CACHE.entries()].sort((a, b) => a[1].at - b[1].at)[0];
     if (oldest) CACHE.delete(oldest[0]);
   }
-  return result;
+  return includeDiagnostics ? { ...result, diagnostics } : result;
 }
 
 module.exports = {