tokentracker-cli 0.79.9 → 0.80.1

package/src/lib/rollout.js

@@ -172,6 +172,7 @@ async function parseRolloutIncremental({
   onProgress,
   source,
   publicRepoResolver,
+  diagnostics,
 }) {
   await ensureDir(path.dirname(queuePath));
   let filesProcessed = 0;
@@ -189,6 +190,7 @@ async function parseRolloutIncremental({
   const projectFreshnessNowMs = Date.now();
   const touchedBuckets = new Set();
   const defaultSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
+  const syncDiagnostics = diagnostics && typeof diagnostics === "object" ? diagnostics : null;
 
   if (!cursors.files || typeof cursors.files !== "object") {
     cursors.files = {};
@@ -199,7 +201,108 @@ async function parseRolloutIncremental({
   // external rewrite of a session file (Codex-Manager's atomic provider-patch on
   // account switch, issue #187) cannot re-count already-counted events.
   const prevCodexHashes = Array.isArray(cursors?.codexHashes) ? cursors.codexHashes : [];
-  const seenCodexEvents = new Set(prevCodexHashes);
+  if (!Array.isArray(cursors.codexHashes)) cursors.codexHashes = prevCodexHashes;
+  let seenCodexEvents = null;
+  let newCodexEventKeys = null;
+  let newCodexEventKeySet = null;
+  let appendOnlyCodexEvents = null;
+  let historicalCodexEvents = null;
+  let cursorSessionPaths = null;
+  if (syncDiagnostics) {
+    const codexParseCandidates = (Array.isArray(rolloutFiles) ? rolloutFiles : []).reduce((count, entry) => {
+      const entrySource = typeof entry === "string"
+        ? defaultSource
+        : normalizeSourceInput(entry?.source) || defaultSource;
+      return count + (entrySource === DEFAULT_SOURCE ? 1 : 0);
+    }, 0);
+    const discoveredRollouts = Number(syncDiagnostics.discovered_rollouts);
+    const cursorKeys = Number(syncDiagnostics.cursor_keys);
+    const coldSkipped = Number(syncDiagnostics.cold_skipped);
+    const parseCandidates = Number(syncDiagnostics.parse_candidates);
+    Object.assign(syncDiagnostics, {
+      discovered_rollouts: Number.isFinite(discoveredRollouts) ? discoveredRollouts : codexParseCandidates,
+      cursor_keys: Number.isFinite(cursorKeys) ? cursorKeys : Object.keys(cursors.files).length,
+      parse_candidates: Number.isFinite(parseCandidates) ? parseCandidates : codexParseCandidates,
+      stat_candidates: 0,
+      cold_skipped: Number.isFinite(coldSkipped) ? coldSkipped : 0,
+      content_files_read: 0,
+      hash_set_constructions: 0,
+      hash_array_materializations: 0,
+      hash_array_materialized_items: 0,
+      codex_hash_count: prevCodexHashes.length,
+    });
+  }
+  const ensureNewCodexEventKeys = () => {
+    if (!newCodexEventKeys) newCodexEventKeys = [];
+    if (!newCodexEventKeySet) newCodexEventKeySet = new Set();
+  };
+  const recordNewCodexEvent = (key) => {
+    ensureNewCodexEventKeys();
+    if (newCodexEventKeySet.has(key)) return;
+    newCodexEventKeySet.add(key);
+    newCodexEventKeys.push(key);
+    if (seenCodexEvents) seenCodexEvents.add(key);
+  };
+  const getAppendOnlyCodexEvents = () => {
+    if (!appendOnlyCodexEvents) {
+      appendOnlyCodexEvents = {
+        has(key) {
+          return Boolean(
+            newCodexEventKeySet?.has(key) ||
+            seenCodexEvents?.has(key)
+          );
+        },
+        add: recordNewCodexEvent,
+      };
+    }
+    return appendOnlyCodexEvents;
+  };
+  const getHistoricalCodexEvents = () => {
+    if (!seenCodexEvents) {
+      seenCodexEvents = new Set(prevCodexHashes);
+      for (const key of newCodexEventKeys || []) seenCodexEvents.add(key);
+      if (syncDiagnostics) syncDiagnostics.hash_set_constructions += 1;
+    }
+    if (!historicalCodexEvents) {
+      historicalCodexEvents = {
+        has: (key) => seenCodexEvents.has(key),
+        add: recordNewCodexEvent,
+      };
+    }
+    return historicalCodexEvents;
+  };
+  const getCursorSessionPaths = () => {
+    if (cursorSessionPaths) return cursorSessionPaths;
+    cursorSessionPaths = new Map();
+    for (const existingPath of Object.keys(cursors.files)) {
+      const sessionId = codexSessionIdFromPath(existingPath);
+      if (!sessionId) continue;
+      if (!cursorSessionPaths.has(sessionId)) cursorSessionPaths.set(sessionId, new Set());
+      cursorSessionPaths.get(sessionId).add(existingPath);
+    }
+    return cursorSessionPaths;
+  };
+  const needsHistoricalCodexDedup = ({
+    filePath,
+    prev,
+    sameInode,
+    truncated,
+    startOffset,
+    rebuildingBaseline,
+  }) => {
+    if (rebuildingBaseline) return true;
+    if (prev && (!sameInode || truncated)) return true;
+    if (sameInode && !truncated && startOffset > 0) return false;
+    if (prev?.lastTotal) return true;
+    const sessionId = codexSessionIdFromPath(filePath);
+    if (!sessionId) return true;
+    const knownPaths = getCursorSessionPaths().get(sessionId);
+    if (!knownPaths) return false;
+    for (const knownPath of knownPaths) {
+      if (knownPath !== filePath) return true;
+    }
+    return false;
+  };
 
   for (let idx = 0; idx < rolloutFiles.length; idx++) {
     const entry = rolloutFiles[idx];
@@ -209,6 +312,7 @@ async function parseRolloutIncremental({
       typeof entry === "string"
         ? defaultSource
         : normalizeSourceInput(entry?.source) || defaultSource;
+    if (syncDiagnostics && fileSource === DEFAULT_SOURCE) syncDiagnostics.stat_candidates += 1;
     const st = await fs.stat(filePath).catch(() => null);
     if (!st || !st.isFile()) continue;
 
@@ -218,8 +322,18 @@ async function parseRolloutIncremental({
     const sameInode = prev && prev.inode === inode;
     const prevOffset = sameInode ? prev.offset || 0 : 0;
     const truncated = sameInode && prevOffset > st.size;
-    const startOffset = sameInode && !truncated ? prevOffset : 0;
-    const lastTotal = sameInode && !truncated ? prev.lastTotal || null : null;
+    const rebuildingCodexBaseline = Boolean(
+      fileSource === DEFAULT_SOURCE &&
+      sameInode &&
+      !truncated &&
+      prevOffset > 0 &&
+      prevOffset < st.size &&
+      (!prev.lastTotal || typeof prev.lastTotal !== "object")
+    );
+    const startOffset = sameInode && !truncated && !rebuildingCodexBaseline ? prevOffset : 0;
+    const lastTotal = sameInode && !truncated && !rebuildingCodexBaseline
+      ? prev.lastTotal || null
+      : null;
     const lastModel = sameInode && !truncated ? prev.lastModel || null : null;
 
     const codexProjectFastPath = projectEnabled && fileSource === DEFAULT_SOURCE;
@@ -270,6 +384,21 @@ async function parseRolloutIncremental({
     const projectRef = projectContext?.projectRef || null;
     const projectKey = projectContext?.projectKey || null;
 
+    if (syncDiagnostics && !projectContextOnlyScan && fileSource === DEFAULT_SOURCE) {
+      syncDiagnostics.content_files_read += 1;
+    }
+    const codexEventTracker = fileSource === DEFAULT_SOURCE
+      ? (needsHistoricalCodexDedup({
+          filePath,
+          prev,
+          sameInode,
+          truncated,
+          startOffset,
+          rebuildingBaseline: rebuildingCodexBaseline,
+        })
+          ? getHistoricalCodexEvents
+          : getAppendOnlyCodexEvents)
+      : null;
     const result = projectContextOnlyScan
       ? await scanRolloutProjectFileContexts({
           filePath,
@@ -299,7 +428,7 @@ async function parseRolloutIncremental({
           publicRepoCache,
           publicRepoResolver,
           projectContext,
-          seenCodexEvents,
+          seenCodexEvents: codexEventTracker,
           sessionId: codexSessionIdFromPath(filePath),
         });
 
@@ -341,7 +470,12 @@ async function parseRolloutIncremental({
   const projectBucketsQueued = projectEnabled
     ? await enqueueTouchedProjectBuckets({ projectQueuePath, projectState, projectTouchedBuckets })
     : 0;
-  cursors.codexHashes = Array.from(seenCodexEvents);
+  for (const key of newCodexEventKeys || []) prevCodexHashes.push(key);
+  if (syncDiagnostics) {
+    syncDiagnostics.codex_hash_count = Array.isArray(cursors.codexHashes)
+      ? cursors.codexHashes.length
+      : prevCodexHashes.length;
+  }
   hourlyState.updatedAt = new Date().toISOString();
   cursors.hourly = hourlyState;
   if (projectState) {
@@ -359,8 +493,23 @@ async function filterColdCodexRolloutFiles({
   auditDue = false,
   nowMs = Date.now(),
   recentDays = DEFAULT_CODEX_COLD_SKIP_RECENT_DAYS,
+  diagnostics = null,
 } = {}) {
   const files = Array.isArray(rolloutFiles) ? rolloutFiles : [];
+  const syncDiagnostics = diagnostics && typeof diagnostics === "object" ? diagnostics : null;
+  const isCodexEntry = (entry) => (
+    typeof entry === "string" || (normalizeSourceInput(entry?.source) || DEFAULT_SOURCE) === DEFAULT_SOURCE
+  );
+  if (syncDiagnostics) {
+    const discoveredRollouts = files.reduce(
+      (count, entry) => count + (isCodexEntry(entry) ? 1 : 0),
+      0,
+    );
+    syncDiagnostics.discovered_rollouts = discoveredRollouts;
+    syncDiagnostics.cursor_keys = Object.keys(cursors?.files || {}).length;
+    syncDiagnostics.cold_skipped = 0;
+    syncDiagnostics.parse_candidates = discoveredRollouts;
+  }
   if (auditDue || !cursors?.files || typeof cursors.files !== "object") {
     return { rolloutFiles: files, skipped: 0 };
   }
@@ -414,6 +563,13 @@ async function filterColdCodexRolloutFiles({
     skipped += 1;
   }
 
+  if (syncDiagnostics) {
+    syncDiagnostics.cold_skipped = skipped;
+    syncDiagnostics.parse_candidates = out.reduce(
+      (count, entry) => count + (isCodexEntry(entry) ? 1 : 0),
+      0,
+    );
+  }
   return { rolloutFiles: out, skipped };
 }
 
@@ -1514,6 +1670,16 @@ async function parseRolloutFile({
   let currentCwd = null;
   let currentDate = null;
   let isForkedRollout = false;
+  // The replay prefix only ever exists at the head of a freshly-forked file. A
+  // resumed scan (startOffset > 0) can never see it — and never re-reads the
+  // session_meta line, so isForkedRollout stays false there anyway — but gate
+  // on the offset explicitly so correctness does not hinge on that distant
+  // invariant. Known accepted limitation: if a sync's read races the fork's
+  // single-flush write and persists a cursor INSIDE the burst, the resumed
+  // scan counts the remaining burst tail (bounded over-count, ms-wide window;
+  // self-heals on any inode-changing rescan and never drops genuine usage).
+  let replayPrefixActive = startOffset === 0;
+  let prevForkedTokenMs = null;
   const rolloutDate = rolloutDateFromPath(filePath);
   let currentProjectRef = projectRef || null;
   let currentProjectKey = projectKey || null;
@@ -1591,8 +1757,50 @@ async function parseRolloutFile({
       totals = totalUsage;
     }
 
-    // date matching is conservative; same-day fork replays are still counted.
-    if (isForkedReplayToken({ isForkedRollout, rolloutDate, currentDate })) continue;
+    // Forked Codex rollouts replay the parent session's entire token history
+    // into the child file the moment the fork is created. The date guard below
+    // (current_date < rolloutDate) catches cross-day forks, but same-day forks
+    // share the parent's current_date and slip through it. The replay is written
+    // in a single flush, so its rows carry near-identical timestamps (sub-ms to a
+    // few ms apart across observed 0.129–0.137 samples), while the first genuine
+    // live turn lands seconds later (≥11s in every sampled fork). We therefore
+    // skip the *leading* run of densely-spaced token_count rows and latch off
+    // permanently at the first multi-second gap: a lone fast turn (always
+    // preceded by a large gap) is never dropped, and an arbitrarily large replay
+    // is fully skipped regardless of length. The threshold sits far above the
+    // flush spacing and well below genuine turn cadence (≥~4.6s observed). The
+    // first replayed row cannot be identified without lookahead, so it is still
+    // counted (a bounded, <1% residual over-count); dropping real usage is the
+    // worse failure, so we bias against it. `totals` is already advanced above,
+    // keeping the cumulative-delta baseline correct for the live turns we keep.
+    // Scoped to forked codex rollouts. (issue #169 follow-up.)
+    let forkedReplaySkip = false;
+    if (isForkedRollout && source === DEFAULT_SOURCE && replayPrefixActive) {
+      const tokenMs = Date.parse(tokenTimestamp);
+      // Fail open on anything the burst heuristic was not measured against:
+      // an unparseable timestamp or a backwards clock step permanently ends
+      // the prefix, so no later row can be skipped off a stale baseline.
+      // Replay flushes are monotonic in every sampled rollout (2,116 gaps,
+      // min 0ms), so this never fires on genuine replays.
+      if (!Number.isFinite(tokenMs) || (prevForkedTokenMs !== null && tokenMs < prevForkedTokenMs)) {
+        replayPrefixActive = false;
+      } else {
+        if (prevForkedTokenMs !== null && tokenMs - prevForkedTokenMs >= CODEX_FORK_REPLAY_GAP_MS) {
+          replayPrefixActive = false;
+        }
+        forkedReplaySkip =
+          replayPrefixActive &&
+          prevForkedTokenMs !== null &&
+          tokenMs - prevForkedTokenMs < CODEX_FORK_REPLAY_GAP_MS;
+        prevForkedTokenMs = tokenMs;
+      }
+    }
+
+    // date matching is conservative; same-day fork replays are caught by the
+    // burst detector above rather than this cross-day date guard.
+    if (forkedReplaySkip || isForkedReplayToken({ isForkedRollout, rolloutDate, currentDate })) {
+      continue;
+    }
 
     const bucketStart = toUtcHalfHourStart(tokenTimestamp);
     if (!bucketStart) continue;
@@ -1613,10 +1821,13 @@ async function parseRolloutFile({
     // rewrite) manages Codex. Other rollout-format sources (e.g. every-code) have
     // their own model re-alignment that legitimately re-reads prior events, which
     // this dedup would otherwise suppress.
-    if (seenCodexEvents && source === "codex") {
+    const codexEvents = typeof seenCodexEvents === "function"
+      ? seenCodexEvents()
+      : seenCodexEvents;
+    if (codexEvents && source === "codex") {
       const dedupKey = `${sessionId || filePath}:${tokenTimestamp}`;
-      if (seenCodexEvents.has(dedupKey)) continue;
-      seenCodexEvents.add(dedupKey);
+      if (codexEvents.has(dedupKey)) continue;
+      codexEvents.add(dedupKey);
     }
 
     const bucket = getHourlyBucket(hourlyState, source, model, bucketStart);
@@ -2694,6 +2905,15 @@ function normalizeIsoDate(value) {
   return match ? match[1] : null;
 }
 
+// Max inter-row gap (ms) still treated as part of a forked rollout's replay
+// burst. Empirically the replay flush spaces rows sub-ms to a few ms apart while
+// genuine live turns arrive ≥~4.6s apart and the replay→live break is ≥11s, so
+// any value in the ~0.25–2s band separates them with a wide margin; 500ms biases
+// toward the low end because merging a real fast turn (under-count) is worse than
+// leaving a slow replay counted (bounded over-count). See the skip site in
+// parseRolloutFile. (issue #169 follow-up.)
+const CODEX_FORK_REPLAY_GAP_MS = 500;
+
 function isForkedReplayToken({ isForkedRollout, rolloutDate, currentDate }) {
   return Boolean(isForkedRollout && rolloutDate && currentDate && currentDate < rolloutDate);
 }

package/src/lib/usage-limits.js

@@ -2823,6 +2823,7 @@ async function fetchUsageLimitsUncached({
   requestFn,
   now = new Date(),
   providerTimeoutMs = DEFAULT_PROVIDER_TIMEOUT_MS,
+  forceRefresh = false,
 } = {}) {
   const nowMs = Date.now();
 
@@ -2870,11 +2871,14 @@ async function fetchUsageLimitsUncached({
   // Skip the upstream Claude call entirely while a 429 cooldown is active — calling again
   // just renews the penalty. The result handling below serves cache or a cooldown message.
   const claudeRetryAtMs = claudeToken ? readClaudeRateLimitRetryAtMs({ home, nowMs }) : null;
-  // Also avoid cross-process hammering after a recent successful read. The macOS app can
-  // restart its embedded Node server or force-refresh the limits page, both of which clear
-  // the in-memory cache; the disk cache keeps those paths from immediately spending
-  // another Claude OAuth usage request.
-  const freshClaudeCache = claudeToken ? readFreshClaudeLimitsCache({ home, nowMs }) : null;
+  // Also avoid cross-process hammering after a recent successful read: embedded-server
+  // restarts and background polls read the disk cache instead of spending another Claude
+  // OAuth usage request. An explicit user refresh (refresh=1 → forceRefresh) punches
+  // through this cache — but never through the 429 cooldown above, which is exactly the
+  // hammering the cooldown exists to prevent.
+  const freshClaudeCache = claudeToken && !forceRefresh
+    ? readFreshClaudeLimitsCache({ home, nowMs })
+    : null;
 
   const providerFetch = withFetchTimeout(fetchImpl, providerTimeoutMs);
   const [claudeResult, codexResult, cursor, kimi, gemini, kiro, antigravity, copilot, grok, zcode, opencodeGo] = await Promise.all([

package/dashboard/dist/assets/DialogDescription-DWs-KIUA.js

@@ -1 +0,0 @@
-import{r as n,bb as a}from"./main-B9m79io-.js";import{u as p}from"./DialogTitle-f7gNmCca.js";import{u as c}from"./useScrollLock-DijV6DYS.js";const x=n.forwardRef(function(e,r){const{render:l,className:u,style:m,id:s,...t}=e,{store:i}=p(),o=c(s);return i.useSyncedValueWithCleanup("descriptionElementId",o),a("p",e,{ref:r,props:[{id:o},t]})});export{x as D};

package/dashboard/dist/assets/check-Dn9BLYEf.js

@@ -1 +0,0 @@
-import{c}from"./main-B9m79io-.js";const e=[["path",{d:"M20 6 9 17l-5-5",key:"1gmf2c"}]],t=c("check",e);export{t as C};

package/dashboard/dist/assets/useTokenFormat-nxdG6gi9.js

@@ -1 +0,0 @@
-import{r as e,b3 as r,ai as a,b4 as n,b5 as s}from"./main-B9m79io-.js";const T=Object.freeze({mode:a.COMPACT,setMode:()=>{},formatTokens:(o,t={})=>s(o,t),formatTokensTooltip:(o,t={})=>n(o,t)});function f(){return e.useContext(r)??T}export{f as u};