tokentracker-cli 0.81.2 → 0.81.3

package/src/lib/diagnostics.js

@@ -3,6 +3,7 @@ const path = require("node:path");
 const fs = require("node:fs/promises");
 
 const { readJson } = require("./fs");
+const { readCursorStateSummary } = require("./cursor-store");
 const { readCodexNotify, readEveryCodeNotify } = require("./codex-config");
 const { areClaudeUsageHooksConfigured, buildClaudeHookCommand } = require("./claude-config");
 const {
@@ -59,7 +60,8 @@ async function collectTrackerDiagnostics({
     path.join(home, ".grok");
 
   const config = await readJson(configPath);
-  const cursors = await readJson(cursorsPath);
+  const cursorSummary = await readCursorStateSummary({ trackerDir, cursorsPath });
+  const cursors = cursorSummary.cursors;
   const queueState = (await readJson(queueStatePath)) || { offset: 0 };
   const uploadThrottle = normalizeUploadState(await readJson(uploadThrottlePath));
   const autoRetry = await readJson(autoRetryPath);
@@ -165,10 +167,11 @@ async function collectTrackerDiagnostics({
     },
     parse: {
       updated_at: typeof cursors?.updatedAt === "string" ? cursors.updatedAt : null,
-      file_count:
-        cursors?.files && typeof cursors.files === "object"
-          ? Object.keys(cursors.files).length
-          : null,
+      file_count: cursorSummary.fileCount,
+      cursor_store: cursorSummary.mode,
+      cursor_store_legacy_drift: cursorSummary.legacyDrift === true,
+      codex_file_count: cursorSummary.codexFileCount,
+      codex_event_count: cursorSummary.codexEventCount,
     },
     queue: {
       size_bytes: queueSize,

package/src/lib/fs.js

@@ -1,4 +1,5 @@
 const fs = require("node:fs/promises");
+const crypto = require("node:crypto");
 const path = require("node:path");
 
 async function ensureDir(p) {
@@ -48,28 +49,265 @@ async function chmod600IfPossible(filePath) {
 }
 
 const LOCK_STALE_MS = 5 * 60 * 1000; // 5 minutes
+const LOCK_HEARTBEAT_MS = 30 * 1000;
+const MAX_RECLAIM_DEPTH = 4;
+const TRANSITION_GUARD_RETRY_MS = 5;
+const TRANSITION_GUARD_ATTEMPTS = 1000;
 
-async function openLock(lockPath, { quietIfLocked }) {
+function parseLockOwner(raw) {
+  try {
+    const parsed = JSON.parse(raw);
+    const pid = Number(parsed?.pid);
+    const token = typeof parsed?.token === "string" ? parsed.token : null;
+    if (!Number.isSafeInteger(pid) || pid <= 0 || !token) return null;
+    return { pid, token };
+  } catch (_e) {
+    return null;
+  }
+}
+
+function heartbeatPathFor(lockPath, token) {
+  const tokenDigest = crypto.createHash("sha256").update(token, "utf8").digest("hex");
+  return `${lockPath}.heartbeat.${tokenDigest}`;
+}
+
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    if (e?.code === "ESRCH") return false;
+    // EPERM means the process exists but belongs to another user. Unknown
+    // errors are treated conservatively so a live lock is never reclaimed.
+    return true;
+  }
+}
+
+async function existingLockCanBeReclaimed(lockPath) {
+  let lockHandle = null;
+  try {
+    // Inspect one opened inode instead of stat-ing a path and then reading
+    // that path. The latter is a TOCTOU window and is also unsafe if a
+    // concurrent reclaimer replaces the lock between the two operations.
+    lockHandle = await fs.open(lockPath, "r");
+    const [stat, raw] = await Promise.all([
+      lockHandle.stat(),
+      lockHandle.readFile({ encoding: "utf8" }),
+    ]);
+    const owner = parseLockOwner(raw);
+    if (owner) {
+      if (!isProcessAlive(owner.pid)) return true;
+
+      // New leases keep a token-specific heartbeat separate from the owner
+      // file. This lets a PID that has since been reused be recognized as an
+      // abandoned lease without reclaiming a live, long-running sync whose
+      // heartbeat is still fresh. Locks from before heartbeat support retain
+      // the conservative live-PID behavior.
+      let heartbeatHandle = null;
+      try {
+        heartbeatHandle = await fs.open(heartbeatPathFor(lockPath, owner.token), "r");
+        const heartbeat = await heartbeatHandle.stat();
+        return Date.now() - heartbeat.mtimeMs > LOCK_STALE_MS;
+      } catch (_e) {
+        return false;
+      } finally {
+        await heartbeatHandle?.close().catch(() => {});
+      }
+    }
+    return Date.now() - stat.mtimeMs > LOCK_STALE_MS;
+  } catch (e) {
+    // The lock disappeared between open and inspection, so retry acquisition.
+    if (e?.code === "ENOENT") return true;
+    return false;
+  } finally {
+    await lockHandle?.close().catch(() => {});
+  }
+}
+
+function startLockHeartbeat(handle, heartbeatHandle) {
+  const beat = async () => {
+    try {
+      const now = new Date();
+      await Promise.all([
+        handle.utimes(now, now),
+        heartbeatHandle.utimes(now, now),
+      ]);
+    } catch (_e) {
+      // A failed heartbeat makes the lease eligible for bounded stale
+      // reclamation. The owning sync will still finish or release normally.
+    }
+  };
+  const timer = setInterval(() => {
+    void beat();
+  }, LOCK_HEARTBEAT_MS);
+  timer.unref?.();
+  return timer;
+}
+
+async function releaseOwnedLock(
+  lockPath,
+  handle,
+  heartbeatHandle,
+  heartbeatTimer,
+  token,
+  {
+    beforeReleaseUnlink = null,
+    reclaimDepth = 0,
+    serializeRelease = true,
+  } = {},
+) {
+  let transitionGuard = null;
+  if (serializeRelease) {
+    // Reclaimers use the same guard while moving a stale lease aside. Holding
+    // it across token validation and unlink makes that ownership transition
+    // indivisible: an old owner cannot delete a replacement installed by a
+    // concurrent stale-lock reclaimer.
+    for (let attempt = 0; attempt < TRANSITION_GUARD_ATTEMPTS; attempt += 1) {
+      transitionGuard = await openLock(`${lockPath}.reclaim`, {
+        quietIfLocked: true,
+        reclaimDepth: reclaimDepth + 1,
+        // This short-lived internal guard is the serialization primitive; its
+        // own release must not recursively acquire another transition guard.
+        serializeRelease: false,
+      });
+      if (transitionGuard) break;
+      await new Promise((resolve) => setTimeout(resolve, TRANSITION_GUARD_RETRY_MS));
+    }
+    if (!transitionGuard) {
+      clearInterval(heartbeatTimer);
+      await handle.close().catch(() => {});
+      await heartbeatHandle?.close().catch(() => {});
+      await fs.unlink(heartbeatPathFor(lockPath, token)).catch(() => {});
+      throw new Error(`Timed out serializing lock release: ${lockPath}`);
+    }
+  }
+
+  try {
+    clearInterval(heartbeatTimer);
+    await handle.close().catch(() => {});
+    await heartbeatHandle?.close().catch(() => {});
+    const owner = parseLockOwner(await fs.readFile(lockPath, "utf8"));
+    if (owner?.token === token) {
+      if (typeof beforeReleaseUnlink === "function") {
+        await beforeReleaseUnlink({ lockPath });
+      }
+      await fs.unlink(lockPath).catch(() => {});
+      await fs.unlink(heartbeatPathFor(lockPath, token)).catch(() => {});
+    }
+  } catch (_e) {
+    // Missing or replaced owner paths are already released from this token's
+    // perspective. Other failures retain the conservative no-delete outcome.
+  } finally {
+    await transitionGuard?.release();
+  }
+}
+
+async function openLock(
+  lockPath,
+  {
+    quietIfLocked = false,
+    beforeReclaim = null,
+    beforeReleaseUnlink = null,
+    reclaimDepth = 0,
+    serializeRelease = true,
+  } = {},
+) {
   try {
     const handle = await fs.open(lockPath, "wx");
-    return {
-      async release() {
-        await handle.close().catch(() => {});
-        await fs.unlink(lockPath).catch(() => {});
-      },
-    };
+    const token = crypto.randomUUID();
+    const heartbeatPath = heartbeatPathFor(lockPath, token);
+    let heartbeatHandle = null;
+    try {
+      await handle.writeFile(
+        JSON.stringify({ pid: process.pid, token, createdAt: new Date().toISOString() }) + "\n",
+        "utf8",
+      );
+      heartbeatHandle = await fs.open(heartbeatPath, "wx");
+      await heartbeatHandle.writeFile(`${token}\n`, "utf8");
+      const heartbeatTimer = startLockHeartbeat(handle, heartbeatHandle);
+      return {
+        async release() {
+          await releaseOwnedLock(
+            lockPath,
+            handle,
+            heartbeatHandle,
+            heartbeatTimer,
+            token,
+            {
+              beforeReleaseUnlink,
+              reclaimDepth,
+              serializeRelease,
+            },
+          );
+        },
+      };
+    } catch (e) {
+      await heartbeatHandle?.close().catch(() => {});
+      await fs.unlink(heartbeatPath).catch(() => {});
+      await handle.close().catch(() => {});
+      await fs.unlink(lockPath).catch(() => {});
+      throw e;
+    }
   } catch (e) {
     if (e && e.code === "EEXIST") {
-      // Check if lock is stale
-      try {
-        const stat = await fs.stat(lockPath);
-        if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
-          await fs.unlink(lockPath).catch(() => {});
-          return openLock(lockPath, { quietIfLocked });
+      if (await existingLockCanBeReclaimed(lockPath)) {
+        if (typeof beforeReclaim === "function") {
+          await beforeReclaim({ lockPath });
+        }
+        if (reclaimDepth >= MAX_RECLAIM_DEPTH) return null;
+
+        // Serialize all reclaimers before re-checking the target. A stale
+        // check can be shared by many contenders; only the holder of this
+        // atomic guard may move the old lease out of the way.
+        const reclaimGuard = await openLock(`${lockPath}.reclaim`, {
+          quietIfLocked: true,
+          reclaimDepth: reclaimDepth + 1,
+          serializeRelease: false,
+        });
+        if (!reclaimGuard) return null;
+
+        let quarantinePath = null;
+        try {
+          // Re-check while holding the guard. A competing reclaimer may have
+          // already replaced the target since our initial stale inspection.
+          if (!(await existingLockCanBeReclaimed(lockPath))) return null;
+
+          quarantinePath = `${lockPath}.stale.${process.pid}.${crypto.randomUUID()}`;
+          try {
+            await fs.rename(lockPath, quarantinePath);
+          } catch (renameError) {
+            if (renameError?.code === "ENOENT") {
+              return openLock(lockPath, {
+                quietIfLocked,
+                beforeReleaseUnlink,
+                reclaimDepth,
+                serializeRelease,
+              });
+            }
+            if (!quietIfLocked) {
+              process.stdout.write("Another sync is already running.\n");
+            }
+            return null;
+          }
+
+          const staleOwner = parseLockOwner(
+            await fs.readFile(quarantinePath, "utf8").catch(() => ""),
+          );
+          if (staleOwner) {
+            // The heartbeat name contains the old token, so it cannot belong
+            // to a replacement lease created after the atomic rename.
+            await fs.unlink(heartbeatPathFor(lockPath, staleOwner.token)).catch(() => {});
+          }
+          return await openLock(lockPath, {
+            quietIfLocked,
+            beforeReleaseUnlink,
+            reclaimDepth,
+            serializeRelease,
+          });
+        } finally {
+          if (quarantinePath) await fs.unlink(quarantinePath).catch(() => {});
+          await reclaimGuard.release();
         }
-      } catch (_statErr) {
-        // Lock file disappeared between checks, retry
-        return openLock(lockPath, { quietIfLocked });
       }
       if (!quietIfLocked) {
         process.stdout.write("Another sync is already running.\n");

package/src/lib/local-api.js

@@ -35,6 +35,7 @@ const {
   getModelPricing,
   computeRowCost,
   ensurePricingLoaded,
+  getPricingRevision,
 } = require("./pricing");
 
 const {
@@ -66,6 +67,33 @@ const CLAUDE_MEM_OBSERVER_PROJECT_KEY = deriveProjectKeyFromRef(
   CLAUDE_MEM_OBSERVER_PROJECT_REF,
 );
 const PROJECT_USAGE_MAX_ENTRIES = 10;
+const MAX_TIME_ZONE_CACHE_ENTRIES = 16;
+
+// The native dashboard fans one refresh out across several local endpoints.
+// Keep one immutable, deduped view of the current queue so those endpoints do
+// not all read and JSON.parse the same append-only file. A file identity +
+// nanosecond timestamp signature invalidates the cache immediately on append,
+// in-place rewrite, or atomic replacement.
+let queueDataCache = null;
+const dailyAggregationCache = new WeakMap();
+const zonedPartsFormatters = new Map();
+
+function boundedCacheSet(cache, key, value, maxEntries) {
+  if (cache.has(key)) cache.delete(key);
+  cache.set(key, value);
+  while (cache.size > maxEntries) {
+    cache.delete(cache.keys().next().value);
+  }
+}
+
+function queueFileSignature(queuePath) {
+  const stat = fs.statSync(queuePath, { bigint: true });
+  return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
+}
+
+function timeZoneContextKey({ timeZone, offsetMinutes } = {}) {
+  return `${timeZone || ""}|${Number.isFinite(offsetMinutes) ? offsetMinutes : ""}|p${getPricingRevision()}`;
+}
 
 // Shared by project-usage-summary and project-usage-detail: reads the
 // deduped project bucket log, drops the claude-mem pseudo project, and
@@ -160,6 +188,21 @@ function normalizeQueueRow(row) {
 }
 
 function readQueueData(queuePath) {
+  let signature;
+  try {
+    signature = queueFileSignature(queuePath);
+  } catch (e) {
+    if (queueDataCache?.queuePath === queuePath) queueDataCache = null;
+    if (e?.code !== "ENOENT") {
+      console.error("[LocalAPI] readQueueData: failed to stat queue:", e?.message || e);
+    }
+    return [];
+  }
+
+  if (queueDataCache?.queuePath === queuePath && queueDataCache.signature === signature) {
+    return queueDataCache.rows;
+  }
+
   let raw;
   try {
     raw = fs.readFileSync(queuePath, "utf8");
@@ -196,7 +239,9 @@ function readQueueData(queuePath) {
     const key = `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}`;
     seen.set(key, normalizeQueueRow(row));
   }
-  return Array.from(seen.values());
+  const rows = Array.from(seen.values());
+  queueDataCache = { queuePath, signature, rows };
+  return rows;
 }
 
 function rowDayKey(row, timeZoneContext) {
@@ -214,8 +259,19 @@ function rowDayKey(row, timeZoneContext) {
 }
 
 function aggregateByDay(rows, timeZoneContext = null) {
+  const normalizedRows = Array.isArray(rows) ? rows : [];
+  const cacheKey = timeZoneContextKey(timeZoneContext || {});
+  let cachedByTimeZone = dailyAggregationCache.get(normalizedRows);
+  if (cachedByTimeZone?.has(cacheKey)) {
+    const cached = cachedByTimeZone.get(cacheKey);
+    // Touch the entry so the small per-queue map behaves as an LRU.
+    cachedByTimeZone.delete(cacheKey);
+    cachedByTimeZone.set(cacheKey, cached);
+    return cached;
+  }
+
   const byDay = new Map();
-  for (const row of rows) {
+  for (const row of normalizedRows) {
     if (!row.hour_start) continue;
     const day = rowDayKey(row, timeZoneContext);
     if (!day) continue;
@@ -250,7 +306,13 @@ function aggregateByDay(rows, timeZoneContext = null) {
     const model = row.model || "unknown";
     a.models[model] = (a.models[model] || 0) + (row.total_tokens || 0);
   }
-  return Array.from(byDay.values()).sort((a, b) => a.day.localeCompare(b.day));
+  const daily = Array.from(byDay.values()).sort((a, b) => a.day.localeCompare(b.day));
+  if (!cachedByTimeZone) {
+    cachedByTimeZone = new Map();
+    dailyAggregationCache.set(normalizedRows, cachedByTimeZone);
+  }
+  boundedCacheSet(cachedByTimeZone, cacheKey, daily, MAX_TIME_ZONE_CACHE_ENTRIES);
+  return daily;
 }
 
 function buildCodexCategoryFallbackFromQueue(queueRows, { from, to, timeZoneContext }) {
@@ -477,17 +539,34 @@ function getZonedParts(date, { timeZone, offsetMinutes } = {}) {
 
   if (timeZone && typeof Intl !== "undefined" && Intl.DateTimeFormat) {
     try {
-      const formatter = new Intl.DateTimeFormat("en-CA", {
-        timeZone,
-        year: "numeric",
-        month: "2-digit",
-        day: "2-digit",
-        hour: "2-digit",
-        minute: "2-digit",
-        second: "2-digit",
-        hourCycle: "h23",
-      });
-      const parts = formatter.formatToParts(dt);
+      let formatter;
+      if (zonedPartsFormatters.has(timeZone)) {
+        formatter = zonedPartsFormatters.get(timeZone);
+      } else {
+        try {
+          formatter = new Intl.DateTimeFormat("en-CA", {
+            timeZone,
+            year: "numeric",
+            month: "2-digit",
+            day: "2-digit",
+            hour: "2-digit",
+            minute: "2-digit",
+            second: "2-digit",
+            hourCycle: "h23",
+          });
+        } catch {
+          // Cache invalid zone ids too; otherwise a bad local query would
+          // throw once per queue row before falling back to its fixed offset.
+          formatter = null;
+        }
+        boundedCacheSet(
+          zonedPartsFormatters,
+          timeZone,
+          formatter,
+          MAX_TIME_ZONE_CACHE_ENTRIES,
+        );
+      }
+      const parts = formatter?.formatToParts(dt) || [];
       const values = parts.reduce((acc, part) => {
         if (part.type && part.value) acc[part.type] = part.value;
         return acc;
@@ -685,6 +764,7 @@ function runSyncCommand(extraEnv = {}, opts = {}) {
     const args = [TRACKER_BIN, "sync"];
     if (opts.auto === true) args.push("--auto");
     if (opts.background === true) args.push("--background");
+    if (opts.publishAccount === true) args.push("--publish-account");
     if (opts.allLocalSources === true) args.push("--all-local-sources");
     if (opts.drain === true) args.push("--drain");
     const child = spawn(process.execPath, args, {
@@ -1696,6 +1776,11 @@ function createLocalApiHandler({ queuePath }) {
         const auto = body.auto === true && !drain;
         const background =
           auto && (body.background === true || body.lightweight === true);
+        // The local server is the trust boundary for cloud-sync preferences.
+        // A persisted device token must not let a native background request
+        // upload after the user has explicitly disabled cloud sync.
+        const publishAccount =
+          background && body.publishAccount === true && getCloudSyncPref();
         const allLocalSources = background && body.allLocalSources === true;
         if (typeof body.deviceToken === "string" && body.deviceToken.trim()) {
           extraEnv.TOKENTRACKER_DEVICE_TOKEN = body.deviceToken.trim();
@@ -1710,7 +1795,10 @@ function createLocalApiHandler({ queuePath }) {
           extraEnv.TOKENTRACKER_INSFORGE_BASE_URL = allowedBaseUrl;
           localSyncBaseUrl = allowedBaseUrl;
         }
-        if (!background && !extraEnv.TOKENTRACKER_DEVICE_TOKEN && getCloudSyncPref() && getRefreshTokenForCloud()) {
+        if ((!background || publishAccount) &&
+            !extraEnv.TOKENTRACKER_DEVICE_TOKEN &&
+            getCloudSyncPref() &&
+            getRefreshTokenForCloud()) {
           let issuedToken = null;
           try {
             issuedToken = await issueDeviceTokenForLocalSync(qp, { baseUrl: localSyncBaseUrl });
@@ -1732,6 +1820,7 @@ function createLocalApiHandler({ queuePath }) {
           drain,
           auto,
           background,
+          publishAccount,
           allLocalSources,
         });
         try {
@@ -1778,7 +1867,8 @@ function createLocalApiHandler({ queuePath }) {
       const to = url.searchParams.get("to") || "";
       const timeZoneContext = getTimeZoneContext(url);
       const { rows, scope, excludedSources } = scopedQueueRows(qp, url);
-      const daily = aggregateByDay(rows, timeZoneContext).filter((d) => d.day >= from && d.day <= to);
+      const allDaily = aggregateByDay(rows, timeZoneContext);
+      const daily = allDaily.filter((d) => d.day >= from && d.day <= to);
       const totals = daily.reduce(
         (acc, r) => {
           acc.total_tokens += r.total_tokens;
@@ -1798,7 +1888,6 @@ function createLocalApiHandler({ queuePath }) {
 
       const todayParts = getZonedParts(new Date(), timeZoneContext);
       const todayStr = formatPartsDayKey(todayParts) || new Date().toISOString().slice(0, 10);
-      const allDaily = aggregateByDay(rows, timeZoneContext);
 
       const shiftDay = (dayStr, delta) => {
         const d = new Date(`${dayStr}T00:00:00Z`);

package/src/lib/pricing/index.js

@@ -37,6 +37,7 @@ const seedRaw = loadSeedSync();
 const state = {
   loaded: false,
   loadingPromise: null,
+  revision: 0,
   litellmRawMap: seedRaw, // raw per-token; field shape from LiteLLM JSON
   litellmPerMillionMap: buildLitellmPerMillionMap(seedRaw), // USD/MTok
   source: Object.keys(seedRaw).length ? "seed-snapshot:sync" : null,
@@ -61,6 +62,7 @@ async function ensurePricingLoaded(opts = {}) {
       state.litellmPerMillionMap = buildLitellmPerMillionMap(state.litellmRawMap);
       state.source = source;
       state.loaded = true;
+      state.revision += 1;
       state.negativeCache.clear();
       return state;
     } finally {
@@ -79,9 +81,14 @@ function resetPricingForTests() {
   state.litellmRawMap = seedRaw;
   state.litellmPerMillionMap = buildLitellmPerMillionMap(seedRaw);
   state.source = Object.keys(seedRaw).length ? "seed-snapshot:sync" : null;
+  state.revision += 1;
   state.negativeCache.clear();
 }
 
+function getPricingRevision() {
+  return state.revision;
+}
+
 function getModelPricing(model, opts = {}) {
   if (!model) return ZERO_PRICING;
   let lookupSource = null;
@@ -133,6 +140,7 @@ const MODEL_PRICING = curatedOverrides.exact;
 
 module.exports = {
   ensurePricingLoaded,
+  getPricingRevision,
   getModelPricing,
   computeRowCost,
   resetPricingForTests,