tokentracker-cli 0.83.1 → 0.83.3

package/src/lib/cursor-store.js

@@ -14,6 +14,7 @@ const STORE_VERSION = 2;
 const STORE_DIRNAME = "cursor-store-v2";
 const MANIFEST_FILENAME = "manifest.json";
 const GENERATIONS_DIRNAME = "generations";
+const DEFERRED_CODEX_AUDIT_FILENAME = "deferred-codex-audit.json";
 const DEFAULT_ACTIVATION_BYTES = 16 * 1024 * 1024;
 const PROJECT_ABSENT_CONTEXT_RESCAN_MS = 24 * 60 * 60 * 1000;
 const CURSOR_STORE_RETRY_CODE = "TOKENTRACKER_CURSOR_STORE_RETRY";
@@ -35,6 +36,7 @@ async function openCursorStore({
   const normalizedCodexRoots = normalizeCodexRoots(codexRoots);
   const legacyFingerprint = await fingerprintFile(cursorsPath);
   let manifest = await readJson(manifestPath);
+  let migratedDuringOpen = false;
 
   if (isManifest(manifest)) {
     const legacyChanged = legacyFingerprint
@@ -49,6 +51,7 @@ async function openCursorStore({
         codexRoots: normalizedCodexRoots,
         failureInjector,
       });
+      migratedDuringOpen = true;
     }
 
     const opened = await openManifestGeneration({
@@ -58,7 +61,10 @@ async function openCursorStore({
       codexRoots: normalizedCodexRoots,
       failureInjector,
     });
-    if (opened) return opened;
+    if (opened) {
+      if (migratedDuringOpen) opened.requiresCommit = true;
+      return opened;
+    }
 
     manifest = await migrateLegacyCursorState({
       cursorsPath,
@@ -68,6 +74,7 @@ async function openCursorStore({
       codexRoots: normalizedCodexRoots,
       failureInjector,
     });
+    migratedDuringOpen = true;
     const recovered = await openManifestGeneration({
       cursorsPath,
       storeRoot,
@@ -75,7 +82,10 @@ async function openCursorStore({
       codexRoots: normalizedCodexRoots,
       failureInjector,
     });
-    if (recovered) return recovered;
+    if (recovered) {
+      recovered.requiresCommit = true;
+      return recovered;
+    }
   }
 
   const legacySize = Number(legacyFingerprint?.size || 0);
@@ -97,6 +107,7 @@ async function openCursorStore({
     codexRoots: normalizedCodexRoots,
     failureInjector,
   });
+  migratedDuringOpen = true;
   const opened = await openManifestGeneration({
     cursorsPath,
     storeRoot,
@@ -105,6 +116,7 @@ async function openCursorStore({
     failureInjector,
   });
   if (!opened) throw new Error("Unable to open migrated cursor store");
+  if (migratedDuringOpen) opened.requiresCommit = true;
   return opened;
 }
 
@@ -180,6 +192,14 @@ class LegacyCursorStore {
     return false;
   }
 
+  async readDeferredCodexAuditSyncs() {
+    return 0;
+  }
+
+  async writeDeferredCodexAuditSyncs() {}
+
+  async clearDeferredCodexAuditSyncs() {}
+
   async commit(cursors = this.cursors) {
     await writeJson(this.cursorsPath, cursors);
   }
@@ -205,12 +225,17 @@ class V2CursorStore {
     this.codexRoots = codexRoots;
     this.cursors = generation.core;
     this.failureInjector = failureInjector;
+    this.deferredCodexAuditPath = path.join(
+      storeRoot,
+      DEFERRED_CODEX_AUDIT_FILENAME,
+    );
     this.loadedFileShards = new Map();
     this.loadedEventSets = new Map();
     this.pendingEventKeys = new Map();
     this.skipValidationCache = new Map();
     this.fileShardLoadCount = 0;
     this.materializedCodexHashes = false;
+    this.requiresCommit = generation.metadata.id !== manifest.current;
 
     if (!this.cursors.files || typeof this.cursors.files !== "object") {
       this.cursors.files = {};
@@ -377,6 +402,29 @@ class V2CursorStore {
     return valid;
   }
 
+  async readDeferredCodexAuditSyncs() {
+    const state = await readJson(this.deferredCodexAuditPath);
+    const count = Number(state?.syncsSinceFullScan || 0);
+    return Number.isSafeInteger(count) && count > 0 ? count : 0;
+  }
+
+  async writeDeferredCodexAuditSyncs(count) {
+    const normalized = Math.max(0, Math.floor(Number(count) || 0));
+    if (normalized === 0) {
+      await this.clearDeferredCodexAuditSyncs();
+      return;
+    }
+    await writeJson(this.deferredCodexAuditPath, {
+      version: 1,
+      syncsSinceFullScan: normalized,
+      updatedAt: new Date().toISOString(),
+    });
+  }
+
+  async clearDeferredCodexAuditSyncs() {
+    await fs.unlink(this.deferredCodexAuditPath).catch(() => {});
+  }
+
   hasCodexEvent(key) {
     const shardKey = codexEventShardKey(key);
     if (this.pendingEventKeys.get(shardKey)?.has(key)) return true;
@@ -464,6 +512,7 @@ class V2CursorStore {
     this.pendingEventKeys.clear();
     this.skipValidationCache.clear();
     this.materializedCodexHashes = false;
+    this.requiresCommit = true;
     return true;
   }
 
@@ -594,6 +643,7 @@ class V2CursorStore {
       this.pendingEventKeys.clear();
       this.skipValidationCache.clear();
       this.materializedCodexHashes = false;
+      this.requiresCommit = false;
       await cleanupGenerations(this.storeRoot, new Set([
         nextManifest.current,
         nextManifest.previous,

package/src/lib/local-api.js

@@ -118,7 +118,32 @@ function readProjectUsageContext(qp, url) {
   return { from, to, timeZoneContext, hasRange, dayInRange, projectRows };
 }
 
+// Same signature-based caching as readQueueData: the project-usage endpoints
+// all fan out from one dashboard refresh, so without this every request
+// re-read and re-parsed the whole append-only project queue.
+let projectQueueDataCache = null;
+
 function readProjectQueueData(projectQueuePath) {
+  let signature;
+  try {
+    signature = queueFileSignature(projectQueuePath);
+  } catch (e) {
+    if (projectQueueDataCache?.queuePath === projectQueuePath) {
+      projectQueueDataCache = null;
+    }
+    if (e?.code !== "ENOENT") {
+      console.error("[LocalAPI] readProjectQueueData: failed to stat:", e?.message || e);
+    }
+    return [];
+  }
+
+  if (
+    projectQueueDataCache?.queuePath === projectQueuePath &&
+    projectQueueDataCache.signature === signature
+  ) {
+    return projectQueueDataCache.rows;
+  }
+
   let raw;
   try {
     raw = fs.readFileSync(projectQueuePath, "utf8");
@@ -141,7 +166,11 @@ function readProjectQueueData(projectQueuePath) {
       // skip malformed
     }
   }
-  return Array.from(seen.values());
+  // Callers treat the result as read-only (sortByHour copies before sorting),
+  // so the cached array can be shared across requests.
+  const rows = Array.from(seen.values());
+  projectQueueDataCache = { queuePath: projectQueuePath, signature, rows };
+  return rows;
 }
 
 function isLegacyInclusiveCodexRow(row) {
@@ -188,9 +217,9 @@ function normalizeQueueRow(row) {
 }
 
 function readQueueData(queuePath) {
-  let signature;
+  let stat;
   try {
-    signature = queueFileSignature(queuePath);
+    stat = fs.statSync(queuePath, { bigint: true });
   } catch (e) {
     if (queueDataCache?.queuePath === queuePath) queueDataCache = null;
     if (e?.code !== "ENOENT") {
@@ -198,49 +227,119 @@ function readQueueData(queuePath) {
     }
     return [];
   }
+  const signature = `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
 
-  if (queueDataCache?.queuePath === queuePath && queueDataCache.signature === signature) {
-    return queueDataCache.rows;
+  const cached = queueDataCache;
+  if (cached?.queuePath === queuePath && cached.signature === signature) {
+    return cached.rows;
   }
 
-  let raw;
-  try {
-    raw = fs.readFileSync(queuePath, "utf8");
-  } catch (e) {
-    // ENOENT is legitimate (never synced yet); anything else is a signal we
-    // don't want to hide behind an empty array forever — the dashboard would
-    // otherwise render "0 tokens" with no clue the queue was unreadable.
-    if (e?.code !== "ENOENT") {
-      console.error("[LocalAPI] readQueueData: failed to read queue:", e?.message || e);
+  // The queue is append-only, so on a same-file append only the new tail is
+  // read and parsed; the deduped row set carries over. A replaced, truncated,
+  // or first-seen file falls back to a full read.
+  const devIno = `${stat.dev}:${stat.ino}`;
+  const fileSize = Number(stat.size);
+  const canAppend =
+    cached != null &&
+    cached.queuePath === queuePath &&
+    cached.devIno === devIno &&
+    fileSize >= cached.consumedBytes;
+  let seen = canAppend ? cached.seen : new Map();
+  let offset = canAppend ? cached.consumedBytes : 0;
+
+  // Integrity probe for the append assumption: repo writers only append or
+  // atomically replace (inode change), but an EXTERNAL in-place rewrite
+  // (cp over the file, shell redirect) keeps the inode with a same/larger
+  // size while changing the prefix. The byte before our offset must be the
+  // newline that terminated the last consumed line — anything else means the
+  // prefix is no longer ours, so fall back to a full read.
+  if (offset > 0) {
+    try {
+      const fd = fs.openSync(queuePath, "r");
+      try {
+        const probe = Buffer.allocUnsafe(1);
+        if (fs.readSync(fd, probe, 0, 1, offset - 1) !== 1 || probe[0] !== 0x0a) {
+          seen = new Map();
+          offset = 0;
+        }
+      } finally {
+        fs.closeSync(fd);
+      }
+    } catch {
+      seen = new Map();
+      offset = 0;
     }
-    return [];
   }
-  const lines = raw.split("\n").filter((l) => l.trim());
+
+  let raw = "";
+  if (fileSize > offset) {
+    try {
+      if (offset === 0) {
+        raw = fs.readFileSync(queuePath, "utf8");
+      } else {
+        const fd = fs.openSync(queuePath, "r");
+        try {
+          const length = fileSize - offset;
+          const buffer = Buffer.allocUnsafe(length);
+          let read = 0;
+          while (read < length) {
+            // A concurrent truncation between stat and open makes readSync hit
+            // EOF early — without the 0-byte break this loop never terminates.
+            const n = fs.readSync(fd, buffer, read, length - read, offset + read);
+            if (n === 0) break;
+            read += n;
+          }
+          raw = buffer.toString("utf8", 0, read);
+        } finally {
+          fs.closeSync(fd);
+        }
+      }
+    } catch (e) {
+      // ENOENT is legitimate (queue deleted between stat and read); anything
+      // else is a signal we don't want to hide behind an empty array forever —
+      // the dashboard would otherwise render "0 tokens" with no clue the queue
+      // was unreadable.
+      if (e?.code !== "ENOENT") {
+        console.error("[LocalAPI] readQueueData: failed to read queue:", e?.message || e);
+      }
+      return canAppend ? cached.rows : [];
+    }
+  }
+
   // Parse row-by-row so a single corrupted line (partial write, disk-full
-  // truncation, …) does not wipe out every other row with it.
-  const parsed = [];
+  // truncation, …) does not wipe out every other row with it. An unterminated
+  // tail is attempted (legacy writers may omit the final newline) but NOT
+  // marked consumed — a mid-append partial line is re-read once it completes.
+  // "\n" (0x0A) never appears inside a multi-byte UTF-8 sequence, so cutting
+  // on the last newline is byte-safe.
+  let consumedBytes = offset;
   let malformed = 0;
-  for (const line of lines) {
-    try {
-      parsed.push(JSON.parse(line));
-    } catch {
-      malformed += 1;
+  if (raw) {
+    const lastNewline = raw.lastIndexOf("\n");
+    if (lastNewline !== -1) {
+      consumedBytes += Buffer.byteLength(raw.slice(0, lastNewline), "utf8") + 1;
+    }
+    const lines = raw.split("\n");
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      try {
+        const row = JSON.parse(line);
+        // Deduplicate: each sync appends cumulative totals per bucket, so for
+        // each (source, model, hour_start) keep only the latest (last) entry.
+        const key = `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}`;
+        seen.set(key, normalizeQueueRow(row));
+      } catch {
+        malformed += 1;
+      }
     }
   }
   if (malformed > 0) {
     console.error(
-      `[LocalAPI] readQueueData: skipped ${malformed}/${lines.length} malformed line(s) in ${queuePath}`,
+      `[LocalAPI] readQueueData: skipped ${malformed} malformed line(s) in ${queuePath}`,
     );
   }
-  // Deduplicate: each sync appends cumulative totals per bucket, so for
-  // each (source, model, hour_start) keep only the latest (last) entry.
-  const seen = new Map();
-  for (const row of parsed) {
-    const key = `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}`;
-    seen.set(key, normalizeQueueRow(row));
-  }
   const rows = Array.from(seen.values());
-  queueDataCache = { queuePath, signature, rows };
+  queueDataCache = { queuePath, signature, devIno, consumedBytes, seen, rows };
   return rows;
 }
 
@@ -802,9 +901,18 @@ function runSyncCommand(extraEnv = {}, opts = {}) {
     });
     child.on("close", (code) => {
       const r = { code: code ?? 1, stdout: trimOutput(stdout), stderr: trimOutput(stderr) };
-      code === 0
-        ? finish(resolve, r)
-        : finish(reject, Object.assign(new Error(r.stderr || r.stdout || `exit ${r.code}`), r));
+      if (code === 0) {
+        finish(resolve, r);
+        return;
+      }
+      const error = Object.assign(
+        new Error(r.stderr || r.stdout || `exit ${r.code}`),
+        r,
+      );
+      if (/\bSYNC_BUSY\b/.test(`${r.stderr}\n${r.stdout}`)) {
+        error.code = "SYNC_BUSY";
+      }
+      finish(reject, error);
     });
   });
 }