tokentracker-cli 1.5.14 → 1.5.16

package/src/lib/qoder-limits.js

@@ -6,6 +6,7 @@ const net = require("node:net");
 const os = require("node:os");
 const path = require("node:path");
 const { fetchQoderCnCliLimits } = require("./qoder-cn-cli-limits");
+const { createLimitsCache } = require("./limits-cache");
 
 const QODER_SITES = {
   international: {
@@ -406,18 +407,40 @@ function writeQoderActivityCache(window, {
   } catch (_error) {}
 }
 
-function qoderLimitsCachePath({ home = os.homedir(), namespace } = {}) {
-  const file = namespace
+// The active region partitions the cache by filename (cn vs global); the
+// record itself stays account-agnostic. A window without a parseable reset is
+// kept only within the 12h unknown-reset TTL, and the global max-age only
+// bounds snapshots with no future-dated window — a dated survivor outlives it.
+// Provenance must describe the data's age, not the transport (#955): a fresh
+// hit re-serves the winning source it was written with, an older hit honestly
+// reports the disk cache.
+const qoderLimitsCache = createLimitsCache({
+  file: ({ namespace }) => (namespace
     ? `qoder-${namespace}-usage-limits-cache.json`
-    : QODER_LIMITS_CACHE_FILE;
-  return path.join(home, ".tokentracker", "tracker", file);
-}
+    : QODER_LIMITS_CACHE_FILE),
+  key: "qoder",
+  maxAgeMs: QODER_LIMITS_CACHE_MAX_AGE_MS,
+  maxAgeGate: "ifNoDatedSurvivor",
+  unknownResetTtlMs: QODER_LIMITS_CACHE_UNKNOWN_RESET_TTL_MS,
+  fields: {
+    plan_label: "string",
+    quota_exceeded: "boolTrue",
+    // Remember the winning source so a fresh re-serve keeps honest provenance
+    // instead of always claiming "disk-cache" (#955).
+    source: { write: (record) => (typeof record?.source === "string" ? record.source : null) },
+  },
+  windowKeys: ["primary_window", "secondary_window", "tertiary_window"],
+  // tertiary_window is written only when present and read back only when it
+  // survives — primary/secondary always appear (null when dead).
+  sparseWindowKeys: ["tertiary_window"],
+  stale: "byAge",
+  freshTtlMs: QODER_LIMITS_CACHE_FRESH_TTL_MS,
+  cachedFlag: true,
+  source: "provenance",
+});
 
-function qoderCachedWindow(window, { cachedAtMs, nowMs } = {}) {
-  if (!window || typeof window !== "object") return null;
-  const resetAtMs = Date.parse(window.reset_at || "");
-  if (Number.isFinite(resetAtMs)) return resetAtMs > nowMs ? window : null;
-  return nowMs - cachedAtMs <= QODER_LIMITS_CACHE_UNKNOWN_RESET_TTL_MS ? window : null;
+function qoderLimitsCachePath({ home = os.homedir(), namespace } = {}) {
+  return qoderLimitsCache.path({ home, namespace });
 }
 
 function readQoderLimitsCache({
@@ -425,44 +448,7 @@ function readQoderLimitsCache({
   nowMs = Date.now(),
   namespace,
 } = {}) {
-  try {
-    const raw = JSON.parse(fs.readFileSync(qoderLimitsCachePath({ home, namespace }), "utf8"))?.qoder;
-    const cachedAtMs = Date.parse(raw?.cached_at || "");
-    if (!Number.isFinite(cachedAtMs) || cachedAtMs > nowMs + 60_000) return null;
-    const primaryWindow = qoderCachedWindow(raw.primary_window, { cachedAtMs, nowMs });
-    const secondaryWindow = qoderCachedWindow(raw.secondary_window, { cachedAtMs, nowMs });
-    const tertiaryWindow = qoderCachedWindow(raw.tertiary_window, { cachedAtMs, nowMs });
-    if (!primaryWindow && !secondaryWindow && !tertiaryWindow) return null;
-    const hasUnexpiredDatedWindow = [primaryWindow, secondaryWindow, tertiaryWindow]
-      .some((window) => Number.isFinite(Date.parse(window?.reset_at || "")));
-    if (nowMs - cachedAtMs > QODER_LIMITS_CACHE_MAX_AGE_MS && !hasUnexpiredDatedWindow) {
-      return null;
-    }
-    // Provenance must describe the data's age, not the transport (#955): a
-    // successful refresh writes this cache and immediately floors later log
-    // scans, so the next refresh re-serves the same payload. Marking every
-    // hit stale/disk-cache made the Limits page permanently partial.
-    const ageMs = nowMs - cachedAtMs;
-    const fresh = ageMs <= QODER_LIMITS_CACHE_FRESH_TTL_MS;
-    const writtenSource = typeof raw.source === "string" && raw.source.trim()
-      ? raw.source.trim()
-      : null;
-    return {
-      configured: true,
-      error: null,
-      plan_label: typeof raw.plan_label === "string" ? raw.plan_label : null,
-      quota_exceeded: raw.quota_exceeded === true,
-      primary_window: primaryWindow,
-      secondary_window: secondaryWindow,
-      ...(tertiaryWindow ? { tertiary_window: tertiaryWindow } : {}),
-      cached: true,
-      stale: !fresh,
-      cached_at: raw.cached_at,
-      source: fresh && writtenSource ? writtenSource : "disk-cache",
-    };
-  } catch (_error) {
-    return null;
-  }
+  return qoderLimitsCache.read({ home, nowMs, namespace });
 }
 
 function writeQoderLimitsCache(limits, {
@@ -470,27 +456,7 @@ function writeQoderLimitsCache(limits, {
   nowMs = Date.now(),
   namespace,
 } = {}) {
-  if (!limits?.configured || limits.error || (!limits.primary_window && !limits.secondary_window && !limits.tertiary_window)) return;
-  const cachePath = qoderLimitsCachePath({ home, namespace });
-  const payload = {
-    qoder: {
-      plan_label: limits.plan_label || null,
-      quota_exceeded: limits.quota_exceeded === true,
-      primary_window: limits.primary_window || null,
-      secondary_window: limits.secondary_window || null,
-      ...(limits.tertiary_window ? { tertiary_window: limits.tertiary_window } : {}),
-      // Remember the winning source so a fresh re-serve keeps honest
-      // provenance instead of always claiming "disk-cache" (#955).
-      source: typeof limits.source === "string" ? limits.source : null,
-      cached_at: new Date(nowMs).toISOString(),
-    },
-  };
-  try {
-    fs.mkdirSync(path.dirname(cachePath), { recursive: true });
-    const tmpPath = `${cachePath}.${process.pid}.tmp`;
-    fs.writeFileSync(tmpPath, JSON.stringify(payload, null, 2), { encoding: "utf8", mode: 0o600 });
-    fs.renameSync(tmpPath, cachePath);
-  } catch (_error) {}
+  qoderLimitsCache.write(limits, { home, nowMs, namespace });
 }
 
 function buildQoderActivityHeaders(authStatus, {

package/src/lib/rollout.js

@@ -11641,26 +11641,14 @@ async function parseWorkbuddyIncremental({
     const startOffset = stat.size < prevSize || inodeChanged ? 0 : prevSize;
     if (stat.size <= startOffset) continue;
 
-    let stream;
-    try {
-      stream = fssync.createReadStream(filePath, {
-        encoding: "utf8",
-        start: startOffset,
-      });
-    } catch { continue; }
-    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
-
-    for await (const line of rl) {
-      if (!line || !line.trim()) continue;
-      let entry;
-      try { entry = JSON.parse(line); } catch { continue; }
-      if (!entry || typeof entry !== "object") continue;
+    const committedEndOffset = await scanOmpLikeTranscript(filePath, startOffset, OMP_LIKE_MAX_RECORD_BYTES, (entry) => {
+      if (!entry || typeof entry !== "object") return;
 
       // Usage is carried on ANY record with providerData.rawUsage — assistant
       // messages AND function_call records. Aggregate them all; dedup per id.
       const provider = entry.providerData;
       const rawUsage = provider && typeof provider === "object" ? provider.rawUsage : null;
-      if (!rawUsage || typeof rawUsage !== "object") continue;
+      if (!rawUsage || typeof rawUsage !== "object") return;
 
       const sessionId =
         typeof entry.sessionId === "string" && entry.sessionId
@@ -11668,10 +11656,10 @@ async function parseWorkbuddyIncremental({
           : path.basename(filePath, ".jsonl");
       const sqliteSession = sqliteSessions[sessionId];
       if (sqliteSession?.detailed || tracedSessionIds.has(sessionId)) {
-        continue;
+        return;
       }
       const tsMs =
-        Number.isFinite(Number(entry.timestamp)) && Number(entry.timestamp) > 0
+        Number.isFinite(new Date(Number(entry.timestamp)).getTime()) && Number(entry.timestamp) > 0
           ? Number(entry.timestamp)
           : null;
       // One usage record per LLM round-trip; providerData.messageId is the
@@ -11688,24 +11676,24 @@ async function parseWorkbuddyIncremental({
                 : tsMs != null
                   ? `${sessionId}:${tsMs}`
                   : null;
-      if (!messageId) continue;
-      if (seenIds.has(messageId)) continue;
+      if (!messageId) return;
+      if (seenIds.has(messageId)) return;
 
       recordsProcessed++;
 
       const delta = workbuddyUsageDelta(rawUsage);
       if (delta.total_tokens === 0) {
         seenIds.add(messageId);
-        continue;
+        return;
       }
 
       if (tsMs == null) {
         seenIds.add(messageId);
-        continue;
+        return;
       }
       const tsIso = new Date(tsMs).toISOString();
       const bucketStart = toUtcHalfHourStart(tsIso);
-      if (!bucketStart) continue;
+      if (!bucketStart) return;
 
       const model =
         normalizeModelInput(provider.model) ||
@@ -11735,14 +11723,14 @@ async function parseWorkbuddyIncremental({
           bucketsQueued: touchedBuckets.size,
         });
       }
-    }
+    }, { recoverReadErrors: true });
 
-    let postStat = stat;
-    try { postStat = fssync.statSync(filePath); } catch {}
+    // As in the project pass, acknowledge only records consumed in full. An
+    // unfinished tail or an append after EOF must remain visible next sync.
     fileOffsets[filePath] = {
-      size: postStat.size,
-      mtimeMs: postStat.mtimeMs,
-      ino: postStat.ino,
+      size: committedEndOffset,
+      mtimeMs: stat.mtimeMs,
+      ino: stat.ino,
     };
   }
 
@@ -11850,7 +11838,7 @@ async function parseWorkbuddyIncremental({
           if (sqliteSession?.detailed || tracedSessionIds.has(sessionId)) return;
 
           const tsMs =
-            Number.isFinite(Number(entry.timestamp)) && Number(entry.timestamp) > 0
+            Number.isFinite(new Date(Number(entry.timestamp)).getTime()) && Number(entry.timestamp) > 0
               ? Number(entry.timestamp)
               : null;
           if (tsMs == null) return;
@@ -11890,6 +11878,7 @@ async function parseWorkbuddyIncremental({
           addTotals(projectBucket.totals, delta);
           projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
         },
+        { recoverReadErrors: true },
       );
 
       // Commit only past records that were read in full: a record appended
@@ -16891,7 +16880,7 @@ function ompEntryTimestampMs(msg, entry) {
 // records are orders of magnitude smaller.
 const OMP_LIKE_MAX_RECORD_BYTES = 64 * 1024 * 1024;
 
-async function scanOmpLikeTranscript(filePath, startOffset, maxRecordBytes, onEntry) {
+async function scanOmpLikeTranscript(filePath, startOffset, maxRecordBytes, onEntry, { recoverReadErrors = false } = {}) {
   let stream;
   try {
     stream = fssync.createReadStream(filePath, { start: startOffset });
@@ -16900,26 +16889,33 @@ async function scanOmpLikeTranscript(filePath, startOffset, maxRecordBytes, onEn
   }
   let scannedEndOffset = startOffset;
   let committedEndOffset = startOffset;
-  for await (const record of physicalJsonlRecords(stream, {
-    invalidUtf8: "record",
-    maxRecordBytes,
-  })) {
-    scannedEndOffset += record.physicalBytes;
-    if (record.terminated) {
-      committedEndOffset = scannedEndOffset;
-      if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) continue;
+  try {
+    for await (const record of physicalJsonlRecords(stream, {
+      invalidUtf8: "record",
+      maxRecordBytes,
+    })) {
+      scannedEndOffset += record.physicalBytes;
+      if (record.terminated) {
+        committedEndOffset = scannedEndOffset;
+        if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) continue;
+        let entry;
+        try { entry = JSON.parse(record.line); } catch { continue; }
+        onEntry(entry);
+        continue;
+      }
+      // A final incomplete record may still be in the process of being written.
+      if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) break;
       let entry;
-      try { entry = JSON.parse(record.line); } catch { continue; }
+      try { entry = JSON.parse(record.line); } catch { break; }
       onEntry(entry);
-      continue;
+      committedEndOffset = scannedEndOffset;
+      break;
     }
-    // A final incomplete record may still be in the process of being written.
-    if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) break;
-    let entry;
-    try { entry = JSON.parse(record.line); } catch { break; }
-    onEntry(entry);
-    committedEndOffset = scannedEndOffset;
-    break;
+  } catch (error) {
+    // WorkBuddy can publish complete records and continue with other files or
+    // fallbacks after a read failure. Unfinished bytes are never acknowledged.
+    // Do not swallow callback/parser errors, or change other readers' policy.
+    if (!recoverReadErrors || stream.errored !== error) throw error;
   }
   return committedEndOffset;
 }