tokentracker-cli 1.3.11 → 1.3.14

package/src/commands/sync.js

@@ -145,6 +145,8 @@ const {
   toUtcHalfHourStart,
   totalsKey,
   claudeMessageDedupKey,
+  hasZcodeCursorState,
+  resolveZcodeInstallPaths,
 } = require("../lib/rollout");
 const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
 const { createProgress, renderBar, formatNumber, formatBytes } = require("../lib/progress");
@@ -1733,8 +1735,23 @@ async function cmdSync(argv, context = {}) {
       const wslZcodeDir = process.platform === "win32" && wsl.shouldProbeWsl(process.env)
         ? wsl.discoverWslHome(".zcode")
         : null;
-      const zcodePaths = resolveInstallPaths({ nativeValue: zcodeNativeValue, wslValue: wslZcodeDir ? path.join(wslZcodeDir, "cli", "db", "db.sqlite") : null });
+      const zcodeWslValue = wslZcodeDir ? path.join(wslZcodeDir, "cli", "db", "db.sqlite") : null;
+      const zcodePaths = resolveZcodeInstallPaths(
+        { nativeValue: zcodeNativeValue, wslValue: zcodeWslValue }, process.env,
+        { cursors },
+      );
       if (zcodePaths.native || zcodePaths.wsl) {
+        const mode = wsl.getWslMode(process.env);
+        if (process.platform === "win32" && ["native-first", "wsl-first"].includes(mode) &&
+            !cursors.zcodeInstallSelection && hasZcodeCursorState(cursors)) {
+          // Preserve legacy ownership before a repair can clear old buckets,
+          // even when the selected database currently has no rows to parse.
+          cursors.zcodeInstallSelection = {
+            mode, zcodeHome: process.env.ZCODE_HOME?.trim() || null,
+            dbPath: zcodePaths.native || zcodePaths.wsl,
+          };
+          cursorStore.requiresCommit = true;
+        }
         if (progress?.enabled) progress.start(`Parsing ZCode ${renderBar(0)} | buckets 0`);
         try {
           const nativeUsageAvailable = [zcodePaths.native, zcodePaths.wsl]
@@ -1761,6 +1778,16 @@ async function cmdSync(argv, context = {}) {
             cursors, getParams: (p) => ({ dbPath: p, readFn: readZcodeDbMessages, source: "zcode", cursorKey: "zcode" }),
             queuePath, projectQueuePath, onProgress: makeProviderProgress("ZCode"),
           });
+          if (zcodeResult.recordsProcessed > 0) {
+            if (process.platform === "win32" && ["native-first", "wsl-first"].includes(mode)) {
+              cursors.zcodeInstallSelection = {
+                mode, zcodeHome: process.env.ZCODE_HOME?.trim() || null,
+                dbPath: zcodePaths.native || zcodePaths.wsl,
+              };
+            } else {
+              delete cursors.zcodeInstallSelection;
+            }
+          }
         } catch (err) { warnProviderParseFailure("ZCode", err, opts); }
       }
     }

package/src/lib/async-queue-reader.js

@@ -0,0 +1,117 @@
+const fs = require("node:fs/promises");
+const { StringDecoder } = require("node:string_decoder");
+const { setImmediate: yieldToRequests } = require("node:timers/promises");
+
+const CHUNK_BYTES = 64 * 1024;
+const PROBE_BYTES = 256;
+
+function signature(stat) {
+  return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
+}
+
+// One in-flight read per path shares the dashboard's request fan-out. Parsing
+// yields between bounded chunks so a cold/replaced queue cannot monopolize the
+// HTTP event loop. Keep committed newline-terminated rows separate from a tail
+// that a concurrent writer may still be completing.
+function createAsyncQueueReader({ parseRow, label, maxEntries = 4 }) {
+  const cache = new Map();
+  const pending = new Map();
+
+  async function probe(file, position, length) {
+    const buffer = Buffer.alloc(length);
+    const { bytesRead } = await file.read(buffer, 0, length, position);
+    return buffer.subarray(0, bytesRead);
+  }
+
+  async function load(queuePath) {
+    const cached = cache.get(queuePath);
+    let file;
+    try {
+      file = await fs.open(queuePath, "r");
+      // Stat the opened file, rather than a path which can be atomically replaced
+      // between stat and open. Its signature always describes the bytes we read.
+      const stat = await file.stat({ bigint: true });
+      const nextSignature = signature(stat);
+      if (cached?.signature === nextSignature) return cached.rows;
+      const identity = `${stat.dev}:${stat.ino}`;
+      const size = Number(stat.size);
+      let offset = 0;
+      let seen = new Map();
+      if (cached && cached.identity === identity && size > cached.size && cached.consumedBytes > 0) {
+        const head = await probe(file, 0, cached.head.length);
+        const anchor = await probe(file, cached.consumedBytes - cached.anchor.length, cached.anchor.length);
+        if (head.equals(cached.head) && anchor.equals(cached.anchor)) {
+          offset = cached.consumedBytes;
+          seen = cached.seen;
+        }
+      }
+
+      const changes = new Map();
+      let malformed = 0;
+      function parse(line, target) {
+        if (!line.trim()) return;
+        try {
+          const entry = parseRow(JSON.parse(line));
+          if (entry) target.set(entry.key, entry.row);
+        } catch { malformed++; }
+      }
+
+      const decoder = new StringDecoder("utf8");
+      const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
+      let remainder = "";
+      let position = offset;
+      let consumedBytes = offset;
+      while (position < size) {
+        const length = Math.min(CHUNK_BYTES, size - position);
+        const { bytesRead } = await file.read(buffer, 0, length, position);
+        if (!bytesRead) throw new Error("Queue truncated during read");
+        const newline = buffer.lastIndexOf(0x0a, bytesRead - 1);
+        if (newline >= 0) consumedBytes = position + newline + 1;
+        position += bytesRead;
+        remainder += decoder.write(buffer.subarray(0, bytesRead));
+        let start = 0;
+        let end;
+        while ((end = remainder.indexOf("\n", start)) !== -1) {
+          parse(remainder.slice(start, end), changes);
+          start = end + 1;
+        }
+        remainder = remainder.slice(start);
+        await yieldToRequests();
+      }
+      remainder += decoder.end();
+      // Capture probes before committing any state: a failed read must not mutate
+      // a snapshot still being used by another endpoint.
+      const head = await probe(file, 0, Math.min(PROBE_BYTES, consumedBytes));
+      const anchor = await probe(file, Math.max(0, consumedBytes - PROBE_BYTES), Math.min(PROBE_BYTES, consumedBytes));
+      for (const [key, row] of changes) seen.set(key, row);
+      const tail = new Map();
+      parse(remainder, tail);
+      // Legacy files without a final newline are readable, but their provisional
+      // key never enters the committed map (completion may change that key).
+      const rows = tail.size ? Array.from(new Map([...seen, ...tail]).values()) : Array.from(seen.values());
+      cache.delete(queuePath);
+      cache.set(queuePath, { signature: nextSignature, identity, size, consumedBytes, head, anchor, seen, rows });
+      while (cache.size > maxEntries) cache.delete(cache.keys().next().value);
+      if (malformed) console.error(`[LocalAPI] ${label}: skipped ${malformed} malformed line(s)`);
+      return rows;
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        cache.delete(queuePath);
+        return [];
+      }
+      console.error(`[LocalAPI] ${label}: failed to read queue:`, error.message);
+      return cached?.rows || [];
+    } finally {
+      if (file) await file.close();
+    }
+  }
+
+  return function read(queuePath) {
+    if (pending.has(queuePath)) return pending.get(queuePath);
+    const task = load(queuePath).finally(() => pending.delete(queuePath));
+    pending.set(queuePath, task);
+    return task;
+  };
+}
+
+module.exports = { createAsyncQueueReader };

package/src/lib/jsonl-lines.js

@@ -5,6 +5,9 @@
  * only LF as a record separator so valid U+2028/U+2029 characters inside JSON
  * strings remain part of the record. Reading raw bytes keeps byte budgets exact
  * and lets malformed UTF-8 fail closed.
+ * Records longer than maxRecordBytes content bytes (excluding LF) are reported
+ * as { line: null, utf8Valid: null, oversized: true } without being buffered
+ * beyond the limit or decoded.
  */
 class PhysicalJsonlLimitError extends Error {
   constructor(maxPhysicalBytes) {
@@ -26,6 +29,7 @@ function decodePhysicalLine(buffer, { stripCr = false } = {}) {
 async function* physicalJsonlRecords(input, {
   invalidUtf8 = "throw",
   maxPhysicalBytes = Infinity,
+  maxRecordBytes = Infinity,
 } = {}) {
   if (invalidUtf8 !== "throw" && invalidUtf8 !== "record") {
     throw new TypeError(`unsupported invalidUtf8 policy: ${invalidUtf8}`);
@@ -33,8 +37,12 @@ async function* physicalJsonlRecords(input, {
   const byteLimit = Number.isFinite(maxPhysicalBytes)
     ? Math.max(0, Number(maxPhysicalBytes))
     : Infinity;
+  const recordLimit = Number.isFinite(maxRecordBytes)
+    ? Math.max(0, Number(maxRecordBytes))
+    : Infinity;
   const fragments = [];
   let fragmentsBytes = 0;
+  let oversized = false;
   let physicalBytesRead = 0;
 
   for await (const value of input) {
@@ -53,6 +61,17 @@ async function* physicalJsonlRecords(input, {
       if (physicalBytesRead + physicalBytes > byteLimit) {
         throw new PhysicalJsonlLimitError(byteLimit);
       }
+      const contentBytes = fragmentsBytes + fragment.length;
+      if (oversized || contentBytes > recordLimit) {
+        fragments.length = 0;
+        fragmentsBytes = 0;
+        oversized = false;
+        physicalBytesRead += physicalBytes;
+        start = newline + 1;
+        newline = chunk.indexOf(0x0a, start);
+        yield { line: null, utf8Valid: null, oversized: true, physicalBytes, terminated: true };
+        continue;
+      }
       const lineBuffer = fragments.length === 0
         ? fragment
         : Buffer.concat([...fragments, fragment], fragmentsBytes + fragment.length);
@@ -66,10 +85,10 @@ async function* physicalJsonlRecords(input, {
         line = decodePhysicalLine(lineBuffer, { stripCr: true });
       } catch (error) {
         if (invalidUtf8 === "throw") throw error;
-        yield { line: null, utf8Valid: false, physicalBytes, terminated: true };
+        yield { line: null, utf8Valid: false, oversized: false, physicalBytes, terminated: true };
         continue;
       }
-      yield { line, utf8Valid: true, physicalBytes, terminated: true };
+      yield { line, utf8Valid: true, oversized: false, physicalBytes, terminated: true };
     }
 
     if (start < chunk.length) {
@@ -77,12 +96,23 @@ async function* physicalJsonlRecords(input, {
       if (physicalBytesRead + fragmentsBytes + fragment.length > byteLimit) {
         throw new PhysicalJsonlLimitError(byteLimit);
       }
-      fragments.push(Buffer.from(fragment));
+      if (!oversized) {
+        if (fragmentsBytes + fragment.length > recordLimit) {
+          fragments.length = 0;
+          oversized = true;
+        } else {
+          fragments.push(Buffer.from(fragment));
+        }
+      }
       fragmentsBytes += fragment.length;
     }
   }
 
   if (fragmentsBytes > 0) {
+    if (oversized) {
+      yield { line: null, utf8Valid: null, oversized: true, physicalBytes: fragmentsBytes, terminated: false };
+      return;
+    }
     const lineBuffer = fragments.length === 1
       ? fragments[0]
       : Buffer.concat(fragments, fragmentsBytes);
@@ -94,7 +124,7 @@ async function* physicalJsonlRecords(input, {
       if (invalidUtf8 === "throw") throw error;
       utf8Valid = false;
     }
-    yield { line, utf8Valid, physicalBytes: fragmentsBytes, terminated: false };
+    yield { line, utf8Valid, oversized: false, physicalBytes: fragmentsBytes, terminated: false };
   }
 }