tokentracker-cli 1.5.14 → 1.5.15

package/src/lib/rollout.js

@@ -11641,26 +11641,14 @@ async function parseWorkbuddyIncremental({
     const startOffset = stat.size < prevSize || inodeChanged ? 0 : prevSize;
     if (stat.size <= startOffset) continue;
 
-    let stream;
-    try {
-      stream = fssync.createReadStream(filePath, {
-        encoding: "utf8",
-        start: startOffset,
-      });
-    } catch { continue; }
-    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
-
-    for await (const line of rl) {
-      if (!line || !line.trim()) continue;
-      let entry;
-      try { entry = JSON.parse(line); } catch { continue; }
-      if (!entry || typeof entry !== "object") continue;
+    const committedEndOffset = await scanOmpLikeTranscript(filePath, startOffset, OMP_LIKE_MAX_RECORD_BYTES, (entry) => {
+      if (!entry || typeof entry !== "object") return;
 
       // Usage is carried on ANY record with providerData.rawUsage — assistant
       // messages AND function_call records. Aggregate them all; dedup per id.
       const provider = entry.providerData;
       const rawUsage = provider && typeof provider === "object" ? provider.rawUsage : null;
-      if (!rawUsage || typeof rawUsage !== "object") continue;
+      if (!rawUsage || typeof rawUsage !== "object") return;
 
       const sessionId =
         typeof entry.sessionId === "string" && entry.sessionId
@@ -11668,10 +11656,10 @@ async function parseWorkbuddyIncremental({
           : path.basename(filePath, ".jsonl");
       const sqliteSession = sqliteSessions[sessionId];
       if (sqliteSession?.detailed || tracedSessionIds.has(sessionId)) {
-        continue;
+        return;
       }
       const tsMs =
-        Number.isFinite(Number(entry.timestamp)) && Number(entry.timestamp) > 0
+        Number.isFinite(new Date(Number(entry.timestamp)).getTime()) && Number(entry.timestamp) > 0
           ? Number(entry.timestamp)
           : null;
       // One usage record per LLM round-trip; providerData.messageId is the
@@ -11688,24 +11676,24 @@ async function parseWorkbuddyIncremental({
                 : tsMs != null
                   ? `${sessionId}:${tsMs}`
                   : null;
-      if (!messageId) continue;
-      if (seenIds.has(messageId)) continue;
+      if (!messageId) return;
+      if (seenIds.has(messageId)) return;
 
       recordsProcessed++;
 
       const delta = workbuddyUsageDelta(rawUsage);
       if (delta.total_tokens === 0) {
         seenIds.add(messageId);
-        continue;
+        return;
       }
 
       if (tsMs == null) {
         seenIds.add(messageId);
-        continue;
+        return;
       }
       const tsIso = new Date(tsMs).toISOString();
       const bucketStart = toUtcHalfHourStart(tsIso);
-      if (!bucketStart) continue;
+      if (!bucketStart) return;
 
       const model =
         normalizeModelInput(provider.model) ||
@@ -11735,14 +11723,14 @@ async function parseWorkbuddyIncremental({
           bucketsQueued: touchedBuckets.size,
         });
       }
-    }
+    }, { recoverReadErrors: true });
 
-    let postStat = stat;
-    try { postStat = fssync.statSync(filePath); } catch {}
+    // As in the project pass, acknowledge only records consumed in full. An
+    // unfinished tail or an append after EOF must remain visible next sync.
     fileOffsets[filePath] = {
-      size: postStat.size,
-      mtimeMs: postStat.mtimeMs,
-      ino: postStat.ino,
+      size: committedEndOffset,
+      mtimeMs: stat.mtimeMs,
+      ino: stat.ino,
     };
   }
 
@@ -11850,7 +11838,7 @@ async function parseWorkbuddyIncremental({
           if (sqliteSession?.detailed || tracedSessionIds.has(sessionId)) return;
 
           const tsMs =
-            Number.isFinite(Number(entry.timestamp)) && Number(entry.timestamp) > 0
+            Number.isFinite(new Date(Number(entry.timestamp)).getTime()) && Number(entry.timestamp) > 0
               ? Number(entry.timestamp)
               : null;
           if (tsMs == null) return;
@@ -11890,6 +11878,7 @@ async function parseWorkbuddyIncremental({
           addTotals(projectBucket.totals, delta);
           projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
         },
+        { recoverReadErrors: true },
       );
 
       // Commit only past records that were read in full: a record appended
@@ -16891,7 +16880,7 @@ function ompEntryTimestampMs(msg, entry) {
 // records are orders of magnitude smaller.
 const OMP_LIKE_MAX_RECORD_BYTES = 64 * 1024 * 1024;
 
-async function scanOmpLikeTranscript(filePath, startOffset, maxRecordBytes, onEntry) {
+async function scanOmpLikeTranscript(filePath, startOffset, maxRecordBytes, onEntry, { recoverReadErrors = false } = {}) {
   let stream;
   try {
     stream = fssync.createReadStream(filePath, { start: startOffset });
@@ -16900,26 +16889,33 @@ async function scanOmpLikeTranscript(filePath, startOffset, maxRecordBytes, onEn
   }
   let scannedEndOffset = startOffset;
   let committedEndOffset = startOffset;
-  for await (const record of physicalJsonlRecords(stream, {
-    invalidUtf8: "record",
-    maxRecordBytes,
-  })) {
-    scannedEndOffset += record.physicalBytes;
-    if (record.terminated) {
-      committedEndOffset = scannedEndOffset;
-      if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) continue;
+  try {
+    for await (const record of physicalJsonlRecords(stream, {
+      invalidUtf8: "record",
+      maxRecordBytes,
+    })) {
+      scannedEndOffset += record.physicalBytes;
+      if (record.terminated) {
+        committedEndOffset = scannedEndOffset;
+        if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) continue;
+        let entry;
+        try { entry = JSON.parse(record.line); } catch { continue; }
+        onEntry(entry);
+        continue;
+      }
+      // A final incomplete record may still be in the process of being written.
+      if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) break;
       let entry;
-      try { entry = JSON.parse(record.line); } catch { continue; }
+      try { entry = JSON.parse(record.line); } catch { break; }
       onEntry(entry);
-      continue;
+      committedEndOffset = scannedEndOffset;
+      break;
     }
-    // A final incomplete record may still be in the process of being written.
-    if (record.oversized || !record.utf8Valid || !record.line || !record.line.trim()) break;
-    let entry;
-    try { entry = JSON.parse(record.line); } catch { break; }
-    onEntry(entry);
-    committedEndOffset = scannedEndOffset;
-    break;
+  } catch (error) {
+    // WorkBuddy can publish complete records and continue with other files or
+    // fallbacks after a read failure. Unfinished bytes are never acknowledged.
+    // Do not swallow callback/parser errors, or change other readers' policy.
+    if (!recoverReadErrors || stream.errored !== error) throw error;
   }
   return committedEndOffset;
 }