tokentracker-cli 0.12.0 → 0.13.0

package/src/lib/rollout.js

@@ -899,10 +899,8 @@ async function parseClaudeFile({
     if (!usage || typeof usage !== "object") continue;
 
     if (seenMessageHashes) {
-      const msgId = obj?.message?.id;
-      const reqId = obj?.requestId;
-      if (msgId && reqId) {
-        const hash = `${msgId}:${reqId}`;
+      const hash = claudeMessageDedupKey(obj);
+      if (hash) {
         if (seenMessageHashes.has(hash)) continue;
         seenMessageHashes.add(hash);
       }
@@ -2260,6 +2258,24 @@ function normalizeUsage(u) {
   return out;
 }
 
+// Stable dedup key for one Claude jsonl entry. Anthropic's official protocol
+// guarantees `message.id` is globally unique per response, so msgId alone is a
+// valid dedup key. Older code required both msgId AND requestId, which short-
+// circuited dedup entirely for jsonl entries where `requestId` is absent
+// (DeepSeek/Kimi/Mimo/MiniMax anthropic-compatible endpoints don't return the
+// `request-id` HTTP header, and Claude Code's sub-agent / thinking transport
+// paths drop the field too). The short-circuit caused 1.6–3.7x overcounting on
+// every affected provider — see issue #64. Falling back to msgId-only keeps
+// backward compatibility for the (msgId, reqId) format already persisted in
+// cursors.claudeHashes (msgId strings don't contain `:`, so the two formats
+// share the same Set without collision).
+function claudeMessageDedupKey(obj) {
+  const msgId = typeof obj?.message?.id === "string" && obj.message.id ? obj.message.id : null;
+  if (!msgId) return null;
+  const reqId = typeof obj?.requestId === "string" && obj.requestId ? obj.requestId : null;
+  return reqId ? `${msgId}:${reqId}` : msgId;
+}
+
 function normalizeClaudeUsage(u) {
   const inputTokens = toNonNegativeInt(u?.input_tokens);
   const outputTokens = toNonNegativeInt(u?.output_tokens);
@@ -2411,6 +2427,7 @@ async function parseOpencodeDbIncremental({
   projectQueuePath,
   onProgress,
   source,
+  cursorKey,
   publicRepoResolver,
 }) {
   await ensureDir(path.dirname(queuePath));
@@ -2426,7 +2443,8 @@ async function parseOpencodeDbIncremental({
   const projectTouchedBuckets = projectEnabled ? new Set() : null;
   const projectMetaCache = projectEnabled ? new Map() : null;
   const publicRepoCache = projectEnabled ? new Map() : null;
-  const opencodeState = normalizeOpencodeState(cursors?.opencode);
+  const cursorNamespace = typeof cursorKey === "string" && cursorKey.length > 0 ? cursorKey : "opencode";
+  const opencodeState = normalizeOpencodeState(cursors?.[cursorNamespace]);
   const messageIndex = opencodeState.messages;
   const touchedBuckets = new Set();
   const defaultSource = normalizeSourceInput(source) || "opencode";
@@ -2545,7 +2563,7 @@ async function parseOpencodeDbIncremental({
   hourlyState.updatedAt = new Date().toISOString();
   cursors.hourly = hourlyState;
   opencodeState.updatedAt = new Date().toISOString();
-  cursors.opencode = opencodeState;
+  cursors[cursorNamespace] = opencodeState;
   if (projectState) {
     projectState.updatedAt = new Date().toISOString();
     cursors.projectHourly = projectState;
@@ -4553,6 +4571,249 @@ function resolveOmpDefaultModel() {
   return "omp-unknown";
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Kilo Code VS Code extension — passive reader for VS Code-family
+// globalStorage/kilocode.kilo-code/tasks/<uuid>/ui_messages.json files.
+//
+// Each task folder contains a ui_messages.json (JSON array, not JSONL). Token
+// usage records are messages where `say == "api_req_started"`; the `text`
+// field is a JSON-stringified payload:
+//
+//   {
+//     "apiProtocol":    "openai" | "anthropic" | ...,
+//     "tokensIn":       28673,    // request input (already excludes cache)
+//     "tokensOut":      31,       // completion
+//     "cacheWrites":    0,
+//     "cacheReads":     5120,
+//     "cost":           0,
+//     "usageMissing":   false,
+//     "inferenceProvider": "Moonshot AI" | "minimax" | ...,
+//   }
+//
+// We scan every supported VS Code-family install (Cursor, Code, CodeBuddy,
+// Windsurf, …) under both Library/Application Support (macOS) and Linux/Win
+// equivalents. Files are small (median ~30KB) and rewritten on each turn — we
+// can't byte-tail them, so we read the whole file on every sync and dedupe by
+// (taskId, ts). Per-file mtime caching skips unchanged files.
+// ─────────────────────────────────────────────────────────────────────────────
+
+function resolveKilocodeRoots(env = process.env) {
+  if (typeof env.TOKENTRACKER_KILOCODE_ROOTS === "string" && env.TOKENTRACKER_KILOCODE_ROOTS.trim()) {
+    return env.TOKENTRACKER_KILOCODE_ROOTS.split(":")
+      .map((r) => r.trim())
+      .filter(Boolean);
+  }
+  const home = env.HOME || require("node:os").homedir();
+  const candidates = [];
+  if (process.platform === "darwin") {
+    const base = path.join(home, "Library", "Application Support");
+    candidates.push(
+      path.join(base, "Code"),
+      path.join(base, "Code - Insiders"),
+      path.join(base, "Cursor"),
+      path.join(base, "CodeBuddy"),
+      path.join(base, "Windsurf"),
+      path.join(base, "VSCodium"),
+      path.join(base, "Trae"),
+      path.join(base, "Trae CN"),
+    );
+  } else if (process.platform === "win32") {
+    const appData = env.APPDATA || path.join(home, "AppData", "Roaming");
+    candidates.push(
+      path.join(appData, "Code"),
+      path.join(appData, "Code - Insiders"),
+      path.join(appData, "Cursor"),
+      path.join(appData, "CodeBuddy"),
+      path.join(appData, "Windsurf"),
+      path.join(appData, "VSCodium"),
+    );
+  } else {
+    const xdg = env.XDG_CONFIG_HOME || path.join(home, ".config");
+    candidates.push(
+      path.join(xdg, "Code"),
+      path.join(xdg, "Code - Insiders"),
+      path.join(xdg, "Cursor"),
+      path.join(xdg, "CodeBuddy"),
+      path.join(xdg, "Windsurf"),
+      path.join(xdg, "VSCodium"),
+    );
+  }
+  return candidates;
+}
+
+function resolveKilocodeTaskFiles(env = process.env) {
+  const roots = resolveKilocodeRoots(env);
+  const out = [];
+  for (const root of roots) {
+    const tasksDir = path.join(root, "User", "globalStorage", "kilocode.kilo-code", "tasks");
+    if (!fssync.existsSync(tasksDir)) continue;
+    let entries;
+    try { entries = fssync.readdirSync(tasksDir); } catch { continue; }
+    for (const taskUuid of entries) {
+      const filePath = path.join(tasksDir, taskUuid, "ui_messages.json");
+      if (!fssync.existsSync(filePath)) continue;
+      out.push({ filePath, taskUuid, ide: path.basename(root) });
+    }
+  }
+  out.sort((a, b) => a.filePath.localeCompare(b.filePath));
+  return out;
+}
+
+// Kilo Code only persists the inference provider (e.g. "minimax",
+// "Moonshot AI", "Stealth") in ui_messages.json — the actual model id is
+// stored in workspace state but isn't attributed to individual turns and may
+// change across sessions, so we cannot map a row back to a model id reliably.
+// We surface the provider explicitly so the dashboard's Model column doesn't
+// imply this is a model.
+function normalizeKilocodeProviderToModel(providerName) {
+  if (typeof providerName !== "string" || !providerName.trim()) return "provider:unknown";
+  const slug = providerName
+    .trim()
+    .toLowerCase()
+    .replace(/\s+/g, "-")
+    .replace(/[^a-z0-9._-]/g, "");
+  // A slug consisting only of separators (dashes/dots/underscores) carries no
+  // information — treat it as unknown.
+  if (!slug || !/[a-z0-9]/.test(slug)) return "provider:unknown";
+  return `provider:${slug}`;
+}
+
+async function parseKilocodeIncremental({
+  taskFiles,
+  cursors,
+  queuePath,
+  onProgress,
+  env,
+} = {}) {
+  await ensureDir(path.dirname(queuePath));
+  const kilocodeState =
+    cursors.kilocode && typeof cursors.kilocode === "object" ? cursors.kilocode : {};
+  const seenIds = new Set(
+    Array.isArray(kilocodeState.seenIds) ? kilocodeState.seenIds : [],
+  );
+  const fileOffsets =
+    kilocodeState.fileOffsets && typeof kilocodeState.fileOffsets === "object"
+      ? { ...kilocodeState.fileOffsets }
+      : {};
+
+  const files = Array.isArray(taskFiles)
+    ? taskFiles
+    : resolveKilocodeTaskFiles(env || process.env);
+
+  if (files.length === 0) {
+    cursors.kilocode = {
+      ...kilocodeState,
+      seenIds: Array.from(seenIds),
+      fileOffsets,
+      updatedAt: new Date().toISOString(),
+    };
+    return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+  }
+
+  const hourlyState = normalizeHourlyState(cursors?.hourly);
+  const touchedBuckets = new Set();
+  const cb = typeof onProgress === "function" ? onProgress : null;
+  let recordsProcessed = 0;
+  let eventsAggregated = 0;
+
+  for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
+    const entry = files[fileIdx];
+    const { filePath, taskUuid } = entry;
+    let stat;
+    try { stat = fssync.statSync(filePath); } catch { continue; }
+
+    const prevEntry = fileOffsets[filePath];
+    if (
+      prevEntry &&
+      Number(prevEntry.size) === stat.size &&
+      Number(prevEntry.mtimeMs) === stat.mtimeMs
+    ) {
+      continue;
+    }
+
+    let raw;
+    try { raw = fssync.readFileSync(filePath, "utf8"); } catch { continue; }
+    let data;
+    try { data = JSON.parse(raw); } catch { continue; }
+    if (!Array.isArray(data)) continue;
+
+    for (const msg of data) {
+      if (!msg || typeof msg !== "object") continue;
+      // `api_req_started` is the live billing record; `api_req_deleted` keeps
+      // the same payload when a user removes a turn from the task (Cline-style
+      // edit-and-retry) — tokens were already consumed by the provider, so we
+      // still count them.
+      if (msg.say !== "api_req_started" && msg.say !== "api_req_deleted") continue;
+      if (typeof msg.text !== "string" || !msg.text.startsWith("{")) continue;
+
+      let payload;
+      try { payload = JSON.parse(msg.text); } catch { continue; }
+      if (!payload || typeof payload !== "object") continue;
+
+      const ts = Number(msg.ts);
+      if (!Number.isFinite(ts) || ts <= 0) continue;
+
+      const dedupKey = `${taskUuid}:${ts}`;
+      recordsProcessed++;
+      if (seenIds.has(dedupKey)) continue;
+
+      const tokensIn = toNonNegativeInt(payload.tokensIn);
+      const tokensOut = toNonNegativeInt(payload.tokensOut);
+      const cacheReads = toNonNegativeInt(payload.cacheReads);
+      const cacheWrites = toNonNegativeInt(payload.cacheWrites);
+      if (tokensIn === 0 && tokensOut === 0 && cacheReads === 0 && cacheWrites === 0) {
+        seenIds.add(dedupKey);
+        continue;
+      }
+
+      const tsIso = new Date(ts).toISOString();
+      const bucketStart = toUtcHalfHourStart(tsIso);
+      if (!bucketStart) continue;
+
+      const delta = {
+        input_tokens: tokensIn,
+        cached_input_tokens: cacheReads,
+        cache_creation_input_tokens: cacheWrites,
+        output_tokens: tokensOut,
+        reasoning_output_tokens: 0,
+        total_tokens: tokensIn + tokensOut + cacheReads + cacheWrites,
+        conversation_count: 1,
+      };
+
+      const model = normalizeKilocodeProviderToModel(payload.inferenceProvider);
+      const bucket = getHourlyBucket(hourlyState, "kilo-code", model, bucketStart);
+      addTotals(bucket.totals, delta);
+      touchedBuckets.add(bucketKey("kilo-code", model, bucketStart));
+      seenIds.add(dedupKey);
+      eventsAggregated++;
+    }
+
+    fileOffsets[filePath] = { size: stat.size, mtimeMs: stat.mtimeMs, ino: stat.ino };
+
+    if (cb) {
+      cb({
+        index: fileIdx + 1,
+        total: files.length,
+        recordsProcessed,
+        eventsAggregated,
+        bucketsQueued: touchedBuckets.size,
+      });
+    }
+  }
+
+  // Cap seenIds to last 50k to bound cursor state size
+  const seenArr = Array.from(seenIds);
+  const cappedSeen = seenArr.length > 50_000 ? seenArr.slice(seenArr.length - 50_000) : seenArr;
+
+  const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
+  const updatedAt = new Date().toISOString();
+  hourlyState.updatedAt = updatedAt;
+  cursors.hourly = hourlyState;
+  cursors.kilocode = { ...kilocodeState, seenIds: cappedSeen, fileOffsets, updatedAt };
+
+  return { recordsProcessed, eventsAggregated, bucketsQueued };
+}
+
 async function parseOmpIncremental({
   sessionFiles,
   cursors,
@@ -5526,6 +5787,10 @@ module.exports = {
   resolveOmpSessionFiles,
   resolveOmpDefaultModel,
   parseOmpIncremental,
+  resolveKilocodeRoots,
+  resolveKilocodeTaskFiles,
+  normalizeKilocodeProviderToModel,
+  parseKilocodeIncremental,
   resolvePiHome,
   resolvePiAgentDir,
   resolvePiSessionFiles,
@@ -5546,5 +5811,6 @@ module.exports = {
   // same key format sync uses elsewhere.
   bucketKey,
   totalsKey,
+  claudeMessageDedupKey,
   groupBucketKey,
 };