tokentracker-cli 0.12.1 → 0.14.0

package/src/lib/rollout.js

@@ -2427,6 +2427,7 @@ async function parseOpencodeDbIncremental({
   projectQueuePath,
   onProgress,
   source,
+  cursorKey,
   publicRepoResolver,
 }) {
   await ensureDir(path.dirname(queuePath));
@@ -2442,7 +2443,8 @@ async function parseOpencodeDbIncremental({
   const projectTouchedBuckets = projectEnabled ? new Set() : null;
   const projectMetaCache = projectEnabled ? new Map() : null;
   const publicRepoCache = projectEnabled ? new Map() : null;
-  const opencodeState = normalizeOpencodeState(cursors?.opencode);
+  const cursorNamespace = typeof cursorKey === "string" && cursorKey.length > 0 ? cursorKey : "opencode";
+  const opencodeState = normalizeOpencodeState(cursors?.[cursorNamespace]);
   const messageIndex = opencodeState.messages;
   const touchedBuckets = new Set();
   const defaultSource = normalizeSourceInput(source) || "opencode";
@@ -2561,7 +2563,7 @@ async function parseOpencodeDbIncremental({
   hourlyState.updatedAt = new Date().toISOString();
   cursors.hourly = hourlyState;
   opencodeState.updatedAt = new Date().toISOString();
-  cursors.opencode = opencodeState;
+  cursors[cursorNamespace] = opencodeState;
   if (projectState) {
     projectState.updatedAt = new Date().toISOString();
     cursors.projectHourly = projectState;
@@ -4569,6 +4571,249 @@ function resolveOmpDefaultModel() {
   return "omp-unknown";
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Kilo Code VS Code extension — passive reader for VS Code-family
+// globalStorage/kilocode.kilo-code/tasks/<uuid>/ui_messages.json files.
+//
+// Each task folder contains a ui_messages.json (JSON array, not JSONL). Token
+// usage records are messages where `say == "api_req_started"`; the `text`
+// field is a JSON-stringified payload:
+//
+//   {
+//     "apiProtocol":    "openai" | "anthropic" | ...,
+//     "tokensIn":       28673,    // request input (already excludes cache)
+//     "tokensOut":      31,       // completion
+//     "cacheWrites":    0,
+//     "cacheReads":     5120,
+//     "cost":           0,
+//     "usageMissing":   false,
+//     "inferenceProvider": "Moonshot AI" | "minimax" | ...,
+//   }
+//
+// We scan every supported VS Code-family install (Cursor, Code, CodeBuddy,
+// Windsurf, …) under both Library/Application Support (macOS) and Linux/Win
+// equivalents. Files are small (median ~30KB) and rewritten on each turn — we
+// can't byte-tail them, so we read the whole file on every sync and dedupe by
+// (taskId, ts). Per-file mtime caching skips unchanged files.
+// ─────────────────────────────────────────────────────────────────────────────
+
+function resolveKilocodeRoots(env = process.env) {
+  if (typeof env.TOKENTRACKER_KILOCODE_ROOTS === "string" && env.TOKENTRACKER_KILOCODE_ROOTS.trim()) {
+    return env.TOKENTRACKER_KILOCODE_ROOTS.split(":")
+      .map((r) => r.trim())
+      .filter(Boolean);
+  }
+  const home = env.HOME || require("node:os").homedir();
+  const candidates = [];
+  if (process.platform === "darwin") {
+    const base = path.join(home, "Library", "Application Support");
+    candidates.push(
+      path.join(base, "Code"),
+      path.join(base, "Code - Insiders"),
+      path.join(base, "Cursor"),
+      path.join(base, "CodeBuddy"),
+      path.join(base, "Windsurf"),
+      path.join(base, "VSCodium"),
+      path.join(base, "Trae"),
+      path.join(base, "Trae CN"),
+    );
+  } else if (process.platform === "win32") {
+    const appData = env.APPDATA || path.join(home, "AppData", "Roaming");
+    candidates.push(
+      path.join(appData, "Code"),
+      path.join(appData, "Code - Insiders"),
+      path.join(appData, "Cursor"),
+      path.join(appData, "CodeBuddy"),
+      path.join(appData, "Windsurf"),
+      path.join(appData, "VSCodium"),
+    );
+  } else {
+    const xdg = env.XDG_CONFIG_HOME || path.join(home, ".config");
+    candidates.push(
+      path.join(xdg, "Code"),
+      path.join(xdg, "Code - Insiders"),
+      path.join(xdg, "Cursor"),
+      path.join(xdg, "CodeBuddy"),
+      path.join(xdg, "Windsurf"),
+      path.join(xdg, "VSCodium"),
+    );
+  }
+  return candidates;
+}
+
+function resolveKilocodeTaskFiles(env = process.env) {
+  const roots = resolveKilocodeRoots(env);
+  const out = [];
+  for (const root of roots) {
+    const tasksDir = path.join(root, "User", "globalStorage", "kilocode.kilo-code", "tasks");
+    if (!fssync.existsSync(tasksDir)) continue;
+    let entries;
+    try { entries = fssync.readdirSync(tasksDir); } catch { continue; }
+    for (const taskUuid of entries) {
+      const filePath = path.join(tasksDir, taskUuid, "ui_messages.json");
+      if (!fssync.existsSync(filePath)) continue;
+      out.push({ filePath, taskUuid, ide: path.basename(root) });
+    }
+  }
+  out.sort((a, b) => a.filePath.localeCompare(b.filePath));
+  return out;
+}
+
+// Kilo Code only persists the inference provider (e.g. "minimax",
+// "Moonshot AI", "Stealth") in ui_messages.json — the actual model id is
+// stored in workspace state but isn't attributed to individual turns and may
+// change across sessions, so we cannot map a row back to a model id reliably.
+// We surface the provider explicitly so the dashboard's Model column doesn't
+// imply this is a model.
+function normalizeKilocodeProviderToModel(providerName) {
+  if (typeof providerName !== "string" || !providerName.trim()) return "provider:unknown";
+  const slug = providerName
+    .trim()
+    .toLowerCase()
+    .replace(/\s+/g, "-")
+    .replace(/[^a-z0-9._-]/g, "");
+  // A slug consisting only of separators (dashes/dots/underscores) carries no
+  // information — treat it as unknown.
+  if (!slug || !/[a-z0-9]/.test(slug)) return "provider:unknown";
+  return `provider:${slug}`;
+}
+
+async function parseKilocodeIncremental({
+  taskFiles,
+  cursors,
+  queuePath,
+  onProgress,
+  env,
+} = {}) {
+  await ensureDir(path.dirname(queuePath));
+  const kilocodeState =
+    cursors.kilocode && typeof cursors.kilocode === "object" ? cursors.kilocode : {};
+  const seenIds = new Set(
+    Array.isArray(kilocodeState.seenIds) ? kilocodeState.seenIds : [],
+  );
+  const fileOffsets =
+    kilocodeState.fileOffsets && typeof kilocodeState.fileOffsets === "object"
+      ? { ...kilocodeState.fileOffsets }
+      : {};
+
+  const files = Array.isArray(taskFiles)
+    ? taskFiles
+    : resolveKilocodeTaskFiles(env || process.env);
+
+  if (files.length === 0) {
+    cursors.kilocode = {
+      ...kilocodeState,
+      seenIds: Array.from(seenIds),
+      fileOffsets,
+      updatedAt: new Date().toISOString(),
+    };
+    return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+  }
+
+  const hourlyState = normalizeHourlyState(cursors?.hourly);
+  const touchedBuckets = new Set();
+  const cb = typeof onProgress === "function" ? onProgress : null;
+  let recordsProcessed = 0;
+  let eventsAggregated = 0;
+
+  for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
+    const entry = files[fileIdx];
+    const { filePath, taskUuid } = entry;
+    let stat;
+    try { stat = fssync.statSync(filePath); } catch { continue; }
+
+    const prevEntry = fileOffsets[filePath];
+    if (
+      prevEntry &&
+      Number(prevEntry.size) === stat.size &&
+      Number(prevEntry.mtimeMs) === stat.mtimeMs
+    ) {
+      continue;
+    }
+
+    let raw;
+    try { raw = fssync.readFileSync(filePath, "utf8"); } catch { continue; }
+    let data;
+    try { data = JSON.parse(raw); } catch { continue; }
+    if (!Array.isArray(data)) continue;
+
+    for (const msg of data) {
+      if (!msg || typeof msg !== "object") continue;
+      // `api_req_started` is the live billing record; `api_req_deleted` keeps
+      // the same payload when a user removes a turn from the task (Cline-style
+      // edit-and-retry) — tokens were already consumed by the provider, so we
+      // still count them.
+      if (msg.say !== "api_req_started" && msg.say !== "api_req_deleted") continue;
+      if (typeof msg.text !== "string" || !msg.text.startsWith("{")) continue;
+
+      let payload;
+      try { payload = JSON.parse(msg.text); } catch { continue; }
+      if (!payload || typeof payload !== "object") continue;
+
+      const ts = Number(msg.ts);
+      if (!Number.isFinite(ts) || ts <= 0) continue;
+
+      const dedupKey = `${taskUuid}:${ts}`;
+      recordsProcessed++;
+      if (seenIds.has(dedupKey)) continue;
+
+      const tokensIn = toNonNegativeInt(payload.tokensIn);
+      const tokensOut = toNonNegativeInt(payload.tokensOut);
+      const cacheReads = toNonNegativeInt(payload.cacheReads);
+      const cacheWrites = toNonNegativeInt(payload.cacheWrites);
+      if (tokensIn === 0 && tokensOut === 0 && cacheReads === 0 && cacheWrites === 0) {
+        seenIds.add(dedupKey);
+        continue;
+      }
+
+      const tsIso = new Date(ts).toISOString();
+      const bucketStart = toUtcHalfHourStart(tsIso);
+      if (!bucketStart) continue;
+
+      const delta = {
+        input_tokens: tokensIn,
+        cached_input_tokens: cacheReads,
+        cache_creation_input_tokens: cacheWrites,
+        output_tokens: tokensOut,
+        reasoning_output_tokens: 0,
+        total_tokens: tokensIn + tokensOut + cacheReads + cacheWrites,
+        conversation_count: 1,
+      };
+
+      const model = normalizeKilocodeProviderToModel(payload.inferenceProvider);
+      const bucket = getHourlyBucket(hourlyState, "kilo-code", model, bucketStart);
+      addTotals(bucket.totals, delta);
+      touchedBuckets.add(bucketKey("kilo-code", model, bucketStart));
+      seenIds.add(dedupKey);
+      eventsAggregated++;
+    }
+
+    fileOffsets[filePath] = { size: stat.size, mtimeMs: stat.mtimeMs, ino: stat.ino };
+
+    if (cb) {
+      cb({
+        index: fileIdx + 1,
+        total: files.length,
+        recordsProcessed,
+        eventsAggregated,
+        bucketsQueued: touchedBuckets.size,
+      });
+    }
+  }
+
+  // Cap seenIds to last 50k to bound cursor state size
+  const seenArr = Array.from(seenIds);
+  const cappedSeen = seenArr.length > 50_000 ? seenArr.slice(seenArr.length - 50_000) : seenArr;
+
+  const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
+  const updatedAt = new Date().toISOString();
+  hourlyState.updatedAt = updatedAt;
+  cursors.hourly = hourlyState;
+  cursors.kilocode = { ...kilocodeState, seenIds: cappedSeen, fileOffsets, updatedAt };
+
+  return { recordsProcessed, eventsAggregated, bucketsQueued };
+}
+
 async function parseOmpIncremental({
   sessionFiles,
   cursors,
@@ -5327,7 +5572,7 @@ async function parseCraftIncremental({
 // ─────────────────────────────────────────────────────────────────────────────
 
 function resolveCopilotOtelPaths(env = process.env) {
-  const home = require("node:os").homedir();
+  const home = env.HOME || require("node:os").homedir();
   const paths = new Set();
   const defaultDir = path.join(home, ".copilot", "otel");
   if (fssync.existsSync(defaultDir)) {
@@ -5345,10 +5590,17 @@ function resolveCopilotOtelPaths(env = process.env) {
 }
 
 function isCopilotChatSpan(record) {
-  if (!record || record.type !== "span") return false;
+  if (!record || typeof record !== "object") return false;
+  // Skip metric records (resource + scopeMetrics) which have no chat usage data
+  if (record.scopeMetrics) return false;
   const opName = record?.attributes?.["gen_ai.operation.name"];
+  // Both Copilot CLI (Span shape with type:"span") and Copilot Chat extension
+  // (OTEL JS SDK LogRecord shape with event.name:"gen_ai.client.inference.operation.details")
+  // mark chat completions with gen_ai.operation.name === "chat".
   if (opName === "chat") return true;
-  if (typeof record.name === "string" && record.name.startsWith("chat ")) return true;
+  if (record.type === "span" && typeof record.name === "string" && record.name.startsWith("chat ")) {
+    return true;
+  }
   return false;
 }
 
@@ -5369,20 +5621,116 @@ function pickCopilotModel(attrs) {
   return null;
 }
 
+const COPILOT_PARSER_VERSION = 2;
+
+function isCopilotV1ChatSpan(record) {
+  if (!record || record.type !== "span") return false;
+  const opName = record?.attributes?.["gen_ai.operation.name"];
+  if (opName === "chat") return true;
+  return typeof record.name === "string" && record.name.startsWith("chat ");
+}
+
+function copilotLineHash(line) {
+  return crypto.createHash("sha256").update(line).digest("hex");
+}
+
+function incrementMapCount(map, key, amount = 1) {
+  map.set(key, (map.get(key) || 0) + amount);
+}
+
+function getCopilotDedupKey(record, attrs = record?.attributes || {}) {
+  const traceId = record?.traceId || record?.spanContext?.traceId || "";
+  const spanId = record?.spanId || record?.spanContext?.spanId || "";
+  const responseId =
+    typeof attrs["gen_ai.response.id"] === "string" ? attrs["gen_ai.response.id"] : "";
+  return traceId && spanId ? `${traceId}:${spanId}` : responseId ? `resp:${responseId}` : null;
+}
+
+// Migration helper: stream the bytes v1 already saw (0 -> prevSize), classify
+// whether the file contains old CLI spans v1 processed, and whether it also
+// contains v2-only chat records v1 skipped. Mixed files must be replayed, but
+// their old CLI lines are skipped by hash so history does not double-count.
+async function scanCopilotV1MigrationFile(filePath, maxBytes) {
+  const result = {
+    v1Processed: false,
+    v2OnlyChat: false,
+    v1LineHashes: new Map(),
+  };
+  if (!maxBytes || maxBytes <= 0) return result;
+  try {
+    const stream = fssync.createReadStream(filePath, {
+      encoding: "utf8",
+      start: 0,
+      end: maxBytes - 1,
+    });
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    try {
+      for await (const line of rl) {
+        if (!line || !line.trim()) continue;
+        let record;
+        try {
+          record = JSON.parse(line);
+        } catch (_e) {
+          continue;
+        }
+        // Must mirror v1's isCopilotChatSpan exactly: BOTH the
+        // gen_ai.operation.name path AND the legacy name-prefix fallback.
+        // Missing the second path lets metric-free files of name-only CLI spans
+        // look like "v1 skipped" -> offset reset -> re-read -> double-count.
+        if (isCopilotV1ChatSpan(record)) {
+          result.v1Processed = true;
+          incrementMapCount(result.v1LineHashes, copilotLineHash(line));
+        } else if (isCopilotChatSpan(record)) {
+          result.v2OnlyChat = true;
+        }
+      }
+    } finally {
+      rl.close();
+      stream.destroy();
+    }
+  } catch (_e) {}
+  return result;
+}
+
 async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgress, env } = {}) {
   await ensureDir(path.dirname(queuePath));
   const copilotState = cursors.copilot && typeof cursors.copilot === "object" ? cursors.copilot : {};
   const seenIds = new Set(Array.isArray(copilotState.seenIds) ? copilotState.seenIds : []);
-  const fileOffsets =
+  const priorVersion = Number(copilotState.version) || 1;
+  const fileOffsetsRaw =
     copilotState.fileOffsets && typeof copilotState.fileOffsets === "object"
-      ? { ...copilotState.fileOffsets }
+      ? copilotState.fileOffsets
       : {};
+  const fileOffsets = { ...fileOffsetsRaw };
+  const migrationSkipLineHashes = new Map();
+  // One-shot v1->v2 migration:
+  // - pure v2-only files: clear offset and re-read all skipped Chat records
+  // - pure v1 CLI files: preserve offset to avoid replaying history beyond seenIds
+  // - mixed files: clear offset, but skip old v1 CLI lines by hash during replay
+  if (priorVersion < COPILOT_PARSER_VERSION) {
+    for (const filePath of Object.keys(fileOffsets)) {
+      const prevSize = Number(fileOffsets[filePath]?.size) || 0;
+      const scan = await scanCopilotV1MigrationFile(filePath, prevSize);
+      if (!scan.v1Processed) {
+        delete fileOffsets[filePath];
+      } else if (scan.v2OnlyChat) {
+        delete fileOffsets[filePath];
+        migrationSkipLineHashes.set(filePath, scan.v1LineHashes);
+      }
+    }
+  }
 
   const files = Array.isArray(otelPaths) && otelPaths.length > 0
     ? otelPaths
     : resolveCopilotOtelPaths(env || process.env);
   if (files.length === 0) {
-    cursors.copilot = { ...copilotState, seenIds: Array.from(seenIds), fileOffsets, updatedAt: new Date().toISOString() };
+    cursors.copilot = {
+      ...copilotState,
+      version: COPILOT_PARSER_VERSION,
+      seenIds: Array.from(seenIds),
+      fileOffsets,
+      updatedAt: new Date().toISOString(),
+    };
     return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
   }
 
@@ -5421,6 +5769,16 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
 
     for await (const line of rl) {
       if (!line || !line.trim()) continue;
+      const skipLineHashes = migrationSkipLineHashes.get(filePath);
+      if (skipLineHashes && skipLineHashes.size > 0) {
+        const lineHash = copilotLineHash(line);
+        const skipCount = skipLineHashes.get(lineHash) || 0;
+        if (skipCount > 0) {
+          if (skipCount === 1) skipLineHashes.delete(lineHash);
+          else skipLineHashes.set(lineHash, skipCount - 1);
+          continue;
+        }
+      }
       let record;
       try {
         record = JSON.parse(line);
@@ -5430,25 +5788,37 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
       recordsProcessed++;
       if (!isCopilotChatSpan(record)) continue;
 
-      const traceId = record?.traceId || "";
-      const spanId = record?.spanId || "";
-      const dedupKey = traceId && spanId ? `${traceId}:${spanId}` : null;
+      const attrs = record.attributes || {};
+      // Dedup: CLI puts traceId/spanId at the top level; the Chat extension
+      // file exporter writes LogRecord-shaped entries without either, but
+      // gen_ai.response.id is per-LLM-call unique.
+      const dedupKey = getCopilotDedupKey(record, attrs);
       if (dedupKey && seenIds.has(dedupKey)) continue;
 
-      const attrs = record.attributes || {};
       const inputRaw = toNonNegativeInt(attrs["gen_ai.usage.input_tokens"]);
       const output = toNonNegativeInt(attrs["gen_ai.usage.output_tokens"]);
       const cacheRead = toNonNegativeInt(attrs["gen_ai.usage.cache_read.input_tokens"]);
-      const cacheWrite = toNonNegativeInt(attrs["gen_ai.usage.cache_write.input_tokens"]);
-      const reasoning = toNonNegativeInt(attrs["gen_ai.usage.reasoning.output_tokens"]);
+      // Copilot CLI: cache_write.input_tokens; Copilot Chat extension: cache_creation.input_tokens
+      const cacheWrite = toNonNegativeInt(
+        attrs["gen_ai.usage.cache_write.input_tokens"] ??
+          attrs["gen_ai.usage.cache_creation.input_tokens"],
+      );
+      // Copilot CLI: reasoning.output_tokens; Copilot Chat extension: reasoning_tokens
+      const reasoning = toNonNegativeInt(
+        attrs["gen_ai.usage.reasoning.output_tokens"] ?? attrs["gen_ai.usage.reasoning_tokens"],
+      );
       // OTEL input_tokens INCLUDES cache_read — subtract per project convention
       const cacheReadClamped = Math.min(cacheRead, inputRaw);
       const input = Math.max(0, inputRaw - cacheReadClamped);
       const totalInteresting = input + output + cacheReadClamped + cacheWrite + reasoning;
-      // Drop empty rows unless cache-only
       if (totalInteresting === 0) continue;
 
-      const tsMs = copilotOtelTimeToMs(record.endTime) || copilotOtelTimeToMs(record.startTime);
+      // CLI Span uses endTime/startTime; Chat extension LogRecord uses hrTime/hrTimeObserved.
+      const tsMs =
+        copilotOtelTimeToMs(record.endTime) ||
+        copilotOtelTimeToMs(record.startTime) ||
+        copilotOtelTimeToMs(record.hrTime) ||
+        copilotOtelTimeToMs(record.hrTimeObserved);
       if (!tsMs) continue;
       const tsIso = new Date(tsMs).toISOString();
       const bucketStart = toUtcHalfHourStart(tsIso);
@@ -5502,7 +5872,13 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
   const updatedAt = new Date().toISOString();
   hourlyState.updatedAt = updatedAt;
   cursors.hourly = hourlyState;
-  cursors.copilot = { ...copilotState, seenIds: cappedSeen, fileOffsets, updatedAt };
+  cursors.copilot = {
+    ...copilotState,
+    version: COPILOT_PARSER_VERSION,
+    seenIds: cappedSeen,
+    fileOffsets,
+    updatedAt,
+  };
 
   return { recordsProcessed, eventsAggregated, bucketsQueued };
 }
@@ -5542,6 +5918,10 @@ module.exports = {
   resolveOmpSessionFiles,
   resolveOmpDefaultModel,
   parseOmpIncremental,
+  resolveKilocodeRoots,
+  resolveKilocodeTaskFiles,
+  normalizeKilocodeProviderToModel,
+  parseKilocodeIncremental,
   resolvePiHome,
   resolvePiAgentDir,
   resolvePiSessionFiles,