tokentracker-cli 0.5.98 → 0.5.100

package/src/lib/proxy-env.js

@@ -0,0 +1,81 @@
+const cp = require("node:child_process");
+
+function hasProxyEnv(env = process.env) {
+  return Boolean(
+    env.HTTPS_PROXY ||
+    env.https_proxy ||
+    env.HTTP_PROXY ||
+    env.http_proxy ||
+    env.ALL_PROXY ||
+    env.all_proxy,
+  );
+}
+
+function parseMacProxyOutput(output) {
+  const values = {};
+  for (const line of String(output || "").split(/\r?\n/)) {
+    const match = line.match(/^\s*([A-Za-z]+)\s*:\s*(.+?)\s*$/);
+    if (match) values[match[1]] = match[2];
+  }
+  if (values.HTTPSEnable !== "1" || !values.HTTPSProxy || !values.HTTPSPort) return null;
+  return `http://${values.HTTPSProxy}:${values.HTTPSPort}`;
+}
+
+function resolveSystemProxyEnv({ env = process.env, platform = process.platform, commandRunner = cp.spawnSync } = {}) {
+  const out = {};
+  if (hasProxyEnv(env)) {
+    out.NODE_USE_ENV_PROXY = env.NODE_USE_ENV_PROXY || "1";
+    return out;
+  }
+
+  if (platform !== "darwin") return null;
+  const result = commandRunner("scutil", ["--proxy"], {
+    encoding: "utf8",
+    timeout: 2000,
+  });
+  if (result?.error || result?.status !== 0) return null;
+  const proxyUrl = parseMacProxyOutput(result.stdout);
+  if (!proxyUrl) return null;
+
+  return {
+    NODE_USE_ENV_PROXY: "1",
+    HTTPS_PROXY: proxyUrl,
+    HTTP_PROXY: proxyUrl,
+  };
+}
+
+function shouldRelaunchForProxy(argv, env = process.env) {
+  if (env.TOKENTRACKER_PROXY_ENV_APPLIED === "1") return false;
+  const command = Array.isArray(argv) ? argv[0] : null;
+  return !command || command === "serve";
+}
+
+function relaunchWithProxyEnvIfNeeded({
+  argv,
+  originalArgv,
+  env = process.env,
+  platform = process.platform,
+  commandRunner = cp.spawnSync,
+  nodePath = process.execPath,
+} = {}) {
+  if (!shouldRelaunchForProxy(argv, env)) return null;
+  const proxyEnv = resolveSystemProxyEnv({ env, platform, commandRunner });
+  if (!proxyEnv || proxyEnv.NODE_USE_ENV_PROXY === env.NODE_USE_ENV_PROXY) return null;
+
+  const childEnv = {
+    ...env,
+    ...proxyEnv,
+    TOKENTRACKER_PROXY_ENV_APPLIED: "1",
+  };
+  return commandRunner(nodePath, originalArgv, {
+    stdio: "inherit",
+    env: childEnv,
+  });
+}
+
+module.exports = {
+  hasProxyEnv,
+  parseMacProxyOutput,
+  resolveSystemProxyEnv,
+  relaunchWithProxyEnvIfNeeded,
+};

package/src/lib/rollout.js

@@ -1244,6 +1244,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
             output_tokens: zeroTotals.output_tokens,
             reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
             total_tokens: zeroTotals.total_tokens,
+            billable_total_tokens: zeroTotals.billable_total_tokens,
             conversation_count: zeroTotals.conversation_count,
           }),
         );
@@ -1265,6 +1266,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
               output_tokens: zeroTotals.output_tokens,
               reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
               total_tokens: zeroTotals.total_tokens,
+              billable_total_tokens: zeroTotals.billable_total_tokens,
               conversation_count: zeroTotals.conversation_count,
             }),
           );
@@ -1292,6 +1294,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
             output_tokens: totals.output_tokens,
             reasoning_output_tokens: totals.reasoning_output_tokens,
             total_tokens: totals.total_tokens,
+            billable_total_tokens: totals.billable_total_tokens ?? totals.total_tokens,
             conversation_count: totals.conversation_count,
           }),
         );
@@ -1318,6 +1321,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
           output_tokens: zeroTotals.output_tokens,
           reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
           total_tokens: zeroTotals.total_tokens,
+          billable_total_tokens: zeroTotals.billable_total_tokens,
           conversation_count: zeroTotals.conversation_count,
         }),
       );
@@ -1340,6 +1344,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
             output_tokens: zeroTotals.output_tokens,
             reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
             total_tokens: zeroTotals.total_tokens,
+            billable_total_tokens: zeroTotals.billable_total_tokens,
             conversation_count: zeroTotals.conversation_count,
           }),
         );
@@ -1361,6 +1366,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
         output_tokens: unknownBucket.totals.output_tokens,
         reasoning_output_tokens: unknownBucket.totals.reasoning_output_tokens,
         total_tokens: unknownBucket.totals.total_tokens,
+        billable_total_tokens: unknownBucket.totals.billable_total_tokens ?? unknownBucket.totals.total_tokens,
         conversation_count: unknownBucket.totals.conversation_count,
       }),
     );
@@ -1407,6 +1413,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
           output_tokens: group.totals.output_tokens,
           reasoning_output_tokens: group.totals.reasoning_output_tokens,
           total_tokens: group.totals.total_tokens,
+          billable_total_tokens: group.totals.billable_total_tokens ?? group.totals.total_tokens,
           conversation_count: group.totals.conversation_count,
         }),
       );
@@ -1461,6 +1468,7 @@ async function enqueueTouchedProjectBuckets({
         output_tokens: totals.output_tokens,
         reasoning_output_tokens: totals.reasoning_output_tokens,
         total_tokens: totals.total_tokens,
+        billable_total_tokens: totals.billable_total_tokens ?? totals.total_tokens,
         conversation_count: totals.conversation_count,
       }),
     );
@@ -1716,6 +1724,7 @@ function initTotals() {
     output_tokens: 0,
     reasoning_output_tokens: 0,
     total_tokens: 0,
+    billable_total_tokens: 0,
     conversation_count: 0,
   };
 }
@@ -1727,6 +1736,7 @@ function addTotals(target, delta) {
   target.output_tokens += delta.output_tokens || 0;
   target.reasoning_output_tokens += delta.reasoning_output_tokens || 0;
   target.total_tokens += delta.total_tokens || 0;
+  target.billable_total_tokens += delta.billable_total_tokens ?? delta.total_tokens ?? 0;
   target.conversation_count += delta.conversation_count || 0;
 }
 
@@ -1738,6 +1748,7 @@ function totalsKey(totals) {
     totals.output_tokens || 0,
     totals.reasoning_output_tokens || 0,
     totals.total_tokens || 0,
+    totals.billable_total_tokens ?? totals.total_tokens ?? 0,
     totals.conversation_count || 0,
   ].join("|");
 }
@@ -2056,7 +2067,9 @@ function normalizeGeminiTokens(tokens) {
   const output = toNonNegativeInt(tokens.output);
   const tool = toNonNegativeInt(tokens.tool);
   const thoughts = toNonNegativeInt(tokens.thoughts);
-  const total = toNonNegativeInt(tokens.total);
+  const reportedTotal = toNonNegativeInt(tokens.total);
+  const computedTotal = input + cached + output + tool + thoughts;
+  const total = Math.max(reportedTotal, computedTotal);
 
   return {
     input_tokens: input,
@@ -2150,26 +2163,18 @@ function pickDelta(lastUsage, totalUsage, prevTotals) {
   const hasTotal = isNonEmptyObject(totalUsage);
   const hasPrevTotals = isNonEmptyObject(prevTotals);
 
-  // NOTE: We used to guard against "duplicate token_count records where
-  // total_token_usage is unchanged" by returning null here. We removed that
-  // guard to align token counts with ccusage exactly (audited against 10 days
-  // of real rollouts). When last_token_usage is present we trust it as the
-  // per-turn delta; when it's absent the cumulative-subtract path naturally
-  // yields an all-zero delta on duplicates and is still filtered below.
-  if (!hasLast && hasTotal && hasPrevTotals && totalsReset(totalUsage, prevTotals)) {
-    const normalized = normalizeUsage(totalUsage);
-    return isAllZeroUsage(normalized) ? null : normalized;
-  }
-
-  if (hasLast) {
-    return normalizeUsage(lastUsage);
-  }
-
   if (hasTotal && hasPrevTotals) {
+    if (totalsReset(totalUsage, prevTotals)) {
+      const resetUsage = hasLast ? lastUsage : totalUsage;
+      const normalized = normalizeUsage(resetUsage);
+      return isAllZeroUsage(normalized) ? null : normalized;
+    }
+
     const delta = {};
     for (const k of [
       "input_tokens",
       "cached_input_tokens",
+      "cache_creation_input_tokens",
       "output_tokens",
       "reasoning_output_tokens",
       "total_tokens",
@@ -2182,6 +2187,11 @@ function pickDelta(lastUsage, totalUsage, prevTotals) {
     return isAllZeroUsage(normalized) ? null : normalized;
   }
 
+  if (hasLast) {
+    const normalized = normalizeUsage(lastUsage);
+    return isAllZeroUsage(normalized) ? null : normalized;
+  }
+
   if (hasTotal) {
     const normalized = normalizeUsage(totalUsage);
     return isAllZeroUsage(normalized) ? null : normalized;
@@ -2554,21 +2564,31 @@ async function parseCursorApiIncremental({
   const hourlyState = normalizeHourlyState(cursors?.hourly);
   const touchedBuckets = new Set();
 
-  // Incremental: skip records we already processed
+  // Cursor's CSV is an account-level API export, not an append-only local log.
+  // Treat the fetched CSV as authoritative so historical backfills and row
+  // corrections replace prior local bucket totals instead of being skipped.
   const lastTs = cursors?.cursorApi?.lastRecordTimestamp || null;
   let latestTs = lastTs;
   let eventsAggregated = 0;
   const cb = typeof onProgress === "function" ? onProgress : null;
   const total = records.length;
 
+  if (records.length > 0) {
+    for (const [key, bucket] of Object.entries(hourlyState.buckets || {})) {
+      const parsed = parseBucketKey(key);
+      const sourceKey = normalizeSourceInput(parsed.source) || DEFAULT_SOURCE;
+      if (sourceKey !== defaultSource) continue;
+      if (!bucket?.totals) continue;
+      bucket.totals = initTotals();
+      touchedBuckets.add(key);
+    }
+  }
+
   for (let i = 0; i < records.length; i++) {
     const record = records[i];
     const recordDate = record.date;
     if (!recordDate) continue;
 
-    // Skip records we already processed (CSV is ordered newest-first)
-    if (lastTs && recordDate <= lastTs) continue;
-
     const { normalizeCursorUsage } = require("./cursor-config");
     const delta = normalizeCursorUsage(record);
     if (isAllZeroUsage(delta)) continue;
@@ -4353,6 +4373,260 @@ async function parseCodebuddyIncremental({
   return { recordsProcessed, eventsAggregated, bucketsQueued };
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// oh-my-pi (omp) — passive JSONL reader (~/.omp/agent/sessions/**/*.jsonl)
+//
+// oh-my-pi writes one append-only JSONL per session:
+//   ~/.omp/agent/sessions/--<cwd-encoded>--/<timestamp>_<sessionId>.jsonl
+//
+// Per-line record types: the first line is type:"session" (header).
+// Only type:"message" lines with message.role=="assistant" carry token usage.
+// The shape (verbatim from oh-my-pi docs/session.md):
+//
+//   {
+//     "type": "message",
+//     "id": "a1b2c3d4",          ← 8-char dedup key
+//     "parentId": "...",
+//     "timestamp": "2026-02-16T10:21:00.000Z",
+//     "message": {
+//       "role": "assistant",
+//       "provider": "anthropic",
+//       "model": "claude-sonnet-4-5",
+//       "usage": {
+//         "input": 100, "output": 20, "cacheRead": 0, "cacheWrite": 0,
+//         "totalTokens": 120, "reasoningTokens": 0
+//       },
+//       "timestamp": 1760000000000   ← ms epoch, preferred for bucketing
+//     }
+//   }
+//
+// oh-my-pi is a router — dispatches to upstream providers (Anthropic, OpenAI,
+// etc.) and records the upstream model name per message. There is no global
+// default model setting; model is always per-message (fallback: "omp-unknown").
+// ─────────────────────────────────────────────────────────────────────────────
+
+function resolveOmpHome(env = process.env) {
+  const home = env.HOME || require("node:os").homedir();
+  // Honor TokenTracker override first, then oh-my-pi upstream env vars.
+  if (env.OMP_HOME) return env.OMP_HOME;
+  if (env.PI_CONFIG_DIR) return path.join(home, env.PI_CONFIG_DIR);
+  return path.join(home, ".omp");
+}
+
+function resolveOmpAgentDir(env = process.env) {
+  if (env.PI_CODING_AGENT_DIR) return env.PI_CODING_AGENT_DIR;
+  return path.join(resolveOmpHome(env), "agent");
+}
+
+function resolveOmpSessionFiles(env = process.env) {
+  const sessionsDir = path.join(resolveOmpAgentDir(env), "sessions");
+  if (!fssync.existsSync(sessionsDir)) return [];
+  const files = [];
+  try {
+    for (const cwdDir of fssync.readdirSync(sessionsDir)) {
+      const cwdPath = path.join(sessionsDir, cwdDir);
+      let stat;
+      try { stat = fssync.statSync(cwdPath); } catch { continue; }
+      if (!stat.isDirectory()) continue;
+      let entries;
+      try { entries = fssync.readdirSync(cwdPath); } catch { continue; }
+      for (const entry of entries) {
+        if (!entry.endsWith(".jsonl")) continue;
+        files.push(path.join(cwdPath, entry));
+      }
+    }
+  } catch {
+    // ignore — return what we have
+  }
+  files.sort((a, b) => a.localeCompare(b));
+  return files;
+}
+
+function resolveOmpDefaultModel() {
+  // oh-my-pi has no global default model setting; model is per-message.
+  return "omp-unknown";
+}
+
+async function parseOmpIncremental({
+  sessionFiles,
+  cursors,
+  queuePath,
+  onProgress,
+  env,
+  defaultModel,
+} = {}) {
+  await ensureDir(path.dirname(queuePath));
+  const ompState = cursors.omp && typeof cursors.omp === "object" ? cursors.omp : {};
+  const seenIds = new Set(Array.isArray(ompState.seenIds) ? ompState.seenIds : []);
+  const fileOffsets =
+    ompState.fileOffsets && typeof ompState.fileOffsets === "object"
+      ? { ...ompState.fileOffsets }
+      : {};
+
+  const files = Array.isArray(sessionFiles)
+    ? sessionFiles
+    : resolveOmpSessionFiles(env || process.env);
+  const fallbackModel = defaultModel || resolveOmpDefaultModel();
+
+  if (files.length === 0) {
+    cursors.omp = {
+      ...ompState,
+      seenIds: Array.from(seenIds),
+      fileOffsets,
+      updatedAt: new Date().toISOString(),
+    };
+    return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+  }
+
+  const hourlyState = normalizeHourlyState(cursors?.hourly);
+  const touchedBuckets = new Set();
+  const cb = typeof onProgress === "function" ? onProgress : null;
+  let recordsProcessed = 0;
+  let eventsAggregated = 0;
+
+  for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
+    const filePath = files[fileIdx];
+    let stat;
+    try { stat = fssync.statSync(filePath); } catch { continue; }
+
+    const prevEntry = fileOffsets[filePath] || {};
+    const prevSize = Number(prevEntry.size) || 0;
+    const prevIno = prevEntry.ino;
+    // Re-read from start if file shrunk (truncate/rewrite) or inode changed.
+    const inodeChanged = typeof prevIno === "number" && prevIno !== stat.ino;
+    const startOffset = stat.size < prevSize || inodeChanged ? 0 : prevSize;
+    if (stat.size <= startOffset) continue;
+
+    let stream;
+    try {
+      stream = fssync.createReadStream(filePath, {
+        encoding: "utf8",
+        start: startOffset,
+      });
+    } catch { continue; }
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+    for await (const line of rl) {
+      if (!line || !line.trim()) continue;
+      let entry;
+      try { entry = JSON.parse(line); } catch { continue; }
+
+      // First line of each file is type:"session" (header) — skip all
+      // non-message records.
+      if (!entry || entry.type !== "message") continue;
+
+      // Only assistant messages carry token usage.
+      const msg = entry.message;
+      if (!msg || msg.role !== "assistant") continue;
+
+      const usage = msg.usage;
+      if (!usage || typeof usage !== "object") continue;
+
+      // Dedup by top-level entry id (8-char string assigned by oh-my-pi).
+      const entryId = typeof entry.id === "string" && entry.id ? entry.id : null;
+      if (!entryId) continue;
+      if (seenIds.has(entryId)) continue;
+
+      recordsProcessed++;
+
+      const input = toNonNegativeInt(usage.input);
+      const output = toNonNegativeInt(usage.output);
+      const cacheRead = toNonNegativeInt(usage.cacheRead);
+      const cacheWrite = toNonNegativeInt(usage.cacheWrite);
+      const reasoningTokens = toNonNegativeInt(usage.reasoningTokens);
+
+      if (input === 0 && output === 0 && cacheRead === 0 && cacheWrite === 0) {
+        seenIds.add(entryId);
+        continue;
+      }
+
+      // Prefer message-level timestamp (ms epoch); fall back to entry-level
+      // ISO string. Entries with no resolvable timestamp are skipped — they
+      // cannot be placed in a bucket.
+      let tsMs = null;
+      if (Number.isFinite(Number(msg.timestamp)) && Number(msg.timestamp) > 0) {
+        tsMs = Number(msg.timestamp);
+      } else if (typeof entry.timestamp === "string" && entry.timestamp) {
+        const parsed = Date.parse(entry.timestamp);
+        if (Number.isFinite(parsed) && parsed > 0) tsMs = parsed;
+      }
+      if (tsMs == null) {
+        seenIds.add(entryId);
+        continue;
+      }
+
+      const tsIso = new Date(tsMs).toISOString();
+      const bucketStart = toUtcHalfHourStart(tsIso);
+      if (!bucketStart) continue;
+
+      // Use provided totalTokens when available; otherwise sum all components.
+      const totalTokens =
+        Number.isFinite(Number(usage.totalTokens)) && Number(usage.totalTokens) > 0
+          ? toNonNegativeInt(usage.totalTokens)
+          : input + output + cacheRead + cacheWrite + reasoningTokens;
+
+      const model = normalizeModelInput(msg.model) || fallbackModel;
+
+      const delta = {
+        input_tokens: input,
+        cached_input_tokens: cacheRead,
+        cache_creation_input_tokens: cacheWrite,
+        output_tokens: output,
+        reasoning_output_tokens: reasoningTokens,
+        total_tokens: totalTokens,
+        conversation_count: 1,
+      };
+
+      const bucket = getHourlyBucket(hourlyState, "omp", model, bucketStart);
+      addTotals(bucket.totals, delta);
+      touchedBuckets.add(bucketKey("omp", model, bucketStart));
+      seenIds.add(entryId);
+      eventsAggregated++;
+
+      if (cb) {
+        cb({
+          index: fileIdx + 1,
+          total: files.length,
+          recordsProcessed,
+          eventsAggregated,
+          bucketsQueued: touchedBuckets.size,
+        });
+      }
+    }
+
+    let postStat = stat;
+    try { postStat = fssync.statSync(filePath); } catch {}
+    fileOffsets[filePath] = {
+      size: postStat.size,
+      mtimeMs: postStat.mtimeMs,
+      ino: postStat.ino,
+    };
+  }
+
+  // Cap dedup set to last 10k IDs to bound cursor state size — same convention
+  // as Kimi/CodeBuddy/Copilot so cursors.json doesn't grow unbounded.
+  const seenArr = Array.from(seenIds);
+  const cappedSeen =
+    seenArr.length > 10_000 ? seenArr.slice(seenArr.length - 10_000) : seenArr;
+
+  const bucketsQueued = await enqueueTouchedBuckets({
+    queuePath,
+    hourlyState,
+    touchedBuckets,
+  });
+  const updatedAt = new Date().toISOString();
+  hourlyState.updatedAt = updatedAt;
+  cursors.hourly = hourlyState;
+  cursors.omp = {
+    ...ompState,
+    seenIds: cappedSeen,
+    fileOffsets,
+    updatedAt,
+  };
+
+  return { recordsProcessed, eventsAggregated, bucketsQueued };
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // GitHub Copilot CLI — OpenTelemetry JSONL exporter
 // User must opt in by setting:
@@ -4573,6 +4847,11 @@ module.exports = {
   resolveKiroCliSessionFiles,
   resolveKiroCliDbPath,
   parseKiroCliIncremental,
+  resolveOmpHome,
+  resolveOmpAgentDir,
+  resolveOmpSessionFiles,
+  resolveOmpDefaultModel,
+  parseOmpIncremental,
   // Exposed for regression tests covering cache-token accounting.
   normalizeGeminiTokens,
   normalizeOpencodeTokens,

package/src/lib/source-metadata.js

@@ -0,0 +1,46 @@
+const ACCOUNT_LEVEL_SOURCES = new Set(["cursor"]);
+
+function normalizeSource(value) {
+  return String(value || "").trim().toLowerCase();
+}
+
+function getSourceScope(source) {
+  return ACCOUNT_LEVEL_SOURCES.has(normalizeSource(source)) ? "account" : "local";
+}
+
+function isAccountLevelSource(source) {
+  return getSourceScope(source) === "account";
+}
+
+function normalizeUsageScope(value) {
+  const raw = String(value || "").trim().toLowerCase();
+  return raw === "all" || raw === "raw" ? "all" : "personal";
+}
+
+function filterRowsByUsageScope(rows, scope = "personal") {
+  const normalizedScope = normalizeUsageScope(scope);
+  if (normalizedScope === "all") return Array.isArray(rows) ? rows : [];
+  return (Array.isArray(rows) ? rows : []).filter((row) => !isAccountLevelSource(row?.source));
+}
+
+function listExcludedSources(rows, scope = "personal") {
+  const normalizedScope = normalizeUsageScope(scope);
+  if (normalizedScope === "all") return [];
+  const seen = new Set();
+  const out = [];
+  for (const row of Array.isArray(rows) ? rows : []) {
+    const source = normalizeSource(row?.source);
+    if (!source || seen.has(source) || !isAccountLevelSource(source)) continue;
+    seen.add(source);
+    out.push({ source, source_scope: getSourceScope(source), reason: "account_level_source" });
+  }
+  return out.sort((a, b) => a.source.localeCompare(b.source));
+}
+
+module.exports = {
+  getSourceScope,
+  isAccountLevelSource,
+  normalizeUsageScope,
+  filterRowsByUsageScope,
+  listExcludedSources,
+};