tokentracker-cli 1.5.6 → 1.5.8

package/src/lib/rollout.js

@@ -1,3 +1,4 @@
+const { preserveLegacyClaudeGroups, preserveLegacyClaudePrefix, createBackendResolver, mergeBackendUsage, claudeLines } = require("./claude-backend");
 const { claudeUserIdentity, collectClaudeForkAliases } = require("./claude-user-identity");
 const fs = require("node:fs/promises");
 const fssync = require("node:fs");
@@ -352,6 +353,7 @@ async function parseRolloutIncremental({
   const projectFreshnessNowMs = Date.now();
   const touchedBuckets = new Set();
   const defaultSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
+  preserveCodexModelBuckets(hourlyState, touchedBuckets);
   const syncDiagnostics = diagnostics && typeof diagnostics === "object" ? diagnostics : null;
 
   if (!cursors.files || typeof cursors.files !== "object") {
@@ -1084,6 +1086,9 @@ async function parseClaudeIncremental({
   const prevHashes = Array.isArray(cursors.claudeHashes) ? cursors.claudeHashes : [];
   const seenMessageHashes = new Set(prevHashes);
   const defaultSource = normalizeSourceInput(source) || "claude";
+  preserveLegacyClaudeGroups(files.filter((entry) =>
+    (typeof entry === "string" ? defaultSource : normalizeSourceInput(entry?.source) || defaultSource) === "claude"
+  ).map((entry) => typeof entry === "string" ? entry : entry?.path), path.dirname(queuePath));
   // Resolve all changed-file lineage before counting: fork files may sort first.
   const forkTimes = new Map();
   const forkAliases = await collectClaudeForkAliases(files, cursors,
@@ -1153,6 +1158,7 @@ async function parseClaudeIncremental({
     const truncated = sameInode && prevOffset > st.size;
     const startOffset = sameInode && !truncated ? prevOffset : 0;
 
+    if (fileSource === "claude") preserveLegacyClaudePrefix(filePath, path.dirname(queuePath), startOffset);
     let fileId = null;
     if (seenFileIds) {
       const cachedFileId =
@@ -1287,6 +1293,7 @@ async function parseClaudeIncremental({
       projectKey,
       seenMessageHashes,
       forkAliases,
+      backendResolver: fileSource === "claude" ? createBackendResolver(filePath, path.dirname(queuePath)) : null,
     });
 
     cursors.files[key] = {
@@ -3134,6 +3141,7 @@ async function parseClaudeFile({
   projectKey,
   seenMessageHashes,
   forkAliases,
+  backendResolver,
 }) {
   const st = fileStat || (await fs.stat(filePath).catch(() => null));
   if (!st || !st.isFile()) return { endOffset: startOffset, eventsAggregated: 0 };
@@ -3141,15 +3149,14 @@ async function parseClaudeFile({
   const endOffset = st.size;
   if (startOffset >= endOffset) return { endOffset, eventsAggregated: 0 };
 
-  const stream = fssync.createReadStream(filePath, { encoding: "utf8", start: startOffset });
-  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
-
   let eventsAggregated = 0;
   // Separator-agnostic: WSL installs are scanned over \\wsl$ UNC paths where
   // path.join produces backslashes, and a forward-slash-only check would
   // misclassify subagent transcripts as main sessions (#307).
   const isMainSession = !/[\\/]subagents[\\/]/.test(filePath);
-  for await (const line of rl) {
+  let byteOffset = startOffset;
+  for await (const { line, start: lineStart, end } of claudeLines(filePath, startOffset, endOffset)) {
+    byteOffset = end;
     if (!line) continue;
 
     // Count user-typed messages as conversations (main sessions only).
@@ -3211,6 +3218,8 @@ async function parseClaudeFile({
 
     if (dedupHash) seenMessageHashes.add(dedupHash);
     delta.conversation_count = 0;
+    const backend = backendResolver?.(obj, lineStart, byteOffset);
+    if (backend) delta.backend_usage = [{ ...delta, ...backend }];
 
     const bucketStart = toUtcHalfHourStart(tokenTimestamp);
     if (!bucketStart) continue;
@@ -3232,9 +3241,7 @@ async function parseClaudeFile({
     eventsAggregated += 1;
   }
 
-  rl.close();
-  stream.close?.();
-  return { endOffset, eventsAggregated };
+  return { endOffset: byteOffset, eventsAggregated };
 }
 
 async function parseGeminiFile({
@@ -3514,6 +3521,40 @@ async function parseOpencodeMessageFile({
   };
 }
 
+// Codex models are observed per rollout/turn. A busier sibling agent in the
+// same half-hour is not evidence for the model of an unnamed request (#927).
+// Re-emit stored buckets once on upgrade, including idle historical groups:
+// their totals still retain the original attribution, so no log replay is
+// needed. Queue readers replace rows by (source, model, hour_start).
+function preserveCodexModelBuckets(hourlyState, touchedBuckets) {
+  const upgrading = hourlyState.modelAttributionPolicy?.codex !== "observed";
+  if (!upgrading && !Object.keys(hourlyState.groupQueued).some(
+    key => key.startsWith(`${DEFAULT_SOURCE}${BUCKET_SEPARATOR}`),
+  )) return;
+  const legacyGroups = new Set();
+  for (const [key, bucket] of Object.entries(hourlyState.buckets || {})) {
+    const parsed = parseBucketKey(key);
+    if (parsed.source !== DEFAULT_SOURCE || !bucket?.totals) continue;
+    const groupKey = groupBucketKey(parsed.source, parsed.hourStart);
+    const legacy = Object.prototype.hasOwnProperty.call(hourlyState.groupQueued, groupKey);
+    if (legacy) legacyGroups.add(groupKey);
+    if (!upgrading && !legacy) continue;
+    bucket.queuedKey = null;
+    bucket.retractedUnknownKey = null;
+    touchedBuckets.add(key);
+  }
+  for (const groupKey of legacyGroups) {
+    const hourStart = groupKey.slice(DEFAULT_SOURCE.length + BUCKET_SEPARATOR.length);
+    // Legacy groups emitted a combined unknown row for multiple named models.
+    // Replace it with the actual unknown total (or zero) before splitting.
+    const unknown = getHourlyBucket(hourlyState, DEFAULT_SOURCE, DEFAULT_MODEL, hourStart);
+    unknown.queuedKey = null;
+    touchedBuckets.add(bucketKey(DEFAULT_SOURCE, DEFAULT_MODEL, hourStart));
+    delete hourlyState.groupQueued[groupKey];
+  }
+  hourlyState.modelAttributionPolicy = { ...hourlyState.modelAttributionPolicy, codex: "observed" };
+}
+
 async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets }) {
   if (!touchedBuckets || touchedBuckets.size === 0) return 0;
 
@@ -3607,10 +3648,10 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
 
   const toAppend = [];
   for (const group of groupedBuckets.values()) {
-    // A local reconstruction may opt into observed-only attribution. Preserve
-    // every model bucket, including unknown, instead of guessing from the
-    // busiest model in the same half-hour. The policy lives in cursor state
-    // so hooks, desktop refreshes, and restarted CLI processes agree.
+    // Codex defaults to observed-only attribution; reconstructions can opt
+    // other sources in. Preserve every model bucket, including unknown,
+    // instead of guessing from the busiest model in the same half-hour.
+    // Persist the policy so hooks, desktop refreshes and restarts agree.
     if (hourlyState.modelAttributionPolicy?.[group.source] === "observed") {
       // A group last aligned under the default policy must not keep the stale
       // dominant attribution downstream. Mirror the default path's alignment
@@ -3662,6 +3703,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
           total_cost_usd: totals.total_cost_usd || 0,
           usage_precision: usagePrecision || undefined,
           ...prioritySubsetOf(totals),
+          ...(totals.backend_usage ? { backend_usage: totals.backend_usage } : {}),
           conversation_count: totals.conversation_count,
         }));
         bucket.queuedKey = key;
@@ -3749,6 +3791,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
             total_cost_usd: totals.total_cost_usd || 0,
             usage_precision: usagePrecision || undefined,
             ...prioritySubsetOf(totals),
+            ...(totals.backend_usage ? { backend_usage: totals.backend_usage } : {}),
             conversation_count: totals.conversation_count,
           }),
         );
@@ -3830,6 +3873,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
         total_cost_usd: unknownBucket.totals.total_cost_usd || 0,
         usage_precision: usagePrecision || undefined,
         ...prioritySubsetOf(unknownBucket.totals),
+        ...(unknownBucket.totals.backend_usage ? { backend_usage: unknownBucket.totals.backend_usage } : {}),
         conversation_count: unknownBucket.totals.conversation_count,
       }),
     );
@@ -3879,6 +3923,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
           billable_total_tokens: group.totals.billable_total_tokens ?? group.totals.total_tokens,
           total_cost_usd: group.totals.total_cost_usd || 0,
           ...prioritySubsetOf(group.totals),
+          ...(group.totals.backend_usage ? { backend_usage: group.totals.backend_usage } : {}),
           conversation_count: group.totals.conversation_count,
         }),
       );
@@ -3926,6 +3971,7 @@ async function enqueueTouchedProjectBuckets({
         project_ref: projectRef,
         project_key: projectKey,
         source: bucket.source,
+        ...(totals.backend_usage ? { backend_usage: totals.backend_usage } : {}),
         hour_start: bucket.hour_start,
         input_tokens: totals.input_tokens,
         cached_input_tokens: totals.cached_input_tokens,
@@ -4038,18 +4084,29 @@ function normalizeHourlyState(raw) {
   const version = Number(state.version || 1);
   const rawBuckets = state.buckets && typeof state.buckets === "object" ? state.buckets : {};
   const buckets = {};
-  const groupQueued = {};
+  const existingGroupQueued =
+    state.groupQueued && typeof state.groupQueued === "object" ? state.groupQueued : {};
   const modelAttributionPolicy = Object.fromEntries(
     Object.entries(state.modelAttributionPolicy || {})
       .filter(([, policy]) => policy === "observed"),
   );
 
   if (!Number.isFinite(version) || version < 2) {
+    const groupQueued = { ...existingGroupQueued };
     for (const [key, value] of Object.entries(rawBuckets)) {
       const parsed = parseBucketKey(key);
       const hourStart = parsed.hourStart;
       if (!hourStart) continue;
       const source = normalizeSourceInput(parsed.source) || DEFAULT_SOURCE;
+      // Older repairs persisted model-aware keys without a schema version.
+      // Preserve them before v1 normalization can collapse sibling models into
+      // unknown; preserveCodexModelBuckets needs the original attribution (#927).
+      const firstSeparator = key.indexOf(BUCKET_SEPARATOR);
+      const hasModel = firstSeparator > 0 && key.indexOf(BUCKET_SEPARATOR, firstSeparator + 1) > 0;
+      if (hasModel) {
+        buckets[bucketKey(source, parsed.model, hourStart)] = value;
+        continue;
+      }
       const normalizedKey = bucketKey(source, DEFAULT_MODEL, hourStart);
       buckets[normalizedKey] = value;
       if (value?.queuedKey) {
@@ -4073,9 +4130,6 @@ function normalizeHourlyState(raw) {
     buckets[normalizedKey] = value;
   }
 
-  const existingGroupQueued =
-    state.groupQueued && typeof state.groupQueued === "object" ? state.groupQueued : {};
-
   return {
     version: 3,
     buckets,
@@ -4529,6 +4583,7 @@ function prioritySubsetOf(totals) {
 }
 
 function addTotals(target, delta) {
+  mergeBackendUsage(target, delta);
   for (const [field] of SPEED_TIER_SUBSET_FIELDS) {
     const value = Number(delta?.[field]) || 0;
     if (value > 0) target[field] = (Number(target[field]) || 0) + value;

package/src/lib/session-analytics.js

@@ -1,4 +1,5 @@
 "use strict";
+const { createBackendResolver, mergeBackendUsage, claudeLines } = require("./claude-backend");
 
 // Metadata-only session analytics sidecar.
 //
@@ -146,6 +147,7 @@ function tokenTotals(usage) {
 }
 
 function addTotals(target, delta) {
+  mergeBackendUsage(target, delta);
   for (const key of ["input_tokens", "cached_input_tokens", "cache_creation_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens"]) {
     target[key] = finite(target[key]) + finite(delta?.[key]);
   }
@@ -226,6 +228,7 @@ function normalizeModelUsageRows(rows) {
       };
       byModel.set(model, row);
     }
+    mergeBackendUsage(row, value);
     for (const field of MODEL_USAGE_SUM_FIELDS) row[field] += finite(value?.[field]);
     row.performance = mergePerformance(row.performance, value?.performance);
     if (Array.isArray(value?.selected_models)) {
@@ -266,7 +269,7 @@ function normalizeModelUsageRows(rows) {
 // most of it the long_context_* counters for sessions with no requests over
 // OPENAI_LONG_CONTEXT_INPUT_THRESHOLD in src/lib/pricing/index.js.
 function serializeModelUsageRow(row) {
-  const out = { model: row.model };
+  const out = { model: row.model, ...(row.backend_usage ? { backend_usage: row.backend_usage } : {}) };
   for (const field of MODEL_USAGE_SUM_FIELDS) {
     const value = finite(row[field]);
     if (value !== 0) out[field] = value;
@@ -299,7 +302,7 @@ function repriceSessionRecord(record) {
     let hasUnpricedUsage = false;
     for (const row of modelUsage) {
       row.cost_usd = computeRowCost({ source: record.source, ...row });
-      row.pricing = getModelPricingInfo(row.model, { source: record.source });
+      row.pricing = getModelPricingInfo(row.model, { ...row, source: record.source });
       if (row.total_tokens > 0 && row.pricing.status === "unpriced") hasUnpricedUsage = true;
     }
     record.model_usage = modelUsage;
@@ -366,7 +369,7 @@ function modelUsageForAggregation(record) {
   }
   const priced = rows.map((row) => denseModelUsageRow({
     ...row,
-    pricing: getModelPricingInfo(row.model, { source: record.source }),
+    pricing: getModelPricingInfo(row.model, { ...row, source: record.source }),
     cost_usd: Number.isFinite(Number(row.cost_usd))
       ? Number(row.cost_usd)
       : computeRowCost({ source: record.source, ...row }),
@@ -548,7 +551,7 @@ function isRecoverableSessionReadError(error) {
   return ["ENOENT", "EACCES", "EPERM", "EISDIR", "EIO", "ESTALE"].includes(error?.code);
 }
 
-async function scanClaudeSession(filePath) {
+async function scanClaudeSession(filePath, { trackerDir, legacyOffsets = {} } = {}) {
   // Native and WSL mounts can disappear independently after discovery. Keep
   // the surviving mirror instead of dropping the whole logical session.
   const filePaths = readableSessionPaths(filePath);
@@ -600,9 +603,8 @@ async function scanClaudeSession(filePath) {
   for (const currentFilePath of filePaths) {
     const currentRecordHashes = new Set();
     try {
-      const input = fs.createReadStream(currentFilePath, { encoding: "utf8" });
-      const lines = readline.createInterface({ input, crlfDelay: Infinity });
-      for await (const line of lines) {
+      const backend = trackerDir ? createBackendResolver(currentFilePath, trackerDir, { legacyOffset: legacyOffsets[currentFilePath] || 0 }) : null;
+      for await (const { line, start, end } of claudeLines(currentFilePath)) {
         if (filePaths.length > 1) {
           const recordHash = crypto.createHash("sha256").update(line).digest("base64url");
           if (priorRecordHashes.has(recordHash)) continue;
@@ -649,6 +651,8 @@ async function scanClaudeSession(filePath) {
         if (candidateModel) model = candidateModel;
         currentTurnModel = model;
         const usage = normalizeClaudeUsage(obj.message.usage);
+        const identity = backend && usage.total_tokens > 0 ? backend(obj, start, end) : null;
+        if (identity) usage.backend_usage = [{ ...usage, ...identity }];
         addTotals(tokens, usage);
         if (usage.total_tokens > 0) {
           const modelUsage = modelUsageByModel.get(model) || { model, ...emptyTotals(), usage_events: 0 };
@@ -2343,6 +2347,8 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
   const retryOmpScan = fs.existsSync(incompleteOmpPath);
   const incompleteDshPath = `${metaPath}.dsh-incomplete`;
   const retryDshScan = fs.existsSync(incompleteDshPath);
+  let provenanceMeta = null;
+  try { provenanceMeta = JSON.parse(fs.readFileSync(metaPath, "utf8")); } catch { /* first scan */ }
   let previousMeta = null;
   if (!force) {
     try {
@@ -2468,7 +2474,12 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
     }
     if (!rows) {
       try {
-        const scanned = await entry.scan(entry.filePath);
+        const legacyOffsets = {};
+        if (entry.source === "claude") {
+          const sizes = provenanceMeta?.files?.[cacheKey]?.stat_key?.split("|") || [];
+          entry.filePath.forEach((file, index) => { legacyOffsets[file] = Number(sizes[index]?.split(":")[0]) || 0; });
+        }
+        const scanned = await entry.scan(entry.filePath, { trackerDir: path.dirname(sidecarPath), legacyOffsets });
         rows = Array.isArray(scanned) ? scanned.filter(Boolean) : (scanned ? [scanned] : []);
         // Batch sources count the files they had to give up on themselves.
         // A dsh batch is one deduplicated inventory, so a single unreadable log
@@ -2659,6 +2670,7 @@ function scopeSessionUsage(row, from, to, context) {
     const modelName = previousModelUsage ? previousModelUsage.model : (row.model || bucket.model);
     const model = byModel.get(modelName) || {
       ...(previousModelUsage || {}),
+      backend_usage: undefined,
       model: modelName,
       ...Object.fromEntries(MODEL_USAGE_SUM_FIELDS.filter((field) => field !== "edit_turns").map((field) => [field, 0])),
       usage_events: 0,

package/src/lib/trae-cn-config.js

@@ -34,6 +34,9 @@ function isTraeCnUsageEnabled(env = process.env) {
 }
 const TRAE_CN_APP_DIRS = ["TRAE SOLO CN", "Trae CN"];
 const TRAE_CN_AUTH_KEY = "iCubeAuthInfo://icube.cloudide";
+const TRAE_CN_AUTH_HELP = "Sign in again in the TRAE CN app, then rerun tokentracker sync --source=trae-cn. " +
+  "If multiple CN apps are installed, set TOKENTRACKER_TRAE_CN_HOME to the signed-in app data root. " +
+  "Readable local auth does not prove API access; TokenTracker cannot refresh the app session.";
 
 const TRAE_CN_USAGE_URL = "https://api.trae.cn/trae/api/v1/pay/query_user_usage_group_by_session";
 // usage_type [7] mirrors the constant the official TRAE client sends when its
@@ -282,6 +285,26 @@ function extractTraeCnToken(auth) {
   throw new Error("Trae CN auth data has no token.");
 }
 
+// Local-only doctor probe. Return an allowlisted state, never credentials or
+// exception text. Opting out also avoids reading/decrypting the auth payload.
+function inspectTraeCnAuth(options = {}) {
+  const enabled = isTraeCnUsageEnabled(options.env ?? process.env);
+  const storagePath = resolveTraeCnStoragePath(options);
+  const result = { usage_opt_in: enabled, storage_path: storagePath, auth: "unchecked" };
+  if (!enabled) return result;
+  try {
+    const auth = readTraeCnAuthFromStorage(options);
+    if (!auth) result.auth = "not-signed-in";
+    else {
+      extractTraeCnToken(auth);
+      result.auth = "readable";
+    }
+  } catch (error) {
+    result.auth = error?.code === "TRAE_CN_STORAGE_UNREADABLE" ? "unreadable" : "malformed";
+  }
+  return result;
+}
+
 // ---------------------------------------------------------------------------
 // Usage API
 // ---------------------------------------------------------------------------
@@ -365,7 +388,7 @@ async function fetchTraeCnUsagePage({
       throw new Error("Trae CN usage API request failed.");
     }
     if (response.status === 401 || response.status === 403) {
-      throw usageError(`Trae CN usage API returned HTTP ${response.status}.`, { status: response.status });
+      throw usageError(`Trae CN usage API returned HTTP ${response.status}; authorization was rejected. ${TRAE_CN_AUTH_HELP}`, { status: response.status });
     }
     if (!response.ok) {
       throw usageError(`Trae CN usage API returned HTTP ${response.status}.`, { status: response.status });
@@ -377,8 +400,13 @@ async function fetchTraeCnUsagePage({
       throw new Error("Trae CN usage API returned a non-JSON response.");
     }
     const apiCode = parsed?.data?.code ?? parsed?.code;
+    if (apiCode !== undefined &&
+        !((typeof apiCode === "number" || (typeof apiCode === "string" && /^\d+$/.test(apiCode))) &&
+          Number.isSafeInteger(Number(apiCode)))) {
+      throw new Error("Trae CN usage API returned an invalid error code.");
+    }
     if (apiCode !== undefined && Number(apiCode) !== 0) {
-      throw usageError(`Trae CN usage API returned error code ${apiCode}.`, {
+      throw usageError(`Trae CN usage API returned error code ${Number(apiCode)}.`, {
         status: response.status,
         apiCode: Number(apiCode),
       });
@@ -543,6 +571,8 @@ module.exports = {
   isTraeCnUsageEnabled,
   TRAE_CN_APP_DIRS,
   TRAE_CN_AUTH_KEY,
+  TRAE_CN_AUTH_HELP,
+  inspectTraeCnAuth,
   TRAE_CN_USAGE_URL,
   TRAE_CN_USAGE_TYPE,
   TRAE_CN_USAGE_PAGE_SIZE,