tokentracker-cli 0.94.0 → 0.95.0

package/src/lib/grok-usage.js

@@ -0,0 +1,95 @@
+"use strict";
+
+// Grok's ACP turn_completed.usage uses camelCase fields. In that shape,
+// inputTokens is the whole prompt (cache reads/writes included) and
+// outputTokens includes reasoningTokens. TokenTracker stores mutually
+// exclusive columns, so both reported totals need to be split before pricing.
+//
+// Grok's headless/CLI summaries may use snake_case fields whose input_tokens is
+// already the non-cached portion. Keep the two shapes distinct instead of
+// subtracting cache tokens twice.
+
+const USD_TICKS_PER_USD = 10_000_000_000;
+
+function nonNegative(value) {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : 0;
+}
+
+function firstBoolean(...values) {
+  for (const value of values) {
+    if (typeof value === "boolean") return value;
+  }
+  return false;
+}
+
+function reportedCostUsd(usage, { costIsPartial, usageIsIncomplete }) {
+  if (costIsPartial || usageIsIncomplete) return null;
+
+  const ticks = Number(
+    usage.costUsdTicks ??
+      usage.totalCostUsdTicks ??
+      usage.cost_usd_ticks ??
+      usage.total_cost_usd_ticks,
+  );
+  if (Number.isFinite(ticks) && ticks >= 0) return ticks / USD_TICKS_PER_USD;
+
+  const usd = Number(usage.costUsd ?? usage.totalCostUsd ?? usage.cost_usd ?? usage.total_cost_usd);
+  return Number.isFinite(usd) && usd >= 0 ? usd : null;
+}
+
+function normalizeGrokUsage(usage) {
+  if (!usage || typeof usage !== "object") return null;
+
+  const camelCaseInput = usage.inputTokens != null;
+  const reportedInput = nonNegative(usage.inputTokens ?? usage.input_tokens);
+  const cachedInput = nonNegative(
+    usage.cachedReadTokens ?? usage.cacheReadInputTokens ?? usage.cache_read_input_tokens ?? usage.cached_input_tokens,
+  );
+  const cacheCreationInput = nonNegative(
+    usage.cacheCreationTokens ??
+      usage.cachedWriteTokens ??
+      usage.cacheWriteInputTokens ??
+      usage.cache_creation_input_tokens,
+  );
+  const rawOutput = nonNegative(usage.outputTokens ?? usage.output_tokens);
+  const reasoningOutput = nonNegative(usage.reasoningTokens ?? usage.reasoning_output_tokens);
+
+  const inputTokens = camelCaseInput
+    ? Math.max(0, reportedInput - cachedInput - cacheCreationInput)
+    : reportedInput;
+  const outputTokens = Math.max(0, rawOutput - reasoningOutput);
+
+  let totalTokens = nonNegative(usage.totalTokens ?? usage.total_tokens);
+  if (totalTokens <= 0) {
+    totalTokens = inputTokens
+      + cachedInput
+      + cacheCreationInput
+      + outputTokens
+      + reasoningOutput;
+  }
+  if (totalTokens <= 0) return null;
+
+  const costIsPartial = firstBoolean(usage.costIsPartial, usage.cost_is_partial);
+  const usageIsIncomplete = firstBoolean(usage.usageIsIncomplete, usage.usage_is_incomplete);
+
+  return {
+    input_tokens: inputTokens,
+    cached_input_tokens: cachedInput,
+    cache_creation_input_tokens: cacheCreationInput,
+    output_tokens: outputTokens,
+    reasoning_output_tokens: reasoningOutput,
+    total_tokens: totalTokens,
+    billable_total_tokens: totalTokens,
+    total_cost_usd: reportedCostUsd(usage, { costIsPartial, usageIsIncomplete }),
+    cost_is_partial: costIsPartial,
+    usage_is_incomplete: usageIsIncomplete,
+    model_calls: nonNegative(usage.modelCalls ?? usage.model_calls),
+    api_duration_ms: nonNegative(usage.apiDurationMs ?? usage.api_duration_ms),
+  };
+}
+
+module.exports = {
+  USD_TICKS_PER_USD,
+  normalizeGrokUsage,
+};

package/src/lib/local-api.js

@@ -14,6 +14,11 @@ const { accountSlugFor, fetchAccountUsage, mintAccessToken } = require("./cloud-
 const { getOrCreateMachineId, computeStableMachineId } = require("./machine-id");
 
 const SYNC_TIMEOUT_MS = 120_000;
+// A failed account-view request should not stall every dashboard refresh
+// while an expired session or unreachable backend keeps timing out. The first
+// request still falls back normally; subsequent fan-out requests reuse that
+// decision for a short cooldown and return local data immediately.
+const ACCOUNT_VIEW_FAILURE_BACKOFF_MS = 2 * 60_000;
 const TRACKER_BIN = path.resolve(__dirname, "../../bin/tracker.js");
 const MAX_DEVICE_NAME_LENGTH = 128;
 
@@ -219,8 +224,12 @@ function normalizeQueueRow(row) {
   // non-zero billable usage (GitHub issue #106). Treat billing and usage as
   // orthogonal: bump billable up to total_tokens at read time so historical
   // queue.jsonl entries render correctly without requiring a file rewrite.
+  // Antigravity had the same symptom from e69e2746a (billable deleted during
+  // the O(N) refactoring, leaving 167 local rows with billable=0 until the
+  // parser fix). Apply the same read-time healing so upgraded installs show
+  // correct heatmap / rolling numbers without a queue rewrite.
   const sourceName = String(normalized.source || "").toLowerCase();
-  if (sourceName === "cursor") {
+  if (sourceName === "cursor" || sourceName === "antigravity") {
     const totalTokens = Number(normalized.total_tokens || 0);
     const billable = Number(normalized.billable_total_tokens || 0);
     if (totalTokens > 0 && billable < totalTokens) {
@@ -1135,6 +1144,30 @@ function createLocalApiHandler({ queuePath }) {
   const cookiePath = path.join(trackerDataDir, "relay-cookies.json");
   const localSyncDeviceTokenCache = new Map();
   const localSyncDeviceTokenInflight = new Map();
+  // A dashboard refresh fans out to several account-view endpoints. Keep the
+  // circuit-breaker state per session instead of one global slot (a successful
+  // request for one account must not clear another account's failure), and
+  // suppress the duplicate fan-out while the first cloud probe is pending.
+  const accountViewFailures = new Map();
+  const accountViewInFlight = new Set();
+
+  function accountViewFailureUntil(failureKey) {
+    const until = accountViewFailures.get(failureKey) || 0;
+    if (until > Date.now()) return until;
+    if (until) accountViewFailures.delete(failureKey);
+    return 0;
+  }
+
+  function rememberAccountViewFailure(failureKey) {
+    accountViewFailures.set(failureKey, Date.now() + ACCOUNT_VIEW_FAILURE_BACKOFF_MS);
+    // Refresh-token rotation or account switching can produce many keys over a
+    // long-lived tray session. Bound the map without affecting the active key.
+    while (accountViewFailures.size > 16) {
+      const oldest = accountViewFailures.keys().next().value;
+      if (!oldest || oldest === failureKey) break;
+      accountViewFailures.delete(oldest);
+    }
+  }
 
   // Load persisted cookies on startup
   try {
@@ -1399,15 +1432,52 @@ function createLocalApiHandler({ queuePath }) {
     }
   }
 
-  // Returns "served" when the cross-device aggregate was written to `res`, or
-  // "fallthrough" when the caller should serve the local single-machine data.
-  // Any failure (not signed in, cloud sync off, network/auth error) falls
-  // through so the popover always renders something.
+  // Why a *classified* fallback: the native popover keeps the last successful
+  // account (cross-device) snapshot on screen when a cloud read fails
+  // transiently, but must switch to this-machine data the moment the user signs
+  // out or turns cloud sync off. Both used to look identical from the outside
+  // (HTTP 200 + local payload + `X-TokenTracker-Account-View: 0`), so a single
+  // timed-out read silently shrank Activity to one device until the next
+  // manual sync. See ACCOUNT_FALLBACK_* below for the vocabulary.
+  const ACCOUNT_FALLBACK_CLOUD_SYNC_OFF = "cloud-sync-off";
+  const ACCOUNT_FALLBACK_SIGNED_OUT = "signed-out";
+
+  // Every transient reason is prefixed "transient-"; clients only need the
+  // prefix, so new reasons can be added without a client change.
+  function classifyAccountFallback(err) {
+    if (!err) return "transient-error";
+    if (err.name === "AbortError" || err.name === "TimeoutError" || err.code === "auth_timeout") {
+      return "transient-timeout";
+    }
+    if (err.code === "auth_rejected" || err.code === "auth_invalid" || err.status === 401 || err.status === 403) {
+      return "transient-auth";
+    }
+    if (err.code === "auth_network") return "transient-network";
+    if (Number.isFinite(err.status) && err.status > 0) return "transient-upstream";
+    return "transient-network";
+  }
+
+  // Returns "served" when the cross-device aggregate was written to `res`,
+  // otherwise a fallback reason (see above) telling the caller — and, through
+  // the `X-TokenTracker-Account-Fallback` header, the popover — why the local
+  // single-machine data is being served instead.
   async function tryServeAccountView(usageSlug, url, res) {
-    if (!getCloudSyncPref()) return "fallthrough";
+    if (!getCloudSyncPref()) return ACCOUNT_FALLBACK_CLOUD_SYNC_OFF;
     const refreshToken = getRefreshTokenForCloud();
-    if (!refreshToken) return "fallthrough";
+    if (!refreshToken) return ACCOUNT_FALLBACK_SIGNED_OUT;
     const runtime = resolveRuntimeConfig();
+    const failureKey = `${runtime.baseUrl}\0${refreshToken}`;
+    const requestKey = `${failureKey}\0${usageSlug}\0${url.searchParams.toString()}`;
+    if (accountViewFailureUntil(failureKey) > Date.now()) {
+      return "transient-backoff";
+    }
+    // Six native dashboard requests normally arrive together. If the cloud is
+    // already being probed for this session, let the remaining requests use
+    // their local fallback instead of opening more timeout-bound sockets.
+    if (accountViewInFlight.has(requestKey)) {
+      return "transient-inflight";
+    }
+    accountViewInFlight.add(requestKey);
     const envTimeout = process.env.TOKENTRACKER_HTTP_TIMEOUT_MS
       ? parseInt(process.env.TOKENTRACKER_HTTP_TIMEOUT_MS, 10)
       : 0;
@@ -1421,9 +1491,16 @@ function createLocalApiHandler({ queuePath }) {
         refreshToken,
         timeoutMs,
       });
-      if (!out || out.data == null) return "fallthrough";
+      // `null` here means the slug has no cloud equivalent, or the session
+      // vanished mid-request — a routing/session fact, not an outage.
+      if (!out) return ACCOUNT_FALLBACK_SIGNED_OUT;
+      if (out.data == null) {
+        rememberAccountViewFailure(failureKey);
+        return "transient-upstream";
+      }
       if (out.rotatedRefreshToken) setRelayRefreshToken(out.rotatedRefreshToken);
       if (out.rotatedCsrfToken) setRelayCsrfToken(out.rotatedCsrfToken);
+      accountViewFailures.delete(failureKey);
       res.writeHead(200, {
         "Content-Type": "application/json",
         "Cache-Control": "no-store",
@@ -1433,11 +1510,17 @@ function createLocalApiHandler({ queuePath }) {
       return "served";
     } catch (e) {
       // Signed in + cloud sync on, but the cloud read failed (offline, token
-      // rejected, edge error, or timeout). Fall back to local data rather than erroring.
+      // rejected, edge error, or timeout). Serve local data rather than
+      // erroring, but say so: this is a *temporary* downgrade and the client
+      // must not treat it as the user's real data scope.
+      const reason = classifyAccountFallback(e);
       if (resolveRuntimeConfig().debug) {
-        console.warn(`[LocalAPI] account view fallback for ${usageSlug}:`, e?.message || e);
+        console.warn(`[LocalAPI] account view fallback (${reason}) for ${usageSlug}:`, e?.message || e);
       }
-      return "fallthrough";
+      rememberAccountViewFailure(failureKey);
+      return reason;
+    } finally {
+      accountViewInFlight.delete(requestKey);
     }
   }
 
@@ -2006,14 +2089,17 @@ function createLocalApiHandler({ queuePath }) {
     // --- account (cross-device) view proxy for the native popover ---
     // When ?account=1 and the user is signed in with cloud sync on, serve the
     // same cross-device aggregate the dashboard shows; otherwise tag the
-    // response (X-TokenTracker-Account-View: 0) so the popover knows it got
-    // local single-machine data, and fall through to the local handler below.
+    // response (X-TokenTracker-Account-View: 0 plus an
+    // X-TokenTracker-Account-Fallback reason) so the popover knows it got local
+    // single-machine data and whether that is permanent (signed out / cloud
+    // sync off) or transient, and fall through to the local handler below.
     if (url.searchParams.get("account") === "1") {
       const usageSlug = p.startsWith("/functions/") ? p.slice("/functions/".length) : "";
       if (accountSlugFor(usageSlug)) {
         const result = await tryServeAccountView(usageSlug, url, res);
         if (result === "served") return true;
         res.setHeader("X-TokenTracker-Account-View", "0");
+        res.setHeader("X-TokenTracker-Account-Fallback", result);
       }
     }
 
@@ -2291,6 +2377,8 @@ function createLocalApiHandler({ queuePath }) {
         ma.totals.cached_input_tokens += row.cached_input_tokens || 0;
         ma.totals.cache_creation_input_tokens += row.cache_creation_input_tokens || 0;
         ma.totals.reasoning_output_tokens += row.reasoning_output_tokens || 0;
+        ma.totals.total_cost_usd = Number(ma.totals.total_cost_usd || 0)
+          + (Number(row.total_cost_usd) || 0);
       }
 
       const sources = Array.from(bySource.values()).map((s) => {

package/src/lib/pricing/index.js

@@ -21,6 +21,11 @@ const PI_SUBSCRIPTION_SOURCES = new Set([
   "prime-agent-github-copilot",
   "prime-agent-copilot",
 ]);
+// LM Studio's server logs describe inference served by the local developer
+// server (including LM Link). Secure Cloud usage has a separate billing path
+// and is not present in these logs.
+const LOCAL_INFERENCE_SOURCES = new Set(["lmstudio"]);
+const SOURCES_WITH_AUTHORITATIVE_COST = new Set(["grok"]);
 const SEED_SNAPSHOT_PATH = path.resolve(__dirname, "seed-snapshot.json");
 const DEEPSEEK_TIME_PRICED_MODELS = [
   "deepseek-v4-flash",
@@ -168,10 +173,20 @@ function getRowPricing(row) {
 // computeRowCost in src/lib/local-api.js. Moved here so vite mock + local
 // server share one source of truth.
 function computeRowCost(row) {
+  if (LOCAL_INFERENCE_SOURCES.has(String(row?.source || "").toLowerCase())) return 0;
   // Pi can route a turn through a subscription-backed Copilot provider. Pi's
   // usage record reports a zero marginal cost for those turns; do not
   // reinterpret the Claude model name as an Anthropic API bill.
   if (PI_SUBSCRIPTION_SOURCES.has(String(row?.source || "").toLowerCase())) return 0;
+  // Some providers (currently Grok) persist an exact server-reported cost on
+  // the usage bucket. Prefer it when positive; zero remains the legacy
+  // "unreported" sentinel and falls through to model pricing.
+  const reportedCost = Number(row?.total_cost_usd);
+  if (
+    SOURCES_WITH_AUTHORITATIVE_COST.has(row?.source) &&
+    Number.isFinite(reportedCost) &&
+    reportedCost > 0
+  ) return reportedCost;
   const pricing = getRowPricing(row);
   const reasoningIncludedInOutput = row.source === "codex" || row.source === "every-code";
   const reasoningCost = reasoningIncludedInOutput

package/src/lib/pricing/matcher.js

@@ -164,6 +164,20 @@ function normalizeWorkbuddyModel(model) {
   return model;
 }
 
+// Unsloth Studio can mix local engines and paid API providers in one database.
+// The parser qualifies metered models with their provider and marks local,
+// subscription-backed, or ambiguous custom routes as unpriced. Returning a
+// sentinel here prevents the generic reverse-substring matcher from mistaking
+// a locally hosted model name for the similarly named cloud SKU.
+function normalizeUnslothModel(model) {
+  if (typeof model !== "string") return model;
+  const lower = model.trim().toLowerCase();
+  if (lower.startsWith("local/") || lower.startsWith("unpriced/")) {
+    return "__tokentracker_unpriced_unsloth_model__";
+  }
+  return model;
+}
+
 // Per-source model-name normalizers, applied at pricing-lookup time only (the
 // raw model name is preserved for storage/display). Add a source here when its
 // model strings don't match the LiteLLM/curated keys verbatim.
@@ -174,6 +188,7 @@ const SOURCE_MODEL_NORMALIZERS = {
   "pi-anthropic": normalizeClaudeModel,
   "prime-agent-anthropic": normalizeClaudeModel,
   zed: normalizeZedModel,
+  unsloth: normalizeUnslothModel,
   workbuddy: normalizeWorkbuddyModel,
 };