tokentracker-cli 0.81.2 → 0.82.0

package/src/lib/local-api.js

@@ -35,6 +35,7 @@ const {
   getModelPricing,
   computeRowCost,
   ensurePricingLoaded,
+  getPricingRevision,
 } = require("./pricing");
 
 const {
@@ -66,6 +67,33 @@ const CLAUDE_MEM_OBSERVER_PROJECT_KEY = deriveProjectKeyFromRef(
   CLAUDE_MEM_OBSERVER_PROJECT_REF,
 );
 const PROJECT_USAGE_MAX_ENTRIES = 10;
+const MAX_TIME_ZONE_CACHE_ENTRIES = 16;
+
+// The native dashboard fans one refresh out across several local endpoints.
+// Keep one immutable, deduped view of the current queue so those endpoints do
+// not all read and JSON.parse the same append-only file. A file identity +
+// nanosecond timestamp signature invalidates the cache immediately on append,
+// in-place rewrite, or atomic replacement.
+let queueDataCache = null;
+const dailyAggregationCache = new WeakMap();
+const zonedPartsFormatters = new Map();
+
+function boundedCacheSet(cache, key, value, maxEntries) {
+  if (cache.has(key)) cache.delete(key);
+  cache.set(key, value);
+  while (cache.size > maxEntries) {
+    cache.delete(cache.keys().next().value);
+  }
+}
+
+function queueFileSignature(queuePath) {
+  const stat = fs.statSync(queuePath, { bigint: true });
+  return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
+}
+
+function timeZoneContextKey({ timeZone, offsetMinutes } = {}) {
+  return `${timeZone || ""}|${Number.isFinite(offsetMinutes) ? offsetMinutes : ""}|p${getPricingRevision()}`;
+}
 
 // Shared by project-usage-summary and project-usage-detail: reads the
 // deduped project bucket log, drops the claude-mem pseudo project, and
@@ -160,6 +188,21 @@ function normalizeQueueRow(row) {
 }
 
 function readQueueData(queuePath) {
+  let signature;
+  try {
+    signature = queueFileSignature(queuePath);
+  } catch (e) {
+    if (queueDataCache?.queuePath === queuePath) queueDataCache = null;
+    if (e?.code !== "ENOENT") {
+      console.error("[LocalAPI] readQueueData: failed to stat queue:", e?.message || e);
+    }
+    return [];
+  }
+
+  if (queueDataCache?.queuePath === queuePath && queueDataCache.signature === signature) {
+    return queueDataCache.rows;
+  }
+
   let raw;
   try {
     raw = fs.readFileSync(queuePath, "utf8");
@@ -196,7 +239,9 @@ function readQueueData(queuePath) {
     const key = `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}`;
     seen.set(key, normalizeQueueRow(row));
   }
-  return Array.from(seen.values());
+  const rows = Array.from(seen.values());
+  queueDataCache = { queuePath, signature, rows };
+  return rows;
 }
 
 function rowDayKey(row, timeZoneContext) {
@@ -214,8 +259,19 @@ function rowDayKey(row, timeZoneContext) {
 }
 
 function aggregateByDay(rows, timeZoneContext = null) {
+  const normalizedRows = Array.isArray(rows) ? rows : [];
+  const cacheKey = timeZoneContextKey(timeZoneContext || {});
+  let cachedByTimeZone = dailyAggregationCache.get(normalizedRows);
+  if (cachedByTimeZone?.has(cacheKey)) {
+    const cached = cachedByTimeZone.get(cacheKey);
+    // Touch the entry so the small per-queue map behaves as an LRU.
+    cachedByTimeZone.delete(cacheKey);
+    cachedByTimeZone.set(cacheKey, cached);
+    return cached;
+  }
+
   const byDay = new Map();
-  for (const row of rows) {
+  for (const row of normalizedRows) {
     if (!row.hour_start) continue;
     const day = rowDayKey(row, timeZoneContext);
     if (!day) continue;
@@ -250,7 +306,13 @@ function aggregateByDay(rows, timeZoneContext = null) {
     const model = row.model || "unknown";
     a.models[model] = (a.models[model] || 0) + (row.total_tokens || 0);
   }
-  return Array.from(byDay.values()).sort((a, b) => a.day.localeCompare(b.day));
+  const daily = Array.from(byDay.values()).sort((a, b) => a.day.localeCompare(b.day));
+  if (!cachedByTimeZone) {
+    cachedByTimeZone = new Map();
+    dailyAggregationCache.set(normalizedRows, cachedByTimeZone);
+  }
+  boundedCacheSet(cachedByTimeZone, cacheKey, daily, MAX_TIME_ZONE_CACHE_ENTRIES);
+  return daily;
 }
 
 function buildCodexCategoryFallbackFromQueue(queueRows, { from, to, timeZoneContext }) {
@@ -477,17 +539,34 @@ function getZonedParts(date, { timeZone, offsetMinutes } = {}) {
 
   if (timeZone && typeof Intl !== "undefined" && Intl.DateTimeFormat) {
     try {
-      const formatter = new Intl.DateTimeFormat("en-CA", {
-        timeZone,
-        year: "numeric",
-        month: "2-digit",
-        day: "2-digit",
-        hour: "2-digit",
-        minute: "2-digit",
-        second: "2-digit",
-        hourCycle: "h23",
-      });
-      const parts = formatter.formatToParts(dt);
+      let formatter;
+      if (zonedPartsFormatters.has(timeZone)) {
+        formatter = zonedPartsFormatters.get(timeZone);
+      } else {
+        try {
+          formatter = new Intl.DateTimeFormat("en-CA", {
+            timeZone,
+            year: "numeric",
+            month: "2-digit",
+            day: "2-digit",
+            hour: "2-digit",
+            minute: "2-digit",
+            second: "2-digit",
+            hourCycle: "h23",
+          });
+        } catch {
+          // Cache invalid zone ids too; otherwise a bad local query would
+          // throw once per queue row before falling back to its fixed offset.
+          formatter = null;
+        }
+        boundedCacheSet(
+          zonedPartsFormatters,
+          timeZone,
+          formatter,
+          MAX_TIME_ZONE_CACHE_ENTRIES,
+        );
+      }
+      const parts = formatter?.formatToParts(dt) || [];
       const values = parts.reduce((acc, part) => {
         if (part.type && part.value) acc[part.type] = part.value;
         return acc;
@@ -685,6 +764,7 @@ function runSyncCommand(extraEnv = {}, opts = {}) {
     const args = [TRACKER_BIN, "sync"];
     if (opts.auto === true) args.push("--auto");
     if (opts.background === true) args.push("--background");
+    if (opts.publishAccount === true) args.push("--publish-account");
     if (opts.allLocalSources === true) args.push("--all-local-sources");
     if (opts.drain === true) args.push("--drain");
     const child = spawn(process.execPath, args, {
@@ -1696,6 +1776,11 @@ function createLocalApiHandler({ queuePath }) {
         const auto = body.auto === true && !drain;
         const background =
           auto && (body.background === true || body.lightweight === true);
+        // The local server is the trust boundary for cloud-sync preferences.
+        // A persisted device token must not let a native background request
+        // upload after the user has explicitly disabled cloud sync.
+        const publishAccount =
+          background && body.publishAccount === true && getCloudSyncPref();
         const allLocalSources = background && body.allLocalSources === true;
         if (typeof body.deviceToken === "string" && body.deviceToken.trim()) {
           extraEnv.TOKENTRACKER_DEVICE_TOKEN = body.deviceToken.trim();
@@ -1710,7 +1795,10 @@ function createLocalApiHandler({ queuePath }) {
           extraEnv.TOKENTRACKER_INSFORGE_BASE_URL = allowedBaseUrl;
           localSyncBaseUrl = allowedBaseUrl;
         }
-        if (!background && !extraEnv.TOKENTRACKER_DEVICE_TOKEN && getCloudSyncPref() && getRefreshTokenForCloud()) {
+        if ((!background || publishAccount) &&
+            !extraEnv.TOKENTRACKER_DEVICE_TOKEN &&
+            getCloudSyncPref() &&
+            getRefreshTokenForCloud()) {
           let issuedToken = null;
           try {
             issuedToken = await issueDeviceTokenForLocalSync(qp, { baseUrl: localSyncBaseUrl });
@@ -1732,6 +1820,7 @@ function createLocalApiHandler({ queuePath }) {
           drain,
           auto,
           background,
+          publishAccount,
           allLocalSources,
         });
         try {
@@ -1778,7 +1867,8 @@ function createLocalApiHandler({ queuePath }) {
       const to = url.searchParams.get("to") || "";
       const timeZoneContext = getTimeZoneContext(url);
       const { rows, scope, excludedSources } = scopedQueueRows(qp, url);
-      const daily = aggregateByDay(rows, timeZoneContext).filter((d) => d.day >= from && d.day <= to);
+      const allDaily = aggregateByDay(rows, timeZoneContext);
+      const daily = allDaily.filter((d) => d.day >= from && d.day <= to);
       const totals = daily.reduce(
         (acc, r) => {
           acc.total_tokens += r.total_tokens;
@@ -1798,7 +1888,6 @@ function createLocalApiHandler({ queuePath }) {
 
       const todayParts = getZonedParts(new Date(), timeZoneContext);
       const todayStr = formatPartsDayKey(todayParts) || new Date().toISOString().slice(0, 10);
-      const allDaily = aggregateByDay(rows, timeZoneContext);
 
       const shiftDay = (dayStr, delta) => {
         const d = new Date(`${dayStr}T00:00:00Z`);
@@ -1860,11 +1949,20 @@ function createLocalApiHandler({ queuePath }) {
       const from = url.searchParams.get("from") || "";
       const to = url.searchParams.get("to") || "";
       const {
-        readOutcomesData,
-        resolveOutcomesPath,
+        readAllOutcomesData,
         computeQualityPerDollar,
       } = require("./outcomes-engine");
-      const outcomes = readOutcomesData(resolveOutcomesPath());
+      // Refresh the independent metadata-only sidecars before joining. Any
+      // scanner failure degrades to the existing manual outcomes file.
+      try {
+        const { buildSessionAnalytics } = require("./session-analytics");
+        const { buildGitOutcomes } = require("./git-outcomes");
+        const sessions = await buildSessionAnalytics();
+        await buildGitOutcomes(sessions);
+      } catch (error) {
+        console.error("[outcomes] automatic Git attribution failed:", error?.message || error);
+      }
+      const outcomes = readAllOutcomesData();
       if (!outcomes.length) {
         json(res, { available: false, from, to, by_model: [], by_tool: [], totals: null });
         return true;
@@ -1875,6 +1973,39 @@ function createLocalApiHandler({ queuePath }) {
       return true;
     }
 
+    // --- metadata-only Claude/Codex session efficiency ---
+    if (p === "/functions/tokentracker-session-insights") {
+      const from = url.searchParams.get("from") || "";
+      const to = url.searchParams.get("to") || "";
+      const refresh = ["1", "true"].includes(url.searchParams.get("refresh"));
+      try {
+        const { buildSessionAnalytics, summarizeSessions, sessionsToCsv } = require("./session-analytics");
+        const sessions = await buildSessionAnalytics({ force: refresh });
+        const wantsCsv = url.searchParams.get("format") === "csv";
+        const includeSessions = wantsCsv || ["1", "true"].includes(url.searchParams.get("include_sessions"));
+        const result = summarizeSessions(sessions, { from, to, includeSessions });
+        if (wantsCsv) {
+          const content = sessionsToCsv(result.sessions);
+          res.statusCode = 200;
+          res.setHeader("Content-Type", "text/csv; charset=utf-8");
+          res.setHeader("Content-Disposition", "attachment; filename=tokentracker-sessions.csv");
+          res.end(content);
+          return true;
+        }
+        json(res, { from, to, ...result });
+      } catch (error) {
+        json(res, { available: false, error: error?.message || "Session analytics failed" }, 500);
+      }
+      return true;
+    }
+
+    // --- fixed context overhead audit (counts and estimates only) ---
+    if (p === "/functions/tokentracker-context-health") {
+      const { computeContextHealth } = require("./context-health");
+      json(res, computeContextHealth({ home: os.homedir(), cwd: process.cwd(), env: process.env }));
+      return true;
+    }
+
     // --- usage-heatmap ---
     if (p === "/functions/tokentracker-usage-heatmap") {
       const weeks = parseInt(url.searchParams.get("weeks") || "52", 10);

package/src/lib/outcomes-engine.js

@@ -27,6 +27,10 @@ function resolveOutcomesPath() {
   return path.join(os.homedir(), ".tokentracker", "tracker", "outcomes.jsonl");
 }
 
+function resolveAutoOutcomesPath() {
+  return path.join(os.homedir(), ".tokentracker", "tracker", "auto-outcomes.jsonl");
+}
+
 // The ONLY fields ever lifted off an outcome record. Anything not on this list
 // — bodies, diffs, messages, arbitrary blobs — is dropped at read time. This is
 // the enforcement point for the metadata-only privacy invariant.
@@ -54,6 +58,10 @@ function sanitizeOutcome(raw) {
   if (typeof raw.task_type === "string" && raw.task_type.trim()) {
     out.task_type = raw.task_type.trim();
   }
+  for (const key of ["status", "session_hash", "commit_hash", "confidence", "methodology"]) {
+    if (typeof raw[key] === "string" && raw[key].trim()) out[key] = raw[key].trim();
+  }
+  if (Number.isFinite(Number(raw.parent_count))) out.parent_count = Number(raw.parent_count);
   return out;
 }
 
@@ -91,6 +99,19 @@ function readOutcomesData(outcomesPath) {
   return out;
 }
 
+function readAllOutcomesData(paths = [resolveOutcomesPath(), resolveAutoOutcomesPath()]) {
+  const dedup = new Map();
+  for (const outcomePath of paths) {
+    for (const row of readOutcomesData(outcomePath)) {
+      const key = row.commit_hash
+        ? `commit:${row.commit_hash}`
+        : `${row.timestamp}|${row.model}|${row.tool}|${row.task_type || ""}`;
+      dedup.set(key, row);
+    }
+  }
+  return [...dedup.values()];
+}
+
 function dayOf(value) {
   return typeof value === "string" ? value.slice(0, 10) : "";
 }
@@ -224,7 +245,9 @@ function computeQualityPerDollar(queueRows, outcomes, { from = "", to = "" } = {
 
 module.exports = {
   resolveOutcomesPath,
+  resolveAutoOutcomesPath,
   sanitizeOutcome,
   readOutcomesData,
+  readAllOutcomesData,
   computeQualityPerDollar,
 };

package/src/lib/pricing/index.js

@@ -37,6 +37,7 @@ const seedRaw = loadSeedSync();
 const state = {
   loaded: false,
   loadingPromise: null,
+  revision: 0,
   litellmRawMap: seedRaw, // raw per-token; field shape from LiteLLM JSON
   litellmPerMillionMap: buildLitellmPerMillionMap(seedRaw), // USD/MTok
   source: Object.keys(seedRaw).length ? "seed-snapshot:sync" : null,
@@ -61,6 +62,7 @@ async function ensurePricingLoaded(opts = {}) {
       state.litellmPerMillionMap = buildLitellmPerMillionMap(state.litellmRawMap);
       state.source = source;
       state.loaded = true;
+      state.revision += 1;
       state.negativeCache.clear();
       return state;
     } finally {
@@ -79,9 +81,14 @@ function resetPricingForTests() {
   state.litellmRawMap = seedRaw;
   state.litellmPerMillionMap = buildLitellmPerMillionMap(seedRaw);
   state.source = Object.keys(seedRaw).length ? "seed-snapshot:sync" : null;
+  state.revision += 1;
   state.negativeCache.clear();
 }
 
+function getPricingRevision() {
+  return state.revision;
+}
+
 function getModelPricing(model, opts = {}) {
   if (!model) return ZERO_PRICING;
   let lookupSource = null;
@@ -133,6 +140,7 @@ const MODEL_PRICING = curatedOverrides.exact;
 
 module.exports = {
   ensurePricingLoaded,
+  getPricingRevision,
   getModelPricing,
   computeRowCost,
   resetPricingForTests,