tokelytics 0.3.5 → 0.3.6

package/bin/tokelytics.mjs

@@ -85,9 +85,20 @@ function mergeDashboardSnapshot(previous, turns, limits, device, now = /* @__PUR
   const next = structuredClone(previous ?? emptyDashboardSnapshot(now));
   next.version = 2;
   if (turns.length > 0) {
-    next.recentTurns = mergeRecentTurns(next.recentTurns, turns);
-    next.sessions = mergeSessions(next.sessions, turns);
-    next.rollups = mergeRollups(next.rollups, turns, now);
+    const previousById = new Map(next.recentTurns.map((turn) => [turn.turnId, turn]));
+    const replacements = /* @__PURE__ */ new Map();
+    const changedTurns = turns.filter((turn) => {
+      const prior = previousById.get(turn.turnId);
+      if (!prior)
+        return true;
+      if (sameTurn(prior, turn))
+        return false;
+      replacements.set(turn.turnId, prior);
+      return true;
+    });
+    next.recentTurns = mergeRecentTurns(next.recentTurns, changedTurns);
+    next.sessions = mergeSessions(next.sessions, changedTurns, replacements);
+    next.rollups = mergeRollups(next.rollups, changedTurns, now, replacements);
   }
   if (limits.length > 0)
     next.limits = mergeLimits(next.limits, limits);
@@ -130,6 +141,9 @@ function mergeRecentTurns(existing, incoming) {
     byId.set(turn.turnId, turn);
   return [...byId.values()].sort((a, b) => a.ts.localeCompare(b.ts)).slice(-SNAPSHOT_RECENT_TURNS);
 }
+function sameTurn(left, right) {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
 function sanitizeTurn(turn) {
   return {
     ...turn,
@@ -156,17 +170,30 @@ function safePath(value) {
   const parts = normalized.split("/").filter((part) => part && part !== "." && part !== ".." && !/^[A-Za-z]:$/.test(part));
   return parts.slice(-3).join("/").slice(-200);
 }
-function mergeSessions(existing, turns) {
+function mergeSessions(existing, turns, replacements = /* @__PURE__ */ new Map()) {
   const byId = new Map(existing.map((session) => [session.sessionId, { ...session }]));
   for (const turn of turns) {
-    const session = byId.get(turn.sessionId) ?? emptySession(turn);
-    session.turnCount++;
-    session.inputTokens += turn.inputTokens;
-    session.outputTokens += turn.outputTokens;
-    session.cacheReadTokens += turn.cacheReadTokens;
-    session.cacheCreationTokens += turn.cacheCreationTokens;
-    session.cachedInputTokens += turn.cachedInputTokens;
-    session.reasoningTokens += turn.reasoningTokens;
+    const prior = replacements.get(turn.turnId);
+    if (prior)
+      applySessionTurn(byId, prior, -1);
+    applySessionTurn(byId, turn, 1);
+  }
+  return [...byId.values()].sort((a, b) => b.lastTs.localeCompare(a.lastTs)).slice(0, SNAPSHOT_SESSIONS);
+}
+function applySessionTurn(byId, turn, direction) {
+  const session = byId.get(turn.sessionId) ?? emptySession(turn);
+  session.turnCount += direction;
+  session.inputTokens = Math.max(0, session.inputTokens + direction * turn.inputTokens);
+  session.outputTokens = Math.max(0, session.outputTokens + direction * turn.outputTokens);
+  session.cacheReadTokens = Math.max(0, session.cacheReadTokens + direction * turn.cacheReadTokens);
+  session.cacheCreationTokens = Math.max(0, session.cacheCreationTokens + direction * turn.cacheCreationTokens);
+  session.cachedInputTokens = Math.max(0, session.cachedInputTokens + direction * turn.cachedInputTokens);
+  session.reasoningTokens = Math.max(0, session.reasoningTokens + direction * turn.reasoningTokens);
+  if (session.turnCount <= 0) {
+    byId.delete(turn.sessionId);
+    return;
+  }
+  if (direction > 0) {
     if (!session.firstTs || turn.ts < session.firstTs)
       session.firstTs = turn.ts;
     if (!session.lastTs || turn.ts > session.lastTs)
@@ -178,9 +205,8 @@ function mergeSessions(existing, turns) {
     if (turn.model && (!session.model || modelPriority(turn.model) > modelPriority(session.model))) {
       session.model = turn.model;
     }
-    byId.set(turn.sessionId, session);
   }
-  return [...byId.values()].sort((a, b) => b.lastTs.localeCompare(a.lastTs)).slice(0, SNAPSHOT_SESSIONS);
+  byId.set(turn.sessionId, session);
 }
 function emptySession(turn) {
   return {
@@ -200,48 +226,62 @@ function emptySession(turn) {
     reasoningTokens: 0
   };
 }
-function mergeRollups(existing, turns, now) {
+function mergeRollups(existing, turns, now, replacements = /* @__PURE__ */ new Map()) {
   const byId = new Map(existing.map((rollup) => [rollup.id, structuredClone(rollup)]));
   for (const turn of turns) {
-    const day = dayOf(turn.ts);
-    if (!day)
-      continue;
-    const id = rollupId(turn.provider, day);
-    const rollup = byId.get(id) ?? { id, provider: turn.provider, day, models: {}, hourly: [] };
-    const model = turn.model || "unknown";
-    const totals = rollup.models[model] ??= {
-      model,
-      turns: 0,
-      inputTokens: 0,
-      outputTokens: 0,
-      cacheReadTokens: 0,
-      cacheCreationTokens: 0,
-      cachedInputTokens: 0,
-      reasoningTokens: 0
-    };
-    totals.turns++;
-    totals.inputTokens += turn.inputTokens;
-    totals.outputTokens += turn.outputTokens;
-    totals.cacheReadTokens += turn.cacheReadTokens;
-    totals.cacheCreationTokens += turn.cacheCreationTokens;
-    totals.cachedInputTokens += turn.cachedInputTokens;
-    totals.reasoningTokens += turn.reasoningTokens;
-    const hour = hourOf(turn.ts);
-    if (hour !== null) {
-      const bucket = rollup.hourly.find((item) => item.hour === hour) ?? { hour, output: 0, turns: 0 };
-      bucket.output += turn.outputTokens;
-      bucket.turns++;
-      if (!rollup.hourly.some((item) => item.hour === hour))
-        rollup.hourly.push(bucket);
-      rollup.hourly.sort((a, b) => a.hour - b.hour);
-    }
-    byId.set(id, rollup);
+    const prior = replacements.get(turn.turnId);
+    if (prior)
+      applyRollupTurn(byId, prior, -1);
+    applyRollupTurn(byId, turn, 1);
   }
   const cutoff = new Date(now);
   cutoff.setUTCDate(cutoff.getUTCDate() - (SNAPSHOT_RETENTION_DAYS - 1));
   const cutoffDay = cutoff.toISOString().slice(0, 10);
   return [...byId.values()].filter((rollup) => rollup.day >= cutoffDay).sort((a, b) => a.day.localeCompare(b.day) || a.provider.localeCompare(b.provider));
 }
+function applyRollupTurn(byId, turn, direction) {
+  const day = dayOf(turn.ts);
+  if (!day)
+    return;
+  const id = rollupId(turn.provider, day);
+  const rollup = byId.get(id) ?? { id, provider: turn.provider, day, models: {}, hourly: [] };
+  const model = turn.model || "unknown";
+  const totals = rollup.models[model] ??= {
+    model,
+    turns: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheCreationTokens: 0,
+    cachedInputTokens: 0,
+    reasoningTokens: 0
+  };
+  totals.turns += direction;
+  totals.inputTokens = Math.max(0, totals.inputTokens + direction * turn.inputTokens);
+  totals.outputTokens = Math.max(0, totals.outputTokens + direction * turn.outputTokens);
+  totals.cacheReadTokens = Math.max(0, totals.cacheReadTokens + direction * turn.cacheReadTokens);
+  totals.cacheCreationTokens = Math.max(0, totals.cacheCreationTokens + direction * turn.cacheCreationTokens);
+  totals.cachedInputTokens = Math.max(0, totals.cachedInputTokens + direction * turn.cachedInputTokens);
+  totals.reasoningTokens = Math.max(0, totals.reasoningTokens + direction * turn.reasoningTokens);
+  if (totals.turns <= 0)
+    delete rollup.models[model];
+  const hour = hourOf(turn.ts);
+  if (hour !== null) {
+    const bucket = rollup.hourly.find((item) => item.hour === hour) ?? { hour, output: 0, turns: 0 };
+    bucket.output = Math.max(0, bucket.output + direction * turn.outputTokens);
+    bucket.turns += direction;
+    if (bucket.turns <= 0) {
+      rollup.hourly = rollup.hourly.filter((item) => item.hour !== hour);
+    } else if (!rollup.hourly.some((item) => item.hour === hour)) {
+      rollup.hourly.push(bucket);
+    }
+    rollup.hourly.sort((a, b) => a.hour - b.hour);
+  }
+  if (Object.keys(rollup.models).length === 0)
+    byId.delete(id);
+  else
+    byId.set(id, rollup);
+}
 function mergeLimits(existing, incoming) {
   const byProvider = new Map(existing.map((limit) => [limit.provider, limit]));
   for (const limit of incoming)
@@ -1802,7 +1842,7 @@ var FirestoreSink = class {
 };
 
 // src/version.ts
-var AGENT_VERSION = "0.3.5";
+var AGENT_VERSION = "0.3.6";
 
 // src/sync.ts
 async function runSync(connectors, sink, state, device, limitCollectors = [], options = {}) {
@@ -1817,15 +1857,11 @@ async function runSync(connectors, sink, state, device, limitCollectors = [], op
   let st = state;
   const collected = [];
   const limits = [];
-  const byProvider = {};
   for (const c of connectors) {
     const { turns: turns2, limits: nativeLimits2 = [], state: next } = await c.collect(st);
     st = next;
     if (refreshLimits) limits.push(...nativeLimits2);
-    for (const t of turns2) {
-      collected.push(t);
-      byProvider[t.provider] = (byProvider[t.provider] ?? 0) + 1;
-    }
+    collected.push(...turns2);
   }
   if (refreshLimits) {
     const providerLimits = await Promise.all(
@@ -1841,7 +1877,16 @@ async function runSync(connectors, sink, state, device, limitCollectors = [], op
   }
   const unique = /* @__PURE__ */ new Map();
   for (const t of collected) unique.set(t.turnId, t);
-  const turns = [...unique.values()];
+  const collectedTurns = [...unique.values()];
+  const previousTurns = new Map((options.dashboard?.recentTurns ?? []).map((turn) => [turn.turnId, turn]));
+  const turns = collectedTurns.filter((turn) => {
+    const previous = previousTurns.get(turn.turnId);
+    return !previous || !sameTurn(previous, turn);
+  });
+  const changedByProvider = {};
+  for (const turn of turns) {
+    changedByProvider[turn.provider] = (changedByProvider[turn.provider] ?? 0) + 1;
+  }
   const nativeLimits = refreshLimits ? dedupeLimits(limits) : [];
   const changedLimits = nativeLimits.filter(
     (limit) => publication.limitFingerprints?.[limit.provider] !== limitFingerprint(limit)
@@ -1907,14 +1952,18 @@ async function runSync(connectors, sink, state, device, limitCollectors = [], op
       publicationError = error;
     }
   }
+  const scheduledCloudWrite = publication.dashboardDirty && hasBudget && !options.suppressCloudPublish ? Number.isFinite(Date.parse(publication.lastCloudWriteAt ?? "")) ? new Date(
+    Math.max(now.getTime(), Date.parse(publication.lastCloudWriteAt) + cloudSyncIntervalMs)
+  ).toISOString() : now.toISOString() : void 0;
   return {
     processedTurns: turns.length,
     processedLimits: changedLimits.length,
-    byProvider,
+    byProvider: changedByProvider,
     state: st,
     dashboard,
     publication,
     published,
+    nextCloudWriteAt: scheduledCloudWrite,
     publicationError
   };
 }
@@ -2005,6 +2054,7 @@ async function createRunner() {
         byProvider: res.byProvider,
         published: res.published,
         cloudWritesToday: res.publication.cloudWritesToday ?? 0,
+        nextCloudWriteAt: res.nextCloudWriteAt,
         publicationError: res.publicationError
       };
     }
@@ -3798,7 +3848,7 @@ function close(server) {
 }
 
 // src/watch.ts
-var DEBOUNCE_MS = 1200;
+var TURN_SETTLE_MS = 2500;
 var LIMIT_REFRESH_MS = 6e4;
 var FALLBACK_SCAN_MS = 5e3;
 var DEVICE_HEARTBEAT_MS = 60 * 6e4;
@@ -3807,6 +3857,7 @@ var MAX_CLOUD_WRITES_PER_DAY = 16;
 var QUOTA_RETRY_MS = 15 * 6e4;
 var QUOTA_RETRY_JITTER_MS = 5 * 6e4;
 async function watch2(runner) {
+  const logState = {};
   let localServer;
   try {
     localServer = await startLocalDashboardServer(() => runner.localSnapshot(), runner.uid);
@@ -3821,7 +3872,7 @@ async function watch2(runner) {
     deviceHeartbeatMs: DEVICE_HEARTBEAT_MS,
     cloudSyncIntervalMs: CLOUD_SYNC_INTERVAL_MS,
     maxCloudWritesPerDay: MAX_CLOUD_WRITES_PER_DAY
-  }, false);
+  }, false, logState);
   const paths = runner.watchPaths();
   const watcher = esm_default.watch(paths, {
     ignoreInitial: true,
@@ -3835,7 +3886,12 @@ async function watch2(runner) {
   let running = false;
   let scheduledAt = 0;
   let cloudBlockedUntil = startupRetryAfter ? Date.now() + startupRetryAfter : 0;
-  const schedule = (request, delay = DEBOUNCE_MS) => {
+  const takePending = () => {
+    const request = pending;
+    pending = void 0;
+    return request;
+  };
+  const schedule = (request, delay = TURN_SETTLE_MS) => {
     pending = mergeRequests(pending, request);
     if (running) return;
     const target = Date.now() + delay;
@@ -3845,21 +3901,19 @@ async function watch2(runner) {
     timer = setTimeout(async () => {
       timer = void 0;
       scheduledAt = 0;
-      const next = pending;
-      pending = void 0;
+      const next = takePending();
       if (!next) return;
       running = true;
-      const retryAfterMs = await safeSync(runner, next, Date.now() < cloudBlockedUntil);
+      const retryAfterMs = await safeSync(runner, next, Date.now() < cloudBlockedUntil, logState);
       running = false;
       if (retryAfterMs) {
         cloudBlockedUntil = Date.now() + retryAfterMs;
       } else if (Date.now() >= cloudBlockedUntil) {
         cloudBlockedUntil = 0;
       }
-      if (pending) {
-        const queued = pending;
-        pending = void 0;
-        schedule(queued, 0);
+      const queued = takePending();
+      if (queued) {
+        schedule(queued, queued.reason === "change" ? TURN_SETTLE_MS : 0);
       }
     }, Math.max(0, target - Date.now()));
   };
@@ -3899,21 +3953,37 @@ Press Ctrl+C to stop.`);
     process.once("SIGTERM", stop);
   });
 }
-async function safeSync(runner, request, suppressCloudPublish) {
+async function safeSync(runner, request, suppressCloudPublish, logState) {
   try {
     const { reason, ...options } = request;
-    const { processedTurns, byProvider, published, cloudWritesToday, publicationError } = await runner.syncOnce({
+    const {
+      processedTurns,
+      byProvider,
+      published,
+      cloudWritesToday,
+      nextCloudWriteAt,
+      publicationError
+    } = await runner.syncOnce({
       ...options,
       suppressCloudPublish
     });
     if (processedTurns > 0) {
       const detail = Object.entries(byProvider).map(([p, n]) => `${p}:${n}`).join(" ");
       console.log(
-        `[${(/* @__PURE__ */ new Date()).toLocaleTimeString()}] processed ${processedTurns} turn(s) locally (${detail}); ` + (published ? `cloud snapshot published (${cloudWritesToday}/${MAX_CLOUD_WRITES_PER_DAY} today)` : "cloud backup queued")
+        `[${(/* @__PURE__ */ new Date()).toLocaleTimeString()}] processed ${processedTurns} completed turn(s) locally (${detail}).`
       );
     } else if (reason === "startup") {
+      console.log(`[${(/* @__PURE__ */ new Date()).toLocaleTimeString()}] local snapshot ready.`);
+    }
+    if (published) {
+      logState.cloudSchedule = void 0;
+      console.log(
+        `[${(/* @__PURE__ */ new Date()).toLocaleTimeString()}] cloud snapshot published (${cloudWritesToday}/${MAX_CLOUD_WRITES_PER_DAY} writes today).`
+      );
+    } else if (nextCloudWriteAt && nextCloudWriteAt !== logState.cloudSchedule) {
+      logState.cloudSchedule = nextCloudWriteAt;
       console.log(
-        `[${(/* @__PURE__ */ new Date()).toLocaleTimeString()}] local snapshot ready; ` + (published ? `cloud snapshot published (${cloudWritesToday}/${MAX_CLOUD_WRITES_PER_DAY} today)` : "cloud backup queued")
+        `Cloud backup scheduled for ${new Date(nextCloudWriteAt).toLocaleTimeString()} (new turns will be combined into this snapshot).`
       );
     }
     if (publicationError) return cloudRetryDelay(publicationError);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokelytics",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Tokelytics sync agent — streams local AI CLI usage logs (Claude Code, Codex) to your Tokelytics dashboard.",
   "license": "MIT",
   "type": "module",