npm - tokentracker-cli - Versions diffs - 0.9.0 → 0.10.1 - Mend

tokentracker-cli 0.9.0 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (11) hide show

package/dashboard/dist/assets/main-Bst6S3yM.css +1 -0
package/dashboard/dist/assets/{main-DDgbwHx4.js → main-Cw4csGy9.js} +204 -190
package/dashboard/dist/index.html +2 -2
package/dashboard/dist/share.html +2 -2
package/package.json +1 -1
package/src/commands/sync.js +181 -0
package/src/lib/claude-categorizer.js +759 -0
package/src/lib/local-api.js +29 -0
package/src/lib/pricing/seed-snapshot.json +1 -1
package/src/lib/rollout.js +5 -0
package/dashboard/dist/assets/main-HLMqEvtH.css +0 -1

package/dashboard/dist/index.html CHANGED Viewed

@@ -210,8 +210,8 @@
         ]
       }
     </script>
-    <script type="module" crossorigin src="/assets/main-DDgbwHx4.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/main-HLMqEvtH.css">
+    <script type="module" crossorigin src="/assets/main-Cw4csGy9.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/main-Bst6S3yM.css">
   </head>
   <body>
     <main class="aeo-seed-content" aria-label="Token Tracker AI-readable summary">

package/dashboard/dist/share.html CHANGED Viewed

@@ -51,8 +51,8 @@
         "description": "Shareable Token Tracker dashboard snapshot."
       }
     </script>
-    <script type="module" crossorigin src="/assets/main-DDgbwHx4.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/main-HLMqEvtH.css">
+    <script type="module" crossorigin src="/assets/main-Cw4csGy9.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/main-Bst6S3yM.css">
   </head>
   <body>
     <main class="aeo-seed-content" aria-label="Token Tracker share page summary">

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "tokentracker-cli",
-  "version": "0.9.0",
+  "version": "0.10.1",
   "description": "Token usage tracker for AI agent CLIs (Claude Code, Codex, Cursor, Gemini, Kiro, OpenCode, OpenClaw, Every Code, Hermes, GitHub Copilot, Kimi Code, CodeBuddy, oh-my-pi, pi, Craft Agents)",
   "main": "src/cli.js",
   "bin": {

package/src/commands/sync.js CHANGED Viewed

@@ -40,7 +40,11 @@ const {
   resolveKiroCliSessionFiles,
   resolveKiroCliDbPath,
   parseKiroCliIncremental,
+  bucketKey,
+  totalsKey,
+  groupBucketKey,
 } = require("../lib/rollout");
+const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
 const { createProgress, renderBar, formatNumber, formatBytes } = require("../lib/progress");
 const {
   normalizeState: normalizeUploadState,
@@ -63,6 +67,11 @@ const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
 const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
 const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
 const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
+// v1 had a cursor-format bug (wrote plain integer instead of {inode, offset,
+// updatedAt}), which made parseClaudeIncremental reread every jsonl from
+// byte 0 on the next sync and double everything. v2 fixes the format and
+// re-runs the repair regardless of whether v1 already applied.
+const CLAUDE_GROUND_TRUTH_REPAIR_KEY = "claudeGroundTruthRepair_2026_05_v2";
 async function cmdSync(argv) {
   const opts = parseArgs(argv);
@@ -178,6 +187,7 @@ async function cmdSync(argv) {
     const claudeFiles = await listClaudeProjectFiles(claudeProjectsDir);
     await reincludeClaudeMemObserverFiles({ cursors, claudeFiles, queuePath });
+    await repairClaudeQueueFromGroundTruth({ cursors, queuePath, queueStatePath });
     let claudeResult = { filesProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
     if (claudeFiles.length > 0) {
       if (progress?.enabled) {
@@ -1196,6 +1206,177 @@ async function migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rollou
   cursors.migrations[ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY] = new Date().toISOString();
 }
+// One-time repair migration: rebuild source=claude rows in queue.jsonl from
+// the actual jsonl files using ccusage's algorithm (msgId+reqId global
+// dedup). Earlier `reincludeClaudeMemObserverFiles` versions (v1/v2/v3) each
+// reset the hash set and re-read observer jsonls, which silently inflated
+// queue.jsonl's claude totals by ~40%. We do an atomic rewrite — keep all
+// non-claude rows verbatim, replace every claude/claude-mem row with the
+// ground-truth set — then reset cursors so the next incremental sync stays
+// in sync, and reset the cloud upload offset so the corrected rows actually
+// reach the cloud (the ingest endpoint upserts by (source, model,
+// hour_start), so re-uploading other sources is idempotent).
+async function repairClaudeQueueFromGroundTruth({ cursors, queuePath, queueStatePath = null }) {
+  if (!cursors || typeof cursors !== "object") return false;
+  const migrations = (cursors.migrations ||= {});
+  if (migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY]) return false;
+  let result;
+  try {
+    result = await computeClaudeGroundTruthBuckets();
+  } catch (e) {
+    console.error("[sync] claude ground-truth repair: scan failed:", e?.message || e);
+    return false;
+  }
+  const { rows, seenHashes, fileList } = result;
+  // 1. Atomic rewrite of queue.jsonl: keep non-claude rows, drop existing
+  //    claude/claude-mem rows, append truth rows. Atomic via tmp + rename.
+  let claudeRowsRemoved = 0;
+  if (typeof queuePath === "string" && queuePath) {
+    let raw = "";
+    try {
+      raw = await fs.readFile(queuePath, "utf8");
+    } catch (e) {
+      if (e?.code !== "ENOENT") throw e;
+    }
+    const keptLines = [];
+    for (const line of raw.split("\n")) {
+      if (!line.trim()) continue;
+      let row;
+      try {
+        row = JSON.parse(line);
+      } catch (_e) {
+        // Preserve unparseable lines verbatim — operator may want to
+        // recover them later.
+        keptLines.push(line);
+        continue;
+      }
+      if (row?.source === "claude" || row?.source === "claude-mem") {
+        claudeRowsRemoved += 1;
+        continue;
+      }
+      keptLines.push(line);
+    }
+    const truthLines = rows.map((r) =>
+      JSON.stringify({
+        source: "claude",
+        model: r.model,
+        hour_start: r.hour_start,
+        input_tokens: r.input_tokens,
+        cached_input_tokens: r.cached_input_tokens,
+        cache_creation_input_tokens: r.cache_creation_input_tokens,
+        output_tokens: r.output_tokens,
+        reasoning_output_tokens: r.reasoning_output_tokens,
+        total_tokens: r.total_tokens,
+        billable_total_tokens: r.billable_total_tokens,
+        conversation_count: r.conversation_count,
+      }),
+    );
+    await ensureDir(path.dirname(queuePath));
+    const out = keptLines.concat(truthLines).join("\n") + "\n";
+    const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
+    await fs.writeFile(tmp, out, "utf8");
+    await fs.rename(tmp, queuePath);
+  }
+  // 2. Reset cursors.hourly.buckets / groupQueued for source=claude (and the
+  //    dead source=claude-mem buckets) so incremental sync's in-memory state
+  //    matches the truth.
+  const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
+  hourly.buckets ||= {};
+  hourly.groupQueued ||= {};
+  let bucketsCleared = 0;
+  for (const k of Object.keys(hourly.buckets)) {
+    if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
+      delete hourly.buckets[k];
+      bucketsCleared += 1;
+    }
+  }
+  for (const k of Object.keys(hourly.groupQueued)) {
+    if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
+      delete hourly.groupQueued[k];
+    }
+  }
+  for (const r of rows) {
+    const totals = {
+      input_tokens: r.input_tokens,
+      cached_input_tokens: r.cached_input_tokens,
+      cache_creation_input_tokens: r.cache_creation_input_tokens,
+      output_tokens: r.output_tokens,
+      reasoning_output_tokens: r.reasoning_output_tokens,
+      total_tokens: r.total_tokens,
+      billable_total_tokens: r.billable_total_tokens,
+      conversation_count: r.conversation_count,
+    };
+    const key = bucketKey("claude", r.model, r.hour_start);
+    hourly.buckets[key] = {
+      totals,
+      queuedKey: totalsKey(totals),
+      source: "claude",
+      hour_start: r.hour_start,
+    };
+    hourly.groupQueued[groupBucketKey("claude", r.hour_start)] = totalsKey(totals);
+  }
+  // 3. Reset per-file cursors so future incremental sync only reads genuinely
+  //    new tail content. Format must match what rollout.js expects:
+  //    { inode, offset, updatedAt }. Setting a plain integer here breaks
+  //    the inode-equality check inside parseClaudeFile, which would treat
+  //    the file as untracked and re-read it from byte 0 — silently doubling
+  //    everything. (That was the actual cause of the regression after the
+  //    first repair attempt.)
+  cursors.files ||= {};
+  let filesReset = 0;
+  const nowIso = new Date().toISOString();
+  for (const fp of fileList) {
+    let st;
+    try {
+      st = fssync.statSync(fp);
+    } catch (_e) {
+      continue;
+    }
+    cursors.files[fp] = {
+      inode: st.ino || 0,
+      offset: st.size,
+      updatedAt: nowIso,
+    };
+    filesReset += 1;
+  }
+  cursors.claudeHashes = seenHashes;
+  // 4. Reset cloud-upload offset so the corrected rows are re-sent. Other
+  //    sources are upserted idempotently by the ingest endpoint, so this is
+  //    safe — just costs one extra round of bandwidth.
+  if (typeof queueStatePath === "string" && queueStatePath) {
+    let uploadState = {};
+    try {
+      uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
+    } catch (_e) {
+      uploadState = {};
+    }
+    uploadState.offset = 0;
+    uploadState.updatedAt = new Date().toISOString();
+    uploadState.note = "reset_after_claude_repair_2026_05_v1";
+    await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
+  }
+  migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY] = {
+    appliedAt: new Date().toISOString(),
+    bucketsWritten: rows.length,
+    bucketsCleared,
+    rowsRemoved: claudeRowsRemoved,
+    filesReset,
+    hashesRetained: seenHashes.length,
+    uploadOffsetReset: typeof queueStatePath === "string" && !!queueStatePath,
+  };
+  return true;
+}
 async function reincludeClaudeMemObserverFiles({ cursors, claudeFiles, queuePath }) {
   if (!cursors || typeof cursors !== "object") return false;
   const migrations = (cursors.migrations ||= {});