tokentracker-cli 0.18.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +5 -4
  2. package/README.zh-CN.md +4 -3
  3. package/dashboard/dist/assets/{Card-8ZPdKuRR.js → Card-jA08WeEw.js} +1 -1
  4. package/dashboard/dist/assets/{DashboardPage-OVP6u_7i.js → DashboardPage-chDVOYmG.js} +1 -1
  5. package/dashboard/dist/assets/{FadeIn-BxnaPv7O.js → FadeIn-DqSYXuUL.js} +1 -1
  6. package/dashboard/dist/assets/{HeaderGithubStar-8z6DrTLD.js → HeaderGithubStar-C11rWv0B.js} +1 -1
  7. package/dashboard/dist/assets/{IpCheckPage-yagKgpi7.js → IpCheckPage-CkEZ9yLK.js} +1 -1
  8. package/dashboard/dist/assets/{LandingPage-Ca72J5F0.js → LandingPage-BgckTHRQ.js} +1 -1
  9. package/dashboard/dist/assets/{LeaderboardPage-CSmW4lBz.js → LeaderboardPage-BCNW7UWp.js} +1 -1
  10. package/dashboard/dist/assets/{LeaderboardProfilePage-BOihURRE.js → LeaderboardProfilePage-BLATxMt-.js} +1 -1
  11. package/dashboard/dist/assets/{LimitsPage-Bq4zB2w9.js → LimitsPage-arF--WgR.js} +1 -1
  12. package/dashboard/dist/assets/{LoginPage-CoB1ZkE6.js → LoginPage-DpoFP0va.js} +1 -1
  13. package/dashboard/dist/assets/{PopoverPopup-D7d5-v70.js → PopoverPopup-kdgc2H6C.js} +1 -1
  14. package/dashboard/dist/assets/{ProviderIcon-DzvUcjPu.js → ProviderIcon-DV5r9qqP.js} +1 -1
  15. package/dashboard/dist/assets/{SettingsPage-BeyW1iTj.js → SettingsPage-Bb22ORmU.js} +1 -1
  16. package/dashboard/dist/assets/{SkillsPage-B6auz1NO.js → SkillsPage-xhtBqVKC.js} +1 -1
  17. package/dashboard/dist/assets/{WidgetsPage-C9t8qw0F.js → WidgetsPage-CUoSVDET.js} +1 -1
  18. package/dashboard/dist/assets/{chevron-down-C8RgL-uJ.js → chevron-down-DYb2EChD.js} +1 -1
  19. package/dashboard/dist/assets/{download-C90EEqc8.js → download-C-_8o6dh.js} +1 -1
  20. package/dashboard/dist/assets/{leaderboard-columns-BgqTAms5.js → leaderboard-columns-BgzBlYo7.js} +1 -1
  21. package/dashboard/dist/assets/{main-DJcfmlDf.js → main-11hApDak.js} +6 -4
  22. package/dashboard/dist/assets/{use-limits-display-prefs-BUBBOUIF.js → use-limits-display-prefs-BeGKWUuk.js} +1 -1
  23. package/dashboard/dist/assets/{use-native-settings-CFUEzyoi.js → use-native-settings-nTTHktn0.js} +1 -1
  24. package/dashboard/dist/assets/{use-reduced-motion-NZDZrVKK.js → use-reduced-motion-DU8Gm6j1.js} +1 -1
  25. package/dashboard/dist/assets/{use-usage-limits-CoOOhZrW.js → use-usage-limits-DTPmEB8Y.js} +1 -1
  26. package/dashboard/dist/brand-logos/codebuddy.svg +1 -0
  27. package/dashboard/dist/brand-logos/every-code.svg +1 -0
  28. package/dashboard/dist/brand-logos/grok.svg +1 -0
  29. package/dashboard/dist/brand-logos/hermes.svg +1 -11
  30. package/dashboard/dist/brand-logos/kilo-code.svg +1 -0
  31. package/dashboard/dist/brand-logos/oh-my-pi.svg +1 -0
  32. package/dashboard/dist/index.html +1 -1
  33. package/dashboard/dist/share.html +1 -1
  34. package/package.json +1 -1
  35. package/src/commands/init.js +9 -5
  36. package/src/commands/serve.js +0 -12
  37. package/src/commands/sync.js +370 -7
  38. package/src/lib/grok-hook.js +86 -7
  39. package/src/lib/pricing/curated-overrides.json +1 -1
  40. package/src/lib/pricing/seed-snapshot.json +1 -1
  41. package/src/lib/rollout.js +438 -148
  42. package/src/lib/subscriptions.js +92 -40
  43. package/src/lib/usage-limits.js +1 -1
@@ -14,7 +14,7 @@ const {
14
14
  readOpencodeDbMessages,
15
15
  resolveKiroDbPath,
16
16
  resolveKiroJsonlPath,
17
- resolveHermesDbPath,
17
+ resolveHermesPath,
18
18
  resolveCopilotOtelPaths,
19
19
  parseRolloutIncremental,
20
20
  parseClaudeIncremental,
@@ -48,7 +48,6 @@ const {
48
48
  parseKilocodeIncremental,
49
49
  bucketKey,
50
50
  totalsKey,
51
- groupBucketKey,
52
51
  claudeMessageDedupKey,
53
52
  } = require("../lib/rollout");
54
53
  const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
@@ -73,6 +72,7 @@ const { resolveRuntimeConfig } = require("../lib/runtime-config");
73
72
  const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
74
73
  const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
75
74
  const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
75
+ const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
76
76
  const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
77
77
  // v1 had a cursor-format bug (wrote plain integer instead of {inode, offset,
78
78
  // updatedAt}), which made parseClaudeIncremental reread every jsonl from
@@ -500,13 +500,13 @@ async function cmdSync(argv) {
500
500
 
501
501
  // ── Hermes Agent (SQLite-based) ──
502
502
  let hermesResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
503
- const hermesDbPath = resolveHermesDbPath();
504
- if (fssync.existsSync(hermesDbPath)) {
503
+ const hermesPath = resolveHermesPath();
504
+ if (fssync.existsSync(hermesPath)) {
505
505
  if (progress?.enabled) {
506
506
  progress.start(`Parsing Hermes ${renderBar(0)} | buckets 0`);
507
507
  }
508
508
  hermesResult = await parseHermesIncremental({
509
- dbPath: hermesDbPath,
509
+ hermesPath,
510
510
  cursors,
511
511
  queuePath,
512
512
  onProgress: (p) => {
@@ -681,18 +681,36 @@ async function cmdSync(argv) {
681
681
  ? grokHookSignal.sessionId.trim()
682
682
  : null;
683
683
  if (hookSessionId) {
684
+ const hookContextTokens =
685
+ grokHookSignal.contextTokensUsed != null
686
+ ? grokHookSignal.contextTokensUsed
687
+ : grokHookSignal.totalTokens;
688
+ const hookTotalTokens =
689
+ grokHookSignal.totalTokens != null
690
+ ? grokHookSignal.totalTokens
691
+ : hookContextTokens;
684
692
  grokSessionInputs.unshift({
685
693
  sessionId: hookSessionId,
694
+ sessionDir:
695
+ typeof grokHookSignal.sessionDir === "string" ? grokHookSignal.sessionDir : undefined,
696
+ updatesPath:
697
+ typeof grokHookSignal.updatesPath === "string" ? grokHookSignal.updatesPath : undefined,
698
+ signalsPath:
699
+ typeof grokHookSignal.signalsPath === "string" ? grokHookSignal.signalsPath : undefined,
700
+ summaryPath:
701
+ typeof grokHookSignal.summaryPath === "string" ? grokHookSignal.summaryPath : undefined,
686
702
  signals: {
687
- contextTokensUsed: grokHookSignal.totalTokens,
703
+ contextTokensUsed: hookContextTokens,
704
+ totalTokens: hookTotalTokens,
705
+ totalTokensBeforeCompaction: grokHookSignal.totalTokensBeforeCompaction,
688
706
  assistantMessageCount: grokHookSignal.messageCount,
689
707
  primaryModelId: grokHookSignal.model,
690
708
  lastActiveAt: grokHookSignal.lastActive,
691
709
  },
692
710
  summary: { updated_at: grokHookSignal.lastActive },
693
711
  });
712
+ grokHookSignalConsumed = true;
694
713
  }
695
- grokHookSignalConsumed = true;
696
714
  }
697
715
  if (grokSessionInputs.length > 0) {
698
716
  if (progress?.enabled) {
@@ -717,6 +735,9 @@ async function cmdSync(argv) {
717
735
  bucketsQueued: grokResult.bucketsQueued + grokScanResult.bucketsQueued,
718
736
  };
719
737
  }
738
+ if (opts.repairGrok) {
739
+ await repairGrokQueueFromSessionSnapshots({ cursors, queuePath, queueStatePath });
740
+ }
720
741
 
721
742
  // ── GitHub Copilot CLI (OTEL JSONL files) ──
722
743
  let copilotResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
@@ -899,6 +920,7 @@ function parseArgs(argv) {
899
920
  fromRetry: false,
900
921
  fromOpenclaw: false,
901
922
  drain: false,
923
+ repairGrok: false,
902
924
  };
903
925
  for (let i = 0; i < argv.length; i++) {
904
926
  const a = argv[i];
@@ -907,6 +929,7 @@ function parseArgs(argv) {
907
929
  else if (a === "--from-retry") out.fromRetry = true;
908
930
  else if (a === "--from-openclaw") out.fromOpenclaw = true;
909
931
  else if (a === "--drain") out.drain = true;
932
+ else if (a === "--repair-grok") out.repairGrok = true;
910
933
  else throw new Error(`Unknown option: ${a}`);
911
934
  }
912
935
  return out;
@@ -917,9 +940,11 @@ module.exports = {
917
940
  migrateCursorUnknownBuckets,
918
941
  migrateRolloutCumulativeDeltaBuckets,
919
942
  reincludeClaudeMemObserverFiles,
943
+ repairGrokQueueFromSessionSnapshots,
920
944
  CURSOR_UNKNOWN_MIGRATION_KEY,
921
945
  ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY,
922
946
  CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
947
+ GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
923
948
  };
924
949
 
925
950
  function normalizeString(value) {
@@ -1289,6 +1314,344 @@ async function readQueueBatch(queuePath, startOffset, maxBuckets) {
1289
1314
  return { buckets: Array.from(bucketMap.values()), nextOffset: offset };
1290
1315
  }
1291
1316
 
1317
+ function normalizeGrokRepairSource(value) {
1318
+ return typeof value === "string" ? value.trim().toLowerCase() : "";
1319
+ }
1320
+
1321
+ function normalizeGrokRepairModel(value) {
1322
+ return typeof value === "string" && value.trim() ? value.trim() : "grok-build";
1323
+ }
1324
+
1325
+ function normalizeGrokRepairNumber(value) {
1326
+ const n = Number(value);
1327
+ return Number.isFinite(n) && n > 0 ? n : 0;
1328
+ }
1329
+
1330
+ function toGrokRepairHalfHourStart(value) {
1331
+ if (value == null) return null;
1332
+ const millis =
1333
+ typeof value === "number"
1334
+ ? value < 10_000_000_000
1335
+ ? value * 1000
1336
+ : value
1337
+ : Date.parse(String(value));
1338
+ if (!Number.isFinite(millis)) return null;
1339
+ const halfHourMs = 30 * 60 * 1000;
1340
+ return new Date(Math.floor(millis / halfHourMs) * halfHourMs).toISOString();
1341
+ }
1342
+
1343
+ function estimateGrokRepairTotals(totalTokens, conversationCount) {
1344
+ const total = Math.trunc(normalizeGrokRepairNumber(totalTokens));
1345
+ const inputTokens = Math.round(total * 0.8);
1346
+ const outputTokens = Math.max(0, total - inputTokens);
1347
+ return {
1348
+ input_tokens: inputTokens,
1349
+ cached_input_tokens: 0,
1350
+ cache_creation_input_tokens: 0,
1351
+ output_tokens: outputTokens,
1352
+ reasoning_output_tokens: 0,
1353
+ total_tokens: total,
1354
+ billable_total_tokens: total,
1355
+ conversation_count: Math.trunc(normalizeGrokRepairNumber(conversationCount)),
1356
+ };
1357
+ }
1358
+
1359
+ function addGrokRepairTotals(target, delta) {
1360
+ target.input_tokens += delta.input_tokens;
1361
+ target.cached_input_tokens += delta.cached_input_tokens;
1362
+ target.cache_creation_input_tokens += delta.cache_creation_input_tokens;
1363
+ target.output_tokens += delta.output_tokens;
1364
+ target.reasoning_output_tokens += delta.reasoning_output_tokens;
1365
+ target.total_tokens += delta.total_tokens;
1366
+ target.billable_total_tokens += delta.billable_total_tokens;
1367
+ target.conversation_count += delta.conversation_count;
1368
+ }
1369
+
1370
+ function buildGrokRepairRowsFromSnapshots(sessionSnapshots) {
1371
+ if (!sessionSnapshots || typeof sessionSnapshots !== "object") return [];
1372
+
1373
+ const buckets = new Map();
1374
+ for (const snapshot of Object.values(sessionSnapshots)) {
1375
+ if (!snapshot || typeof snapshot !== "object") continue;
1376
+ const totalTokens = Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens));
1377
+ if (totalTokens <= 0) continue;
1378
+
1379
+ const hourStart = toGrokRepairHalfHourStart(
1380
+ snapshot.lastEventTimestamp || snapshot.updatedAt,
1381
+ );
1382
+ if (!hourStart) continue;
1383
+
1384
+ const model = normalizeGrokRepairModel(snapshot.model);
1385
+ const key = bucketKey("grok", model, hourStart);
1386
+ let totals = buckets.get(key);
1387
+ if (!totals) {
1388
+ totals = {
1389
+ source: "grok",
1390
+ model,
1391
+ hour_start: hourStart,
1392
+ input_tokens: 0,
1393
+ cached_input_tokens: 0,
1394
+ cache_creation_input_tokens: 0,
1395
+ output_tokens: 0,
1396
+ reasoning_output_tokens: 0,
1397
+ total_tokens: 0,
1398
+ billable_total_tokens: 0,
1399
+ conversation_count: 0,
1400
+ };
1401
+ buckets.set(key, totals);
1402
+ }
1403
+ addGrokRepairTotals(
1404
+ totals,
1405
+ estimateGrokRepairTotals(totalTokens, snapshot.messageCount),
1406
+ );
1407
+ }
1408
+
1409
+ return Array.from(buckets.values()).sort((a, b) => {
1410
+ const timeCompare = a.hour_start.localeCompare(b.hour_start);
1411
+ return timeCompare || a.model.localeCompare(b.model);
1412
+ });
1413
+ }
1414
+
1415
+ function applyGrokRepairHourlyState(cursors, rows) {
1416
+ const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
1417
+ const buckets = hourly.buckets && typeof hourly.buckets === "object" ? hourly.buckets : {};
1418
+ const groupQueued =
1419
+ hourly.groupQueued && typeof hourly.groupQueued === "object" ? hourly.groupQueued : {};
1420
+
1421
+ for (const key of Object.keys(buckets)) {
1422
+ if (key.startsWith("grok|")) {
1423
+ delete buckets[key];
1424
+ }
1425
+ }
1426
+ for (const key of Object.keys(groupQueued)) {
1427
+ if (key.startsWith("grok|")) {
1428
+ delete groupQueued[key];
1429
+ }
1430
+ }
1431
+
1432
+ for (const row of rows) {
1433
+ const totals = {
1434
+ input_tokens: row.input_tokens,
1435
+ cached_input_tokens: row.cached_input_tokens,
1436
+ cache_creation_input_tokens: row.cache_creation_input_tokens,
1437
+ output_tokens: row.output_tokens,
1438
+ reasoning_output_tokens: row.reasoning_output_tokens,
1439
+ total_tokens: row.total_tokens,
1440
+ billable_total_tokens: row.billable_total_tokens,
1441
+ conversation_count: row.conversation_count,
1442
+ };
1443
+ buckets[bucketKey("grok", row.model, row.hour_start)] = {
1444
+ totals,
1445
+ queuedKey: totalsKey(totals),
1446
+ source: "grok",
1447
+ hour_start: row.hour_start,
1448
+ };
1449
+ }
1450
+
1451
+ cursors.hourly = {
1452
+ ...hourly,
1453
+ version: 3,
1454
+ buckets,
1455
+ groupQueued,
1456
+ updatedAt: typeof hourly.updatedAt === "string" ? hourly.updatedAt : null,
1457
+ };
1458
+ }
1459
+
1460
+ async function resetGrokRepairUploadOffset(queueStatePath) {
1461
+ if (typeof queueStatePath !== "string" || !queueStatePath) return false;
1462
+ let state = {};
1463
+ try {
1464
+ state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
1465
+ } catch (_e) {
1466
+ state = {};
1467
+ }
1468
+ state.offset = 0;
1469
+ state.updatedAt = new Date().toISOString();
1470
+ state.note = "reset_after_grok_append_only_repair_2026_05_v4";
1471
+ await ensureDir(path.dirname(queueStatePath));
1472
+ await fs.writeFile(queueStatePath, JSON.stringify(state, null, 2) + "\n", "utf8");
1473
+ return true;
1474
+ }
1475
+
1476
+ function hasAppliedGrokRepairMigration(value) {
1477
+ if (!value) return false;
1478
+ if (value === true) return true;
1479
+ if (value && typeof value === "object") {
1480
+ if (value.status === "applied" || value.status === "noop") return true;
1481
+ if (value.status) return false;
1482
+ return value.rowsWritten != null || value.rowsRemoved != null;
1483
+ }
1484
+ return false;
1485
+ }
1486
+
1487
+ function serializeGrokRepairRow(row) {
1488
+ return JSON.stringify({
1489
+ source: "grok",
1490
+ model: normalizeGrokRepairModel(row.model),
1491
+ hour_start: row.hour_start,
1492
+ input_tokens: row.input_tokens || 0,
1493
+ cached_input_tokens: row.cached_input_tokens || 0,
1494
+ cache_creation_input_tokens: row.cache_creation_input_tokens || 0,
1495
+ output_tokens: row.output_tokens || 0,
1496
+ reasoning_output_tokens: row.reasoning_output_tokens || 0,
1497
+ total_tokens: row.total_tokens || 0,
1498
+ billable_total_tokens: row.billable_total_tokens || 0,
1499
+ conversation_count: row.conversation_count || 0,
1500
+ });
1501
+ }
1502
+
1503
+ async function backupExistingFile(filePath) {
1504
+ if (typeof filePath !== "string" || !filePath) return null;
1505
+ try {
1506
+ const stat = await fs.stat(filePath);
1507
+ if (!stat.isFile()) return null;
1508
+ } catch (e) {
1509
+ if (e?.code === "ENOENT" || e?.code === "ENOTDIR") return null;
1510
+ throw e;
1511
+ }
1512
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");
1513
+ const backupPath = `${filePath}.bak.${stamp}`;
1514
+ await fs.copyFile(filePath, backupPath);
1515
+ return backupPath;
1516
+ }
1517
+
1518
+ async function repairGrokQueueFromSessionSnapshots({ cursors, queuePath, queueStatePath } = {}) {
1519
+ if (!cursors || typeof cursors !== "object") return false;
1520
+ const grokState = (cursors.grok ||= {});
1521
+ const migrations = (grokState.migrations ||= {});
1522
+ if (hasAppliedGrokRepairMigration(migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY])) {
1523
+ return false;
1524
+ }
1525
+
1526
+ let raw = "";
1527
+ try {
1528
+ raw = await fs.readFile(queuePath, "utf8");
1529
+ } catch (e) {
1530
+ if (e?.code !== "ENOENT") throw e;
1531
+ }
1532
+
1533
+ const latestGrokRows = new Map();
1534
+ let existingGrokRows = 0;
1535
+ for (const line of raw.split("\n")) {
1536
+ if (!line.trim()) continue;
1537
+ let row;
1538
+ try {
1539
+ row = JSON.parse(line);
1540
+ } catch (_e) {
1541
+ continue;
1542
+ }
1543
+
1544
+ if (normalizeGrokRepairSource(row?.source) === "grok") {
1545
+ const model = normalizeGrokRepairModel(row.model);
1546
+ const hourStart = typeof row.hour_start === "string" ? row.hour_start : null;
1547
+ if (!hourStart) continue;
1548
+ existingGrokRows += 1;
1549
+ latestGrokRows.set(bucketKey("grok", model, hourStart), {
1550
+ ...row,
1551
+ source: "grok",
1552
+ model,
1553
+ hour_start: hourStart,
1554
+ });
1555
+ }
1556
+ }
1557
+
1558
+ if (existingGrokRows === 0) {
1559
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
1560
+ status: "noop",
1561
+ appliedAt: new Date().toISOString(),
1562
+ existingGrokRows: 0,
1563
+ rowsWritten: 0,
1564
+ snapshotsUsed: 0,
1565
+ uploadOffsetReset: false,
1566
+ };
1567
+ return false;
1568
+ }
1569
+
1570
+ const repairRows = buildGrokRepairRowsFromSnapshots(grokState.sessionSnapshots);
1571
+ if (repairRows.length === 0) {
1572
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
1573
+ status: "skipped",
1574
+ appliedAt: new Date().toISOString(),
1575
+ reason: "missing-session-snapshots",
1576
+ existingGrokRows,
1577
+ rowsWritten: 0,
1578
+ snapshotsUsed: 0,
1579
+ uploadOffsetReset: false,
1580
+ };
1581
+ return false;
1582
+ }
1583
+
1584
+ applyGrokRepairHourlyState(cursors, repairRows);
1585
+
1586
+ const repairLines = [];
1587
+ const repairKeys = new Set();
1588
+ for (const row of repairRows) {
1589
+ const key = bucketKey("grok", row.model, row.hour_start);
1590
+ repairKeys.add(key);
1591
+ const current = latestGrokRows.get(key);
1592
+ if (current && totalsKey(current) === totalsKey(row)) continue;
1593
+ repairLines.push(serializeGrokRepairRow(row));
1594
+ }
1595
+
1596
+ const zeroTotals = {
1597
+ input_tokens: 0,
1598
+ cached_input_tokens: 0,
1599
+ cache_creation_input_tokens: 0,
1600
+ output_tokens: 0,
1601
+ reasoning_output_tokens: 0,
1602
+ total_tokens: 0,
1603
+ billable_total_tokens: 0,
1604
+ conversation_count: 0,
1605
+ };
1606
+ let staleRowsRetracted = 0;
1607
+ for (const [key, row] of latestGrokRows.entries()) {
1608
+ if (repairKeys.has(key)) continue;
1609
+ if (totalsKey(row) === totalsKey(zeroTotals)) continue;
1610
+ staleRowsRetracted += 1;
1611
+ repairLines.push(serializeGrokRepairRow({
1612
+ ...zeroTotals,
1613
+ model: row.model,
1614
+ hour_start: row.hour_start,
1615
+ }));
1616
+ }
1617
+
1618
+ if (repairLines.length === 0) {
1619
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
1620
+ status: "noop",
1621
+ appliedAt: new Date().toISOString(),
1622
+ existingGrokRows,
1623
+ rowsWritten: 0,
1624
+ staleRowsRetracted,
1625
+ snapshotsUsed: repairRows.length,
1626
+ uploadOffsetReset: false,
1627
+ };
1628
+ return false;
1629
+ }
1630
+
1631
+ await ensureDir(path.dirname(queuePath));
1632
+ const queueBackupPath = await backupExistingFile(queuePath);
1633
+ const queueStateBackupPath = await backupExistingFile(queueStatePath);
1634
+ await fs.appendFile(queuePath, `${repairLines.join("\n")}\n`, "utf8");
1635
+
1636
+ const uploadOffsetReset = await resetGrokRepairUploadOffset(queueStatePath);
1637
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
1638
+ status: "applied",
1639
+ appliedAt: new Date().toISOString(),
1640
+ existingGrokRows,
1641
+ rowsWritten: repairLines.length,
1642
+ staleRowsRetracted,
1643
+ snapshotsUsed: Object.values(grokState.sessionSnapshots || {}).filter((snapshot) => {
1644
+ if (!snapshot || typeof snapshot !== "object") return false;
1645
+ if (Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens)) <= 0) return false;
1646
+ return Boolean(toGrokRepairHalfHourStart(snapshot.lastEventTimestamp || snapshot.updatedAt));
1647
+ }).length,
1648
+ uploadOffsetReset,
1649
+ queueBackupPath,
1650
+ queueStateBackupPath,
1651
+ };
1652
+ return true;
1653
+ }
1654
+
1292
1655
  async function migrateCursorUnknownBuckets({ cursors, queuePath }) {
1293
1656
  if (!cursors || typeof cursors !== "object") return;
1294
1657
  cursors.migrations = cursors.migrations || {};
@@ -94,14 +94,15 @@ function buildGrokSessionEndHandler({ trackerDir }) {
94
94
  // It must be self-contained enough or rely on the copied runtime.
95
95
  // For simplicity and reliability we write a small script that:
96
96
  // 1. Reads GROK_SESSION_ID + GROK_WORKSPACE_ROOT from env
97
- // 2. Locates the signals.json
98
- // 3. Extracts usage
97
+ // 2. Locates the session metadata files
98
+ // 3. Extracts usage metadata only
99
99
  // 4. Writes a signal file under trackerDir that sync.js will pick up on next run
100
100
 
101
101
  return `#!/usr/bin/env node
102
102
  const fs = require('node:fs');
103
103
  const path = require('node:path');
104
104
  const os = require('node:os');
105
+ const readline = require('node:readline');
105
106
 
106
107
  const GROK_HOME = process.env.TOKENTRACKER_GROK_HOME || process.env.GROK_HOME || path.join(os.homedir(), '.grok');
107
108
  const SESSION_ID = process.env.GROK_SESSION_ID;
@@ -120,14 +121,17 @@ const encodedCwd = encodeGrokCwd(WORKSPACE_ROOT);
120
121
  const sessionDir = path.join(GROK_HOME, 'sessions', encodedCwd, SESSION_ID);
121
122
  const signalsPath = path.join(sessionDir, 'signals.json');
122
123
  const summaryPath = path.join(sessionDir, 'summary.json');
124
+ const updatesPath = path.join(sessionDir, 'updates.jsonl');
123
125
 
124
- let signals = null;
126
+ let signals = {};
125
127
  try {
126
128
  const raw = fs.readFileSync(signalsPath, 'utf8');
127
129
  signals = JSON.parse(raw);
128
130
  } catch (err) {
129
- // Session may still be active or signals not written yet; exit quietly
130
- process.exit(0);
131
+ signals = {};
132
+ }
133
+ if (!signals || typeof signals !== 'object') {
134
+ signals = {};
131
135
  }
132
136
 
133
137
  const summary = (() => {
@@ -143,10 +147,74 @@ function toNonNegativeFiniteNumber(value) {
143
147
  return Number.isFinite(number) && number > 0 ? number : 0;
144
148
  }
145
149
 
146
- const totalTokens = toNonNegativeFiniteNumber(signals.contextTokensUsed);
150
+ function timestampToIso(value) {
151
+ if (value == null) return null;
152
+ if (typeof value === 'number') {
153
+ if (!Number.isFinite(value) || value <= 0) return null;
154
+ const millis = value < 10000000000 ? value * 1000 : value;
155
+ const dt = new Date(millis);
156
+ return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
157
+ }
158
+ if (typeof value === 'string') {
159
+ const trimmed = value.trim();
160
+ if (!trimmed) return null;
161
+ if (/^[0-9]+(?:\\.[0-9]+)?$/.test(trimmed)) return timestampToIso(Number(trimmed));
162
+ const dt = new Date(trimmed);
163
+ return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
164
+ }
165
+ return null;
166
+ }
167
+
168
+ async function readUpdateTelemetry(filePath) {
169
+ let totalTokens = 0;
170
+ let lastEventId = null;
171
+ let lastEventTimestamp = null;
172
+ let lineNumber = 0;
173
+
174
+ try {
175
+ const input = fs.createReadStream(filePath, { encoding: 'utf8' });
176
+ const rl = readline.createInterface({ input, crlfDelay: Infinity });
177
+ try {
178
+ for await (const line of rl) {
179
+ lineNumber += 1;
180
+ if (!line || !line.trim()) continue;
181
+ let record = null;
182
+ try {
183
+ record = JSON.parse(line);
184
+ } catch {
185
+ continue;
186
+ }
187
+ const meta = record && record.params && record.params._meta ? record.params._meta : record && record._meta;
188
+ if (!meta || typeof meta !== 'object') continue;
189
+ const nextTotal = toNonNegativeFiniteNumber(meta.totalTokens);
190
+ if (nextTotal <= 0) continue;
191
+ totalTokens = Math.max(totalTokens, nextTotal);
192
+ lastEventId = meta.eventId != null ? String(meta.eventId) : String(lineNumber);
193
+ lastEventTimestamp =
194
+ timestampToIso(meta.agentTimestampMs) ||
195
+ timestampToIso(meta.timestampMs) ||
196
+ timestampToIso(record.timestamp_ms) ||
197
+ timestampToIso(record.timestamp) ||
198
+ lastEventTimestamp;
199
+ }
200
+ } finally {
201
+ rl.close();
202
+ }
203
+ } catch {
204
+ return { totalTokens, lastEventId, lastEventTimestamp };
205
+ }
206
+ return { totalTokens, lastEventId, lastEventTimestamp };
207
+ }
208
+
209
+ async function main() {
210
+ const contextTokensUsed = toNonNegativeFiniteNumber(signals.contextTokensUsed ?? signals.totalTokens);
211
+ const totalTokensBeforeCompaction = toNonNegativeFiniteNumber(signals.totalTokensBeforeCompaction);
212
+ const signalTotalTokens = totalTokensBeforeCompaction + contextTokensUsed;
213
+ const updateTelemetry = await readUpdateTelemetry(updatesPath);
214
+ const totalTokens = Math.max(updateTelemetry.totalTokens, signalTotalTokens);
147
215
  const messageCount = toNonNegativeFiniteNumber(signals.assistantMessageCount || signals.num_chat_messages);
148
216
  const model = signals.primaryModelId || (Array.isArray(signals.modelsUsed) ? signals.modelsUsed[0] : 'grok-build');
149
- const lastActive = signals.lastActiveAt || summary.updated_at || new Date().toISOString();
217
+ const lastActive = signals.lastActiveAt || updateTelemetry.lastEventTimestamp || summary.updated_at || new Date().toISOString();
150
218
 
151
219
  if (totalTokens <= 0) {
152
220
  process.exit(0);
@@ -163,8 +231,16 @@ const signal = {
163
231
  cwd: WORKSPACE_ROOT,
164
232
  model,
165
233
  totalTokens,
234
+ contextTokensUsed,
235
+ totalTokensBeforeCompaction,
166
236
  messageCount,
167
237
  lastActive,
238
+ sessionDir,
239
+ updatesPath,
240
+ signalsPath,
241
+ summaryPath,
242
+ lastEventId: updateTelemetry.lastEventId,
243
+ lastEventTimestamp: updateTelemetry.lastEventTimestamp,
168
244
  capturedAt: new Date().toISOString()
169
245
  };
170
246
 
@@ -177,6 +253,9 @@ try {
177
253
  } catch {}
178
254
 
179
255
  process.exit(0);
256
+ }
257
+
258
+ main().catch(() => process.exit(0));
180
259
  `;
181
260
  }
182
261
 
@@ -18,7 +18,7 @@
18
18
  "deepseek-v4-flash":{ "input": 0.14, "output": 0.28, "cache_read": 0.0028, "cache_write": 0.14 },
19
19
  "deepseek-v4-pro": { "input": 0.435,"output": 0.87, "cache_read": 0.003625, "cache_write": 0.435 },
20
20
  "deepseek-chat": { "input": 0.14, "output": 0.28, "cache_read": 0.0028, "cache_write": 0.14 },
21
- "grok-build": { "input": 1.25, "output": 2.50, "cache_read": 0.20, "note": "Grok Build TUI estimate. signals.json currently exposes contextTokensUsed snapshots, so TokenTracker estimates input/output split until Grok exposes per-call telemetry." },
21
+ "grok-build": { "input": 1.25, "output": 2.50, "cache_read": 0.20, "note": "Grok Build TUI estimate. Local telemetry currently exposes totalTokens without a stable prompt/output/cache split, so TokenTracker estimates input/output split until Grok exposes per-call usage details." },
22
22
  "grok-4-0709": { "input": 3.00, "output": 15.00, "cache_read": 0.75 },
23
23
  "grok-4": { "input": 3.00, "output": 15.00, "cache_read": 0.75 },
24
24
  "grok-4-latest": { "input": 3.00, "output": 15.00, "cache_read": 0.75 },