tokentracker-cli 0.18.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/README.zh-CN.md +4 -3
- package/dashboard/dist/assets/{Card-8ZPdKuRR.js → Card-jA08WeEw.js} +1 -1
- package/dashboard/dist/assets/{DashboardPage-OVP6u_7i.js → DashboardPage-chDVOYmG.js} +1 -1
- package/dashboard/dist/assets/{FadeIn-BxnaPv7O.js → FadeIn-DqSYXuUL.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-8z6DrTLD.js → HeaderGithubStar-C11rWv0B.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-yagKgpi7.js → IpCheckPage-CkEZ9yLK.js} +1 -1
- package/dashboard/dist/assets/{LandingPage-Ca72J5F0.js → LandingPage-BgckTHRQ.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-CSmW4lBz.js → LeaderboardPage-BCNW7UWp.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardProfilePage-BOihURRE.js → LeaderboardProfilePage-BLATxMt-.js} +1 -1
- package/dashboard/dist/assets/{LimitsPage-Bq4zB2w9.js → LimitsPage-arF--WgR.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-CoB1ZkE6.js → LoginPage-DpoFP0va.js} +1 -1
- package/dashboard/dist/assets/{PopoverPopup-D7d5-v70.js → PopoverPopup-kdgc2H6C.js} +1 -1
- package/dashboard/dist/assets/{ProviderIcon-DzvUcjPu.js → ProviderIcon-DV5r9qqP.js} +1 -1
- package/dashboard/dist/assets/{SettingsPage-BeyW1iTj.js → SettingsPage-Bb22ORmU.js} +1 -1
- package/dashboard/dist/assets/{SkillsPage-B6auz1NO.js → SkillsPage-xhtBqVKC.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-C9t8qw0F.js → WidgetsPage-CUoSVDET.js} +1 -1
- package/dashboard/dist/assets/{chevron-down-C8RgL-uJ.js → chevron-down-DYb2EChD.js} +1 -1
- package/dashboard/dist/assets/{download-C90EEqc8.js → download-C-_8o6dh.js} +1 -1
- package/dashboard/dist/assets/{leaderboard-columns-BgqTAms5.js → leaderboard-columns-BgzBlYo7.js} +1 -1
- package/dashboard/dist/assets/{main-DJcfmlDf.js → main-11hApDak.js} +6 -4
- package/dashboard/dist/assets/{use-limits-display-prefs-BUBBOUIF.js → use-limits-display-prefs-BeGKWUuk.js} +1 -1
- package/dashboard/dist/assets/{use-native-settings-CFUEzyoi.js → use-native-settings-nTTHktn0.js} +1 -1
- package/dashboard/dist/assets/{use-reduced-motion-NZDZrVKK.js → use-reduced-motion-DU8Gm6j1.js} +1 -1
- package/dashboard/dist/assets/{use-usage-limits-CoOOhZrW.js → use-usage-limits-DTPmEB8Y.js} +1 -1
- package/dashboard/dist/brand-logos/codebuddy.svg +1 -0
- package/dashboard/dist/brand-logos/every-code.svg +1 -0
- package/dashboard/dist/brand-logos/grok.svg +1 -0
- package/dashboard/dist/brand-logos/hermes.svg +1 -11
- package/dashboard/dist/brand-logos/kilo-code.svg +1 -0
- package/dashboard/dist/brand-logos/oh-my-pi.svg +1 -0
- package/dashboard/dist/index.html +1 -1
- package/dashboard/dist/share.html +1 -1
- package/package.json +1 -1
- package/src/commands/init.js +9 -5
- package/src/commands/serve.js +0 -12
- package/src/commands/sync.js +370 -7
- package/src/lib/grok-hook.js +86 -7
- package/src/lib/pricing/curated-overrides.json +1 -1
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +438 -148
- package/src/lib/subscriptions.js +92 -40
- package/src/lib/usage-limits.js +1 -1
package/src/commands/sync.js
CHANGED
|
@@ -14,7 +14,7 @@ const {
|
|
|
14
14
|
readOpencodeDbMessages,
|
|
15
15
|
resolveKiroDbPath,
|
|
16
16
|
resolveKiroJsonlPath,
|
|
17
|
-
|
|
17
|
+
resolveHermesPath,
|
|
18
18
|
resolveCopilotOtelPaths,
|
|
19
19
|
parseRolloutIncremental,
|
|
20
20
|
parseClaudeIncremental,
|
|
@@ -48,7 +48,6 @@ const {
|
|
|
48
48
|
parseKilocodeIncremental,
|
|
49
49
|
bucketKey,
|
|
50
50
|
totalsKey,
|
|
51
|
-
groupBucketKey,
|
|
52
51
|
claudeMessageDedupKey,
|
|
53
52
|
} = require("../lib/rollout");
|
|
54
53
|
const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
|
|
@@ -73,6 +72,7 @@ const { resolveRuntimeConfig } = require("../lib/runtime-config");
|
|
|
73
72
|
const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
|
|
74
73
|
const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
|
|
75
74
|
const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
|
|
75
|
+
const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
|
|
76
76
|
const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
|
|
77
77
|
// v1 had a cursor-format bug (wrote plain integer instead of {inode, offset,
|
|
78
78
|
// updatedAt}), which made parseClaudeIncremental reread every jsonl from
|
|
@@ -500,13 +500,13 @@ async function cmdSync(argv) {
|
|
|
500
500
|
|
|
501
501
|
// ── Hermes Agent (SQLite-based) ──
|
|
502
502
|
let hermesResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
503
|
-
const
|
|
504
|
-
if (fssync.existsSync(
|
|
503
|
+
const hermesPath = resolveHermesPath();
|
|
504
|
+
if (fssync.existsSync(hermesPath)) {
|
|
505
505
|
if (progress?.enabled) {
|
|
506
506
|
progress.start(`Parsing Hermes ${renderBar(0)} | buckets 0`);
|
|
507
507
|
}
|
|
508
508
|
hermesResult = await parseHermesIncremental({
|
|
509
|
-
|
|
509
|
+
hermesPath,
|
|
510
510
|
cursors,
|
|
511
511
|
queuePath,
|
|
512
512
|
onProgress: (p) => {
|
|
@@ -681,18 +681,36 @@ async function cmdSync(argv) {
|
|
|
681
681
|
? grokHookSignal.sessionId.trim()
|
|
682
682
|
: null;
|
|
683
683
|
if (hookSessionId) {
|
|
684
|
+
const hookContextTokens =
|
|
685
|
+
grokHookSignal.contextTokensUsed != null
|
|
686
|
+
? grokHookSignal.contextTokensUsed
|
|
687
|
+
: grokHookSignal.totalTokens;
|
|
688
|
+
const hookTotalTokens =
|
|
689
|
+
grokHookSignal.totalTokens != null
|
|
690
|
+
? grokHookSignal.totalTokens
|
|
691
|
+
: hookContextTokens;
|
|
684
692
|
grokSessionInputs.unshift({
|
|
685
693
|
sessionId: hookSessionId,
|
|
694
|
+
sessionDir:
|
|
695
|
+
typeof grokHookSignal.sessionDir === "string" ? grokHookSignal.sessionDir : undefined,
|
|
696
|
+
updatesPath:
|
|
697
|
+
typeof grokHookSignal.updatesPath === "string" ? grokHookSignal.updatesPath : undefined,
|
|
698
|
+
signalsPath:
|
|
699
|
+
typeof grokHookSignal.signalsPath === "string" ? grokHookSignal.signalsPath : undefined,
|
|
700
|
+
summaryPath:
|
|
701
|
+
typeof grokHookSignal.summaryPath === "string" ? grokHookSignal.summaryPath : undefined,
|
|
686
702
|
signals: {
|
|
687
|
-
contextTokensUsed:
|
|
703
|
+
contextTokensUsed: hookContextTokens,
|
|
704
|
+
totalTokens: hookTotalTokens,
|
|
705
|
+
totalTokensBeforeCompaction: grokHookSignal.totalTokensBeforeCompaction,
|
|
688
706
|
assistantMessageCount: grokHookSignal.messageCount,
|
|
689
707
|
primaryModelId: grokHookSignal.model,
|
|
690
708
|
lastActiveAt: grokHookSignal.lastActive,
|
|
691
709
|
},
|
|
692
710
|
summary: { updated_at: grokHookSignal.lastActive },
|
|
693
711
|
});
|
|
712
|
+
grokHookSignalConsumed = true;
|
|
694
713
|
}
|
|
695
|
-
grokHookSignalConsumed = true;
|
|
696
714
|
}
|
|
697
715
|
if (grokSessionInputs.length > 0) {
|
|
698
716
|
if (progress?.enabled) {
|
|
@@ -717,6 +735,9 @@ async function cmdSync(argv) {
|
|
|
717
735
|
bucketsQueued: grokResult.bucketsQueued + grokScanResult.bucketsQueued,
|
|
718
736
|
};
|
|
719
737
|
}
|
|
738
|
+
if (opts.repairGrok) {
|
|
739
|
+
await repairGrokQueueFromSessionSnapshots({ cursors, queuePath, queueStatePath });
|
|
740
|
+
}
|
|
720
741
|
|
|
721
742
|
// ── GitHub Copilot CLI (OTEL JSONL files) ──
|
|
722
743
|
let copilotResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
@@ -899,6 +920,7 @@ function parseArgs(argv) {
|
|
|
899
920
|
fromRetry: false,
|
|
900
921
|
fromOpenclaw: false,
|
|
901
922
|
drain: false,
|
|
923
|
+
repairGrok: false,
|
|
902
924
|
};
|
|
903
925
|
for (let i = 0; i < argv.length; i++) {
|
|
904
926
|
const a = argv[i];
|
|
@@ -907,6 +929,7 @@ function parseArgs(argv) {
|
|
|
907
929
|
else if (a === "--from-retry") out.fromRetry = true;
|
|
908
930
|
else if (a === "--from-openclaw") out.fromOpenclaw = true;
|
|
909
931
|
else if (a === "--drain") out.drain = true;
|
|
932
|
+
else if (a === "--repair-grok") out.repairGrok = true;
|
|
910
933
|
else throw new Error(`Unknown option: ${a}`);
|
|
911
934
|
}
|
|
912
935
|
return out;
|
|
@@ -917,9 +940,11 @@ module.exports = {
|
|
|
917
940
|
migrateCursorUnknownBuckets,
|
|
918
941
|
migrateRolloutCumulativeDeltaBuckets,
|
|
919
942
|
reincludeClaudeMemObserverFiles,
|
|
943
|
+
repairGrokQueueFromSessionSnapshots,
|
|
920
944
|
CURSOR_UNKNOWN_MIGRATION_KEY,
|
|
921
945
|
ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY,
|
|
922
946
|
CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
|
|
947
|
+
GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
|
|
923
948
|
};
|
|
924
949
|
|
|
925
950
|
function normalizeString(value) {
|
|
@@ -1289,6 +1314,344 @@ async function readQueueBatch(queuePath, startOffset, maxBuckets) {
|
|
|
1289
1314
|
return { buckets: Array.from(bucketMap.values()), nextOffset: offset };
|
|
1290
1315
|
}
|
|
1291
1316
|
|
|
1317
|
+
function normalizeGrokRepairSource(value) {
|
|
1318
|
+
return typeof value === "string" ? value.trim().toLowerCase() : "";
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
function normalizeGrokRepairModel(value) {
|
|
1322
|
+
return typeof value === "string" && value.trim() ? value.trim() : "grok-build";
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
function normalizeGrokRepairNumber(value) {
|
|
1326
|
+
const n = Number(value);
|
|
1327
|
+
return Number.isFinite(n) && n > 0 ? n : 0;
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
function toGrokRepairHalfHourStart(value) {
|
|
1331
|
+
if (value == null) return null;
|
|
1332
|
+
const millis =
|
|
1333
|
+
typeof value === "number"
|
|
1334
|
+
? value < 10_000_000_000
|
|
1335
|
+
? value * 1000
|
|
1336
|
+
: value
|
|
1337
|
+
: Date.parse(String(value));
|
|
1338
|
+
if (!Number.isFinite(millis)) return null;
|
|
1339
|
+
const halfHourMs = 30 * 60 * 1000;
|
|
1340
|
+
return new Date(Math.floor(millis / halfHourMs) * halfHourMs).toISOString();
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
function estimateGrokRepairTotals(totalTokens, conversationCount) {
|
|
1344
|
+
const total = Math.trunc(normalizeGrokRepairNumber(totalTokens));
|
|
1345
|
+
const inputTokens = Math.round(total * 0.8);
|
|
1346
|
+
const outputTokens = Math.max(0, total - inputTokens);
|
|
1347
|
+
return {
|
|
1348
|
+
input_tokens: inputTokens,
|
|
1349
|
+
cached_input_tokens: 0,
|
|
1350
|
+
cache_creation_input_tokens: 0,
|
|
1351
|
+
output_tokens: outputTokens,
|
|
1352
|
+
reasoning_output_tokens: 0,
|
|
1353
|
+
total_tokens: total,
|
|
1354
|
+
billable_total_tokens: total,
|
|
1355
|
+
conversation_count: Math.trunc(normalizeGrokRepairNumber(conversationCount)),
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
function addGrokRepairTotals(target, delta) {
|
|
1360
|
+
target.input_tokens += delta.input_tokens;
|
|
1361
|
+
target.cached_input_tokens += delta.cached_input_tokens;
|
|
1362
|
+
target.cache_creation_input_tokens += delta.cache_creation_input_tokens;
|
|
1363
|
+
target.output_tokens += delta.output_tokens;
|
|
1364
|
+
target.reasoning_output_tokens += delta.reasoning_output_tokens;
|
|
1365
|
+
target.total_tokens += delta.total_tokens;
|
|
1366
|
+
target.billable_total_tokens += delta.billable_total_tokens;
|
|
1367
|
+
target.conversation_count += delta.conversation_count;
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
function buildGrokRepairRowsFromSnapshots(sessionSnapshots) {
|
|
1371
|
+
if (!sessionSnapshots || typeof sessionSnapshots !== "object") return [];
|
|
1372
|
+
|
|
1373
|
+
const buckets = new Map();
|
|
1374
|
+
for (const snapshot of Object.values(sessionSnapshots)) {
|
|
1375
|
+
if (!snapshot || typeof snapshot !== "object") continue;
|
|
1376
|
+
const totalTokens = Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens));
|
|
1377
|
+
if (totalTokens <= 0) continue;
|
|
1378
|
+
|
|
1379
|
+
const hourStart = toGrokRepairHalfHourStart(
|
|
1380
|
+
snapshot.lastEventTimestamp || snapshot.updatedAt,
|
|
1381
|
+
);
|
|
1382
|
+
if (!hourStart) continue;
|
|
1383
|
+
|
|
1384
|
+
const model = normalizeGrokRepairModel(snapshot.model);
|
|
1385
|
+
const key = bucketKey("grok", model, hourStart);
|
|
1386
|
+
let totals = buckets.get(key);
|
|
1387
|
+
if (!totals) {
|
|
1388
|
+
totals = {
|
|
1389
|
+
source: "grok",
|
|
1390
|
+
model,
|
|
1391
|
+
hour_start: hourStart,
|
|
1392
|
+
input_tokens: 0,
|
|
1393
|
+
cached_input_tokens: 0,
|
|
1394
|
+
cache_creation_input_tokens: 0,
|
|
1395
|
+
output_tokens: 0,
|
|
1396
|
+
reasoning_output_tokens: 0,
|
|
1397
|
+
total_tokens: 0,
|
|
1398
|
+
billable_total_tokens: 0,
|
|
1399
|
+
conversation_count: 0,
|
|
1400
|
+
};
|
|
1401
|
+
buckets.set(key, totals);
|
|
1402
|
+
}
|
|
1403
|
+
addGrokRepairTotals(
|
|
1404
|
+
totals,
|
|
1405
|
+
estimateGrokRepairTotals(totalTokens, snapshot.messageCount),
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
return Array.from(buckets.values()).sort((a, b) => {
|
|
1410
|
+
const timeCompare = a.hour_start.localeCompare(b.hour_start);
|
|
1411
|
+
return timeCompare || a.model.localeCompare(b.model);
|
|
1412
|
+
});
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
function applyGrokRepairHourlyState(cursors, rows) {
|
|
1416
|
+
const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
|
|
1417
|
+
const buckets = hourly.buckets && typeof hourly.buckets === "object" ? hourly.buckets : {};
|
|
1418
|
+
const groupQueued =
|
|
1419
|
+
hourly.groupQueued && typeof hourly.groupQueued === "object" ? hourly.groupQueued : {};
|
|
1420
|
+
|
|
1421
|
+
for (const key of Object.keys(buckets)) {
|
|
1422
|
+
if (key.startsWith("grok|")) {
|
|
1423
|
+
delete buckets[key];
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
for (const key of Object.keys(groupQueued)) {
|
|
1427
|
+
if (key.startsWith("grok|")) {
|
|
1428
|
+
delete groupQueued[key];
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
for (const row of rows) {
|
|
1433
|
+
const totals = {
|
|
1434
|
+
input_tokens: row.input_tokens,
|
|
1435
|
+
cached_input_tokens: row.cached_input_tokens,
|
|
1436
|
+
cache_creation_input_tokens: row.cache_creation_input_tokens,
|
|
1437
|
+
output_tokens: row.output_tokens,
|
|
1438
|
+
reasoning_output_tokens: row.reasoning_output_tokens,
|
|
1439
|
+
total_tokens: row.total_tokens,
|
|
1440
|
+
billable_total_tokens: row.billable_total_tokens,
|
|
1441
|
+
conversation_count: row.conversation_count,
|
|
1442
|
+
};
|
|
1443
|
+
buckets[bucketKey("grok", row.model, row.hour_start)] = {
|
|
1444
|
+
totals,
|
|
1445
|
+
queuedKey: totalsKey(totals),
|
|
1446
|
+
source: "grok",
|
|
1447
|
+
hour_start: row.hour_start,
|
|
1448
|
+
};
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
cursors.hourly = {
|
|
1452
|
+
...hourly,
|
|
1453
|
+
version: 3,
|
|
1454
|
+
buckets,
|
|
1455
|
+
groupQueued,
|
|
1456
|
+
updatedAt: typeof hourly.updatedAt === "string" ? hourly.updatedAt : null,
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
async function resetGrokRepairUploadOffset(queueStatePath) {
|
|
1461
|
+
if (typeof queueStatePath !== "string" || !queueStatePath) return false;
|
|
1462
|
+
let state = {};
|
|
1463
|
+
try {
|
|
1464
|
+
state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
|
|
1465
|
+
} catch (_e) {
|
|
1466
|
+
state = {};
|
|
1467
|
+
}
|
|
1468
|
+
state.offset = 0;
|
|
1469
|
+
state.updatedAt = new Date().toISOString();
|
|
1470
|
+
state.note = "reset_after_grok_append_only_repair_2026_05_v4";
|
|
1471
|
+
await ensureDir(path.dirname(queueStatePath));
|
|
1472
|
+
await fs.writeFile(queueStatePath, JSON.stringify(state, null, 2) + "\n", "utf8");
|
|
1473
|
+
return true;
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
function hasAppliedGrokRepairMigration(value) {
|
|
1477
|
+
if (!value) return false;
|
|
1478
|
+
if (value === true) return true;
|
|
1479
|
+
if (value && typeof value === "object") {
|
|
1480
|
+
if (value.status === "applied" || value.status === "noop") return true;
|
|
1481
|
+
if (value.status) return false;
|
|
1482
|
+
return value.rowsWritten != null || value.rowsRemoved != null;
|
|
1483
|
+
}
|
|
1484
|
+
return false;
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
function serializeGrokRepairRow(row) {
|
|
1488
|
+
return JSON.stringify({
|
|
1489
|
+
source: "grok",
|
|
1490
|
+
model: normalizeGrokRepairModel(row.model),
|
|
1491
|
+
hour_start: row.hour_start,
|
|
1492
|
+
input_tokens: row.input_tokens || 0,
|
|
1493
|
+
cached_input_tokens: row.cached_input_tokens || 0,
|
|
1494
|
+
cache_creation_input_tokens: row.cache_creation_input_tokens || 0,
|
|
1495
|
+
output_tokens: row.output_tokens || 0,
|
|
1496
|
+
reasoning_output_tokens: row.reasoning_output_tokens || 0,
|
|
1497
|
+
total_tokens: row.total_tokens || 0,
|
|
1498
|
+
billable_total_tokens: row.billable_total_tokens || 0,
|
|
1499
|
+
conversation_count: row.conversation_count || 0,
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
async function backupExistingFile(filePath) {
|
|
1504
|
+
if (typeof filePath !== "string" || !filePath) return null;
|
|
1505
|
+
try {
|
|
1506
|
+
const stat = await fs.stat(filePath);
|
|
1507
|
+
if (!stat.isFile()) return null;
|
|
1508
|
+
} catch (e) {
|
|
1509
|
+
if (e?.code === "ENOENT" || e?.code === "ENOTDIR") return null;
|
|
1510
|
+
throw e;
|
|
1511
|
+
}
|
|
1512
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
1513
|
+
const backupPath = `${filePath}.bak.${stamp}`;
|
|
1514
|
+
await fs.copyFile(filePath, backupPath);
|
|
1515
|
+
return backupPath;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
async function repairGrokQueueFromSessionSnapshots({ cursors, queuePath, queueStatePath } = {}) {
|
|
1519
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
1520
|
+
const grokState = (cursors.grok ||= {});
|
|
1521
|
+
const migrations = (grokState.migrations ||= {});
|
|
1522
|
+
if (hasAppliedGrokRepairMigration(migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY])) {
|
|
1523
|
+
return false;
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
let raw = "";
|
|
1527
|
+
try {
|
|
1528
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
1529
|
+
} catch (e) {
|
|
1530
|
+
if (e?.code !== "ENOENT") throw e;
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1533
|
+
const latestGrokRows = new Map();
|
|
1534
|
+
let existingGrokRows = 0;
|
|
1535
|
+
for (const line of raw.split("\n")) {
|
|
1536
|
+
if (!line.trim()) continue;
|
|
1537
|
+
let row;
|
|
1538
|
+
try {
|
|
1539
|
+
row = JSON.parse(line);
|
|
1540
|
+
} catch (_e) {
|
|
1541
|
+
continue;
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
if (normalizeGrokRepairSource(row?.source) === "grok") {
|
|
1545
|
+
const model = normalizeGrokRepairModel(row.model);
|
|
1546
|
+
const hourStart = typeof row.hour_start === "string" ? row.hour_start : null;
|
|
1547
|
+
if (!hourStart) continue;
|
|
1548
|
+
existingGrokRows += 1;
|
|
1549
|
+
latestGrokRows.set(bucketKey("grok", model, hourStart), {
|
|
1550
|
+
...row,
|
|
1551
|
+
source: "grok",
|
|
1552
|
+
model,
|
|
1553
|
+
hour_start: hourStart,
|
|
1554
|
+
});
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
if (existingGrokRows === 0) {
|
|
1559
|
+
migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
|
|
1560
|
+
status: "noop",
|
|
1561
|
+
appliedAt: new Date().toISOString(),
|
|
1562
|
+
existingGrokRows: 0,
|
|
1563
|
+
rowsWritten: 0,
|
|
1564
|
+
snapshotsUsed: 0,
|
|
1565
|
+
uploadOffsetReset: false,
|
|
1566
|
+
};
|
|
1567
|
+
return false;
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
const repairRows = buildGrokRepairRowsFromSnapshots(grokState.sessionSnapshots);
|
|
1571
|
+
if (repairRows.length === 0) {
|
|
1572
|
+
migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
|
|
1573
|
+
status: "skipped",
|
|
1574
|
+
appliedAt: new Date().toISOString(),
|
|
1575
|
+
reason: "missing-session-snapshots",
|
|
1576
|
+
existingGrokRows,
|
|
1577
|
+
rowsWritten: 0,
|
|
1578
|
+
snapshotsUsed: 0,
|
|
1579
|
+
uploadOffsetReset: false,
|
|
1580
|
+
};
|
|
1581
|
+
return false;
|
|
1582
|
+
}
|
|
1583
|
+
|
|
1584
|
+
applyGrokRepairHourlyState(cursors, repairRows);
|
|
1585
|
+
|
|
1586
|
+
const repairLines = [];
|
|
1587
|
+
const repairKeys = new Set();
|
|
1588
|
+
for (const row of repairRows) {
|
|
1589
|
+
const key = bucketKey("grok", row.model, row.hour_start);
|
|
1590
|
+
repairKeys.add(key);
|
|
1591
|
+
const current = latestGrokRows.get(key);
|
|
1592
|
+
if (current && totalsKey(current) === totalsKey(row)) continue;
|
|
1593
|
+
repairLines.push(serializeGrokRepairRow(row));
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
const zeroTotals = {
|
|
1597
|
+
input_tokens: 0,
|
|
1598
|
+
cached_input_tokens: 0,
|
|
1599
|
+
cache_creation_input_tokens: 0,
|
|
1600
|
+
output_tokens: 0,
|
|
1601
|
+
reasoning_output_tokens: 0,
|
|
1602
|
+
total_tokens: 0,
|
|
1603
|
+
billable_total_tokens: 0,
|
|
1604
|
+
conversation_count: 0,
|
|
1605
|
+
};
|
|
1606
|
+
let staleRowsRetracted = 0;
|
|
1607
|
+
for (const [key, row] of latestGrokRows.entries()) {
|
|
1608
|
+
if (repairKeys.has(key)) continue;
|
|
1609
|
+
if (totalsKey(row) === totalsKey(zeroTotals)) continue;
|
|
1610
|
+
staleRowsRetracted += 1;
|
|
1611
|
+
repairLines.push(serializeGrokRepairRow({
|
|
1612
|
+
...zeroTotals,
|
|
1613
|
+
model: row.model,
|
|
1614
|
+
hour_start: row.hour_start,
|
|
1615
|
+
}));
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
if (repairLines.length === 0) {
|
|
1619
|
+
migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
|
|
1620
|
+
status: "noop",
|
|
1621
|
+
appliedAt: new Date().toISOString(),
|
|
1622
|
+
existingGrokRows,
|
|
1623
|
+
rowsWritten: 0,
|
|
1624
|
+
staleRowsRetracted,
|
|
1625
|
+
snapshotsUsed: repairRows.length,
|
|
1626
|
+
uploadOffsetReset: false,
|
|
1627
|
+
};
|
|
1628
|
+
return false;
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
await ensureDir(path.dirname(queuePath));
|
|
1632
|
+
const queueBackupPath = await backupExistingFile(queuePath);
|
|
1633
|
+
const queueStateBackupPath = await backupExistingFile(queueStatePath);
|
|
1634
|
+
await fs.appendFile(queuePath, `${repairLines.join("\n")}\n`, "utf8");
|
|
1635
|
+
|
|
1636
|
+
const uploadOffsetReset = await resetGrokRepairUploadOffset(queueStatePath);
|
|
1637
|
+
migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
|
|
1638
|
+
status: "applied",
|
|
1639
|
+
appliedAt: new Date().toISOString(),
|
|
1640
|
+
existingGrokRows,
|
|
1641
|
+
rowsWritten: repairLines.length,
|
|
1642
|
+
staleRowsRetracted,
|
|
1643
|
+
snapshotsUsed: Object.values(grokState.sessionSnapshots || {}).filter((snapshot) => {
|
|
1644
|
+
if (!snapshot || typeof snapshot !== "object") return false;
|
|
1645
|
+
if (Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens)) <= 0) return false;
|
|
1646
|
+
return Boolean(toGrokRepairHalfHourStart(snapshot.lastEventTimestamp || snapshot.updatedAt));
|
|
1647
|
+
}).length,
|
|
1648
|
+
uploadOffsetReset,
|
|
1649
|
+
queueBackupPath,
|
|
1650
|
+
queueStateBackupPath,
|
|
1651
|
+
};
|
|
1652
|
+
return true;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1292
1655
|
async function migrateCursorUnknownBuckets({ cursors, queuePath }) {
|
|
1293
1656
|
if (!cursors || typeof cursors !== "object") return;
|
|
1294
1657
|
cursors.migrations = cursors.migrations || {};
|
package/src/lib/grok-hook.js
CHANGED
|
@@ -94,14 +94,15 @@ function buildGrokSessionEndHandler({ trackerDir }) {
|
|
|
94
94
|
// It must be self-contained enough or rely on the copied runtime.
|
|
95
95
|
// For simplicity and reliability we write a small script that:
|
|
96
96
|
// 1. Reads GROK_SESSION_ID + GROK_WORKSPACE_ROOT from env
|
|
97
|
-
// 2. Locates the
|
|
98
|
-
// 3. Extracts usage
|
|
97
|
+
// 2. Locates the session metadata files
|
|
98
|
+
// 3. Extracts usage metadata only
|
|
99
99
|
// 4. Writes a signal file under trackerDir that sync.js will pick up on next run
|
|
100
100
|
|
|
101
101
|
return `#!/usr/bin/env node
|
|
102
102
|
const fs = require('node:fs');
|
|
103
103
|
const path = require('node:path');
|
|
104
104
|
const os = require('node:os');
|
|
105
|
+
const readline = require('node:readline');
|
|
105
106
|
|
|
106
107
|
const GROK_HOME = process.env.TOKENTRACKER_GROK_HOME || process.env.GROK_HOME || path.join(os.homedir(), '.grok');
|
|
107
108
|
const SESSION_ID = process.env.GROK_SESSION_ID;
|
|
@@ -120,14 +121,17 @@ const encodedCwd = encodeGrokCwd(WORKSPACE_ROOT);
|
|
|
120
121
|
const sessionDir = path.join(GROK_HOME, 'sessions', encodedCwd, SESSION_ID);
|
|
121
122
|
const signalsPath = path.join(sessionDir, 'signals.json');
|
|
122
123
|
const summaryPath = path.join(sessionDir, 'summary.json');
|
|
124
|
+
const updatesPath = path.join(sessionDir, 'updates.jsonl');
|
|
123
125
|
|
|
124
|
-
let signals =
|
|
126
|
+
let signals = {};
|
|
125
127
|
try {
|
|
126
128
|
const raw = fs.readFileSync(signalsPath, 'utf8');
|
|
127
129
|
signals = JSON.parse(raw);
|
|
128
130
|
} catch (err) {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
+
signals = {};
|
|
132
|
+
}
|
|
133
|
+
if (!signals || typeof signals !== 'object') {
|
|
134
|
+
signals = {};
|
|
131
135
|
}
|
|
132
136
|
|
|
133
137
|
const summary = (() => {
|
|
@@ -143,10 +147,74 @@ function toNonNegativeFiniteNumber(value) {
|
|
|
143
147
|
return Number.isFinite(number) && number > 0 ? number : 0;
|
|
144
148
|
}
|
|
145
149
|
|
|
146
|
-
|
|
150
|
+
function timestampToIso(value) {
|
|
151
|
+
if (value == null) return null;
|
|
152
|
+
if (typeof value === 'number') {
|
|
153
|
+
if (!Number.isFinite(value) || value <= 0) return null;
|
|
154
|
+
const millis = value < 10000000000 ? value * 1000 : value;
|
|
155
|
+
const dt = new Date(millis);
|
|
156
|
+
return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
|
|
157
|
+
}
|
|
158
|
+
if (typeof value === 'string') {
|
|
159
|
+
const trimmed = value.trim();
|
|
160
|
+
if (!trimmed) return null;
|
|
161
|
+
if (/^[0-9]+(?:\\.[0-9]+)?$/.test(trimmed)) return timestampToIso(Number(trimmed));
|
|
162
|
+
const dt = new Date(trimmed);
|
|
163
|
+
return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
|
|
164
|
+
}
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async function readUpdateTelemetry(filePath) {
|
|
169
|
+
let totalTokens = 0;
|
|
170
|
+
let lastEventId = null;
|
|
171
|
+
let lastEventTimestamp = null;
|
|
172
|
+
let lineNumber = 0;
|
|
173
|
+
|
|
174
|
+
try {
|
|
175
|
+
const input = fs.createReadStream(filePath, { encoding: 'utf8' });
|
|
176
|
+
const rl = readline.createInterface({ input, crlfDelay: Infinity });
|
|
177
|
+
try {
|
|
178
|
+
for await (const line of rl) {
|
|
179
|
+
lineNumber += 1;
|
|
180
|
+
if (!line || !line.trim()) continue;
|
|
181
|
+
let record = null;
|
|
182
|
+
try {
|
|
183
|
+
record = JSON.parse(line);
|
|
184
|
+
} catch {
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const meta = record && record.params && record.params._meta ? record.params._meta : record && record._meta;
|
|
188
|
+
if (!meta || typeof meta !== 'object') continue;
|
|
189
|
+
const nextTotal = toNonNegativeFiniteNumber(meta.totalTokens);
|
|
190
|
+
if (nextTotal <= 0) continue;
|
|
191
|
+
totalTokens = Math.max(totalTokens, nextTotal);
|
|
192
|
+
lastEventId = meta.eventId != null ? String(meta.eventId) : String(lineNumber);
|
|
193
|
+
lastEventTimestamp =
|
|
194
|
+
timestampToIso(meta.agentTimestampMs) ||
|
|
195
|
+
timestampToIso(meta.timestampMs) ||
|
|
196
|
+
timestampToIso(record.timestamp_ms) ||
|
|
197
|
+
timestampToIso(record.timestamp) ||
|
|
198
|
+
lastEventTimestamp;
|
|
199
|
+
}
|
|
200
|
+
} finally {
|
|
201
|
+
rl.close();
|
|
202
|
+
}
|
|
203
|
+
} catch {
|
|
204
|
+
return { totalTokens, lastEventId, lastEventTimestamp };
|
|
205
|
+
}
|
|
206
|
+
return { totalTokens, lastEventId, lastEventTimestamp };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async function main() {
|
|
210
|
+
const contextTokensUsed = toNonNegativeFiniteNumber(signals.contextTokensUsed ?? signals.totalTokens);
|
|
211
|
+
const totalTokensBeforeCompaction = toNonNegativeFiniteNumber(signals.totalTokensBeforeCompaction);
|
|
212
|
+
const signalTotalTokens = totalTokensBeforeCompaction + contextTokensUsed;
|
|
213
|
+
const updateTelemetry = await readUpdateTelemetry(updatesPath);
|
|
214
|
+
const totalTokens = Math.max(updateTelemetry.totalTokens, signalTotalTokens);
|
|
147
215
|
const messageCount = toNonNegativeFiniteNumber(signals.assistantMessageCount || signals.num_chat_messages);
|
|
148
216
|
const model = signals.primaryModelId || (Array.isArray(signals.modelsUsed) ? signals.modelsUsed[0] : 'grok-build');
|
|
149
|
-
const lastActive = signals.lastActiveAt || summary.updated_at || new Date().toISOString();
|
|
217
|
+
const lastActive = signals.lastActiveAt || updateTelemetry.lastEventTimestamp || summary.updated_at || new Date().toISOString();
|
|
150
218
|
|
|
151
219
|
if (totalTokens <= 0) {
|
|
152
220
|
process.exit(0);
|
|
@@ -163,8 +231,16 @@ const signal = {
|
|
|
163
231
|
cwd: WORKSPACE_ROOT,
|
|
164
232
|
model,
|
|
165
233
|
totalTokens,
|
|
234
|
+
contextTokensUsed,
|
|
235
|
+
totalTokensBeforeCompaction,
|
|
166
236
|
messageCount,
|
|
167
237
|
lastActive,
|
|
238
|
+
sessionDir,
|
|
239
|
+
updatesPath,
|
|
240
|
+
signalsPath,
|
|
241
|
+
summaryPath,
|
|
242
|
+
lastEventId: updateTelemetry.lastEventId,
|
|
243
|
+
lastEventTimestamp: updateTelemetry.lastEventTimestamp,
|
|
168
244
|
capturedAt: new Date().toISOString()
|
|
169
245
|
};
|
|
170
246
|
|
|
@@ -177,6 +253,9 @@ try {
|
|
|
177
253
|
} catch {}
|
|
178
254
|
|
|
179
255
|
process.exit(0);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
main().catch(() => process.exit(0));
|
|
180
259
|
`;
|
|
181
260
|
}
|
|
182
261
|
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"deepseek-v4-flash":{ "input": 0.14, "output": 0.28, "cache_read": 0.0028, "cache_write": 0.14 },
|
|
19
19
|
"deepseek-v4-pro": { "input": 0.435,"output": 0.87, "cache_read": 0.003625, "cache_write": 0.435 },
|
|
20
20
|
"deepseek-chat": { "input": 0.14, "output": 0.28, "cache_read": 0.0028, "cache_write": 0.14 },
|
|
21
|
-
"grok-build": { "input": 1.25, "output": 2.50, "cache_read": 0.20, "note": "Grok Build TUI estimate.
|
|
21
|
+
"grok-build": { "input": 1.25, "output": 2.50, "cache_read": 0.20, "note": "Grok Build TUI estimate. Local telemetry currently exposes totalTokens without a stable prompt/output/cache split, so TokenTracker estimates input/output split until Grok exposes per-call usage details." },
|
|
22
22
|
"grok-4-0709": { "input": 3.00, "output": 15.00, "cache_read": 0.75 },
|
|
23
23
|
"grok-4": { "input": 3.00, "output": 15.00, "cache_read": 0.75 },
|
|
24
24
|
"grok-4-latest": { "input": 3.00, "output": 15.00, "cache_read": 0.75 },
|