tokenusage-cli 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/README.md +52 -11
  3. package/bin/tokenusage.js +14 -2
  4. package/package.json +1 -2
  5. package/src/cli.js +325 -78
  6. package/src/commands/device-login.js +3 -3
  7. package/src/commands/init.js +343 -153
  8. package/src/commands/pause.js +11 -23
  9. package/src/commands/reconcile.js +485 -0
  10. package/src/commands/sessions.js +57 -14
  11. package/src/commands/sync.js +69 -15
  12. package/src/commands/uninstall.js +6 -9
  13. package/src/commands/upload.js +147 -0
  14. package/src/lib/cli-errors.js +33 -0
  15. package/src/lib/cli-ui.js +93 -0
  16. package/src/lib/queue-row-normalize.js +55 -0
  17. package/src/lib/scheduled-task.js +597 -75
  18. package/src/lib/antigravity-paths.js +0 -48
  19. package/src/lib/async-queue-reader.js +0 -138
  20. package/src/lib/atomic-json.js +0 -27
  21. package/src/lib/bearer-token.js +0 -11
  22. package/src/lib/cloud-account.js +0 -541
  23. package/src/lib/codex-context-breakdown.js +0 -806
  24. package/src/lib/context-health.js +0 -124
  25. package/src/lib/grok-context-breakdown.js +0 -890
  26. package/src/lib/heatmap-compact.js +0 -114
  27. package/src/lib/local-api.js +0 -3574
  28. package/src/lib/outcomes-engine.js +0 -253
  29. package/src/lib/pet-packages.js +0 -722
  30. package/src/lib/skill-usage.js +0 -267
  31. package/src/lib/skills-manager.js +0 -1591
  32. package/src/lib/source-metadata.js +0 -48
  33. package/src/lib/subscription-manager.js +0 -293
  34. package/src/lib/wrapped-aggregator.js +0 -236
  35. package/test/audit-fixes.test.js +0 -249
  36. package/test/branding.test.js +0 -24
  37. package/test/cli-options.test.js +0 -43
  38. package/test/cli-smoke.test.js +0 -53
  39. package/test/cli-ui.test.js +0 -47
  40. package/test/collection-controls.test.js +0 -255
  41. package/test/ingest-sanitize.test.js +0 -162
  42. package/test/packaging.test.js +0 -107
  43. package/test/quiet-warnings.test.js +0 -53
  44. package/test/runtime-copy.test.js +0 -111
  45. package/test/session-upload.test.js +0 -427
  46. package/test/trae-db.test.js +0 -67
  47. package/test/zstd-native.test.js +0 -36
@@ -1,890 +0,0 @@
1
- // Grok Build "Context Breakdown" — Codex-style tool-oriented view.
2
- //
3
- // Privacy: tokens + timestamps + tool names only. Never return prompt text,
4
- // assistant text, tool arguments, or file contents.
5
- //
6
- // Data source: ~/.grok/sessions/**/updates.jsonl
7
- // Authoritative billable totals come from turn_completed.usage. Tool/exec
8
- // attribution is heuristic: the turn's non-reasoning tokens are split evenly
9
- // across tools observed in that turn (same approach as Codex).
10
-
11
- "use strict";
12
-
13
- const crypto = require("node:crypto");
14
- const fs = require("node:fs");
15
- const os = require("node:os");
16
- const path = require("node:path");
17
- const readline = require("node:readline");
18
-
19
- const {
20
- emptyTotals,
21
- addInto,
22
- roundTotals,
23
- buildExecStatsEntry,
24
- categorizeTool,
25
- inferExecCommandKind,
26
- sanitizeCommandSignature,
27
- getExecutableName,
28
- } = require("./categorizer-utils");
29
- const { normalizeGrokUsage } = require("./grok-usage");
30
-
31
- const CACHE_SCHEMA_VERSION = "grok-context-v3";
32
- const CACHE = new Map();
33
- // Result-level TTL. Per-file parse cache is the main win when flipping tabs.
34
- const CACHE_TTL_MS = 5 * 60_000;
35
- // Deduplicate concurrent scans when the dashboard remounts the panel quickly.
36
- const IN_FLIGHT = new Map();
37
- // Per updates.jsonl parse cache — mirrors Codex PARSED_GROUP_CACHE so switching
38
- // back to Grok does not re-read every session log on each open.
39
- const FILE_PARSE_CACHE = new Map();
40
- const MAX_FILE_PARSE_CACHE_ENTRIES = 512;
41
-
42
- // ---------------------------------------------------------------------------
43
- // Time helpers
44
- // ---------------------------------------------------------------------------
45
-
46
- function dayKeyToIsoBounds(from, to) {
47
- if (!from && !to) return { fromIso: null, toIso: null };
48
- const fromDate = from ? new Date(`${from}T00:00:00Z`) : null;
49
- const toDate = to ? new Date(`${to}T23:59:59Z`) : null;
50
- if (fromDate && Number.isFinite(fromDate.getTime())) {
51
- fromDate.setUTCHours(fromDate.getUTCHours() - 14);
52
- }
53
- if (toDate && Number.isFinite(toDate.getTime())) {
54
- toDate.setUTCHours(toDate.getUTCHours() + 14);
55
- }
56
- return {
57
- fromIso: fromDate ? fromDate.toISOString() : null,
58
- toIso: toDate ? toDate.toISOString() : null,
59
- };
60
- }
61
-
62
- function timestampToIso(value) {
63
- if (value == null) return null;
64
- if (typeof value === "number") {
65
- if (!Number.isFinite(value) || value <= 0) return null;
66
- const millis = value < 10_000_000_000 ? value * 1000 : value;
67
- const dt = new Date(millis);
68
- return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
69
- }
70
- if (typeof value === "string") {
71
- const trimmed = value.trim();
72
- if (!trimmed) return null;
73
- if (/^[0-9]+(?:\.[0-9]+)?$/.test(trimmed)) return timestampToIso(Number(trimmed));
74
- const dt = new Date(trimmed);
75
- return Number.isFinite(dt.getTime()) ? dt.toISOString() : null;
76
- }
77
- return null;
78
- }
79
-
80
- function dayKeyFromIso(iso, timeZoneContext) {
81
- if (!iso) return "";
82
- const { timeZone, offsetMinutes } = timeZoneContext || {};
83
- const dt = new Date(iso);
84
- if (!Number.isFinite(dt.getTime())) return iso.slice(0, 10);
85
-
86
- if (timeZone && typeof Intl !== "undefined" && Intl.DateTimeFormat) {
87
- try {
88
- const parts = new Intl.DateTimeFormat("en-CA", {
89
- timeZone,
90
- year: "numeric",
91
- month: "2-digit",
92
- day: "2-digit",
93
- hourCycle: "h23",
94
- }).formatToParts(dt);
95
- const values = {};
96
- for (const part of parts) {
97
- if (part.type !== "literal") values[part.type] = part.value;
98
- }
99
- if (values.year && values.month && values.day) {
100
- return `${values.year}-${values.month}-${values.day}`;
101
- }
102
- } catch {
103
- // fall through
104
- }
105
- }
106
- if (Number.isFinite(offsetMinutes)) {
107
- const shifted = new Date(dt.getTime() - Number(offsetMinutes) * 60_000);
108
- return `${String(shifted.getUTCFullYear()).padStart(4, "0")}-${String(shifted.getUTCMonth() + 1).padStart(2, "0")}-${String(shifted.getUTCDate()).padStart(2, "0")}`;
109
- }
110
- return iso.slice(0, 10);
111
- }
112
-
113
- function isIsoInRange(iso, { from, to, timeZoneContext } = {}) {
114
- if (!from && !to) return true;
115
- const day = dayKeyFromIso(iso, timeZoneContext);
116
- if (!day) return false;
117
- if (from && day < from) return false;
118
- if (to && day > to) return false;
119
- return true;
120
- }
121
-
122
- // ---------------------------------------------------------------------------
123
- // Grok path discovery
124
- // ---------------------------------------------------------------------------
125
-
126
- function resolveGrokHome(env = process.env) {
127
- if (env.TOKENUSAGE_GROK_HOME) return env.TOKENUSAGE_GROK_HOME;
128
- if (env.GROK_HOME) return env.GROK_HOME;
129
- return path.join(os.homedir(), ".grok");
130
- }
131
-
132
- function discoverGrokSessionFiles(env = process.env) {
133
- const home = resolveGrokHome(env);
134
- const sessionsRoot = path.join(home, "sessions");
135
- if (!fs.existsSync(sessionsRoot)) return [];
136
-
137
- const out = [];
138
- let cwdDirs = [];
139
- try {
140
- cwdDirs = fs.readdirSync(sessionsRoot);
141
- } catch {
142
- return [];
143
- }
144
-
145
- for (const cwdDir of cwdDirs) {
146
- const cwdPath = path.join(sessionsRoot, cwdDir);
147
- let st;
148
- try {
149
- st = fs.statSync(cwdPath);
150
- } catch {
151
- continue;
152
- }
153
- if (!st.isDirectory()) continue;
154
-
155
- let sessionIds = [];
156
- try {
157
- sessionIds = fs.readdirSync(cwdPath);
158
- } catch {
159
- continue;
160
- }
161
-
162
- for (const sid of sessionIds) {
163
- const sessionDir = path.join(cwdPath, sid);
164
- const updatesPath = path.join(sessionDir, "updates.jsonl");
165
- let ust;
166
- try {
167
- ust = fs.statSync(updatesPath);
168
- } catch {
169
- continue;
170
- }
171
- if (!ust.isFile() || ust.size <= 0) continue;
172
- out.push({
173
- sessionId: sid,
174
- sessionDir,
175
- updatesPath,
176
- mtimeMs: ust.mtimeMs,
177
- size: ust.size,
178
- });
179
- }
180
- }
181
- out.sort((a, b) => a.updatesPath.localeCompare(b.updatesPath));
182
- return out;
183
- }
184
-
185
- // ---------------------------------------------------------------------------
186
- // Tool / exec naming
187
- // ---------------------------------------------------------------------------
188
-
189
- function mapGrokToolName(rawName) {
190
- const name = String(rawName || "").trim();
191
- if (!name) return "unknown";
192
- const lower = name.toLowerCase();
193
- if (lower === "read_file" || lower === "read") return "Read";
194
- if (lower === "write" || lower === "write_file") return "Write";
195
- if (lower === "search_replace" || lower === "str_replace" || lower === "edit") return "Edit";
196
- if (lower === "grep") return "Grep";
197
- if (lower === "list_dir" || lower === "glob" || lower === "find") return "Glob";
198
- if (lower === "run_terminal_command" || lower === "bash" || lower === "shell") return "Bash";
199
- if (lower === "todo_write" || lower === "todowrite") return "TodoWrite";
200
- if (lower === "web_search" || lower === "websearch") return "WebSearch";
201
- if (lower === "web_fetch" || lower === "webfetch") return "WebFetch";
202
- if (lower.startsWith("mcp_") || lower.startsWith("mcp-")) {
203
- return name.replace(/^mcp[_-]/i, "mcp__").replace(/-/g, "_");
204
- }
205
- return name;
206
- }
207
-
208
- function extractToolName(update, meta) {
209
- const fromMetaTool =
210
- update?._meta?.["x.ai/tool"]?.name ||
211
- update?._meta?.["x.ai/tool"]?.label ||
212
- null;
213
- const fromUpdateParams = meta?.updateParams?.title || meta?.updateParams?.name || null;
214
- const fromUpdate = update?.title || update?.name || null;
215
- return mapGrokToolName(fromMetaTool || fromUpdateParams || fromUpdate || "unknown");
216
- }
217
-
218
- function extractExecCommand(update) {
219
- const raw = update?.rawInput;
220
- if (!raw || typeof raw !== "object") return null;
221
- const command =
222
- typeof raw.command === "string"
223
- ? raw.command
224
- : typeof raw.cmd === "string"
225
- ? raw.cmd
226
- : null;
227
- if (!command || !command.trim()) return null;
228
- return command.trim();
229
- }
230
-
231
- function durationBucket(ms) {
232
- const n = Number(ms || 0);
233
- if (n < 100) return "<100ms";
234
- if (n < 500) return "100-500ms";
235
- if (n < 2000) return "0.5-2s";
236
- if (n < 10000) return "2-10s";
237
- if (n < 60000) return "10-60s";
238
- return ">=60s";
239
- }
240
-
241
- function outputSizeBucket(lines, chars) {
242
- if (lines >= 500 || chars >= 50_000) return "huge";
243
- if (lines >= 100 || chars >= 10_000) return "large";
244
- if (lines >= 20 || chars >= 2_000) return "medium";
245
- if (lines > 0 || chars > 0) return "small";
246
- return "empty";
247
- }
248
-
249
- function readUsageTotals(usage) {
250
- return normalizeGrokUsage(usage);
251
- }
252
-
253
- // ---------------------------------------------------------------------------
254
- // Per-session parse
255
- // ---------------------------------------------------------------------------
256
-
257
- function ensureTool(map, name) {
258
- if (!map.has(name)) map.set(name, { name, raw_name: name, calls: 0, totals: emptyTotals() });
259
- return map.get(name);
260
- }
261
-
262
- function ensureExec(map, key) {
263
- if (!map.has(key)) map.set(key, { name: key, ...buildExecStatsEntry() });
264
- return map.get(key);
265
- }
266
-
267
- function absorbExecStats(map, key, details) {
268
- const row = ensureExec(map, key);
269
- row.calls += 1;
270
- row.duration_ms += details.dur;
271
- row.max_duration_ms = Math.max(row.max_duration_ms, details.dur);
272
- row.output_chars += details.outputChars;
273
- row.output_lines += details.outputLines;
274
- if (details.failed) row.failures += 1;
275
- }
276
-
277
- function finalizeToolRows(map) {
278
- return Array.from(map.values())
279
- .map((row) => ({
280
- name: row.name,
281
- raw_name: row.raw_name || row.name,
282
- calls: row.calls,
283
- totals: roundTotals(row.totals),
284
- }))
285
- .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
286
- }
287
-
288
- function finalizeExecRows(map) {
289
- return Array.from(map.values())
290
- .map((row) => ({
291
- name: row.name,
292
- calls: row.calls,
293
- failures: row.failures,
294
- duration_ms: row.duration_ms,
295
- max_duration_ms: row.max_duration_ms,
296
- output_chars: row.output_chars,
297
- output_lines: row.output_lines,
298
- totals: roundTotals(row.totals),
299
- }))
300
- .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
301
- }
302
-
303
- function fileParseCacheKey(updatesPath, { from, to, timeZoneContext } = {}, stat) {
304
- return [
305
- CACHE_SCHEMA_VERSION,
306
- updatesPath,
307
- from || "",
308
- to || "",
309
- timeZoneContext?.timeZone || "",
310
- timeZoneContext?.offsetMinutes ?? "",
311
- Number(stat?.size || 0),
312
- Math.floor(Number(stat?.mtimeMs || 0)),
313
- ].join("|");
314
- }
315
-
316
- function rememberFileParse(key, value) {
317
- FILE_PARSE_CACHE.delete(key);
318
- FILE_PARSE_CACHE.set(key, value);
319
- while (FILE_PARSE_CACHE.size > MAX_FILE_PARSE_CACHE_ENTRIES) {
320
- const oldest = FILE_PARSE_CACHE.keys().next().value;
321
- if (oldest == null) break;
322
- FILE_PARSE_CACHE.delete(oldest);
323
- }
324
- }
325
-
326
- async function parseGrokUpdatesFile(updatesPath, { from, to, timeZoneContext } = {}) {
327
- let stat = null;
328
- try {
329
- stat = fs.statSync(updatesPath);
330
- } catch {
331
- return null;
332
- }
333
- if (!stat.isFile() || stat.size <= 0) return null;
334
-
335
- const cacheKey = fileParseCacheKey(updatesPath, { from, to, timeZoneContext }, stat);
336
- const hit = FILE_PARSE_CACHE.get(cacheKey);
337
- if (hit) {
338
- rememberFileParse(cacheKey, hit);
339
- return hit;
340
- }
341
-
342
- const byTool = new Map();
343
- const byExecKind = new Map();
344
- const byExecExit = new Map();
345
- const byExecExecutable = new Map();
346
- const byExecCommand = new Map();
347
- const byExecDuration = new Map();
348
- const byExecOutput = new Map();
349
- const totals = emptyTotals();
350
- let turnCount = 0;
351
-
352
- // promptId -> { tools: string[], execs: details[], sawThought: bool }
353
- const turns = new Map();
354
-
355
- function getTurn(key) {
356
- if (!turns.has(key)) {
357
- turns.set(key, { tools: [], execs: [], sawThought: false, sawMessage: false });
358
- }
359
- return turns.get(key);
360
- }
361
-
362
- function buildExecDetails(command, status) {
363
- const kind = inferExecCommandKind(command);
364
- const failed = String(status || "").toLowerCase() === "failed" || String(status || "").toLowerCase() === "error";
365
- return {
366
- kind,
367
- exitKey: failed ? "failed:unknown" : "completed:0",
368
- executable: getExecutableName(command),
369
- command: sanitizeCommandSignature(command),
370
- duration: durationBucket(0),
371
- output: outputSizeBucket(0, 0),
372
- dur: 0,
373
- outputChars: 0,
374
- outputLines: 0,
375
- failed,
376
- };
377
- }
378
-
379
- function attributeTurn(delta, turnState) {
380
- if (!delta || delta.total_tokens <= 0) return;
381
- turnCount += 1;
382
-
383
- const uniqueTools = [...new Set((turnState?.tools || []).filter(Boolean))];
384
- const tools = uniqueTools.length > 0 ? uniqueTools : ["text_response"];
385
- const share = 1 / tools.length;
386
-
387
- for (const name of tools) {
388
- const row = ensureTool(byTool, name);
389
- row.calls += share;
390
- addInto(row.totals, {
391
- input_tokens: delta.input_tokens * share,
392
- cached_input_tokens: delta.cached_input_tokens * share,
393
- cache_creation_input_tokens: delta.cache_creation_input_tokens * share,
394
- output_tokens: delta.output_tokens * share,
395
- reasoning_output_tokens: delta.reasoning_output_tokens * share,
396
- total_tokens: delta.total_tokens * share,
397
- });
398
- }
399
-
400
- // Prefer Bash bucket share when present (Codex attaches exec tokens only to
401
- // the shell tool's fraction of the turn).
402
- const bashShare = tools.includes("Bash") ? share : 0;
403
- const execDelta =
404
- bashShare > 0
405
- ? {
406
- input_tokens: delta.input_tokens * bashShare,
407
- cached_input_tokens: delta.cached_input_tokens * bashShare,
408
- cache_creation_input_tokens: delta.cache_creation_input_tokens * bashShare,
409
- output_tokens: delta.output_tokens * bashShare,
410
- reasoning_output_tokens: delta.reasoning_output_tokens * bashShare,
411
- total_tokens: delta.total_tokens * bashShare,
412
- }
413
- : null;
414
-
415
- const pendingExecDetails = turnState?.execs || [];
416
- if (execDelta && pendingExecDetails.length > 0) {
417
- const per = 1 / pendingExecDetails.length;
418
- for (const details of pendingExecDetails) {
419
- const attributed = {
420
- input_tokens: execDelta.input_tokens * per,
421
- cached_input_tokens: execDelta.cached_input_tokens * per,
422
- cache_creation_input_tokens: execDelta.cache_creation_input_tokens * per,
423
- output_tokens: execDelta.output_tokens * per,
424
- reasoning_output_tokens: execDelta.reasoning_output_tokens * per,
425
- total_tokens: execDelta.total_tokens * per,
426
- };
427
- addInto(ensureExec(byExecKind, details.kind).totals, attributed);
428
- addInto(ensureExec(byExecExit, details.exitKey).totals, attributed);
429
- addInto(ensureExec(byExecExecutable, details.executable).totals, attributed);
430
- addInto(ensureExec(byExecCommand, details.command).totals, attributed);
431
- addInto(ensureExec(byExecDuration, details.duration).totals, attributed);
432
- addInto(ensureExec(byExecOutput, details.output).totals, attributed);
433
- absorbExecStats(byExecKind, details.kind, details);
434
- absorbExecStats(byExecExit, details.exitKey, details);
435
- absorbExecStats(byExecExecutable, details.executable, details);
436
- absorbExecStats(byExecCommand, details.command, details);
437
- absorbExecStats(byExecDuration, details.duration, details);
438
- absorbExecStats(byExecOutput, details.output, details);
439
- }
440
- } else {
441
- for (const details of pendingExecDetails) {
442
- absorbExecStats(byExecKind, details.kind, details);
443
- absorbExecStats(byExecExit, details.exitKey, details);
444
- absorbExecStats(byExecExecutable, details.executable, details);
445
- absorbExecStats(byExecCommand, details.command, details);
446
- absorbExecStats(byExecDuration, details.duration, details);
447
- absorbExecStats(byExecOutput, details.output, details);
448
- }
449
- }
450
-
451
- addInto(totals, delta);
452
- }
453
-
454
- if (!fs.existsSync(updatesPath)) {
455
- return emptySessionResult();
456
- }
457
-
458
- const input = fs.createReadStream(updatesPath, { encoding: "utf8" });
459
- const rl = readline.createInterface({ input, crlfDelay: Infinity });
460
- try {
461
- for await (const line of rl) {
462
- if (!line || !line.trim()) continue;
463
- let record;
464
- try {
465
- record = JSON.parse(line);
466
- } catch {
467
- continue;
468
- }
469
- const params = record?.params || {};
470
- const meta = params._meta || record?._meta || {};
471
- const update = params.update;
472
- if (!update || typeof update !== "object") continue;
473
-
474
- const sessionUpdate = update.sessionUpdate || meta.updateType || "";
475
- const promptId =
476
- meta.promptId ||
477
- update.prompt_id ||
478
- (meta.turnStartMs != null ? `turn-${meta.turnStartMs}` : null) ||
479
- "unknown";
480
- const ts =
481
- timestampToIso(meta.agentTimestampMs) ||
482
- timestampToIso(meta.timestampMs) ||
483
- timestampToIso(record.timestamp) ||
484
- null;
485
-
486
- if (sessionUpdate === "tool_call" || meta.updateType === "ToolCall") {
487
- const turn = getTurn(promptId);
488
- const toolName = extractToolName(update, meta);
489
- turn.tools.push(toolName);
490
- if (toolName === "Bash") {
491
- const cmd = extractExecCommand(update);
492
- if (cmd) {
493
- const status = update.status || meta.updateParams?.status || "completed";
494
- turn.execs.push(buildExecDetails(cmd, status));
495
- }
496
- }
497
- continue;
498
- }
499
-
500
- if (sessionUpdate === "agent_thought_chunk" || meta.updateType === "AgentThoughtChunk") {
501
- getTurn(promptId).sawThought = true;
502
- continue;
503
- }
504
-
505
- if (
506
- sessionUpdate === "agent_message_chunk" ||
507
- sessionUpdate === "user_message_chunk" ||
508
- meta.updateType === "AgentMessageChunk"
509
- ) {
510
- getTurn(promptId).sawMessage = true;
511
- continue;
512
- }
513
-
514
- if (sessionUpdate !== "turn_completed") continue;
515
- if (ts && !isIsoInRange(ts, { from, to, timeZoneContext })) {
516
- turns.delete(promptId);
517
- continue;
518
- }
519
-
520
- const usage = readUsageTotals(update.usage);
521
- if (!usage) {
522
- turns.delete(promptId);
523
- continue;
524
- }
525
-
526
- // Prefer per-model split when present (sum models into one turn total —
527
- // already aggregated in usage.totalTokens).
528
- const turnState = turns.get(promptId) || { tools: [], execs: [], sawThought: false, sawMessage: false };
529
- attributeTurn(usage, turnState);
530
- turns.delete(promptId);
531
- }
532
- } finally {
533
- rl.close();
534
- input.destroy?.();
535
- }
536
-
537
- const parsedResult = {
538
- totals: roundTotals(totals),
539
- turnCount,
540
- toolBreakdown: { tool_rows: finalizeToolRows(byTool) },
541
- execCommandBreakdown: {
542
- by_type: finalizeExecRows(byExecKind),
543
- by_executable: finalizeExecRows(byExecExecutable),
544
- by_command: finalizeExecRows(byExecCommand),
545
- by_duration: finalizeExecRows(byExecDuration),
546
- by_output: finalizeExecRows(byExecOutput),
547
- by_exit: finalizeExecRows(byExecExit),
548
- },
549
- };
550
- rememberFileParse(cacheKey, parsedResult);
551
- return parsedResult;
552
- }
553
-
554
- function emptySessionResult() {
555
- return {
556
- totals: emptyTotals(),
557
- turnCount: 0,
558
- toolBreakdown: { tool_rows: [] },
559
- execCommandBreakdown: {
560
- by_type: [],
561
- by_executable: [],
562
- by_command: [],
563
- by_duration: [],
564
- by_output: [],
565
- by_exit: [],
566
- },
567
- };
568
- }
569
-
570
- // ---------------------------------------------------------------------------
571
- // Merge + display assembly (mirrors codex-context-breakdown)
572
- // ---------------------------------------------------------------------------
573
-
574
- function mergeRows(map, rows) {
575
- for (const row of rows || []) {
576
- const name = row?.name ? String(row.name) : "";
577
- const rawName = row?.raw_name ? String(row.raw_name) : name;
578
- const key = rawName || name;
579
- if (!key) continue;
580
- if (!map.has(key)) {
581
- map.set(key, { name, raw_name: rawName, calls: 0, totals: emptyTotals() });
582
- }
583
- const cur = map.get(key);
584
- cur.name = name;
585
- cur.raw_name = rawName;
586
- cur.calls += Number(row.calls || 0);
587
- addInto(cur.totals, row.totals || {});
588
- }
589
- }
590
-
591
- function mergeExecRows(map, rows) {
592
- for (const row of rows || []) {
593
- const name = row?.name ? String(row.name) : "";
594
- if (!name) continue;
595
- if (!map.has(name)) map.set(name, { name, ...buildExecStatsEntry() });
596
- const cur = map.get(name);
597
- cur.calls += Number(row.calls || 0);
598
- cur.failures += Number(row.failures || 0);
599
- cur.duration_ms += Number(row.duration_ms || 0);
600
- cur.max_duration_ms = Math.max(cur.max_duration_ms, Number(row.max_duration_ms || 0));
601
- cur.output_chars += Number(row.output_chars || 0);
602
- cur.output_lines += Number(row.output_lines || 0);
603
- addInto(cur.totals, row.totals || {});
604
- }
605
- }
606
-
607
- function buildGrokInventorySignature(sessions) {
608
- return crypto
609
- .createHash("sha256")
610
- .update(
611
- (sessions || [])
612
- .map((s) => `${s.updatesPath}:${s.size}:${Math.floor(s.mtimeMs)}`)
613
- .join("|"),
614
- )
615
- .digest("hex");
616
- }
617
-
618
- function buildGrokResultCacheKey({ from, to, top, timeZoneContext, inventorySig }) {
619
- // Include inventorySig so a changed session file cannot serve a stale aggregate.
620
- // Unchanged files still skip re-parse via FILE_PARSE_CACHE (Codex-style).
621
- return [
622
- CACHE_SCHEMA_VERSION,
623
- from || "",
624
- to || "",
625
- top,
626
- timeZoneContext?.timeZone || "",
627
- timeZoneContext?.offsetMinutes ?? "",
628
- inventorySig || "",
629
- ].join("|");
630
- }
631
-
632
- async function computeGrokContextBreakdown(options = {}) {
633
- const {
634
- from = null,
635
- to = null,
636
- top = 50,
637
- timeZoneContext = null,
638
- env = process.env,
639
- } = options;
640
- const limitedTop = Number.isFinite(top) && top > 0 ? Math.min(Math.floor(top), 200) : 50;
641
- const sessions = discoverGrokSessionFiles(env);
642
- const inventorySig = buildGrokInventorySignature(sessions);
643
- const cacheKey = buildGrokResultCacheKey({
644
- from,
645
- to,
646
- top: limitedTop,
647
- timeZoneContext,
648
- inventorySig,
649
- });
650
-
651
- const cached = CACHE.get(cacheKey);
652
- if (cached && Date.now() - cached.at < CACHE_TTL_MS) {
653
- return cached.value;
654
- }
655
-
656
- const existing = IN_FLIGHT.get(cacheKey);
657
- if (existing) return existing;
658
-
659
- const run = computeGrokContextBreakdownUncached({
660
- from,
661
- to,
662
- top: limitedTop,
663
- timeZoneContext,
664
- env,
665
- sessions,
666
- cacheKey,
667
- });
668
- IN_FLIGHT.set(cacheKey, run);
669
- try {
670
- return await run;
671
- } finally {
672
- if (IN_FLIGHT.get(cacheKey) === run) IN_FLIGHT.delete(cacheKey);
673
- }
674
- }
675
-
676
- async function computeGrokContextBreakdownUncached({
677
- from = null,
678
- to = null,
679
- top = 50,
680
- timeZoneContext = null,
681
- env = process.env,
682
- sessions = null,
683
- cacheKey = null,
684
- } = {}) {
685
- const limitedTop = Number.isFinite(top) && top > 0 ? Math.min(Math.floor(top), 200) : 50;
686
- const sessionList = sessions || discoverGrokSessionFiles(env);
687
- const resultCacheKey =
688
- cacheKey ||
689
- buildGrokResultCacheKey({
690
- from,
691
- to,
692
- top: limitedTop,
693
- timeZoneContext,
694
- inventorySig: buildGrokInventorySignature(sessionList),
695
- });
696
-
697
- const grand = emptyTotals();
698
- const byTool = new Map();
699
- const byExecKind = new Map();
700
- const byExecExit = new Map();
701
- const byExecExecutable = new Map();
702
- const byExecCommand = new Map();
703
- const byExecDuration = new Map();
704
- const byExecOutput = new Map();
705
- let messageCount = 0;
706
- let sessionCount = 0;
707
-
708
- for (const sess of sessionList) {
709
- const parsed = await parseGrokUpdatesFile(sess.updatesPath, {
710
- from,
711
- to,
712
- timeZoneContext,
713
- });
714
- if (!parsed?.totals?.total_tokens) continue;
715
- sessionCount += 1;
716
- messageCount += Number(parsed.turnCount || 0);
717
- addInto(grand, parsed.totals);
718
- mergeRows(byTool, parsed.toolBreakdown?.tool_rows);
719
- mergeExecRows(byExecKind, parsed.execCommandBreakdown?.by_type);
720
- mergeExecRows(byExecExecutable, parsed.execCommandBreakdown?.by_executable);
721
- mergeExecRows(byExecCommand, parsed.execCommandBreakdown?.by_command);
722
- mergeExecRows(byExecDuration, parsed.execCommandBreakdown?.by_duration);
723
- mergeExecRows(byExecOutput, parsed.execCommandBreakdown?.by_output);
724
- mergeExecRows(byExecExit, parsed.execCommandBreakdown?.by_exit);
725
- }
726
-
727
- const toolRows = finalizeToolRows(byTool);
728
- const toolRowsLimited = toolRows.slice(0, limitedTop);
729
-
730
- // Category rows from tool names (File Ops / Search / Execution / ...)
731
- const byCategory = new Map();
732
- for (const row of toolRows) {
733
- if (row.name === "text_response") continue;
734
- const cat = categorizeTool(row.raw_name || row.name);
735
- if (!byCategory.has(cat)) {
736
- byCategory.set(cat, { name: cat, calls: 0, totals: emptyTotals(), tools: [] });
737
- }
738
- const target = byCategory.get(cat);
739
- target.calls += Number(row.calls || 0);
740
- addInto(target.totals, row.totals || {});
741
- target.tools.push({
742
- name: row.name,
743
- calls: row.calls,
744
- totals: roundTotals(row.totals),
745
- });
746
- }
747
- const categoryRows = Array.from(byCategory.values())
748
- .map((c) => ({
749
- name: c.name,
750
- calls: Math.round(c.calls || 0),
751
- totals: roundTotals(c.totals),
752
- tools: (c.tools || [])
753
- .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0))
754
- .slice(0, limitedTop),
755
- }))
756
- .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
757
-
758
- // Message breakdown uses the same mutually exclusive columns as the parser.
759
- // Grok's raw outputTokens includes reasoning, but readUsageTotals has already
760
- // split it, so subtracting reasoning here would undercount assistant output.
761
- const reasoning = Number(grand.reasoning_output_tokens || 0);
762
- const assistantOut = Number(grand.output_tokens || 0);
763
- // For Grok, cached reads dominate conversation history; non-cached input is
764
- // closer to "current user/context injection". Heuristic only — matches Codex
765
- // message_breakdown intent without reading message bodies.
766
- const cachedHistoryTokens = Number(grand.cached_input_tokens || 0);
767
- const cacheCreationTokens = Number(grand.cache_creation_input_tokens || 0);
768
- const historyTokens = cachedHistoryTokens + cacheCreationTokens;
769
- const userish = Number(grand.input_tokens || 0);
770
- const messageBreakdown = [
771
- {
772
- key: "user_input",
773
- name: "User input",
774
- totals: roundTotals({
775
- input_tokens: userish,
776
- cached_input_tokens: 0,
777
- cache_creation_input_tokens: 0,
778
- output_tokens: 0,
779
- reasoning_output_tokens: 0,
780
- total_tokens: userish,
781
- }),
782
- },
783
- {
784
- key: "conversation_history",
785
- name: "Conversation history",
786
- totals: roundTotals({
787
- input_tokens: 0,
788
- cached_input_tokens: cachedHistoryTokens,
789
- cache_creation_input_tokens: cacheCreationTokens,
790
- output_tokens: 0,
791
- reasoning_output_tokens: 0,
792
- total_tokens: historyTokens,
793
- }),
794
- },
795
- {
796
- key: "assistant_response",
797
- name: "Assistant response",
798
- totals: roundTotals({
799
- input_tokens: 0,
800
- cached_input_tokens: 0,
801
- cache_creation_input_tokens: 0,
802
- output_tokens: assistantOut,
803
- reasoning_output_tokens: 0,
804
- total_tokens: assistantOut,
805
- }),
806
- },
807
- {
808
- key: "reasoning",
809
- name: "Reasoning",
810
- totals: roundTotals({
811
- input_tokens: 0,
812
- cached_input_tokens: 0,
813
- cache_creation_input_tokens: 0,
814
- output_tokens: 0,
815
- reasoning_output_tokens: reasoning,
816
- total_tokens: reasoning,
817
- }),
818
- },
819
- ].sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
820
-
821
- const serializeExecRows = (rows) =>
822
- (rows || []).slice(0, limitedTop).map((r) => ({
823
- name: r.name,
824
- calls: r.calls,
825
- failures: r.failures,
826
- duration_ms: r.duration_ms,
827
- max_duration_ms: r.max_duration_ms,
828
- output_chars: r.output_chars,
829
- output_lines: r.output_lines,
830
- totals: roundTotals(r.totals),
831
- }));
832
-
833
- const result = {
834
- source: "grok",
835
- scope: "supported",
836
- breakdown_status: "ok",
837
- totals: roundTotals(grand),
838
- session_count: sessionCount,
839
- message_count: messageCount,
840
- message_breakdown: {
841
- categories: messageBreakdown,
842
- privacy: {
843
- includes_content: false,
844
- note: "Aggregated message token categories only; prompt and assistant text are never returned.",
845
- },
846
- },
847
- tool_calls_breakdown: {
848
- total_calls: Math.round(toolRows.reduce((a, r) => a + Number(r.calls || 0), 0)),
849
- tools: toolRowsLimited,
850
- categories: categoryRows.slice(0, limitedTop),
851
- tools_total: toolRows.reduce((a, r) => a + Math.round(r.totals?.total_tokens || 0), 0),
852
- privacy: {
853
- includes_inputs: false,
854
- note: "Aggregated tool names only; no tool arguments or outputs are included.",
855
- },
856
- },
857
- skills_breakdown: {
858
- total_calls: 0,
859
- skills: [],
860
- privacy: {
861
- includes_inputs: false,
862
- note: "Grok Build does not currently expose a dedicated skill usage channel in turn logs.",
863
- },
864
- },
865
- exec_command_breakdown: {
866
- by_type: serializeExecRows(finalizeExecRows(byExecKind)),
867
- by_executable: serializeExecRows(finalizeExecRows(byExecExecutable)),
868
- by_command: serializeExecRows(finalizeExecRows(byExecCommand)),
869
- by_duration: serializeExecRows(finalizeExecRows(byExecDuration)),
870
- by_output: serializeExecRows(finalizeExecRows(byExecOutput)),
871
- by_exit: serializeExecRows(finalizeExecRows(byExecExit)),
872
- },
873
- };
874
-
875
- CACHE.set(resultCacheKey, { at: Date.now(), value: result });
876
- while (CACHE.size > 32) {
877
- const oldest = [...CACHE.entries()].sort((a, b) => a[1].at - b[1].at)[0];
878
- if (oldest) CACHE.delete(oldest[0]);
879
- }
880
- return result;
881
- }
882
-
883
- module.exports = {
884
- computeGrokContextBreakdown,
885
- // test helpers
886
- mapGrokToolName,
887
- readUsageTotals,
888
- discoverGrokSessionFiles,
889
- parseGrokUpdatesFile,
890
- };