tokenusage-cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/LICENSE +22 -0
  3. package/README.md +94 -0
  4. package/THIRD_PARTY_NOTICES.md +140 -0
  5. package/bin/tokenusage.js +69 -0
  6. package/package.json +59 -0
  7. package/src/cli.js +134 -0
  8. package/src/commands/device-login.js +205 -0
  9. package/src/commands/diagnostics.js +38 -0
  10. package/src/commands/doctor.js +110 -0
  11. package/src/commands/init.js +2436 -0
  12. package/src/commands/sessions.js +29 -0
  13. package/src/commands/status.js +1747 -0
  14. package/src/commands/sync.js +7934 -0
  15. package/src/commands/uninstall.js +312 -0
  16. package/src/lib/account-usage-dedup.js +267 -0
  17. package/src/lib/antigravity-paths.js +48 -0
  18. package/src/lib/ark-agent-plan-limits.js +438 -0
  19. package/src/lib/ark-coding-plan-limits.js +537 -0
  20. package/src/lib/async-queue-reader.js +138 -0
  21. package/src/lib/atomic-json.js +27 -0
  22. package/src/lib/bearer-token.js +11 -0
  23. package/src/lib/categorizer-utils.js +232 -0
  24. package/src/lib/claude-categorizer.js +1343 -0
  25. package/src/lib/claude-config.js +279 -0
  26. package/src/lib/claude-user-identity.js +56 -0
  27. package/src/lib/cli-ui.js +179 -0
  28. package/src/lib/cloud-account.js +541 -0
  29. package/src/lib/cloud-device-token.js +141 -0
  30. package/src/lib/cloud-sync-prefs.js +14 -0
  31. package/src/lib/cloud-usage-reset.js +115 -0
  32. package/src/lib/codex-config.js +595 -0
  33. package/src/lib/codex-context-breakdown.js +806 -0
  34. package/src/lib/codex-model-attribution.js +128 -0
  35. package/src/lib/codex-rollout-parser.js +1471 -0
  36. package/src/lib/codex-service-tier.js +49 -0
  37. package/src/lib/codex-token-refresh.js +140 -0
  38. package/src/lib/codex-token-usage.js +195 -0
  39. package/src/lib/codex-usage-record.js +195 -0
  40. package/src/lib/command-runner.js +313 -0
  41. package/src/lib/commandcode-limits.js +317 -0
  42. package/src/lib/context-health.js +124 -0
  43. package/src/lib/cursor-config.js +605 -0
  44. package/src/lib/cursor-store.js +1918 -0
  45. package/src/lib/debug-flags.js +7 -0
  46. package/src/lib/devin-limits.js +249 -0
  47. package/src/lib/devin-usage.js +212 -0
  48. package/src/lib/diagnostics.js +383 -0
  49. package/src/lib/doctor.js +385 -0
  50. package/src/lib/dumate.js +95 -0
  51. package/src/lib/fs.js +455 -0
  52. package/src/lib/function-url.js +14 -0
  53. package/src/lib/gemini-config.js +291 -0
  54. package/src/lib/git-outcomes.js +243 -0
  55. package/src/lib/grok-context-breakdown.js +890 -0
  56. package/src/lib/grok-hook.js +360 -0
  57. package/src/lib/grok-limits.js +865 -0
  58. package/src/lib/grok-usage.js +98 -0
  59. package/src/lib/heatmap-compact.js +114 -0
  60. package/src/lib/ingest-sanitize.js +41 -0
  61. package/src/lib/init-flow.js +42 -0
  62. package/src/lib/install-resolver.js +170 -0
  63. package/src/lib/jsonl-lines.js +134 -0
  64. package/src/lib/kimi-model-repair.js +173 -0
  65. package/src/lib/kimi-profile.js +61 -0
  66. package/src/lib/kimi-session.js +216 -0
  67. package/src/lib/local-api.js +3588 -0
  68. package/src/lib/machine-id.js +238 -0
  69. package/src/lib/multi-install-parser.js +113 -0
  70. package/src/lib/notify-command.js +38 -0
  71. package/src/lib/omp-hook.js +589 -0
  72. package/src/lib/openclaw-cli.js +104 -0
  73. package/src/lib/openclaw-hook.js +375 -0
  74. package/src/lib/openclaw-session-plugin.js +560 -0
  75. package/src/lib/opencode-config.js +121 -0
  76. package/src/lib/opencode-go-limits.js +807 -0
  77. package/src/lib/outcomes-engine.js +253 -0
  78. package/src/lib/passive-mode.js +219 -0
  79. package/src/lib/pet-packages.js +722 -0
  80. package/src/lib/pricing/curated-overrides.json +822 -0
  81. package/src/lib/pricing/index.js +379 -0
  82. package/src/lib/pricing/litellm-fetcher.js +172 -0
  83. package/src/lib/pricing/matcher.js +456 -0
  84. package/src/lib/pricing/seed-snapshot.json +1 -0
  85. package/src/lib/progress.js +76 -0
  86. package/src/lib/project-usage-purge.js +117 -0
  87. package/src/lib/prompt.js +24 -0
  88. package/src/lib/provider-status.js +104 -0
  89. package/src/lib/proxy-env.js +489 -0
  90. package/src/lib/proxy-settings.js +163 -0
  91. package/src/lib/qoder-cn-cli-limits.js +119 -0
  92. package/src/lib/qoder-limits.js +880 -0
  93. package/src/lib/quiet-warnings.js +26 -0
  94. package/src/lib/rollout.js +25966 -0
  95. package/src/lib/runtime-config.js +259 -0
  96. package/src/lib/scan-roots.js +277 -0
  97. package/src/lib/sentry.js +231 -0
  98. package/src/lib/session-analytics.js +2972 -0
  99. package/src/lib/session-performance.js +237 -0
  100. package/src/lib/session-upload.js +379 -0
  101. package/src/lib/skill-usage.js +267 -0
  102. package/src/lib/skills-manager.js +1591 -0
  103. package/src/lib/source-metadata.js +48 -0
  104. package/src/lib/sqlite-reader.js +215 -0
  105. package/src/lib/star-cta.js +73 -0
  106. package/src/lib/subscription-manager.js +293 -0
  107. package/src/lib/subscriptions.js +508 -0
  108. package/src/lib/telemetry.js +193 -0
  109. package/src/lib/tracker-paths.js +16 -0
  110. package/src/lib/trae-cn-config.js +548 -0
  111. package/src/lib/trae-db.js +127 -0
  112. package/src/lib/trae-snapshot.js +230 -0
  113. package/src/lib/trae-sqlite.js +143 -0
  114. package/src/lib/trae-usage.js +121 -0
  115. package/src/lib/upload-queue.js +134 -0
  116. package/src/lib/upload-throttle.js +171 -0
  117. package/src/lib/usage-limits.js +4552 -0
  118. package/src/lib/wrapped-aggregator.js +236 -0
  119. package/src/lib/wsl-probe.js +281 -0
  120. package/src/lib/zcode-limits.js +1172 -0
  121. package/src/lib/zcode-reset-credits.js +66 -0
  122. package/test/cli-smoke.test.js +53 -0
  123. package/test/ingest-sanitize.test.js +162 -0
  124. package/test/quiet-warnings.test.js +53 -0
  125. package/test/session-upload.test.js +427 -0
@@ -0,0 +1,806 @@
1
+ // Codex CLI "Context Breakdown" — tool-oriented view.
2
+ //
3
+ // Privacy commitment: tokens + timestamps only. We do not return prompt text,
4
+ // assistant text, tool outputs, file contents, or exec_command arguments.
5
+ //
6
+ // Data source: ~/.codex/sessions/**/rollout-*.jsonl
7
+ // We treat each token_count event as the authoritative delta and attribute
8
+ // that delta to "turn" activity since the last token_count. Tool attribution
9
+ // is heuristic: delta is split evenly across tools used in that turn.
10
+
11
+ "use strict";
12
+
13
+ const crypto = require("node:crypto");
14
+ const fs = require("node:fs");
15
+ const os = require("node:os");
16
+ const path = require("node:path");
17
+
18
+ const { resolveEnvRoot } = require("./scan-roots");
19
+
20
+ const {
21
+ emptyTotals,
22
+ addInto,
23
+ roundTotals,
24
+ buildExecStatsEntry,
25
+ allocateByLargestRemainder,
26
+ categorizeTool,
27
+ } = require("./categorizer-utils");
28
+
29
+ const {
30
+ parseCodexRolloutFile,
31
+ discoverCodexSessionFiles,
32
+ rolloutSessionIdFromPath,
33
+ finalizeToolRows,
34
+ finalizeSkillRows,
35
+ finalizeExecRows,
36
+ buildSkillStatsEntry,
37
+ isCodexResumeInvalidated,
38
+ isRecoverableGroupedReadError,
39
+ } = require("./codex-rollout-parser");
40
+
41
+ // ---------------------------------------------------------------------------
42
+ // Merge helpers
43
+ // ---------------------------------------------------------------------------
44
+
45
+ function mergeRollupTotals(target, add) {
46
+ addInto(target, add);
47
+ }
48
+
49
+ function mergeRows(map, rows) {
50
+ for (const row of rows || []) {
51
+ const name = row?.name ? String(row.name) : "";
52
+ const rawName = row?.raw_name ? String(row.raw_name) : name;
53
+ const key = rawName || name;
54
+ if (!key) continue;
55
+ if (!map.has(key)) {
56
+ map.set(key, { name, raw_name: rawName, calls: 0, totals: emptyTotals() });
57
+ }
58
+ const cur = map.get(key);
59
+ cur.name = name;
60
+ cur.raw_name = rawName;
61
+ cur.calls += Number(row.calls || 0);
62
+ mergeRollupTotals(cur.totals, row.totals || {});
63
+ }
64
+ }
65
+
66
+ function mergeSkillRows(map, rows) {
67
+ for (const row of rows || []) {
68
+ const name = row?.name ? String(row.name) : "";
69
+ if (!name) continue;
70
+ if (!map.has(name)) map.set(name, buildSkillStatsEntry(name));
71
+ const cur = map.get(name);
72
+ cur.calls += Number(row.calls || 0);
73
+ mergeRollupTotals(cur.totals, row.totals || {});
74
+ }
75
+ }
76
+
77
+ function mergeExecRows(map, rows) {
78
+ for (const row of rows || []) {
79
+ const name = row?.name ? String(row.name) : "";
80
+ if (!name) continue;
81
+ if (!map.has(name)) map.set(name, { name, ...buildExecStatsEntry() });
82
+ const cur = map.get(name);
83
+ cur.calls += Number(row.calls || 0);
84
+ cur.failures += Number(row.failures || 0);
85
+ cur.duration_ms += Number(row.duration_ms || 0);
86
+ cur.max_duration_ms = Math.max(cur.max_duration_ms, Number(row.max_duration_ms || 0));
87
+ cur.output_chars += Number(row.output_chars || 0);
88
+ cur.output_lines += Number(row.output_lines || 0);
89
+ mergeRollupTotals(cur.totals, row.totals || {});
90
+ }
91
+ }
92
+
93
+ function mergeParsedSessionResults(previous, delta) {
94
+ if (!previous) return delta;
95
+ if (!delta) return previous;
96
+
97
+ const totals = emptyTotals();
98
+ mergeRollupTotals(totals, previous.totals || {});
99
+ mergeRollupTotals(totals, delta.totals || {});
100
+
101
+ const tools = new Map();
102
+ mergeRows(tools, previous.toolBreakdown?.tool_rows);
103
+ mergeRows(tools, delta.toolBreakdown?.tool_rows);
104
+ const skills = new Map();
105
+ mergeSkillRows(skills, previous.skillsBreakdown?.skill_rows);
106
+ mergeSkillRows(skills, delta.skillsBreakdown?.skill_rows);
107
+
108
+ const mergeExecGroup = (key) => {
109
+ const rows = new Map();
110
+ mergeExecRows(rows, previous.execCommandBreakdown?.[key]);
111
+ mergeExecRows(rows, delta.execCommandBreakdown?.[key]);
112
+ return finalizeExecRows(rows);
113
+ };
114
+
115
+ return {
116
+ sessionId: delta.sessionId || previous.sessionId,
117
+ cwd: delta.cwd || previous.cwd,
118
+ model: delta.model || previous.model,
119
+ provider: delta.provider || previous.provider,
120
+ version: delta.version || previous.version,
121
+ filePath: previous.filePath || delta.filePath,
122
+ turnCount: Number(previous.turnCount || 0) + Number(delta.turnCount || 0),
123
+ totals,
124
+ toolBreakdown: { tool_rows: finalizeToolRows(tools) },
125
+ skillsBreakdown: { skill_rows: finalizeSkillRows(skills) },
126
+ execCommandBreakdown: {
127
+ byType: mergeExecGroup("byType"),
128
+ byExit: mergeExecGroup("byExit"),
129
+ byExecutable: mergeExecGroup("byExecutable"),
130
+ byCommand: mergeExecGroup("byCommand"),
131
+ byDuration: mergeExecGroup("byDuration"),
132
+ byOutput: mergeExecGroup("byOutput"),
133
+ },
134
+ };
135
+ }
136
+
137
+ // ---------------------------------------------------------------------------
138
+ // Period helpers
139
+ // ---------------------------------------------------------------------------
140
+
141
+ function normalizePeriod(period) {
142
+ const p = String(period || "").trim().toLowerCase();
143
+ if (!p) return null;
144
+ if (["day", "week", "month", "total"].includes(p)) return p;
145
+ return null;
146
+ }
147
+
148
+ function buildDateRange({ period, date }) {
149
+ const anchor = date ? new Date(`${date}T00:00:00Z`) : new Date();
150
+ if (!Number.isFinite(anchor.getTime())) return null;
151
+ const end = new Date(`${anchor.toISOString().slice(0, 10)}T23:59:59Z`);
152
+ if (!Number.isFinite(end.getTime())) return null;
153
+
154
+ let start;
155
+ if (period === "day") start = new Date(`${anchor.toISOString().slice(0, 10)}T00:00:00Z`);
156
+ else if (period === "week") start = new Date(end.getTime() - 6 * 86400_000);
157
+ else if (period === "month") start = new Date(end.getTime() - 29 * 86400_000);
158
+ else if (period === "total") start = null;
159
+ else return null;
160
+
161
+ return {
162
+ from: start ? start.toISOString().slice(0, 10) : null,
163
+ to: end.toISOString().slice(0, 10),
164
+ };
165
+ }
166
+
167
+ // ---------------------------------------------------------------------------
168
+ // Cache
169
+ // ---------------------------------------------------------------------------
170
+
171
+ const CACHE = new Map();
172
+ const PARSED_GROUP_CACHE = new Map();
173
+ const CACHE_TTL_MS = 60_000;
174
+ const CACHE_SCHEMA_VERSION = "codex-context-v6";
175
+ const MAX_PARSED_GROUP_CACHE_ENTRIES = 4096;
176
+ const PREFIX_HASH_READ_BYTES = 1024 * 1024;
177
+
178
+ function cacheTimeZoneKey(timeZoneContext) {
179
+ if (!timeZoneContext) return "";
180
+ return `${timeZoneContext.timeZone || ""}|${Number.isFinite(timeZoneContext.offsetMinutes) ? timeZoneContext.offsetMinutes : ""}`;
181
+ }
182
+
183
+ function statIdentity(filePath, stat) {
184
+ return JSON.stringify([filePath, stat.dev, stat.ino, stat.size, stat.mtimeMs, stat.ctimeMs]);
185
+ }
186
+
187
+ function contentStatIdentity(filePath, stat) {
188
+ return JSON.stringify([
189
+ filePath,
190
+ stat.dev,
191
+ stat.ino,
192
+ stat.size,
193
+ stat.mtimeMs,
194
+ ]);
195
+ }
196
+
197
+ function isUnlinkedResumeSnapshot(filePath, before, after) {
198
+ return Boolean(
199
+ before &&
200
+ after &&
201
+ Number(after.nlink) === 0 &&
202
+ contentStatIdentity(filePath, before) === contentStatIdentity(filePath, after),
203
+ );
204
+ }
205
+
206
+ function inventorySignature(entries) {
207
+ const hash = crypto.createHash("sha256");
208
+ for (const { filePath, stat, candidate } of entries) {
209
+ hash.update(statIdentity(filePath, stat));
210
+ hash.update(candidate ? "1" : "0");
211
+ hash.update("\0");
212
+ }
213
+ return hash.digest("hex");
214
+ }
215
+
216
+ function parsedGroupCacheKey(entries, { from, to, timeZoneContext }) {
217
+ const hash = crypto.createHash("sha256");
218
+ hash.update(CACHE_SCHEMA_VERSION);
219
+ hash.update(from || "");
220
+ hash.update("\0");
221
+ hash.update(to || "");
222
+ hash.update("\0");
223
+ hash.update(cacheTimeZoneKey(timeZoneContext));
224
+ for (const { filePath } of entries) {
225
+ hash.update(filePath);
226
+ hash.update("\0");
227
+ }
228
+ return hash.digest("hex");
229
+ }
230
+
231
+ function parsedGroupSignature(entries) {
232
+ const hash = crypto.createHash("sha256");
233
+ for (const { filePath, stat } of entries) {
234
+ hash.update(statIdentity(filePath, stat));
235
+ hash.update("\0");
236
+ }
237
+ return hash.digest("hex");
238
+ }
239
+
240
+ function selectAppendHashPaths(candidateGroups) {
241
+ // Only the newest single-file group pays the cold hash cost. Any file that
242
+ // later grows becomes newest, takes one conservative full parse, and keeps
243
+ // its hash state for subsequent append-only refreshes.
244
+ let newest = null;
245
+ for (const entries of candidateGroups.values()) {
246
+ if (
247
+ entries.length !== 1 ||
248
+ !rolloutSessionIdFromPath(entries[0].filePath)
249
+ ) continue;
250
+ const entry = entries[0];
251
+ if (!newest) {
252
+ newest = entry;
253
+ continue;
254
+ }
255
+ const mtimeDelta = Number(entry.stat?.mtimeMs || 0) -
256
+ Number(newest.stat?.mtimeMs || 0);
257
+ const ctimeDelta = Number(entry.stat?.ctimeMs || 0) -
258
+ Number(newest.stat?.ctimeMs || 0);
259
+ if (
260
+ mtimeDelta > 0 ||
261
+ (mtimeDelta === 0 && ctimeDelta > 0) ||
262
+ (mtimeDelta === 0 && ctimeDelta === 0 &&
263
+ entry.filePath.localeCompare(newest.filePath) < 0)
264
+ ) {
265
+ newest = entry;
266
+ }
267
+ }
268
+ return newest ? new Set([newest.filePath]) : new Set();
269
+ }
270
+
271
+ async function digestFilePrefix(
272
+ sourceHandle,
273
+ endOffset,
274
+ diagnostics = null,
275
+ ) {
276
+ const end = Math.max(0, Number(endOffset) || 0);
277
+ const hash = crypto.createHash("sha256");
278
+ if (end === 0) return hash.digest("hex");
279
+ let bytesRead = 0;
280
+ const buffer = Buffer.allocUnsafe(PREFIX_HASH_READ_BYTES);
281
+ while (bytesRead < end) {
282
+ const length = Math.min(buffer.length, end - bytesRead);
283
+ const result = await sourceHandle.read(buffer, 0, length, bytesRead);
284
+ if (!result.bytesRead) break;
285
+ hash.update(buffer.subarray(0, result.bytesRead));
286
+ bytesRead += result.bytesRead;
287
+ }
288
+ if (diagnostics) {
289
+ diagnostics.prefix_validation_bytes =
290
+ Number(diagnostics.prefix_validation_bytes || 0) + bytesRead;
291
+ }
292
+ return bytesRead === end ? hash.digest("hex") : null;
293
+ }
294
+
295
+ async function openValidatedResumeFile(cached, entry, diagnostics = null) {
296
+ if (
297
+ !cached?.resumeState ||
298
+ !cached?.stat ||
299
+ !cached?.contentHashState ||
300
+ typeof cached.contentHashState.copy !== "function" ||
301
+ cached.appendable !== true ||
302
+ cached.filePath !== entry.filePath
303
+ ) return null;
304
+ const previous = cached.stat;
305
+ const current = entry.stat;
306
+ if (
307
+ Number(previous.dev) !== Number(current.dev) ||
308
+ Number(previous.ino) !== Number(current.ino) ||
309
+ Number(current.size) <= Number(previous.size) ||
310
+ Number(cached.endOffset) < 0 ||
311
+ Number(cached.endOffset) > Number(previous.size)
312
+ ) {
313
+ return null;
314
+ }
315
+ let handle = null;
316
+ let keepOpen = false;
317
+ try {
318
+ // Validation and suffix parsing share this handle so a path replacement
319
+ // cannot splice cached aggregates onto bytes from a different file.
320
+ handle = await fs.promises.open(entry.filePath, "r");
321
+ if (diagnostics) diagnostics.stat_calls += 1;
322
+ const opened = await handle.stat();
323
+ if (
324
+ statIdentity(entry.filePath, opened) !==
325
+ statIdentity(entry.filePath, current)
326
+ ) return null;
327
+
328
+ const expected = cached.contentHashState.copy().digest("hex");
329
+ const actual = await digestFilePrefix(
330
+ handle,
331
+ cached.endOffset,
332
+ diagnostics,
333
+ );
334
+ if (actual !== expected) return null;
335
+ if (diagnostics) diagnostics.stat_calls += 1;
336
+ const afterValidation = await handle.stat();
337
+ if (
338
+ statIdentity(entry.filePath, current) !==
339
+ statIdentity(entry.filePath, afterValidation) &&
340
+ !isUnlinkedResumeSnapshot(entry.filePath, current, afterValidation)
341
+ ) return null;
342
+ keepOpen = true;
343
+ return handle;
344
+ } catch {
345
+ return null;
346
+ } finally {
347
+ if (handle && !keepOpen) {
348
+ try {
349
+ await handle.close();
350
+ } catch {
351
+ // A failed validation already forces the conservative full parse.
352
+ }
353
+ }
354
+ }
355
+ }
356
+
357
+ function rememberParsedGroup(key, value) {
358
+ PARSED_GROUP_CACHE.delete(key);
359
+ PARSED_GROUP_CACHE.set(key, value);
360
+ while (PARSED_GROUP_CACHE.size > MAX_PARSED_GROUP_CACHE_ENTRIES) {
361
+ const oldest = PARSED_GROUP_CACHE.keys().next().value;
362
+ if (!oldest) break;
363
+ PARSED_GROUP_CACHE.delete(oldest);
364
+ }
365
+ }
366
+
367
+ function splitCapturedParse(parsed) {
368
+ const {
369
+ resumeState = null,
370
+ endOffset = 0,
371
+ appendable = false,
372
+ contentHashState = null,
373
+ ...result
374
+ } = parsed;
375
+ return { parsed: result, resumeState, endOffset, appendable, contentHashState };
376
+ }
377
+
378
+ // ---------------------------------------------------------------------------
379
+ // Main entry
380
+ // ---------------------------------------------------------------------------
381
+
382
+ async function computeCodexContextBreakdown({
383
+ from = null,
384
+ to = null,
385
+ period = null,
386
+ date = null,
387
+ codexDir = null,
388
+ top = 20,
389
+ timeZoneContext = null,
390
+ exhaustive = false,
391
+ nowMs = Date.now(),
392
+ includeDiagnostics = false,
393
+ } = {}) {
394
+ let fromKey = from;
395
+ let toKey = to;
396
+ if ((!fromKey && !toKey) && normalizePeriod(period)) {
397
+ const range = buildDateRange({ period: normalizePeriod(period), date });
398
+ fromKey = range?.from || null;
399
+ toKey = range?.to || null;
400
+ }
401
+
402
+ const codexHome = resolveEnvRoot("codex", { env: process.env, home: os.homedir() }) || path.join(os.homedir(), ".codex");
403
+ const roots = codexDir
404
+ ? [codexDir]
405
+ : [path.join(codexHome, "sessions"), path.join(codexHome, "archived_sessions")];
406
+ const baseDir = roots.join(path.delimiter);
407
+ const diagnostics = {
408
+ cache_hit: false,
409
+ full_metadata_audit: false,
410
+ discovered_files: 0,
411
+ candidate_files: 0,
412
+ stat_calls: 0,
413
+ metadata_cache_hits: 0,
414
+ opened_files: 0,
415
+ parsed_files: 0,
416
+ parse_cache_hits: 0,
417
+ incremental_parse_hits: 0,
418
+ incremental_parse_fallbacks: 0,
419
+ bytes_read: 0,
420
+ prefix_validation_bytes: 0,
421
+ json_parse_calls: 0,
422
+ };
423
+ const discovered = discoverCodexSessionFiles(roots, {
424
+ from: fromKey,
425
+ to: toKey,
426
+ timeZoneContext,
427
+ exhaustive,
428
+ nowMs,
429
+ diagnostics,
430
+ });
431
+ const candidates = discovered.filter((entry) => entry.candidate);
432
+ const fileSignature = inventorySignature(discovered);
433
+ const cacheKey = [
434
+ CACHE_SCHEMA_VERSION,
435
+ baseDir,
436
+ fromKey || "",
437
+ toKey || "",
438
+ cacheTimeZoneKey(timeZoneContext),
439
+ Number.isFinite(top) ? top : 20,
440
+ exhaustive ? "exhaustive" : "bounded",
441
+ fileSignature,
442
+ ].join("|");
443
+ const cached = CACHE.get(cacheKey);
444
+ if (cached && Date.now() - cached.at < CACHE_TTL_MS) {
445
+ diagnostics.cache_hit = true;
446
+ return includeDiagnostics ? { ...cached.value, diagnostics } : cached.value;
447
+ }
448
+
449
+ const sessions = [];
450
+ const seenTokenEvents = new Set();
451
+
452
+ const candidateGroups = new Map();
453
+ for (const entry of candidates) {
454
+ const sessionKey = rolloutSessionIdFromPath(entry.filePath) || entry.filePath;
455
+ if (!candidateGroups.has(sessionKey)) candidateGroups.set(sessionKey, []);
456
+ candidateGroups.get(sessionKey).push(entry);
457
+ }
458
+ const appendHashPaths = selectAppendHashPaths(candidateGroups);
459
+
460
+ for (const entries of candidateGroups.values()) {
461
+ const filePaths = entries.map((entry) => entry.filePath);
462
+ const cacheable = Boolean(rolloutSessionIdFromPath(filePaths[0]));
463
+ const parsedCacheKey = cacheable
464
+ ? parsedGroupCacheKey(entries, { from: fromKey, to: toKey, timeZoneContext })
465
+ : null;
466
+ const signature = parsedGroupSignature(entries);
467
+ const cachedGroup = parsedCacheKey ? PARSED_GROUP_CACHE.get(parsedCacheKey) : null;
468
+ let parsed = null;
469
+ if (cachedGroup?.signature === signature) {
470
+ // Cacheable groups are partitioned by rollout UUID, so their token event
471
+ // keys cannot overlap and cache hits need not backfill seenTokenEvents.
472
+ diagnostics.parse_cache_hits += 1;
473
+ parsed = cachedGroup.parsed;
474
+ PARSED_GROUP_CACHE.delete(parsedCacheKey);
475
+ PARSED_GROUP_CACHE.set(parsedCacheKey, cachedGroup);
476
+ } else {
477
+ try {
478
+ let captured = null;
479
+ const singleEntry = entries.length === 1 ? entries[0] : null;
480
+ const appendHashEligible = Boolean(
481
+ singleEntry && appendHashPaths.has(singleEntry.filePath),
482
+ );
483
+ const resumeHandle = appendHashEligible
484
+ ? await openValidatedResumeFile(cachedGroup, singleEntry, diagnostics)
485
+ : null;
486
+ if (resumeHandle) {
487
+ try {
488
+ const resumed = await parseCodexRolloutFile(singleEntry.filePath, {
489
+ from: fromKey,
490
+ to: toKey,
491
+ timeZoneContext,
492
+ diagnostics,
493
+ seenTokenEvents: new Set(),
494
+ startOffset: cachedGroup.endOffset,
495
+ endOffset: singleEntry.stat.size,
496
+ resumeState: cachedGroup.resumeState,
497
+ captureResumeState: true,
498
+ captureContentHash: true,
499
+ contentHashState: cachedGroup.contentHashState,
500
+ sourceHandle: resumeHandle,
501
+ });
502
+ if (diagnostics) diagnostics.stat_calls += 1;
503
+ const afterParse = await resumeHandle.stat().catch(() => null);
504
+ let stableAfterParse = Boolean(
505
+ afterParse &&
506
+ statIdentity(singleEntry.filePath, afterParse) ===
507
+ statIdentity(singleEntry.filePath, singleEntry.stat),
508
+ );
509
+ if (
510
+ !stableAfterParse &&
511
+ isUnlinkedResumeSnapshot(
512
+ singleEntry.filePath,
513
+ singleEntry.stat,
514
+ afterParse,
515
+ )
516
+ ) {
517
+ try {
518
+ const expectedSnapshot = resumed.contentHashState.copy().digest("hex");
519
+ const actualSnapshot = await digestFilePrefix(
520
+ resumeHandle,
521
+ singleEntry.stat.size,
522
+ diagnostics,
523
+ );
524
+ stableAfterParse = actualSnapshot === expectedSnapshot;
525
+ } catch {
526
+ stableAfterParse = false;
527
+ }
528
+ }
529
+ if (stableAfterParse) {
530
+ const delta = splitCapturedParse(resumed);
531
+ captured = {
532
+ parsed: mergeParsedSessionResults(cachedGroup.parsed, delta.parsed),
533
+ resumeState: delta.resumeState,
534
+ endOffset: delta.endOffset,
535
+ appendable: delta.appendable,
536
+ contentHashState: delta.contentHashState,
537
+ };
538
+ diagnostics.incremental_parse_hits += 1;
539
+ } else {
540
+ diagnostics.incremental_parse_fallbacks += 1;
541
+ }
542
+ } catch (error) {
543
+ if (!isCodexResumeInvalidated(error) && !isRecoverableGroupedReadError(error)) throw error;
544
+ diagnostics.incremental_parse_fallbacks += 1;
545
+ } finally {
546
+ await resumeHandle.close();
547
+ }
548
+ }
549
+
550
+ if (!captured) {
551
+ let freshHandle = null;
552
+ let freshEndOffset = singleEntry?.stat?.size ?? null;
553
+ try {
554
+ if (resumeHandle && singleEntry) {
555
+ freshHandle = await fs.promises.open(singleEntry.filePath, "r");
556
+ if (diagnostics) diagnostics.stat_calls += 1;
557
+ freshEndOffset = (await freshHandle.stat()).size;
558
+ }
559
+ const fresh = await parseCodexRolloutFile(
560
+ filePaths.length === 1 ? filePaths[0] : filePaths,
561
+ {
562
+ from: fromKey,
563
+ to: toKey,
564
+ timeZoneContext,
565
+ diagnostics,
566
+ seenTokenEvents: filePaths.length === 1 ? new Set() : seenTokenEvents,
567
+ endOffset: freshEndOffset,
568
+ captureResumeState: Boolean(singleEntry),
569
+ captureContentHash: appendHashEligible,
570
+ sourceHandle: freshHandle,
571
+ },
572
+ );
573
+ captured = splitCapturedParse(fresh);
574
+ } finally {
575
+ await freshHandle?.close();
576
+ }
577
+ }
578
+
579
+ parsed = captured.parsed;
580
+ if (parsedCacheKey) {
581
+ rememberParsedGroup(parsedCacheKey, {
582
+ signature,
583
+ parsed,
584
+ resumeState: captured.resumeState,
585
+ endOffset: captured.endOffset,
586
+ appendable: captured.appendable === true,
587
+ contentHashState: captured.contentHashState,
588
+ filePath: singleEntry?.filePath || null,
589
+ stat: singleEntry?.stat || null,
590
+ });
591
+ }
592
+ } catch (error) {
593
+ // One unreadable archive/session must not fail the whole breakdown.
594
+ if (!isRecoverableGroupedReadError(error)) throw error;
595
+ continue;
596
+ }
597
+ }
598
+ if (!parsed || !parsed.totals || !parsed.totals.total_tokens) continue;
599
+ sessions.push(parsed);
600
+ }
601
+
602
+ const grand = emptyTotals();
603
+ const byTool = new Map();
604
+ const bySkill = new Map();
605
+ const byExecType = new Map();
606
+ const byExecExit = new Map();
607
+ const byExecExecutable = new Map();
608
+ const byExecCommand = new Map();
609
+ const byExecDuration = new Map();
610
+ const byExecOutput = new Map();
611
+
612
+ for (const s of sessions) {
613
+ mergeRollupTotals(grand, s.totals);
614
+ mergeRows(byTool, s.toolBreakdown?.tool_rows);
615
+ mergeSkillRows(bySkill, s.skillsBreakdown?.skill_rows);
616
+ mergeExecRows(byExecType, s.execCommandBreakdown?.byType);
617
+ mergeExecRows(byExecExit, s.execCommandBreakdown?.byExit);
618
+ mergeExecRows(byExecExecutable, s.execCommandBreakdown?.byExecutable);
619
+ mergeExecRows(byExecCommand, s.execCommandBreakdown?.byCommand);
620
+ mergeExecRows(byExecDuration, s.execCommandBreakdown?.byDuration);
621
+ mergeExecRows(byExecOutput, s.execCommandBreakdown?.byOutput);
622
+ }
623
+
624
+ const toolRows = finalizeToolRows(new Map([...byTool.entries()].map(([k, v]) => [k, v])));
625
+ const skillRows = finalizeSkillRows(new Map([...bySkill.entries()].map(([k, v]) => [k, v])));
626
+ const execTypeRows = finalizeExecRows(new Map([...byExecType.entries()].map(([k, v]) => [k, v])));
627
+ const execExitRows = finalizeExecRows(new Map([...byExecExit.entries()].map(([k, v]) => [k, v])));
628
+ const execExecutableRows = finalizeExecRows(new Map([...byExecExecutable.entries()].map(([k, v]) => [k, v])));
629
+ const execCommandRows = finalizeExecRows(new Map([...byExecCommand.entries()].map(([k, v]) => [k, v])));
630
+ const execDurationRows = finalizeExecRows(new Map([...byExecDuration.entries()].map(([k, v]) => [k, v])));
631
+ const execOutputRows = finalizeExecRows(new Map([...byExecOutput.entries()].map(([k, v]) => [k, v])));
632
+ const limitedTop = Number.isFinite(top) ? top : 20;
633
+ const toolRowsLimited = toolRows.slice(0, limitedTop).map((r) => ({
634
+ name: r.name,
635
+ calls: Math.round(r.calls || 0),
636
+ totals: roundTotals(r.totals),
637
+ }));
638
+ const skillRowsLimited = skillRows.slice(0, limitedTop).map((r) => ({
639
+ name: r.name,
640
+ calls: Math.round(r.calls || 0),
641
+ totals: roundTotals(r.totals),
642
+ }));
643
+
644
+ const byCategory = new Map(); // category -> {name,calls,totals,tools:Map}
645
+ for (const row of toolRows) {
646
+ const cat = categorizeTool(row.raw_name || row.name);
647
+ if (!byCategory.has(cat)) {
648
+ byCategory.set(cat, { name: cat, calls: 0, totals: emptyTotals(), tools: new Map() });
649
+ }
650
+ const target = byCategory.get(cat);
651
+ target.calls += row.calls || 0;
652
+ mergeRollupTotals(target.totals, row.totals || {});
653
+ target.tools.set(row.raw_name || row.name, row);
654
+ }
655
+ const categoryRows = Array.from(byCategory.values())
656
+ .map((c) => ({
657
+ name: c.name,
658
+ calls: Math.round(c.calls || 0),
659
+ totals: roundTotals(c.totals),
660
+ tools: finalizeToolRows(new Map([...c.tools.entries()].map(([k, v]) => [k, v])))
661
+ .slice(0, limitedTop)
662
+ .map((r) => ({
663
+ name: r.name,
664
+ calls: Math.round(r.calls || 0),
665
+ totals: roundTotals(r.totals),
666
+ })),
667
+ }))
668
+ .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
669
+
670
+ const textResponse = toolRows.find((row) => (row.raw_name || row.name) === "text_response");
671
+ const textResponseTotals = textResponse?.totals || emptyTotals();
672
+ const textResponseHistoryWeight = Math.max(
673
+ 0,
674
+ Number(textResponseTotals.cached_input_tokens || 0) + Number(textResponseTotals.cache_creation_input_tokens || 0),
675
+ );
676
+ const textResponseInputWeight = Math.max(0, Number(textResponseTotals.input_tokens || 0));
677
+ const textResponseOutputWeight = Math.max(
678
+ 0,
679
+ Number(textResponseTotals.output_tokens || 0) - Number(textResponseTotals.reasoning_output_tokens || 0),
680
+ );
681
+ const displayedMessageTotal =
682
+ textResponseInputWeight + textResponseHistoryWeight + textResponseOutputWeight;
683
+ const messageAlloc = allocateByLargestRemainder(
684
+ displayedMessageTotal,
685
+ {
686
+ user_input: textResponseInputWeight,
687
+ conversation_history: textResponseHistoryWeight,
688
+ assistant_response: textResponseOutputWeight,
689
+ },
690
+ ["user_input", "conversation_history", "assistant_response"],
691
+ );
692
+ const historyAlloc = allocateByLargestRemainder(
693
+ messageAlloc.conversation_history || 0,
694
+ {
695
+ cached_input_tokens: Math.max(0, Number(textResponseTotals.cached_input_tokens || 0)),
696
+ cache_creation_input_tokens: Math.max(0, Number(textResponseTotals.cache_creation_input_tokens || 0)),
697
+ },
698
+ ["cached_input_tokens", "cache_creation_input_tokens"],
699
+ );
700
+ const textResponseInput = messageAlloc.user_input || 0;
701
+ const textResponseHistory = messageAlloc.conversation_history || 0;
702
+ const textResponseOutput = messageAlloc.assistant_response || 0;
703
+ const messageBreakdown = [
704
+ {
705
+ key: "user_input",
706
+ name: "User input",
707
+ totals: roundTotals({
708
+ input_tokens: textResponseInput,
709
+ cached_input_tokens: 0,
710
+ cache_creation_input_tokens: 0,
711
+ output_tokens: 0,
712
+ reasoning_output_tokens: 0,
713
+ total_tokens: textResponseInput,
714
+ }),
715
+ },
716
+ {
717
+ key: "conversation_history",
718
+ name: "Conversation history",
719
+ totals: roundTotals({
720
+ input_tokens: 0,
721
+ cached_input_tokens: historyAlloc.cached_input_tokens || 0,
722
+ cache_creation_input_tokens: historyAlloc.cache_creation_input_tokens || 0,
723
+ output_tokens: 0,
724
+ reasoning_output_tokens: 0,
725
+ total_tokens: textResponseHistory,
726
+ }),
727
+ },
728
+ {
729
+ key: "assistant_response",
730
+ name: "Assistant response",
731
+ totals: roundTotals({
732
+ input_tokens: 0,
733
+ cached_input_tokens: 0,
734
+ cache_creation_input_tokens: 0,
735
+ output_tokens: textResponseOutput,
736
+ reasoning_output_tokens: 0,
737
+ total_tokens: textResponseOutput,
738
+ }),
739
+ },
740
+ ].sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
741
+
742
+ const serializeExecRows = (rows) => rows.slice(0, limitedTop).map((r) => ({
743
+ name: r.name,
744
+ calls: r.calls,
745
+ failures: r.failures,
746
+ duration_ms: r.duration_ms,
747
+ max_duration_ms: r.max_duration_ms,
748
+ output_chars: r.output_chars,
749
+ output_lines: r.output_lines,
750
+ totals: roundTotals(r.totals),
751
+ }));
752
+
753
+ const result = {
754
+ source: "codex",
755
+ scope: "supported",
756
+ breakdown_status: "ok",
757
+ totals: grand,
758
+ session_count: new Set(sessions.map((session) => session.sessionId || session.filePath)).size,
759
+ message_count: sessions.reduce((a, s) => a + (s.turnCount || 0), 0),
760
+ message_breakdown: {
761
+ categories: messageBreakdown,
762
+ privacy: {
763
+ includes_content: false,
764
+ note: "Aggregated message token categories only; prompt and assistant text are never returned.",
765
+ },
766
+ },
767
+ tool_calls_breakdown: {
768
+ total_calls: Math.round(toolRows.reduce((a, r) => a + Number(r.calls || 0), 0)),
769
+ tools: toolRowsLimited,
770
+ categories: categoryRows.slice(0, limitedTop),
771
+ tools_total: toolRows.reduce((a, r) => a + Math.round(r.totals?.total_tokens || 0), 0),
772
+ privacy: {
773
+ includes_inputs: false,
774
+ note: "Aggregated tool names only; no tool arguments or outputs are included.",
775
+ },
776
+ },
777
+ skills_breakdown: {
778
+ total_calls: Math.round(skillRows.reduce((a, r) => a + Number(r.calls || 0), 0)),
779
+ skills: skillRowsLimited,
780
+ privacy: {
781
+ includes_inputs: false,
782
+ note: "Codex skill use is inferred from exec_command reads of SKILL.md; command arguments are not returned.",
783
+ },
784
+ },
785
+ exec_command_breakdown: {
786
+ by_type: serializeExecRows(execTypeRows),
787
+ by_executable: serializeExecRows(execExecutableRows),
788
+ by_command: serializeExecRows(execCommandRows),
789
+ by_duration: serializeExecRows(execDurationRows),
790
+ by_output: serializeExecRows(execOutputRows),
791
+ by_exit: serializeExecRows(execExitRows),
792
+ },
793
+ };
794
+
795
+ const cacheValue = { ...result };
796
+ CACHE.set(cacheKey, { at: Date.now(), value: cacheValue });
797
+ while (CACHE.size > 32) {
798
+ const oldest = [...CACHE.entries()].sort((a, b) => a[1].at - b[1].at)[0];
799
+ if (oldest) CACHE.delete(oldest[0]);
800
+ }
801
+ return includeDiagnostics ? { ...result, diagnostics } : result;
802
+ }
803
+
804
+ module.exports = {
805
+ computeCodexContextBreakdown,
806
+ };