tokenusage-cli 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +46 -19
  2. package/LICENSE +22 -22
  3. package/README.md +123 -94
  4. package/THIRD_PARTY_NOTICES.md +140 -140
  5. package/bin/tokenusage.js +69 -69
  6. package/package.json +59 -59
  7. package/src/cli.js +146 -134
  8. package/src/commands/device-login.js +205 -205
  9. package/src/commands/diagnostics.js +38 -38
  10. package/src/commands/doctor.js +110 -110
  11. package/src/commands/init.js +2484 -2436
  12. package/src/commands/pause.js +77 -0
  13. package/src/commands/sessions.js +29 -29
  14. package/src/commands/status.js +1758 -1747
  15. package/src/commands/sync.js +8048 -7934
  16. package/src/commands/uninstall.js +322 -312
  17. package/src/lib/account-usage-dedup.js +267 -267
  18. package/src/lib/antigravity-paths.js +48 -48
  19. package/src/lib/ark-agent-plan-limits.js +438 -438
  20. package/src/lib/ark-coding-plan-limits.js +537 -537
  21. package/src/lib/async-queue-reader.js +138 -138
  22. package/src/lib/atomic-json.js +27 -27
  23. package/src/lib/bearer-token.js +11 -11
  24. package/src/lib/categorizer-utils.js +232 -232
  25. package/src/lib/claude-categorizer.js +1343 -1343
  26. package/src/lib/claude-config.js +279 -279
  27. package/src/lib/claude-user-identity.js +56 -56
  28. package/src/lib/cli-ui.js +179 -179
  29. package/src/lib/cloud-account.js +541 -541
  30. package/src/lib/cloud-device-token.js +141 -141
  31. package/src/lib/cloud-sync-prefs.js +14 -14
  32. package/src/lib/cloud-usage-reset.js +115 -115
  33. package/src/lib/codex-config.js +595 -595
  34. package/src/lib/codex-context-breakdown.js +806 -806
  35. package/src/lib/codex-model-attribution.js +128 -128
  36. package/src/lib/codex-rollout-parser.js +1471 -1471
  37. package/src/lib/codex-service-tier.js +49 -49
  38. package/src/lib/codex-token-refresh.js +140 -140
  39. package/src/lib/codex-token-usage.js +195 -195
  40. package/src/lib/codex-usage-record.js +195 -195
  41. package/src/lib/collection-pause.js +77 -0
  42. package/src/lib/command-runner.js +313 -313
  43. package/src/lib/commandcode-limits.js +317 -317
  44. package/src/lib/context-health.js +124 -124
  45. package/src/lib/cursor-config.js +605 -605
  46. package/src/lib/cursor-store.js +1918 -1918
  47. package/src/lib/debug-flags.js +7 -7
  48. package/src/lib/devin-limits.js +249 -249
  49. package/src/lib/devin-usage.js +212 -212
  50. package/src/lib/diagnostics.js +383 -383
  51. package/src/lib/doctor.js +385 -385
  52. package/src/lib/dumate.js +95 -95
  53. package/src/lib/fs.js +455 -455
  54. package/src/lib/function-url.js +14 -14
  55. package/src/lib/gemini-config.js +291 -291
  56. package/src/lib/git-outcomes.js +243 -243
  57. package/src/lib/grok-context-breakdown.js +890 -890
  58. package/src/lib/grok-hook.js +360 -360
  59. package/src/lib/grok-limits.js +865 -865
  60. package/src/lib/grok-usage.js +98 -98
  61. package/src/lib/heatmap-compact.js +114 -114
  62. package/src/lib/ingest-sanitize.js +41 -41
  63. package/src/lib/init-flow.js +42 -42
  64. package/src/lib/install-resolver.js +170 -170
  65. package/src/lib/jsonl-lines.js +134 -134
  66. package/src/lib/kimi-model-repair.js +173 -173
  67. package/src/lib/kimi-profile.js +61 -61
  68. package/src/lib/kimi-session.js +216 -216
  69. package/src/lib/local-api.js +3588 -3588
  70. package/src/lib/machine-id.js +238 -238
  71. package/src/lib/multi-install-parser.js +113 -113
  72. package/src/lib/notify-command.js +38 -38
  73. package/src/lib/omp-hook.js +589 -589
  74. package/src/lib/openclaw-cli.js +104 -104
  75. package/src/lib/openclaw-hook.js +375 -375
  76. package/src/lib/openclaw-session-plugin.js +560 -560
  77. package/src/lib/opencode-config.js +121 -121
  78. package/src/lib/opencode-go-limits.js +807 -807
  79. package/src/lib/outcomes-engine.js +253 -253
  80. package/src/lib/passive-mode.js +219 -219
  81. package/src/lib/pet-packages.js +722 -722
  82. package/src/lib/pricing/curated-overrides.json +822 -822
  83. package/src/lib/pricing/index.js +379 -379
  84. package/src/lib/pricing/litellm-fetcher.js +172 -172
  85. package/src/lib/pricing/matcher.js +456 -456
  86. package/src/lib/pricing/seed-snapshot.json +1 -1
  87. package/src/lib/progress.js +76 -76
  88. package/src/lib/project-usage-purge.js +117 -117
  89. package/src/lib/prompt.js +24 -24
  90. package/src/lib/provider-status.js +104 -104
  91. package/src/lib/proxy-env.js +489 -489
  92. package/src/lib/proxy-settings.js +163 -163
  93. package/src/lib/qoder-cn-cli-limits.js +119 -119
  94. package/src/lib/qoder-limits.js +880 -880
  95. package/src/lib/quiet-warnings.js +26 -26
  96. package/src/lib/rollout.js +26005 -25966
  97. package/src/lib/runtime-config.js +9 -4
  98. package/src/lib/scan-roots.js +277 -277
  99. package/src/lib/scheduled-task.js +145 -0
  100. package/src/lib/sentry.js +231 -231
  101. package/src/lib/session-analytics.js +2972 -2972
  102. package/src/lib/session-performance.js +237 -237
  103. package/src/lib/session-upload.js +379 -379
  104. package/src/lib/skill-usage.js +267 -267
  105. package/src/lib/skills-manager.js +1591 -1591
  106. package/src/lib/source-metadata.js +48 -48
  107. package/src/lib/sqlite-reader.js +215 -215
  108. package/src/lib/star-cta.js +73 -73
  109. package/src/lib/subscription-manager.js +293 -293
  110. package/src/lib/subscriptions.js +508 -508
  111. package/src/lib/sync-report.js +138 -0
  112. package/src/lib/telemetry.js +193 -193
  113. package/src/lib/tracker-paths.js +16 -16
  114. package/src/lib/trae-cn-config.js +548 -548
  115. package/src/lib/trae-db.js +127 -127
  116. package/src/lib/trae-snapshot.js +230 -230
  117. package/src/lib/trae-sqlite.js +143 -143
  118. package/src/lib/trae-usage.js +121 -121
  119. package/src/lib/upload-queue.js +134 -134
  120. package/src/lib/upload-throttle.js +171 -171
  121. package/src/lib/usage-limits.js +4552 -4552
  122. package/src/lib/wrapped-aggregator.js +236 -236
  123. package/src/lib/wsl-probe.js +281 -281
  124. package/src/lib/zcode-limits.js +1172 -1172
  125. package/src/lib/zcode-reset-credits.js +66 -66
  126. package/test/audit-fixes.test.js +200 -0
  127. package/test/cli-smoke.test.js +53 -53
  128. package/test/collection-controls.test.js +210 -0
  129. package/test/ingest-sanitize.test.js +162 -162
  130. package/test/quiet-warnings.test.js +53 -53
  131. package/test/session-upload.test.js +427 -427
@@ -1,1471 +1,1471 @@
1
- // Codex rollout JSONL parser — extracted from codex-context-breakdown.js.
2
- //
3
- // Handles file discovery and per-file parsing. Does NOT hold any aggregation
4
- // state; callers (computeCodexContextBreakdown) own the merge step.
5
-
6
- "use strict";
7
-
8
- const crypto = require("node:crypto");
9
- const fs = require("node:fs");
10
- const path = require("node:path");
11
-
12
- const { listRolloutFiles } = require("./rollout");
13
- const {
14
- consumeUsageDelta,
15
- createUsageDeltaState,
16
- snapshotUsageBaselines,
17
- totalsReset,
18
- } = require("./codex-token-usage");
19
- const {
20
- emptyTotals,
21
- addInto,
22
- inferExecCommandKind,
23
- sanitizeCommandSignature,
24
- getExecutableName,
25
- buildExecStatsEntry,
26
- } = require("./categorizer-utils");
27
- const {
28
- applyCodexModelEvent,
29
- createCodexModelAttributionState,
30
- currentCodexModel,
31
- snapshotCodexModelAttributionState,
32
- } = require("./codex-model-attribution");
33
- const { readCodexServiceTier, isPriorityServiceTier } = require("./codex-service-tier");
34
- const {
35
- createUsageRecordState,
36
- extractTokenUsageRecord,
37
- noteLineAfterUsageRecord,
38
- noteTokenCount,
39
- noteUsageRecord,
40
- snapshotUsageRecordState,
41
- takeCompactionOnTokenCount,
42
- } = require("./codex-usage-record");
43
-
44
- const DISCOVERY_FULL_AUDIT_INTERVAL_MS = 60 * 1000;
45
- const CODEX_ROLLOUT_TEXT_MAX_BYTES = 256 * 1024 * 1024;
46
-
47
- async function readCodexRolloutText(filePath, { maxOutputBytes = CODEX_ROLLOUT_TEXT_MAX_BYTES } = {}) {
48
- if (typeof filePath !== "string" || !/\.jsonl\.zstd?$/i.test(filePath)) {
49
- return fs.readFile(filePath, "utf8");
50
- }
51
- // The snapshot decoder lives in rollout.js (reused by the sync parser);
52
- // require lazily so this module stays loadable without pulling the whole
53
- // rollout graph at import time.
54
- const rollout = require("./rollout");
55
- const snapshot = await rollout.readCodexRolloutSnapshot(filePath, { maxOutputBytes });
56
- if (!snapshot) {
57
- // Treat undecodable / vanished archives as recoverable I/O so grouped
58
- // session reads (live + archive) can still use a readable sibling copy.
59
- const error = new Error(`unreadable Codex archive ${filePath}`);
60
- error.code = "EIO";
61
- throw error;
62
- }
63
- // Snapshot lines keep their "\n" terminators; the tail element may be an
64
- // unterminated final line — join preserves exactly that.
65
- return snapshot.lines.join("\n");
66
- }
67
- const MAX_DISCOVERY_INVENTORIES = 32;
68
- const DISCOVERY_INVENTORIES = new Map();
69
- const ZONED_DAY_FORMATTERS = new Map();
70
- const CODEX_RESUME_INVALIDATED = "TOKENUSAGE_CODEX_RESUME_INVALIDATED";
71
- // Match src/lib/pricing/index.js. Classify individual requests using raw
72
- // input (including cache), not the accumulated session or day totals.
73
- const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272_000;
74
-
75
- // ---------------------------------------------------------------------------
76
- // Timezone helpers
77
- // ---------------------------------------------------------------------------
78
-
79
- function dayKeyToIsoBounds(from, to) {
80
- if (!from && !to) return { fromIso: null, toIso: null };
81
- const fromDate = from ? new Date(`${from}T00:00:00Z`) : null;
82
- const toDate = to ? new Date(`${to}T23:59:59Z`) : null;
83
- if (fromDate && Number.isFinite(fromDate.getTime())) fromDate.setUTCHours(fromDate.getUTCHours() - 14);
84
- if (toDate && Number.isFinite(toDate.getTime())) toDate.setUTCHours(toDate.getUTCHours() + 14);
85
- return {
86
- fromIso: fromDate ? fromDate.toISOString() : null,
87
- toIso: toDate ? toDate.toISOString() : null,
88
- };
89
- }
90
-
91
- function formatPartsDayKey(parts) {
92
- if (!parts) return "";
93
- const values = {};
94
- for (const part of parts) {
95
- if (part.type !== "literal") values[part.type] = part.value;
96
- }
97
- if (!values.year || !values.month || !values.day) return "";
98
- return `${values.year}-${values.month}-${values.day}`;
99
- }
100
-
101
- function getZonedParts(date, timeZoneContext = {}) {
102
- const { timeZone, offsetMinutes } = timeZoneContext || {};
103
- const dt = date instanceof Date ? date : new Date(date);
104
- if (!Number.isFinite(dt.getTime())) return null;
105
-
106
- if (timeZone && typeof Intl !== "undefined" && Intl.DateTimeFormat) {
107
- try {
108
- let formatter = ZONED_DAY_FORMATTERS.get(timeZone);
109
- if (!formatter) {
110
- formatter = new Intl.DateTimeFormat("en-CA", {
111
- timeZone,
112
- year: "numeric",
113
- month: "2-digit",
114
- day: "2-digit",
115
- hourCycle: "h23",
116
- });
117
- ZONED_DAY_FORMATTERS.set(timeZone, formatter);
118
- }
119
- return formatter.formatToParts(dt);
120
- } catch {
121
- // Fall through to offset handling.
122
- }
123
- }
124
-
125
- if (Number.isFinite(offsetMinutes)) {
126
- const shifted = new Date(dt.getTime() - Number(offsetMinutes) * 60_000);
127
- return [
128
- { type: "year", value: String(shifted.getUTCFullYear()).padStart(4, "0") },
129
- { type: "month", value: String(shifted.getUTCMonth() + 1).padStart(2, "0") },
130
- { type: "day", value: String(shifted.getUTCDate()).padStart(2, "0") },
131
- ];
132
- }
133
-
134
- return null;
135
- }
136
-
137
- function timestampDayKey(timestamp, timeZoneContext) {
138
- const ts = typeof timestamp === "string" ? timestamp : "";
139
- if (!ts) return "";
140
- const parts = getZonedParts(ts, timeZoneContext);
141
- const zoned = formatPartsDayKey(parts);
142
- if (zoned) return zoned;
143
- return ts.slice(0, 10);
144
- }
145
-
146
- function isTimestampInRequestedDayRange(timestamp, { from, to, timeZoneContext } = {}) {
147
- if (!from && !to) return true;
148
- const day = timestampDayKey(timestamp, timeZoneContext);
149
- if (!day) return false;
150
- if (from && day < from) return false;
151
- if (to && day > to) return false;
152
- return true;
153
- }
154
-
155
- // ---------------------------------------------------------------------------
156
- // File discovery
157
- // ---------------------------------------------------------------------------
158
-
159
- function safeJsonParse(str, diagnostics = null) {
160
- try {
161
- if (diagnostics) diagnostics.json_parse_calls += 1;
162
- return JSON.parse(str);
163
- } catch {
164
- return null;
165
- }
166
- }
167
-
168
- function isRolloutCandidateFileName(name) {
169
- if (typeof name !== "string") return false;
170
- return name.endsWith(".jsonl") || /^rollout-.*\.jsonl\.zstd?$/.test(name);
171
- }
172
-
173
- function listJsonlFiles(rootDir) {
174
- const out = [];
175
- const stack = [rootDir];
176
-
177
- while (stack.length > 0) {
178
- const dir = stack.pop();
179
- let entries;
180
- try {
181
- entries = fs.readdirSync(dir, { withFileTypes: true });
182
- } catch {
183
- continue;
184
- }
185
-
186
- for (const entry of entries) {
187
- const filePath = path.join(dir, entry.name);
188
- if (entry.isDirectory()) {
189
- stack.push(filePath);
190
- } else if (entry.isFile() && isRolloutCandidateFileName(entry.name)) {
191
- out.push(filePath);
192
- }
193
- }
194
- }
195
-
196
- out.sort((a, b) => a.localeCompare(b));
197
- return out;
198
- }
199
-
200
- function rolloutPathDay(filePath) {
201
- const normalized = String(filePath || "").replaceAll("\\", "/");
202
- const fileMatch = path.basename(normalized).match(/(?:^|rollout-)(\d{4})-(\d{2})-(\d{2})/);
203
- if (fileMatch) return `${fileMatch[1]}-${fileMatch[2]}-${fileMatch[3]}`;
204
- const directoryMatch = normalized.match(/.*\/(\d{4})\/(\d{2})\/(\d{2})(?:\/|$)/);
205
- return directoryMatch
206
- ? `${directoryMatch[1]}-${directoryMatch[2]}-${directoryMatch[3]}`
207
- : null;
208
- }
209
-
210
- function nextDayKey(dayKey) {
211
- if (!dayKey) return null;
212
- const date = new Date(`${dayKey}T00:00:00.000Z`);
213
- if (!Number.isFinite(date.getTime())) return null;
214
- date.setUTCDate(date.getUTCDate() + 1);
215
- return date.toISOString().slice(0, 10);
216
- }
217
-
218
- function metadataRangeBounds(from, to, timeZoneContext) {
219
- const offsetMinutes = timeZoneContext?.offsetMinutes;
220
- if (Number.isFinite(offsetMinutes)) {
221
- const dstGuardMs = 2 * 60 * 60 * 1000;
222
- const fromUtc = from ? Date.parse(`${from}T00:00:00.000Z`) : null;
223
- const afterTo = to ? nextDayKey(to) : null;
224
- const toUtcExclusive = afterTo ? Date.parse(`${afterTo}T00:00:00.000Z`) : null;
225
- return {
226
- fromMs: Number.isFinite(fromUtc)
227
- ? fromUtc + offsetMinutes * 60_000 - dstGuardMs
228
- : null,
229
- toMs: Number.isFinite(toUtcExclusive)
230
- ? toUtcExclusive + offsetMinutes * 60_000 + dstGuardMs
231
- : null,
232
- };
233
- }
234
- const { fromIso, toIso } = dayKeyToIsoBounds(from, to);
235
- return {
236
- fromMs: fromIso ? Date.parse(fromIso) : null,
237
- toMs: toIso ? Date.parse(toIso) : null,
238
- };
239
- }
240
-
241
- function isConservativeRangeCandidate(filePath, stat, {
242
- from = null,
243
- to = null,
244
- timeZoneContext = null,
245
- metadataBounds = null,
246
- } = {}) {
247
- if (!from && !to) return true;
248
- // With no lower bound, a later-path rewrite can still contain an event at or
249
- // before `to`. There is no metadata-only exclusion that can prove otherwise.
250
- if (!from) return true;
251
- const pathDay = rolloutPathDay(filePath);
252
- if (!pathDay) return true;
253
-
254
- const pathOverlaps = (!from || pathDay >= from) && (!to || pathDay <= to);
255
- if (pathOverlaps) return true;
256
-
257
- // Old sessions can be appended or rewritten. A post-range mtime or ctime is
258
- // therefore a reason to parse, never a reason to exclude. ctime also catches
259
- // cold-process inode replacement/truncation when a writer preserves mtime.
260
- // Future-dated decoys are kept conservatively; correctness wins over an open.
261
- const bounds = metadataBounds || metadataRangeBounds(from, to, timeZoneContext);
262
- const fromMs = bounds.fromMs;
263
- const toMs = bounds.toMs;
264
- const mtimeMs = Number(stat?.mtimeMs);
265
- const ctimeMs = Number(stat?.ctimeMs);
266
- const changedAfterStart = (value) => (
267
- Number.isFinite(value) && (!Number.isFinite(fromMs) || value >= fromMs)
268
- );
269
- const changedInsideRange = (value) => (
270
- changedAfterStart(value) && (!Number.isFinite(toMs) || value <= toMs)
271
- );
272
- if (pathDay < from) {
273
- return changedAfterStart(mtimeMs) || changedAfterStart(ctimeMs);
274
- }
275
-
276
- // A path one day ahead can still overlap the requested browser day because
277
- // rollout folders follow the writer's local date. Its current mtime may be
278
- // later if the same session kept growing, so metadata alone cannot exclude it.
279
- if (to && pathDay === nextDayKey(to)) return true;
280
-
281
- // A host-local path day can be later than the requested browser day when
282
- // their timezones differ. Keep it only when filesystem metadata maps back
283
- // into the requested day range; do not pull every genuinely future session
284
- // into a historical query.
285
- return changedInsideRange(mtimeMs) || changedInsideRange(ctimeMs);
286
- }
287
-
288
- function isRolloutPathInRequestedRange(filePath, { from = null, to = null } = {}) {
289
- const pathDay = rolloutPathDay(filePath);
290
- if (!pathDay) return true;
291
- if (from && pathDay < from) return false;
292
- if (to && pathDay > to) return false;
293
- return true;
294
- }
295
-
296
- function discoveryInventoryKey(roots) {
297
- return roots.map((root) => path.resolve(root)).sort().join("\0");
298
- }
299
-
300
- function getDiscoveryInventory(roots, nowMs) {
301
- const key = discoveryInventoryKey(roots);
302
- let inventory = DISCOVERY_INVENTORIES.get(key);
303
- if (!inventory) {
304
- inventory = { files: new Map(), lastFullAuditAtMs: 0, lastUsedAtMs: nowMs };
305
- DISCOVERY_INVENTORIES.set(key, inventory);
306
- }
307
- inventory.lastUsedAtMs = nowMs;
308
- while (DISCOVERY_INVENTORIES.size > MAX_DISCOVERY_INVENTORIES) {
309
- let oldestKey = null;
310
- let oldestAt = Infinity;
311
- for (const [candidateKey, candidate] of DISCOVERY_INVENTORIES) {
312
- if (candidateKey === key) continue;
313
- if (candidate.lastUsedAtMs < oldestAt) {
314
- oldestAt = candidate.lastUsedAtMs;
315
- oldestKey = candidateKey;
316
- }
317
- }
318
- if (!oldestKey) break;
319
- DISCOVERY_INVENTORIES.delete(oldestKey);
320
- }
321
- return inventory;
322
- }
323
-
324
- function discoverCodexSessionFiles(rootDirs, options = {}) {
325
- const diagnostics = options.diagnostics || null;
326
- const roots = Array.isArray(rootDirs) ? rootDirs : [rootDirs];
327
- const nowMs = Number.isFinite(Number(options.nowMs)) ? Number(options.nowMs) : Date.now();
328
- const inventory = getDiscoveryInventory(roots.filter(Boolean), nowMs);
329
- const candidateOptions = {
330
- ...options,
331
- metadataBounds: metadataRangeBounds(options.from, options.to, options.timeZoneContext),
332
- };
333
- const bounded = Boolean(options.from || options.to);
334
- const auditAgeMs = nowMs - Number(inventory.lastFullAuditAtMs || 0);
335
- const fullMetadataAudit = Boolean(
336
- options.exhaustive ||
337
- !bounded ||
338
- !Number.isFinite(auditAgeMs) ||
339
- auditAgeMs < 0 ||
340
- auditAgeMs >= DISCOVERY_FULL_AUDIT_INTERVAL_MS
341
- );
342
- if (diagnostics) diagnostics.full_metadata_audit = fullMetadataAudit;
343
- const entries = [];
344
- const visited = new Set();
345
- const stack = roots.filter(Boolean);
346
-
347
- while (stack.length > 0) {
348
- const dir = stack.pop();
349
- let children;
350
- try {
351
- children = fs.readdirSync(dir, { withFileTypes: true });
352
- } catch {
353
- continue;
354
- }
355
- for (const child of children) {
356
- const filePath = path.join(dir, child.name);
357
- if (child.isDirectory()) {
358
- stack.push(filePath);
359
- continue;
360
- }
361
- if (!child.isFile() || !isRolloutCandidateFileName(child.name) || visited.has(filePath)) continue;
362
- visited.add(filePath);
363
- if (diagnostics) diagnostics.discovered_files += 1;
364
- const cached = inventory.files.get(filePath) || null;
365
- const cachedCandidate = cached
366
- ? isConservativeRangeCandidate(filePath, cached, candidateOptions)
367
- : false;
368
- const shouldStat = Boolean(
369
- fullMetadataAudit ||
370
- !cached ||
371
- cachedCandidate ||
372
- isRolloutPathInRequestedRange(filePath, options)
373
- );
374
- let stat = cached;
375
- if (shouldStat) {
376
- try {
377
- if (diagnostics) diagnostics.stat_calls += 1;
378
- stat = fs.statSync(filePath);
379
- inventory.files.set(filePath, stat);
380
- } catch {
381
- inventory.files.delete(filePath);
382
- continue;
383
- }
384
- } else if (diagnostics) {
385
- diagnostics.metadata_cache_hits += 1;
386
- }
387
- const candidate = Boolean(
388
- options.exhaustive ||
389
- isConservativeRangeCandidate(filePath, stat, candidateOptions),
390
- );
391
- entries.push({ filePath, stat, candidate });
392
- if (candidate && diagnostics) diagnostics.candidate_files += 1;
393
- }
394
- }
395
-
396
- for (const filePath of inventory.files.keys()) {
397
- if (!visited.has(filePath)) inventory.files.delete(filePath);
398
- }
399
- if (fullMetadataAudit) inventory.lastFullAuditAtMs = nowMs;
400
-
401
- return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
402
- }
403
-
404
- async function listCodexSessionFiles(baseDir) {
405
- const rolloutFiles = await listRolloutFiles(baseDir).catch(() => []);
406
- const allJsonlFiles = listJsonlFiles(baseDir);
407
- if (allJsonlFiles.length === 0) return rolloutFiles;
408
- if (rolloutFiles.length === 0) return allJsonlFiles;
409
-
410
- const merged = new Set(rolloutFiles);
411
- for (const filePath of allJsonlFiles) merged.add(filePath);
412
- return Array.from(merged).sort((a, b) => a.localeCompare(b));
413
- }
414
-
415
- // ---------------------------------------------------------------------------
416
- // Token count extraction
417
- // ---------------------------------------------------------------------------
418
-
419
- function extractTokenCount(obj) {
420
- const payload = obj?.payload;
421
- if (!payload || obj?.type !== "event_msg") return null;
422
- if (payload.type === "token_count") {
423
- return { info: payload.info || null, timestamp: obj?.timestamp || null };
424
- }
425
- const msg = payload.msg;
426
- if (msg && msg.type === "token_count") {
427
- return { info: msg.info || null, timestamp: obj?.timestamp || null };
428
- }
429
- return null;
430
- }
431
-
432
- // ---------------------------------------------------------------------------
433
- // Tool name helpers
434
- // ---------------------------------------------------------------------------
435
-
436
- function normalizeToolName(payload) {
437
- const name = payload?.name || "";
438
- const ns = payload?.namespace || null;
439
- if (ns && typeof ns === "string" && ns.startsWith("mcp__")) return `${ns}${name}`;
440
- return name || "unknown";
441
- }
442
-
443
- function extractSkillNameFromFunctionCall(payload, diagnostics = null) {
444
- if (!payload || payload.name !== "exec_command") return null;
445
- const args = safeJsonParse(payload.arguments || "{}", diagnostics) || {};
446
- const cmd = String(args.cmd || "");
447
- const match = cmd.match(/(?:^|\/)skills\/(?:\.system\/)?([^/\s]+)\/SKILL\.md\b/);
448
- return match ? match[1] : null;
449
- }
450
-
451
- function formatToolDisplayName(name) {
452
- if (typeof name !== "string" || !name.startsWith("mcp__")) return name;
453
- const parts = name.split("__");
454
- if (parts.length < 3) return name;
455
- const server = String(parts[1] || "").replace(/^plugin_/, "").replace(/_/g, "-");
456
- const tool = parts.slice(2).join("__") || name;
457
- return server ? `${server}/${tool}` : tool;
458
- }
459
-
460
- // ---------------------------------------------------------------------------
461
- // Usage normalization
462
- // ---------------------------------------------------------------------------
463
-
464
- function normalizeUsage(u) {
465
- const out = {};
466
- for (const k of [
467
- "input_tokens",
468
- "cached_input_tokens",
469
- "cache_creation_input_tokens",
470
- "output_tokens",
471
- "reasoning_output_tokens",
472
- "total_tokens",
473
- ]) {
474
- const n = Number(u?.[k] || 0);
475
- out[k] = Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
476
- }
477
- // Codex reports input inclusive of cached_input_tokens; keep our schema
478
- // convention: non-cached input and cached input tracked separately.
479
- out.input_tokens = Math.max(0, out.input_tokens - out.cached_input_tokens);
480
- out.total_tokens =
481
- out.input_tokens +
482
- out.cached_input_tokens +
483
- out.cache_creation_input_tokens +
484
- out.output_tokens;
485
- return out;
486
- }
487
-
488
- function pickDelta(lastUsage, totalUsage, prevTotals) {
489
- const state = createUsageDeltaState({ lastTotal: prevTotals });
490
- const delta = consumeUsageDelta(state, lastUsage, totalUsage);
491
- return delta ? normalizeUsage(delta) : null;
492
- }
493
-
494
- // ---------------------------------------------------------------------------
495
- // Exec stats helpers (local to parser, not shared)
496
- // ---------------------------------------------------------------------------
497
-
498
- function durationBucket(ms) {
499
- const n = Number(ms || 0);
500
- if (!Number.isFinite(n) || n <= 0) return "unknown";
501
- if (n < 1000) return "<1s";
502
- if (n < 10_000) return "1-10s";
503
- if (n < 60_000) return "10-60s";
504
- if (n < 300_000) return "1-5m";
505
- return ">5m";
506
- }
507
-
508
- function outputSizeBucket(lines, chars) {
509
- const l = Number(lines || 0);
510
- const c = Number(chars || 0);
511
- if (!l && !c) return "quiet";
512
- if (l <= 20 && c <= 2_000) return "small";
513
- if (l <= 200 && c <= 20_000) return "medium";
514
- if (l <= 1000 && c <= 100_000) return "large";
515
- return "very_large";
516
- }
517
-
518
- function buildToolStatsEntry() {
519
- return { calls: 0, totals: emptyTotals() };
520
- }
521
-
522
- function buildSkillStatsEntry(name) {
523
- return { name, calls: 0, totals: emptyTotals() };
524
- }
525
-
526
- // ---------------------------------------------------------------------------
527
- // Finalize helpers
528
- // ---------------------------------------------------------------------------
529
-
530
- function finalizeToolRows(map) {
531
- return Array.from(map.values())
532
- .map((row) => {
533
- const rawName = row.raw_name || row.name;
534
- return {
535
- name: formatToolDisplayName(rawName),
536
- raw_name: rawName,
537
- calls: row.calls,
538
- totals: row.totals,
539
- };
540
- })
541
- .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
542
- }
543
-
544
- function finalizeSkillRows(map) {
545
- return Array.from(map.values())
546
- .map((row) => ({
547
- name: row.name,
548
- calls: row.calls,
549
- totals: row.totals,
550
- }))
551
- .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
552
- }
553
-
554
- function finalizeExecRows(map) {
555
- return Array.from(map.values())
556
- .map((row) => ({
557
- name: row.name,
558
- calls: row.calls,
559
- failures: row.failures,
560
- duration_ms: row.duration_ms,
561
- max_duration_ms: row.max_duration_ms,
562
- output_chars: row.output_chars,
563
- output_lines: row.output_lines,
564
- totals: row.totals,
565
- }))
566
- .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
567
- }
568
-
569
- function finalizeModelUsageRows(map) {
570
- return Array.from(map.values())
571
- .map((row) => ({
572
- ...row,
573
- selected_models: Array.from(row.selected_models).sort(),
574
- reroute_reasons: Array.from(row.reroute_reasons).sort(),
575
- model_attribution: row.rerouted_usage_events > 0 ? "effective" : "selected",
576
- }))
577
- .sort((a, b) => b.total_tokens - a.total_tokens || a.model.localeCompare(b.model));
578
- }
579
-
580
- // ---------------------------------------------------------------------------
581
- // Main parser
582
- // ---------------------------------------------------------------------------
583
-
584
- async function parseCodexRolloutFile(filePath, {
585
- from = null,
586
- to = null,
587
- timeZoneContext = null,
588
- diagnostics = null,
589
- seenTokenEvents = null,
590
- startOffset = 0,
591
- endOffset = null,
592
- resumeState = null,
593
- captureResumeState = false,
594
- captureContentHash = false,
595
- contentHashState = null,
596
- sourceHandle = null,
597
- collectBreakdowns = true,
598
- collectModelUsage = false,
599
- onObject = null,
600
- onUsage = null,
601
- } = {}) {
602
- const filePaths = (Array.isArray(filePath) ? filePath : [filePath]).filter(Boolean);
603
- const primaryFilePath = filePaths[0] || String(filePath || "");
604
- const isResuming = Boolean(resumeState && filePaths.length === 1);
605
- const initialOffset = Math.max(0, Number(startOffset) || 0);
606
- const readProgress = {
607
- endOffset: initialOffset,
608
- lastByte: initialOffset > 0 ? 0x0a : null,
609
- };
610
- let contentHasher = null;
611
- if (captureResumeState && captureContentHash && filePaths.length === 1) {
612
- if (contentHashState && typeof contentHashState.copy === "function") {
613
- try {
614
- contentHasher = contentHashState.copy();
615
- } catch {
616
- contentHasher = null;
617
- }
618
- }
619
- if (initialOffset > 0 && !contentHasher) {
620
- const error = new Error("Codex rollout append is missing its prefix hash state");
621
- error.code = CODEX_RESUME_INVALIDATED;
622
- throw error;
623
- }
624
- if (!contentHasher) contentHasher = crypto.createHash("sha256");
625
- }
626
-
627
- let sessionId = isResuming ? resumeState.sessionId || null : null;
628
- let cwd = isResuming ? resumeState.cwd || null : null;
629
- let model = isResuming ? resumeState.model || null : null;
630
- const modelAttributionState = createCodexModelAttributionState(
631
- isResuming ? resumeState.modelAttributionState || { model } : {},
632
- );
633
- // Codex writes thread_settings_applied immediately before the turn_context of
634
- // the turn it applies to, and token_count rows carry no turn id, so each one
635
- // belongs to the most recent preceding tier record. Sessions that never emit
636
- // the event (most CLI sessions) stay null and are billed at Standard.
637
- let serviceTier = isResuming ? resumeState.serviceTier || null : null;
638
- let provider = isResuming ? resumeState.provider || null : null;
639
- let cliVersion = isResuming ? resumeState.cliVersion || null : null;
640
- let forkedFromId = isResuming ? resumeState.forkedFromId || null : null;
641
- let parentThreadId = isResuming ? resumeState.parentThreadId || null : null;
642
- let threadSource = isResuming ? resumeState.threadSource || null : null;
643
- let agentNickname = isResuming ? resumeState.agentNickname || null : null;
644
- let agentRole = isResuming ? resumeState.agentRole || null : null;
645
- let sessionLineageCaptured = isResuming
646
- ? Boolean(resumeState.sessionLineageCaptured || resumeState.sessionId)
647
- : false;
648
-
649
- const usageDeltaState = createUsageDeltaState({
650
- lastTotal: isResuming ? resumeState.prevTotals || null : null,
651
- baselines: isResuming ? resumeState.tokenUsageBaselines || null : null,
652
- });
653
- let pendingToolNames = isResuming
654
- ? Array.from(resumeState.pendingToolNames || [])
655
- : [];
656
- let pendingSkills = isResuming ? Array.from(resumeState.pendingSkills || []) : [];
657
- let pendingExecDetails = isResuming
658
- ? Array.from(resumeState.pendingExecDetails || [])
659
- : [];
660
- let lastTokenTimestamp = isResuming
661
- ? String(resumeState.lastTokenTimestamp || "") || null
662
- : null;
663
- let lastTimestampEventSignatures = new Set(
664
- isResuming ? resumeState.lastTimestampEventSignatures || [] : [],
665
- );
666
- let lastTimestampLastUsage = null;
667
- let lastTimestampTotalUsage = null;
668
- let hasUnmaterializedLastTimestampUsage = false;
669
-
670
- const totals = emptyTotals();
671
- const byTool = new Map(); // tool_name -> {name,calls,totals}
672
- const bySkill = new Map(); // skill_name -> {name,calls,totals}
673
- const byExecKind = new Map(); // kind -> stats
674
- const byExecExit = new Map(); // status:exit -> stats
675
- const byExecExecutable = new Map(); // executable -> stats
676
- const byExecCommand = new Map(); // sanitized executable + subcommand -> stats
677
- const byExecDuration = new Map(); // duration bucket -> stats
678
- const byExecOutput = new Map(); // output size bucket -> stats
679
- const byModel = new Map(); // effective model -> observed token usage
680
-
681
- let turnCount = 0;
682
-
683
- // Compaction records (issue #652, rules in codex-usage-record.js). A resume
684
- // state from before #652 with prevTotals has seen a token_count.
685
- const usageRecordState = createUsageRecordState(
686
- isResuming
687
- ? resumeState.usageRecordState || (resumeState.prevTotals ? { sawTokenCount: true } : null)
688
- : null,
689
- );
690
- // Compactions counted in this session, by response_id: the files merged for
691
- // a session can repeat a record (an archived copy, a split fragment), and a
692
- // resumed parse must not count one from an earlier chunk again.
693
- const compactionResponseIds = new Set(
694
- isResuming && Array.isArray(resumeState.compactionResponseIds)
695
- ? resumeState.compactionResponseIds
696
- : [],
697
- );
698
-
699
- // Attribution as of now. A compaction record is counted later than it is
700
- // read, so it carries the attribution captured when it was read.
701
- function currentUsageAttribution() {
702
- return {
703
- model: currentCodexModel(modelAttributionState) || "unknown",
704
- selectedModel: modelAttributionState.selectedModel || null,
705
- rerouted: Boolean(modelAttributionState.rerouted),
706
- rerouteReason: modelAttributionState.rerouteReason || null,
707
- serviceTier,
708
- };
709
- }
710
-
711
- function recordModelUsage(delta, rawRequestUsage, attribution = currentUsageAttribution()) {
712
- if (!collectModelUsage || !delta || delta.total_tokens <= 0) return;
713
- const effectiveModel = attribution.model;
714
- let row = byModel.get(effectiveModel);
715
- if (!row) {
716
- row = {
717
- model: effectiveModel,
718
- ...emptyTotals(),
719
- long_context_input_tokens: 0,
720
- long_context_cached_input_tokens: 0,
721
- long_context_cache_creation_input_tokens: 0,
722
- long_context_output_tokens: 0,
723
- long_context_reasoning_output_tokens: 0,
724
- priority_input_tokens: 0,
725
- priority_cached_input_tokens: 0,
726
- priority_cache_creation_input_tokens: 0,
727
- priority_output_tokens: 0,
728
- priority_reasoning_output_tokens: 0,
729
- priority_long_context_input_tokens: 0,
730
- priority_long_context_cached_input_tokens: 0,
731
- priority_long_context_cache_creation_input_tokens: 0,
732
- priority_long_context_output_tokens: 0,
733
- priority_long_context_reasoning_output_tokens: 0,
734
- usage_events: 0,
735
- rerouted_usage_events: 0,
736
- long_context_usage_events: 0,
737
- priority_usage_events: 0,
738
- selected_models: new Set(),
739
- reroute_reasons: new Set(),
740
- };
741
- byModel.set(effectiveModel, row);
742
- }
743
- const subsetKeys = Object.keys(row).filter((key) => /^(?:long_context_|priority_)|^(?:usage_events|rerouted_usage_events)$/.test(key));
744
- const before = Object.fromEntries(subsetKeys.map((key) => [key, row[key]]));
745
- addInto(row, delta);
746
- row.usage_events += 1;
747
- if (attribution.selectedModel) {
748
- row.selected_models.add(attribution.selectedModel);
749
- }
750
- if (attribution.rerouted) {
751
- row.rerouted_usage_events += 1;
752
- if (attribution.rerouteReason) {
753
- row.reroute_reasons.add(attribution.rerouteReason);
754
- }
755
- }
756
- // OpenAI applies the long-context tier to the whole request when its raw
757
- // (cache-inclusive) input exceeds 272K tokens. Preserve the exact subset
758
- // so pricing can be recomputed later without rescanning private logs.
759
- const isLongContext =
760
- Number(rawRequestUsage?.input_tokens || 0) > OPENAI_LONG_CONTEXT_INPUT_THRESHOLD;
761
- if (isLongContext) {
762
- row.long_context_usage_events += 1;
763
- row.long_context_input_tokens += delta.input_tokens;
764
- row.long_context_cached_input_tokens += delta.cached_input_tokens;
765
- row.long_context_cache_creation_input_tokens += delta.cache_creation_input_tokens;
766
- row.long_context_output_tokens += delta.output_tokens;
767
- row.long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
768
- }
769
- // Astra Fast bills at a flat multiple of whichever context tier the request
770
- // was on, so the priority premium multiplies the long-context premium
771
- // instead of replacing it. The two subsets alone cannot say how much usage
772
- // was on BOTH, so record the intersection explicitly — otherwise a long
773
- // Fast request prices at 3x Standard input where it should be 4x.
774
- if (isPriorityServiceTier(attribution.serviceTier)) {
775
- row.priority_usage_events += 1;
776
- row.priority_input_tokens += delta.input_tokens;
777
- row.priority_cached_input_tokens += delta.cached_input_tokens;
778
- row.priority_cache_creation_input_tokens += delta.cache_creation_input_tokens;
779
- row.priority_output_tokens += delta.output_tokens;
780
- row.priority_reasoning_output_tokens += delta.reasoning_output_tokens;
781
- if (isLongContext) {
782
- row.priority_long_context_input_tokens += delta.input_tokens;
783
- row.priority_long_context_cached_input_tokens += delta.cached_input_tokens;
784
- row.priority_long_context_cache_creation_input_tokens +=
785
- delta.cache_creation_input_tokens;
786
- row.priority_long_context_output_tokens += delta.output_tokens;
787
- row.priority_long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
788
- }
789
- }
790
- return { ...delta, ...Object.fromEntries(subsetKeys.map((key) => [key, row[key] - before[key]])),
791
- selected_models: attribution.selectedModel ? [attribution.selectedModel] : [],
792
- reroute_reasons: attribution.rerouteReason ? [attribution.rerouteReason] : [],
793
- model_attribution: attribution.rerouted ? "effective" : "selected" };
794
- }
795
-
796
- function ensureTool(name) {
797
- if (!byTool.has(name)) {
798
- byTool.set(name, { name, ...buildToolStatsEntry() });
799
- }
800
- return byTool.get(name);
801
- }
802
-
803
- function ensureExec(map, key) {
804
- if (!map.has(key)) map.set(key, { name: key, ...buildExecStatsEntry() });
805
- return map.get(key);
806
- }
807
-
808
- function ensureSkill(name) {
809
- if (!bySkill.has(name)) bySkill.set(name, buildSkillStatsEntry(name));
810
- return bySkill.get(name);
811
- }
812
-
813
- function getExecKeys(p) {
814
- if (!p || typeof p !== "object") return;
815
- const cmdArr = Array.isArray(p.command) ? p.command : null;
816
- const cmd = cmdArr && cmdArr.length > 0 ? String(cmdArr[cmdArr.length - 1] || "") : "";
817
- const kind = p.parsed_cmd?.[0]?.type && p.parsed_cmd[0].type !== "unknown"
818
- ? p.parsed_cmd[0].type
819
- : inferExecCommandKind(cmd);
820
-
821
- const status = String(p.status || "unknown");
822
- const exit = Number.isFinite(Number(p.exit_code)) ? Number(p.exit_code) : null;
823
- const exitKey = `${status}:${exit === null ? "unknown" : exit}`;
824
-
825
- const dur = p.duration ? Math.round((Number(p.duration.secs || 0) * 1000) + Number(p.duration.nanos || 0) / 1e6) : 0;
826
- const output = String(p.aggregated_output || p.stdout || "");
827
- const outputChars = output.length;
828
- const outputLines = output ? output.split("\n").length : 0;
829
- return {
830
- kind,
831
- exitKey,
832
- executable: getExecutableName(cmd),
833
- command: sanitizeCommandSignature(cmd),
834
- duration: durationBucket(dur),
835
- output: outputSizeBucket(outputLines, outputChars),
836
- dur,
837
- outputChars,
838
- outputLines,
839
- failed: status !== "completed" || exit !== 0,
840
- };
841
- }
842
-
843
- function absorbExecStats(map, key, details) {
844
- const row = ensureExec(map, key);
845
- row.calls += 1;
846
- row.duration_ms += details.dur;
847
- row.max_duration_ms = Math.max(row.max_duration_ms, details.dur);
848
- row.output_chars += details.outputChars;
849
- row.output_lines += details.outputLines;
850
- if (details.failed) row.failures += 1;
851
- }
852
-
853
- function absorbExecEnd(details) {
854
- absorbExecStats(byExecKind, details.kind, details);
855
- absorbExecStats(byExecExit, details.exitKey, details);
856
- absorbExecStats(byExecExecutable, details.executable, details);
857
- absorbExecStats(byExecCommand, details.command, details);
858
- absorbExecStats(byExecDuration, details.duration, details);
859
- absorbExecStats(byExecOutput, details.output, details);
860
- }
861
-
862
- function attributeTurn(delta) {
863
- if (!delta || delta.total_tokens <= 0) {
864
- for (const details of pendingExecDetails) absorbExecEnd(details);
865
- pendingToolNames = [];
866
- pendingSkills = [];
867
- pendingExecDetails = [];
868
- return;
869
- }
870
- turnCount += 1;
871
-
872
- if (!collectBreakdowns) {
873
- addInto(totals, delta);
874
- pendingToolNames = [];
875
- pendingSkills = [];
876
- pendingExecDetails = [];
877
- return;
878
- }
879
-
880
- const unique = [...new Set(pendingToolNames.filter(Boolean))];
881
- const tools = unique.length > 0 ? unique : ["text_response"];
882
- const share = 1 / tools.length;
883
-
884
- for (const name of tools) {
885
- const row = ensureTool(name);
886
- row.calls += share;
887
- addInto(row.totals, {
888
- input_tokens: delta.input_tokens * share,
889
- cached_input_tokens: delta.cached_input_tokens * share,
890
- cache_creation_input_tokens: delta.cache_creation_input_tokens * share,
891
- output_tokens: delta.output_tokens * share,
892
- reasoning_output_tokens: delta.reasoning_output_tokens * share,
893
- total_tokens: delta.total_tokens * share,
894
- });
895
- }
896
-
897
- const uniqueSkills = [...new Set(pendingSkills.filter(Boolean))];
898
- if (uniqueSkills.length > 0) {
899
- const skillShare = 1 / uniqueSkills.length;
900
- for (const name of uniqueSkills) {
901
- const row = ensureSkill(name);
902
- row.calls += skillShare;
903
- addInto(row.totals, {
904
- input_tokens: delta.input_tokens * skillShare,
905
- cached_input_tokens: delta.cached_input_tokens * skillShare,
906
- cache_creation_input_tokens: delta.cache_creation_input_tokens * skillShare,
907
- output_tokens: delta.output_tokens * skillShare,
908
- reasoning_output_tokens: delta.reasoning_output_tokens * skillShare,
909
- total_tokens: delta.total_tokens * skillShare,
910
- });
911
- }
912
- }
913
-
914
- // Attribute exec_command_end rows to exec stats; note these are not a
915
- // token source — we attach the same tool-shared delta to the command
916
- // classifier so users can find high-cost command families.
917
- const execToolShare = tools.includes("exec_command") ? (1 / tools.length) : 0;
918
- const execDelta = execToolShare > 0 ? {
919
- input_tokens: delta.input_tokens * execToolShare,
920
- cached_input_tokens: delta.cached_input_tokens * execToolShare,
921
- cache_creation_input_tokens: delta.cache_creation_input_tokens * execToolShare,
922
- output_tokens: delta.output_tokens * execToolShare,
923
- reasoning_output_tokens: delta.reasoning_output_tokens * execToolShare,
924
- total_tokens: delta.total_tokens * execToolShare,
925
- } : null;
926
-
927
- if (execDelta && pendingExecDetails.length > 0) {
928
- const perExecShare = 1 / pendingExecDetails.length;
929
- for (const details of pendingExecDetails) {
930
- const attributed = {
931
- input_tokens: execDelta.input_tokens * perExecShare,
932
- cached_input_tokens: execDelta.cached_input_tokens * perExecShare,
933
- cache_creation_input_tokens: execDelta.cache_creation_input_tokens * perExecShare,
934
- output_tokens: execDelta.output_tokens * perExecShare,
935
- reasoning_output_tokens: execDelta.reasoning_output_tokens * perExecShare,
936
- total_tokens: execDelta.total_tokens * perExecShare,
937
- };
938
-
939
- addInto(ensureExec(byExecKind, details.kind).totals, attributed);
940
- addInto(ensureExec(byExecExit, details.exitKey).totals, attributed);
941
- addInto(ensureExec(byExecExecutable, details.executable).totals, attributed);
942
- addInto(ensureExec(byExecCommand, details.command).totals, attributed);
943
- addInto(ensureExec(byExecDuration, details.duration).totals, attributed);
944
- addInto(ensureExec(byExecOutput, details.output).totals, attributed);
945
-
946
- absorbExecEnd(details);
947
- }
948
- } else {
949
- // Still ingest exec end stats without token attribution so the drill-down works.
950
- for (const details of pendingExecDetails) absorbExecEnd(details);
951
- }
952
-
953
- addInto(totals, delta);
954
- pendingToolNames = [];
955
- pendingSkills = [];
956
- pendingExecDetails = [];
957
- }
958
-
959
- function countCompactionRecord(record) {
960
- if (compactionResponseIds.has(record.responseId)) return;
961
- compactionResponseIds.add(record.responseId);
962
- if (!record.inRequestedRange) return;
963
- const pricedDelta = recordModelUsage(record.delta, record.rawUsage, record.attribution);
964
- if (typeof onUsage === "function") onUsage({ timestamp: record.timestamp,
965
- delta: pricedDelta || record.delta, rawUsage: record.rawUsage, model: record.attribution.model, origin: "compaction" });
966
- attributeTurn(record.delta);
967
- }
968
-
969
- function usageEventSignature(lastUsage, totalUsage) {
970
- return JSON.stringify({ lastUsage, totalUsage });
971
- }
972
-
973
- function materializeLastTimestampSignatures() {
974
- if (hasUnmaterializedLastTimestampUsage) {
975
- lastTimestampEventSignatures.add(
976
- usageEventSignature(lastTimestampLastUsage, lastTimestampTotalUsage),
977
- );
978
- hasUnmaterializedLastTimestampUsage = false;
979
- }
980
- return lastTimestampEventSignatures;
981
- }
982
-
983
- function noteTokenEvent(timestamp, lastUsage, totalUsage) {
984
- if (!isResuming) {
985
- if (timestamp !== lastTokenTimestamp) {
986
- lastTokenTimestamp = timestamp;
987
- lastTimestampEventSignatures = new Set();
988
- hasUnmaterializedLastTimestampUsage = false;
989
- } else {
990
- materializeLastTimestampSignatures();
991
- }
992
- lastTimestampLastUsage = lastUsage;
993
- lastTimestampTotalUsage = totalUsage;
994
- hasUnmaterializedLastTimestampUsage = true;
995
- return true;
996
- }
997
- if (isResuming && lastTokenTimestamp && timestamp < lastTokenTimestamp) {
998
- const error = new Error("Codex rollout append is not timestamp-monotonic");
999
- error.code = CODEX_RESUME_INVALIDATED;
1000
- throw error;
1001
- }
1002
- if (!lastTokenTimestamp || timestamp > lastTokenTimestamp) {
1003
- lastTokenTimestamp = timestamp;
1004
- lastTimestampEventSignatures = new Set();
1005
- lastTimestampLastUsage = lastUsage;
1006
- lastTimestampTotalUsage = totalUsage;
1007
- hasUnmaterializedLastTimestampUsage = true;
1008
- return true;
1009
- }
1010
- if (timestamp === lastTokenTimestamp) {
1011
- const signature = usageEventSignature(lastUsage, totalUsage);
1012
- const signatures = materializeLastTimestampSignatures();
1013
- if (signatures.has(signature)) return false;
1014
- signatures.add(signature);
1015
- }
1016
- return true;
1017
- }
1018
-
1019
- for await (const obj of iterateCodexObjects(filePaths, diagnostics, {
1020
- startOffset,
1021
- endOffset,
1022
- readProgress,
1023
- contentHasher,
1024
- sourceHandle,
1025
- })) {
1026
- const modelEvent = applyCodexModelEvent(modelAttributionState, obj);
1027
- const attributedModel = currentCodexModel(modelAttributionState);
1028
- model = attributedModel || model;
1029
- // onObject runs AFTER the attribution state machine, and is handed the
1030
- // model it just decided. A consumer sharing this pass (the delivery-signal
1031
- // collector, which attributes edit turns) therefore sees exactly the model
1032
- // recordModelUsage() bills the turn's tokens to. Letting it re-read
1033
- // turn_context.payload.model itself would make codex-model-attribution.js
1034
- // one of two authorities on the same question.
1035
- if (typeof onObject === "function") onObject(obj, attributedModel);
1036
- const usageRecord = extractTokenUsageRecord(obj);
1037
- if (!usageRecord) noteLineAfterUsageRecord(usageRecordState, obj?.type === "compacted");
1038
- const ts = typeof obj?.timestamp === "string" ? obj.timestamp : null;
1039
- if (!ts) continue;
1040
- const inRequestedRange = isTimestampInRequestedDayRange(ts, { from, to, timeZoneContext });
1041
-
1042
- if (obj.type === "session_meta") {
1043
- const p = obj.payload || {};
1044
- // Forked rollouts replay historical session_meta rows after their own
1045
- // child metadata. Only the first row describes this file's lineage.
1046
- if (!sessionLineageCaptured) {
1047
- sessionId = p.id || sessionId;
1048
- cwd = p.cwd || cwd;
1049
- cliVersion = p.cli_version || cliVersion;
1050
- provider = p.model_provider || provider;
1051
- forkedFromId = p.forked_from_id || null;
1052
- parentThreadId = p.parent_thread_id || null;
1053
- threadSource = p.thread_source || p.source || null;
1054
- agentNickname = p.agent_nickname || null;
1055
- agentRole = p.agent_role || null;
1056
- sessionLineageCaptured = true;
1057
- } else if (!sessionId) {
1058
- sessionId = p.id || null;
1059
- }
1060
- }
1061
-
1062
- const appliedServiceTier = readCodexServiceTier(obj);
1063
- if (appliedServiceTier) {
1064
- serviceTier = appliedServiceTier;
1065
- continue;
1066
- }
1067
-
1068
- if (obj.type === "turn_context") {
1069
- const p = obj.payload || {};
1070
- if (typeof p.cwd === "string") cwd = p.cwd;
1071
- continue;
1072
- }
1073
-
1074
- if (modelEvent?.type === "model_rerouted") continue;
1075
-
1076
- if (collectBreakdowns && obj.type === "response_item" && obj.payload?.type === "function_call") {
1077
- pendingToolNames.push(normalizeToolName(obj.payload));
1078
- const skill = extractSkillNameFromFunctionCall(obj.payload, diagnostics);
1079
- if (skill) pendingSkills.push(skill);
1080
- continue;
1081
- }
1082
-
1083
- if (collectBreakdowns && obj.type === "event_msg" && obj.payload?.type === "exec_command_end") {
1084
- const details = getExecKeys(obj.payload);
1085
- if (details) pendingExecDetails.push(details);
1086
- continue;
1087
- }
1088
-
1089
- if (usageRecord) {
1090
- const delta = normalizeUsage(usageRecord.usage);
1091
- noteUsageRecord(
1092
- usageRecordState,
1093
- // Without a response_id a compaction cannot be deduplicated.
1094
- delta.total_tokens > 0 && usageRecord.responseId
1095
- ? {
1096
- responseId: usageRecord.responseId,
1097
- timestamp: usageRecord.timestamp,
1098
- inRequestedRange,
1099
- rawUsage: usageRecord.usage,
1100
- delta,
1101
- attribution: currentUsageAttribution(),
1102
- }
1103
- : null,
1104
- );
1105
- continue;
1106
- }
1107
-
1108
- const tokenCount = extractTokenCount(obj);
1109
- if (tokenCount) {
1110
- const info = tokenCount.info;
1111
- const lastUsage = info?.last_token_usage;
1112
- const totalUsage = info?.total_token_usage;
1113
- let compaction = null;
1114
- if (info && typeof info === "object") {
1115
- noteTokenCount(usageRecordState);
1116
- compaction = takeCompactionOnTokenCount(
1117
- usageRecordState,
1118
- usageDeltaState.lastTotal,
1119
- totalUsage,
1120
- );
1121
- }
1122
- const rawDelta = consumeUsageDelta(usageDeltaState, lastUsage, totalUsage);
1123
- const delta = rawDelta ? normalizeUsage(rawDelta) : null;
1124
- // Timing consumers receive canonical, deduplicated usage below. A null
1125
- // delta is a snapshot, not another model response.
1126
- if (compaction) countCompactionRecord(compaction);
1127
- const isNewResumeEvent = noteTokenEvent(ts, lastUsage, totalUsage);
1128
- if (inRequestedRange) {
1129
- const eventSessionId = sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath;
1130
- const eventKey = `${eventSessionId}:${ts}:${usageEventSignature(lastUsage, totalUsage)}`;
1131
- if (!isNewResumeEvent || (seenTokenEvents && seenTokenEvents.has(eventKey))) {
1132
- if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: null, rawUsage: lastUsage, model: attributedModel });
1133
- attributeTurn(null);
1134
- } else {
1135
- if (seenTokenEvents && delta?.total_tokens > 0) seenTokenEvents.add(eventKey);
1136
- const pricedDelta = recordModelUsage(delta, lastUsage || rawDelta);
1137
- if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: pricedDelta || delta, rawUsage: lastUsage || rawDelta, model: attributedModel });
1138
- attributeTurn(delta);
1139
- }
1140
- } else {
1141
- pendingToolNames = [];
1142
- pendingSkills = [];
1143
- pendingExecDetails = [];
1144
- }
1145
- continue;
1146
- }
1147
- }
1148
-
1149
- const result = {
1150
- sessionId: sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath,
1151
- cwd,
1152
- model: currentCodexModel(modelAttributionState) || model || provider,
1153
- provider,
1154
- version: cliVersion,
1155
- forkedFromId,
1156
- parentThreadId,
1157
- threadSource,
1158
- agentNickname,
1159
- agentRole,
1160
- filePath: primaryFilePath,
1161
- turnCount,
1162
- totals,
1163
- modelUsage: finalizeModelUsageRows(byModel),
1164
- toolBreakdown: {
1165
- tool_rows: finalizeToolRows(byTool),
1166
- },
1167
- skillsBreakdown: {
1168
- skill_rows: finalizeSkillRows(bySkill),
1169
- },
1170
- execCommandBreakdown: {
1171
- byType: finalizeExecRows(byExecKind),
1172
- byExit: finalizeExecRows(byExecExit),
1173
- byExecutable: finalizeExecRows(byExecExecutable),
1174
- byCommand: finalizeExecRows(byExecCommand),
1175
- byDuration: finalizeExecRows(byExecDuration),
1176
- byOutput: finalizeExecRows(byExecOutput),
1177
- },
1178
- };
1179
- if (captureResumeState && filePaths.length === 1) {
1180
- result.resumeState = {
1181
- sessionId,
1182
- cwd,
1183
- model,
1184
- modelAttributionState: snapshotCodexModelAttributionState(modelAttributionState),
1185
- serviceTier,
1186
- provider,
1187
- cliVersion,
1188
- forkedFromId,
1189
- parentThreadId,
1190
- threadSource,
1191
- agentNickname,
1192
- agentRole,
1193
- sessionLineageCaptured,
1194
- prevTotals: usageDeltaState.lastTotal,
1195
- tokenUsageBaselines: snapshotUsageBaselines(usageDeltaState),
1196
- pendingToolNames,
1197
- pendingSkills,
1198
- pendingExecDetails,
1199
- lastTokenTimestamp,
1200
- lastTimestampEventSignatures: Array.from(materializeLastTimestampSignatures()),
1201
- usageRecordState: snapshotUsageRecordState(usageRecordState),
1202
- compactionResponseIds: Array.from(compactionResponseIds),
1203
- };
1204
- result.endOffset = readProgress.endOffset;
1205
- // Compressed snapshots are not append-only logs: endOffset stays at the
1206
- // pre-decode cursor (often 0) and must never be marked appendable, or the
1207
- // next growth would "resume" by parsing compressed bytes as JSONL.
1208
- const hasCompressedArchive = filePaths.some(
1209
- (candidate) => typeof candidate === "string" && /\.jsonl\.zstd?$/i.test(candidate),
1210
- );
1211
- result.appendable = Boolean(contentHasher) && !hasCompressedArchive && (
1212
- readProgress.endOffset === 0 || readProgress.lastByte === 0x0a
1213
- );
1214
- result.contentHashState = contentHasher;
1215
- }
1216
- return result;
1217
- }
1218
-
1219
- const fatalUtf8Decoder = new TextDecoder("utf-8", { fatal: true });
1220
-
1221
- function parseCodexLine(lineBuffer, diagnostics) {
1222
- let content = lineBuffer;
1223
- if (content.length > 0 && content[content.length - 1] === 0x0d) {
1224
- content = content.subarray(0, content.length - 1);
1225
- }
1226
- if (content.length === 0) return null;
1227
- try {
1228
- if (diagnostics) diagnostics.json_parse_calls += 1;
1229
- return JSON.parse(fatalUtf8Decoder.decode(content));
1230
- } catch {
1231
- return null;
1232
- }
1233
- }
1234
-
1235
- async function* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
1236
- startOffset = 0,
1237
- endOffset = null,
1238
- readProgress = null,
1239
- contentHasher = null,
1240
- sourceHandle = null,
1241
- } = {}) {
1242
- const start = Math.max(0, Number(startOffset) || 0);
1243
- const requestedEnd = Number(endOffset);
1244
- const hasBoundedEnd = endOffset !== null && endOffset !== undefined &&
1245
- Number.isFinite(requestedEnd) && requestedEnd >= 0;
1246
- if (hasBoundedEnd && requestedEnd <= start) {
1247
- if (readProgress) readProgress.endOffset = start;
1248
- return;
1249
- }
1250
-
1251
- if (diagnostics) diagnostics.opened_files += 1;
1252
- const stream = sourceHandle && typeof sourceHandle.read === "function"
1253
- ? null
1254
- : fs.createReadStream(filePath, {
1255
- start,
1256
- ...(hasBoundedEnd ? { end: requestedEnd - 1 } : {}),
1257
- });
1258
- const chunks = stream || (async function* readStableHandleChunks() {
1259
- const buffer = Buffer.allocUnsafe(64 * 1024);
1260
- const limit = hasBoundedEnd ? requestedEnd : Number.MAX_SAFE_INTEGER;
1261
- let offset = start;
1262
- while (offset < limit) {
1263
- const length = Math.min(buffer.length, limit - offset);
1264
- const result = await sourceHandle.read(buffer, 0, length, offset);
1265
- if (!result.bytesRead) break;
1266
- offset += result.bytesRead;
1267
- yield buffer.subarray(0, result.bytesRead);
1268
- }
1269
- })();
1270
- const fragments = [];
1271
- let fragmentsBytes = 0;
1272
- let nextChunkOffset = start;
1273
- let lineStartOffset = start;
1274
- let lineIndex = 0;
1275
-
1276
- try {
1277
- for await (const value of chunks) {
1278
- const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
1279
- const chunkStart = nextChunkOffset;
1280
- nextChunkOffset += chunk.length;
1281
- if (contentHasher) contentHasher.update(chunk);
1282
- if (readProgress && chunk.length > 0) {
1283
- readProgress.lastByte = chunk[chunk.length - 1];
1284
- }
1285
- if (diagnostics) {
1286
- diagnostics.bytes_read = Number(diagnostics.bytes_read || 0) + chunk.length;
1287
- }
1288
-
1289
- let cursor = 0;
1290
- while (cursor < chunk.length) {
1291
- const newline = chunk.indexOf(0x0a, cursor);
1292
- if (newline < 0) {
1293
- const tail = chunk.subarray(cursor);
1294
- if (tail.length > 0) {
1295
- // Stable-handle reads reuse their buffer, so retained fragments
1296
- // must own their bytes before the next read overwrites it.
1297
- fragments.push(Buffer.from(tail));
1298
- fragmentsBytes += tail.length;
1299
- }
1300
- break;
1301
- }
1302
-
1303
- const tail = chunk.subarray(cursor, newline);
1304
- const lineBuffer = fragments.length > 0
1305
- ? Buffer.concat([...fragments, tail], fragmentsBytes + tail.length)
1306
- : tail;
1307
- fragments.length = 0;
1308
- fragmentsBytes = 0;
1309
-
1310
- const absoluteEnd = chunkStart + newline + 1;
1311
- if (readProgress) readProgress.endOffset = absoluteEnd;
1312
- lineStartOffset = absoluteEnd;
1313
- const obj = parseCodexLine(lineBuffer, diagnostics);
1314
- if (obj) {
1315
- yield { obj, fileIndex, lineIndex };
1316
- lineIndex += 1;
1317
- }
1318
- cursor = newline + 1;
1319
- }
1320
- }
1321
-
1322
- if (fragmentsBytes > 0) {
1323
- const lineBuffer = fragments.length === 1
1324
- ? fragments[0]
1325
- : Buffer.concat(fragments, fragmentsBytes);
1326
- const obj = parseCodexLine(lineBuffer, diagnostics);
1327
- if (obj) {
1328
- if (readProgress) readProgress.endOffset = lineStartOffset + fragmentsBytes;
1329
- yield { obj, fileIndex, lineIndex };
1330
- }
1331
- }
1332
- } finally {
1333
- if (readProgress) readProgress.endOffset = nextChunkOffset;
1334
- stream?.destroy();
1335
- if (diagnostics) diagnostics.parsed_files += 1;
1336
- }
1337
- }
1338
-
1339
- async function* readCodexObjectsFull(filePath, diagnostics, fileIndex, {
1340
- startOffset = 0,
1341
- endOffset = null,
1342
- readProgress = null,
1343
- contentHasher = null,
1344
- } = {}) {
1345
- // Compressed archives are immutable snapshots, not append-only logs: byte
1346
- // offsets are meaningless inside them, so decode the whole text and split
1347
- // into lines. `readProgress` stays null so callers never persist a fake
1348
- // byte cursor for them.
1349
- if (typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath)) {
1350
- const text = await readCodexRolloutText(filePath);
1351
- let lineIndex = 0;
1352
- const lines = text.split("\n");
1353
- for (let index = 0; index < lines.length; index += 1) {
1354
- const isLast = index + 1 >= lines.length;
1355
- if (isLast && lines[index] === "") continue;
1356
- const lineBuffer = Buffer.from(lines[index], "utf8");
1357
- const obj = parseCodexLine(lineBuffer, diagnostics);
1358
- if (obj) {
1359
- yield { obj, fileIndex, lineIndex };
1360
- lineIndex += 1;
1361
- }
1362
- }
1363
- if (diagnostics) diagnostics.parsed_files += 1;
1364
- return;
1365
- }
1366
- yield* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
1367
- startOffset,
1368
- endOffset,
1369
- readProgress,
1370
- contentHasher,
1371
- });
1372
- }
1373
-
1374
- async function* readCodexObjects(filePath, diagnostics, fileIndex, options = {}) {
1375
- const start = Math.max(0, Number(options.startOffset) || 0);
1376
- const reader = start > 0 || options.sourceHandle
1377
- ? readCodexObjectsIncremental
1378
- : readCodexObjectsFull;
1379
- yield* reader(filePath, diagnostics, fileIndex, options);
1380
- }
1381
-
1382
- function isRecoverableGroupedReadError(error) {
1383
- return ["ENOENT", "EACCES", "EPERM", "EISDIR", "EIO", "ESTALE"].includes(error?.code);
1384
- }
1385
-
1386
- async function* iterateCodexObjects(filePaths, diagnostics, options = {}) {
1387
- if (filePaths.length <= 1) {
1388
- if (!filePaths[0]) return;
1389
- for await (const record of readCodexObjects(filePaths[0], diagnostics, 0, options)) {
1390
- yield record.obj;
1391
- }
1392
- return;
1393
- }
1394
-
1395
- // A live rollout can overlap its archived copy or be split into prefix/suffix
1396
- // fragments. Merge only duplicate-session groups, order by event time, and
1397
- // suppress byte-semantically identical records from different files so the
1398
- // cumulative baseline is established exactly once.
1399
- const records = [];
1400
- let scannedFiles = 0;
1401
- for (let fileIndex = 0; fileIndex < filePaths.length; fileIndex += 1) {
1402
- const fileRecords = [];
1403
- try {
1404
- for await (const record of readCodexObjects(filePaths[fileIndex], diagnostics, fileIndex)) {
1405
- fileRecords.push(record);
1406
- }
1407
- for (const record of fileRecords) records.push(record);
1408
- scannedFiles += 1;
1409
- } catch (error) {
1410
- if (!isRecoverableGroupedReadError(error)) throw error;
1411
- }
1412
- }
1413
- if (!scannedFiles) {
1414
- const error = new Error("all grouped Codex rollout files failed during read");
1415
- error.code = "EIO";
1416
- throw error;
1417
- }
1418
- records.sort((a, b) => {
1419
- const aTimestamp = typeof a.obj?.timestamp === "string" ? a.obj.timestamp : "";
1420
- const bTimestamp = typeof b.obj?.timestamp === "string" ? b.obj.timestamp : "";
1421
- return aTimestamp.localeCompare(bTimestamp) || a.fileIndex - b.fileIndex || a.lineIndex - b.lineIndex;
1422
- });
1423
- const firstFileByRecord = new Map();
1424
- for (const record of records) {
1425
- const semanticKey = JSON.stringify(record.obj);
1426
- const firstFile = firstFileByRecord.get(semanticKey);
1427
- if (firstFile !== undefined && firstFile !== record.fileIndex) continue;
1428
- if (firstFile === undefined) firstFileByRecord.set(semanticKey, record.fileIndex);
1429
- yield record.obj;
1430
- }
1431
- }
1432
-
1433
- function rolloutSessionIdFromPath(filePath) {
1434
- const match = path.basename(String(filePath || "")).match(
1435
- /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl(?:\.zstd?)?$/i,
1436
- );
1437
- return match ? match[1] : null;
1438
- }
1439
-
1440
- function isCodexResumeInvalidated(error) {
1441
- return error?.code === CODEX_RESUME_INVALIDATED;
1442
- }
1443
-
1444
- module.exports = {
1445
- parseCodexRolloutFile,
1446
- extractTokenCount,
1447
- extractSkillNameFromFunctionCall,
1448
- formatToolDisplayName,
1449
- normalizeToolName,
1450
- pickDelta,
1451
- normalizeUsage,
1452
- totalsReset,
1453
- listJsonlFiles,
1454
- listCodexSessionFiles,
1455
- discoverCodexSessionFiles,
1456
- rolloutPathDay,
1457
- isConservativeRangeCandidate,
1458
- safeJsonParse,
1459
- dayKeyToIsoBounds,
1460
- formatPartsDayKey,
1461
- getZonedParts,
1462
- timestampDayKey,
1463
- isTimestampInRequestedDayRange,
1464
- rolloutSessionIdFromPath,
1465
- isCodexResumeInvalidated,
1466
- isRecoverableGroupedReadError,
1467
- finalizeToolRows,
1468
- finalizeSkillRows,
1469
- finalizeExecRows,
1470
- buildSkillStatsEntry,
1471
- };
1
+ // Codex rollout JSONL parser — extracted from codex-context-breakdown.js.
2
+ //
3
+ // Handles file discovery and per-file parsing. Does NOT hold any aggregation
4
+ // state; callers (computeCodexContextBreakdown) own the merge step.
5
+
6
+ "use strict";
7
+
8
+ const crypto = require("node:crypto");
9
+ const fs = require("node:fs");
10
+ const path = require("node:path");
11
+
12
+ const { listRolloutFiles } = require("./rollout");
13
+ const {
14
+ consumeUsageDelta,
15
+ createUsageDeltaState,
16
+ snapshotUsageBaselines,
17
+ totalsReset,
18
+ } = require("./codex-token-usage");
19
+ const {
20
+ emptyTotals,
21
+ addInto,
22
+ inferExecCommandKind,
23
+ sanitizeCommandSignature,
24
+ getExecutableName,
25
+ buildExecStatsEntry,
26
+ } = require("./categorizer-utils");
27
+ const {
28
+ applyCodexModelEvent,
29
+ createCodexModelAttributionState,
30
+ currentCodexModel,
31
+ snapshotCodexModelAttributionState,
32
+ } = require("./codex-model-attribution");
33
+ const { readCodexServiceTier, isPriorityServiceTier } = require("./codex-service-tier");
34
+ const {
35
+ createUsageRecordState,
36
+ extractTokenUsageRecord,
37
+ noteLineAfterUsageRecord,
38
+ noteTokenCount,
39
+ noteUsageRecord,
40
+ snapshotUsageRecordState,
41
+ takeCompactionOnTokenCount,
42
+ } = require("./codex-usage-record");
43
+
44
+ const DISCOVERY_FULL_AUDIT_INTERVAL_MS = 60 * 1000;
45
+ const CODEX_ROLLOUT_TEXT_MAX_BYTES = 256 * 1024 * 1024;
46
+
47
+ async function readCodexRolloutText(filePath, { maxOutputBytes = CODEX_ROLLOUT_TEXT_MAX_BYTES } = {}) {
48
+ if (typeof filePath !== "string" || !/\.jsonl\.zstd?$/i.test(filePath)) {
49
+ return fs.readFile(filePath, "utf8");
50
+ }
51
+ // The snapshot decoder lives in rollout.js (reused by the sync parser);
52
+ // require lazily so this module stays loadable without pulling the whole
53
+ // rollout graph at import time.
54
+ const rollout = require("./rollout");
55
+ const snapshot = await rollout.readCodexRolloutSnapshot(filePath, { maxOutputBytes });
56
+ if (!snapshot) {
57
+ // Treat undecodable / vanished archives as recoverable I/O so grouped
58
+ // session reads (live + archive) can still use a readable sibling copy.
59
+ const error = new Error(`unreadable Codex archive ${filePath}`);
60
+ error.code = "EIO";
61
+ throw error;
62
+ }
63
+ // Snapshot lines keep their "\n" terminators; the tail element may be an
64
+ // unterminated final line — join preserves exactly that.
65
+ return snapshot.lines.join("\n");
66
+ }
67
+ const MAX_DISCOVERY_INVENTORIES = 32;
68
+ const DISCOVERY_INVENTORIES = new Map();
69
+ const ZONED_DAY_FORMATTERS = new Map();
70
+ const CODEX_RESUME_INVALIDATED = "TOKENUSAGE_CODEX_RESUME_INVALIDATED";
71
+ // Match src/lib/pricing/index.js. Classify individual requests using raw
72
+ // input (including cache), not the accumulated session or day totals.
73
+ const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272_000;
74
+
75
+ // ---------------------------------------------------------------------------
76
+ // Timezone helpers
77
+ // ---------------------------------------------------------------------------
78
+
79
+ function dayKeyToIsoBounds(from, to) {
80
+ if (!from && !to) return { fromIso: null, toIso: null };
81
+ const fromDate = from ? new Date(`${from}T00:00:00Z`) : null;
82
+ const toDate = to ? new Date(`${to}T23:59:59Z`) : null;
83
+ if (fromDate && Number.isFinite(fromDate.getTime())) fromDate.setUTCHours(fromDate.getUTCHours() - 14);
84
+ if (toDate && Number.isFinite(toDate.getTime())) toDate.setUTCHours(toDate.getUTCHours() + 14);
85
+ return {
86
+ fromIso: fromDate ? fromDate.toISOString() : null,
87
+ toIso: toDate ? toDate.toISOString() : null,
88
+ };
89
+ }
90
+
91
+ function formatPartsDayKey(parts) {
92
+ if (!parts) return "";
93
+ const values = {};
94
+ for (const part of parts) {
95
+ if (part.type !== "literal") values[part.type] = part.value;
96
+ }
97
+ if (!values.year || !values.month || !values.day) return "";
98
+ return `${values.year}-${values.month}-${values.day}`;
99
+ }
100
+
101
+ function getZonedParts(date, timeZoneContext = {}) {
102
+ const { timeZone, offsetMinutes } = timeZoneContext || {};
103
+ const dt = date instanceof Date ? date : new Date(date);
104
+ if (!Number.isFinite(dt.getTime())) return null;
105
+
106
+ if (timeZone && typeof Intl !== "undefined" && Intl.DateTimeFormat) {
107
+ try {
108
+ let formatter = ZONED_DAY_FORMATTERS.get(timeZone);
109
+ if (!formatter) {
110
+ formatter = new Intl.DateTimeFormat("en-CA", {
111
+ timeZone,
112
+ year: "numeric",
113
+ month: "2-digit",
114
+ day: "2-digit",
115
+ hourCycle: "h23",
116
+ });
117
+ ZONED_DAY_FORMATTERS.set(timeZone, formatter);
118
+ }
119
+ return formatter.formatToParts(dt);
120
+ } catch {
121
+ // Fall through to offset handling.
122
+ }
123
+ }
124
+
125
+ if (Number.isFinite(offsetMinutes)) {
126
+ const shifted = new Date(dt.getTime() - Number(offsetMinutes) * 60_000);
127
+ return [
128
+ { type: "year", value: String(shifted.getUTCFullYear()).padStart(4, "0") },
129
+ { type: "month", value: String(shifted.getUTCMonth() + 1).padStart(2, "0") },
130
+ { type: "day", value: String(shifted.getUTCDate()).padStart(2, "0") },
131
+ ];
132
+ }
133
+
134
+ return null;
135
+ }
136
+
137
+ function timestampDayKey(timestamp, timeZoneContext) {
138
+ const ts = typeof timestamp === "string" ? timestamp : "";
139
+ if (!ts) return "";
140
+ const parts = getZonedParts(ts, timeZoneContext);
141
+ const zoned = formatPartsDayKey(parts);
142
+ if (zoned) return zoned;
143
+ return ts.slice(0, 10);
144
+ }
145
+
146
+ function isTimestampInRequestedDayRange(timestamp, { from, to, timeZoneContext } = {}) {
147
+ if (!from && !to) return true;
148
+ const day = timestampDayKey(timestamp, timeZoneContext);
149
+ if (!day) return false;
150
+ if (from && day < from) return false;
151
+ if (to && day > to) return false;
152
+ return true;
153
+ }
154
+
155
+ // ---------------------------------------------------------------------------
156
+ // File discovery
157
+ // ---------------------------------------------------------------------------
158
+
159
+ function safeJsonParse(str, diagnostics = null) {
160
+ try {
161
+ if (diagnostics) diagnostics.json_parse_calls += 1;
162
+ return JSON.parse(str);
163
+ } catch {
164
+ return null;
165
+ }
166
+ }
167
+
168
+ function isRolloutCandidateFileName(name) {
169
+ if (typeof name !== "string") return false;
170
+ return name.endsWith(".jsonl") || /^rollout-.*\.jsonl\.zstd?$/.test(name);
171
+ }
172
+
173
+ function listJsonlFiles(rootDir) {
174
+ const out = [];
175
+ const stack = [rootDir];
176
+
177
+ while (stack.length > 0) {
178
+ const dir = stack.pop();
179
+ let entries;
180
+ try {
181
+ entries = fs.readdirSync(dir, { withFileTypes: true });
182
+ } catch {
183
+ continue;
184
+ }
185
+
186
+ for (const entry of entries) {
187
+ const filePath = path.join(dir, entry.name);
188
+ if (entry.isDirectory()) {
189
+ stack.push(filePath);
190
+ } else if (entry.isFile() && isRolloutCandidateFileName(entry.name)) {
191
+ out.push(filePath);
192
+ }
193
+ }
194
+ }
195
+
196
+ out.sort((a, b) => a.localeCompare(b));
197
+ return out;
198
+ }
199
+
200
+ function rolloutPathDay(filePath) {
201
+ const normalized = String(filePath || "").replaceAll("\\", "/");
202
+ const fileMatch = path.basename(normalized).match(/(?:^|rollout-)(\d{4})-(\d{2})-(\d{2})/);
203
+ if (fileMatch) return `${fileMatch[1]}-${fileMatch[2]}-${fileMatch[3]}`;
204
+ const directoryMatch = normalized.match(/.*\/(\d{4})\/(\d{2})\/(\d{2})(?:\/|$)/);
205
+ return directoryMatch
206
+ ? `${directoryMatch[1]}-${directoryMatch[2]}-${directoryMatch[3]}`
207
+ : null;
208
+ }
209
+
210
+ function nextDayKey(dayKey) {
211
+ if (!dayKey) return null;
212
+ const date = new Date(`${dayKey}T00:00:00.000Z`);
213
+ if (!Number.isFinite(date.getTime())) return null;
214
+ date.setUTCDate(date.getUTCDate() + 1);
215
+ return date.toISOString().slice(0, 10);
216
+ }
217
+
218
+ function metadataRangeBounds(from, to, timeZoneContext) {
219
+ const offsetMinutes = timeZoneContext?.offsetMinutes;
220
+ if (Number.isFinite(offsetMinutes)) {
221
+ const dstGuardMs = 2 * 60 * 60 * 1000;
222
+ const fromUtc = from ? Date.parse(`${from}T00:00:00.000Z`) : null;
223
+ const afterTo = to ? nextDayKey(to) : null;
224
+ const toUtcExclusive = afterTo ? Date.parse(`${afterTo}T00:00:00.000Z`) : null;
225
+ return {
226
+ fromMs: Number.isFinite(fromUtc)
227
+ ? fromUtc + offsetMinutes * 60_000 - dstGuardMs
228
+ : null,
229
+ toMs: Number.isFinite(toUtcExclusive)
230
+ ? toUtcExclusive + offsetMinutes * 60_000 + dstGuardMs
231
+ : null,
232
+ };
233
+ }
234
+ const { fromIso, toIso } = dayKeyToIsoBounds(from, to);
235
+ return {
236
+ fromMs: fromIso ? Date.parse(fromIso) : null,
237
+ toMs: toIso ? Date.parse(toIso) : null,
238
+ };
239
+ }
240
+
241
+ function isConservativeRangeCandidate(filePath, stat, {
242
+ from = null,
243
+ to = null,
244
+ timeZoneContext = null,
245
+ metadataBounds = null,
246
+ } = {}) {
247
+ if (!from && !to) return true;
248
+ // With no lower bound, a later-path rewrite can still contain an event at or
249
+ // before `to`. There is no metadata-only exclusion that can prove otherwise.
250
+ if (!from) return true;
251
+ const pathDay = rolloutPathDay(filePath);
252
+ if (!pathDay) return true;
253
+
254
+ const pathOverlaps = (!from || pathDay >= from) && (!to || pathDay <= to);
255
+ if (pathOverlaps) return true;
256
+
257
+ // Old sessions can be appended or rewritten. A post-range mtime or ctime is
258
+ // therefore a reason to parse, never a reason to exclude. ctime also catches
259
+ // cold-process inode replacement/truncation when a writer preserves mtime.
260
+ // Future-dated decoys are kept conservatively; correctness wins over an open.
261
+ const bounds = metadataBounds || metadataRangeBounds(from, to, timeZoneContext);
262
+ const fromMs = bounds.fromMs;
263
+ const toMs = bounds.toMs;
264
+ const mtimeMs = Number(stat?.mtimeMs);
265
+ const ctimeMs = Number(stat?.ctimeMs);
266
+ const changedAfterStart = (value) => (
267
+ Number.isFinite(value) && (!Number.isFinite(fromMs) || value >= fromMs)
268
+ );
269
+ const changedInsideRange = (value) => (
270
+ changedAfterStart(value) && (!Number.isFinite(toMs) || value <= toMs)
271
+ );
272
+ if (pathDay < from) {
273
+ return changedAfterStart(mtimeMs) || changedAfterStart(ctimeMs);
274
+ }
275
+
276
+ // A path one day ahead can still overlap the requested browser day because
277
+ // rollout folders follow the writer's local date. Its current mtime may be
278
+ // later if the same session kept growing, so metadata alone cannot exclude it.
279
+ if (to && pathDay === nextDayKey(to)) return true;
280
+
281
+ // A host-local path day can be later than the requested browser day when
282
+ // their timezones differ. Keep it only when filesystem metadata maps back
283
+ // into the requested day range; do not pull every genuinely future session
284
+ // into a historical query.
285
+ return changedInsideRange(mtimeMs) || changedInsideRange(ctimeMs);
286
+ }
287
+
288
+ function isRolloutPathInRequestedRange(filePath, { from = null, to = null } = {}) {
289
+ const pathDay = rolloutPathDay(filePath);
290
+ if (!pathDay) return true;
291
+ if (from && pathDay < from) return false;
292
+ if (to && pathDay > to) return false;
293
+ return true;
294
+ }
295
+
296
+ function discoveryInventoryKey(roots) {
297
+ return roots.map((root) => path.resolve(root)).sort().join("\0");
298
+ }
299
+
300
+ function getDiscoveryInventory(roots, nowMs) {
301
+ const key = discoveryInventoryKey(roots);
302
+ let inventory = DISCOVERY_INVENTORIES.get(key);
303
+ if (!inventory) {
304
+ inventory = { files: new Map(), lastFullAuditAtMs: 0, lastUsedAtMs: nowMs };
305
+ DISCOVERY_INVENTORIES.set(key, inventory);
306
+ }
307
+ inventory.lastUsedAtMs = nowMs;
308
+ while (DISCOVERY_INVENTORIES.size > MAX_DISCOVERY_INVENTORIES) {
309
+ let oldestKey = null;
310
+ let oldestAt = Infinity;
311
+ for (const [candidateKey, candidate] of DISCOVERY_INVENTORIES) {
312
+ if (candidateKey === key) continue;
313
+ if (candidate.lastUsedAtMs < oldestAt) {
314
+ oldestAt = candidate.lastUsedAtMs;
315
+ oldestKey = candidateKey;
316
+ }
317
+ }
318
+ if (!oldestKey) break;
319
+ DISCOVERY_INVENTORIES.delete(oldestKey);
320
+ }
321
+ return inventory;
322
+ }
323
+
324
+ function discoverCodexSessionFiles(rootDirs, options = {}) {
325
+ const diagnostics = options.diagnostics || null;
326
+ const roots = Array.isArray(rootDirs) ? rootDirs : [rootDirs];
327
+ const nowMs = Number.isFinite(Number(options.nowMs)) ? Number(options.nowMs) : Date.now();
328
+ const inventory = getDiscoveryInventory(roots.filter(Boolean), nowMs);
329
+ const candidateOptions = {
330
+ ...options,
331
+ metadataBounds: metadataRangeBounds(options.from, options.to, options.timeZoneContext),
332
+ };
333
+ const bounded = Boolean(options.from || options.to);
334
+ const auditAgeMs = nowMs - Number(inventory.lastFullAuditAtMs || 0);
335
+ const fullMetadataAudit = Boolean(
336
+ options.exhaustive ||
337
+ !bounded ||
338
+ !Number.isFinite(auditAgeMs) ||
339
+ auditAgeMs < 0 ||
340
+ auditAgeMs >= DISCOVERY_FULL_AUDIT_INTERVAL_MS
341
+ );
342
+ if (diagnostics) diagnostics.full_metadata_audit = fullMetadataAudit;
343
+ const entries = [];
344
+ const visited = new Set();
345
+ const stack = roots.filter(Boolean);
346
+
347
+ while (stack.length > 0) {
348
+ const dir = stack.pop();
349
+ let children;
350
+ try {
351
+ children = fs.readdirSync(dir, { withFileTypes: true });
352
+ } catch {
353
+ continue;
354
+ }
355
+ for (const child of children) {
356
+ const filePath = path.join(dir, child.name);
357
+ if (child.isDirectory()) {
358
+ stack.push(filePath);
359
+ continue;
360
+ }
361
+ if (!child.isFile() || !isRolloutCandidateFileName(child.name) || visited.has(filePath)) continue;
362
+ visited.add(filePath);
363
+ if (diagnostics) diagnostics.discovered_files += 1;
364
+ const cached = inventory.files.get(filePath) || null;
365
+ const cachedCandidate = cached
366
+ ? isConservativeRangeCandidate(filePath, cached, candidateOptions)
367
+ : false;
368
+ const shouldStat = Boolean(
369
+ fullMetadataAudit ||
370
+ !cached ||
371
+ cachedCandidate ||
372
+ isRolloutPathInRequestedRange(filePath, options)
373
+ );
374
+ let stat = cached;
375
+ if (shouldStat) {
376
+ try {
377
+ if (diagnostics) diagnostics.stat_calls += 1;
378
+ stat = fs.statSync(filePath);
379
+ inventory.files.set(filePath, stat);
380
+ } catch {
381
+ inventory.files.delete(filePath);
382
+ continue;
383
+ }
384
+ } else if (diagnostics) {
385
+ diagnostics.metadata_cache_hits += 1;
386
+ }
387
+ const candidate = Boolean(
388
+ options.exhaustive ||
389
+ isConservativeRangeCandidate(filePath, stat, candidateOptions),
390
+ );
391
+ entries.push({ filePath, stat, candidate });
392
+ if (candidate && diagnostics) diagnostics.candidate_files += 1;
393
+ }
394
+ }
395
+
396
+ for (const filePath of inventory.files.keys()) {
397
+ if (!visited.has(filePath)) inventory.files.delete(filePath);
398
+ }
399
+ if (fullMetadataAudit) inventory.lastFullAuditAtMs = nowMs;
400
+
401
+ return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
402
+ }
403
+
404
+ async function listCodexSessionFiles(baseDir) {
405
+ const rolloutFiles = await listRolloutFiles(baseDir).catch(() => []);
406
+ const allJsonlFiles = listJsonlFiles(baseDir);
407
+ if (allJsonlFiles.length === 0) return rolloutFiles;
408
+ if (rolloutFiles.length === 0) return allJsonlFiles;
409
+
410
+ const merged = new Set(rolloutFiles);
411
+ for (const filePath of allJsonlFiles) merged.add(filePath);
412
+ return Array.from(merged).sort((a, b) => a.localeCompare(b));
413
+ }
414
+
415
+ // ---------------------------------------------------------------------------
416
+ // Token count extraction
417
+ // ---------------------------------------------------------------------------
418
+
419
+ function extractTokenCount(obj) {
420
+ const payload = obj?.payload;
421
+ if (!payload || obj?.type !== "event_msg") return null;
422
+ if (payload.type === "token_count") {
423
+ return { info: payload.info || null, timestamp: obj?.timestamp || null };
424
+ }
425
+ const msg = payload.msg;
426
+ if (msg && msg.type === "token_count") {
427
+ return { info: msg.info || null, timestamp: obj?.timestamp || null };
428
+ }
429
+ return null;
430
+ }
431
+
432
+ // ---------------------------------------------------------------------------
433
+ // Tool name helpers
434
+ // ---------------------------------------------------------------------------
435
+
436
+ function normalizeToolName(payload) {
437
+ const name = payload?.name || "";
438
+ const ns = payload?.namespace || null;
439
+ if (ns && typeof ns === "string" && ns.startsWith("mcp__")) return `${ns}${name}`;
440
+ return name || "unknown";
441
+ }
442
+
443
+ function extractSkillNameFromFunctionCall(payload, diagnostics = null) {
444
+ if (!payload || payload.name !== "exec_command") return null;
445
+ const args = safeJsonParse(payload.arguments || "{}", diagnostics) || {};
446
+ const cmd = String(args.cmd || "");
447
+ const match = cmd.match(/(?:^|\/)skills\/(?:\.system\/)?([^/\s]+)\/SKILL\.md\b/);
448
+ return match ? match[1] : null;
449
+ }
450
+
451
+ function formatToolDisplayName(name) {
452
+ if (typeof name !== "string" || !name.startsWith("mcp__")) return name;
453
+ const parts = name.split("__");
454
+ if (parts.length < 3) return name;
455
+ const server = String(parts[1] || "").replace(/^plugin_/, "").replace(/_/g, "-");
456
+ const tool = parts.slice(2).join("__") || name;
457
+ return server ? `${server}/${tool}` : tool;
458
+ }
459
+
460
+ // ---------------------------------------------------------------------------
461
+ // Usage normalization
462
+ // ---------------------------------------------------------------------------
463
+
464
+ function normalizeUsage(u) {
465
+ const out = {};
466
+ for (const k of [
467
+ "input_tokens",
468
+ "cached_input_tokens",
469
+ "cache_creation_input_tokens",
470
+ "output_tokens",
471
+ "reasoning_output_tokens",
472
+ "total_tokens",
473
+ ]) {
474
+ const n = Number(u?.[k] || 0);
475
+ out[k] = Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
476
+ }
477
+ // Codex reports input inclusive of cached_input_tokens; keep our schema
478
+ // convention: non-cached input and cached input tracked separately.
479
+ out.input_tokens = Math.max(0, out.input_tokens - out.cached_input_tokens);
480
+ out.total_tokens =
481
+ out.input_tokens +
482
+ out.cached_input_tokens +
483
+ out.cache_creation_input_tokens +
484
+ out.output_tokens;
485
+ return out;
486
+ }
487
+
488
+ function pickDelta(lastUsage, totalUsage, prevTotals) {
489
+ const state = createUsageDeltaState({ lastTotal: prevTotals });
490
+ const delta = consumeUsageDelta(state, lastUsage, totalUsage);
491
+ return delta ? normalizeUsage(delta) : null;
492
+ }
493
+
494
+ // ---------------------------------------------------------------------------
495
+ // Exec stats helpers (local to parser, not shared)
496
+ // ---------------------------------------------------------------------------
497
+
498
+ function durationBucket(ms) {
499
+ const n = Number(ms || 0);
500
+ if (!Number.isFinite(n) || n <= 0) return "unknown";
501
+ if (n < 1000) return "<1s";
502
+ if (n < 10_000) return "1-10s";
503
+ if (n < 60_000) return "10-60s";
504
+ if (n < 300_000) return "1-5m";
505
+ return ">5m";
506
+ }
507
+
508
+ function outputSizeBucket(lines, chars) {
509
+ const l = Number(lines || 0);
510
+ const c = Number(chars || 0);
511
+ if (!l && !c) return "quiet";
512
+ if (l <= 20 && c <= 2_000) return "small";
513
+ if (l <= 200 && c <= 20_000) return "medium";
514
+ if (l <= 1000 && c <= 100_000) return "large";
515
+ return "very_large";
516
+ }
517
+
518
+ function buildToolStatsEntry() {
519
+ return { calls: 0, totals: emptyTotals() };
520
+ }
521
+
522
+ function buildSkillStatsEntry(name) {
523
+ return { name, calls: 0, totals: emptyTotals() };
524
+ }
525
+
526
+ // ---------------------------------------------------------------------------
527
+ // Finalize helpers
528
+ // ---------------------------------------------------------------------------
529
+
530
+ function finalizeToolRows(map) {
531
+ return Array.from(map.values())
532
+ .map((row) => {
533
+ const rawName = row.raw_name || row.name;
534
+ return {
535
+ name: formatToolDisplayName(rawName),
536
+ raw_name: rawName,
537
+ calls: row.calls,
538
+ totals: row.totals,
539
+ };
540
+ })
541
+ .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
542
+ }
543
+
544
+ function finalizeSkillRows(map) {
545
+ return Array.from(map.values())
546
+ .map((row) => ({
547
+ name: row.name,
548
+ calls: row.calls,
549
+ totals: row.totals,
550
+ }))
551
+ .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
552
+ }
553
+
554
+ function finalizeExecRows(map) {
555
+ return Array.from(map.values())
556
+ .map((row) => ({
557
+ name: row.name,
558
+ calls: row.calls,
559
+ failures: row.failures,
560
+ duration_ms: row.duration_ms,
561
+ max_duration_ms: row.max_duration_ms,
562
+ output_chars: row.output_chars,
563
+ output_lines: row.output_lines,
564
+ totals: row.totals,
565
+ }))
566
+ .sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
567
+ }
568
+
569
+ function finalizeModelUsageRows(map) {
570
+ return Array.from(map.values())
571
+ .map((row) => ({
572
+ ...row,
573
+ selected_models: Array.from(row.selected_models).sort(),
574
+ reroute_reasons: Array.from(row.reroute_reasons).sort(),
575
+ model_attribution: row.rerouted_usage_events > 0 ? "effective" : "selected",
576
+ }))
577
+ .sort((a, b) => b.total_tokens - a.total_tokens || a.model.localeCompare(b.model));
578
+ }
579
+
580
+ // ---------------------------------------------------------------------------
581
+ // Main parser
582
+ // ---------------------------------------------------------------------------
583
+
584
+ async function parseCodexRolloutFile(filePath, {
585
+ from = null,
586
+ to = null,
587
+ timeZoneContext = null,
588
+ diagnostics = null,
589
+ seenTokenEvents = null,
590
+ startOffset = 0,
591
+ endOffset = null,
592
+ resumeState = null,
593
+ captureResumeState = false,
594
+ captureContentHash = false,
595
+ contentHashState = null,
596
+ sourceHandle = null,
597
+ collectBreakdowns = true,
598
+ collectModelUsage = false,
599
+ onObject = null,
600
+ onUsage = null,
601
+ } = {}) {
602
+ const filePaths = (Array.isArray(filePath) ? filePath : [filePath]).filter(Boolean);
603
+ const primaryFilePath = filePaths[0] || String(filePath || "");
604
+ const isResuming = Boolean(resumeState && filePaths.length === 1);
605
+ const initialOffset = Math.max(0, Number(startOffset) || 0);
606
+ const readProgress = {
607
+ endOffset: initialOffset,
608
+ lastByte: initialOffset > 0 ? 0x0a : null,
609
+ };
610
+ let contentHasher = null;
611
+ if (captureResumeState && captureContentHash && filePaths.length === 1) {
612
+ if (contentHashState && typeof contentHashState.copy === "function") {
613
+ try {
614
+ contentHasher = contentHashState.copy();
615
+ } catch {
616
+ contentHasher = null;
617
+ }
618
+ }
619
+ if (initialOffset > 0 && !contentHasher) {
620
+ const error = new Error("Codex rollout append is missing its prefix hash state");
621
+ error.code = CODEX_RESUME_INVALIDATED;
622
+ throw error;
623
+ }
624
+ if (!contentHasher) contentHasher = crypto.createHash("sha256");
625
+ }
626
+
627
+ let sessionId = isResuming ? resumeState.sessionId || null : null;
628
+ let cwd = isResuming ? resumeState.cwd || null : null;
629
+ let model = isResuming ? resumeState.model || null : null;
630
+ const modelAttributionState = createCodexModelAttributionState(
631
+ isResuming ? resumeState.modelAttributionState || { model } : {},
632
+ );
633
+ // Codex writes thread_settings_applied immediately before the turn_context of
634
+ // the turn it applies to, and token_count rows carry no turn id, so each one
635
+ // belongs to the most recent preceding tier record. Sessions that never emit
636
+ // the event (most CLI sessions) stay null and are billed at Standard.
637
+ let serviceTier = isResuming ? resumeState.serviceTier || null : null;
638
+ let provider = isResuming ? resumeState.provider || null : null;
639
+ let cliVersion = isResuming ? resumeState.cliVersion || null : null;
640
+ let forkedFromId = isResuming ? resumeState.forkedFromId || null : null;
641
+ let parentThreadId = isResuming ? resumeState.parentThreadId || null : null;
642
+ let threadSource = isResuming ? resumeState.threadSource || null : null;
643
+ let agentNickname = isResuming ? resumeState.agentNickname || null : null;
644
+ let agentRole = isResuming ? resumeState.agentRole || null : null;
645
+ let sessionLineageCaptured = isResuming
646
+ ? Boolean(resumeState.sessionLineageCaptured || resumeState.sessionId)
647
+ : false;
648
+
649
+ const usageDeltaState = createUsageDeltaState({
650
+ lastTotal: isResuming ? resumeState.prevTotals || null : null,
651
+ baselines: isResuming ? resumeState.tokenUsageBaselines || null : null,
652
+ });
653
+ let pendingToolNames = isResuming
654
+ ? Array.from(resumeState.pendingToolNames || [])
655
+ : [];
656
+ let pendingSkills = isResuming ? Array.from(resumeState.pendingSkills || []) : [];
657
+ let pendingExecDetails = isResuming
658
+ ? Array.from(resumeState.pendingExecDetails || [])
659
+ : [];
660
+ let lastTokenTimestamp = isResuming
661
+ ? String(resumeState.lastTokenTimestamp || "") || null
662
+ : null;
663
+ let lastTimestampEventSignatures = new Set(
664
+ isResuming ? resumeState.lastTimestampEventSignatures || [] : [],
665
+ );
666
+ let lastTimestampLastUsage = null;
667
+ let lastTimestampTotalUsage = null;
668
+ let hasUnmaterializedLastTimestampUsage = false;
669
+
670
+ const totals = emptyTotals();
671
+ const byTool = new Map(); // tool_name -> {name,calls,totals}
672
+ const bySkill = new Map(); // skill_name -> {name,calls,totals}
673
+ const byExecKind = new Map(); // kind -> stats
674
+ const byExecExit = new Map(); // status:exit -> stats
675
+ const byExecExecutable = new Map(); // executable -> stats
676
+ const byExecCommand = new Map(); // sanitized executable + subcommand -> stats
677
+ const byExecDuration = new Map(); // duration bucket -> stats
678
+ const byExecOutput = new Map(); // output size bucket -> stats
679
+ const byModel = new Map(); // effective model -> observed token usage
680
+
681
+ let turnCount = 0;
682
+
683
+ // Compaction records (issue #652, rules in codex-usage-record.js). A resume
684
+ // state from before #652 with prevTotals has seen a token_count.
685
+ const usageRecordState = createUsageRecordState(
686
+ isResuming
687
+ ? resumeState.usageRecordState || (resumeState.prevTotals ? { sawTokenCount: true } : null)
688
+ : null,
689
+ );
690
+ // Compactions counted in this session, by response_id: the files merged for
691
+ // a session can repeat a record (an archived copy, a split fragment), and a
692
+ // resumed parse must not count one from an earlier chunk again.
693
+ const compactionResponseIds = new Set(
694
+ isResuming && Array.isArray(resumeState.compactionResponseIds)
695
+ ? resumeState.compactionResponseIds
696
+ : [],
697
+ );
698
+
699
+ // Attribution as of now. A compaction record is counted later than it is
700
+ // read, so it carries the attribution captured when it was read.
701
+ function currentUsageAttribution() {
702
+ return {
703
+ model: currentCodexModel(modelAttributionState) || "unknown",
704
+ selectedModel: modelAttributionState.selectedModel || null,
705
+ rerouted: Boolean(modelAttributionState.rerouted),
706
+ rerouteReason: modelAttributionState.rerouteReason || null,
707
+ serviceTier,
708
+ };
709
+ }
710
+
711
+ function recordModelUsage(delta, rawRequestUsage, attribution = currentUsageAttribution()) {
712
+ if (!collectModelUsage || !delta || delta.total_tokens <= 0) return;
713
+ const effectiveModel = attribution.model;
714
+ let row = byModel.get(effectiveModel);
715
+ if (!row) {
716
+ row = {
717
+ model: effectiveModel,
718
+ ...emptyTotals(),
719
+ long_context_input_tokens: 0,
720
+ long_context_cached_input_tokens: 0,
721
+ long_context_cache_creation_input_tokens: 0,
722
+ long_context_output_tokens: 0,
723
+ long_context_reasoning_output_tokens: 0,
724
+ priority_input_tokens: 0,
725
+ priority_cached_input_tokens: 0,
726
+ priority_cache_creation_input_tokens: 0,
727
+ priority_output_tokens: 0,
728
+ priority_reasoning_output_tokens: 0,
729
+ priority_long_context_input_tokens: 0,
730
+ priority_long_context_cached_input_tokens: 0,
731
+ priority_long_context_cache_creation_input_tokens: 0,
732
+ priority_long_context_output_tokens: 0,
733
+ priority_long_context_reasoning_output_tokens: 0,
734
+ usage_events: 0,
735
+ rerouted_usage_events: 0,
736
+ long_context_usage_events: 0,
737
+ priority_usage_events: 0,
738
+ selected_models: new Set(),
739
+ reroute_reasons: new Set(),
740
+ };
741
+ byModel.set(effectiveModel, row);
742
+ }
743
+ const subsetKeys = Object.keys(row).filter((key) => /^(?:long_context_|priority_)|^(?:usage_events|rerouted_usage_events)$/.test(key));
744
+ const before = Object.fromEntries(subsetKeys.map((key) => [key, row[key]]));
745
+ addInto(row, delta);
746
+ row.usage_events += 1;
747
+ if (attribution.selectedModel) {
748
+ row.selected_models.add(attribution.selectedModel);
749
+ }
750
+ if (attribution.rerouted) {
751
+ row.rerouted_usage_events += 1;
752
+ if (attribution.rerouteReason) {
753
+ row.reroute_reasons.add(attribution.rerouteReason);
754
+ }
755
+ }
756
+ // OpenAI applies the long-context tier to the whole request when its raw
757
+ // (cache-inclusive) input exceeds 272K tokens. Preserve the exact subset
758
+ // so pricing can be recomputed later without rescanning private logs.
759
+ const isLongContext =
760
+ Number(rawRequestUsage?.input_tokens || 0) > OPENAI_LONG_CONTEXT_INPUT_THRESHOLD;
761
+ if (isLongContext) {
762
+ row.long_context_usage_events += 1;
763
+ row.long_context_input_tokens += delta.input_tokens;
764
+ row.long_context_cached_input_tokens += delta.cached_input_tokens;
765
+ row.long_context_cache_creation_input_tokens += delta.cache_creation_input_tokens;
766
+ row.long_context_output_tokens += delta.output_tokens;
767
+ row.long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
768
+ }
769
+ // Astra Fast bills at a flat multiple of whichever context tier the request
770
+ // was on, so the priority premium multiplies the long-context premium
771
+ // instead of replacing it. The two subsets alone cannot say how much usage
772
+ // was on BOTH, so record the intersection explicitly — otherwise a long
773
+ // Fast request prices at 3x Standard input where it should be 4x.
774
+ if (isPriorityServiceTier(attribution.serviceTier)) {
775
+ row.priority_usage_events += 1;
776
+ row.priority_input_tokens += delta.input_tokens;
777
+ row.priority_cached_input_tokens += delta.cached_input_tokens;
778
+ row.priority_cache_creation_input_tokens += delta.cache_creation_input_tokens;
779
+ row.priority_output_tokens += delta.output_tokens;
780
+ row.priority_reasoning_output_tokens += delta.reasoning_output_tokens;
781
+ if (isLongContext) {
782
+ row.priority_long_context_input_tokens += delta.input_tokens;
783
+ row.priority_long_context_cached_input_tokens += delta.cached_input_tokens;
784
+ row.priority_long_context_cache_creation_input_tokens +=
785
+ delta.cache_creation_input_tokens;
786
+ row.priority_long_context_output_tokens += delta.output_tokens;
787
+ row.priority_long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
788
+ }
789
+ }
790
+ return { ...delta, ...Object.fromEntries(subsetKeys.map((key) => [key, row[key] - before[key]])),
791
+ selected_models: attribution.selectedModel ? [attribution.selectedModel] : [],
792
+ reroute_reasons: attribution.rerouteReason ? [attribution.rerouteReason] : [],
793
+ model_attribution: attribution.rerouted ? "effective" : "selected" };
794
+ }
795
+
796
+ function ensureTool(name) {
797
+ if (!byTool.has(name)) {
798
+ byTool.set(name, { name, ...buildToolStatsEntry() });
799
+ }
800
+ return byTool.get(name);
801
+ }
802
+
803
+ function ensureExec(map, key) {
804
+ if (!map.has(key)) map.set(key, { name: key, ...buildExecStatsEntry() });
805
+ return map.get(key);
806
+ }
807
+
808
+ function ensureSkill(name) {
809
+ if (!bySkill.has(name)) bySkill.set(name, buildSkillStatsEntry(name));
810
+ return bySkill.get(name);
811
+ }
812
+
813
+ function getExecKeys(p) {
814
+ if (!p || typeof p !== "object") return;
815
+ const cmdArr = Array.isArray(p.command) ? p.command : null;
816
+ const cmd = cmdArr && cmdArr.length > 0 ? String(cmdArr[cmdArr.length - 1] || "") : "";
817
+ const kind = p.parsed_cmd?.[0]?.type && p.parsed_cmd[0].type !== "unknown"
818
+ ? p.parsed_cmd[0].type
819
+ : inferExecCommandKind(cmd);
820
+
821
+ const status = String(p.status || "unknown");
822
+ const exit = Number.isFinite(Number(p.exit_code)) ? Number(p.exit_code) : null;
823
+ const exitKey = `${status}:${exit === null ? "unknown" : exit}`;
824
+
825
+ const dur = p.duration ? Math.round((Number(p.duration.secs || 0) * 1000) + Number(p.duration.nanos || 0) / 1e6) : 0;
826
+ const output = String(p.aggregated_output || p.stdout || "");
827
+ const outputChars = output.length;
828
+ const outputLines = output ? output.split("\n").length : 0;
829
+ return {
830
+ kind,
831
+ exitKey,
832
+ executable: getExecutableName(cmd),
833
+ command: sanitizeCommandSignature(cmd),
834
+ duration: durationBucket(dur),
835
+ output: outputSizeBucket(outputLines, outputChars),
836
+ dur,
837
+ outputChars,
838
+ outputLines,
839
+ failed: status !== "completed" || exit !== 0,
840
+ };
841
+ }
842
+
843
+ function absorbExecStats(map, key, details) {
844
+ const row = ensureExec(map, key);
845
+ row.calls += 1;
846
+ row.duration_ms += details.dur;
847
+ row.max_duration_ms = Math.max(row.max_duration_ms, details.dur);
848
+ row.output_chars += details.outputChars;
849
+ row.output_lines += details.outputLines;
850
+ if (details.failed) row.failures += 1;
851
+ }
852
+
853
+ function absorbExecEnd(details) {
854
+ absorbExecStats(byExecKind, details.kind, details);
855
+ absorbExecStats(byExecExit, details.exitKey, details);
856
+ absorbExecStats(byExecExecutable, details.executable, details);
857
+ absorbExecStats(byExecCommand, details.command, details);
858
+ absorbExecStats(byExecDuration, details.duration, details);
859
+ absorbExecStats(byExecOutput, details.output, details);
860
+ }
861
+
862
+ function attributeTurn(delta) {
863
+ if (!delta || delta.total_tokens <= 0) {
864
+ for (const details of pendingExecDetails) absorbExecEnd(details);
865
+ pendingToolNames = [];
866
+ pendingSkills = [];
867
+ pendingExecDetails = [];
868
+ return;
869
+ }
870
+ turnCount += 1;
871
+
872
+ if (!collectBreakdowns) {
873
+ addInto(totals, delta);
874
+ pendingToolNames = [];
875
+ pendingSkills = [];
876
+ pendingExecDetails = [];
877
+ return;
878
+ }
879
+
880
+ const unique = [...new Set(pendingToolNames.filter(Boolean))];
881
+ const tools = unique.length > 0 ? unique : ["text_response"];
882
+ const share = 1 / tools.length;
883
+
884
+ for (const name of tools) {
885
+ const row = ensureTool(name);
886
+ row.calls += share;
887
+ addInto(row.totals, {
888
+ input_tokens: delta.input_tokens * share,
889
+ cached_input_tokens: delta.cached_input_tokens * share,
890
+ cache_creation_input_tokens: delta.cache_creation_input_tokens * share,
891
+ output_tokens: delta.output_tokens * share,
892
+ reasoning_output_tokens: delta.reasoning_output_tokens * share,
893
+ total_tokens: delta.total_tokens * share,
894
+ });
895
+ }
896
+
897
+ const uniqueSkills = [...new Set(pendingSkills.filter(Boolean))];
898
+ if (uniqueSkills.length > 0) {
899
+ const skillShare = 1 / uniqueSkills.length;
900
+ for (const name of uniqueSkills) {
901
+ const row = ensureSkill(name);
902
+ row.calls += skillShare;
903
+ addInto(row.totals, {
904
+ input_tokens: delta.input_tokens * skillShare,
905
+ cached_input_tokens: delta.cached_input_tokens * skillShare,
906
+ cache_creation_input_tokens: delta.cache_creation_input_tokens * skillShare,
907
+ output_tokens: delta.output_tokens * skillShare,
908
+ reasoning_output_tokens: delta.reasoning_output_tokens * skillShare,
909
+ total_tokens: delta.total_tokens * skillShare,
910
+ });
911
+ }
912
+ }
913
+
914
+ // Attribute exec_command_end rows to exec stats; note these are not a
915
+ // token source — we attach the same tool-shared delta to the command
916
+ // classifier so users can find high-cost command families.
917
+ const execToolShare = tools.includes("exec_command") ? (1 / tools.length) : 0;
918
+ const execDelta = execToolShare > 0 ? {
919
+ input_tokens: delta.input_tokens * execToolShare,
920
+ cached_input_tokens: delta.cached_input_tokens * execToolShare,
921
+ cache_creation_input_tokens: delta.cache_creation_input_tokens * execToolShare,
922
+ output_tokens: delta.output_tokens * execToolShare,
923
+ reasoning_output_tokens: delta.reasoning_output_tokens * execToolShare,
924
+ total_tokens: delta.total_tokens * execToolShare,
925
+ } : null;
926
+
927
+ if (execDelta && pendingExecDetails.length > 0) {
928
+ const perExecShare = 1 / pendingExecDetails.length;
929
+ for (const details of pendingExecDetails) {
930
+ const attributed = {
931
+ input_tokens: execDelta.input_tokens * perExecShare,
932
+ cached_input_tokens: execDelta.cached_input_tokens * perExecShare,
933
+ cache_creation_input_tokens: execDelta.cache_creation_input_tokens * perExecShare,
934
+ output_tokens: execDelta.output_tokens * perExecShare,
935
+ reasoning_output_tokens: execDelta.reasoning_output_tokens * perExecShare,
936
+ total_tokens: execDelta.total_tokens * perExecShare,
937
+ };
938
+
939
+ addInto(ensureExec(byExecKind, details.kind).totals, attributed);
940
+ addInto(ensureExec(byExecExit, details.exitKey).totals, attributed);
941
+ addInto(ensureExec(byExecExecutable, details.executable).totals, attributed);
942
+ addInto(ensureExec(byExecCommand, details.command).totals, attributed);
943
+ addInto(ensureExec(byExecDuration, details.duration).totals, attributed);
944
+ addInto(ensureExec(byExecOutput, details.output).totals, attributed);
945
+
946
+ absorbExecEnd(details);
947
+ }
948
+ } else {
949
+ // Still ingest exec end stats without token attribution so the drill-down works.
950
+ for (const details of pendingExecDetails) absorbExecEnd(details);
951
+ }
952
+
953
+ addInto(totals, delta);
954
+ pendingToolNames = [];
955
+ pendingSkills = [];
956
+ pendingExecDetails = [];
957
+ }
958
+
959
+ function countCompactionRecord(record) {
960
+ if (compactionResponseIds.has(record.responseId)) return;
961
+ compactionResponseIds.add(record.responseId);
962
+ if (!record.inRequestedRange) return;
963
+ const pricedDelta = recordModelUsage(record.delta, record.rawUsage, record.attribution);
964
+ if (typeof onUsage === "function") onUsage({ timestamp: record.timestamp,
965
+ delta: pricedDelta || record.delta, rawUsage: record.rawUsage, model: record.attribution.model, origin: "compaction" });
966
+ attributeTurn(record.delta);
967
+ }
968
+
969
+ function usageEventSignature(lastUsage, totalUsage) {
970
+ return JSON.stringify({ lastUsage, totalUsage });
971
+ }
972
+
973
+ function materializeLastTimestampSignatures() {
974
+ if (hasUnmaterializedLastTimestampUsage) {
975
+ lastTimestampEventSignatures.add(
976
+ usageEventSignature(lastTimestampLastUsage, lastTimestampTotalUsage),
977
+ );
978
+ hasUnmaterializedLastTimestampUsage = false;
979
+ }
980
+ return lastTimestampEventSignatures;
981
+ }
982
+
983
+ function noteTokenEvent(timestamp, lastUsage, totalUsage) {
984
+ if (!isResuming) {
985
+ if (timestamp !== lastTokenTimestamp) {
986
+ lastTokenTimestamp = timestamp;
987
+ lastTimestampEventSignatures = new Set();
988
+ hasUnmaterializedLastTimestampUsage = false;
989
+ } else {
990
+ materializeLastTimestampSignatures();
991
+ }
992
+ lastTimestampLastUsage = lastUsage;
993
+ lastTimestampTotalUsage = totalUsage;
994
+ hasUnmaterializedLastTimestampUsage = true;
995
+ return true;
996
+ }
997
+ if (isResuming && lastTokenTimestamp && timestamp < lastTokenTimestamp) {
998
+ const error = new Error("Codex rollout append is not timestamp-monotonic");
999
+ error.code = CODEX_RESUME_INVALIDATED;
1000
+ throw error;
1001
+ }
1002
+ if (!lastTokenTimestamp || timestamp > lastTokenTimestamp) {
1003
+ lastTokenTimestamp = timestamp;
1004
+ lastTimestampEventSignatures = new Set();
1005
+ lastTimestampLastUsage = lastUsage;
1006
+ lastTimestampTotalUsage = totalUsage;
1007
+ hasUnmaterializedLastTimestampUsage = true;
1008
+ return true;
1009
+ }
1010
+ if (timestamp === lastTokenTimestamp) {
1011
+ const signature = usageEventSignature(lastUsage, totalUsage);
1012
+ const signatures = materializeLastTimestampSignatures();
1013
+ if (signatures.has(signature)) return false;
1014
+ signatures.add(signature);
1015
+ }
1016
+ return true;
1017
+ }
1018
+
1019
+ for await (const obj of iterateCodexObjects(filePaths, diagnostics, {
1020
+ startOffset,
1021
+ endOffset,
1022
+ readProgress,
1023
+ contentHasher,
1024
+ sourceHandle,
1025
+ })) {
1026
+ const modelEvent = applyCodexModelEvent(modelAttributionState, obj);
1027
+ const attributedModel = currentCodexModel(modelAttributionState);
1028
+ model = attributedModel || model;
1029
+ // onObject runs AFTER the attribution state machine, and is handed the
1030
+ // model it just decided. A consumer sharing this pass (the delivery-signal
1031
+ // collector, which attributes edit turns) therefore sees exactly the model
1032
+ // recordModelUsage() bills the turn's tokens to. Letting it re-read
1033
+ // turn_context.payload.model itself would make codex-model-attribution.js
1034
+ // one of two authorities on the same question.
1035
+ if (typeof onObject === "function") onObject(obj, attributedModel);
1036
+ const usageRecord = extractTokenUsageRecord(obj);
1037
+ if (!usageRecord) noteLineAfterUsageRecord(usageRecordState, obj?.type === "compacted");
1038
+ const ts = typeof obj?.timestamp === "string" ? obj.timestamp : null;
1039
+ if (!ts) continue;
1040
+ const inRequestedRange = isTimestampInRequestedDayRange(ts, { from, to, timeZoneContext });
1041
+
1042
+ if (obj.type === "session_meta") {
1043
+ const p = obj.payload || {};
1044
+ // Forked rollouts replay historical session_meta rows after their own
1045
+ // child metadata. Only the first row describes this file's lineage.
1046
+ if (!sessionLineageCaptured) {
1047
+ sessionId = p.id || sessionId;
1048
+ cwd = p.cwd || cwd;
1049
+ cliVersion = p.cli_version || cliVersion;
1050
+ provider = p.model_provider || provider;
1051
+ forkedFromId = p.forked_from_id || null;
1052
+ parentThreadId = p.parent_thread_id || null;
1053
+ threadSource = p.thread_source || p.source || null;
1054
+ agentNickname = p.agent_nickname || null;
1055
+ agentRole = p.agent_role || null;
1056
+ sessionLineageCaptured = true;
1057
+ } else if (!sessionId) {
1058
+ sessionId = p.id || null;
1059
+ }
1060
+ }
1061
+
1062
+ const appliedServiceTier = readCodexServiceTier(obj);
1063
+ if (appliedServiceTier) {
1064
+ serviceTier = appliedServiceTier;
1065
+ continue;
1066
+ }
1067
+
1068
+ if (obj.type === "turn_context") {
1069
+ const p = obj.payload || {};
1070
+ if (typeof p.cwd === "string") cwd = p.cwd;
1071
+ continue;
1072
+ }
1073
+
1074
+ if (modelEvent?.type === "model_rerouted") continue;
1075
+
1076
+ if (collectBreakdowns && obj.type === "response_item" && obj.payload?.type === "function_call") {
1077
+ pendingToolNames.push(normalizeToolName(obj.payload));
1078
+ const skill = extractSkillNameFromFunctionCall(obj.payload, diagnostics);
1079
+ if (skill) pendingSkills.push(skill);
1080
+ continue;
1081
+ }
1082
+
1083
+ if (collectBreakdowns && obj.type === "event_msg" && obj.payload?.type === "exec_command_end") {
1084
+ const details = getExecKeys(obj.payload);
1085
+ if (details) pendingExecDetails.push(details);
1086
+ continue;
1087
+ }
1088
+
1089
+ if (usageRecord) {
1090
+ const delta = normalizeUsage(usageRecord.usage);
1091
+ noteUsageRecord(
1092
+ usageRecordState,
1093
+ // Without a response_id a compaction cannot be deduplicated.
1094
+ delta.total_tokens > 0 && usageRecord.responseId
1095
+ ? {
1096
+ responseId: usageRecord.responseId,
1097
+ timestamp: usageRecord.timestamp,
1098
+ inRequestedRange,
1099
+ rawUsage: usageRecord.usage,
1100
+ delta,
1101
+ attribution: currentUsageAttribution(),
1102
+ }
1103
+ : null,
1104
+ );
1105
+ continue;
1106
+ }
1107
+
1108
+ const tokenCount = extractTokenCount(obj);
1109
+ if (tokenCount) {
1110
+ const info = tokenCount.info;
1111
+ const lastUsage = info?.last_token_usage;
1112
+ const totalUsage = info?.total_token_usage;
1113
+ let compaction = null;
1114
+ if (info && typeof info === "object") {
1115
+ noteTokenCount(usageRecordState);
1116
+ compaction = takeCompactionOnTokenCount(
1117
+ usageRecordState,
1118
+ usageDeltaState.lastTotal,
1119
+ totalUsage,
1120
+ );
1121
+ }
1122
+ const rawDelta = consumeUsageDelta(usageDeltaState, lastUsage, totalUsage);
1123
+ const delta = rawDelta ? normalizeUsage(rawDelta) : null;
1124
+ // Timing consumers receive canonical, deduplicated usage below. A null
1125
+ // delta is a snapshot, not another model response.
1126
+ if (compaction) countCompactionRecord(compaction);
1127
+ const isNewResumeEvent = noteTokenEvent(ts, lastUsage, totalUsage);
1128
+ if (inRequestedRange) {
1129
+ const eventSessionId = sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath;
1130
+ const eventKey = `${eventSessionId}:${ts}:${usageEventSignature(lastUsage, totalUsage)}`;
1131
+ if (!isNewResumeEvent || (seenTokenEvents && seenTokenEvents.has(eventKey))) {
1132
+ if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: null, rawUsage: lastUsage, model: attributedModel });
1133
+ attributeTurn(null);
1134
+ } else {
1135
+ if (seenTokenEvents && delta?.total_tokens > 0) seenTokenEvents.add(eventKey);
1136
+ const pricedDelta = recordModelUsage(delta, lastUsage || rawDelta);
1137
+ if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: pricedDelta || delta, rawUsage: lastUsage || rawDelta, model: attributedModel });
1138
+ attributeTurn(delta);
1139
+ }
1140
+ } else {
1141
+ pendingToolNames = [];
1142
+ pendingSkills = [];
1143
+ pendingExecDetails = [];
1144
+ }
1145
+ continue;
1146
+ }
1147
+ }
1148
+
1149
+ const result = {
1150
+ sessionId: sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath,
1151
+ cwd,
1152
+ model: currentCodexModel(modelAttributionState) || model || provider,
1153
+ provider,
1154
+ version: cliVersion,
1155
+ forkedFromId,
1156
+ parentThreadId,
1157
+ threadSource,
1158
+ agentNickname,
1159
+ agentRole,
1160
+ filePath: primaryFilePath,
1161
+ turnCount,
1162
+ totals,
1163
+ modelUsage: finalizeModelUsageRows(byModel),
1164
+ toolBreakdown: {
1165
+ tool_rows: finalizeToolRows(byTool),
1166
+ },
1167
+ skillsBreakdown: {
1168
+ skill_rows: finalizeSkillRows(bySkill),
1169
+ },
1170
+ execCommandBreakdown: {
1171
+ byType: finalizeExecRows(byExecKind),
1172
+ byExit: finalizeExecRows(byExecExit),
1173
+ byExecutable: finalizeExecRows(byExecExecutable),
1174
+ byCommand: finalizeExecRows(byExecCommand),
1175
+ byDuration: finalizeExecRows(byExecDuration),
1176
+ byOutput: finalizeExecRows(byExecOutput),
1177
+ },
1178
+ };
1179
+ if (captureResumeState && filePaths.length === 1) {
1180
+ result.resumeState = {
1181
+ sessionId,
1182
+ cwd,
1183
+ model,
1184
+ modelAttributionState: snapshotCodexModelAttributionState(modelAttributionState),
1185
+ serviceTier,
1186
+ provider,
1187
+ cliVersion,
1188
+ forkedFromId,
1189
+ parentThreadId,
1190
+ threadSource,
1191
+ agentNickname,
1192
+ agentRole,
1193
+ sessionLineageCaptured,
1194
+ prevTotals: usageDeltaState.lastTotal,
1195
+ tokenUsageBaselines: snapshotUsageBaselines(usageDeltaState),
1196
+ pendingToolNames,
1197
+ pendingSkills,
1198
+ pendingExecDetails,
1199
+ lastTokenTimestamp,
1200
+ lastTimestampEventSignatures: Array.from(materializeLastTimestampSignatures()),
1201
+ usageRecordState: snapshotUsageRecordState(usageRecordState),
1202
+ compactionResponseIds: Array.from(compactionResponseIds),
1203
+ };
1204
+ result.endOffset = readProgress.endOffset;
1205
+ // Compressed snapshots are not append-only logs: endOffset stays at the
1206
+ // pre-decode cursor (often 0) and must never be marked appendable, or the
1207
+ // next growth would "resume" by parsing compressed bytes as JSONL.
1208
+ const hasCompressedArchive = filePaths.some(
1209
+ (candidate) => typeof candidate === "string" && /\.jsonl\.zstd?$/i.test(candidate),
1210
+ );
1211
+ result.appendable = Boolean(contentHasher) && !hasCompressedArchive && (
1212
+ readProgress.endOffset === 0 || readProgress.lastByte === 0x0a
1213
+ );
1214
+ result.contentHashState = contentHasher;
1215
+ }
1216
+ return result;
1217
+ }
1218
+
1219
+ const fatalUtf8Decoder = new TextDecoder("utf-8", { fatal: true });
1220
+
1221
+ function parseCodexLine(lineBuffer, diagnostics) {
1222
+ let content = lineBuffer;
1223
+ if (content.length > 0 && content[content.length - 1] === 0x0d) {
1224
+ content = content.subarray(0, content.length - 1);
1225
+ }
1226
+ if (content.length === 0) return null;
1227
+ try {
1228
+ if (diagnostics) diagnostics.json_parse_calls += 1;
1229
+ return JSON.parse(fatalUtf8Decoder.decode(content));
1230
+ } catch {
1231
+ return null;
1232
+ }
1233
+ }
1234
+
1235
+ async function* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
1236
+ startOffset = 0,
1237
+ endOffset = null,
1238
+ readProgress = null,
1239
+ contentHasher = null,
1240
+ sourceHandle = null,
1241
+ } = {}) {
1242
+ const start = Math.max(0, Number(startOffset) || 0);
1243
+ const requestedEnd = Number(endOffset);
1244
+ const hasBoundedEnd = endOffset !== null && endOffset !== undefined &&
1245
+ Number.isFinite(requestedEnd) && requestedEnd >= 0;
1246
+ if (hasBoundedEnd && requestedEnd <= start) {
1247
+ if (readProgress) readProgress.endOffset = start;
1248
+ return;
1249
+ }
1250
+
1251
+ if (diagnostics) diagnostics.opened_files += 1;
1252
+ const stream = sourceHandle && typeof sourceHandle.read === "function"
1253
+ ? null
1254
+ : fs.createReadStream(filePath, {
1255
+ start,
1256
+ ...(hasBoundedEnd ? { end: requestedEnd - 1 } : {}),
1257
+ });
1258
+ const chunks = stream || (async function* readStableHandleChunks() {
1259
+ const buffer = Buffer.allocUnsafe(64 * 1024);
1260
+ const limit = hasBoundedEnd ? requestedEnd : Number.MAX_SAFE_INTEGER;
1261
+ let offset = start;
1262
+ while (offset < limit) {
1263
+ const length = Math.min(buffer.length, limit - offset);
1264
+ const result = await sourceHandle.read(buffer, 0, length, offset);
1265
+ if (!result.bytesRead) break;
1266
+ offset += result.bytesRead;
1267
+ yield buffer.subarray(0, result.bytesRead);
1268
+ }
1269
+ })();
1270
+ const fragments = [];
1271
+ let fragmentsBytes = 0;
1272
+ let nextChunkOffset = start;
1273
+ let lineStartOffset = start;
1274
+ let lineIndex = 0;
1275
+
1276
+ try {
1277
+ for await (const value of chunks) {
1278
+ const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
1279
+ const chunkStart = nextChunkOffset;
1280
+ nextChunkOffset += chunk.length;
1281
+ if (contentHasher) contentHasher.update(chunk);
1282
+ if (readProgress && chunk.length > 0) {
1283
+ readProgress.lastByte = chunk[chunk.length - 1];
1284
+ }
1285
+ if (diagnostics) {
1286
+ diagnostics.bytes_read = Number(diagnostics.bytes_read || 0) + chunk.length;
1287
+ }
1288
+
1289
+ let cursor = 0;
1290
+ while (cursor < chunk.length) {
1291
+ const newline = chunk.indexOf(0x0a, cursor);
1292
+ if (newline < 0) {
1293
+ const tail = chunk.subarray(cursor);
1294
+ if (tail.length > 0) {
1295
+ // Stable-handle reads reuse their buffer, so retained fragments
1296
+ // must own their bytes before the next read overwrites it.
1297
+ fragments.push(Buffer.from(tail));
1298
+ fragmentsBytes += tail.length;
1299
+ }
1300
+ break;
1301
+ }
1302
+
1303
+ const tail = chunk.subarray(cursor, newline);
1304
+ const lineBuffer = fragments.length > 0
1305
+ ? Buffer.concat([...fragments, tail], fragmentsBytes + tail.length)
1306
+ : tail;
1307
+ fragments.length = 0;
1308
+ fragmentsBytes = 0;
1309
+
1310
+ const absoluteEnd = chunkStart + newline + 1;
1311
+ if (readProgress) readProgress.endOffset = absoluteEnd;
1312
+ lineStartOffset = absoluteEnd;
1313
+ const obj = parseCodexLine(lineBuffer, diagnostics);
1314
+ if (obj) {
1315
+ yield { obj, fileIndex, lineIndex };
1316
+ lineIndex += 1;
1317
+ }
1318
+ cursor = newline + 1;
1319
+ }
1320
+ }
1321
+
1322
+ if (fragmentsBytes > 0) {
1323
+ const lineBuffer = fragments.length === 1
1324
+ ? fragments[0]
1325
+ : Buffer.concat(fragments, fragmentsBytes);
1326
+ const obj = parseCodexLine(lineBuffer, diagnostics);
1327
+ if (obj) {
1328
+ if (readProgress) readProgress.endOffset = lineStartOffset + fragmentsBytes;
1329
+ yield { obj, fileIndex, lineIndex };
1330
+ }
1331
+ }
1332
+ } finally {
1333
+ if (readProgress) readProgress.endOffset = nextChunkOffset;
1334
+ stream?.destroy();
1335
+ if (diagnostics) diagnostics.parsed_files += 1;
1336
+ }
1337
+ }
1338
+
1339
+ async function* readCodexObjectsFull(filePath, diagnostics, fileIndex, {
1340
+ startOffset = 0,
1341
+ endOffset = null,
1342
+ readProgress = null,
1343
+ contentHasher = null,
1344
+ } = {}) {
1345
+ // Compressed archives are immutable snapshots, not append-only logs: byte
1346
+ // offsets are meaningless inside them, so decode the whole text and split
1347
+ // into lines. `readProgress` stays null so callers never persist a fake
1348
+ // byte cursor for them.
1349
+ if (typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath)) {
1350
+ const text = await readCodexRolloutText(filePath);
1351
+ let lineIndex = 0;
1352
+ const lines = text.split("\n");
1353
+ for (let index = 0; index < lines.length; index += 1) {
1354
+ const isLast = index + 1 >= lines.length;
1355
+ if (isLast && lines[index] === "") continue;
1356
+ const lineBuffer = Buffer.from(lines[index], "utf8");
1357
+ const obj = parseCodexLine(lineBuffer, diagnostics);
1358
+ if (obj) {
1359
+ yield { obj, fileIndex, lineIndex };
1360
+ lineIndex += 1;
1361
+ }
1362
+ }
1363
+ if (diagnostics) diagnostics.parsed_files += 1;
1364
+ return;
1365
+ }
1366
+ yield* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
1367
+ startOffset,
1368
+ endOffset,
1369
+ readProgress,
1370
+ contentHasher,
1371
+ });
1372
+ }
1373
+
1374
+ async function* readCodexObjects(filePath, diagnostics, fileIndex, options = {}) {
1375
+ const start = Math.max(0, Number(options.startOffset) || 0);
1376
+ const reader = start > 0 || options.sourceHandle
1377
+ ? readCodexObjectsIncremental
1378
+ : readCodexObjectsFull;
1379
+ yield* reader(filePath, diagnostics, fileIndex, options);
1380
+ }
1381
+
1382
+ function isRecoverableGroupedReadError(error) {
1383
+ return ["ENOENT", "EACCES", "EPERM", "EISDIR", "EIO", "ESTALE"].includes(error?.code);
1384
+ }
1385
+
1386
+ async function* iterateCodexObjects(filePaths, diagnostics, options = {}) {
1387
+ if (filePaths.length <= 1) {
1388
+ if (!filePaths[0]) return;
1389
+ for await (const record of readCodexObjects(filePaths[0], diagnostics, 0, options)) {
1390
+ yield record.obj;
1391
+ }
1392
+ return;
1393
+ }
1394
+
1395
+ // A live rollout can overlap its archived copy or be split into prefix/suffix
1396
+ // fragments. Merge only duplicate-session groups, order by event time, and
1397
+ // suppress byte-semantically identical records from different files so the
1398
+ // cumulative baseline is established exactly once.
1399
+ const records = [];
1400
+ let scannedFiles = 0;
1401
+ for (let fileIndex = 0; fileIndex < filePaths.length; fileIndex += 1) {
1402
+ const fileRecords = [];
1403
+ try {
1404
+ for await (const record of readCodexObjects(filePaths[fileIndex], diagnostics, fileIndex)) {
1405
+ fileRecords.push(record);
1406
+ }
1407
+ for (const record of fileRecords) records.push(record);
1408
+ scannedFiles += 1;
1409
+ } catch (error) {
1410
+ if (!isRecoverableGroupedReadError(error)) throw error;
1411
+ }
1412
+ }
1413
+ if (!scannedFiles) {
1414
+ const error = new Error("all grouped Codex rollout files failed during read");
1415
+ error.code = "EIO";
1416
+ throw error;
1417
+ }
1418
+ records.sort((a, b) => {
1419
+ const aTimestamp = typeof a.obj?.timestamp === "string" ? a.obj.timestamp : "";
1420
+ const bTimestamp = typeof b.obj?.timestamp === "string" ? b.obj.timestamp : "";
1421
+ return aTimestamp.localeCompare(bTimestamp) || a.fileIndex - b.fileIndex || a.lineIndex - b.lineIndex;
1422
+ });
1423
+ const firstFileByRecord = new Map();
1424
+ for (const record of records) {
1425
+ const semanticKey = JSON.stringify(record.obj);
1426
+ const firstFile = firstFileByRecord.get(semanticKey);
1427
+ if (firstFile !== undefined && firstFile !== record.fileIndex) continue;
1428
+ if (firstFile === undefined) firstFileByRecord.set(semanticKey, record.fileIndex);
1429
+ yield record.obj;
1430
+ }
1431
+ }
1432
+
1433
+ function rolloutSessionIdFromPath(filePath) {
1434
+ const match = path.basename(String(filePath || "")).match(
1435
+ /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl(?:\.zstd?)?$/i,
1436
+ );
1437
+ return match ? match[1] : null;
1438
+ }
1439
+
1440
+ function isCodexResumeInvalidated(error) {
1441
+ return error?.code === CODEX_RESUME_INVALIDATED;
1442
+ }
1443
+
1444
+ module.exports = {
1445
+ parseCodexRolloutFile,
1446
+ extractTokenCount,
1447
+ extractSkillNameFromFunctionCall,
1448
+ formatToolDisplayName,
1449
+ normalizeToolName,
1450
+ pickDelta,
1451
+ normalizeUsage,
1452
+ totalsReset,
1453
+ listJsonlFiles,
1454
+ listCodexSessionFiles,
1455
+ discoverCodexSessionFiles,
1456
+ rolloutPathDay,
1457
+ isConservativeRangeCandidate,
1458
+ safeJsonParse,
1459
+ dayKeyToIsoBounds,
1460
+ formatPartsDayKey,
1461
+ getZonedParts,
1462
+ timestampDayKey,
1463
+ isTimestampInRequestedDayRange,
1464
+ rolloutSessionIdFromPath,
1465
+ isCodexResumeInvalidated,
1466
+ isRecoverableGroupedReadError,
1467
+ finalizeToolRows,
1468
+ finalizeSkillRows,
1469
+ finalizeExecRows,
1470
+ buildSkillStatsEntry,
1471
+ };