tokenusage-cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -19
- package/LICENSE +22 -22
- package/README.md +94 -94
- package/THIRD_PARTY_NOTICES.md +140 -140
- package/bin/tokenusage.js +69 -69
- package/package.json +59 -59
- package/src/cli.js +146 -134
- package/src/commands/device-login.js +205 -205
- package/src/commands/diagnostics.js +38 -38
- package/src/commands/doctor.js +110 -110
- package/src/commands/init.js +2484 -2436
- package/src/commands/pause.js +77 -0
- package/src/commands/sessions.js +29 -29
- package/src/commands/status.js +1758 -1747
- package/src/commands/sync.js +8048 -7934
- package/src/commands/uninstall.js +322 -312
- package/src/lib/account-usage-dedup.js +267 -267
- package/src/lib/antigravity-paths.js +48 -48
- package/src/lib/ark-agent-plan-limits.js +438 -438
- package/src/lib/ark-coding-plan-limits.js +537 -537
- package/src/lib/async-queue-reader.js +138 -138
- package/src/lib/atomic-json.js +27 -27
- package/src/lib/bearer-token.js +11 -11
- package/src/lib/categorizer-utils.js +232 -232
- package/src/lib/claude-categorizer.js +1343 -1343
- package/src/lib/claude-config.js +279 -279
- package/src/lib/claude-user-identity.js +56 -56
- package/src/lib/cli-ui.js +179 -179
- package/src/lib/cloud-account.js +541 -541
- package/src/lib/cloud-device-token.js +141 -141
- package/src/lib/cloud-sync-prefs.js +14 -14
- package/src/lib/cloud-usage-reset.js +115 -115
- package/src/lib/codex-config.js +595 -595
- package/src/lib/codex-context-breakdown.js +806 -806
- package/src/lib/codex-model-attribution.js +128 -128
- package/src/lib/codex-rollout-parser.js +1471 -1471
- package/src/lib/codex-service-tier.js +49 -49
- package/src/lib/codex-token-refresh.js +140 -140
- package/src/lib/codex-token-usage.js +195 -195
- package/src/lib/codex-usage-record.js +195 -195
- package/src/lib/collection-pause.js +77 -0
- package/src/lib/command-runner.js +313 -313
- package/src/lib/commandcode-limits.js +317 -317
- package/src/lib/context-health.js +124 -124
- package/src/lib/cursor-config.js +605 -605
- package/src/lib/cursor-store.js +1918 -1918
- package/src/lib/debug-flags.js +7 -7
- package/src/lib/devin-limits.js +249 -249
- package/src/lib/devin-usage.js +212 -212
- package/src/lib/diagnostics.js +383 -383
- package/src/lib/doctor.js +385 -385
- package/src/lib/dumate.js +95 -95
- package/src/lib/fs.js +455 -455
- package/src/lib/function-url.js +14 -14
- package/src/lib/gemini-config.js +291 -291
- package/src/lib/git-outcomes.js +243 -243
- package/src/lib/grok-context-breakdown.js +890 -890
- package/src/lib/grok-hook.js +360 -360
- package/src/lib/grok-limits.js +865 -865
- package/src/lib/grok-usage.js +98 -98
- package/src/lib/heatmap-compact.js +114 -114
- package/src/lib/ingest-sanitize.js +41 -41
- package/src/lib/init-flow.js +42 -42
- package/src/lib/install-resolver.js +170 -170
- package/src/lib/jsonl-lines.js +134 -134
- package/src/lib/kimi-model-repair.js +173 -173
- package/src/lib/kimi-profile.js +61 -61
- package/src/lib/kimi-session.js +216 -216
- package/src/lib/local-api.js +3588 -3588
- package/src/lib/machine-id.js +238 -238
- package/src/lib/multi-install-parser.js +113 -113
- package/src/lib/notify-command.js +38 -38
- package/src/lib/omp-hook.js +589 -589
- package/src/lib/openclaw-cli.js +104 -104
- package/src/lib/openclaw-hook.js +375 -375
- package/src/lib/openclaw-session-plugin.js +560 -560
- package/src/lib/opencode-config.js +121 -121
- package/src/lib/opencode-go-limits.js +807 -807
- package/src/lib/outcomes-engine.js +253 -253
- package/src/lib/passive-mode.js +219 -219
- package/src/lib/pet-packages.js +722 -722
- package/src/lib/pricing/curated-overrides.json +822 -822
- package/src/lib/pricing/index.js +379 -379
- package/src/lib/pricing/litellm-fetcher.js +172 -172
- package/src/lib/pricing/matcher.js +456 -456
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/progress.js +76 -76
- package/src/lib/project-usage-purge.js +117 -117
- package/src/lib/prompt.js +24 -24
- package/src/lib/provider-status.js +104 -104
- package/src/lib/proxy-env.js +489 -489
- package/src/lib/proxy-settings.js +163 -163
- package/src/lib/qoder-cn-cli-limits.js +119 -119
- package/src/lib/qoder-limits.js +880 -880
- package/src/lib/quiet-warnings.js +26 -26
- package/src/lib/rollout.js +26005 -25966
- package/src/lib/runtime-config.js +9 -4
- package/src/lib/scan-roots.js +277 -277
- package/src/lib/scheduled-task.js +145 -0
- package/src/lib/sentry.js +231 -231
- package/src/lib/session-analytics.js +2972 -2972
- package/src/lib/session-performance.js +237 -237
- package/src/lib/session-upload.js +379 -379
- package/src/lib/skill-usage.js +267 -267
- package/src/lib/skills-manager.js +1591 -1591
- package/src/lib/source-metadata.js +48 -48
- package/src/lib/sqlite-reader.js +215 -215
- package/src/lib/star-cta.js +73 -73
- package/src/lib/subscription-manager.js +293 -293
- package/src/lib/subscriptions.js +508 -508
- package/src/lib/sync-report.js +138 -0
- package/src/lib/telemetry.js +193 -193
- package/src/lib/tracker-paths.js +16 -16
- package/src/lib/trae-cn-config.js +548 -548
- package/src/lib/trae-db.js +127 -127
- package/src/lib/trae-snapshot.js +230 -230
- package/src/lib/trae-sqlite.js +143 -143
- package/src/lib/trae-usage.js +121 -121
- package/src/lib/upload-queue.js +134 -134
- package/src/lib/upload-throttle.js +171 -171
- package/src/lib/usage-limits.js +4552 -4552
- package/src/lib/wrapped-aggregator.js +236 -236
- package/src/lib/wsl-probe.js +281 -281
- package/src/lib/zcode-limits.js +1172 -1172
- package/src/lib/zcode-reset-credits.js +66 -66
- package/test/audit-fixes.test.js +200 -0
- package/test/cli-smoke.test.js +53 -53
- package/test/collection-controls.test.js +181 -0
- package/test/ingest-sanitize.test.js +162 -162
- package/test/quiet-warnings.test.js +53 -53
- package/test/session-upload.test.js +427 -427
|
@@ -1,1471 +1,1471 @@
|
|
|
1
|
-
// Codex rollout JSONL parser — extracted from codex-context-breakdown.js.
|
|
2
|
-
//
|
|
3
|
-
// Handles file discovery and per-file parsing. Does NOT hold any aggregation
|
|
4
|
-
// state; callers (computeCodexContextBreakdown) own the merge step.
|
|
5
|
-
|
|
6
|
-
"use strict";
|
|
7
|
-
|
|
8
|
-
const crypto = require("node:crypto");
|
|
9
|
-
const fs = require("node:fs");
|
|
10
|
-
const path = require("node:path");
|
|
11
|
-
|
|
12
|
-
const { listRolloutFiles } = require("./rollout");
|
|
13
|
-
const {
|
|
14
|
-
consumeUsageDelta,
|
|
15
|
-
createUsageDeltaState,
|
|
16
|
-
snapshotUsageBaselines,
|
|
17
|
-
totalsReset,
|
|
18
|
-
} = require("./codex-token-usage");
|
|
19
|
-
const {
|
|
20
|
-
emptyTotals,
|
|
21
|
-
addInto,
|
|
22
|
-
inferExecCommandKind,
|
|
23
|
-
sanitizeCommandSignature,
|
|
24
|
-
getExecutableName,
|
|
25
|
-
buildExecStatsEntry,
|
|
26
|
-
} = require("./categorizer-utils");
|
|
27
|
-
const {
|
|
28
|
-
applyCodexModelEvent,
|
|
29
|
-
createCodexModelAttributionState,
|
|
30
|
-
currentCodexModel,
|
|
31
|
-
snapshotCodexModelAttributionState,
|
|
32
|
-
} = require("./codex-model-attribution");
|
|
33
|
-
const { readCodexServiceTier, isPriorityServiceTier } = require("./codex-service-tier");
|
|
34
|
-
const {
|
|
35
|
-
createUsageRecordState,
|
|
36
|
-
extractTokenUsageRecord,
|
|
37
|
-
noteLineAfterUsageRecord,
|
|
38
|
-
noteTokenCount,
|
|
39
|
-
noteUsageRecord,
|
|
40
|
-
snapshotUsageRecordState,
|
|
41
|
-
takeCompactionOnTokenCount,
|
|
42
|
-
} = require("./codex-usage-record");
|
|
43
|
-
|
|
44
|
-
const DISCOVERY_FULL_AUDIT_INTERVAL_MS = 60 * 1000;
|
|
45
|
-
const CODEX_ROLLOUT_TEXT_MAX_BYTES = 256 * 1024 * 1024;
|
|
46
|
-
|
|
47
|
-
async function readCodexRolloutText(filePath, { maxOutputBytes = CODEX_ROLLOUT_TEXT_MAX_BYTES } = {}) {
|
|
48
|
-
if (typeof filePath !== "string" || !/\.jsonl\.zstd?$/i.test(filePath)) {
|
|
49
|
-
return fs.readFile(filePath, "utf8");
|
|
50
|
-
}
|
|
51
|
-
// The snapshot decoder lives in rollout.js (reused by the sync parser);
|
|
52
|
-
// require lazily so this module stays loadable without pulling the whole
|
|
53
|
-
// rollout graph at import time.
|
|
54
|
-
const rollout = require("./rollout");
|
|
55
|
-
const snapshot = await rollout.readCodexRolloutSnapshot(filePath, { maxOutputBytes });
|
|
56
|
-
if (!snapshot) {
|
|
57
|
-
// Treat undecodable / vanished archives as recoverable I/O so grouped
|
|
58
|
-
// session reads (live + archive) can still use a readable sibling copy.
|
|
59
|
-
const error = new Error(`unreadable Codex archive ${filePath}`);
|
|
60
|
-
error.code = "EIO";
|
|
61
|
-
throw error;
|
|
62
|
-
}
|
|
63
|
-
// Snapshot lines keep their "\n" terminators; the tail element may be an
|
|
64
|
-
// unterminated final line — join preserves exactly that.
|
|
65
|
-
return snapshot.lines.join("\n");
|
|
66
|
-
}
|
|
67
|
-
const MAX_DISCOVERY_INVENTORIES = 32;
|
|
68
|
-
const DISCOVERY_INVENTORIES = new Map();
|
|
69
|
-
const ZONED_DAY_FORMATTERS = new Map();
|
|
70
|
-
const CODEX_RESUME_INVALIDATED = "TOKENUSAGE_CODEX_RESUME_INVALIDATED";
|
|
71
|
-
// Match src/lib/pricing/index.js. Classify individual requests using raw
|
|
72
|
-
// input (including cache), not the accumulated session or day totals.
|
|
73
|
-
const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272_000;
|
|
74
|
-
|
|
75
|
-
// ---------------------------------------------------------------------------
|
|
76
|
-
// Timezone helpers
|
|
77
|
-
// ---------------------------------------------------------------------------
|
|
78
|
-
|
|
79
|
-
function dayKeyToIsoBounds(from, to) {
|
|
80
|
-
if (!from && !to) return { fromIso: null, toIso: null };
|
|
81
|
-
const fromDate = from ? new Date(`${from}T00:00:00Z`) : null;
|
|
82
|
-
const toDate = to ? new Date(`${to}T23:59:59Z`) : null;
|
|
83
|
-
if (fromDate && Number.isFinite(fromDate.getTime())) fromDate.setUTCHours(fromDate.getUTCHours() - 14);
|
|
84
|
-
if (toDate && Number.isFinite(toDate.getTime())) toDate.setUTCHours(toDate.getUTCHours() + 14);
|
|
85
|
-
return {
|
|
86
|
-
fromIso: fromDate ? fromDate.toISOString() : null,
|
|
87
|
-
toIso: toDate ? toDate.toISOString() : null,
|
|
88
|
-
};
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function formatPartsDayKey(parts) {
|
|
92
|
-
if (!parts) return "";
|
|
93
|
-
const values = {};
|
|
94
|
-
for (const part of parts) {
|
|
95
|
-
if (part.type !== "literal") values[part.type] = part.value;
|
|
96
|
-
}
|
|
97
|
-
if (!values.year || !values.month || !values.day) return "";
|
|
98
|
-
return `${values.year}-${values.month}-${values.day}`;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
function getZonedParts(date, timeZoneContext = {}) {
|
|
102
|
-
const { timeZone, offsetMinutes } = timeZoneContext || {};
|
|
103
|
-
const dt = date instanceof Date ? date : new Date(date);
|
|
104
|
-
if (!Number.isFinite(dt.getTime())) return null;
|
|
105
|
-
|
|
106
|
-
if (timeZone && typeof Intl !== "undefined" && Intl.DateTimeFormat) {
|
|
107
|
-
try {
|
|
108
|
-
let formatter = ZONED_DAY_FORMATTERS.get(timeZone);
|
|
109
|
-
if (!formatter) {
|
|
110
|
-
formatter = new Intl.DateTimeFormat("en-CA", {
|
|
111
|
-
timeZone,
|
|
112
|
-
year: "numeric",
|
|
113
|
-
month: "2-digit",
|
|
114
|
-
day: "2-digit",
|
|
115
|
-
hourCycle: "h23",
|
|
116
|
-
});
|
|
117
|
-
ZONED_DAY_FORMATTERS.set(timeZone, formatter);
|
|
118
|
-
}
|
|
119
|
-
return formatter.formatToParts(dt);
|
|
120
|
-
} catch {
|
|
121
|
-
// Fall through to offset handling.
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
if (Number.isFinite(offsetMinutes)) {
|
|
126
|
-
const shifted = new Date(dt.getTime() - Number(offsetMinutes) * 60_000);
|
|
127
|
-
return [
|
|
128
|
-
{ type: "year", value: String(shifted.getUTCFullYear()).padStart(4, "0") },
|
|
129
|
-
{ type: "month", value: String(shifted.getUTCMonth() + 1).padStart(2, "0") },
|
|
130
|
-
{ type: "day", value: String(shifted.getUTCDate()).padStart(2, "0") },
|
|
131
|
-
];
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
return null;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
function timestampDayKey(timestamp, timeZoneContext) {
|
|
138
|
-
const ts = typeof timestamp === "string" ? timestamp : "";
|
|
139
|
-
if (!ts) return "";
|
|
140
|
-
const parts = getZonedParts(ts, timeZoneContext);
|
|
141
|
-
const zoned = formatPartsDayKey(parts);
|
|
142
|
-
if (zoned) return zoned;
|
|
143
|
-
return ts.slice(0, 10);
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
function isTimestampInRequestedDayRange(timestamp, { from, to, timeZoneContext } = {}) {
|
|
147
|
-
if (!from && !to) return true;
|
|
148
|
-
const day = timestampDayKey(timestamp, timeZoneContext);
|
|
149
|
-
if (!day) return false;
|
|
150
|
-
if (from && day < from) return false;
|
|
151
|
-
if (to && day > to) return false;
|
|
152
|
-
return true;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// ---------------------------------------------------------------------------
|
|
156
|
-
// File discovery
|
|
157
|
-
// ---------------------------------------------------------------------------
|
|
158
|
-
|
|
159
|
-
function safeJsonParse(str, diagnostics = null) {
|
|
160
|
-
try {
|
|
161
|
-
if (diagnostics) diagnostics.json_parse_calls += 1;
|
|
162
|
-
return JSON.parse(str);
|
|
163
|
-
} catch {
|
|
164
|
-
return null;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
function isRolloutCandidateFileName(name) {
|
|
169
|
-
if (typeof name !== "string") return false;
|
|
170
|
-
return name.endsWith(".jsonl") || /^rollout-.*\.jsonl\.zstd?$/.test(name);
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
function listJsonlFiles(rootDir) {
|
|
174
|
-
const out = [];
|
|
175
|
-
const stack = [rootDir];
|
|
176
|
-
|
|
177
|
-
while (stack.length > 0) {
|
|
178
|
-
const dir = stack.pop();
|
|
179
|
-
let entries;
|
|
180
|
-
try {
|
|
181
|
-
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
182
|
-
} catch {
|
|
183
|
-
continue;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
for (const entry of entries) {
|
|
187
|
-
const filePath = path.join(dir, entry.name);
|
|
188
|
-
if (entry.isDirectory()) {
|
|
189
|
-
stack.push(filePath);
|
|
190
|
-
} else if (entry.isFile() && isRolloutCandidateFileName(entry.name)) {
|
|
191
|
-
out.push(filePath);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
out.sort((a, b) => a.localeCompare(b));
|
|
197
|
-
return out;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
function rolloutPathDay(filePath) {
|
|
201
|
-
const normalized = String(filePath || "").replaceAll("\\", "/");
|
|
202
|
-
const fileMatch = path.basename(normalized).match(/(?:^|rollout-)(\d{4})-(\d{2})-(\d{2})/);
|
|
203
|
-
if (fileMatch) return `${fileMatch[1]}-${fileMatch[2]}-${fileMatch[3]}`;
|
|
204
|
-
const directoryMatch = normalized.match(/.*\/(\d{4})\/(\d{2})\/(\d{2})(?:\/|$)/);
|
|
205
|
-
return directoryMatch
|
|
206
|
-
? `${directoryMatch[1]}-${directoryMatch[2]}-${directoryMatch[3]}`
|
|
207
|
-
: null;
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
function nextDayKey(dayKey) {
|
|
211
|
-
if (!dayKey) return null;
|
|
212
|
-
const date = new Date(`${dayKey}T00:00:00.000Z`);
|
|
213
|
-
if (!Number.isFinite(date.getTime())) return null;
|
|
214
|
-
date.setUTCDate(date.getUTCDate() + 1);
|
|
215
|
-
return date.toISOString().slice(0, 10);
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
function metadataRangeBounds(from, to, timeZoneContext) {
|
|
219
|
-
const offsetMinutes = timeZoneContext?.offsetMinutes;
|
|
220
|
-
if (Number.isFinite(offsetMinutes)) {
|
|
221
|
-
const dstGuardMs = 2 * 60 * 60 * 1000;
|
|
222
|
-
const fromUtc = from ? Date.parse(`${from}T00:00:00.000Z`) : null;
|
|
223
|
-
const afterTo = to ? nextDayKey(to) : null;
|
|
224
|
-
const toUtcExclusive = afterTo ? Date.parse(`${afterTo}T00:00:00.000Z`) : null;
|
|
225
|
-
return {
|
|
226
|
-
fromMs: Number.isFinite(fromUtc)
|
|
227
|
-
? fromUtc + offsetMinutes * 60_000 - dstGuardMs
|
|
228
|
-
: null,
|
|
229
|
-
toMs: Number.isFinite(toUtcExclusive)
|
|
230
|
-
? toUtcExclusive + offsetMinutes * 60_000 + dstGuardMs
|
|
231
|
-
: null,
|
|
232
|
-
};
|
|
233
|
-
}
|
|
234
|
-
const { fromIso, toIso } = dayKeyToIsoBounds(from, to);
|
|
235
|
-
return {
|
|
236
|
-
fromMs: fromIso ? Date.parse(fromIso) : null,
|
|
237
|
-
toMs: toIso ? Date.parse(toIso) : null,
|
|
238
|
-
};
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
function isConservativeRangeCandidate(filePath, stat, {
|
|
242
|
-
from = null,
|
|
243
|
-
to = null,
|
|
244
|
-
timeZoneContext = null,
|
|
245
|
-
metadataBounds = null,
|
|
246
|
-
} = {}) {
|
|
247
|
-
if (!from && !to) return true;
|
|
248
|
-
// With no lower bound, a later-path rewrite can still contain an event at or
|
|
249
|
-
// before `to`. There is no metadata-only exclusion that can prove otherwise.
|
|
250
|
-
if (!from) return true;
|
|
251
|
-
const pathDay = rolloutPathDay(filePath);
|
|
252
|
-
if (!pathDay) return true;
|
|
253
|
-
|
|
254
|
-
const pathOverlaps = (!from || pathDay >= from) && (!to || pathDay <= to);
|
|
255
|
-
if (pathOverlaps) return true;
|
|
256
|
-
|
|
257
|
-
// Old sessions can be appended or rewritten. A post-range mtime or ctime is
|
|
258
|
-
// therefore a reason to parse, never a reason to exclude. ctime also catches
|
|
259
|
-
// cold-process inode replacement/truncation when a writer preserves mtime.
|
|
260
|
-
// Future-dated decoys are kept conservatively; correctness wins over an open.
|
|
261
|
-
const bounds = metadataBounds || metadataRangeBounds(from, to, timeZoneContext);
|
|
262
|
-
const fromMs = bounds.fromMs;
|
|
263
|
-
const toMs = bounds.toMs;
|
|
264
|
-
const mtimeMs = Number(stat?.mtimeMs);
|
|
265
|
-
const ctimeMs = Number(stat?.ctimeMs);
|
|
266
|
-
const changedAfterStart = (value) => (
|
|
267
|
-
Number.isFinite(value) && (!Number.isFinite(fromMs) || value >= fromMs)
|
|
268
|
-
);
|
|
269
|
-
const changedInsideRange = (value) => (
|
|
270
|
-
changedAfterStart(value) && (!Number.isFinite(toMs) || value <= toMs)
|
|
271
|
-
);
|
|
272
|
-
if (pathDay < from) {
|
|
273
|
-
return changedAfterStart(mtimeMs) || changedAfterStart(ctimeMs);
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
// A path one day ahead can still overlap the requested browser day because
|
|
277
|
-
// rollout folders follow the writer's local date. Its current mtime may be
|
|
278
|
-
// later if the same session kept growing, so metadata alone cannot exclude it.
|
|
279
|
-
if (to && pathDay === nextDayKey(to)) return true;
|
|
280
|
-
|
|
281
|
-
// A host-local path day can be later than the requested browser day when
|
|
282
|
-
// their timezones differ. Keep it only when filesystem metadata maps back
|
|
283
|
-
// into the requested day range; do not pull every genuinely future session
|
|
284
|
-
// into a historical query.
|
|
285
|
-
return changedInsideRange(mtimeMs) || changedInsideRange(ctimeMs);
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
function isRolloutPathInRequestedRange(filePath, { from = null, to = null } = {}) {
|
|
289
|
-
const pathDay = rolloutPathDay(filePath);
|
|
290
|
-
if (!pathDay) return true;
|
|
291
|
-
if (from && pathDay < from) return false;
|
|
292
|
-
if (to && pathDay > to) return false;
|
|
293
|
-
return true;
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
function discoveryInventoryKey(roots) {
|
|
297
|
-
return roots.map((root) => path.resolve(root)).sort().join("\0");
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
function getDiscoveryInventory(roots, nowMs) {
|
|
301
|
-
const key = discoveryInventoryKey(roots);
|
|
302
|
-
let inventory = DISCOVERY_INVENTORIES.get(key);
|
|
303
|
-
if (!inventory) {
|
|
304
|
-
inventory = { files: new Map(), lastFullAuditAtMs: 0, lastUsedAtMs: nowMs };
|
|
305
|
-
DISCOVERY_INVENTORIES.set(key, inventory);
|
|
306
|
-
}
|
|
307
|
-
inventory.lastUsedAtMs = nowMs;
|
|
308
|
-
while (DISCOVERY_INVENTORIES.size > MAX_DISCOVERY_INVENTORIES) {
|
|
309
|
-
let oldestKey = null;
|
|
310
|
-
let oldestAt = Infinity;
|
|
311
|
-
for (const [candidateKey, candidate] of DISCOVERY_INVENTORIES) {
|
|
312
|
-
if (candidateKey === key) continue;
|
|
313
|
-
if (candidate.lastUsedAtMs < oldestAt) {
|
|
314
|
-
oldestAt = candidate.lastUsedAtMs;
|
|
315
|
-
oldestKey = candidateKey;
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
if (!oldestKey) break;
|
|
319
|
-
DISCOVERY_INVENTORIES.delete(oldestKey);
|
|
320
|
-
}
|
|
321
|
-
return inventory;
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
function discoverCodexSessionFiles(rootDirs, options = {}) {
|
|
325
|
-
const diagnostics = options.diagnostics || null;
|
|
326
|
-
const roots = Array.isArray(rootDirs) ? rootDirs : [rootDirs];
|
|
327
|
-
const nowMs = Number.isFinite(Number(options.nowMs)) ? Number(options.nowMs) : Date.now();
|
|
328
|
-
const inventory = getDiscoveryInventory(roots.filter(Boolean), nowMs);
|
|
329
|
-
const candidateOptions = {
|
|
330
|
-
...options,
|
|
331
|
-
metadataBounds: metadataRangeBounds(options.from, options.to, options.timeZoneContext),
|
|
332
|
-
};
|
|
333
|
-
const bounded = Boolean(options.from || options.to);
|
|
334
|
-
const auditAgeMs = nowMs - Number(inventory.lastFullAuditAtMs || 0);
|
|
335
|
-
const fullMetadataAudit = Boolean(
|
|
336
|
-
options.exhaustive ||
|
|
337
|
-
!bounded ||
|
|
338
|
-
!Number.isFinite(auditAgeMs) ||
|
|
339
|
-
auditAgeMs < 0 ||
|
|
340
|
-
auditAgeMs >= DISCOVERY_FULL_AUDIT_INTERVAL_MS
|
|
341
|
-
);
|
|
342
|
-
if (diagnostics) diagnostics.full_metadata_audit = fullMetadataAudit;
|
|
343
|
-
const entries = [];
|
|
344
|
-
const visited = new Set();
|
|
345
|
-
const stack = roots.filter(Boolean);
|
|
346
|
-
|
|
347
|
-
while (stack.length > 0) {
|
|
348
|
-
const dir = stack.pop();
|
|
349
|
-
let children;
|
|
350
|
-
try {
|
|
351
|
-
children = fs.readdirSync(dir, { withFileTypes: true });
|
|
352
|
-
} catch {
|
|
353
|
-
continue;
|
|
354
|
-
}
|
|
355
|
-
for (const child of children) {
|
|
356
|
-
const filePath = path.join(dir, child.name);
|
|
357
|
-
if (child.isDirectory()) {
|
|
358
|
-
stack.push(filePath);
|
|
359
|
-
continue;
|
|
360
|
-
}
|
|
361
|
-
if (!child.isFile() || !isRolloutCandidateFileName(child.name) || visited.has(filePath)) continue;
|
|
362
|
-
visited.add(filePath);
|
|
363
|
-
if (diagnostics) diagnostics.discovered_files += 1;
|
|
364
|
-
const cached = inventory.files.get(filePath) || null;
|
|
365
|
-
const cachedCandidate = cached
|
|
366
|
-
? isConservativeRangeCandidate(filePath, cached, candidateOptions)
|
|
367
|
-
: false;
|
|
368
|
-
const shouldStat = Boolean(
|
|
369
|
-
fullMetadataAudit ||
|
|
370
|
-
!cached ||
|
|
371
|
-
cachedCandidate ||
|
|
372
|
-
isRolloutPathInRequestedRange(filePath, options)
|
|
373
|
-
);
|
|
374
|
-
let stat = cached;
|
|
375
|
-
if (shouldStat) {
|
|
376
|
-
try {
|
|
377
|
-
if (diagnostics) diagnostics.stat_calls += 1;
|
|
378
|
-
stat = fs.statSync(filePath);
|
|
379
|
-
inventory.files.set(filePath, stat);
|
|
380
|
-
} catch {
|
|
381
|
-
inventory.files.delete(filePath);
|
|
382
|
-
continue;
|
|
383
|
-
}
|
|
384
|
-
} else if (diagnostics) {
|
|
385
|
-
diagnostics.metadata_cache_hits += 1;
|
|
386
|
-
}
|
|
387
|
-
const candidate = Boolean(
|
|
388
|
-
options.exhaustive ||
|
|
389
|
-
isConservativeRangeCandidate(filePath, stat, candidateOptions),
|
|
390
|
-
);
|
|
391
|
-
entries.push({ filePath, stat, candidate });
|
|
392
|
-
if (candidate && diagnostics) diagnostics.candidate_files += 1;
|
|
393
|
-
}
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
for (const filePath of inventory.files.keys()) {
|
|
397
|
-
if (!visited.has(filePath)) inventory.files.delete(filePath);
|
|
398
|
-
}
|
|
399
|
-
if (fullMetadataAudit) inventory.lastFullAuditAtMs = nowMs;
|
|
400
|
-
|
|
401
|
-
return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
async function listCodexSessionFiles(baseDir) {
|
|
405
|
-
const rolloutFiles = await listRolloutFiles(baseDir).catch(() => []);
|
|
406
|
-
const allJsonlFiles = listJsonlFiles(baseDir);
|
|
407
|
-
if (allJsonlFiles.length === 0) return rolloutFiles;
|
|
408
|
-
if (rolloutFiles.length === 0) return allJsonlFiles;
|
|
409
|
-
|
|
410
|
-
const merged = new Set(rolloutFiles);
|
|
411
|
-
for (const filePath of allJsonlFiles) merged.add(filePath);
|
|
412
|
-
return Array.from(merged).sort((a, b) => a.localeCompare(b));
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
// ---------------------------------------------------------------------------
|
|
416
|
-
// Token count extraction
|
|
417
|
-
// ---------------------------------------------------------------------------
|
|
418
|
-
|
|
419
|
-
function extractTokenCount(obj) {
|
|
420
|
-
const payload = obj?.payload;
|
|
421
|
-
if (!payload || obj?.type !== "event_msg") return null;
|
|
422
|
-
if (payload.type === "token_count") {
|
|
423
|
-
return { info: payload.info || null, timestamp: obj?.timestamp || null };
|
|
424
|
-
}
|
|
425
|
-
const msg = payload.msg;
|
|
426
|
-
if (msg && msg.type === "token_count") {
|
|
427
|
-
return { info: msg.info || null, timestamp: obj?.timestamp || null };
|
|
428
|
-
}
|
|
429
|
-
return null;
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
// ---------------------------------------------------------------------------
|
|
433
|
-
// Tool name helpers
|
|
434
|
-
// ---------------------------------------------------------------------------
|
|
435
|
-
|
|
436
|
-
function normalizeToolName(payload) {
|
|
437
|
-
const name = payload?.name || "";
|
|
438
|
-
const ns = payload?.namespace || null;
|
|
439
|
-
if (ns && typeof ns === "string" && ns.startsWith("mcp__")) return `${ns}${name}`;
|
|
440
|
-
return name || "unknown";
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
function extractSkillNameFromFunctionCall(payload, diagnostics = null) {
|
|
444
|
-
if (!payload || payload.name !== "exec_command") return null;
|
|
445
|
-
const args = safeJsonParse(payload.arguments || "{}", diagnostics) || {};
|
|
446
|
-
const cmd = String(args.cmd || "");
|
|
447
|
-
const match = cmd.match(/(?:^|\/)skills\/(?:\.system\/)?([^/\s]+)\/SKILL\.md\b/);
|
|
448
|
-
return match ? match[1] : null;
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
function formatToolDisplayName(name) {
|
|
452
|
-
if (typeof name !== "string" || !name.startsWith("mcp__")) return name;
|
|
453
|
-
const parts = name.split("__");
|
|
454
|
-
if (parts.length < 3) return name;
|
|
455
|
-
const server = String(parts[1] || "").replace(/^plugin_/, "").replace(/_/g, "-");
|
|
456
|
-
const tool = parts.slice(2).join("__") || name;
|
|
457
|
-
return server ? `${server}/${tool}` : tool;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
// ---------------------------------------------------------------------------
|
|
461
|
-
// Usage normalization
|
|
462
|
-
// ---------------------------------------------------------------------------
|
|
463
|
-
|
|
464
|
-
function normalizeUsage(u) {
|
|
465
|
-
const out = {};
|
|
466
|
-
for (const k of [
|
|
467
|
-
"input_tokens",
|
|
468
|
-
"cached_input_tokens",
|
|
469
|
-
"cache_creation_input_tokens",
|
|
470
|
-
"output_tokens",
|
|
471
|
-
"reasoning_output_tokens",
|
|
472
|
-
"total_tokens",
|
|
473
|
-
]) {
|
|
474
|
-
const n = Number(u?.[k] || 0);
|
|
475
|
-
out[k] = Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
|
|
476
|
-
}
|
|
477
|
-
// Codex reports input inclusive of cached_input_tokens; keep our schema
|
|
478
|
-
// convention: non-cached input and cached input tracked separately.
|
|
479
|
-
out.input_tokens = Math.max(0, out.input_tokens - out.cached_input_tokens);
|
|
480
|
-
out.total_tokens =
|
|
481
|
-
out.input_tokens +
|
|
482
|
-
out.cached_input_tokens +
|
|
483
|
-
out.cache_creation_input_tokens +
|
|
484
|
-
out.output_tokens;
|
|
485
|
-
return out;
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
function pickDelta(lastUsage, totalUsage, prevTotals) {
|
|
489
|
-
const state = createUsageDeltaState({ lastTotal: prevTotals });
|
|
490
|
-
const delta = consumeUsageDelta(state, lastUsage, totalUsage);
|
|
491
|
-
return delta ? normalizeUsage(delta) : null;
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
// ---------------------------------------------------------------------------
|
|
495
|
-
// Exec stats helpers (local to parser, not shared)
|
|
496
|
-
// ---------------------------------------------------------------------------
|
|
497
|
-
|
|
498
|
-
function durationBucket(ms) {
|
|
499
|
-
const n = Number(ms || 0);
|
|
500
|
-
if (!Number.isFinite(n) || n <= 0) return "unknown";
|
|
501
|
-
if (n < 1000) return "<1s";
|
|
502
|
-
if (n < 10_000) return "1-10s";
|
|
503
|
-
if (n < 60_000) return "10-60s";
|
|
504
|
-
if (n < 300_000) return "1-5m";
|
|
505
|
-
return ">5m";
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
function outputSizeBucket(lines, chars) {
|
|
509
|
-
const l = Number(lines || 0);
|
|
510
|
-
const c = Number(chars || 0);
|
|
511
|
-
if (!l && !c) return "quiet";
|
|
512
|
-
if (l <= 20 && c <= 2_000) return "small";
|
|
513
|
-
if (l <= 200 && c <= 20_000) return "medium";
|
|
514
|
-
if (l <= 1000 && c <= 100_000) return "large";
|
|
515
|
-
return "very_large";
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
function buildToolStatsEntry() {
|
|
519
|
-
return { calls: 0, totals: emptyTotals() };
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
function buildSkillStatsEntry(name) {
|
|
523
|
-
return { name, calls: 0, totals: emptyTotals() };
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
// ---------------------------------------------------------------------------
|
|
527
|
-
// Finalize helpers
|
|
528
|
-
// ---------------------------------------------------------------------------
|
|
529
|
-
|
|
530
|
-
function finalizeToolRows(map) {
|
|
531
|
-
return Array.from(map.values())
|
|
532
|
-
.map((row) => {
|
|
533
|
-
const rawName = row.raw_name || row.name;
|
|
534
|
-
return {
|
|
535
|
-
name: formatToolDisplayName(rawName),
|
|
536
|
-
raw_name: rawName,
|
|
537
|
-
calls: row.calls,
|
|
538
|
-
totals: row.totals,
|
|
539
|
-
};
|
|
540
|
-
})
|
|
541
|
-
.sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
function finalizeSkillRows(map) {
|
|
545
|
-
return Array.from(map.values())
|
|
546
|
-
.map((row) => ({
|
|
547
|
-
name: row.name,
|
|
548
|
-
calls: row.calls,
|
|
549
|
-
totals: row.totals,
|
|
550
|
-
}))
|
|
551
|
-
.sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
function finalizeExecRows(map) {
|
|
555
|
-
return Array.from(map.values())
|
|
556
|
-
.map((row) => ({
|
|
557
|
-
name: row.name,
|
|
558
|
-
calls: row.calls,
|
|
559
|
-
failures: row.failures,
|
|
560
|
-
duration_ms: row.duration_ms,
|
|
561
|
-
max_duration_ms: row.max_duration_ms,
|
|
562
|
-
output_chars: row.output_chars,
|
|
563
|
-
output_lines: row.output_lines,
|
|
564
|
-
totals: row.totals,
|
|
565
|
-
}))
|
|
566
|
-
.sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
function finalizeModelUsageRows(map) {
|
|
570
|
-
return Array.from(map.values())
|
|
571
|
-
.map((row) => ({
|
|
572
|
-
...row,
|
|
573
|
-
selected_models: Array.from(row.selected_models).sort(),
|
|
574
|
-
reroute_reasons: Array.from(row.reroute_reasons).sort(),
|
|
575
|
-
model_attribution: row.rerouted_usage_events > 0 ? "effective" : "selected",
|
|
576
|
-
}))
|
|
577
|
-
.sort((a, b) => b.total_tokens - a.total_tokens || a.model.localeCompare(b.model));
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
// ---------------------------------------------------------------------------
|
|
581
|
-
// Main parser
|
|
582
|
-
// ---------------------------------------------------------------------------
|
|
583
|
-
|
|
584
|
-
async function parseCodexRolloutFile(filePath, {
|
|
585
|
-
from = null,
|
|
586
|
-
to = null,
|
|
587
|
-
timeZoneContext = null,
|
|
588
|
-
diagnostics = null,
|
|
589
|
-
seenTokenEvents = null,
|
|
590
|
-
startOffset = 0,
|
|
591
|
-
endOffset = null,
|
|
592
|
-
resumeState = null,
|
|
593
|
-
captureResumeState = false,
|
|
594
|
-
captureContentHash = false,
|
|
595
|
-
contentHashState = null,
|
|
596
|
-
sourceHandle = null,
|
|
597
|
-
collectBreakdowns = true,
|
|
598
|
-
collectModelUsage = false,
|
|
599
|
-
onObject = null,
|
|
600
|
-
onUsage = null,
|
|
601
|
-
} = {}) {
|
|
602
|
-
const filePaths = (Array.isArray(filePath) ? filePath : [filePath]).filter(Boolean);
|
|
603
|
-
const primaryFilePath = filePaths[0] || String(filePath || "");
|
|
604
|
-
const isResuming = Boolean(resumeState && filePaths.length === 1);
|
|
605
|
-
const initialOffset = Math.max(0, Number(startOffset) || 0);
|
|
606
|
-
const readProgress = {
|
|
607
|
-
endOffset: initialOffset,
|
|
608
|
-
lastByte: initialOffset > 0 ? 0x0a : null,
|
|
609
|
-
};
|
|
610
|
-
let contentHasher = null;
|
|
611
|
-
if (captureResumeState && captureContentHash && filePaths.length === 1) {
|
|
612
|
-
if (contentHashState && typeof contentHashState.copy === "function") {
|
|
613
|
-
try {
|
|
614
|
-
contentHasher = contentHashState.copy();
|
|
615
|
-
} catch {
|
|
616
|
-
contentHasher = null;
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
if (initialOffset > 0 && !contentHasher) {
|
|
620
|
-
const error = new Error("Codex rollout append is missing its prefix hash state");
|
|
621
|
-
error.code = CODEX_RESUME_INVALIDATED;
|
|
622
|
-
throw error;
|
|
623
|
-
}
|
|
624
|
-
if (!contentHasher) contentHasher = crypto.createHash("sha256");
|
|
625
|
-
}
|
|
626
|
-
|
|
627
|
-
let sessionId = isResuming ? resumeState.sessionId || null : null;
|
|
628
|
-
let cwd = isResuming ? resumeState.cwd || null : null;
|
|
629
|
-
let model = isResuming ? resumeState.model || null : null;
|
|
630
|
-
const modelAttributionState = createCodexModelAttributionState(
|
|
631
|
-
isResuming ? resumeState.modelAttributionState || { model } : {},
|
|
632
|
-
);
|
|
633
|
-
// Codex writes thread_settings_applied immediately before the turn_context of
|
|
634
|
-
// the turn it applies to, and token_count rows carry no turn id, so each one
|
|
635
|
-
// belongs to the most recent preceding tier record. Sessions that never emit
|
|
636
|
-
// the event (most CLI sessions) stay null and are billed at Standard.
|
|
637
|
-
let serviceTier = isResuming ? resumeState.serviceTier || null : null;
|
|
638
|
-
let provider = isResuming ? resumeState.provider || null : null;
|
|
639
|
-
let cliVersion = isResuming ? resumeState.cliVersion || null : null;
|
|
640
|
-
let forkedFromId = isResuming ? resumeState.forkedFromId || null : null;
|
|
641
|
-
let parentThreadId = isResuming ? resumeState.parentThreadId || null : null;
|
|
642
|
-
let threadSource = isResuming ? resumeState.threadSource || null : null;
|
|
643
|
-
let agentNickname = isResuming ? resumeState.agentNickname || null : null;
|
|
644
|
-
let agentRole = isResuming ? resumeState.agentRole || null : null;
|
|
645
|
-
let sessionLineageCaptured = isResuming
|
|
646
|
-
? Boolean(resumeState.sessionLineageCaptured || resumeState.sessionId)
|
|
647
|
-
: false;
|
|
648
|
-
|
|
649
|
-
const usageDeltaState = createUsageDeltaState({
|
|
650
|
-
lastTotal: isResuming ? resumeState.prevTotals || null : null,
|
|
651
|
-
baselines: isResuming ? resumeState.tokenUsageBaselines || null : null,
|
|
652
|
-
});
|
|
653
|
-
let pendingToolNames = isResuming
|
|
654
|
-
? Array.from(resumeState.pendingToolNames || [])
|
|
655
|
-
: [];
|
|
656
|
-
let pendingSkills = isResuming ? Array.from(resumeState.pendingSkills || []) : [];
|
|
657
|
-
let pendingExecDetails = isResuming
|
|
658
|
-
? Array.from(resumeState.pendingExecDetails || [])
|
|
659
|
-
: [];
|
|
660
|
-
let lastTokenTimestamp = isResuming
|
|
661
|
-
? String(resumeState.lastTokenTimestamp || "") || null
|
|
662
|
-
: null;
|
|
663
|
-
let lastTimestampEventSignatures = new Set(
|
|
664
|
-
isResuming ? resumeState.lastTimestampEventSignatures || [] : [],
|
|
665
|
-
);
|
|
666
|
-
let lastTimestampLastUsage = null;
|
|
667
|
-
let lastTimestampTotalUsage = null;
|
|
668
|
-
let hasUnmaterializedLastTimestampUsage = false;
|
|
669
|
-
|
|
670
|
-
const totals = emptyTotals();
|
|
671
|
-
const byTool = new Map(); // tool_name -> {name,calls,totals}
|
|
672
|
-
const bySkill = new Map(); // skill_name -> {name,calls,totals}
|
|
673
|
-
const byExecKind = new Map(); // kind -> stats
|
|
674
|
-
const byExecExit = new Map(); // status:exit -> stats
|
|
675
|
-
const byExecExecutable = new Map(); // executable -> stats
|
|
676
|
-
const byExecCommand = new Map(); // sanitized executable + subcommand -> stats
|
|
677
|
-
const byExecDuration = new Map(); // duration bucket -> stats
|
|
678
|
-
const byExecOutput = new Map(); // output size bucket -> stats
|
|
679
|
-
const byModel = new Map(); // effective model -> observed token usage
|
|
680
|
-
|
|
681
|
-
let turnCount = 0;
|
|
682
|
-
|
|
683
|
-
// Compaction records (issue #652, rules in codex-usage-record.js). A resume
|
|
684
|
-
// state from before #652 with prevTotals has seen a token_count.
|
|
685
|
-
const usageRecordState = createUsageRecordState(
|
|
686
|
-
isResuming
|
|
687
|
-
? resumeState.usageRecordState || (resumeState.prevTotals ? { sawTokenCount: true } : null)
|
|
688
|
-
: null,
|
|
689
|
-
);
|
|
690
|
-
// Compactions counted in this session, by response_id: the files merged for
|
|
691
|
-
// a session can repeat a record (an archived copy, a split fragment), and a
|
|
692
|
-
// resumed parse must not count one from an earlier chunk again.
|
|
693
|
-
const compactionResponseIds = new Set(
|
|
694
|
-
isResuming && Array.isArray(resumeState.compactionResponseIds)
|
|
695
|
-
? resumeState.compactionResponseIds
|
|
696
|
-
: [],
|
|
697
|
-
);
|
|
698
|
-
|
|
699
|
-
// Attribution as of now. A compaction record is counted later than it is
|
|
700
|
-
// read, so it carries the attribution captured when it was read.
|
|
701
|
-
function currentUsageAttribution() {
|
|
702
|
-
return {
|
|
703
|
-
model: currentCodexModel(modelAttributionState) || "unknown",
|
|
704
|
-
selectedModel: modelAttributionState.selectedModel || null,
|
|
705
|
-
rerouted: Boolean(modelAttributionState.rerouted),
|
|
706
|
-
rerouteReason: modelAttributionState.rerouteReason || null,
|
|
707
|
-
serviceTier,
|
|
708
|
-
};
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
function recordModelUsage(delta, rawRequestUsage, attribution = currentUsageAttribution()) {
|
|
712
|
-
if (!collectModelUsage || !delta || delta.total_tokens <= 0) return;
|
|
713
|
-
const effectiveModel = attribution.model;
|
|
714
|
-
let row = byModel.get(effectiveModel);
|
|
715
|
-
if (!row) {
|
|
716
|
-
row = {
|
|
717
|
-
model: effectiveModel,
|
|
718
|
-
...emptyTotals(),
|
|
719
|
-
long_context_input_tokens: 0,
|
|
720
|
-
long_context_cached_input_tokens: 0,
|
|
721
|
-
long_context_cache_creation_input_tokens: 0,
|
|
722
|
-
long_context_output_tokens: 0,
|
|
723
|
-
long_context_reasoning_output_tokens: 0,
|
|
724
|
-
priority_input_tokens: 0,
|
|
725
|
-
priority_cached_input_tokens: 0,
|
|
726
|
-
priority_cache_creation_input_tokens: 0,
|
|
727
|
-
priority_output_tokens: 0,
|
|
728
|
-
priority_reasoning_output_tokens: 0,
|
|
729
|
-
priority_long_context_input_tokens: 0,
|
|
730
|
-
priority_long_context_cached_input_tokens: 0,
|
|
731
|
-
priority_long_context_cache_creation_input_tokens: 0,
|
|
732
|
-
priority_long_context_output_tokens: 0,
|
|
733
|
-
priority_long_context_reasoning_output_tokens: 0,
|
|
734
|
-
usage_events: 0,
|
|
735
|
-
rerouted_usage_events: 0,
|
|
736
|
-
long_context_usage_events: 0,
|
|
737
|
-
priority_usage_events: 0,
|
|
738
|
-
selected_models: new Set(),
|
|
739
|
-
reroute_reasons: new Set(),
|
|
740
|
-
};
|
|
741
|
-
byModel.set(effectiveModel, row);
|
|
742
|
-
}
|
|
743
|
-
const subsetKeys = Object.keys(row).filter((key) => /^(?:long_context_|priority_)|^(?:usage_events|rerouted_usage_events)$/.test(key));
|
|
744
|
-
const before = Object.fromEntries(subsetKeys.map((key) => [key, row[key]]));
|
|
745
|
-
addInto(row, delta);
|
|
746
|
-
row.usage_events += 1;
|
|
747
|
-
if (attribution.selectedModel) {
|
|
748
|
-
row.selected_models.add(attribution.selectedModel);
|
|
749
|
-
}
|
|
750
|
-
if (attribution.rerouted) {
|
|
751
|
-
row.rerouted_usage_events += 1;
|
|
752
|
-
if (attribution.rerouteReason) {
|
|
753
|
-
row.reroute_reasons.add(attribution.rerouteReason);
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
// OpenAI applies the long-context tier to the whole request when its raw
|
|
757
|
-
// (cache-inclusive) input exceeds 272K tokens. Preserve the exact subset
|
|
758
|
-
// so pricing can be recomputed later without rescanning private logs.
|
|
759
|
-
const isLongContext =
|
|
760
|
-
Number(rawRequestUsage?.input_tokens || 0) > OPENAI_LONG_CONTEXT_INPUT_THRESHOLD;
|
|
761
|
-
if (isLongContext) {
|
|
762
|
-
row.long_context_usage_events += 1;
|
|
763
|
-
row.long_context_input_tokens += delta.input_tokens;
|
|
764
|
-
row.long_context_cached_input_tokens += delta.cached_input_tokens;
|
|
765
|
-
row.long_context_cache_creation_input_tokens += delta.cache_creation_input_tokens;
|
|
766
|
-
row.long_context_output_tokens += delta.output_tokens;
|
|
767
|
-
row.long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
|
|
768
|
-
}
|
|
769
|
-
// Astra Fast bills at a flat multiple of whichever context tier the request
|
|
770
|
-
// was on, so the priority premium multiplies the long-context premium
|
|
771
|
-
// instead of replacing it. The two subsets alone cannot say how much usage
|
|
772
|
-
// was on BOTH, so record the intersection explicitly — otherwise a long
|
|
773
|
-
// Fast request prices at 3x Standard input where it should be 4x.
|
|
774
|
-
if (isPriorityServiceTier(attribution.serviceTier)) {
|
|
775
|
-
row.priority_usage_events += 1;
|
|
776
|
-
row.priority_input_tokens += delta.input_tokens;
|
|
777
|
-
row.priority_cached_input_tokens += delta.cached_input_tokens;
|
|
778
|
-
row.priority_cache_creation_input_tokens += delta.cache_creation_input_tokens;
|
|
779
|
-
row.priority_output_tokens += delta.output_tokens;
|
|
780
|
-
row.priority_reasoning_output_tokens += delta.reasoning_output_tokens;
|
|
781
|
-
if (isLongContext) {
|
|
782
|
-
row.priority_long_context_input_tokens += delta.input_tokens;
|
|
783
|
-
row.priority_long_context_cached_input_tokens += delta.cached_input_tokens;
|
|
784
|
-
row.priority_long_context_cache_creation_input_tokens +=
|
|
785
|
-
delta.cache_creation_input_tokens;
|
|
786
|
-
row.priority_long_context_output_tokens += delta.output_tokens;
|
|
787
|
-
row.priority_long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
|
|
788
|
-
}
|
|
789
|
-
}
|
|
790
|
-
return { ...delta, ...Object.fromEntries(subsetKeys.map((key) => [key, row[key] - before[key]])),
|
|
791
|
-
selected_models: attribution.selectedModel ? [attribution.selectedModel] : [],
|
|
792
|
-
reroute_reasons: attribution.rerouteReason ? [attribution.rerouteReason] : [],
|
|
793
|
-
model_attribution: attribution.rerouted ? "effective" : "selected" };
|
|
794
|
-
}
|
|
795
|
-
|
|
796
|
-
function ensureTool(name) {
|
|
797
|
-
if (!byTool.has(name)) {
|
|
798
|
-
byTool.set(name, { name, ...buildToolStatsEntry() });
|
|
799
|
-
}
|
|
800
|
-
return byTool.get(name);
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
function ensureExec(map, key) {
|
|
804
|
-
if (!map.has(key)) map.set(key, { name: key, ...buildExecStatsEntry() });
|
|
805
|
-
return map.get(key);
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
function ensureSkill(name) {
|
|
809
|
-
if (!bySkill.has(name)) bySkill.set(name, buildSkillStatsEntry(name));
|
|
810
|
-
return bySkill.get(name);
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
function getExecKeys(p) {
|
|
814
|
-
if (!p || typeof p !== "object") return;
|
|
815
|
-
const cmdArr = Array.isArray(p.command) ? p.command : null;
|
|
816
|
-
const cmd = cmdArr && cmdArr.length > 0 ? String(cmdArr[cmdArr.length - 1] || "") : "";
|
|
817
|
-
const kind = p.parsed_cmd?.[0]?.type && p.parsed_cmd[0].type !== "unknown"
|
|
818
|
-
? p.parsed_cmd[0].type
|
|
819
|
-
: inferExecCommandKind(cmd);
|
|
820
|
-
|
|
821
|
-
const status = String(p.status || "unknown");
|
|
822
|
-
const exit = Number.isFinite(Number(p.exit_code)) ? Number(p.exit_code) : null;
|
|
823
|
-
const exitKey = `${status}:${exit === null ? "unknown" : exit}`;
|
|
824
|
-
|
|
825
|
-
const dur = p.duration ? Math.round((Number(p.duration.secs || 0) * 1000) + Number(p.duration.nanos || 0) / 1e6) : 0;
|
|
826
|
-
const output = String(p.aggregated_output || p.stdout || "");
|
|
827
|
-
const outputChars = output.length;
|
|
828
|
-
const outputLines = output ? output.split("\n").length : 0;
|
|
829
|
-
return {
|
|
830
|
-
kind,
|
|
831
|
-
exitKey,
|
|
832
|
-
executable: getExecutableName(cmd),
|
|
833
|
-
command: sanitizeCommandSignature(cmd),
|
|
834
|
-
duration: durationBucket(dur),
|
|
835
|
-
output: outputSizeBucket(outputLines, outputChars),
|
|
836
|
-
dur,
|
|
837
|
-
outputChars,
|
|
838
|
-
outputLines,
|
|
839
|
-
failed: status !== "completed" || exit !== 0,
|
|
840
|
-
};
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
function absorbExecStats(map, key, details) {
|
|
844
|
-
const row = ensureExec(map, key);
|
|
845
|
-
row.calls += 1;
|
|
846
|
-
row.duration_ms += details.dur;
|
|
847
|
-
row.max_duration_ms = Math.max(row.max_duration_ms, details.dur);
|
|
848
|
-
row.output_chars += details.outputChars;
|
|
849
|
-
row.output_lines += details.outputLines;
|
|
850
|
-
if (details.failed) row.failures += 1;
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
function absorbExecEnd(details) {
|
|
854
|
-
absorbExecStats(byExecKind, details.kind, details);
|
|
855
|
-
absorbExecStats(byExecExit, details.exitKey, details);
|
|
856
|
-
absorbExecStats(byExecExecutable, details.executable, details);
|
|
857
|
-
absorbExecStats(byExecCommand, details.command, details);
|
|
858
|
-
absorbExecStats(byExecDuration, details.duration, details);
|
|
859
|
-
absorbExecStats(byExecOutput, details.output, details);
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
function attributeTurn(delta) {
|
|
863
|
-
if (!delta || delta.total_tokens <= 0) {
|
|
864
|
-
for (const details of pendingExecDetails) absorbExecEnd(details);
|
|
865
|
-
pendingToolNames = [];
|
|
866
|
-
pendingSkills = [];
|
|
867
|
-
pendingExecDetails = [];
|
|
868
|
-
return;
|
|
869
|
-
}
|
|
870
|
-
turnCount += 1;
|
|
871
|
-
|
|
872
|
-
if (!collectBreakdowns) {
|
|
873
|
-
addInto(totals, delta);
|
|
874
|
-
pendingToolNames = [];
|
|
875
|
-
pendingSkills = [];
|
|
876
|
-
pendingExecDetails = [];
|
|
877
|
-
return;
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
const unique = [...new Set(pendingToolNames.filter(Boolean))];
|
|
881
|
-
const tools = unique.length > 0 ? unique : ["text_response"];
|
|
882
|
-
const share = 1 / tools.length;
|
|
883
|
-
|
|
884
|
-
for (const name of tools) {
|
|
885
|
-
const row = ensureTool(name);
|
|
886
|
-
row.calls += share;
|
|
887
|
-
addInto(row.totals, {
|
|
888
|
-
input_tokens: delta.input_tokens * share,
|
|
889
|
-
cached_input_tokens: delta.cached_input_tokens * share,
|
|
890
|
-
cache_creation_input_tokens: delta.cache_creation_input_tokens * share,
|
|
891
|
-
output_tokens: delta.output_tokens * share,
|
|
892
|
-
reasoning_output_tokens: delta.reasoning_output_tokens * share,
|
|
893
|
-
total_tokens: delta.total_tokens * share,
|
|
894
|
-
});
|
|
895
|
-
}
|
|
896
|
-
|
|
897
|
-
const uniqueSkills = [...new Set(pendingSkills.filter(Boolean))];
|
|
898
|
-
if (uniqueSkills.length > 0) {
|
|
899
|
-
const skillShare = 1 / uniqueSkills.length;
|
|
900
|
-
for (const name of uniqueSkills) {
|
|
901
|
-
const row = ensureSkill(name);
|
|
902
|
-
row.calls += skillShare;
|
|
903
|
-
addInto(row.totals, {
|
|
904
|
-
input_tokens: delta.input_tokens * skillShare,
|
|
905
|
-
cached_input_tokens: delta.cached_input_tokens * skillShare,
|
|
906
|
-
cache_creation_input_tokens: delta.cache_creation_input_tokens * skillShare,
|
|
907
|
-
output_tokens: delta.output_tokens * skillShare,
|
|
908
|
-
reasoning_output_tokens: delta.reasoning_output_tokens * skillShare,
|
|
909
|
-
total_tokens: delta.total_tokens * skillShare,
|
|
910
|
-
});
|
|
911
|
-
}
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
// Attribute exec_command_end rows to exec stats; note these are not a
|
|
915
|
-
// token source — we attach the same tool-shared delta to the command
|
|
916
|
-
// classifier so users can find high-cost command families.
|
|
917
|
-
const execToolShare = tools.includes("exec_command") ? (1 / tools.length) : 0;
|
|
918
|
-
const execDelta = execToolShare > 0 ? {
|
|
919
|
-
input_tokens: delta.input_tokens * execToolShare,
|
|
920
|
-
cached_input_tokens: delta.cached_input_tokens * execToolShare,
|
|
921
|
-
cache_creation_input_tokens: delta.cache_creation_input_tokens * execToolShare,
|
|
922
|
-
output_tokens: delta.output_tokens * execToolShare,
|
|
923
|
-
reasoning_output_tokens: delta.reasoning_output_tokens * execToolShare,
|
|
924
|
-
total_tokens: delta.total_tokens * execToolShare,
|
|
925
|
-
} : null;
|
|
926
|
-
|
|
927
|
-
if (execDelta && pendingExecDetails.length > 0) {
|
|
928
|
-
const perExecShare = 1 / pendingExecDetails.length;
|
|
929
|
-
for (const details of pendingExecDetails) {
|
|
930
|
-
const attributed = {
|
|
931
|
-
input_tokens: execDelta.input_tokens * perExecShare,
|
|
932
|
-
cached_input_tokens: execDelta.cached_input_tokens * perExecShare,
|
|
933
|
-
cache_creation_input_tokens: execDelta.cache_creation_input_tokens * perExecShare,
|
|
934
|
-
output_tokens: execDelta.output_tokens * perExecShare,
|
|
935
|
-
reasoning_output_tokens: execDelta.reasoning_output_tokens * perExecShare,
|
|
936
|
-
total_tokens: execDelta.total_tokens * perExecShare,
|
|
937
|
-
};
|
|
938
|
-
|
|
939
|
-
addInto(ensureExec(byExecKind, details.kind).totals, attributed);
|
|
940
|
-
addInto(ensureExec(byExecExit, details.exitKey).totals, attributed);
|
|
941
|
-
addInto(ensureExec(byExecExecutable, details.executable).totals, attributed);
|
|
942
|
-
addInto(ensureExec(byExecCommand, details.command).totals, attributed);
|
|
943
|
-
addInto(ensureExec(byExecDuration, details.duration).totals, attributed);
|
|
944
|
-
addInto(ensureExec(byExecOutput, details.output).totals, attributed);
|
|
945
|
-
|
|
946
|
-
absorbExecEnd(details);
|
|
947
|
-
}
|
|
948
|
-
} else {
|
|
949
|
-
// Still ingest exec end stats without token attribution so the drill-down works.
|
|
950
|
-
for (const details of pendingExecDetails) absorbExecEnd(details);
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
addInto(totals, delta);
|
|
954
|
-
pendingToolNames = [];
|
|
955
|
-
pendingSkills = [];
|
|
956
|
-
pendingExecDetails = [];
|
|
957
|
-
}
|
|
958
|
-
|
|
959
|
-
function countCompactionRecord(record) {
|
|
960
|
-
if (compactionResponseIds.has(record.responseId)) return;
|
|
961
|
-
compactionResponseIds.add(record.responseId);
|
|
962
|
-
if (!record.inRequestedRange) return;
|
|
963
|
-
const pricedDelta = recordModelUsage(record.delta, record.rawUsage, record.attribution);
|
|
964
|
-
if (typeof onUsage === "function") onUsage({ timestamp: record.timestamp,
|
|
965
|
-
delta: pricedDelta || record.delta, rawUsage: record.rawUsage, model: record.attribution.model, origin: "compaction" });
|
|
966
|
-
attributeTurn(record.delta);
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
function usageEventSignature(lastUsage, totalUsage) {
|
|
970
|
-
return JSON.stringify({ lastUsage, totalUsage });
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
function materializeLastTimestampSignatures() {
|
|
974
|
-
if (hasUnmaterializedLastTimestampUsage) {
|
|
975
|
-
lastTimestampEventSignatures.add(
|
|
976
|
-
usageEventSignature(lastTimestampLastUsage, lastTimestampTotalUsage),
|
|
977
|
-
);
|
|
978
|
-
hasUnmaterializedLastTimestampUsage = false;
|
|
979
|
-
}
|
|
980
|
-
return lastTimestampEventSignatures;
|
|
981
|
-
}
|
|
982
|
-
|
|
983
|
-
function noteTokenEvent(timestamp, lastUsage, totalUsage) {
|
|
984
|
-
if (!isResuming) {
|
|
985
|
-
if (timestamp !== lastTokenTimestamp) {
|
|
986
|
-
lastTokenTimestamp = timestamp;
|
|
987
|
-
lastTimestampEventSignatures = new Set();
|
|
988
|
-
hasUnmaterializedLastTimestampUsage = false;
|
|
989
|
-
} else {
|
|
990
|
-
materializeLastTimestampSignatures();
|
|
991
|
-
}
|
|
992
|
-
lastTimestampLastUsage = lastUsage;
|
|
993
|
-
lastTimestampTotalUsage = totalUsage;
|
|
994
|
-
hasUnmaterializedLastTimestampUsage = true;
|
|
995
|
-
return true;
|
|
996
|
-
}
|
|
997
|
-
if (isResuming && lastTokenTimestamp && timestamp < lastTokenTimestamp) {
|
|
998
|
-
const error = new Error("Codex rollout append is not timestamp-monotonic");
|
|
999
|
-
error.code = CODEX_RESUME_INVALIDATED;
|
|
1000
|
-
throw error;
|
|
1001
|
-
}
|
|
1002
|
-
if (!lastTokenTimestamp || timestamp > lastTokenTimestamp) {
|
|
1003
|
-
lastTokenTimestamp = timestamp;
|
|
1004
|
-
lastTimestampEventSignatures = new Set();
|
|
1005
|
-
lastTimestampLastUsage = lastUsage;
|
|
1006
|
-
lastTimestampTotalUsage = totalUsage;
|
|
1007
|
-
hasUnmaterializedLastTimestampUsage = true;
|
|
1008
|
-
return true;
|
|
1009
|
-
}
|
|
1010
|
-
if (timestamp === lastTokenTimestamp) {
|
|
1011
|
-
const signature = usageEventSignature(lastUsage, totalUsage);
|
|
1012
|
-
const signatures = materializeLastTimestampSignatures();
|
|
1013
|
-
if (signatures.has(signature)) return false;
|
|
1014
|
-
signatures.add(signature);
|
|
1015
|
-
}
|
|
1016
|
-
return true;
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
for await (const obj of iterateCodexObjects(filePaths, diagnostics, {
|
|
1020
|
-
startOffset,
|
|
1021
|
-
endOffset,
|
|
1022
|
-
readProgress,
|
|
1023
|
-
contentHasher,
|
|
1024
|
-
sourceHandle,
|
|
1025
|
-
})) {
|
|
1026
|
-
const modelEvent = applyCodexModelEvent(modelAttributionState, obj);
|
|
1027
|
-
const attributedModel = currentCodexModel(modelAttributionState);
|
|
1028
|
-
model = attributedModel || model;
|
|
1029
|
-
// onObject runs AFTER the attribution state machine, and is handed the
|
|
1030
|
-
// model it just decided. A consumer sharing this pass (the delivery-signal
|
|
1031
|
-
// collector, which attributes edit turns) therefore sees exactly the model
|
|
1032
|
-
// recordModelUsage() bills the turn's tokens to. Letting it re-read
|
|
1033
|
-
// turn_context.payload.model itself would make codex-model-attribution.js
|
|
1034
|
-
// one of two authorities on the same question.
|
|
1035
|
-
if (typeof onObject === "function") onObject(obj, attributedModel);
|
|
1036
|
-
const usageRecord = extractTokenUsageRecord(obj);
|
|
1037
|
-
if (!usageRecord) noteLineAfterUsageRecord(usageRecordState, obj?.type === "compacted");
|
|
1038
|
-
const ts = typeof obj?.timestamp === "string" ? obj.timestamp : null;
|
|
1039
|
-
if (!ts) continue;
|
|
1040
|
-
const inRequestedRange = isTimestampInRequestedDayRange(ts, { from, to, timeZoneContext });
|
|
1041
|
-
|
|
1042
|
-
if (obj.type === "session_meta") {
|
|
1043
|
-
const p = obj.payload || {};
|
|
1044
|
-
// Forked rollouts replay historical session_meta rows after their own
|
|
1045
|
-
// child metadata. Only the first row describes this file's lineage.
|
|
1046
|
-
if (!sessionLineageCaptured) {
|
|
1047
|
-
sessionId = p.id || sessionId;
|
|
1048
|
-
cwd = p.cwd || cwd;
|
|
1049
|
-
cliVersion = p.cli_version || cliVersion;
|
|
1050
|
-
provider = p.model_provider || provider;
|
|
1051
|
-
forkedFromId = p.forked_from_id || null;
|
|
1052
|
-
parentThreadId = p.parent_thread_id || null;
|
|
1053
|
-
threadSource = p.thread_source || p.source || null;
|
|
1054
|
-
agentNickname = p.agent_nickname || null;
|
|
1055
|
-
agentRole = p.agent_role || null;
|
|
1056
|
-
sessionLineageCaptured = true;
|
|
1057
|
-
} else if (!sessionId) {
|
|
1058
|
-
sessionId = p.id || null;
|
|
1059
|
-
}
|
|
1060
|
-
}
|
|
1061
|
-
|
|
1062
|
-
const appliedServiceTier = readCodexServiceTier(obj);
|
|
1063
|
-
if (appliedServiceTier) {
|
|
1064
|
-
serviceTier = appliedServiceTier;
|
|
1065
|
-
continue;
|
|
1066
|
-
}
|
|
1067
|
-
|
|
1068
|
-
if (obj.type === "turn_context") {
|
|
1069
|
-
const p = obj.payload || {};
|
|
1070
|
-
if (typeof p.cwd === "string") cwd = p.cwd;
|
|
1071
|
-
continue;
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
if (modelEvent?.type === "model_rerouted") continue;
|
|
1075
|
-
|
|
1076
|
-
if (collectBreakdowns && obj.type === "response_item" && obj.payload?.type === "function_call") {
|
|
1077
|
-
pendingToolNames.push(normalizeToolName(obj.payload));
|
|
1078
|
-
const skill = extractSkillNameFromFunctionCall(obj.payload, diagnostics);
|
|
1079
|
-
if (skill) pendingSkills.push(skill);
|
|
1080
|
-
continue;
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
if (collectBreakdowns && obj.type === "event_msg" && obj.payload?.type === "exec_command_end") {
|
|
1084
|
-
const details = getExecKeys(obj.payload);
|
|
1085
|
-
if (details) pendingExecDetails.push(details);
|
|
1086
|
-
continue;
|
|
1087
|
-
}
|
|
1088
|
-
|
|
1089
|
-
if (usageRecord) {
|
|
1090
|
-
const delta = normalizeUsage(usageRecord.usage);
|
|
1091
|
-
noteUsageRecord(
|
|
1092
|
-
usageRecordState,
|
|
1093
|
-
// Without a response_id a compaction cannot be deduplicated.
|
|
1094
|
-
delta.total_tokens > 0 && usageRecord.responseId
|
|
1095
|
-
? {
|
|
1096
|
-
responseId: usageRecord.responseId,
|
|
1097
|
-
timestamp: usageRecord.timestamp,
|
|
1098
|
-
inRequestedRange,
|
|
1099
|
-
rawUsage: usageRecord.usage,
|
|
1100
|
-
delta,
|
|
1101
|
-
attribution: currentUsageAttribution(),
|
|
1102
|
-
}
|
|
1103
|
-
: null,
|
|
1104
|
-
);
|
|
1105
|
-
continue;
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
const tokenCount = extractTokenCount(obj);
|
|
1109
|
-
if (tokenCount) {
|
|
1110
|
-
const info = tokenCount.info;
|
|
1111
|
-
const lastUsage = info?.last_token_usage;
|
|
1112
|
-
const totalUsage = info?.total_token_usage;
|
|
1113
|
-
let compaction = null;
|
|
1114
|
-
if (info && typeof info === "object") {
|
|
1115
|
-
noteTokenCount(usageRecordState);
|
|
1116
|
-
compaction = takeCompactionOnTokenCount(
|
|
1117
|
-
usageRecordState,
|
|
1118
|
-
usageDeltaState.lastTotal,
|
|
1119
|
-
totalUsage,
|
|
1120
|
-
);
|
|
1121
|
-
}
|
|
1122
|
-
const rawDelta = consumeUsageDelta(usageDeltaState, lastUsage, totalUsage);
|
|
1123
|
-
const delta = rawDelta ? normalizeUsage(rawDelta) : null;
|
|
1124
|
-
// Timing consumers receive canonical, deduplicated usage below. A null
|
|
1125
|
-
// delta is a snapshot, not another model response.
|
|
1126
|
-
if (compaction) countCompactionRecord(compaction);
|
|
1127
|
-
const isNewResumeEvent = noteTokenEvent(ts, lastUsage, totalUsage);
|
|
1128
|
-
if (inRequestedRange) {
|
|
1129
|
-
const eventSessionId = sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath;
|
|
1130
|
-
const eventKey = `${eventSessionId}:${ts}:${usageEventSignature(lastUsage, totalUsage)}`;
|
|
1131
|
-
if (!isNewResumeEvent || (seenTokenEvents && seenTokenEvents.has(eventKey))) {
|
|
1132
|
-
if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: null, rawUsage: lastUsage, model: attributedModel });
|
|
1133
|
-
attributeTurn(null);
|
|
1134
|
-
} else {
|
|
1135
|
-
if (seenTokenEvents && delta?.total_tokens > 0) seenTokenEvents.add(eventKey);
|
|
1136
|
-
const pricedDelta = recordModelUsage(delta, lastUsage || rawDelta);
|
|
1137
|
-
if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: pricedDelta || delta, rawUsage: lastUsage || rawDelta, model: attributedModel });
|
|
1138
|
-
attributeTurn(delta);
|
|
1139
|
-
}
|
|
1140
|
-
} else {
|
|
1141
|
-
pendingToolNames = [];
|
|
1142
|
-
pendingSkills = [];
|
|
1143
|
-
pendingExecDetails = [];
|
|
1144
|
-
}
|
|
1145
|
-
continue;
|
|
1146
|
-
}
|
|
1147
|
-
}
|
|
1148
|
-
|
|
1149
|
-
const result = {
|
|
1150
|
-
sessionId: sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath,
|
|
1151
|
-
cwd,
|
|
1152
|
-
model: currentCodexModel(modelAttributionState) || model || provider,
|
|
1153
|
-
provider,
|
|
1154
|
-
version: cliVersion,
|
|
1155
|
-
forkedFromId,
|
|
1156
|
-
parentThreadId,
|
|
1157
|
-
threadSource,
|
|
1158
|
-
agentNickname,
|
|
1159
|
-
agentRole,
|
|
1160
|
-
filePath: primaryFilePath,
|
|
1161
|
-
turnCount,
|
|
1162
|
-
totals,
|
|
1163
|
-
modelUsage: finalizeModelUsageRows(byModel),
|
|
1164
|
-
toolBreakdown: {
|
|
1165
|
-
tool_rows: finalizeToolRows(byTool),
|
|
1166
|
-
},
|
|
1167
|
-
skillsBreakdown: {
|
|
1168
|
-
skill_rows: finalizeSkillRows(bySkill),
|
|
1169
|
-
},
|
|
1170
|
-
execCommandBreakdown: {
|
|
1171
|
-
byType: finalizeExecRows(byExecKind),
|
|
1172
|
-
byExit: finalizeExecRows(byExecExit),
|
|
1173
|
-
byExecutable: finalizeExecRows(byExecExecutable),
|
|
1174
|
-
byCommand: finalizeExecRows(byExecCommand),
|
|
1175
|
-
byDuration: finalizeExecRows(byExecDuration),
|
|
1176
|
-
byOutput: finalizeExecRows(byExecOutput),
|
|
1177
|
-
},
|
|
1178
|
-
};
|
|
1179
|
-
if (captureResumeState && filePaths.length === 1) {
|
|
1180
|
-
result.resumeState = {
|
|
1181
|
-
sessionId,
|
|
1182
|
-
cwd,
|
|
1183
|
-
model,
|
|
1184
|
-
modelAttributionState: snapshotCodexModelAttributionState(modelAttributionState),
|
|
1185
|
-
serviceTier,
|
|
1186
|
-
provider,
|
|
1187
|
-
cliVersion,
|
|
1188
|
-
forkedFromId,
|
|
1189
|
-
parentThreadId,
|
|
1190
|
-
threadSource,
|
|
1191
|
-
agentNickname,
|
|
1192
|
-
agentRole,
|
|
1193
|
-
sessionLineageCaptured,
|
|
1194
|
-
prevTotals: usageDeltaState.lastTotal,
|
|
1195
|
-
tokenUsageBaselines: snapshotUsageBaselines(usageDeltaState),
|
|
1196
|
-
pendingToolNames,
|
|
1197
|
-
pendingSkills,
|
|
1198
|
-
pendingExecDetails,
|
|
1199
|
-
lastTokenTimestamp,
|
|
1200
|
-
lastTimestampEventSignatures: Array.from(materializeLastTimestampSignatures()),
|
|
1201
|
-
usageRecordState: snapshotUsageRecordState(usageRecordState),
|
|
1202
|
-
compactionResponseIds: Array.from(compactionResponseIds),
|
|
1203
|
-
};
|
|
1204
|
-
result.endOffset = readProgress.endOffset;
|
|
1205
|
-
// Compressed snapshots are not append-only logs: endOffset stays at the
|
|
1206
|
-
// pre-decode cursor (often 0) and must never be marked appendable, or the
|
|
1207
|
-
// next growth would "resume" by parsing compressed bytes as JSONL.
|
|
1208
|
-
const hasCompressedArchive = filePaths.some(
|
|
1209
|
-
(candidate) => typeof candidate === "string" && /\.jsonl\.zstd?$/i.test(candidate),
|
|
1210
|
-
);
|
|
1211
|
-
result.appendable = Boolean(contentHasher) && !hasCompressedArchive && (
|
|
1212
|
-
readProgress.endOffset === 0 || readProgress.lastByte === 0x0a
|
|
1213
|
-
);
|
|
1214
|
-
result.contentHashState = contentHasher;
|
|
1215
|
-
}
|
|
1216
|
-
return result;
|
|
1217
|
-
}
|
|
1218
|
-
|
|
1219
|
-
const fatalUtf8Decoder = new TextDecoder("utf-8", { fatal: true });
|
|
1220
|
-
|
|
1221
|
-
function parseCodexLine(lineBuffer, diagnostics) {
|
|
1222
|
-
let content = lineBuffer;
|
|
1223
|
-
if (content.length > 0 && content[content.length - 1] === 0x0d) {
|
|
1224
|
-
content = content.subarray(0, content.length - 1);
|
|
1225
|
-
}
|
|
1226
|
-
if (content.length === 0) return null;
|
|
1227
|
-
try {
|
|
1228
|
-
if (diagnostics) diagnostics.json_parse_calls += 1;
|
|
1229
|
-
return JSON.parse(fatalUtf8Decoder.decode(content));
|
|
1230
|
-
} catch {
|
|
1231
|
-
return null;
|
|
1232
|
-
}
|
|
1233
|
-
}
|
|
1234
|
-
|
|
1235
|
-
async function* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
|
|
1236
|
-
startOffset = 0,
|
|
1237
|
-
endOffset = null,
|
|
1238
|
-
readProgress = null,
|
|
1239
|
-
contentHasher = null,
|
|
1240
|
-
sourceHandle = null,
|
|
1241
|
-
} = {}) {
|
|
1242
|
-
const start = Math.max(0, Number(startOffset) || 0);
|
|
1243
|
-
const requestedEnd = Number(endOffset);
|
|
1244
|
-
const hasBoundedEnd = endOffset !== null && endOffset !== undefined &&
|
|
1245
|
-
Number.isFinite(requestedEnd) && requestedEnd >= 0;
|
|
1246
|
-
if (hasBoundedEnd && requestedEnd <= start) {
|
|
1247
|
-
if (readProgress) readProgress.endOffset = start;
|
|
1248
|
-
return;
|
|
1249
|
-
}
|
|
1250
|
-
|
|
1251
|
-
if (diagnostics) diagnostics.opened_files += 1;
|
|
1252
|
-
const stream = sourceHandle && typeof sourceHandle.read === "function"
|
|
1253
|
-
? null
|
|
1254
|
-
: fs.createReadStream(filePath, {
|
|
1255
|
-
start,
|
|
1256
|
-
...(hasBoundedEnd ? { end: requestedEnd - 1 } : {}),
|
|
1257
|
-
});
|
|
1258
|
-
const chunks = stream || (async function* readStableHandleChunks() {
|
|
1259
|
-
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
1260
|
-
const limit = hasBoundedEnd ? requestedEnd : Number.MAX_SAFE_INTEGER;
|
|
1261
|
-
let offset = start;
|
|
1262
|
-
while (offset < limit) {
|
|
1263
|
-
const length = Math.min(buffer.length, limit - offset);
|
|
1264
|
-
const result = await sourceHandle.read(buffer, 0, length, offset);
|
|
1265
|
-
if (!result.bytesRead) break;
|
|
1266
|
-
offset += result.bytesRead;
|
|
1267
|
-
yield buffer.subarray(0, result.bytesRead);
|
|
1268
|
-
}
|
|
1269
|
-
})();
|
|
1270
|
-
const fragments = [];
|
|
1271
|
-
let fragmentsBytes = 0;
|
|
1272
|
-
let nextChunkOffset = start;
|
|
1273
|
-
let lineStartOffset = start;
|
|
1274
|
-
let lineIndex = 0;
|
|
1275
|
-
|
|
1276
|
-
try {
|
|
1277
|
-
for await (const value of chunks) {
|
|
1278
|
-
const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
|
1279
|
-
const chunkStart = nextChunkOffset;
|
|
1280
|
-
nextChunkOffset += chunk.length;
|
|
1281
|
-
if (contentHasher) contentHasher.update(chunk);
|
|
1282
|
-
if (readProgress && chunk.length > 0) {
|
|
1283
|
-
readProgress.lastByte = chunk[chunk.length - 1];
|
|
1284
|
-
}
|
|
1285
|
-
if (diagnostics) {
|
|
1286
|
-
diagnostics.bytes_read = Number(diagnostics.bytes_read || 0) + chunk.length;
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
let cursor = 0;
|
|
1290
|
-
while (cursor < chunk.length) {
|
|
1291
|
-
const newline = chunk.indexOf(0x0a, cursor);
|
|
1292
|
-
if (newline < 0) {
|
|
1293
|
-
const tail = chunk.subarray(cursor);
|
|
1294
|
-
if (tail.length > 0) {
|
|
1295
|
-
// Stable-handle reads reuse their buffer, so retained fragments
|
|
1296
|
-
// must own their bytes before the next read overwrites it.
|
|
1297
|
-
fragments.push(Buffer.from(tail));
|
|
1298
|
-
fragmentsBytes += tail.length;
|
|
1299
|
-
}
|
|
1300
|
-
break;
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
const tail = chunk.subarray(cursor, newline);
|
|
1304
|
-
const lineBuffer = fragments.length > 0
|
|
1305
|
-
? Buffer.concat([...fragments, tail], fragmentsBytes + tail.length)
|
|
1306
|
-
: tail;
|
|
1307
|
-
fragments.length = 0;
|
|
1308
|
-
fragmentsBytes = 0;
|
|
1309
|
-
|
|
1310
|
-
const absoluteEnd = chunkStart + newline + 1;
|
|
1311
|
-
if (readProgress) readProgress.endOffset = absoluteEnd;
|
|
1312
|
-
lineStartOffset = absoluteEnd;
|
|
1313
|
-
const obj = parseCodexLine(lineBuffer, diagnostics);
|
|
1314
|
-
if (obj) {
|
|
1315
|
-
yield { obj, fileIndex, lineIndex };
|
|
1316
|
-
lineIndex += 1;
|
|
1317
|
-
}
|
|
1318
|
-
cursor = newline + 1;
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1321
|
-
|
|
1322
|
-
if (fragmentsBytes > 0) {
|
|
1323
|
-
const lineBuffer = fragments.length === 1
|
|
1324
|
-
? fragments[0]
|
|
1325
|
-
: Buffer.concat(fragments, fragmentsBytes);
|
|
1326
|
-
const obj = parseCodexLine(lineBuffer, diagnostics);
|
|
1327
|
-
if (obj) {
|
|
1328
|
-
if (readProgress) readProgress.endOffset = lineStartOffset + fragmentsBytes;
|
|
1329
|
-
yield { obj, fileIndex, lineIndex };
|
|
1330
|
-
}
|
|
1331
|
-
}
|
|
1332
|
-
} finally {
|
|
1333
|
-
if (readProgress) readProgress.endOffset = nextChunkOffset;
|
|
1334
|
-
stream?.destroy();
|
|
1335
|
-
if (diagnostics) diagnostics.parsed_files += 1;
|
|
1336
|
-
}
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
async function* readCodexObjectsFull(filePath, diagnostics, fileIndex, {
|
|
1340
|
-
startOffset = 0,
|
|
1341
|
-
endOffset = null,
|
|
1342
|
-
readProgress = null,
|
|
1343
|
-
contentHasher = null,
|
|
1344
|
-
} = {}) {
|
|
1345
|
-
// Compressed archives are immutable snapshots, not append-only logs: byte
|
|
1346
|
-
// offsets are meaningless inside them, so decode the whole text and split
|
|
1347
|
-
// into lines. `readProgress` stays null so callers never persist a fake
|
|
1348
|
-
// byte cursor for them.
|
|
1349
|
-
if (typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath)) {
|
|
1350
|
-
const text = await readCodexRolloutText(filePath);
|
|
1351
|
-
let lineIndex = 0;
|
|
1352
|
-
const lines = text.split("\n");
|
|
1353
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
1354
|
-
const isLast = index + 1 >= lines.length;
|
|
1355
|
-
if (isLast && lines[index] === "") continue;
|
|
1356
|
-
const lineBuffer = Buffer.from(lines[index], "utf8");
|
|
1357
|
-
const obj = parseCodexLine(lineBuffer, diagnostics);
|
|
1358
|
-
if (obj) {
|
|
1359
|
-
yield { obj, fileIndex, lineIndex };
|
|
1360
|
-
lineIndex += 1;
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1363
|
-
if (diagnostics) diagnostics.parsed_files += 1;
|
|
1364
|
-
return;
|
|
1365
|
-
}
|
|
1366
|
-
yield* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
|
|
1367
|
-
startOffset,
|
|
1368
|
-
endOffset,
|
|
1369
|
-
readProgress,
|
|
1370
|
-
contentHasher,
|
|
1371
|
-
});
|
|
1372
|
-
}
|
|
1373
|
-
|
|
1374
|
-
async function* readCodexObjects(filePath, diagnostics, fileIndex, options = {}) {
|
|
1375
|
-
const start = Math.max(0, Number(options.startOffset) || 0);
|
|
1376
|
-
const reader = start > 0 || options.sourceHandle
|
|
1377
|
-
? readCodexObjectsIncremental
|
|
1378
|
-
: readCodexObjectsFull;
|
|
1379
|
-
yield* reader(filePath, diagnostics, fileIndex, options);
|
|
1380
|
-
}
|
|
1381
|
-
|
|
1382
|
-
function isRecoverableGroupedReadError(error) {
|
|
1383
|
-
return ["ENOENT", "EACCES", "EPERM", "EISDIR", "EIO", "ESTALE"].includes(error?.code);
|
|
1384
|
-
}
|
|
1385
|
-
|
|
1386
|
-
async function* iterateCodexObjects(filePaths, diagnostics, options = {}) {
|
|
1387
|
-
if (filePaths.length <= 1) {
|
|
1388
|
-
if (!filePaths[0]) return;
|
|
1389
|
-
for await (const record of readCodexObjects(filePaths[0], diagnostics, 0, options)) {
|
|
1390
|
-
yield record.obj;
|
|
1391
|
-
}
|
|
1392
|
-
return;
|
|
1393
|
-
}
|
|
1394
|
-
|
|
1395
|
-
// A live rollout can overlap its archived copy or be split into prefix/suffix
|
|
1396
|
-
// fragments. Merge only duplicate-session groups, order by event time, and
|
|
1397
|
-
// suppress byte-semantically identical records from different files so the
|
|
1398
|
-
// cumulative baseline is established exactly once.
|
|
1399
|
-
const records = [];
|
|
1400
|
-
let scannedFiles = 0;
|
|
1401
|
-
for (let fileIndex = 0; fileIndex < filePaths.length; fileIndex += 1) {
|
|
1402
|
-
const fileRecords = [];
|
|
1403
|
-
try {
|
|
1404
|
-
for await (const record of readCodexObjects(filePaths[fileIndex], diagnostics, fileIndex)) {
|
|
1405
|
-
fileRecords.push(record);
|
|
1406
|
-
}
|
|
1407
|
-
for (const record of fileRecords) records.push(record);
|
|
1408
|
-
scannedFiles += 1;
|
|
1409
|
-
} catch (error) {
|
|
1410
|
-
if (!isRecoverableGroupedReadError(error)) throw error;
|
|
1411
|
-
}
|
|
1412
|
-
}
|
|
1413
|
-
if (!scannedFiles) {
|
|
1414
|
-
const error = new Error("all grouped Codex rollout files failed during read");
|
|
1415
|
-
error.code = "EIO";
|
|
1416
|
-
throw error;
|
|
1417
|
-
}
|
|
1418
|
-
records.sort((a, b) => {
|
|
1419
|
-
const aTimestamp = typeof a.obj?.timestamp === "string" ? a.obj.timestamp : "";
|
|
1420
|
-
const bTimestamp = typeof b.obj?.timestamp === "string" ? b.obj.timestamp : "";
|
|
1421
|
-
return aTimestamp.localeCompare(bTimestamp) || a.fileIndex - b.fileIndex || a.lineIndex - b.lineIndex;
|
|
1422
|
-
});
|
|
1423
|
-
const firstFileByRecord = new Map();
|
|
1424
|
-
for (const record of records) {
|
|
1425
|
-
const semanticKey = JSON.stringify(record.obj);
|
|
1426
|
-
const firstFile = firstFileByRecord.get(semanticKey);
|
|
1427
|
-
if (firstFile !== undefined && firstFile !== record.fileIndex) continue;
|
|
1428
|
-
if (firstFile === undefined) firstFileByRecord.set(semanticKey, record.fileIndex);
|
|
1429
|
-
yield record.obj;
|
|
1430
|
-
}
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
function rolloutSessionIdFromPath(filePath) {
|
|
1434
|
-
const match = path.basename(String(filePath || "")).match(
|
|
1435
|
-
/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl(?:\.zstd?)?$/i,
|
|
1436
|
-
);
|
|
1437
|
-
return match ? match[1] : null;
|
|
1438
|
-
}
|
|
1439
|
-
|
|
1440
|
-
function isCodexResumeInvalidated(error) {
|
|
1441
|
-
return error?.code === CODEX_RESUME_INVALIDATED;
|
|
1442
|
-
}
|
|
1443
|
-
|
|
1444
|
-
module.exports = {
|
|
1445
|
-
parseCodexRolloutFile,
|
|
1446
|
-
extractTokenCount,
|
|
1447
|
-
extractSkillNameFromFunctionCall,
|
|
1448
|
-
formatToolDisplayName,
|
|
1449
|
-
normalizeToolName,
|
|
1450
|
-
pickDelta,
|
|
1451
|
-
normalizeUsage,
|
|
1452
|
-
totalsReset,
|
|
1453
|
-
listJsonlFiles,
|
|
1454
|
-
listCodexSessionFiles,
|
|
1455
|
-
discoverCodexSessionFiles,
|
|
1456
|
-
rolloutPathDay,
|
|
1457
|
-
isConservativeRangeCandidate,
|
|
1458
|
-
safeJsonParse,
|
|
1459
|
-
dayKeyToIsoBounds,
|
|
1460
|
-
formatPartsDayKey,
|
|
1461
|
-
getZonedParts,
|
|
1462
|
-
timestampDayKey,
|
|
1463
|
-
isTimestampInRequestedDayRange,
|
|
1464
|
-
rolloutSessionIdFromPath,
|
|
1465
|
-
isCodexResumeInvalidated,
|
|
1466
|
-
isRecoverableGroupedReadError,
|
|
1467
|
-
finalizeToolRows,
|
|
1468
|
-
finalizeSkillRows,
|
|
1469
|
-
finalizeExecRows,
|
|
1470
|
-
buildSkillStatsEntry,
|
|
1471
|
-
};
|
|
1
|
+
// Codex rollout JSONL parser — extracted from codex-context-breakdown.js.
|
|
2
|
+
//
|
|
3
|
+
// Handles file discovery and per-file parsing. Does NOT hold any aggregation
|
|
4
|
+
// state; callers (computeCodexContextBreakdown) own the merge step.
|
|
5
|
+
|
|
6
|
+
"use strict";
|
|
7
|
+
|
|
8
|
+
const crypto = require("node:crypto");
|
|
9
|
+
const fs = require("node:fs");
|
|
10
|
+
const path = require("node:path");
|
|
11
|
+
|
|
12
|
+
const { listRolloutFiles } = require("./rollout");
|
|
13
|
+
const {
|
|
14
|
+
consumeUsageDelta,
|
|
15
|
+
createUsageDeltaState,
|
|
16
|
+
snapshotUsageBaselines,
|
|
17
|
+
totalsReset,
|
|
18
|
+
} = require("./codex-token-usage");
|
|
19
|
+
const {
|
|
20
|
+
emptyTotals,
|
|
21
|
+
addInto,
|
|
22
|
+
inferExecCommandKind,
|
|
23
|
+
sanitizeCommandSignature,
|
|
24
|
+
getExecutableName,
|
|
25
|
+
buildExecStatsEntry,
|
|
26
|
+
} = require("./categorizer-utils");
|
|
27
|
+
const {
|
|
28
|
+
applyCodexModelEvent,
|
|
29
|
+
createCodexModelAttributionState,
|
|
30
|
+
currentCodexModel,
|
|
31
|
+
snapshotCodexModelAttributionState,
|
|
32
|
+
} = require("./codex-model-attribution");
|
|
33
|
+
const { readCodexServiceTier, isPriorityServiceTier } = require("./codex-service-tier");
|
|
34
|
+
const {
|
|
35
|
+
createUsageRecordState,
|
|
36
|
+
extractTokenUsageRecord,
|
|
37
|
+
noteLineAfterUsageRecord,
|
|
38
|
+
noteTokenCount,
|
|
39
|
+
noteUsageRecord,
|
|
40
|
+
snapshotUsageRecordState,
|
|
41
|
+
takeCompactionOnTokenCount,
|
|
42
|
+
} = require("./codex-usage-record");
|
|
43
|
+
|
|
44
|
+
const DISCOVERY_FULL_AUDIT_INTERVAL_MS = 60 * 1000;
|
|
45
|
+
const CODEX_ROLLOUT_TEXT_MAX_BYTES = 256 * 1024 * 1024;
|
|
46
|
+
|
|
47
|
+
async function readCodexRolloutText(filePath, { maxOutputBytes = CODEX_ROLLOUT_TEXT_MAX_BYTES } = {}) {
|
|
48
|
+
if (typeof filePath !== "string" || !/\.jsonl\.zstd?$/i.test(filePath)) {
|
|
49
|
+
return fs.readFile(filePath, "utf8");
|
|
50
|
+
}
|
|
51
|
+
// The snapshot decoder lives in rollout.js (reused by the sync parser);
|
|
52
|
+
// require lazily so this module stays loadable without pulling the whole
|
|
53
|
+
// rollout graph at import time.
|
|
54
|
+
const rollout = require("./rollout");
|
|
55
|
+
const snapshot = await rollout.readCodexRolloutSnapshot(filePath, { maxOutputBytes });
|
|
56
|
+
if (!snapshot) {
|
|
57
|
+
// Treat undecodable / vanished archives as recoverable I/O so grouped
|
|
58
|
+
// session reads (live + archive) can still use a readable sibling copy.
|
|
59
|
+
const error = new Error(`unreadable Codex archive ${filePath}`);
|
|
60
|
+
error.code = "EIO";
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
// Snapshot lines keep their "\n" terminators; the tail element may be an
|
|
64
|
+
// unterminated final line — join preserves exactly that.
|
|
65
|
+
return snapshot.lines.join("\n");
|
|
66
|
+
}
|
|
67
|
+
const MAX_DISCOVERY_INVENTORIES = 32;
|
|
68
|
+
const DISCOVERY_INVENTORIES = new Map();
|
|
69
|
+
const ZONED_DAY_FORMATTERS = new Map();
|
|
70
|
+
const CODEX_RESUME_INVALIDATED = "TOKENUSAGE_CODEX_RESUME_INVALIDATED";
|
|
71
|
+
// Match src/lib/pricing/index.js. Classify individual requests using raw
|
|
72
|
+
// input (including cache), not the accumulated session or day totals.
|
|
73
|
+
const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272_000;
|
|
74
|
+
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
// Timezone helpers
|
|
77
|
+
// ---------------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
function dayKeyToIsoBounds(from, to) {
|
|
80
|
+
if (!from && !to) return { fromIso: null, toIso: null };
|
|
81
|
+
const fromDate = from ? new Date(`${from}T00:00:00Z`) : null;
|
|
82
|
+
const toDate = to ? new Date(`${to}T23:59:59Z`) : null;
|
|
83
|
+
if (fromDate && Number.isFinite(fromDate.getTime())) fromDate.setUTCHours(fromDate.getUTCHours() - 14);
|
|
84
|
+
if (toDate && Number.isFinite(toDate.getTime())) toDate.setUTCHours(toDate.getUTCHours() + 14);
|
|
85
|
+
return {
|
|
86
|
+
fromIso: fromDate ? fromDate.toISOString() : null,
|
|
87
|
+
toIso: toDate ? toDate.toISOString() : null,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function formatPartsDayKey(parts) {
|
|
92
|
+
if (!parts) return "";
|
|
93
|
+
const values = {};
|
|
94
|
+
for (const part of parts) {
|
|
95
|
+
if (part.type !== "literal") values[part.type] = part.value;
|
|
96
|
+
}
|
|
97
|
+
if (!values.year || !values.month || !values.day) return "";
|
|
98
|
+
return `${values.year}-${values.month}-${values.day}`;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function getZonedParts(date, timeZoneContext = {}) {
|
|
102
|
+
const { timeZone, offsetMinutes } = timeZoneContext || {};
|
|
103
|
+
const dt = date instanceof Date ? date : new Date(date);
|
|
104
|
+
if (!Number.isFinite(dt.getTime())) return null;
|
|
105
|
+
|
|
106
|
+
if (timeZone && typeof Intl !== "undefined" && Intl.DateTimeFormat) {
|
|
107
|
+
try {
|
|
108
|
+
let formatter = ZONED_DAY_FORMATTERS.get(timeZone);
|
|
109
|
+
if (!formatter) {
|
|
110
|
+
formatter = new Intl.DateTimeFormat("en-CA", {
|
|
111
|
+
timeZone,
|
|
112
|
+
year: "numeric",
|
|
113
|
+
month: "2-digit",
|
|
114
|
+
day: "2-digit",
|
|
115
|
+
hourCycle: "h23",
|
|
116
|
+
});
|
|
117
|
+
ZONED_DAY_FORMATTERS.set(timeZone, formatter);
|
|
118
|
+
}
|
|
119
|
+
return formatter.formatToParts(dt);
|
|
120
|
+
} catch {
|
|
121
|
+
// Fall through to offset handling.
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (Number.isFinite(offsetMinutes)) {
|
|
126
|
+
const shifted = new Date(dt.getTime() - Number(offsetMinutes) * 60_000);
|
|
127
|
+
return [
|
|
128
|
+
{ type: "year", value: String(shifted.getUTCFullYear()).padStart(4, "0") },
|
|
129
|
+
{ type: "month", value: String(shifted.getUTCMonth() + 1).padStart(2, "0") },
|
|
130
|
+
{ type: "day", value: String(shifted.getUTCDate()).padStart(2, "0") },
|
|
131
|
+
];
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function timestampDayKey(timestamp, timeZoneContext) {
|
|
138
|
+
const ts = typeof timestamp === "string" ? timestamp : "";
|
|
139
|
+
if (!ts) return "";
|
|
140
|
+
const parts = getZonedParts(ts, timeZoneContext);
|
|
141
|
+
const zoned = formatPartsDayKey(parts);
|
|
142
|
+
if (zoned) return zoned;
|
|
143
|
+
return ts.slice(0, 10);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function isTimestampInRequestedDayRange(timestamp, { from, to, timeZoneContext } = {}) {
|
|
147
|
+
if (!from && !to) return true;
|
|
148
|
+
const day = timestampDayKey(timestamp, timeZoneContext);
|
|
149
|
+
if (!day) return false;
|
|
150
|
+
if (from && day < from) return false;
|
|
151
|
+
if (to && day > to) return false;
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ---------------------------------------------------------------------------
|
|
156
|
+
// File discovery
|
|
157
|
+
// ---------------------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
function safeJsonParse(str, diagnostics = null) {
|
|
160
|
+
try {
|
|
161
|
+
if (diagnostics) diagnostics.json_parse_calls += 1;
|
|
162
|
+
return JSON.parse(str);
|
|
163
|
+
} catch {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function isRolloutCandidateFileName(name) {
|
|
169
|
+
if (typeof name !== "string") return false;
|
|
170
|
+
return name.endsWith(".jsonl") || /^rollout-.*\.jsonl\.zstd?$/.test(name);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function listJsonlFiles(rootDir) {
|
|
174
|
+
const out = [];
|
|
175
|
+
const stack = [rootDir];
|
|
176
|
+
|
|
177
|
+
while (stack.length > 0) {
|
|
178
|
+
const dir = stack.pop();
|
|
179
|
+
let entries;
|
|
180
|
+
try {
|
|
181
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
182
|
+
} catch {
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
for (const entry of entries) {
|
|
187
|
+
const filePath = path.join(dir, entry.name);
|
|
188
|
+
if (entry.isDirectory()) {
|
|
189
|
+
stack.push(filePath);
|
|
190
|
+
} else if (entry.isFile() && isRolloutCandidateFileName(entry.name)) {
|
|
191
|
+
out.push(filePath);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
out.sort((a, b) => a.localeCompare(b));
|
|
197
|
+
return out;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function rolloutPathDay(filePath) {
|
|
201
|
+
const normalized = String(filePath || "").replaceAll("\\", "/");
|
|
202
|
+
const fileMatch = path.basename(normalized).match(/(?:^|rollout-)(\d{4})-(\d{2})-(\d{2})/);
|
|
203
|
+
if (fileMatch) return `${fileMatch[1]}-${fileMatch[2]}-${fileMatch[3]}`;
|
|
204
|
+
const directoryMatch = normalized.match(/.*\/(\d{4})\/(\d{2})\/(\d{2})(?:\/|$)/);
|
|
205
|
+
return directoryMatch
|
|
206
|
+
? `${directoryMatch[1]}-${directoryMatch[2]}-${directoryMatch[3]}`
|
|
207
|
+
: null;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function nextDayKey(dayKey) {
|
|
211
|
+
if (!dayKey) return null;
|
|
212
|
+
const date = new Date(`${dayKey}T00:00:00.000Z`);
|
|
213
|
+
if (!Number.isFinite(date.getTime())) return null;
|
|
214
|
+
date.setUTCDate(date.getUTCDate() + 1);
|
|
215
|
+
return date.toISOString().slice(0, 10);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function metadataRangeBounds(from, to, timeZoneContext) {
|
|
219
|
+
const offsetMinutes = timeZoneContext?.offsetMinutes;
|
|
220
|
+
if (Number.isFinite(offsetMinutes)) {
|
|
221
|
+
const dstGuardMs = 2 * 60 * 60 * 1000;
|
|
222
|
+
const fromUtc = from ? Date.parse(`${from}T00:00:00.000Z`) : null;
|
|
223
|
+
const afterTo = to ? nextDayKey(to) : null;
|
|
224
|
+
const toUtcExclusive = afterTo ? Date.parse(`${afterTo}T00:00:00.000Z`) : null;
|
|
225
|
+
return {
|
|
226
|
+
fromMs: Number.isFinite(fromUtc)
|
|
227
|
+
? fromUtc + offsetMinutes * 60_000 - dstGuardMs
|
|
228
|
+
: null,
|
|
229
|
+
toMs: Number.isFinite(toUtcExclusive)
|
|
230
|
+
? toUtcExclusive + offsetMinutes * 60_000 + dstGuardMs
|
|
231
|
+
: null,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
const { fromIso, toIso } = dayKeyToIsoBounds(from, to);
|
|
235
|
+
return {
|
|
236
|
+
fromMs: fromIso ? Date.parse(fromIso) : null,
|
|
237
|
+
toMs: toIso ? Date.parse(toIso) : null,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function isConservativeRangeCandidate(filePath, stat, {
|
|
242
|
+
from = null,
|
|
243
|
+
to = null,
|
|
244
|
+
timeZoneContext = null,
|
|
245
|
+
metadataBounds = null,
|
|
246
|
+
} = {}) {
|
|
247
|
+
if (!from && !to) return true;
|
|
248
|
+
// With no lower bound, a later-path rewrite can still contain an event at or
|
|
249
|
+
// before `to`. There is no metadata-only exclusion that can prove otherwise.
|
|
250
|
+
if (!from) return true;
|
|
251
|
+
const pathDay = rolloutPathDay(filePath);
|
|
252
|
+
if (!pathDay) return true;
|
|
253
|
+
|
|
254
|
+
const pathOverlaps = (!from || pathDay >= from) && (!to || pathDay <= to);
|
|
255
|
+
if (pathOverlaps) return true;
|
|
256
|
+
|
|
257
|
+
// Old sessions can be appended or rewritten. A post-range mtime or ctime is
|
|
258
|
+
// therefore a reason to parse, never a reason to exclude. ctime also catches
|
|
259
|
+
// cold-process inode replacement/truncation when a writer preserves mtime.
|
|
260
|
+
// Future-dated decoys are kept conservatively; correctness wins over an open.
|
|
261
|
+
const bounds = metadataBounds || metadataRangeBounds(from, to, timeZoneContext);
|
|
262
|
+
const fromMs = bounds.fromMs;
|
|
263
|
+
const toMs = bounds.toMs;
|
|
264
|
+
const mtimeMs = Number(stat?.mtimeMs);
|
|
265
|
+
const ctimeMs = Number(stat?.ctimeMs);
|
|
266
|
+
const changedAfterStart = (value) => (
|
|
267
|
+
Number.isFinite(value) && (!Number.isFinite(fromMs) || value >= fromMs)
|
|
268
|
+
);
|
|
269
|
+
const changedInsideRange = (value) => (
|
|
270
|
+
changedAfterStart(value) && (!Number.isFinite(toMs) || value <= toMs)
|
|
271
|
+
);
|
|
272
|
+
if (pathDay < from) {
|
|
273
|
+
return changedAfterStart(mtimeMs) || changedAfterStart(ctimeMs);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// A path one day ahead can still overlap the requested browser day because
|
|
277
|
+
// rollout folders follow the writer's local date. Its current mtime may be
|
|
278
|
+
// later if the same session kept growing, so metadata alone cannot exclude it.
|
|
279
|
+
if (to && pathDay === nextDayKey(to)) return true;
|
|
280
|
+
|
|
281
|
+
// A host-local path day can be later than the requested browser day when
|
|
282
|
+
// their timezones differ. Keep it only when filesystem metadata maps back
|
|
283
|
+
// into the requested day range; do not pull every genuinely future session
|
|
284
|
+
// into a historical query.
|
|
285
|
+
return changedInsideRange(mtimeMs) || changedInsideRange(ctimeMs);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function isRolloutPathInRequestedRange(filePath, { from = null, to = null } = {}) {
|
|
289
|
+
const pathDay = rolloutPathDay(filePath);
|
|
290
|
+
if (!pathDay) return true;
|
|
291
|
+
if (from && pathDay < from) return false;
|
|
292
|
+
if (to && pathDay > to) return false;
|
|
293
|
+
return true;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function discoveryInventoryKey(roots) {
|
|
297
|
+
return roots.map((root) => path.resolve(root)).sort().join("\0");
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function getDiscoveryInventory(roots, nowMs) {
|
|
301
|
+
const key = discoveryInventoryKey(roots);
|
|
302
|
+
let inventory = DISCOVERY_INVENTORIES.get(key);
|
|
303
|
+
if (!inventory) {
|
|
304
|
+
inventory = { files: new Map(), lastFullAuditAtMs: 0, lastUsedAtMs: nowMs };
|
|
305
|
+
DISCOVERY_INVENTORIES.set(key, inventory);
|
|
306
|
+
}
|
|
307
|
+
inventory.lastUsedAtMs = nowMs;
|
|
308
|
+
while (DISCOVERY_INVENTORIES.size > MAX_DISCOVERY_INVENTORIES) {
|
|
309
|
+
let oldestKey = null;
|
|
310
|
+
let oldestAt = Infinity;
|
|
311
|
+
for (const [candidateKey, candidate] of DISCOVERY_INVENTORIES) {
|
|
312
|
+
if (candidateKey === key) continue;
|
|
313
|
+
if (candidate.lastUsedAtMs < oldestAt) {
|
|
314
|
+
oldestAt = candidate.lastUsedAtMs;
|
|
315
|
+
oldestKey = candidateKey;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (!oldestKey) break;
|
|
319
|
+
DISCOVERY_INVENTORIES.delete(oldestKey);
|
|
320
|
+
}
|
|
321
|
+
return inventory;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function discoverCodexSessionFiles(rootDirs, options = {}) {
|
|
325
|
+
const diagnostics = options.diagnostics || null;
|
|
326
|
+
const roots = Array.isArray(rootDirs) ? rootDirs : [rootDirs];
|
|
327
|
+
const nowMs = Number.isFinite(Number(options.nowMs)) ? Number(options.nowMs) : Date.now();
|
|
328
|
+
const inventory = getDiscoveryInventory(roots.filter(Boolean), nowMs);
|
|
329
|
+
const candidateOptions = {
|
|
330
|
+
...options,
|
|
331
|
+
metadataBounds: metadataRangeBounds(options.from, options.to, options.timeZoneContext),
|
|
332
|
+
};
|
|
333
|
+
const bounded = Boolean(options.from || options.to);
|
|
334
|
+
const auditAgeMs = nowMs - Number(inventory.lastFullAuditAtMs || 0);
|
|
335
|
+
const fullMetadataAudit = Boolean(
|
|
336
|
+
options.exhaustive ||
|
|
337
|
+
!bounded ||
|
|
338
|
+
!Number.isFinite(auditAgeMs) ||
|
|
339
|
+
auditAgeMs < 0 ||
|
|
340
|
+
auditAgeMs >= DISCOVERY_FULL_AUDIT_INTERVAL_MS
|
|
341
|
+
);
|
|
342
|
+
if (diagnostics) diagnostics.full_metadata_audit = fullMetadataAudit;
|
|
343
|
+
const entries = [];
|
|
344
|
+
const visited = new Set();
|
|
345
|
+
const stack = roots.filter(Boolean);
|
|
346
|
+
|
|
347
|
+
while (stack.length > 0) {
|
|
348
|
+
const dir = stack.pop();
|
|
349
|
+
let children;
|
|
350
|
+
try {
|
|
351
|
+
children = fs.readdirSync(dir, { withFileTypes: true });
|
|
352
|
+
} catch {
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
for (const child of children) {
|
|
356
|
+
const filePath = path.join(dir, child.name);
|
|
357
|
+
if (child.isDirectory()) {
|
|
358
|
+
stack.push(filePath);
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
if (!child.isFile() || !isRolloutCandidateFileName(child.name) || visited.has(filePath)) continue;
|
|
362
|
+
visited.add(filePath);
|
|
363
|
+
if (diagnostics) diagnostics.discovered_files += 1;
|
|
364
|
+
const cached = inventory.files.get(filePath) || null;
|
|
365
|
+
const cachedCandidate = cached
|
|
366
|
+
? isConservativeRangeCandidate(filePath, cached, candidateOptions)
|
|
367
|
+
: false;
|
|
368
|
+
const shouldStat = Boolean(
|
|
369
|
+
fullMetadataAudit ||
|
|
370
|
+
!cached ||
|
|
371
|
+
cachedCandidate ||
|
|
372
|
+
isRolloutPathInRequestedRange(filePath, options)
|
|
373
|
+
);
|
|
374
|
+
let stat = cached;
|
|
375
|
+
if (shouldStat) {
|
|
376
|
+
try {
|
|
377
|
+
if (diagnostics) diagnostics.stat_calls += 1;
|
|
378
|
+
stat = fs.statSync(filePath);
|
|
379
|
+
inventory.files.set(filePath, stat);
|
|
380
|
+
} catch {
|
|
381
|
+
inventory.files.delete(filePath);
|
|
382
|
+
continue;
|
|
383
|
+
}
|
|
384
|
+
} else if (diagnostics) {
|
|
385
|
+
diagnostics.metadata_cache_hits += 1;
|
|
386
|
+
}
|
|
387
|
+
const candidate = Boolean(
|
|
388
|
+
options.exhaustive ||
|
|
389
|
+
isConservativeRangeCandidate(filePath, stat, candidateOptions),
|
|
390
|
+
);
|
|
391
|
+
entries.push({ filePath, stat, candidate });
|
|
392
|
+
if (candidate && diagnostics) diagnostics.candidate_files += 1;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
for (const filePath of inventory.files.keys()) {
|
|
397
|
+
if (!visited.has(filePath)) inventory.files.delete(filePath);
|
|
398
|
+
}
|
|
399
|
+
if (fullMetadataAudit) inventory.lastFullAuditAtMs = nowMs;
|
|
400
|
+
|
|
401
|
+
return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
async function listCodexSessionFiles(baseDir) {
|
|
405
|
+
const rolloutFiles = await listRolloutFiles(baseDir).catch(() => []);
|
|
406
|
+
const allJsonlFiles = listJsonlFiles(baseDir);
|
|
407
|
+
if (allJsonlFiles.length === 0) return rolloutFiles;
|
|
408
|
+
if (rolloutFiles.length === 0) return allJsonlFiles;
|
|
409
|
+
|
|
410
|
+
const merged = new Set(rolloutFiles);
|
|
411
|
+
for (const filePath of allJsonlFiles) merged.add(filePath);
|
|
412
|
+
return Array.from(merged).sort((a, b) => a.localeCompare(b));
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// ---------------------------------------------------------------------------
|
|
416
|
+
// Token count extraction
|
|
417
|
+
// ---------------------------------------------------------------------------
|
|
418
|
+
|
|
419
|
+
function extractTokenCount(obj) {
|
|
420
|
+
const payload = obj?.payload;
|
|
421
|
+
if (!payload || obj?.type !== "event_msg") return null;
|
|
422
|
+
if (payload.type === "token_count") {
|
|
423
|
+
return { info: payload.info || null, timestamp: obj?.timestamp || null };
|
|
424
|
+
}
|
|
425
|
+
const msg = payload.msg;
|
|
426
|
+
if (msg && msg.type === "token_count") {
|
|
427
|
+
return { info: msg.info || null, timestamp: obj?.timestamp || null };
|
|
428
|
+
}
|
|
429
|
+
return null;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// ---------------------------------------------------------------------------
|
|
433
|
+
// Tool name helpers
|
|
434
|
+
// ---------------------------------------------------------------------------
|
|
435
|
+
|
|
436
|
+
function normalizeToolName(payload) {
|
|
437
|
+
const name = payload?.name || "";
|
|
438
|
+
const ns = payload?.namespace || null;
|
|
439
|
+
if (ns && typeof ns === "string" && ns.startsWith("mcp__")) return `${ns}${name}`;
|
|
440
|
+
return name || "unknown";
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function extractSkillNameFromFunctionCall(payload, diagnostics = null) {
|
|
444
|
+
if (!payload || payload.name !== "exec_command") return null;
|
|
445
|
+
const args = safeJsonParse(payload.arguments || "{}", diagnostics) || {};
|
|
446
|
+
const cmd = String(args.cmd || "");
|
|
447
|
+
const match = cmd.match(/(?:^|\/)skills\/(?:\.system\/)?([^/\s]+)\/SKILL\.md\b/);
|
|
448
|
+
return match ? match[1] : null;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
function formatToolDisplayName(name) {
|
|
452
|
+
if (typeof name !== "string" || !name.startsWith("mcp__")) return name;
|
|
453
|
+
const parts = name.split("__");
|
|
454
|
+
if (parts.length < 3) return name;
|
|
455
|
+
const server = String(parts[1] || "").replace(/^plugin_/, "").replace(/_/g, "-");
|
|
456
|
+
const tool = parts.slice(2).join("__") || name;
|
|
457
|
+
return server ? `${server}/${tool}` : tool;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// ---------------------------------------------------------------------------
|
|
461
|
+
// Usage normalization
|
|
462
|
+
// ---------------------------------------------------------------------------
|
|
463
|
+
|
|
464
|
+
function normalizeUsage(u) {
|
|
465
|
+
const out = {};
|
|
466
|
+
for (const k of [
|
|
467
|
+
"input_tokens",
|
|
468
|
+
"cached_input_tokens",
|
|
469
|
+
"cache_creation_input_tokens",
|
|
470
|
+
"output_tokens",
|
|
471
|
+
"reasoning_output_tokens",
|
|
472
|
+
"total_tokens",
|
|
473
|
+
]) {
|
|
474
|
+
const n = Number(u?.[k] || 0);
|
|
475
|
+
out[k] = Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
|
|
476
|
+
}
|
|
477
|
+
// Codex reports input inclusive of cached_input_tokens; keep our schema
|
|
478
|
+
// convention: non-cached input and cached input tracked separately.
|
|
479
|
+
out.input_tokens = Math.max(0, out.input_tokens - out.cached_input_tokens);
|
|
480
|
+
out.total_tokens =
|
|
481
|
+
out.input_tokens +
|
|
482
|
+
out.cached_input_tokens +
|
|
483
|
+
out.cache_creation_input_tokens +
|
|
484
|
+
out.output_tokens;
|
|
485
|
+
return out;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function pickDelta(lastUsage, totalUsage, prevTotals) {
|
|
489
|
+
const state = createUsageDeltaState({ lastTotal: prevTotals });
|
|
490
|
+
const delta = consumeUsageDelta(state, lastUsage, totalUsage);
|
|
491
|
+
return delta ? normalizeUsage(delta) : null;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// ---------------------------------------------------------------------------
|
|
495
|
+
// Exec stats helpers (local to parser, not shared)
|
|
496
|
+
// ---------------------------------------------------------------------------
|
|
497
|
+
|
|
498
|
+
function durationBucket(ms) {
|
|
499
|
+
const n = Number(ms || 0);
|
|
500
|
+
if (!Number.isFinite(n) || n <= 0) return "unknown";
|
|
501
|
+
if (n < 1000) return "<1s";
|
|
502
|
+
if (n < 10_000) return "1-10s";
|
|
503
|
+
if (n < 60_000) return "10-60s";
|
|
504
|
+
if (n < 300_000) return "1-5m";
|
|
505
|
+
return ">5m";
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
function outputSizeBucket(lines, chars) {
|
|
509
|
+
const l = Number(lines || 0);
|
|
510
|
+
const c = Number(chars || 0);
|
|
511
|
+
if (!l && !c) return "quiet";
|
|
512
|
+
if (l <= 20 && c <= 2_000) return "small";
|
|
513
|
+
if (l <= 200 && c <= 20_000) return "medium";
|
|
514
|
+
if (l <= 1000 && c <= 100_000) return "large";
|
|
515
|
+
return "very_large";
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
function buildToolStatsEntry() {
|
|
519
|
+
return { calls: 0, totals: emptyTotals() };
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function buildSkillStatsEntry(name) {
|
|
523
|
+
return { name, calls: 0, totals: emptyTotals() };
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// ---------------------------------------------------------------------------
|
|
527
|
+
// Finalize helpers
|
|
528
|
+
// ---------------------------------------------------------------------------
|
|
529
|
+
|
|
530
|
+
function finalizeToolRows(map) {
|
|
531
|
+
return Array.from(map.values())
|
|
532
|
+
.map((row) => {
|
|
533
|
+
const rawName = row.raw_name || row.name;
|
|
534
|
+
return {
|
|
535
|
+
name: formatToolDisplayName(rawName),
|
|
536
|
+
raw_name: rawName,
|
|
537
|
+
calls: row.calls,
|
|
538
|
+
totals: row.totals,
|
|
539
|
+
};
|
|
540
|
+
})
|
|
541
|
+
.sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
function finalizeSkillRows(map) {
|
|
545
|
+
return Array.from(map.values())
|
|
546
|
+
.map((row) => ({
|
|
547
|
+
name: row.name,
|
|
548
|
+
calls: row.calls,
|
|
549
|
+
totals: row.totals,
|
|
550
|
+
}))
|
|
551
|
+
.sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function finalizeExecRows(map) {
|
|
555
|
+
return Array.from(map.values())
|
|
556
|
+
.map((row) => ({
|
|
557
|
+
name: row.name,
|
|
558
|
+
calls: row.calls,
|
|
559
|
+
failures: row.failures,
|
|
560
|
+
duration_ms: row.duration_ms,
|
|
561
|
+
max_duration_ms: row.max_duration_ms,
|
|
562
|
+
output_chars: row.output_chars,
|
|
563
|
+
output_lines: row.output_lines,
|
|
564
|
+
totals: row.totals,
|
|
565
|
+
}))
|
|
566
|
+
.sort((a, b) => (b.totals?.total_tokens || 0) - (a.totals?.total_tokens || 0));
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
function finalizeModelUsageRows(map) {
|
|
570
|
+
return Array.from(map.values())
|
|
571
|
+
.map((row) => ({
|
|
572
|
+
...row,
|
|
573
|
+
selected_models: Array.from(row.selected_models).sort(),
|
|
574
|
+
reroute_reasons: Array.from(row.reroute_reasons).sort(),
|
|
575
|
+
model_attribution: row.rerouted_usage_events > 0 ? "effective" : "selected",
|
|
576
|
+
}))
|
|
577
|
+
.sort((a, b) => b.total_tokens - a.total_tokens || a.model.localeCompare(b.model));
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// ---------------------------------------------------------------------------
|
|
581
|
+
// Main parser
|
|
582
|
+
// ---------------------------------------------------------------------------
|
|
583
|
+
|
|
584
|
+
async function parseCodexRolloutFile(filePath, {
|
|
585
|
+
from = null,
|
|
586
|
+
to = null,
|
|
587
|
+
timeZoneContext = null,
|
|
588
|
+
diagnostics = null,
|
|
589
|
+
seenTokenEvents = null,
|
|
590
|
+
startOffset = 0,
|
|
591
|
+
endOffset = null,
|
|
592
|
+
resumeState = null,
|
|
593
|
+
captureResumeState = false,
|
|
594
|
+
captureContentHash = false,
|
|
595
|
+
contentHashState = null,
|
|
596
|
+
sourceHandle = null,
|
|
597
|
+
collectBreakdowns = true,
|
|
598
|
+
collectModelUsage = false,
|
|
599
|
+
onObject = null,
|
|
600
|
+
onUsage = null,
|
|
601
|
+
} = {}) {
|
|
602
|
+
const filePaths = (Array.isArray(filePath) ? filePath : [filePath]).filter(Boolean);
|
|
603
|
+
const primaryFilePath = filePaths[0] || String(filePath || "");
|
|
604
|
+
const isResuming = Boolean(resumeState && filePaths.length === 1);
|
|
605
|
+
const initialOffset = Math.max(0, Number(startOffset) || 0);
|
|
606
|
+
const readProgress = {
|
|
607
|
+
endOffset: initialOffset,
|
|
608
|
+
lastByte: initialOffset > 0 ? 0x0a : null,
|
|
609
|
+
};
|
|
610
|
+
let contentHasher = null;
|
|
611
|
+
if (captureResumeState && captureContentHash && filePaths.length === 1) {
|
|
612
|
+
if (contentHashState && typeof contentHashState.copy === "function") {
|
|
613
|
+
try {
|
|
614
|
+
contentHasher = contentHashState.copy();
|
|
615
|
+
} catch {
|
|
616
|
+
contentHasher = null;
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
if (initialOffset > 0 && !contentHasher) {
|
|
620
|
+
const error = new Error("Codex rollout append is missing its prefix hash state");
|
|
621
|
+
error.code = CODEX_RESUME_INVALIDATED;
|
|
622
|
+
throw error;
|
|
623
|
+
}
|
|
624
|
+
if (!contentHasher) contentHasher = crypto.createHash("sha256");
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
let sessionId = isResuming ? resumeState.sessionId || null : null;
|
|
628
|
+
let cwd = isResuming ? resumeState.cwd || null : null;
|
|
629
|
+
let model = isResuming ? resumeState.model || null : null;
|
|
630
|
+
const modelAttributionState = createCodexModelAttributionState(
|
|
631
|
+
isResuming ? resumeState.modelAttributionState || { model } : {},
|
|
632
|
+
);
|
|
633
|
+
// Codex writes thread_settings_applied immediately before the turn_context of
|
|
634
|
+
// the turn it applies to, and token_count rows carry no turn id, so each one
|
|
635
|
+
// belongs to the most recent preceding tier record. Sessions that never emit
|
|
636
|
+
// the event (most CLI sessions) stay null and are billed at Standard.
|
|
637
|
+
let serviceTier = isResuming ? resumeState.serviceTier || null : null;
|
|
638
|
+
let provider = isResuming ? resumeState.provider || null : null;
|
|
639
|
+
let cliVersion = isResuming ? resumeState.cliVersion || null : null;
|
|
640
|
+
let forkedFromId = isResuming ? resumeState.forkedFromId || null : null;
|
|
641
|
+
let parentThreadId = isResuming ? resumeState.parentThreadId || null : null;
|
|
642
|
+
let threadSource = isResuming ? resumeState.threadSource || null : null;
|
|
643
|
+
let agentNickname = isResuming ? resumeState.agentNickname || null : null;
|
|
644
|
+
let agentRole = isResuming ? resumeState.agentRole || null : null;
|
|
645
|
+
let sessionLineageCaptured = isResuming
|
|
646
|
+
? Boolean(resumeState.sessionLineageCaptured || resumeState.sessionId)
|
|
647
|
+
: false;
|
|
648
|
+
|
|
649
|
+
const usageDeltaState = createUsageDeltaState({
|
|
650
|
+
lastTotal: isResuming ? resumeState.prevTotals || null : null,
|
|
651
|
+
baselines: isResuming ? resumeState.tokenUsageBaselines || null : null,
|
|
652
|
+
});
|
|
653
|
+
let pendingToolNames = isResuming
|
|
654
|
+
? Array.from(resumeState.pendingToolNames || [])
|
|
655
|
+
: [];
|
|
656
|
+
let pendingSkills = isResuming ? Array.from(resumeState.pendingSkills || []) : [];
|
|
657
|
+
let pendingExecDetails = isResuming
|
|
658
|
+
? Array.from(resumeState.pendingExecDetails || [])
|
|
659
|
+
: [];
|
|
660
|
+
let lastTokenTimestamp = isResuming
|
|
661
|
+
? String(resumeState.lastTokenTimestamp || "") || null
|
|
662
|
+
: null;
|
|
663
|
+
let lastTimestampEventSignatures = new Set(
|
|
664
|
+
isResuming ? resumeState.lastTimestampEventSignatures || [] : [],
|
|
665
|
+
);
|
|
666
|
+
let lastTimestampLastUsage = null;
|
|
667
|
+
let lastTimestampTotalUsage = null;
|
|
668
|
+
let hasUnmaterializedLastTimestampUsage = false;
|
|
669
|
+
|
|
670
|
+
const totals = emptyTotals();
|
|
671
|
+
const byTool = new Map(); // tool_name -> {name,calls,totals}
|
|
672
|
+
const bySkill = new Map(); // skill_name -> {name,calls,totals}
|
|
673
|
+
const byExecKind = new Map(); // kind -> stats
|
|
674
|
+
const byExecExit = new Map(); // status:exit -> stats
|
|
675
|
+
const byExecExecutable = new Map(); // executable -> stats
|
|
676
|
+
const byExecCommand = new Map(); // sanitized executable + subcommand -> stats
|
|
677
|
+
const byExecDuration = new Map(); // duration bucket -> stats
|
|
678
|
+
const byExecOutput = new Map(); // output size bucket -> stats
|
|
679
|
+
const byModel = new Map(); // effective model -> observed token usage
|
|
680
|
+
|
|
681
|
+
let turnCount = 0;
|
|
682
|
+
|
|
683
|
+
// Compaction records (issue #652, rules in codex-usage-record.js). A resume
|
|
684
|
+
// state from before #652 with prevTotals has seen a token_count.
|
|
685
|
+
const usageRecordState = createUsageRecordState(
|
|
686
|
+
isResuming
|
|
687
|
+
? resumeState.usageRecordState || (resumeState.prevTotals ? { sawTokenCount: true } : null)
|
|
688
|
+
: null,
|
|
689
|
+
);
|
|
690
|
+
// Compactions counted in this session, by response_id: the files merged for
|
|
691
|
+
// a session can repeat a record (an archived copy, a split fragment), and a
|
|
692
|
+
// resumed parse must not count one from an earlier chunk again.
|
|
693
|
+
const compactionResponseIds = new Set(
|
|
694
|
+
isResuming && Array.isArray(resumeState.compactionResponseIds)
|
|
695
|
+
? resumeState.compactionResponseIds
|
|
696
|
+
: [],
|
|
697
|
+
);
|
|
698
|
+
|
|
699
|
+
// Attribution as of now. A compaction record is counted later than it is
|
|
700
|
+
// read, so it carries the attribution captured when it was read.
|
|
701
|
+
function currentUsageAttribution() {
|
|
702
|
+
return {
|
|
703
|
+
model: currentCodexModel(modelAttributionState) || "unknown",
|
|
704
|
+
selectedModel: modelAttributionState.selectedModel || null,
|
|
705
|
+
rerouted: Boolean(modelAttributionState.rerouted),
|
|
706
|
+
rerouteReason: modelAttributionState.rerouteReason || null,
|
|
707
|
+
serviceTier,
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
function recordModelUsage(delta, rawRequestUsage, attribution = currentUsageAttribution()) {
|
|
712
|
+
if (!collectModelUsage || !delta || delta.total_tokens <= 0) return;
|
|
713
|
+
const effectiveModel = attribution.model;
|
|
714
|
+
let row = byModel.get(effectiveModel);
|
|
715
|
+
if (!row) {
|
|
716
|
+
row = {
|
|
717
|
+
model: effectiveModel,
|
|
718
|
+
...emptyTotals(),
|
|
719
|
+
long_context_input_tokens: 0,
|
|
720
|
+
long_context_cached_input_tokens: 0,
|
|
721
|
+
long_context_cache_creation_input_tokens: 0,
|
|
722
|
+
long_context_output_tokens: 0,
|
|
723
|
+
long_context_reasoning_output_tokens: 0,
|
|
724
|
+
priority_input_tokens: 0,
|
|
725
|
+
priority_cached_input_tokens: 0,
|
|
726
|
+
priority_cache_creation_input_tokens: 0,
|
|
727
|
+
priority_output_tokens: 0,
|
|
728
|
+
priority_reasoning_output_tokens: 0,
|
|
729
|
+
priority_long_context_input_tokens: 0,
|
|
730
|
+
priority_long_context_cached_input_tokens: 0,
|
|
731
|
+
priority_long_context_cache_creation_input_tokens: 0,
|
|
732
|
+
priority_long_context_output_tokens: 0,
|
|
733
|
+
priority_long_context_reasoning_output_tokens: 0,
|
|
734
|
+
usage_events: 0,
|
|
735
|
+
rerouted_usage_events: 0,
|
|
736
|
+
long_context_usage_events: 0,
|
|
737
|
+
priority_usage_events: 0,
|
|
738
|
+
selected_models: new Set(),
|
|
739
|
+
reroute_reasons: new Set(),
|
|
740
|
+
};
|
|
741
|
+
byModel.set(effectiveModel, row);
|
|
742
|
+
}
|
|
743
|
+
const subsetKeys = Object.keys(row).filter((key) => /^(?:long_context_|priority_)|^(?:usage_events|rerouted_usage_events)$/.test(key));
|
|
744
|
+
const before = Object.fromEntries(subsetKeys.map((key) => [key, row[key]]));
|
|
745
|
+
addInto(row, delta);
|
|
746
|
+
row.usage_events += 1;
|
|
747
|
+
if (attribution.selectedModel) {
|
|
748
|
+
row.selected_models.add(attribution.selectedModel);
|
|
749
|
+
}
|
|
750
|
+
if (attribution.rerouted) {
|
|
751
|
+
row.rerouted_usage_events += 1;
|
|
752
|
+
if (attribution.rerouteReason) {
|
|
753
|
+
row.reroute_reasons.add(attribution.rerouteReason);
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
// OpenAI applies the long-context tier to the whole request when its raw
|
|
757
|
+
// (cache-inclusive) input exceeds 272K tokens. Preserve the exact subset
|
|
758
|
+
// so pricing can be recomputed later without rescanning private logs.
|
|
759
|
+
const isLongContext =
|
|
760
|
+
Number(rawRequestUsage?.input_tokens || 0) > OPENAI_LONG_CONTEXT_INPUT_THRESHOLD;
|
|
761
|
+
if (isLongContext) {
|
|
762
|
+
row.long_context_usage_events += 1;
|
|
763
|
+
row.long_context_input_tokens += delta.input_tokens;
|
|
764
|
+
row.long_context_cached_input_tokens += delta.cached_input_tokens;
|
|
765
|
+
row.long_context_cache_creation_input_tokens += delta.cache_creation_input_tokens;
|
|
766
|
+
row.long_context_output_tokens += delta.output_tokens;
|
|
767
|
+
row.long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
|
|
768
|
+
}
|
|
769
|
+
// Astra Fast bills at a flat multiple of whichever context tier the request
|
|
770
|
+
// was on, so the priority premium multiplies the long-context premium
|
|
771
|
+
// instead of replacing it. The two subsets alone cannot say how much usage
|
|
772
|
+
// was on BOTH, so record the intersection explicitly — otherwise a long
|
|
773
|
+
// Fast request prices at 3x Standard input where it should be 4x.
|
|
774
|
+
if (isPriorityServiceTier(attribution.serviceTier)) {
|
|
775
|
+
row.priority_usage_events += 1;
|
|
776
|
+
row.priority_input_tokens += delta.input_tokens;
|
|
777
|
+
row.priority_cached_input_tokens += delta.cached_input_tokens;
|
|
778
|
+
row.priority_cache_creation_input_tokens += delta.cache_creation_input_tokens;
|
|
779
|
+
row.priority_output_tokens += delta.output_tokens;
|
|
780
|
+
row.priority_reasoning_output_tokens += delta.reasoning_output_tokens;
|
|
781
|
+
if (isLongContext) {
|
|
782
|
+
row.priority_long_context_input_tokens += delta.input_tokens;
|
|
783
|
+
row.priority_long_context_cached_input_tokens += delta.cached_input_tokens;
|
|
784
|
+
row.priority_long_context_cache_creation_input_tokens +=
|
|
785
|
+
delta.cache_creation_input_tokens;
|
|
786
|
+
row.priority_long_context_output_tokens += delta.output_tokens;
|
|
787
|
+
row.priority_long_context_reasoning_output_tokens += delta.reasoning_output_tokens;
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
return { ...delta, ...Object.fromEntries(subsetKeys.map((key) => [key, row[key] - before[key]])),
|
|
791
|
+
selected_models: attribution.selectedModel ? [attribution.selectedModel] : [],
|
|
792
|
+
reroute_reasons: attribution.rerouteReason ? [attribution.rerouteReason] : [],
|
|
793
|
+
model_attribution: attribution.rerouted ? "effective" : "selected" };
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
function ensureTool(name) {
|
|
797
|
+
if (!byTool.has(name)) {
|
|
798
|
+
byTool.set(name, { name, ...buildToolStatsEntry() });
|
|
799
|
+
}
|
|
800
|
+
return byTool.get(name);
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
function ensureExec(map, key) {
|
|
804
|
+
if (!map.has(key)) map.set(key, { name: key, ...buildExecStatsEntry() });
|
|
805
|
+
return map.get(key);
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
function ensureSkill(name) {
|
|
809
|
+
if (!bySkill.has(name)) bySkill.set(name, buildSkillStatsEntry(name));
|
|
810
|
+
return bySkill.get(name);
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
function getExecKeys(p) {
|
|
814
|
+
if (!p || typeof p !== "object") return;
|
|
815
|
+
const cmdArr = Array.isArray(p.command) ? p.command : null;
|
|
816
|
+
const cmd = cmdArr && cmdArr.length > 0 ? String(cmdArr[cmdArr.length - 1] || "") : "";
|
|
817
|
+
const kind = p.parsed_cmd?.[0]?.type && p.parsed_cmd[0].type !== "unknown"
|
|
818
|
+
? p.parsed_cmd[0].type
|
|
819
|
+
: inferExecCommandKind(cmd);
|
|
820
|
+
|
|
821
|
+
const status = String(p.status || "unknown");
|
|
822
|
+
const exit = Number.isFinite(Number(p.exit_code)) ? Number(p.exit_code) : null;
|
|
823
|
+
const exitKey = `${status}:${exit === null ? "unknown" : exit}`;
|
|
824
|
+
|
|
825
|
+
const dur = p.duration ? Math.round((Number(p.duration.secs || 0) * 1000) + Number(p.duration.nanos || 0) / 1e6) : 0;
|
|
826
|
+
const output = String(p.aggregated_output || p.stdout || "");
|
|
827
|
+
const outputChars = output.length;
|
|
828
|
+
const outputLines = output ? output.split("\n").length : 0;
|
|
829
|
+
return {
|
|
830
|
+
kind,
|
|
831
|
+
exitKey,
|
|
832
|
+
executable: getExecutableName(cmd),
|
|
833
|
+
command: sanitizeCommandSignature(cmd),
|
|
834
|
+
duration: durationBucket(dur),
|
|
835
|
+
output: outputSizeBucket(outputLines, outputChars),
|
|
836
|
+
dur,
|
|
837
|
+
outputChars,
|
|
838
|
+
outputLines,
|
|
839
|
+
failed: status !== "completed" || exit !== 0,
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
function absorbExecStats(map, key, details) {
|
|
844
|
+
const row = ensureExec(map, key);
|
|
845
|
+
row.calls += 1;
|
|
846
|
+
row.duration_ms += details.dur;
|
|
847
|
+
row.max_duration_ms = Math.max(row.max_duration_ms, details.dur);
|
|
848
|
+
row.output_chars += details.outputChars;
|
|
849
|
+
row.output_lines += details.outputLines;
|
|
850
|
+
if (details.failed) row.failures += 1;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
function absorbExecEnd(details) {
|
|
854
|
+
absorbExecStats(byExecKind, details.kind, details);
|
|
855
|
+
absorbExecStats(byExecExit, details.exitKey, details);
|
|
856
|
+
absorbExecStats(byExecExecutable, details.executable, details);
|
|
857
|
+
absorbExecStats(byExecCommand, details.command, details);
|
|
858
|
+
absorbExecStats(byExecDuration, details.duration, details);
|
|
859
|
+
absorbExecStats(byExecOutput, details.output, details);
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
function attributeTurn(delta) {
|
|
863
|
+
if (!delta || delta.total_tokens <= 0) {
|
|
864
|
+
for (const details of pendingExecDetails) absorbExecEnd(details);
|
|
865
|
+
pendingToolNames = [];
|
|
866
|
+
pendingSkills = [];
|
|
867
|
+
pendingExecDetails = [];
|
|
868
|
+
return;
|
|
869
|
+
}
|
|
870
|
+
turnCount += 1;
|
|
871
|
+
|
|
872
|
+
if (!collectBreakdowns) {
|
|
873
|
+
addInto(totals, delta);
|
|
874
|
+
pendingToolNames = [];
|
|
875
|
+
pendingSkills = [];
|
|
876
|
+
pendingExecDetails = [];
|
|
877
|
+
return;
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
const unique = [...new Set(pendingToolNames.filter(Boolean))];
|
|
881
|
+
const tools = unique.length > 0 ? unique : ["text_response"];
|
|
882
|
+
const share = 1 / tools.length;
|
|
883
|
+
|
|
884
|
+
for (const name of tools) {
|
|
885
|
+
const row = ensureTool(name);
|
|
886
|
+
row.calls += share;
|
|
887
|
+
addInto(row.totals, {
|
|
888
|
+
input_tokens: delta.input_tokens * share,
|
|
889
|
+
cached_input_tokens: delta.cached_input_tokens * share,
|
|
890
|
+
cache_creation_input_tokens: delta.cache_creation_input_tokens * share,
|
|
891
|
+
output_tokens: delta.output_tokens * share,
|
|
892
|
+
reasoning_output_tokens: delta.reasoning_output_tokens * share,
|
|
893
|
+
total_tokens: delta.total_tokens * share,
|
|
894
|
+
});
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
const uniqueSkills = [...new Set(pendingSkills.filter(Boolean))];
|
|
898
|
+
if (uniqueSkills.length > 0) {
|
|
899
|
+
const skillShare = 1 / uniqueSkills.length;
|
|
900
|
+
for (const name of uniqueSkills) {
|
|
901
|
+
const row = ensureSkill(name);
|
|
902
|
+
row.calls += skillShare;
|
|
903
|
+
addInto(row.totals, {
|
|
904
|
+
input_tokens: delta.input_tokens * skillShare,
|
|
905
|
+
cached_input_tokens: delta.cached_input_tokens * skillShare,
|
|
906
|
+
cache_creation_input_tokens: delta.cache_creation_input_tokens * skillShare,
|
|
907
|
+
output_tokens: delta.output_tokens * skillShare,
|
|
908
|
+
reasoning_output_tokens: delta.reasoning_output_tokens * skillShare,
|
|
909
|
+
total_tokens: delta.total_tokens * skillShare,
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
// Attribute exec_command_end rows to exec stats; note these are not a
|
|
915
|
+
// token source — we attach the same tool-shared delta to the command
|
|
916
|
+
// classifier so users can find high-cost command families.
|
|
917
|
+
const execToolShare = tools.includes("exec_command") ? (1 / tools.length) : 0;
|
|
918
|
+
const execDelta = execToolShare > 0 ? {
|
|
919
|
+
input_tokens: delta.input_tokens * execToolShare,
|
|
920
|
+
cached_input_tokens: delta.cached_input_tokens * execToolShare,
|
|
921
|
+
cache_creation_input_tokens: delta.cache_creation_input_tokens * execToolShare,
|
|
922
|
+
output_tokens: delta.output_tokens * execToolShare,
|
|
923
|
+
reasoning_output_tokens: delta.reasoning_output_tokens * execToolShare,
|
|
924
|
+
total_tokens: delta.total_tokens * execToolShare,
|
|
925
|
+
} : null;
|
|
926
|
+
|
|
927
|
+
if (execDelta && pendingExecDetails.length > 0) {
|
|
928
|
+
const perExecShare = 1 / pendingExecDetails.length;
|
|
929
|
+
for (const details of pendingExecDetails) {
|
|
930
|
+
const attributed = {
|
|
931
|
+
input_tokens: execDelta.input_tokens * perExecShare,
|
|
932
|
+
cached_input_tokens: execDelta.cached_input_tokens * perExecShare,
|
|
933
|
+
cache_creation_input_tokens: execDelta.cache_creation_input_tokens * perExecShare,
|
|
934
|
+
output_tokens: execDelta.output_tokens * perExecShare,
|
|
935
|
+
reasoning_output_tokens: execDelta.reasoning_output_tokens * perExecShare,
|
|
936
|
+
total_tokens: execDelta.total_tokens * perExecShare,
|
|
937
|
+
};
|
|
938
|
+
|
|
939
|
+
addInto(ensureExec(byExecKind, details.kind).totals, attributed);
|
|
940
|
+
addInto(ensureExec(byExecExit, details.exitKey).totals, attributed);
|
|
941
|
+
addInto(ensureExec(byExecExecutable, details.executable).totals, attributed);
|
|
942
|
+
addInto(ensureExec(byExecCommand, details.command).totals, attributed);
|
|
943
|
+
addInto(ensureExec(byExecDuration, details.duration).totals, attributed);
|
|
944
|
+
addInto(ensureExec(byExecOutput, details.output).totals, attributed);
|
|
945
|
+
|
|
946
|
+
absorbExecEnd(details);
|
|
947
|
+
}
|
|
948
|
+
} else {
|
|
949
|
+
// Still ingest exec end stats without token attribution so the drill-down works.
|
|
950
|
+
for (const details of pendingExecDetails) absorbExecEnd(details);
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
addInto(totals, delta);
|
|
954
|
+
pendingToolNames = [];
|
|
955
|
+
pendingSkills = [];
|
|
956
|
+
pendingExecDetails = [];
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
function countCompactionRecord(record) {
|
|
960
|
+
if (compactionResponseIds.has(record.responseId)) return;
|
|
961
|
+
compactionResponseIds.add(record.responseId);
|
|
962
|
+
if (!record.inRequestedRange) return;
|
|
963
|
+
const pricedDelta = recordModelUsage(record.delta, record.rawUsage, record.attribution);
|
|
964
|
+
if (typeof onUsage === "function") onUsage({ timestamp: record.timestamp,
|
|
965
|
+
delta: pricedDelta || record.delta, rawUsage: record.rawUsage, model: record.attribution.model, origin: "compaction" });
|
|
966
|
+
attributeTurn(record.delta);
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
function usageEventSignature(lastUsage, totalUsage) {
|
|
970
|
+
return JSON.stringify({ lastUsage, totalUsage });
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
function materializeLastTimestampSignatures() {
|
|
974
|
+
if (hasUnmaterializedLastTimestampUsage) {
|
|
975
|
+
lastTimestampEventSignatures.add(
|
|
976
|
+
usageEventSignature(lastTimestampLastUsage, lastTimestampTotalUsage),
|
|
977
|
+
);
|
|
978
|
+
hasUnmaterializedLastTimestampUsage = false;
|
|
979
|
+
}
|
|
980
|
+
return lastTimestampEventSignatures;
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
function noteTokenEvent(timestamp, lastUsage, totalUsage) {
|
|
984
|
+
if (!isResuming) {
|
|
985
|
+
if (timestamp !== lastTokenTimestamp) {
|
|
986
|
+
lastTokenTimestamp = timestamp;
|
|
987
|
+
lastTimestampEventSignatures = new Set();
|
|
988
|
+
hasUnmaterializedLastTimestampUsage = false;
|
|
989
|
+
} else {
|
|
990
|
+
materializeLastTimestampSignatures();
|
|
991
|
+
}
|
|
992
|
+
lastTimestampLastUsage = lastUsage;
|
|
993
|
+
lastTimestampTotalUsage = totalUsage;
|
|
994
|
+
hasUnmaterializedLastTimestampUsage = true;
|
|
995
|
+
return true;
|
|
996
|
+
}
|
|
997
|
+
if (isResuming && lastTokenTimestamp && timestamp < lastTokenTimestamp) {
|
|
998
|
+
const error = new Error("Codex rollout append is not timestamp-monotonic");
|
|
999
|
+
error.code = CODEX_RESUME_INVALIDATED;
|
|
1000
|
+
throw error;
|
|
1001
|
+
}
|
|
1002
|
+
if (!lastTokenTimestamp || timestamp > lastTokenTimestamp) {
|
|
1003
|
+
lastTokenTimestamp = timestamp;
|
|
1004
|
+
lastTimestampEventSignatures = new Set();
|
|
1005
|
+
lastTimestampLastUsage = lastUsage;
|
|
1006
|
+
lastTimestampTotalUsage = totalUsage;
|
|
1007
|
+
hasUnmaterializedLastTimestampUsage = true;
|
|
1008
|
+
return true;
|
|
1009
|
+
}
|
|
1010
|
+
if (timestamp === lastTokenTimestamp) {
|
|
1011
|
+
const signature = usageEventSignature(lastUsage, totalUsage);
|
|
1012
|
+
const signatures = materializeLastTimestampSignatures();
|
|
1013
|
+
if (signatures.has(signature)) return false;
|
|
1014
|
+
signatures.add(signature);
|
|
1015
|
+
}
|
|
1016
|
+
return true;
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
for await (const obj of iterateCodexObjects(filePaths, diagnostics, {
|
|
1020
|
+
startOffset,
|
|
1021
|
+
endOffset,
|
|
1022
|
+
readProgress,
|
|
1023
|
+
contentHasher,
|
|
1024
|
+
sourceHandle,
|
|
1025
|
+
})) {
|
|
1026
|
+
const modelEvent = applyCodexModelEvent(modelAttributionState, obj);
|
|
1027
|
+
const attributedModel = currentCodexModel(modelAttributionState);
|
|
1028
|
+
model = attributedModel || model;
|
|
1029
|
+
// onObject runs AFTER the attribution state machine, and is handed the
|
|
1030
|
+
// model it just decided. A consumer sharing this pass (the delivery-signal
|
|
1031
|
+
// collector, which attributes edit turns) therefore sees exactly the model
|
|
1032
|
+
// recordModelUsage() bills the turn's tokens to. Letting it re-read
|
|
1033
|
+
// turn_context.payload.model itself would make codex-model-attribution.js
|
|
1034
|
+
// one of two authorities on the same question.
|
|
1035
|
+
if (typeof onObject === "function") onObject(obj, attributedModel);
|
|
1036
|
+
const usageRecord = extractTokenUsageRecord(obj);
|
|
1037
|
+
if (!usageRecord) noteLineAfterUsageRecord(usageRecordState, obj?.type === "compacted");
|
|
1038
|
+
const ts = typeof obj?.timestamp === "string" ? obj.timestamp : null;
|
|
1039
|
+
if (!ts) continue;
|
|
1040
|
+
const inRequestedRange = isTimestampInRequestedDayRange(ts, { from, to, timeZoneContext });
|
|
1041
|
+
|
|
1042
|
+
if (obj.type === "session_meta") {
|
|
1043
|
+
const p = obj.payload || {};
|
|
1044
|
+
// Forked rollouts replay historical session_meta rows after their own
|
|
1045
|
+
// child metadata. Only the first row describes this file's lineage.
|
|
1046
|
+
if (!sessionLineageCaptured) {
|
|
1047
|
+
sessionId = p.id || sessionId;
|
|
1048
|
+
cwd = p.cwd || cwd;
|
|
1049
|
+
cliVersion = p.cli_version || cliVersion;
|
|
1050
|
+
provider = p.model_provider || provider;
|
|
1051
|
+
forkedFromId = p.forked_from_id || null;
|
|
1052
|
+
parentThreadId = p.parent_thread_id || null;
|
|
1053
|
+
threadSource = p.thread_source || p.source || null;
|
|
1054
|
+
agentNickname = p.agent_nickname || null;
|
|
1055
|
+
agentRole = p.agent_role || null;
|
|
1056
|
+
sessionLineageCaptured = true;
|
|
1057
|
+
} else if (!sessionId) {
|
|
1058
|
+
sessionId = p.id || null;
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
const appliedServiceTier = readCodexServiceTier(obj);
|
|
1063
|
+
if (appliedServiceTier) {
|
|
1064
|
+
serviceTier = appliedServiceTier;
|
|
1065
|
+
continue;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
if (obj.type === "turn_context") {
|
|
1069
|
+
const p = obj.payload || {};
|
|
1070
|
+
if (typeof p.cwd === "string") cwd = p.cwd;
|
|
1071
|
+
continue;
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
if (modelEvent?.type === "model_rerouted") continue;
|
|
1075
|
+
|
|
1076
|
+
if (collectBreakdowns && obj.type === "response_item" && obj.payload?.type === "function_call") {
|
|
1077
|
+
pendingToolNames.push(normalizeToolName(obj.payload));
|
|
1078
|
+
const skill = extractSkillNameFromFunctionCall(obj.payload, diagnostics);
|
|
1079
|
+
if (skill) pendingSkills.push(skill);
|
|
1080
|
+
continue;
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
if (collectBreakdowns && obj.type === "event_msg" && obj.payload?.type === "exec_command_end") {
|
|
1084
|
+
const details = getExecKeys(obj.payload);
|
|
1085
|
+
if (details) pendingExecDetails.push(details);
|
|
1086
|
+
continue;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
if (usageRecord) {
|
|
1090
|
+
const delta = normalizeUsage(usageRecord.usage);
|
|
1091
|
+
noteUsageRecord(
|
|
1092
|
+
usageRecordState,
|
|
1093
|
+
// Without a response_id a compaction cannot be deduplicated.
|
|
1094
|
+
delta.total_tokens > 0 && usageRecord.responseId
|
|
1095
|
+
? {
|
|
1096
|
+
responseId: usageRecord.responseId,
|
|
1097
|
+
timestamp: usageRecord.timestamp,
|
|
1098
|
+
inRequestedRange,
|
|
1099
|
+
rawUsage: usageRecord.usage,
|
|
1100
|
+
delta,
|
|
1101
|
+
attribution: currentUsageAttribution(),
|
|
1102
|
+
}
|
|
1103
|
+
: null,
|
|
1104
|
+
);
|
|
1105
|
+
continue;
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
const tokenCount = extractTokenCount(obj);
|
|
1109
|
+
if (tokenCount) {
|
|
1110
|
+
const info = tokenCount.info;
|
|
1111
|
+
const lastUsage = info?.last_token_usage;
|
|
1112
|
+
const totalUsage = info?.total_token_usage;
|
|
1113
|
+
let compaction = null;
|
|
1114
|
+
if (info && typeof info === "object") {
|
|
1115
|
+
noteTokenCount(usageRecordState);
|
|
1116
|
+
compaction = takeCompactionOnTokenCount(
|
|
1117
|
+
usageRecordState,
|
|
1118
|
+
usageDeltaState.lastTotal,
|
|
1119
|
+
totalUsage,
|
|
1120
|
+
);
|
|
1121
|
+
}
|
|
1122
|
+
const rawDelta = consumeUsageDelta(usageDeltaState, lastUsage, totalUsage);
|
|
1123
|
+
const delta = rawDelta ? normalizeUsage(rawDelta) : null;
|
|
1124
|
+
// Timing consumers receive canonical, deduplicated usage below. A null
|
|
1125
|
+
// delta is a snapshot, not another model response.
|
|
1126
|
+
if (compaction) countCompactionRecord(compaction);
|
|
1127
|
+
const isNewResumeEvent = noteTokenEvent(ts, lastUsage, totalUsage);
|
|
1128
|
+
if (inRequestedRange) {
|
|
1129
|
+
const eventSessionId = sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath;
|
|
1130
|
+
const eventKey = `${eventSessionId}:${ts}:${usageEventSignature(lastUsage, totalUsage)}`;
|
|
1131
|
+
if (!isNewResumeEvent || (seenTokenEvents && seenTokenEvents.has(eventKey))) {
|
|
1132
|
+
if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: null, rawUsage: lastUsage, model: attributedModel });
|
|
1133
|
+
attributeTurn(null);
|
|
1134
|
+
} else {
|
|
1135
|
+
if (seenTokenEvents && delta?.total_tokens > 0) seenTokenEvents.add(eventKey);
|
|
1136
|
+
const pricedDelta = recordModelUsage(delta, lastUsage || rawDelta);
|
|
1137
|
+
if (typeof onUsage === "function") onUsage({ timestamp: ts, delta: pricedDelta || delta, rawUsage: lastUsage || rawDelta, model: attributedModel });
|
|
1138
|
+
attributeTurn(delta);
|
|
1139
|
+
}
|
|
1140
|
+
} else {
|
|
1141
|
+
pendingToolNames = [];
|
|
1142
|
+
pendingSkills = [];
|
|
1143
|
+
pendingExecDetails = [];
|
|
1144
|
+
}
|
|
1145
|
+
continue;
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
const result = {
|
|
1150
|
+
sessionId: sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath,
|
|
1151
|
+
cwd,
|
|
1152
|
+
model: currentCodexModel(modelAttributionState) || model || provider,
|
|
1153
|
+
provider,
|
|
1154
|
+
version: cliVersion,
|
|
1155
|
+
forkedFromId,
|
|
1156
|
+
parentThreadId,
|
|
1157
|
+
threadSource,
|
|
1158
|
+
agentNickname,
|
|
1159
|
+
agentRole,
|
|
1160
|
+
filePath: primaryFilePath,
|
|
1161
|
+
turnCount,
|
|
1162
|
+
totals,
|
|
1163
|
+
modelUsage: finalizeModelUsageRows(byModel),
|
|
1164
|
+
toolBreakdown: {
|
|
1165
|
+
tool_rows: finalizeToolRows(byTool),
|
|
1166
|
+
},
|
|
1167
|
+
skillsBreakdown: {
|
|
1168
|
+
skill_rows: finalizeSkillRows(bySkill),
|
|
1169
|
+
},
|
|
1170
|
+
execCommandBreakdown: {
|
|
1171
|
+
byType: finalizeExecRows(byExecKind),
|
|
1172
|
+
byExit: finalizeExecRows(byExecExit),
|
|
1173
|
+
byExecutable: finalizeExecRows(byExecExecutable),
|
|
1174
|
+
byCommand: finalizeExecRows(byExecCommand),
|
|
1175
|
+
byDuration: finalizeExecRows(byExecDuration),
|
|
1176
|
+
byOutput: finalizeExecRows(byExecOutput),
|
|
1177
|
+
},
|
|
1178
|
+
};
|
|
1179
|
+
if (captureResumeState && filePaths.length === 1) {
|
|
1180
|
+
result.resumeState = {
|
|
1181
|
+
sessionId,
|
|
1182
|
+
cwd,
|
|
1183
|
+
model,
|
|
1184
|
+
modelAttributionState: snapshotCodexModelAttributionState(modelAttributionState),
|
|
1185
|
+
serviceTier,
|
|
1186
|
+
provider,
|
|
1187
|
+
cliVersion,
|
|
1188
|
+
forkedFromId,
|
|
1189
|
+
parentThreadId,
|
|
1190
|
+
threadSource,
|
|
1191
|
+
agentNickname,
|
|
1192
|
+
agentRole,
|
|
1193
|
+
sessionLineageCaptured,
|
|
1194
|
+
prevTotals: usageDeltaState.lastTotal,
|
|
1195
|
+
tokenUsageBaselines: snapshotUsageBaselines(usageDeltaState),
|
|
1196
|
+
pendingToolNames,
|
|
1197
|
+
pendingSkills,
|
|
1198
|
+
pendingExecDetails,
|
|
1199
|
+
lastTokenTimestamp,
|
|
1200
|
+
lastTimestampEventSignatures: Array.from(materializeLastTimestampSignatures()),
|
|
1201
|
+
usageRecordState: snapshotUsageRecordState(usageRecordState),
|
|
1202
|
+
compactionResponseIds: Array.from(compactionResponseIds),
|
|
1203
|
+
};
|
|
1204
|
+
result.endOffset = readProgress.endOffset;
|
|
1205
|
+
// Compressed snapshots are not append-only logs: endOffset stays at the
|
|
1206
|
+
// pre-decode cursor (often 0) and must never be marked appendable, or the
|
|
1207
|
+
// next growth would "resume" by parsing compressed bytes as JSONL.
|
|
1208
|
+
const hasCompressedArchive = filePaths.some(
|
|
1209
|
+
(candidate) => typeof candidate === "string" && /\.jsonl\.zstd?$/i.test(candidate),
|
|
1210
|
+
);
|
|
1211
|
+
result.appendable = Boolean(contentHasher) && !hasCompressedArchive && (
|
|
1212
|
+
readProgress.endOffset === 0 || readProgress.lastByte === 0x0a
|
|
1213
|
+
);
|
|
1214
|
+
result.contentHashState = contentHasher;
|
|
1215
|
+
}
|
|
1216
|
+
return result;
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
const fatalUtf8Decoder = new TextDecoder("utf-8", { fatal: true });
|
|
1220
|
+
|
|
1221
|
+
function parseCodexLine(lineBuffer, diagnostics) {
|
|
1222
|
+
let content = lineBuffer;
|
|
1223
|
+
if (content.length > 0 && content[content.length - 1] === 0x0d) {
|
|
1224
|
+
content = content.subarray(0, content.length - 1);
|
|
1225
|
+
}
|
|
1226
|
+
if (content.length === 0) return null;
|
|
1227
|
+
try {
|
|
1228
|
+
if (diagnostics) diagnostics.json_parse_calls += 1;
|
|
1229
|
+
return JSON.parse(fatalUtf8Decoder.decode(content));
|
|
1230
|
+
} catch {
|
|
1231
|
+
return null;
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
async function* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
|
|
1236
|
+
startOffset = 0,
|
|
1237
|
+
endOffset = null,
|
|
1238
|
+
readProgress = null,
|
|
1239
|
+
contentHasher = null,
|
|
1240
|
+
sourceHandle = null,
|
|
1241
|
+
} = {}) {
|
|
1242
|
+
const start = Math.max(0, Number(startOffset) || 0);
|
|
1243
|
+
const requestedEnd = Number(endOffset);
|
|
1244
|
+
const hasBoundedEnd = endOffset !== null && endOffset !== undefined &&
|
|
1245
|
+
Number.isFinite(requestedEnd) && requestedEnd >= 0;
|
|
1246
|
+
if (hasBoundedEnd && requestedEnd <= start) {
|
|
1247
|
+
if (readProgress) readProgress.endOffset = start;
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
if (diagnostics) diagnostics.opened_files += 1;
|
|
1252
|
+
const stream = sourceHandle && typeof sourceHandle.read === "function"
|
|
1253
|
+
? null
|
|
1254
|
+
: fs.createReadStream(filePath, {
|
|
1255
|
+
start,
|
|
1256
|
+
...(hasBoundedEnd ? { end: requestedEnd - 1 } : {}),
|
|
1257
|
+
});
|
|
1258
|
+
const chunks = stream || (async function* readStableHandleChunks() {
|
|
1259
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
1260
|
+
const limit = hasBoundedEnd ? requestedEnd : Number.MAX_SAFE_INTEGER;
|
|
1261
|
+
let offset = start;
|
|
1262
|
+
while (offset < limit) {
|
|
1263
|
+
const length = Math.min(buffer.length, limit - offset);
|
|
1264
|
+
const result = await sourceHandle.read(buffer, 0, length, offset);
|
|
1265
|
+
if (!result.bytesRead) break;
|
|
1266
|
+
offset += result.bytesRead;
|
|
1267
|
+
yield buffer.subarray(0, result.bytesRead);
|
|
1268
|
+
}
|
|
1269
|
+
})();
|
|
1270
|
+
const fragments = [];
|
|
1271
|
+
let fragmentsBytes = 0;
|
|
1272
|
+
let nextChunkOffset = start;
|
|
1273
|
+
let lineStartOffset = start;
|
|
1274
|
+
let lineIndex = 0;
|
|
1275
|
+
|
|
1276
|
+
try {
|
|
1277
|
+
for await (const value of chunks) {
|
|
1278
|
+
const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
|
1279
|
+
const chunkStart = nextChunkOffset;
|
|
1280
|
+
nextChunkOffset += chunk.length;
|
|
1281
|
+
if (contentHasher) contentHasher.update(chunk);
|
|
1282
|
+
if (readProgress && chunk.length > 0) {
|
|
1283
|
+
readProgress.lastByte = chunk[chunk.length - 1];
|
|
1284
|
+
}
|
|
1285
|
+
if (diagnostics) {
|
|
1286
|
+
diagnostics.bytes_read = Number(diagnostics.bytes_read || 0) + chunk.length;
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
let cursor = 0;
|
|
1290
|
+
while (cursor < chunk.length) {
|
|
1291
|
+
const newline = chunk.indexOf(0x0a, cursor);
|
|
1292
|
+
if (newline < 0) {
|
|
1293
|
+
const tail = chunk.subarray(cursor);
|
|
1294
|
+
if (tail.length > 0) {
|
|
1295
|
+
// Stable-handle reads reuse their buffer, so retained fragments
|
|
1296
|
+
// must own their bytes before the next read overwrites it.
|
|
1297
|
+
fragments.push(Buffer.from(tail));
|
|
1298
|
+
fragmentsBytes += tail.length;
|
|
1299
|
+
}
|
|
1300
|
+
break;
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
const tail = chunk.subarray(cursor, newline);
|
|
1304
|
+
const lineBuffer = fragments.length > 0
|
|
1305
|
+
? Buffer.concat([...fragments, tail], fragmentsBytes + tail.length)
|
|
1306
|
+
: tail;
|
|
1307
|
+
fragments.length = 0;
|
|
1308
|
+
fragmentsBytes = 0;
|
|
1309
|
+
|
|
1310
|
+
const absoluteEnd = chunkStart + newline + 1;
|
|
1311
|
+
if (readProgress) readProgress.endOffset = absoluteEnd;
|
|
1312
|
+
lineStartOffset = absoluteEnd;
|
|
1313
|
+
const obj = parseCodexLine(lineBuffer, diagnostics);
|
|
1314
|
+
if (obj) {
|
|
1315
|
+
yield { obj, fileIndex, lineIndex };
|
|
1316
|
+
lineIndex += 1;
|
|
1317
|
+
}
|
|
1318
|
+
cursor = newline + 1;
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
if (fragmentsBytes > 0) {
|
|
1323
|
+
const lineBuffer = fragments.length === 1
|
|
1324
|
+
? fragments[0]
|
|
1325
|
+
: Buffer.concat(fragments, fragmentsBytes);
|
|
1326
|
+
const obj = parseCodexLine(lineBuffer, diagnostics);
|
|
1327
|
+
if (obj) {
|
|
1328
|
+
if (readProgress) readProgress.endOffset = lineStartOffset + fragmentsBytes;
|
|
1329
|
+
yield { obj, fileIndex, lineIndex };
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
} finally {
|
|
1333
|
+
if (readProgress) readProgress.endOffset = nextChunkOffset;
|
|
1334
|
+
stream?.destroy();
|
|
1335
|
+
if (diagnostics) diagnostics.parsed_files += 1;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
async function* readCodexObjectsFull(filePath, diagnostics, fileIndex, {
|
|
1340
|
+
startOffset = 0,
|
|
1341
|
+
endOffset = null,
|
|
1342
|
+
readProgress = null,
|
|
1343
|
+
contentHasher = null,
|
|
1344
|
+
} = {}) {
|
|
1345
|
+
// Compressed archives are immutable snapshots, not append-only logs: byte
|
|
1346
|
+
// offsets are meaningless inside them, so decode the whole text and split
|
|
1347
|
+
// into lines. `readProgress` stays null so callers never persist a fake
|
|
1348
|
+
// byte cursor for them.
|
|
1349
|
+
if (typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath)) {
|
|
1350
|
+
const text = await readCodexRolloutText(filePath);
|
|
1351
|
+
let lineIndex = 0;
|
|
1352
|
+
const lines = text.split("\n");
|
|
1353
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
1354
|
+
const isLast = index + 1 >= lines.length;
|
|
1355
|
+
if (isLast && lines[index] === "") continue;
|
|
1356
|
+
const lineBuffer = Buffer.from(lines[index], "utf8");
|
|
1357
|
+
const obj = parseCodexLine(lineBuffer, diagnostics);
|
|
1358
|
+
if (obj) {
|
|
1359
|
+
yield { obj, fileIndex, lineIndex };
|
|
1360
|
+
lineIndex += 1;
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
if (diagnostics) diagnostics.parsed_files += 1;
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
yield* readCodexObjectsIncremental(filePath, diagnostics, fileIndex, {
|
|
1367
|
+
startOffset,
|
|
1368
|
+
endOffset,
|
|
1369
|
+
readProgress,
|
|
1370
|
+
contentHasher,
|
|
1371
|
+
});
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
async function* readCodexObjects(filePath, diagnostics, fileIndex, options = {}) {
|
|
1375
|
+
const start = Math.max(0, Number(options.startOffset) || 0);
|
|
1376
|
+
const reader = start > 0 || options.sourceHandle
|
|
1377
|
+
? readCodexObjectsIncremental
|
|
1378
|
+
: readCodexObjectsFull;
|
|
1379
|
+
yield* reader(filePath, diagnostics, fileIndex, options);
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
function isRecoverableGroupedReadError(error) {
|
|
1383
|
+
return ["ENOENT", "EACCES", "EPERM", "EISDIR", "EIO", "ESTALE"].includes(error?.code);
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
async function* iterateCodexObjects(filePaths, diagnostics, options = {}) {
|
|
1387
|
+
if (filePaths.length <= 1) {
|
|
1388
|
+
if (!filePaths[0]) return;
|
|
1389
|
+
for await (const record of readCodexObjects(filePaths[0], diagnostics, 0, options)) {
|
|
1390
|
+
yield record.obj;
|
|
1391
|
+
}
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
// A live rollout can overlap its archived copy or be split into prefix/suffix
|
|
1396
|
+
// fragments. Merge only duplicate-session groups, order by event time, and
|
|
1397
|
+
// suppress byte-semantically identical records from different files so the
|
|
1398
|
+
// cumulative baseline is established exactly once.
|
|
1399
|
+
const records = [];
|
|
1400
|
+
let scannedFiles = 0;
|
|
1401
|
+
for (let fileIndex = 0; fileIndex < filePaths.length; fileIndex += 1) {
|
|
1402
|
+
const fileRecords = [];
|
|
1403
|
+
try {
|
|
1404
|
+
for await (const record of readCodexObjects(filePaths[fileIndex], diagnostics, fileIndex)) {
|
|
1405
|
+
fileRecords.push(record);
|
|
1406
|
+
}
|
|
1407
|
+
for (const record of fileRecords) records.push(record);
|
|
1408
|
+
scannedFiles += 1;
|
|
1409
|
+
} catch (error) {
|
|
1410
|
+
if (!isRecoverableGroupedReadError(error)) throw error;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
if (!scannedFiles) {
|
|
1414
|
+
const error = new Error("all grouped Codex rollout files failed during read");
|
|
1415
|
+
error.code = "EIO";
|
|
1416
|
+
throw error;
|
|
1417
|
+
}
|
|
1418
|
+
records.sort((a, b) => {
|
|
1419
|
+
const aTimestamp = typeof a.obj?.timestamp === "string" ? a.obj.timestamp : "";
|
|
1420
|
+
const bTimestamp = typeof b.obj?.timestamp === "string" ? b.obj.timestamp : "";
|
|
1421
|
+
return aTimestamp.localeCompare(bTimestamp) || a.fileIndex - b.fileIndex || a.lineIndex - b.lineIndex;
|
|
1422
|
+
});
|
|
1423
|
+
const firstFileByRecord = new Map();
|
|
1424
|
+
for (const record of records) {
|
|
1425
|
+
const semanticKey = JSON.stringify(record.obj);
|
|
1426
|
+
const firstFile = firstFileByRecord.get(semanticKey);
|
|
1427
|
+
if (firstFile !== undefined && firstFile !== record.fileIndex) continue;
|
|
1428
|
+
if (firstFile === undefined) firstFileByRecord.set(semanticKey, record.fileIndex);
|
|
1429
|
+
yield record.obj;
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
function rolloutSessionIdFromPath(filePath) {
|
|
1434
|
+
const match = path.basename(String(filePath || "")).match(
|
|
1435
|
+
/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl(?:\.zstd?)?$/i,
|
|
1436
|
+
);
|
|
1437
|
+
return match ? match[1] : null;
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
function isCodexResumeInvalidated(error) {
|
|
1441
|
+
return error?.code === CODEX_RESUME_INVALIDATED;
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
module.exports = {
|
|
1445
|
+
parseCodexRolloutFile,
|
|
1446
|
+
extractTokenCount,
|
|
1447
|
+
extractSkillNameFromFunctionCall,
|
|
1448
|
+
formatToolDisplayName,
|
|
1449
|
+
normalizeToolName,
|
|
1450
|
+
pickDelta,
|
|
1451
|
+
normalizeUsage,
|
|
1452
|
+
totalsReset,
|
|
1453
|
+
listJsonlFiles,
|
|
1454
|
+
listCodexSessionFiles,
|
|
1455
|
+
discoverCodexSessionFiles,
|
|
1456
|
+
rolloutPathDay,
|
|
1457
|
+
isConservativeRangeCandidate,
|
|
1458
|
+
safeJsonParse,
|
|
1459
|
+
dayKeyToIsoBounds,
|
|
1460
|
+
formatPartsDayKey,
|
|
1461
|
+
getZonedParts,
|
|
1462
|
+
timestampDayKey,
|
|
1463
|
+
isTimestampInRequestedDayRange,
|
|
1464
|
+
rolloutSessionIdFromPath,
|
|
1465
|
+
isCodexResumeInvalidated,
|
|
1466
|
+
isRecoverableGroupedReadError,
|
|
1467
|
+
finalizeToolRows,
|
|
1468
|
+
finalizeSkillRows,
|
|
1469
|
+
finalizeExecRows,
|
|
1470
|
+
buildSkillStatsEntry,
|
|
1471
|
+
};
|