tledger 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -67
- package/bin/token-ledger-rates.mjs +62 -0
- package/bin/token-ledger-terminal.mjs +97 -154
- package/bin/token-ledger-trend-image.mjs +945 -0
- package/bin/token-ledger-trend-terminal.mjs +609 -0
- package/bin/token-ledger-trend.mjs +745 -0
- package/bin/token-ledger-tui.mjs +15 -21
- package/bin/token-ledger.mjs +390 -102
- package/lib/{token-ledger-collector.mjs → token-ledger-importer.mjs} +244 -407
- package/package.json +17 -10
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Token Ledger local collector
|
|
5
5
|
*
|
|
6
6
|
* Reads Codex's local JSONL rollouts and metadata database, then writes a
|
|
7
|
-
* privacy-reduced snapshot for the Token Ledger
|
|
7
|
+
* privacy-reduced snapshot for the Token Ledger site. It never exports message
|
|
8
8
|
* bodies, tool arguments/results, reasoning text, instructions, credential
|
|
9
9
|
* fields, or full local paths in the generated snapshot.
|
|
10
10
|
*
|
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
stat,
|
|
24
24
|
writeFile,
|
|
25
25
|
} from "node:fs/promises";
|
|
26
|
-
import {
|
|
26
|
+
import { homedir } from "node:os";
|
|
27
27
|
import {
|
|
28
28
|
basename,
|
|
29
29
|
dirname,
|
|
@@ -32,63 +32,33 @@ import {
|
|
|
32
32
|
} from "node:path";
|
|
33
33
|
import { pathToFileURL } from "node:url";
|
|
34
34
|
import { createInterface } from "node:readline";
|
|
35
|
-
import {
|
|
36
|
-
isMainThread,
|
|
37
|
-
parentPort,
|
|
38
|
-
Worker,
|
|
39
|
-
workerData,
|
|
40
|
-
} from "node:worker_threads";
|
|
35
|
+
import { DatabaseSync } from "node:sqlite";
|
|
41
36
|
|
|
42
37
|
const SCHEMA_VERSION = 1;
|
|
38
|
+
const RATE_CARD_AS_OF = "2026-08-17";
|
|
39
|
+
const FAST_MODE_MULTIPLIER = 1.5;
|
|
40
|
+
const RATE_CARD_URL = "https://help.openai.com/en/articles/20001106";
|
|
43
41
|
const WEEK_MINUTES = 10_080;
|
|
44
|
-
const DEFAULT_MAX_SCAN_WORKERS = 4;
|
|
45
|
-
const MAX_SCAN_WORKERS = 6;
|
|
46
|
-
const SCAN_WORKER_MODE = "token-ledger-rollout-scanner";
|
|
47
42
|
const UUID_AT_END =
|
|
48
43
|
/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
"
|
|
53
|
-
"
|
|
54
|
-
"
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
if (!databaseSyncPromise) {
|
|
62
|
-
databaseSyncPromise = (async () => {
|
|
63
|
-
const originalEmitWarning = process.emitWarning;
|
|
64
|
-
process.emitWarning = function emitWarning(warning, ...arguments_) {
|
|
65
|
-
const message = warning instanceof Error ? warning.message : String(warning);
|
|
66
|
-
const type = typeof arguments_[0] === "string"
|
|
67
|
-
? arguments_[0]
|
|
68
|
-
: arguments_[0]?.type;
|
|
69
|
-
if (
|
|
70
|
-
type === "ExperimentalWarning" &&
|
|
71
|
-
message === "SQLite is an experimental feature and might change at any time"
|
|
72
|
-
) {
|
|
73
|
-
return;
|
|
74
|
-
}
|
|
75
|
-
return originalEmitWarning.call(this, warning, ...arguments_);
|
|
76
|
-
};
|
|
77
|
-
try {
|
|
78
|
-
return (await import("node:sqlite")).DatabaseSync;
|
|
79
|
-
} finally {
|
|
80
|
-
process.emitWarning = originalEmitWarning;
|
|
81
|
-
}
|
|
82
|
-
})();
|
|
83
|
-
}
|
|
84
|
-
return databaseSyncPromise;
|
|
85
|
-
}
|
|
44
|
+
|
|
45
|
+
const RATE_CARD = {
|
|
46
|
+
"gpt-5.6-sol": { input: 125, cached: 12.5, output: 750 },
|
|
47
|
+
"gpt-5.6-terra": { input: 50, cached: 5, output: 300 },
|
|
48
|
+
"gpt-5.6-luna": { input: 5, cached: 0.5, output: 30 },
|
|
49
|
+
"gpt-5.5": { input: 125, cached: 12.5, output: 750 },
|
|
50
|
+
"gpt-5.5-cyber": { input: 500, cached: 50, output: 3_000 },
|
|
51
|
+
"gpt-5.4": { input: 62.5, cached: 6.25, output: 375 },
|
|
52
|
+
"gpt-5.4-mini": { input: 18.75, cached: 1.875, output: 113 },
|
|
53
|
+
"gpt-5.3-codex": { input: 43.75, cached: 4.375, output: 350 },
|
|
54
|
+
"gpt-5.2": { input: 43.75, cached: 4.375, output: 350 },
|
|
55
|
+
};
|
|
86
56
|
|
|
87
57
|
function usage() {
|
|
88
58
|
return `Token Ledger local collector
|
|
89
59
|
|
|
90
60
|
Usage:
|
|
91
|
-
node
|
|
61
|
+
node token-ledger-importer.mjs [options]
|
|
92
62
|
|
|
93
63
|
Options:
|
|
94
64
|
--output <file> Snapshot destination (default: token-ledger-snapshot.json)
|
|
@@ -97,9 +67,10 @@ Options:
|
|
|
97
67
|
--no-archived Skip archived_sessions
|
|
98
68
|
--help Show this help
|
|
99
69
|
|
|
100
|
-
The snapshot contains
|
|
101
|
-
contains
|
|
102
|
-
|
|
70
|
+
The snapshot contains usage metadata and Codex display titles only. It never
|
|
71
|
+
contains message bodies, tool payloads, reasoning text, credential fields, or
|
|
72
|
+
full local paths in its output. Codex display titles may contain user-written
|
|
73
|
+
text.`;
|
|
103
74
|
}
|
|
104
75
|
|
|
105
76
|
function parseArgs(argv) {
|
|
@@ -141,45 +112,6 @@ function hash(value, length = 24) {
|
|
|
141
112
|
return createHash("sha256").update(String(value)).digest("hex").slice(0, length);
|
|
142
113
|
}
|
|
143
114
|
|
|
144
|
-
export function sourceFingerprint(codexHome, includeArchived = true) {
|
|
145
|
-
return hash(JSON.stringify({
|
|
146
|
-
codexHome: resolve(codexHome),
|
|
147
|
-
includeArchived,
|
|
148
|
-
}));
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function sanitizeLabel(value, fallback = "unknown") {
|
|
152
|
-
const label = String(value ?? "")
|
|
153
|
-
.replace(/\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g, "")
|
|
154
|
-
.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
155
|
-
.replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ")
|
|
156
|
-
.replace(/\s+/g, " ")
|
|
157
|
-
.trim();
|
|
158
|
-
return (label || fallback).slice(0, 160);
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
export async function writePrivateSnapshot(output, snapshot) {
|
|
162
|
-
const destination = resolve(output);
|
|
163
|
-
const directory = dirname(destination);
|
|
164
|
-
const temporary = resolve(
|
|
165
|
-
directory,
|
|
166
|
-
`.token-ledger-${process.pid}-${randomUUID()}.tmp`,
|
|
167
|
-
);
|
|
168
|
-
await mkdir(directory, { recursive: true });
|
|
169
|
-
try {
|
|
170
|
-
await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
|
|
171
|
-
encoding: "utf8",
|
|
172
|
-
flag: "wx",
|
|
173
|
-
mode: 0o600,
|
|
174
|
-
});
|
|
175
|
-
await chmod(temporary, 0o600);
|
|
176
|
-
await rename(temporary, destination);
|
|
177
|
-
await chmod(destination, 0o600);
|
|
178
|
-
} finally {
|
|
179
|
-
await rm(temporary, { force: true });
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
115
|
function asFiniteNumber(value) {
|
|
184
116
|
const number = Number(value ?? 0);
|
|
185
117
|
return Number.isFinite(number) ? number : 0;
|
|
@@ -230,9 +162,11 @@ function hasDetailedBreakdown(usage) {
|
|
|
230
162
|
}
|
|
231
163
|
|
|
232
164
|
function normalizeModel(model) {
|
|
233
|
-
const value =
|
|
165
|
+
const value = String(model || "unknown")
|
|
166
|
+
.trim()
|
|
234
167
|
.toLowerCase()
|
|
235
168
|
.replaceAll("_", "-");
|
|
169
|
+
if (RATE_CARD[value]) return value;
|
|
236
170
|
if (value.startsWith("gpt-5.6-sol")) return "gpt-5.6-sol";
|
|
237
171
|
if (value.startsWith("gpt-5.6-terra")) return "gpt-5.6-terra";
|
|
238
172
|
if (value.startsWith("gpt-5.6-luna")) return "gpt-5.6-luna";
|
|
@@ -245,6 +179,20 @@ function normalizeModel(model) {
|
|
|
245
179
|
return value || "unknown";
|
|
246
180
|
}
|
|
247
181
|
|
|
182
|
+
function creditsForUsage(model, usage) {
|
|
183
|
+
if (!hasDetailedBreakdown(usage)) return null;
|
|
184
|
+
const rate = RATE_CARD[normalizeModel(model)];
|
|
185
|
+
if (!rate) return null;
|
|
186
|
+
const cached = Math.min(usage.inputTokens, usage.cachedInputTokens);
|
|
187
|
+
const uncached = Math.max(0, usage.inputTokens - cached);
|
|
188
|
+
return (
|
|
189
|
+
(uncached * rate.input +
|
|
190
|
+
cached * rate.cached +
|
|
191
|
+
usage.outputTokens * rate.output) /
|
|
192
|
+
1_000_000
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
248
196
|
function parseStructuredSource(value) {
|
|
249
197
|
if (!value) return null;
|
|
250
198
|
if (typeof value === "object") return value;
|
|
@@ -275,15 +223,9 @@ function sourceLabels(threadSource, rawSource) {
|
|
|
275
223
|
if (source === "exec") return { source: "cli", useType: "cli" };
|
|
276
224
|
if (source === "vscode") return { source: "desktop", useType: "interactive" };
|
|
277
225
|
if (typeof source === "string" && source) {
|
|
278
|
-
return {
|
|
279
|
-
source: sanitizeLabel(source).slice(0, 40),
|
|
280
|
-
useType: sanitizeLabel(threadSource || "interactive").slice(0, 40),
|
|
281
|
-
};
|
|
226
|
+
return { source: source.slice(0, 40), useType: threadSource || "interactive" };
|
|
282
227
|
}
|
|
283
|
-
return {
|
|
284
|
-
source: "unknown",
|
|
285
|
-
useType: sanitizeLabel(threadSource || "unknown").slice(0, 40),
|
|
286
|
-
};
|
|
228
|
+
return { source: "unknown", useType: threadSource || "unknown" };
|
|
287
229
|
}
|
|
288
230
|
|
|
289
231
|
function cleanRemote(value) {
|
|
@@ -291,15 +233,15 @@ function cleanRemote(value) {
|
|
|
291
233
|
const remote = String(value).trim();
|
|
292
234
|
const scp = remote.match(/^[^@]+@([^:]+):(.+)$/);
|
|
293
235
|
if (scp) {
|
|
294
|
-
return
|
|
236
|
+
return `${scp[1]}/${scp[2].replace(/\.git$/i, "")}`;
|
|
295
237
|
}
|
|
296
238
|
try {
|
|
297
239
|
const url = new URL(remote);
|
|
298
240
|
const path = url.pathname.replace(/^\/+/, "").replace(/\.git$/i, "");
|
|
299
|
-
return
|
|
241
|
+
return path ? `${url.hostname}/${path}` : url.hostname;
|
|
300
242
|
} catch {
|
|
301
243
|
const withoutCredentials = remote.replace(/\/\/[^/@]+@/, "//");
|
|
302
|
-
return
|
|
244
|
+
return withoutCredentials.replace(/\.git$/i, "").slice(0, 160);
|
|
303
245
|
}
|
|
304
246
|
}
|
|
305
247
|
|
|
@@ -307,15 +249,37 @@ function projectLabel(cwd, gitOrigin) {
|
|
|
307
249
|
const remote = cleanRemote(gitOrigin);
|
|
308
250
|
if (remote) {
|
|
309
251
|
const parts = remote.split("/").filter(Boolean);
|
|
310
|
-
return
|
|
252
|
+
return parts.slice(-2).join("/") || "Unknown project";
|
|
311
253
|
}
|
|
312
254
|
const path = String(cwd || "").replaceAll("\\", "/").replace(/\/+$/, "");
|
|
313
255
|
const worktree = path.match(/\/\.codex\/worktrees\/[^/]+\/([^/]+)$/);
|
|
314
256
|
if (worktree) return worktree[1];
|
|
315
257
|
const name = basename(path);
|
|
316
|
-
return name && name !== "." && name !== "/"
|
|
317
|
-
|
|
318
|
-
|
|
258
|
+
return name && name !== "." && name !== "/" ? name : "Unknown project";
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function safeTitle(row, sessionTitle) {
|
|
262
|
+
const candidate = String(sessionTitle || row?.title || row?.name || "").trim();
|
|
263
|
+
const subagent =
|
|
264
|
+
row?.thread_source === "subagent" ||
|
|
265
|
+
String(row?.source || "").includes('"subagent"');
|
|
266
|
+
if (
|
|
267
|
+
!candidate ||
|
|
268
|
+
candidate.includes("<codex_delegation>") ||
|
|
269
|
+
(subagent && candidate.length > 120)
|
|
270
|
+
) {
|
|
271
|
+
if (row?.agent_nickname) return `Subagent · ${row.agent_nickname}`;
|
|
272
|
+
return subagent ? `Subagent · ${String(row?.id || "").slice(0, 8)}` : "Untitled task";
|
|
273
|
+
}
|
|
274
|
+
return candidate
|
|
275
|
+
.replace(/\/Users\/[^\s"'`]+/g, "[local path]")
|
|
276
|
+
.replace(/\/(?:private\/)?tmp\/[^\s"'`]+/g, "[temporary path]")
|
|
277
|
+
.replace(
|
|
278
|
+
/\b(?:sk-[A-Za-z0-9_-]{16,}|lin_api_[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_-]{16,})\b/g,
|
|
279
|
+
"[redacted credential-like text]",
|
|
280
|
+
)
|
|
281
|
+
.replace(/\s+/g, " ")
|
|
282
|
+
.slice(0, 180);
|
|
319
283
|
}
|
|
320
284
|
|
|
321
285
|
async function pathExists(path) {
|
|
@@ -327,6 +291,28 @@ async function pathExists(path) {
|
|
|
327
291
|
}
|
|
328
292
|
}
|
|
329
293
|
|
|
294
|
+
export async function writePrivateSnapshot(output, snapshot) {
|
|
295
|
+
const destination = resolve(output);
|
|
296
|
+
const directory = dirname(destination);
|
|
297
|
+
const temporary = resolve(
|
|
298
|
+
directory,
|
|
299
|
+
`.token-ledger-${process.pid}-${randomUUID()}.tmp`,
|
|
300
|
+
);
|
|
301
|
+
await mkdir(directory, { recursive: true });
|
|
302
|
+
try {
|
|
303
|
+
await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
|
|
304
|
+
encoding: "utf8",
|
|
305
|
+
flag: "wx",
|
|
306
|
+
mode: 0o600,
|
|
307
|
+
});
|
|
308
|
+
await chmod(temporary, 0o600);
|
|
309
|
+
await rename(temporary, destination);
|
|
310
|
+
await chmod(destination, 0o600);
|
|
311
|
+
} finally {
|
|
312
|
+
await rm(temporary, { force: true });
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
330
316
|
export async function listJsonlFiles(root) {
|
|
331
317
|
if (!(await pathExists(root))) return [];
|
|
332
318
|
const found = [];
|
|
@@ -343,33 +329,51 @@ export async function listJsonlFiles(root) {
|
|
|
343
329
|
return found;
|
|
344
330
|
}
|
|
345
331
|
|
|
346
|
-
export async function
|
|
332
|
+
export async function latestSourceModifiedAt(codexHome, includeArchived = true) {
|
|
347
333
|
const roots = [resolve(codexHome, "sessions")];
|
|
348
334
|
if (includeArchived) {
|
|
349
335
|
roots.push(resolve(codexHome, "archived_sessions"));
|
|
350
336
|
}
|
|
351
337
|
const files = (await Promise.all(roots.map((root) => listJsonlFiles(root))))
|
|
352
338
|
.flat();
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
if (!sourceFiles.length) {
|
|
362
|
-
return { latestMtimeMs: 0, fileCount: 0 };
|
|
339
|
+
const metadataFiles = [
|
|
340
|
+
resolve(codexHome, "session_index.jsonl"),
|
|
341
|
+
resolve(codexHome, "state_5.sqlite"),
|
|
342
|
+
resolve(codexHome, "sqlite", "state_5.sqlite"),
|
|
343
|
+
];
|
|
344
|
+
const existingMetadataFiles = [];
|
|
345
|
+
for (const path of metadataFiles) {
|
|
346
|
+
if (await pathExists(path)) existingMetadataFiles.push(path);
|
|
363
347
|
}
|
|
348
|
+
const sourceFiles = [...files, ...existingMetadataFiles];
|
|
349
|
+
if (!sourceFiles.length) return 0;
|
|
364
350
|
const stats = await Promise.all(sourceFiles.map((path) => stat(path)));
|
|
365
|
-
return
|
|
366
|
-
latestMtimeMs: Math.max(...stats.map((entry) => entry.mtimeMs)),
|
|
367
|
-
fileCount: sourceFiles.length,
|
|
368
|
-
};
|
|
351
|
+
return Math.max(...stats.map((entry) => entry.mtimeMs));
|
|
369
352
|
}
|
|
370
353
|
|
|
371
|
-
|
|
372
|
-
|
|
354
|
+
async function readSessionTitles(path) {
|
|
355
|
+
const titles = new Map();
|
|
356
|
+
if (!(await pathExists(path))) return titles;
|
|
357
|
+
const input = createReadStream(path, { encoding: "utf8" });
|
|
358
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
359
|
+
for await (const line of lines) {
|
|
360
|
+
if (!line.trim()) continue;
|
|
361
|
+
try {
|
|
362
|
+
const record = JSON.parse(line);
|
|
363
|
+
if (!record?.id || !record?.thread_name) continue;
|
|
364
|
+
const timestamp = new Date(record.updated_at || 0).getTime();
|
|
365
|
+
const current = titles.get(record.id);
|
|
366
|
+
if (!current || timestamp >= current.timestamp) {
|
|
367
|
+
titles.set(record.id, {
|
|
368
|
+
title: String(record.thread_name).replace(/\s+/g, " ").slice(0, 180),
|
|
369
|
+
timestamp,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
} catch {
|
|
373
|
+
// The thread index is optional; malformed lines do not affect token totals.
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return titles;
|
|
373
377
|
}
|
|
374
378
|
|
|
375
379
|
async function readState(codexHome) {
|
|
@@ -384,13 +388,13 @@ async function readState(codexHome) {
|
|
|
384
388
|
const parents = new Map();
|
|
385
389
|
if (!path) return { path: null, rows, parents };
|
|
386
390
|
|
|
387
|
-
const DatabaseSync = await loadDatabaseSync();
|
|
388
391
|
const database = new DatabaseSync(path, { readOnly: true });
|
|
389
392
|
try {
|
|
390
393
|
const threadRows = database
|
|
391
394
|
.prepare(
|
|
392
|
-
`SELECT id, created_at, updated_at, source, cwd,
|
|
393
|
-
tokens_used,
|
|
395
|
+
`SELECT id, created_at, updated_at, source, cwd, title, name,
|
|
396
|
+
tokens_used, git_sha, git_branch, git_origin_url,
|
|
397
|
+
agent_nickname, agent_role, model, reasoning_effort,
|
|
394
398
|
thread_source
|
|
395
399
|
FROM threads`,
|
|
396
400
|
)
|
|
@@ -431,6 +435,7 @@ function taskStartCandidate(record, threadId, stateRow, fileContext) {
|
|
|
431
435
|
cwd: fileContext.cwd || stateRow?.cwd || "",
|
|
432
436
|
gitOrigin: fileContext.gitOrigin || stateRow?.git_origin_url || null,
|
|
433
437
|
rawSource: fileContext.rawSource || stateRow?.source || null,
|
|
438
|
+
serviceTier: fileContext.serviceTier,
|
|
434
439
|
};
|
|
435
440
|
}
|
|
436
441
|
|
|
@@ -455,19 +460,38 @@ function rememberQuota(quotaMap, rateLimits, occurrence) {
|
|
|
455
460
|
usedPercent,
|
|
456
461
|
windowMinutes,
|
|
457
462
|
resetsAt,
|
|
458
|
-
|
|
463
|
+
planType: String(rateLimits.plan_type || "unknown"),
|
|
464
|
+
limitKey: hash(limitKey, 16),
|
|
465
|
+
limitName: rateLimits.limit_name
|
|
466
|
+
? String(rateLimits.limit_name).slice(0, 80)
|
|
467
|
+
: null,
|
|
459
468
|
source: "log",
|
|
460
469
|
turnId: occurrence.turnId || null,
|
|
461
470
|
originalLikely: occurrence.originalLikely,
|
|
462
471
|
};
|
|
463
|
-
|
|
472
|
+
const current = quotaMap.get(key);
|
|
473
|
+
if (
|
|
474
|
+
!current ||
|
|
475
|
+
(!current.originalLikely && candidate.originalLikely) ||
|
|
476
|
+
(current.originalLikely === candidate.originalLikely &&
|
|
477
|
+
candidate.timestamp < current.timestamp)
|
|
478
|
+
) {
|
|
479
|
+
quotaMap.set(key, candidate);
|
|
480
|
+
}
|
|
464
481
|
}
|
|
465
482
|
}
|
|
466
483
|
|
|
467
484
|
function responseCall(record) {
|
|
468
485
|
if (record.type !== "response_item") return null;
|
|
469
486
|
const payload = record.payload;
|
|
470
|
-
|
|
487
|
+
const allowed = new Set([
|
|
488
|
+
"function_call",
|
|
489
|
+
"custom_tool_call",
|
|
490
|
+
"tool_search_call",
|
|
491
|
+
"web_search_call",
|
|
492
|
+
"image_generation_call",
|
|
493
|
+
]);
|
|
494
|
+
if (!payload || !allowed.has(payload.type)) return null;
|
|
471
495
|
return {
|
|
472
496
|
type: payload.type,
|
|
473
497
|
name: String(payload.name || payload.namespace || payload.type).slice(0, 80),
|
|
@@ -475,97 +499,17 @@ function responseCall(record) {
|
|
|
475
499
|
};
|
|
476
500
|
}
|
|
477
501
|
|
|
478
|
-
|
|
479
|
-
return RELEVANT_RECORD_TYPE.test(line);
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
function rolloutThreadId(path) {
|
|
502
|
+
async function scanRollout(path, context) {
|
|
483
503
|
const match = path.match(UUID_AT_END);
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
function createScanFragment() {
|
|
488
|
-
return {
|
|
489
|
-
parents: new Map(),
|
|
490
|
-
origins: new Map(),
|
|
491
|
-
tokens: new Map(),
|
|
492
|
-
quotas: new Map(),
|
|
493
|
-
calls: new Map(),
|
|
494
|
-
parseErrors: 0,
|
|
495
|
-
duplicateEventsSkipped: 0,
|
|
496
|
-
correctionIntervals: 0,
|
|
497
|
-
};
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
function rememberOrigin(originMap, turnId, candidate) {
|
|
501
|
-
const current = originMap.get(turnId);
|
|
502
|
-
if (!current || candidate.deltaMs < current.deltaMs) {
|
|
503
|
-
originMap.set(turnId, candidate);
|
|
504
|
-
}
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
function rememberCall(callMap, key, candidate) {
|
|
508
|
-
const current = callMap.get(key);
|
|
509
|
-
if (!current || (!current.originalLikely && candidate.originalLikely)) {
|
|
510
|
-
callMap.set(key, candidate);
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
function rememberToken(context, key, candidate) {
|
|
515
|
-
const current = context.tokens.get(key);
|
|
516
|
-
if (!current) {
|
|
517
|
-
context.tokens.set(key, candidate);
|
|
518
|
-
return;
|
|
519
|
-
}
|
|
520
|
-
context.duplicateEventsSkipped += 1;
|
|
521
|
-
if (!current.originalLikely && candidate.originalLikely) {
|
|
522
|
-
current.occurrence = candidate.occurrence;
|
|
523
|
-
current.originalLikely = true;
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
function rememberQuotaCandidate(quotaMap, key, candidate) {
|
|
528
|
-
const current = quotaMap.get(key);
|
|
529
|
-
if (
|
|
530
|
-
!current ||
|
|
531
|
-
(!current.originalLikely && candidate.originalLikely) ||
|
|
532
|
-
(current.originalLikely === candidate.originalLikely &&
|
|
533
|
-
candidate.timestamp < current.timestamp)
|
|
534
|
-
) {
|
|
535
|
-
quotaMap.set(key, candidate);
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
function mergeScanFragment(context, fragment) {
|
|
540
|
-
for (const [threadId, parentThreadId] of fragment.parents) {
|
|
541
|
-
context.parents.set(threadId, parentThreadId);
|
|
542
|
-
}
|
|
543
|
-
for (const [turnId, candidate] of fragment.origins) {
|
|
544
|
-
rememberOrigin(context.origins, turnId, candidate);
|
|
545
|
-
}
|
|
546
|
-
for (const [key, candidate] of fragment.tokens) {
|
|
547
|
-
rememberToken(context, key, candidate);
|
|
548
|
-
}
|
|
549
|
-
for (const [key, candidate] of fragment.quotas) {
|
|
550
|
-
rememberQuotaCandidate(context.quotas, key, candidate);
|
|
551
|
-
}
|
|
552
|
-
for (const [key, candidate] of fragment.calls) {
|
|
553
|
-
rememberCall(context.calls, key, candidate);
|
|
554
|
-
}
|
|
555
|
-
context.parseErrors += fragment.parseErrors;
|
|
556
|
-
context.duplicateEventsSkipped += fragment.duplicateEventsSkipped;
|
|
557
|
-
context.correctionIntervals += fragment.correctionIntervals;
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
async function scanRollout(path, stateRow) {
|
|
561
|
-
const context = createScanFragment();
|
|
562
|
-
const threadId = rolloutThreadId(path);
|
|
504
|
+
const threadId = match?.[1] || `file-${hash(path)}`;
|
|
505
|
+
const stateRow = context.stateRows.get(threadId);
|
|
563
506
|
const fileContext = {
|
|
564
507
|
model: stateRow?.model || "unknown",
|
|
565
508
|
effort: stateRow?.reasoning_effort || "unknown",
|
|
566
509
|
cwd: stateRow?.cwd || "",
|
|
567
510
|
gitOrigin: stateRow?.git_origin_url || null,
|
|
568
511
|
rawSource: stateRow?.source || null,
|
|
512
|
+
serviceTier: null,
|
|
569
513
|
};
|
|
570
514
|
const callOrdinals = new Map();
|
|
571
515
|
let currentTurnId = "";
|
|
@@ -575,7 +519,7 @@ async function scanRollout(path, stateRow) {
|
|
|
575
519
|
const input = createReadStream(path, { encoding: "utf8" });
|
|
576
520
|
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
577
521
|
for await (const line of lines) {
|
|
578
|
-
if (!line
|
|
522
|
+
if (!line.trim()) continue;
|
|
579
523
|
let record;
|
|
580
524
|
try {
|
|
581
525
|
record = JSON.parse(line);
|
|
@@ -613,7 +557,10 @@ async function scanRollout(path, stateRow) {
|
|
|
613
557
|
stateRow,
|
|
614
558
|
fileContext,
|
|
615
559
|
);
|
|
616
|
-
|
|
560
|
+
const currentBest = context.origins.get(currentTurnId);
|
|
561
|
+
if (!currentBest || currentCandidate.deltaMs < currentBest.deltaMs) {
|
|
562
|
+
context.origins.set(currentTurnId, currentCandidate);
|
|
563
|
+
}
|
|
617
564
|
}
|
|
618
565
|
continue;
|
|
619
566
|
}
|
|
@@ -638,6 +585,10 @@ async function scanRollout(path, stateRow) {
|
|
|
638
585
|
const settings = record.payload?.thread_settings;
|
|
639
586
|
fileContext.model = settings?.model || fileContext.model;
|
|
640
587
|
fileContext.effort = settings?.reasoning_effort || fileContext.effort;
|
|
588
|
+
const serviceTier = String(settings?.service_tier ?? "").trim();
|
|
589
|
+
fileContext.serviceTier = serviceTier
|
|
590
|
+
? serviceTier.slice(0, 40)
|
|
591
|
+
: null;
|
|
641
592
|
continue;
|
|
642
593
|
}
|
|
643
594
|
|
|
@@ -655,6 +606,7 @@ async function scanRollout(path, stateRow) {
|
|
|
655
606
|
cwd: fileContext.cwd,
|
|
656
607
|
gitOrigin: fileContext.gitOrigin,
|
|
657
608
|
rawSource: fileContext.rawSource,
|
|
609
|
+
serviceTier: fileContext.serviceTier,
|
|
658
610
|
};
|
|
659
611
|
|
|
660
612
|
const call = responseCall(record);
|
|
@@ -665,11 +617,14 @@ async function scanRollout(path, stateRow) {
|
|
|
665
617
|
const callKey = call.stableId
|
|
666
618
|
? `id|${call.stableId}`
|
|
667
619
|
: `ordinal|${ordinalBase}|${ordinal}`;
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
620
|
+
const existing = context.calls.get(callKey);
|
|
621
|
+
if (!existing || (!existing.originalLikely && originalLikely)) {
|
|
622
|
+
context.calls.set(callKey, {
|
|
623
|
+
turnId: currentTurnId,
|
|
624
|
+
threadId,
|
|
625
|
+
originalLikely,
|
|
626
|
+
});
|
|
627
|
+
}
|
|
673
628
|
continue;
|
|
674
629
|
}
|
|
675
630
|
|
|
@@ -702,7 +657,17 @@ async function scanRollout(path, stateRow) {
|
|
|
702
657
|
}
|
|
703
658
|
previousCumulative = totalTuple[5];
|
|
704
659
|
|
|
705
|
-
|
|
660
|
+
const existing = context.tokens.get(eventKey);
|
|
661
|
+
if (existing) {
|
|
662
|
+
context.duplicateEventsSkipped += 1;
|
|
663
|
+
if (!existing.originalLikely && originalLikely) {
|
|
664
|
+
existing.occurrence = occurrence;
|
|
665
|
+
existing.originalLikely = true;
|
|
666
|
+
}
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
context.tokens.set(eventKey, {
|
|
706
671
|
key: eventKey,
|
|
707
672
|
turnId: currentTurnId,
|
|
708
673
|
usage: usageFromTuple(lastTuple),
|
|
@@ -711,172 +676,24 @@ async function scanRollout(path, stateRow) {
|
|
|
711
676
|
dedupeQuality: currentTurnId ? "turn-exact" : "legacy-heuristic",
|
|
712
677
|
});
|
|
713
678
|
}
|
|
714
|
-
return context;
|
|
715
679
|
}
|
|
716
680
|
|
|
717
|
-
|
|
718
|
-
fileCount,
|
|
719
|
-
requestedWorkers = null,
|
|
720
|
-
parallelism = availableParallelism(),
|
|
721
|
-
) {
|
|
722
|
-
if (!Number.isInteger(fileCount) || fileCount < 0) {
|
|
723
|
-
throw new Error("fileCount must be a non-negative integer.");
|
|
724
|
-
}
|
|
725
|
-
if (fileCount === 0) return 0;
|
|
726
|
-
if (
|
|
727
|
-
requestedWorkers !== null &&
|
|
728
|
-
(!Number.isInteger(requestedWorkers) ||
|
|
729
|
-
requestedWorkers < 1 ||
|
|
730
|
-
requestedWorkers > MAX_SCAN_WORKERS)
|
|
731
|
-
) {
|
|
732
|
-
throw new Error(`workers must be an integer from 1 to ${MAX_SCAN_WORKERS}.`);
|
|
733
|
-
}
|
|
734
|
-
const detectedParallelism =
|
|
735
|
-
Number.isInteger(parallelism) && parallelism > 0 ? parallelism : 1;
|
|
736
|
-
const automaticWorkers = Math.min(
|
|
737
|
-
DEFAULT_MAX_SCAN_WORKERS,
|
|
738
|
-
Math.max(1, detectedParallelism - 1),
|
|
739
|
-
);
|
|
740
|
-
return Math.min(fileCount, requestedWorkers ?? automaticWorkers);
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
function reportScanProgress(onProgress, current, total, path) {
|
|
744
|
-
if (current === 1 || current === total || current % 10 === 0) {
|
|
745
|
-
onProgress({ current, total, path });
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
|
-
|
|
749
|
-
function workerFailure(message) {
|
|
750
|
-
const error = new Error(message?.error?.message || "Rollout scan worker failed.");
|
|
751
|
-
error.name = message?.error?.name || "Error";
|
|
752
|
-
if (message?.error?.code) error.code = message.error.code;
|
|
753
|
-
return error;
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
function scanWithWorker(worker, job) {
|
|
757
|
-
return new Promise((resolveJob, rejectJob) => {
|
|
758
|
-
const cleanup = () => {
|
|
759
|
-
worker.off("message", onMessage);
|
|
760
|
-
worker.off("error", onError);
|
|
761
|
-
worker.off("exit", onExit);
|
|
762
|
-
};
|
|
763
|
-
const onMessage = (message) => {
|
|
764
|
-
cleanup();
|
|
765
|
-
if (message?.jobId !== job.index) {
|
|
766
|
-
rejectJob(new Error("Rollout scan worker returned an unexpected job."));
|
|
767
|
-
} else if (message.type === "error") {
|
|
768
|
-
rejectJob(workerFailure(message));
|
|
769
|
-
} else if (message.type === "result") {
|
|
770
|
-
resolveJob(message.fragment);
|
|
771
|
-
} else {
|
|
772
|
-
rejectJob(new Error("Rollout scan worker returned an unknown response."));
|
|
773
|
-
}
|
|
774
|
-
};
|
|
775
|
-
const onError = (error) => {
|
|
776
|
-
cleanup();
|
|
777
|
-
rejectJob(error);
|
|
778
|
-
};
|
|
779
|
-
const onExit = (code) => {
|
|
780
|
-
cleanup();
|
|
781
|
-
rejectJob(
|
|
782
|
-
new Error(`Rollout scan worker exited before completing its job (${code}).`),
|
|
783
|
-
);
|
|
784
|
-
};
|
|
785
|
-
worker.once("message", onMessage);
|
|
786
|
-
worker.once("error", onError);
|
|
787
|
-
worker.once("exit", onExit);
|
|
788
|
-
try {
|
|
789
|
-
worker.postMessage({
|
|
790
|
-
type: "scan",
|
|
791
|
-
jobId: job.index,
|
|
792
|
-
path: job.path,
|
|
793
|
-
stateRow: job.stateRow,
|
|
794
|
-
});
|
|
795
|
-
} catch (error) {
|
|
796
|
-
cleanup();
|
|
797
|
-
rejectJob(error);
|
|
798
|
-
}
|
|
799
|
-
});
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
async function scanRolloutsSequential(jobs, onProgress) {
|
|
803
|
-
const fragments = new Array(jobs.length);
|
|
804
|
-
for (let index = 0; index < jobs.length; index += 1) {
|
|
805
|
-
const job = jobs[index];
|
|
806
|
-
fragments[job.index] = await scanRollout(job.path, job.stateRow);
|
|
807
|
-
reportScanProgress(onProgress, index + 1, jobs.length, job.path);
|
|
808
|
-
}
|
|
809
|
-
return fragments;
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
async function scanRolloutsInParallel(jobs, workerCount, onProgress) {
|
|
813
|
-
const scheduled = [...jobs].sort(
|
|
814
|
-
(left, right) => right.size - left.size || left.index - right.index,
|
|
815
|
-
);
|
|
816
|
-
const fragments = new Array(jobs.length);
|
|
817
|
-
const workers = Array.from(
|
|
818
|
-
{ length: workerCount },
|
|
819
|
-
() => new Worker(new URL(import.meta.url), {
|
|
820
|
-
execArgv: [],
|
|
821
|
-
workerData: { mode: SCAN_WORKER_MODE },
|
|
822
|
-
}),
|
|
823
|
-
);
|
|
824
|
-
let nextJob = 0;
|
|
825
|
-
let completed = 0;
|
|
826
|
-
try {
|
|
827
|
-
await Promise.all(workers.map(async (worker) => {
|
|
828
|
-
while (nextJob < scheduled.length) {
|
|
829
|
-
const job = scheduled[nextJob];
|
|
830
|
-
nextJob += 1;
|
|
831
|
-
fragments[job.index] = await scanWithWorker(worker, job);
|
|
832
|
-
completed += 1;
|
|
833
|
-
reportScanProgress(onProgress, completed, jobs.length, job.path);
|
|
834
|
-
}
|
|
835
|
-
}));
|
|
836
|
-
return fragments;
|
|
837
|
-
} finally {
|
|
838
|
-
await Promise.allSettled(workers.map((worker) => worker.terminate()));
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
|
|
842
|
-
async function runScanWorker() {
|
|
843
|
-
parentPort.on("message", async (message) => {
|
|
844
|
-
if (message?.type !== "scan") return;
|
|
845
|
-
try {
|
|
846
|
-
const fragment = await scanRollout(message.path, message.stateRow);
|
|
847
|
-
parentPort.postMessage({
|
|
848
|
-
type: "result",
|
|
849
|
-
jobId: message.jobId,
|
|
850
|
-
fragment,
|
|
851
|
-
});
|
|
852
|
-
} catch (error) {
|
|
853
|
-
parentPort.postMessage({
|
|
854
|
-
type: "error",
|
|
855
|
-
jobId: message.jobId,
|
|
856
|
-
error: {
|
|
857
|
-
name: error instanceof Error ? error.name : "Error",
|
|
858
|
-
message: error instanceof Error ? error.message : String(error),
|
|
859
|
-
code: error?.code || null,
|
|
860
|
-
},
|
|
861
|
-
});
|
|
862
|
-
}
|
|
863
|
-
});
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
function threadMetadata(threadId, stateRows, parents, fallback = {}) {
|
|
681
|
+
function threadMetadata(threadId, stateRows, titles, parents, fallback = {}) {
|
|
867
682
|
const row = stateRows.get(threadId);
|
|
683
|
+
const sessionTitle = titles.get(threadId)?.title;
|
|
868
684
|
const labels = sourceLabels(
|
|
869
685
|
row?.thread_source,
|
|
870
686
|
fallback.rawSource || row?.source,
|
|
871
687
|
);
|
|
872
688
|
return {
|
|
873
689
|
id: threadId,
|
|
690
|
+
title: safeTitle(row, sessionTitle),
|
|
874
691
|
project: projectLabel(
|
|
875
692
|
fallback.cwd || row?.cwd,
|
|
876
693
|
fallback.gitOrigin || row?.git_origin_url,
|
|
877
694
|
),
|
|
878
695
|
model: normalizeModel(fallback.model || row?.model || "unknown"),
|
|
879
|
-
effort:
|
|
696
|
+
effort: String(
|
|
880
697
|
fallback.effort || row?.reasoning_effort || "unknown",
|
|
881
698
|
).slice(0, 40),
|
|
882
699
|
source: labels.source,
|
|
@@ -891,7 +708,7 @@ function threadMetadata(threadId, stateRows, parents, fallback = {}) {
|
|
|
891
708
|
};
|
|
892
709
|
}
|
|
893
710
|
|
|
894
|
-
function buildSnapshot(context, options) {
|
|
711
|
+
function buildSnapshot(context, options, titles) {
|
|
895
712
|
const events = [];
|
|
896
713
|
for (const token of context.tokens.values()) {
|
|
897
714
|
const origin = token.turnId ? context.origins.get(token.turnId) : null;
|
|
@@ -907,6 +724,7 @@ function buildSnapshot(context, options) {
|
|
|
907
724
|
const metadata = threadMetadata(
|
|
908
725
|
threadId,
|
|
909
726
|
context.stateRows,
|
|
727
|
+
titles,
|
|
910
728
|
context.parents,
|
|
911
729
|
origin || occurrence,
|
|
912
730
|
);
|
|
@@ -918,11 +736,14 @@ function buildSnapshot(context, options) {
|
|
|
918
736
|
continue;
|
|
919
737
|
}
|
|
920
738
|
const breakdownAvailable = hasDetailedBreakdown(token.usage);
|
|
739
|
+
const serviceTier = occurrence.serviceTier || null;
|
|
740
|
+
const baseCredits = creditsForUsage(metadata.model, token.usage);
|
|
921
741
|
events.push({
|
|
922
742
|
...token.usage,
|
|
923
743
|
id: `evt-${hash(token.key)}`,
|
|
924
744
|
timestamp,
|
|
925
745
|
threadId,
|
|
746
|
+
threadTitle: metadata.title,
|
|
926
747
|
project: metadata.project,
|
|
927
748
|
model: metadata.model,
|
|
928
749
|
effort: metadata.effort,
|
|
@@ -930,6 +751,13 @@ function buildSnapshot(context, options) {
|
|
|
930
751
|
useType: metadata.useType,
|
|
931
752
|
turnId: token.turnId || "",
|
|
932
753
|
toolCalls: 0,
|
|
754
|
+
serviceTier,
|
|
755
|
+
rateCardCredits:
|
|
756
|
+
baseCredits === null
|
|
757
|
+
? null
|
|
758
|
+
: serviceTier === "priority"
|
|
759
|
+
? baseCredits * FAST_MODE_MULTIPLIER
|
|
760
|
+
: baseCredits,
|
|
933
761
|
breakdownAvailable,
|
|
934
762
|
dedupeQuality: token.dedupeQuality,
|
|
935
763
|
});
|
|
@@ -978,6 +806,7 @@ function buildSnapshot(context, options) {
|
|
|
978
806
|
const metadata = threadMetadata(
|
|
979
807
|
threadId,
|
|
980
808
|
context.stateRows,
|
|
809
|
+
titles,
|
|
981
810
|
context.parents,
|
|
982
811
|
origin || first || {},
|
|
983
812
|
);
|
|
@@ -991,6 +820,10 @@ function buildSnapshot(context, options) {
|
|
|
991
820
|
sum.toolCalls += event.toolCalls;
|
|
992
821
|
if (event.breakdownAvailable) sum.detailedTokens += event.totalTokens;
|
|
993
822
|
else sum.unknownBreakdownTokens += event.totalTokens;
|
|
823
|
+
if (event.rateCardCredits !== null) {
|
|
824
|
+
sum.rateCardCredits += event.rateCardCredits;
|
|
825
|
+
sum.ratedTokens += event.totalTokens;
|
|
826
|
+
}
|
|
994
827
|
return sum;
|
|
995
828
|
},
|
|
996
829
|
{
|
|
@@ -1002,6 +835,8 @@ function buildSnapshot(context, options) {
|
|
|
1002
835
|
toolCalls: 0,
|
|
1003
836
|
detailedTokens: 0,
|
|
1004
837
|
unknownBreakdownTokens: 0,
|
|
838
|
+
rateCardCredits: 0,
|
|
839
|
+
ratedTokens: 0,
|
|
1005
840
|
},
|
|
1006
841
|
);
|
|
1007
842
|
if (rows.length === 0 && !(metadata.reportedCumulativeTokens > 0)) continue;
|
|
@@ -1015,6 +850,7 @@ function buildSnapshot(context, options) {
|
|
|
1015
850
|
: "total-only";
|
|
1016
851
|
threads.push({
|
|
1017
852
|
id: threadId,
|
|
853
|
+
title: metadata.title,
|
|
1018
854
|
project: metadata.project,
|
|
1019
855
|
model: metadata.model,
|
|
1020
856
|
effort: metadata.effort,
|
|
@@ -1031,6 +867,11 @@ function buildSnapshot(context, options) {
|
|
|
1031
867
|
cachedInputTokens: totals.cachedInputTokens,
|
|
1032
868
|
outputTokens: totals.outputTokens,
|
|
1033
869
|
reasoningTokens: totals.reasoningTokens,
|
|
870
|
+
rateCardCredits:
|
|
871
|
+
totals.totalTokens > 0 && totals.ratedTokens === totals.totalTokens
|
|
872
|
+
? totals.rateCardCredits
|
|
873
|
+
: null,
|
|
874
|
+
ratedTokens: totals.ratedTokens,
|
|
1034
875
|
toolCalls: totals.toolCalls,
|
|
1035
876
|
eventCount: rows.length,
|
|
1036
877
|
coverage,
|
|
@@ -1078,7 +919,7 @@ function buildSnapshot(context, options) {
|
|
|
1078
919
|
(quota) => quota.windowMinutes === WEEK_MINUTES,
|
|
1079
920
|
);
|
|
1080
921
|
const accountWideWeekly = weeklyCandidates.filter(
|
|
1081
|
-
(quota) => quota.
|
|
922
|
+
(quota) => !quota.limitName,
|
|
1082
923
|
);
|
|
1083
924
|
const weekly = [
|
|
1084
925
|
...(accountWideWeekly.length ? accountWideWeekly : weeklyCandidates),
|
|
@@ -1111,16 +952,13 @@ function buildSnapshot(context, options) {
|
|
|
1111
952
|
label: "Local Codex snapshot",
|
|
1112
953
|
provenance: {
|
|
1113
954
|
kind: "codex-local-metadata",
|
|
1114
|
-
sourceFingerprint: sourceFingerprint(
|
|
1115
|
-
options.codexHome,
|
|
1116
|
-
options.includeArchived,
|
|
1117
|
-
),
|
|
1118
955
|
privacy:
|
|
1119
|
-
"Contains token metadata and
|
|
956
|
+
"Contains token metadata and Codex display titles only; credential fields, message bodies, reasoning text, tool payloads, and full local paths are not exported. Display titles may contain user-written text.",
|
|
957
|
+
rateCardAsOf: RATE_CARD_AS_OF,
|
|
958
|
+
rateCardUrl: RATE_CARD_URL,
|
|
1120
959
|
},
|
|
1121
960
|
coverage: {
|
|
1122
961
|
filesScanned: context.filesScanned,
|
|
1123
|
-
sourceFileCount: context.sourceFileCount,
|
|
1124
962
|
bytesScanned: context.bytesScanned,
|
|
1125
963
|
parseErrors: context.parseErrors,
|
|
1126
964
|
duplicateEventsSkipped: context.duplicateEventsSkipped,
|
|
@@ -1148,6 +986,9 @@ function buildSnapshot(context, options) {
|
|
|
1148
986
|
|
|
1149
987
|
export async function collectUsage(options, onProgress = () => {}) {
|
|
1150
988
|
const state = await readState(options.codexHome);
|
|
989
|
+
const titles = await readSessionTitles(
|
|
990
|
+
resolve(options.codexHome, "session_index.jsonl"),
|
|
991
|
+
);
|
|
1151
992
|
const roots = [resolve(options.codexHome, "sessions")];
|
|
1152
993
|
if (options.includeArchived) {
|
|
1153
994
|
roots.push(resolve(options.codexHome, "archived_sessions"));
|
|
@@ -1158,12 +999,6 @@ export async function collectUsage(options, onProgress = () => {}) {
|
|
|
1158
999
|
.flat()
|
|
1159
1000
|
.sort();
|
|
1160
1001
|
const sizes = await Promise.all(files.map((path) => stat(path)));
|
|
1161
|
-
const jobs = files.map((path, index) => ({
|
|
1162
|
-
index,
|
|
1163
|
-
path,
|
|
1164
|
-
size: sizes[index].size,
|
|
1165
|
-
stateRow: state.rows.get(rolloutThreadId(path)) || null,
|
|
1166
|
-
}));
|
|
1167
1002
|
|
|
1168
1003
|
const context = {
|
|
1169
1004
|
stateRows: state.rows,
|
|
@@ -1173,24 +1008,30 @@ export async function collectUsage(options, onProgress = () => {}) {
|
|
|
1173
1008
|
quotas: new Map(),
|
|
1174
1009
|
calls: new Map(),
|
|
1175
1010
|
filesScanned: 0,
|
|
1176
|
-
sourceFileCount: files.length + (state.path ? 1 : 0),
|
|
1177
1011
|
bytesScanned: 0,
|
|
1178
1012
|
parseErrors: 0,
|
|
1179
1013
|
duplicateEventsSkipped: 0,
|
|
1180
1014
|
correctionIntervals: 0,
|
|
1181
1015
|
};
|
|
1182
1016
|
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1017
|
+
for (let index = 0; index < files.length; index += 1) {
|
|
1018
|
+
await scanRollout(files[index], context);
|
|
1019
|
+
context.filesScanned += 1;
|
|
1020
|
+
context.bytesScanned += sizes[index].size;
|
|
1021
|
+
if (
|
|
1022
|
+
index === 0 ||
|
|
1023
|
+
index === files.length - 1 ||
|
|
1024
|
+
(index + 1) % 10 === 0
|
|
1025
|
+
) {
|
|
1026
|
+
onProgress({
|
|
1027
|
+
current: index + 1,
|
|
1028
|
+
total: files.length,
|
|
1029
|
+
path: files[index],
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1189
1032
|
}
|
|
1190
|
-
context.filesScanned = files.length;
|
|
1191
|
-
context.bytesScanned = sizes.reduce((sum, entry) => sum + entry.size, 0);
|
|
1192
1033
|
|
|
1193
|
-
return buildSnapshot(context, options);
|
|
1034
|
+
return buildSnapshot(context, options, titles);
|
|
1194
1035
|
}
|
|
1195
1036
|
|
|
1196
1037
|
async function main() {
|
|
@@ -1198,7 +1039,7 @@ async function main() {
|
|
|
1198
1039
|
try {
|
|
1199
1040
|
options = parseArgs(process.argv.slice(2));
|
|
1200
1041
|
} catch (error) {
|
|
1201
|
-
process.stderr.write(`${
|
|
1042
|
+
process.stderr.write(`${error.message}\n\n${usage()}\n`);
|
|
1202
1043
|
process.exitCode = 1;
|
|
1203
1044
|
return;
|
|
1204
1045
|
}
|
|
@@ -1207,9 +1048,7 @@ async function main() {
|
|
|
1207
1048
|
return;
|
|
1208
1049
|
}
|
|
1209
1050
|
if (!(await pathExists(options.codexHome))) {
|
|
1210
|
-
throw new Error(
|
|
1211
|
-
`Codex data directory not found: ${sanitizeLabel(options.codexHome)}`,
|
|
1212
|
-
);
|
|
1051
|
+
throw new Error(`Codex data directory not found: ${options.codexHome}`);
|
|
1213
1052
|
}
|
|
1214
1053
|
|
|
1215
1054
|
process.stdout.write("Token Ledger: scanning local Codex metadata…\n");
|
|
@@ -1223,7 +1062,7 @@ async function main() {
|
|
|
1223
1062
|
const outputSize = (await stat(options.output)).size;
|
|
1224
1063
|
process.stdout.write(
|
|
1225
1064
|
[
|
|
1226
|
-
`Snapshot: ${
|
|
1065
|
+
`Snapshot: ${options.output}`,
|
|
1227
1066
|
`Observed model-call tokens: ${snapshot.coverage.observedTokens.toLocaleString()}`,
|
|
1228
1067
|
`Unique events: ${snapshot.events.length.toLocaleString()}`,
|
|
1229
1068
|
`Duplicate/copied events skipped: ${snapshot.coverage.duplicateEventsSkipped.toLocaleString()}`,
|
|
@@ -1234,13 +1073,11 @@ async function main() {
|
|
|
1234
1073
|
}
|
|
1235
1074
|
|
|
1236
1075
|
const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : "";
|
|
1237
|
-
if (
|
|
1238
|
-
runScanWorker();
|
|
1239
|
-
} else if (isMainThread && import.meta.url === invokedPath) {
|
|
1076
|
+
if (import.meta.url === invokedPath) {
|
|
1240
1077
|
main().catch((error) => {
|
|
1241
1078
|
process.stderr.write(
|
|
1242
1079
|
`Token Ledger collector failed: ${
|
|
1243
|
-
error instanceof Error ?
|
|
1080
|
+
error instanceof Error ? error.message : String(error)
|
|
1244
1081
|
}\n`,
|
|
1245
1082
|
);
|
|
1246
1083
|
process.exitCode = 1;
|