tledger 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -80
- package/bin/token-ledger-rates.mjs +62 -0
- package/bin/token-ledger-terminal.mjs +133 -257
- package/bin/token-ledger-trend-image.mjs +945 -0
- package/bin/token-ledger-trend-terminal.mjs +609 -0
- package/bin/token-ledger-trend.mjs +745 -0
- package/bin/token-ledger-tui.mjs +15 -21
- package/bin/token-ledger.mjs +408 -248
- package/lib/{token-ledger-collector.mjs → token-ledger-importer.mjs} +256 -409
- package/package.json +18 -14
- package/lib/token-ledger-models.mjs +0 -113
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Token Ledger local collector
|
|
5
5
|
*
|
|
6
6
|
* Reads Codex's local JSONL rollouts and metadata database, then writes a
|
|
7
|
-
* privacy-reduced snapshot for the Token Ledger
|
|
7
|
+
* privacy-reduced snapshot for the Token Ledger site. It never exports message
|
|
8
8
|
* bodies, tool arguments/results, reasoning text, instructions, credential
|
|
9
9
|
* fields, or full local paths in the generated snapshot.
|
|
10
10
|
*
|
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
stat,
|
|
24
24
|
writeFile,
|
|
25
25
|
} from "node:fs/promises";
|
|
26
|
-
import {
|
|
26
|
+
import { homedir } from "node:os";
|
|
27
27
|
import {
|
|
28
28
|
basename,
|
|
29
29
|
dirname,
|
|
@@ -32,65 +32,33 @@ import {
|
|
|
32
32
|
} from "node:path";
|
|
33
33
|
import { pathToFileURL } from "node:url";
|
|
34
34
|
import { createInterface } from "node:readline";
|
|
35
|
-
import {
|
|
36
|
-
isMainThread,
|
|
37
|
-
parentPort,
|
|
38
|
-
Worker,
|
|
39
|
-
workerData,
|
|
40
|
-
} from "node:worker_threads";
|
|
41
|
-
|
|
42
|
-
import { normalizeModelIdentifier } from "./token-ledger-models.mjs";
|
|
35
|
+
import { DatabaseSync } from "node:sqlite";
|
|
43
36
|
|
|
44
37
|
const SCHEMA_VERSION = 1;
|
|
38
|
+
const RATE_CARD_AS_OF = "2026-08-17";
|
|
39
|
+
const FAST_MODE_MULTIPLIER = 1.5;
|
|
40
|
+
const RATE_CARD_URL = "https://help.openai.com/en/articles/20001106";
|
|
45
41
|
const WEEK_MINUTES = 10_080;
|
|
46
|
-
const DEFAULT_MAX_SCAN_WORKERS = 4;
|
|
47
|
-
const MAX_SCAN_WORKERS = 6;
|
|
48
|
-
const SCAN_WORKER_MODE = "token-ledger-rollout-scanner";
|
|
49
42
|
const UUID_AT_END =
|
|
50
43
|
/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
"
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if (!databaseSyncPromise) {
|
|
64
|
-
databaseSyncPromise = (async () => {
|
|
65
|
-
const originalEmitWarning = process.emitWarning;
|
|
66
|
-
process.emitWarning = function emitWarning(warning, ...arguments_) {
|
|
67
|
-
const message = warning instanceof Error ? warning.message : String(warning);
|
|
68
|
-
const type = typeof arguments_[0] === "string"
|
|
69
|
-
? arguments_[0]
|
|
70
|
-
: arguments_[0]?.type;
|
|
71
|
-
if (
|
|
72
|
-
type === "ExperimentalWarning" &&
|
|
73
|
-
message === "SQLite is an experimental feature and might change at any time"
|
|
74
|
-
) {
|
|
75
|
-
return;
|
|
76
|
-
}
|
|
77
|
-
return originalEmitWarning.call(this, warning, ...arguments_);
|
|
78
|
-
};
|
|
79
|
-
try {
|
|
80
|
-
return (await import("node:sqlite")).DatabaseSync;
|
|
81
|
-
} finally {
|
|
82
|
-
process.emitWarning = originalEmitWarning;
|
|
83
|
-
}
|
|
84
|
-
})();
|
|
85
|
-
}
|
|
86
|
-
return databaseSyncPromise;
|
|
87
|
-
}
|
|
44
|
+
|
|
45
|
+
const RATE_CARD = {
|
|
46
|
+
"gpt-5.6-sol": { input: 125, cached: 12.5, output: 750 },
|
|
47
|
+
"gpt-5.6-terra": { input: 50, cached: 5, output: 300 },
|
|
48
|
+
"gpt-5.6-luna": { input: 5, cached: 0.5, output: 30 },
|
|
49
|
+
"gpt-5.5": { input: 125, cached: 12.5, output: 750 },
|
|
50
|
+
"gpt-5.5-cyber": { input: 500, cached: 50, output: 3_000 },
|
|
51
|
+
"gpt-5.4": { input: 62.5, cached: 6.25, output: 375 },
|
|
52
|
+
"gpt-5.4-mini": { input: 18.75, cached: 1.875, output: 113 },
|
|
53
|
+
"gpt-5.3-codex": { input: 43.75, cached: 4.375, output: 350 },
|
|
54
|
+
"gpt-5.2": { input: 43.75, cached: 4.375, output: 350 },
|
|
55
|
+
};
|
|
88
56
|
|
|
89
57
|
function usage() {
|
|
90
58
|
return `Token Ledger local collector
|
|
91
59
|
|
|
92
60
|
Usage:
|
|
93
|
-
node
|
|
61
|
+
node token-ledger-importer.mjs [options]
|
|
94
62
|
|
|
95
63
|
Options:
|
|
96
64
|
--output <file> Snapshot destination (default: token-ledger-snapshot.json)
|
|
@@ -99,9 +67,10 @@ Options:
|
|
|
99
67
|
--no-archived Skip archived_sessions
|
|
100
68
|
--help Show this help
|
|
101
69
|
|
|
102
|
-
The snapshot contains
|
|
103
|
-
contains
|
|
104
|
-
|
|
70
|
+
The snapshot contains usage metadata and Codex display titles only. It never
|
|
71
|
+
contains message bodies, tool payloads, reasoning text, credential fields, or
|
|
72
|
+
full local paths in its output. Codex display titles may contain user-written
|
|
73
|
+
text.`;
|
|
105
74
|
}
|
|
106
75
|
|
|
107
76
|
function parseArgs(argv) {
|
|
@@ -143,45 +112,6 @@ function hash(value, length = 24) {
|
|
|
143
112
|
return createHash("sha256").update(String(value)).digest("hex").slice(0, length);
|
|
144
113
|
}
|
|
145
114
|
|
|
146
|
-
export function sourceFingerprint(codexHome, includeArchived = true) {
|
|
147
|
-
return hash(JSON.stringify({
|
|
148
|
-
codexHome: resolve(codexHome),
|
|
149
|
-
includeArchived,
|
|
150
|
-
}));
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
function sanitizeLabel(value, fallback = "unknown") {
|
|
154
|
-
const label = String(value ?? "")
|
|
155
|
-
.replace(/\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g, "")
|
|
156
|
-
.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
157
|
-
.replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ")
|
|
158
|
-
.replace(/\s+/g, " ")
|
|
159
|
-
.trim();
|
|
160
|
-
return (label || fallback).slice(0, 160);
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
export async function writePrivateSnapshot(output, snapshot) {
|
|
164
|
-
const destination = resolve(output);
|
|
165
|
-
const directory = dirname(destination);
|
|
166
|
-
const temporary = resolve(
|
|
167
|
-
directory,
|
|
168
|
-
`.token-ledger-${process.pid}-${randomUUID()}.tmp`,
|
|
169
|
-
);
|
|
170
|
-
await mkdir(directory, { recursive: true });
|
|
171
|
-
try {
|
|
172
|
-
await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
|
|
173
|
-
encoding: "utf8",
|
|
174
|
-
flag: "wx",
|
|
175
|
-
mode: 0o600,
|
|
176
|
-
});
|
|
177
|
-
await chmod(temporary, 0o600);
|
|
178
|
-
await rename(temporary, destination);
|
|
179
|
-
await chmod(destination, 0o600);
|
|
180
|
-
} finally {
|
|
181
|
-
await rm(temporary, { force: true });
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
|
|
185
115
|
function asFiniteNumber(value) {
|
|
186
116
|
const number = Number(value ?? 0);
|
|
187
117
|
return Number.isFinite(number) ? number : 0;
|
|
@@ -232,7 +162,35 @@ function hasDetailedBreakdown(usage) {
|
|
|
232
162
|
}
|
|
233
163
|
|
|
234
164
|
function normalizeModel(model) {
|
|
235
|
-
|
|
165
|
+
const value = String(model || "unknown")
|
|
166
|
+
.trim()
|
|
167
|
+
.toLowerCase()
|
|
168
|
+
.replaceAll("_", "-");
|
|
169
|
+
if (RATE_CARD[value]) return value;
|
|
170
|
+
if (value.startsWith("gpt-5.6-sol")) return "gpt-5.6-sol";
|
|
171
|
+
if (value.startsWith("gpt-5.6-terra")) return "gpt-5.6-terra";
|
|
172
|
+
if (value.startsWith("gpt-5.6-luna")) return "gpt-5.6-luna";
|
|
173
|
+
if (value.startsWith("gpt-5.5-cyber")) return "gpt-5.5-cyber";
|
|
174
|
+
if (value.startsWith("gpt-5.5")) return "gpt-5.5";
|
|
175
|
+
if (value.startsWith("gpt-5.4-mini")) return "gpt-5.4-mini";
|
|
176
|
+
if (value.startsWith("gpt-5.4")) return "gpt-5.4";
|
|
177
|
+
if (value.startsWith("gpt-5.3-codex")) return "gpt-5.3-codex";
|
|
178
|
+
if (value.startsWith("gpt-5.2")) return "gpt-5.2";
|
|
179
|
+
return value || "unknown";
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function creditsForUsage(model, usage) {
|
|
183
|
+
if (!hasDetailedBreakdown(usage)) return null;
|
|
184
|
+
const rate = RATE_CARD[normalizeModel(model)];
|
|
185
|
+
if (!rate) return null;
|
|
186
|
+
const cached = Math.min(usage.inputTokens, usage.cachedInputTokens);
|
|
187
|
+
const uncached = Math.max(0, usage.inputTokens - cached);
|
|
188
|
+
return (
|
|
189
|
+
(uncached * rate.input +
|
|
190
|
+
cached * rate.cached +
|
|
191
|
+
usage.outputTokens * rate.output) /
|
|
192
|
+
1_000_000
|
|
193
|
+
);
|
|
236
194
|
}
|
|
237
195
|
|
|
238
196
|
function parseStructuredSource(value) {
|
|
@@ -265,15 +223,9 @@ function sourceLabels(threadSource, rawSource) {
|
|
|
265
223
|
if (source === "exec") return { source: "cli", useType: "cli" };
|
|
266
224
|
if (source === "vscode") return { source: "desktop", useType: "interactive" };
|
|
267
225
|
if (typeof source === "string" && source) {
|
|
268
|
-
return {
|
|
269
|
-
source: sanitizeLabel(source).slice(0, 40),
|
|
270
|
-
useType: sanitizeLabel(threadSource || "interactive").slice(0, 40),
|
|
271
|
-
};
|
|
226
|
+
return { source: source.slice(0, 40), useType: threadSource || "interactive" };
|
|
272
227
|
}
|
|
273
|
-
return {
|
|
274
|
-
source: "unknown",
|
|
275
|
-
useType: sanitizeLabel(threadSource || "unknown").slice(0, 40),
|
|
276
|
-
};
|
|
228
|
+
return { source: "unknown", useType: threadSource || "unknown" };
|
|
277
229
|
}
|
|
278
230
|
|
|
279
231
|
function cleanRemote(value) {
|
|
@@ -281,15 +233,15 @@ function cleanRemote(value) {
|
|
|
281
233
|
const remote = String(value).trim();
|
|
282
234
|
const scp = remote.match(/^[^@]+@([^:]+):(.+)$/);
|
|
283
235
|
if (scp) {
|
|
284
|
-
return
|
|
236
|
+
return `${scp[1]}/${scp[2].replace(/\.git$/i, "")}`;
|
|
285
237
|
}
|
|
286
238
|
try {
|
|
287
239
|
const url = new URL(remote);
|
|
288
240
|
const path = url.pathname.replace(/^\/+/, "").replace(/\.git$/i, "");
|
|
289
|
-
return
|
|
241
|
+
return path ? `${url.hostname}/${path}` : url.hostname;
|
|
290
242
|
} catch {
|
|
291
243
|
const withoutCredentials = remote.replace(/\/\/[^/@]+@/, "//");
|
|
292
|
-
return
|
|
244
|
+
return withoutCredentials.replace(/\.git$/i, "").slice(0, 160);
|
|
293
245
|
}
|
|
294
246
|
}
|
|
295
247
|
|
|
@@ -297,15 +249,37 @@ function projectLabel(cwd, gitOrigin) {
|
|
|
297
249
|
const remote = cleanRemote(gitOrigin);
|
|
298
250
|
if (remote) {
|
|
299
251
|
const parts = remote.split("/").filter(Boolean);
|
|
300
|
-
return
|
|
252
|
+
return parts.slice(-2).join("/") || "Unknown project";
|
|
301
253
|
}
|
|
302
254
|
const path = String(cwd || "").replaceAll("\\", "/").replace(/\/+$/, "");
|
|
303
255
|
const worktree = path.match(/\/\.codex\/worktrees\/[^/]+\/([^/]+)$/);
|
|
304
256
|
if (worktree) return worktree[1];
|
|
305
257
|
const name = basename(path);
|
|
306
|
-
return name && name !== "." && name !== "/"
|
|
307
|
-
|
|
308
|
-
|
|
258
|
+
return name && name !== "." && name !== "/" ? name : "Unknown project";
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function safeTitle(row, sessionTitle) {
|
|
262
|
+
const candidate = String(sessionTitle || row?.title || row?.name || "").trim();
|
|
263
|
+
const subagent =
|
|
264
|
+
row?.thread_source === "subagent" ||
|
|
265
|
+
String(row?.source || "").includes('"subagent"');
|
|
266
|
+
if (
|
|
267
|
+
!candidate ||
|
|
268
|
+
candidate.includes("<codex_delegation>") ||
|
|
269
|
+
(subagent && candidate.length > 120)
|
|
270
|
+
) {
|
|
271
|
+
if (row?.agent_nickname) return `Subagent · ${row.agent_nickname}`;
|
|
272
|
+
return subagent ? `Subagent · ${String(row?.id || "").slice(0, 8)}` : "Untitled task";
|
|
273
|
+
}
|
|
274
|
+
return candidate
|
|
275
|
+
.replace(/\/Users\/[^\s"'`]+/g, "[local path]")
|
|
276
|
+
.replace(/\/(?:private\/)?tmp\/[^\s"'`]+/g, "[temporary path]")
|
|
277
|
+
.replace(
|
|
278
|
+
/\b(?:sk-[A-Za-z0-9_-]{16,}|lin_api_[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_-]{16,})\b/g,
|
|
279
|
+
"[redacted credential-like text]",
|
|
280
|
+
)
|
|
281
|
+
.replace(/\s+/g, " ")
|
|
282
|
+
.slice(0, 180);
|
|
309
283
|
}
|
|
310
284
|
|
|
311
285
|
async function pathExists(path) {
|
|
@@ -317,6 +291,28 @@ async function pathExists(path) {
|
|
|
317
291
|
}
|
|
318
292
|
}
|
|
319
293
|
|
|
294
|
+
export async function writePrivateSnapshot(output, snapshot) {
|
|
295
|
+
const destination = resolve(output);
|
|
296
|
+
const directory = dirname(destination);
|
|
297
|
+
const temporary = resolve(
|
|
298
|
+
directory,
|
|
299
|
+
`.token-ledger-${process.pid}-${randomUUID()}.tmp`,
|
|
300
|
+
);
|
|
301
|
+
await mkdir(directory, { recursive: true });
|
|
302
|
+
try {
|
|
303
|
+
await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
|
|
304
|
+
encoding: "utf8",
|
|
305
|
+
flag: "wx",
|
|
306
|
+
mode: 0o600,
|
|
307
|
+
});
|
|
308
|
+
await chmod(temporary, 0o600);
|
|
309
|
+
await rename(temporary, destination);
|
|
310
|
+
await chmod(destination, 0o600);
|
|
311
|
+
} finally {
|
|
312
|
+
await rm(temporary, { force: true });
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
320
316
|
export async function listJsonlFiles(root) {
|
|
321
317
|
if (!(await pathExists(root))) return [];
|
|
322
318
|
const found = [];
|
|
@@ -333,33 +329,51 @@ export async function listJsonlFiles(root) {
|
|
|
333
329
|
return found;
|
|
334
330
|
}
|
|
335
331
|
|
|
336
|
-
export async function
|
|
332
|
+
export async function latestSourceModifiedAt(codexHome, includeArchived = true) {
|
|
337
333
|
const roots = [resolve(codexHome, "sessions")];
|
|
338
334
|
if (includeArchived) {
|
|
339
335
|
roots.push(resolve(codexHome, "archived_sessions"));
|
|
340
336
|
}
|
|
341
337
|
const files = (await Promise.all(roots.map((root) => listJsonlFiles(root))))
|
|
342
338
|
.flat();
|
|
343
|
-
const
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
if (!sourceFiles.length) {
|
|
352
|
-
return { latestMtimeMs: 0, fileCount: 0 };
|
|
339
|
+
const metadataFiles = [
|
|
340
|
+
resolve(codexHome, "session_index.jsonl"),
|
|
341
|
+
resolve(codexHome, "state_5.sqlite"),
|
|
342
|
+
resolve(codexHome, "sqlite", "state_5.sqlite"),
|
|
343
|
+
];
|
|
344
|
+
const existingMetadataFiles = [];
|
|
345
|
+
for (const path of metadataFiles) {
|
|
346
|
+
if (await pathExists(path)) existingMetadataFiles.push(path);
|
|
353
347
|
}
|
|
348
|
+
const sourceFiles = [...files, ...existingMetadataFiles];
|
|
349
|
+
if (!sourceFiles.length) return 0;
|
|
354
350
|
const stats = await Promise.all(sourceFiles.map((path) => stat(path)));
|
|
355
|
-
return
|
|
356
|
-
latestMtimeMs: Math.max(...stats.map((entry) => entry.mtimeMs)),
|
|
357
|
-
fileCount: sourceFiles.length,
|
|
358
|
-
};
|
|
351
|
+
return Math.max(...stats.map((entry) => entry.mtimeMs));
|
|
359
352
|
}
|
|
360
353
|
|
|
361
|
-
|
|
362
|
-
|
|
354
|
+
async function readSessionTitles(path) {
|
|
355
|
+
const titles = new Map();
|
|
356
|
+
if (!(await pathExists(path))) return titles;
|
|
357
|
+
const input = createReadStream(path, { encoding: "utf8" });
|
|
358
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
359
|
+
for await (const line of lines) {
|
|
360
|
+
if (!line.trim()) continue;
|
|
361
|
+
try {
|
|
362
|
+
const record = JSON.parse(line);
|
|
363
|
+
if (!record?.id || !record?.thread_name) continue;
|
|
364
|
+
const timestamp = new Date(record.updated_at || 0).getTime();
|
|
365
|
+
const current = titles.get(record.id);
|
|
366
|
+
if (!current || timestamp >= current.timestamp) {
|
|
367
|
+
titles.set(record.id, {
|
|
368
|
+
title: String(record.thread_name).replace(/\s+/g, " ").slice(0, 180),
|
|
369
|
+
timestamp,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
} catch {
|
|
373
|
+
// The thread index is optional; malformed lines do not affect token totals.
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return titles;
|
|
363
377
|
}
|
|
364
378
|
|
|
365
379
|
async function readState(codexHome) {
|
|
@@ -374,13 +388,13 @@ async function readState(codexHome) {
|
|
|
374
388
|
const parents = new Map();
|
|
375
389
|
if (!path) return { path: null, rows, parents };
|
|
376
390
|
|
|
377
|
-
const DatabaseSync = await loadDatabaseSync();
|
|
378
391
|
const database = new DatabaseSync(path, { readOnly: true });
|
|
379
392
|
try {
|
|
380
393
|
const threadRows = database
|
|
381
394
|
.prepare(
|
|
382
|
-
`SELECT id, created_at, updated_at, source, cwd,
|
|
383
|
-
tokens_used,
|
|
395
|
+
`SELECT id, created_at, updated_at, source, cwd, title, name,
|
|
396
|
+
tokens_used, git_sha, git_branch, git_origin_url,
|
|
397
|
+
agent_nickname, agent_role, model, reasoning_effort,
|
|
384
398
|
thread_source
|
|
385
399
|
FROM threads`,
|
|
386
400
|
)
|
|
@@ -421,6 +435,7 @@ function taskStartCandidate(record, threadId, stateRow, fileContext) {
|
|
|
421
435
|
cwd: fileContext.cwd || stateRow?.cwd || "",
|
|
422
436
|
gitOrigin: fileContext.gitOrigin || stateRow?.git_origin_url || null,
|
|
423
437
|
rawSource: fileContext.rawSource || stateRow?.source || null,
|
|
438
|
+
serviceTier: fileContext.serviceTier,
|
|
424
439
|
};
|
|
425
440
|
}
|
|
426
441
|
|
|
@@ -445,19 +460,38 @@ function rememberQuota(quotaMap, rateLimits, occurrence) {
|
|
|
445
460
|
usedPercent,
|
|
446
461
|
windowMinutes,
|
|
447
462
|
resetsAt,
|
|
448
|
-
|
|
463
|
+
planType: String(rateLimits.plan_type || "unknown"),
|
|
464
|
+
limitKey: hash(limitKey, 16),
|
|
465
|
+
limitName: rateLimits.limit_name
|
|
466
|
+
? String(rateLimits.limit_name).slice(0, 80)
|
|
467
|
+
: null,
|
|
449
468
|
source: "log",
|
|
450
469
|
turnId: occurrence.turnId || null,
|
|
451
470
|
originalLikely: occurrence.originalLikely,
|
|
452
471
|
};
|
|
453
|
-
|
|
472
|
+
const current = quotaMap.get(key);
|
|
473
|
+
if (
|
|
474
|
+
!current ||
|
|
475
|
+
(!current.originalLikely && candidate.originalLikely) ||
|
|
476
|
+
(current.originalLikely === candidate.originalLikely &&
|
|
477
|
+
candidate.timestamp < current.timestamp)
|
|
478
|
+
) {
|
|
479
|
+
quotaMap.set(key, candidate);
|
|
480
|
+
}
|
|
454
481
|
}
|
|
455
482
|
}
|
|
456
483
|
|
|
457
484
|
function responseCall(record) {
|
|
458
485
|
if (record.type !== "response_item") return null;
|
|
459
486
|
const payload = record.payload;
|
|
460
|
-
|
|
487
|
+
const allowed = new Set([
|
|
488
|
+
"function_call",
|
|
489
|
+
"custom_tool_call",
|
|
490
|
+
"tool_search_call",
|
|
491
|
+
"web_search_call",
|
|
492
|
+
"image_generation_call",
|
|
493
|
+
]);
|
|
494
|
+
if (!payload || !allowed.has(payload.type)) return null;
|
|
461
495
|
return {
|
|
462
496
|
type: payload.type,
|
|
463
497
|
name: String(payload.name || payload.namespace || payload.type).slice(0, 80),
|
|
@@ -465,97 +499,17 @@ function responseCall(record) {
|
|
|
465
499
|
};
|
|
466
500
|
}
|
|
467
501
|
|
|
468
|
-
|
|
469
|
-
return RELEVANT_RECORD_TYPE.test(line);
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
function rolloutThreadId(path) {
|
|
502
|
+
async function scanRollout(path, context) {
|
|
473
503
|
const match = path.match(UUID_AT_END);
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
function createScanFragment() {
|
|
478
|
-
return {
|
|
479
|
-
parents: new Map(),
|
|
480
|
-
origins: new Map(),
|
|
481
|
-
tokens: new Map(),
|
|
482
|
-
quotas: new Map(),
|
|
483
|
-
calls: new Map(),
|
|
484
|
-
parseErrors: 0,
|
|
485
|
-
duplicateEventsSkipped: 0,
|
|
486
|
-
correctionIntervals: 0,
|
|
487
|
-
};
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
function rememberOrigin(originMap, turnId, candidate) {
|
|
491
|
-
const current = originMap.get(turnId);
|
|
492
|
-
if (!current || candidate.deltaMs < current.deltaMs) {
|
|
493
|
-
originMap.set(turnId, candidate);
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
function rememberCall(callMap, key, candidate) {
|
|
498
|
-
const current = callMap.get(key);
|
|
499
|
-
if (!current || (!current.originalLikely && candidate.originalLikely)) {
|
|
500
|
-
callMap.set(key, candidate);
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
function rememberToken(context, key, candidate) {
|
|
505
|
-
const current = context.tokens.get(key);
|
|
506
|
-
if (!current) {
|
|
507
|
-
context.tokens.set(key, candidate);
|
|
508
|
-
return;
|
|
509
|
-
}
|
|
510
|
-
context.duplicateEventsSkipped += 1;
|
|
511
|
-
if (!current.originalLikely && candidate.originalLikely) {
|
|
512
|
-
current.occurrence = candidate.occurrence;
|
|
513
|
-
current.originalLikely = true;
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
function rememberQuotaCandidate(quotaMap, key, candidate) {
|
|
518
|
-
const current = quotaMap.get(key);
|
|
519
|
-
if (
|
|
520
|
-
!current ||
|
|
521
|
-
(!current.originalLikely && candidate.originalLikely) ||
|
|
522
|
-
(current.originalLikely === candidate.originalLikely &&
|
|
523
|
-
candidate.timestamp < current.timestamp)
|
|
524
|
-
) {
|
|
525
|
-
quotaMap.set(key, candidate);
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
function mergeScanFragment(context, fragment) {
|
|
530
|
-
for (const [threadId, parentThreadId] of fragment.parents) {
|
|
531
|
-
context.parents.set(threadId, parentThreadId);
|
|
532
|
-
}
|
|
533
|
-
for (const [turnId, candidate] of fragment.origins) {
|
|
534
|
-
rememberOrigin(context.origins, turnId, candidate);
|
|
535
|
-
}
|
|
536
|
-
for (const [key, candidate] of fragment.tokens) {
|
|
537
|
-
rememberToken(context, key, candidate);
|
|
538
|
-
}
|
|
539
|
-
for (const [key, candidate] of fragment.quotas) {
|
|
540
|
-
rememberQuotaCandidate(context.quotas, key, candidate);
|
|
541
|
-
}
|
|
542
|
-
for (const [key, candidate] of fragment.calls) {
|
|
543
|
-
rememberCall(context.calls, key, candidate);
|
|
544
|
-
}
|
|
545
|
-
context.parseErrors += fragment.parseErrors;
|
|
546
|
-
context.duplicateEventsSkipped += fragment.duplicateEventsSkipped;
|
|
547
|
-
context.correctionIntervals += fragment.correctionIntervals;
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
async function scanRollout(path, stateRow) {
|
|
551
|
-
const context = createScanFragment();
|
|
552
|
-
const threadId = rolloutThreadId(path);
|
|
504
|
+
const threadId = match?.[1] || `file-${hash(path)}`;
|
|
505
|
+
const stateRow = context.stateRows.get(threadId);
|
|
553
506
|
const fileContext = {
|
|
554
507
|
model: stateRow?.model || "unknown",
|
|
555
508
|
effort: stateRow?.reasoning_effort || "unknown",
|
|
556
509
|
cwd: stateRow?.cwd || "",
|
|
557
510
|
gitOrigin: stateRow?.git_origin_url || null,
|
|
558
511
|
rawSource: stateRow?.source || null,
|
|
512
|
+
serviceTier: null,
|
|
559
513
|
};
|
|
560
514
|
const callOrdinals = new Map();
|
|
561
515
|
let currentTurnId = "";
|
|
@@ -565,7 +519,7 @@ async function scanRollout(path, stateRow) {
|
|
|
565
519
|
const input = createReadStream(path, { encoding: "utf8" });
|
|
566
520
|
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
567
521
|
for await (const line of lines) {
|
|
568
|
-
if (!line
|
|
522
|
+
if (!line.trim()) continue;
|
|
569
523
|
let record;
|
|
570
524
|
try {
|
|
571
525
|
record = JSON.parse(line);
|
|
@@ -603,7 +557,10 @@ async function scanRollout(path, stateRow) {
|
|
|
603
557
|
stateRow,
|
|
604
558
|
fileContext,
|
|
605
559
|
);
|
|
606
|
-
|
|
560
|
+
const currentBest = context.origins.get(currentTurnId);
|
|
561
|
+
if (!currentBest || currentCandidate.deltaMs < currentBest.deltaMs) {
|
|
562
|
+
context.origins.set(currentTurnId, currentCandidate);
|
|
563
|
+
}
|
|
607
564
|
}
|
|
608
565
|
continue;
|
|
609
566
|
}
|
|
@@ -628,6 +585,10 @@ async function scanRollout(path, stateRow) {
|
|
|
628
585
|
const settings = record.payload?.thread_settings;
|
|
629
586
|
fileContext.model = settings?.model || fileContext.model;
|
|
630
587
|
fileContext.effort = settings?.reasoning_effort || fileContext.effort;
|
|
588
|
+
const serviceTier = String(settings?.service_tier ?? "").trim();
|
|
589
|
+
fileContext.serviceTier = serviceTier
|
|
590
|
+
? serviceTier.slice(0, 40)
|
|
591
|
+
: null;
|
|
631
592
|
continue;
|
|
632
593
|
}
|
|
633
594
|
|
|
@@ -645,6 +606,7 @@ async function scanRollout(path, stateRow) {
|
|
|
645
606
|
cwd: fileContext.cwd,
|
|
646
607
|
gitOrigin: fileContext.gitOrigin,
|
|
647
608
|
rawSource: fileContext.rawSource,
|
|
609
|
+
serviceTier: fileContext.serviceTier,
|
|
648
610
|
};
|
|
649
611
|
|
|
650
612
|
const call = responseCall(record);
|
|
@@ -655,11 +617,14 @@ async function scanRollout(path, stateRow) {
|
|
|
655
617
|
const callKey = call.stableId
|
|
656
618
|
? `id|${call.stableId}`
|
|
657
619
|
: `ordinal|${ordinalBase}|${ordinal}`;
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
620
|
+
const existing = context.calls.get(callKey);
|
|
621
|
+
if (!existing || (!existing.originalLikely && originalLikely)) {
|
|
622
|
+
context.calls.set(callKey, {
|
|
623
|
+
turnId: currentTurnId,
|
|
624
|
+
threadId,
|
|
625
|
+
originalLikely,
|
|
626
|
+
});
|
|
627
|
+
}
|
|
663
628
|
continue;
|
|
664
629
|
}
|
|
665
630
|
|
|
@@ -692,7 +657,17 @@ async function scanRollout(path, stateRow) {
|
|
|
692
657
|
}
|
|
693
658
|
previousCumulative = totalTuple[5];
|
|
694
659
|
|
|
695
|
-
|
|
660
|
+
const existing = context.tokens.get(eventKey);
|
|
661
|
+
if (existing) {
|
|
662
|
+
context.duplicateEventsSkipped += 1;
|
|
663
|
+
if (!existing.originalLikely && originalLikely) {
|
|
664
|
+
existing.occurrence = occurrence;
|
|
665
|
+
existing.originalLikely = true;
|
|
666
|
+
}
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
context.tokens.set(eventKey, {
|
|
696
671
|
key: eventKey,
|
|
697
672
|
turnId: currentTurnId,
|
|
698
673
|
usage: usageFromTuple(lastTuple),
|
|
@@ -701,172 +676,24 @@ async function scanRollout(path, stateRow) {
|
|
|
701
676
|
dedupeQuality: currentTurnId ? "turn-exact" : "legacy-heuristic",
|
|
702
677
|
});
|
|
703
678
|
}
|
|
704
|
-
return context;
|
|
705
679
|
}
|
|
706
680
|
|
|
707
|
-
|
|
708
|
-
fileCount,
|
|
709
|
-
requestedWorkers = null,
|
|
710
|
-
parallelism = availableParallelism(),
|
|
711
|
-
) {
|
|
712
|
-
if (!Number.isInteger(fileCount) || fileCount < 0) {
|
|
713
|
-
throw new Error("fileCount must be a non-negative integer.");
|
|
714
|
-
}
|
|
715
|
-
if (fileCount === 0) return 0;
|
|
716
|
-
if (
|
|
717
|
-
requestedWorkers !== null &&
|
|
718
|
-
(!Number.isInteger(requestedWorkers) ||
|
|
719
|
-
requestedWorkers < 1 ||
|
|
720
|
-
requestedWorkers > MAX_SCAN_WORKERS)
|
|
721
|
-
) {
|
|
722
|
-
throw new Error(`workers must be an integer from 1 to ${MAX_SCAN_WORKERS}.`);
|
|
723
|
-
}
|
|
724
|
-
const detectedParallelism =
|
|
725
|
-
Number.isInteger(parallelism) && parallelism > 0 ? parallelism : 1;
|
|
726
|
-
const automaticWorkers = Math.min(
|
|
727
|
-
DEFAULT_MAX_SCAN_WORKERS,
|
|
728
|
-
Math.max(1, detectedParallelism - 1),
|
|
729
|
-
);
|
|
730
|
-
return Math.min(fileCount, requestedWorkers ?? automaticWorkers);
|
|
731
|
-
}
|
|
732
|
-
|
|
733
|
-
function reportScanProgress(onProgress, current, total, path) {
|
|
734
|
-
if (current === 1 || current === total || current % 10 === 0) {
|
|
735
|
-
onProgress({ current, total, path });
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
function workerFailure(message) {
|
|
740
|
-
const error = new Error(message?.error?.message || "Rollout scan worker failed.");
|
|
741
|
-
error.name = message?.error?.name || "Error";
|
|
742
|
-
if (message?.error?.code) error.code = message.error.code;
|
|
743
|
-
return error;
|
|
744
|
-
}
|
|
745
|
-
|
|
746
|
-
function scanWithWorker(worker, job) {
|
|
747
|
-
return new Promise((resolveJob, rejectJob) => {
|
|
748
|
-
const cleanup = () => {
|
|
749
|
-
worker.off("message", onMessage);
|
|
750
|
-
worker.off("error", onError);
|
|
751
|
-
worker.off("exit", onExit);
|
|
752
|
-
};
|
|
753
|
-
const onMessage = (message) => {
|
|
754
|
-
cleanup();
|
|
755
|
-
if (message?.jobId !== job.index) {
|
|
756
|
-
rejectJob(new Error("Rollout scan worker returned an unexpected job."));
|
|
757
|
-
} else if (message.type === "error") {
|
|
758
|
-
rejectJob(workerFailure(message));
|
|
759
|
-
} else if (message.type === "result") {
|
|
760
|
-
resolveJob(message.fragment);
|
|
761
|
-
} else {
|
|
762
|
-
rejectJob(new Error("Rollout scan worker returned an unknown response."));
|
|
763
|
-
}
|
|
764
|
-
};
|
|
765
|
-
const onError = (error) => {
|
|
766
|
-
cleanup();
|
|
767
|
-
rejectJob(error);
|
|
768
|
-
};
|
|
769
|
-
const onExit = (code) => {
|
|
770
|
-
cleanup();
|
|
771
|
-
rejectJob(
|
|
772
|
-
new Error(`Rollout scan worker exited before completing its job (${code}).`),
|
|
773
|
-
);
|
|
774
|
-
};
|
|
775
|
-
worker.once("message", onMessage);
|
|
776
|
-
worker.once("error", onError);
|
|
777
|
-
worker.once("exit", onExit);
|
|
778
|
-
try {
|
|
779
|
-
worker.postMessage({
|
|
780
|
-
type: "scan",
|
|
781
|
-
jobId: job.index,
|
|
782
|
-
path: job.path,
|
|
783
|
-
stateRow: job.stateRow,
|
|
784
|
-
});
|
|
785
|
-
} catch (error) {
|
|
786
|
-
cleanup();
|
|
787
|
-
rejectJob(error);
|
|
788
|
-
}
|
|
789
|
-
});
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
async function scanRolloutsSequential(jobs, onProgress) {
|
|
793
|
-
const fragments = new Array(jobs.length);
|
|
794
|
-
for (let index = 0; index < jobs.length; index += 1) {
|
|
795
|
-
const job = jobs[index];
|
|
796
|
-
fragments[job.index] = await scanRollout(job.path, job.stateRow);
|
|
797
|
-
reportScanProgress(onProgress, index + 1, jobs.length, job.path);
|
|
798
|
-
}
|
|
799
|
-
return fragments;
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
async function scanRolloutsInParallel(jobs, workerCount, onProgress) {
|
|
803
|
-
const scheduled = [...jobs].sort(
|
|
804
|
-
(left, right) => right.size - left.size || left.index - right.index,
|
|
805
|
-
);
|
|
806
|
-
const fragments = new Array(jobs.length);
|
|
807
|
-
const workers = Array.from(
|
|
808
|
-
{ length: workerCount },
|
|
809
|
-
() => new Worker(new URL(import.meta.url), {
|
|
810
|
-
execArgv: [],
|
|
811
|
-
workerData: { mode: SCAN_WORKER_MODE },
|
|
812
|
-
}),
|
|
813
|
-
);
|
|
814
|
-
let nextJob = 0;
|
|
815
|
-
let completed = 0;
|
|
816
|
-
try {
|
|
817
|
-
await Promise.all(workers.map(async (worker) => {
|
|
818
|
-
while (nextJob < scheduled.length) {
|
|
819
|
-
const job = scheduled[nextJob];
|
|
820
|
-
nextJob += 1;
|
|
821
|
-
fragments[job.index] = await scanWithWorker(worker, job);
|
|
822
|
-
completed += 1;
|
|
823
|
-
reportScanProgress(onProgress, completed, jobs.length, job.path);
|
|
824
|
-
}
|
|
825
|
-
}));
|
|
826
|
-
return fragments;
|
|
827
|
-
} finally {
|
|
828
|
-
await Promise.allSettled(workers.map((worker) => worker.terminate()));
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
async function runScanWorker() {
|
|
833
|
-
parentPort.on("message", async (message) => {
|
|
834
|
-
if (message?.type !== "scan") return;
|
|
835
|
-
try {
|
|
836
|
-
const fragment = await scanRollout(message.path, message.stateRow);
|
|
837
|
-
parentPort.postMessage({
|
|
838
|
-
type: "result",
|
|
839
|
-
jobId: message.jobId,
|
|
840
|
-
fragment,
|
|
841
|
-
});
|
|
842
|
-
} catch (error) {
|
|
843
|
-
parentPort.postMessage({
|
|
844
|
-
type: "error",
|
|
845
|
-
jobId: message.jobId,
|
|
846
|
-
error: {
|
|
847
|
-
name: error instanceof Error ? error.name : "Error",
|
|
848
|
-
message: error instanceof Error ? error.message : String(error),
|
|
849
|
-
code: error?.code || null,
|
|
850
|
-
},
|
|
851
|
-
});
|
|
852
|
-
}
|
|
853
|
-
});
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
function threadMetadata(threadId, stateRows, parents, fallback = {}) {
|
|
681
|
+
function threadMetadata(threadId, stateRows, titles, parents, fallback = {}) {
|
|
857
682
|
const row = stateRows.get(threadId);
|
|
683
|
+
const sessionTitle = titles.get(threadId)?.title;
|
|
858
684
|
const labels = sourceLabels(
|
|
859
685
|
row?.thread_source,
|
|
860
686
|
fallback.rawSource || row?.source,
|
|
861
687
|
);
|
|
862
688
|
return {
|
|
863
689
|
id: threadId,
|
|
690
|
+
title: safeTitle(row, sessionTitle),
|
|
864
691
|
project: projectLabel(
|
|
865
692
|
fallback.cwd || row?.cwd,
|
|
866
693
|
fallback.gitOrigin || row?.git_origin_url,
|
|
867
694
|
),
|
|
868
695
|
model: normalizeModel(fallback.model || row?.model || "unknown"),
|
|
869
|
-
effort:
|
|
696
|
+
effort: String(
|
|
870
697
|
fallback.effort || row?.reasoning_effort || "unknown",
|
|
871
698
|
).slice(0, 40),
|
|
872
699
|
source: labels.source,
|
|
@@ -881,7 +708,7 @@ function threadMetadata(threadId, stateRows, parents, fallback = {}) {
|
|
|
881
708
|
};
|
|
882
709
|
}
|
|
883
710
|
|
|
884
|
-
function buildSnapshot(context, options) {
|
|
711
|
+
function buildSnapshot(context, options, titles) {
|
|
885
712
|
const events = [];
|
|
886
713
|
for (const token of context.tokens.values()) {
|
|
887
714
|
const origin = token.turnId ? context.origins.get(token.turnId) : null;
|
|
@@ -897,6 +724,7 @@ function buildSnapshot(context, options) {
|
|
|
897
724
|
const metadata = threadMetadata(
|
|
898
725
|
threadId,
|
|
899
726
|
context.stateRows,
|
|
727
|
+
titles,
|
|
900
728
|
context.parents,
|
|
901
729
|
origin || occurrence,
|
|
902
730
|
);
|
|
@@ -908,11 +736,14 @@ function buildSnapshot(context, options) {
|
|
|
908
736
|
continue;
|
|
909
737
|
}
|
|
910
738
|
const breakdownAvailable = hasDetailedBreakdown(token.usage);
|
|
739
|
+
const serviceTier = occurrence.serviceTier || null;
|
|
740
|
+
const baseCredits = creditsForUsage(metadata.model, token.usage);
|
|
911
741
|
events.push({
|
|
912
742
|
...token.usage,
|
|
913
743
|
id: `evt-${hash(token.key)}`,
|
|
914
744
|
timestamp,
|
|
915
745
|
threadId,
|
|
746
|
+
threadTitle: metadata.title,
|
|
916
747
|
project: metadata.project,
|
|
917
748
|
model: metadata.model,
|
|
918
749
|
effort: metadata.effort,
|
|
@@ -920,6 +751,13 @@ function buildSnapshot(context, options) {
|
|
|
920
751
|
useType: metadata.useType,
|
|
921
752
|
turnId: token.turnId || "",
|
|
922
753
|
toolCalls: 0,
|
|
754
|
+
serviceTier,
|
|
755
|
+
rateCardCredits:
|
|
756
|
+
baseCredits === null
|
|
757
|
+
? null
|
|
758
|
+
: serviceTier === "priority"
|
|
759
|
+
? baseCredits * FAST_MODE_MULTIPLIER
|
|
760
|
+
: baseCredits,
|
|
923
761
|
breakdownAvailable,
|
|
924
762
|
dedupeQuality: token.dedupeQuality,
|
|
925
763
|
});
|
|
@@ -968,6 +806,7 @@ function buildSnapshot(context, options) {
|
|
|
968
806
|
const metadata = threadMetadata(
|
|
969
807
|
threadId,
|
|
970
808
|
context.stateRows,
|
|
809
|
+
titles,
|
|
971
810
|
context.parents,
|
|
972
811
|
origin || first || {},
|
|
973
812
|
);
|
|
@@ -981,6 +820,10 @@ function buildSnapshot(context, options) {
|
|
|
981
820
|
sum.toolCalls += event.toolCalls;
|
|
982
821
|
if (event.breakdownAvailable) sum.detailedTokens += event.totalTokens;
|
|
983
822
|
else sum.unknownBreakdownTokens += event.totalTokens;
|
|
823
|
+
if (event.rateCardCredits !== null) {
|
|
824
|
+
sum.rateCardCredits += event.rateCardCredits;
|
|
825
|
+
sum.ratedTokens += event.totalTokens;
|
|
826
|
+
}
|
|
984
827
|
return sum;
|
|
985
828
|
},
|
|
986
829
|
{
|
|
@@ -992,6 +835,8 @@ function buildSnapshot(context, options) {
|
|
|
992
835
|
toolCalls: 0,
|
|
993
836
|
detailedTokens: 0,
|
|
994
837
|
unknownBreakdownTokens: 0,
|
|
838
|
+
rateCardCredits: 0,
|
|
839
|
+
ratedTokens: 0,
|
|
995
840
|
},
|
|
996
841
|
);
|
|
997
842
|
if (rows.length === 0 && !(metadata.reportedCumulativeTokens > 0)) continue;
|
|
@@ -1005,6 +850,7 @@ function buildSnapshot(context, options) {
|
|
|
1005
850
|
: "total-only";
|
|
1006
851
|
threads.push({
|
|
1007
852
|
id: threadId,
|
|
853
|
+
title: metadata.title,
|
|
1008
854
|
project: metadata.project,
|
|
1009
855
|
model: metadata.model,
|
|
1010
856
|
effort: metadata.effort,
|
|
@@ -1021,6 +867,11 @@ function buildSnapshot(context, options) {
|
|
|
1021
867
|
cachedInputTokens: totals.cachedInputTokens,
|
|
1022
868
|
outputTokens: totals.outputTokens,
|
|
1023
869
|
reasoningTokens: totals.reasoningTokens,
|
|
870
|
+
rateCardCredits:
|
|
871
|
+
totals.totalTokens > 0 && totals.ratedTokens === totals.totalTokens
|
|
872
|
+
? totals.rateCardCredits
|
|
873
|
+
: null,
|
|
874
|
+
ratedTokens: totals.ratedTokens,
|
|
1024
875
|
toolCalls: totals.toolCalls,
|
|
1025
876
|
eventCount: rows.length,
|
|
1026
877
|
coverage,
|
|
@@ -1068,7 +919,7 @@ function buildSnapshot(context, options) {
|
|
|
1068
919
|
(quota) => quota.windowMinutes === WEEK_MINUTES,
|
|
1069
920
|
);
|
|
1070
921
|
const accountWideWeekly = weeklyCandidates.filter(
|
|
1071
|
-
(quota) => quota.
|
|
922
|
+
(quota) => !quota.limitName,
|
|
1072
923
|
);
|
|
1073
924
|
const weekly = [
|
|
1074
925
|
...(accountWideWeekly.length ? accountWideWeekly : weeklyCandidates),
|
|
@@ -1101,16 +952,13 @@ function buildSnapshot(context, options) {
|
|
|
1101
952
|
label: "Local Codex snapshot",
|
|
1102
953
|
provenance: {
|
|
1103
954
|
kind: "codex-local-metadata",
|
|
1104
|
-
sourceFingerprint: sourceFingerprint(
|
|
1105
|
-
options.codexHome,
|
|
1106
|
-
options.includeArchived,
|
|
1107
|
-
),
|
|
1108
955
|
privacy:
|
|
1109
|
-
"Contains token metadata and
|
|
956
|
+
"Contains token metadata and Codex display titles only; credential fields, message bodies, reasoning text, tool payloads, and full local paths are not exported. Display titles may contain user-written text.",
|
|
957
|
+
rateCardAsOf: RATE_CARD_AS_OF,
|
|
958
|
+
rateCardUrl: RATE_CARD_URL,
|
|
1110
959
|
},
|
|
1111
960
|
coverage: {
|
|
1112
961
|
filesScanned: context.filesScanned,
|
|
1113
|
-
sourceFileCount: context.sourceFileCount,
|
|
1114
962
|
bytesScanned: context.bytesScanned,
|
|
1115
963
|
parseErrors: context.parseErrors,
|
|
1116
964
|
duplicateEventsSkipped: context.duplicateEventsSkipped,
|
|
@@ -1138,6 +986,9 @@ function buildSnapshot(context, options) {
|
|
|
1138
986
|
|
|
1139
987
|
export async function collectUsage(options, onProgress = () => {}) {
|
|
1140
988
|
const state = await readState(options.codexHome);
|
|
989
|
+
const titles = await readSessionTitles(
|
|
990
|
+
resolve(options.codexHome, "session_index.jsonl"),
|
|
991
|
+
);
|
|
1141
992
|
const roots = [resolve(options.codexHome, "sessions")];
|
|
1142
993
|
if (options.includeArchived) {
|
|
1143
994
|
roots.push(resolve(options.codexHome, "archived_sessions"));
|
|
@@ -1148,12 +999,6 @@ export async function collectUsage(options, onProgress = () => {}) {
|
|
|
1148
999
|
.flat()
|
|
1149
1000
|
.sort();
|
|
1150
1001
|
const sizes = await Promise.all(files.map((path) => stat(path)));
|
|
1151
|
-
const jobs = files.map((path, index) => ({
|
|
1152
|
-
index,
|
|
1153
|
-
path,
|
|
1154
|
-
size: sizes[index].size,
|
|
1155
|
-
stateRow: state.rows.get(rolloutThreadId(path)) || null,
|
|
1156
|
-
}));
|
|
1157
1002
|
|
|
1158
1003
|
const context = {
|
|
1159
1004
|
stateRows: state.rows,
|
|
@@ -1163,24 +1008,30 @@ export async function collectUsage(options, onProgress = () => {}) {
|
|
|
1163
1008
|
quotas: new Map(),
|
|
1164
1009
|
calls: new Map(),
|
|
1165
1010
|
filesScanned: 0,
|
|
1166
|
-
sourceFileCount: files.length + (state.path ? 1 : 0),
|
|
1167
1011
|
bytesScanned: 0,
|
|
1168
1012
|
parseErrors: 0,
|
|
1169
1013
|
duplicateEventsSkipped: 0,
|
|
1170
1014
|
correctionIntervals: 0,
|
|
1171
1015
|
};
|
|
1172
1016
|
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1017
|
+
for (let index = 0; index < files.length; index += 1) {
|
|
1018
|
+
await scanRollout(files[index], context);
|
|
1019
|
+
context.filesScanned += 1;
|
|
1020
|
+
context.bytesScanned += sizes[index].size;
|
|
1021
|
+
if (
|
|
1022
|
+
index === 0 ||
|
|
1023
|
+
index === files.length - 1 ||
|
|
1024
|
+
(index + 1) % 10 === 0
|
|
1025
|
+
) {
|
|
1026
|
+
onProgress({
|
|
1027
|
+
current: index + 1,
|
|
1028
|
+
total: files.length,
|
|
1029
|
+
path: files[index],
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1179
1032
|
}
|
|
1180
|
-
context.filesScanned = files.length;
|
|
1181
|
-
context.bytesScanned = sizes.reduce((sum, entry) => sum + entry.size, 0);
|
|
1182
1033
|
|
|
1183
|
-
return buildSnapshot(context, options);
|
|
1034
|
+
return buildSnapshot(context, options, titles);
|
|
1184
1035
|
}
|
|
1185
1036
|
|
|
1186
1037
|
async function main() {
|
|
@@ -1188,7 +1039,7 @@ async function main() {
|
|
|
1188
1039
|
try {
|
|
1189
1040
|
options = parseArgs(process.argv.slice(2));
|
|
1190
1041
|
} catch (error) {
|
|
1191
|
-
process.stderr.write(`${
|
|
1042
|
+
process.stderr.write(`${error.message}\n\n${usage()}\n`);
|
|
1192
1043
|
process.exitCode = 1;
|
|
1193
1044
|
return;
|
|
1194
1045
|
}
|
|
@@ -1197,9 +1048,7 @@ async function main() {
|
|
|
1197
1048
|
return;
|
|
1198
1049
|
}
|
|
1199
1050
|
if (!(await pathExists(options.codexHome))) {
|
|
1200
|
-
throw new Error(
|
|
1201
|
-
`Codex data directory not found: ${sanitizeLabel(options.codexHome)}`,
|
|
1202
|
-
);
|
|
1051
|
+
throw new Error(`Codex data directory not found: ${options.codexHome}`);
|
|
1203
1052
|
}
|
|
1204
1053
|
|
|
1205
1054
|
process.stdout.write("Token Ledger: scanning local Codex metadata…\n");
|
|
@@ -1213,7 +1062,7 @@ async function main() {
|
|
|
1213
1062
|
const outputSize = (await stat(options.output)).size;
|
|
1214
1063
|
process.stdout.write(
|
|
1215
1064
|
[
|
|
1216
|
-
`Snapshot: ${
|
|
1065
|
+
`Snapshot: ${options.output}`,
|
|
1217
1066
|
`Observed model-call tokens: ${snapshot.coverage.observedTokens.toLocaleString()}`,
|
|
1218
1067
|
`Unique events: ${snapshot.events.length.toLocaleString()}`,
|
|
1219
1068
|
`Duplicate/copied events skipped: ${snapshot.coverage.duplicateEventsSkipped.toLocaleString()}`,
|
|
@@ -1224,13 +1073,11 @@ async function main() {
|
|
|
1224
1073
|
}
|
|
1225
1074
|
|
|
1226
1075
|
const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : "";
|
|
1227
|
-
if (
|
|
1228
|
-
runScanWorker();
|
|
1229
|
-
} else if (isMainThread && import.meta.url === invokedPath) {
|
|
1076
|
+
if (import.meta.url === invokedPath) {
|
|
1230
1077
|
main().catch((error) => {
|
|
1231
1078
|
process.stderr.write(
|
|
1232
1079
|
`Token Ledger collector failed: ${
|
|
1233
|
-
error instanceof Error ?
|
|
1080
|
+
error instanceof Error ? error.message : String(error)
|
|
1234
1081
|
}\n`,
|
|
1235
1082
|
);
|
|
1236
1083
|
process.exitCode = 1;
|