tledger 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@
4
4
  * Token Ledger local collector
5
5
  *
6
6
  * Reads Codex's local JSONL rollouts and metadata database, then writes a
7
- * privacy-reduced snapshot for the Token Ledger CLI. It never exports message
7
+ * privacy-reduced snapshot for the Token Ledger site. It never exports message
8
8
  * bodies, tool arguments/results, reasoning text, instructions, credential
9
9
  * fields, or full local paths in the generated snapshot.
10
10
  *
@@ -23,7 +23,7 @@ import {
23
23
  stat,
24
24
  writeFile,
25
25
  } from "node:fs/promises";
26
- import { availableParallelism, homedir } from "node:os";
26
+ import { homedir } from "node:os";
27
27
  import {
28
28
  basename,
29
29
  dirname,
@@ -32,63 +32,33 @@ import {
32
32
  } from "node:path";
33
33
  import { pathToFileURL } from "node:url";
34
34
  import { createInterface } from "node:readline";
35
- import {
36
- isMainThread,
37
- parentPort,
38
- Worker,
39
- workerData,
40
- } from "node:worker_threads";
35
+ import { DatabaseSync } from "node:sqlite";
41
36
 
42
37
  const SCHEMA_VERSION = 1;
38
+ const RATE_CARD_AS_OF = "2026-08-17";
39
+ const FAST_MODE_MULTIPLIER = 1.5;
40
+ const RATE_CARD_URL = "https://help.openai.com/en/articles/20001106";
43
41
  const WEEK_MINUTES = 10_080;
44
- const DEFAULT_MAX_SCAN_WORKERS = 4;
45
- const MAX_SCAN_WORKERS = 6;
46
- const SCAN_WORKER_MODE = "token-ledger-rollout-scanner";
47
42
  const UUID_AT_END =
48
43
  /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
49
- const RELEVANT_RECORD_TYPE =
50
- /"type"\s*:\s*"(?:session_meta|turn_context|task_started|thread_settings_applied|token_count|function_call|custom_tool_call|tool_search_call|web_search_call|image_generation_call)"/;
51
- const RESPONSE_CALL_TYPES = new Set([
52
- "function_call",
53
- "custom_tool_call",
54
- "tool_search_call",
55
- "web_search_call",
56
- "image_generation_call",
57
- ]);
58
- let databaseSyncPromise;
59
-
60
- function loadDatabaseSync() {
61
- if (!databaseSyncPromise) {
62
- databaseSyncPromise = (async () => {
63
- const originalEmitWarning = process.emitWarning;
64
- process.emitWarning = function emitWarning(warning, ...arguments_) {
65
- const message = warning instanceof Error ? warning.message : String(warning);
66
- const type = typeof arguments_[0] === "string"
67
- ? arguments_[0]
68
- : arguments_[0]?.type;
69
- if (
70
- type === "ExperimentalWarning" &&
71
- message === "SQLite is an experimental feature and might change at any time"
72
- ) {
73
- return;
74
- }
75
- return originalEmitWarning.call(this, warning, ...arguments_);
76
- };
77
- try {
78
- return (await import("node:sqlite")).DatabaseSync;
79
- } finally {
80
- process.emitWarning = originalEmitWarning;
81
- }
82
- })();
83
- }
84
- return databaseSyncPromise;
85
- }
44
+
45
+ const RATE_CARD = {
46
+ "gpt-5.6-sol": { input: 125, cached: 12.5, output: 750 },
47
+ "gpt-5.6-terra": { input: 50, cached: 5, output: 300 },
48
+ "gpt-5.6-luna": { input: 5, cached: 0.5, output: 30 },
49
+ "gpt-5.5": { input: 125, cached: 12.5, output: 750 },
50
+ "gpt-5.5-cyber": { input: 500, cached: 50, output: 3_000 },
51
+ "gpt-5.4": { input: 62.5, cached: 6.25, output: 375 },
52
+ "gpt-5.4-mini": { input: 18.75, cached: 1.875, output: 113 },
53
+ "gpt-5.3-codex": { input: 43.75, cached: 4.375, output: 350 },
54
+ "gpt-5.2": { input: 43.75, cached: 4.375, output: 350 },
55
+ };
86
56
 
87
57
  function usage() {
88
58
  return `Token Ledger local collector
89
59
 
90
60
  Usage:
91
- node lib/token-ledger-collector.mjs [options]
61
+ node token-ledger-importer.mjs [options]
92
62
 
93
63
  Options:
94
64
  --output <file> Snapshot destination (default: token-ledger-snapshot.json)
@@ -97,9 +67,10 @@ Options:
97
67
  --no-archived Skip archived_sessions
98
68
  --help Show this help
99
69
 
100
- The snapshot contains local token metadata and project labels only. It never
101
- contains display titles, message bodies, tool payloads, reasoning text,
102
- credential fields, or full local paths. Project labels can still be sensitive.`;
70
+ The snapshot contains usage metadata and Codex display titles only. It never
71
+ contains message bodies, tool payloads, reasoning text, credential fields, or
72
+ full local paths in its output. Codex display titles may contain user-written
73
+ text.`;
103
74
  }
104
75
 
105
76
  function parseArgs(argv) {
@@ -141,45 +112,6 @@ function hash(value, length = 24) {
141
112
  return createHash("sha256").update(String(value)).digest("hex").slice(0, length);
142
113
  }
143
114
 
144
- export function sourceFingerprint(codexHome, includeArchived = true) {
145
- return hash(JSON.stringify({
146
- codexHome: resolve(codexHome),
147
- includeArchived,
148
- }));
149
- }
150
-
151
- function sanitizeLabel(value, fallback = "unknown") {
152
- const label = String(value ?? "")
153
- .replace(/\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g, "")
154
- .replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
155
- .replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ")
156
- .replace(/\s+/g, " ")
157
- .trim();
158
- return (label || fallback).slice(0, 160);
159
- }
160
-
161
- export async function writePrivateSnapshot(output, snapshot) {
162
- const destination = resolve(output);
163
- const directory = dirname(destination);
164
- const temporary = resolve(
165
- directory,
166
- `.token-ledger-${process.pid}-${randomUUID()}.tmp`,
167
- );
168
- await mkdir(directory, { recursive: true });
169
- try {
170
- await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
171
- encoding: "utf8",
172
- flag: "wx",
173
- mode: 0o600,
174
- });
175
- await chmod(temporary, 0o600);
176
- await rename(temporary, destination);
177
- await chmod(destination, 0o600);
178
- } finally {
179
- await rm(temporary, { force: true });
180
- }
181
- }
182
-
183
115
  function asFiniteNumber(value) {
184
116
  const number = Number(value ?? 0);
185
117
  return Number.isFinite(number) ? number : 0;
@@ -230,9 +162,11 @@ function hasDetailedBreakdown(usage) {
230
162
  }
231
163
 
232
164
  function normalizeModel(model) {
233
- const value = sanitizeLabel(model, "unknown")
165
+ const value = String(model || "unknown")
166
+ .trim()
234
167
  .toLowerCase()
235
168
  .replaceAll("_", "-");
169
+ if (RATE_CARD[value]) return value;
236
170
  if (value.startsWith("gpt-5.6-sol")) return "gpt-5.6-sol";
237
171
  if (value.startsWith("gpt-5.6-terra")) return "gpt-5.6-terra";
238
172
  if (value.startsWith("gpt-5.6-luna")) return "gpt-5.6-luna";
@@ -245,6 +179,20 @@ function normalizeModel(model) {
245
179
  return value || "unknown";
246
180
  }
247
181
 
182
+ function creditsForUsage(model, usage) {
183
+ if (!hasDetailedBreakdown(usage)) return null;
184
+ const rate = RATE_CARD[normalizeModel(model)];
185
+ if (!rate) return null;
186
+ const cached = Math.min(usage.inputTokens, usage.cachedInputTokens);
187
+ const uncached = Math.max(0, usage.inputTokens - cached);
188
+ return (
189
+ (uncached * rate.input +
190
+ cached * rate.cached +
191
+ usage.outputTokens * rate.output) /
192
+ 1_000_000
193
+ );
194
+ }
195
+
248
196
  function parseStructuredSource(value) {
249
197
  if (!value) return null;
250
198
  if (typeof value === "object") return value;
@@ -275,15 +223,9 @@ function sourceLabels(threadSource, rawSource) {
275
223
  if (source === "exec") return { source: "cli", useType: "cli" };
276
224
  if (source === "vscode") return { source: "desktop", useType: "interactive" };
277
225
  if (typeof source === "string" && source) {
278
- return {
279
- source: sanitizeLabel(source).slice(0, 40),
280
- useType: sanitizeLabel(threadSource || "interactive").slice(0, 40),
281
- };
226
+ return { source: source.slice(0, 40), useType: threadSource || "interactive" };
282
227
  }
283
- return {
284
- source: "unknown",
285
- useType: sanitizeLabel(threadSource || "unknown").slice(0, 40),
286
- };
228
+ return { source: "unknown", useType: threadSource || "unknown" };
287
229
  }
288
230
 
289
231
  function cleanRemote(value) {
@@ -291,15 +233,15 @@ function cleanRemote(value) {
291
233
  const remote = String(value).trim();
292
234
  const scp = remote.match(/^[^@]+@([^:]+):(.+)$/);
293
235
  if (scp) {
294
- return sanitizeLabel(`${scp[1]}/${scp[2].replace(/\.git$/i, "")}`);
236
+ return `${scp[1]}/${scp[2].replace(/\.git$/i, "")}`;
295
237
  }
296
238
  try {
297
239
  const url = new URL(remote);
298
240
  const path = url.pathname.replace(/^\/+/, "").replace(/\.git$/i, "");
299
- return sanitizeLabel(path ? `${url.hostname}/${path}` : url.hostname);
241
+ return path ? `${url.hostname}/${path}` : url.hostname;
300
242
  } catch {
301
243
  const withoutCredentials = remote.replace(/\/\/[^/@]+@/, "//");
302
- return sanitizeLabel(withoutCredentials.replace(/\.git$/i, ""));
244
+ return withoutCredentials.replace(/\.git$/i, "").slice(0, 160);
303
245
  }
304
246
  }
305
247
 
@@ -307,15 +249,37 @@ function projectLabel(cwd, gitOrigin) {
307
249
  const remote = cleanRemote(gitOrigin);
308
250
  if (remote) {
309
251
  const parts = remote.split("/").filter(Boolean);
310
- return sanitizeLabel(parts.slice(-2).join("/"), "Unknown project");
252
+ return parts.slice(-2).join("/") || "Unknown project";
311
253
  }
312
254
  const path = String(cwd || "").replaceAll("\\", "/").replace(/\/+$/, "");
313
255
  const worktree = path.match(/\/\.codex\/worktrees\/[^/]+\/([^/]+)$/);
314
256
  if (worktree) return worktree[1];
315
257
  const name = basename(path);
316
- return name && name !== "." && name !== "/"
317
- ? sanitizeLabel(name, "Unknown project")
318
- : "Unknown project";
258
+ return name && name !== "." && name !== "/" ? name : "Unknown project";
259
+ }
260
+
261
+ function safeTitle(row, sessionTitle) {
262
+ const candidate = String(sessionTitle || row?.title || row?.name || "").trim();
263
+ const subagent =
264
+ row?.thread_source === "subagent" ||
265
+ String(row?.source || "").includes('"subagent"');
266
+ if (
267
+ !candidate ||
268
+ candidate.includes("<codex_delegation>") ||
269
+ (subagent && candidate.length > 120)
270
+ ) {
271
+ if (row?.agent_nickname) return `Subagent · ${row.agent_nickname}`;
272
+ return subagent ? `Subagent · ${String(row?.id || "").slice(0, 8)}` : "Untitled task";
273
+ }
274
+ return candidate
275
+ .replace(/\/Users\/[^\s"'`]+/g, "[local path]")
276
+ .replace(/\/(?:private\/)?tmp\/[^\s"'`]+/g, "[temporary path]")
277
+ .replace(
278
+ /\b(?:sk-[A-Za-z0-9_-]{16,}|lin_api_[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_-]{16,})\b/g,
279
+ "[redacted credential-like text]",
280
+ )
281
+ .replace(/\s+/g, " ")
282
+ .slice(0, 180);
319
283
  }
320
284
 
321
285
  async function pathExists(path) {
@@ -327,6 +291,28 @@ async function pathExists(path) {
327
291
  }
328
292
  }
329
293
 
294
+ export async function writePrivateSnapshot(output, snapshot) {
295
+ const destination = resolve(output);
296
+ const directory = dirname(destination);
297
+ const temporary = resolve(
298
+ directory,
299
+ `.token-ledger-${process.pid}-${randomUUID()}.tmp`,
300
+ );
301
+ await mkdir(directory, { recursive: true });
302
+ try {
303
+ await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
304
+ encoding: "utf8",
305
+ flag: "wx",
306
+ mode: 0o600,
307
+ });
308
+ await chmod(temporary, 0o600);
309
+ await rename(temporary, destination);
310
+ await chmod(destination, 0o600);
311
+ } finally {
312
+ await rm(temporary, { force: true });
313
+ }
314
+ }
315
+
330
316
  export async function listJsonlFiles(root) {
331
317
  if (!(await pathExists(root))) return [];
332
318
  const found = [];
@@ -343,33 +329,51 @@ export async function listJsonlFiles(root) {
343
329
  return found;
344
330
  }
345
331
 
346
- export async function sourceState(codexHome, includeArchived = true) {
332
+ export async function latestSourceModifiedAt(codexHome, includeArchived = true) {
347
333
  const roots = [resolve(codexHome, "sessions")];
348
334
  if (includeArchived) {
349
335
  roots.push(resolve(codexHome, "archived_sessions"));
350
336
  }
351
337
  const files = (await Promise.all(roots.map((root) => listJsonlFiles(root))))
352
338
  .flat();
353
- const preferredState = resolve(codexHome, "state_5.sqlite");
354
- const legacyState = resolve(codexHome, "sqlite", "state_5.sqlite");
355
- const statePath = (await pathExists(preferredState))
356
- ? preferredState
357
- : (await pathExists(legacyState))
358
- ? legacyState
359
- : null;
360
- const sourceFiles = statePath ? [...files, statePath] : files;
361
- if (!sourceFiles.length) {
362
- return { latestMtimeMs: 0, fileCount: 0 };
339
+ const metadataFiles = [
340
+ resolve(codexHome, "session_index.jsonl"),
341
+ resolve(codexHome, "state_5.sqlite"),
342
+ resolve(codexHome, "sqlite", "state_5.sqlite"),
343
+ ];
344
+ const existingMetadataFiles = [];
345
+ for (const path of metadataFiles) {
346
+ if (await pathExists(path)) existingMetadataFiles.push(path);
363
347
  }
348
+ const sourceFiles = [...files, ...existingMetadataFiles];
349
+ if (!sourceFiles.length) return 0;
364
350
  const stats = await Promise.all(sourceFiles.map((path) => stat(path)));
365
- return {
366
- latestMtimeMs: Math.max(...stats.map((entry) => entry.mtimeMs)),
367
- fileCount: sourceFiles.length,
368
- };
351
+ return Math.max(...stats.map((entry) => entry.mtimeMs));
369
352
  }
370
353
 
371
- export async function latestSourceModifiedAt(codexHome, includeArchived = true) {
372
- return (await sourceState(codexHome, includeArchived)).latestMtimeMs;
354
+ async function readSessionTitles(path) {
355
+ const titles = new Map();
356
+ if (!(await pathExists(path))) return titles;
357
+ const input = createReadStream(path, { encoding: "utf8" });
358
+ const lines = createInterface({ input, crlfDelay: Infinity });
359
+ for await (const line of lines) {
360
+ if (!line.trim()) continue;
361
+ try {
362
+ const record = JSON.parse(line);
363
+ if (!record?.id || !record?.thread_name) continue;
364
+ const timestamp = new Date(record.updated_at || 0).getTime();
365
+ const current = titles.get(record.id);
366
+ if (!current || timestamp >= current.timestamp) {
367
+ titles.set(record.id, {
368
+ title: String(record.thread_name).replace(/\s+/g, " ").slice(0, 180),
369
+ timestamp,
370
+ });
371
+ }
372
+ } catch {
373
+ // The thread index is optional; malformed lines do not affect token totals.
374
+ }
375
+ }
376
+ return titles;
373
377
  }
374
378
 
375
379
  async function readState(codexHome) {
@@ -384,13 +388,13 @@ async function readState(codexHome) {
384
388
  const parents = new Map();
385
389
  if (!path) return { path: null, rows, parents };
386
390
 
387
- const DatabaseSync = await loadDatabaseSync();
388
391
  const database = new DatabaseSync(path, { readOnly: true });
389
392
  try {
390
393
  const threadRows = database
391
394
  .prepare(
392
- `SELECT id, created_at, updated_at, source, cwd,
393
- tokens_used, git_origin_url, model, reasoning_effort,
395
+ `SELECT id, created_at, updated_at, source, cwd, title, name,
396
+ tokens_used, git_sha, git_branch, git_origin_url,
397
+ agent_nickname, agent_role, model, reasoning_effort,
394
398
  thread_source
395
399
  FROM threads`,
396
400
  )
@@ -431,6 +435,7 @@ function taskStartCandidate(record, threadId, stateRow, fileContext) {
431
435
  cwd: fileContext.cwd || stateRow?.cwd || "",
432
436
  gitOrigin: fileContext.gitOrigin || stateRow?.git_origin_url || null,
433
437
  rawSource: fileContext.rawSource || stateRow?.source || null,
438
+ serviceTier: fileContext.serviceTier,
434
439
  };
435
440
  }
436
441
 
@@ -455,19 +460,38 @@ function rememberQuota(quotaMap, rateLimits, occurrence) {
455
460
  usedPercent,
456
461
  windowMinutes,
457
462
  resetsAt,
458
- scope: rateLimits.limit_name ? "named" : "account",
463
+ planType: String(rateLimits.plan_type || "unknown"),
464
+ limitKey: hash(limitKey, 16),
465
+ limitName: rateLimits.limit_name
466
+ ? String(rateLimits.limit_name).slice(0, 80)
467
+ : null,
459
468
  source: "log",
460
469
  turnId: occurrence.turnId || null,
461
470
  originalLikely: occurrence.originalLikely,
462
471
  };
463
- rememberQuotaCandidate(quotaMap, key, candidate);
472
+ const current = quotaMap.get(key);
473
+ if (
474
+ !current ||
475
+ (!current.originalLikely && candidate.originalLikely) ||
476
+ (current.originalLikely === candidate.originalLikely &&
477
+ candidate.timestamp < current.timestamp)
478
+ ) {
479
+ quotaMap.set(key, candidate);
480
+ }
464
481
  }
465
482
  }
466
483
 
467
484
  function responseCall(record) {
468
485
  if (record.type !== "response_item") return null;
469
486
  const payload = record.payload;
470
- if (!payload || !RESPONSE_CALL_TYPES.has(payload.type)) return null;
487
+ const allowed = new Set([
488
+ "function_call",
489
+ "custom_tool_call",
490
+ "tool_search_call",
491
+ "web_search_call",
492
+ "image_generation_call",
493
+ ]);
494
+ if (!payload || !allowed.has(payload.type)) return null;
471
495
  return {
472
496
  type: payload.type,
473
497
  name: String(payload.name || payload.namespace || payload.type).slice(0, 80),
@@ -475,97 +499,17 @@ function responseCall(record) {
475
499
  };
476
500
  }
477
501
 
478
- export function rolloutLineMayAffectUsage(line) {
479
- return RELEVANT_RECORD_TYPE.test(line);
480
- }
481
-
482
- function rolloutThreadId(path) {
502
+ async function scanRollout(path, context) {
483
503
  const match = path.match(UUID_AT_END);
484
- return match?.[1] || `file-${hash(path)}`;
485
- }
486
-
487
- function createScanFragment() {
488
- return {
489
- parents: new Map(),
490
- origins: new Map(),
491
- tokens: new Map(),
492
- quotas: new Map(),
493
- calls: new Map(),
494
- parseErrors: 0,
495
- duplicateEventsSkipped: 0,
496
- correctionIntervals: 0,
497
- };
498
- }
499
-
500
- function rememberOrigin(originMap, turnId, candidate) {
501
- const current = originMap.get(turnId);
502
- if (!current || candidate.deltaMs < current.deltaMs) {
503
- originMap.set(turnId, candidate);
504
- }
505
- }
506
-
507
- function rememberCall(callMap, key, candidate) {
508
- const current = callMap.get(key);
509
- if (!current || (!current.originalLikely && candidate.originalLikely)) {
510
- callMap.set(key, candidate);
511
- }
512
- }
513
-
514
- function rememberToken(context, key, candidate) {
515
- const current = context.tokens.get(key);
516
- if (!current) {
517
- context.tokens.set(key, candidate);
518
- return;
519
- }
520
- context.duplicateEventsSkipped += 1;
521
- if (!current.originalLikely && candidate.originalLikely) {
522
- current.occurrence = candidate.occurrence;
523
- current.originalLikely = true;
524
- }
525
- }
526
-
527
- function rememberQuotaCandidate(quotaMap, key, candidate) {
528
- const current = quotaMap.get(key);
529
- if (
530
- !current ||
531
- (!current.originalLikely && candidate.originalLikely) ||
532
- (current.originalLikely === candidate.originalLikely &&
533
- candidate.timestamp < current.timestamp)
534
- ) {
535
- quotaMap.set(key, candidate);
536
- }
537
- }
538
-
539
- function mergeScanFragment(context, fragment) {
540
- for (const [threadId, parentThreadId] of fragment.parents) {
541
- context.parents.set(threadId, parentThreadId);
542
- }
543
- for (const [turnId, candidate] of fragment.origins) {
544
- rememberOrigin(context.origins, turnId, candidate);
545
- }
546
- for (const [key, candidate] of fragment.tokens) {
547
- rememberToken(context, key, candidate);
548
- }
549
- for (const [key, candidate] of fragment.quotas) {
550
- rememberQuotaCandidate(context.quotas, key, candidate);
551
- }
552
- for (const [key, candidate] of fragment.calls) {
553
- rememberCall(context.calls, key, candidate);
554
- }
555
- context.parseErrors += fragment.parseErrors;
556
- context.duplicateEventsSkipped += fragment.duplicateEventsSkipped;
557
- context.correctionIntervals += fragment.correctionIntervals;
558
- }
559
-
560
- async function scanRollout(path, stateRow) {
561
- const context = createScanFragment();
562
- const threadId = rolloutThreadId(path);
504
+ const threadId = match?.[1] || `file-${hash(path)}`;
505
+ const stateRow = context.stateRows.get(threadId);
563
506
  const fileContext = {
564
507
  model: stateRow?.model || "unknown",
565
508
  effort: stateRow?.reasoning_effort || "unknown",
566
509
  cwd: stateRow?.cwd || "",
567
510
  gitOrigin: stateRow?.git_origin_url || null,
568
511
  rawSource: stateRow?.source || null,
512
+ serviceTier: null,
569
513
  };
570
514
  const callOrdinals = new Map();
571
515
  let currentTurnId = "";
@@ -575,7 +519,7 @@ async function scanRollout(path, stateRow) {
575
519
  const input = createReadStream(path, { encoding: "utf8" });
576
520
  const lines = createInterface({ input, crlfDelay: Infinity });
577
521
  for await (const line of lines) {
578
- if (!line || !rolloutLineMayAffectUsage(line)) continue;
522
+ if (!line.trim()) continue;
579
523
  let record;
580
524
  try {
581
525
  record = JSON.parse(line);
@@ -613,7 +557,10 @@ async function scanRollout(path, stateRow) {
613
557
  stateRow,
614
558
  fileContext,
615
559
  );
616
- rememberOrigin(context.origins, currentTurnId, currentCandidate);
560
+ const currentBest = context.origins.get(currentTurnId);
561
+ if (!currentBest || currentCandidate.deltaMs < currentBest.deltaMs) {
562
+ context.origins.set(currentTurnId, currentCandidate);
563
+ }
617
564
  }
618
565
  continue;
619
566
  }
@@ -638,6 +585,10 @@ async function scanRollout(path, stateRow) {
638
585
  const settings = record.payload?.thread_settings;
639
586
  fileContext.model = settings?.model || fileContext.model;
640
587
  fileContext.effort = settings?.reasoning_effort || fileContext.effort;
588
+ const serviceTier = String(settings?.service_tier ?? "").trim();
589
+ fileContext.serviceTier = serviceTier
590
+ ? serviceTier.slice(0, 40)
591
+ : null;
641
592
  continue;
642
593
  }
643
594
 
@@ -655,6 +606,7 @@ async function scanRollout(path, stateRow) {
655
606
  cwd: fileContext.cwd,
656
607
  gitOrigin: fileContext.gitOrigin,
657
608
  rawSource: fileContext.rawSource,
609
+ serviceTier: fileContext.serviceTier,
658
610
  };
659
611
 
660
612
  const call = responseCall(record);
@@ -665,11 +617,14 @@ async function scanRollout(path, stateRow) {
665
617
  const callKey = call.stableId
666
618
  ? `id|${call.stableId}`
667
619
  : `ordinal|${ordinalBase}|${ordinal}`;
668
- rememberCall(context.calls, callKey, {
669
- turnId: currentTurnId,
670
- threadId,
671
- originalLikely,
672
- });
620
+ const existing = context.calls.get(callKey);
621
+ if (!existing || (!existing.originalLikely && originalLikely)) {
622
+ context.calls.set(callKey, {
623
+ turnId: currentTurnId,
624
+ threadId,
625
+ originalLikely,
626
+ });
627
+ }
673
628
  continue;
674
629
  }
675
630
 
@@ -702,7 +657,17 @@ async function scanRollout(path, stateRow) {
702
657
  }
703
658
  previousCumulative = totalTuple[5];
704
659
 
705
- rememberToken(context, eventKey, {
660
+ const existing = context.tokens.get(eventKey);
661
+ if (existing) {
662
+ context.duplicateEventsSkipped += 1;
663
+ if (!existing.originalLikely && originalLikely) {
664
+ existing.occurrence = occurrence;
665
+ existing.originalLikely = true;
666
+ }
667
+ continue;
668
+ }
669
+
670
+ context.tokens.set(eventKey, {
706
671
  key: eventKey,
707
672
  turnId: currentTurnId,
708
673
  usage: usageFromTuple(lastTuple),
@@ -711,172 +676,24 @@ async function scanRollout(path, stateRow) {
711
676
  dedupeQuality: currentTurnId ? "turn-exact" : "legacy-heuristic",
712
677
  });
713
678
  }
714
- return context;
715
679
  }
716
680
 
717
- export function scanWorkerCount(
718
- fileCount,
719
- requestedWorkers = null,
720
- parallelism = availableParallelism(),
721
- ) {
722
- if (!Number.isInteger(fileCount) || fileCount < 0) {
723
- throw new Error("fileCount must be a non-negative integer.");
724
- }
725
- if (fileCount === 0) return 0;
726
- if (
727
- requestedWorkers !== null &&
728
- (!Number.isInteger(requestedWorkers) ||
729
- requestedWorkers < 1 ||
730
- requestedWorkers > MAX_SCAN_WORKERS)
731
- ) {
732
- throw new Error(`workers must be an integer from 1 to ${MAX_SCAN_WORKERS}.`);
733
- }
734
- const detectedParallelism =
735
- Number.isInteger(parallelism) && parallelism > 0 ? parallelism : 1;
736
- const automaticWorkers = Math.min(
737
- DEFAULT_MAX_SCAN_WORKERS,
738
- Math.max(1, detectedParallelism - 1),
739
- );
740
- return Math.min(fileCount, requestedWorkers ?? automaticWorkers);
741
- }
742
-
743
- function reportScanProgress(onProgress, current, total, path) {
744
- if (current === 1 || current === total || current % 10 === 0) {
745
- onProgress({ current, total, path });
746
- }
747
- }
748
-
749
- function workerFailure(message) {
750
- const error = new Error(message?.error?.message || "Rollout scan worker failed.");
751
- error.name = message?.error?.name || "Error";
752
- if (message?.error?.code) error.code = message.error.code;
753
- return error;
754
- }
755
-
756
- function scanWithWorker(worker, job) {
757
- return new Promise((resolveJob, rejectJob) => {
758
- const cleanup = () => {
759
- worker.off("message", onMessage);
760
- worker.off("error", onError);
761
- worker.off("exit", onExit);
762
- };
763
- const onMessage = (message) => {
764
- cleanup();
765
- if (message?.jobId !== job.index) {
766
- rejectJob(new Error("Rollout scan worker returned an unexpected job."));
767
- } else if (message.type === "error") {
768
- rejectJob(workerFailure(message));
769
- } else if (message.type === "result") {
770
- resolveJob(message.fragment);
771
- } else {
772
- rejectJob(new Error("Rollout scan worker returned an unknown response."));
773
- }
774
- };
775
- const onError = (error) => {
776
- cleanup();
777
- rejectJob(error);
778
- };
779
- const onExit = (code) => {
780
- cleanup();
781
- rejectJob(
782
- new Error(`Rollout scan worker exited before completing its job (${code}).`),
783
- );
784
- };
785
- worker.once("message", onMessage);
786
- worker.once("error", onError);
787
- worker.once("exit", onExit);
788
- try {
789
- worker.postMessage({
790
- type: "scan",
791
- jobId: job.index,
792
- path: job.path,
793
- stateRow: job.stateRow,
794
- });
795
- } catch (error) {
796
- cleanup();
797
- rejectJob(error);
798
- }
799
- });
800
- }
801
-
802
- async function scanRolloutsSequential(jobs, onProgress) {
803
- const fragments = new Array(jobs.length);
804
- for (let index = 0; index < jobs.length; index += 1) {
805
- const job = jobs[index];
806
- fragments[job.index] = await scanRollout(job.path, job.stateRow);
807
- reportScanProgress(onProgress, index + 1, jobs.length, job.path);
808
- }
809
- return fragments;
810
- }
811
-
812
- async function scanRolloutsInParallel(jobs, workerCount, onProgress) {
813
- const scheduled = [...jobs].sort(
814
- (left, right) => right.size - left.size || left.index - right.index,
815
- );
816
- const fragments = new Array(jobs.length);
817
- const workers = Array.from(
818
- { length: workerCount },
819
- () => new Worker(new URL(import.meta.url), {
820
- execArgv: [],
821
- workerData: { mode: SCAN_WORKER_MODE },
822
- }),
823
- );
824
- let nextJob = 0;
825
- let completed = 0;
826
- try {
827
- await Promise.all(workers.map(async (worker) => {
828
- while (nextJob < scheduled.length) {
829
- const job = scheduled[nextJob];
830
- nextJob += 1;
831
- fragments[job.index] = await scanWithWorker(worker, job);
832
- completed += 1;
833
- reportScanProgress(onProgress, completed, jobs.length, job.path);
834
- }
835
- }));
836
- return fragments;
837
- } finally {
838
- await Promise.allSettled(workers.map((worker) => worker.terminate()));
839
- }
840
- }
841
-
842
- async function runScanWorker() {
843
- parentPort.on("message", async (message) => {
844
- if (message?.type !== "scan") return;
845
- try {
846
- const fragment = await scanRollout(message.path, message.stateRow);
847
- parentPort.postMessage({
848
- type: "result",
849
- jobId: message.jobId,
850
- fragment,
851
- });
852
- } catch (error) {
853
- parentPort.postMessage({
854
- type: "error",
855
- jobId: message.jobId,
856
- error: {
857
- name: error instanceof Error ? error.name : "Error",
858
- message: error instanceof Error ? error.message : String(error),
859
- code: error?.code || null,
860
- },
861
- });
862
- }
863
- });
864
- }
865
-
866
- function threadMetadata(threadId, stateRows, parents, fallback = {}) {
681
+ function threadMetadata(threadId, stateRows, titles, parents, fallback = {}) {
867
682
  const row = stateRows.get(threadId);
683
+ const sessionTitle = titles.get(threadId)?.title;
868
684
  const labels = sourceLabels(
869
685
  row?.thread_source,
870
686
  fallback.rawSource || row?.source,
871
687
  );
872
688
  return {
873
689
  id: threadId,
690
+ title: safeTitle(row, sessionTitle),
874
691
  project: projectLabel(
875
692
  fallback.cwd || row?.cwd,
876
693
  fallback.gitOrigin || row?.git_origin_url,
877
694
  ),
878
695
  model: normalizeModel(fallback.model || row?.model || "unknown"),
879
- effort: sanitizeLabel(
696
+ effort: String(
880
697
  fallback.effort || row?.reasoning_effort || "unknown",
881
698
  ).slice(0, 40),
882
699
  source: labels.source,
@@ -891,7 +708,7 @@ function threadMetadata(threadId, stateRows, parents, fallback = {}) {
891
708
  };
892
709
  }
893
710
 
894
- function buildSnapshot(context, options) {
711
+ function buildSnapshot(context, options, titles) {
895
712
  const events = [];
896
713
  for (const token of context.tokens.values()) {
897
714
  const origin = token.turnId ? context.origins.get(token.turnId) : null;
@@ -907,6 +724,7 @@ function buildSnapshot(context, options) {
907
724
  const metadata = threadMetadata(
908
725
  threadId,
909
726
  context.stateRows,
727
+ titles,
910
728
  context.parents,
911
729
  origin || occurrence,
912
730
  );
@@ -918,11 +736,14 @@ function buildSnapshot(context, options) {
918
736
  continue;
919
737
  }
920
738
  const breakdownAvailable = hasDetailedBreakdown(token.usage);
739
+ const serviceTier = occurrence.serviceTier || null;
740
+ const baseCredits = creditsForUsage(metadata.model, token.usage);
921
741
  events.push({
922
742
  ...token.usage,
923
743
  id: `evt-${hash(token.key)}`,
924
744
  timestamp,
925
745
  threadId,
746
+ threadTitle: metadata.title,
926
747
  project: metadata.project,
927
748
  model: metadata.model,
928
749
  effort: metadata.effort,
@@ -930,6 +751,13 @@ function buildSnapshot(context, options) {
930
751
  useType: metadata.useType,
931
752
  turnId: token.turnId || "",
932
753
  toolCalls: 0,
754
+ serviceTier,
755
+ rateCardCredits:
756
+ baseCredits === null
757
+ ? null
758
+ : serviceTier === "priority"
759
+ ? baseCredits * FAST_MODE_MULTIPLIER
760
+ : baseCredits,
933
761
  breakdownAvailable,
934
762
  dedupeQuality: token.dedupeQuality,
935
763
  });
@@ -978,6 +806,7 @@ function buildSnapshot(context, options) {
978
806
  const metadata = threadMetadata(
979
807
  threadId,
980
808
  context.stateRows,
809
+ titles,
981
810
  context.parents,
982
811
  origin || first || {},
983
812
  );
@@ -991,6 +820,10 @@ function buildSnapshot(context, options) {
991
820
  sum.toolCalls += event.toolCalls;
992
821
  if (event.breakdownAvailable) sum.detailedTokens += event.totalTokens;
993
822
  else sum.unknownBreakdownTokens += event.totalTokens;
823
+ if (event.rateCardCredits !== null) {
824
+ sum.rateCardCredits += event.rateCardCredits;
825
+ sum.ratedTokens += event.totalTokens;
826
+ }
994
827
  return sum;
995
828
  },
996
829
  {
@@ -1002,6 +835,8 @@ function buildSnapshot(context, options) {
1002
835
  toolCalls: 0,
1003
836
  detailedTokens: 0,
1004
837
  unknownBreakdownTokens: 0,
838
+ rateCardCredits: 0,
839
+ ratedTokens: 0,
1005
840
  },
1006
841
  );
1007
842
  if (rows.length === 0 && !(metadata.reportedCumulativeTokens > 0)) continue;
@@ -1015,6 +850,7 @@ function buildSnapshot(context, options) {
1015
850
  : "total-only";
1016
851
  threads.push({
1017
852
  id: threadId,
853
+ title: metadata.title,
1018
854
  project: metadata.project,
1019
855
  model: metadata.model,
1020
856
  effort: metadata.effort,
@@ -1031,6 +867,11 @@ function buildSnapshot(context, options) {
1031
867
  cachedInputTokens: totals.cachedInputTokens,
1032
868
  outputTokens: totals.outputTokens,
1033
869
  reasoningTokens: totals.reasoningTokens,
870
+ rateCardCredits:
871
+ totals.totalTokens > 0 && totals.ratedTokens === totals.totalTokens
872
+ ? totals.rateCardCredits
873
+ : null,
874
+ ratedTokens: totals.ratedTokens,
1034
875
  toolCalls: totals.toolCalls,
1035
876
  eventCount: rows.length,
1036
877
  coverage,
@@ -1078,7 +919,7 @@ function buildSnapshot(context, options) {
1078
919
  (quota) => quota.windowMinutes === WEEK_MINUTES,
1079
920
  );
1080
921
  const accountWideWeekly = weeklyCandidates.filter(
1081
- (quota) => quota.scope !== "named",
922
+ (quota) => !quota.limitName,
1082
923
  );
1083
924
  const weekly = [
1084
925
  ...(accountWideWeekly.length ? accountWideWeekly : weeklyCandidates),
@@ -1111,16 +952,13 @@ function buildSnapshot(context, options) {
1111
952
  label: "Local Codex snapshot",
1112
953
  provenance: {
1113
954
  kind: "codex-local-metadata",
1114
- sourceFingerprint: sourceFingerprint(
1115
- options.codexHome,
1116
- options.includeArchived,
1117
- ),
1118
955
  privacy:
1119
- "Contains token metadata and project labels only; display titles, credential fields, message bodies, reasoning text, tool payloads, and full local paths are not exported. Project labels can still be sensitive.",
956
+ "Contains token metadata and Codex display titles only; credential fields, message bodies, reasoning text, tool payloads, and full local paths are not exported. Display titles may contain user-written text.",
957
+ rateCardAsOf: RATE_CARD_AS_OF,
958
+ rateCardUrl: RATE_CARD_URL,
1120
959
  },
1121
960
  coverage: {
1122
961
  filesScanned: context.filesScanned,
1123
- sourceFileCount: context.sourceFileCount,
1124
962
  bytesScanned: context.bytesScanned,
1125
963
  parseErrors: context.parseErrors,
1126
964
  duplicateEventsSkipped: context.duplicateEventsSkipped,
@@ -1148,6 +986,9 @@ function buildSnapshot(context, options) {
1148
986
 
1149
987
  export async function collectUsage(options, onProgress = () => {}) {
1150
988
  const state = await readState(options.codexHome);
989
+ const titles = await readSessionTitles(
990
+ resolve(options.codexHome, "session_index.jsonl"),
991
+ );
1151
992
  const roots = [resolve(options.codexHome, "sessions")];
1152
993
  if (options.includeArchived) {
1153
994
  roots.push(resolve(options.codexHome, "archived_sessions"));
@@ -1158,12 +999,6 @@ export async function collectUsage(options, onProgress = () => {}) {
1158
999
  .flat()
1159
1000
  .sort();
1160
1001
  const sizes = await Promise.all(files.map((path) => stat(path)));
1161
- const jobs = files.map((path, index) => ({
1162
- index,
1163
- path,
1164
- size: sizes[index].size,
1165
- stateRow: state.rows.get(rolloutThreadId(path)) || null,
1166
- }));
1167
1002
 
1168
1003
  const context = {
1169
1004
  stateRows: state.rows,
@@ -1173,24 +1008,30 @@ export async function collectUsage(options, onProgress = () => {}) {
1173
1008
  quotas: new Map(),
1174
1009
  calls: new Map(),
1175
1010
  filesScanned: 0,
1176
- sourceFileCount: files.length + (state.path ? 1 : 0),
1177
1011
  bytesScanned: 0,
1178
1012
  parseErrors: 0,
1179
1013
  duplicateEventsSkipped: 0,
1180
1014
  correctionIntervals: 0,
1181
1015
  };
1182
1016
 
1183
- const workerCount = scanWorkerCount(files.length, options.workers);
1184
- const fragments = workerCount > 1
1185
- ? await scanRolloutsInParallel(jobs, workerCount, onProgress)
1186
- : await scanRolloutsSequential(jobs, onProgress);
1187
- for (const fragment of fragments) {
1188
- mergeScanFragment(context, fragment);
1017
+ for (let index = 0; index < files.length; index += 1) {
1018
+ await scanRollout(files[index], context);
1019
+ context.filesScanned += 1;
1020
+ context.bytesScanned += sizes[index].size;
1021
+ if (
1022
+ index === 0 ||
1023
+ index === files.length - 1 ||
1024
+ (index + 1) % 10 === 0
1025
+ ) {
1026
+ onProgress({
1027
+ current: index + 1,
1028
+ total: files.length,
1029
+ path: files[index],
1030
+ });
1031
+ }
1189
1032
  }
1190
- context.filesScanned = files.length;
1191
- context.bytesScanned = sizes.reduce((sum, entry) => sum + entry.size, 0);
1192
1033
 
1193
- return buildSnapshot(context, options);
1034
+ return buildSnapshot(context, options, titles);
1194
1035
  }
1195
1036
 
1196
1037
  async function main() {
@@ -1198,7 +1039,7 @@ async function main() {
1198
1039
  try {
1199
1040
  options = parseArgs(process.argv.slice(2));
1200
1041
  } catch (error) {
1201
- process.stderr.write(`${sanitizeLabel(error.message)}\n\n${usage()}\n`);
1042
+ process.stderr.write(`${error.message}\n\n${usage()}\n`);
1202
1043
  process.exitCode = 1;
1203
1044
  return;
1204
1045
  }
@@ -1207,9 +1048,7 @@ async function main() {
1207
1048
  return;
1208
1049
  }
1209
1050
  if (!(await pathExists(options.codexHome))) {
1210
- throw new Error(
1211
- `Codex data directory not found: ${sanitizeLabel(options.codexHome)}`,
1212
- );
1051
+ throw new Error(`Codex data directory not found: ${options.codexHome}`);
1213
1052
  }
1214
1053
 
1215
1054
  process.stdout.write("Token Ledger: scanning local Codex metadata…\n");
@@ -1223,7 +1062,7 @@ async function main() {
1223
1062
  const outputSize = (await stat(options.output)).size;
1224
1063
  process.stdout.write(
1225
1064
  [
1226
- `Snapshot: ${sanitizeLabel(options.output)}`,
1065
+ `Snapshot: ${options.output}`,
1227
1066
  `Observed model-call tokens: ${snapshot.coverage.observedTokens.toLocaleString()}`,
1228
1067
  `Unique events: ${snapshot.events.length.toLocaleString()}`,
1229
1068
  `Duplicate/copied events skipped: ${snapshot.coverage.duplicateEventsSkipped.toLocaleString()}`,
@@ -1234,13 +1073,11 @@ async function main() {
1234
1073
  }
1235
1074
 
1236
1075
  const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : "";
1237
- if (!isMainThread && workerData?.mode === SCAN_WORKER_MODE) {
1238
- runScanWorker();
1239
- } else if (isMainThread && import.meta.url === invokedPath) {
1076
+ if (import.meta.url === invokedPath) {
1240
1077
  main().catch((error) => {
1241
1078
  process.stderr.write(
1242
1079
  `Token Ledger collector failed: ${
1243
- error instanceof Error ? sanitizeLabel(error.message) : sanitizeLabel(error)
1080
+ error instanceof Error ? error.message : String(error)
1244
1081
  }\n`,
1245
1082
  );
1246
1083
  process.exitCode = 1;