tledger 0.1.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@
4
4
  * Token Ledger local collector
5
5
  *
6
6
  * Reads Codex's local JSONL rollouts and metadata database, then writes a
7
- * privacy-reduced snapshot for the Token Ledger CLI. It never exports message
7
+ * privacy-reduced snapshot for the Token Ledger site. It never exports message
8
8
  * bodies, tool arguments/results, reasoning text, instructions, credential
9
9
  * fields, or full local paths in the generated snapshot.
10
10
  *
@@ -23,7 +23,7 @@ import {
23
23
  stat,
24
24
  writeFile,
25
25
  } from "node:fs/promises";
26
- import { availableParallelism, homedir } from "node:os";
26
+ import { homedir } from "node:os";
27
27
  import {
28
28
  basename,
29
29
  dirname,
@@ -32,65 +32,33 @@ import {
32
32
  } from "node:path";
33
33
  import { pathToFileURL } from "node:url";
34
34
  import { createInterface } from "node:readline";
35
- import {
36
- isMainThread,
37
- parentPort,
38
- Worker,
39
- workerData,
40
- } from "node:worker_threads";
41
-
42
- import { normalizeModelIdentifier } from "./token-ledger-models.mjs";
35
+ import { DatabaseSync } from "node:sqlite";
43
36
 
44
37
  const SCHEMA_VERSION = 1;
38
+ const RATE_CARD_AS_OF = "2026-08-17";
39
+ const FAST_MODE_MULTIPLIER = 1.5;
40
+ const RATE_CARD_URL = "https://help.openai.com/en/articles/20001106";
45
41
  const WEEK_MINUTES = 10_080;
46
- const DEFAULT_MAX_SCAN_WORKERS = 4;
47
- const MAX_SCAN_WORKERS = 6;
48
- const SCAN_WORKER_MODE = "token-ledger-rollout-scanner";
49
42
  const UUID_AT_END =
50
43
  /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
51
- const RELEVANT_RECORD_TYPE =
52
- /"type"\s*:\s*"(?:session_meta|turn_context|task_started|thread_settings_applied|token_count|function_call|custom_tool_call|tool_search_call|web_search_call|image_generation_call)"/;
53
- const RESPONSE_CALL_TYPES = new Set([
54
- "function_call",
55
- "custom_tool_call",
56
- "tool_search_call",
57
- "web_search_call",
58
- "image_generation_call",
59
- ]);
60
- let databaseSyncPromise;
61
-
62
- function loadDatabaseSync() {
63
- if (!databaseSyncPromise) {
64
- databaseSyncPromise = (async () => {
65
- const originalEmitWarning = process.emitWarning;
66
- process.emitWarning = function emitWarning(warning, ...arguments_) {
67
- const message = warning instanceof Error ? warning.message : String(warning);
68
- const type = typeof arguments_[0] === "string"
69
- ? arguments_[0]
70
- : arguments_[0]?.type;
71
- if (
72
- type === "ExperimentalWarning" &&
73
- message === "SQLite is an experimental feature and might change at any time"
74
- ) {
75
- return;
76
- }
77
- return originalEmitWarning.call(this, warning, ...arguments_);
78
- };
79
- try {
80
- return (await import("node:sqlite")).DatabaseSync;
81
- } finally {
82
- process.emitWarning = originalEmitWarning;
83
- }
84
- })();
85
- }
86
- return databaseSyncPromise;
87
- }
44
+
45
+ const RATE_CARD = {
46
+ "gpt-5.6-sol": { input: 125, cached: 12.5, output: 750 },
47
+ "gpt-5.6-terra": { input: 50, cached: 5, output: 300 },
48
+ "gpt-5.6-luna": { input: 5, cached: 0.5, output: 30 },
49
+ "gpt-5.5": { input: 125, cached: 12.5, output: 750 },
50
+ "gpt-5.5-cyber": { input: 500, cached: 50, output: 3_000 },
51
+ "gpt-5.4": { input: 62.5, cached: 6.25, output: 375 },
52
+ "gpt-5.4-mini": { input: 18.75, cached: 1.875, output: 113 },
53
+ "gpt-5.3-codex": { input: 43.75, cached: 4.375, output: 350 },
54
+ "gpt-5.2": { input: 43.75, cached: 4.375, output: 350 },
55
+ };
88
56
 
89
57
  function usage() {
90
58
  return `Token Ledger local collector
91
59
 
92
60
  Usage:
93
- node lib/token-ledger-collector.mjs [options]
61
+ node token-ledger-importer.mjs [options]
94
62
 
95
63
  Options:
96
64
  --output <file> Snapshot destination (default: token-ledger-snapshot.json)
@@ -99,9 +67,10 @@ Options:
99
67
  --no-archived Skip archived_sessions
100
68
  --help Show this help
101
69
 
102
- The snapshot contains local token metadata and project labels only. It never
103
- contains display titles, message bodies, tool payloads, reasoning text,
104
- credential fields, or full local paths. Project labels can still be sensitive.`;
70
+ The snapshot contains usage metadata and Codex display titles only. It never
71
+ contains message bodies, tool payloads, reasoning text, credential fields, or
72
+ full local paths in its output. Codex display titles may contain user-written
73
+ text.`;
105
74
  }
106
75
 
107
76
  function parseArgs(argv) {
@@ -143,45 +112,6 @@ function hash(value, length = 24) {
143
112
  return createHash("sha256").update(String(value)).digest("hex").slice(0, length);
144
113
  }
145
114
 
146
- export function sourceFingerprint(codexHome, includeArchived = true) {
147
- return hash(JSON.stringify({
148
- codexHome: resolve(codexHome),
149
- includeArchived,
150
- }));
151
- }
152
-
153
- function sanitizeLabel(value, fallback = "unknown") {
154
- const label = String(value ?? "")
155
- .replace(/\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g, "")
156
- .replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
157
- .replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ")
158
- .replace(/\s+/g, " ")
159
- .trim();
160
- return (label || fallback).slice(0, 160);
161
- }
162
-
163
- export async function writePrivateSnapshot(output, snapshot) {
164
- const destination = resolve(output);
165
- const directory = dirname(destination);
166
- const temporary = resolve(
167
- directory,
168
- `.token-ledger-${process.pid}-${randomUUID()}.tmp`,
169
- );
170
- await mkdir(directory, { recursive: true });
171
- try {
172
- await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
173
- encoding: "utf8",
174
- flag: "wx",
175
- mode: 0o600,
176
- });
177
- await chmod(temporary, 0o600);
178
- await rename(temporary, destination);
179
- await chmod(destination, 0o600);
180
- } finally {
181
- await rm(temporary, { force: true });
182
- }
183
- }
184
-
185
115
  function asFiniteNumber(value) {
186
116
  const number = Number(value ?? 0);
187
117
  return Number.isFinite(number) ? number : 0;
@@ -232,7 +162,35 @@ function hasDetailedBreakdown(usage) {
232
162
  }
233
163
 
234
164
  function normalizeModel(model) {
235
- return normalizeModelIdentifier(sanitizeLabel(model, "unknown"));
165
+ const value = String(model || "unknown")
166
+ .trim()
167
+ .toLowerCase()
168
+ .replaceAll("_", "-");
169
+ if (RATE_CARD[value]) return value;
170
+ if (value.startsWith("gpt-5.6-sol")) return "gpt-5.6-sol";
171
+ if (value.startsWith("gpt-5.6-terra")) return "gpt-5.6-terra";
172
+ if (value.startsWith("gpt-5.6-luna")) return "gpt-5.6-luna";
173
+ if (value.startsWith("gpt-5.5-cyber")) return "gpt-5.5-cyber";
174
+ if (value.startsWith("gpt-5.5")) return "gpt-5.5";
175
+ if (value.startsWith("gpt-5.4-mini")) return "gpt-5.4-mini";
176
+ if (value.startsWith("gpt-5.4")) return "gpt-5.4";
177
+ if (value.startsWith("gpt-5.3-codex")) return "gpt-5.3-codex";
178
+ if (value.startsWith("gpt-5.2")) return "gpt-5.2";
179
+ return value || "unknown";
180
+ }
181
+
182
+ function creditsForUsage(model, usage) {
183
+ if (!hasDetailedBreakdown(usage)) return null;
184
+ const rate = RATE_CARD[normalizeModel(model)];
185
+ if (!rate) return null;
186
+ const cached = Math.min(usage.inputTokens, usage.cachedInputTokens);
187
+ const uncached = Math.max(0, usage.inputTokens - cached);
188
+ return (
189
+ (uncached * rate.input +
190
+ cached * rate.cached +
191
+ usage.outputTokens * rate.output) /
192
+ 1_000_000
193
+ );
236
194
  }
237
195
 
238
196
  function parseStructuredSource(value) {
@@ -265,15 +223,9 @@ function sourceLabels(threadSource, rawSource) {
265
223
  if (source === "exec") return { source: "cli", useType: "cli" };
266
224
  if (source === "vscode") return { source: "desktop", useType: "interactive" };
267
225
  if (typeof source === "string" && source) {
268
- return {
269
- source: sanitizeLabel(source).slice(0, 40),
270
- useType: sanitizeLabel(threadSource || "interactive").slice(0, 40),
271
- };
226
+ return { source: source.slice(0, 40), useType: threadSource || "interactive" };
272
227
  }
273
- return {
274
- source: "unknown",
275
- useType: sanitizeLabel(threadSource || "unknown").slice(0, 40),
276
- };
228
+ return { source: "unknown", useType: threadSource || "unknown" };
277
229
  }
278
230
 
279
231
  function cleanRemote(value) {
@@ -281,15 +233,15 @@ function cleanRemote(value) {
281
233
  const remote = String(value).trim();
282
234
  const scp = remote.match(/^[^@]+@([^:]+):(.+)$/);
283
235
  if (scp) {
284
- return sanitizeLabel(`${scp[1]}/${scp[2].replace(/\.git$/i, "")}`);
236
+ return `${scp[1]}/${scp[2].replace(/\.git$/i, "")}`;
285
237
  }
286
238
  try {
287
239
  const url = new URL(remote);
288
240
  const path = url.pathname.replace(/^\/+/, "").replace(/\.git$/i, "");
289
- return sanitizeLabel(path ? `${url.hostname}/${path}` : url.hostname);
241
+ return path ? `${url.hostname}/${path}` : url.hostname;
290
242
  } catch {
291
243
  const withoutCredentials = remote.replace(/\/\/[^/@]+@/, "//");
292
- return sanitizeLabel(withoutCredentials.replace(/\.git$/i, ""));
244
+ return withoutCredentials.replace(/\.git$/i, "").slice(0, 160);
293
245
  }
294
246
  }
295
247
 
@@ -297,15 +249,37 @@ function projectLabel(cwd, gitOrigin) {
297
249
  const remote = cleanRemote(gitOrigin);
298
250
  if (remote) {
299
251
  const parts = remote.split("/").filter(Boolean);
300
- return sanitizeLabel(parts.slice(-2).join("/"), "Unknown project");
252
+ return parts.slice(-2).join("/") || "Unknown project";
301
253
  }
302
254
  const path = String(cwd || "").replaceAll("\\", "/").replace(/\/+$/, "");
303
255
  const worktree = path.match(/\/\.codex\/worktrees\/[^/]+\/([^/]+)$/);
304
256
  if (worktree) return worktree[1];
305
257
  const name = basename(path);
306
- return name && name !== "." && name !== "/"
307
- ? sanitizeLabel(name, "Unknown project")
308
- : "Unknown project";
258
+ return name && name !== "." && name !== "/" ? name : "Unknown project";
259
+ }
260
+
261
+ function safeTitle(row, sessionTitle) {
262
+ const candidate = String(sessionTitle || row?.title || row?.name || "").trim();
263
+ const subagent =
264
+ row?.thread_source === "subagent" ||
265
+ String(row?.source || "").includes('"subagent"');
266
+ if (
267
+ !candidate ||
268
+ candidate.includes("<codex_delegation>") ||
269
+ (subagent && candidate.length > 120)
270
+ ) {
271
+ if (row?.agent_nickname) return `Subagent · ${row.agent_nickname}`;
272
+ return subagent ? `Subagent · ${String(row?.id || "").slice(0, 8)}` : "Untitled task";
273
+ }
274
+ return candidate
275
+ .replace(/\/Users\/[^\s"'`]+/g, "[local path]")
276
+ .replace(/\/(?:private\/)?tmp\/[^\s"'`]+/g, "[temporary path]")
277
+ .replace(
278
+ /\b(?:sk-[A-Za-z0-9_-]{16,}|lin_api_[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_-]{16,})\b/g,
279
+ "[redacted credential-like text]",
280
+ )
281
+ .replace(/\s+/g, " ")
282
+ .slice(0, 180);
309
283
  }
310
284
 
311
285
  async function pathExists(path) {
@@ -317,6 +291,28 @@ async function pathExists(path) {
317
291
  }
318
292
  }
319
293
 
294
+ export async function writePrivateSnapshot(output, snapshot) {
295
+ const destination = resolve(output);
296
+ const directory = dirname(destination);
297
+ const temporary = resolve(
298
+ directory,
299
+ `.token-ledger-${process.pid}-${randomUUID()}.tmp`,
300
+ );
301
+ await mkdir(directory, { recursive: true });
302
+ try {
303
+ await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, {
304
+ encoding: "utf8",
305
+ flag: "wx",
306
+ mode: 0o600,
307
+ });
308
+ await chmod(temporary, 0o600);
309
+ await rename(temporary, destination);
310
+ await chmod(destination, 0o600);
311
+ } finally {
312
+ await rm(temporary, { force: true });
313
+ }
314
+ }
315
+
320
316
  export async function listJsonlFiles(root) {
321
317
  if (!(await pathExists(root))) return [];
322
318
  const found = [];
@@ -333,33 +329,51 @@ export async function listJsonlFiles(root) {
333
329
  return found;
334
330
  }
335
331
 
336
- export async function sourceState(codexHome, includeArchived = true) {
332
+ export async function latestSourceModifiedAt(codexHome, includeArchived = true) {
337
333
  const roots = [resolve(codexHome, "sessions")];
338
334
  if (includeArchived) {
339
335
  roots.push(resolve(codexHome, "archived_sessions"));
340
336
  }
341
337
  const files = (await Promise.all(roots.map((root) => listJsonlFiles(root))))
342
338
  .flat();
343
- const preferredState = resolve(codexHome, "state_5.sqlite");
344
- const legacyState = resolve(codexHome, "sqlite", "state_5.sqlite");
345
- const statePath = (await pathExists(preferredState))
346
- ? preferredState
347
- : (await pathExists(legacyState))
348
- ? legacyState
349
- : null;
350
- const sourceFiles = statePath ? [...files, statePath] : files;
351
- if (!sourceFiles.length) {
352
- return { latestMtimeMs: 0, fileCount: 0 };
339
+ const metadataFiles = [
340
+ resolve(codexHome, "session_index.jsonl"),
341
+ resolve(codexHome, "state_5.sqlite"),
342
+ resolve(codexHome, "sqlite", "state_5.sqlite"),
343
+ ];
344
+ const existingMetadataFiles = [];
345
+ for (const path of metadataFiles) {
346
+ if (await pathExists(path)) existingMetadataFiles.push(path);
353
347
  }
348
+ const sourceFiles = [...files, ...existingMetadataFiles];
349
+ if (!sourceFiles.length) return 0;
354
350
  const stats = await Promise.all(sourceFiles.map((path) => stat(path)));
355
- return {
356
- latestMtimeMs: Math.max(...stats.map((entry) => entry.mtimeMs)),
357
- fileCount: sourceFiles.length,
358
- };
351
+ return Math.max(...stats.map((entry) => entry.mtimeMs));
359
352
  }
360
353
 
361
- export async function latestSourceModifiedAt(codexHome, includeArchived = true) {
362
- return (await sourceState(codexHome, includeArchived)).latestMtimeMs;
354
+ async function readSessionTitles(path) {
355
+ const titles = new Map();
356
+ if (!(await pathExists(path))) return titles;
357
+ const input = createReadStream(path, { encoding: "utf8" });
358
+ const lines = createInterface({ input, crlfDelay: Infinity });
359
+ for await (const line of lines) {
360
+ if (!line.trim()) continue;
361
+ try {
362
+ const record = JSON.parse(line);
363
+ if (!record?.id || !record?.thread_name) continue;
364
+ const timestamp = new Date(record.updated_at || 0).getTime();
365
+ const current = titles.get(record.id);
366
+ if (!current || timestamp >= current.timestamp) {
367
+ titles.set(record.id, {
368
+ title: String(record.thread_name).replace(/\s+/g, " ").slice(0, 180),
369
+ timestamp,
370
+ });
371
+ }
372
+ } catch {
373
+ // The thread index is optional; malformed lines do not affect token totals.
374
+ }
375
+ }
376
+ return titles;
363
377
  }
364
378
 
365
379
  async function readState(codexHome) {
@@ -374,13 +388,13 @@ async function readState(codexHome) {
374
388
  const parents = new Map();
375
389
  if (!path) return { path: null, rows, parents };
376
390
 
377
- const DatabaseSync = await loadDatabaseSync();
378
391
  const database = new DatabaseSync(path, { readOnly: true });
379
392
  try {
380
393
  const threadRows = database
381
394
  .prepare(
382
- `SELECT id, created_at, updated_at, source, cwd,
383
- tokens_used, git_origin_url, model, reasoning_effort,
395
+ `SELECT id, created_at, updated_at, source, cwd, title, name,
396
+ tokens_used, git_sha, git_branch, git_origin_url,
397
+ agent_nickname, agent_role, model, reasoning_effort,
384
398
  thread_source
385
399
  FROM threads`,
386
400
  )
@@ -421,6 +435,7 @@ function taskStartCandidate(record, threadId, stateRow, fileContext) {
421
435
  cwd: fileContext.cwd || stateRow?.cwd || "",
422
436
  gitOrigin: fileContext.gitOrigin || stateRow?.git_origin_url || null,
423
437
  rawSource: fileContext.rawSource || stateRow?.source || null,
438
+ serviceTier: fileContext.serviceTier,
424
439
  };
425
440
  }
426
441
 
@@ -445,19 +460,38 @@ function rememberQuota(quotaMap, rateLimits, occurrence) {
445
460
  usedPercent,
446
461
  windowMinutes,
447
462
  resetsAt,
448
- scope: rateLimits.limit_name ? "named" : "account",
463
+ planType: String(rateLimits.plan_type || "unknown"),
464
+ limitKey: hash(limitKey, 16),
465
+ limitName: rateLimits.limit_name
466
+ ? String(rateLimits.limit_name).slice(0, 80)
467
+ : null,
449
468
  source: "log",
450
469
  turnId: occurrence.turnId || null,
451
470
  originalLikely: occurrence.originalLikely,
452
471
  };
453
- rememberQuotaCandidate(quotaMap, key, candidate);
472
+ const current = quotaMap.get(key);
473
+ if (
474
+ !current ||
475
+ (!current.originalLikely && candidate.originalLikely) ||
476
+ (current.originalLikely === candidate.originalLikely &&
477
+ candidate.timestamp < current.timestamp)
478
+ ) {
479
+ quotaMap.set(key, candidate);
480
+ }
454
481
  }
455
482
  }
456
483
 
457
484
  function responseCall(record) {
458
485
  if (record.type !== "response_item") return null;
459
486
  const payload = record.payload;
460
- if (!payload || !RESPONSE_CALL_TYPES.has(payload.type)) return null;
487
+ const allowed = new Set([
488
+ "function_call",
489
+ "custom_tool_call",
490
+ "tool_search_call",
491
+ "web_search_call",
492
+ "image_generation_call",
493
+ ]);
494
+ if (!payload || !allowed.has(payload.type)) return null;
461
495
  return {
462
496
  type: payload.type,
463
497
  name: String(payload.name || payload.namespace || payload.type).slice(0, 80),
@@ -465,97 +499,17 @@ function responseCall(record) {
465
499
  };
466
500
  }
467
501
 
468
- export function rolloutLineMayAffectUsage(line) {
469
- return RELEVANT_RECORD_TYPE.test(line);
470
- }
471
-
472
- function rolloutThreadId(path) {
502
+ async function scanRollout(path, context) {
473
503
  const match = path.match(UUID_AT_END);
474
- return match?.[1] || `file-${hash(path)}`;
475
- }
476
-
477
- function createScanFragment() {
478
- return {
479
- parents: new Map(),
480
- origins: new Map(),
481
- tokens: new Map(),
482
- quotas: new Map(),
483
- calls: new Map(),
484
- parseErrors: 0,
485
- duplicateEventsSkipped: 0,
486
- correctionIntervals: 0,
487
- };
488
- }
489
-
490
- function rememberOrigin(originMap, turnId, candidate) {
491
- const current = originMap.get(turnId);
492
- if (!current || candidate.deltaMs < current.deltaMs) {
493
- originMap.set(turnId, candidate);
494
- }
495
- }
496
-
497
- function rememberCall(callMap, key, candidate) {
498
- const current = callMap.get(key);
499
- if (!current || (!current.originalLikely && candidate.originalLikely)) {
500
- callMap.set(key, candidate);
501
- }
502
- }
503
-
504
- function rememberToken(context, key, candidate) {
505
- const current = context.tokens.get(key);
506
- if (!current) {
507
- context.tokens.set(key, candidate);
508
- return;
509
- }
510
- context.duplicateEventsSkipped += 1;
511
- if (!current.originalLikely && candidate.originalLikely) {
512
- current.occurrence = candidate.occurrence;
513
- current.originalLikely = true;
514
- }
515
- }
516
-
517
- function rememberQuotaCandidate(quotaMap, key, candidate) {
518
- const current = quotaMap.get(key);
519
- if (
520
- !current ||
521
- (!current.originalLikely && candidate.originalLikely) ||
522
- (current.originalLikely === candidate.originalLikely &&
523
- candidate.timestamp < current.timestamp)
524
- ) {
525
- quotaMap.set(key, candidate);
526
- }
527
- }
528
-
529
- function mergeScanFragment(context, fragment) {
530
- for (const [threadId, parentThreadId] of fragment.parents) {
531
- context.parents.set(threadId, parentThreadId);
532
- }
533
- for (const [turnId, candidate] of fragment.origins) {
534
- rememberOrigin(context.origins, turnId, candidate);
535
- }
536
- for (const [key, candidate] of fragment.tokens) {
537
- rememberToken(context, key, candidate);
538
- }
539
- for (const [key, candidate] of fragment.quotas) {
540
- rememberQuotaCandidate(context.quotas, key, candidate);
541
- }
542
- for (const [key, candidate] of fragment.calls) {
543
- rememberCall(context.calls, key, candidate);
544
- }
545
- context.parseErrors += fragment.parseErrors;
546
- context.duplicateEventsSkipped += fragment.duplicateEventsSkipped;
547
- context.correctionIntervals += fragment.correctionIntervals;
548
- }
549
-
550
- async function scanRollout(path, stateRow) {
551
- const context = createScanFragment();
552
- const threadId = rolloutThreadId(path);
504
+ const threadId = match?.[1] || `file-${hash(path)}`;
505
+ const stateRow = context.stateRows.get(threadId);
553
506
  const fileContext = {
554
507
  model: stateRow?.model || "unknown",
555
508
  effort: stateRow?.reasoning_effort || "unknown",
556
509
  cwd: stateRow?.cwd || "",
557
510
  gitOrigin: stateRow?.git_origin_url || null,
558
511
  rawSource: stateRow?.source || null,
512
+ serviceTier: null,
559
513
  };
560
514
  const callOrdinals = new Map();
561
515
  let currentTurnId = "";
@@ -565,7 +519,7 @@ async function scanRollout(path, stateRow) {
565
519
  const input = createReadStream(path, { encoding: "utf8" });
566
520
  const lines = createInterface({ input, crlfDelay: Infinity });
567
521
  for await (const line of lines) {
568
- if (!line || !rolloutLineMayAffectUsage(line)) continue;
522
+ if (!line.trim()) continue;
569
523
  let record;
570
524
  try {
571
525
  record = JSON.parse(line);
@@ -603,7 +557,10 @@ async function scanRollout(path, stateRow) {
603
557
  stateRow,
604
558
  fileContext,
605
559
  );
606
- rememberOrigin(context.origins, currentTurnId, currentCandidate);
560
+ const currentBest = context.origins.get(currentTurnId);
561
+ if (!currentBest || currentCandidate.deltaMs < currentBest.deltaMs) {
562
+ context.origins.set(currentTurnId, currentCandidate);
563
+ }
607
564
  }
608
565
  continue;
609
566
  }
@@ -628,6 +585,10 @@ async function scanRollout(path, stateRow) {
628
585
  const settings = record.payload?.thread_settings;
629
586
  fileContext.model = settings?.model || fileContext.model;
630
587
  fileContext.effort = settings?.reasoning_effort || fileContext.effort;
588
+ const serviceTier = String(settings?.service_tier ?? "").trim();
589
+ fileContext.serviceTier = serviceTier
590
+ ? serviceTier.slice(0, 40)
591
+ : null;
631
592
  continue;
632
593
  }
633
594
 
@@ -645,6 +606,7 @@ async function scanRollout(path, stateRow) {
645
606
  cwd: fileContext.cwd,
646
607
  gitOrigin: fileContext.gitOrigin,
647
608
  rawSource: fileContext.rawSource,
609
+ serviceTier: fileContext.serviceTier,
648
610
  };
649
611
 
650
612
  const call = responseCall(record);
@@ -655,11 +617,14 @@ async function scanRollout(path, stateRow) {
655
617
  const callKey = call.stableId
656
618
  ? `id|${call.stableId}`
657
619
  : `ordinal|${ordinalBase}|${ordinal}`;
658
- rememberCall(context.calls, callKey, {
659
- turnId: currentTurnId,
660
- threadId,
661
- originalLikely,
662
- });
620
+ const existing = context.calls.get(callKey);
621
+ if (!existing || (!existing.originalLikely && originalLikely)) {
622
+ context.calls.set(callKey, {
623
+ turnId: currentTurnId,
624
+ threadId,
625
+ originalLikely,
626
+ });
627
+ }
663
628
  continue;
664
629
  }
665
630
 
@@ -692,7 +657,17 @@ async function scanRollout(path, stateRow) {
692
657
  }
693
658
  previousCumulative = totalTuple[5];
694
659
 
695
- rememberToken(context, eventKey, {
660
+ const existing = context.tokens.get(eventKey);
661
+ if (existing) {
662
+ context.duplicateEventsSkipped += 1;
663
+ if (!existing.originalLikely && originalLikely) {
664
+ existing.occurrence = occurrence;
665
+ existing.originalLikely = true;
666
+ }
667
+ continue;
668
+ }
669
+
670
+ context.tokens.set(eventKey, {
696
671
  key: eventKey,
697
672
  turnId: currentTurnId,
698
673
  usage: usageFromTuple(lastTuple),
@@ -701,172 +676,24 @@ async function scanRollout(path, stateRow) {
701
676
  dedupeQuality: currentTurnId ? "turn-exact" : "legacy-heuristic",
702
677
  });
703
678
  }
704
- return context;
705
679
  }
706
680
 
707
- export function scanWorkerCount(
708
- fileCount,
709
- requestedWorkers = null,
710
- parallelism = availableParallelism(),
711
- ) {
712
- if (!Number.isInteger(fileCount) || fileCount < 0) {
713
- throw new Error("fileCount must be a non-negative integer.");
714
- }
715
- if (fileCount === 0) return 0;
716
- if (
717
- requestedWorkers !== null &&
718
- (!Number.isInteger(requestedWorkers) ||
719
- requestedWorkers < 1 ||
720
- requestedWorkers > MAX_SCAN_WORKERS)
721
- ) {
722
- throw new Error(`workers must be an integer from 1 to ${MAX_SCAN_WORKERS}.`);
723
- }
724
- const detectedParallelism =
725
- Number.isInteger(parallelism) && parallelism > 0 ? parallelism : 1;
726
- const automaticWorkers = Math.min(
727
- DEFAULT_MAX_SCAN_WORKERS,
728
- Math.max(1, detectedParallelism - 1),
729
- );
730
- return Math.min(fileCount, requestedWorkers ?? automaticWorkers);
731
- }
732
-
733
- function reportScanProgress(onProgress, current, total, path) {
734
- if (current === 1 || current === total || current % 10 === 0) {
735
- onProgress({ current, total, path });
736
- }
737
- }
738
-
739
- function workerFailure(message) {
740
- const error = new Error(message?.error?.message || "Rollout scan worker failed.");
741
- error.name = message?.error?.name || "Error";
742
- if (message?.error?.code) error.code = message.error.code;
743
- return error;
744
- }
745
-
746
- function scanWithWorker(worker, job) {
747
- return new Promise((resolveJob, rejectJob) => {
748
- const cleanup = () => {
749
- worker.off("message", onMessage);
750
- worker.off("error", onError);
751
- worker.off("exit", onExit);
752
- };
753
- const onMessage = (message) => {
754
- cleanup();
755
- if (message?.jobId !== job.index) {
756
- rejectJob(new Error("Rollout scan worker returned an unexpected job."));
757
- } else if (message.type === "error") {
758
- rejectJob(workerFailure(message));
759
- } else if (message.type === "result") {
760
- resolveJob(message.fragment);
761
- } else {
762
- rejectJob(new Error("Rollout scan worker returned an unknown response."));
763
- }
764
- };
765
- const onError = (error) => {
766
- cleanup();
767
- rejectJob(error);
768
- };
769
- const onExit = (code) => {
770
- cleanup();
771
- rejectJob(
772
- new Error(`Rollout scan worker exited before completing its job (${code}).`),
773
- );
774
- };
775
- worker.once("message", onMessage);
776
- worker.once("error", onError);
777
- worker.once("exit", onExit);
778
- try {
779
- worker.postMessage({
780
- type: "scan",
781
- jobId: job.index,
782
- path: job.path,
783
- stateRow: job.stateRow,
784
- });
785
- } catch (error) {
786
- cleanup();
787
- rejectJob(error);
788
- }
789
- });
790
- }
791
-
792
- async function scanRolloutsSequential(jobs, onProgress) {
793
- const fragments = new Array(jobs.length);
794
- for (let index = 0; index < jobs.length; index += 1) {
795
- const job = jobs[index];
796
- fragments[job.index] = await scanRollout(job.path, job.stateRow);
797
- reportScanProgress(onProgress, index + 1, jobs.length, job.path);
798
- }
799
- return fragments;
800
- }
801
-
802
- async function scanRolloutsInParallel(jobs, workerCount, onProgress) {
803
- const scheduled = [...jobs].sort(
804
- (left, right) => right.size - left.size || left.index - right.index,
805
- );
806
- const fragments = new Array(jobs.length);
807
- const workers = Array.from(
808
- { length: workerCount },
809
- () => new Worker(new URL(import.meta.url), {
810
- execArgv: [],
811
- workerData: { mode: SCAN_WORKER_MODE },
812
- }),
813
- );
814
- let nextJob = 0;
815
- let completed = 0;
816
- try {
817
- await Promise.all(workers.map(async (worker) => {
818
- while (nextJob < scheduled.length) {
819
- const job = scheduled[nextJob];
820
- nextJob += 1;
821
- fragments[job.index] = await scanWithWorker(worker, job);
822
- completed += 1;
823
- reportScanProgress(onProgress, completed, jobs.length, job.path);
824
- }
825
- }));
826
- return fragments;
827
- } finally {
828
- await Promise.allSettled(workers.map((worker) => worker.terminate()));
829
- }
830
- }
831
-
832
- async function runScanWorker() {
833
- parentPort.on("message", async (message) => {
834
- if (message?.type !== "scan") return;
835
- try {
836
- const fragment = await scanRollout(message.path, message.stateRow);
837
- parentPort.postMessage({
838
- type: "result",
839
- jobId: message.jobId,
840
- fragment,
841
- });
842
- } catch (error) {
843
- parentPort.postMessage({
844
- type: "error",
845
- jobId: message.jobId,
846
- error: {
847
- name: error instanceof Error ? error.name : "Error",
848
- message: error instanceof Error ? error.message : String(error),
849
- code: error?.code || null,
850
- },
851
- });
852
- }
853
- });
854
- }
855
-
856
- function threadMetadata(threadId, stateRows, parents, fallback = {}) {
681
+ function threadMetadata(threadId, stateRows, titles, parents, fallback = {}) {
857
682
  const row = stateRows.get(threadId);
683
+ const sessionTitle = titles.get(threadId)?.title;
858
684
  const labels = sourceLabels(
859
685
  row?.thread_source,
860
686
  fallback.rawSource || row?.source,
861
687
  );
862
688
  return {
863
689
  id: threadId,
690
+ title: safeTitle(row, sessionTitle),
864
691
  project: projectLabel(
865
692
  fallback.cwd || row?.cwd,
866
693
  fallback.gitOrigin || row?.git_origin_url,
867
694
  ),
868
695
  model: normalizeModel(fallback.model || row?.model || "unknown"),
869
- effort: sanitizeLabel(
696
+ effort: String(
870
697
  fallback.effort || row?.reasoning_effort || "unknown",
871
698
  ).slice(0, 40),
872
699
  source: labels.source,
@@ -881,7 +708,7 @@ function threadMetadata(threadId, stateRows, parents, fallback = {}) {
881
708
  };
882
709
  }
883
710
 
884
- function buildSnapshot(context, options) {
711
+ function buildSnapshot(context, options, titles) {
885
712
  const events = [];
886
713
  for (const token of context.tokens.values()) {
887
714
  const origin = token.turnId ? context.origins.get(token.turnId) : null;
@@ -897,6 +724,7 @@ function buildSnapshot(context, options) {
897
724
  const metadata = threadMetadata(
898
725
  threadId,
899
726
  context.stateRows,
727
+ titles,
900
728
  context.parents,
901
729
  origin || occurrence,
902
730
  );
@@ -908,11 +736,14 @@ function buildSnapshot(context, options) {
908
736
  continue;
909
737
  }
910
738
  const breakdownAvailable = hasDetailedBreakdown(token.usage);
739
+ const serviceTier = occurrence.serviceTier || null;
740
+ const baseCredits = creditsForUsage(metadata.model, token.usage);
911
741
  events.push({
912
742
  ...token.usage,
913
743
  id: `evt-${hash(token.key)}`,
914
744
  timestamp,
915
745
  threadId,
746
+ threadTitle: metadata.title,
916
747
  project: metadata.project,
917
748
  model: metadata.model,
918
749
  effort: metadata.effort,
@@ -920,6 +751,13 @@ function buildSnapshot(context, options) {
920
751
  useType: metadata.useType,
921
752
  turnId: token.turnId || "",
922
753
  toolCalls: 0,
754
+ serviceTier,
755
+ rateCardCredits:
756
+ baseCredits === null
757
+ ? null
758
+ : serviceTier === "priority"
759
+ ? baseCredits * FAST_MODE_MULTIPLIER
760
+ : baseCredits,
923
761
  breakdownAvailable,
924
762
  dedupeQuality: token.dedupeQuality,
925
763
  });
@@ -968,6 +806,7 @@ function buildSnapshot(context, options) {
968
806
  const metadata = threadMetadata(
969
807
  threadId,
970
808
  context.stateRows,
809
+ titles,
971
810
  context.parents,
972
811
  origin || first || {},
973
812
  );
@@ -981,6 +820,10 @@ function buildSnapshot(context, options) {
981
820
  sum.toolCalls += event.toolCalls;
982
821
  if (event.breakdownAvailable) sum.detailedTokens += event.totalTokens;
983
822
  else sum.unknownBreakdownTokens += event.totalTokens;
823
+ if (event.rateCardCredits !== null) {
824
+ sum.rateCardCredits += event.rateCardCredits;
825
+ sum.ratedTokens += event.totalTokens;
826
+ }
984
827
  return sum;
985
828
  },
986
829
  {
@@ -992,6 +835,8 @@ function buildSnapshot(context, options) {
992
835
  toolCalls: 0,
993
836
  detailedTokens: 0,
994
837
  unknownBreakdownTokens: 0,
838
+ rateCardCredits: 0,
839
+ ratedTokens: 0,
995
840
  },
996
841
  );
997
842
  if (rows.length === 0 && !(metadata.reportedCumulativeTokens > 0)) continue;
@@ -1005,6 +850,7 @@ function buildSnapshot(context, options) {
1005
850
  : "total-only";
1006
851
  threads.push({
1007
852
  id: threadId,
853
+ title: metadata.title,
1008
854
  project: metadata.project,
1009
855
  model: metadata.model,
1010
856
  effort: metadata.effort,
@@ -1021,6 +867,11 @@ function buildSnapshot(context, options) {
1021
867
  cachedInputTokens: totals.cachedInputTokens,
1022
868
  outputTokens: totals.outputTokens,
1023
869
  reasoningTokens: totals.reasoningTokens,
870
+ rateCardCredits:
871
+ totals.totalTokens > 0 && totals.ratedTokens === totals.totalTokens
872
+ ? totals.rateCardCredits
873
+ : null,
874
+ ratedTokens: totals.ratedTokens,
1024
875
  toolCalls: totals.toolCalls,
1025
876
  eventCount: rows.length,
1026
877
  coverage,
@@ -1068,7 +919,7 @@ function buildSnapshot(context, options) {
1068
919
  (quota) => quota.windowMinutes === WEEK_MINUTES,
1069
920
  );
1070
921
  const accountWideWeekly = weeklyCandidates.filter(
1071
- (quota) => quota.scope !== "named",
922
+ (quota) => !quota.limitName,
1072
923
  );
1073
924
  const weekly = [
1074
925
  ...(accountWideWeekly.length ? accountWideWeekly : weeklyCandidates),
@@ -1101,16 +952,13 @@ function buildSnapshot(context, options) {
1101
952
  label: "Local Codex snapshot",
1102
953
  provenance: {
1103
954
  kind: "codex-local-metadata",
1104
- sourceFingerprint: sourceFingerprint(
1105
- options.codexHome,
1106
- options.includeArchived,
1107
- ),
1108
955
  privacy:
1109
- "Contains token metadata and project labels only; display titles, credential fields, message bodies, reasoning text, tool payloads, and full local paths are not exported. Project labels can still be sensitive.",
956
+ "Contains token metadata and Codex display titles only; credential fields, message bodies, reasoning text, tool payloads, and full local paths are not exported. Display titles may contain user-written text.",
957
+ rateCardAsOf: RATE_CARD_AS_OF,
958
+ rateCardUrl: RATE_CARD_URL,
1110
959
  },
1111
960
  coverage: {
1112
961
  filesScanned: context.filesScanned,
1113
- sourceFileCount: context.sourceFileCount,
1114
962
  bytesScanned: context.bytesScanned,
1115
963
  parseErrors: context.parseErrors,
1116
964
  duplicateEventsSkipped: context.duplicateEventsSkipped,
@@ -1138,6 +986,9 @@ function buildSnapshot(context, options) {
1138
986
 
1139
987
  export async function collectUsage(options, onProgress = () => {}) {
1140
988
  const state = await readState(options.codexHome);
989
+ const titles = await readSessionTitles(
990
+ resolve(options.codexHome, "session_index.jsonl"),
991
+ );
1141
992
  const roots = [resolve(options.codexHome, "sessions")];
1142
993
  if (options.includeArchived) {
1143
994
  roots.push(resolve(options.codexHome, "archived_sessions"));
@@ -1148,12 +999,6 @@ export async function collectUsage(options, onProgress = () => {}) {
1148
999
  .flat()
1149
1000
  .sort();
1150
1001
  const sizes = await Promise.all(files.map((path) => stat(path)));
1151
- const jobs = files.map((path, index) => ({
1152
- index,
1153
- path,
1154
- size: sizes[index].size,
1155
- stateRow: state.rows.get(rolloutThreadId(path)) || null,
1156
- }));
1157
1002
 
1158
1003
  const context = {
1159
1004
  stateRows: state.rows,
@@ -1163,24 +1008,30 @@ export async function collectUsage(options, onProgress = () => {}) {
1163
1008
  quotas: new Map(),
1164
1009
  calls: new Map(),
1165
1010
  filesScanned: 0,
1166
- sourceFileCount: files.length + (state.path ? 1 : 0),
1167
1011
  bytesScanned: 0,
1168
1012
  parseErrors: 0,
1169
1013
  duplicateEventsSkipped: 0,
1170
1014
  correctionIntervals: 0,
1171
1015
  };
1172
1016
 
1173
- const workerCount = scanWorkerCount(files.length, options.workers);
1174
- const fragments = workerCount > 1
1175
- ? await scanRolloutsInParallel(jobs, workerCount, onProgress)
1176
- : await scanRolloutsSequential(jobs, onProgress);
1177
- for (const fragment of fragments) {
1178
- mergeScanFragment(context, fragment);
1017
+ for (let index = 0; index < files.length; index += 1) {
1018
+ await scanRollout(files[index], context);
1019
+ context.filesScanned += 1;
1020
+ context.bytesScanned += sizes[index].size;
1021
+ if (
1022
+ index === 0 ||
1023
+ index === files.length - 1 ||
1024
+ (index + 1) % 10 === 0
1025
+ ) {
1026
+ onProgress({
1027
+ current: index + 1,
1028
+ total: files.length,
1029
+ path: files[index],
1030
+ });
1031
+ }
1179
1032
  }
1180
- context.filesScanned = files.length;
1181
- context.bytesScanned = sizes.reduce((sum, entry) => sum + entry.size, 0);
1182
1033
 
1183
- return buildSnapshot(context, options);
1034
+ return buildSnapshot(context, options, titles);
1184
1035
  }
1185
1036
 
1186
1037
  async function main() {
@@ -1188,7 +1039,7 @@ async function main() {
1188
1039
  try {
1189
1040
  options = parseArgs(process.argv.slice(2));
1190
1041
  } catch (error) {
1191
- process.stderr.write(`${sanitizeLabel(error.message)}\n\n${usage()}\n`);
1042
+ process.stderr.write(`${error.message}\n\n${usage()}\n`);
1192
1043
  process.exitCode = 1;
1193
1044
  return;
1194
1045
  }
@@ -1197,9 +1048,7 @@ async function main() {
1197
1048
  return;
1198
1049
  }
1199
1050
  if (!(await pathExists(options.codexHome))) {
1200
- throw new Error(
1201
- `Codex data directory not found: ${sanitizeLabel(options.codexHome)}`,
1202
- );
1051
+ throw new Error(`Codex data directory not found: ${options.codexHome}`);
1203
1052
  }
1204
1053
 
1205
1054
  process.stdout.write("Token Ledger: scanning local Codex metadata…\n");
@@ -1213,7 +1062,7 @@ async function main() {
1213
1062
  const outputSize = (await stat(options.output)).size;
1214
1063
  process.stdout.write(
1215
1064
  [
1216
- `Snapshot: ${sanitizeLabel(options.output)}`,
1065
+ `Snapshot: ${options.output}`,
1217
1066
  `Observed model-call tokens: ${snapshot.coverage.observedTokens.toLocaleString()}`,
1218
1067
  `Unique events: ${snapshot.events.length.toLocaleString()}`,
1219
1068
  `Duplicate/copied events skipped: ${snapshot.coverage.duplicateEventsSkipped.toLocaleString()}`,
@@ -1224,13 +1073,11 @@ async function main() {
1224
1073
  }
1225
1074
 
1226
1075
  const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : "";
1227
- if (!isMainThread && workerData?.mode === SCAN_WORKER_MODE) {
1228
- runScanWorker();
1229
- } else if (isMainThread && import.meta.url === invokedPath) {
1076
+ if (import.meta.url === invokedPath) {
1230
1077
  main().catch((error) => {
1231
1078
  process.stderr.write(
1232
1079
  `Token Ledger collector failed: ${
1233
- error instanceof Error ? sanitizeLabel(error.message) : sanitizeLabel(error)
1080
+ error instanceof Error ? error.message : String(error)
1234
1081
  }\n`,
1235
1082
  );
1236
1083
  process.exitCode = 1;