tledger 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +283 -157
- package/bin/token-ledger-cache-data.mjs +492 -0
- package/bin/token-ledger-cache-image.mjs +7 -1147
- package/bin/token-ledger-cache-sections.mjs +848 -0
- package/bin/token-ledger-cost-terminal.mjs +234 -0
- package/bin/token-ledger-image-layout.mjs +20 -0
- package/bin/token-ledger-image-primitives.mjs +192 -0
- package/bin/token-ledger-report-data.mjs +1159 -0
- package/bin/token-ledger-source-status.mjs +31 -0
- package/bin/token-ledger-terminal.mjs +245 -69
- package/bin/token-ledger-trend-image.mjs +2025 -1724
- package/bin/token-ledger-trend-terminal.mjs +273 -150
- package/bin/token-ledger-trend.mjs +204 -198
- package/bin/token-ledger-tui.mjs +180 -51
- package/bin/token-ledger.mjs +735 -208
- package/docs/durable-ledger-operations.md +198 -0
- package/docs/release-notes-0.4.0.md +41 -0
- package/docs/token-ledger-report-7-day.png +0 -0
- package/lib/token-ledger-calendar.mjs +225 -0
- package/lib/token-ledger-collection.mjs +100 -0
- package/lib/token-ledger-importer.mjs +3329 -488
- package/lib/token-ledger-labels.mjs +66 -0
- package/lib/token-ledger-ledger.mjs +6056 -0
- package/lib/token-ledger-quota-contract.mjs +38 -0
- package/lib/token-ledger-range-analysis.mjs +120 -0
- package/lib/token-ledger-rates.mjs +330 -0
- package/lib/token-ledger-snapshot.mjs +336 -35
- package/lib/token-ledger-terminal-text.mjs +11 -0
- package/lib/token-ledger-usage.mjs +339 -33
- package/package.json +13 -10
- package/bin/token-ledger-rates.mjs +0 -65
package/bin/token-ledger.mjs
CHANGED
|
@@ -4,7 +4,6 @@ import { spawnSync } from "node:child_process";
|
|
|
4
4
|
import { existsSync, realpathSync } from "node:fs";
|
|
5
5
|
import {
|
|
6
6
|
mkdir,
|
|
7
|
-
stat,
|
|
8
7
|
} from "node:fs/promises";
|
|
9
8
|
import { homedir } from "node:os";
|
|
10
9
|
import { basename, dirname, resolve } from "node:path";
|
|
@@ -14,30 +13,67 @@ import {
|
|
|
14
13
|
MODEL_COLORS as TERMINAL_MODEL_COLORS,
|
|
15
14
|
renderTerminal,
|
|
16
15
|
} from "./token-ledger-terminal.mjs";
|
|
17
|
-
import { buildUsageTrend, multiDayBounds } from "./token-ledger-trend.mjs";
|
|
18
16
|
import {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
buildRangeAnalysis,
|
|
18
|
+
buildUsageTrend,
|
|
19
|
+
multiDayBounds,
|
|
20
|
+
priorPeriodBounds,
|
|
21
|
+
} from "./token-ledger-trend.mjs";
|
|
22
|
+
import {
|
|
23
|
+
resolveEffectiveEnd,
|
|
24
|
+
} from "./token-ledger-report-data.mjs";
|
|
23
25
|
import { renderTrendCombo } from "./token-ledger-trend-terminal.mjs";
|
|
26
|
+
import { renderCostTerminal } from "./token-ledger-cost-terminal.mjs";
|
|
24
27
|
import { startInteractive } from "./token-ledger-tui.mjs";
|
|
28
|
+
import {
|
|
29
|
+
snapshotFreshnessDetail,
|
|
30
|
+
sourceStatusLine,
|
|
31
|
+
} from "./token-ledger-source-status.mjs";
|
|
32
|
+
import {
|
|
33
|
+
createTimeZoneFormatter,
|
|
34
|
+
formatCalendarDate,
|
|
35
|
+
localDateBoundary,
|
|
36
|
+
shiftCalendarDate,
|
|
37
|
+
todayInTimeZone,
|
|
38
|
+
validateTimeZone,
|
|
39
|
+
} from "../lib/token-ledger-calendar.mjs";
|
|
25
40
|
import {
|
|
26
41
|
readPrivateSnapshot,
|
|
27
|
-
|
|
42
|
+
stagePrivateSnapshot,
|
|
28
43
|
} from "../lib/token-ledger-snapshot.mjs";
|
|
44
|
+
import {
|
|
45
|
+
collectionScope,
|
|
46
|
+
historyScopeLabel,
|
|
47
|
+
normalizeCollectionSince,
|
|
48
|
+
snapshotCollectionCutoffMs,
|
|
49
|
+
snapshotCollectionScope,
|
|
50
|
+
snapshotMatchesCollectionScope,
|
|
51
|
+
} from "../lib/token-ledger-collection.mjs";
|
|
29
52
|
import {
|
|
30
53
|
SNAPSHOT_SCHEMA_VERSION,
|
|
54
|
+
checkedFiniteAdd,
|
|
55
|
+
checkedTokenAdd,
|
|
56
|
+
scaledOutputTokens,
|
|
57
|
+
tokenValue,
|
|
58
|
+
MAX_SAFE_TOKEN_COUNT,
|
|
31
59
|
usageBuckets,
|
|
32
60
|
usageBucketsInRange,
|
|
33
61
|
usageCallCount,
|
|
34
62
|
usageThreadIds,
|
|
35
63
|
} from "../lib/token-ledger-usage.mjs";
|
|
64
|
+
import { calculateCodexPurchasedCredits } from "../lib/token-ledger-rates.mjs";
|
|
65
|
+
import { sanitizeTerminalText } from "../lib/token-ledger-terminal-text.mjs";
|
|
66
|
+
import {
|
|
67
|
+
QUOTA_IDENTITY_CONTRACT_VERSION,
|
|
68
|
+
snapshotHasCurrentQuotaIdentityContract,
|
|
69
|
+
} from "../lib/token-ledger-quota-contract.mjs";
|
|
70
|
+
|
|
71
|
+
export { sanitizeTerminalText };
|
|
36
72
|
|
|
37
73
|
export const DEFAULT_SNAPSHOT = resolve(
|
|
38
74
|
homedir(),
|
|
39
75
|
".token-ledger",
|
|
40
|
-
"token-ledger-snapshot-
|
|
76
|
+
"token-ledger-snapshot-v3.json.gz",
|
|
41
77
|
);
|
|
42
78
|
const DEFAULT_TOP = 10;
|
|
43
79
|
const DEFAULT_TIME_ZONE = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
@@ -47,6 +83,7 @@ const MAX_ROLLING_DAYS = 3_650;
|
|
|
47
83
|
const DURATION_ALIAS = /^(\d+)(d|w)$/i;
|
|
48
84
|
const ANSI_RESET = "\u001b[0m";
|
|
49
85
|
const MODEL_COLORS = {
|
|
86
|
+
astra: TERMINAL_MODEL_COLORS.astra,
|
|
50
87
|
sol: TERMINAL_MODEL_COLORS.sol,
|
|
51
88
|
luna: TERMINAL_MODEL_COLORS.luna,
|
|
52
89
|
terra: TERMINAL_MODEL_COLORS.terra,
|
|
@@ -55,6 +92,38 @@ const MODEL_COLORS = {
|
|
|
55
92
|
other: TERMINAL_MODEL_COLORS.other,
|
|
56
93
|
};
|
|
57
94
|
|
|
95
|
+
function createSharedTokenScale() {
|
|
96
|
+
return {
|
|
97
|
+
scale: 1,
|
|
98
|
+
totalTokens: 0,
|
|
99
|
+
targets: new Set(),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function addSharedTokenContribution(state, contribution, targets) {
|
|
104
|
+
const tokens = tokenValue(contribution, { allowFractional: true });
|
|
105
|
+
if (!(tokens > 0)) return;
|
|
106
|
+
const scaledTokens = tokens / state.scale;
|
|
107
|
+
for (const target of targets) {
|
|
108
|
+
state.targets.add(target);
|
|
109
|
+
target.totalTokens += scaledTokens;
|
|
110
|
+
}
|
|
111
|
+
state.totalTokens += scaledTokens;
|
|
112
|
+
const scaleFactor = Math.max(
|
|
113
|
+
1,
|
|
114
|
+
state.totalTokens / MAX_SAFE_TOKEN_COUNT,
|
|
115
|
+
);
|
|
116
|
+
if (scaleFactor === 1) return;
|
|
117
|
+
for (const target of state.targets) {
|
|
118
|
+
target.totalTokens /= scaleFactor;
|
|
119
|
+
if (target.totalTokens >= MAX_SAFE_TOKEN_COUNT - 2) {
|
|
120
|
+
target.totalTokens = MAX_SAFE_TOKEN_COUNT;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
state.totalTokens = MAX_SAFE_TOKEN_COUNT;
|
|
124
|
+
state.scale *= scaleFactor;
|
|
125
|
+
}
|
|
126
|
+
|
|
58
127
|
export function usage() {
|
|
59
128
|
return `Token Ledger
|
|
60
129
|
|
|
@@ -62,12 +131,16 @@ Usage:
|
|
|
62
131
|
tledger 1d Last 24 hours in the terminal
|
|
63
132
|
tledger week Last 7 calendar days in the terminal
|
|
64
133
|
tledger 30d Rolling 30 days in the terminal
|
|
134
|
+
tledger cost 7d --basis api-usd Hypothetical API-equivalent USD estimate
|
|
135
|
+
tledger cost week --basis codex-credits
|
|
136
|
+
Codex purchased-credit estimate
|
|
65
137
|
tledger report 7d Write the 7-day PNG report
|
|
66
138
|
tledger report 7d --cache-rate Write the cache-only PNG report
|
|
67
139
|
|
|
68
140
|
Common options:
|
|
69
141
|
--static Print once instead of opening the dashboard
|
|
70
142
|
--refresh Rebuild the local usage cache
|
|
143
|
+
--since <ISO timestamp> Collect history at or after this timestamp
|
|
71
144
|
--image-output <file> Choose where to save a PNG
|
|
72
145
|
--no-open Do not open a generated PNG
|
|
73
146
|
-h, --help Show this quick guide
|
|
@@ -87,6 +160,12 @@ Terminal commands:
|
|
|
87
160
|
tledger week [end-day] Seven local calendar days
|
|
88
161
|
tledger trend [Nd|Nw] Multi-day terminal trend
|
|
89
162
|
|
|
163
|
+
Cost commands (basis is required):
|
|
164
|
+
tledger cost <1d|Nd|Nw|week> --basis api-usd
|
|
165
|
+
Hypothetical API-equivalent USD estimate
|
|
166
|
+
tledger cost <1d|Nd|Nw|week> --basis codex-credits
|
|
167
|
+
Codex purchased-credit estimate
|
|
168
|
+
|
|
90
169
|
Report commands:
|
|
91
170
|
tledger report [Nd|Nw] Write the usage dashboard PNG
|
|
92
171
|
tledger report [Nd|Nw] --cache-rate
|
|
@@ -102,6 +181,7 @@ Data and refresh:
|
|
|
102
181
|
--refresh Rebuild the default snapshot from local Codex data
|
|
103
182
|
--no-refresh Use the cached snapshot without checking source files
|
|
104
183
|
--codex-home <dir> Codex data root used when refreshing
|
|
184
|
+
--since <ISO timestamp> Collect history at or after this timestamp
|
|
105
185
|
--no-archived Skip archived sessions when refreshing
|
|
106
186
|
|
|
107
187
|
Terminal output:
|
|
@@ -114,6 +194,7 @@ Terminal output:
|
|
|
114
194
|
--youplot Use the legacy single-series renderer
|
|
115
195
|
|
|
116
196
|
Report output:
|
|
197
|
+
--private Hide project names in the report
|
|
117
198
|
--drain Chart estimated meter drain instead of token volume
|
|
118
199
|
--cache-rate Write the cache-only report (report command only)
|
|
119
200
|
--image Write the trend view as a PNG
|
|
@@ -125,7 +206,8 @@ Help:
|
|
|
125
206
|
-h, --help Show the quick guide
|
|
126
207
|
--help-all Show this complete reference
|
|
127
208
|
|
|
128
|
-
The default snapshot is ~/.token-ledger/token-ledger-snapshot-
|
|
209
|
+
The default snapshot is ~/.token-ledger/token-ledger-snapshot-v3.json.gz and
|
|
210
|
+
the durable ledger is ~/.token-ledger/token-ledger-ledger.sqlite.
|
|
129
211
|
Token Ledger reads local Codex data only. It does not upload your usage.`;
|
|
130
212
|
}
|
|
131
213
|
|
|
@@ -172,10 +254,22 @@ function readOption(argv, index, name) {
|
|
|
172
254
|
|
|
173
255
|
export function parseArgs(argv) {
|
|
174
256
|
const helpCommand = argv[0] === "help";
|
|
257
|
+
const costCommand = argv[0] === "cost";
|
|
258
|
+
const costRangeValue = costCommand ? argv[1] : null;
|
|
259
|
+
const costHelpWithoutRange = costCommand &&
|
|
260
|
+
["--help", "-h", "--help-all"].includes(costRangeValue);
|
|
261
|
+
const costAlias = costCommand && costRangeValue !== "week" && !costHelpWithoutRange
|
|
262
|
+
? durationAlias(costRangeValue)
|
|
263
|
+
: null;
|
|
264
|
+
const costRangeValid = costRangeValue === "week" || Boolean(costAlias);
|
|
175
265
|
const alias = durationAlias(argv[0]);
|
|
176
|
-
const rolling24hCommand = argv[0] === "1d"
|
|
177
|
-
|
|
178
|
-
const
|
|
266
|
+
const rolling24hCommand = argv[0] === "1d" ||
|
|
267
|
+
(costCommand && costAlias?.days === 1);
|
|
268
|
+
const rollingDurationCommand = (Boolean(alias) && argv[0] !== "1d") ||
|
|
269
|
+
(costCommand && Boolean(costAlias) && costAlias.days !== 1);
|
|
270
|
+
const command = costCommand && costRangeValue === "week"
|
|
271
|
+
? "week"
|
|
272
|
+
: rolling24hCommand
|
|
179
273
|
? "rolling24h"
|
|
180
274
|
: rollingDurationCommand
|
|
181
275
|
? "rolling"
|
|
@@ -186,13 +280,14 @@ export function parseArgs(argv) {
|
|
|
186
280
|
: "day";
|
|
187
281
|
const options = {
|
|
188
282
|
range: command,
|
|
189
|
-
view: command === "trend" ? "trend" : "projects",
|
|
283
|
+
view: costCommand ? "cost" : command === "trend" ? "trend" : "projects",
|
|
190
284
|
rolling24h: rolling24hCommand,
|
|
191
285
|
rollingDuration: rollingDurationCommand,
|
|
192
|
-
rollingDays: alias?.days ?? (rolling24hCommand ? 1 : null),
|
|
193
|
-
rollingAmount: alias?.amount ?? (rolling24hCommand ? 1 : null),
|
|
194
|
-
rollingUnit: alias?.unit ?? (rolling24hCommand ? "d" : null),
|
|
195
|
-
rollingLabel: alias?.label ?? "1 day",
|
|
286
|
+
rollingDays: (costCommand ? costAlias?.days : alias?.days) ?? (rolling24hCommand ? 1 : null),
|
|
287
|
+
rollingAmount: (costCommand ? costAlias?.amount : alias?.amount) ?? (rolling24hCommand ? 1 : null),
|
|
288
|
+
rollingUnit: (costCommand ? costAlias?.unit : alias?.unit) ?? (rolling24hCommand ? "d" : null),
|
|
289
|
+
rollingLabel: (costCommand ? costAlias?.label : alias?.label) ?? "1 day",
|
|
290
|
+
basis: null,
|
|
196
291
|
report: argv[0] === "report",
|
|
197
292
|
trendDays: 7,
|
|
198
293
|
date: null,
|
|
@@ -201,6 +296,7 @@ export function parseArgs(argv) {
|
|
|
201
296
|
refresh: false,
|
|
202
297
|
autoRefresh: true,
|
|
203
298
|
codexHome: resolve(process.env.CODEX_HOME || `${homedir()}/.codex`),
|
|
299
|
+
since: null,
|
|
204
300
|
includeArchived: true,
|
|
205
301
|
timeZone: DEFAULT_TIME_ZONE,
|
|
206
302
|
top: DEFAULT_TOP,
|
|
@@ -208,20 +304,24 @@ export function parseArgs(argv) {
|
|
|
208
304
|
rawProjects: false,
|
|
209
305
|
plain: false,
|
|
210
306
|
ascii: false,
|
|
211
|
-
static:
|
|
307
|
+
static: costCommand,
|
|
212
308
|
image: false,
|
|
213
309
|
imageOutput: null,
|
|
214
310
|
imageWidth: null,
|
|
215
311
|
openImage: true,
|
|
216
312
|
drain: false,
|
|
217
313
|
cacheRate: false,
|
|
314
|
+
private: false,
|
|
218
315
|
legacyPlot: false,
|
|
219
316
|
help: argv.length === 0 || helpCommand,
|
|
220
317
|
helpAll: false,
|
|
221
318
|
};
|
|
222
319
|
|
|
223
320
|
let trendPeriodSeen = false;
|
|
224
|
-
let
|
|
321
|
+
let basisSeen = false;
|
|
322
|
+
let index = costCommand
|
|
323
|
+
? costHelpWithoutRange ? 1 : 2
|
|
324
|
+
: alias || ["day", "week", "trend", "report", "help"].includes(argv[0]) ? 1 : 0;
|
|
225
325
|
for (; index < argv.length; index += 1) {
|
|
226
326
|
const argument = argv[index];
|
|
227
327
|
if (argument === "--help" || argument === "-h") {
|
|
@@ -249,6 +349,18 @@ export function parseArgs(argv) {
|
|
|
249
349
|
options.trendDays = period.days;
|
|
250
350
|
trendPeriodSeen = true;
|
|
251
351
|
index += 1;
|
|
352
|
+
} else if (argument === "--basis") {
|
|
353
|
+
if (options.view !== "cost") {
|
|
354
|
+
throw new Error("--basis is only available with the cost command.");
|
|
355
|
+
}
|
|
356
|
+
if (basisSeen) throw new Error("Cost basis can only be specified once.");
|
|
357
|
+
const value = readOption(argv, index, "--basis");
|
|
358
|
+
if (value !== "codex-credits" && value !== "api-usd") {
|
|
359
|
+
throw new Error("--basis must be codex-credits or api-usd.");
|
|
360
|
+
}
|
|
361
|
+
options.basis = value;
|
|
362
|
+
basisSeen = true;
|
|
363
|
+
index += 1;
|
|
252
364
|
} else if (argument === "--input") {
|
|
253
365
|
options.input = resolve(readOption(argv, index, "--input"));
|
|
254
366
|
options.inputExplicit = true;
|
|
@@ -260,10 +372,17 @@ export function parseArgs(argv) {
|
|
|
260
372
|
} else if (argument === "--codex-home") {
|
|
261
373
|
options.codexHome = resolve(readOption(argv, index, "--codex-home"));
|
|
262
374
|
index += 1;
|
|
375
|
+
} else if (argument === "--since") {
|
|
376
|
+
const value = readOption(argv, index, "--since");
|
|
377
|
+
options.since = new Date(normalizeCollectionSince(value));
|
|
378
|
+
index += 1;
|
|
263
379
|
} else if (argument === "--tz") {
|
|
264
380
|
options.timeZone = readOption(argv, index, "--tz");
|
|
265
381
|
index += 1;
|
|
266
382
|
} else if (argument === "--top") {
|
|
383
|
+
if (options.view === "cost") {
|
|
384
|
+
throw new Error("--top is not available with the cost command.");
|
|
385
|
+
}
|
|
267
386
|
const value = Number(readOption(argv, index, "--top"));
|
|
268
387
|
if (!Number.isInteger(value) || value < 1 || value > 100) {
|
|
269
388
|
throw new Error("--top must be an integer from 1 to 100.");
|
|
@@ -271,6 +390,9 @@ export function parseArgs(argv) {
|
|
|
271
390
|
options.top = value;
|
|
272
391
|
index += 1;
|
|
273
392
|
} else if (argument === "--width") {
|
|
393
|
+
if (options.view === "cost") {
|
|
394
|
+
throw new Error("--width is not available with the cost command.");
|
|
395
|
+
}
|
|
274
396
|
const value = Number(readOption(argv, index, "--width"));
|
|
275
397
|
if (!Number.isInteger(value) || value < 40 || value > 200) {
|
|
276
398
|
throw new Error("--width must be an integer from 40 to 200.");
|
|
@@ -284,6 +406,9 @@ export function parseArgs(argv) {
|
|
|
284
406
|
} else if (argument === "--plain") {
|
|
285
407
|
options.plain = true;
|
|
286
408
|
} else if (argument === "--ascii") {
|
|
409
|
+
if (options.view === "cost") {
|
|
410
|
+
throw new Error("--ascii is not available with the cost command.");
|
|
411
|
+
}
|
|
287
412
|
options.ascii = true;
|
|
288
413
|
} else if (argument === "--static") {
|
|
289
414
|
options.static = true;
|
|
@@ -292,6 +417,11 @@ export function parseArgs(argv) {
|
|
|
292
417
|
throw new Error("--drain is only available for the trend view.");
|
|
293
418
|
}
|
|
294
419
|
options.drain = true;
|
|
420
|
+
} else if (argument === "--private") {
|
|
421
|
+
if (!options.report) {
|
|
422
|
+
throw new Error("--private is only available with the report command.");
|
|
423
|
+
}
|
|
424
|
+
options.private = true;
|
|
295
425
|
} else if (argument === "--cache-rate") {
|
|
296
426
|
if (!options.report) {
|
|
297
427
|
throw new Error("--cache-rate is only available with the report command.");
|
|
@@ -364,6 +494,12 @@ export function parseArgs(argv) {
|
|
|
364
494
|
}
|
|
365
495
|
|
|
366
496
|
if (options.report) options.image = true;
|
|
497
|
+
if (!options.help && costCommand && !costRangeValid) {
|
|
498
|
+
throw new Error("Cost range must be 1d, Nd, Nw, or week.");
|
|
499
|
+
}
|
|
500
|
+
if (!options.help && costCommand && !options.basis) {
|
|
501
|
+
throw new Error("The cost command requires --basis codex-credits or --basis api-usd.");
|
|
502
|
+
}
|
|
367
503
|
if (!options.help && (options.rolling24h || options.rollingDuration) && options.date) {
|
|
368
504
|
throw new Error(`${options.rollingLabel} does not accept --date; its rolling window ends now.`);
|
|
369
505
|
}
|
|
@@ -382,92 +518,24 @@ export function parseArgs(argv) {
|
|
|
382
518
|
if (!options.help && options.view === "trend" && options.legacyPlot) {
|
|
383
519
|
throw new Error("--youplot is only available for the project view.");
|
|
384
520
|
}
|
|
521
|
+
if (!options.help && options.view === "cost" && options.legacyPlot) {
|
|
522
|
+
throw new Error("--youplot is not available with the cost command.");
|
|
523
|
+
}
|
|
524
|
+
if (!options.help && options.view === "cost" && options.rawProjects) {
|
|
525
|
+
throw new Error("--raw-projects is not available with the cost command.");
|
|
526
|
+
}
|
|
385
527
|
if (!options.help && options.cacheRate && options.drain) {
|
|
386
528
|
throw new Error("--cache-rate cannot be combined with --drain.");
|
|
387
529
|
}
|
|
388
530
|
return options;
|
|
389
531
|
}
|
|
390
532
|
|
|
391
|
-
function numericDateParts(date) {
|
|
392
|
-
const parts = new Intl.DateTimeFormat("en-US", {
|
|
393
|
-
timeZone: date.timeZone,
|
|
394
|
-
year: "numeric",
|
|
395
|
-
month: "2-digit",
|
|
396
|
-
day: "2-digit",
|
|
397
|
-
}).formatToParts(date.value);
|
|
398
|
-
const values = Object.fromEntries(
|
|
399
|
-
parts
|
|
400
|
-
.filter((part) => part.type !== "literal")
|
|
401
|
-
.map((part) => [part.type, Number(part.value)]),
|
|
402
|
-
);
|
|
403
|
-
return values;
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
function dateStringFromParts(parts) {
|
|
407
|
-
return [parts.year, parts.month, parts.day]
|
|
408
|
-
.map((value, index) => (index === 0 ? String(value) : String(value).padStart(2, "0")))
|
|
409
|
-
.join("-");
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
function shiftCalendarDate(value, amount) {
|
|
413
|
-
const date = new Date(`${value}T00:00:00.000Z`);
|
|
414
|
-
date.setUTCDate(date.getUTCDate() + amount);
|
|
415
|
-
return date.toISOString().slice(0, 10);
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
function validateTimeZone(timeZone) {
|
|
419
|
-
try {
|
|
420
|
-
new Intl.DateTimeFormat("en-US", { timeZone }).format();
|
|
421
|
-
} catch {
|
|
422
|
-
throw new Error(`Unknown IANA timezone: ${timeZone}`);
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
function offsetAt(instant, timeZone) {
|
|
427
|
-
const parts = new Intl.DateTimeFormat("en-US", {
|
|
428
|
-
timeZone,
|
|
429
|
-
year: "numeric",
|
|
430
|
-
month: "2-digit",
|
|
431
|
-
day: "2-digit",
|
|
432
|
-
hour: "2-digit",
|
|
433
|
-
minute: "2-digit",
|
|
434
|
-
second: "2-digit",
|
|
435
|
-
hourCycle: "h23",
|
|
436
|
-
}).formatToParts(instant);
|
|
437
|
-
const values = Object.fromEntries(
|
|
438
|
-
parts
|
|
439
|
-
.filter((part) => part.type !== "literal")
|
|
440
|
-
.map((part) => [part.type, Number(part.value)]),
|
|
441
|
-
);
|
|
442
|
-
return (
|
|
443
|
-
Date.UTC(
|
|
444
|
-
values.year,
|
|
445
|
-
values.month - 1,
|
|
446
|
-
values.day,
|
|
447
|
-
values.hour,
|
|
448
|
-
values.minute,
|
|
449
|
-
values.second,
|
|
450
|
-
) - instant.getTime()
|
|
451
|
-
);
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
function zonedMidnight(dateString, timeZone) {
|
|
455
|
-
const [year, month, day] = dateString.split("-").map(Number);
|
|
456
|
-
const utcGuess = Date.UTC(year, month - 1, day);
|
|
457
|
-
let instant = new Date(utcGuess - offsetAt(new Date(utcGuess), timeZone));
|
|
458
|
-
const refinedOffset = offsetAt(instant, timeZone);
|
|
459
|
-
instant = new Date(utcGuess - refinedOffset);
|
|
460
|
-
return instant;
|
|
461
|
-
}
|
|
462
|
-
|
|
463
533
|
export function dayBounds(value, timeZone) {
|
|
464
534
|
validateTimeZone(timeZone);
|
|
535
|
+
const formatter = createTimeZoneFormatter(timeZone);
|
|
465
536
|
let dateString = value;
|
|
466
537
|
if (value === "today" || value === "yesterday") {
|
|
467
|
-
const today =
|
|
468
|
-
value: new Date(),
|
|
469
|
-
timeZone,
|
|
470
|
-
}));
|
|
538
|
+
const today = todayInTimeZone(timeZone, formatter);
|
|
471
539
|
dateString = value === "today" ? today : shiftCalendarDate(today, -1);
|
|
472
540
|
}
|
|
473
541
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(dateString)) {
|
|
@@ -483,8 +551,8 @@ export function dayBounds(value, timeZone) {
|
|
|
483
551
|
throw new Error(`Invalid calendar day: ${dateString}`);
|
|
484
552
|
}
|
|
485
553
|
const nextDateString = shiftCalendarDate(dateString, 1);
|
|
486
|
-
const start =
|
|
487
|
-
const end =
|
|
554
|
+
const start = localDateBoundary(dateString, timeZone, formatter);
|
|
555
|
+
const end = localDateBoundary(nextDateString, timeZone, formatter);
|
|
488
556
|
return { dateString, start, end, timeZone };
|
|
489
557
|
}
|
|
490
558
|
|
|
@@ -495,7 +563,7 @@ export function weekBounds(value, timeZone) {
|
|
|
495
563
|
...endDay,
|
|
496
564
|
startDateString,
|
|
497
565
|
endDateString: endDay.dateString,
|
|
498
|
-
start:
|
|
566
|
+
start: localDateBoundary(startDateString, timeZone),
|
|
499
567
|
rangeDays: 7,
|
|
500
568
|
};
|
|
501
569
|
}
|
|
@@ -529,13 +597,6 @@ export function rolling24hBounds(value = new Date(), timeZone = DEFAULT_TIME_ZON
|
|
|
529
597
|
return rollingDurationBounds(value, timeZone, 1);
|
|
530
598
|
}
|
|
531
599
|
|
|
532
|
-
export function sanitizeTerminalText(value) {
|
|
533
|
-
return String(value ?? "")
|
|
534
|
-
.replace(/\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g, "")
|
|
535
|
-
.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
536
|
-
.replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ");
|
|
537
|
-
}
|
|
538
|
-
|
|
539
600
|
function cleanLabel(value, fallback) {
|
|
540
601
|
const label = sanitizeTerminalText(value)
|
|
541
602
|
.replace(/\s+/g, " ")
|
|
@@ -548,6 +609,12 @@ const QUOTED_ABSOLUTE_PATH =
|
|
|
548
609
|
const UNQUOTED_ABSOLUTE_PATH =
|
|
549
610
|
/(^|[\s([{=])((?:\/(?!\/)|\/\/|\\\\|[A-Za-z]:[\\/])[^\s"'`)\]},;]+)/g;
|
|
550
611
|
|
|
612
|
+
const QUOTED_FILE_URL_PATH = /(["'])(file:\/\/\/[^"'\r\n]*)\1/gi;
|
|
613
|
+
const UNQUOTED_FILE_URL_PATH =
|
|
614
|
+
/(^|[\s([{=:])(file:\/\/\/[^\s"'`)\]},;]+)/gi;
|
|
615
|
+
const COLON_PREFIXED_LOCAL_PATH =
|
|
616
|
+
/(^|:)((?:\/(?!\/)|\\\\|[A-Za-z]:[\\/])[^\s"'`)\]},;]+)/g;
|
|
617
|
+
|
|
551
618
|
function isAbsoluteLocalPath(path) {
|
|
552
619
|
return (
|
|
553
620
|
path.startsWith("/") ||
|
|
@@ -580,6 +647,39 @@ export function redactLocalPaths(value, paths = []) {
|
|
|
580
647
|
.filter((path) => path && path !== "/")
|
|
581
648
|
.sort((left, right) => right.length - left.length);
|
|
582
649
|
|
|
650
|
+
const detectedPathLabel = (detectedPath) => {
|
|
651
|
+
const candidates = [detectedPath];
|
|
652
|
+
if (/^file:\/\/\//i.test(detectedPath)) {
|
|
653
|
+
const filePath = detectedPath.slice("file://".length);
|
|
654
|
+
candidates.push(filePath);
|
|
655
|
+
try {
|
|
656
|
+
candidates.push(decodeURIComponent(filePath));
|
|
657
|
+
} catch {
|
|
658
|
+
// Keep the raw file URL as the only candidate when its escape syntax
|
|
659
|
+
// is malformed; it is still safe to redact as an implicit path.
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
const explicit = [...explicitPaths].find((path) => candidates.includes(path));
|
|
663
|
+
return explicit ? safeDisplayLabel(explicit) : "[local path]";
|
|
664
|
+
};
|
|
665
|
+
|
|
666
|
+
// These forms occur in Node/worker diagnostics but are not covered by the
|
|
667
|
+
// ordinary absolute-path boundaries below. Handle them before replacing an
|
|
668
|
+
// explicit path substring so an explicit input still gets its safe filename.
|
|
669
|
+
redacted = redacted
|
|
670
|
+
.replace(
|
|
671
|
+
QUOTED_FILE_URL_PATH,
|
|
672
|
+
(_match, quote, path) => `${quote}${detectedPathLabel(path)}${quote}`,
|
|
673
|
+
)
|
|
674
|
+
.replace(
|
|
675
|
+
UNQUOTED_FILE_URL_PATH,
|
|
676
|
+
(_match, prefix, path) => `${prefix}${detectedPathLabel(path)}`,
|
|
677
|
+
)
|
|
678
|
+
.replace(
|
|
679
|
+
COLON_PREFIXED_LOCAL_PATH,
|
|
680
|
+
(_match, prefix, path) => `${prefix}${detectedPathLabel(path)}`,
|
|
681
|
+
);
|
|
682
|
+
|
|
583
683
|
for (const path of pathsToRedact) {
|
|
584
684
|
redacted = redacted.replaceAll(
|
|
585
685
|
path,
|
|
@@ -605,7 +705,7 @@ function displayLabel(value) {
|
|
|
605
705
|
return `${label.slice(0, 14)}…${label.slice(-13)}`;
|
|
606
706
|
}
|
|
607
707
|
|
|
608
|
-
export function oneOffProjects(snapshot) {
|
|
708
|
+
export function oneOffProjects(snapshot, events = null) {
|
|
609
709
|
const threadIdsByProject = new Map();
|
|
610
710
|
const add = (project, threadId) => {
|
|
611
711
|
if (!project || !threadId) return;
|
|
@@ -614,7 +714,7 @@ export function oneOffProjects(snapshot) {
|
|
|
614
714
|
ids.add(threadId);
|
|
615
715
|
threadIdsByProject.set(normalizedProject, ids);
|
|
616
716
|
};
|
|
617
|
-
for (const bucket of usageBuckets(snapshot)) {
|
|
717
|
+
for (const bucket of events ?? usageBuckets(snapshot)) {
|
|
618
718
|
for (const threadId of usageThreadIds(bucket)) add(bucket.project, threadId);
|
|
619
719
|
}
|
|
620
720
|
for (const thread of snapshot.threads ?? []) add(thread.project, thread.id);
|
|
@@ -628,6 +728,7 @@ export function oneOffProjects(snapshot) {
|
|
|
628
728
|
function modelLabel(value) {
|
|
629
729
|
const model = cleanLabel(value, "Unknown model");
|
|
630
730
|
const lower = model.toLowerCase();
|
|
731
|
+
if (lower.includes("astra")) return "Astra";
|
|
631
732
|
if (lower.includes("sol")) return "Sol";
|
|
632
733
|
if (lower.includes("luna")) return "Luna";
|
|
633
734
|
if (lower.includes("terra")) return "Terra";
|
|
@@ -636,17 +737,33 @@ function modelLabel(value) {
|
|
|
636
737
|
return model;
|
|
637
738
|
}
|
|
638
739
|
|
|
639
|
-
|
|
740
|
+
function currentRateCardCredits(event) {
|
|
741
|
+
return calculateCodexPurchasedCredits({
|
|
742
|
+
model: event?.rateCardModel ?? event?.model,
|
|
743
|
+
serviceTier: event?.serviceTier,
|
|
744
|
+
usage: event,
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
export function filterDayEvents(snapshot, bounds, analysis = null) {
|
|
749
|
+
if (analysis !== null) return analysis.currentEvents;
|
|
640
750
|
const start = bounds.start.getTime();
|
|
641
751
|
const end = bounds.end.getTime();
|
|
642
752
|
return usageBucketsInRange(snapshot, start, end);
|
|
643
753
|
}
|
|
644
754
|
|
|
645
|
-
export function aggregateProjects(snapshot, events, options = {}) {
|
|
646
|
-
const singletonProjects = options.rawProjects
|
|
755
|
+
export function aggregateProjects(snapshot, events, options = {}, analysis = null) {
|
|
756
|
+
const singletonProjects = options.rawProjects
|
|
757
|
+
? new Set()
|
|
758
|
+
: oneOffProjects(snapshot, analysis?.allEvents);
|
|
647
759
|
const grouped = new Map();
|
|
760
|
+
const sharedTokenScale = createSharedTokenScale();
|
|
648
761
|
|
|
649
762
|
for (const event of events) {
|
|
763
|
+
if (event?.invalidTokenRecord === true) continue;
|
|
764
|
+
const allowFractional = event?.rangeAllocationEstimated === true;
|
|
765
|
+
const totalTokens = tokenValue(event.totalTokens, { allowFractional });
|
|
766
|
+
const estimatedTokenContribution = allowFractional && totalTokens > 0;
|
|
650
767
|
const rawProject = cleanLabel(event.project, "Unlabelled activity");
|
|
651
768
|
const project =
|
|
652
769
|
!options.rawProjects && singletonProjects.has(rawProject)
|
|
@@ -665,16 +782,36 @@ export function aggregateProjects(snapshot, events, options = {}) {
|
|
|
665
782
|
rateCardCredits: 0,
|
|
666
783
|
knownCreditTokens: 0,
|
|
667
784
|
models: new Map(),
|
|
785
|
+
estimated: false,
|
|
668
786
|
};
|
|
669
|
-
row.
|
|
670
|
-
row.outputTokens
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
787
|
+
row.estimated ||= estimatedTokenContribution;
|
|
788
|
+
row.outputTokens = checkedTokenAdd(
|
|
789
|
+
row.outputTokens,
|
|
790
|
+
tokenValue(event.outputTokens, { allowFractional }),
|
|
791
|
+
{ allowFractional },
|
|
792
|
+
);
|
|
793
|
+
row.reasoningTokens = checkedTokenAdd(
|
|
794
|
+
row.reasoningTokens,
|
|
795
|
+
tokenValue(event.reasoningTokens, { allowFractional }),
|
|
796
|
+
{ allowFractional },
|
|
797
|
+
);
|
|
798
|
+
row.toolCalls = checkedTokenAdd(
|
|
799
|
+
row.toolCalls,
|
|
800
|
+
tokenValue(event.toolCalls, { allowFractional }),
|
|
801
|
+
{ allowFractional },
|
|
802
|
+
);
|
|
803
|
+
row.events = checkedTokenAdd(row.events, usageCallCount(event), {
|
|
804
|
+
allowFractional,
|
|
805
|
+
});
|
|
674
806
|
for (const threadId of usageThreadIds(event)) row.threadIds.add(threadId);
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
row.
|
|
807
|
+
const rateCardCredits = currentRateCardCredits(event);
|
|
808
|
+
if (Number.isFinite(rateCardCredits)) {
|
|
809
|
+
row.rateCardCredits = checkedFiniteAdd(row.rateCardCredits, rateCardCredits);
|
|
810
|
+
row.knownCreditTokens = checkedTokenAdd(
|
|
811
|
+
row.knownCreditTokens,
|
|
812
|
+
tokenValue(event.totalTokens, { allowFractional }),
|
|
813
|
+
{ allowFractional },
|
|
814
|
+
);
|
|
678
815
|
}
|
|
679
816
|
|
|
680
817
|
const model = modelLabel(event.model);
|
|
@@ -683,11 +820,22 @@ export function aggregateProjects(snapshot, events, options = {}) {
|
|
|
683
820
|
totalTokens: 0,
|
|
684
821
|
events: 0,
|
|
685
822
|
rateCardCredits: 0,
|
|
823
|
+
estimated: false,
|
|
686
824
|
};
|
|
687
|
-
modelRow.
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
825
|
+
modelRow.estimated ||= estimatedTokenContribution;
|
|
826
|
+
addSharedTokenContribution(
|
|
827
|
+
sharedTokenScale,
|
|
828
|
+
totalTokens,
|
|
829
|
+
[row, modelRow],
|
|
830
|
+
);
|
|
831
|
+
modelRow.events = checkedTokenAdd(modelRow.events, usageCallCount(event), {
|
|
832
|
+
allowFractional,
|
|
833
|
+
});
|
|
834
|
+
if (Number.isFinite(rateCardCredits)) {
|
|
835
|
+
modelRow.rateCardCredits = checkedFiniteAdd(
|
|
836
|
+
modelRow.rateCardCredits,
|
|
837
|
+
rateCardCredits,
|
|
838
|
+
);
|
|
691
839
|
}
|
|
692
840
|
row.models.set(model, modelRow);
|
|
693
841
|
grouped.set(project, row);
|
|
@@ -709,19 +857,33 @@ export function aggregateProjects(snapshot, events, options = {}) {
|
|
|
709
857
|
});
|
|
710
858
|
}
|
|
711
859
|
|
|
712
|
-
function totalSummary(events) {
|
|
713
|
-
|
|
860
|
+
function totalSummary(events, projectRows) {
|
|
861
|
+
const summary = events.reduce(
|
|
714
862
|
(summary, event) => {
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
summary.toolCalls
|
|
718
|
-
|
|
863
|
+
if (event?.invalidTokenRecord === true) return summary;
|
|
864
|
+
const allowFractional = event?.rangeAllocationEstimated === true;
|
|
865
|
+
summary.toolCalls = checkedTokenAdd(
|
|
866
|
+
summary.toolCalls,
|
|
867
|
+
tokenValue(event.toolCalls, { allowFractional }),
|
|
868
|
+
{ allowFractional },
|
|
869
|
+
);
|
|
870
|
+
summary.calls = checkedTokenAdd(summary.calls, usageCallCount(event), {
|
|
871
|
+
allowFractional,
|
|
872
|
+
});
|
|
719
873
|
for (const threadId of usageThreadIds(event)) {
|
|
720
874
|
summary.threadIds.add(threadId);
|
|
721
875
|
}
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
summary.
|
|
876
|
+
const rateCardCredits = currentRateCardCredits(event);
|
|
877
|
+
if (Number.isFinite(rateCardCredits)) {
|
|
878
|
+
summary.rateCardCredits = checkedFiniteAdd(
|
|
879
|
+
summary.rateCardCredits,
|
|
880
|
+
rateCardCredits,
|
|
881
|
+
);
|
|
882
|
+
summary.knownCreditTokens = checkedTokenAdd(
|
|
883
|
+
summary.knownCreditTokens,
|
|
884
|
+
tokenValue(event.totalTokens, { allowFractional }),
|
|
885
|
+
{ allowFractional },
|
|
886
|
+
);
|
|
725
887
|
}
|
|
726
888
|
return summary;
|
|
727
889
|
},
|
|
@@ -735,6 +897,12 @@ function totalSummary(events) {
|
|
|
735
897
|
threadIds: new Set(),
|
|
736
898
|
},
|
|
737
899
|
);
|
|
900
|
+
summary.totalTokens = projectRows.reduce(
|
|
901
|
+
(sum, row) => sum + row.totalTokens,
|
|
902
|
+
0,
|
|
903
|
+
);
|
|
904
|
+
summary.outputTokens = scaledOutputTokens(events, summary.totalTokens);
|
|
905
|
+
return summary;
|
|
738
906
|
}
|
|
739
907
|
|
|
740
908
|
function compact(value, digits = 2) {
|
|
@@ -786,6 +954,7 @@ function colorize(value, code, enabled) {
|
|
|
786
954
|
|
|
787
955
|
function modelColor(model) {
|
|
788
956
|
const lower = model.toLowerCase();
|
|
957
|
+
if (lower.includes("astra")) return MODEL_COLORS.astra;
|
|
789
958
|
if (lower.includes("sol")) return MODEL_COLORS.sol;
|
|
790
959
|
if (lower.includes("luna")) return MODEL_COLORS.luna;
|
|
791
960
|
if (lower.includes("terra")) return MODEL_COLORS.terra;
|
|
@@ -819,7 +988,7 @@ function runYouPlot(rows, options, dateLabel, unit) {
|
|
|
819
988
|
const chartInput = [
|
|
820
989
|
"project\tvalue",
|
|
821
990
|
...rows.map(
|
|
822
|
-
(row) => `${row.displayProject.replace(/[\t\r\n]+/g, " ")}\t${chartNumber(row.totalTokens, unit.divisor)}`,
|
|
991
|
+
(row) => `${sanitizeTerminalText(row.displayProject).replace(/[\t\r\n]+/g, " ")}\t${chartNumber(row.totalTokens, unit.divisor)}`,
|
|
823
992
|
),
|
|
824
993
|
].join("\n");
|
|
825
994
|
const terminalWidth = Number(process.stdout.columns) || 100;
|
|
@@ -881,6 +1050,77 @@ async function readSnapshot(snapshotPath) {
|
|
|
881
1050
|
return parsed;
|
|
882
1051
|
}
|
|
883
1052
|
|
|
1053
|
+
function snapshotScopeMatchesOptions(snapshot, options) {
|
|
1054
|
+
const requested = collectionScope(options);
|
|
1055
|
+
const actual = snapshotCollectionScope(snapshot);
|
|
1056
|
+
if (!actual) {
|
|
1057
|
+
// Explicit snapshots are deliberate inputs. Preserve the existing
|
|
1058
|
+
// fixture/export workflow when no collection filter was requested, while
|
|
1059
|
+
// refusing to claim that an unknown snapshot satisfies a filter.
|
|
1060
|
+
return requested.since === null && requested.includeArchived;
|
|
1061
|
+
}
|
|
1062
|
+
return snapshotMatchesCollectionScope(snapshot, requested);
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
function snapshotScopeError(snapshot, options) {
|
|
1066
|
+
const requested = collectionScope(options);
|
|
1067
|
+
const actual = snapshotCollectionScope(snapshot);
|
|
1068
|
+
const describe = (scope) => {
|
|
1069
|
+
if (!scope) return "unknown collection scope";
|
|
1070
|
+
const since = scope.since === null ? "full history" : `since ${scope.since}`;
|
|
1071
|
+
const archives = scope.includeArchived
|
|
1072
|
+
? "archived sessions included"
|
|
1073
|
+
: "archived sessions excluded";
|
|
1074
|
+
return `${since}; ${archives}`;
|
|
1075
|
+
};
|
|
1076
|
+
const remedy = options.inputExplicit
|
|
1077
|
+
? "For --input, supply matching --since/--no-archived filters or rebuild that file with the collector; --refresh cannot be combined with --input."
|
|
1078
|
+
: "Rebuild it with --refresh.";
|
|
1079
|
+
return new Error(
|
|
1080
|
+
`Snapshot collection scope does not match the requested filters (requested ${describe(requested)}; found ${describe(actual)}). ${remedy}`,
|
|
1081
|
+
);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
async function snapshotAllowsStaleFallback(snapshot, options) {
|
|
1085
|
+
if (!snapshotMatchesCollectionScope(snapshot, collectionScope(options))) {
|
|
1086
|
+
return false;
|
|
1087
|
+
}
|
|
1088
|
+
const { codexHomeFingerprint } = await import(
|
|
1089
|
+
"../lib/token-ledger-ledger.mjs"
|
|
1090
|
+
);
|
|
1091
|
+
return primitiveString(
|
|
1092
|
+
snapshot.metadata?.durableLedger?.codexHomeFingerprint,
|
|
1093
|
+
) === codexHomeFingerprint(options.codexHome);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
function snapshotWithUnavailableUntrustedQuota(snapshot, force = false) {
|
|
1097
|
+
if (!force && snapshotHasCurrentQuotaIdentityContract(snapshot)) return snapshot;
|
|
1098
|
+
return {
|
|
1099
|
+
...snapshot,
|
|
1100
|
+
coverage: {
|
|
1101
|
+
...snapshot.coverage,
|
|
1102
|
+
quotaMeterUnavailableReason: "quota-contract-unverified",
|
|
1103
|
+
},
|
|
1104
|
+
quotaObservations: [],
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
async function snapshotAndLedgerHaveCurrentQuotaContract(snapshot, options) {
|
|
1109
|
+
if (!snapshotHasCurrentQuotaIdentityContract(snapshot)) return false;
|
|
1110
|
+
const {
|
|
1111
|
+
readDurableLedgerCacheState,
|
|
1112
|
+
resolveDurableLedgerPath,
|
|
1113
|
+
} = await import("../lib/token-ledger-ledger.mjs");
|
|
1114
|
+
const ledgerState = await readDurableLedgerCacheState(
|
|
1115
|
+
resolveDurableLedgerPath({
|
|
1116
|
+
codexHome: options.codexHome,
|
|
1117
|
+
output: options.input,
|
|
1118
|
+
}),
|
|
1119
|
+
);
|
|
1120
|
+
return ledgerState?.quotaIdentityContract ===
|
|
1121
|
+
QUOTA_IDENTITY_CONTRACT_VERSION;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
884
1124
|
async function refreshSnapshot(options) {
|
|
885
1125
|
if (!existsSync(options.codexHome)) {
|
|
886
1126
|
throw new Error(
|
|
@@ -889,25 +1129,42 @@ async function refreshSnapshot(options) {
|
|
|
889
1129
|
}
|
|
890
1130
|
let progressStarted = false;
|
|
891
1131
|
try {
|
|
892
|
-
const {
|
|
1132
|
+
const {
|
|
1133
|
+
collectUsage,
|
|
1134
|
+
sourceInventory,
|
|
1135
|
+
sourceWatermarksEqual,
|
|
1136
|
+
} = await import(
|
|
893
1137
|
"../lib/token-ledger-importer.mjs"
|
|
894
1138
|
);
|
|
895
1139
|
process.stderr.write("Token Ledger: refreshing local snapshot…\n");
|
|
896
1140
|
progressStarted = true;
|
|
1141
|
+
let writeResult = null;
|
|
897
1142
|
const snapshot = await collectUsage(
|
|
898
1143
|
{
|
|
899
1144
|
output: options.input,
|
|
900
1145
|
codexHome: options.codexHome,
|
|
901
1146
|
includeArchived: options.includeArchived,
|
|
902
|
-
since:
|
|
1147
|
+
since: options.since,
|
|
1148
|
+
stageSnapshot: async (candidate) => {
|
|
1149
|
+
writeResult = await stagePrivateSnapshot(
|
|
1150
|
+
options.input,
|
|
1151
|
+
candidate,
|
|
1152
|
+
options.snapshotWriteOptions,
|
|
1153
|
+
);
|
|
1154
|
+
return writeResult;
|
|
1155
|
+
},
|
|
903
1156
|
},
|
|
904
1157
|
({ current, total }) => {
|
|
905
1158
|
process.stderr.write(`\rToken Ledger: scanned ${current}/${total} rollout files`);
|
|
906
1159
|
},
|
|
907
1160
|
);
|
|
908
1161
|
process.stderr.write("\n");
|
|
909
|
-
const
|
|
910
|
-
|
|
1162
|
+
const storedSnapshot = snapshot;
|
|
1163
|
+
if (storedSnapshot.coverage.filesReused > 0) {
|
|
1164
|
+
process.stderr.write(
|
|
1165
|
+
`Token Ledger: reused ${storedSnapshot.coverage.filesReused.toLocaleString()} unchanged rollout files from the durable ledger.\n`,
|
|
1166
|
+
);
|
|
1167
|
+
}
|
|
911
1168
|
process.stderr.write(
|
|
912
1169
|
`Token Ledger: cached ${(writeResult.bytesWritten / 1_000_000).toFixed(1)} MB ${writeResult.encoding} snapshot (${(writeResult.jsonBytes / 1_000_000).toFixed(1)} MB JSON before encoding; ${storedSnapshot.events.length.toLocaleString()} buckets for ${storedSnapshot.coverage.observedModelCalls.toLocaleString()} calls; ${(writeResult.maxBytes / 1_000_000).toFixed(1)} MB limit).\n`,
|
|
913
1170
|
);
|
|
@@ -916,23 +1173,163 @@ async function refreshSnapshot(options) {
|
|
|
916
1173
|
"Token Ledger: snapshot is above 70% of its safety limit; older buckets will compact automatically as it grows.\n",
|
|
917
1174
|
);
|
|
918
1175
|
}
|
|
919
|
-
|
|
1176
|
+
let sourceStatus = "unchecked-cache";
|
|
1177
|
+
try {
|
|
1178
|
+
const currentInventory = await sourceInventory(
|
|
1179
|
+
options.codexHome,
|
|
1180
|
+
options.includeArchived,
|
|
1181
|
+
);
|
|
1182
|
+
sourceStatus = refreshedSnapshotSourceStatus(
|
|
1183
|
+
storedSnapshot.sourceWatermark,
|
|
1184
|
+
currentInventory.watermark,
|
|
1185
|
+
sourceWatermarksEqual,
|
|
1186
|
+
);
|
|
1187
|
+
} catch (error) {
|
|
1188
|
+
if (!postRefreshStatusAllowsUnchecked(error)) throw error;
|
|
1189
|
+
}
|
|
1190
|
+
return {
|
|
1191
|
+
snapshot: storedSnapshot,
|
|
1192
|
+
sourceStatus,
|
|
1193
|
+
};
|
|
920
1194
|
} catch (error) {
|
|
921
1195
|
if (progressStarted) process.stderr.write("\n");
|
|
922
1196
|
if (error?.code === "ERR_SNAPSHOT_SIZE_LIMIT" && existsSync(options.input)) {
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1197
|
+
try {
|
|
1198
|
+
const previous = await readSnapshot(options.input);
|
|
1199
|
+
if (await snapshotAllowsStaleFallback(previous, options)) {
|
|
1200
|
+
const quotaContractTrusted =
|
|
1201
|
+
await snapshotAndLedgerHaveCurrentQuotaContract(
|
|
1202
|
+
previous,
|
|
1203
|
+
options,
|
|
1204
|
+
);
|
|
1205
|
+
process.stderr.write(
|
|
1206
|
+
"Token Ledger: refresh exceeded the safety limit; continuing with the previous cache, which may be stale.\n",
|
|
1207
|
+
);
|
|
1208
|
+
return {
|
|
1209
|
+
snapshot: snapshotWithUnavailableUntrustedQuota(
|
|
1210
|
+
previous,
|
|
1211
|
+
!quotaContractTrusted,
|
|
1212
|
+
),
|
|
1213
|
+
sourceStatus: "stale-fallback",
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
} catch {
|
|
1217
|
+
// Preserve the original refresh error when the previous cache cannot
|
|
1218
|
+
// prove its exact collection scope and Codex-home identity.
|
|
1219
|
+
}
|
|
927
1220
|
}
|
|
928
|
-
|
|
1221
|
+
const wrapped = new Error(
|
|
929
1222
|
`Could not refresh local snapshot: ${safeErrorMessage(error, [options.input, options.codexHome])}`,
|
|
1223
|
+
{ cause: error },
|
|
930
1224
|
);
|
|
1225
|
+
wrapped.code = error?.code;
|
|
1226
|
+
throw wrapped;
|
|
931
1227
|
}
|
|
932
1228
|
}
|
|
933
1229
|
|
|
934
|
-
export function
|
|
935
|
-
|
|
1230
|
+
export function refreshedSnapshotSourceStatus(
|
|
1231
|
+
snapshotWatermark,
|
|
1232
|
+
currentWatermark,
|
|
1233
|
+
watermarksEqual,
|
|
1234
|
+
) {
|
|
1235
|
+
return watermarksEqual(snapshotWatermark, currentWatermark)
|
|
1236
|
+
? "verified-current"
|
|
1237
|
+
: "unchecked-cache";
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
export function postRefreshStatusAllowsUnchecked(error) {
|
|
1241
|
+
return ["ENOENT", "ENOTDIR"].includes(primitiveString(error?.code));
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
const RECOVERABLE_REFRESH_ERROR_CODES = new Set([
|
|
1245
|
+
"ERR_SNAPSHOT_SIZE_LIMIT",
|
|
1246
|
+
"ERR_SOURCE_CHANGED_DURING_COLLECTION",
|
|
1247
|
+
"SQLITE_BUSY",
|
|
1248
|
+
"SQLITE_LOCKED",
|
|
1249
|
+
]);
|
|
1250
|
+
const RECOVERABLE_SQLITE_ERROR_CODES = new Set([5, 6]);
|
|
1251
|
+
const RECOVERABLE_SQLITE_ERROR_NAMES = new Set([
|
|
1252
|
+
"SQLITE_BUSY",
|
|
1253
|
+
"SQLITE_LOCKED",
|
|
1254
|
+
]);
|
|
1255
|
+
const HARD_SQLITE_ERROR_CODES = new Set([11, 26]);
|
|
1256
|
+
const HARD_SQLITE_ERROR_NAMES = new Set([
|
|
1257
|
+
"SQLITE_CORRUPT",
|
|
1258
|
+
"SQLITE_NOTADB",
|
|
1259
|
+
]);
|
|
1260
|
+
|
|
1261
|
+
export function refreshFailureAllowsStaleFallback(error) {
|
|
1262
|
+
const seen = new Set();
|
|
1263
|
+
let recoverable = false;
|
|
1264
|
+
let current = error;
|
|
1265
|
+
while (current instanceof Error) {
|
|
1266
|
+
if (seen.has(current)) return false;
|
|
1267
|
+
seen.add(current);
|
|
1268
|
+
const code = primitiveString(current.code);
|
|
1269
|
+
if (RECOVERABLE_REFRESH_ERROR_CODES.has(code)) recoverable = true;
|
|
1270
|
+
if (
|
|
1271
|
+
code?.startsWith("ERR_DURABLE_LEDGER_") ||
|
|
1272
|
+
(
|
|
1273
|
+
code?.startsWith("ERR_SNAPSHOT_") &&
|
|
1274
|
+
code !== "ERR_SNAPSHOT_SIZE_LIMIT"
|
|
1275
|
+
) ||
|
|
1276
|
+
code === "ERR_BUFFER_TOO_LARGE" ||
|
|
1277
|
+
HARD_SQLITE_ERROR_NAMES.has(code)
|
|
1278
|
+
) {
|
|
1279
|
+
return false;
|
|
1280
|
+
}
|
|
1281
|
+
if (code === "ERR_SQLITE_ERROR") {
|
|
1282
|
+
if (RECOVERABLE_SQLITE_ERROR_CODES.has(Number(current.errcode))) {
|
|
1283
|
+
recoverable = true;
|
|
1284
|
+
}
|
|
1285
|
+
if (HARD_SQLITE_ERROR_CODES.has(Number(current.errcode))) {
|
|
1286
|
+
return false;
|
|
1287
|
+
}
|
|
1288
|
+
const name = primitiveString(current.errstr)?.toUpperCase();
|
|
1289
|
+
if (RECOVERABLE_SQLITE_ERROR_NAMES.has(name)) recoverable = true;
|
|
1290
|
+
if (HARD_SQLITE_ERROR_NAMES.has(name)) return false;
|
|
1291
|
+
}
|
|
1292
|
+
current = current.cause;
|
|
1293
|
+
}
|
|
1294
|
+
return recoverable;
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
async function refreshSnapshotOrUseStaleFallback(
|
|
1298
|
+
options,
|
|
1299
|
+
cached,
|
|
1300
|
+
{ quotaContractTrusted = true } = {},
|
|
1301
|
+
) {
|
|
1302
|
+
try {
|
|
1303
|
+
const refreshed = await refreshSnapshot(options);
|
|
1304
|
+
if (
|
|
1305
|
+
refreshed.sourceStatus === "stale-fallback" &&
|
|
1306
|
+
!quotaContractTrusted
|
|
1307
|
+
) {
|
|
1308
|
+
return {
|
|
1309
|
+
...refreshed,
|
|
1310
|
+
snapshot: snapshotWithUnavailableUntrustedQuota(
|
|
1311
|
+
refreshed.snapshot,
|
|
1312
|
+
true,
|
|
1313
|
+
),
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
return refreshed;
|
|
1317
|
+
} catch (error) {
|
|
1318
|
+
// Default to surfacing refresh failures. Only bounded cache growth, source
|
|
1319
|
+
// races, and SQLite lock contention are safe reasons to reuse known-good
|
|
1320
|
+
// cached data; migration, schema, corruption, and unknown failures are not.
|
|
1321
|
+
if (
|
|
1322
|
+
!refreshFailureAllowsStaleFallback(error) ||
|
|
1323
|
+
!(await snapshotAllowsStaleFallback(cached, options))
|
|
1324
|
+
) throw error;
|
|
1325
|
+
return {
|
|
1326
|
+
snapshot: snapshotWithUnavailableUntrustedQuota(
|
|
1327
|
+
cached,
|
|
1328
|
+
!quotaContractTrusted,
|
|
1329
|
+
),
|
|
1330
|
+
sourceStatus: "stale-fallback",
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
936
1333
|
}
|
|
937
1334
|
|
|
938
1335
|
export function snapshotCacheIsFresh(
|
|
@@ -949,14 +1346,10 @@ export function snapshotCacheIsFresh(
|
|
|
949
1346
|
|
|
950
1347
|
export function shouldCheckSourceFreshness(
|
|
951
1348
|
options = {},
|
|
952
|
-
snapshotMtimeMs,
|
|
953
|
-
nowMs = Date.now(),
|
|
954
1349
|
) {
|
|
955
|
-
//
|
|
956
|
-
//
|
|
957
|
-
|
|
958
|
-
return Boolean(options.view === "trend" && options.image) ||
|
|
959
|
-
!snapshotCacheIsFresh(snapshotMtimeMs, nowMs);
|
|
1350
|
+
// The source manifest is cheap to stat and is the cache's validity anchor.
|
|
1351
|
+
// The full collector only runs when the persisted watermark changes.
|
|
1352
|
+
return options.autoRefresh !== false && options.inputExplicit !== true;
|
|
960
1353
|
}
|
|
961
1354
|
|
|
962
1355
|
function snapshotAgeLabel(ageMs) {
|
|
@@ -993,7 +1386,10 @@ export function snapshotFreshness(snapshot = {}, nowMs = Date.now()) {
|
|
|
993
1386
|
};
|
|
994
1387
|
}
|
|
995
1388
|
|
|
996
|
-
|
|
1389
|
+
// Resolve the snapshot together with the evidence available for its report
|
|
1390
|
+
// cutoff. A cache can be readable without being current, so callers must keep
|
|
1391
|
+
// this status separate from the age label shown in terminal output.
|
|
1392
|
+
export async function loadSnapshot(options) {
|
|
997
1393
|
if (options.refresh) {
|
|
998
1394
|
return refreshSnapshot(options);
|
|
999
1395
|
}
|
|
@@ -1005,26 +1401,31 @@ async function loadSnapshot(options) {
|
|
|
1005
1401
|
}
|
|
1006
1402
|
return refreshSnapshot(options);
|
|
1007
1403
|
}
|
|
1008
|
-
|
|
1009
|
-
|
|
1404
|
+
const cached = await readSnapshot(options.input);
|
|
1405
|
+
if (!snapshotScopeMatchesOptions(cached, options)) {
|
|
1406
|
+
if (options.inputExplicit || !options.autoRefresh) {
|
|
1407
|
+
throw snapshotScopeError(cached, options);
|
|
1408
|
+
}
|
|
1409
|
+
return refreshSnapshot(options);
|
|
1010
1410
|
}
|
|
1011
1411
|
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
snapshotStat = await stat(options.input);
|
|
1015
|
-
} catch (error) {
|
|
1016
|
-
throw new Error(
|
|
1017
|
-
`Could not inspect snapshot ${safeDisplayLabel(options.input, "snapshot")}: ${safeErrorMessage(error, [options.input])}`,
|
|
1018
|
-
);
|
|
1412
|
+
if (options.inputExplicit) {
|
|
1413
|
+
return { snapshot: cached, sourceStatus: "explicit-snapshot" };
|
|
1019
1414
|
}
|
|
1020
|
-
if (!
|
|
1021
|
-
return
|
|
1415
|
+
if (!options.autoRefresh) {
|
|
1416
|
+
return { snapshot: cached, sourceStatus: "unchecked-cache" };
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
if (!shouldCheckSourceFreshness(options)) {
|
|
1420
|
+
return { snapshot: cached, sourceStatus: "unchecked-cache" };
|
|
1022
1421
|
}
|
|
1023
1422
|
|
|
1024
|
-
const {
|
|
1025
|
-
|
|
1423
|
+
const { sourceInventory, sourceWatermarksEqual } = await import(
|
|
1424
|
+
"../lib/token-ledger-importer.mjs"
|
|
1425
|
+
);
|
|
1426
|
+
let inventory;
|
|
1026
1427
|
try {
|
|
1027
|
-
|
|
1428
|
+
inventory = await sourceInventory(
|
|
1028
1429
|
options.codexHome,
|
|
1029
1430
|
options.includeArchived,
|
|
1030
1431
|
);
|
|
@@ -1033,13 +1434,53 @@ async function loadSnapshot(options) {
|
|
|
1033
1434
|
`Could not inspect local Codex source: ${safeErrorMessage(error, [options.codexHome])}`,
|
|
1034
1435
|
);
|
|
1035
1436
|
}
|
|
1036
|
-
|
|
1437
|
+
const {
|
|
1438
|
+
codexHomeFingerprint,
|
|
1439
|
+
readDurableLedgerCacheState,
|
|
1440
|
+
resolveDurableLedgerPath,
|
|
1441
|
+
} = await import("../lib/token-ledger-ledger.mjs");
|
|
1442
|
+
const cachedCodexHomeFingerprint = primitiveString(
|
|
1443
|
+
cached.metadata?.durableLedger?.codexHomeFingerprint,
|
|
1444
|
+
);
|
|
1445
|
+
if (
|
|
1446
|
+
cachedCodexHomeFingerprint !== codexHomeFingerprint(options.codexHome)
|
|
1447
|
+
) {
|
|
1037
1448
|
return refreshSnapshot(options);
|
|
1038
1449
|
}
|
|
1039
|
-
|
|
1450
|
+
const ledgerState = await readDurableLedgerCacheState(
|
|
1451
|
+
resolveDurableLedgerPath({
|
|
1452
|
+
codexHome: options.codexHome,
|
|
1453
|
+
output: options.input,
|
|
1454
|
+
}),
|
|
1455
|
+
);
|
|
1456
|
+
const quotaContractTrusted =
|
|
1457
|
+
ledgerState?.quotaIdentityContract === QUOTA_IDENTITY_CONTRACT_VERSION &&
|
|
1458
|
+
snapshotHasCurrentQuotaIdentityContract(cached);
|
|
1459
|
+
if (!sourceWatermarksEqual(cached.sourceWatermark, inventory.watermark)) {
|
|
1460
|
+
return refreshSnapshotOrUseStaleFallback(options, cached, {
|
|
1461
|
+
quotaContractTrusted,
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
const snapshotRevision = Number(
|
|
1465
|
+
cached.metadata?.durableLedger?.revision,
|
|
1466
|
+
);
|
|
1467
|
+
if (
|
|
1468
|
+
!Number.isSafeInteger(snapshotRevision) ||
|
|
1469
|
+
ledgerState?.revision !== snapshotRevision
|
|
1470
|
+
) {
|
|
1471
|
+
return refreshSnapshotOrUseStaleFallback(options, cached, {
|
|
1472
|
+
quotaContractTrusted,
|
|
1473
|
+
});
|
|
1474
|
+
}
|
|
1475
|
+
if (!quotaContractTrusted) {
|
|
1476
|
+
return refreshSnapshotOrUseStaleFallback(options, cached, {
|
|
1477
|
+
quotaContractTrusted: false,
|
|
1478
|
+
});
|
|
1479
|
+
}
|
|
1480
|
+
return { snapshot: cached, sourceStatus: "verified-current" };
|
|
1040
1481
|
}
|
|
1041
1482
|
|
|
1042
|
-
function render(
|
|
1483
|
+
async function render(
|
|
1043
1484
|
options,
|
|
1044
1485
|
snapshot,
|
|
1045
1486
|
bounds,
|
|
@@ -1047,26 +1488,48 @@ function render(
|
|
|
1047
1488
|
rows,
|
|
1048
1489
|
allRows,
|
|
1049
1490
|
freshness,
|
|
1050
|
-
|
|
1491
|
+
report = {},
|
|
1492
|
+
analysis,
|
|
1051
1493
|
) {
|
|
1494
|
+
if (options.view === "cost") {
|
|
1495
|
+
return renderCostTerminal({
|
|
1496
|
+
events,
|
|
1497
|
+
bounds,
|
|
1498
|
+
basis: options.basis,
|
|
1499
|
+
snapshotFreshness: freshness,
|
|
1500
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1501
|
+
});
|
|
1502
|
+
}
|
|
1052
1503
|
if (options.view === "trend") {
|
|
1053
1504
|
if (options.image && options.cacheRate) {
|
|
1505
|
+
const { renderCacheReportImage } = await import(
|
|
1506
|
+
"./token-ledger-cache-image.mjs"
|
|
1507
|
+
);
|
|
1054
1508
|
return renderCacheReportImage({
|
|
1055
1509
|
snapshot,
|
|
1056
1510
|
bounds,
|
|
1057
1511
|
days: options.trendDays,
|
|
1058
1512
|
options,
|
|
1513
|
+
analysis,
|
|
1514
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1059
1515
|
});
|
|
1060
1516
|
}
|
|
1061
|
-
const trend = buildUsageTrend(snapshot, bounds);
|
|
1517
|
+
const trend = buildUsageTrend(snapshot, bounds, { analysis });
|
|
1062
1518
|
if (options.image) {
|
|
1519
|
+
const { renderTrendImage } = await import(
|
|
1520
|
+
"./token-ledger-trend-image.mjs"
|
|
1521
|
+
);
|
|
1063
1522
|
return renderTrendImage({
|
|
1064
1523
|
snapshot,
|
|
1065
1524
|
bounds,
|
|
1066
1525
|
trend,
|
|
1067
1526
|
days: options.trendDays,
|
|
1068
|
-
options
|
|
1527
|
+
options,
|
|
1069
1528
|
projectRows: allRows,
|
|
1529
|
+
reportTimeMs: report.reportTimeMs ?? null,
|
|
1530
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1531
|
+
analysis,
|
|
1532
|
+
reportEvents: events,
|
|
1070
1533
|
});
|
|
1071
1534
|
}
|
|
1072
1535
|
return renderTrendCombo({
|
|
@@ -1075,6 +1538,9 @@ function render(
|
|
|
1075
1538
|
trend,
|
|
1076
1539
|
days: options.trendDays,
|
|
1077
1540
|
options,
|
|
1541
|
+
analysis,
|
|
1542
|
+
snapshotFreshness: freshness,
|
|
1543
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1078
1544
|
});
|
|
1079
1545
|
}
|
|
1080
1546
|
if (!options.legacyPlot) {
|
|
@@ -1082,6 +1548,7 @@ function render(
|
|
|
1082
1548
|
options,
|
|
1083
1549
|
snapshot,
|
|
1084
1550
|
snapshotFreshness: freshness,
|
|
1551
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1085
1552
|
bounds,
|
|
1086
1553
|
events,
|
|
1087
1554
|
rows,
|
|
@@ -1089,35 +1556,38 @@ function render(
|
|
|
1089
1556
|
});
|
|
1090
1557
|
}
|
|
1091
1558
|
const enabled = !options.plain && !process.env.NO_COLOR && Boolean(process.stdout.isTTY);
|
|
1092
|
-
const summary = totalSummary(events);
|
|
1559
|
+
const summary = totalSummary(events, allRows);
|
|
1093
1560
|
const totalTokens = summary.totalTokens;
|
|
1094
1561
|
const dateLabel = options.range === "rolling24h"
|
|
1095
1562
|
? "last 24 hours"
|
|
1096
1563
|
: options.range === "rolling"
|
|
1097
1564
|
? `last ${options.rollingLabel}`
|
|
1098
1565
|
: options.range === "week"
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
}).format(bounds.start);
|
|
1566
|
+
? `${bounds.startDateString} through ${bounds.endDateString}`
|
|
1567
|
+
: formatCalendarDate(bounds.dateString, {
|
|
1568
|
+
weekday: "short",
|
|
1569
|
+
month: "short",
|
|
1570
|
+
day: "numeric",
|
|
1571
|
+
year: "numeric",
|
|
1572
|
+
});
|
|
1107
1573
|
const unit = chartUnit(rows[0]?.totalTokens ?? 0);
|
|
1108
1574
|
const shares = rows.map((row) =>
|
|
1109
1575
|
totalTokens > 0 ? (row.totalTokens / totalTokens) * 100 : 0,
|
|
1110
1576
|
);
|
|
1111
1577
|
const chart = runYouPlot(rows, options, dateLabel, unit).trimEnd();
|
|
1578
|
+
const historyScope = historyScopeLabel(snapshot);
|
|
1112
1579
|
|
|
1113
1580
|
const header = [
|
|
1114
1581
|
`Token Ledger · ${dateLabel} · ${bounds.timeZone}`,
|
|
1115
1582
|
`${compact(totalTokens)} tokens · ${summary.threadIds.size.toLocaleString()} threads · ${summary.calls.toLocaleString()} calls · ${compact(summary.outputTokens)} output`,
|
|
1583
|
+
...(historyScope ? [`History: ${historyScope}`] : []),
|
|
1584
|
+
`Snapshot: ${snapshotFreshnessDetail(freshness)}`,
|
|
1585
|
+
sourceStatusLine(report.sourceStatus ?? "unchecked-cache"),
|
|
1116
1586
|
`Source: ${sourceLabel(options.input, snapshot)}`,
|
|
1117
1587
|
"",
|
|
1118
1588
|
chart,
|
|
1119
1589
|
"",
|
|
1120
|
-
`Model mix · colors: ${colorize("Sol", MODEL_COLORS.sol, enabled)} ${colorize("Luna", MODEL_COLORS.luna, enabled)} ${colorize("Terra", MODEL_COLORS.terra, enabled)} ${colorize("GPT", MODEL_COLORS["gpt-5.5"], enabled)} ${colorize("Other", MODEL_COLORS.other, enabled)}`,
|
|
1590
|
+
`Model mix · colors: ${colorize("Astra", MODEL_COLORS.astra, enabled)} ${colorize("Sol", MODEL_COLORS.sol, enabled)} ${colorize("Luna", MODEL_COLORS.luna, enabled)} ${colorize("Terra", MODEL_COLORS.terra, enabled)} ${colorize("GPT", MODEL_COLORS["gpt-5.5"], enabled)} ${colorize("Other", MODEL_COLORS.other, enabled)}`,
|
|
1121
1591
|
];
|
|
1122
1592
|
|
|
1123
1593
|
const details = rows.map((row, index) => {
|
|
@@ -1125,7 +1595,7 @@ function render(
|
|
|
1125
1595
|
summary.rateCardCredits > 0 && row.rateCardCredits > 0
|
|
1126
1596
|
? ` · ${percent((row.rateCardCredits / summary.rateCardCredits) * 100)} credits`
|
|
1127
1597
|
: "";
|
|
1128
|
-
return `${String(index + 1).padStart(2, " ")} ${row.displayProject} · ${compact(row.totalTokens)} · ${percent(shares[index])} · ${row.threads.toLocaleString()} threads${knownCreditShare}\n ${modelMix(row, enabled)}`;
|
|
1598
|
+
return `${String(index + 1).padStart(2, " ")} ${sanitizeTerminalText(row.displayProject)} · ${compact(row.totalTokens)} · ${percent(shares[index])} · ${row.threads.toLocaleString()} threads${knownCreditShare}\n ${modelMix(row, enabled)}`;
|
|
1129
1599
|
});
|
|
1130
1600
|
|
|
1131
1601
|
return `${header.join("\n")}\n\n${details.join("\n")}`;
|
|
@@ -1157,29 +1627,85 @@ function rangeDescription(options, bounds) {
|
|
|
1157
1627
|
return bounds.dateString;
|
|
1158
1628
|
}
|
|
1159
1629
|
|
|
1630
|
+
function emptyRangeMessage(
|
|
1631
|
+
options,
|
|
1632
|
+
snapshot,
|
|
1633
|
+
bounds,
|
|
1634
|
+
{
|
|
1635
|
+
sourceStatus = "unchecked-cache",
|
|
1636
|
+
reportTimeMs = Date.now(),
|
|
1637
|
+
} = {},
|
|
1638
|
+
) {
|
|
1639
|
+
const range = rangeDescription(options, bounds);
|
|
1640
|
+
const cutoffMs = snapshotCollectionCutoffMs(snapshot);
|
|
1641
|
+
const hasUncollectedHistory =
|
|
1642
|
+
Number.isFinite(cutoffMs) && bounds.start.getTime() < cutoffMs;
|
|
1643
|
+
const lines = [
|
|
1644
|
+
hasUncollectedHistory
|
|
1645
|
+
? `No model-call events were collected for ${range} (${bounds.timeZone}); history before the snapshot cutoff is outside this snapshot and is not a verified zero.`
|
|
1646
|
+
: `No model-call events found for ${range} (${bounds.timeZone}).`,
|
|
1647
|
+
];
|
|
1648
|
+
const scope = historyScopeLabel(snapshot);
|
|
1649
|
+
if (scope) lines.push(`History: ${scope}`);
|
|
1650
|
+
lines.push(
|
|
1651
|
+
`Snapshot: ${snapshotFreshnessDetail(snapshotFreshness(snapshot, reportTimeMs))}`,
|
|
1652
|
+
);
|
|
1653
|
+
lines.push(sourceStatusLine(sourceStatus));
|
|
1654
|
+
lines.push(`Source: ${sourceLabel(options.input, snapshot)}`);
|
|
1655
|
+
return lines.join("\n");
|
|
1656
|
+
}
|
|
1657
|
+
|
|
1160
1658
|
export async function run(options, { nowMs } = {}) {
|
|
1161
1659
|
const hasInjectedNow = nowMs !== undefined;
|
|
1162
1660
|
const now = new Date(hasInjectedNow ? nowMs : Date.now());
|
|
1661
|
+
const reportTimeMs = now.getTime();
|
|
1163
1662
|
const bounds = boundsForOptions(options, now);
|
|
1164
|
-
const snapshot = await loadSnapshot(options);
|
|
1165
|
-
const
|
|
1663
|
+
const { snapshot, sourceStatus } = await loadSnapshot(options);
|
|
1664
|
+
const freshnessNowMs = hasInjectedNow ? reportTimeMs : Date.now();
|
|
1665
|
+
const analysis = buildRangeAnalysis(
|
|
1666
|
+
snapshot,
|
|
1667
|
+
bounds,
|
|
1668
|
+
{
|
|
1669
|
+
priorBounds: options.view === "trend"
|
|
1670
|
+
? priorPeriodBounds(bounds, options.trendDays)
|
|
1671
|
+
: null,
|
|
1672
|
+
includeTrend: options.view === "trend" && !options.cacheRate,
|
|
1673
|
+
},
|
|
1674
|
+
);
|
|
1675
|
+
let events = filterDayEvents(snapshot, bounds, analysis);
|
|
1166
1676
|
const writingImage = options.view === "trend" && options.image;
|
|
1167
1677
|
const writingEmptyCacheReport = writingImage && options.cacheRate;
|
|
1678
|
+
if (writingImage && !options.cacheRate) {
|
|
1679
|
+
const effectiveEndMs = resolveEffectiveEnd({
|
|
1680
|
+
snapshot,
|
|
1681
|
+
bounds,
|
|
1682
|
+
reportTimeMs,
|
|
1683
|
+
sourceStatus,
|
|
1684
|
+
});
|
|
1685
|
+
// Keep the terminal aggregation and the report's project breakdown on the
|
|
1686
|
+
// same captured event window. The report view model independently applies
|
|
1687
|
+
// the same bound to raw snapshot events for its other panels.
|
|
1688
|
+
events = usageBucketsInRange(
|
|
1689
|
+
{ events },
|
|
1690
|
+
bounds.start.getTime(),
|
|
1691
|
+
effectiveEndMs,
|
|
1692
|
+
);
|
|
1693
|
+
}
|
|
1168
1694
|
if (events.length === 0 && !writingEmptyCacheReport) {
|
|
1169
|
-
return
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1695
|
+
return emptyRangeMessage(options, snapshot, bounds, {
|
|
1696
|
+
sourceStatus,
|
|
1697
|
+
reportTimeMs,
|
|
1698
|
+
});
|
|
1173
1699
|
}
|
|
1174
|
-
const allRows = options.cacheRate
|
|
1700
|
+
const allRows = options.cacheRate || options.view === "cost"
|
|
1175
1701
|
? []
|
|
1176
|
-
: aggregateProjects(snapshot, events, options);
|
|
1702
|
+
: aggregateProjects(snapshot, events, options, analysis);
|
|
1177
1703
|
const rows = allRows.slice(0, options.top);
|
|
1178
1704
|
const outputPath = writingImage
|
|
1179
1705
|
? options.imageOutput ??
|
|
1180
1706
|
resolve(
|
|
1181
1707
|
process.cwd(),
|
|
1182
|
-
`token-ledger-${options.cacheRate ? "cache-report" : options.report ? "report" : "trend"}-${options.trendDays}d.png`,
|
|
1708
|
+
`token-ledger-${options.cacheRate ? "cache-report" : options.report ? "report" : "trend"}-${options.trendDays}d${options.private ? "-private" : ""}.png`,
|
|
1183
1709
|
)
|
|
1184
1710
|
: null;
|
|
1185
1711
|
const imageLabel = options.cacheRate
|
|
@@ -1190,11 +1716,7 @@ export async function run(options, { nowMs } = {}) {
|
|
|
1190
1716
|
if (writingImage) {
|
|
1191
1717
|
process.stderr.write(`Token Ledger: generating ${imageLabel} PNG…\n`);
|
|
1192
1718
|
}
|
|
1193
|
-
const
|
|
1194
|
-
const verifiedSourceTimeMs = options.autoRefresh && !options.inputExplicit
|
|
1195
|
-
? reportTimeMs
|
|
1196
|
-
: undefined;
|
|
1197
|
-
const output = render(
|
|
1719
|
+
const output = await render(
|
|
1198
1720
|
options,
|
|
1199
1721
|
snapshot,
|
|
1200
1722
|
bounds,
|
|
@@ -1203,13 +1725,15 @@ export async function run(options, { nowMs } = {}) {
|
|
|
1203
1725
|
allRows,
|
|
1204
1726
|
snapshotFreshness(
|
|
1205
1727
|
snapshot,
|
|
1206
|
-
|
|
1728
|
+
freshnessNowMs,
|
|
1207
1729
|
),
|
|
1208
|
-
|
|
1730
|
+
{ sourceStatus, reportTimeMs },
|
|
1731
|
+
analysis,
|
|
1209
1732
|
);
|
|
1210
1733
|
if (writingImage) {
|
|
1211
1734
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
1212
1735
|
process.stderr.write(`Token Ledger: encoding ${imageLabel} PNG…\n`);
|
|
1736
|
+
const { writeTrendPng } = await import("./token-ledger-trend-image.mjs");
|
|
1213
1737
|
await writeTrendPng(output, outputPath);
|
|
1214
1738
|
process.stderr.write(`Token Ledger: finished ${imageLabel} PNG.\n`);
|
|
1215
1739
|
const lines = [
|
|
@@ -1246,6 +1770,7 @@ function shouldUseInteractive(options) {
|
|
|
1246
1770
|
return Boolean(
|
|
1247
1771
|
!options.static &&
|
|
1248
1772
|
options.view !== "trend" &&
|
|
1773
|
+
options.view !== "cost" &&
|
|
1249
1774
|
!options.plain &&
|
|
1250
1775
|
!options.legacyPlot &&
|
|
1251
1776
|
!process.env.NO_COLOR &&
|
|
@@ -1256,21 +1781,23 @@ function shouldUseInteractive(options) {
|
|
|
1256
1781
|
|
|
1257
1782
|
async function runInteractive(options) {
|
|
1258
1783
|
const bounds = boundsForOptions(options);
|
|
1259
|
-
const snapshot = await loadSnapshot(options);
|
|
1260
|
-
const
|
|
1784
|
+
const { snapshot, sourceStatus } = await loadSnapshot(options);
|
|
1785
|
+
const reportTimeMs = Date.now();
|
|
1786
|
+
const analysis = buildRangeAnalysis(snapshot, bounds, { includeTrend: false });
|
|
1787
|
+
const events = filterDayEvents(snapshot, bounds, analysis);
|
|
1261
1788
|
if (events.length === 0) {
|
|
1262
|
-
process.stdout.write(
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
].join("\n"));
|
|
1789
|
+
process.stdout.write(`${emptyRangeMessage(options, snapshot, bounds, {
|
|
1790
|
+
sourceStatus,
|
|
1791
|
+
reportTimeMs,
|
|
1792
|
+
})}\n`);
|
|
1267
1793
|
return;
|
|
1268
1794
|
}
|
|
1269
|
-
const allRows = aggregateProjects(snapshot, events, options);
|
|
1795
|
+
const allRows = aggregateProjects(snapshot, events, options, analysis);
|
|
1270
1796
|
await startInteractive({
|
|
1271
1797
|
options,
|
|
1272
1798
|
snapshot,
|
|
1273
|
-
snapshotFreshness: snapshotFreshness(snapshot),
|
|
1799
|
+
snapshotFreshness: snapshotFreshness(snapshot, reportTimeMs),
|
|
1800
|
+
sourceStatus,
|
|
1274
1801
|
bounds,
|
|
1275
1802
|
events,
|
|
1276
1803
|
rows: allRows.slice(0, options.top),
|