tledger 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +283 -155
- package/bin/token-ledger-cache-data.mjs +492 -0
- package/bin/token-ledger-cache-image.mjs +7 -1147
- package/bin/token-ledger-cache-sections.mjs +848 -0
- package/bin/token-ledger-cost-terminal.mjs +234 -0
- package/bin/token-ledger-image-layout.mjs +20 -0
- package/bin/token-ledger-image-primitives.mjs +192 -0
- package/bin/token-ledger-report-data.mjs +1159 -0
- package/bin/token-ledger-source-status.mjs +31 -0
- package/bin/token-ledger-terminal.mjs +245 -69
- package/bin/token-ledger-trend-image.mjs +2039 -1599
- package/bin/token-ledger-trend-terminal.mjs +273 -150
- package/bin/token-ledger-trend.mjs +262 -210
- package/bin/token-ledger-tui.mjs +180 -51
- package/bin/token-ledger.mjs +747 -194
- package/docs/durable-ledger-operations.md +198 -0
- package/docs/release-notes-0.4.0.md +41 -0
- package/docs/token-ledger-report-7-day.png +0 -0
- package/lib/token-ledger-calendar.mjs +225 -0
- package/lib/token-ledger-collection.mjs +100 -0
- package/lib/token-ledger-importer.mjs +3334 -480
- package/lib/token-ledger-labels.mjs +66 -0
- package/lib/token-ledger-ledger.mjs +6056 -0
- package/lib/token-ledger-quota-contract.mjs +38 -0
- package/lib/token-ledger-range-analysis.mjs +120 -0
- package/lib/token-ledger-rates.mjs +330 -0
- package/lib/token-ledger-snapshot.mjs +336 -35
- package/lib/token-ledger-terminal-text.mjs +11 -0
- package/lib/token-ledger-usage.mjs +339 -33
- package/package.json +13 -10
- package/bin/token-ledger-rates.mjs +0 -65
package/bin/token-ledger.mjs
CHANGED
|
@@ -4,7 +4,6 @@ import { spawnSync } from "node:child_process";
|
|
|
4
4
|
import { existsSync, realpathSync } from "node:fs";
|
|
5
5
|
import {
|
|
6
6
|
mkdir,
|
|
7
|
-
stat,
|
|
8
7
|
} from "node:fs/promises";
|
|
9
8
|
import { homedir } from "node:os";
|
|
10
9
|
import { basename, dirname, resolve } from "node:path";
|
|
@@ -14,30 +13,67 @@ import {
|
|
|
14
13
|
MODEL_COLORS as TERMINAL_MODEL_COLORS,
|
|
15
14
|
renderTerminal,
|
|
16
15
|
} from "./token-ledger-terminal.mjs";
|
|
17
|
-
import { buildUsageTrend, multiDayBounds } from "./token-ledger-trend.mjs";
|
|
18
16
|
import {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
buildRangeAnalysis,
|
|
18
|
+
buildUsageTrend,
|
|
19
|
+
multiDayBounds,
|
|
20
|
+
priorPeriodBounds,
|
|
21
|
+
} from "./token-ledger-trend.mjs";
|
|
22
|
+
import {
|
|
23
|
+
resolveEffectiveEnd,
|
|
24
|
+
} from "./token-ledger-report-data.mjs";
|
|
23
25
|
import { renderTrendCombo } from "./token-ledger-trend-terminal.mjs";
|
|
26
|
+
import { renderCostTerminal } from "./token-ledger-cost-terminal.mjs";
|
|
24
27
|
import { startInteractive } from "./token-ledger-tui.mjs";
|
|
28
|
+
import {
|
|
29
|
+
snapshotFreshnessDetail,
|
|
30
|
+
sourceStatusLine,
|
|
31
|
+
} from "./token-ledger-source-status.mjs";
|
|
32
|
+
import {
|
|
33
|
+
createTimeZoneFormatter,
|
|
34
|
+
formatCalendarDate,
|
|
35
|
+
localDateBoundary,
|
|
36
|
+
shiftCalendarDate,
|
|
37
|
+
todayInTimeZone,
|
|
38
|
+
validateTimeZone,
|
|
39
|
+
} from "../lib/token-ledger-calendar.mjs";
|
|
25
40
|
import {
|
|
26
41
|
readPrivateSnapshot,
|
|
27
|
-
|
|
42
|
+
stagePrivateSnapshot,
|
|
28
43
|
} from "../lib/token-ledger-snapshot.mjs";
|
|
44
|
+
import {
|
|
45
|
+
collectionScope,
|
|
46
|
+
historyScopeLabel,
|
|
47
|
+
normalizeCollectionSince,
|
|
48
|
+
snapshotCollectionCutoffMs,
|
|
49
|
+
snapshotCollectionScope,
|
|
50
|
+
snapshotMatchesCollectionScope,
|
|
51
|
+
} from "../lib/token-ledger-collection.mjs";
|
|
29
52
|
import {
|
|
30
53
|
SNAPSHOT_SCHEMA_VERSION,
|
|
54
|
+
checkedFiniteAdd,
|
|
55
|
+
checkedTokenAdd,
|
|
56
|
+
scaledOutputTokens,
|
|
57
|
+
tokenValue,
|
|
58
|
+
MAX_SAFE_TOKEN_COUNT,
|
|
31
59
|
usageBuckets,
|
|
32
60
|
usageBucketsInRange,
|
|
33
61
|
usageCallCount,
|
|
34
62
|
usageThreadIds,
|
|
35
63
|
} from "../lib/token-ledger-usage.mjs";
|
|
64
|
+
import { calculateCodexPurchasedCredits } from "../lib/token-ledger-rates.mjs";
|
|
65
|
+
import { sanitizeTerminalText } from "../lib/token-ledger-terminal-text.mjs";
|
|
66
|
+
import {
|
|
67
|
+
QUOTA_IDENTITY_CONTRACT_VERSION,
|
|
68
|
+
snapshotHasCurrentQuotaIdentityContract,
|
|
69
|
+
} from "../lib/token-ledger-quota-contract.mjs";
|
|
70
|
+
|
|
71
|
+
export { sanitizeTerminalText };
|
|
36
72
|
|
|
37
73
|
export const DEFAULT_SNAPSHOT = resolve(
|
|
38
74
|
homedir(),
|
|
39
75
|
".token-ledger",
|
|
40
|
-
"token-ledger-snapshot-
|
|
76
|
+
"token-ledger-snapshot-v3.json.gz",
|
|
41
77
|
);
|
|
42
78
|
const DEFAULT_TOP = 10;
|
|
43
79
|
const DEFAULT_TIME_ZONE = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
@@ -47,6 +83,7 @@ const MAX_ROLLING_DAYS = 3_650;
|
|
|
47
83
|
const DURATION_ALIAS = /^(\d+)(d|w)$/i;
|
|
48
84
|
const ANSI_RESET = "\u001b[0m";
|
|
49
85
|
const MODEL_COLORS = {
|
|
86
|
+
astra: TERMINAL_MODEL_COLORS.astra,
|
|
50
87
|
sol: TERMINAL_MODEL_COLORS.sol,
|
|
51
88
|
luna: TERMINAL_MODEL_COLORS.luna,
|
|
52
89
|
terra: TERMINAL_MODEL_COLORS.terra,
|
|
@@ -55,6 +92,38 @@ const MODEL_COLORS = {
|
|
|
55
92
|
other: TERMINAL_MODEL_COLORS.other,
|
|
56
93
|
};
|
|
57
94
|
|
|
95
|
+
function createSharedTokenScale() {
|
|
96
|
+
return {
|
|
97
|
+
scale: 1,
|
|
98
|
+
totalTokens: 0,
|
|
99
|
+
targets: new Set(),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function addSharedTokenContribution(state, contribution, targets) {
|
|
104
|
+
const tokens = tokenValue(contribution, { allowFractional: true });
|
|
105
|
+
if (!(tokens > 0)) return;
|
|
106
|
+
const scaledTokens = tokens / state.scale;
|
|
107
|
+
for (const target of targets) {
|
|
108
|
+
state.targets.add(target);
|
|
109
|
+
target.totalTokens += scaledTokens;
|
|
110
|
+
}
|
|
111
|
+
state.totalTokens += scaledTokens;
|
|
112
|
+
const scaleFactor = Math.max(
|
|
113
|
+
1,
|
|
114
|
+
state.totalTokens / MAX_SAFE_TOKEN_COUNT,
|
|
115
|
+
);
|
|
116
|
+
if (scaleFactor === 1) return;
|
|
117
|
+
for (const target of state.targets) {
|
|
118
|
+
target.totalTokens /= scaleFactor;
|
|
119
|
+
if (target.totalTokens >= MAX_SAFE_TOKEN_COUNT - 2) {
|
|
120
|
+
target.totalTokens = MAX_SAFE_TOKEN_COUNT;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
state.totalTokens = MAX_SAFE_TOKEN_COUNT;
|
|
124
|
+
state.scale *= scaleFactor;
|
|
125
|
+
}
|
|
126
|
+
|
|
58
127
|
export function usage() {
|
|
59
128
|
return `Token Ledger
|
|
60
129
|
|
|
@@ -62,12 +131,16 @@ Usage:
|
|
|
62
131
|
tledger 1d Last 24 hours in the terminal
|
|
63
132
|
tledger week Last 7 calendar days in the terminal
|
|
64
133
|
tledger 30d Rolling 30 days in the terminal
|
|
134
|
+
tledger cost 7d --basis api-usd Hypothetical API-equivalent USD estimate
|
|
135
|
+
tledger cost week --basis codex-credits
|
|
136
|
+
Codex purchased-credit estimate
|
|
65
137
|
tledger report 7d Write the 7-day PNG report
|
|
66
138
|
tledger report 7d --cache-rate Write the cache-only PNG report
|
|
67
139
|
|
|
68
140
|
Common options:
|
|
69
141
|
--static Print once instead of opening the dashboard
|
|
70
142
|
--refresh Rebuild the local usage cache
|
|
143
|
+
--since <ISO timestamp> Collect history at or after this timestamp
|
|
71
144
|
--image-output <file> Choose where to save a PNG
|
|
72
145
|
--no-open Do not open a generated PNG
|
|
73
146
|
-h, --help Show this quick guide
|
|
@@ -87,6 +160,12 @@ Terminal commands:
|
|
|
87
160
|
tledger week [end-day] Seven local calendar days
|
|
88
161
|
tledger trend [Nd|Nw] Multi-day terminal trend
|
|
89
162
|
|
|
163
|
+
Cost commands (basis is required):
|
|
164
|
+
tledger cost <1d|Nd|Nw|week> --basis api-usd
|
|
165
|
+
Hypothetical API-equivalent USD estimate
|
|
166
|
+
tledger cost <1d|Nd|Nw|week> --basis codex-credits
|
|
167
|
+
Codex purchased-credit estimate
|
|
168
|
+
|
|
90
169
|
Report commands:
|
|
91
170
|
tledger report [Nd|Nw] Write the usage dashboard PNG
|
|
92
171
|
tledger report [Nd|Nw] --cache-rate
|
|
@@ -102,6 +181,7 @@ Data and refresh:
|
|
|
102
181
|
--refresh Rebuild the default snapshot from local Codex data
|
|
103
182
|
--no-refresh Use the cached snapshot without checking source files
|
|
104
183
|
--codex-home <dir> Codex data root used when refreshing
|
|
184
|
+
--since <ISO timestamp> Collect history at or after this timestamp
|
|
105
185
|
--no-archived Skip archived sessions when refreshing
|
|
106
186
|
|
|
107
187
|
Terminal output:
|
|
@@ -114,6 +194,7 @@ Terminal output:
|
|
|
114
194
|
--youplot Use the legacy single-series renderer
|
|
115
195
|
|
|
116
196
|
Report output:
|
|
197
|
+
--private Hide project names in the report
|
|
117
198
|
--drain Chart estimated meter drain instead of token volume
|
|
118
199
|
--cache-rate Write the cache-only report (report command only)
|
|
119
200
|
--image Write the trend view as a PNG
|
|
@@ -125,7 +206,8 @@ Help:
|
|
|
125
206
|
-h, --help Show the quick guide
|
|
126
207
|
--help-all Show this complete reference
|
|
127
208
|
|
|
128
|
-
The default snapshot is ~/.token-ledger/token-ledger-snapshot-
|
|
209
|
+
The default snapshot is ~/.token-ledger/token-ledger-snapshot-v3.json.gz and
|
|
210
|
+
the durable ledger is ~/.token-ledger/token-ledger-ledger.sqlite.
|
|
129
211
|
Token Ledger reads local Codex data only. It does not upload your usage.`;
|
|
130
212
|
}
|
|
131
213
|
|
|
@@ -172,10 +254,22 @@ function readOption(argv, index, name) {
|
|
|
172
254
|
|
|
173
255
|
export function parseArgs(argv) {
|
|
174
256
|
const helpCommand = argv[0] === "help";
|
|
257
|
+
const costCommand = argv[0] === "cost";
|
|
258
|
+
const costRangeValue = costCommand ? argv[1] : null;
|
|
259
|
+
const costHelpWithoutRange = costCommand &&
|
|
260
|
+
["--help", "-h", "--help-all"].includes(costRangeValue);
|
|
261
|
+
const costAlias = costCommand && costRangeValue !== "week" && !costHelpWithoutRange
|
|
262
|
+
? durationAlias(costRangeValue)
|
|
263
|
+
: null;
|
|
264
|
+
const costRangeValid = costRangeValue === "week" || Boolean(costAlias);
|
|
175
265
|
const alias = durationAlias(argv[0]);
|
|
176
|
-
const rolling24hCommand = argv[0] === "1d"
|
|
177
|
-
|
|
178
|
-
const
|
|
266
|
+
const rolling24hCommand = argv[0] === "1d" ||
|
|
267
|
+
(costCommand && costAlias?.days === 1);
|
|
268
|
+
const rollingDurationCommand = (Boolean(alias) && argv[0] !== "1d") ||
|
|
269
|
+
(costCommand && Boolean(costAlias) && costAlias.days !== 1);
|
|
270
|
+
const command = costCommand && costRangeValue === "week"
|
|
271
|
+
? "week"
|
|
272
|
+
: rolling24hCommand
|
|
179
273
|
? "rolling24h"
|
|
180
274
|
: rollingDurationCommand
|
|
181
275
|
? "rolling"
|
|
@@ -186,13 +280,14 @@ export function parseArgs(argv) {
|
|
|
186
280
|
: "day";
|
|
187
281
|
const options = {
|
|
188
282
|
range: command,
|
|
189
|
-
view: command === "trend" ? "trend" : "projects",
|
|
283
|
+
view: costCommand ? "cost" : command === "trend" ? "trend" : "projects",
|
|
190
284
|
rolling24h: rolling24hCommand,
|
|
191
285
|
rollingDuration: rollingDurationCommand,
|
|
192
|
-
rollingDays: alias?.days ?? (rolling24hCommand ? 1 : null),
|
|
193
|
-
rollingAmount: alias?.amount ?? (rolling24hCommand ? 1 : null),
|
|
194
|
-
rollingUnit: alias?.unit ?? (rolling24hCommand ? "d" : null),
|
|
195
|
-
rollingLabel: alias?.label ?? "1 day",
|
|
286
|
+
rollingDays: (costCommand ? costAlias?.days : alias?.days) ?? (rolling24hCommand ? 1 : null),
|
|
287
|
+
rollingAmount: (costCommand ? costAlias?.amount : alias?.amount) ?? (rolling24hCommand ? 1 : null),
|
|
288
|
+
rollingUnit: (costCommand ? costAlias?.unit : alias?.unit) ?? (rolling24hCommand ? "d" : null),
|
|
289
|
+
rollingLabel: (costCommand ? costAlias?.label : alias?.label) ?? "1 day",
|
|
290
|
+
basis: null,
|
|
196
291
|
report: argv[0] === "report",
|
|
197
292
|
trendDays: 7,
|
|
198
293
|
date: null,
|
|
@@ -201,6 +296,7 @@ export function parseArgs(argv) {
|
|
|
201
296
|
refresh: false,
|
|
202
297
|
autoRefresh: true,
|
|
203
298
|
codexHome: resolve(process.env.CODEX_HOME || `${homedir()}/.codex`),
|
|
299
|
+
since: null,
|
|
204
300
|
includeArchived: true,
|
|
205
301
|
timeZone: DEFAULT_TIME_ZONE,
|
|
206
302
|
top: DEFAULT_TOP,
|
|
@@ -208,20 +304,24 @@ export function parseArgs(argv) {
|
|
|
208
304
|
rawProjects: false,
|
|
209
305
|
plain: false,
|
|
210
306
|
ascii: false,
|
|
211
|
-
static:
|
|
307
|
+
static: costCommand,
|
|
212
308
|
image: false,
|
|
213
309
|
imageOutput: null,
|
|
214
310
|
imageWidth: null,
|
|
215
311
|
openImage: true,
|
|
216
312
|
drain: false,
|
|
217
313
|
cacheRate: false,
|
|
314
|
+
private: false,
|
|
218
315
|
legacyPlot: false,
|
|
219
316
|
help: argv.length === 0 || helpCommand,
|
|
220
317
|
helpAll: false,
|
|
221
318
|
};
|
|
222
319
|
|
|
223
320
|
let trendPeriodSeen = false;
|
|
224
|
-
let
|
|
321
|
+
let basisSeen = false;
|
|
322
|
+
let index = costCommand
|
|
323
|
+
? costHelpWithoutRange ? 1 : 2
|
|
324
|
+
: alias || ["day", "week", "trend", "report", "help"].includes(argv[0]) ? 1 : 0;
|
|
225
325
|
for (; index < argv.length; index += 1) {
|
|
226
326
|
const argument = argv[index];
|
|
227
327
|
if (argument === "--help" || argument === "-h") {
|
|
@@ -249,6 +349,18 @@ export function parseArgs(argv) {
|
|
|
249
349
|
options.trendDays = period.days;
|
|
250
350
|
trendPeriodSeen = true;
|
|
251
351
|
index += 1;
|
|
352
|
+
} else if (argument === "--basis") {
|
|
353
|
+
if (options.view !== "cost") {
|
|
354
|
+
throw new Error("--basis is only available with the cost command.");
|
|
355
|
+
}
|
|
356
|
+
if (basisSeen) throw new Error("Cost basis can only be specified once.");
|
|
357
|
+
const value = readOption(argv, index, "--basis");
|
|
358
|
+
if (value !== "codex-credits" && value !== "api-usd") {
|
|
359
|
+
throw new Error("--basis must be codex-credits or api-usd.");
|
|
360
|
+
}
|
|
361
|
+
options.basis = value;
|
|
362
|
+
basisSeen = true;
|
|
363
|
+
index += 1;
|
|
252
364
|
} else if (argument === "--input") {
|
|
253
365
|
options.input = resolve(readOption(argv, index, "--input"));
|
|
254
366
|
options.inputExplicit = true;
|
|
@@ -260,10 +372,17 @@ export function parseArgs(argv) {
|
|
|
260
372
|
} else if (argument === "--codex-home") {
|
|
261
373
|
options.codexHome = resolve(readOption(argv, index, "--codex-home"));
|
|
262
374
|
index += 1;
|
|
375
|
+
} else if (argument === "--since") {
|
|
376
|
+
const value = readOption(argv, index, "--since");
|
|
377
|
+
options.since = new Date(normalizeCollectionSince(value));
|
|
378
|
+
index += 1;
|
|
263
379
|
} else if (argument === "--tz") {
|
|
264
380
|
options.timeZone = readOption(argv, index, "--tz");
|
|
265
381
|
index += 1;
|
|
266
382
|
} else if (argument === "--top") {
|
|
383
|
+
if (options.view === "cost") {
|
|
384
|
+
throw new Error("--top is not available with the cost command.");
|
|
385
|
+
}
|
|
267
386
|
const value = Number(readOption(argv, index, "--top"));
|
|
268
387
|
if (!Number.isInteger(value) || value < 1 || value > 100) {
|
|
269
388
|
throw new Error("--top must be an integer from 1 to 100.");
|
|
@@ -271,6 +390,9 @@ export function parseArgs(argv) {
|
|
|
271
390
|
options.top = value;
|
|
272
391
|
index += 1;
|
|
273
392
|
} else if (argument === "--width") {
|
|
393
|
+
if (options.view === "cost") {
|
|
394
|
+
throw new Error("--width is not available with the cost command.");
|
|
395
|
+
}
|
|
274
396
|
const value = Number(readOption(argv, index, "--width"));
|
|
275
397
|
if (!Number.isInteger(value) || value < 40 || value > 200) {
|
|
276
398
|
throw new Error("--width must be an integer from 40 to 200.");
|
|
@@ -284,6 +406,9 @@ export function parseArgs(argv) {
|
|
|
284
406
|
} else if (argument === "--plain") {
|
|
285
407
|
options.plain = true;
|
|
286
408
|
} else if (argument === "--ascii") {
|
|
409
|
+
if (options.view === "cost") {
|
|
410
|
+
throw new Error("--ascii is not available with the cost command.");
|
|
411
|
+
}
|
|
287
412
|
options.ascii = true;
|
|
288
413
|
} else if (argument === "--static") {
|
|
289
414
|
options.static = true;
|
|
@@ -292,6 +417,11 @@ export function parseArgs(argv) {
|
|
|
292
417
|
throw new Error("--drain is only available for the trend view.");
|
|
293
418
|
}
|
|
294
419
|
options.drain = true;
|
|
420
|
+
} else if (argument === "--private") {
|
|
421
|
+
if (!options.report) {
|
|
422
|
+
throw new Error("--private is only available with the report command.");
|
|
423
|
+
}
|
|
424
|
+
options.private = true;
|
|
295
425
|
} else if (argument === "--cache-rate") {
|
|
296
426
|
if (!options.report) {
|
|
297
427
|
throw new Error("--cache-rate is only available with the report command.");
|
|
@@ -364,6 +494,12 @@ export function parseArgs(argv) {
|
|
|
364
494
|
}
|
|
365
495
|
|
|
366
496
|
if (options.report) options.image = true;
|
|
497
|
+
if (!options.help && costCommand && !costRangeValid) {
|
|
498
|
+
throw new Error("Cost range must be 1d, Nd, Nw, or week.");
|
|
499
|
+
}
|
|
500
|
+
if (!options.help && costCommand && !options.basis) {
|
|
501
|
+
throw new Error("The cost command requires --basis codex-credits or --basis api-usd.");
|
|
502
|
+
}
|
|
367
503
|
if (!options.help && (options.rolling24h || options.rollingDuration) && options.date) {
|
|
368
504
|
throw new Error(`${options.rollingLabel} does not accept --date; its rolling window ends now.`);
|
|
369
505
|
}
|
|
@@ -382,92 +518,24 @@ export function parseArgs(argv) {
|
|
|
382
518
|
if (!options.help && options.view === "trend" && options.legacyPlot) {
|
|
383
519
|
throw new Error("--youplot is only available for the project view.");
|
|
384
520
|
}
|
|
521
|
+
if (!options.help && options.view === "cost" && options.legacyPlot) {
|
|
522
|
+
throw new Error("--youplot is not available with the cost command.");
|
|
523
|
+
}
|
|
524
|
+
if (!options.help && options.view === "cost" && options.rawProjects) {
|
|
525
|
+
throw new Error("--raw-projects is not available with the cost command.");
|
|
526
|
+
}
|
|
385
527
|
if (!options.help && options.cacheRate && options.drain) {
|
|
386
528
|
throw new Error("--cache-rate cannot be combined with --drain.");
|
|
387
529
|
}
|
|
388
530
|
return options;
|
|
389
531
|
}
|
|
390
532
|
|
|
391
|
-
function numericDateParts(date) {
|
|
392
|
-
const parts = new Intl.DateTimeFormat("en-US", {
|
|
393
|
-
timeZone: date.timeZone,
|
|
394
|
-
year: "numeric",
|
|
395
|
-
month: "2-digit",
|
|
396
|
-
day: "2-digit",
|
|
397
|
-
}).formatToParts(date.value);
|
|
398
|
-
const values = Object.fromEntries(
|
|
399
|
-
parts
|
|
400
|
-
.filter((part) => part.type !== "literal")
|
|
401
|
-
.map((part) => [part.type, Number(part.value)]),
|
|
402
|
-
);
|
|
403
|
-
return values;
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
function dateStringFromParts(parts) {
|
|
407
|
-
return [parts.year, parts.month, parts.day]
|
|
408
|
-
.map((value, index) => (index === 0 ? String(value) : String(value).padStart(2, "0")))
|
|
409
|
-
.join("-");
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
function shiftCalendarDate(value, amount) {
|
|
413
|
-
const date = new Date(`${value}T00:00:00.000Z`);
|
|
414
|
-
date.setUTCDate(date.getUTCDate() + amount);
|
|
415
|
-
return date.toISOString().slice(0, 10);
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
function validateTimeZone(timeZone) {
|
|
419
|
-
try {
|
|
420
|
-
new Intl.DateTimeFormat("en-US", { timeZone }).format();
|
|
421
|
-
} catch {
|
|
422
|
-
throw new Error(`Unknown IANA timezone: ${timeZone}`);
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
function offsetAt(instant, timeZone) {
|
|
427
|
-
const parts = new Intl.DateTimeFormat("en-US", {
|
|
428
|
-
timeZone,
|
|
429
|
-
year: "numeric",
|
|
430
|
-
month: "2-digit",
|
|
431
|
-
day: "2-digit",
|
|
432
|
-
hour: "2-digit",
|
|
433
|
-
minute: "2-digit",
|
|
434
|
-
second: "2-digit",
|
|
435
|
-
hourCycle: "h23",
|
|
436
|
-
}).formatToParts(instant);
|
|
437
|
-
const values = Object.fromEntries(
|
|
438
|
-
parts
|
|
439
|
-
.filter((part) => part.type !== "literal")
|
|
440
|
-
.map((part) => [part.type, Number(part.value)]),
|
|
441
|
-
);
|
|
442
|
-
return (
|
|
443
|
-
Date.UTC(
|
|
444
|
-
values.year,
|
|
445
|
-
values.month - 1,
|
|
446
|
-
values.day,
|
|
447
|
-
values.hour,
|
|
448
|
-
values.minute,
|
|
449
|
-
values.second,
|
|
450
|
-
) - instant.getTime()
|
|
451
|
-
);
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
function zonedMidnight(dateString, timeZone) {
|
|
455
|
-
const [year, month, day] = dateString.split("-").map(Number);
|
|
456
|
-
const utcGuess = Date.UTC(year, month - 1, day);
|
|
457
|
-
let instant = new Date(utcGuess - offsetAt(new Date(utcGuess), timeZone));
|
|
458
|
-
const refinedOffset = offsetAt(instant, timeZone);
|
|
459
|
-
instant = new Date(utcGuess - refinedOffset);
|
|
460
|
-
return instant;
|
|
461
|
-
}
|
|
462
|
-
|
|
463
533
|
export function dayBounds(value, timeZone) {
|
|
464
534
|
validateTimeZone(timeZone);
|
|
535
|
+
const formatter = createTimeZoneFormatter(timeZone);
|
|
465
536
|
let dateString = value;
|
|
466
537
|
if (value === "today" || value === "yesterday") {
|
|
467
|
-
const today =
|
|
468
|
-
value: new Date(),
|
|
469
|
-
timeZone,
|
|
470
|
-
}));
|
|
538
|
+
const today = todayInTimeZone(timeZone, formatter);
|
|
471
539
|
dateString = value === "today" ? today : shiftCalendarDate(today, -1);
|
|
472
540
|
}
|
|
473
541
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(dateString)) {
|
|
@@ -483,8 +551,8 @@ export function dayBounds(value, timeZone) {
|
|
|
483
551
|
throw new Error(`Invalid calendar day: ${dateString}`);
|
|
484
552
|
}
|
|
485
553
|
const nextDateString = shiftCalendarDate(dateString, 1);
|
|
486
|
-
const start =
|
|
487
|
-
const end =
|
|
554
|
+
const start = localDateBoundary(dateString, timeZone, formatter);
|
|
555
|
+
const end = localDateBoundary(nextDateString, timeZone, formatter);
|
|
488
556
|
return { dateString, start, end, timeZone };
|
|
489
557
|
}
|
|
490
558
|
|
|
@@ -495,7 +563,7 @@ export function weekBounds(value, timeZone) {
|
|
|
495
563
|
...endDay,
|
|
496
564
|
startDateString,
|
|
497
565
|
endDateString: endDay.dateString,
|
|
498
|
-
start:
|
|
566
|
+
start: localDateBoundary(startDateString, timeZone),
|
|
499
567
|
rangeDays: 7,
|
|
500
568
|
};
|
|
501
569
|
}
|
|
@@ -529,13 +597,6 @@ export function rolling24hBounds(value = new Date(), timeZone = DEFAULT_TIME_ZON
|
|
|
529
597
|
return rollingDurationBounds(value, timeZone, 1);
|
|
530
598
|
}
|
|
531
599
|
|
|
532
|
-
export function sanitizeTerminalText(value) {
|
|
533
|
-
return String(value ?? "")
|
|
534
|
-
.replace(/\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g, "")
|
|
535
|
-
.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
536
|
-
.replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ");
|
|
537
|
-
}
|
|
538
|
-
|
|
539
600
|
function cleanLabel(value, fallback) {
|
|
540
601
|
const label = sanitizeTerminalText(value)
|
|
541
602
|
.replace(/\s+/g, " ")
|
|
@@ -548,6 +609,12 @@ const QUOTED_ABSOLUTE_PATH =
|
|
|
548
609
|
const UNQUOTED_ABSOLUTE_PATH =
|
|
549
610
|
/(^|[\s([{=])((?:\/(?!\/)|\/\/|\\\\|[A-Za-z]:[\\/])[^\s"'`)\]},;]+)/g;
|
|
550
611
|
|
|
612
|
+
const QUOTED_FILE_URL_PATH = /(["'])(file:\/\/\/[^"'\r\n]*)\1/gi;
|
|
613
|
+
const UNQUOTED_FILE_URL_PATH =
|
|
614
|
+
/(^|[\s([{=:])(file:\/\/\/[^\s"'`)\]},;]+)/gi;
|
|
615
|
+
const COLON_PREFIXED_LOCAL_PATH =
|
|
616
|
+
/(^|:)((?:\/(?!\/)|\\\\|[A-Za-z]:[\\/])[^\s"'`)\]},;]+)/g;
|
|
617
|
+
|
|
551
618
|
function isAbsoluteLocalPath(path) {
|
|
552
619
|
return (
|
|
553
620
|
path.startsWith("/") ||
|
|
@@ -580,6 +647,39 @@ export function redactLocalPaths(value, paths = []) {
|
|
|
580
647
|
.filter((path) => path && path !== "/")
|
|
581
648
|
.sort((left, right) => right.length - left.length);
|
|
582
649
|
|
|
650
|
+
const detectedPathLabel = (detectedPath) => {
|
|
651
|
+
const candidates = [detectedPath];
|
|
652
|
+
if (/^file:\/\/\//i.test(detectedPath)) {
|
|
653
|
+
const filePath = detectedPath.slice("file://".length);
|
|
654
|
+
candidates.push(filePath);
|
|
655
|
+
try {
|
|
656
|
+
candidates.push(decodeURIComponent(filePath));
|
|
657
|
+
} catch {
|
|
658
|
+
// Keep the raw file URL as the only candidate when its escape syntax
|
|
659
|
+
// is malformed; it is still safe to redact as an implicit path.
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
const explicit = [...explicitPaths].find((path) => candidates.includes(path));
|
|
663
|
+
return explicit ? safeDisplayLabel(explicit) : "[local path]";
|
|
664
|
+
};
|
|
665
|
+
|
|
666
|
+
// These forms occur in Node/worker diagnostics but are not covered by the
|
|
667
|
+
// ordinary absolute-path boundaries below. Handle them before replacing an
|
|
668
|
+
// explicit path substring so an explicit input still gets its safe filename.
|
|
669
|
+
redacted = redacted
|
|
670
|
+
.replace(
|
|
671
|
+
QUOTED_FILE_URL_PATH,
|
|
672
|
+
(_match, quote, path) => `${quote}${detectedPathLabel(path)}${quote}`,
|
|
673
|
+
)
|
|
674
|
+
.replace(
|
|
675
|
+
UNQUOTED_FILE_URL_PATH,
|
|
676
|
+
(_match, prefix, path) => `${prefix}${detectedPathLabel(path)}`,
|
|
677
|
+
)
|
|
678
|
+
.replace(
|
|
679
|
+
COLON_PREFIXED_LOCAL_PATH,
|
|
680
|
+
(_match, prefix, path) => `${prefix}${detectedPathLabel(path)}`,
|
|
681
|
+
);
|
|
682
|
+
|
|
583
683
|
for (const path of pathsToRedact) {
|
|
584
684
|
redacted = redacted.replaceAll(
|
|
585
685
|
path,
|
|
@@ -605,7 +705,7 @@ function displayLabel(value) {
|
|
|
605
705
|
return `${label.slice(0, 14)}…${label.slice(-13)}`;
|
|
606
706
|
}
|
|
607
707
|
|
|
608
|
-
export function oneOffProjects(snapshot) {
|
|
708
|
+
export function oneOffProjects(snapshot, events = null) {
|
|
609
709
|
const threadIdsByProject = new Map();
|
|
610
710
|
const add = (project, threadId) => {
|
|
611
711
|
if (!project || !threadId) return;
|
|
@@ -614,7 +714,7 @@ export function oneOffProjects(snapshot) {
|
|
|
614
714
|
ids.add(threadId);
|
|
615
715
|
threadIdsByProject.set(normalizedProject, ids);
|
|
616
716
|
};
|
|
617
|
-
for (const bucket of usageBuckets(snapshot)) {
|
|
717
|
+
for (const bucket of events ?? usageBuckets(snapshot)) {
|
|
618
718
|
for (const threadId of usageThreadIds(bucket)) add(bucket.project, threadId);
|
|
619
719
|
}
|
|
620
720
|
for (const thread of snapshot.threads ?? []) add(thread.project, thread.id);
|
|
@@ -628,6 +728,7 @@ export function oneOffProjects(snapshot) {
|
|
|
628
728
|
function modelLabel(value) {
|
|
629
729
|
const model = cleanLabel(value, "Unknown model");
|
|
630
730
|
const lower = model.toLowerCase();
|
|
731
|
+
if (lower.includes("astra")) return "Astra";
|
|
631
732
|
if (lower.includes("sol")) return "Sol";
|
|
632
733
|
if (lower.includes("luna")) return "Luna";
|
|
633
734
|
if (lower.includes("terra")) return "Terra";
|
|
@@ -636,17 +737,33 @@ function modelLabel(value) {
|
|
|
636
737
|
return model;
|
|
637
738
|
}
|
|
638
739
|
|
|
639
|
-
|
|
740
|
+
function currentRateCardCredits(event) {
|
|
741
|
+
return calculateCodexPurchasedCredits({
|
|
742
|
+
model: event?.rateCardModel ?? event?.model,
|
|
743
|
+
serviceTier: event?.serviceTier,
|
|
744
|
+
usage: event,
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
export function filterDayEvents(snapshot, bounds, analysis = null) {
|
|
749
|
+
if (analysis !== null) return analysis.currentEvents;
|
|
640
750
|
const start = bounds.start.getTime();
|
|
641
751
|
const end = bounds.end.getTime();
|
|
642
752
|
return usageBucketsInRange(snapshot, start, end);
|
|
643
753
|
}
|
|
644
754
|
|
|
645
|
-
export function aggregateProjects(snapshot, events, options = {}) {
|
|
646
|
-
const singletonProjects = options.rawProjects
|
|
755
|
+
export function aggregateProjects(snapshot, events, options = {}, analysis = null) {
|
|
756
|
+
const singletonProjects = options.rawProjects
|
|
757
|
+
? new Set()
|
|
758
|
+
: oneOffProjects(snapshot, analysis?.allEvents);
|
|
647
759
|
const grouped = new Map();
|
|
760
|
+
const sharedTokenScale = createSharedTokenScale();
|
|
648
761
|
|
|
649
762
|
for (const event of events) {
|
|
763
|
+
if (event?.invalidTokenRecord === true) continue;
|
|
764
|
+
const allowFractional = event?.rangeAllocationEstimated === true;
|
|
765
|
+
const totalTokens = tokenValue(event.totalTokens, { allowFractional });
|
|
766
|
+
const estimatedTokenContribution = allowFractional && totalTokens > 0;
|
|
650
767
|
const rawProject = cleanLabel(event.project, "Unlabelled activity");
|
|
651
768
|
const project =
|
|
652
769
|
!options.rawProjects && singletonProjects.has(rawProject)
|
|
@@ -665,16 +782,36 @@ export function aggregateProjects(snapshot, events, options = {}) {
|
|
|
665
782
|
rateCardCredits: 0,
|
|
666
783
|
knownCreditTokens: 0,
|
|
667
784
|
models: new Map(),
|
|
785
|
+
estimated: false,
|
|
668
786
|
};
|
|
669
|
-
row.
|
|
670
|
-
row.outputTokens
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
787
|
+
row.estimated ||= estimatedTokenContribution;
|
|
788
|
+
row.outputTokens = checkedTokenAdd(
|
|
789
|
+
row.outputTokens,
|
|
790
|
+
tokenValue(event.outputTokens, { allowFractional }),
|
|
791
|
+
{ allowFractional },
|
|
792
|
+
);
|
|
793
|
+
row.reasoningTokens = checkedTokenAdd(
|
|
794
|
+
row.reasoningTokens,
|
|
795
|
+
tokenValue(event.reasoningTokens, { allowFractional }),
|
|
796
|
+
{ allowFractional },
|
|
797
|
+
);
|
|
798
|
+
row.toolCalls = checkedTokenAdd(
|
|
799
|
+
row.toolCalls,
|
|
800
|
+
tokenValue(event.toolCalls, { allowFractional }),
|
|
801
|
+
{ allowFractional },
|
|
802
|
+
);
|
|
803
|
+
row.events = checkedTokenAdd(row.events, usageCallCount(event), {
|
|
804
|
+
allowFractional,
|
|
805
|
+
});
|
|
674
806
|
for (const threadId of usageThreadIds(event)) row.threadIds.add(threadId);
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
row.
|
|
807
|
+
const rateCardCredits = currentRateCardCredits(event);
|
|
808
|
+
if (Number.isFinite(rateCardCredits)) {
|
|
809
|
+
row.rateCardCredits = checkedFiniteAdd(row.rateCardCredits, rateCardCredits);
|
|
810
|
+
row.knownCreditTokens = checkedTokenAdd(
|
|
811
|
+
row.knownCreditTokens,
|
|
812
|
+
tokenValue(event.totalTokens, { allowFractional }),
|
|
813
|
+
{ allowFractional },
|
|
814
|
+
);
|
|
678
815
|
}
|
|
679
816
|
|
|
680
817
|
const model = modelLabel(event.model);
|
|
@@ -683,11 +820,22 @@ export function aggregateProjects(snapshot, events, options = {}) {
|
|
|
683
820
|
totalTokens: 0,
|
|
684
821
|
events: 0,
|
|
685
822
|
rateCardCredits: 0,
|
|
823
|
+
estimated: false,
|
|
686
824
|
};
|
|
687
|
-
modelRow.
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
825
|
+
modelRow.estimated ||= estimatedTokenContribution;
|
|
826
|
+
addSharedTokenContribution(
|
|
827
|
+
sharedTokenScale,
|
|
828
|
+
totalTokens,
|
|
829
|
+
[row, modelRow],
|
|
830
|
+
);
|
|
831
|
+
modelRow.events = checkedTokenAdd(modelRow.events, usageCallCount(event), {
|
|
832
|
+
allowFractional,
|
|
833
|
+
});
|
|
834
|
+
if (Number.isFinite(rateCardCredits)) {
|
|
835
|
+
modelRow.rateCardCredits = checkedFiniteAdd(
|
|
836
|
+
modelRow.rateCardCredits,
|
|
837
|
+
rateCardCredits,
|
|
838
|
+
);
|
|
691
839
|
}
|
|
692
840
|
row.models.set(model, modelRow);
|
|
693
841
|
grouped.set(project, row);
|
|
@@ -709,19 +857,33 @@ export function aggregateProjects(snapshot, events, options = {}) {
|
|
|
709
857
|
});
|
|
710
858
|
}
|
|
711
859
|
|
|
712
|
-
function totalSummary(events) {
|
|
713
|
-
|
|
860
|
+
function totalSummary(events, projectRows) {
|
|
861
|
+
const summary = events.reduce(
|
|
714
862
|
(summary, event) => {
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
summary.toolCalls
|
|
718
|
-
|
|
863
|
+
if (event?.invalidTokenRecord === true) return summary;
|
|
864
|
+
const allowFractional = event?.rangeAllocationEstimated === true;
|
|
865
|
+
summary.toolCalls = checkedTokenAdd(
|
|
866
|
+
summary.toolCalls,
|
|
867
|
+
tokenValue(event.toolCalls, { allowFractional }),
|
|
868
|
+
{ allowFractional },
|
|
869
|
+
);
|
|
870
|
+
summary.calls = checkedTokenAdd(summary.calls, usageCallCount(event), {
|
|
871
|
+
allowFractional,
|
|
872
|
+
});
|
|
719
873
|
for (const threadId of usageThreadIds(event)) {
|
|
720
874
|
summary.threadIds.add(threadId);
|
|
721
875
|
}
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
summary.
|
|
876
|
+
const rateCardCredits = currentRateCardCredits(event);
|
|
877
|
+
if (Number.isFinite(rateCardCredits)) {
|
|
878
|
+
summary.rateCardCredits = checkedFiniteAdd(
|
|
879
|
+
summary.rateCardCredits,
|
|
880
|
+
rateCardCredits,
|
|
881
|
+
);
|
|
882
|
+
summary.knownCreditTokens = checkedTokenAdd(
|
|
883
|
+
summary.knownCreditTokens,
|
|
884
|
+
tokenValue(event.totalTokens, { allowFractional }),
|
|
885
|
+
{ allowFractional },
|
|
886
|
+
);
|
|
725
887
|
}
|
|
726
888
|
return summary;
|
|
727
889
|
},
|
|
@@ -735,6 +897,12 @@ function totalSummary(events) {
|
|
|
735
897
|
threadIds: new Set(),
|
|
736
898
|
},
|
|
737
899
|
);
|
|
900
|
+
summary.totalTokens = projectRows.reduce(
|
|
901
|
+
(sum, row) => sum + row.totalTokens,
|
|
902
|
+
0,
|
|
903
|
+
);
|
|
904
|
+
summary.outputTokens = scaledOutputTokens(events, summary.totalTokens);
|
|
905
|
+
return summary;
|
|
738
906
|
}
|
|
739
907
|
|
|
740
908
|
function compact(value, digits = 2) {
|
|
@@ -786,6 +954,7 @@ function colorize(value, code, enabled) {
|
|
|
786
954
|
|
|
787
955
|
function modelColor(model) {
|
|
788
956
|
const lower = model.toLowerCase();
|
|
957
|
+
if (lower.includes("astra")) return MODEL_COLORS.astra;
|
|
789
958
|
if (lower.includes("sol")) return MODEL_COLORS.sol;
|
|
790
959
|
if (lower.includes("luna")) return MODEL_COLORS.luna;
|
|
791
960
|
if (lower.includes("terra")) return MODEL_COLORS.terra;
|
|
@@ -819,7 +988,7 @@ function runYouPlot(rows, options, dateLabel, unit) {
|
|
|
819
988
|
const chartInput = [
|
|
820
989
|
"project\tvalue",
|
|
821
990
|
...rows.map(
|
|
822
|
-
(row) => `${row.displayProject.replace(/[\t\r\n]+/g, " ")}\t${chartNumber(row.totalTokens, unit.divisor)}`,
|
|
991
|
+
(row) => `${sanitizeTerminalText(row.displayProject).replace(/[\t\r\n]+/g, " ")}\t${chartNumber(row.totalTokens, unit.divisor)}`,
|
|
823
992
|
),
|
|
824
993
|
].join("\n");
|
|
825
994
|
const terminalWidth = Number(process.stdout.columns) || 100;
|
|
@@ -881,6 +1050,77 @@ async function readSnapshot(snapshotPath) {
|
|
|
881
1050
|
return parsed;
|
|
882
1051
|
}
|
|
883
1052
|
|
|
1053
|
+
function snapshotScopeMatchesOptions(snapshot, options) {
|
|
1054
|
+
const requested = collectionScope(options);
|
|
1055
|
+
const actual = snapshotCollectionScope(snapshot);
|
|
1056
|
+
if (!actual) {
|
|
1057
|
+
// Explicit snapshots are deliberate inputs. Preserve the existing
|
|
1058
|
+
// fixture/export workflow when no collection filter was requested, while
|
|
1059
|
+
// refusing to claim that an unknown snapshot satisfies a filter.
|
|
1060
|
+
return requested.since === null && requested.includeArchived;
|
|
1061
|
+
}
|
|
1062
|
+
return snapshotMatchesCollectionScope(snapshot, requested);
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
function snapshotScopeError(snapshot, options) {
|
|
1066
|
+
const requested = collectionScope(options);
|
|
1067
|
+
const actual = snapshotCollectionScope(snapshot);
|
|
1068
|
+
const describe = (scope) => {
|
|
1069
|
+
if (!scope) return "unknown collection scope";
|
|
1070
|
+
const since = scope.since === null ? "full history" : `since ${scope.since}`;
|
|
1071
|
+
const archives = scope.includeArchived
|
|
1072
|
+
? "archived sessions included"
|
|
1073
|
+
: "archived sessions excluded";
|
|
1074
|
+
return `${since}; ${archives}`;
|
|
1075
|
+
};
|
|
1076
|
+
const remedy = options.inputExplicit
|
|
1077
|
+
? "For --input, supply matching --since/--no-archived filters or rebuild that file with the collector; --refresh cannot be combined with --input."
|
|
1078
|
+
: "Rebuild it with --refresh.";
|
|
1079
|
+
return new Error(
|
|
1080
|
+
`Snapshot collection scope does not match the requested filters (requested ${describe(requested)}; found ${describe(actual)}). ${remedy}`,
|
|
1081
|
+
);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
async function snapshotAllowsStaleFallback(snapshot, options) {
|
|
1085
|
+
if (!snapshotMatchesCollectionScope(snapshot, collectionScope(options))) {
|
|
1086
|
+
return false;
|
|
1087
|
+
}
|
|
1088
|
+
const { codexHomeFingerprint } = await import(
|
|
1089
|
+
"../lib/token-ledger-ledger.mjs"
|
|
1090
|
+
);
|
|
1091
|
+
return primitiveString(
|
|
1092
|
+
snapshot.metadata?.durableLedger?.codexHomeFingerprint,
|
|
1093
|
+
) === codexHomeFingerprint(options.codexHome);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
function snapshotWithUnavailableUntrustedQuota(snapshot, force = false) {
|
|
1097
|
+
if (!force && snapshotHasCurrentQuotaIdentityContract(snapshot)) return snapshot;
|
|
1098
|
+
return {
|
|
1099
|
+
...snapshot,
|
|
1100
|
+
coverage: {
|
|
1101
|
+
...snapshot.coverage,
|
|
1102
|
+
quotaMeterUnavailableReason: "quota-contract-unverified",
|
|
1103
|
+
},
|
|
1104
|
+
quotaObservations: [],
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
async function snapshotAndLedgerHaveCurrentQuotaContract(snapshot, options) {
|
|
1109
|
+
if (!snapshotHasCurrentQuotaIdentityContract(snapshot)) return false;
|
|
1110
|
+
const {
|
|
1111
|
+
readDurableLedgerCacheState,
|
|
1112
|
+
resolveDurableLedgerPath,
|
|
1113
|
+
} = await import("../lib/token-ledger-ledger.mjs");
|
|
1114
|
+
const ledgerState = await readDurableLedgerCacheState(
|
|
1115
|
+
resolveDurableLedgerPath({
|
|
1116
|
+
codexHome: options.codexHome,
|
|
1117
|
+
output: options.input,
|
|
1118
|
+
}),
|
|
1119
|
+
);
|
|
1120
|
+
return ledgerState?.quotaIdentityContract ===
|
|
1121
|
+
QUOTA_IDENTITY_CONTRACT_VERSION;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
884
1124
|
async function refreshSnapshot(options) {
|
|
885
1125
|
if (!existsSync(options.codexHome)) {
|
|
886
1126
|
throw new Error(
|
|
@@ -889,25 +1129,42 @@ async function refreshSnapshot(options) {
|
|
|
889
1129
|
}
|
|
890
1130
|
let progressStarted = false;
|
|
891
1131
|
try {
|
|
892
|
-
const {
|
|
1132
|
+
const {
|
|
1133
|
+
collectUsage,
|
|
1134
|
+
sourceInventory,
|
|
1135
|
+
sourceWatermarksEqual,
|
|
1136
|
+
} = await import(
|
|
893
1137
|
"../lib/token-ledger-importer.mjs"
|
|
894
1138
|
);
|
|
895
1139
|
process.stderr.write("Token Ledger: refreshing local snapshot…\n");
|
|
896
1140
|
progressStarted = true;
|
|
1141
|
+
let writeResult = null;
|
|
897
1142
|
const snapshot = await collectUsage(
|
|
898
1143
|
{
|
|
899
1144
|
output: options.input,
|
|
900
1145
|
codexHome: options.codexHome,
|
|
901
1146
|
includeArchived: options.includeArchived,
|
|
902
|
-
since:
|
|
1147
|
+
since: options.since,
|
|
1148
|
+
stageSnapshot: async (candidate) => {
|
|
1149
|
+
writeResult = await stagePrivateSnapshot(
|
|
1150
|
+
options.input,
|
|
1151
|
+
candidate,
|
|
1152
|
+
options.snapshotWriteOptions,
|
|
1153
|
+
);
|
|
1154
|
+
return writeResult;
|
|
1155
|
+
},
|
|
903
1156
|
},
|
|
904
1157
|
({ current, total }) => {
|
|
905
1158
|
process.stderr.write(`\rToken Ledger: scanned ${current}/${total} rollout files`);
|
|
906
1159
|
},
|
|
907
1160
|
);
|
|
908
1161
|
process.stderr.write("\n");
|
|
909
|
-
const
|
|
910
|
-
|
|
1162
|
+
const storedSnapshot = snapshot;
|
|
1163
|
+
if (storedSnapshot.coverage.filesReused > 0) {
|
|
1164
|
+
process.stderr.write(
|
|
1165
|
+
`Token Ledger: reused ${storedSnapshot.coverage.filesReused.toLocaleString()} unchanged rollout files from the durable ledger.\n`,
|
|
1166
|
+
);
|
|
1167
|
+
}
|
|
911
1168
|
process.stderr.write(
|
|
912
1169
|
`Token Ledger: cached ${(writeResult.bytesWritten / 1_000_000).toFixed(1)} MB ${writeResult.encoding} snapshot (${(writeResult.jsonBytes / 1_000_000).toFixed(1)} MB JSON before encoding; ${storedSnapshot.events.length.toLocaleString()} buckets for ${storedSnapshot.coverage.observedModelCalls.toLocaleString()} calls; ${(writeResult.maxBytes / 1_000_000).toFixed(1)} MB limit).\n`,
|
|
913
1170
|
);
|
|
@@ -916,23 +1173,163 @@ async function refreshSnapshot(options) {
|
|
|
916
1173
|
"Token Ledger: snapshot is above 70% of its safety limit; older buckets will compact automatically as it grows.\n",
|
|
917
1174
|
);
|
|
918
1175
|
}
|
|
919
|
-
|
|
1176
|
+
let sourceStatus = "unchecked-cache";
|
|
1177
|
+
try {
|
|
1178
|
+
const currentInventory = await sourceInventory(
|
|
1179
|
+
options.codexHome,
|
|
1180
|
+
options.includeArchived,
|
|
1181
|
+
);
|
|
1182
|
+
sourceStatus = refreshedSnapshotSourceStatus(
|
|
1183
|
+
storedSnapshot.sourceWatermark,
|
|
1184
|
+
currentInventory.watermark,
|
|
1185
|
+
sourceWatermarksEqual,
|
|
1186
|
+
);
|
|
1187
|
+
} catch (error) {
|
|
1188
|
+
if (!postRefreshStatusAllowsUnchecked(error)) throw error;
|
|
1189
|
+
}
|
|
1190
|
+
return {
|
|
1191
|
+
snapshot: storedSnapshot,
|
|
1192
|
+
sourceStatus,
|
|
1193
|
+
};
|
|
920
1194
|
} catch (error) {
|
|
921
1195
|
if (progressStarted) process.stderr.write("\n");
|
|
922
1196
|
if (error?.code === "ERR_SNAPSHOT_SIZE_LIMIT" && existsSync(options.input)) {
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1197
|
+
try {
|
|
1198
|
+
const previous = await readSnapshot(options.input);
|
|
1199
|
+
if (await snapshotAllowsStaleFallback(previous, options)) {
|
|
1200
|
+
const quotaContractTrusted =
|
|
1201
|
+
await snapshotAndLedgerHaveCurrentQuotaContract(
|
|
1202
|
+
previous,
|
|
1203
|
+
options,
|
|
1204
|
+
);
|
|
1205
|
+
process.stderr.write(
|
|
1206
|
+
"Token Ledger: refresh exceeded the safety limit; continuing with the previous cache, which may be stale.\n",
|
|
1207
|
+
);
|
|
1208
|
+
return {
|
|
1209
|
+
snapshot: snapshotWithUnavailableUntrustedQuota(
|
|
1210
|
+
previous,
|
|
1211
|
+
!quotaContractTrusted,
|
|
1212
|
+
),
|
|
1213
|
+
sourceStatus: "stale-fallback",
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
} catch {
|
|
1217
|
+
// Preserve the original refresh error when the previous cache cannot
|
|
1218
|
+
// prove its exact collection scope and Codex-home identity.
|
|
1219
|
+
}
|
|
927
1220
|
}
|
|
928
|
-
|
|
1221
|
+
const wrapped = new Error(
|
|
929
1222
|
`Could not refresh local snapshot: ${safeErrorMessage(error, [options.input, options.codexHome])}`,
|
|
1223
|
+
{ cause: error },
|
|
930
1224
|
);
|
|
1225
|
+
wrapped.code = error?.code;
|
|
1226
|
+
throw wrapped;
|
|
931
1227
|
}
|
|
932
1228
|
}
|
|
933
1229
|
|
|
934
|
-
export function
|
|
935
|
-
|
|
1230
|
+
export function refreshedSnapshotSourceStatus(
|
|
1231
|
+
snapshotWatermark,
|
|
1232
|
+
currentWatermark,
|
|
1233
|
+
watermarksEqual,
|
|
1234
|
+
) {
|
|
1235
|
+
return watermarksEqual(snapshotWatermark, currentWatermark)
|
|
1236
|
+
? "verified-current"
|
|
1237
|
+
: "unchecked-cache";
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
export function postRefreshStatusAllowsUnchecked(error) {
|
|
1241
|
+
return ["ENOENT", "ENOTDIR"].includes(primitiveString(error?.code));
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
const RECOVERABLE_REFRESH_ERROR_CODES = new Set([
|
|
1245
|
+
"ERR_SNAPSHOT_SIZE_LIMIT",
|
|
1246
|
+
"ERR_SOURCE_CHANGED_DURING_COLLECTION",
|
|
1247
|
+
"SQLITE_BUSY",
|
|
1248
|
+
"SQLITE_LOCKED",
|
|
1249
|
+
]);
|
|
1250
|
+
const RECOVERABLE_SQLITE_ERROR_CODES = new Set([5, 6]);
|
|
1251
|
+
const RECOVERABLE_SQLITE_ERROR_NAMES = new Set([
|
|
1252
|
+
"SQLITE_BUSY",
|
|
1253
|
+
"SQLITE_LOCKED",
|
|
1254
|
+
]);
|
|
1255
|
+
const HARD_SQLITE_ERROR_CODES = new Set([11, 26]);
|
|
1256
|
+
const HARD_SQLITE_ERROR_NAMES = new Set([
|
|
1257
|
+
"SQLITE_CORRUPT",
|
|
1258
|
+
"SQLITE_NOTADB",
|
|
1259
|
+
]);
|
|
1260
|
+
|
|
1261
|
+
export function refreshFailureAllowsStaleFallback(error) {
|
|
1262
|
+
const seen = new Set();
|
|
1263
|
+
let recoverable = false;
|
|
1264
|
+
let current = error;
|
|
1265
|
+
while (current instanceof Error) {
|
|
1266
|
+
if (seen.has(current)) return false;
|
|
1267
|
+
seen.add(current);
|
|
1268
|
+
const code = primitiveString(current.code);
|
|
1269
|
+
if (RECOVERABLE_REFRESH_ERROR_CODES.has(code)) recoverable = true;
|
|
1270
|
+
if (
|
|
1271
|
+
code?.startsWith("ERR_DURABLE_LEDGER_") ||
|
|
1272
|
+
(
|
|
1273
|
+
code?.startsWith("ERR_SNAPSHOT_") &&
|
|
1274
|
+
code !== "ERR_SNAPSHOT_SIZE_LIMIT"
|
|
1275
|
+
) ||
|
|
1276
|
+
code === "ERR_BUFFER_TOO_LARGE" ||
|
|
1277
|
+
HARD_SQLITE_ERROR_NAMES.has(code)
|
|
1278
|
+
) {
|
|
1279
|
+
return false;
|
|
1280
|
+
}
|
|
1281
|
+
if (code === "ERR_SQLITE_ERROR") {
|
|
1282
|
+
if (RECOVERABLE_SQLITE_ERROR_CODES.has(Number(current.errcode))) {
|
|
1283
|
+
recoverable = true;
|
|
1284
|
+
}
|
|
1285
|
+
if (HARD_SQLITE_ERROR_CODES.has(Number(current.errcode))) {
|
|
1286
|
+
return false;
|
|
1287
|
+
}
|
|
1288
|
+
const name = primitiveString(current.errstr)?.toUpperCase();
|
|
1289
|
+
if (RECOVERABLE_SQLITE_ERROR_NAMES.has(name)) recoverable = true;
|
|
1290
|
+
if (HARD_SQLITE_ERROR_NAMES.has(name)) return false;
|
|
1291
|
+
}
|
|
1292
|
+
current = current.cause;
|
|
1293
|
+
}
|
|
1294
|
+
return recoverable;
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
async function refreshSnapshotOrUseStaleFallback(
|
|
1298
|
+
options,
|
|
1299
|
+
cached,
|
|
1300
|
+
{ quotaContractTrusted = true } = {},
|
|
1301
|
+
) {
|
|
1302
|
+
try {
|
|
1303
|
+
const refreshed = await refreshSnapshot(options);
|
|
1304
|
+
if (
|
|
1305
|
+
refreshed.sourceStatus === "stale-fallback" &&
|
|
1306
|
+
!quotaContractTrusted
|
|
1307
|
+
) {
|
|
1308
|
+
return {
|
|
1309
|
+
...refreshed,
|
|
1310
|
+
snapshot: snapshotWithUnavailableUntrustedQuota(
|
|
1311
|
+
refreshed.snapshot,
|
|
1312
|
+
true,
|
|
1313
|
+
),
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
return refreshed;
|
|
1317
|
+
} catch (error) {
|
|
1318
|
+
// Default to surfacing refresh failures. Only bounded cache growth, source
|
|
1319
|
+
// races, and SQLite lock contention are safe reasons to reuse known-good
|
|
1320
|
+
// cached data; migration, schema, corruption, and unknown failures are not.
|
|
1321
|
+
if (
|
|
1322
|
+
!refreshFailureAllowsStaleFallback(error) ||
|
|
1323
|
+
!(await snapshotAllowsStaleFallback(cached, options))
|
|
1324
|
+
) throw error;
|
|
1325
|
+
return {
|
|
1326
|
+
snapshot: snapshotWithUnavailableUntrustedQuota(
|
|
1327
|
+
cached,
|
|
1328
|
+
!quotaContractTrusted,
|
|
1329
|
+
),
|
|
1330
|
+
sourceStatus: "stale-fallback",
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
936
1333
|
}
|
|
937
1334
|
|
|
938
1335
|
export function snapshotCacheIsFresh(
|
|
@@ -947,6 +1344,14 @@ export function snapshotCacheIsFresh(
|
|
|
947
1344
|
);
|
|
948
1345
|
}
|
|
949
1346
|
|
|
1347
|
+
export function shouldCheckSourceFreshness(
|
|
1348
|
+
options = {},
|
|
1349
|
+
) {
|
|
1350
|
+
// The source manifest is cheap to stat and is the cache's validity anchor.
|
|
1351
|
+
// The full collector only runs when the persisted watermark changes.
|
|
1352
|
+
return options.autoRefresh !== false && options.inputExplicit !== true;
|
|
1353
|
+
}
|
|
1354
|
+
|
|
950
1355
|
function snapshotAgeLabel(ageMs) {
|
|
951
1356
|
if (ageMs < 60 * 1_000) return "now";
|
|
952
1357
|
const minutes = Math.floor(ageMs / (60 * 1_000));
|
|
@@ -981,7 +1386,10 @@ export function snapshotFreshness(snapshot = {}, nowMs = Date.now()) {
|
|
|
981
1386
|
};
|
|
982
1387
|
}
|
|
983
1388
|
|
|
984
|
-
|
|
1389
|
+
// Resolve the snapshot together with the evidence available for its report
|
|
1390
|
+
// cutoff. A cache can be readable without being current, so callers must keep
|
|
1391
|
+
// this status separate from the age label shown in terminal output.
|
|
1392
|
+
export async function loadSnapshot(options) {
|
|
985
1393
|
if (options.refresh) {
|
|
986
1394
|
return refreshSnapshot(options);
|
|
987
1395
|
}
|
|
@@ -993,26 +1401,31 @@ async function loadSnapshot(options) {
|
|
|
993
1401
|
}
|
|
994
1402
|
return refreshSnapshot(options);
|
|
995
1403
|
}
|
|
996
|
-
|
|
997
|
-
|
|
1404
|
+
const cached = await readSnapshot(options.input);
|
|
1405
|
+
if (!snapshotScopeMatchesOptions(cached, options)) {
|
|
1406
|
+
if (options.inputExplicit || !options.autoRefresh) {
|
|
1407
|
+
throw snapshotScopeError(cached, options);
|
|
1408
|
+
}
|
|
1409
|
+
return refreshSnapshot(options);
|
|
998
1410
|
}
|
|
999
1411
|
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
`Could not inspect snapshot ${safeDisplayLabel(options.input, "snapshot")}: ${safeErrorMessage(error, [options.input])}`,
|
|
1006
|
-
);
|
|
1412
|
+
if (options.inputExplicit) {
|
|
1413
|
+
return { snapshot: cached, sourceStatus: "explicit-snapshot" };
|
|
1414
|
+
}
|
|
1415
|
+
if (!options.autoRefresh) {
|
|
1416
|
+
return { snapshot: cached, sourceStatus: "unchecked-cache" };
|
|
1007
1417
|
}
|
|
1008
|
-
|
|
1009
|
-
|
|
1418
|
+
|
|
1419
|
+
if (!shouldCheckSourceFreshness(options)) {
|
|
1420
|
+
return { snapshot: cached, sourceStatus: "unchecked-cache" };
|
|
1010
1421
|
}
|
|
1011
1422
|
|
|
1012
|
-
const {
|
|
1013
|
-
|
|
1423
|
+
const { sourceInventory, sourceWatermarksEqual } = await import(
|
|
1424
|
+
"../lib/token-ledger-importer.mjs"
|
|
1425
|
+
);
|
|
1426
|
+
let inventory;
|
|
1014
1427
|
try {
|
|
1015
|
-
|
|
1428
|
+
inventory = await sourceInventory(
|
|
1016
1429
|
options.codexHome,
|
|
1017
1430
|
options.includeArchived,
|
|
1018
1431
|
);
|
|
@@ -1021,24 +1434,91 @@ async function loadSnapshot(options) {
|
|
|
1021
1434
|
`Could not inspect local Codex source: ${safeErrorMessage(error, [options.codexHome])}`,
|
|
1022
1435
|
);
|
|
1023
1436
|
}
|
|
1024
|
-
|
|
1437
|
+
const {
|
|
1438
|
+
codexHomeFingerprint,
|
|
1439
|
+
readDurableLedgerCacheState,
|
|
1440
|
+
resolveDurableLedgerPath,
|
|
1441
|
+
} = await import("../lib/token-ledger-ledger.mjs");
|
|
1442
|
+
const cachedCodexHomeFingerprint = primitiveString(
|
|
1443
|
+
cached.metadata?.durableLedger?.codexHomeFingerprint,
|
|
1444
|
+
);
|
|
1445
|
+
if (
|
|
1446
|
+
cachedCodexHomeFingerprint !== codexHomeFingerprint(options.codexHome)
|
|
1447
|
+
) {
|
|
1025
1448
|
return refreshSnapshot(options);
|
|
1026
1449
|
}
|
|
1027
|
-
|
|
1450
|
+
const ledgerState = await readDurableLedgerCacheState(
|
|
1451
|
+
resolveDurableLedgerPath({
|
|
1452
|
+
codexHome: options.codexHome,
|
|
1453
|
+
output: options.input,
|
|
1454
|
+
}),
|
|
1455
|
+
);
|
|
1456
|
+
const quotaContractTrusted =
|
|
1457
|
+
ledgerState?.quotaIdentityContract === QUOTA_IDENTITY_CONTRACT_VERSION &&
|
|
1458
|
+
snapshotHasCurrentQuotaIdentityContract(cached);
|
|
1459
|
+
if (!sourceWatermarksEqual(cached.sourceWatermark, inventory.watermark)) {
|
|
1460
|
+
return refreshSnapshotOrUseStaleFallback(options, cached, {
|
|
1461
|
+
quotaContractTrusted,
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
const snapshotRevision = Number(
|
|
1465
|
+
cached.metadata?.durableLedger?.revision,
|
|
1466
|
+
);
|
|
1467
|
+
if (
|
|
1468
|
+
!Number.isSafeInteger(snapshotRevision) ||
|
|
1469
|
+
ledgerState?.revision !== snapshotRevision
|
|
1470
|
+
) {
|
|
1471
|
+
return refreshSnapshotOrUseStaleFallback(options, cached, {
|
|
1472
|
+
quotaContractTrusted,
|
|
1473
|
+
});
|
|
1474
|
+
}
|
|
1475
|
+
if (!quotaContractTrusted) {
|
|
1476
|
+
return refreshSnapshotOrUseStaleFallback(options, cached, {
|
|
1477
|
+
quotaContractTrusted: false,
|
|
1478
|
+
});
|
|
1479
|
+
}
|
|
1480
|
+
return { snapshot: cached, sourceStatus: "verified-current" };
|
|
1028
1481
|
}
|
|
1029
1482
|
|
|
1030
|
-
function render(
|
|
1483
|
+
async function render(
|
|
1484
|
+
options,
|
|
1485
|
+
snapshot,
|
|
1486
|
+
bounds,
|
|
1487
|
+
events,
|
|
1488
|
+
rows,
|
|
1489
|
+
allRows,
|
|
1490
|
+
freshness,
|
|
1491
|
+
report = {},
|
|
1492
|
+
analysis,
|
|
1493
|
+
) {
|
|
1494
|
+
if (options.view === "cost") {
|
|
1495
|
+
return renderCostTerminal({
|
|
1496
|
+
events,
|
|
1497
|
+
bounds,
|
|
1498
|
+
basis: options.basis,
|
|
1499
|
+
snapshotFreshness: freshness,
|
|
1500
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1501
|
+
});
|
|
1502
|
+
}
|
|
1031
1503
|
if (options.view === "trend") {
|
|
1032
1504
|
if (options.image && options.cacheRate) {
|
|
1505
|
+
const { renderCacheReportImage } = await import(
|
|
1506
|
+
"./token-ledger-cache-image.mjs"
|
|
1507
|
+
);
|
|
1033
1508
|
return renderCacheReportImage({
|
|
1034
1509
|
snapshot,
|
|
1035
1510
|
bounds,
|
|
1036
1511
|
days: options.trendDays,
|
|
1037
1512
|
options,
|
|
1513
|
+
analysis,
|
|
1514
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1038
1515
|
});
|
|
1039
1516
|
}
|
|
1040
|
-
const trend = buildUsageTrend(snapshot, bounds);
|
|
1517
|
+
const trend = buildUsageTrend(snapshot, bounds, { analysis });
|
|
1041
1518
|
if (options.image) {
|
|
1519
|
+
const { renderTrendImage } = await import(
|
|
1520
|
+
"./token-ledger-trend-image.mjs"
|
|
1521
|
+
);
|
|
1042
1522
|
return renderTrendImage({
|
|
1043
1523
|
snapshot,
|
|
1044
1524
|
bounds,
|
|
@@ -1046,6 +1526,10 @@ function render(options, snapshot, bounds, events, rows, allRows, freshness) {
|
|
|
1046
1526
|
days: options.trendDays,
|
|
1047
1527
|
options,
|
|
1048
1528
|
projectRows: allRows,
|
|
1529
|
+
reportTimeMs: report.reportTimeMs ?? null,
|
|
1530
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1531
|
+
analysis,
|
|
1532
|
+
reportEvents: events,
|
|
1049
1533
|
});
|
|
1050
1534
|
}
|
|
1051
1535
|
return renderTrendCombo({
|
|
@@ -1054,6 +1538,9 @@ function render(options, snapshot, bounds, events, rows, allRows, freshness) {
|
|
|
1054
1538
|
trend,
|
|
1055
1539
|
days: options.trendDays,
|
|
1056
1540
|
options,
|
|
1541
|
+
analysis,
|
|
1542
|
+
snapshotFreshness: freshness,
|
|
1543
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1057
1544
|
});
|
|
1058
1545
|
}
|
|
1059
1546
|
if (!options.legacyPlot) {
|
|
@@ -1061,6 +1548,7 @@ function render(options, snapshot, bounds, events, rows, allRows, freshness) {
|
|
|
1061
1548
|
options,
|
|
1062
1549
|
snapshot,
|
|
1063
1550
|
snapshotFreshness: freshness,
|
|
1551
|
+
sourceStatus: report.sourceStatus ?? "unchecked-cache",
|
|
1064
1552
|
bounds,
|
|
1065
1553
|
events,
|
|
1066
1554
|
rows,
|
|
@@ -1068,35 +1556,38 @@ function render(options, snapshot, bounds, events, rows, allRows, freshness) {
|
|
|
1068
1556
|
});
|
|
1069
1557
|
}
|
|
1070
1558
|
const enabled = !options.plain && !process.env.NO_COLOR && Boolean(process.stdout.isTTY);
|
|
1071
|
-
const summary = totalSummary(events);
|
|
1559
|
+
const summary = totalSummary(events, allRows);
|
|
1072
1560
|
const totalTokens = summary.totalTokens;
|
|
1073
1561
|
const dateLabel = options.range === "rolling24h"
|
|
1074
1562
|
? "last 24 hours"
|
|
1075
1563
|
: options.range === "rolling"
|
|
1076
1564
|
? `last ${options.rollingLabel}`
|
|
1077
1565
|
: options.range === "week"
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
}).format(bounds.start);
|
|
1566
|
+
? `${bounds.startDateString} through ${bounds.endDateString}`
|
|
1567
|
+
: formatCalendarDate(bounds.dateString, {
|
|
1568
|
+
weekday: "short",
|
|
1569
|
+
month: "short",
|
|
1570
|
+
day: "numeric",
|
|
1571
|
+
year: "numeric",
|
|
1572
|
+
});
|
|
1086
1573
|
const unit = chartUnit(rows[0]?.totalTokens ?? 0);
|
|
1087
1574
|
const shares = rows.map((row) =>
|
|
1088
1575
|
totalTokens > 0 ? (row.totalTokens / totalTokens) * 100 : 0,
|
|
1089
1576
|
);
|
|
1090
1577
|
const chart = runYouPlot(rows, options, dateLabel, unit).trimEnd();
|
|
1578
|
+
const historyScope = historyScopeLabel(snapshot);
|
|
1091
1579
|
|
|
1092
1580
|
const header = [
|
|
1093
1581
|
`Token Ledger · ${dateLabel} · ${bounds.timeZone}`,
|
|
1094
1582
|
`${compact(totalTokens)} tokens · ${summary.threadIds.size.toLocaleString()} threads · ${summary.calls.toLocaleString()} calls · ${compact(summary.outputTokens)} output`,
|
|
1583
|
+
...(historyScope ? [`History: ${historyScope}`] : []),
|
|
1584
|
+
`Snapshot: ${snapshotFreshnessDetail(freshness)}`,
|
|
1585
|
+
sourceStatusLine(report.sourceStatus ?? "unchecked-cache"),
|
|
1095
1586
|
`Source: ${sourceLabel(options.input, snapshot)}`,
|
|
1096
1587
|
"",
|
|
1097
1588
|
chart,
|
|
1098
1589
|
"",
|
|
1099
|
-
`Model mix · colors: ${colorize("Sol", MODEL_COLORS.sol, enabled)} ${colorize("Luna", MODEL_COLORS.luna, enabled)} ${colorize("Terra", MODEL_COLORS.terra, enabled)} ${colorize("GPT", MODEL_COLORS["gpt-5.5"], enabled)} ${colorize("Other", MODEL_COLORS.other, enabled)}`,
|
|
1590
|
+
`Model mix · colors: ${colorize("Astra", MODEL_COLORS.astra, enabled)} ${colorize("Sol", MODEL_COLORS.sol, enabled)} ${colorize("Luna", MODEL_COLORS.luna, enabled)} ${colorize("Terra", MODEL_COLORS.terra, enabled)} ${colorize("GPT", MODEL_COLORS["gpt-5.5"], enabled)} ${colorize("Other", MODEL_COLORS.other, enabled)}`,
|
|
1100
1591
|
];
|
|
1101
1592
|
|
|
1102
1593
|
const details = rows.map((row, index) => {
|
|
@@ -1104,7 +1595,7 @@ function render(options, snapshot, bounds, events, rows, allRows, freshness) {
|
|
|
1104
1595
|
summary.rateCardCredits > 0 && row.rateCardCredits > 0
|
|
1105
1596
|
? ` · ${percent((row.rateCardCredits / summary.rateCardCredits) * 100)} credits`
|
|
1106
1597
|
: "";
|
|
1107
|
-
return `${String(index + 1).padStart(2, " ")} ${row.displayProject} · ${compact(row.totalTokens)} · ${percent(shares[index])} · ${row.threads.toLocaleString()} threads${knownCreditShare}\n ${modelMix(row, enabled)}`;
|
|
1598
|
+
return `${String(index + 1).padStart(2, " ")} ${sanitizeTerminalText(row.displayProject)} · ${compact(row.totalTokens)} · ${percent(shares[index])} · ${row.threads.toLocaleString()} threads${knownCreditShare}\n ${modelMix(row, enabled)}`;
|
|
1108
1599
|
});
|
|
1109
1600
|
|
|
1110
1601
|
return `${header.join("\n")}\n\n${details.join("\n")}`;
|
|
@@ -1136,29 +1627,85 @@ function rangeDescription(options, bounds) {
|
|
|
1136
1627
|
return bounds.dateString;
|
|
1137
1628
|
}
|
|
1138
1629
|
|
|
1630
|
+
function emptyRangeMessage(
|
|
1631
|
+
options,
|
|
1632
|
+
snapshot,
|
|
1633
|
+
bounds,
|
|
1634
|
+
{
|
|
1635
|
+
sourceStatus = "unchecked-cache",
|
|
1636
|
+
reportTimeMs = Date.now(),
|
|
1637
|
+
} = {},
|
|
1638
|
+
) {
|
|
1639
|
+
const range = rangeDescription(options, bounds);
|
|
1640
|
+
const cutoffMs = snapshotCollectionCutoffMs(snapshot);
|
|
1641
|
+
const hasUncollectedHistory =
|
|
1642
|
+
Number.isFinite(cutoffMs) && bounds.start.getTime() < cutoffMs;
|
|
1643
|
+
const lines = [
|
|
1644
|
+
hasUncollectedHistory
|
|
1645
|
+
? `No model-call events were collected for ${range} (${bounds.timeZone}); history before the snapshot cutoff is outside this snapshot and is not a verified zero.`
|
|
1646
|
+
: `No model-call events found for ${range} (${bounds.timeZone}).`,
|
|
1647
|
+
];
|
|
1648
|
+
const scope = historyScopeLabel(snapshot);
|
|
1649
|
+
if (scope) lines.push(`History: ${scope}`);
|
|
1650
|
+
lines.push(
|
|
1651
|
+
`Snapshot: ${snapshotFreshnessDetail(snapshotFreshness(snapshot, reportTimeMs))}`,
|
|
1652
|
+
);
|
|
1653
|
+
lines.push(sourceStatusLine(sourceStatus));
|
|
1654
|
+
lines.push(`Source: ${sourceLabel(options.input, snapshot)}`);
|
|
1655
|
+
return lines.join("\n");
|
|
1656
|
+
}
|
|
1657
|
+
|
|
1139
1658
|
export async function run(options, { nowMs } = {}) {
|
|
1140
1659
|
const hasInjectedNow = nowMs !== undefined;
|
|
1141
1660
|
const now = new Date(hasInjectedNow ? nowMs : Date.now());
|
|
1661
|
+
const reportTimeMs = now.getTime();
|
|
1142
1662
|
const bounds = boundsForOptions(options, now);
|
|
1143
|
-
const snapshot = await loadSnapshot(options);
|
|
1144
|
-
const
|
|
1663
|
+
const { snapshot, sourceStatus } = await loadSnapshot(options);
|
|
1664
|
+
const freshnessNowMs = hasInjectedNow ? reportTimeMs : Date.now();
|
|
1665
|
+
const analysis = buildRangeAnalysis(
|
|
1666
|
+
snapshot,
|
|
1667
|
+
bounds,
|
|
1668
|
+
{
|
|
1669
|
+
priorBounds: options.view === "trend"
|
|
1670
|
+
? priorPeriodBounds(bounds, options.trendDays)
|
|
1671
|
+
: null,
|
|
1672
|
+
includeTrend: options.view === "trend" && !options.cacheRate,
|
|
1673
|
+
},
|
|
1674
|
+
);
|
|
1675
|
+
let events = filterDayEvents(snapshot, bounds, analysis);
|
|
1145
1676
|
const writingImage = options.view === "trend" && options.image;
|
|
1146
1677
|
const writingEmptyCacheReport = writingImage && options.cacheRate;
|
|
1678
|
+
if (writingImage && !options.cacheRate) {
|
|
1679
|
+
const effectiveEndMs = resolveEffectiveEnd({
|
|
1680
|
+
snapshot,
|
|
1681
|
+
bounds,
|
|
1682
|
+
reportTimeMs,
|
|
1683
|
+
sourceStatus,
|
|
1684
|
+
});
|
|
1685
|
+
// Keep the terminal aggregation and the report's project breakdown on the
|
|
1686
|
+
// same captured event window. The report view model independently applies
|
|
1687
|
+
// the same bound to raw snapshot events for its other panels.
|
|
1688
|
+
events = usageBucketsInRange(
|
|
1689
|
+
{ events },
|
|
1690
|
+
bounds.start.getTime(),
|
|
1691
|
+
effectiveEndMs,
|
|
1692
|
+
);
|
|
1693
|
+
}
|
|
1147
1694
|
if (events.length === 0 && !writingEmptyCacheReport) {
|
|
1148
|
-
return
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1695
|
+
return emptyRangeMessage(options, snapshot, bounds, {
|
|
1696
|
+
sourceStatus,
|
|
1697
|
+
reportTimeMs,
|
|
1698
|
+
});
|
|
1152
1699
|
}
|
|
1153
|
-
const allRows = options.cacheRate
|
|
1700
|
+
const allRows = options.cacheRate || options.view === "cost"
|
|
1154
1701
|
? []
|
|
1155
|
-
: aggregateProjects(snapshot, events, options);
|
|
1702
|
+
: aggregateProjects(snapshot, events, options, analysis);
|
|
1156
1703
|
const rows = allRows.slice(0, options.top);
|
|
1157
1704
|
const outputPath = writingImage
|
|
1158
1705
|
? options.imageOutput ??
|
|
1159
1706
|
resolve(
|
|
1160
1707
|
process.cwd(),
|
|
1161
|
-
`token-ledger-${options.cacheRate ? "cache-report" : options.report ? "report" : "trend"}-${options.trendDays}d.png`,
|
|
1708
|
+
`token-ledger-${options.cacheRate ? "cache-report" : options.report ? "report" : "trend"}-${options.trendDays}d${options.private ? "-private" : ""}.png`,
|
|
1162
1709
|
)
|
|
1163
1710
|
: null;
|
|
1164
1711
|
const imageLabel = options.cacheRate
|
|
@@ -1169,7 +1716,7 @@ export async function run(options, { nowMs } = {}) {
|
|
|
1169
1716
|
if (writingImage) {
|
|
1170
1717
|
process.stderr.write(`Token Ledger: generating ${imageLabel} PNG…\n`);
|
|
1171
1718
|
}
|
|
1172
|
-
const output = render(
|
|
1719
|
+
const output = await render(
|
|
1173
1720
|
options,
|
|
1174
1721
|
snapshot,
|
|
1175
1722
|
bounds,
|
|
@@ -1178,12 +1725,15 @@ export async function run(options, { nowMs } = {}) {
|
|
|
1178
1725
|
allRows,
|
|
1179
1726
|
snapshotFreshness(
|
|
1180
1727
|
snapshot,
|
|
1181
|
-
|
|
1728
|
+
freshnessNowMs,
|
|
1182
1729
|
),
|
|
1730
|
+
{ sourceStatus, reportTimeMs },
|
|
1731
|
+
analysis,
|
|
1183
1732
|
);
|
|
1184
1733
|
if (writingImage) {
|
|
1185
1734
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
1186
1735
|
process.stderr.write(`Token Ledger: encoding ${imageLabel} PNG…\n`);
|
|
1736
|
+
const { writeTrendPng } = await import("./token-ledger-trend-image.mjs");
|
|
1187
1737
|
await writeTrendPng(output, outputPath);
|
|
1188
1738
|
process.stderr.write(`Token Ledger: finished ${imageLabel} PNG.\n`);
|
|
1189
1739
|
const lines = [
|
|
@@ -1220,6 +1770,7 @@ function shouldUseInteractive(options) {
|
|
|
1220
1770
|
return Boolean(
|
|
1221
1771
|
!options.static &&
|
|
1222
1772
|
options.view !== "trend" &&
|
|
1773
|
+
options.view !== "cost" &&
|
|
1223
1774
|
!options.plain &&
|
|
1224
1775
|
!options.legacyPlot &&
|
|
1225
1776
|
!process.env.NO_COLOR &&
|
|
@@ -1230,21 +1781,23 @@ function shouldUseInteractive(options) {
|
|
|
1230
1781
|
|
|
1231
1782
|
async function runInteractive(options) {
|
|
1232
1783
|
const bounds = boundsForOptions(options);
|
|
1233
|
-
const snapshot = await loadSnapshot(options);
|
|
1234
|
-
const
|
|
1784
|
+
const { snapshot, sourceStatus } = await loadSnapshot(options);
|
|
1785
|
+
const reportTimeMs = Date.now();
|
|
1786
|
+
const analysis = buildRangeAnalysis(snapshot, bounds, { includeTrend: false });
|
|
1787
|
+
const events = filterDayEvents(snapshot, bounds, analysis);
|
|
1235
1788
|
if (events.length === 0) {
|
|
1236
|
-
process.stdout.write(
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
].join("\n"));
|
|
1789
|
+
process.stdout.write(`${emptyRangeMessage(options, snapshot, bounds, {
|
|
1790
|
+
sourceStatus,
|
|
1791
|
+
reportTimeMs,
|
|
1792
|
+
})}\n`);
|
|
1241
1793
|
return;
|
|
1242
1794
|
}
|
|
1243
|
-
const allRows = aggregateProjects(snapshot, events, options);
|
|
1795
|
+
const allRows = aggregateProjects(snapshot, events, options, analysis);
|
|
1244
1796
|
await startInteractive({
|
|
1245
1797
|
options,
|
|
1246
1798
|
snapshot,
|
|
1247
|
-
snapshotFreshness: snapshotFreshness(snapshot),
|
|
1799
|
+
snapshotFreshness: snapshotFreshness(snapshot, reportTimeMs),
|
|
1800
|
+
sourceStatus,
|
|
1248
1801
|
bounds,
|
|
1249
1802
|
events,
|
|
1250
1803
|
rows: allRows.slice(0, options.top),
|