tledger 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +283 -155
- package/bin/token-ledger-cache-data.mjs +492 -0
- package/bin/token-ledger-cache-image.mjs +7 -1147
- package/bin/token-ledger-cache-sections.mjs +848 -0
- package/bin/token-ledger-cost-terminal.mjs +234 -0
- package/bin/token-ledger-image-layout.mjs +20 -0
- package/bin/token-ledger-image-primitives.mjs +192 -0
- package/bin/token-ledger-report-data.mjs +1159 -0
- package/bin/token-ledger-source-status.mjs +31 -0
- package/bin/token-ledger-terminal.mjs +245 -69
- package/bin/token-ledger-trend-image.mjs +2039 -1599
- package/bin/token-ledger-trend-terminal.mjs +273 -150
- package/bin/token-ledger-trend.mjs +262 -210
- package/bin/token-ledger-tui.mjs +180 -51
- package/bin/token-ledger.mjs +747 -194
- package/docs/durable-ledger-operations.md +198 -0
- package/docs/release-notes-0.4.0.md +41 -0
- package/docs/token-ledger-report-7-day.png +0 -0
- package/lib/token-ledger-calendar.mjs +225 -0
- package/lib/token-ledger-collection.mjs +100 -0
- package/lib/token-ledger-importer.mjs +3334 -480
- package/lib/token-ledger-labels.mjs +66 -0
- package/lib/token-ledger-ledger.mjs +6056 -0
- package/lib/token-ledger-quota-contract.mjs +38 -0
- package/lib/token-ledger-range-analysis.mjs +120 -0
- package/lib/token-ledger-rates.mjs +330 -0
- package/lib/token-ledger-snapshot.mjs +336 -35
- package/lib/token-ledger-terminal-text.mjs +11 -0
- package/lib/token-ledger-usage.mjs +339 -33
- package/package.json +13 -10
- package/bin/token-ledger-rates.mjs +0 -65
|
@@ -2,7 +2,7 @@ const MINUTE_MS = 60 * 1_000;
|
|
|
2
2
|
const HOUR_MS = 60 * MINUTE_MS;
|
|
3
3
|
const DAY_MS = 24 * HOUR_MS;
|
|
4
4
|
|
|
5
|
-
export const SNAPSHOT_SCHEMA_VERSION =
|
|
5
|
+
export const SNAPSHOT_SCHEMA_VERSION = 3;
|
|
6
6
|
export const ADAPTIVE_USAGE_RESOLUTIONS_SECONDS = Object.freeze([
|
|
7
7
|
5 * 60,
|
|
8
8
|
15 * 60,
|
|
@@ -21,6 +21,7 @@ export const DEFAULT_USAGE_RESOLUTION_POLICY = Object.freeze([
|
|
|
21
21
|
|
|
22
22
|
const COMPACT_DURING_BUILD_BUCKET_COUNT = 50_000;
|
|
23
23
|
const MAX_BUILD_BUCKET_COUNT = 100_000;
|
|
24
|
+
export const MAX_SAFE_TOKEN_COUNT = Number.MAX_SAFE_INTEGER;
|
|
24
25
|
|
|
25
26
|
const NUMERIC_USAGE_FIELDS = Object.freeze([
|
|
26
27
|
"inputTokens",
|
|
@@ -37,6 +38,242 @@ const FRACTIONAL_COUNT_FIELDS = Object.freeze([
|
|
|
37
38
|
"inputCallCount",
|
|
38
39
|
]);
|
|
39
40
|
|
|
41
|
+
function primitiveNumber(value) {
|
|
42
|
+
try {
|
|
43
|
+
const number = Number.prototype.valueOf.call(value);
|
|
44
|
+
return number === value ? number : null;
|
|
45
|
+
} catch {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function parsedTokenValue(value, allowFractional = false) {
|
|
51
|
+
const number = primitiveNumber(value);
|
|
52
|
+
if (
|
|
53
|
+
number === null ||
|
|
54
|
+
!Number.isFinite(number) ||
|
|
55
|
+
number < 0 ||
|
|
56
|
+
number > MAX_SAFE_TOKEN_COUNT ||
|
|
57
|
+
(!allowFractional && !Number.isSafeInteger(number))
|
|
58
|
+
) {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
return number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function parsedOptionalTokenValue(value, allowFractional = false) {
|
|
65
|
+
return value === undefined
|
|
66
|
+
? 0
|
|
67
|
+
: parsedTokenValue(value, allowFractional);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function tokenValue(value, { allowFractional = false } = {}) {
|
|
71
|
+
return parsedTokenValue(value, allowFractional) ?? 0;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function isValidTokenValue(value, { allowFractional = false } = {}) {
|
|
75
|
+
return parsedTokenValue(value, allowFractional) !== null;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function checkedTokenAdd(
|
|
79
|
+
current,
|
|
80
|
+
contribution,
|
|
81
|
+
{ allowFractional = false } = {},
|
|
82
|
+
) {
|
|
83
|
+
const left = tokenValue(current, { allowFractional: true });
|
|
84
|
+
const right = tokenValue(contribution, { allowFractional });
|
|
85
|
+
const sum = left + right;
|
|
86
|
+
return Number.isFinite(sum) && sum <= MAX_SAFE_TOKEN_COUNT
|
|
87
|
+
? sum
|
|
88
|
+
: MAX_SAFE_TOKEN_COUNT;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Output rendered next to a scaled overall total must shrink by the same
|
|
92
|
+
// factor; saturating it independently would overstate the output share.
|
|
93
|
+
export function scaledOutputTokens(events, totalTokens) {
|
|
94
|
+
const totals = { totalTokens: 0, outputTokens: 0, scale: 1 };
|
|
95
|
+
for (const event of events) {
|
|
96
|
+
if (event?.invalidTokenRecord === true) continue;
|
|
97
|
+
const allowFractional = event?.rangeAllocationEstimated === true;
|
|
98
|
+
const nextTotal = totals.totalTokens +
|
|
99
|
+
tokenValue(event.totalTokens, { allowFractional }) / totals.scale;
|
|
100
|
+
const nextOutput = totals.outputTokens +
|
|
101
|
+
tokenValue(event.outputTokens, { allowFractional }) / totals.scale;
|
|
102
|
+
const scaleFactor = Math.max(
|
|
103
|
+
1,
|
|
104
|
+
nextTotal / MAX_SAFE_TOKEN_COUNT,
|
|
105
|
+
nextOutput / MAX_SAFE_TOKEN_COUNT,
|
|
106
|
+
);
|
|
107
|
+
totals.totalTokens = nextTotal / scaleFactor;
|
|
108
|
+
totals.outputTokens = nextOutput / scaleFactor;
|
|
109
|
+
totals.scale *= scaleFactor;
|
|
110
|
+
}
|
|
111
|
+
if (totals.scale === 1) return totals.outputTokens;
|
|
112
|
+
return totals.totalTokens > 0 && totalTokens > 0
|
|
113
|
+
? (totals.outputTokens / totals.totalTokens) * totalTokens
|
|
114
|
+
: totals.outputTokens;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const TOKEN_PARTITION_SCALE = Symbol("tokenPartitionScale");
|
|
118
|
+
|
|
119
|
+
function tokenPartitionScale(target) {
|
|
120
|
+
return Number.isFinite(target[TOKEN_PARTITION_SCALE]) &&
|
|
121
|
+
target[TOKEN_PARTITION_SCALE] >= 1
|
|
122
|
+
? target[TOKEN_PARTITION_SCALE]
|
|
123
|
+
: 1;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function setTokenPartitionScale(target, scale) {
|
|
127
|
+
Object.defineProperty(target, TOKEN_PARTITION_SCALE, {
|
|
128
|
+
configurable: true,
|
|
129
|
+
enumerable: false,
|
|
130
|
+
value: scale,
|
|
131
|
+
writable: true,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function checkedTokenPartitionAdd(
|
|
136
|
+
target,
|
|
137
|
+
contribution,
|
|
138
|
+
{ detailed = false } = {},
|
|
139
|
+
) {
|
|
140
|
+
const targetScale = tokenPartitionScale(target);
|
|
141
|
+
const value =
|
|
142
|
+
tokenValue(contribution, { allowFractional: true }) / targetScale;
|
|
143
|
+
let nextDetailed = tokenValue(target.detailedTokens, {
|
|
144
|
+
allowFractional: true,
|
|
145
|
+
});
|
|
146
|
+
let nextUnknown = tokenValue(target.unknownBreakdownTokens, {
|
|
147
|
+
allowFractional: true,
|
|
148
|
+
});
|
|
149
|
+
if (detailed) nextDetailed += value;
|
|
150
|
+
else nextUnknown += value;
|
|
151
|
+
const scaleFactor = Math.max(
|
|
152
|
+
1,
|
|
153
|
+
(nextDetailed + nextUnknown) / MAX_SAFE_TOKEN_COUNT,
|
|
154
|
+
);
|
|
155
|
+
target.detailedTokens = nextDetailed / scaleFactor;
|
|
156
|
+
target.unknownBreakdownTokens = nextUnknown / scaleFactor;
|
|
157
|
+
setTokenPartitionScale(target, targetScale * scaleFactor);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function nonNegativeFiniteValue(value) {
|
|
161
|
+
const number = primitiveNumber(value);
|
|
162
|
+
return number !== null && Number.isFinite(number) && number >= 0
|
|
163
|
+
? number
|
|
164
|
+
: 0;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export function isNonNegativeFiniteValue(value) {
|
|
168
|
+
const number = primitiveNumber(value);
|
|
169
|
+
return number !== null && Number.isFinite(number) && number >= 0;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function checkedFiniteAdd(current, contribution) {
|
|
173
|
+
const sum =
|
|
174
|
+
nonNegativeFiniteValue(current) + nonNegativeFiniteValue(contribution);
|
|
175
|
+
return Number.isFinite(sum) ? sum : Number.MAX_VALUE;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export function tokenTotalsReconcile(
|
|
179
|
+
totalTokens,
|
|
180
|
+
inputTokens,
|
|
181
|
+
outputTokens,
|
|
182
|
+
allowFractional,
|
|
183
|
+
) {
|
|
184
|
+
const componentTotal = inputTokens + outputTokens;
|
|
185
|
+
if (
|
|
186
|
+
!Number.isFinite(componentTotal) ||
|
|
187
|
+
componentTotal > MAX_SAFE_TOKEN_COUNT
|
|
188
|
+
) {
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
if (componentTotal === totalTokens) return true;
|
|
192
|
+
if (!allowFractional) return false;
|
|
193
|
+
const tolerance =
|
|
194
|
+
Number.EPSILON * Math.max(1, componentTotal, totalTokens) * 16;
|
|
195
|
+
return Math.abs(componentTotal - totalTokens) <= tolerance;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function invalidTokenUsage(row) {
|
|
199
|
+
const source = Object(row) === row ? { ...row } : {};
|
|
200
|
+
return {
|
|
201
|
+
...source,
|
|
202
|
+
inputTokens: 0,
|
|
203
|
+
cachedInputTokens: 0,
|
|
204
|
+
cacheWriteInputTokens: 0,
|
|
205
|
+
outputTokens: 0,
|
|
206
|
+
reasoningTokens: 0,
|
|
207
|
+
totalTokens: 0,
|
|
208
|
+
toolCalls: 0,
|
|
209
|
+
breakdownAvailable: false,
|
|
210
|
+
invalidTokenRecord: true,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export function normalizeTokenUsage(row) {
|
|
215
|
+
if (!row || Object(row) !== row) return null;
|
|
216
|
+
if (row.invalidTokenRecord === true) return invalidTokenUsage(row);
|
|
217
|
+
|
|
218
|
+
const allowFractional = row.rangeAllocationEstimated === true;
|
|
219
|
+
const totalTokens = parsedTokenValue(row.totalTokens, allowFractional);
|
|
220
|
+
if (totalTokens === null) return invalidTokenUsage(row);
|
|
221
|
+
|
|
222
|
+
const componentValues = [
|
|
223
|
+
parsedOptionalTokenValue(row.inputTokens, allowFractional),
|
|
224
|
+
parsedOptionalTokenValue(row.cachedInputTokens, allowFractional),
|
|
225
|
+
parsedOptionalTokenValue(row.cacheWriteInputTokens, allowFractional),
|
|
226
|
+
parsedOptionalTokenValue(row.outputTokens, allowFractional),
|
|
227
|
+
parsedOptionalTokenValue(row.reasoningTokens, allowFractional),
|
|
228
|
+
];
|
|
229
|
+
const componentsValid = componentValues.every((value) => value !== null);
|
|
230
|
+
const inputTokens = componentValues[0] ?? 0;
|
|
231
|
+
const cachedInputTokens = Math.min(inputTokens, componentValues[1] ?? 0);
|
|
232
|
+
const cacheWriteInputTokens = componentValues[2] ?? 0;
|
|
233
|
+
const outputTokens = componentValues[3] ?? 0;
|
|
234
|
+
const reasoningTokens = Math.min(outputTokens, componentValues[4] ?? 0);
|
|
235
|
+
const componentTotal = inputTokens + outputTokens;
|
|
236
|
+
const breakdownAvailable =
|
|
237
|
+
row.breakdownAvailable !== false &&
|
|
238
|
+
componentsValid &&
|
|
239
|
+
(totalTokens === 0
|
|
240
|
+
? componentTotal === 0
|
|
241
|
+
: tokenTotalsReconcile(
|
|
242
|
+
totalTokens,
|
|
243
|
+
inputTokens,
|
|
244
|
+
outputTokens,
|
|
245
|
+
allowFractional,
|
|
246
|
+
) &&
|
|
247
|
+
(inputTokens > 0 || outputTokens > 0));
|
|
248
|
+
|
|
249
|
+
const normalized = {
|
|
250
|
+
...row,
|
|
251
|
+
inputTokens,
|
|
252
|
+
cachedInputTokens,
|
|
253
|
+
cacheWriteInputTokens,
|
|
254
|
+
outputTokens,
|
|
255
|
+
reasoningTokens,
|
|
256
|
+
totalTokens,
|
|
257
|
+
toolCalls: tokenValue(row.toolCalls, { allowFractional }),
|
|
258
|
+
breakdownAvailable,
|
|
259
|
+
};
|
|
260
|
+
// Older snapshots could retain a positive detailed-call count on a
|
|
261
|
+
// total-only bucket because the source supplied a valid total but no
|
|
262
|
+
// input/output partition. Canonicalize the exported count here so every
|
|
263
|
+
// downstream consumer sees the same coverage contract. A valid explicit
|
|
264
|
+
// zero is meaningful partial coverage and must not fall back to callCount;
|
|
265
|
+
// omitted or malformed legacy values retain the detailed fallback.
|
|
266
|
+
const storedDetailedCallCount = nonNegativeStoredCount(
|
|
267
|
+
row.detailedCallCount,
|
|
268
|
+
allowFractional,
|
|
269
|
+
);
|
|
270
|
+
const callCount = usageCallCount(normalized);
|
|
271
|
+
normalized.detailedCallCount = breakdownAvailable
|
|
272
|
+
? Math.min(callCount, storedDetailedCallCount ?? callCount)
|
|
273
|
+
: 0;
|
|
274
|
+
return normalized;
|
|
275
|
+
}
|
|
276
|
+
|
|
40
277
|
function finiteTimestamp(value) {
|
|
41
278
|
try {
|
|
42
279
|
const timestampMs = Date.parse(String(value ?? ""));
|
|
@@ -56,19 +293,20 @@ function nonNegativeNumber(value) {
|
|
|
56
293
|
}
|
|
57
294
|
|
|
58
295
|
function positiveSafeInteger(value) {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
return Number.isSafeInteger(number) && number > 0 ? number : 0;
|
|
62
|
-
} catch {
|
|
63
|
-
return 0;
|
|
64
|
-
}
|
|
296
|
+
const number = parsedTokenValue(value);
|
|
297
|
+
return number !== null && number > 0 ? number : 0;
|
|
65
298
|
}
|
|
66
299
|
|
|
67
300
|
function positiveNumber(value) {
|
|
68
|
-
const number =
|
|
301
|
+
const number = nonNegativeFiniteValue(value);
|
|
69
302
|
return number > 0 ? number : 0;
|
|
70
303
|
}
|
|
71
304
|
|
|
305
|
+
function nonNegativeStoredCount(value, allowFractional = false) {
|
|
306
|
+
if (value === undefined || value === null) return null;
|
|
307
|
+
return parsedTokenValue(value, allowFractional);
|
|
308
|
+
}
|
|
309
|
+
|
|
72
310
|
function primitiveString(value) {
|
|
73
311
|
try {
|
|
74
312
|
const text = String.prototype.valueOf.call(value);
|
|
@@ -101,6 +339,7 @@ function groupingKey(row, bucketIndex, uniqueKey = "") {
|
|
|
101
339
|
uniqueKey,
|
|
102
340
|
String(row.project ?? "Unlabelled activity"),
|
|
103
341
|
String(row.model ?? "unknown"),
|
|
342
|
+
String(row.rateCardModel ?? row.model ?? "unknown"),
|
|
104
343
|
String(row.effort ?? "unknown"),
|
|
105
344
|
String(row.source ?? "unknown"),
|
|
106
345
|
String(row.useType ?? "unknown"),
|
|
@@ -114,7 +353,7 @@ function newAggregate(row, timestampMs, resolutionMs) {
|
|
|
114
353
|
const rateCardCredits = row.rateCardCredits == null
|
|
115
354
|
? null
|
|
116
355
|
: 0;
|
|
117
|
-
const callCount =
|
|
356
|
+
const callCount = usageCallCount(row);
|
|
118
357
|
const aggregate = {
|
|
119
358
|
timestampMeanMs: timestampMs,
|
|
120
359
|
timestampWeight: callCount,
|
|
@@ -122,6 +361,7 @@ function newAggregate(row, timestampMs, resolutionMs) {
|
|
|
122
361
|
endMs: finiteTimestamp(row.endAt) ?? timestampMs,
|
|
123
362
|
project: String(row.project ?? "Unlabelled activity"),
|
|
124
363
|
model: String(row.model ?? "unknown"),
|
|
364
|
+
rateCardModel: String(row.rateCardModel ?? row.model ?? "unknown"),
|
|
125
365
|
effort: String(row.effort ?? "unknown"),
|
|
126
366
|
source: String(row.source ?? "unknown"),
|
|
127
367
|
useType: String(row.useType ?? "unknown"),
|
|
@@ -138,6 +378,7 @@ function newAggregate(row, timestampMs, resolutionMs) {
|
|
|
138
378
|
callCount: 0,
|
|
139
379
|
detailedCallCount: 0,
|
|
140
380
|
inputCallCount: 0,
|
|
381
|
+
rangeAllocationEstimated: false,
|
|
141
382
|
resolutionSeconds: Math.max(
|
|
142
383
|
nonNegativeNumber(row.resolutionSeconds),
|
|
143
384
|
resolutionMs / 1_000,
|
|
@@ -149,7 +390,9 @@ function newAggregate(row, timestampMs, resolutionMs) {
|
|
|
149
390
|
}
|
|
150
391
|
|
|
151
392
|
function addToAggregate(aggregate, row, timestampMs) {
|
|
152
|
-
const
|
|
393
|
+
const estimated = row.rangeAllocationEstimated === true;
|
|
394
|
+
aggregate.rangeAllocationEstimated ||= estimated;
|
|
395
|
+
const callCount = usageCallCount(row);
|
|
153
396
|
if (aggregate.callCount > 0) {
|
|
154
397
|
const combinedWeight = aggregate.timestampWeight + callCount;
|
|
155
398
|
aggregate.timestampMeanMs +=
|
|
@@ -165,19 +408,46 @@ function addToAggregate(aggregate, row, timestampMs) {
|
|
|
165
408
|
finiteTimestamp(row.endAt) ?? timestampMs,
|
|
166
409
|
);
|
|
167
410
|
for (const field of NUMERIC_USAGE_FIELDS) {
|
|
168
|
-
aggregate[field]
|
|
411
|
+
aggregate[field] = checkedTokenAdd(aggregate[field], row[field], {
|
|
412
|
+
allowFractional: estimated,
|
|
413
|
+
});
|
|
169
414
|
}
|
|
170
|
-
aggregate.
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
415
|
+
aggregate.cachedInputTokens = Math.min(
|
|
416
|
+
aggregate.inputTokens,
|
|
417
|
+
aggregate.cachedInputTokens,
|
|
418
|
+
);
|
|
419
|
+
aggregate.reasoningTokens = Math.min(
|
|
420
|
+
aggregate.outputTokens,
|
|
421
|
+
aggregate.reasoningTokens,
|
|
422
|
+
);
|
|
423
|
+
aggregate.breakdownAvailable =
|
|
424
|
+
aggregate.breakdownAvailable &&
|
|
425
|
+
tokenTotalsReconcile(
|
|
426
|
+
aggregate.totalTokens,
|
|
427
|
+
aggregate.inputTokens,
|
|
428
|
+
aggregate.outputTokens,
|
|
429
|
+
aggregate.rangeAllocationEstimated,
|
|
430
|
+
);
|
|
431
|
+
aggregate.callCount = checkedTokenAdd(aggregate.callCount, callCount, {
|
|
432
|
+
allowFractional: estimated,
|
|
433
|
+
});
|
|
434
|
+
const detailedCallCount = usageDetailedCallCount(row);
|
|
435
|
+
aggregate.detailedCallCount = checkedTokenAdd(
|
|
436
|
+
aggregate.detailedCallCount,
|
|
437
|
+
detailedCallCount,
|
|
438
|
+
{ allowFractional: estimated },
|
|
174
439
|
);
|
|
175
|
-
const inputCallCount =
|
|
176
|
-
aggregate.inputCallCount
|
|
177
|
-
|
|
440
|
+
const inputCallCount = usageInputCallCount(row);
|
|
441
|
+
aggregate.inputCallCount = checkedTokenAdd(
|
|
442
|
+
aggregate.inputCallCount,
|
|
443
|
+
inputCallCount,
|
|
444
|
+
{ allowFractional: estimated },
|
|
178
445
|
);
|
|
179
446
|
if (aggregate.rateCardCredits !== null) {
|
|
180
|
-
aggregate.rateCardCredits
|
|
447
|
+
aggregate.rateCardCredits = checkedFiniteAdd(
|
|
448
|
+
aggregate.rateCardCredits,
|
|
449
|
+
row.rateCardCredits,
|
|
450
|
+
);
|
|
181
451
|
}
|
|
182
452
|
aggregate.resolutionSeconds = Math.max(
|
|
183
453
|
aggregate.resolutionSeconds,
|
|
@@ -187,12 +457,13 @@ function addToAggregate(aggregate, row, timestampMs) {
|
|
|
187
457
|
}
|
|
188
458
|
|
|
189
459
|
function finishAggregate(aggregate) {
|
|
190
|
-
|
|
460
|
+
const result = {
|
|
191
461
|
timestamp: new Date(Math.round(aggregate.timestampMeanMs)).toISOString(),
|
|
192
462
|
startAt: new Date(aggregate.startMs).toISOString(),
|
|
193
463
|
endAt: new Date(aggregate.endMs).toISOString(),
|
|
194
464
|
project: aggregate.project,
|
|
195
465
|
model: aggregate.model,
|
|
466
|
+
rateCardModel: aggregate.rateCardModel,
|
|
196
467
|
effort: aggregate.effort,
|
|
197
468
|
source: aggregate.source,
|
|
198
469
|
useType: aggregate.useType,
|
|
@@ -212,6 +483,10 @@ function finishAggregate(aggregate) {
|
|
|
212
483
|
threadIds: [...aggregate.threadIds].sort(),
|
|
213
484
|
resolutionSeconds: aggregate.resolutionSeconds,
|
|
214
485
|
};
|
|
486
|
+
if (aggregate.rangeAllocationEstimated) {
|
|
487
|
+
result.rangeAllocationEstimated = true;
|
|
488
|
+
}
|
|
489
|
+
return result;
|
|
215
490
|
}
|
|
216
491
|
|
|
217
492
|
function addRowToGroups(
|
|
@@ -233,7 +508,12 @@ function addRowToGroups(
|
|
|
233
508
|
function aggregateRows(rows, resolutionForRow) {
|
|
234
509
|
const groups = new Map();
|
|
235
510
|
let index = 0;
|
|
236
|
-
for (const
|
|
511
|
+
for (const sourceRow of rows) {
|
|
512
|
+
const row = normalizeTokenUsage(sourceRow);
|
|
513
|
+
if (!row || row.invalidTokenRecord === true) {
|
|
514
|
+
index += 1;
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
237
517
|
const timestampMs = finiteTimestamp(row?.timestamp);
|
|
238
518
|
if (timestampMs === null) {
|
|
239
519
|
index += 1;
|
|
@@ -292,7 +572,12 @@ export function buildUsageBuckets(
|
|
|
292
572
|
let minimumResolutionMs = 0;
|
|
293
573
|
let nextResolutionIndex = 0;
|
|
294
574
|
let index = 0;
|
|
295
|
-
for (const
|
|
575
|
+
for (const sourceRow of rows) {
|
|
576
|
+
const row = normalizeTokenUsage(sourceRow);
|
|
577
|
+
if (!row || row.invalidTokenRecord === true) {
|
|
578
|
+
index += 1;
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
296
581
|
const timestampMs = finiteTimestamp(row?.timestamp);
|
|
297
582
|
if (timestampMs === null) {
|
|
298
583
|
index += 1;
|
|
@@ -349,7 +634,13 @@ export function coarsenUsageBuckets(buckets, resolutionSeconds) {
|
|
|
349
634
|
}
|
|
350
635
|
|
|
351
636
|
export function usageBuckets(snapshot = {}) {
|
|
352
|
-
|
|
637
|
+
if (!Array.isArray(snapshot.events)) return [];
|
|
638
|
+
return [...snapshot.events].flatMap((row) => {
|
|
639
|
+
const normalized = normalizeTokenUsage(row);
|
|
640
|
+
return normalized && normalized.invalidTokenRecord !== true
|
|
641
|
+
? [normalized]
|
|
642
|
+
: [];
|
|
643
|
+
});
|
|
353
644
|
}
|
|
354
645
|
|
|
355
646
|
function usageBucketInterval(bucket) {
|
|
@@ -377,6 +668,11 @@ function firstBoundaryAfter(boundaries, value) {
|
|
|
377
668
|
}
|
|
378
669
|
|
|
379
670
|
function sliceUsageBucket(bucket, startMs, endMs, fraction) {
|
|
671
|
+
const rangeAllocationOrigin = bucket?.rangeAllocationOrigin ?? {
|
|
672
|
+
inputTokens: nonNegativeNumber(bucket?.inputTokens),
|
|
673
|
+
totalTokens: nonNegativeNumber(bucket?.totalTokens),
|
|
674
|
+
callCount: usageCallCount(bucket),
|
|
675
|
+
};
|
|
380
676
|
const fragment = {
|
|
381
677
|
...bucket,
|
|
382
678
|
timestamp: new Date(Math.round(startMs + (endMs - startMs - 1) / 2))
|
|
@@ -386,12 +682,13 @@ function sliceUsageBucket(bucket, startMs, endMs, fraction) {
|
|
|
386
682
|
rangeAllocationEstimated: true,
|
|
387
683
|
rangeAllocationFraction:
|
|
388
684
|
(positiveNumber(bucket?.rangeAllocationFraction) || 1) * fraction,
|
|
685
|
+
rangeAllocationOrigin,
|
|
389
686
|
};
|
|
390
687
|
for (const field of NUMERIC_USAGE_FIELDS) {
|
|
391
|
-
fragment[field] =
|
|
688
|
+
fragment[field] = tokenValue(bucket?.[field], { allowFractional: true }) * fraction;
|
|
392
689
|
}
|
|
393
690
|
if (bucket?.rateCardCredits != null) {
|
|
394
|
-
fragment.rateCardCredits =
|
|
691
|
+
fragment.rateCardCredits = nonNegativeFiniteValue(bucket.rateCardCredits) * fraction;
|
|
395
692
|
}
|
|
396
693
|
for (const field of FRACTIONAL_COUNT_FIELDS) {
|
|
397
694
|
const value = field === "callCount"
|
|
@@ -418,7 +715,9 @@ export function splitUsageBucketsAtBoundaries(buckets, boundaryValues) {
|
|
|
418
715
|
if (boundaries.length === 0) return Array.isArray(buckets) ? [...buckets] : [];
|
|
419
716
|
|
|
420
717
|
const fragments = [];
|
|
421
|
-
for (const
|
|
718
|
+
for (const sourceBucket of Array.isArray(buckets) ? buckets : []) {
|
|
719
|
+
const bucket = normalizeTokenUsage(sourceBucket);
|
|
720
|
+
if (!bucket || bucket.invalidTokenRecord === true) continue;
|
|
422
721
|
const interval = usageBucketInterval(bucket);
|
|
423
722
|
if (interval === null) {
|
|
424
723
|
fragments.push(bucket);
|
|
@@ -476,6 +775,7 @@ export function usageBucketsInRange(snapshot, startValue, endValue) {
|
|
|
476
775
|
|
|
477
776
|
export function usageCallCount(bucket) {
|
|
478
777
|
if (bucket == null || Object(bucket) !== bucket) return 0;
|
|
778
|
+
if (bucket.invalidTokenRecord === true) return 0;
|
|
479
779
|
const stored = bucket.rangeAllocationEstimated === true
|
|
480
780
|
? positiveNumber(bucket.callCount)
|
|
481
781
|
: positiveSafeInteger(bucket.callCount);
|
|
@@ -483,17 +783,18 @@ export function usageCallCount(bucket) {
|
|
|
483
783
|
}
|
|
484
784
|
|
|
485
785
|
export function usageDetailedCallCount(bucket) {
|
|
786
|
+
if (bucket?.invalidTokenRecord === true) return 0;
|
|
787
|
+
if (bucket?.breakdownAvailable === false) return 0;
|
|
486
788
|
const callCount = usageCallCount(bucket);
|
|
487
|
-
const stored =
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
return Math.min(
|
|
491
|
-
callCount,
|
|
492
|
-
stored || (bucket?.breakdownAvailable === false ? 0 : callCount),
|
|
789
|
+
const stored = nonNegativeStoredCount(
|
|
790
|
+
bucket?.detailedCallCount,
|
|
791
|
+
bucket?.rangeAllocationEstimated === true,
|
|
493
792
|
);
|
|
793
|
+
return Math.min(callCount, stored ?? callCount);
|
|
494
794
|
}
|
|
495
795
|
|
|
496
796
|
export function usageInputCallCount(bucket) {
|
|
797
|
+
if (bucket?.invalidTokenRecord === true) return 0;
|
|
497
798
|
const callCount = usageCallCount(bucket);
|
|
498
799
|
const stored = bucket?.rangeAllocationEstimated === true
|
|
499
800
|
? positiveNumber(bucket.inputCallCount)
|
|
@@ -512,7 +813,12 @@ export function usageBucketStats(buckets) {
|
|
|
512
813
|
const rows = Array.isArray(buckets) ? buckets : [];
|
|
513
814
|
return {
|
|
514
815
|
bucketCount: rows.length,
|
|
515
|
-
callCount: rows.reduce(
|
|
816
|
+
callCount: rows.reduce(
|
|
817
|
+
(sum, row) => checkedTokenAdd(sum, usageCallCount(row), {
|
|
818
|
+
allowFractional: row?.rangeAllocationEstimated === true,
|
|
819
|
+
}),
|
|
820
|
+
0,
|
|
821
|
+
),
|
|
516
822
|
maximumResolutionSeconds: rows.reduce(
|
|
517
823
|
(maximum, row) => Math.max(
|
|
518
824
|
maximum,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tledger",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "A local-only terminal dashboard for Codex token usage",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -36,20 +36,23 @@
|
|
|
36
36
|
},
|
|
37
37
|
"scripts": {
|
|
38
38
|
"test": "node --test tests/*.test.mjs",
|
|
39
|
-
"
|
|
40
|
-
"
|
|
41
|
-
"
|
|
39
|
+
"test:stress": "node --test tests/stress/*.test.mjs",
|
|
40
|
+
"test:all": "node tools/run-tests.mjs all",
|
|
41
|
+
"cli:test": "node --test tests/cli.smoke.test.mjs",
|
|
42
|
+
"lint": "eslint .",
|
|
43
|
+
"usage:snapshot": "node lib/token-ledger-importer.mjs --output outputs/token-ledger-snapshot-v3.json.gz",
|
|
42
44
|
"usage:day": "node bin/token-ledger.mjs day",
|
|
43
45
|
"usage:week": "node bin/token-ledger.mjs week",
|
|
44
46
|
"verify:release": "node tools/verify-release.mjs",
|
|
45
|
-
"prepublishOnly": "npm test && npm run verify:release",
|
|
46
|
-
"
|
|
47
|
-
"
|
|
47
|
+
"prepublishOnly": "npm run test:all && npm run lint && npm run verify:release",
|
|
48
|
+
"benchmark:refresh": "node tools/benchmark-importer.mjs",
|
|
49
|
+
"test:fast": "node tools/run-tests.mjs fast",
|
|
50
|
+
"test:integration": "node tools/run-tests.mjs integration",
|
|
51
|
+
"check": "npm test && npm run lint"
|
|
48
52
|
},
|
|
49
53
|
"devDependencies": {
|
|
50
|
-
"@
|
|
51
|
-
"eslint": "9.39.4"
|
|
52
|
-
"oxlint": "1.79.0"
|
|
54
|
+
"@eslint/js": "9.39.4",
|
|
55
|
+
"eslint": "9.39.4"
|
|
53
56
|
},
|
|
54
57
|
"type": "module",
|
|
55
58
|
"dependencies": {
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
// Pricing weights are used only for the separate attribution lens. They are
|
|
2
|
-
// never used to scale or relabel the actual-token columns.
|
|
3
|
-
export const RATE_CARD_AS_OF = "2026-08-17";
|
|
4
|
-
|
|
5
|
-
// Fast mode (service tier "priority") debits the plan limit at a higher rate.
|
|
6
|
-
export const FAST_MODE_MULTIPLIER = 1.5;
|
|
7
|
-
|
|
8
|
-
export const RATE_CARD = {
|
|
9
|
-
"gpt-5.6-sol": { input: 125, cached: 12.5, output: 750 },
|
|
10
|
-
"gpt-5.6-terra": { input: 50, cached: 5, output: 300 },
|
|
11
|
-
"gpt-5.6-luna": { input: 5, cached: 0.5, output: 30 },
|
|
12
|
-
"gpt-5.5": { input: 125, cached: 12.5, output: 750 },
|
|
13
|
-
"gpt-5.5-cyber": { input: 500, cached: 50, output: 3_000 },
|
|
14
|
-
"gpt-5.4": { input: 62.5, cached: 6.25, output: 375 },
|
|
15
|
-
"gpt-5.4-mini": { input: 18.75, cached: 1.875, output: 113 },
|
|
16
|
-
"gpt-5.3-codex": { input: 43.75, cached: 4.375, output: 350 },
|
|
17
|
-
"gpt-5.2": { input: 43.75, cached: 4.375, output: 350 },
|
|
18
|
-
};
|
|
19
|
-
|
|
20
|
-
export function normalizeModel(model) {
|
|
21
|
-
// Collapse underscore and whitespace separators to dashes so variants like
|
|
22
|
-
// "gpt-5.4 mini" resolve to their own rate-card entry. Keep in lockstep
|
|
23
|
-
// with normalizeModel in lib/token-ledger-importer.mjs.
|
|
24
|
-
const value = String(model || "unknown")
|
|
25
|
-
.trim()
|
|
26
|
-
.toLowerCase()
|
|
27
|
-
.replace(/[\s_]+/g, "-");
|
|
28
|
-
if (RATE_CARD[value]) return value;
|
|
29
|
-
if (value.startsWith("gpt-5.6-sol")) return "gpt-5.6-sol";
|
|
30
|
-
if (value.startsWith("gpt-5.6-terra")) return "gpt-5.6-terra";
|
|
31
|
-
if (value.startsWith("gpt-5.6-luna")) return "gpt-5.6-luna";
|
|
32
|
-
if (value.startsWith("gpt-5.5-cyber")) return "gpt-5.5-cyber";
|
|
33
|
-
if (value.startsWith("gpt-5.5")) return "gpt-5.5";
|
|
34
|
-
if (value.startsWith("gpt-5.4-mini")) return "gpt-5.4-mini";
|
|
35
|
-
if (value.startsWith("gpt-5.4")) return "gpt-5.4";
|
|
36
|
-
if (value.startsWith("gpt-5.3-codex")) return "gpt-5.3-codex";
|
|
37
|
-
if (value.startsWith("gpt-5.2")) return "gpt-5.2";
|
|
38
|
-
return value || "unknown";
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
function hasDetailedBreakdown(usage) {
|
|
42
|
-
const totalTokens = Number(usage.totalTokens) || 0;
|
|
43
|
-
const inputTokens = Number(usage.inputTokens) || 0;
|
|
44
|
-
const outputTokens = Number(usage.outputTokens) || 0;
|
|
45
|
-
if (totalTokens === 0) return true;
|
|
46
|
-
return (
|
|
47
|
-
inputTokens + outputTokens === totalTokens &&
|
|
48
|
-
(inputTokens > 0 || outputTokens > 0)
|
|
49
|
-
);
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export function creditsForUsage(model, usage) {
|
|
53
|
-
if (!hasDetailedBreakdown(usage)) return null;
|
|
54
|
-
const rate = RATE_CARD[normalizeModel(model)];
|
|
55
|
-
if (!rate) return null;
|
|
56
|
-
const inputTokens = Math.max(0, Number(usage.inputTokens) || 0);
|
|
57
|
-
const cachedInputTokens = Math.max(0, Number(usage.cachedInputTokens) || 0);
|
|
58
|
-
const outputTokens = Math.max(0, Number(usage.outputTokens) || 0);
|
|
59
|
-
const cached = Math.min(inputTokens, cachedInputTokens);
|
|
60
|
-
const uncached = Math.max(0, inputTokens - cached);
|
|
61
|
-
return (
|
|
62
|
-
(uncached * rate.input + cached * rate.cached + outputTokens * rate.output) /
|
|
63
|
-
1_000_000
|
|
64
|
-
);
|
|
65
|
-
}
|