tokenusage-cli 0.2.6 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +52 -11
  3. package/bin/tokenusage.js +14 -2
  4. package/package.json +1 -2
  5. package/src/cli.js +325 -78
  6. package/src/commands/device-login.js +3 -3
  7. package/src/commands/init.js +343 -153
  8. package/src/commands/pause.js +11 -23
  9. package/src/commands/reconcile.js +485 -0
  10. package/src/commands/sessions.js +57 -14
  11. package/src/commands/status.js +5 -2
  12. package/src/commands/sync.js +77 -18
  13. package/src/commands/uninstall.js +6 -9
  14. package/src/commands/upload.js +147 -0
  15. package/src/lib/cli-errors.js +33 -0
  16. package/src/lib/cli-ui.js +93 -0
  17. package/src/lib/ingest-sanitize.js +9 -0
  18. package/src/lib/queue-row-normalize.js +55 -0
  19. package/src/lib/rollout.js +39 -2
  20. package/src/lib/scheduled-task.js +597 -75
  21. package/src/lib/session-analytics.js +171 -3
  22. package/src/lib/session-upload.js +6 -0
  23. package/src/lib/sync-report.js +8 -2
  24. package/src/lib/trae-db.js +6 -1
  25. package/src/lib/antigravity-paths.js +0 -48
  26. package/src/lib/async-queue-reader.js +0 -138
  27. package/src/lib/atomic-json.js +0 -27
  28. package/src/lib/bearer-token.js +0 -11
  29. package/src/lib/cloud-account.js +0 -541
  30. package/src/lib/codex-context-breakdown.js +0 -806
  31. package/src/lib/context-health.js +0 -124
  32. package/src/lib/grok-context-breakdown.js +0 -890
  33. package/src/lib/heatmap-compact.js +0 -114
  34. package/src/lib/local-api.js +0 -3574
  35. package/src/lib/outcomes-engine.js +0 -253
  36. package/src/lib/pet-packages.js +0 -722
  37. package/src/lib/skill-usage.js +0 -267
  38. package/src/lib/skills-manager.js +0 -1591
  39. package/src/lib/source-metadata.js +0 -48
  40. package/src/lib/subscription-manager.js +0 -293
  41. package/src/lib/wrapped-aggregator.js +0 -236
  42. package/test/audit-fixes.test.js +0 -249
  43. package/test/branding.test.js +0 -24
  44. package/test/cli-options.test.js +0 -43
  45. package/test/cli-smoke.test.js +0 -53
  46. package/test/cli-ui.test.js +0 -47
  47. package/test/collection-controls.test.js +0 -255
  48. package/test/ingest-sanitize.test.js +0 -162
  49. package/test/packaging.test.js +0 -107
  50. package/test/quiet-warnings.test.js +0 -53
  51. package/test/runtime-copy.test.js +0 -111
  52. package/test/session-upload.test.js +0 -427
  53. package/test/zstd-native.test.js +0 -36
@@ -1,427 +0,0 @@
1
- "use strict";
2
-
3
- // Golden tests for the session metadata upload against the ingest contract
4
- // (docs/cli-contract.md §2.4 `sessions`): field whitelist, token identity,
5
- // account-level sources, batching/ordering and watermark semantics. The wire
6
- // shape asserted here is what the server validates — any drift on either side
7
- // must fail one of these tests.
8
-
9
- const assert = require("node:assert/strict");
10
- const fs = require("node:fs");
11
- const os = require("node:os");
12
- const path = require("node:path");
13
- const { test, beforeEach, afterEach } = require("node:test");
14
-
15
- const {
16
- basenameKey,
17
- buildSessionUploadRows,
18
- loadWatermarks,
19
- mapSessionRow,
20
- MAX_SESSIONS_PER_BATCH,
21
- selectSessionsForUpload,
22
- uploadSessions,
23
- } = require("../src/lib/session-upload");
24
-
25
- let tmpDir;
26
-
27
- beforeEach(() => {
28
- tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tokenusage-session-upload-"));
29
- });
30
-
31
- afterEach(() => {
32
- fs.rmSync(tmpDir, { recursive: true, force: true });
33
- });
34
-
35
- function validRow(overrides = {}) {
36
- return {
37
- session_id: "abc-123",
38
- source: "claude",
39
- started_at: "2026-10-25T01:00:00.000Z",
40
- ended_at: "2026-10-25T02:00:00.000Z",
41
- project_key: "token-usage",
42
- // Sidecar shape: counters nested in row.tokens, total mirrored on top.
43
- tokens: {
44
- input_tokens: 100,
45
- cached_input_tokens: 10,
46
- cache_creation_input_tokens: 5,
47
- output_tokens: 50,
48
- reasoning_output_tokens: 20,
49
- total_tokens: 165, // 100 + 10 + 5 + 50
50
- },
51
- total_tokens: 165,
52
- model_usage: [
53
- {
54
- model: "glm-5.3",
55
- input_tokens: 100,
56
- cached_input_tokens: 10,
57
- cache_creation_input_tokens: 5,
58
- output_tokens: 50,
59
- reasoning_output_tokens: 20,
60
- total_tokens: 165,
61
- },
62
- ],
63
- // Fields that must NEVER reach the wire:
64
- title: "secret agent-authored title",
65
- project_ref: "/home/user/secret-cwd",
66
- _cache_key: "sha:whatever",
67
- ...overrides,
68
- };
69
- }
70
-
71
- function rowsOnly(request) {
72
- return request.body.sessions;
73
- }
74
-
75
- test("maps a valid row onto the contract whitelist", () => {
76
- const { rows, skipped } = buildSessionUploadRows([validRow()]);
77
- assert.equal(skipped, 0);
78
- assert.equal(rows.length, 1);
79
- const row = rows[0];
80
- assert.deepEqual(Object.keys(row).sort(), [
81
- "billable_total_tokens",
82
- "cache_creation_input_tokens",
83
- "cached_input_tokens",
84
- "ended_at",
85
- "input_tokens",
86
- "models",
87
- "output_tokens",
88
- "project_key",
89
- "reasoning_output_tokens",
90
- "session_id",
91
- "source",
92
- "started_at",
93
- "total_tokens",
94
- ]);
95
- assert.equal(row.total_tokens, 165);
96
- assert.equal(row.billable_total_tokens, null);
97
- assert.deepEqual(row.models, [
98
- {
99
- model: "glm-5.3",
100
- input_tokens: 100,
101
- cached_input_tokens: 10,
102
- cache_creation_input_tokens: 5,
103
- output_tokens: 50,
104
- reasoning_output_tokens: 20,
105
- total_tokens: 165,
106
- },
107
- ]);
108
- });
109
-
110
- test("privacy: title, project_ref and cache keys never reach the payload", () => {
111
- const { rows } = buildSessionUploadRows([validRow()]);
112
- const json = JSON.stringify(rows);
113
- assert.ok(!json.includes("secret agent-authored title"));
114
- assert.ok(!json.includes("/home/user/secret-cwd"));
115
- assert.ok(!json.includes("sha:whatever"));
116
- });
117
-
118
- test("drops account-level sources entirely", () => {
119
- const { rows } = buildSessionUploadRows([
120
- validRow({ source: "cursor", session_id: "c1" }),
121
- validRow({ source: "TRAE-CN ", session_id: "t1" }),
122
- validRow({ source: "claude", session_id: "c2" }),
123
- ]);
124
- assert.deepEqual(rows.map((r) => r.session_id), ["c2"]);
125
- });
126
-
127
- test("drops rows failing the token identity instead of failing the batch", () => {
128
- const { rows, skipped } = buildSessionUploadRows([
129
- validRow({ total_tokens: 999, session_id: "bad-identity" }),
130
- validRow({ session_id: "good" }),
131
- ]);
132
- assert.equal(rows.length, 1);
133
- assert.equal(rows[0].session_id, "good");
134
- assert.equal(skipped, 1);
135
- });
136
-
137
- test("folds zcode/opencode reasoning-outside-output rows into the contract identity", () => {
138
- // Vendor shape (zcode/opencode): total includes reasoning, output does not.
139
- // The contract requires reasoning ⊆ output, so the mapper folds it in and
140
- // total_tokens survives untouched (185 = 100 + 10 + 5 + 50 + 20).
141
- const skewedTokens = {
142
- input_tokens: 100,
143
- cached_input_tokens: 10,
144
- cache_creation_input_tokens: 5,
145
- output_tokens: 50,
146
- reasoning_output_tokens: 20,
147
- total_tokens: 185,
148
- };
149
- const row = validRow({
150
- tokens: skewedTokens,
151
- total_tokens: 185,
152
- model_usage: [{ model: "glm-5.3", ...skewedTokens }],
153
- });
154
- const mapped = mapSessionRow(row);
155
- assert.ok(mapped, "skewed-but-reconcilable row must not be dropped");
156
- assert.equal(mapped.total_tokens, 185);
157
- assert.equal(mapped.output_tokens, 70);
158
- assert.equal(mapped.reasoning_output_tokens, 20);
159
- assert.equal(mapped.models.length, 1);
160
- assert.equal(mapped.models[0].output_tokens, 70);
161
- assert.equal(mapped.models[0].total_tokens, 185);
162
-
163
- // An unreconcilable skew (total matches neither identity shape) still drops.
164
- assert.equal(
165
- mapSessionRow(validRow({ total_tokens: 186, session_id: "bad-skew" })),
166
- null
167
- );
168
- });
169
-
170
- test("drops rows with missing/invalid identity fields", () => {
171
- const badTokens = (patch) => ({ tokens: { ...validRow().tokens, ...patch } });
172
- const cases = [
173
- validRow({ session_id: " " }),
174
- validRow({ session_id: "a/b".repeat(200) }), // path separators + too long
175
- validRow({ session_id: "line\nbreak" }),
176
- validRow({ started_at: "not-a-date" }),
177
- validRow({ ended_at: "" }),
178
- validRow({ started_at: "2026-10-25T03:00:00Z" }), // ended before started
179
- validRow(badTokens({ input_tokens: -1 })),
180
- validRow(badTokens({ input_tokens: 1.5 })),
181
- validRow({ total_tokens: null }),
182
- ];
183
- for (const row of cases) {
184
- assert.equal(mapSessionRow(row), null, JSON.stringify(row.session_id || row));
185
- }
186
- });
187
-
188
- test("falls back to the stable session_hash when no vendor session_id exists", () => {
189
- const { rows } = buildSessionUploadRows([
190
- validRow({ session_id: undefined, session_hash: "af80fc70a69f557" }),
191
- ]);
192
- assert.equal(rows.length, 1);
193
- assert.equal(rows[0].session_id, "af80fc70a69f557");
194
- // And a row with neither identifier is dropped.
195
- const none = buildSessionUploadRows([validRow({ session_id: undefined })]);
196
- assert.equal(none.rows.length, 0);
197
- });
198
-
199
- test("project_key falls back to basename and handles empty forms", () => {
200
- assert.equal(basenameKey("/home/u/proj/"), "proj");
201
- assert.equal(basenameKey("C:\\home\\u\\proj\\"), "proj");
202
- assert.equal(basenameKey("."), null);
203
- assert.equal(basenameKey(".."), null);
204
- assert.equal(basenameKey(""), null);
205
-
206
- const { rows } = buildSessionUploadRows([validRow({ project_key: "/a/b/c" })]);
207
- assert.equal(rows[0].project_key, "c");
208
- const empty = buildSessionUploadRows([validRow({ project_key: "." })]).rows[0];
209
- assert.equal(empty.project_key, null);
210
- });
211
-
212
- test("models are omitted when the sum disagrees with the session total", () => {
213
- const row = validRow({
214
- model_usage: [
215
- { model: "a", input_tokens: 50, cached_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 25, reasoning_output_tokens: 0, total_tokens: 75 },
216
- { model: "b", input_tokens: 50, cached_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 25, reasoning_output_tokens: 0, total_tokens: 90 },
217
- ],
218
- });
219
- const mapped = mapSessionRow(row);
220
- assert.equal(mapped.models, undefined);
221
- });
222
-
223
- test("elided zero-value counters on model rows fill as 0 (fresh sidecars omit them)", () => {
224
- // zcode sidecar rows drop cache_creation/reasoning keys entirely; the
225
- // identity still holds and the wire must carry explicit zeros.
226
- const row = validRow({
227
- model_usage: [
228
- {
229
- model: "glm-5.3",
230
- input_tokens: 100,
231
- cached_input_tokens: 15,
232
- output_tokens: 50,
233
- total_tokens: 165,
234
- },
235
- ],
236
- });
237
- const mapped = mapSessionRow(row);
238
- assert.equal(mapped.models.length, 1);
239
- assert.equal(mapped.models[0].cache_creation_input_tokens, 0);
240
- assert.equal(mapped.models[0].reasoning_output_tokens, 0);
241
- assert.equal(mapped.models[0].total_tokens, 165);
242
- });
243
-
244
- test("sessions with more than 32 models omit the models array (contract cannot accept a partial sum)", () => {
245
- const usage = Array.from({ length: 40 }, (_, i) => ({
246
- model: `model-${i}`,
247
- input_tokens: i + 1,
248
- cached_input_tokens: 0,
249
- cache_creation_input_tokens: 0,
250
- output_tokens: 0,
251
- reasoning_output_tokens: 0,
252
- total_tokens: i + 1,
253
- }));
254
- const row = validRow({
255
- tokens: {
256
- input_tokens: 820, // sum of 1..40
257
- cached_input_tokens: 0,
258
- cache_creation_input_tokens: 0,
259
- output_tokens: 0,
260
- reasoning_output_tokens: 0,
261
- total_tokens: 820,
262
- },
263
- total_tokens: 820,
264
- model_usage: usage,
265
- });
266
- const mapped = mapSessionRow(row);
267
- assert.equal(mapped.models, undefined);
268
- });
269
-
270
- test("watermarks skip equal/older snapshots and admit strictly newer ones", () => {
271
- const watermarkPath = path.join(tmpDir, "watermarks.json");
272
- fs.writeFileSync(
273
- watermarkPath,
274
- JSON.stringify({ "claude|abc-123": "2026-10-25T02:00:00.000Z" }),
275
- );
276
- const watermarks = loadWatermarks(watermarkPath);
277
- const pending = selectSessionsForUpload(
278
- [
279
- validRow(), // equal ended_at → skip
280
- validRow({ ended_at: "2026-10-25T01:00:00.000Z" }), // older → skip
281
- validRow({ ended_at: "2026-10-25T03:00:00.000Z", session_id: "abc-123" }),
282
- validRow({ session_id: "other" }),
283
- ],
284
- watermarks,
285
- );
286
- assert.deepEqual(pending.map((r) => r.session_id + "@" + r.ended_at), [
287
- "other@2026-10-25T02:00:00.000Z",
288
- "abc-123@2026-10-25T03:00:00.000Z",
289
- ]);
290
- });
291
-
292
- test("upload batches at 200 ascending by ended_at and advances watermarks only on success", async () => {
293
- const watermarkPath = path.join(tmpDir, "watermarks.json");
294
- const sessions = [];
295
- for (let i = 0; i < MAX_SESSIONS_PER_BATCH + 50; i++) {
296
- sessions.push(validRow({
297
- session_id: `s-${String(i).padStart(4, "0")}`,
298
- started_at: "2026-10-25T00:00:00.000Z",
299
- ended_at: new Date(Date.parse("2026-10-25T00:00:00.000Z") + i * 1000).toISOString(),
300
- }));
301
- }
302
-
303
- const requests = [];
304
- const fetchImpl = async (url, init) => {
305
- const body = JSON.parse(init.body);
306
- requests.push({ url, body });
307
- return {
308
- ok: true,
309
- json: async () => ({ inserted: 0, sessions_upserted: body.sessions.length, sessions_skipped: 0 }),
310
- };
311
- };
312
-
313
- const result = await uploadSessions({
314
- baseUrl: "https://example.test",
315
- anonKey: null,
316
- deviceToken: "token-1",
317
- sessions,
318
- watermarkPath,
319
- fetchImpl,
320
- });
321
-
322
- assert.equal(result.pending, MAX_SESSIONS_PER_BATCH + 50);
323
- assert.equal(result.uploaded, MAX_SESSIONS_PER_BATCH + 50);
324
- assert.equal(requests.length, 2);
325
- assert.ok(requests[0].url.endsWith("/api/functions/token-usage-ingest"));
326
- assert.equal(rowsOnly(requests[0]).length, MAX_SESSIONS_PER_BATCH);
327
- assert.equal(rowsOnly(requests[1]).length, 50);
328
- // Ascending ended_at across the whole upload.
329
- const all = [...rowsOnly(requests[0]), ...rowsOnly(requests[1])];
330
- for (let i = 1; i < all.length; i++) {
331
- assert.ok(all[i - 1].ended_at <= all[i].ended_at);
332
- }
333
- assert.equal(requests[0].body.upload_id, requests[1].body.upload_id);
334
-
335
- const marks = loadWatermarks(watermarkPath);
336
- assert.equal(marks.size, MAX_SESSIONS_PER_BATCH + 50);
337
- assert.equal(marks.get("claude|s-0000"), all[0].ended_at);
338
- });
339
-
340
- test("a failed batch keeps earlier watermark advances and reports progress", async () => {
341
- const watermarkPath = path.join(tmpDir, "watermarks.json");
342
- const sessions = [
343
- validRow({ session_id: "ok-1" }),
344
- validRow({ session_id: "ok-2" }),
345
- validRow({ session_id: "doomed" }),
346
- ];
347
- let call = 0;
348
- const fetchImpl = async () => {
349
- call += 1;
350
- if (call === 3) return { ok: false, status: 400, text: async () => '{"error":"Invalid session"}' };
351
- return { ok: true, json: async () => ({ inserted: 0, sessions_upserted: 1, sessions_skipped: 0 }) };
352
- };
353
-
354
- await assert.rejects(
355
- uploadSessions({
356
- baseUrl: "https://example.test", deviceToken: "t", sessions, watermarkPath, fetchImpl, batchSize: 1,
357
- }),
358
- (error) => error.status === 400 && error.uploaded === 2,
359
- );
360
-
361
- const marks = loadWatermarks(watermarkPath);
362
- assert.equal(marks.size, 2);
363
- assert.ok(marks.has("claude|ok-1"));
364
- assert.ok(marks.has("claude|ok-2"));
365
- assert.ok(!marks.has("claude|doomed"));
366
- });
367
-
368
- test("batches cut on the 1 MiB body limit as well as the row cap", () => {
369
- const { buildBatches } = require("../src/lib/session-upload");
370
- // Rows padded past 500 KiB each force one row per batch regardless of the
371
- // 200-row cap (a real batch of model-heavy sessions hit 413 this way).
372
- const fatRow = (i) => validRow({
373
- session_id: `fat-${i}`,
374
- project_key: "x".repeat(300 * 1024),
375
- });
376
- const batches = buildBatches([fatRow(1), fatRow(2), fatRow(3)], 200);
377
- assert.equal(batches.length, 3);
378
- for (const batch of batches) assert.equal(batch.length, 1);
379
-
380
- // Small rows still group up to the row cap.
381
- const small = Array.from({ length: 250 }, (_, i) => validRow({ session_id: `s-${i}` }));
382
- const grouped = buildBatches(small, 200);
383
- assert.deepEqual(grouped.map((b) => b.length), [200, 50]);
384
- });
385
-
386
- test("uploadSessions maps RAW sidecar rows internally (no unmapped field ever reaches the wire)", async () => {
387
- const watermarkPath = path.join(tmpDir, "watermarks.json");
388
- const rawSidecarRow = {
389
- ...validRow(),
390
- tokens: validRow().tokens, // counters nested (raw shape)
391
- title: "secret title",
392
- project_ref: "/home/u/secret",
393
- _cache_key: "sha:abc",
394
- };
395
- const seen = [];
396
- const fetchImpl = async (url, init) => {
397
- seen.push(JSON.parse(init.body));
398
- return { ok: true, json: async () => ({ inserted: 0, sessions_upserted: 1 }) };
399
- };
400
- const result = await uploadSessions({
401
- baseUrl: "https://example.test", deviceToken: "t",
402
- sessions: [rawSidecarRow], watermarkPath, fetchImpl,
403
- });
404
- assert.equal(result.uploaded, 1);
405
- const payload = seen[0].sessions[0];
406
- assert.equal(payload.input_tokens, 100); // unnested
407
- assert.ok(!("title" in payload));
408
- assert.ok(!("project_ref" in payload));
409
- assert.ok(!("_cache_key" in payload));
410
- assert.ok(!("tokens" in payload));
411
- });
412
-
413
- test("nothing is sent when every row is already watermarked", async () => {
414
- const watermarkPath = path.join(tmpDir, "watermarks.json");
415
- let called = 0;
416
- const fetchImpl = async () => {
417
- called += 1;
418
- return { ok: true, json: async () => ({}) };
419
- };
420
- const { rows } = buildSessionUploadRows([validRow()]);
421
- fs.writeFileSync(watermarkPath, JSON.stringify({ [`claude|${rows[0].session_id}`]: rows[0].ended_at }));
422
- const result = await uploadSessions({
423
- baseUrl: "https://example.test", deviceToken: "t", sessions: rows, watermarkPath, fetchImpl,
424
- });
425
- assert.equal(called, 0);
426
- assert.equal(result.uploaded, 0);
427
- });
@@ -1,36 +0,0 @@
1
- "use strict";
2
-
3
- // 0.2.5 dropped @mongodb-js/zstd: the package engine floor is >=22.19, so
4
- // node:zlib can always decode zstd. These tests exercise the two real decode
5
- // paths (Zed thread blobs and concatenated DeepSeek Harness frames) so a
6
- // future dependency/engine change cannot silently remove zstd support.
7
-
8
- const assert = require("node:assert/strict");
9
- const zlib = require("node:zlib");
10
- const { test } = require("node:test");
11
-
12
- const { decodeDshZstd, decodeZedThreadBlob } = require("../src/lib/rollout");
13
-
14
- test("the supported Node runtime has a built-in zstd decoder", () => {
15
- assert.equal(typeof zlib.zstdDecompressSync, "function");
16
- });
17
-
18
- test("Zed zstd thread blobs decode through the built-in decoder", async () => {
19
- const text = JSON.stringify({ tokens: { input: 10, output: 4 }, model: "zed-test" });
20
- const decoded = await decodeZedThreadBlob({
21
- dataType: "zstd",
22
- data: zlib.zstdCompressSync(Buffer.from(text)),
23
- });
24
- assert.equal(decoded, text);
25
- });
26
-
27
- test("concatenated DeepSeek Harness zstd frames are decoded one by one", async () => {
28
- const first = "{\"seq\":1}\n";
29
- const second = "{\"seq\":2}\n";
30
- const payload = Buffer.concat([
31
- zlib.zstdCompressSync(Buffer.from(first)),
32
- zlib.zstdCompressSync(Buffer.from(second)),
33
- ]);
34
- const decoded = await decodeDshZstd(payload, { maxOutputBytes: 4096 });
35
- assert.equal(decoded.toString("utf8"), first + second);
36
- });