tokenusage-cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +39 -19
  2. package/LICENSE +22 -22
  3. package/README.md +94 -94
  4. package/THIRD_PARTY_NOTICES.md +140 -140
  5. package/bin/tokenusage.js +69 -69
  6. package/package.json +59 -59
  7. package/src/cli.js +146 -134
  8. package/src/commands/device-login.js +205 -205
  9. package/src/commands/diagnostics.js +38 -38
  10. package/src/commands/doctor.js +110 -110
  11. package/src/commands/init.js +2484 -2436
  12. package/src/commands/pause.js +77 -0
  13. package/src/commands/sessions.js +29 -29
  14. package/src/commands/status.js +1758 -1747
  15. package/src/commands/sync.js +8048 -7934
  16. package/src/commands/uninstall.js +322 -312
  17. package/src/lib/account-usage-dedup.js +267 -267
  18. package/src/lib/antigravity-paths.js +48 -48
  19. package/src/lib/ark-agent-plan-limits.js +438 -438
  20. package/src/lib/ark-coding-plan-limits.js +537 -537
  21. package/src/lib/async-queue-reader.js +138 -138
  22. package/src/lib/atomic-json.js +27 -27
  23. package/src/lib/bearer-token.js +11 -11
  24. package/src/lib/categorizer-utils.js +232 -232
  25. package/src/lib/claude-categorizer.js +1343 -1343
  26. package/src/lib/claude-config.js +279 -279
  27. package/src/lib/claude-user-identity.js +56 -56
  28. package/src/lib/cli-ui.js +179 -179
  29. package/src/lib/cloud-account.js +541 -541
  30. package/src/lib/cloud-device-token.js +141 -141
  31. package/src/lib/cloud-sync-prefs.js +14 -14
  32. package/src/lib/cloud-usage-reset.js +115 -115
  33. package/src/lib/codex-config.js +595 -595
  34. package/src/lib/codex-context-breakdown.js +806 -806
  35. package/src/lib/codex-model-attribution.js +128 -128
  36. package/src/lib/codex-rollout-parser.js +1471 -1471
  37. package/src/lib/codex-service-tier.js +49 -49
  38. package/src/lib/codex-token-refresh.js +140 -140
  39. package/src/lib/codex-token-usage.js +195 -195
  40. package/src/lib/codex-usage-record.js +195 -195
  41. package/src/lib/collection-pause.js +77 -0
  42. package/src/lib/command-runner.js +313 -313
  43. package/src/lib/commandcode-limits.js +317 -317
  44. package/src/lib/context-health.js +124 -124
  45. package/src/lib/cursor-config.js +605 -605
  46. package/src/lib/cursor-store.js +1918 -1918
  47. package/src/lib/debug-flags.js +7 -7
  48. package/src/lib/devin-limits.js +249 -249
  49. package/src/lib/devin-usage.js +212 -212
  50. package/src/lib/diagnostics.js +383 -383
  51. package/src/lib/doctor.js +385 -385
  52. package/src/lib/dumate.js +95 -95
  53. package/src/lib/fs.js +455 -455
  54. package/src/lib/function-url.js +14 -14
  55. package/src/lib/gemini-config.js +291 -291
  56. package/src/lib/git-outcomes.js +243 -243
  57. package/src/lib/grok-context-breakdown.js +890 -890
  58. package/src/lib/grok-hook.js +360 -360
  59. package/src/lib/grok-limits.js +865 -865
  60. package/src/lib/grok-usage.js +98 -98
  61. package/src/lib/heatmap-compact.js +114 -114
  62. package/src/lib/ingest-sanitize.js +41 -41
  63. package/src/lib/init-flow.js +42 -42
  64. package/src/lib/install-resolver.js +170 -170
  65. package/src/lib/jsonl-lines.js +134 -134
  66. package/src/lib/kimi-model-repair.js +173 -173
  67. package/src/lib/kimi-profile.js +61 -61
  68. package/src/lib/kimi-session.js +216 -216
  69. package/src/lib/local-api.js +3588 -3588
  70. package/src/lib/machine-id.js +238 -238
  71. package/src/lib/multi-install-parser.js +113 -113
  72. package/src/lib/notify-command.js +38 -38
  73. package/src/lib/omp-hook.js +589 -589
  74. package/src/lib/openclaw-cli.js +104 -104
  75. package/src/lib/openclaw-hook.js +375 -375
  76. package/src/lib/openclaw-session-plugin.js +560 -560
  77. package/src/lib/opencode-config.js +121 -121
  78. package/src/lib/opencode-go-limits.js +807 -807
  79. package/src/lib/outcomes-engine.js +253 -253
  80. package/src/lib/passive-mode.js +219 -219
  81. package/src/lib/pet-packages.js +722 -722
  82. package/src/lib/pricing/curated-overrides.json +822 -822
  83. package/src/lib/pricing/index.js +379 -379
  84. package/src/lib/pricing/litellm-fetcher.js +172 -172
  85. package/src/lib/pricing/matcher.js +456 -456
  86. package/src/lib/pricing/seed-snapshot.json +1 -1
  87. package/src/lib/progress.js +76 -76
  88. package/src/lib/project-usage-purge.js +117 -117
  89. package/src/lib/prompt.js +24 -24
  90. package/src/lib/provider-status.js +104 -104
  91. package/src/lib/proxy-env.js +489 -489
  92. package/src/lib/proxy-settings.js +163 -163
  93. package/src/lib/qoder-cn-cli-limits.js +119 -119
  94. package/src/lib/qoder-limits.js +880 -880
  95. package/src/lib/quiet-warnings.js +26 -26
  96. package/src/lib/rollout.js +26005 -25966
  97. package/src/lib/runtime-config.js +9 -4
  98. package/src/lib/scan-roots.js +277 -277
  99. package/src/lib/scheduled-task.js +145 -0
  100. package/src/lib/sentry.js +231 -231
  101. package/src/lib/session-analytics.js +2972 -2972
  102. package/src/lib/session-performance.js +237 -237
  103. package/src/lib/session-upload.js +379 -379
  104. package/src/lib/skill-usage.js +267 -267
  105. package/src/lib/skills-manager.js +1591 -1591
  106. package/src/lib/source-metadata.js +48 -48
  107. package/src/lib/sqlite-reader.js +215 -215
  108. package/src/lib/star-cta.js +73 -73
  109. package/src/lib/subscription-manager.js +293 -293
  110. package/src/lib/subscriptions.js +508 -508
  111. package/src/lib/sync-report.js +138 -0
  112. package/src/lib/telemetry.js +193 -193
  113. package/src/lib/tracker-paths.js +16 -16
  114. package/src/lib/trae-cn-config.js +548 -548
  115. package/src/lib/trae-db.js +127 -127
  116. package/src/lib/trae-snapshot.js +230 -230
  117. package/src/lib/trae-sqlite.js +143 -143
  118. package/src/lib/trae-usage.js +121 -121
  119. package/src/lib/upload-queue.js +134 -134
  120. package/src/lib/upload-throttle.js +171 -171
  121. package/src/lib/usage-limits.js +4552 -4552
  122. package/src/lib/wrapped-aggregator.js +236 -236
  123. package/src/lib/wsl-probe.js +281 -281
  124. package/src/lib/zcode-limits.js +1172 -1172
  125. package/src/lib/zcode-reset-credits.js +66 -66
  126. package/test/audit-fixes.test.js +200 -0
  127. package/test/cli-smoke.test.js +53 -53
  128. package/test/collection-controls.test.js +181 -0
  129. package/test/ingest-sanitize.test.js +162 -162
  130. package/test/quiet-warnings.test.js +53 -53
  131. package/test/session-upload.test.js +427 -427
@@ -1,427 +1,427 @@
1
- "use strict";
2
-
3
- // Golden tests for the session metadata upload against the ingest contract
4
- // (docs/cli-contract.md §2.4 `sessions`): field whitelist, token identity,
5
- // account-level sources, batching/ordering and watermark semantics. The wire
6
- // shape asserted here is what the server validates — any drift on either side
7
- // must fail one of these tests.
8
-
9
- const assert = require("node:assert/strict");
10
- const fs = require("node:fs");
11
- const os = require("node:os");
12
- const path = require("node:path");
13
- const { test, beforeEach, afterEach } = require("node:test");
14
-
15
- const {
16
- basenameKey,
17
- buildSessionUploadRows,
18
- loadWatermarks,
19
- mapSessionRow,
20
- MAX_SESSIONS_PER_BATCH,
21
- selectSessionsForUpload,
22
- uploadSessions,
23
- } = require("../src/lib/session-upload");
24
-
25
- let tmpDir;
26
-
27
- beforeEach(() => {
28
- tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tokenusage-session-upload-"));
29
- });
30
-
31
- afterEach(() => {
32
- fs.rmSync(tmpDir, { recursive: true, force: true });
33
- });
34
-
35
- function validRow(overrides = {}) {
36
- return {
37
- session_id: "abc-123",
38
- source: "claude",
39
- started_at: "2026-10-25T01:00:00.000Z",
40
- ended_at: "2026-10-25T02:00:00.000Z",
41
- project_key: "token-usage",
42
- // Sidecar shape: counters nested in row.tokens, total mirrored on top.
43
- tokens: {
44
- input_tokens: 100,
45
- cached_input_tokens: 10,
46
- cache_creation_input_tokens: 5,
47
- output_tokens: 50,
48
- reasoning_output_tokens: 20,
49
- total_tokens: 165, // 100 + 10 + 5 + 50
50
- },
51
- total_tokens: 165,
52
- model_usage: [
53
- {
54
- model: "glm-5.3",
55
- input_tokens: 100,
56
- cached_input_tokens: 10,
57
- cache_creation_input_tokens: 5,
58
- output_tokens: 50,
59
- reasoning_output_tokens: 20,
60
- total_tokens: 165,
61
- },
62
- ],
63
- // Fields that must NEVER reach the wire:
64
- title: "secret agent-authored title",
65
- project_ref: "/home/user/secret-cwd",
66
- _cache_key: "sha:whatever",
67
- ...overrides,
68
- };
69
- }
70
-
71
- function rowsOnly(request) {
72
- return request.body.sessions;
73
- }
74
-
75
- test("maps a valid row onto the contract whitelist", () => {
76
- const { rows, skipped } = buildSessionUploadRows([validRow()]);
77
- assert.equal(skipped, 0);
78
- assert.equal(rows.length, 1);
79
- const row = rows[0];
80
- assert.deepEqual(Object.keys(row).sort(), [
81
- "billable_total_tokens",
82
- "cache_creation_input_tokens",
83
- "cached_input_tokens",
84
- "ended_at",
85
- "input_tokens",
86
- "models",
87
- "output_tokens",
88
- "project_key",
89
- "reasoning_output_tokens",
90
- "session_id",
91
- "source",
92
- "started_at",
93
- "total_tokens",
94
- ]);
95
- assert.equal(row.total_tokens, 165);
96
- assert.equal(row.billable_total_tokens, null);
97
- assert.deepEqual(row.models, [
98
- {
99
- model: "glm-5.3",
100
- input_tokens: 100,
101
- cached_input_tokens: 10,
102
- cache_creation_input_tokens: 5,
103
- output_tokens: 50,
104
- reasoning_output_tokens: 20,
105
- total_tokens: 165,
106
- },
107
- ]);
108
- });
109
-
110
- test("privacy: title, project_ref and cache keys never reach the payload", () => {
111
- const { rows } = buildSessionUploadRows([validRow()]);
112
- const json = JSON.stringify(rows);
113
- assert.ok(!json.includes("secret agent-authored title"));
114
- assert.ok(!json.includes("/home/user/secret-cwd"));
115
- assert.ok(!json.includes("sha:whatever"));
116
- });
117
-
118
- test("drops account-level sources entirely", () => {
119
- const { rows } = buildSessionUploadRows([
120
- validRow({ source: "cursor", session_id: "c1" }),
121
- validRow({ source: "TRAE-CN ", session_id: "t1" }),
122
- validRow({ source: "claude", session_id: "c2" }),
123
- ]);
124
- assert.deepEqual(rows.map((r) => r.session_id), ["c2"]);
125
- });
126
-
127
- test("drops rows failing the token identity instead of failing the batch", () => {
128
- const { rows, skipped } = buildSessionUploadRows([
129
- validRow({ total_tokens: 999, session_id: "bad-identity" }),
130
- validRow({ session_id: "good" }),
131
- ]);
132
- assert.equal(rows.length, 1);
133
- assert.equal(rows[0].session_id, "good");
134
- assert.equal(skipped, 1);
135
- });
136
-
137
- test("folds zcode/opencode reasoning-outside-output rows into the contract identity", () => {
138
- // Vendor shape (zcode/opencode): total includes reasoning, output does not.
139
- // The contract requires reasoning ⊆ output, so the mapper folds it in and
140
- // total_tokens survives untouched (185 = 100 + 10 + 5 + 50 + 20).
141
- const skewedTokens = {
142
- input_tokens: 100,
143
- cached_input_tokens: 10,
144
- cache_creation_input_tokens: 5,
145
- output_tokens: 50,
146
- reasoning_output_tokens: 20,
147
- total_tokens: 185,
148
- };
149
- const row = validRow({
150
- tokens: skewedTokens,
151
- total_tokens: 185,
152
- model_usage: [{ model: "glm-5.3", ...skewedTokens }],
153
- });
154
- const mapped = mapSessionRow(row);
155
- assert.ok(mapped, "skewed-but-reconcilable row must not be dropped");
156
- assert.equal(mapped.total_tokens, 185);
157
- assert.equal(mapped.output_tokens, 70);
158
- assert.equal(mapped.reasoning_output_tokens, 20);
159
- assert.equal(mapped.models.length, 1);
160
- assert.equal(mapped.models[0].output_tokens, 70);
161
- assert.equal(mapped.models[0].total_tokens, 185);
162
-
163
- // An unreconcilable skew (total matches neither identity shape) still drops.
164
- assert.equal(
165
- mapSessionRow(validRow({ total_tokens: 186, session_id: "bad-skew" })),
166
- null
167
- );
168
- });
169
-
170
- test("drops rows with missing/invalid identity fields", () => {
171
- const badTokens = (patch) => ({ tokens: { ...validRow().tokens, ...patch } });
172
- const cases = [
173
- validRow({ session_id: " " }),
174
- validRow({ session_id: "a/b".repeat(200) }), // path separators + too long
175
- validRow({ session_id: "line\nbreak" }),
176
- validRow({ started_at: "not-a-date" }),
177
- validRow({ ended_at: "" }),
178
- validRow({ started_at: "2026-10-25T03:00:00Z" }), // ended before started
179
- validRow(badTokens({ input_tokens: -1 })),
180
- validRow(badTokens({ input_tokens: 1.5 })),
181
- validRow({ total_tokens: null }),
182
- ];
183
- for (const row of cases) {
184
- assert.equal(mapSessionRow(row), null, JSON.stringify(row.session_id || row));
185
- }
186
- });
187
-
188
- test("falls back to the stable session_hash when no vendor session_id exists", () => {
189
- const { rows } = buildSessionUploadRows([
190
- validRow({ session_id: undefined, session_hash: "af80fc70a69f557" }),
191
- ]);
192
- assert.equal(rows.length, 1);
193
- assert.equal(rows[0].session_id, "af80fc70a69f557");
194
- // And a row with neither identifier is dropped.
195
- const none = buildSessionUploadRows([validRow({ session_id: undefined })]);
196
- assert.equal(none.rows.length, 0);
197
- });
198
-
199
- test("project_key falls back to basename and handles empty forms", () => {
200
- assert.equal(basenameKey("/home/u/proj/"), "proj");
201
- assert.equal(basenameKey("C:\\home\\u\\proj\\"), "proj");
202
- assert.equal(basenameKey("."), null);
203
- assert.equal(basenameKey(".."), null);
204
- assert.equal(basenameKey(""), null);
205
-
206
- const { rows } = buildSessionUploadRows([validRow({ project_key: "/a/b/c" })]);
207
- assert.equal(rows[0].project_key, "c");
208
- const empty = buildSessionUploadRows([validRow({ project_key: "." })]).rows[0];
209
- assert.equal(empty.project_key, null);
210
- });
211
-
212
- test("models are omitted when the sum disagrees with the session total", () => {
213
- const row = validRow({
214
- model_usage: [
215
- { model: "a", input_tokens: 50, cached_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 25, reasoning_output_tokens: 0, total_tokens: 75 },
216
- { model: "b", input_tokens: 50, cached_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 25, reasoning_output_tokens: 0, total_tokens: 90 },
217
- ],
218
- });
219
- const mapped = mapSessionRow(row);
220
- assert.equal(mapped.models, undefined);
221
- });
222
-
223
- test("elided zero-value counters on model rows fill as 0 (fresh sidecars omit them)", () => {
224
- // zcode sidecar rows drop cache_creation/reasoning keys entirely; the
225
- // identity still holds and the wire must carry explicit zeros.
226
- const row = validRow({
227
- model_usage: [
228
- {
229
- model: "glm-5.3",
230
- input_tokens: 100,
231
- cached_input_tokens: 15,
232
- output_tokens: 50,
233
- total_tokens: 165,
234
- },
235
- ],
236
- });
237
- const mapped = mapSessionRow(row);
238
- assert.equal(mapped.models.length, 1);
239
- assert.equal(mapped.models[0].cache_creation_input_tokens, 0);
240
- assert.equal(mapped.models[0].reasoning_output_tokens, 0);
241
- assert.equal(mapped.models[0].total_tokens, 165);
242
- });
243
-
244
- test("sessions with more than 32 models omit the models array (contract cannot accept a partial sum)", () => {
245
- const usage = Array.from({ length: 40 }, (_, i) => ({
246
- model: `model-${i}`,
247
- input_tokens: i + 1,
248
- cached_input_tokens: 0,
249
- cache_creation_input_tokens: 0,
250
- output_tokens: 0,
251
- reasoning_output_tokens: 0,
252
- total_tokens: i + 1,
253
- }));
254
- const row = validRow({
255
- tokens: {
256
- input_tokens: 820, // sum of 1..40
257
- cached_input_tokens: 0,
258
- cache_creation_input_tokens: 0,
259
- output_tokens: 0,
260
- reasoning_output_tokens: 0,
261
- total_tokens: 820,
262
- },
263
- total_tokens: 820,
264
- model_usage: usage,
265
- });
266
- const mapped = mapSessionRow(row);
267
- assert.equal(mapped.models, undefined);
268
- });
269
-
270
- test("watermarks skip equal/older snapshots and admit strictly newer ones", () => {
271
- const watermarkPath = path.join(tmpDir, "watermarks.json");
272
- fs.writeFileSync(
273
- watermarkPath,
274
- JSON.stringify({ "claude|abc-123": "2026-10-25T02:00:00.000Z" }),
275
- );
276
- const watermarks = loadWatermarks(watermarkPath);
277
- const pending = selectSessionsForUpload(
278
- [
279
- validRow(), // equal ended_at → skip
280
- validRow({ ended_at: "2026-10-25T01:00:00.000Z" }), // older → skip
281
- validRow({ ended_at: "2026-10-25T03:00:00.000Z", session_id: "abc-123" }),
282
- validRow({ session_id: "other" }),
283
- ],
284
- watermarks,
285
- );
286
- assert.deepEqual(pending.map((r) => r.session_id + "@" + r.ended_at), [
287
- "other@2026-10-25T02:00:00.000Z",
288
- "abc-123@2026-10-25T03:00:00.000Z",
289
- ]);
290
- });
291
-
292
- test("upload batches at 200 ascending by ended_at and advances watermarks only on success", async () => {
293
- const watermarkPath = path.join(tmpDir, "watermarks.json");
294
- const sessions = [];
295
- for (let i = 0; i < MAX_SESSIONS_PER_BATCH + 50; i++) {
296
- sessions.push(validRow({
297
- session_id: `s-${String(i).padStart(4, "0")}`,
298
- started_at: "2026-10-25T00:00:00.000Z",
299
- ended_at: new Date(Date.parse("2026-10-25T00:00:00.000Z") + i * 1000).toISOString(),
300
- }));
301
- }
302
-
303
- const requests = [];
304
- const fetchImpl = async (url, init) => {
305
- const body = JSON.parse(init.body);
306
- requests.push({ url, body });
307
- return {
308
- ok: true,
309
- json: async () => ({ inserted: 0, sessions_upserted: body.sessions.length, sessions_skipped: 0 }),
310
- };
311
- };
312
-
313
- const result = await uploadSessions({
314
- baseUrl: "https://example.test",
315
- anonKey: null,
316
- deviceToken: "token-1",
317
- sessions,
318
- watermarkPath,
319
- fetchImpl,
320
- });
321
-
322
- assert.equal(result.pending, MAX_SESSIONS_PER_BATCH + 50);
323
- assert.equal(result.uploaded, MAX_SESSIONS_PER_BATCH + 50);
324
- assert.equal(requests.length, 2);
325
- assert.ok(requests[0].url.endsWith("/api/functions/token-usage-ingest"));
326
- assert.equal(rowsOnly(requests[0]).length, MAX_SESSIONS_PER_BATCH);
327
- assert.equal(rowsOnly(requests[1]).length, 50);
328
- // Ascending ended_at across the whole upload.
329
- const all = [...rowsOnly(requests[0]), ...rowsOnly(requests[1])];
330
- for (let i = 1; i < all.length; i++) {
331
- assert.ok(all[i - 1].ended_at <= all[i].ended_at);
332
- }
333
- assert.equal(requests[0].body.upload_id, requests[1].body.upload_id);
334
-
335
- const marks = loadWatermarks(watermarkPath);
336
- assert.equal(marks.size, MAX_SESSIONS_PER_BATCH + 50);
337
- assert.equal(marks.get("claude|s-0000"), all[0].ended_at);
338
- });
339
-
340
- test("a failed batch keeps earlier watermark advances and reports progress", async () => {
341
- const watermarkPath = path.join(tmpDir, "watermarks.json");
342
- const sessions = [
343
- validRow({ session_id: "ok-1" }),
344
- validRow({ session_id: "ok-2" }),
345
- validRow({ session_id: "doomed" }),
346
- ];
347
- let call = 0;
348
- const fetchImpl = async () => {
349
- call += 1;
350
- if (call === 3) return { ok: false, status: 400, text: async () => '{"error":"Invalid session"}' };
351
- return { ok: true, json: async () => ({ inserted: 0, sessions_upserted: 1, sessions_skipped: 0 }) };
352
- };
353
-
354
- await assert.rejects(
355
- uploadSessions({
356
- baseUrl: "https://example.test", deviceToken: "t", sessions, watermarkPath, fetchImpl, batchSize: 1,
357
- }),
358
- (error) => error.status === 400 && error.uploaded === 2,
359
- );
360
-
361
- const marks = loadWatermarks(watermarkPath);
362
- assert.equal(marks.size, 2);
363
- assert.ok(marks.has("claude|ok-1"));
364
- assert.ok(marks.has("claude|ok-2"));
365
- assert.ok(!marks.has("claude|doomed"));
366
- });
367
-
368
- test("batches cut on the 1 MiB body limit as well as the row cap", () => {
369
- const { buildBatches } = require("../src/lib/session-upload");
370
- // Rows padded past 500 KiB each force one row per batch regardless of the
371
- // 200-row cap (a real batch of model-heavy sessions hit 413 this way).
372
- const fatRow = (i) => validRow({
373
- session_id: `fat-${i}`,
374
- project_key: "x".repeat(300 * 1024),
375
- });
376
- const batches = buildBatches([fatRow(1), fatRow(2), fatRow(3)], 200);
377
- assert.equal(batches.length, 3);
378
- for (const batch of batches) assert.equal(batch.length, 1);
379
-
380
- // Small rows still group up to the row cap.
381
- const small = Array.from({ length: 250 }, (_, i) => validRow({ session_id: `s-${i}` }));
382
- const grouped = buildBatches(small, 200);
383
- assert.deepEqual(grouped.map((b) => b.length), [200, 50]);
384
- });
385
-
386
- test("uploadSessions maps RAW sidecar rows internally (no unmapped field ever reaches the wire)", async () => {
387
- const watermarkPath = path.join(tmpDir, "watermarks.json");
388
- const rawSidecarRow = {
389
- ...validRow(),
390
- tokens: validRow().tokens, // counters nested (raw shape)
391
- title: "secret title",
392
- project_ref: "/home/u/secret",
393
- _cache_key: "sha:abc",
394
- };
395
- const seen = [];
396
- const fetchImpl = async (url, init) => {
397
- seen.push(JSON.parse(init.body));
398
- return { ok: true, json: async () => ({ inserted: 0, sessions_upserted: 1 }) };
399
- };
400
- const result = await uploadSessions({
401
- baseUrl: "https://example.test", deviceToken: "t",
402
- sessions: [rawSidecarRow], watermarkPath, fetchImpl,
403
- });
404
- assert.equal(result.uploaded, 1);
405
- const payload = seen[0].sessions[0];
406
- assert.equal(payload.input_tokens, 100); // unnested
407
- assert.ok(!("title" in payload));
408
- assert.ok(!("project_ref" in payload));
409
- assert.ok(!("_cache_key" in payload));
410
- assert.ok(!("tokens" in payload));
411
- });
412
-
413
- test("nothing is sent when every row is already watermarked", async () => {
414
- const watermarkPath = path.join(tmpDir, "watermarks.json");
415
- let called = 0;
416
- const fetchImpl = async () => {
417
- called += 1;
418
- return { ok: true, json: async () => ({}) };
419
- };
420
- const { rows } = buildSessionUploadRows([validRow()]);
421
- fs.writeFileSync(watermarkPath, JSON.stringify({ [`claude|${rows[0].session_id}`]: rows[0].ended_at }));
422
- const result = await uploadSessions({
423
- baseUrl: "https://example.test", deviceToken: "t", sessions: rows, watermarkPath, fetchImpl,
424
- });
425
- assert.equal(called, 0);
426
- assert.equal(result.uploaded, 0);
427
- });
1
+ "use strict";
2
+
3
+ // Golden tests for the session metadata upload against the ingest contract
4
+ // (docs/cli-contract.md §2.4 `sessions`): field whitelist, token identity,
5
+ // account-level sources, batching/ordering and watermark semantics. The wire
6
+ // shape asserted here is what the server validates — any drift on either side
7
+ // must fail one of these tests.
8
+
9
+ const assert = require("node:assert/strict");
10
+ const fs = require("node:fs");
11
+ const os = require("node:os");
12
+ const path = require("node:path");
13
+ const { test, beforeEach, afterEach } = require("node:test");
14
+
15
+ const {
16
+ basenameKey,
17
+ buildSessionUploadRows,
18
+ loadWatermarks,
19
+ mapSessionRow,
20
+ MAX_SESSIONS_PER_BATCH,
21
+ selectSessionsForUpload,
22
+ uploadSessions,
23
+ } = require("../src/lib/session-upload");
24
+
25
+ let tmpDir;
26
+
27
+ beforeEach(() => {
28
+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tokenusage-session-upload-"));
29
+ });
30
+
31
+ afterEach(() => {
32
+ fs.rmSync(tmpDir, { recursive: true, force: true });
33
+ });
34
+
35
+ function validRow(overrides = {}) {
36
+ return {
37
+ session_id: "abc-123",
38
+ source: "claude",
39
+ started_at: "2026-10-25T01:00:00.000Z",
40
+ ended_at: "2026-10-25T02:00:00.000Z",
41
+ project_key: "token-usage",
42
+ // Sidecar shape: counters nested in row.tokens, total mirrored on top.
43
+ tokens: {
44
+ input_tokens: 100,
45
+ cached_input_tokens: 10,
46
+ cache_creation_input_tokens: 5,
47
+ output_tokens: 50,
48
+ reasoning_output_tokens: 20,
49
+ total_tokens: 165, // 100 + 10 + 5 + 50
50
+ },
51
+ total_tokens: 165,
52
+ model_usage: [
53
+ {
54
+ model: "glm-5.3",
55
+ input_tokens: 100,
56
+ cached_input_tokens: 10,
57
+ cache_creation_input_tokens: 5,
58
+ output_tokens: 50,
59
+ reasoning_output_tokens: 20,
60
+ total_tokens: 165,
61
+ },
62
+ ],
63
+ // Fields that must NEVER reach the wire:
64
+ title: "secret agent-authored title",
65
+ project_ref: "/home/user/secret-cwd",
66
+ _cache_key: "sha:whatever",
67
+ ...overrides,
68
+ };
69
+ }
70
+
71
+ function rowsOnly(request) {
72
+ return request.body.sessions;
73
+ }
74
+
75
+ test("maps a valid row onto the contract whitelist", () => {
76
+ const { rows, skipped } = buildSessionUploadRows([validRow()]);
77
+ assert.equal(skipped, 0);
78
+ assert.equal(rows.length, 1);
79
+ const row = rows[0];
80
+ assert.deepEqual(Object.keys(row).sort(), [
81
+ "billable_total_tokens",
82
+ "cache_creation_input_tokens",
83
+ "cached_input_tokens",
84
+ "ended_at",
85
+ "input_tokens",
86
+ "models",
87
+ "output_tokens",
88
+ "project_key",
89
+ "reasoning_output_tokens",
90
+ "session_id",
91
+ "source",
92
+ "started_at",
93
+ "total_tokens",
94
+ ]);
95
+ assert.equal(row.total_tokens, 165);
96
+ assert.equal(row.billable_total_tokens, null);
97
+ assert.deepEqual(row.models, [
98
+ {
99
+ model: "glm-5.3",
100
+ input_tokens: 100,
101
+ cached_input_tokens: 10,
102
+ cache_creation_input_tokens: 5,
103
+ output_tokens: 50,
104
+ reasoning_output_tokens: 20,
105
+ total_tokens: 165,
106
+ },
107
+ ]);
108
+ });
109
+
110
+ test("privacy: title, project_ref and cache keys never reach the payload", () => {
111
+ const { rows } = buildSessionUploadRows([validRow()]);
112
+ const json = JSON.stringify(rows);
113
+ assert.ok(!json.includes("secret agent-authored title"));
114
+ assert.ok(!json.includes("/home/user/secret-cwd"));
115
+ assert.ok(!json.includes("sha:whatever"));
116
+ });
117
+
118
+ test("drops account-level sources entirely", () => {
119
+ const { rows } = buildSessionUploadRows([
120
+ validRow({ source: "cursor", session_id: "c1" }),
121
+ validRow({ source: "TRAE-CN ", session_id: "t1" }),
122
+ validRow({ source: "claude", session_id: "c2" }),
123
+ ]);
124
+ assert.deepEqual(rows.map((r) => r.session_id), ["c2"]);
125
+ });
126
+
127
+ test("drops rows failing the token identity instead of failing the batch", () => {
128
+ const { rows, skipped } = buildSessionUploadRows([
129
+ validRow({ total_tokens: 999, session_id: "bad-identity" }),
130
+ validRow({ session_id: "good" }),
131
+ ]);
132
+ assert.equal(rows.length, 1);
133
+ assert.equal(rows[0].session_id, "good");
134
+ assert.equal(skipped, 1);
135
+ });
136
+
137
+ test("folds zcode/opencode reasoning-outside-output rows into the contract identity", () => {
138
+ // Vendor shape (zcode/opencode): total includes reasoning, output does not.
139
+ // The contract requires reasoning ⊆ output, so the mapper folds it in and
140
+ // total_tokens survives untouched (185 = 100 + 10 + 5 + 50 + 20).
141
+ const skewedTokens = {
142
+ input_tokens: 100,
143
+ cached_input_tokens: 10,
144
+ cache_creation_input_tokens: 5,
145
+ output_tokens: 50,
146
+ reasoning_output_tokens: 20,
147
+ total_tokens: 185,
148
+ };
149
+ const row = validRow({
150
+ tokens: skewedTokens,
151
+ total_tokens: 185,
152
+ model_usage: [{ model: "glm-5.3", ...skewedTokens }],
153
+ });
154
+ const mapped = mapSessionRow(row);
155
+ assert.ok(mapped, "skewed-but-reconcilable row must not be dropped");
156
+ assert.equal(mapped.total_tokens, 185);
157
+ assert.equal(mapped.output_tokens, 70);
158
+ assert.equal(mapped.reasoning_output_tokens, 20);
159
+ assert.equal(mapped.models.length, 1);
160
+ assert.equal(mapped.models[0].output_tokens, 70);
161
+ assert.equal(mapped.models[0].total_tokens, 185);
162
+
163
+ // An unreconcilable skew (total matches neither identity shape) still drops.
164
+ assert.equal(
165
+ mapSessionRow(validRow({ total_tokens: 186, session_id: "bad-skew" })),
166
+ null
167
+ );
168
+ });
169
+
170
+ test("drops rows with missing/invalid identity fields", () => {
171
+ const badTokens = (patch) => ({ tokens: { ...validRow().tokens, ...patch } });
172
+ const cases = [
173
+ validRow({ session_id: " " }),
174
+ validRow({ session_id: "a/b".repeat(200) }), // path separators + too long
175
+ validRow({ session_id: "line\nbreak" }),
176
+ validRow({ started_at: "not-a-date" }),
177
+ validRow({ ended_at: "" }),
178
+ validRow({ started_at: "2026-10-25T03:00:00Z" }), // ended before started
179
+ validRow(badTokens({ input_tokens: -1 })),
180
+ validRow(badTokens({ input_tokens: 1.5 })),
181
+ validRow({ total_tokens: null }),
182
+ ];
183
+ for (const row of cases) {
184
+ assert.equal(mapSessionRow(row), null, JSON.stringify(row.session_id || row));
185
+ }
186
+ });
187
+
188
+ test("falls back to the stable session_hash when no vendor session_id exists", () => {
189
+ const { rows } = buildSessionUploadRows([
190
+ validRow({ session_id: undefined, session_hash: "af80fc70a69f557" }),
191
+ ]);
192
+ assert.equal(rows.length, 1);
193
+ assert.equal(rows[0].session_id, "af80fc70a69f557");
194
+ // And a row with neither identifier is dropped.
195
+ const none = buildSessionUploadRows([validRow({ session_id: undefined })]);
196
+ assert.equal(none.rows.length, 0);
197
+ });
198
+
199
+ test("project_key falls back to basename and handles empty forms", () => {
200
+ assert.equal(basenameKey("/home/u/proj/"), "proj");
201
+ assert.equal(basenameKey("C:\\home\\u\\proj\\"), "proj");
202
+ assert.equal(basenameKey("."), null);
203
+ assert.equal(basenameKey(".."), null);
204
+ assert.equal(basenameKey(""), null);
205
+
206
+ const { rows } = buildSessionUploadRows([validRow({ project_key: "/a/b/c" })]);
207
+ assert.equal(rows[0].project_key, "c");
208
+ const empty = buildSessionUploadRows([validRow({ project_key: "." })]).rows[0];
209
+ assert.equal(empty.project_key, null);
210
+ });
211
+
212
+ test("models are omitted when the sum disagrees with the session total", () => {
213
+ const row = validRow({
214
+ model_usage: [
215
+ { model: "a", input_tokens: 50, cached_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 25, reasoning_output_tokens: 0, total_tokens: 75 },
216
+ { model: "b", input_tokens: 50, cached_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 25, reasoning_output_tokens: 0, total_tokens: 90 },
217
+ ],
218
+ });
219
+ const mapped = mapSessionRow(row);
220
+ assert.equal(mapped.models, undefined);
221
+ });
222
+
223
+ test("elided zero-value counters on model rows fill as 0 (fresh sidecars omit them)", () => {
224
+ // zcode sidecar rows drop cache_creation/reasoning keys entirely; the
225
+ // identity still holds and the wire must carry explicit zeros.
226
+ const row = validRow({
227
+ model_usage: [
228
+ {
229
+ model: "glm-5.3",
230
+ input_tokens: 100,
231
+ cached_input_tokens: 15,
232
+ output_tokens: 50,
233
+ total_tokens: 165,
234
+ },
235
+ ],
236
+ });
237
+ const mapped = mapSessionRow(row);
238
+ assert.equal(mapped.models.length, 1);
239
+ assert.equal(mapped.models[0].cache_creation_input_tokens, 0);
240
+ assert.equal(mapped.models[0].reasoning_output_tokens, 0);
241
+ assert.equal(mapped.models[0].total_tokens, 165);
242
+ });
243
+
244
+ test("sessions with more than 32 models omit the models array (contract cannot accept a partial sum)", () => {
245
+ const usage = Array.from({ length: 40 }, (_, i) => ({
246
+ model: `model-${i}`,
247
+ input_tokens: i + 1,
248
+ cached_input_tokens: 0,
249
+ cache_creation_input_tokens: 0,
250
+ output_tokens: 0,
251
+ reasoning_output_tokens: 0,
252
+ total_tokens: i + 1,
253
+ }));
254
+ const row = validRow({
255
+ tokens: {
256
+ input_tokens: 820, // sum of 1..40
257
+ cached_input_tokens: 0,
258
+ cache_creation_input_tokens: 0,
259
+ output_tokens: 0,
260
+ reasoning_output_tokens: 0,
261
+ total_tokens: 820,
262
+ },
263
+ total_tokens: 820,
264
+ model_usage: usage,
265
+ });
266
+ const mapped = mapSessionRow(row);
267
+ assert.equal(mapped.models, undefined);
268
+ });
269
+
270
+ test("watermarks skip equal/older snapshots and admit strictly newer ones", () => {
271
+ const watermarkPath = path.join(tmpDir, "watermarks.json");
272
+ fs.writeFileSync(
273
+ watermarkPath,
274
+ JSON.stringify({ "claude|abc-123": "2026-10-25T02:00:00.000Z" }),
275
+ );
276
+ const watermarks = loadWatermarks(watermarkPath);
277
+ const pending = selectSessionsForUpload(
278
+ [
279
+ validRow(), // equal ended_at → skip
280
+ validRow({ ended_at: "2026-10-25T01:00:00.000Z" }), // older → skip
281
+ validRow({ ended_at: "2026-10-25T03:00:00.000Z", session_id: "abc-123" }),
282
+ validRow({ session_id: "other" }),
283
+ ],
284
+ watermarks,
285
+ );
286
+ assert.deepEqual(pending.map((r) => r.session_id + "@" + r.ended_at), [
287
+ "other@2026-10-25T02:00:00.000Z",
288
+ "abc-123@2026-10-25T03:00:00.000Z",
289
+ ]);
290
+ });
291
+
292
+ test("upload batches at 200 ascending by ended_at and advances watermarks only on success", async () => {
293
+ const watermarkPath = path.join(tmpDir, "watermarks.json");
294
+ const sessions = [];
295
+ for (let i = 0; i < MAX_SESSIONS_PER_BATCH + 50; i++) {
296
+ sessions.push(validRow({
297
+ session_id: `s-${String(i).padStart(4, "0")}`,
298
+ started_at: "2026-10-25T00:00:00.000Z",
299
+ ended_at: new Date(Date.parse("2026-10-25T00:00:00.000Z") + i * 1000).toISOString(),
300
+ }));
301
+ }
302
+
303
+ const requests = [];
304
+ const fetchImpl = async (url, init) => {
305
+ const body = JSON.parse(init.body);
306
+ requests.push({ url, body });
307
+ return {
308
+ ok: true,
309
+ json: async () => ({ inserted: 0, sessions_upserted: body.sessions.length, sessions_skipped: 0 }),
310
+ };
311
+ };
312
+
313
+ const result = await uploadSessions({
314
+ baseUrl: "https://example.test",
315
+ anonKey: null,
316
+ deviceToken: "token-1",
317
+ sessions,
318
+ watermarkPath,
319
+ fetchImpl,
320
+ });
321
+
322
+ assert.equal(result.pending, MAX_SESSIONS_PER_BATCH + 50);
323
+ assert.equal(result.uploaded, MAX_SESSIONS_PER_BATCH + 50);
324
+ assert.equal(requests.length, 2);
325
+ assert.ok(requests[0].url.endsWith("/api/functions/token-usage-ingest"));
326
+ assert.equal(rowsOnly(requests[0]).length, MAX_SESSIONS_PER_BATCH);
327
+ assert.equal(rowsOnly(requests[1]).length, 50);
328
+ // Ascending ended_at across the whole upload.
329
+ const all = [...rowsOnly(requests[0]), ...rowsOnly(requests[1])];
330
+ for (let i = 1; i < all.length; i++) {
331
+ assert.ok(all[i - 1].ended_at <= all[i].ended_at);
332
+ }
333
+ assert.equal(requests[0].body.upload_id, requests[1].body.upload_id);
334
+
335
+ const marks = loadWatermarks(watermarkPath);
336
+ assert.equal(marks.size, MAX_SESSIONS_PER_BATCH + 50);
337
+ assert.equal(marks.get("claude|s-0000"), all[0].ended_at);
338
+ });
339
+
340
+ test("a failed batch keeps earlier watermark advances and reports progress", async () => {
341
+ const watermarkPath = path.join(tmpDir, "watermarks.json");
342
+ const sessions = [
343
+ validRow({ session_id: "ok-1" }),
344
+ validRow({ session_id: "ok-2" }),
345
+ validRow({ session_id: "doomed" }),
346
+ ];
347
+ let call = 0;
348
+ const fetchImpl = async () => {
349
+ call += 1;
350
+ if (call === 3) return { ok: false, status: 400, text: async () => '{"error":"Invalid session"}' };
351
+ return { ok: true, json: async () => ({ inserted: 0, sessions_upserted: 1, sessions_skipped: 0 }) };
352
+ };
353
+
354
+ await assert.rejects(
355
+ uploadSessions({
356
+ baseUrl: "https://example.test", deviceToken: "t", sessions, watermarkPath, fetchImpl, batchSize: 1,
357
+ }),
358
+ (error) => error.status === 400 && error.uploaded === 2,
359
+ );
360
+
361
+ const marks = loadWatermarks(watermarkPath);
362
+ assert.equal(marks.size, 2);
363
+ assert.ok(marks.has("claude|ok-1"));
364
+ assert.ok(marks.has("claude|ok-2"));
365
+ assert.ok(!marks.has("claude|doomed"));
366
+ });
367
+
368
+ test("batches cut on the 1 MiB body limit as well as the row cap", () => {
369
+ const { buildBatches } = require("../src/lib/session-upload");
370
+ // Rows padded past 500 KiB each force one row per batch regardless of the
371
+ // 200-row cap (a real batch of model-heavy sessions hit 413 this way).
372
+ const fatRow = (i) => validRow({
373
+ session_id: `fat-${i}`,
374
+ project_key: "x".repeat(300 * 1024),
375
+ });
376
+ const batches = buildBatches([fatRow(1), fatRow(2), fatRow(3)], 200);
377
+ assert.equal(batches.length, 3);
378
+ for (const batch of batches) assert.equal(batch.length, 1);
379
+
380
+ // Small rows still group up to the row cap.
381
+ const small = Array.from({ length: 250 }, (_, i) => validRow({ session_id: `s-${i}` }));
382
+ const grouped = buildBatches(small, 200);
383
+ assert.deepEqual(grouped.map((b) => b.length), [200, 50]);
384
+ });
385
+
386
+ test("uploadSessions maps RAW sidecar rows internally (no unmapped field ever reaches the wire)", async () => {
387
+ const watermarkPath = path.join(tmpDir, "watermarks.json");
388
+ const rawSidecarRow = {
389
+ ...validRow(),
390
+ tokens: validRow().tokens, // counters nested (raw shape)
391
+ title: "secret title",
392
+ project_ref: "/home/u/secret",
393
+ _cache_key: "sha:abc",
394
+ };
395
+ const seen = [];
396
+ const fetchImpl = async (url, init) => {
397
+ seen.push(JSON.parse(init.body));
398
+ return { ok: true, json: async () => ({ inserted: 0, sessions_upserted: 1 }) };
399
+ };
400
+ const result = await uploadSessions({
401
+ baseUrl: "https://example.test", deviceToken: "t",
402
+ sessions: [rawSidecarRow], watermarkPath, fetchImpl,
403
+ });
404
+ assert.equal(result.uploaded, 1);
405
+ const payload = seen[0].sessions[0];
406
+ assert.equal(payload.input_tokens, 100); // unnested
407
+ assert.ok(!("title" in payload));
408
+ assert.ok(!("project_ref" in payload));
409
+ assert.ok(!("_cache_key" in payload));
410
+ assert.ok(!("tokens" in payload));
411
+ });
412
+
413
+ test("nothing is sent when every row is already watermarked", async () => {
414
+ const watermarkPath = path.join(tmpDir, "watermarks.json");
415
+ let called = 0;
416
+ const fetchImpl = async () => {
417
+ called += 1;
418
+ return { ok: true, json: async () => ({}) };
419
+ };
420
+ const { rows } = buildSessionUploadRows([validRow()]);
421
+ fs.writeFileSync(watermarkPath, JSON.stringify({ [`claude|${rows[0].session_id}`]: rows[0].ended_at }));
422
+ const result = await uploadSessions({
423
+ baseUrl: "https://example.test", deviceToken: "t", sessions: rows, watermarkPath, fetchImpl,
424
+ });
425
+ assert.equal(called, 0);
426
+ assert.equal(result.uploaded, 0);
427
+ });