tokenusage-cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +39 -19
  2. package/LICENSE +22 -22
  3. package/README.md +94 -94
  4. package/THIRD_PARTY_NOTICES.md +140 -140
  5. package/bin/tokenusage.js +69 -69
  6. package/package.json +59 -59
  7. package/src/cli.js +146 -134
  8. package/src/commands/device-login.js +205 -205
  9. package/src/commands/diagnostics.js +38 -38
  10. package/src/commands/doctor.js +110 -110
  11. package/src/commands/init.js +2484 -2436
  12. package/src/commands/pause.js +77 -0
  13. package/src/commands/sessions.js +29 -29
  14. package/src/commands/status.js +1758 -1747
  15. package/src/commands/sync.js +8048 -7934
  16. package/src/commands/uninstall.js +322 -312
  17. package/src/lib/account-usage-dedup.js +267 -267
  18. package/src/lib/antigravity-paths.js +48 -48
  19. package/src/lib/ark-agent-plan-limits.js +438 -438
  20. package/src/lib/ark-coding-plan-limits.js +537 -537
  21. package/src/lib/async-queue-reader.js +138 -138
  22. package/src/lib/atomic-json.js +27 -27
  23. package/src/lib/bearer-token.js +11 -11
  24. package/src/lib/categorizer-utils.js +232 -232
  25. package/src/lib/claude-categorizer.js +1343 -1343
  26. package/src/lib/claude-config.js +279 -279
  27. package/src/lib/claude-user-identity.js +56 -56
  28. package/src/lib/cli-ui.js +179 -179
  29. package/src/lib/cloud-account.js +541 -541
  30. package/src/lib/cloud-device-token.js +141 -141
  31. package/src/lib/cloud-sync-prefs.js +14 -14
  32. package/src/lib/cloud-usage-reset.js +115 -115
  33. package/src/lib/codex-config.js +595 -595
  34. package/src/lib/codex-context-breakdown.js +806 -806
  35. package/src/lib/codex-model-attribution.js +128 -128
  36. package/src/lib/codex-rollout-parser.js +1471 -1471
  37. package/src/lib/codex-service-tier.js +49 -49
  38. package/src/lib/codex-token-refresh.js +140 -140
  39. package/src/lib/codex-token-usage.js +195 -195
  40. package/src/lib/codex-usage-record.js +195 -195
  41. package/src/lib/collection-pause.js +77 -0
  42. package/src/lib/command-runner.js +313 -313
  43. package/src/lib/commandcode-limits.js +317 -317
  44. package/src/lib/context-health.js +124 -124
  45. package/src/lib/cursor-config.js +605 -605
  46. package/src/lib/cursor-store.js +1918 -1918
  47. package/src/lib/debug-flags.js +7 -7
  48. package/src/lib/devin-limits.js +249 -249
  49. package/src/lib/devin-usage.js +212 -212
  50. package/src/lib/diagnostics.js +383 -383
  51. package/src/lib/doctor.js +385 -385
  52. package/src/lib/dumate.js +95 -95
  53. package/src/lib/fs.js +455 -455
  54. package/src/lib/function-url.js +14 -14
  55. package/src/lib/gemini-config.js +291 -291
  56. package/src/lib/git-outcomes.js +243 -243
  57. package/src/lib/grok-context-breakdown.js +890 -890
  58. package/src/lib/grok-hook.js +360 -360
  59. package/src/lib/grok-limits.js +865 -865
  60. package/src/lib/grok-usage.js +98 -98
  61. package/src/lib/heatmap-compact.js +114 -114
  62. package/src/lib/ingest-sanitize.js +41 -41
  63. package/src/lib/init-flow.js +42 -42
  64. package/src/lib/install-resolver.js +170 -170
  65. package/src/lib/jsonl-lines.js +134 -134
  66. package/src/lib/kimi-model-repair.js +173 -173
  67. package/src/lib/kimi-profile.js +61 -61
  68. package/src/lib/kimi-session.js +216 -216
  69. package/src/lib/local-api.js +3588 -3588
  70. package/src/lib/machine-id.js +238 -238
  71. package/src/lib/multi-install-parser.js +113 -113
  72. package/src/lib/notify-command.js +38 -38
  73. package/src/lib/omp-hook.js +589 -589
  74. package/src/lib/openclaw-cli.js +104 -104
  75. package/src/lib/openclaw-hook.js +375 -375
  76. package/src/lib/openclaw-session-plugin.js +560 -560
  77. package/src/lib/opencode-config.js +121 -121
  78. package/src/lib/opencode-go-limits.js +807 -807
  79. package/src/lib/outcomes-engine.js +253 -253
  80. package/src/lib/passive-mode.js +219 -219
  81. package/src/lib/pet-packages.js +722 -722
  82. package/src/lib/pricing/curated-overrides.json +822 -822
  83. package/src/lib/pricing/index.js +379 -379
  84. package/src/lib/pricing/litellm-fetcher.js +172 -172
  85. package/src/lib/pricing/matcher.js +456 -456
  86. package/src/lib/pricing/seed-snapshot.json +1 -1
  87. package/src/lib/progress.js +76 -76
  88. package/src/lib/project-usage-purge.js +117 -117
  89. package/src/lib/prompt.js +24 -24
  90. package/src/lib/provider-status.js +104 -104
  91. package/src/lib/proxy-env.js +489 -489
  92. package/src/lib/proxy-settings.js +163 -163
  93. package/src/lib/qoder-cn-cli-limits.js +119 -119
  94. package/src/lib/qoder-limits.js +880 -880
  95. package/src/lib/quiet-warnings.js +26 -26
  96. package/src/lib/rollout.js +26005 -25966
  97. package/src/lib/runtime-config.js +9 -4
  98. package/src/lib/scan-roots.js +277 -277
  99. package/src/lib/scheduled-task.js +145 -0
  100. package/src/lib/sentry.js +231 -231
  101. package/src/lib/session-analytics.js +2972 -2972
  102. package/src/lib/session-performance.js +237 -237
  103. package/src/lib/session-upload.js +379 -379
  104. package/src/lib/skill-usage.js +267 -267
  105. package/src/lib/skills-manager.js +1591 -1591
  106. package/src/lib/source-metadata.js +48 -48
  107. package/src/lib/sqlite-reader.js +215 -215
  108. package/src/lib/star-cta.js +73 -73
  109. package/src/lib/subscription-manager.js +293 -293
  110. package/src/lib/subscriptions.js +508 -508
  111. package/src/lib/sync-report.js +138 -0
  112. package/src/lib/telemetry.js +193 -193
  113. package/src/lib/tracker-paths.js +16 -16
  114. package/src/lib/trae-cn-config.js +548 -548
  115. package/src/lib/trae-db.js +127 -127
  116. package/src/lib/trae-snapshot.js +230 -230
  117. package/src/lib/trae-sqlite.js +143 -143
  118. package/src/lib/trae-usage.js +121 -121
  119. package/src/lib/upload-queue.js +134 -134
  120. package/src/lib/upload-throttle.js +171 -171
  121. package/src/lib/usage-limits.js +4552 -4552
  122. package/src/lib/wrapped-aggregator.js +236 -236
  123. package/src/lib/wsl-probe.js +281 -281
  124. package/src/lib/zcode-limits.js +1172 -1172
  125. package/src/lib/zcode-reset-credits.js +66 -66
  126. package/test/audit-fixes.test.js +200 -0
  127. package/test/cli-smoke.test.js +53 -53
  128. package/test/collection-controls.test.js +181 -0
  129. package/test/ingest-sanitize.test.js +162 -162
  130. package/test/quiet-warnings.test.js +53 -53
  131. package/test/session-upload.test.js +427 -427
@@ -1,379 +1,379 @@
1
- "use strict";
2
-
3
- // Session metadata upload for the token-usage ingest (docs/cli-contract.md
4
- // §2.4 `sessions`). Maps the local session-analytics sidecar rows onto the
5
- // contract's wire shape, filters through per-(source, session_id) ended_at
6
- // watermarks so repeat syncs are increments, and POSTs batches of <=200 to
7
- // the same ingest endpoint the hourly buckets use.
8
- //
9
- // Privacy boundary (docs/cli-contract.md §2.4): only the contract's whitelisted
10
- // fields are sent. The sidecar's title, project_ref (local cwd), resume
11
- // identifiers and cache keys never leave the process — buildSessionUploadRows
12
- // constructs each row field-by-field rather than spreading.
13
-
14
- const fs = require("node:fs");
15
- const fsp = require("node:fs/promises");
16
- const path = require("node:path");
17
- const crypto = require("node:crypto");
18
-
19
- const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
20
-
21
- const INGEST_SLUG = "token-usage-ingest";
22
- const MAX_SESSIONS_PER_BATCH = 200;
23
- // Contract: source is trimmed + lowercased; these two are account-level and
24
- // the server skips them wholesale — the CLI just never uploads them.
25
- const ACCOUNT_LEVEL_SOURCES = new Set(["cursor", "trae-cn"]);
26
- // Keep the watermark map bounded: sessions older than the newest N never
27
- // re-upload anyway (their ended_at watermark only moves forward).
28
- const WATERMARK_MAX_ENTRIES = 5000;
29
-
30
- // Six token counters ride along everywhere; total must equal the sum of the
31
- // four that count toward it (reasoning is a sub-view of output, single-column).
32
- const TOKEN_FIELDS = [
33
- "input_tokens",
34
- "cached_input_tokens",
35
- "cache_creation_input_tokens",
36
- "output_tokens",
37
- "reasoning_output_tokens",
38
- "total_tokens",
39
- ];
40
-
41
- function safeInt(value) {
42
- if (value === null || value === undefined) return null;
43
- const n = Number(value);
44
- if (!Number.isSafeInteger(n) || n < 0) return null;
45
- return n;
46
- }
47
-
48
- // zcode/opencode report reasoning outside output_tokens (vendor semantics:
49
- // total = identity + reasoning), but the contract's identity requires
50
- // reasoning to sit inside output (docs/cli-contract.md §2.4: total = input +
51
- // cached + creation + output, reasoning single-column). Fold that exact shape
52
- // so the row satisfies the identity; any other mismatch still fails it and
53
- // the row is dropped — guessing a fix for unknown skews would silently
54
- // rewrite totals the vendor did report.
55
- function foldReasoningIntoOutput(tokens) {
56
- const identity =
57
- tokens.input_tokens +
58
- tokens.cached_input_tokens +
59
- tokens.cache_creation_input_tokens +
60
- tokens.output_tokens;
61
- if (
62
- tokens.total_tokens !== identity &&
63
- tokens.total_tokens === identity + tokens.reasoning_output_tokens
64
- ) {
65
- tokens.output_tokens += tokens.reasoning_output_tokens;
66
- }
67
- }
68
-
69
- function basenameKey(value) {
70
- const raw = String(value || "").replace(/[\\/]+$/, "");
71
- if (!raw || raw === "." || raw === "..") return null;
72
- const base = raw.split(/[\\/]/).pop().trim();
73
- if (!base || base === "." || base === "..") return null;
74
- // Control characters are rejected server-side; strip instead of failing.
75
- // eslint-disable-next-line no-control-regex
76
- const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
77
- return clean || null;
78
- }
79
-
80
- function validSessionId(value) {
81
- if (typeof value !== "string") return false;
82
- const id = value.trim();
83
- if (!id || id.length > 256) return false;
84
- if (id.includes("/") || id.includes("\\")) return false;
85
- // Full Unicode Cc range — the server validator is /\p{Cc}/u, which also
86
- // covers C1 (U+0080–U+009F); a narrower client check would let such an id
87
- // through to a whole-batch 400 that stalls the sessions upload.
88
- // eslint-disable-next-line no-control-regex
89
- if (/[\u0000-\u001f\u007f-\u009f]/.test(id)) return false;
90
- return true;
91
- }
92
-
93
- /**
94
- * Map one session-analytics sidecar row onto the contract wire shape.
95
- * Returns null when the row must not be uploaded (account-level source,
96
- * missing identity/timestamps, or a token identity that the server would
97
- * reject with a whole-batch 400 — dropping one bad row beats failing 200).
98
- *
99
- * Field locations on the sidecar row: the six token counters live nested in
100
- * row.tokens (the top level only mirrors total_tokens), and the stable id is
101
- * row.session_id when the vendor log carried one (Claude/Codex), else the
102
- * locally-computed sha256 fingerprint row.session_hash (source\0id, 24 hex)
103
- * — both are opaque stable keys without path leakage, valid as the contract
104
- * session_id.
105
- */
106
- function mapSessionRow(row) {
107
- if (!row || typeof row !== "object") return null;
108
- const source = String(row.source || "").trim().toLowerCase();
109
- if (!source || ACCOUNT_LEVEL_SOURCES.has(source)) return null;
110
- const sessionId = typeof row.session_id === "string" && row.session_id.trim()
111
- ? row.session_id.trim()
112
- : typeof row.session_hash === "string" && row.session_hash.trim()
113
- ? row.session_hash.trim()
114
- : null;
115
- if (!validSessionId(sessionId)) return null;
116
-
117
- const startedMs = Date.parse(row.started_at || "");
118
- const endedMs = Date.parse(row.ended_at || "");
119
- if (!Number.isFinite(startedMs) || !Number.isFinite(endedMs)) return null;
120
- if (endedMs < startedMs) return null;
121
-
122
- // Sidecar rows omit zero-valued counters on some sources (zcode drops
123
- // cache_creation/reasoning keys entirely). The contract requires every
124
- // field present, so missing keys fill as 0 — the identity check below is
125
- // the real gate; dropping a whole models array over an elided zero would
126
- // let a later last-wins upload erase previously stored models.
127
- const nested = row.tokens && typeof row.tokens === "object" ? row.tokens : {};
128
- const tokens = {};
129
- for (const field of TOKEN_FIELDS) {
130
- const raw = field === "total_tokens" ? row.total_tokens : nested[field];
131
- const value = safeInt(raw === undefined || raw === null ? 0 : raw);
132
- if (value === null) return null;
133
- tokens[field] = value;
134
- }
135
- foldReasoningIntoOutput(tokens);
136
- const identity =
137
- tokens.input_tokens +
138
- tokens.cached_input_tokens +
139
- tokens.cache_creation_input_tokens +
140
- tokens.output_tokens;
141
- if (tokens.total_tokens !== identity) return null;
142
-
143
- // Per-model rows follow the same identity rule. The contract caps models
144
- // at 32 AND requires sum(models.total_tokens) === session total, so a
145
- // session with more than 32 models cannot carry a valid array at all —
146
- // omit it (the contract allows missing models; the session prices as 0).
147
- let models = [];
148
- const usageRows = Array.isArray(row.model_usage) ? row.model_usage : [];
149
- let modelsValid = usageRows.length > 0 && usageRows.length <= 32;
150
- let modelsSum = 0;
151
- if (modelsValid) {
152
- for (const usage of usageRows) {
153
- const modelTokens = {};
154
- for (const field of TOKEN_FIELDS) {
155
- const raw = usage?.[field];
156
- const value = safeInt(raw === undefined || raw === null ? 0 : raw);
157
- if (value === null) {
158
- modelsValid = false;
159
- break;
160
- }
161
- modelTokens[field] = value;
162
- }
163
- if (!modelsValid) break;
164
- foldReasoningIntoOutput(modelTokens);
165
- const modelIdentity =
166
- modelTokens.input_tokens +
167
- modelTokens.cached_input_tokens +
168
- modelTokens.cache_creation_input_tokens +
169
- modelTokens.output_tokens;
170
- if (modelTokens.total_tokens !== modelIdentity) {
171
- modelsValid = false;
172
- break;
173
- }
174
- modelsSum += modelTokens.total_tokens;
175
- models.push({ model: String(usage.model || "").trim() || "unknown", ...modelTokens });
176
- }
177
- if (modelsValid && modelsSum !== tokens.total_tokens) modelsValid = false;
178
- }
179
- if (!modelsValid) models = [];
180
-
181
- return {
182
- session_id: sessionId,
183
- source,
184
- started_at: new Date(startedMs).toISOString(),
185
- ended_at: new Date(endedMs).toISOString(),
186
- project_key: basenameKey(row.project_key ?? row.project_ref),
187
- ...tokens,
188
- billable_total_tokens: null,
189
- ...(models.length > 0 ? { models } : {}),
190
- };
191
- }
192
-
193
- /** Map + filter a sidecar rows array; invalid rows are counted, not fatal. */
194
- function buildSessionUploadRows(rows) {
195
- const out = [];
196
- let skipped = 0;
197
- for (const row of Array.isArray(rows) ? rows : []) {
198
- const mapped = mapSessionRow(row);
199
- if (mapped) out.push(mapped);
200
- else skipped += 1;
201
- }
202
- return { rows: out, skipped };
203
- }
204
-
205
- function watermarkKey(source, sessionId) {
206
- return `${source}|${sessionId}`;
207
- }
208
-
209
- function loadWatermarks(filePath) {
210
- try {
211
- const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
212
- const map = new Map();
213
- for (const [key, value] of Object.entries(
214
- parsed && typeof parsed === "object" ? parsed : {},
215
- )) {
216
- if (typeof key === "string" && typeof value === "string" && key.includes("|")) {
217
- map.set(key, value);
218
- }
219
- }
220
- return map;
221
- } catch {
222
- return new Map();
223
- }
224
- }
225
-
226
- function pruneWatermarks(map) {
227
- if (map.size <= WATERMARK_MAX_ENTRIES) return map;
228
- const entries = [...map.entries()].sort((a, b) =>
229
- String(a[1]).localeCompare(String(b[1])),
230
- );
231
- return new Map(entries.slice(entries.length - WATERMARK_MAX_ENTRIES));
232
- }
233
-
234
- async function saveWatermarks(filePath, map) {
235
- const pruned = pruneWatermarks(map);
236
- const json = JSON.stringify(
237
- Object.fromEntries([...pruned.entries()].sort((a, b) => a[0].localeCompare(b[0]))),
238
- null,
239
- 2,
240
- ) + "\n";
241
- await fsp.mkdir(path.dirname(filePath), { recursive: true });
242
- const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
243
- await fsp.writeFile(tmp, json, { mode: 0o600 });
244
- await fsp.rename(tmp, filePath);
245
- await fsp.chmod(filePath, 0o600).catch(() => {});
246
- }
247
-
248
- /** Rows strictly newer than their watermark, ascending by ended_at. */
249
- function selectSessionsForUpload(rows, watermarks) {
250
- const pending = rows.filter((row) => {
251
- const mark = watermarks.get(watermarkKey(row.source, row.session_id));
252
- return !mark || String(row.ended_at) > mark;
253
- });
254
- pending.sort((a, b) => String(a.ended_at).localeCompare(String(b.ended_at)));
255
- return pending;
256
- }
257
-
258
- /**
259
- * Upload pending session rows in ascending ended_at batches of <=200. The
260
- * watermark file only advances for batches the server accepted, so a failed
261
- * batch leaves earlier successes committed and the failure retried next sync.
262
- */
263
- /**
264
- * Split pending rows into request batches. Two independent contract limits
265
- * apply (§1.2/§2.4): <=200 rows AND <=1 MiB body — a 200-row batch of real
266
- * sessions with models easily exceeds the byte cap, so batches are cut on
267
- * whichever limit hits first (bytes measured on the serialized row, with a
268
- * conservative envelope leaving room for upload_id + JSON framing).
269
- */
270
- function buildBatches(pending, maxRows) {
271
- const BYTE_BUDGET = 500 * 1024;
272
- const batches = [];
273
- let current = [];
274
- let bytes = 0;
275
-
276
- for (const row of pending) {
277
- const rowBytes = Buffer.byteLength(JSON.stringify(row), "utf8") + 2;
278
- if (current.length > 0 && (current.length >= maxRows || bytes + rowBytes > BYTE_BUDGET)) {
279
- batches.push(current);
280
- current = [];
281
- bytes = 0;
282
- }
283
- current.push(row);
284
- bytes += rowBytes;
285
- }
286
- if (current.length > 0) batches.push(current);
287
-
288
- return batches;
289
- }
290
-
291
- async function uploadSessions({
292
- baseUrl,
293
- anonKey,
294
- deviceToken,
295
- sessions,
296
- watermarkPath,
297
- batchSize = MAX_SESSIONS_PER_BATCH,
298
- canUpload = () => true,
299
- fetchImpl = fetch,
300
- onProgress = () => {},
301
- }) {
302
- // Map first: callers hand us RAW session-analytics rows (tokens nested in
303
- // row.tokens, sidecar-only fields present); the wire shape is built here so
304
- // no call site can bypass the whitelist/identity checks.
305
- const { rows: mapped, skipped: skippedLocal } = buildSessionUploadRows(sessions);
306
- const watermarks = loadWatermarks(watermarkPath);
307
- const pending = selectSessionsForUpload(mapped, watermarks);
308
- const maxRows = Math.min(Math.max(1, batchSize), MAX_SESSIONS_PER_BATCH);
309
- const uploadId = crypto.randomUUID();
310
- let upserted = 0;
311
- let skippedServer = 0;
312
- let batches = 0;
313
- let uploaded = 0;
314
-
315
- for (const batch of buildBatches(pending, maxRows)) {
316
- if (!canUpload()) break;
317
- const res = await fetchFunctionResponse(
318
- functionUrlFor(baseUrl, INGEST_SLUG),
319
- {
320
- method: "POST",
321
- headers: {
322
- "Content-Type": "application/json",
323
- Accept: "application/json",
324
- Authorization: `Bearer ${deviceToken}`,
325
- ...(anonKey ? { apikey: anonKey } : {}),
326
- },
327
- body: JSON.stringify({
328
- upload_id: uploadId,
329
- sessions: batch,
330
- }),
331
- },
332
- fetchImpl,
333
- );
334
- if (!res.ok) {
335
- const text = await res.text().catch(() => "");
336
- const error = new Error(
337
- `sessions upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`,
338
- );
339
- error.status = res.status;
340
- error.uploaded = uploaded;
341
- error.upserted = upserted;
342
- throw error;
343
- }
344
- const data = await res.json().catch(() => ({}));
345
- batches += 1;
346
- uploaded += batch.length;
347
- upserted += Number(data?.sessions_upserted || 0);
348
- skippedServer += Number(data?.sessions_skipped || 0);
349
- for (const row of batch) {
350
- watermarks.set(watermarkKey(row.source, row.session_id), row.ended_at);
351
- }
352
- await saveWatermarks(watermarkPath, watermarks);
353
- onProgress({ batches, uploaded, upserted });
354
- }
355
-
356
- return {
357
- pending: pending.length,
358
- uploaded,
359
- upserted,
360
- skippedServer,
361
- skippedLocal,
362
- batches,
363
- };
364
- }
365
-
366
- module.exports = {
367
- ACCOUNT_LEVEL_SOURCES,
368
- MAX_SESSIONS_PER_BATCH,
369
- basenameKey,
370
- buildBatches,
371
- buildSessionUploadRows,
372
- loadWatermarks,
373
- mapSessionRow,
374
- saveWatermarks,
375
- selectSessionsForUpload,
376
- uploadSessions,
377
- validSessionId,
378
- watermarkKey,
379
- };
1
+ "use strict";
2
+
3
+ // Session metadata upload for the token-usage ingest (docs/cli-contract.md
4
+ // §2.4 `sessions`). Maps the local session-analytics sidecar rows onto the
5
+ // contract's wire shape, filters through per-(source, session_id) ended_at
6
+ // watermarks so repeat syncs are increments, and POSTs batches of <=200 to
7
+ // the same ingest endpoint the hourly buckets use.
8
+ //
9
+ // Privacy boundary (docs/cli-contract.md §2.4): only the contract's whitelisted
10
+ // fields are sent. The sidecar's title, project_ref (local cwd), resume
11
+ // identifiers and cache keys never leave the process — buildSessionUploadRows
12
+ // constructs each row field-by-field rather than spreading.
13
+
14
+ const fs = require("node:fs");
15
+ const fsp = require("node:fs/promises");
16
+ const path = require("node:path");
17
+ const crypto = require("node:crypto");
18
+
19
+ const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
20
+
21
+ const INGEST_SLUG = "token-usage-ingest";
22
+ const MAX_SESSIONS_PER_BATCH = 200;
23
+ // Contract: source is trimmed + lowercased; these two are account-level and
24
+ // the server skips them wholesale — the CLI just never uploads them.
25
+ const ACCOUNT_LEVEL_SOURCES = new Set(["cursor", "trae-cn"]);
26
+ // Keep the watermark map bounded: sessions older than the newest N never
27
+ // re-upload anyway (their ended_at watermark only moves forward).
28
+ const WATERMARK_MAX_ENTRIES = 5000;
29
+
30
+ // Six token counters ride along everywhere; total must equal the sum of the
31
+ // four that count toward it (reasoning is a sub-view of output, single-column).
32
+ const TOKEN_FIELDS = [
33
+ "input_tokens",
34
+ "cached_input_tokens",
35
+ "cache_creation_input_tokens",
36
+ "output_tokens",
37
+ "reasoning_output_tokens",
38
+ "total_tokens",
39
+ ];
40
+
41
+ function safeInt(value) {
42
+ if (value === null || value === undefined) return null;
43
+ const n = Number(value);
44
+ if (!Number.isSafeInteger(n) || n < 0) return null;
45
+ return n;
46
+ }
47
+
48
+ // zcode/opencode report reasoning outside output_tokens (vendor semantics:
49
+ // total = identity + reasoning), but the contract's identity requires
50
+ // reasoning to sit inside output (docs/cli-contract.md §2.4: total = input +
51
+ // cached + creation + output, reasoning single-column). Fold that exact shape
52
+ // so the row satisfies the identity; any other mismatch still fails it and
53
+ // the row is dropped — guessing a fix for unknown skews would silently
54
+ // rewrite totals the vendor did report.
55
+ function foldReasoningIntoOutput(tokens) {
56
+ const identity =
57
+ tokens.input_tokens +
58
+ tokens.cached_input_tokens +
59
+ tokens.cache_creation_input_tokens +
60
+ tokens.output_tokens;
61
+ if (
62
+ tokens.total_tokens !== identity &&
63
+ tokens.total_tokens === identity + tokens.reasoning_output_tokens
64
+ ) {
65
+ tokens.output_tokens += tokens.reasoning_output_tokens;
66
+ }
67
+ }
68
+
69
+ function basenameKey(value) {
70
+ const raw = String(value || "").replace(/[\\/]+$/, "");
71
+ if (!raw || raw === "." || raw === "..") return null;
72
+ const base = raw.split(/[\\/]/).pop().trim();
73
+ if (!base || base === "." || base === "..") return null;
74
+ // Control characters are rejected server-side; strip instead of failing.
75
+ // eslint-disable-next-line no-control-regex
76
+ const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
77
+ return clean || null;
78
+ }
79
+
80
+ function validSessionId(value) {
81
+ if (typeof value !== "string") return false;
82
+ const id = value.trim();
83
+ if (!id || id.length > 256) return false;
84
+ if (id.includes("/") || id.includes("\\")) return false;
85
+ // Full Unicode Cc range — the server validator is /\p{Cc}/u, which also
86
+ // covers C1 (U+0080–U+009F); a narrower client check would let such an id
87
+ // through to a whole-batch 400 that stalls the sessions upload.
88
+ // eslint-disable-next-line no-control-regex
89
+ if (/[\u0000-\u001f\u007f-\u009f]/.test(id)) return false;
90
+ return true;
91
+ }
92
+
93
+ /**
94
+ * Map one session-analytics sidecar row onto the contract wire shape.
95
+ * Returns null when the row must not be uploaded (account-level source,
96
+ * missing identity/timestamps, or a token identity that the server would
97
+ * reject with a whole-batch 400 — dropping one bad row beats failing 200).
98
+ *
99
+ * Field locations on the sidecar row: the six token counters live nested in
100
+ * row.tokens (the top level only mirrors total_tokens), and the stable id is
101
+ * row.session_id when the vendor log carried one (Claude/Codex), else the
102
+ * locally-computed sha256 fingerprint row.session_hash (source\0id, 24 hex)
103
+ * — both are opaque stable keys without path leakage, valid as the contract
104
+ * session_id.
105
+ */
106
+ function mapSessionRow(row) {
107
+ if (!row || typeof row !== "object") return null;
108
+ const source = String(row.source || "").trim().toLowerCase();
109
+ if (!source || ACCOUNT_LEVEL_SOURCES.has(source)) return null;
110
+ const sessionId = typeof row.session_id === "string" && row.session_id.trim()
111
+ ? row.session_id.trim()
112
+ : typeof row.session_hash === "string" && row.session_hash.trim()
113
+ ? row.session_hash.trim()
114
+ : null;
115
+ if (!validSessionId(sessionId)) return null;
116
+
117
+ const startedMs = Date.parse(row.started_at || "");
118
+ const endedMs = Date.parse(row.ended_at || "");
119
+ if (!Number.isFinite(startedMs) || !Number.isFinite(endedMs)) return null;
120
+ if (endedMs < startedMs) return null;
121
+
122
+ // Sidecar rows omit zero-valued counters on some sources (zcode drops
123
+ // cache_creation/reasoning keys entirely). The contract requires every
124
+ // field present, so missing keys fill as 0 — the identity check below is
125
+ // the real gate; dropping a whole models array over an elided zero would
126
+ // let a later last-wins upload erase previously stored models.
127
+ const nested = row.tokens && typeof row.tokens === "object" ? row.tokens : {};
128
+ const tokens = {};
129
+ for (const field of TOKEN_FIELDS) {
130
+ const raw = field === "total_tokens" ? row.total_tokens : nested[field];
131
+ const value = safeInt(raw === undefined || raw === null ? 0 : raw);
132
+ if (value === null) return null;
133
+ tokens[field] = value;
134
+ }
135
+ foldReasoningIntoOutput(tokens);
136
+ const identity =
137
+ tokens.input_tokens +
138
+ tokens.cached_input_tokens +
139
+ tokens.cache_creation_input_tokens +
140
+ tokens.output_tokens;
141
+ if (tokens.total_tokens !== identity) return null;
142
+
143
+ // Per-model rows follow the same identity rule. The contract caps models
144
+ // at 32 AND requires sum(models.total_tokens) === session total, so a
145
+ // session with more than 32 models cannot carry a valid array at all —
146
+ // omit it (the contract allows missing models; the session prices as 0).
147
+ let models = [];
148
+ const usageRows = Array.isArray(row.model_usage) ? row.model_usage : [];
149
+ let modelsValid = usageRows.length > 0 && usageRows.length <= 32;
150
+ let modelsSum = 0;
151
+ if (modelsValid) {
152
+ for (const usage of usageRows) {
153
+ const modelTokens = {};
154
+ for (const field of TOKEN_FIELDS) {
155
+ const raw = usage?.[field];
156
+ const value = safeInt(raw === undefined || raw === null ? 0 : raw);
157
+ if (value === null) {
158
+ modelsValid = false;
159
+ break;
160
+ }
161
+ modelTokens[field] = value;
162
+ }
163
+ if (!modelsValid) break;
164
+ foldReasoningIntoOutput(modelTokens);
165
+ const modelIdentity =
166
+ modelTokens.input_tokens +
167
+ modelTokens.cached_input_tokens +
168
+ modelTokens.cache_creation_input_tokens +
169
+ modelTokens.output_tokens;
170
+ if (modelTokens.total_tokens !== modelIdentity) {
171
+ modelsValid = false;
172
+ break;
173
+ }
174
+ modelsSum += modelTokens.total_tokens;
175
+ models.push({ model: String(usage.model || "").trim() || "unknown", ...modelTokens });
176
+ }
177
+ if (modelsValid && modelsSum !== tokens.total_tokens) modelsValid = false;
178
+ }
179
+ if (!modelsValid) models = [];
180
+
181
+ return {
182
+ session_id: sessionId,
183
+ source,
184
+ started_at: new Date(startedMs).toISOString(),
185
+ ended_at: new Date(endedMs).toISOString(),
186
+ project_key: basenameKey(row.project_key ?? row.project_ref),
187
+ ...tokens,
188
+ billable_total_tokens: null,
189
+ ...(models.length > 0 ? { models } : {}),
190
+ };
191
+ }
192
+
193
+ /** Map + filter a sidecar rows array; invalid rows are counted, not fatal. */
194
+ function buildSessionUploadRows(rows) {
195
+ const out = [];
196
+ let skipped = 0;
197
+ for (const row of Array.isArray(rows) ? rows : []) {
198
+ const mapped = mapSessionRow(row);
199
+ if (mapped) out.push(mapped);
200
+ else skipped += 1;
201
+ }
202
+ return { rows: out, skipped };
203
+ }
204
+
205
+ function watermarkKey(source, sessionId) {
206
+ return `${source}|${sessionId}`;
207
+ }
208
+
209
+ function loadWatermarks(filePath) {
210
+ try {
211
+ const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
212
+ const map = new Map();
213
+ for (const [key, value] of Object.entries(
214
+ parsed && typeof parsed === "object" ? parsed : {},
215
+ )) {
216
+ if (typeof key === "string" && typeof value === "string" && key.includes("|")) {
217
+ map.set(key, value);
218
+ }
219
+ }
220
+ return map;
221
+ } catch {
222
+ return new Map();
223
+ }
224
+ }
225
+
226
+ function pruneWatermarks(map) {
227
+ if (map.size <= WATERMARK_MAX_ENTRIES) return map;
228
+ const entries = [...map.entries()].sort((a, b) =>
229
+ String(a[1]).localeCompare(String(b[1])),
230
+ );
231
+ return new Map(entries.slice(entries.length - WATERMARK_MAX_ENTRIES));
232
+ }
233
+
234
+ async function saveWatermarks(filePath, map) {
235
+ const pruned = pruneWatermarks(map);
236
+ const json = JSON.stringify(
237
+ Object.fromEntries([...pruned.entries()].sort((a, b) => a[0].localeCompare(b[0]))),
238
+ null,
239
+ 2,
240
+ ) + "\n";
241
+ await fsp.mkdir(path.dirname(filePath), { recursive: true });
242
+ const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
243
+ await fsp.writeFile(tmp, json, { mode: 0o600 });
244
+ await fsp.rename(tmp, filePath);
245
+ await fsp.chmod(filePath, 0o600).catch(() => {});
246
+ }
247
+
248
+ /** Rows strictly newer than their watermark, ascending by ended_at. */
249
+ function selectSessionsForUpload(rows, watermarks) {
250
+ const pending = rows.filter((row) => {
251
+ const mark = watermarks.get(watermarkKey(row.source, row.session_id));
252
+ return !mark || String(row.ended_at) > mark;
253
+ });
254
+ pending.sort((a, b) => String(a.ended_at).localeCompare(String(b.ended_at)));
255
+ return pending;
256
+ }
257
+
258
+ /**
259
+ * Upload pending session rows in ascending ended_at batches of <=200. The
260
+ * watermark file only advances for batches the server accepted, so a failed
261
+ * batch leaves earlier successes committed and the failure retried next sync.
262
+ */
263
+ /**
264
+ * Split pending rows into request batches. Two independent contract limits
265
+ * apply (§1.2/§2.4): <=200 rows AND <=1 MiB body — a 200-row batch of real
266
+ * sessions with models easily exceeds the byte cap, so batches are cut on
267
+ * whichever limit hits first (bytes measured on the serialized row, with a
268
+ * conservative envelope leaving room for upload_id + JSON framing).
269
+ */
270
+ function buildBatches(pending, maxRows) {
271
+ const BYTE_BUDGET = 500 * 1024;
272
+ const batches = [];
273
+ let current = [];
274
+ let bytes = 0;
275
+
276
+ for (const row of pending) {
277
+ const rowBytes = Buffer.byteLength(JSON.stringify(row), "utf8") + 2;
278
+ if (current.length > 0 && (current.length >= maxRows || bytes + rowBytes > BYTE_BUDGET)) {
279
+ batches.push(current);
280
+ current = [];
281
+ bytes = 0;
282
+ }
283
+ current.push(row);
284
+ bytes += rowBytes;
285
+ }
286
+ if (current.length > 0) batches.push(current);
287
+
288
+ return batches;
289
+ }
290
+
291
+ async function uploadSessions({
292
+ baseUrl,
293
+ anonKey,
294
+ deviceToken,
295
+ sessions,
296
+ watermarkPath,
297
+ batchSize = MAX_SESSIONS_PER_BATCH,
298
+ canUpload = () => true,
299
+ fetchImpl = fetch,
300
+ onProgress = () => {},
301
+ }) {
302
+ // Map first: callers hand us RAW session-analytics rows (tokens nested in
303
+ // row.tokens, sidecar-only fields present); the wire shape is built here so
304
+ // no call site can bypass the whitelist/identity checks.
305
+ const { rows: mapped, skipped: skippedLocal } = buildSessionUploadRows(sessions);
306
+ const watermarks = loadWatermarks(watermarkPath);
307
+ const pending = selectSessionsForUpload(mapped, watermarks);
308
+ const maxRows = Math.min(Math.max(1, batchSize), MAX_SESSIONS_PER_BATCH);
309
+ const uploadId = crypto.randomUUID();
310
+ let upserted = 0;
311
+ let skippedServer = 0;
312
+ let batches = 0;
313
+ let uploaded = 0;
314
+
315
+ for (const batch of buildBatches(pending, maxRows)) {
316
+ if (!canUpload()) break;
317
+ const res = await fetchFunctionResponse(
318
+ functionUrlFor(baseUrl, INGEST_SLUG),
319
+ {
320
+ method: "POST",
321
+ headers: {
322
+ "Content-Type": "application/json",
323
+ Accept: "application/json",
324
+ Authorization: `Bearer ${deviceToken}`,
325
+ ...(anonKey ? { apikey: anonKey } : {}),
326
+ },
327
+ body: JSON.stringify({
328
+ upload_id: uploadId,
329
+ sessions: batch,
330
+ }),
331
+ },
332
+ fetchImpl,
333
+ );
334
+ if (!res.ok) {
335
+ const text = await res.text().catch(() => "");
336
+ const error = new Error(
337
+ `sessions upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`,
338
+ );
339
+ error.status = res.status;
340
+ error.uploaded = uploaded;
341
+ error.upserted = upserted;
342
+ throw error;
343
+ }
344
+ const data = await res.json().catch(() => ({}));
345
+ batches += 1;
346
+ uploaded += batch.length;
347
+ upserted += Number(data?.sessions_upserted || 0);
348
+ skippedServer += Number(data?.sessions_skipped || 0);
349
+ for (const row of batch) {
350
+ watermarks.set(watermarkKey(row.source, row.session_id), row.ended_at);
351
+ }
352
+ await saveWatermarks(watermarkPath, watermarks);
353
+ onProgress({ batches, uploaded, upserted });
354
+ }
355
+
356
+ return {
357
+ pending: pending.length,
358
+ uploaded,
359
+ upserted,
360
+ skippedServer,
361
+ skippedLocal,
362
+ batches,
363
+ };
364
+ }
365
+
366
+ module.exports = {
367
+ ACCOUNT_LEVEL_SOURCES,
368
+ MAX_SESSIONS_PER_BATCH,
369
+ basenameKey,
370
+ buildBatches,
371
+ buildSessionUploadRows,
372
+ loadWatermarks,
373
+ mapSessionRow,
374
+ saveWatermarks,
375
+ selectSessionsForUpload,
376
+ uploadSessions,
377
+ validSessionId,
378
+ watermarkKey,
379
+ };