tokenusage-cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/LICENSE +22 -0
  3. package/README.md +94 -0
  4. package/THIRD_PARTY_NOTICES.md +140 -0
  5. package/bin/tokenusage.js +69 -0
  6. package/package.json +59 -0
  7. package/src/cli.js +134 -0
  8. package/src/commands/device-login.js +205 -0
  9. package/src/commands/diagnostics.js +38 -0
  10. package/src/commands/doctor.js +110 -0
  11. package/src/commands/init.js +2436 -0
  12. package/src/commands/sessions.js +29 -0
  13. package/src/commands/status.js +1747 -0
  14. package/src/commands/sync.js +7934 -0
  15. package/src/commands/uninstall.js +312 -0
  16. package/src/lib/account-usage-dedup.js +267 -0
  17. package/src/lib/antigravity-paths.js +48 -0
  18. package/src/lib/ark-agent-plan-limits.js +438 -0
  19. package/src/lib/ark-coding-plan-limits.js +537 -0
  20. package/src/lib/async-queue-reader.js +138 -0
  21. package/src/lib/atomic-json.js +27 -0
  22. package/src/lib/bearer-token.js +11 -0
  23. package/src/lib/categorizer-utils.js +232 -0
  24. package/src/lib/claude-categorizer.js +1343 -0
  25. package/src/lib/claude-config.js +279 -0
  26. package/src/lib/claude-user-identity.js +56 -0
  27. package/src/lib/cli-ui.js +179 -0
  28. package/src/lib/cloud-account.js +541 -0
  29. package/src/lib/cloud-device-token.js +141 -0
  30. package/src/lib/cloud-sync-prefs.js +14 -0
  31. package/src/lib/cloud-usage-reset.js +115 -0
  32. package/src/lib/codex-config.js +595 -0
  33. package/src/lib/codex-context-breakdown.js +806 -0
  34. package/src/lib/codex-model-attribution.js +128 -0
  35. package/src/lib/codex-rollout-parser.js +1471 -0
  36. package/src/lib/codex-service-tier.js +49 -0
  37. package/src/lib/codex-token-refresh.js +140 -0
  38. package/src/lib/codex-token-usage.js +195 -0
  39. package/src/lib/codex-usage-record.js +195 -0
  40. package/src/lib/command-runner.js +313 -0
  41. package/src/lib/commandcode-limits.js +317 -0
  42. package/src/lib/context-health.js +124 -0
  43. package/src/lib/cursor-config.js +605 -0
  44. package/src/lib/cursor-store.js +1918 -0
  45. package/src/lib/debug-flags.js +7 -0
  46. package/src/lib/devin-limits.js +249 -0
  47. package/src/lib/devin-usage.js +212 -0
  48. package/src/lib/diagnostics.js +383 -0
  49. package/src/lib/doctor.js +385 -0
  50. package/src/lib/dumate.js +95 -0
  51. package/src/lib/fs.js +455 -0
  52. package/src/lib/function-url.js +14 -0
  53. package/src/lib/gemini-config.js +291 -0
  54. package/src/lib/git-outcomes.js +243 -0
  55. package/src/lib/grok-context-breakdown.js +890 -0
  56. package/src/lib/grok-hook.js +360 -0
  57. package/src/lib/grok-limits.js +865 -0
  58. package/src/lib/grok-usage.js +98 -0
  59. package/src/lib/heatmap-compact.js +114 -0
  60. package/src/lib/ingest-sanitize.js +41 -0
  61. package/src/lib/init-flow.js +42 -0
  62. package/src/lib/install-resolver.js +170 -0
  63. package/src/lib/jsonl-lines.js +134 -0
  64. package/src/lib/kimi-model-repair.js +173 -0
  65. package/src/lib/kimi-profile.js +61 -0
  66. package/src/lib/kimi-session.js +216 -0
  67. package/src/lib/local-api.js +3588 -0
  68. package/src/lib/machine-id.js +238 -0
  69. package/src/lib/multi-install-parser.js +113 -0
  70. package/src/lib/notify-command.js +38 -0
  71. package/src/lib/omp-hook.js +589 -0
  72. package/src/lib/openclaw-cli.js +104 -0
  73. package/src/lib/openclaw-hook.js +375 -0
  74. package/src/lib/openclaw-session-plugin.js +560 -0
  75. package/src/lib/opencode-config.js +121 -0
  76. package/src/lib/opencode-go-limits.js +807 -0
  77. package/src/lib/outcomes-engine.js +253 -0
  78. package/src/lib/passive-mode.js +219 -0
  79. package/src/lib/pet-packages.js +722 -0
  80. package/src/lib/pricing/curated-overrides.json +822 -0
  81. package/src/lib/pricing/index.js +379 -0
  82. package/src/lib/pricing/litellm-fetcher.js +172 -0
  83. package/src/lib/pricing/matcher.js +456 -0
  84. package/src/lib/pricing/seed-snapshot.json +1 -0
  85. package/src/lib/progress.js +76 -0
  86. package/src/lib/project-usage-purge.js +117 -0
  87. package/src/lib/prompt.js +24 -0
  88. package/src/lib/provider-status.js +104 -0
  89. package/src/lib/proxy-env.js +489 -0
  90. package/src/lib/proxy-settings.js +163 -0
  91. package/src/lib/qoder-cn-cli-limits.js +119 -0
  92. package/src/lib/qoder-limits.js +880 -0
  93. package/src/lib/quiet-warnings.js +26 -0
  94. package/src/lib/rollout.js +25966 -0
  95. package/src/lib/runtime-config.js +259 -0
  96. package/src/lib/scan-roots.js +277 -0
  97. package/src/lib/sentry.js +231 -0
  98. package/src/lib/session-analytics.js +2972 -0
  99. package/src/lib/session-performance.js +237 -0
  100. package/src/lib/session-upload.js +379 -0
  101. package/src/lib/skill-usage.js +267 -0
  102. package/src/lib/skills-manager.js +1591 -0
  103. package/src/lib/source-metadata.js +48 -0
  104. package/src/lib/sqlite-reader.js +215 -0
  105. package/src/lib/star-cta.js +73 -0
  106. package/src/lib/subscription-manager.js +293 -0
  107. package/src/lib/subscriptions.js +508 -0
  108. package/src/lib/telemetry.js +193 -0
  109. package/src/lib/tracker-paths.js +16 -0
  110. package/src/lib/trae-cn-config.js +548 -0
  111. package/src/lib/trae-db.js +127 -0
  112. package/src/lib/trae-snapshot.js +230 -0
  113. package/src/lib/trae-sqlite.js +143 -0
  114. package/src/lib/trae-usage.js +121 -0
  115. package/src/lib/upload-queue.js +134 -0
  116. package/src/lib/upload-throttle.js +171 -0
  117. package/src/lib/usage-limits.js +4552 -0
  118. package/src/lib/wrapped-aggregator.js +236 -0
  119. package/src/lib/wsl-probe.js +281 -0
  120. package/src/lib/zcode-limits.js +1172 -0
  121. package/src/lib/zcode-reset-credits.js +66 -0
  122. package/test/cli-smoke.test.js +53 -0
  123. package/test/ingest-sanitize.test.js +162 -0
  124. package/test/quiet-warnings.test.js +53 -0
  125. package/test/session-upload.test.js +427 -0
@@ -0,0 +1,237 @@
1
+ "use strict";
2
+
3
+ const { canonicalUsage } = require("./codex-token-usage");
4
+
5
+ const SUM_FIELDS = [
6
+ "estimated_output_tokens", "estimated_duration_ms", "estimated_request_count",
7
+ "first_response_total_ms", "first_response_sample_count",
8
+ ];
9
+ const MIN_OUTPUT = 200;
10
+ const MIN_DURATION_MS = 1000;
11
+ const MAX_DURATION_MS = 60 * 60 * 1000;
12
+ const SAME_FLUSH_MS = 100;
13
+
14
+ function nonNegative(value) {
15
+ const number = Number(value);
16
+ return Number.isFinite(number) && number >= 0 ? number : 0;
17
+ }
18
+
19
+ function normalizePerformance(value) {
20
+ const out = Object.fromEntries(SUM_FIELDS.map((field) => [field, nonNegative(value?.[field])]));
21
+ out.estimated_tokens_per_second = out.estimated_request_count > 0 && out.estimated_output_tokens > 0 && out.estimated_duration_ms > 0
22
+ ? out.estimated_output_tokens / (out.estimated_duration_ms / 1000) : null;
23
+ out.first_response_ms = out.first_response_sample_count > 0
24
+ ? out.first_response_total_ms / out.first_response_sample_count : null;
25
+ return out;
26
+ }
27
+
28
+ function mergePerformance(...values) {
29
+ const out = normalizePerformance();
30
+ for (const value of values) {
31
+ for (const field of SUM_FIELDS) out[field] += nonNegative(value?.[field]);
32
+ }
33
+ return normalizePerformance(out);
34
+ }
35
+
36
+ function timestampMs(value) {
37
+ if (typeof value !== "string") return null;
38
+ const number = Date.parse(value);
39
+ return Number.isFinite(number) ? number : null;
40
+ }
41
+
42
+ function accumulator() {
43
+ let total = normalizePerformance();
44
+ const byModel = new Map();
45
+ function add(model, delta) {
46
+ total = mergePerformance(total, delta);
47
+ if (model) byModel.set(model, mergePerformance(byModel.get(model), delta));
48
+ }
49
+ function request(model, outputTokens, start, end) {
50
+ const duration = start !== null && end !== null ? end - start : 0;
51
+ record(model, outputTokens, duration);
52
+ }
53
+ function requestDuration(model, outputTokens, durationMs) {
54
+ record(model, outputTokens, durationMs);
55
+ }
56
+ function record(model, outputTokens, duration) {
57
+ if (outputTokens < MIN_OUTPUT || duration < MIN_DURATION_MS || duration > MAX_DURATION_MS) return;
58
+ add(model, { estimated_output_tokens: outputTokens, estimated_duration_ms: duration, estimated_request_count: 1 });
59
+ }
60
+ return { add, request, requestDuration, finish: () => ({ performance: total, byModel }) };
61
+ }
62
+
63
+ function createClaudePerformanceCollector() {
64
+ const nodes = new Map();
65
+ const messages = new Map();
66
+ const totals = accumulator();
67
+
68
+ function consume(obj, key, countedUsage, model) {
69
+ const timestamp = timestampMs(obj?.timestamp || obj?.message?.timestamp);
70
+ if (typeof obj?.uuid === "string" && !nodes.has(obj.uuid)) {
71
+ nodes.set(obj.uuid, {
72
+ parent: obj.parentUuid, timestamp, type: obj.type,
73
+ invalid: Boolean(obj.isApiErrorMessage || obj.isMeta),
74
+ });
75
+ }
76
+ if (obj?.type !== "assistant" || !key) return;
77
+ let timing = messages.get(key);
78
+ if (!timing) {
79
+ // Tokens come from scanClaudeSession's canonical, once-counted delta.
80
+ // Later content blocks extend only the timing, never the numerator.
81
+ timing = {
82
+ parent: obj.parentUuid, startsAtFirstBlock: obj.apiBlockIndex == null || obj.apiBlockIndex === 0,
83
+ end: null, complete: false, invalid: false, model,
84
+ output: nonNegative(countedUsage?.output_tokens) + nonNegative(countedUsage?.reasoning_output_tokens),
85
+ };
86
+ messages.set(key, timing);
87
+ }
88
+ if (timestamp !== null) timing.end = timing.end === null ? timestamp : Math.max(timing.end, timestamp);
89
+ timing.complete ||= ["end_turn", "tool_use", "max_tokens", "stop_sequence"].includes(obj.message?.stop_reason);
90
+ timing.invalid ||= Boolean(obj.isApiErrorMessage || obj.message?.error);
91
+ }
92
+
93
+ function finish() {
94
+ for (const timing of messages.values()) {
95
+ if (!timing.complete || timing.invalid || !timing.startsAtFirstBlock) continue;
96
+ let parent = timing.parent;
97
+ for (let hop = 0; hop < 32 && parent; hop += 1) {
98
+ const node = nodes.get(parent);
99
+ if (!node) break;
100
+ if (node.type === "attachment") { parent = node.parent; continue; }
101
+ // A user row includes tool results. Their completion timestamp keeps
102
+ // time spent executing the tool outside the next inference window.
103
+ if (node.type === "user" && !node.invalid) totals.request(timing.model, timing.output, node.timestamp, timing.end);
104
+ break;
105
+ }
106
+ }
107
+ return totals.finish();
108
+ }
109
+ return { consume, finish };
110
+ }
111
+
112
+ function createCodexPerformanceCollector() {
113
+ const totals = accumulator();
114
+ let boundary = null;
115
+ let lastTokenCount = null;
116
+ let start = null;
117
+ let lastOutput = null;
118
+ let toolAfterOutput = null;
119
+ let usageRecord = null;
120
+ let invalid = false;
121
+ let turnKey = null;
122
+ let turnModels = new Set();
123
+ const seenFirstResponses = new Set();
124
+
125
+ function reset() {
126
+ start = null;
127
+ lastOutput = null;
128
+ toolAfterOutput = null;
129
+ usageRecord = null;
130
+ invalid = false;
131
+ }
132
+ function noteBoundary(ms) {
133
+ if (ms !== null) boundary = boundary === null ? ms : Math.max(boundary, ms);
134
+ }
135
+ function consume(obj, model) {
136
+ const ms = timestampMs(obj?.timestamp);
137
+ const payload = obj?.payload || {};
138
+ if (obj?.type === "compacted") {
139
+ reset();
140
+ noteBoundary(ms);
141
+ return;
142
+ }
143
+ if (obj?.type === "event_msg" && ["task_started", "turn_aborted"].includes(payload.type)) {
144
+ reset();
145
+ noteBoundary(ms);
146
+ turnModels = new Set();
147
+ turnKey = payload.turn_id || null;
148
+ if (payload.type === "turn_aborted") invalid = true;
149
+ }
150
+ if (obj?.type === "turn_context") {
151
+ const nextKey = payload.turn_id || null;
152
+ if (turnKey && nextKey && turnKey !== nextKey) { reset(); turnModels = new Set(); }
153
+ turnKey = nextKey || turnKey;
154
+ noteBoundary(ms);
155
+ }
156
+ const responseType = String(payload.type || "");
157
+ const isModelOutput = obj?.type === "response_item" && (responseType === "reasoning"
158
+ || responseType === "agent_message" || responseType.endsWith("_call")
159
+ || (responseType === "message" && payload.role === "assistant"));
160
+ if (model && (obj?.type === "turn_context" || isModelOutput
161
+ || payload.type === "model_rerouted")) turnModels.add(model);
162
+ if (obj?.type === "event_msg" && ["error", "api_error"].includes(payload.type)) invalid = true;
163
+ if (obj?.type === "event_msg" && payload.type === "task_complete") {
164
+ const raw = payload.time_to_first_token_ms;
165
+ const key = payload.turn_id || turnKey || obj.timestamp;
166
+ if (!invalid && typeof raw === "number" && Number.isFinite(raw) && raw >= 0 && !seenFirstResponses.has(key)) {
167
+ seenFirstResponses.add(key);
168
+ totals.add(turnModels.size === 1 ? [...turnModels][0] : null, { first_response_total_ms: raw, first_response_sample_count: 1 });
169
+ }
170
+ reset();
171
+ return;
172
+ }
173
+ if (obj?.type === "token_usage_record") {
174
+ usageRecord = { end: ms, usage: canonicalUsage(payload.usage) };
175
+ return;
176
+ }
177
+ if (obj?.type !== "response_item") return;
178
+ const type = String(payload.type || "");
179
+ if (type.endsWith("_output")) {
180
+ noteBoundary(ms);
181
+ if (lastOutput !== null) toolAfterOutput = ms;
182
+ } else if ((type === "message" && payload.role !== "assistant") || type === "user_message") {
183
+ noteBoundary(ms);
184
+ } else if (type === "reasoning" || type === "agent_message" || type.endsWith("_call") || (type === "message" && payload.role === "assistant")) {
185
+ // Old rollouts flush an output item just after its token_count.
186
+ if (ms === null || (start === null && lastTokenCount !== null && ms >= lastTokenCount && ms - lastTokenCount <= SAME_FLUSH_MS)) return;
187
+ if (start === null) start = boundary;
188
+ lastOutput = ms;
189
+ toolAfterOutput = null;
190
+ }
191
+ }
192
+ function consumeUsage({ timestamp, delta, rawUsage, model }) {
193
+ // A repeated cumulative snapshot can be written while the next response
194
+ // is still generating. It is not a completion boundary for that response.
195
+ if (!delta || delta.total_tokens <= 0) return;
196
+ const ms = timestampMs(timestamp);
197
+ const raw = canonicalUsage(rawUsage);
198
+ const matches = raw && usageRecord?.usage && ["input_tokens", "cached_input_tokens", "output_tokens"]
199
+ .every((field) => raw[field] === usageRecord.usage[field]);
200
+ const waitedForTools = toolAfterOutput !== null && ms !== null && ms >= toolAfterOutput;
201
+ // Once a tool result follows the final output marker, the token_count's
202
+ // timestamp contains execution time. The output marker is the available
203
+ // response boundary even if the count was flushed much later.
204
+ const end = matches ? usageRecord.end : waitedForTools ? lastOutput : ms;
205
+ // Missing output markers cannot distinguish response timing from a
206
+ // snapshot or inherited context, so even a positive token delta is skipped.
207
+ if (!invalid && lastOutput !== null) totals.request(model, nonNegative(delta.output_tokens), start, end);
208
+ noteBoundary(ms);
209
+ lastTokenCount = ms;
210
+ reset();
211
+ }
212
+ return { consume, consumeUsage, finish: totals.finish };
213
+ }
214
+
215
+ // ZCode's CLI DB stores per-request wall-clock counters directly in its
216
+ // model_usage table (duration_ms, time_to_first_token_ms) instead of an event
217
+ // stream, so this collector is a thin adapter: each completed request becomes
218
+ // one duration-bounded sample, never a token total (those come from the
219
+ // shared usage reader so the browser cannot drift from the usage card).
220
+ function createZcodePerformanceCollector() {
221
+ const totals = accumulator();
222
+ function consumeUsage({ model, outputTokens, durationMs, timeToFirstTokenMs } = {}) {
223
+ totals.requestDuration(model, nonNegative(outputTokens), nonNegative(durationMs));
224
+ if (typeof timeToFirstTokenMs === "number" && Number.isFinite(timeToFirstTokenMs) && timeToFirstTokenMs >= 0) {
225
+ totals.add(null, { first_response_total_ms: timeToFirstTokenMs, first_response_sample_count: 1 });
226
+ }
227
+ }
228
+ return { consumeUsage, finish: totals.finish };
229
+ }
230
+
231
+ module.exports = {
232
+ normalizePerformance,
233
+ mergePerformance,
234
+ createClaudePerformanceCollector,
235
+ createCodexPerformanceCollector,
236
+ createZcodePerformanceCollector,
237
+ };
@@ -0,0 +1,379 @@
1
+ "use strict";
2
+
3
+ // Session metadata upload for the token-usage ingest (docs/cli-contract.md
4
+ // §2.4 `sessions`). Maps the local session-analytics sidecar rows onto the
5
+ // contract's wire shape, filters through per-(source, session_id) ended_at
6
+ // watermarks so repeat syncs are increments, and POSTs batches of <=200 to
7
+ // the same ingest endpoint the hourly buckets use.
8
+ //
9
+ // Privacy boundary (docs/cli-contract.md §2.4): only the contract's whitelisted
10
+ // fields are sent. The sidecar's title, project_ref (local cwd), resume
11
+ // identifiers and cache keys never leave the process — buildSessionUploadRows
12
+ // constructs each row field-by-field rather than spreading.
13
+
14
+ const fs = require("node:fs");
15
+ const fsp = require("node:fs/promises");
16
+ const path = require("node:path");
17
+ const crypto = require("node:crypto");
18
+
19
+ const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
20
+
21
+ const INGEST_SLUG = "token-usage-ingest";
22
+ const MAX_SESSIONS_PER_BATCH = 200;
23
+ // Contract: source is trimmed + lowercased; these two are account-level and
24
+ // the server skips them wholesale — the CLI just never uploads them.
25
+ const ACCOUNT_LEVEL_SOURCES = new Set(["cursor", "trae-cn"]);
26
+ // Keep the watermark map bounded: sessions older than the newest N never
27
+ // re-upload anyway (their ended_at watermark only moves forward).
28
+ const WATERMARK_MAX_ENTRIES = 5000;
29
+
30
+ // Six token counters ride along everywhere; total must equal the sum of the
31
+ // four that count toward it (reasoning is a sub-view of output, single-column).
32
+ const TOKEN_FIELDS = [
33
+ "input_tokens",
34
+ "cached_input_tokens",
35
+ "cache_creation_input_tokens",
36
+ "output_tokens",
37
+ "reasoning_output_tokens",
38
+ "total_tokens",
39
+ ];
40
+
41
+ function safeInt(value) {
42
+ if (value === null || value === undefined) return null;
43
+ const n = Number(value);
44
+ if (!Number.isSafeInteger(n) || n < 0) return null;
45
+ return n;
46
+ }
47
+
48
+ // zcode/opencode report reasoning outside output_tokens (vendor semantics:
49
+ // total = identity + reasoning), but the contract's identity requires
50
+ // reasoning to sit inside output (docs/cli-contract.md §2.4: total = input +
51
+ // cached + creation + output, reasoning single-column). Fold that exact shape
52
+ // so the row satisfies the identity; any other mismatch still fails it and
53
+ // the row is dropped — guessing a fix for unknown skews would silently
54
+ // rewrite totals the vendor did report.
55
+ function foldReasoningIntoOutput(tokens) {
56
+ const identity =
57
+ tokens.input_tokens +
58
+ tokens.cached_input_tokens +
59
+ tokens.cache_creation_input_tokens +
60
+ tokens.output_tokens;
61
+ if (
62
+ tokens.total_tokens !== identity &&
63
+ tokens.total_tokens === identity + tokens.reasoning_output_tokens
64
+ ) {
65
+ tokens.output_tokens += tokens.reasoning_output_tokens;
66
+ }
67
+ }
68
+
69
+ function basenameKey(value) {
70
+ const raw = String(value || "").replace(/[\\/]+$/, "");
71
+ if (!raw || raw === "." || raw === "..") return null;
72
+ const base = raw.split(/[\\/]/).pop().trim();
73
+ if (!base || base === "." || base === "..") return null;
74
+ // Control characters are rejected server-side; strip instead of failing.
75
+ // eslint-disable-next-line no-control-regex
76
+ const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
77
+ return clean || null;
78
+ }
79
+
80
+ function validSessionId(value) {
81
+ if (typeof value !== "string") return false;
82
+ const id = value.trim();
83
+ if (!id || id.length > 256) return false;
84
+ if (id.includes("/") || id.includes("\\")) return false;
85
+ // Full Unicode Cc range — the server validator is /\p{Cc}/u, which also
86
+ // covers C1 (U+0080–U+009F); a narrower client check would let such an id
87
+ // through to a whole-batch 400 that stalls the sessions upload.
88
+ // eslint-disable-next-line no-control-regex
89
+ if (/[\u0000-\u001f\u007f-\u009f]/.test(id)) return false;
90
+ return true;
91
+ }
92
+
93
+ /**
94
+ * Map one session-analytics sidecar row onto the contract wire shape.
95
+ * Returns null when the row must not be uploaded (account-level source,
96
+ * missing identity/timestamps, or a token identity that the server would
97
+ * reject with a whole-batch 400 — dropping one bad row beats failing 200).
98
+ *
99
+ * Field locations on the sidecar row: the six token counters live nested in
100
+ * row.tokens (the top level only mirrors total_tokens), and the stable id is
101
+ * row.session_id when the vendor log carried one (Claude/Codex), else the
102
+ * locally-computed sha256 fingerprint row.session_hash (source\0id, 24 hex)
103
+ * — both are opaque stable keys without path leakage, valid as the contract
104
+ * session_id.
105
+ */
106
+ function mapSessionRow(row) {
107
+ if (!row || typeof row !== "object") return null;
108
+ const source = String(row.source || "").trim().toLowerCase();
109
+ if (!source || ACCOUNT_LEVEL_SOURCES.has(source)) return null;
110
+ const sessionId = typeof row.session_id === "string" && row.session_id.trim()
111
+ ? row.session_id.trim()
112
+ : typeof row.session_hash === "string" && row.session_hash.trim()
113
+ ? row.session_hash.trim()
114
+ : null;
115
+ if (!validSessionId(sessionId)) return null;
116
+
117
+ const startedMs = Date.parse(row.started_at || "");
118
+ const endedMs = Date.parse(row.ended_at || "");
119
+ if (!Number.isFinite(startedMs) || !Number.isFinite(endedMs)) return null;
120
+ if (endedMs < startedMs) return null;
121
+
122
+ // Sidecar rows omit zero-valued counters on some sources (zcode drops
123
+ // cache_creation/reasoning keys entirely). The contract requires every
124
+ // field present, so missing keys fill as 0 — the identity check below is
125
+ // the real gate; dropping a whole models array over an elided zero would
126
+ // let a later last-wins upload erase previously stored models.
127
+ const nested = row.tokens && typeof row.tokens === "object" ? row.tokens : {};
128
+ const tokens = {};
129
+ for (const field of TOKEN_FIELDS) {
130
+ const raw = field === "total_tokens" ? row.total_tokens : nested[field];
131
+ const value = safeInt(raw === undefined || raw === null ? 0 : raw);
132
+ if (value === null) return null;
133
+ tokens[field] = value;
134
+ }
135
+ foldReasoningIntoOutput(tokens);
136
+ const identity =
137
+ tokens.input_tokens +
138
+ tokens.cached_input_tokens +
139
+ tokens.cache_creation_input_tokens +
140
+ tokens.output_tokens;
141
+ if (tokens.total_tokens !== identity) return null;
142
+
143
+ // Per-model rows follow the same identity rule. The contract caps models
144
+ // at 32 AND requires sum(models.total_tokens) === session total, so a
145
+ // session with more than 32 models cannot carry a valid array at all —
146
+ // omit it (the contract allows missing models; the session prices as 0).
147
+ let models = [];
148
+ const usageRows = Array.isArray(row.model_usage) ? row.model_usage : [];
149
+ let modelsValid = usageRows.length > 0 && usageRows.length <= 32;
150
+ let modelsSum = 0;
151
+ if (modelsValid) {
152
+ for (const usage of usageRows) {
153
+ const modelTokens = {};
154
+ for (const field of TOKEN_FIELDS) {
155
+ const raw = usage?.[field];
156
+ const value = safeInt(raw === undefined || raw === null ? 0 : raw);
157
+ if (value === null) {
158
+ modelsValid = false;
159
+ break;
160
+ }
161
+ modelTokens[field] = value;
162
+ }
163
+ if (!modelsValid) break;
164
+ foldReasoningIntoOutput(modelTokens);
165
+ const modelIdentity =
166
+ modelTokens.input_tokens +
167
+ modelTokens.cached_input_tokens +
168
+ modelTokens.cache_creation_input_tokens +
169
+ modelTokens.output_tokens;
170
+ if (modelTokens.total_tokens !== modelIdentity) {
171
+ modelsValid = false;
172
+ break;
173
+ }
174
+ modelsSum += modelTokens.total_tokens;
175
+ models.push({ model: String(usage.model || "").trim() || "unknown", ...modelTokens });
176
+ }
177
+ if (modelsValid && modelsSum !== tokens.total_tokens) modelsValid = false;
178
+ }
179
+ if (!modelsValid) models = [];
180
+
181
+ return {
182
+ session_id: sessionId,
183
+ source,
184
+ started_at: new Date(startedMs).toISOString(),
185
+ ended_at: new Date(endedMs).toISOString(),
186
+ project_key: basenameKey(row.project_key ?? row.project_ref),
187
+ ...tokens,
188
+ billable_total_tokens: null,
189
+ ...(models.length > 0 ? { models } : {}),
190
+ };
191
+ }
192
+
193
+ /** Map + filter a sidecar rows array; invalid rows are counted, not fatal. */
194
+ function buildSessionUploadRows(rows) {
195
+ const out = [];
196
+ let skipped = 0;
197
+ for (const row of Array.isArray(rows) ? rows : []) {
198
+ const mapped = mapSessionRow(row);
199
+ if (mapped) out.push(mapped);
200
+ else skipped += 1;
201
+ }
202
+ return { rows: out, skipped };
203
+ }
204
+
205
+ function watermarkKey(source, sessionId) {
206
+ return `${source}|${sessionId}`;
207
+ }
208
+
209
+ function loadWatermarks(filePath) {
210
+ try {
211
+ const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
212
+ const map = new Map();
213
+ for (const [key, value] of Object.entries(
214
+ parsed && typeof parsed === "object" ? parsed : {},
215
+ )) {
216
+ if (typeof key === "string" && typeof value === "string" && key.includes("|")) {
217
+ map.set(key, value);
218
+ }
219
+ }
220
+ return map;
221
+ } catch {
222
+ return new Map();
223
+ }
224
+ }
225
+
226
+ function pruneWatermarks(map) {
227
+ if (map.size <= WATERMARK_MAX_ENTRIES) return map;
228
+ const entries = [...map.entries()].sort((a, b) =>
229
+ String(a[1]).localeCompare(String(b[1])),
230
+ );
231
+ return new Map(entries.slice(entries.length - WATERMARK_MAX_ENTRIES));
232
+ }
233
+
234
+ async function saveWatermarks(filePath, map) {
235
+ const pruned = pruneWatermarks(map);
236
+ const json = JSON.stringify(
237
+ Object.fromEntries([...pruned.entries()].sort((a, b) => a[0].localeCompare(b[0]))),
238
+ null,
239
+ 2,
240
+ ) + "\n";
241
+ await fsp.mkdir(path.dirname(filePath), { recursive: true });
242
+ const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
243
+ await fsp.writeFile(tmp, json, { mode: 0o600 });
244
+ await fsp.rename(tmp, filePath);
245
+ await fsp.chmod(filePath, 0o600).catch(() => {});
246
+ }
247
+
248
+ /** Rows strictly newer than their watermark, ascending by ended_at. */
249
+ function selectSessionsForUpload(rows, watermarks) {
250
+ const pending = rows.filter((row) => {
251
+ const mark = watermarks.get(watermarkKey(row.source, row.session_id));
252
+ return !mark || String(row.ended_at) > mark;
253
+ });
254
+ pending.sort((a, b) => String(a.ended_at).localeCompare(String(b.ended_at)));
255
+ return pending;
256
+ }
257
+
258
+ /**
259
+ * Upload pending session rows in ascending ended_at batches of <=200. The
260
+ * watermark file only advances for batches the server accepted, so a failed
261
+ * batch leaves earlier successes committed and the failure retried next sync.
262
+ */
263
+ /**
264
+ * Split pending rows into request batches. Two independent contract limits
265
+ * apply (§1.2/§2.4): <=200 rows AND <=1 MiB body — a 200-row batch of real
266
+ * sessions with models easily exceeds the byte cap, so batches are cut on
267
+ * whichever limit hits first (bytes measured on the serialized row, with a
268
+ * conservative envelope leaving room for upload_id + JSON framing).
269
+ */
270
+ function buildBatches(pending, maxRows) {
271
+ const BYTE_BUDGET = 500 * 1024;
272
+ const batches = [];
273
+ let current = [];
274
+ let bytes = 0;
275
+
276
+ for (const row of pending) {
277
+ const rowBytes = Buffer.byteLength(JSON.stringify(row), "utf8") + 2;
278
+ if (current.length > 0 && (current.length >= maxRows || bytes + rowBytes > BYTE_BUDGET)) {
279
+ batches.push(current);
280
+ current = [];
281
+ bytes = 0;
282
+ }
283
+ current.push(row);
284
+ bytes += rowBytes;
285
+ }
286
+ if (current.length > 0) batches.push(current);
287
+
288
+ return batches;
289
+ }
290
+
291
+ async function uploadSessions({
292
+ baseUrl,
293
+ anonKey,
294
+ deviceToken,
295
+ sessions,
296
+ watermarkPath,
297
+ batchSize = MAX_SESSIONS_PER_BATCH,
298
+ canUpload = () => true,
299
+ fetchImpl = fetch,
300
+ onProgress = () => {},
301
+ }) {
302
+ // Map first: callers hand us RAW session-analytics rows (tokens nested in
303
+ // row.tokens, sidecar-only fields present); the wire shape is built here so
304
+ // no call site can bypass the whitelist/identity checks.
305
+ const { rows: mapped, skipped: skippedLocal } = buildSessionUploadRows(sessions);
306
+ const watermarks = loadWatermarks(watermarkPath);
307
+ const pending = selectSessionsForUpload(mapped, watermarks);
308
+ const maxRows = Math.min(Math.max(1, batchSize), MAX_SESSIONS_PER_BATCH);
309
+ const uploadId = crypto.randomUUID();
310
+ let upserted = 0;
311
+ let skippedServer = 0;
312
+ let batches = 0;
313
+ let uploaded = 0;
314
+
315
+ for (const batch of buildBatches(pending, maxRows)) {
316
+ if (!canUpload()) break;
317
+ const res = await fetchFunctionResponse(
318
+ functionUrlFor(baseUrl, INGEST_SLUG),
319
+ {
320
+ method: "POST",
321
+ headers: {
322
+ "Content-Type": "application/json",
323
+ Accept: "application/json",
324
+ Authorization: `Bearer ${deviceToken}`,
325
+ ...(anonKey ? { apikey: anonKey } : {}),
326
+ },
327
+ body: JSON.stringify({
328
+ upload_id: uploadId,
329
+ sessions: batch,
330
+ }),
331
+ },
332
+ fetchImpl,
333
+ );
334
+ if (!res.ok) {
335
+ const text = await res.text().catch(() => "");
336
+ const error = new Error(
337
+ `sessions upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`,
338
+ );
339
+ error.status = res.status;
340
+ error.uploaded = uploaded;
341
+ error.upserted = upserted;
342
+ throw error;
343
+ }
344
+ const data = await res.json().catch(() => ({}));
345
+ batches += 1;
346
+ uploaded += batch.length;
347
+ upserted += Number(data?.sessions_upserted || 0);
348
+ skippedServer += Number(data?.sessions_skipped || 0);
349
+ for (const row of batch) {
350
+ watermarks.set(watermarkKey(row.source, row.session_id), row.ended_at);
351
+ }
352
+ await saveWatermarks(watermarkPath, watermarks);
353
+ onProgress({ batches, uploaded, upserted });
354
+ }
355
+
356
+ return {
357
+ pending: pending.length,
358
+ uploaded,
359
+ upserted,
360
+ skippedServer,
361
+ skippedLocal,
362
+ batches,
363
+ };
364
+ }
365
+
366
+ module.exports = {
367
+ ACCOUNT_LEVEL_SOURCES,
368
+ MAX_SESSIONS_PER_BATCH,
369
+ basenameKey,
370
+ buildBatches,
371
+ buildSessionUploadRows,
372
+ loadWatermarks,
373
+ mapSessionRow,
374
+ saveWatermarks,
375
+ selectSessionsForUpload,
376
+ uploadSessions,
377
+ validSessionId,
378
+ watermarkKey,
379
+ };