tokenusage-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +22 -0
- package/README.md +94 -0
- package/THIRD_PARTY_NOTICES.md +140 -0
- package/bin/tokenusage.js +69 -0
- package/package.json +59 -0
- package/src/cli.js +134 -0
- package/src/commands/device-login.js +205 -0
- package/src/commands/diagnostics.js +38 -0
- package/src/commands/doctor.js +110 -0
- package/src/commands/init.js +2436 -0
- package/src/commands/sessions.js +29 -0
- package/src/commands/status.js +1747 -0
- package/src/commands/sync.js +7934 -0
- package/src/commands/uninstall.js +312 -0
- package/src/lib/account-usage-dedup.js +267 -0
- package/src/lib/antigravity-paths.js +48 -0
- package/src/lib/ark-agent-plan-limits.js +438 -0
- package/src/lib/ark-coding-plan-limits.js +537 -0
- package/src/lib/async-queue-reader.js +138 -0
- package/src/lib/atomic-json.js +27 -0
- package/src/lib/bearer-token.js +11 -0
- package/src/lib/categorizer-utils.js +232 -0
- package/src/lib/claude-categorizer.js +1343 -0
- package/src/lib/claude-config.js +279 -0
- package/src/lib/claude-user-identity.js +56 -0
- package/src/lib/cli-ui.js +179 -0
- package/src/lib/cloud-account.js +541 -0
- package/src/lib/cloud-device-token.js +141 -0
- package/src/lib/cloud-sync-prefs.js +14 -0
- package/src/lib/cloud-usage-reset.js +115 -0
- package/src/lib/codex-config.js +595 -0
- package/src/lib/codex-context-breakdown.js +806 -0
- package/src/lib/codex-model-attribution.js +128 -0
- package/src/lib/codex-rollout-parser.js +1471 -0
- package/src/lib/codex-service-tier.js +49 -0
- package/src/lib/codex-token-refresh.js +140 -0
- package/src/lib/codex-token-usage.js +195 -0
- package/src/lib/codex-usage-record.js +195 -0
- package/src/lib/command-runner.js +313 -0
- package/src/lib/commandcode-limits.js +317 -0
- package/src/lib/context-health.js +124 -0
- package/src/lib/cursor-config.js +605 -0
- package/src/lib/cursor-store.js +1918 -0
- package/src/lib/debug-flags.js +7 -0
- package/src/lib/devin-limits.js +249 -0
- package/src/lib/devin-usage.js +212 -0
- package/src/lib/diagnostics.js +383 -0
- package/src/lib/doctor.js +385 -0
- package/src/lib/dumate.js +95 -0
- package/src/lib/fs.js +455 -0
- package/src/lib/function-url.js +14 -0
- package/src/lib/gemini-config.js +291 -0
- package/src/lib/git-outcomes.js +243 -0
- package/src/lib/grok-context-breakdown.js +890 -0
- package/src/lib/grok-hook.js +360 -0
- package/src/lib/grok-limits.js +865 -0
- package/src/lib/grok-usage.js +98 -0
- package/src/lib/heatmap-compact.js +114 -0
- package/src/lib/ingest-sanitize.js +41 -0
- package/src/lib/init-flow.js +42 -0
- package/src/lib/install-resolver.js +170 -0
- package/src/lib/jsonl-lines.js +134 -0
- package/src/lib/kimi-model-repair.js +173 -0
- package/src/lib/kimi-profile.js +61 -0
- package/src/lib/kimi-session.js +216 -0
- package/src/lib/local-api.js +3588 -0
- package/src/lib/machine-id.js +238 -0
- package/src/lib/multi-install-parser.js +113 -0
- package/src/lib/notify-command.js +38 -0
- package/src/lib/omp-hook.js +589 -0
- package/src/lib/openclaw-cli.js +104 -0
- package/src/lib/openclaw-hook.js +375 -0
- package/src/lib/openclaw-session-plugin.js +560 -0
- package/src/lib/opencode-config.js +121 -0
- package/src/lib/opencode-go-limits.js +807 -0
- package/src/lib/outcomes-engine.js +253 -0
- package/src/lib/passive-mode.js +219 -0
- package/src/lib/pet-packages.js +722 -0
- package/src/lib/pricing/curated-overrides.json +822 -0
- package/src/lib/pricing/index.js +379 -0
- package/src/lib/pricing/litellm-fetcher.js +172 -0
- package/src/lib/pricing/matcher.js +456 -0
- package/src/lib/pricing/seed-snapshot.json +1 -0
- package/src/lib/progress.js +76 -0
- package/src/lib/project-usage-purge.js +117 -0
- package/src/lib/prompt.js +24 -0
- package/src/lib/provider-status.js +104 -0
- package/src/lib/proxy-env.js +489 -0
- package/src/lib/proxy-settings.js +163 -0
- package/src/lib/qoder-cn-cli-limits.js +119 -0
- package/src/lib/qoder-limits.js +880 -0
- package/src/lib/quiet-warnings.js +26 -0
- package/src/lib/rollout.js +25966 -0
- package/src/lib/runtime-config.js +259 -0
- package/src/lib/scan-roots.js +277 -0
- package/src/lib/sentry.js +231 -0
- package/src/lib/session-analytics.js +2972 -0
- package/src/lib/session-performance.js +237 -0
- package/src/lib/session-upload.js +379 -0
- package/src/lib/skill-usage.js +267 -0
- package/src/lib/skills-manager.js +1591 -0
- package/src/lib/source-metadata.js +48 -0
- package/src/lib/sqlite-reader.js +215 -0
- package/src/lib/star-cta.js +73 -0
- package/src/lib/subscription-manager.js +293 -0
- package/src/lib/subscriptions.js +508 -0
- package/src/lib/telemetry.js +193 -0
- package/src/lib/tracker-paths.js +16 -0
- package/src/lib/trae-cn-config.js +548 -0
- package/src/lib/trae-db.js +127 -0
- package/src/lib/trae-snapshot.js +230 -0
- package/src/lib/trae-sqlite.js +143 -0
- package/src/lib/trae-usage.js +121 -0
- package/src/lib/upload-queue.js +134 -0
- package/src/lib/upload-throttle.js +171 -0
- package/src/lib/usage-limits.js +4552 -0
- package/src/lib/wrapped-aggregator.js +236 -0
- package/src/lib/wsl-probe.js +281 -0
- package/src/lib/zcode-limits.js +1172 -0
- package/src/lib/zcode-reset-credits.js +66 -0
- package/test/cli-smoke.test.js +53 -0
- package/test/ingest-sanitize.test.js +162 -0
- package/test/quiet-warnings.test.js +53 -0
- package/test/session-upload.test.js +427 -0
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { canonicalUsage } = require("./codex-token-usage");
|
|
4
|
+
|
|
5
|
+
const SUM_FIELDS = [
|
|
6
|
+
"estimated_output_tokens", "estimated_duration_ms", "estimated_request_count",
|
|
7
|
+
"first_response_total_ms", "first_response_sample_count",
|
|
8
|
+
];
|
|
9
|
+
const MIN_OUTPUT = 200;
|
|
10
|
+
const MIN_DURATION_MS = 1000;
|
|
11
|
+
const MAX_DURATION_MS = 60 * 60 * 1000;
|
|
12
|
+
const SAME_FLUSH_MS = 100;
|
|
13
|
+
|
|
14
|
+
function nonNegative(value) {
|
|
15
|
+
const number = Number(value);
|
|
16
|
+
return Number.isFinite(number) && number >= 0 ? number : 0;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function normalizePerformance(value) {
|
|
20
|
+
const out = Object.fromEntries(SUM_FIELDS.map((field) => [field, nonNegative(value?.[field])]));
|
|
21
|
+
out.estimated_tokens_per_second = out.estimated_request_count > 0 && out.estimated_output_tokens > 0 && out.estimated_duration_ms > 0
|
|
22
|
+
? out.estimated_output_tokens / (out.estimated_duration_ms / 1000) : null;
|
|
23
|
+
out.first_response_ms = out.first_response_sample_count > 0
|
|
24
|
+
? out.first_response_total_ms / out.first_response_sample_count : null;
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function mergePerformance(...values) {
|
|
29
|
+
const out = normalizePerformance();
|
|
30
|
+
for (const value of values) {
|
|
31
|
+
for (const field of SUM_FIELDS) out[field] += nonNegative(value?.[field]);
|
|
32
|
+
}
|
|
33
|
+
return normalizePerformance(out);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function timestampMs(value) {
|
|
37
|
+
if (typeof value !== "string") return null;
|
|
38
|
+
const number = Date.parse(value);
|
|
39
|
+
return Number.isFinite(number) ? number : null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function accumulator() {
|
|
43
|
+
let total = normalizePerformance();
|
|
44
|
+
const byModel = new Map();
|
|
45
|
+
function add(model, delta) {
|
|
46
|
+
total = mergePerformance(total, delta);
|
|
47
|
+
if (model) byModel.set(model, mergePerformance(byModel.get(model), delta));
|
|
48
|
+
}
|
|
49
|
+
function request(model, outputTokens, start, end) {
|
|
50
|
+
const duration = start !== null && end !== null ? end - start : 0;
|
|
51
|
+
record(model, outputTokens, duration);
|
|
52
|
+
}
|
|
53
|
+
function requestDuration(model, outputTokens, durationMs) {
|
|
54
|
+
record(model, outputTokens, durationMs);
|
|
55
|
+
}
|
|
56
|
+
function record(model, outputTokens, duration) {
|
|
57
|
+
if (outputTokens < MIN_OUTPUT || duration < MIN_DURATION_MS || duration > MAX_DURATION_MS) return;
|
|
58
|
+
add(model, { estimated_output_tokens: outputTokens, estimated_duration_ms: duration, estimated_request_count: 1 });
|
|
59
|
+
}
|
|
60
|
+
return { add, request, requestDuration, finish: () => ({ performance: total, byModel }) };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function createClaudePerformanceCollector() {
|
|
64
|
+
const nodes = new Map();
|
|
65
|
+
const messages = new Map();
|
|
66
|
+
const totals = accumulator();
|
|
67
|
+
|
|
68
|
+
function consume(obj, key, countedUsage, model) {
|
|
69
|
+
const timestamp = timestampMs(obj?.timestamp || obj?.message?.timestamp);
|
|
70
|
+
if (typeof obj?.uuid === "string" && !nodes.has(obj.uuid)) {
|
|
71
|
+
nodes.set(obj.uuid, {
|
|
72
|
+
parent: obj.parentUuid, timestamp, type: obj.type,
|
|
73
|
+
invalid: Boolean(obj.isApiErrorMessage || obj.isMeta),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
if (obj?.type !== "assistant" || !key) return;
|
|
77
|
+
let timing = messages.get(key);
|
|
78
|
+
if (!timing) {
|
|
79
|
+
// Tokens come from scanClaudeSession's canonical, once-counted delta.
|
|
80
|
+
// Later content blocks extend only the timing, never the numerator.
|
|
81
|
+
timing = {
|
|
82
|
+
parent: obj.parentUuid, startsAtFirstBlock: obj.apiBlockIndex == null || obj.apiBlockIndex === 0,
|
|
83
|
+
end: null, complete: false, invalid: false, model,
|
|
84
|
+
output: nonNegative(countedUsage?.output_tokens) + nonNegative(countedUsage?.reasoning_output_tokens),
|
|
85
|
+
};
|
|
86
|
+
messages.set(key, timing);
|
|
87
|
+
}
|
|
88
|
+
if (timestamp !== null) timing.end = timing.end === null ? timestamp : Math.max(timing.end, timestamp);
|
|
89
|
+
timing.complete ||= ["end_turn", "tool_use", "max_tokens", "stop_sequence"].includes(obj.message?.stop_reason);
|
|
90
|
+
timing.invalid ||= Boolean(obj.isApiErrorMessage || obj.message?.error);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function finish() {
|
|
94
|
+
for (const timing of messages.values()) {
|
|
95
|
+
if (!timing.complete || timing.invalid || !timing.startsAtFirstBlock) continue;
|
|
96
|
+
let parent = timing.parent;
|
|
97
|
+
for (let hop = 0; hop < 32 && parent; hop += 1) {
|
|
98
|
+
const node = nodes.get(parent);
|
|
99
|
+
if (!node) break;
|
|
100
|
+
if (node.type === "attachment") { parent = node.parent; continue; }
|
|
101
|
+
// A user row includes tool results. Their completion timestamp keeps
|
|
102
|
+
// time spent executing the tool outside the next inference window.
|
|
103
|
+
if (node.type === "user" && !node.invalid) totals.request(timing.model, timing.output, node.timestamp, timing.end);
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return totals.finish();
|
|
108
|
+
}
|
|
109
|
+
return { consume, finish };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function createCodexPerformanceCollector() {
|
|
113
|
+
const totals = accumulator();
|
|
114
|
+
let boundary = null;
|
|
115
|
+
let lastTokenCount = null;
|
|
116
|
+
let start = null;
|
|
117
|
+
let lastOutput = null;
|
|
118
|
+
let toolAfterOutput = null;
|
|
119
|
+
let usageRecord = null;
|
|
120
|
+
let invalid = false;
|
|
121
|
+
let turnKey = null;
|
|
122
|
+
let turnModels = new Set();
|
|
123
|
+
const seenFirstResponses = new Set();
|
|
124
|
+
|
|
125
|
+
function reset() {
|
|
126
|
+
start = null;
|
|
127
|
+
lastOutput = null;
|
|
128
|
+
toolAfterOutput = null;
|
|
129
|
+
usageRecord = null;
|
|
130
|
+
invalid = false;
|
|
131
|
+
}
|
|
132
|
+
function noteBoundary(ms) {
|
|
133
|
+
if (ms !== null) boundary = boundary === null ? ms : Math.max(boundary, ms);
|
|
134
|
+
}
|
|
135
|
+
function consume(obj, model) {
|
|
136
|
+
const ms = timestampMs(obj?.timestamp);
|
|
137
|
+
const payload = obj?.payload || {};
|
|
138
|
+
if (obj?.type === "compacted") {
|
|
139
|
+
reset();
|
|
140
|
+
noteBoundary(ms);
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (obj?.type === "event_msg" && ["task_started", "turn_aborted"].includes(payload.type)) {
|
|
144
|
+
reset();
|
|
145
|
+
noteBoundary(ms);
|
|
146
|
+
turnModels = new Set();
|
|
147
|
+
turnKey = payload.turn_id || null;
|
|
148
|
+
if (payload.type === "turn_aborted") invalid = true;
|
|
149
|
+
}
|
|
150
|
+
if (obj?.type === "turn_context") {
|
|
151
|
+
const nextKey = payload.turn_id || null;
|
|
152
|
+
if (turnKey && nextKey && turnKey !== nextKey) { reset(); turnModels = new Set(); }
|
|
153
|
+
turnKey = nextKey || turnKey;
|
|
154
|
+
noteBoundary(ms);
|
|
155
|
+
}
|
|
156
|
+
const responseType = String(payload.type || "");
|
|
157
|
+
const isModelOutput = obj?.type === "response_item" && (responseType === "reasoning"
|
|
158
|
+
|| responseType === "agent_message" || responseType.endsWith("_call")
|
|
159
|
+
|| (responseType === "message" && payload.role === "assistant"));
|
|
160
|
+
if (model && (obj?.type === "turn_context" || isModelOutput
|
|
161
|
+
|| payload.type === "model_rerouted")) turnModels.add(model);
|
|
162
|
+
if (obj?.type === "event_msg" && ["error", "api_error"].includes(payload.type)) invalid = true;
|
|
163
|
+
if (obj?.type === "event_msg" && payload.type === "task_complete") {
|
|
164
|
+
const raw = payload.time_to_first_token_ms;
|
|
165
|
+
const key = payload.turn_id || turnKey || obj.timestamp;
|
|
166
|
+
if (!invalid && typeof raw === "number" && Number.isFinite(raw) && raw >= 0 && !seenFirstResponses.has(key)) {
|
|
167
|
+
seenFirstResponses.add(key);
|
|
168
|
+
totals.add(turnModels.size === 1 ? [...turnModels][0] : null, { first_response_total_ms: raw, first_response_sample_count: 1 });
|
|
169
|
+
}
|
|
170
|
+
reset();
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
if (obj?.type === "token_usage_record") {
|
|
174
|
+
usageRecord = { end: ms, usage: canonicalUsage(payload.usage) };
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
if (obj?.type !== "response_item") return;
|
|
178
|
+
const type = String(payload.type || "");
|
|
179
|
+
if (type.endsWith("_output")) {
|
|
180
|
+
noteBoundary(ms);
|
|
181
|
+
if (lastOutput !== null) toolAfterOutput = ms;
|
|
182
|
+
} else if ((type === "message" && payload.role !== "assistant") || type === "user_message") {
|
|
183
|
+
noteBoundary(ms);
|
|
184
|
+
} else if (type === "reasoning" || type === "agent_message" || type.endsWith("_call") || (type === "message" && payload.role === "assistant")) {
|
|
185
|
+
// Old rollouts flush an output item just after its token_count.
|
|
186
|
+
if (ms === null || (start === null && lastTokenCount !== null && ms >= lastTokenCount && ms - lastTokenCount <= SAME_FLUSH_MS)) return;
|
|
187
|
+
if (start === null) start = boundary;
|
|
188
|
+
lastOutput = ms;
|
|
189
|
+
toolAfterOutput = null;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
function consumeUsage({ timestamp, delta, rawUsage, model }) {
|
|
193
|
+
// A repeated cumulative snapshot can be written while the next response
|
|
194
|
+
// is still generating. It is not a completion boundary for that response.
|
|
195
|
+
if (!delta || delta.total_tokens <= 0) return;
|
|
196
|
+
const ms = timestampMs(timestamp);
|
|
197
|
+
const raw = canonicalUsage(rawUsage);
|
|
198
|
+
const matches = raw && usageRecord?.usage && ["input_tokens", "cached_input_tokens", "output_tokens"]
|
|
199
|
+
.every((field) => raw[field] === usageRecord.usage[field]);
|
|
200
|
+
const waitedForTools = toolAfterOutput !== null && ms !== null && ms >= toolAfterOutput;
|
|
201
|
+
// Once a tool result follows the final output marker, the token_count's
|
|
202
|
+
// timestamp contains execution time. The output marker is the available
|
|
203
|
+
// response boundary even if the count was flushed much later.
|
|
204
|
+
const end = matches ? usageRecord.end : waitedForTools ? lastOutput : ms;
|
|
205
|
+
// Missing output markers cannot distinguish response timing from a
|
|
206
|
+
// snapshot or inherited context, so even a positive token delta is skipped.
|
|
207
|
+
if (!invalid && lastOutput !== null) totals.request(model, nonNegative(delta.output_tokens), start, end);
|
|
208
|
+
noteBoundary(ms);
|
|
209
|
+
lastTokenCount = ms;
|
|
210
|
+
reset();
|
|
211
|
+
}
|
|
212
|
+
return { consume, consumeUsage, finish: totals.finish };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// ZCode's CLI DB stores per-request wall-clock counters directly in its
|
|
216
|
+
// model_usage table (duration_ms, time_to_first_token_ms) instead of an event
|
|
217
|
+
// stream, so this collector is a thin adapter: each completed request becomes
|
|
218
|
+
// one duration-bounded sample, never a token total (those come from the
|
|
219
|
+
// shared usage reader so the browser cannot drift from the usage card).
|
|
220
|
+
function createZcodePerformanceCollector() {
|
|
221
|
+
const totals = accumulator();
|
|
222
|
+
function consumeUsage({ model, outputTokens, durationMs, timeToFirstTokenMs } = {}) {
|
|
223
|
+
totals.requestDuration(model, nonNegative(outputTokens), nonNegative(durationMs));
|
|
224
|
+
if (typeof timeToFirstTokenMs === "number" && Number.isFinite(timeToFirstTokenMs) && timeToFirstTokenMs >= 0) {
|
|
225
|
+
totals.add(null, { first_response_total_ms: timeToFirstTokenMs, first_response_sample_count: 1 });
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return { consumeUsage, finish: totals.finish };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
module.exports = {
|
|
232
|
+
normalizePerformance,
|
|
233
|
+
mergePerformance,
|
|
234
|
+
createClaudePerformanceCollector,
|
|
235
|
+
createCodexPerformanceCollector,
|
|
236
|
+
createZcodePerformanceCollector,
|
|
237
|
+
};
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Session metadata upload for the token-usage ingest (docs/cli-contract.md
|
|
4
|
+
// §2.4 `sessions`). Maps the local session-analytics sidecar rows onto the
|
|
5
|
+
// contract's wire shape, filters through per-(source, session_id) ended_at
|
|
6
|
+
// watermarks so repeat syncs are increments, and POSTs batches of <=200 to
|
|
7
|
+
// the same ingest endpoint the hourly buckets use.
|
|
8
|
+
//
|
|
9
|
+
// Privacy boundary (docs/cli-contract.md §2.4): only the contract's whitelisted
|
|
10
|
+
// fields are sent. The sidecar's title, project_ref (local cwd), resume
|
|
11
|
+
// identifiers and cache keys never leave the process — buildSessionUploadRows
|
|
12
|
+
// constructs each row field-by-field rather than spreading.
|
|
13
|
+
|
|
14
|
+
const fs = require("node:fs");
|
|
15
|
+
const fsp = require("node:fs/promises");
|
|
16
|
+
const path = require("node:path");
|
|
17
|
+
const crypto = require("node:crypto");
|
|
18
|
+
|
|
19
|
+
const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
|
|
20
|
+
|
|
21
|
+
const INGEST_SLUG = "token-usage-ingest";
|
|
22
|
+
const MAX_SESSIONS_PER_BATCH = 200;
|
|
23
|
+
// Contract: source is trimmed + lowercased; these two are account-level and
|
|
24
|
+
// the server skips them wholesale — the CLI just never uploads them.
|
|
25
|
+
const ACCOUNT_LEVEL_SOURCES = new Set(["cursor", "trae-cn"]);
|
|
26
|
+
// Keep the watermark map bounded: sessions older than the newest N never
|
|
27
|
+
// re-upload anyway (their ended_at watermark only moves forward).
|
|
28
|
+
const WATERMARK_MAX_ENTRIES = 5000;
|
|
29
|
+
|
|
30
|
+
// Six token counters ride along everywhere; total must equal the sum of the
|
|
31
|
+
// four that count toward it (reasoning is a sub-view of output, single-column).
|
|
32
|
+
const TOKEN_FIELDS = [
|
|
33
|
+
"input_tokens",
|
|
34
|
+
"cached_input_tokens",
|
|
35
|
+
"cache_creation_input_tokens",
|
|
36
|
+
"output_tokens",
|
|
37
|
+
"reasoning_output_tokens",
|
|
38
|
+
"total_tokens",
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
function safeInt(value) {
|
|
42
|
+
if (value === null || value === undefined) return null;
|
|
43
|
+
const n = Number(value);
|
|
44
|
+
if (!Number.isSafeInteger(n) || n < 0) return null;
|
|
45
|
+
return n;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// zcode/opencode report reasoning outside output_tokens (vendor semantics:
|
|
49
|
+
// total = identity + reasoning), but the contract's identity requires
|
|
50
|
+
// reasoning to sit inside output (docs/cli-contract.md §2.4: total = input +
|
|
51
|
+
// cached + creation + output, reasoning single-column). Fold that exact shape
|
|
52
|
+
// so the row satisfies the identity; any other mismatch still fails it and
|
|
53
|
+
// the row is dropped — guessing a fix for unknown skews would silently
|
|
54
|
+
// rewrite totals the vendor did report.
|
|
55
|
+
function foldReasoningIntoOutput(tokens) {
|
|
56
|
+
const identity =
|
|
57
|
+
tokens.input_tokens +
|
|
58
|
+
tokens.cached_input_tokens +
|
|
59
|
+
tokens.cache_creation_input_tokens +
|
|
60
|
+
tokens.output_tokens;
|
|
61
|
+
if (
|
|
62
|
+
tokens.total_tokens !== identity &&
|
|
63
|
+
tokens.total_tokens === identity + tokens.reasoning_output_tokens
|
|
64
|
+
) {
|
|
65
|
+
tokens.output_tokens += tokens.reasoning_output_tokens;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function basenameKey(value) {
|
|
70
|
+
const raw = String(value || "").replace(/[\\/]+$/, "");
|
|
71
|
+
if (!raw || raw === "." || raw === "..") return null;
|
|
72
|
+
const base = raw.split(/[\\/]/).pop().trim();
|
|
73
|
+
if (!base || base === "." || base === "..") return null;
|
|
74
|
+
// Control characters are rejected server-side; strip instead of failing.
|
|
75
|
+
// eslint-disable-next-line no-control-regex
|
|
76
|
+
const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
|
|
77
|
+
return clean || null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function validSessionId(value) {
|
|
81
|
+
if (typeof value !== "string") return false;
|
|
82
|
+
const id = value.trim();
|
|
83
|
+
if (!id || id.length > 256) return false;
|
|
84
|
+
if (id.includes("/") || id.includes("\\")) return false;
|
|
85
|
+
// Full Unicode Cc range — the server validator is /\p{Cc}/u, which also
|
|
86
|
+
// covers C1 (U+0080–U+009F); a narrower client check would let such an id
|
|
87
|
+
// through to a whole-batch 400 that stalls the sessions upload.
|
|
88
|
+
// eslint-disable-next-line no-control-regex
|
|
89
|
+
if (/[\u0000-\u001f\u007f-\u009f]/.test(id)) return false;
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Map one session-analytics sidecar row onto the contract wire shape.
|
|
95
|
+
* Returns null when the row must not be uploaded (account-level source,
|
|
96
|
+
* missing identity/timestamps, or a token identity that the server would
|
|
97
|
+
* reject with a whole-batch 400 — dropping one bad row beats failing 200).
|
|
98
|
+
*
|
|
99
|
+
* Field locations on the sidecar row: the six token counters live nested in
|
|
100
|
+
* row.tokens (the top level only mirrors total_tokens), and the stable id is
|
|
101
|
+
* row.session_id when the vendor log carried one (Claude/Codex), else the
|
|
102
|
+
* locally-computed sha256 fingerprint row.session_hash (source\0id, 24 hex)
|
|
103
|
+
* — both are opaque stable keys without path leakage, valid as the contract
|
|
104
|
+
* session_id.
|
|
105
|
+
*/
|
|
106
|
+
function mapSessionRow(row) {
|
|
107
|
+
if (!row || typeof row !== "object") return null;
|
|
108
|
+
const source = String(row.source || "").trim().toLowerCase();
|
|
109
|
+
if (!source || ACCOUNT_LEVEL_SOURCES.has(source)) return null;
|
|
110
|
+
const sessionId = typeof row.session_id === "string" && row.session_id.trim()
|
|
111
|
+
? row.session_id.trim()
|
|
112
|
+
: typeof row.session_hash === "string" && row.session_hash.trim()
|
|
113
|
+
? row.session_hash.trim()
|
|
114
|
+
: null;
|
|
115
|
+
if (!validSessionId(sessionId)) return null;
|
|
116
|
+
|
|
117
|
+
const startedMs = Date.parse(row.started_at || "");
|
|
118
|
+
const endedMs = Date.parse(row.ended_at || "");
|
|
119
|
+
if (!Number.isFinite(startedMs) || !Number.isFinite(endedMs)) return null;
|
|
120
|
+
if (endedMs < startedMs) return null;
|
|
121
|
+
|
|
122
|
+
// Sidecar rows omit zero-valued counters on some sources (zcode drops
|
|
123
|
+
// cache_creation/reasoning keys entirely). The contract requires every
|
|
124
|
+
// field present, so missing keys fill as 0 — the identity check below is
|
|
125
|
+
// the real gate; dropping a whole models array over an elided zero would
|
|
126
|
+
// let a later last-wins upload erase previously stored models.
|
|
127
|
+
const nested = row.tokens && typeof row.tokens === "object" ? row.tokens : {};
|
|
128
|
+
const tokens = {};
|
|
129
|
+
for (const field of TOKEN_FIELDS) {
|
|
130
|
+
const raw = field === "total_tokens" ? row.total_tokens : nested[field];
|
|
131
|
+
const value = safeInt(raw === undefined || raw === null ? 0 : raw);
|
|
132
|
+
if (value === null) return null;
|
|
133
|
+
tokens[field] = value;
|
|
134
|
+
}
|
|
135
|
+
foldReasoningIntoOutput(tokens);
|
|
136
|
+
const identity =
|
|
137
|
+
tokens.input_tokens +
|
|
138
|
+
tokens.cached_input_tokens +
|
|
139
|
+
tokens.cache_creation_input_tokens +
|
|
140
|
+
tokens.output_tokens;
|
|
141
|
+
if (tokens.total_tokens !== identity) return null;
|
|
142
|
+
|
|
143
|
+
// Per-model rows follow the same identity rule. The contract caps models
|
|
144
|
+
// at 32 AND requires sum(models.total_tokens) === session total, so a
|
|
145
|
+
// session with more than 32 models cannot carry a valid array at all —
|
|
146
|
+
// omit it (the contract allows missing models; the session prices as 0).
|
|
147
|
+
let models = [];
|
|
148
|
+
const usageRows = Array.isArray(row.model_usage) ? row.model_usage : [];
|
|
149
|
+
let modelsValid = usageRows.length > 0 && usageRows.length <= 32;
|
|
150
|
+
let modelsSum = 0;
|
|
151
|
+
if (modelsValid) {
|
|
152
|
+
for (const usage of usageRows) {
|
|
153
|
+
const modelTokens = {};
|
|
154
|
+
for (const field of TOKEN_FIELDS) {
|
|
155
|
+
const raw = usage?.[field];
|
|
156
|
+
const value = safeInt(raw === undefined || raw === null ? 0 : raw);
|
|
157
|
+
if (value === null) {
|
|
158
|
+
modelsValid = false;
|
|
159
|
+
break;
|
|
160
|
+
}
|
|
161
|
+
modelTokens[field] = value;
|
|
162
|
+
}
|
|
163
|
+
if (!modelsValid) break;
|
|
164
|
+
foldReasoningIntoOutput(modelTokens);
|
|
165
|
+
const modelIdentity =
|
|
166
|
+
modelTokens.input_tokens +
|
|
167
|
+
modelTokens.cached_input_tokens +
|
|
168
|
+
modelTokens.cache_creation_input_tokens +
|
|
169
|
+
modelTokens.output_tokens;
|
|
170
|
+
if (modelTokens.total_tokens !== modelIdentity) {
|
|
171
|
+
modelsValid = false;
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
modelsSum += modelTokens.total_tokens;
|
|
175
|
+
models.push({ model: String(usage.model || "").trim() || "unknown", ...modelTokens });
|
|
176
|
+
}
|
|
177
|
+
if (modelsValid && modelsSum !== tokens.total_tokens) modelsValid = false;
|
|
178
|
+
}
|
|
179
|
+
if (!modelsValid) models = [];
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
session_id: sessionId,
|
|
183
|
+
source,
|
|
184
|
+
started_at: new Date(startedMs).toISOString(),
|
|
185
|
+
ended_at: new Date(endedMs).toISOString(),
|
|
186
|
+
project_key: basenameKey(row.project_key ?? row.project_ref),
|
|
187
|
+
...tokens,
|
|
188
|
+
billable_total_tokens: null,
|
|
189
|
+
...(models.length > 0 ? { models } : {}),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Map + filter a sidecar rows array; invalid rows are counted, not fatal. */
|
|
194
|
+
function buildSessionUploadRows(rows) {
|
|
195
|
+
const out = [];
|
|
196
|
+
let skipped = 0;
|
|
197
|
+
for (const row of Array.isArray(rows) ? rows : []) {
|
|
198
|
+
const mapped = mapSessionRow(row);
|
|
199
|
+
if (mapped) out.push(mapped);
|
|
200
|
+
else skipped += 1;
|
|
201
|
+
}
|
|
202
|
+
return { rows: out, skipped };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function watermarkKey(source, sessionId) {
|
|
206
|
+
return `${source}|${sessionId}`;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function loadWatermarks(filePath) {
|
|
210
|
+
try {
|
|
211
|
+
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
212
|
+
const map = new Map();
|
|
213
|
+
for (const [key, value] of Object.entries(
|
|
214
|
+
parsed && typeof parsed === "object" ? parsed : {},
|
|
215
|
+
)) {
|
|
216
|
+
if (typeof key === "string" && typeof value === "string" && key.includes("|")) {
|
|
217
|
+
map.set(key, value);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return map;
|
|
221
|
+
} catch {
|
|
222
|
+
return new Map();
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function pruneWatermarks(map) {
|
|
227
|
+
if (map.size <= WATERMARK_MAX_ENTRIES) return map;
|
|
228
|
+
const entries = [...map.entries()].sort((a, b) =>
|
|
229
|
+
String(a[1]).localeCompare(String(b[1])),
|
|
230
|
+
);
|
|
231
|
+
return new Map(entries.slice(entries.length - WATERMARK_MAX_ENTRIES));
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function saveWatermarks(filePath, map) {
|
|
235
|
+
const pruned = pruneWatermarks(map);
|
|
236
|
+
const json = JSON.stringify(
|
|
237
|
+
Object.fromEntries([...pruned.entries()].sort((a, b) => a[0].localeCompare(b[0]))),
|
|
238
|
+
null,
|
|
239
|
+
2,
|
|
240
|
+
) + "\n";
|
|
241
|
+
await fsp.mkdir(path.dirname(filePath), { recursive: true });
|
|
242
|
+
const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
|
|
243
|
+
await fsp.writeFile(tmp, json, { mode: 0o600 });
|
|
244
|
+
await fsp.rename(tmp, filePath);
|
|
245
|
+
await fsp.chmod(filePath, 0o600).catch(() => {});
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Rows strictly newer than their watermark, ascending by ended_at. */
|
|
249
|
+
function selectSessionsForUpload(rows, watermarks) {
|
|
250
|
+
const pending = rows.filter((row) => {
|
|
251
|
+
const mark = watermarks.get(watermarkKey(row.source, row.session_id));
|
|
252
|
+
return !mark || String(row.ended_at) > mark;
|
|
253
|
+
});
|
|
254
|
+
pending.sort((a, b) => String(a.ended_at).localeCompare(String(b.ended_at)));
|
|
255
|
+
return pending;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Upload pending session rows in ascending ended_at batches of <=200. The
|
|
260
|
+
* watermark file only advances for batches the server accepted, so a failed
|
|
261
|
+
* batch leaves earlier successes committed and the failure retried next sync.
|
|
262
|
+
*/
|
|
263
|
+
/**
|
|
264
|
+
* Split pending rows into request batches. Two independent contract limits
|
|
265
|
+
* apply (§1.2/§2.4): <=200 rows AND <=1 MiB body — a 200-row batch of real
|
|
266
|
+
* sessions with models easily exceeds the byte cap, so batches are cut on
|
|
267
|
+
* whichever limit hits first (bytes measured on the serialized row, with a
|
|
268
|
+
* conservative envelope leaving room for upload_id + JSON framing).
|
|
269
|
+
*/
|
|
270
|
+
function buildBatches(pending, maxRows) {
|
|
271
|
+
const BYTE_BUDGET = 500 * 1024;
|
|
272
|
+
const batches = [];
|
|
273
|
+
let current = [];
|
|
274
|
+
let bytes = 0;
|
|
275
|
+
|
|
276
|
+
for (const row of pending) {
|
|
277
|
+
const rowBytes = Buffer.byteLength(JSON.stringify(row), "utf8") + 2;
|
|
278
|
+
if (current.length > 0 && (current.length >= maxRows || bytes + rowBytes > BYTE_BUDGET)) {
|
|
279
|
+
batches.push(current);
|
|
280
|
+
current = [];
|
|
281
|
+
bytes = 0;
|
|
282
|
+
}
|
|
283
|
+
current.push(row);
|
|
284
|
+
bytes += rowBytes;
|
|
285
|
+
}
|
|
286
|
+
if (current.length > 0) batches.push(current);
|
|
287
|
+
|
|
288
|
+
return batches;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async function uploadSessions({
|
|
292
|
+
baseUrl,
|
|
293
|
+
anonKey,
|
|
294
|
+
deviceToken,
|
|
295
|
+
sessions,
|
|
296
|
+
watermarkPath,
|
|
297
|
+
batchSize = MAX_SESSIONS_PER_BATCH,
|
|
298
|
+
canUpload = () => true,
|
|
299
|
+
fetchImpl = fetch,
|
|
300
|
+
onProgress = () => {},
|
|
301
|
+
}) {
|
|
302
|
+
// Map first: callers hand us RAW session-analytics rows (tokens nested in
|
|
303
|
+
// row.tokens, sidecar-only fields present); the wire shape is built here so
|
|
304
|
+
// no call site can bypass the whitelist/identity checks.
|
|
305
|
+
const { rows: mapped, skipped: skippedLocal } = buildSessionUploadRows(sessions);
|
|
306
|
+
const watermarks = loadWatermarks(watermarkPath);
|
|
307
|
+
const pending = selectSessionsForUpload(mapped, watermarks);
|
|
308
|
+
const maxRows = Math.min(Math.max(1, batchSize), MAX_SESSIONS_PER_BATCH);
|
|
309
|
+
const uploadId = crypto.randomUUID();
|
|
310
|
+
let upserted = 0;
|
|
311
|
+
let skippedServer = 0;
|
|
312
|
+
let batches = 0;
|
|
313
|
+
let uploaded = 0;
|
|
314
|
+
|
|
315
|
+
for (const batch of buildBatches(pending, maxRows)) {
|
|
316
|
+
if (!canUpload()) break;
|
|
317
|
+
const res = await fetchFunctionResponse(
|
|
318
|
+
functionUrlFor(baseUrl, INGEST_SLUG),
|
|
319
|
+
{
|
|
320
|
+
method: "POST",
|
|
321
|
+
headers: {
|
|
322
|
+
"Content-Type": "application/json",
|
|
323
|
+
Accept: "application/json",
|
|
324
|
+
Authorization: `Bearer ${deviceToken}`,
|
|
325
|
+
...(anonKey ? { apikey: anonKey } : {}),
|
|
326
|
+
},
|
|
327
|
+
body: JSON.stringify({
|
|
328
|
+
upload_id: uploadId,
|
|
329
|
+
sessions: batch,
|
|
330
|
+
}),
|
|
331
|
+
},
|
|
332
|
+
fetchImpl,
|
|
333
|
+
);
|
|
334
|
+
if (!res.ok) {
|
|
335
|
+
const text = await res.text().catch(() => "");
|
|
336
|
+
const error = new Error(
|
|
337
|
+
`sessions upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`,
|
|
338
|
+
);
|
|
339
|
+
error.status = res.status;
|
|
340
|
+
error.uploaded = uploaded;
|
|
341
|
+
error.upserted = upserted;
|
|
342
|
+
throw error;
|
|
343
|
+
}
|
|
344
|
+
const data = await res.json().catch(() => ({}));
|
|
345
|
+
batches += 1;
|
|
346
|
+
uploaded += batch.length;
|
|
347
|
+
upserted += Number(data?.sessions_upserted || 0);
|
|
348
|
+
skippedServer += Number(data?.sessions_skipped || 0);
|
|
349
|
+
for (const row of batch) {
|
|
350
|
+
watermarks.set(watermarkKey(row.source, row.session_id), row.ended_at);
|
|
351
|
+
}
|
|
352
|
+
await saveWatermarks(watermarkPath, watermarks);
|
|
353
|
+
onProgress({ batches, uploaded, upserted });
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
return {
|
|
357
|
+
pending: pending.length,
|
|
358
|
+
uploaded,
|
|
359
|
+
upserted,
|
|
360
|
+
skippedServer,
|
|
361
|
+
skippedLocal,
|
|
362
|
+
batches,
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
module.exports = {
|
|
367
|
+
ACCOUNT_LEVEL_SOURCES,
|
|
368
|
+
MAX_SESSIONS_PER_BATCH,
|
|
369
|
+
basenameKey,
|
|
370
|
+
buildBatches,
|
|
371
|
+
buildSessionUploadRows,
|
|
372
|
+
loadWatermarks,
|
|
373
|
+
mapSessionRow,
|
|
374
|
+
saveWatermarks,
|
|
375
|
+
selectSessionsForUpload,
|
|
376
|
+
uploadSessions,
|
|
377
|
+
validSessionId,
|
|
378
|
+
watermarkKey,
|
|
379
|
+
};
|