tokenusage-cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -19
- package/LICENSE +22 -22
- package/README.md +94 -94
- package/THIRD_PARTY_NOTICES.md +140 -140
- package/bin/tokenusage.js +69 -69
- package/package.json +59 -59
- package/src/cli.js +146 -134
- package/src/commands/device-login.js +205 -205
- package/src/commands/diagnostics.js +38 -38
- package/src/commands/doctor.js +110 -110
- package/src/commands/init.js +2484 -2436
- package/src/commands/pause.js +77 -0
- package/src/commands/sessions.js +29 -29
- package/src/commands/status.js +1758 -1747
- package/src/commands/sync.js +8048 -7934
- package/src/commands/uninstall.js +322 -312
- package/src/lib/account-usage-dedup.js +267 -267
- package/src/lib/antigravity-paths.js +48 -48
- package/src/lib/ark-agent-plan-limits.js +438 -438
- package/src/lib/ark-coding-plan-limits.js +537 -537
- package/src/lib/async-queue-reader.js +138 -138
- package/src/lib/atomic-json.js +27 -27
- package/src/lib/bearer-token.js +11 -11
- package/src/lib/categorizer-utils.js +232 -232
- package/src/lib/claude-categorizer.js +1343 -1343
- package/src/lib/claude-config.js +279 -279
- package/src/lib/claude-user-identity.js +56 -56
- package/src/lib/cli-ui.js +179 -179
- package/src/lib/cloud-account.js +541 -541
- package/src/lib/cloud-device-token.js +141 -141
- package/src/lib/cloud-sync-prefs.js +14 -14
- package/src/lib/cloud-usage-reset.js +115 -115
- package/src/lib/codex-config.js +595 -595
- package/src/lib/codex-context-breakdown.js +806 -806
- package/src/lib/codex-model-attribution.js +128 -128
- package/src/lib/codex-rollout-parser.js +1471 -1471
- package/src/lib/codex-service-tier.js +49 -49
- package/src/lib/codex-token-refresh.js +140 -140
- package/src/lib/codex-token-usage.js +195 -195
- package/src/lib/codex-usage-record.js +195 -195
- package/src/lib/collection-pause.js +77 -0
- package/src/lib/command-runner.js +313 -313
- package/src/lib/commandcode-limits.js +317 -317
- package/src/lib/context-health.js +124 -124
- package/src/lib/cursor-config.js +605 -605
- package/src/lib/cursor-store.js +1918 -1918
- package/src/lib/debug-flags.js +7 -7
- package/src/lib/devin-limits.js +249 -249
- package/src/lib/devin-usage.js +212 -212
- package/src/lib/diagnostics.js +383 -383
- package/src/lib/doctor.js +385 -385
- package/src/lib/dumate.js +95 -95
- package/src/lib/fs.js +455 -455
- package/src/lib/function-url.js +14 -14
- package/src/lib/gemini-config.js +291 -291
- package/src/lib/git-outcomes.js +243 -243
- package/src/lib/grok-context-breakdown.js +890 -890
- package/src/lib/grok-hook.js +360 -360
- package/src/lib/grok-limits.js +865 -865
- package/src/lib/grok-usage.js +98 -98
- package/src/lib/heatmap-compact.js +114 -114
- package/src/lib/ingest-sanitize.js +41 -41
- package/src/lib/init-flow.js +42 -42
- package/src/lib/install-resolver.js +170 -170
- package/src/lib/jsonl-lines.js +134 -134
- package/src/lib/kimi-model-repair.js +173 -173
- package/src/lib/kimi-profile.js +61 -61
- package/src/lib/kimi-session.js +216 -216
- package/src/lib/local-api.js +3588 -3588
- package/src/lib/machine-id.js +238 -238
- package/src/lib/multi-install-parser.js +113 -113
- package/src/lib/notify-command.js +38 -38
- package/src/lib/omp-hook.js +589 -589
- package/src/lib/openclaw-cli.js +104 -104
- package/src/lib/openclaw-hook.js +375 -375
- package/src/lib/openclaw-session-plugin.js +560 -560
- package/src/lib/opencode-config.js +121 -121
- package/src/lib/opencode-go-limits.js +807 -807
- package/src/lib/outcomes-engine.js +253 -253
- package/src/lib/passive-mode.js +219 -219
- package/src/lib/pet-packages.js +722 -722
- package/src/lib/pricing/curated-overrides.json +822 -822
- package/src/lib/pricing/index.js +379 -379
- package/src/lib/pricing/litellm-fetcher.js +172 -172
- package/src/lib/pricing/matcher.js +456 -456
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/progress.js +76 -76
- package/src/lib/project-usage-purge.js +117 -117
- package/src/lib/prompt.js +24 -24
- package/src/lib/provider-status.js +104 -104
- package/src/lib/proxy-env.js +489 -489
- package/src/lib/proxy-settings.js +163 -163
- package/src/lib/qoder-cn-cli-limits.js +119 -119
- package/src/lib/qoder-limits.js +880 -880
- package/src/lib/quiet-warnings.js +26 -26
- package/src/lib/rollout.js +26005 -25966
- package/src/lib/runtime-config.js +9 -4
- package/src/lib/scan-roots.js +277 -277
- package/src/lib/scheduled-task.js +145 -0
- package/src/lib/sentry.js +231 -231
- package/src/lib/session-analytics.js +2972 -2972
- package/src/lib/session-performance.js +237 -237
- package/src/lib/session-upload.js +379 -379
- package/src/lib/skill-usage.js +267 -267
- package/src/lib/skills-manager.js +1591 -1591
- package/src/lib/source-metadata.js +48 -48
- package/src/lib/sqlite-reader.js +215 -215
- package/src/lib/star-cta.js +73 -73
- package/src/lib/subscription-manager.js +293 -293
- package/src/lib/subscriptions.js +508 -508
- package/src/lib/sync-report.js +138 -0
- package/src/lib/telemetry.js +193 -193
- package/src/lib/tracker-paths.js +16 -16
- package/src/lib/trae-cn-config.js +548 -548
- package/src/lib/trae-db.js +127 -127
- package/src/lib/trae-snapshot.js +230 -230
- package/src/lib/trae-sqlite.js +143 -143
- package/src/lib/trae-usage.js +121 -121
- package/src/lib/upload-queue.js +134 -134
- package/src/lib/upload-throttle.js +171 -171
- package/src/lib/usage-limits.js +4552 -4552
- package/src/lib/wrapped-aggregator.js +236 -236
- package/src/lib/wsl-probe.js +281 -281
- package/src/lib/zcode-limits.js +1172 -1172
- package/src/lib/zcode-reset-credits.js +66 -66
- package/test/audit-fixes.test.js +200 -0
- package/test/cli-smoke.test.js +53 -53
- package/test/collection-controls.test.js +181 -0
- package/test/ingest-sanitize.test.js +162 -162
- package/test/quiet-warnings.test.js +53 -53
- package/test/session-upload.test.js +427 -427
|
@@ -1,379 +1,379 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
// Session metadata upload for the token-usage ingest (docs/cli-contract.md
|
|
4
|
-
// §2.4 `sessions`). Maps the local session-analytics sidecar rows onto the
|
|
5
|
-
// contract's wire shape, filters through per-(source, session_id) ended_at
|
|
6
|
-
// watermarks so repeat syncs are increments, and POSTs batches of <=200 to
|
|
7
|
-
// the same ingest endpoint the hourly buckets use.
|
|
8
|
-
//
|
|
9
|
-
// Privacy boundary (docs/cli-contract.md §2.4): only the contract's whitelisted
|
|
10
|
-
// fields are sent. The sidecar's title, project_ref (local cwd), resume
|
|
11
|
-
// identifiers and cache keys never leave the process — buildSessionUploadRows
|
|
12
|
-
// constructs each row field-by-field rather than spreading.
|
|
13
|
-
|
|
14
|
-
const fs = require("node:fs");
|
|
15
|
-
const fsp = require("node:fs/promises");
|
|
16
|
-
const path = require("node:path");
|
|
17
|
-
const crypto = require("node:crypto");
|
|
18
|
-
|
|
19
|
-
const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
|
|
20
|
-
|
|
21
|
-
const INGEST_SLUG = "token-usage-ingest";
|
|
22
|
-
const MAX_SESSIONS_PER_BATCH = 200;
|
|
23
|
-
// Contract: source is trimmed + lowercased; these two are account-level and
|
|
24
|
-
// the server skips them wholesale — the CLI just never uploads them.
|
|
25
|
-
const ACCOUNT_LEVEL_SOURCES = new Set(["cursor", "trae-cn"]);
|
|
26
|
-
// Keep the watermark map bounded: sessions older than the newest N never
|
|
27
|
-
// re-upload anyway (their ended_at watermark only moves forward).
|
|
28
|
-
const WATERMARK_MAX_ENTRIES = 5000;
|
|
29
|
-
|
|
30
|
-
// Six token counters ride along everywhere; total must equal the sum of the
|
|
31
|
-
// four that count toward it (reasoning is a sub-view of output, single-column).
|
|
32
|
-
const TOKEN_FIELDS = [
|
|
33
|
-
"input_tokens",
|
|
34
|
-
"cached_input_tokens",
|
|
35
|
-
"cache_creation_input_tokens",
|
|
36
|
-
"output_tokens",
|
|
37
|
-
"reasoning_output_tokens",
|
|
38
|
-
"total_tokens",
|
|
39
|
-
];
|
|
40
|
-
|
|
41
|
-
function safeInt(value) {
|
|
42
|
-
if (value === null || value === undefined) return null;
|
|
43
|
-
const n = Number(value);
|
|
44
|
-
if (!Number.isSafeInteger(n) || n < 0) return null;
|
|
45
|
-
return n;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
// zcode/opencode report reasoning outside output_tokens (vendor semantics:
|
|
49
|
-
// total = identity + reasoning), but the contract's identity requires
|
|
50
|
-
// reasoning to sit inside output (docs/cli-contract.md §2.4: total = input +
|
|
51
|
-
// cached + creation + output, reasoning single-column). Fold that exact shape
|
|
52
|
-
// so the row satisfies the identity; any other mismatch still fails it and
|
|
53
|
-
// the row is dropped — guessing a fix for unknown skews would silently
|
|
54
|
-
// rewrite totals the vendor did report.
|
|
55
|
-
function foldReasoningIntoOutput(tokens) {
|
|
56
|
-
const identity =
|
|
57
|
-
tokens.input_tokens +
|
|
58
|
-
tokens.cached_input_tokens +
|
|
59
|
-
tokens.cache_creation_input_tokens +
|
|
60
|
-
tokens.output_tokens;
|
|
61
|
-
if (
|
|
62
|
-
tokens.total_tokens !== identity &&
|
|
63
|
-
tokens.total_tokens === identity + tokens.reasoning_output_tokens
|
|
64
|
-
) {
|
|
65
|
-
tokens.output_tokens += tokens.reasoning_output_tokens;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function basenameKey(value) {
|
|
70
|
-
const raw = String(value || "").replace(/[\\/]+$/, "");
|
|
71
|
-
if (!raw || raw === "." || raw === "..") return null;
|
|
72
|
-
const base = raw.split(/[\\/]/).pop().trim();
|
|
73
|
-
if (!base || base === "." || base === "..") return null;
|
|
74
|
-
// Control characters are rejected server-side; strip instead of failing.
|
|
75
|
-
// eslint-disable-next-line no-control-regex
|
|
76
|
-
const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
|
|
77
|
-
return clean || null;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
function validSessionId(value) {
|
|
81
|
-
if (typeof value !== "string") return false;
|
|
82
|
-
const id = value.trim();
|
|
83
|
-
if (!id || id.length > 256) return false;
|
|
84
|
-
if (id.includes("/") || id.includes("\\")) return false;
|
|
85
|
-
// Full Unicode Cc range — the server validator is /\p{Cc}/u, which also
|
|
86
|
-
// covers C1 (U+0080–U+009F); a narrower client check would let such an id
|
|
87
|
-
// through to a whole-batch 400 that stalls the sessions upload.
|
|
88
|
-
// eslint-disable-next-line no-control-regex
|
|
89
|
-
if (/[\u0000-\u001f\u007f-\u009f]/.test(id)) return false;
|
|
90
|
-
return true;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Map one session-analytics sidecar row onto the contract wire shape.
|
|
95
|
-
* Returns null when the row must not be uploaded (account-level source,
|
|
96
|
-
* missing identity/timestamps, or a token identity that the server would
|
|
97
|
-
* reject with a whole-batch 400 — dropping one bad row beats failing 200).
|
|
98
|
-
*
|
|
99
|
-
* Field locations on the sidecar row: the six token counters live nested in
|
|
100
|
-
* row.tokens (the top level only mirrors total_tokens), and the stable id is
|
|
101
|
-
* row.session_id when the vendor log carried one (Claude/Codex), else the
|
|
102
|
-
* locally-computed sha256 fingerprint row.session_hash (source\0id, 24 hex)
|
|
103
|
-
* — both are opaque stable keys without path leakage, valid as the contract
|
|
104
|
-
* session_id.
|
|
105
|
-
*/
|
|
106
|
-
function mapSessionRow(row) {
|
|
107
|
-
if (!row || typeof row !== "object") return null;
|
|
108
|
-
const source = String(row.source || "").trim().toLowerCase();
|
|
109
|
-
if (!source || ACCOUNT_LEVEL_SOURCES.has(source)) return null;
|
|
110
|
-
const sessionId = typeof row.session_id === "string" && row.session_id.trim()
|
|
111
|
-
? row.session_id.trim()
|
|
112
|
-
: typeof row.session_hash === "string" && row.session_hash.trim()
|
|
113
|
-
? row.session_hash.trim()
|
|
114
|
-
: null;
|
|
115
|
-
if (!validSessionId(sessionId)) return null;
|
|
116
|
-
|
|
117
|
-
const startedMs = Date.parse(row.started_at || "");
|
|
118
|
-
const endedMs = Date.parse(row.ended_at || "");
|
|
119
|
-
if (!Number.isFinite(startedMs) || !Number.isFinite(endedMs)) return null;
|
|
120
|
-
if (endedMs < startedMs) return null;
|
|
121
|
-
|
|
122
|
-
// Sidecar rows omit zero-valued counters on some sources (zcode drops
|
|
123
|
-
// cache_creation/reasoning keys entirely). The contract requires every
|
|
124
|
-
// field present, so missing keys fill as 0 — the identity check below is
|
|
125
|
-
// the real gate; dropping a whole models array over an elided zero would
|
|
126
|
-
// let a later last-wins upload erase previously stored models.
|
|
127
|
-
const nested = row.tokens && typeof row.tokens === "object" ? row.tokens : {};
|
|
128
|
-
const tokens = {};
|
|
129
|
-
for (const field of TOKEN_FIELDS) {
|
|
130
|
-
const raw = field === "total_tokens" ? row.total_tokens : nested[field];
|
|
131
|
-
const value = safeInt(raw === undefined || raw === null ? 0 : raw);
|
|
132
|
-
if (value === null) return null;
|
|
133
|
-
tokens[field] = value;
|
|
134
|
-
}
|
|
135
|
-
foldReasoningIntoOutput(tokens);
|
|
136
|
-
const identity =
|
|
137
|
-
tokens.input_tokens +
|
|
138
|
-
tokens.cached_input_tokens +
|
|
139
|
-
tokens.cache_creation_input_tokens +
|
|
140
|
-
tokens.output_tokens;
|
|
141
|
-
if (tokens.total_tokens !== identity) return null;
|
|
142
|
-
|
|
143
|
-
// Per-model rows follow the same identity rule. The contract caps models
|
|
144
|
-
// at 32 AND requires sum(models.total_tokens) === session total, so a
|
|
145
|
-
// session with more than 32 models cannot carry a valid array at all —
|
|
146
|
-
// omit it (the contract allows missing models; the session prices as 0).
|
|
147
|
-
let models = [];
|
|
148
|
-
const usageRows = Array.isArray(row.model_usage) ? row.model_usage : [];
|
|
149
|
-
let modelsValid = usageRows.length > 0 && usageRows.length <= 32;
|
|
150
|
-
let modelsSum = 0;
|
|
151
|
-
if (modelsValid) {
|
|
152
|
-
for (const usage of usageRows) {
|
|
153
|
-
const modelTokens = {};
|
|
154
|
-
for (const field of TOKEN_FIELDS) {
|
|
155
|
-
const raw = usage?.[field];
|
|
156
|
-
const value = safeInt(raw === undefined || raw === null ? 0 : raw);
|
|
157
|
-
if (value === null) {
|
|
158
|
-
modelsValid = false;
|
|
159
|
-
break;
|
|
160
|
-
}
|
|
161
|
-
modelTokens[field] = value;
|
|
162
|
-
}
|
|
163
|
-
if (!modelsValid) break;
|
|
164
|
-
foldReasoningIntoOutput(modelTokens);
|
|
165
|
-
const modelIdentity =
|
|
166
|
-
modelTokens.input_tokens +
|
|
167
|
-
modelTokens.cached_input_tokens +
|
|
168
|
-
modelTokens.cache_creation_input_tokens +
|
|
169
|
-
modelTokens.output_tokens;
|
|
170
|
-
if (modelTokens.total_tokens !== modelIdentity) {
|
|
171
|
-
modelsValid = false;
|
|
172
|
-
break;
|
|
173
|
-
}
|
|
174
|
-
modelsSum += modelTokens.total_tokens;
|
|
175
|
-
models.push({ model: String(usage.model || "").trim() || "unknown", ...modelTokens });
|
|
176
|
-
}
|
|
177
|
-
if (modelsValid && modelsSum !== tokens.total_tokens) modelsValid = false;
|
|
178
|
-
}
|
|
179
|
-
if (!modelsValid) models = [];
|
|
180
|
-
|
|
181
|
-
return {
|
|
182
|
-
session_id: sessionId,
|
|
183
|
-
source,
|
|
184
|
-
started_at: new Date(startedMs).toISOString(),
|
|
185
|
-
ended_at: new Date(endedMs).toISOString(),
|
|
186
|
-
project_key: basenameKey(row.project_key ?? row.project_ref),
|
|
187
|
-
...tokens,
|
|
188
|
-
billable_total_tokens: null,
|
|
189
|
-
...(models.length > 0 ? { models } : {}),
|
|
190
|
-
};
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
/** Map + filter a sidecar rows array; invalid rows are counted, not fatal. */
|
|
194
|
-
function buildSessionUploadRows(rows) {
|
|
195
|
-
const out = [];
|
|
196
|
-
let skipped = 0;
|
|
197
|
-
for (const row of Array.isArray(rows) ? rows : []) {
|
|
198
|
-
const mapped = mapSessionRow(row);
|
|
199
|
-
if (mapped) out.push(mapped);
|
|
200
|
-
else skipped += 1;
|
|
201
|
-
}
|
|
202
|
-
return { rows: out, skipped };
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
function watermarkKey(source, sessionId) {
|
|
206
|
-
return `${source}|${sessionId}`;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
function loadWatermarks(filePath) {
|
|
210
|
-
try {
|
|
211
|
-
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
212
|
-
const map = new Map();
|
|
213
|
-
for (const [key, value] of Object.entries(
|
|
214
|
-
parsed && typeof parsed === "object" ? parsed : {},
|
|
215
|
-
)) {
|
|
216
|
-
if (typeof key === "string" && typeof value === "string" && key.includes("|")) {
|
|
217
|
-
map.set(key, value);
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
return map;
|
|
221
|
-
} catch {
|
|
222
|
-
return new Map();
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
function pruneWatermarks(map) {
|
|
227
|
-
if (map.size <= WATERMARK_MAX_ENTRIES) return map;
|
|
228
|
-
const entries = [...map.entries()].sort((a, b) =>
|
|
229
|
-
String(a[1]).localeCompare(String(b[1])),
|
|
230
|
-
);
|
|
231
|
-
return new Map(entries.slice(entries.length - WATERMARK_MAX_ENTRIES));
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
async function saveWatermarks(filePath, map) {
|
|
235
|
-
const pruned = pruneWatermarks(map);
|
|
236
|
-
const json = JSON.stringify(
|
|
237
|
-
Object.fromEntries([...pruned.entries()].sort((a, b) => a[0].localeCompare(b[0]))),
|
|
238
|
-
null,
|
|
239
|
-
2,
|
|
240
|
-
) + "\n";
|
|
241
|
-
await fsp.mkdir(path.dirname(filePath), { recursive: true });
|
|
242
|
-
const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
|
|
243
|
-
await fsp.writeFile(tmp, json, { mode: 0o600 });
|
|
244
|
-
await fsp.rename(tmp, filePath);
|
|
245
|
-
await fsp.chmod(filePath, 0o600).catch(() => {});
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
/** Rows strictly newer than their watermark, ascending by ended_at. */
|
|
249
|
-
function selectSessionsForUpload(rows, watermarks) {
|
|
250
|
-
const pending = rows.filter((row) => {
|
|
251
|
-
const mark = watermarks.get(watermarkKey(row.source, row.session_id));
|
|
252
|
-
return !mark || String(row.ended_at) > mark;
|
|
253
|
-
});
|
|
254
|
-
pending.sort((a, b) => String(a.ended_at).localeCompare(String(b.ended_at)));
|
|
255
|
-
return pending;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/**
|
|
259
|
-
* Upload pending session rows in ascending ended_at batches of <=200. The
|
|
260
|
-
* watermark file only advances for batches the server accepted, so a failed
|
|
261
|
-
* batch leaves earlier successes committed and the failure retried next sync.
|
|
262
|
-
*/
|
|
263
|
-
/**
|
|
264
|
-
* Split pending rows into request batches. Two independent contract limits
|
|
265
|
-
* apply (§1.2/§2.4): <=200 rows AND <=1 MiB body — a 200-row batch of real
|
|
266
|
-
* sessions with models easily exceeds the byte cap, so batches are cut on
|
|
267
|
-
* whichever limit hits first (bytes measured on the serialized row, with a
|
|
268
|
-
* conservative envelope leaving room for upload_id + JSON framing).
|
|
269
|
-
*/
|
|
270
|
-
function buildBatches(pending, maxRows) {
|
|
271
|
-
const BYTE_BUDGET = 500 * 1024;
|
|
272
|
-
const batches = [];
|
|
273
|
-
let current = [];
|
|
274
|
-
let bytes = 0;
|
|
275
|
-
|
|
276
|
-
for (const row of pending) {
|
|
277
|
-
const rowBytes = Buffer.byteLength(JSON.stringify(row), "utf8") + 2;
|
|
278
|
-
if (current.length > 0 && (current.length >= maxRows || bytes + rowBytes > BYTE_BUDGET)) {
|
|
279
|
-
batches.push(current);
|
|
280
|
-
current = [];
|
|
281
|
-
bytes = 0;
|
|
282
|
-
}
|
|
283
|
-
current.push(row);
|
|
284
|
-
bytes += rowBytes;
|
|
285
|
-
}
|
|
286
|
-
if (current.length > 0) batches.push(current);
|
|
287
|
-
|
|
288
|
-
return batches;
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
async function uploadSessions({
|
|
292
|
-
baseUrl,
|
|
293
|
-
anonKey,
|
|
294
|
-
deviceToken,
|
|
295
|
-
sessions,
|
|
296
|
-
watermarkPath,
|
|
297
|
-
batchSize = MAX_SESSIONS_PER_BATCH,
|
|
298
|
-
canUpload = () => true,
|
|
299
|
-
fetchImpl = fetch,
|
|
300
|
-
onProgress = () => {},
|
|
301
|
-
}) {
|
|
302
|
-
// Map first: callers hand us RAW session-analytics rows (tokens nested in
|
|
303
|
-
// row.tokens, sidecar-only fields present); the wire shape is built here so
|
|
304
|
-
// no call site can bypass the whitelist/identity checks.
|
|
305
|
-
const { rows: mapped, skipped: skippedLocal } = buildSessionUploadRows(sessions);
|
|
306
|
-
const watermarks = loadWatermarks(watermarkPath);
|
|
307
|
-
const pending = selectSessionsForUpload(mapped, watermarks);
|
|
308
|
-
const maxRows = Math.min(Math.max(1, batchSize), MAX_SESSIONS_PER_BATCH);
|
|
309
|
-
const uploadId = crypto.randomUUID();
|
|
310
|
-
let upserted = 0;
|
|
311
|
-
let skippedServer = 0;
|
|
312
|
-
let batches = 0;
|
|
313
|
-
let uploaded = 0;
|
|
314
|
-
|
|
315
|
-
for (const batch of buildBatches(pending, maxRows)) {
|
|
316
|
-
if (!canUpload()) break;
|
|
317
|
-
const res = await fetchFunctionResponse(
|
|
318
|
-
functionUrlFor(baseUrl, INGEST_SLUG),
|
|
319
|
-
{
|
|
320
|
-
method: "POST",
|
|
321
|
-
headers: {
|
|
322
|
-
"Content-Type": "application/json",
|
|
323
|
-
Accept: "application/json",
|
|
324
|
-
Authorization: `Bearer ${deviceToken}`,
|
|
325
|
-
...(anonKey ? { apikey: anonKey } : {}),
|
|
326
|
-
},
|
|
327
|
-
body: JSON.stringify({
|
|
328
|
-
upload_id: uploadId,
|
|
329
|
-
sessions: batch,
|
|
330
|
-
}),
|
|
331
|
-
},
|
|
332
|
-
fetchImpl,
|
|
333
|
-
);
|
|
334
|
-
if (!res.ok) {
|
|
335
|
-
const text = await res.text().catch(() => "");
|
|
336
|
-
const error = new Error(
|
|
337
|
-
`sessions upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`,
|
|
338
|
-
);
|
|
339
|
-
error.status = res.status;
|
|
340
|
-
error.uploaded = uploaded;
|
|
341
|
-
error.upserted = upserted;
|
|
342
|
-
throw error;
|
|
343
|
-
}
|
|
344
|
-
const data = await res.json().catch(() => ({}));
|
|
345
|
-
batches += 1;
|
|
346
|
-
uploaded += batch.length;
|
|
347
|
-
upserted += Number(data?.sessions_upserted || 0);
|
|
348
|
-
skippedServer += Number(data?.sessions_skipped || 0);
|
|
349
|
-
for (const row of batch) {
|
|
350
|
-
watermarks.set(watermarkKey(row.source, row.session_id), row.ended_at);
|
|
351
|
-
}
|
|
352
|
-
await saveWatermarks(watermarkPath, watermarks);
|
|
353
|
-
onProgress({ batches, uploaded, upserted });
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
return {
|
|
357
|
-
pending: pending.length,
|
|
358
|
-
uploaded,
|
|
359
|
-
upserted,
|
|
360
|
-
skippedServer,
|
|
361
|
-
skippedLocal,
|
|
362
|
-
batches,
|
|
363
|
-
};
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
module.exports = {
|
|
367
|
-
ACCOUNT_LEVEL_SOURCES,
|
|
368
|
-
MAX_SESSIONS_PER_BATCH,
|
|
369
|
-
basenameKey,
|
|
370
|
-
buildBatches,
|
|
371
|
-
buildSessionUploadRows,
|
|
372
|
-
loadWatermarks,
|
|
373
|
-
mapSessionRow,
|
|
374
|
-
saveWatermarks,
|
|
375
|
-
selectSessionsForUpload,
|
|
376
|
-
uploadSessions,
|
|
377
|
-
validSessionId,
|
|
378
|
-
watermarkKey,
|
|
379
|
-
};
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Session metadata upload for the token-usage ingest (docs/cli-contract.md
|
|
4
|
+
// §2.4 `sessions`). Maps the local session-analytics sidecar rows onto the
|
|
5
|
+
// contract's wire shape, filters through per-(source, session_id) ended_at
|
|
6
|
+
// watermarks so repeat syncs are increments, and POSTs batches of <=200 to
|
|
7
|
+
// the same ingest endpoint the hourly buckets use.
|
|
8
|
+
//
|
|
9
|
+
// Privacy boundary (docs/cli-contract.md §2.4): only the contract's whitelisted
|
|
10
|
+
// fields are sent. The sidecar's title, project_ref (local cwd), resume
|
|
11
|
+
// identifiers and cache keys never leave the process — buildSessionUploadRows
|
|
12
|
+
// constructs each row field-by-field rather than spreading.
|
|
13
|
+
|
|
14
|
+
const fs = require("node:fs");
|
|
15
|
+
const fsp = require("node:fs/promises");
|
|
16
|
+
const path = require("node:path");
|
|
17
|
+
const crypto = require("node:crypto");
|
|
18
|
+
|
|
19
|
+
const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
|
|
20
|
+
|
|
21
|
+
const INGEST_SLUG = "token-usage-ingest";
|
|
22
|
+
const MAX_SESSIONS_PER_BATCH = 200;
|
|
23
|
+
// Contract: source is trimmed + lowercased; these two are account-level and
|
|
24
|
+
// the server skips them wholesale — the CLI just never uploads them.
|
|
25
|
+
const ACCOUNT_LEVEL_SOURCES = new Set(["cursor", "trae-cn"]);
|
|
26
|
+
// Keep the watermark map bounded: sessions older than the newest N never
|
|
27
|
+
// re-upload anyway (their ended_at watermark only moves forward).
|
|
28
|
+
const WATERMARK_MAX_ENTRIES = 5000;
|
|
29
|
+
|
|
30
|
+
// Six token counters ride along everywhere; total must equal the sum of the
|
|
31
|
+
// four that count toward it (reasoning is a sub-view of output, single-column).
|
|
32
|
+
const TOKEN_FIELDS = [
|
|
33
|
+
"input_tokens",
|
|
34
|
+
"cached_input_tokens",
|
|
35
|
+
"cache_creation_input_tokens",
|
|
36
|
+
"output_tokens",
|
|
37
|
+
"reasoning_output_tokens",
|
|
38
|
+
"total_tokens",
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
function safeInt(value) {
|
|
42
|
+
if (value === null || value === undefined) return null;
|
|
43
|
+
const n = Number(value);
|
|
44
|
+
if (!Number.isSafeInteger(n) || n < 0) return null;
|
|
45
|
+
return n;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// zcode/opencode report reasoning outside output_tokens (vendor semantics:
|
|
49
|
+
// total = identity + reasoning), but the contract's identity requires
|
|
50
|
+
// reasoning to sit inside output (docs/cli-contract.md §2.4: total = input +
|
|
51
|
+
// cached + creation + output, reasoning single-column). Fold that exact shape
|
|
52
|
+
// so the row satisfies the identity; any other mismatch still fails it and
|
|
53
|
+
// the row is dropped — guessing a fix for unknown skews would silently
|
|
54
|
+
// rewrite totals the vendor did report.
|
|
55
|
+
function foldReasoningIntoOutput(tokens) {
|
|
56
|
+
const identity =
|
|
57
|
+
tokens.input_tokens +
|
|
58
|
+
tokens.cached_input_tokens +
|
|
59
|
+
tokens.cache_creation_input_tokens +
|
|
60
|
+
tokens.output_tokens;
|
|
61
|
+
if (
|
|
62
|
+
tokens.total_tokens !== identity &&
|
|
63
|
+
tokens.total_tokens === identity + tokens.reasoning_output_tokens
|
|
64
|
+
) {
|
|
65
|
+
tokens.output_tokens += tokens.reasoning_output_tokens;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function basenameKey(value) {
|
|
70
|
+
const raw = String(value || "").replace(/[\\/]+$/, "");
|
|
71
|
+
if (!raw || raw === "." || raw === "..") return null;
|
|
72
|
+
const base = raw.split(/[\\/]/).pop().trim();
|
|
73
|
+
if (!base || base === "." || base === "..") return null;
|
|
74
|
+
// Control characters are rejected server-side; strip instead of failing.
|
|
75
|
+
// eslint-disable-next-line no-control-regex
|
|
76
|
+
const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
|
|
77
|
+
return clean || null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function validSessionId(value) {
|
|
81
|
+
if (typeof value !== "string") return false;
|
|
82
|
+
const id = value.trim();
|
|
83
|
+
if (!id || id.length > 256) return false;
|
|
84
|
+
if (id.includes("/") || id.includes("\\")) return false;
|
|
85
|
+
// Full Unicode Cc range — the server validator is /\p{Cc}/u, which also
|
|
86
|
+
// covers C1 (U+0080–U+009F); a narrower client check would let such an id
|
|
87
|
+
// through to a whole-batch 400 that stalls the sessions upload.
|
|
88
|
+
// eslint-disable-next-line no-control-regex
|
|
89
|
+
if (/[\u0000-\u001f\u007f-\u009f]/.test(id)) return false;
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Map one session-analytics sidecar row onto the contract wire shape.
|
|
95
|
+
* Returns null when the row must not be uploaded (account-level source,
|
|
96
|
+
* missing identity/timestamps, or a token identity that the server would
|
|
97
|
+
* reject with a whole-batch 400 — dropping one bad row beats failing 200).
|
|
98
|
+
*
|
|
99
|
+
* Field locations on the sidecar row: the six token counters live nested in
|
|
100
|
+
* row.tokens (the top level only mirrors total_tokens), and the stable id is
|
|
101
|
+
* row.session_id when the vendor log carried one (Claude/Codex), else the
|
|
102
|
+
* locally-computed sha256 fingerprint row.session_hash (source\0id, 24 hex)
|
|
103
|
+
* — both are opaque stable keys without path leakage, valid as the contract
|
|
104
|
+
* session_id.
|
|
105
|
+
*/
|
|
106
|
+
function mapSessionRow(row) {
|
|
107
|
+
if (!row || typeof row !== "object") return null;
|
|
108
|
+
const source = String(row.source || "").trim().toLowerCase();
|
|
109
|
+
if (!source || ACCOUNT_LEVEL_SOURCES.has(source)) return null;
|
|
110
|
+
const sessionId = typeof row.session_id === "string" && row.session_id.trim()
|
|
111
|
+
? row.session_id.trim()
|
|
112
|
+
: typeof row.session_hash === "string" && row.session_hash.trim()
|
|
113
|
+
? row.session_hash.trim()
|
|
114
|
+
: null;
|
|
115
|
+
if (!validSessionId(sessionId)) return null;
|
|
116
|
+
|
|
117
|
+
const startedMs = Date.parse(row.started_at || "");
|
|
118
|
+
const endedMs = Date.parse(row.ended_at || "");
|
|
119
|
+
if (!Number.isFinite(startedMs) || !Number.isFinite(endedMs)) return null;
|
|
120
|
+
if (endedMs < startedMs) return null;
|
|
121
|
+
|
|
122
|
+
// Sidecar rows omit zero-valued counters on some sources (zcode drops
|
|
123
|
+
// cache_creation/reasoning keys entirely). The contract requires every
|
|
124
|
+
// field present, so missing keys fill as 0 — the identity check below is
|
|
125
|
+
// the real gate; dropping a whole models array over an elided zero would
|
|
126
|
+
// let a later last-wins upload erase previously stored models.
|
|
127
|
+
const nested = row.tokens && typeof row.tokens === "object" ? row.tokens : {};
|
|
128
|
+
const tokens = {};
|
|
129
|
+
for (const field of TOKEN_FIELDS) {
|
|
130
|
+
const raw = field === "total_tokens" ? row.total_tokens : nested[field];
|
|
131
|
+
const value = safeInt(raw === undefined || raw === null ? 0 : raw);
|
|
132
|
+
if (value === null) return null;
|
|
133
|
+
tokens[field] = value;
|
|
134
|
+
}
|
|
135
|
+
foldReasoningIntoOutput(tokens);
|
|
136
|
+
const identity =
|
|
137
|
+
tokens.input_tokens +
|
|
138
|
+
tokens.cached_input_tokens +
|
|
139
|
+
tokens.cache_creation_input_tokens +
|
|
140
|
+
tokens.output_tokens;
|
|
141
|
+
if (tokens.total_tokens !== identity) return null;
|
|
142
|
+
|
|
143
|
+
// Per-model rows follow the same identity rule. The contract caps models
|
|
144
|
+
// at 32 AND requires sum(models.total_tokens) === session total, so a
|
|
145
|
+
// session with more than 32 models cannot carry a valid array at all —
|
|
146
|
+
// omit it (the contract allows missing models; the session prices as 0).
|
|
147
|
+
let models = [];
|
|
148
|
+
const usageRows = Array.isArray(row.model_usage) ? row.model_usage : [];
|
|
149
|
+
let modelsValid = usageRows.length > 0 && usageRows.length <= 32;
|
|
150
|
+
let modelsSum = 0;
|
|
151
|
+
if (modelsValid) {
|
|
152
|
+
for (const usage of usageRows) {
|
|
153
|
+
const modelTokens = {};
|
|
154
|
+
for (const field of TOKEN_FIELDS) {
|
|
155
|
+
const raw = usage?.[field];
|
|
156
|
+
const value = safeInt(raw === undefined || raw === null ? 0 : raw);
|
|
157
|
+
if (value === null) {
|
|
158
|
+
modelsValid = false;
|
|
159
|
+
break;
|
|
160
|
+
}
|
|
161
|
+
modelTokens[field] = value;
|
|
162
|
+
}
|
|
163
|
+
if (!modelsValid) break;
|
|
164
|
+
foldReasoningIntoOutput(modelTokens);
|
|
165
|
+
const modelIdentity =
|
|
166
|
+
modelTokens.input_tokens +
|
|
167
|
+
modelTokens.cached_input_tokens +
|
|
168
|
+
modelTokens.cache_creation_input_tokens +
|
|
169
|
+
modelTokens.output_tokens;
|
|
170
|
+
if (modelTokens.total_tokens !== modelIdentity) {
|
|
171
|
+
modelsValid = false;
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
modelsSum += modelTokens.total_tokens;
|
|
175
|
+
models.push({ model: String(usage.model || "").trim() || "unknown", ...modelTokens });
|
|
176
|
+
}
|
|
177
|
+
if (modelsValid && modelsSum !== tokens.total_tokens) modelsValid = false;
|
|
178
|
+
}
|
|
179
|
+
if (!modelsValid) models = [];
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
session_id: sessionId,
|
|
183
|
+
source,
|
|
184
|
+
started_at: new Date(startedMs).toISOString(),
|
|
185
|
+
ended_at: new Date(endedMs).toISOString(),
|
|
186
|
+
project_key: basenameKey(row.project_key ?? row.project_ref),
|
|
187
|
+
...tokens,
|
|
188
|
+
billable_total_tokens: null,
|
|
189
|
+
...(models.length > 0 ? { models } : {}),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Map + filter a sidecar rows array; invalid rows are counted, not fatal. */
|
|
194
|
+
function buildSessionUploadRows(rows) {
|
|
195
|
+
const out = [];
|
|
196
|
+
let skipped = 0;
|
|
197
|
+
for (const row of Array.isArray(rows) ? rows : []) {
|
|
198
|
+
const mapped = mapSessionRow(row);
|
|
199
|
+
if (mapped) out.push(mapped);
|
|
200
|
+
else skipped += 1;
|
|
201
|
+
}
|
|
202
|
+
return { rows: out, skipped };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function watermarkKey(source, sessionId) {
|
|
206
|
+
return `${source}|${sessionId}`;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function loadWatermarks(filePath) {
|
|
210
|
+
try {
|
|
211
|
+
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
212
|
+
const map = new Map();
|
|
213
|
+
for (const [key, value] of Object.entries(
|
|
214
|
+
parsed && typeof parsed === "object" ? parsed : {},
|
|
215
|
+
)) {
|
|
216
|
+
if (typeof key === "string" && typeof value === "string" && key.includes("|")) {
|
|
217
|
+
map.set(key, value);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return map;
|
|
221
|
+
} catch {
|
|
222
|
+
return new Map();
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function pruneWatermarks(map) {
|
|
227
|
+
if (map.size <= WATERMARK_MAX_ENTRIES) return map;
|
|
228
|
+
const entries = [...map.entries()].sort((a, b) =>
|
|
229
|
+
String(a[1]).localeCompare(String(b[1])),
|
|
230
|
+
);
|
|
231
|
+
return new Map(entries.slice(entries.length - WATERMARK_MAX_ENTRIES));
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function saveWatermarks(filePath, map) {
|
|
235
|
+
const pruned = pruneWatermarks(map);
|
|
236
|
+
const json = JSON.stringify(
|
|
237
|
+
Object.fromEntries([...pruned.entries()].sort((a, b) => a[0].localeCompare(b[0]))),
|
|
238
|
+
null,
|
|
239
|
+
2,
|
|
240
|
+
) + "\n";
|
|
241
|
+
await fsp.mkdir(path.dirname(filePath), { recursive: true });
|
|
242
|
+
const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
|
|
243
|
+
await fsp.writeFile(tmp, json, { mode: 0o600 });
|
|
244
|
+
await fsp.rename(tmp, filePath);
|
|
245
|
+
await fsp.chmod(filePath, 0o600).catch(() => {});
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Rows strictly newer than their watermark, ascending by ended_at. */
|
|
249
|
+
function selectSessionsForUpload(rows, watermarks) {
|
|
250
|
+
const pending = rows.filter((row) => {
|
|
251
|
+
const mark = watermarks.get(watermarkKey(row.source, row.session_id));
|
|
252
|
+
return !mark || String(row.ended_at) > mark;
|
|
253
|
+
});
|
|
254
|
+
pending.sort((a, b) => String(a.ended_at).localeCompare(String(b.ended_at)));
|
|
255
|
+
return pending;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Upload pending session rows in ascending ended_at batches of <=200. The
|
|
260
|
+
* watermark file only advances for batches the server accepted, so a failed
|
|
261
|
+
* batch leaves earlier successes committed and the failure retried next sync.
|
|
262
|
+
*/
|
|
263
|
+
/**
|
|
264
|
+
* Split pending rows into request batches. Two independent contract limits
|
|
265
|
+
* apply (§1.2/§2.4): <=200 rows AND <=1 MiB body — a 200-row batch of real
|
|
266
|
+
* sessions with models easily exceeds the byte cap, so batches are cut on
|
|
267
|
+
* whichever limit hits first (bytes measured on the serialized row, with a
|
|
268
|
+
* conservative envelope leaving room for upload_id + JSON framing).
|
|
269
|
+
*/
|
|
270
|
+
function buildBatches(pending, maxRows) {
|
|
271
|
+
const BYTE_BUDGET = 500 * 1024;
|
|
272
|
+
const batches = [];
|
|
273
|
+
let current = [];
|
|
274
|
+
let bytes = 0;
|
|
275
|
+
|
|
276
|
+
for (const row of pending) {
|
|
277
|
+
const rowBytes = Buffer.byteLength(JSON.stringify(row), "utf8") + 2;
|
|
278
|
+
if (current.length > 0 && (current.length >= maxRows || bytes + rowBytes > BYTE_BUDGET)) {
|
|
279
|
+
batches.push(current);
|
|
280
|
+
current = [];
|
|
281
|
+
bytes = 0;
|
|
282
|
+
}
|
|
283
|
+
current.push(row);
|
|
284
|
+
bytes += rowBytes;
|
|
285
|
+
}
|
|
286
|
+
if (current.length > 0) batches.push(current);
|
|
287
|
+
|
|
288
|
+
return batches;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async function uploadSessions({
|
|
292
|
+
baseUrl,
|
|
293
|
+
anonKey,
|
|
294
|
+
deviceToken,
|
|
295
|
+
sessions,
|
|
296
|
+
watermarkPath,
|
|
297
|
+
batchSize = MAX_SESSIONS_PER_BATCH,
|
|
298
|
+
canUpload = () => true,
|
|
299
|
+
fetchImpl = fetch,
|
|
300
|
+
onProgress = () => {},
|
|
301
|
+
}) {
|
|
302
|
+
// Map first: callers hand us RAW session-analytics rows (tokens nested in
|
|
303
|
+
// row.tokens, sidecar-only fields present); the wire shape is built here so
|
|
304
|
+
// no call site can bypass the whitelist/identity checks.
|
|
305
|
+
const { rows: mapped, skipped: skippedLocal } = buildSessionUploadRows(sessions);
|
|
306
|
+
const watermarks = loadWatermarks(watermarkPath);
|
|
307
|
+
const pending = selectSessionsForUpload(mapped, watermarks);
|
|
308
|
+
const maxRows = Math.min(Math.max(1, batchSize), MAX_SESSIONS_PER_BATCH);
|
|
309
|
+
const uploadId = crypto.randomUUID();
|
|
310
|
+
let upserted = 0;
|
|
311
|
+
let skippedServer = 0;
|
|
312
|
+
let batches = 0;
|
|
313
|
+
let uploaded = 0;
|
|
314
|
+
|
|
315
|
+
for (const batch of buildBatches(pending, maxRows)) {
|
|
316
|
+
if (!canUpload()) break;
|
|
317
|
+
const res = await fetchFunctionResponse(
|
|
318
|
+
functionUrlFor(baseUrl, INGEST_SLUG),
|
|
319
|
+
{
|
|
320
|
+
method: "POST",
|
|
321
|
+
headers: {
|
|
322
|
+
"Content-Type": "application/json",
|
|
323
|
+
Accept: "application/json",
|
|
324
|
+
Authorization: `Bearer ${deviceToken}`,
|
|
325
|
+
...(anonKey ? { apikey: anonKey } : {}),
|
|
326
|
+
},
|
|
327
|
+
body: JSON.stringify({
|
|
328
|
+
upload_id: uploadId,
|
|
329
|
+
sessions: batch,
|
|
330
|
+
}),
|
|
331
|
+
},
|
|
332
|
+
fetchImpl,
|
|
333
|
+
);
|
|
334
|
+
if (!res.ok) {
|
|
335
|
+
const text = await res.text().catch(() => "");
|
|
336
|
+
const error = new Error(
|
|
337
|
+
`sessions upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`,
|
|
338
|
+
);
|
|
339
|
+
error.status = res.status;
|
|
340
|
+
error.uploaded = uploaded;
|
|
341
|
+
error.upserted = upserted;
|
|
342
|
+
throw error;
|
|
343
|
+
}
|
|
344
|
+
const data = await res.json().catch(() => ({}));
|
|
345
|
+
batches += 1;
|
|
346
|
+
uploaded += batch.length;
|
|
347
|
+
upserted += Number(data?.sessions_upserted || 0);
|
|
348
|
+
skippedServer += Number(data?.sessions_skipped || 0);
|
|
349
|
+
for (const row of batch) {
|
|
350
|
+
watermarks.set(watermarkKey(row.source, row.session_id), row.ended_at);
|
|
351
|
+
}
|
|
352
|
+
await saveWatermarks(watermarkPath, watermarks);
|
|
353
|
+
onProgress({ batches, uploaded, upserted });
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
return {
|
|
357
|
+
pending: pending.length,
|
|
358
|
+
uploaded,
|
|
359
|
+
upserted,
|
|
360
|
+
skippedServer,
|
|
361
|
+
skippedLocal,
|
|
362
|
+
batches,
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
module.exports = {
|
|
367
|
+
ACCOUNT_LEVEL_SOURCES,
|
|
368
|
+
MAX_SESSIONS_PER_BATCH,
|
|
369
|
+
basenameKey,
|
|
370
|
+
buildBatches,
|
|
371
|
+
buildSessionUploadRows,
|
|
372
|
+
loadWatermarks,
|
|
373
|
+
mapSessionRow,
|
|
374
|
+
saveWatermarks,
|
|
375
|
+
selectSessionsForUpload,
|
|
376
|
+
uploadSessions,
|
|
377
|
+
validSessionId,
|
|
378
|
+
watermarkKey,
|
|
379
|
+
};
|