tokenusage-cli 0.5.5 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +7 -6
- package/package.json +1 -1
- package/src/cli.js +32 -5
- package/src/commands/device-login.js +16 -17
- package/src/commands/diagnostics.js +16 -16
- package/src/commands/doctor.js +14 -11
- package/src/commands/pause.js +34 -3
- package/src/commands/project-profiles.js +5 -2
- package/src/commands/sessions.js +18 -30
- package/src/commands/status.js +22 -16
- package/src/commands/sync.js +117 -3614
- package/src/commands/uninstall.js +10 -9
- package/src/commands/upload.js +18 -12
- package/src/lib/ark-agent-plan-limits.js +4 -13
- package/src/lib/ark-coding-plan-limits.js +4 -13
- package/src/lib/cli-flags.js +109 -0
- package/src/lib/commandcode-limits.js +11 -38
- package/src/lib/doctor.js +91 -1
- package/src/lib/fs.js +10 -0
- package/src/lib/grok-limits.js +5 -18
- package/src/lib/limits-core.js +64 -0
- package/src/lib/qoder-limits.js +4 -13
- package/src/lib/queue-backup.js +24 -0
- package/src/lib/queue-drain.js +388 -0
- package/src/lib/runtime-config.js +26 -7
- package/src/lib/usage-limits.js +22 -29
- package/src/lib/workbuddy-ipc.js +191 -0
- package/src/lib/workbuddy-limits.js +656 -0
- package/src/lib/zcode-limits.js +5 -18
- package/src/repairs/claude.js +442 -0
- package/src/repairs/codebuddy.js +246 -0
- package/src/repairs/codex.js +1023 -0
- package/src/repairs/droid.js +266 -0
- package/src/repairs/grok.js +352 -0
- package/src/repairs/legacy-migrations.js +135 -0
- package/src/repairs/mimo.js +161 -0
- package/src/repairs/workbuddy.js +209 -0
- package/src/repairs/zcode.js +647 -0
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Queue drain + batch reading, split out of commands/sync.js: one upload pass
|
|
4
|
+
// over queue.jsonl (bounded by batches, bytes and wall clock), the reject-strike
|
|
5
|
+
// accounting that quarantines a batch the backend keeps refusing, and the
|
|
6
|
+
// shared batch reader.
|
|
7
|
+
|
|
8
|
+
const {
|
|
9
|
+
readJson,
|
|
10
|
+
safeStatSize,
|
|
11
|
+
writeJson,
|
|
12
|
+
} = require("./fs");
|
|
13
|
+
const {
|
|
14
|
+
fetchFunctionResponse,
|
|
15
|
+
functionUrlFor,
|
|
16
|
+
} = require("./function-url");
|
|
17
|
+
const {
|
|
18
|
+
sanitizeBucketForUpload,
|
|
19
|
+
} = require("./ingest-sanitize");
|
|
20
|
+
const {
|
|
21
|
+
normalizeQueueRow,
|
|
22
|
+
} = require("./queue-row-normalize");
|
|
23
|
+
const {
|
|
24
|
+
iterateQueueLines,
|
|
25
|
+
sessionSnapshotMs,
|
|
26
|
+
} = require("./upload-queue");
|
|
27
|
+
const {
|
|
28
|
+
parseRetryAfterMs,
|
|
29
|
+
} = require("./upload-throttle");
|
|
30
|
+
const fs = require("node:fs/promises");
|
|
31
|
+
const path = require("node:path");
|
|
32
|
+
|
|
33
|
+
const UPLOAD_BUDGET_MS = 90_000;
|
|
34
|
+
const INGEST_SLUG = "token-usage-ingest";
|
|
35
|
+
const MAX_INGEST_BUCKETS = 500;
|
|
36
|
+
|
|
37
|
+
async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200, maxDurationMs = UPLOAD_BUDGET_MS, canUpload = () => true, fetchImpl = fetch }) {
|
|
38
|
+
const state = (await readJson(queueStatePath)) || { offset: 0 };
|
|
39
|
+
let offset = Number(state.offset || 0);
|
|
40
|
+
let inserted = 0;
|
|
41
|
+
let skipped = 0;
|
|
42
|
+
let batches = 0;
|
|
43
|
+
let tokensIngested = 0;
|
|
44
|
+
let creditsIngested = 0;
|
|
45
|
+
let cloudAccess = null;
|
|
46
|
+
let rejectStrikes = readRejectStrikes(state);
|
|
47
|
+
const uploadId = crypto.randomUUID();
|
|
48
|
+
const deadline = Date.now() + maxDurationMs;
|
|
49
|
+
let batchBudget = Math.min(maxBatches, 100);
|
|
50
|
+
let requestSignal = null;
|
|
51
|
+
const resultSoFar = () => ({ inserted, skipped, batches, tokens: tokensIngested, credits: creditsIngested, ...(cloudAccess?.next_allowed_at ? { cloudAccess } : {}) });
|
|
52
|
+
|
|
53
|
+
const queueSize = await safeStatSize(queuePath);
|
|
54
|
+
const limit = Math.min(Math.max(1, Math.floor(Number(batchSize || 200))), MAX_INGEST_BUCKETS);
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
for (let batch = 0; batch < batchBudget; batch++) {
|
|
58
|
+
if (Date.now() >= deadline) break;
|
|
59
|
+
if (offset >= queueSize) break;
|
|
60
|
+
const result = await readQueueBatch(queuePath, offset, limit);
|
|
61
|
+
// A states-only tail (e.g. a retry batch whose bucket rows already
|
|
62
|
+
// uploaded) is still a meaningful upload: it advances the canonical
|
|
63
|
+
// session corrections cloud-side.
|
|
64
|
+
if (result.buckets.length === 0 && result.sessionStates.length === 0) {
|
|
65
|
+
// Complete ignored control/malformed rows need no cloud request, but
|
|
66
|
+
// must not leave a permanent pending tail. Incomplete bytes stay put.
|
|
67
|
+
if (result.nextOffset > offset) {
|
|
68
|
+
offset = result.nextOffset;
|
|
69
|
+
state.offset = offset;
|
|
70
|
+
state.updatedAt = new Date().toISOString();
|
|
71
|
+
await writeJson(queueStatePath, state);
|
|
72
|
+
}
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const root = baseUrl.replace(/\/$/, "");
|
|
77
|
+
const headers = {
|
|
78
|
+
"Content-Type": "application/json",
|
|
79
|
+
Accept: "application/json",
|
|
80
|
+
Authorization: `Bearer ${deviceToken}`,
|
|
81
|
+
};
|
|
82
|
+
if (anonKey) headers.apikey = anonKey;
|
|
83
|
+
// Re-read after parsing/each batch so switching off stops an active drain.
|
|
84
|
+
if (!canUpload()) break;
|
|
85
|
+
const remainingMs = deadline - Date.now();
|
|
86
|
+
if (remainingMs <= 0) break;
|
|
87
|
+
requestSignal = AbortSignal.timeout(Math.max(1, Math.ceil(remainingMs)));
|
|
88
|
+
const res = await fetchFunctionResponse(functionUrlFor(root, INGEST_SLUG), {
|
|
89
|
+
method: "POST",
|
|
90
|
+
headers,
|
|
91
|
+
signal: requestSignal,
|
|
92
|
+
body: JSON.stringify({
|
|
93
|
+
upload_id: uploadId,
|
|
94
|
+
hourly: result.buckets,
|
|
95
|
+
...(result.sessionStates.length > 0
|
|
96
|
+
? { account_session_states: result.sessionStates }
|
|
97
|
+
: {}),
|
|
98
|
+
}),
|
|
99
|
+
}, fetchImpl);
|
|
100
|
+
|
|
101
|
+
const rawText = await res.text();
|
|
102
|
+
let data = {};
|
|
103
|
+
try { data = JSON.parse(rawText); } catch { data = {}; }
|
|
104
|
+
if (!res.ok) {
|
|
105
|
+
const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
|
|
106
|
+
err.status = res.status;
|
|
107
|
+
err.code = res.status === 401 ? "CLOUD_DEVICE_TOKEN_REJECTED"
|
|
108
|
+
: typeof data?.code === "string" ? data.code
|
|
109
|
+
: res.status === 403 ? "CLOUD_UPLOAD_FORBIDDEN" : "CLOUD_UPLOAD_FAILED";
|
|
110
|
+
const retryAfter = res.headers?.get?.("Retry-After") ?? null;
|
|
111
|
+
const retryAfterMs = parseRetryAfterMs(retryAfter);
|
|
112
|
+
if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
|
|
113
|
+
// 400/413 are deterministic client-shape rejections: retrying the
|
|
114
|
+
// identical batch can never succeed, and the offset only moves on
|
|
115
|
+
// success. Count strikes per offset (persisted, so the count survives
|
|
116
|
+
// process exits between scheduled syncs); after three, quarantine the
|
|
117
|
+
// batch and advance past it — a loud, inspectable drop instead of a
|
|
118
|
+
// silent stall. Quarantined rows stay in queue.jsonl, so
|
|
119
|
+
// `tokenusage reconcile` keeps reporting them as drift for a human
|
|
120
|
+
// to judge.
|
|
121
|
+
if (err.status === 400 || err.status === 413) {
|
|
122
|
+
const strikes = rejectStrikes.offset === offset ? rejectStrikes.count + 1 : 1;
|
|
123
|
+
rejectStrikes = { count: strikes, offset };
|
|
124
|
+
if (strikes >= REJECT_VALVE_LIMIT) {
|
|
125
|
+
const quarantinePath = `${queuePath}.quarantine.jsonl`;
|
|
126
|
+
const quarantined = await quarantineBatchAt({ queuePath, offset, limit, quarantinePath });
|
|
127
|
+
offset = quarantined.nextOffset;
|
|
128
|
+
state.offset = offset;
|
|
129
|
+
delete state.reject;
|
|
130
|
+
state.updatedAt = new Date().toISOString();
|
|
131
|
+
await writeJson(queueStatePath, state);
|
|
132
|
+
process.stderr.write(
|
|
133
|
+
`Upload escape valve: HTTP ${err.status} rejected the same batch ${strikes} times; ` +
|
|
134
|
+
`quarantined ${quarantined.rows} row(s) to ${quarantinePath} and advanced past them. ` +
|
|
135
|
+
`Inspect the quarantine file — \`tokenusage reconcile\` will flag these keys as drift.\n`,
|
|
136
|
+
);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
state.reject = { count: strikes, offset };
|
|
140
|
+
state.updatedAt = new Date().toISOString();
|
|
141
|
+
await writeJson(queueStatePath, state);
|
|
142
|
+
}
|
|
143
|
+
throw err;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (!isConfirmedIngestResponse(data)) {
|
|
147
|
+
const err = new Error(
|
|
148
|
+
`Unconfirmed ingest response (HTTP ${res.status}): ${rawText.substring(0, 300)}`,
|
|
149
|
+
);
|
|
150
|
+
err.status = res.status;
|
|
151
|
+
err.code = "CLOUD_UPLOAD_UNCONFIRMED";
|
|
152
|
+
throw err;
|
|
153
|
+
}
|
|
154
|
+
cloudAccess = data;
|
|
155
|
+
if (data.upload_scope === "community") batchBudget = Math.min(batchBudget, 10);
|
|
156
|
+
inserted += Number(data?.inserted || 0);
|
|
157
|
+
skipped += Number(data?.skipped || 0);
|
|
158
|
+
// Client-side §2.11 telemetry: the total_tokens of the buckets this
|
|
159
|
+
// request carried (a larger replay may last-wins-override them later;
|
|
160
|
+
// this figure never participates in reconciliation). Credits sum the
|
|
161
|
+
// same way — the display-only dimension (docs/sources-capture-audit.md
|
|
162
|
+
// §2) — rounded to 6 decimals to keep float noise out of the ledger.
|
|
163
|
+
for (const bucketRow of result.buckets) {
|
|
164
|
+
tokensIngested += Number(bucketRow?.total_tokens) || 0;
|
|
165
|
+
creditsIngested =
|
|
166
|
+
Math.round((creditsIngested + (Number(bucketRow?.credits) || 0)) * 1e6) / 1e6;
|
|
167
|
+
}
|
|
168
|
+
batches += 1;
|
|
169
|
+
|
|
170
|
+
offset = result.nextOffset;
|
|
171
|
+
state.offset = offset;
|
|
172
|
+
delete state.reject;
|
|
173
|
+
state.updatedAt = new Date().toISOString();
|
|
174
|
+
await writeJson(queueStatePath, state);
|
|
175
|
+
}
|
|
176
|
+
} catch (error) {
|
|
177
|
+
// A bounded drain can finish partially without discarding successful
|
|
178
|
+
// offsets or the server cooldown. Other failures retain their retry path.
|
|
179
|
+
const requestTimedOut = requestSignal?.aborted && requestSignal.reason?.name === "TimeoutError" &&
|
|
180
|
+
(error === requestSignal.reason || error.name === "TimeoutError" || error.name === "AbortError");
|
|
181
|
+
if (!requestTimedOut || batches === 0) {
|
|
182
|
+
error.uploadResult = resultSoFar();
|
|
183
|
+
throw error;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return resultSoFar();
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// A 200 is not proof the ingest committed: captive portals / reverse proxies
|
|
190
|
+
// return 200 HTML and maintenance pages return an empty body. Advancing the
|
|
191
|
+
// queue offset on those silently drops the batch forever, so the drain only
|
|
192
|
+
// accepts the contract's success shape (docs/cli-contract.md §2.4).
|
|
193
|
+
function isCount(value) {
|
|
194
|
+
return (
|
|
195
|
+
(typeof value === "number" || typeof value === "string") &&
|
|
196
|
+
value !== "" &&
|
|
197
|
+
Number.isFinite(Number(value))
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function isConfirmedIngestResponse(data) {
|
|
202
|
+
return Boolean(data) && typeof data === "object" && !Array.isArray(data) &&
|
|
203
|
+
isCount(data.inserted) && isCount(data.skipped);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Consecutive client-shape rejections (400/413) tolerated for one batch
|
|
207
|
+
// offset before the escape valve quarantines it. Three strikes mean the batch
|
|
208
|
+
// failed across three separate sync passes — no transient server state
|
|
209
|
+
// explains it.
|
|
210
|
+
const REJECT_VALVE_LIMIT = 3;
|
|
211
|
+
|
|
212
|
+
function readRejectStrikes(state) {
|
|
213
|
+
const raw = state && typeof state.reject === "object" && state.reject !== null
|
|
214
|
+
? state.reject
|
|
215
|
+
: {};
|
|
216
|
+
const count = Number(raw.count);
|
|
217
|
+
const offset = Number(raw.offset);
|
|
218
|
+
|
|
219
|
+
return {
|
|
220
|
+
count: Number.isFinite(count) && count > 0 ? Math.floor(count) : 0,
|
|
221
|
+
offset: Number.isFinite(offset) ? offset : null,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// Re-read the batch at `offset` through the exact same pipeline the upload
|
|
226
|
+
// used, append the rows verbatim to the quarantine file, and report where the
|
|
227
|
+
// offset should jump to. The queue itself is never rewritten: quarantined rows
|
|
228
|
+
// stay in queue.jsonl, so reconcile keeps flagging their keys as
|
|
229
|
+
// missing_on_server drift for a human to judge.
|
|
230
|
+
async function quarantineBatchAt({ queuePath, offset, limit, quarantinePath }) {
|
|
231
|
+
const result = await readQueueBatch(queuePath, offset, limit);
|
|
232
|
+
const rows = [...result.buckets, ...result.sessionStates];
|
|
233
|
+
|
|
234
|
+
if (rows.length > 0) {
|
|
235
|
+
const payload = rows.map((row) => JSON.stringify(row)).join("\n") + "\n";
|
|
236
|
+
await fs.appendFile(quarantinePath, payload, "utf8");
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return { rows: rows.length, nextOffset: result.nextOffset };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async function readQueueBatch(queuePath, startOffset, maxBuckets) {
|
|
243
|
+
const st = await fs.stat(queuePath).catch(() => null);
|
|
244
|
+
if (!st || !st.isFile()) return { buckets: [], sessionStates: [], nextOffset: startOffset };
|
|
245
|
+
if (startOffset >= st.size) return { buckets: [], sessionStates: [], nextOffset: startOffset };
|
|
246
|
+
|
|
247
|
+
const handle = await fs.open(queuePath, "r");
|
|
248
|
+
|
|
249
|
+
const bucketMap = new Map();
|
|
250
|
+
// Account session states (kind: "account_session_state") are queue
|
|
251
|
+
// control records, not usage rows: collect them separately so they ride
|
|
252
|
+
// the same upload (after the bucket rows in file order) without entering
|
|
253
|
+
// the bucket stream. Keyed by (source, session_id), use the strictly newer
|
|
254
|
+
// snapshot_verified_at and retain the first row on ties, matching account
|
|
255
|
+
// truth even after a clock rollback. Only one observation per session may
|
|
256
|
+
// reach the ingest batch (duplicates would affect one ON CONFLICT row twice).
|
|
257
|
+
// snapshot_verified_at rides along verbatim from the ORIGINAL fetch, so a
|
|
258
|
+
// transport retry never fakes logical freshness.
|
|
259
|
+
//
|
|
260
|
+
// Batching: session-state records count toward the SAME per-batch record
|
|
261
|
+
// cap (maxBuckets) as bucket rows. The ingest edge rejects a request with
|
|
262
|
+
// more than 500 states (HTTP 400); before they counted, a states-heavy
|
|
263
|
+
// queue (a fresh device's first 30-day TRAE sync appends one record per
|
|
264
|
+
// observed session) was read in ONE uncapped batch - the 400 left the
|
|
265
|
+
// queue offset untouched and every retry re-read the identical oversized
|
|
266
|
+
// request, a permanent upload failure. With the default batchSize of 200
|
|
267
|
+
// (and the MAX_INGEST_BUCKETS=500 clamp above), every request stays under
|
|
268
|
+
// the edge cap while batching follows the existing design.
|
|
269
|
+
const sessionStateMap = new Map();
|
|
270
|
+
const sessionStateStamps = new Map();
|
|
271
|
+
let offset = startOffset;
|
|
272
|
+
let linesRead = 0;
|
|
273
|
+
try {
|
|
274
|
+
let batchBytes = 0;
|
|
275
|
+
for await (const record of iterateQueueLines(handle, startOffset, st.size)) {
|
|
276
|
+
// A producer may still be writing the last row; retry it once complete.
|
|
277
|
+
if (!record.complete) break;
|
|
278
|
+
// Byte budget: cut before the row that would overflow, never on the
|
|
279
|
+
// batch's first row (a single oversized line cannot be split anyway,
|
|
280
|
+
// and the whitelist keeps its wire shape small).
|
|
281
|
+
if (linesRead > 0 && batchBytes + record.bytes.length > MAX_BATCH_BYTES) break;
|
|
282
|
+
const line = record.bytes.toString("utf8");
|
|
283
|
+
offset += record.bytes.length;
|
|
284
|
+
batchBytes += record.bytes.length;
|
|
285
|
+
if (!line.trim()) continue;
|
|
286
|
+
let bucket;
|
|
287
|
+
try {
|
|
288
|
+
bucket = JSON.parse(line);
|
|
289
|
+
} catch (_e) {
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
if (bucket?.kind === "account_session_state") {
|
|
293
|
+
if (typeof bucket.source === "string" && typeof bucket.session_id === "string" && bucket.session_id.trim()) {
|
|
294
|
+
const verifiedMs = sessionSnapshotMs(bucket);
|
|
295
|
+
// The edge validates before writing and 400s the WHOLE batch on one
|
|
296
|
+
// invalid state ("Invalid account session state"); the offset only
|
|
297
|
+
// advances on success, so uploading a poison row would stall the
|
|
298
|
+
// queue head-of-line forever. Drop it here instead — nothing is
|
|
299
|
+
// recorded for the key, so a later, valid snapshot for the same
|
|
300
|
+
// session is still admitted.
|
|
301
|
+
if (verifiedMs === null) continue;
|
|
302
|
+
const ssSource = bucket.source.trim().toLowerCase();
|
|
303
|
+
const stateKey = ssSource + "|" + bucket.session_id.trim();
|
|
304
|
+
const sessionState = {
|
|
305
|
+
source: ssSource,
|
|
306
|
+
session_id: bucket.session_id.trim(),
|
|
307
|
+
model: typeof bucket.model === "string" ? bucket.model : "",
|
|
308
|
+
bucket_start: typeof bucket.bucket_start === "string" ? bucket.bucket_start : "",
|
|
309
|
+
input_tokens: Number(bucket.input_tokens) || 0,
|
|
310
|
+
output_tokens: Number(bucket.output_tokens) || 0,
|
|
311
|
+
cached_input_tokens: Number(bucket.cached_input_tokens) || 0,
|
|
312
|
+
cache_creation_input_tokens: Number(bucket.cache_creation_input_tokens) || 0,
|
|
313
|
+
reasoning_output_tokens: Number(bucket.reasoning_output_tokens) || 0,
|
|
314
|
+
total_tokens: Number(bucket.total_tokens) || 0,
|
|
315
|
+
...(typeof bucket.snapshot_verified_at === "string" && bucket.snapshot_verified_at
|
|
316
|
+
? { snapshot_verified_at: bucket.snapshot_verified_at }
|
|
317
|
+
: {}),
|
|
318
|
+
};
|
|
319
|
+
const priorState = sessionStateMap.get(stateKey);
|
|
320
|
+
const priorVerifiedMs = sessionStateStamps.get(stateKey) ?? null;
|
|
321
|
+
if (!priorState || priorVerifiedMs === null || verifiedMs > priorVerifiedMs) {
|
|
322
|
+
sessionStateMap.set(stateKey, sessionState);
|
|
323
|
+
sessionStateStamps.set(stateKey, verifiedMs);
|
|
324
|
+
}
|
|
325
|
+
// Counted like a bucket row: the record rides THIS batch and its
|
|
326
|
+
// bytes are covered by nextOffset, so the next batch resumes right
|
|
327
|
+
// after it.
|
|
328
|
+
linesRead += 1;
|
|
329
|
+
if (linesRead >= maxBuckets) break;
|
|
330
|
+
}
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
// Pre-release watermark records from this feature branch's dev queues
|
|
334
|
+
// are obsolete control records: drop them instead of letting them enter
|
|
335
|
+
// the bucket stream (they carry no hour_start and would be skipped
|
|
336
|
+
// downstream anyway, but be explicit).
|
|
337
|
+
if (bucket?.kind === "account_sync_watermark") continue;
|
|
338
|
+
const hourStart = typeof bucket?.hour_start === "string" ? bucket.hour_start : null;
|
|
339
|
+
if (!hourStart || !Number.isFinite(Date.parse(hourStart))) continue;
|
|
340
|
+
const source = (typeof bucket?.source === "string" ? bucket.source.trim().toLowerCase() : "") || "codex";
|
|
341
|
+
const model = (typeof bucket?.model === "string" ? bucket.model.trim() : "") || "unknown";
|
|
342
|
+
bucket.source = source;
|
|
343
|
+
bucket.model = model;
|
|
344
|
+
// Apply the same legacy-row corrections every local reader applies (the
|
|
345
|
+
// project queue and the wrapped aggregator use this module too). Without
|
|
346
|
+
// this the cloud permanently kept the RAW legacy values — e.g. old Codex
|
|
347
|
+
// rows whose input_tokens still include cached tokens (6-7x inflated) —
|
|
348
|
+
// while local readers showed corrected numbers.
|
|
349
|
+
bucket = normalizeQueueRow(bucket);
|
|
350
|
+
// Contract-shape coercion, poison-row drop (see lib/ingest-sanitize.js):
|
|
351
|
+
// a fractional Grok counter or a negative OpenCode repair delta would
|
|
352
|
+
// otherwise 400 the whole ingest batch and stall the queue.
|
|
353
|
+
const sanitized = sanitizeBucketForUpload(bucket);
|
|
354
|
+
if (!sanitized) continue;
|
|
355
|
+
// Contract v2: project_key is part of the bucket identity. Split hours
|
|
356
|
+
// carry several rows with the same (source, model, hour) under different
|
|
357
|
+
// project keys (plus the '' residual); omitting it here collapsed them
|
|
358
|
+
// to the last row and then advanced the offset past the rest, losing the
|
|
359
|
+
// attributed usage permanently. New queue rows already carry the
|
|
360
|
+
// server-normalized basename (rollout's appendProjectSplitRows); old
|
|
361
|
+
// owner/repo rows stay distinct here and let the server's own basename
|
|
362
|
+
// dedupe pick the larger total instead of collapsing them locally.
|
|
363
|
+
const projectKey =
|
|
364
|
+
typeof sanitized.project_key === "string" ? sanitized.project_key.trim() : "";
|
|
365
|
+
bucketMap.set(`${source}|${model}|${hourStart}|${projectKey}`, sanitized);
|
|
366
|
+
linesRead += 1;
|
|
367
|
+
if (linesRead >= maxBuckets) break;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
} finally {
|
|
371
|
+
await handle.close();
|
|
372
|
+
}
|
|
373
|
+
return {
|
|
374
|
+
buckets: Array.from(bucketMap.values()),
|
|
375
|
+
sessionStates: Array.from(sessionStateMap.values()),
|
|
376
|
+
nextOffset: offset,
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const MAX_BATCH_BYTES = 512 * 1024;
|
|
381
|
+
|
|
382
|
+
module.exports = {
|
|
383
|
+
INGEST_SLUG,
|
|
384
|
+
MAX_INGEST_BUCKETS,
|
|
385
|
+
UPLOAD_BUDGET_MS,
|
|
386
|
+
drainQueueToCloud,
|
|
387
|
+
readQueueBatch,
|
|
388
|
+
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
// token-usage production deployment (
|
|
2
|
-
//
|
|
1
|
+
// token-usage production deployment (moved to app.token-usage.cc with
|
|
2
|
+
// operator approval 2026-11-17). Override per invocation with --base-url or
|
|
3
3
|
// TOKENUSAGE_BASE_URL.
|
|
4
|
-
const DEFAULT_BASE_URL = "https://token-usage.
|
|
4
|
+
const DEFAULT_BASE_URL = "https://app.token-usage.cc";
|
|
5
5
|
// Retired hosts from the upstream (InsForge) era; a persisted config pointing
|
|
6
6
|
// at one of these must fall back to the current default instead of uploading
|
|
7
7
|
// into a dead backend.
|
|
@@ -9,9 +9,12 @@ const LEGACY_INSFORGE_HOSTS = new Set([
|
|
|
9
9
|
"b46ug8xu.us-east.insforge.app",
|
|
10
10
|
"srctyff5.us-east.insforge.app",
|
|
11
11
|
]);
|
|
12
|
-
//
|
|
13
|
-
// to the
|
|
14
|
-
const LEGACY_TOKEN_USAGE_HOSTS = new Set([
|
|
12
|
+
// Retired hosts a persisted config may still point at; such installs fall
|
|
13
|
+
// back to the current default instead of uploading into a dead backend.
|
|
14
|
+
const LEGACY_TOKEN_USAGE_HOSTS = new Set([
|
|
15
|
+
"token-usage-delta.vercel.app",
|
|
16
|
+
"token-usage.baiwumm.com", // retired 2026-11-17, moved to app.token-usage.cc
|
|
17
|
+
]);
|
|
15
18
|
const DEFAULT_DASHBOARD_URL = DEFAULT_BASE_URL;
|
|
16
19
|
const DEFAULT_HTTP_TIMEOUT_MS = 20_000;
|
|
17
20
|
// Public InsForge anon key (JWT, role=anon). Mirrors dashboard/src/lib/insforge-config.ts
|
|
@@ -57,7 +60,7 @@ function resolveRuntimeConfig({ cli = {}, config = {}, env = process.env, defaul
|
|
|
57
60
|
);
|
|
58
61
|
const dashboardUrl = pickString(
|
|
59
62
|
cli.dashboardUrl,
|
|
60
|
-
configBase === selectedBase ? config.dashboardUrl : undefined,
|
|
63
|
+
configBase === selectedBase ? normalizePersistedDashboardUrl(config.dashboardUrl) : undefined,
|
|
61
64
|
envBase === selectedBase ? env?.TOKENUSAGE_DASHBOARD_URL : undefined,
|
|
62
65
|
defaultBase === selectedBase ? defaults.dashboardUrl : undefined,
|
|
63
66
|
custom ? baseUrl.value : DEFAULT_DASHBOARD_URL,
|
|
@@ -211,6 +214,22 @@ function normalizePersistedBaseUrl(value) {
|
|
|
211
214
|
return normalized;
|
|
212
215
|
}
|
|
213
216
|
|
|
217
|
+
// A persisted dashboardUrl aimed at a retired host is treated as unset so it
|
|
218
|
+
// falls back to the current default, mirroring normalizePersistedBaseUrl.
|
|
219
|
+
function normalizePersistedDashboardUrl(value) {
|
|
220
|
+
const normalized = normalizeString(value);
|
|
221
|
+
if (normalized === undefined) return undefined;
|
|
222
|
+
try {
|
|
223
|
+
const hostname = new URL(normalized).hostname.toLowerCase();
|
|
224
|
+
if (LEGACY_INSFORGE_HOSTS.has(hostname) || LEGACY_TOKEN_USAGE_HOSTS.has(hostname)) {
|
|
225
|
+
return undefined;
|
|
226
|
+
}
|
|
227
|
+
} catch {
|
|
228
|
+
// Preserve the existing resolver behavior for arbitrary custom values.
|
|
229
|
+
}
|
|
230
|
+
return normalized;
|
|
231
|
+
}
|
|
232
|
+
|
|
214
233
|
// True when the value points at a retired InsForge project (dead backend).
|
|
215
234
|
// Callers use this to trigger the one-time config repair in sync.
|
|
216
235
|
function isLegacyInsforgeBaseUrl(value) {
|
package/src/lib/usage-limits.js
CHANGED
|
@@ -32,6 +32,7 @@ const { fetchZcodeLimits } = require("./zcode-limits");
|
|
|
32
32
|
const { fetchOpencodeGoLimits } = require("./opencode-go-limits");
|
|
33
33
|
const { fetchCommandcodeLimits } = require("./commandcode-limits");
|
|
34
34
|
const { fetchDevinLimits } = require("./devin-limits");
|
|
35
|
+
const { fetchWorkbuddyLimits, fetchWorkbuddyAiLimits } = require("./workbuddy-limits");
|
|
35
36
|
const { fetchDeepseekLimits } = require("./deepseek-limits");
|
|
36
37
|
const { fetchQoderLimits, fetchQoderCnLimits } = require("./qoder-limits");
|
|
37
38
|
const { fetchArkCodingPlanLimits } = require("./ark-coding-plan-limits");
|
|
@@ -45,6 +46,12 @@ const {
|
|
|
45
46
|
isBinaryAvailable,
|
|
46
47
|
} = require("./command-runner");
|
|
47
48
|
|
|
49
|
+
const {
|
|
50
|
+
clampPercent,
|
|
51
|
+
toFiniteNumberOrNull,
|
|
52
|
+
buildWindow,
|
|
53
|
+
} = require("./limits-core");
|
|
54
|
+
|
|
48
55
|
const execFileAsync = promisify(cp.execFile);
|
|
49
56
|
|
|
50
57
|
// 2-minute in-memory cache. It also expires early at the earliest upcoming window
|
|
@@ -169,34 +176,6 @@ const KIRO_CREDITS_SIDECAR_FILE = "kiro-credits.json";
|
|
|
169
176
|
const CLAUDE_RATE_LIMIT_DEFAULT_COOLDOWN_SEC = 5 * 60;
|
|
170
177
|
const CLAUDE_RATE_LIMIT_MAX_COOLDOWN_SEC = 60 * 60;
|
|
171
178
|
|
|
172
|
-
function clampPercent(value) {
|
|
173
|
-
if (value === null || value === undefined || value === "") return null;
|
|
174
|
-
const n = Number(value);
|
|
175
|
-
if (!Number.isFinite(n)) return null;
|
|
176
|
-
if (n <= 0) return 0;
|
|
177
|
-
if (n >= 100) return 100;
|
|
178
|
-
return n;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
function toFiniteNumberOrNull(value) {
|
|
182
|
-
if (value === null || value === undefined || value === "") return null;
|
|
183
|
-
const n = Number(value);
|
|
184
|
-
return Number.isFinite(n) ? n : null;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
function buildWindow({ usedPercent, resetAt, windowSeconds = null }) {
|
|
188
|
-
const pct = clampPercent(usedPercent);
|
|
189
|
-
if (pct === null) return null;
|
|
190
|
-
const window = {
|
|
191
|
-
used_percent: pct,
|
|
192
|
-
reset_at: typeof resetAt === "string" && resetAt ? resetAt : null,
|
|
193
|
-
};
|
|
194
|
-
if (Number.isFinite(windowSeconds) && windowSeconds > 0) {
|
|
195
|
-
window.limit_window_seconds = windowSeconds;
|
|
196
|
-
}
|
|
197
|
-
return window;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
179
|
function decodeJwtPayload(token) {
|
|
201
180
|
if (typeof token !== "string" || token.length === 0) return null;
|
|
202
181
|
const parts = token.split(".");
|
|
@@ -4449,7 +4428,7 @@ async function fetchUsageLimitsUncached({
|
|
|
4449
4428
|
: null;
|
|
4450
4429
|
|
|
4451
4430
|
const providerFetch = withFetchTimeout(fetchImpl, providerTimeoutMs);
|
|
4452
|
-
const [claudeResult, codexResult, cursor, kimi, gemini, kiro, antigravity, copilot, grok, zcode, opencodeGoRaw, qoder, qoderCn, codingPlan, agentPlan, commandCodeRaw, devinRaw, deepseek, claudeServiceStatus] = await Promise.all([
|
|
4431
|
+
const [claudeResult, codexResult, cursor, kimi, gemini, kiro, antigravity, copilot, grok, zcode, opencodeGoRaw, qoder, qoderCn, codingPlan, agentPlan, commandCodeRaw, devinRaw, workbuddy, workbuddyAi, deepseek, claudeServiceStatus] = await Promise.all([
|
|
4453
4432
|
claudeToken && !freshClaudeCache && !claudeRetryAtMs
|
|
4454
4433
|
? withProviderTimeout(fetchClaudeUsageLimits(claudeToken, { fetchImpl: providerFetch, maxAttempts: 1 }), "Claude", providerTimeoutMs).then(
|
|
4455
4434
|
(value) => ({ status: "fulfilled", value }),
|
|
@@ -4574,6 +4553,14 @@ async function fetchUsageLimitsUncached({
|
|
|
4574
4553
|
(value) => ({ status: "fulfilled", value }),
|
|
4575
4554
|
(reason) => ({ status: "rejected", reason }),
|
|
4576
4555
|
),
|
|
4556
|
+
// WorkBuddy CN and WorkBuddy AI are separate editions: each one only talks
|
|
4557
|
+
// to its own local authenticated billing proxy (wbipc) and never falls back
|
|
4558
|
+
// to the other edition's socket. Abort tears down the IPC socket when the
|
|
4559
|
+
// provider budget expires; reward credits are not a plan window.
|
|
4560
|
+
withAbortableProviderTimeout((signal) => fetchWorkbuddyLimits({ home, env, fetchImpl: providerFetch, signal }), "WorkBuddy", providerTimeoutMs)
|
|
4561
|
+
.catch((reason) => ({ configured: true, error: reason?.message || "Unknown error" })),
|
|
4562
|
+
withAbortableProviderTimeout((signal) => fetchWorkbuddyAiLimits({ home, env, fetchImpl: providerFetch, signal }), "WorkBuddy AI", providerTimeoutMs)
|
|
4563
|
+
.catch((reason) => ({ configured: true, error: reason?.message || "Unknown error" })),
|
|
4577
4564
|
// DeepSeek (platform.deepseek.com, pay-as-you-go): the balance endpoint is
|
|
4578
4565
|
// the only documented programmatic source, so this slot yields a balance
|
|
4579
4566
|
// card, not quota windows (#960). The last-good disk-cache fallback with a
|
|
@@ -4973,6 +4960,12 @@ async function fetchUsageLimitsUncached({
|
|
|
4973
4960
|
// Devin's planName ("Pro", "Max") is already the official display name —
|
|
4974
4961
|
// Title-Case normalization is still harmless and strips any brand prefix.
|
|
4975
4962
|
devin: withPlanLabel(devinObj, devinObj?.plan_label, "Devin"),
|
|
4963
|
+
// WorkBuddy's plan labels are the desktop edition's own English strings
|
|
4964
|
+
// (Ultimate / Advanced / STANDARD / Free). The shared normalization would
|
|
4965
|
+
// Title-Case them and drop "Free" outright, so pass them through verbatim
|
|
4966
|
+
// like grok / commandCode.
|
|
4967
|
+
workbuddy,
|
|
4968
|
+
workbuddyAi,
|
|
4976
4969
|
qoder: withPlanLabel(qoder, qoder?.plan_label, "Qoder"),
|
|
4977
4970
|
qoderCn: withPlanLabel(qoderCn, qoderCn?.plan_label, "Qoder CN"),
|
|
4978
4971
|
codingPlan: withPlanLabel(codingPlan, codingPlan?.plan_label, "Ark Coding Plan"),
|