tokenusage-cli 0.5.1 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -6,6 +6,27 @@
6
6
  > 0.4.x 之前的条目为压缩摘要;细节与当时的验收记录见 `docs/cli-release.md`
7
7
  > 与 `docs/archive/`。
8
8
 
9
+ ## 0.5.2
10
+
11
+ 2026-11-08。上传队列防线:彻底消除「毒批/超大批次导致队头永久卡死」类故障。零契约变更,
12
+ 服务端无需同步升级;对账基线(`queue.jsonl` last-wins)与队列文件格式不变。
13
+
14
+ - **上传行白名单重组**:`sanitizeBucketForUpload` 从「整行展开」改为按契约字段重建——
15
+ 生产者私有附加字段(`queuedKey` 指纹、解析器暂存字段等)不再上车,单一未知大字段无法
16
+ 再把请求体推向 1 MiB 上限;计数器钳制到 `Number.MAX_SAFE_INTEGER`,超界值不再换来
17
+ 确定性 400。
18
+ - **桶批字节预算**:`readQueueBatch` 在 512 KiB 原始字节处切批(会话通道 500 KiB 预算的
19
+ 同款先例),行数上限不再是唯一闸门。
20
+ - **400/413 连败逃生阀**:同一批次偏移连续 3 次收到 400/413(计数持久化在
21
+ `queue.state.json`,跨进程累计)后,该批隔离到 `queue.quarantine.jsonl` 并推进偏移——
22
+ 从「无声卡死」降级为「有声丢弃」,`tokenusage reconcile` 会把这些键持续报为 drift 供
23
+ 人工判断;401/403/5xx 仍走常规退避重试,任意成功即清零计数。
24
+ - **models 省略可见**:sync 摘要新增「models omitted on N」计数(>32 模型或单模型恒等式
25
+ 失败而丢弃模型明细的会话数),这类会话此前静默按 0 计成本。
26
+
27
+ 升级说明:无需任何动作,随下次计划任务自动生效;若历史上已有卡死批次,升级后第一轮
28
+ `sync` 即触发逃生阀完成隔离(检查 `queue.quarantine.jsonl` 并按需人工处理)。
29
+
9
30
  ## 0.5.1
10
31
 
11
32
  2026-11-06。修复 Windows 计划任务在非提升权限安装下的静默降级 + `doctor` 新增计划任务自检。
package/README.md CHANGED
@@ -38,7 +38,7 @@ tokenusage sync --drain # 首次/积压补传(更大的批处理预算)
38
38
  | `tokenusage reconcile` | 只读对账:本地 `queue.jsonl` 的 last-wins 基线 vs 云端该设备的桶明细,按 `(hour_start, source, model)` 逐键比对,有漂移退出码 1 |
39
39
  | `tokenusage pause` / `resume` | 暂停/恢复自动采集:定时任务、hook、`--watch` 全部静默停手,手动 `sync` 不受影响 |
40
40
  | `tokenusage status` | 本地状态:Base URL、令牌、设备身份、采集状态、队列积压、最近解析/上传时间 |
41
- | `tokenusage doctor` | 体检(运行时 / 网络 / 各工具配置),critical 时退出码非 0 |
41
+ | `tokenusage doctor` | 体检(运行时 / 网络 / 各工具配置 / 计划任务是否降级),critical 时退出码非 0 |
42
42
  | `tokenusage diagnostics` | 诊断信息导出(家目录路径脱敏为 `~`) |
43
43
  | `tokenusage sessions` | 本地导出会话分析(JSON/CSV,元数据级) |
44
44
  | `tokenusage uninstall [--purge]` | 移除 hook、计划任务与运行时副本;`--purge` 连数据目录一起删(保留机器身份) |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tokenusage-cli",
3
- "version": "0.5.1",
3
+ "version": "0.5.2",
4
4
  "description": "Self-hosted AI coding token usage collector for the Token Usage dashboard.",
5
5
  "keywords": [
6
6
  "token-usage",
@@ -611,8 +611,11 @@ function buildSyncSummaryLines({
611
611
  lines.push(`- Credits: ${formatNumber(uploadedCredits)}`);
612
612
  }
613
613
  if (sessionUploadSummary) {
614
+ const omittedSuffix = sessionUploadSummary.modelsOmitted > 0
615
+ ? `, models omitted on ${formatNumber(sessionUploadSummary.modelsOmitted)}`
616
+ : "";
614
617
  lines.push(
615
- `- Sessions: ${formatNumber(sessionUploadSummary.uploaded)}/${formatNumber(sessionUploadSummary.pending)} uploaded (upserted ${formatNumber(sessionUploadSummary.upserted)}, skipped ${formatNumber(sessionUploadSummary.skipped)})`,
618
+ `- Sessions: ${formatNumber(sessionUploadSummary.uploaded)}/${formatNumber(sessionUploadSummary.pending)} uploaded (upserted ${formatNumber(sessionUploadSummary.upserted)}, skipped ${formatNumber(sessionUploadSummary.skipped)}${omittedSuffix})`,
616
619
  );
617
620
  }
618
621
  if (hasDeviceToken && pendingBytes > 0 && !drain) {
@@ -3894,6 +3897,7 @@ async function cmdSync(argv, context = {}) {
3894
3897
  pending: result.pending,
3895
3898
  upserted: result.upserted,
3896
3899
  skipped: (result.skippedServer || 0) + (result.skippedLocal || 0),
3900
+ modelsOmitted: result.modelsOmitted || 0,
3897
3901
  };
3898
3902
  // Auto runs keep the rolling log line; a manual run reports the same
3899
3903
  // counters inside the final summary block (printing here would put
@@ -3903,7 +3907,11 @@ async function cmdSync(argv, context = {}) {
3903
3907
  result.upserted || result.skippedServer
3904
3908
  ? ` (upserted ${result.upserted}, skipped ${result.skippedServer})`
3905
3909
  : "";
3906
- autoLog(`Sessions: ${result.uploaded}/${result.pending} uploaded${sessionSuffix}`);
3910
+ const omittedSuffix =
3911
+ sessionUploadSummary.modelsOmitted > 0
3912
+ ? `, models omitted on ${sessionUploadSummary.modelsOmitted}`
3913
+ : "";
3914
+ autoLog(`Sessions: ${result.uploaded}/${result.pending} uploaded${sessionSuffix}${omittedSuffix}`);
3907
3915
  }
3908
3916
  }
3909
3917
  syncRun?.noteSessions({
@@ -4745,8 +4753,14 @@ const { buildSessionAnalytics } = require("../lib/session-analytics");
4745
4753
  const { uploadSessions } = require("../lib/session-upload");
4746
4754
  const { uploadProjectRepos } = require("../lib/project-repos");
4747
4755
  const MAX_INGEST_BUCKETS = 500;
4748
-
4749
- async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200, maxDurationMs = UPLOAD_BUDGET_MS, canUpload = () => true }) {
4756
+ // Sessions split batches on a 500 KiB byte budget (lib/session-upload.js);
4757
+ // the hourly drain used to rely on the row cap alone. Raw queue-line bytes
4758
+ // upper-bound the sanitized wire rows (the whitelist in ingest-sanitize only
4759
+ // shrinks them), so cutting on raw bytes keeps every request comfortably
4760
+ // under the contract's 1 MiB body cap (§1.2 → 413).
4761
+ const MAX_BATCH_BYTES = 512 * 1024;
4762
+
4763
+ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200, maxDurationMs = UPLOAD_BUDGET_MS, canUpload = () => true, fetchImpl = fetch }) {
4750
4764
  const state = (await readJson(queueStatePath)) || { offset: 0 };
4751
4765
  let offset = Number(state.offset || 0);
4752
4766
  let inserted = 0;
@@ -4755,6 +4769,7 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
4755
4769
  let tokensIngested = 0;
4756
4770
  let creditsIngested = 0;
4757
4771
  let cloudAccess = null;
4772
+ let rejectStrikes = readRejectStrikes(state);
4758
4773
  const uploadId = crypto.randomUUID();
4759
4774
  const deadline = Date.now() + maxDurationMs;
4760
4775
  let batchBudget = Math.min(maxBatches, 100);
@@ -4807,7 +4822,7 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
4807
4822
  ? { account_session_states: result.sessionStates }
4808
4823
  : {}),
4809
4824
  }),
4810
- });
4825
+ }, fetchImpl);
4811
4826
 
4812
4827
  const rawText = await res.text();
4813
4828
  let data = {};
@@ -4821,6 +4836,36 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
4821
4836
  const retryAfter = res.headers?.get?.("Retry-After") ?? null;
4822
4837
  const retryAfterMs = parseRetryAfterMs(retryAfter);
4823
4838
  if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
4839
+ // 400/413 are deterministic client-shape rejections: retrying the
4840
+ // identical batch can never succeed, and the offset only moves on
4841
+ // success. Count strikes per offset (persisted, so the count survives
4842
+ // process exits between scheduled syncs); after three, quarantine the
4843
+ // batch and advance past it — a loud, inspectable drop instead of a
4844
+ // silent stall. Quarantined rows stay in queue.jsonl, so
4845
+ // `tokenusage reconcile` keeps reporting them as drift for a human
4846
+ // to judge.
4847
+ if (err.status === 400 || err.status === 413) {
4848
+ const strikes = rejectStrikes.offset === offset ? rejectStrikes.count + 1 : 1;
4849
+ rejectStrikes = { count: strikes, offset };
4850
+ if (strikes >= REJECT_VALVE_LIMIT) {
4851
+ const quarantinePath = `${queuePath}.quarantine.jsonl`;
4852
+ const quarantined = await quarantineBatchAt({ queuePath, offset, limit, quarantinePath });
4853
+ offset = quarantined.nextOffset;
4854
+ state.offset = offset;
4855
+ delete state.reject;
4856
+ state.updatedAt = new Date().toISOString();
4857
+ await writeJson(queueStatePath, state);
4858
+ process.stderr.write(
4859
+ `Upload escape valve: HTTP ${err.status} rejected the same batch ${strikes} times; ` +
4860
+ `quarantined ${quarantined.rows} row(s) to ${quarantinePath} and advanced past them. ` +
4861
+ `Inspect the quarantine file — \`tokenusage reconcile\` will flag these keys as drift.\n`,
4862
+ );
4863
+ continue;
4864
+ }
4865
+ state.reject = { count: strikes, offset };
4866
+ state.updatedAt = new Date().toISOString();
4867
+ await writeJson(queueStatePath, state);
4868
+ }
4824
4869
  throw err;
4825
4870
  }
4826
4871
 
@@ -4850,6 +4895,7 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
4850
4895
 
4851
4896
  offset = result.nextOffset;
4852
4897
  state.offset = offset;
4898
+ delete state.reject;
4853
4899
  state.updatedAt = new Date().toISOString();
4854
4900
  await writeJson(queueStatePath, state);
4855
4901
  }
@@ -4883,6 +4929,42 @@ function isConfirmedIngestResponse(data) {
4883
4929
  isCount(data.inserted) && isCount(data.skipped);
4884
4930
  }
4885
4931
 
4932
+ // Consecutive client-shape rejections (400/413) tolerated for one batch
4933
+ // offset before the escape valve quarantines it. Three strikes mean the batch
4934
+ // failed across three separate sync passes — no transient server state
4935
+ // explains it.
4936
+ const REJECT_VALVE_LIMIT = 3;
4937
+
4938
+ function readRejectStrikes(state) {
4939
+ const raw = state && typeof state.reject === "object" && state.reject !== null
4940
+ ? state.reject
4941
+ : {};
4942
+ const count = Number(raw.count);
4943
+ const offset = Number(raw.offset);
4944
+
4945
+ return {
4946
+ count: Number.isFinite(count) && count > 0 ? Math.floor(count) : 0,
4947
+ offset: Number.isFinite(offset) ? offset : null,
4948
+ };
4949
+ }
4950
+
4951
+ // Re-read the batch at `offset` through the exact same pipeline the upload
4952
+ // used, append the rows verbatim to the quarantine file, and report where the
4953
+ // offset should jump to. The queue itself is never rewritten: quarantined rows
4954
+ // stay in queue.jsonl, so reconcile keeps flagging their keys as
4955
+ // missing_on_server drift for a human to judge.
4956
+ async function quarantineBatchAt({ queuePath, offset, limit, quarantinePath }) {
4957
+ const result = await readQueueBatch(queuePath, offset, limit);
4958
+ const rows = [...result.buckets, ...result.sessionStates];
4959
+
4960
+ if (rows.length > 0) {
4961
+ const payload = rows.map((row) => JSON.stringify(row)).join("\n") + "\n";
4962
+ await fs.appendFile(quarantinePath, payload, "utf8");
4963
+ }
4964
+
4965
+ return { rows: rows.length, nextOffset: result.nextOffset };
4966
+ }
4967
+
4886
4968
  async function readQueueBatch(queuePath, startOffset, maxBuckets) {
4887
4969
  const st = await fs.stat(queuePath).catch(() => null);
4888
4970
  if (!st || !st.isFile()) return { buckets: [], sessionStates: [], nextOffset: startOffset };
@@ -4915,11 +4997,17 @@ async function readQueueBatch(queuePath, startOffset, maxBuckets) {
4915
4997
  let offset = startOffset;
4916
4998
  let linesRead = 0;
4917
4999
  try {
5000
+ let batchBytes = 0;
4918
5001
  for await (const record of iterateQueueLines(handle, startOffset, st.size)) {
4919
5002
  // A producer may still be writing the last row; retry it once complete.
4920
5003
  if (!record.complete) break;
5004
+ // Byte budget: cut before the row that would overflow, never on the
5005
+ // batch's first row (a single oversized line cannot be split anyway,
5006
+ // and the whitelist keeps its wire shape small).
5007
+ if (linesRead > 0 && batchBytes + record.bytes.length > MAX_BATCH_BYTES) break;
4921
5008
  const line = record.bytes.toString("utf8");
4922
5009
  offset += record.bytes.length;
5010
+ batchBytes += record.bytes.length;
4923
5011
  if (!line.trim()) continue;
4924
5012
  let bucket;
4925
5013
  try {
@@ -9,12 +9,27 @@
9
9
  // Rows are therefore coerced into the contract shape here, and dropped
10
10
  // (return null) when they cannot be repaired — losing one row beats losing
11
11
  // the whole queue behind it.
12
+ //
13
+ // The coercion is a WHITELIST REBUILD, not a spread: queue rows accumulate
14
+ // producer-private extras over time (queuedKey fingerprints, parser scratch
15
+ // fields, repair bookkeeping). Spreading them onto the wire bloated request
16
+ // bodies toward the contract's 1 MiB cap (§1.2 → 413, which with the
17
+ // success-only offset used to mean an unretryable batch), and a producer that
18
+ // wrote a huge value into an unknown key could grow a row without bound.
19
+ // Only the contract's fields ride the wire — the server ignores unknown keys
20
+ // anyway, and no consumer of the sanitized row (sync upload, reconcile
21
+ // comparison, drain telemetry) reads anything else.
12
22
 
13
23
  const { TOKEN_COLUMNS } = require("./account-usage-dedup");
14
24
 
25
+ // Contract §2.4: counters must be non-negative SAFE integers, or the edge
26
+ // 400s the whole batch. Clamping to Number.MAX_SAFE_INTEGER (instead of only
27
+ // flooring) keeps a producer that wrote an unsafe value from stalling the
28
+ // queue on a deterministic 400.
15
29
  function nonNegativeInteger(value) {
16
30
  const n = Number(value);
17
- return Number.isFinite(n) ? Math.max(0, Math.floor(n)) : 0;
31
+ if (!Number.isFinite(n)) return 0;
32
+ return Math.min(Math.max(0, Math.floor(n)), Number.MAX_SAFE_INTEGER);
18
33
  }
19
34
 
20
35
  function nonNegativeNumber(value) {
@@ -25,7 +40,11 @@ function nonNegativeNumber(value) {
25
40
  function sanitizeBucketForUpload(bucket) {
26
41
  if (!bucket || typeof bucket !== "object" || Array.isArray(bucket)) return null;
27
42
  if (typeof bucket.hour_start !== "string" || !Number.isFinite(Date.parse(bucket.hour_start))) return null;
28
- const sanitized = { ...bucket };
43
+ const sanitized = {
44
+ hour_start: bucket.hour_start,
45
+ source: bucket.source,
46
+ model: bucket.model,
47
+ };
29
48
  for (const column of TOKEN_COLUMNS) {
30
49
  sanitized[column] = nonNegativeInteger(bucket[column]);
31
50
  }
@@ -41,8 +60,6 @@ function sanitizeBucketForUpload(bucket) {
41
60
  const credits = Number(bucket.credits);
42
61
  if (Number.isFinite(credits) && credits > 0) {
43
62
  sanitized.credits = Math.round(credits * 1e6) / 1e6;
44
- } else {
45
- delete sanitized.credits;
46
63
  }
47
64
  // project_key (contract v2, 2026-11-03): optional pass-through, string only
48
65
  // (anything else is a producer bug — drop the field, not the row); the
@@ -50,8 +67,6 @@ function sanitizeBucketForUpload(bucket) {
50
67
  // dropped so unattributed rows keep the old wire shape.
51
68
  if (typeof bucket.project_key === "string" && bucket.project_key.trim()) {
52
69
  sanitized.project_key = bucket.project_key;
53
- } else {
54
- delete sanitized.project_key;
55
70
  }
56
71
  return sanitized;
57
72
  }
@@ -238,6 +238,18 @@
238
238
  "cache_read": 0.06,
239
239
  "cache_write": 0
240
240
  },
241
+ "minimax-m3.1-flash": {
242
+ "input": 0.1494,
243
+ "output": 0.5976,
244
+ "cache_read": 0.0149,
245
+ "note": "MiniMax M3.1 Flash (released 2026-10-19). Official MiniMax rates as listed on models.dev (minimax-cn provider): $0.1494/$0.5976 per MTok input/output, $0.0149 cache read; no published cache-write rate (retrieved 2026-11-08). Pinned so the bare id reported by collectors does not fall through to LiteLLM prefix-strip, which would pick a third-party provider entry deterministically."
246
+ },
247
+ "ling-3.1-flash": {
248
+ "input": 0.3,
249
+ "output": 0.9,
250
+ "cache_read": 0.06,
251
+ "note": "InclusionAI Ant Ling 3.1 Flash (released 2026-09-29). Novita USD list via models.dev: $0.30/$0.90 per MTok input/output, $0.06 cache read; no published cache-write rate (retrieved 2026-11-08). Queue rows report the bare id; LiteLLM only carries ling-3.0 variants under provider-qualified keys."
252
+ },
241
253
  "deepseek-v4-flash": {
242
254
  "input": 0.44,
243
255
  "output": 1.32,
@@ -190,12 +190,22 @@ function mapSessionRow(row) {
190
190
  function buildSessionUploadRows(rows) {
191
191
  const out = [];
192
192
  let skipped = 0;
193
+ let modelsOmitted = 0;
193
194
  for (const row of Array.isArray(rows) ? rows : []) {
195
+ const carriedModels = Array.isArray(row?.model_usage) && row.model_usage.length > 0;
194
196
  const mapped = mapSessionRow(row);
195
- if (mapped) out.push(mapped);
196
- else skipped += 1;
197
+ if (mapped) {
198
+ // Sessions whose model detail was dropped (>32 models, or a per-model
199
+ // identity/sum failure — see mapSessionRow) still upload but price as
200
+ // 0 on the read side. Count them so the sync summary makes the
201
+ // trade-off visible instead of silently costing nothing.
202
+ if (carriedModels && !Array.isArray(mapped.models)) modelsOmitted += 1;
203
+ out.push(mapped);
204
+ } else {
205
+ skipped += 1;
206
+ }
197
207
  }
198
- return { rows: out, skipped };
208
+ return { rows: out, skipped, modelsOmitted };
199
209
  }
200
210
 
201
211
  function watermarkKey(source, sessionId) {
@@ -298,7 +308,7 @@ async function uploadSessions({
298
308
  // Map first: callers hand us RAW session-analytics rows (tokens nested in
299
309
  // row.tokens, sidecar-only fields present); the wire shape is built here so
300
310
  // no call site can bypass the whitelist/identity checks.
301
- const { rows: mapped, skipped: skippedLocal } = buildSessionUploadRows(sessions);
311
+ const { rows: mapped, skipped: skippedLocal, modelsOmitted } = buildSessionUploadRows(sessions);
302
312
  const watermarks = loadWatermarks(watermarkPath);
303
313
  const pending = selectSessionsForUpload(mapped, watermarks);
304
314
  const maxRows = Math.min(Math.max(1, batchSize), MAX_SESSIONS_PER_BATCH);
@@ -378,6 +388,7 @@ async function uploadSessions({
378
388
  upserted,
379
389
  skippedServer,
380
390
  skippedLocal,
391
+ modelsOmitted,
381
392
  batches,
382
393
  };
383
394
  }