tokenusage-cli 0.4.4 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -3,6 +3,27 @@
3
3
  本包(`tokenusage-cli`)的用户可见变更记录。发版时在顶部新增一节,
4
4
  `## <版本号>` 行会被 release workflow 取作 GitHub Release 正文。
5
5
 
6
+ ## 0.4.5
7
+
8
+ 2026-11-04。一轮审计修复:`sync` 读队列时把契约 v2 的项目拆分行压回余量行、随后照旧推进
9
+ 上传偏移,被归因的用量会永久丢失;上传确认、Windows 隐藏启动器编码与首同步统计也各有一处
10
+ 缺陷。无契约变更,服务端无需同步升级。
11
+
12
+ - **队列批次不再丢项目行**:`readQueueBatch` 的去重键原来只有 `source|model|hour`,
13
+ 契约 v2 的拆分行(同键不同 `project_key`)会被合并成余量行并随偏移推进永久丢失;
14
+ 现键含 `project_key`,每行独立上传。
15
+ - **拆分行按服务端 basename 身份合并**:写出/合并项目拆分行时统一走共享的
16
+ `lib/project-key.js`(`basenameKey()`),与服务端请求去重口径一致;同名不同 owner 的
17
+ 仓库不再互相覆盖。`reconcile` 对旧的 `owner/repo` 行做同样的归一,避免把正常数据报成漂移。
18
+ - **上传确认后才推进进度**:`ingest` 需响应含 `inserted` + `skipped`、session 上传需含
19
+ `sessions_upserted` + `sessions_skipped`,才推进队列偏移 / watermark;未确认的 2xx
20
+ 响应按失败处理,下次 `sync` 重传而不是静默跳数据。
21
+ - **Windows 隐藏启动器改 UTF-16LE + BOM**:`run-sync-hidden.vbs` 按 wscript 期望的编码
22
+ 写入,含非 ASCII 字符的安装路径不再乱码或被截断。
23
+ - **首同步统计修正**:`init` 的首次同步统计跳过 `kind` 控制行,并把 `project_key` 纳入
24
+ last-wins 键,项目用量不再被余量行盖掉。
25
+ - 回归测试:拆分行往返、basename 归一、VBS 编码、未确认 2xx 不推进偏移/watermark。
26
+
6
27
  ## 0.4.4
7
28
 
8
29
  2026-11-04。契约 v2 落地:小时用量桶按项目名拆行(`project_key`),项目键向上回退到真实
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tokenusage-cli",
3
- "version": "0.4.4",
3
+ "version": "0.4.5",
4
4
  "description": "Self-hosted AI coding token usage collector for the Token Usage dashboard.",
5
5
  "keywords": [
6
6
  "token-usage",
@@ -2807,7 +2807,14 @@ function readFirstSyncTotals(trackerDir) {
2807
2807
  if (!trimmed) continue;
2808
2808
  try {
2809
2809
  const row = JSON.parse(trimmed);
2810
- const key = `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}`;
2810
+ // Control records (account session states / watermarks) carry a
2811
+ // `total_tokens` too; they are not usage buckets and must not inflate
2812
+ // the first-sync counter.
2813
+ if (!row || typeof row !== "object" || Object.hasOwn(row, "kind")) continue;
2814
+ // Contract v2: project_key joins the last-wins identity. Without it the
2815
+ // residual row (appended last) hides every attributed project row, and a
2816
+ // fully attributed hour would display as 0 tokens.
2817
+ const key = `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}|${row.project_key || ""}`;
2811
2818
  latest.set(key, row);
2812
2819
  } catch {
2813
2820
  // skip malformed
@@ -40,6 +40,7 @@ const { functionUrlFor, fetchFunctionResponse } = require("../lib/function-url")
40
40
  const { iterateQueueLines } = require("../lib/upload-queue");
41
41
  const { normalizeQueueRow } = require("../lib/queue-row-normalize");
42
42
  const { sanitizeBucketForUpload } = require("../lib/ingest-sanitize");
43
+ const { basenameKey } = require("../lib/project-key");
43
44
  const { TOKEN_COLUMNS } = require("../lib/account-usage-dedup");
44
45
  const { UsageError, unknownOptionError } = require("../lib/cli-errors");
45
46
 
@@ -208,7 +209,13 @@ async function collectBuckets({ queuePath, fromMs, toMs, start = 0 }) {
208
209
  });
209
210
  const sanitized = sanitizeBucketForUpload(row);
210
211
  if (!sanitized) continue;
211
- const project = typeof sanitized.project_key === "string" ? sanitized.project_key.trim() : "";
212
+ // The server stores only the basename (§2.4). Normalizing here keeps
213
+ // pre-v2 queues whose rows still carry `owner/repo` comparable with the
214
+ // cloud instead of reporting permanent missing_on_server drift.
215
+ const project =
216
+ typeof sanitized.project_key === "string"
217
+ ? basenameKey(sanitized.project_key) || ""
218
+ : "";
212
219
  buckets.set(bucketKey(hourMs, source, model, project), sanitized);
213
220
  }
214
221
  } finally {
@@ -4721,6 +4721,14 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
4721
4721
  throw err;
4722
4722
  }
4723
4723
 
4724
+ if (!isConfirmedIngestResponse(data)) {
4725
+ const err = new Error(
4726
+ `Unconfirmed ingest response (HTTP ${res.status}): ${rawText.substring(0, 300)}`,
4727
+ );
4728
+ err.status = res.status;
4729
+ err.code = "CLOUD_UPLOAD_UNCONFIRMED";
4730
+ throw err;
4731
+ }
4724
4732
  cloudAccess = data;
4725
4733
  if (data.upload_scope === "community") batchBudget = Math.min(batchBudget, 10);
4726
4734
  inserted += Number(data?.inserted || 0);
@@ -4755,6 +4763,23 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
4755
4763
  return resultSoFar();
4756
4764
  }
4757
4765
 
4766
+ // A 200 is not proof the ingest committed: captive portals / reverse proxies
4767
+ // return 200 HTML and maintenance pages return an empty body. Advancing the
4768
+ // queue offset on those silently drops the batch forever, so the drain only
4769
+ // accepts the contract's success shape (docs/cli-contract.md §2.4).
4770
+ function isCount(value) {
4771
+ return (
4772
+ (typeof value === "number" || typeof value === "string") &&
4773
+ value !== "" &&
4774
+ Number.isFinite(Number(value))
4775
+ );
4776
+ }
4777
+
4778
+ function isConfirmedIngestResponse(data) {
4779
+ return Boolean(data) && typeof data === "object" && !Array.isArray(data) &&
4780
+ isCount(data.inserted) && isCount(data.skipped);
4781
+ }
4782
+
4758
4783
  async function readQueueBatch(queuePath, startOffset, maxBuckets) {
4759
4784
  const st = await fs.stat(queuePath).catch(() => null);
4760
4785
  if (!st || !st.isFile()) return { buckets: [], sessionStates: [], nextOffset: startOffset };
@@ -4862,7 +4887,17 @@ async function readQueueBatch(queuePath, startOffset, maxBuckets) {
4862
4887
  // otherwise 400 the whole ingest batch and stall the queue.
4863
4888
  const sanitized = sanitizeBucketForUpload(bucket);
4864
4889
  if (!sanitized) continue;
4865
- bucketMap.set(`${source}|${model}|${hourStart}`, sanitized);
4890
+ // Contract v2: project_key is part of the bucket identity. Split hours
4891
+ // carry several rows with the same (source, model, hour) under different
4892
+ // project keys (plus the '' residual); omitting it here collapsed them
4893
+ // to the last row and then advanced the offset past the rest, losing the
4894
+ // attributed usage permanently. New queue rows already carry the
4895
+ // server-normalized basename (rollout's appendProjectSplitRows); old
4896
+ // owner/repo rows stay distinct here and let the server's own basename
4897
+ // dedupe pick the larger total instead of collapsing them locally.
4898
+ const projectKey =
4899
+ typeof sanitized.project_key === "string" ? sanitized.project_key.trim() : "";
4900
+ bucketMap.set(`${source}|${model}|${hourStart}|${projectKey}`, sanitized);
4866
4901
  linesRead += 1;
4867
4902
  if (linesRead >= maxBuckets) break;
4868
4903
  }
@@ -0,0 +1,23 @@
1
+ "use strict";
2
+
3
+ // Contract v2 (docs/cli-contract.md §2.4): the server keeps only the project
4
+ // basename for both hourly buckets and sessions. The CLI must apply the same
5
+ // rule before it writes a queue row or keys a reconciliation baseline —
6
+ // otherwise `owner/repo` and `repo` are two different local identities while
7
+ // the cloud stores one, which under-counts same-basename repos and makes
8
+ // reconcile report permanent false drift.
9
+ //
10
+ // This is the single implementation shared by the hourly writer, the session
11
+ // uploader and reconcile.
12
+ function basenameKey(value) {
13
+ const raw = String(value || "").replace(/[\\/]+$/, "");
14
+ if (!raw || raw === "." || raw === "..") return null;
15
+ const base = raw.split(/[\\/]/).pop().trim();
16
+ if (!base || base === "." || base === "..") return null;
17
+ // Control characters are rejected server-side; strip instead of failing.
18
+ // eslint-disable-next-line no-control-regex
19
+ const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
20
+ return clean || null;
21
+ }
22
+
23
+ module.exports = { basenameKey };
@@ -13,6 +13,7 @@ const { readSqliteJsonRows, readSqliteJsonRowsAsync } = require("./sqlite-reader
13
13
  const wsl = require("./wsl-probe");
14
14
  const { resolveInstallPaths } = require("./install-resolver");
15
15
  const { expandHome } = require("./scan-roots");
16
+ const { basenameKey } = require("./project-key");
16
17
  const {
17
18
  consumeUsageDelta,
18
19
  createUsageDeltaState,
@@ -3575,27 +3576,48 @@ function appendProjectSplitRows({ group, model, aggregateBucket, projects, toApp
3575
3576
  if (!projects || projects.size === 0) return;
3576
3577
  const attributed = initTotals();
3577
3578
  const projectKeys = [...projects.keys()].sort();
3579
+ // The server stores only the basename (docs/cli-contract.md §2.4), so two
3580
+ // local projects such as `alice/api` and `bob/api` share one cloud identity.
3581
+ // Merge their totals under that normalized key here: emitting two rows with
3582
+ // the same wire key would make the server's request dedupe keep just one of
3583
+ // them and silently drop the other project's usage.
3584
+ const normalizedProjects = new Map();
3578
3585
  for (const projectKey of projectKeys) {
3579
3586
  const projectBucket = projects.get(projectKey);
3580
3587
  if (!projectBucket.totals) continue;
3581
3588
  addTotals(attributed, projectBucket.totals);
3582
- const totals = projectBucket.totals;
3583
- const usagePrecision = projectBucket.usage_precision || aggregateBucket.usage_precision || null;
3589
+ const normalizedKey = basenameKey(projectKey);
3590
+ if (!normalizedKey) continue;
3591
+ let entry = normalizedProjects.get(normalizedKey);
3592
+ if (!entry) {
3593
+ entry = { totals: initTotals(), usagePrecision: null, buckets: [] };
3594
+ normalizedProjects.set(normalizedKey, entry);
3595
+ }
3596
+ addTotals(entry.totals, projectBucket.totals);
3597
+ entry.usagePrecision = entry.usagePrecision || projectBucket.usage_precision || null;
3598
+ entry.buckets.push(projectBucket);
3599
+ }
3600
+
3601
+ for (const [normalizedKey, entry] of normalizedProjects) {
3602
+ const totals = entry.totals;
3603
+ const usagePrecision = entry.usagePrecision || aggregateBucket.usage_precision || null;
3584
3604
  const marker = usagePrecision ? `${totalsKey(totals)}|${usagePrecision}` : totalsKey(totals);
3585
- if (projectBucket.queuedHourlyKey === marker) continue;
3605
+ // Every contributing bucket must agree that this normalized row was
3606
+ // already queued; one changed contributor re-emits the merged total.
3607
+ if (entry.buckets.every((bucket) => bucket.queuedHourlyKey === marker)) continue;
3586
3608
  toAppend.push(
3587
3609
  JSON.stringify({
3588
3610
  source: group.source,
3589
3611
  model,
3590
3612
  hour_start: group.hourStart,
3591
- project_key: projectKey,
3613
+ project_key: normalizedKey,
3592
3614
  ...totalsRowFields(totals),
3593
3615
  usage_precision: usagePrecision || undefined,
3594
3616
  ...prioritySubsetOf(totals),
3595
3617
  ...creditsSubsetOf(totals),
3596
3618
  }),
3597
3619
  );
3598
- projectBucket.queuedHourlyKey = marker;
3620
+ for (const bucket of entry.buckets) bucket.queuedHourlyKey = marker;
3599
3621
  }
3600
3622
 
3601
3623
  const residual = cloneTotals(aggregateBucket.totals);
@@ -158,7 +158,13 @@ function writeHiddenRunner({
158
158
  }) {
159
159
  fs.mkdirSync(binDir, { recursive: true });
160
160
  const vbsPath = vbsPathFor(binDir);
161
- fs.writeFileSync(vbsPath, buildHiddenRunnerVbs({ nodeExe, cliJs, fallbackCliJs }), { mode: 0o600 });
161
+ // wscript.exe reads a BOM-less .vbs with the system ANSI code page (GBK on
162
+ // Chinese Windows), so a UTF-8 path containing a non-ASCII username turns
163
+ // into mojibake and the hourly run cannot find node. UTF-16LE + BOM is what
164
+ // Windows Script Host expects; the XML definition already follows the same
165
+ // rule.
166
+ const body = buildHiddenRunnerVbs({ nodeExe, cliJs, fallbackCliJs });
167
+ fs.writeFileSync(vbsPath, Buffer.from(`\uFEFF${body}`, "utf16le"), { mode: 0o600 });
162
168
  return vbsPath;
163
169
  }
164
170
 
@@ -17,6 +17,7 @@ const path = require("node:path");
17
17
  const crypto = require("node:crypto");
18
18
 
19
19
  const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
20
+ const { basenameKey } = require("./project-key");
20
21
 
21
22
  const INGEST_SLUG = "token-usage-ingest";
22
23
  const MAX_SESSIONS_PER_BATCH = 200;
@@ -66,17 +67,6 @@ function foldReasoningIntoOutput(tokens) {
66
67
  }
67
68
  }
68
69
 
69
- function basenameKey(value) {
70
- const raw = String(value || "").replace(/[\\/]+$/, "");
71
- if (!raw || raw === "." || raw === "..") return null;
72
- const base = raw.split(/[\\/]/).pop().trim();
73
- if (!base || base === "." || base === "..") return null;
74
- // Control characters are rejected server-side; strip instead of failing.
75
- // eslint-disable-next-line no-control-regex
76
- const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
77
- return clean || null;
78
- }
79
-
80
70
  function validSessionId(value) {
81
71
  if (typeof value !== "string") return false;
82
72
  const id = value.trim();
@@ -347,7 +337,30 @@ async function uploadSessions({
347
337
  error.upserted = upserted;
348
338
  throw error;
349
339
  }
350
- const data = await res.json().catch(() => ({}));
340
+ // A 2xx is not a confirmation: captive portals and reverse proxies answer
341
+ // with 200 HTML/empty bodies. Advancing the watermark on those would skip
342
+ // the batch forever, so require the contract's counter fields
343
+ // (docs/cli-contract.md §2.4) before recording progress.
344
+ const data = await res.json().catch(() => null);
345
+ const isCount = (value) =>
346
+ (typeof value === "number" || typeof value === "string") &&
347
+ value !== "" &&
348
+ Number.isFinite(Number(value));
349
+
350
+ if (
351
+ !data ||
352
+ typeof data !== "object" ||
353
+ !isCount(data.sessions_upserted) ||
354
+ !isCount(data.sessions_skipped)
355
+ ) {
356
+ const error = new Error(
357
+ `sessions upload was not confirmed (HTTP ${res.status}): unexpected response body`,
358
+ );
359
+ error.status = res.status;
360
+ error.uploaded = uploaded;
361
+ error.upserted = upserted;
362
+ throw error;
363
+ }
351
364
  batches += 1;
352
365
  uploaded += batch.length;
353
366
  upserted += Number(data?.sessions_upserted || 0);