tokenusage-cli 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/package.json +1 -1
- package/src/commands/init.js +8 -1
- package/src/commands/reconcile.js +8 -1
- package/src/commands/sync.js +36 -1
- package/src/lib/project-key.js +23 -0
- package/src/lib/rollout.js +27 -5
- package/src/lib/scheduled-task.js +7 -1
- package/src/lib/session-upload.js +25 -12
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,27 @@
|
|
|
3
3
|
本包(`tokenusage-cli`)的用户可见变更记录。发版时在顶部新增一节,
|
|
4
4
|
`## <版本号>` 行会被 release workflow 取作 GitHub Release 正文。
|
|
5
5
|
|
|
6
|
+
## 0.4.5
|
|
7
|
+
|
|
8
|
+
2026-11-04。一轮审计修复:`sync` 读队列时把契约 v2 的项目拆分行压回余量行、随后照旧推进
|
|
9
|
+
上传偏移,被归因的用量会永久丢失;上传确认、Windows 隐藏启动器编码与首同步统计也各有一处
|
|
10
|
+
缺陷。无契约变更,服务端无需同步升级。
|
|
11
|
+
|
|
12
|
+
- **队列批次不再丢项目行**:`readQueueBatch` 的去重键原来只有 `source|model|hour`,
|
|
13
|
+
契约 v2 的拆分行(同键不同 `project_key`)会被合并成余量行并随偏移推进永久丢失;
|
|
14
|
+
现键含 `project_key`,每行独立上传。
|
|
15
|
+
- **拆分行按服务端 basename 身份合并**:写出/合并项目拆分行时统一走共享的
|
|
16
|
+
`lib/project-key.js`(`basenameKey()`),与服务端请求去重口径一致;同名不同 owner 的
|
|
17
|
+
仓库不再互相覆盖。`reconcile` 对旧的 `owner/repo` 行做同样的归一,避免把正常数据报成漂移。
|
|
18
|
+
- **上传确认后才推进进度**:`ingest` 需响应含 `inserted` + `skipped`、session 上传需含
|
|
19
|
+
`sessions_upserted` + `sessions_skipped`,才推进队列偏移 / watermark;未确认的 2xx
|
|
20
|
+
响应按失败处理,下次 `sync` 重传而不是静默跳数据。
|
|
21
|
+
- **Windows 隐藏启动器改 UTF-16LE + BOM**:`run-sync-hidden.vbs` 按 wscript 期望的编码
|
|
22
|
+
写入,含非 ASCII 字符的安装路径不再乱码或被截断。
|
|
23
|
+
- **首同步统计修正**:`init` 的首次同步统计跳过 `kind` 控制行,并把 `project_key` 纳入
|
|
24
|
+
last-wins 键,项目用量不再被余量行盖掉。
|
|
25
|
+
- 回归测试:拆分行往返、basename 归一、VBS 编码、未确认 2xx 不推进偏移/watermark。
|
|
26
|
+
|
|
6
27
|
## 0.4.4
|
|
7
28
|
|
|
8
29
|
2026-11-04。契约 v2 落地:小时用量桶按项目名拆行(`project_key`),项目键向上回退到真实
|
package/package.json
CHANGED
package/src/commands/init.js
CHANGED
|
@@ -2807,7 +2807,14 @@ function readFirstSyncTotals(trackerDir) {
|
|
|
2807
2807
|
if (!trimmed) continue;
|
|
2808
2808
|
try {
|
|
2809
2809
|
const row = JSON.parse(trimmed);
|
|
2810
|
-
|
|
2810
|
+
// Control records (account session states / watermarks) carry a
|
|
2811
|
+
// `total_tokens` too; they are not usage buckets and must not inflate
|
|
2812
|
+
// the first-sync counter.
|
|
2813
|
+
if (!row || typeof row !== "object" || Object.hasOwn(row, "kind")) continue;
|
|
2814
|
+
// Contract v2: project_key joins the last-wins identity. Without it the
|
|
2815
|
+
// residual row (appended last) hides every attributed project row, and a
|
|
2816
|
+
// fully attributed hour would display as 0 tokens.
|
|
2817
|
+
const key = `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}|${row.project_key || ""}`;
|
|
2811
2818
|
latest.set(key, row);
|
|
2812
2819
|
} catch {
|
|
2813
2820
|
// skip malformed
|
|
@@ -40,6 +40,7 @@ const { functionUrlFor, fetchFunctionResponse } = require("../lib/function-url")
|
|
|
40
40
|
const { iterateQueueLines } = require("../lib/upload-queue");
|
|
41
41
|
const { normalizeQueueRow } = require("../lib/queue-row-normalize");
|
|
42
42
|
const { sanitizeBucketForUpload } = require("../lib/ingest-sanitize");
|
|
43
|
+
const { basenameKey } = require("../lib/project-key");
|
|
43
44
|
const { TOKEN_COLUMNS } = require("../lib/account-usage-dedup");
|
|
44
45
|
const { UsageError, unknownOptionError } = require("../lib/cli-errors");
|
|
45
46
|
|
|
@@ -208,7 +209,13 @@ async function collectBuckets({ queuePath, fromMs, toMs, start = 0 }) {
|
|
|
208
209
|
});
|
|
209
210
|
const sanitized = sanitizeBucketForUpload(row);
|
|
210
211
|
if (!sanitized) continue;
|
|
211
|
-
|
|
212
|
+
// The server stores only the basename (§2.4). Normalizing here keeps
|
|
213
|
+
// pre-v2 queues whose rows still carry `owner/repo` comparable with the
|
|
214
|
+
// cloud instead of reporting permanent missing_on_server drift.
|
|
215
|
+
const project =
|
|
216
|
+
typeof sanitized.project_key === "string"
|
|
217
|
+
? basenameKey(sanitized.project_key) || ""
|
|
218
|
+
: "";
|
|
212
219
|
buckets.set(bucketKey(hourMs, source, model, project), sanitized);
|
|
213
220
|
}
|
|
214
221
|
} finally {
|
package/src/commands/sync.js
CHANGED
|
@@ -4721,6 +4721,14 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
|
|
|
4721
4721
|
throw err;
|
|
4722
4722
|
}
|
|
4723
4723
|
|
|
4724
|
+
if (!isConfirmedIngestResponse(data)) {
|
|
4725
|
+
const err = new Error(
|
|
4726
|
+
`Unconfirmed ingest response (HTTP ${res.status}): ${rawText.substring(0, 300)}`,
|
|
4727
|
+
);
|
|
4728
|
+
err.status = res.status;
|
|
4729
|
+
err.code = "CLOUD_UPLOAD_UNCONFIRMED";
|
|
4730
|
+
throw err;
|
|
4731
|
+
}
|
|
4724
4732
|
cloudAccess = data;
|
|
4725
4733
|
if (data.upload_scope === "community") batchBudget = Math.min(batchBudget, 10);
|
|
4726
4734
|
inserted += Number(data?.inserted || 0);
|
|
@@ -4755,6 +4763,23 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
|
|
|
4755
4763
|
return resultSoFar();
|
|
4756
4764
|
}
|
|
4757
4765
|
|
|
4766
|
+
// A 200 is not proof the ingest committed: captive portals / reverse proxies
|
|
4767
|
+
// return 200 HTML and maintenance pages return an empty body. Advancing the
|
|
4768
|
+
// queue offset on those silently drops the batch forever, so the drain only
|
|
4769
|
+
// accepts the contract's success shape (docs/cli-contract.md §2.4).
|
|
4770
|
+
function isCount(value) {
|
|
4771
|
+
return (
|
|
4772
|
+
(typeof value === "number" || typeof value === "string") &&
|
|
4773
|
+
value !== "" &&
|
|
4774
|
+
Number.isFinite(Number(value))
|
|
4775
|
+
);
|
|
4776
|
+
}
|
|
4777
|
+
|
|
4778
|
+
function isConfirmedIngestResponse(data) {
|
|
4779
|
+
return Boolean(data) && typeof data === "object" && !Array.isArray(data) &&
|
|
4780
|
+
isCount(data.inserted) && isCount(data.skipped);
|
|
4781
|
+
}
|
|
4782
|
+
|
|
4758
4783
|
async function readQueueBatch(queuePath, startOffset, maxBuckets) {
|
|
4759
4784
|
const st = await fs.stat(queuePath).catch(() => null);
|
|
4760
4785
|
if (!st || !st.isFile()) return { buckets: [], sessionStates: [], nextOffset: startOffset };
|
|
@@ -4862,7 +4887,17 @@ async function readQueueBatch(queuePath, startOffset, maxBuckets) {
|
|
|
4862
4887
|
// otherwise 400 the whole ingest batch and stall the queue.
|
|
4863
4888
|
const sanitized = sanitizeBucketForUpload(bucket);
|
|
4864
4889
|
if (!sanitized) continue;
|
|
4865
|
-
|
|
4890
|
+
// Contract v2: project_key is part of the bucket identity. Split hours
|
|
4891
|
+
// carry several rows with the same (source, model, hour) under different
|
|
4892
|
+
// project keys (plus the '' residual); omitting it here collapsed them
|
|
4893
|
+
// to the last row and then advanced the offset past the rest, losing the
|
|
4894
|
+
// attributed usage permanently. New queue rows already carry the
|
|
4895
|
+
// server-normalized basename (rollout's appendProjectSplitRows); old
|
|
4896
|
+
// owner/repo rows stay distinct here and let the server's own basename
|
|
4897
|
+
// dedupe pick the larger total instead of collapsing them locally.
|
|
4898
|
+
const projectKey =
|
|
4899
|
+
typeof sanitized.project_key === "string" ? sanitized.project_key.trim() : "";
|
|
4900
|
+
bucketMap.set(`${source}|${model}|${hourStart}|${projectKey}`, sanitized);
|
|
4866
4901
|
linesRead += 1;
|
|
4867
4902
|
if (linesRead >= maxBuckets) break;
|
|
4868
4903
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Contract v2 (docs/cli-contract.md §2.4): the server keeps only the project
|
|
4
|
+
// basename for both hourly buckets and sessions. The CLI must apply the same
|
|
5
|
+
// rule before it writes a queue row or keys a reconciliation baseline —
|
|
6
|
+
// otherwise `owner/repo` and `repo` are two different local identities while
|
|
7
|
+
// the cloud stores one, which under-counts same-basename repos and makes
|
|
8
|
+
// reconcile report permanent false drift.
|
|
9
|
+
//
|
|
10
|
+
// This is the single implementation shared by the hourly writer, the session
|
|
11
|
+
// uploader and reconcile.
|
|
12
|
+
function basenameKey(value) {
|
|
13
|
+
const raw = String(value || "").replace(/[\\/]+$/, "");
|
|
14
|
+
if (!raw || raw === "." || raw === "..") return null;
|
|
15
|
+
const base = raw.split(/[\\/]/).pop().trim();
|
|
16
|
+
if (!base || base === "." || base === "..") return null;
|
|
17
|
+
// Control characters are rejected server-side; strip instead of failing.
|
|
18
|
+
// eslint-disable-next-line no-control-regex
|
|
19
|
+
const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
|
|
20
|
+
return clean || null;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
module.exports = { basenameKey };
|
package/src/lib/rollout.js
CHANGED
|
@@ -13,6 +13,7 @@ const { readSqliteJsonRows, readSqliteJsonRowsAsync } = require("./sqlite-reader
|
|
|
13
13
|
const wsl = require("./wsl-probe");
|
|
14
14
|
const { resolveInstallPaths } = require("./install-resolver");
|
|
15
15
|
const { expandHome } = require("./scan-roots");
|
|
16
|
+
const { basenameKey } = require("./project-key");
|
|
16
17
|
const {
|
|
17
18
|
consumeUsageDelta,
|
|
18
19
|
createUsageDeltaState,
|
|
@@ -3575,27 +3576,48 @@ function appendProjectSplitRows({ group, model, aggregateBucket, projects, toApp
|
|
|
3575
3576
|
if (!projects || projects.size === 0) return;
|
|
3576
3577
|
const attributed = initTotals();
|
|
3577
3578
|
const projectKeys = [...projects.keys()].sort();
|
|
3579
|
+
// The server stores only the basename (docs/cli-contract.md §2.4), so two
|
|
3580
|
+
// local projects such as `alice/api` and `bob/api` share one cloud identity.
|
|
3581
|
+
// Merge their totals under that normalized key here: emitting two rows with
|
|
3582
|
+
// the same wire key would make the server's request dedupe keep just one of
|
|
3583
|
+
// them and silently drop the other project's usage.
|
|
3584
|
+
const normalizedProjects = new Map();
|
|
3578
3585
|
for (const projectKey of projectKeys) {
|
|
3579
3586
|
const projectBucket = projects.get(projectKey);
|
|
3580
3587
|
if (!projectBucket.totals) continue;
|
|
3581
3588
|
addTotals(attributed, projectBucket.totals);
|
|
3582
|
-
const
|
|
3583
|
-
|
|
3589
|
+
const normalizedKey = basenameKey(projectKey);
|
|
3590
|
+
if (!normalizedKey) continue;
|
|
3591
|
+
let entry = normalizedProjects.get(normalizedKey);
|
|
3592
|
+
if (!entry) {
|
|
3593
|
+
entry = { totals: initTotals(), usagePrecision: null, buckets: [] };
|
|
3594
|
+
normalizedProjects.set(normalizedKey, entry);
|
|
3595
|
+
}
|
|
3596
|
+
addTotals(entry.totals, projectBucket.totals);
|
|
3597
|
+
entry.usagePrecision = entry.usagePrecision || projectBucket.usage_precision || null;
|
|
3598
|
+
entry.buckets.push(projectBucket);
|
|
3599
|
+
}
|
|
3600
|
+
|
|
3601
|
+
for (const [normalizedKey, entry] of normalizedProjects) {
|
|
3602
|
+
const totals = entry.totals;
|
|
3603
|
+
const usagePrecision = entry.usagePrecision || aggregateBucket.usage_precision || null;
|
|
3584
3604
|
const marker = usagePrecision ? `${totalsKey(totals)}|${usagePrecision}` : totalsKey(totals);
|
|
3585
|
-
|
|
3605
|
+
// Every contributing bucket must agree that this normalized row was
|
|
3606
|
+
// already queued; one changed contributor re-emits the merged total.
|
|
3607
|
+
if (entry.buckets.every((bucket) => bucket.queuedHourlyKey === marker)) continue;
|
|
3586
3608
|
toAppend.push(
|
|
3587
3609
|
JSON.stringify({
|
|
3588
3610
|
source: group.source,
|
|
3589
3611
|
model,
|
|
3590
3612
|
hour_start: group.hourStart,
|
|
3591
|
-
project_key:
|
|
3613
|
+
project_key: normalizedKey,
|
|
3592
3614
|
...totalsRowFields(totals),
|
|
3593
3615
|
usage_precision: usagePrecision || undefined,
|
|
3594
3616
|
...prioritySubsetOf(totals),
|
|
3595
3617
|
...creditsSubsetOf(totals),
|
|
3596
3618
|
}),
|
|
3597
3619
|
);
|
|
3598
|
-
|
|
3620
|
+
for (const bucket of entry.buckets) bucket.queuedHourlyKey = marker;
|
|
3599
3621
|
}
|
|
3600
3622
|
|
|
3601
3623
|
const residual = cloneTotals(aggregateBucket.totals);
|
|
@@ -158,7 +158,13 @@ function writeHiddenRunner({
|
|
|
158
158
|
}) {
|
|
159
159
|
fs.mkdirSync(binDir, { recursive: true });
|
|
160
160
|
const vbsPath = vbsPathFor(binDir);
|
|
161
|
-
|
|
161
|
+
// wscript.exe reads a BOM-less .vbs with the system ANSI code page (GBK on
|
|
162
|
+
// Chinese Windows), so a UTF-8 path containing a non-ASCII username turns
|
|
163
|
+
// into mojibake and the hourly run cannot find node. UTF-16LE + BOM is what
|
|
164
|
+
// Windows Script Host expects; the XML definition already follows the same
|
|
165
|
+
// rule.
|
|
166
|
+
const body = buildHiddenRunnerVbs({ nodeExe, cliJs, fallbackCliJs });
|
|
167
|
+
fs.writeFileSync(vbsPath, Buffer.from(`\uFEFF${body}`, "utf16le"), { mode: 0o600 });
|
|
162
168
|
return vbsPath;
|
|
163
169
|
}
|
|
164
170
|
|
|
@@ -17,6 +17,7 @@ const path = require("node:path");
|
|
|
17
17
|
const crypto = require("node:crypto");
|
|
18
18
|
|
|
19
19
|
const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
|
|
20
|
+
const { basenameKey } = require("./project-key");
|
|
20
21
|
|
|
21
22
|
const INGEST_SLUG = "token-usage-ingest";
|
|
22
23
|
const MAX_SESSIONS_PER_BATCH = 200;
|
|
@@ -66,17 +67,6 @@ function foldReasoningIntoOutput(tokens) {
|
|
|
66
67
|
}
|
|
67
68
|
}
|
|
68
69
|
|
|
69
|
-
function basenameKey(value) {
|
|
70
|
-
const raw = String(value || "").replace(/[\\/]+$/, "");
|
|
71
|
-
if (!raw || raw === "." || raw === "..") return null;
|
|
72
|
-
const base = raw.split(/[\\/]/).pop().trim();
|
|
73
|
-
if (!base || base === "." || base === "..") return null;
|
|
74
|
-
// Control characters are rejected server-side; strip instead of failing.
|
|
75
|
-
// eslint-disable-next-line no-control-regex
|
|
76
|
-
const clean = base.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, 120);
|
|
77
|
-
return clean || null;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
70
|
function validSessionId(value) {
|
|
81
71
|
if (typeof value !== "string") return false;
|
|
82
72
|
const id = value.trim();
|
|
@@ -347,7 +337,30 @@ async function uploadSessions({
|
|
|
347
337
|
error.upserted = upserted;
|
|
348
338
|
throw error;
|
|
349
339
|
}
|
|
350
|
-
|
|
340
|
+
// A 2xx is not a confirmation: captive portals and reverse proxies answer
|
|
341
|
+
// with 200 HTML/empty bodies. Advancing the watermark on those would skip
|
|
342
|
+
// the batch forever, so require the contract's counter fields
|
|
343
|
+
// (docs/cli-contract.md §2.4) before recording progress.
|
|
344
|
+
const data = await res.json().catch(() => null);
|
|
345
|
+
const isCount = (value) =>
|
|
346
|
+
(typeof value === "number" || typeof value === "string") &&
|
|
347
|
+
value !== "" &&
|
|
348
|
+
Number.isFinite(Number(value));
|
|
349
|
+
|
|
350
|
+
if (
|
|
351
|
+
!data ||
|
|
352
|
+
typeof data !== "object" ||
|
|
353
|
+
!isCount(data.sessions_upserted) ||
|
|
354
|
+
!isCount(data.sessions_skipped)
|
|
355
|
+
) {
|
|
356
|
+
const error = new Error(
|
|
357
|
+
`sessions upload was not confirmed (HTTP ${res.status}): unexpected response body`,
|
|
358
|
+
);
|
|
359
|
+
error.status = res.status;
|
|
360
|
+
error.uploaded = uploaded;
|
|
361
|
+
error.upserted = upserted;
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
351
364
|
batches += 1;
|
|
352
365
|
uploaded += batch.length;
|
|
353
366
|
upserted += Number(data?.sessions_upserted || 0);
|