tokenusage-cli 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/README.md +52 -11
  3. package/bin/tokenusage.js +14 -2
  4. package/package.json +1 -2
  5. package/src/cli.js +325 -78
  6. package/src/commands/device-login.js +3 -3
  7. package/src/commands/init.js +343 -153
  8. package/src/commands/pause.js +11 -23
  9. package/src/commands/reconcile.js +485 -0
  10. package/src/commands/sessions.js +57 -14
  11. package/src/commands/sync.js +69 -15
  12. package/src/commands/uninstall.js +6 -9
  13. package/src/commands/upload.js +147 -0
  14. package/src/lib/cli-errors.js +33 -0
  15. package/src/lib/cli-ui.js +93 -0
  16. package/src/lib/queue-row-normalize.js +55 -0
  17. package/src/lib/scheduled-task.js +597 -75
  18. package/src/lib/antigravity-paths.js +0 -48
  19. package/src/lib/async-queue-reader.js +0 -138
  20. package/src/lib/atomic-json.js +0 -27
  21. package/src/lib/bearer-token.js +0 -11
  22. package/src/lib/cloud-account.js +0 -541
  23. package/src/lib/codex-context-breakdown.js +0 -806
  24. package/src/lib/context-health.js +0 -124
  25. package/src/lib/grok-context-breakdown.js +0 -890
  26. package/src/lib/heatmap-compact.js +0 -114
  27. package/src/lib/local-api.js +0 -3574
  28. package/src/lib/outcomes-engine.js +0 -253
  29. package/src/lib/pet-packages.js +0 -722
  30. package/src/lib/skill-usage.js +0 -267
  31. package/src/lib/skills-manager.js +0 -1591
  32. package/src/lib/source-metadata.js +0 -48
  33. package/src/lib/subscription-manager.js +0 -293
  34. package/src/lib/wrapped-aggregator.js +0 -236
  35. package/test/audit-fixes.test.js +0 -249
  36. package/test/branding.test.js +0 -24
  37. package/test/cli-options.test.js +0 -43
  38. package/test/cli-smoke.test.js +0 -53
  39. package/test/cli-ui.test.js +0 -47
  40. package/test/collection-controls.test.js +0 -255
  41. package/test/ingest-sanitize.test.js +0 -162
  42. package/test/packaging.test.js +0 -107
  43. package/test/quiet-warnings.test.js +0 -53
  44. package/test/runtime-copy.test.js +0 -111
  45. package/test/session-upload.test.js +0 -427
  46. package/test/trae-db.test.js +0 -67
  47. package/test/zstd-native.test.js +0 -36
@@ -1,162 +0,0 @@
1
- "use strict";
2
-
3
- // Regression tests for the upload-path poison-row defense. The ingest edge
4
- // validates every row before writing and rejects the WHOLE batch on the
5
- // first contract-invalid row; the queue offset only advances on success, so
6
- // one bad row would otherwise stall every source behind it forever. These
7
- // tests pin: sanitizeBucketForUpload coerces buckets into the contract shape
8
- // (docs/cli-contract.md §2.4), readQueueBatch drops rows it cannot repair,
9
- // and a dropped row never occupies its dedup key.
10
-
11
- const assert = require("node:assert/strict");
12
- const fs = require("node:fs");
13
- const os = require("node:os");
14
- const path = require("node:path");
15
- const { test, beforeEach, afterEach } = require("node:test");
16
-
17
- const { sanitizeBucketForUpload } = require("../src/lib/ingest-sanitize");
18
- const { readQueueBatch } = require("../src/commands/sync");
19
- const { validSessionId } = require("../src/lib/session-upload");
20
-
21
- let tmpDir;
22
-
23
- beforeEach(() => {
24
- tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tokenusage-ingest-sanitize-"));
25
- });
26
-
27
- afterEach(() => {
28
- fs.rmSync(tmpDir, { recursive: true, force: true });
29
- });
30
-
31
- function writeQueue(lines) {
32
- const queuePath = path.join(tmpDir, "queue.jsonl");
33
- fs.writeFileSync(queuePath, lines.map((line) => JSON.stringify(line)).join("\n") + "\n");
34
- return queuePath;
35
- }
36
-
37
- function validBucket(overrides = {}) {
38
- return {
39
- hour_start: "2026-10-26T01:00:00.000Z",
40
- source: "claude",
41
- model: "claude-x",
42
- input_tokens: 10,
43
- cached_input_tokens: 2,
44
- cache_creation_input_tokens: 3,
45
- output_tokens: 4,
46
- reasoning_output_tokens: 1,
47
- total_tokens: 19,
48
- billable_total_tokens: 19,
49
- conversation_count: 1,
50
- total_cost_usd: 0.5,
51
- ...overrides,
52
- };
53
- }
54
-
55
- function validState(overrides = {}) {
56
- return {
57
- kind: "account_session_state",
58
- source: "trae-cn",
59
- session_id: "session-1",
60
- model: "kimi-k2",
61
- bucket_start: "2026-10-26T01:00:00.000Z",
62
- input_tokens: 100,
63
- output_tokens: 20,
64
- cached_input_tokens: 30,
65
- cache_creation_input_tokens: 10,
66
- reasoning_output_tokens: 5,
67
- total_tokens: 160,
68
- snapshot_verified_at: "2026-10-26T02:00:00.000Z",
69
- ...overrides,
70
- };
71
- }
72
-
73
- test("sanitizeBucketForUpload floors fractional counters and clamps negatives to 0", () => {
74
- const out = sanitizeBucketForUpload(validBucket({
75
- input_tokens: 10.75,
76
- output_tokens: -4,
77
- reasoning_output_tokens: Number.NaN,
78
- total_cost_usd: -0.5,
79
- }));
80
- assert.equal(out.input_tokens, 10);
81
- assert.equal(out.output_tokens, 0);
82
- assert.equal(out.reasoning_output_tokens, 0);
83
- assert.equal(out.total_cost_usd, 0);
84
- });
85
-
86
- test("sanitizeBucketForUpload keeps a valid row verbatim, incl. explicit billable null", () => {
87
- const row = validBucket({ billable_total_tokens: null });
88
- assert.deepEqual(sanitizeBucketForUpload(row), row);
89
- });
90
-
91
- test("sanitizeBucketForUpload drops rows without a parseable hour_start and non-objects", () => {
92
- assert.equal(sanitizeBucketForUpload(validBucket({ hour_start: "garbage" })), null);
93
- assert.equal(sanitizeBucketForUpload(validBucket({ hour_start: 17 })), null);
94
- assert.equal(sanitizeBucketForUpload(null), null);
95
- assert.equal(sanitizeBucketForUpload([validBucket()]), null);
96
- });
97
-
98
- test("sanitizeBucketForUpload fills missing counters with 0 and cost with 0", () => {
99
- const out = sanitizeBucketForUpload({
100
- hour_start: "2026-10-26T01:00:00.000Z",
101
- source: "x",
102
- model: "m",
103
- total_tokens: 5,
104
- });
105
- assert.equal(out.input_tokens, 0);
106
- assert.equal(out.total_tokens, 5);
107
- assert.equal(out.billable_total_tokens, null);
108
- assert.equal(out.total_cost_usd, 0);
109
- });
110
-
111
- test("readQueueBatch drops poison account_session_states and keeps the valid one", async () => {
112
- const tooLong = validState({ session_id: "s".repeat(300) });
113
- const brokenSum = validState({ session_id: "broken", total_tokens: 99 });
114
- const good = validState({ session_id: "good" });
115
- const queuePath = writeQueue([validBucket(), tooLong, brokenSum, good]);
116
- const result = await readQueueBatch(queuePath, 0, 500);
117
- assert.equal(result.buckets.length, 1);
118
- assert.equal(result.sessionStates.length, 1);
119
- assert.equal(result.sessionStates[0].session_id, "good");
120
- assert.equal(result.nextOffset, fs.statSync(queuePath).size);
121
- });
122
-
123
- test("a poison row does not occupy its dedup key — a later valid snapshot is admitted", async () => {
124
- const poison = validState({ total_tokens: 99 });
125
- const good = validState({ total_tokens: 160 });
126
- const queuePath = writeQueue([poison, good]);
127
- const result = await readQueueBatch(queuePath, 0, 500);
128
- assert.equal(result.sessionStates.length, 1);
129
- assert.equal(result.sessionStates[0].total_tokens, 160);
130
- });
131
-
132
- test("readQueueBatch drops hourly rows the edge would 400 and floors fractional counters", async () => {
133
- const badHour = validBucket({ hour_start: "garbage" });
134
- const fractional = validBucket({ hour_start: "2026-10-26T02:00:00.000Z", total_tokens: 19.5 });
135
- const good = validBucket({ hour_start: "2026-10-26T03:00:00.000Z" });
136
- const queuePath = writeQueue([badHour, fractional, good]);
137
- const result = await readQueueBatch(queuePath, 0, 500);
138
- assert.equal(result.buckets.length, 2);
139
- assert.equal(result.buckets.find((b) => b.hour_start === "2026-10-26T02:00:00.000Z").total_tokens, 19);
140
- assert.equal(result.buckets.find((b) => b.hour_start === "2026-10-26T03:00:00.000Z").total_tokens, 19);
141
- assert.equal(result.nextOffset, fs.statSync(queuePath).size);
142
- });
143
-
144
- test("a segment of only-poison rows reads empty with the offset advanced", async () => {
145
- const queuePath = writeQueue([
146
- validBucket({ hour_start: "garbage" }),
147
- validState({ total_tokens: 99 }),
148
- ]);
149
- const result = await readQueueBatch(queuePath, 0, 500);
150
- assert.deepEqual(result.buckets, []);
151
- assert.deepEqual(result.sessionStates, []);
152
- assert.equal(result.nextOffset, fs.statSync(queuePath).size);
153
- });
154
-
155
- test("validSessionId rejects the full Unicode Cc range (incl. C1), matching the server", () => {
156
- assert.equal(validSessionId("ok-id"), true);
157
- assert.equal(validSessionId("bad\u0085-id"), false); // NEL, C1
158
- assert.equal(validSessionId("bad\u001f-id"), false); // C0
159
- assert.equal(validSessionId("bad\u007f-id"), false); // DEL
160
- assert.equal(validSessionId("with/slash"), false);
161
- assert.equal(validSessionId("with\\slash"), false);
162
- });
@@ -1,107 +0,0 @@
1
- // Packaging guards: what `npm pack` ships must actually boot. 0.1.x shipped
2
- // without bin/tracker.js (installer still pointed at the pre-fork name) and
3
- // local-api's dormant local server still resolved that missing path. The
4
- // upstream install-count heartbeat was also never ported to this backend.
5
-
6
- const assert = require("node:assert/strict");
7
- const fs = require("node:fs");
8
- const path = require("node:path");
9
- const { test } = require("node:test");
10
-
11
- const PACKAGE_ROOT = path.join(__dirname, "..");
12
- const SRC = path.join(PACKAGE_ROOT, "src");
13
- const BIN = path.join(PACKAGE_ROOT, "bin");
14
-
15
- function readSource(...segments) {
16
- return fs.readFileSync(path.join(PACKAGE_ROOT, ...segments), "utf8");
17
- }
18
-
19
- test("package bin points at the CLI entry that actually ships", () => {
20
- const pkg = JSON.parse(readSource("package.json"));
21
- assert.equal(pkg.bin.tokenusage, "bin/tokenusage.js");
22
- assert.ok(
23
- fs.existsSync(path.join(PACKAGE_ROOT, pkg.bin.tokenusage)),
24
- "the declared bin file must exist in the tarball",
25
- );
26
- });
27
-
28
- test("local-api spawns the package bin entry, not the missing bin/tracker.js", () => {
29
- const source = readSource("src", "lib", "local-api.js");
30
- assert.ok(!source.includes('"../../bin/tracker.js"'), "bin/tracker.js never existed after the fork");
31
- const match = source.match(/TRACKER_BIN = path\.resolve\(__dirname, "([^"]+)"\)/);
32
- assert.ok(match, "TRACKER_BIN must stay a single resolved path");
33
- assert.ok(
34
- fs.existsSync(path.resolve(path.join(SRC, "lib"), match[1])),
35
- `local-api spawn target does not resolve: ${match[1]}`,
36
- );
37
- });
38
-
39
- test("the unused Sentry error-reporting SDK stays out of the package", () => {
40
- // 0.2.6 removed @sentry/node: it was only initialized (captureException had no
41
- // call sites) and shipped 44 MB into the runtime copy plus a build-time
42
- // bundler-plugin chain into every npm install. Keep it out unless the
43
- // reporting path is actually wired and budgeted again.
44
- const pkg = JSON.parse(readSource("package.json"));
45
- const declared = { ...(pkg.dependencies || {}), ...(pkg.optionalDependencies || {}) };
46
- assert.ok(
47
- !Object.keys(declared).some((name) => name === "@sentry/node" || name.startsWith("@sentry/")),
48
- "@sentry/* must not be a declared dependency",
49
- );
50
- assert.ok(
51
- !fs.existsSync(path.join(SRC, "lib", "sentry.js")),
52
- "the inert sentry bootstrap must stay deleted",
53
- );
54
- assert.ok(
55
- !fs.existsSync(path.join(SRC, "lib", "telemetry-pref.js")),
56
- "the opt-out helper only existed to gate Sentry",
57
- );
58
- const roots = [SRC, BIN];
59
- for (const root of roots) {
60
- const sourceFiles = fs
61
- .readdirSync(root, { recursive: true })
62
- .filter((rel) => String(rel).endsWith(".js"));
63
- for (const rel of sourceFiles) {
64
- const source = fs.readFileSync(path.join(root, String(rel)), "utf8");
65
- assert.ok(!source.includes("@sentry/"), `${rel} must not require the removed SDK`);
66
- }
67
- }
68
- });
69
-
70
- test("the unported install heartbeat stays removed", () => {
71
- assert.ok(
72
- !fs.existsSync(path.join(SRC, "lib", "telemetry.js")),
73
- "the backend has no /api/functions/tokenusage-telemetry route, so nothing may post to it",
74
- );
75
- const sync = readSource("src", "commands", "sync.js");
76
- assert.ok(!sync.includes("maybeSendHeartbeat"));
77
- assert.ok(!sync.includes('require("../lib/telemetry")'));
78
- });
79
-
80
- test("every auto-sync spawn point gates on the runtime-copy readiness marker", () => {
81
- // 0.2.4: a runtime copy without its marker is a half copy (missing deps) and
82
- // dies with MODULE_NOT_FOUND inside an ignored stdio stream — collection
83
- // looks alive while nothing is collected. Each generated/hidden launcher must
84
- // check the marker before choosing it.
85
- const init = readSource("src", "commands", "init.js");
86
- assert.ok(init.includes("runtimeReadyMarkerPath"), "init.js must write and probe the marker");
87
-
88
- const notify = readSource("src", "commands", "init.js");
89
- assert.ok(
90
- notify.includes("const hasLocalRuntime = fs.existsSync(runtimeReadyPath) && fs.existsSync(trackerBinPath);"),
91
- "the notify handler must require both the launcher and the marker",
92
- );
93
-
94
- for (const file of [["src", "lib", "openclaw-hook.js"], ["src", "lib", "openclaw-session-plugin.js"]]) {
95
- const source = readSource(...file);
96
- assert.ok(
97
- source.includes("runtimeReadyPath") && source.includes("packageEntryPath"),
98
- `${file.join("/")} must bake the marker + installed-package fallback`,
99
- );
100
- }
101
-
102
- const sync = readSource("src", "commands", "sync.js");
103
- assert.ok(
104
- sync.includes("runtimeReadyPath") && sync.includes("packageEntryPath"),
105
- "the auto-retry script must use the same launcher selection",
106
- );
107
- });
@@ -1,53 +0,0 @@
1
- "use strict";
2
-
3
- // The bin entry installs the node:sqlite ExperimentalWarning filter before
4
- // any require can pull node:sqlite in; redirected scheduled-task logs grow
5
- // by one identical warning per cycle otherwise. Pin the filter's contract:
6
- // drop only that warning, pass everything else through, stay idempotent.
7
-
8
- const assert = require("node:assert/strict");
9
- const { test } = require("node:test");
10
-
11
- const { suppressSqliteExperimentalWarning } = require("../src/lib/quiet-warnings");
12
-
13
- test("filters the node:sqlite experimental warning and passes everything else through", () => {
14
- const seen = [];
15
- const saved = process.emitWarning;
16
- process.emitWarning = (warning, ...args) => {
17
- seen.push(warning);
18
- };
19
- try {
20
- suppressSqliteExperimentalWarning();
21
-
22
- process.emitWarning(
23
- "SQLite is an experimental feature and might change at any time",
24
- { type: "ExperimentalWarning", code: "ExperimentalWarnings" },
25
- );
26
- assert.equal(seen.length, 0);
27
-
28
- const other = new Error("some other experimental feature");
29
- other.name = "ExperimentalWarning";
30
- process.emitWarning(other);
31
- assert.equal(seen.length, 1);
32
- assert.equal(seen[0], other);
33
-
34
- process.emitWarning("plain string warning");
35
- assert.equal(seen.length, 2);
36
- assert.equal(seen[1], "plain string warning");
37
- } finally {
38
- process.emitWarning = saved;
39
- }
40
- });
41
-
42
- test("suppressSqliteExperimentalWarning is idempotent (does not stack filters)", () => {
43
- const saved = process.emitWarning;
44
- process.emitWarning = () => {};
45
- try {
46
- suppressSqliteExperimentalWarning();
47
- const onceFiltered = process.emitWarning;
48
- suppressSqliteExperimentalWarning();
49
- assert.equal(process.emitWarning, onceFiltered);
50
- } finally {
51
- process.emitWarning = saved;
52
- }
53
- });
@@ -1,111 +0,0 @@
1
- "use strict";
2
-
3
- // The init-copied runtime (`<trackerDir>/app`) is the launcher behind notify
4
- // hooks, the hidden scheduled task and the auto-retry child. 0.2.4 made its
5
- // readiness marker the single gate: a copy that never booted must never be
6
- // spawned (it dies with MODULE_NOT_FOUND inside an ignored stdio stream, which
7
- // is indistinguishable from "collection is running"), and `doctor` must say so.
8
-
9
- const assert = require("node:assert/strict");
10
- const fs = require("node:fs");
11
- const os = require("node:os");
12
- const path = require("node:path");
13
- const { test } = require("node:test");
14
-
15
- const { buildNotifyHandler } = require("../src/commands/init");
16
- const { buildDoctorReport } = require("../src/lib/doctor");
17
- const { runtimeAppDir, runtimeBinPath, runtimeReadyMarkerPath, RUNTIME_READY_MARKER } = require("../src/lib/runtime-entry");
18
-
19
- const CLI_VERSION = require("../package.json").version;
20
-
21
- async function withTrackerDir(fn) {
22
- const home = fs.mkdtempSync(path.join(os.tmpdir(), "tu-runtime-copy-"));
23
- const trackerDir = path.join(home, "tracker");
24
- fs.mkdirSync(trackerDir, { recursive: true });
25
- try {
26
- return await fn({ home, trackerDir });
27
- } finally {
28
- fs.rmSync(home, { recursive: true, force: true });
29
- }
30
- }
31
-
32
- async function runtimeCopyCheck(trackerDir) {
33
- const report = await buildDoctorReport({
34
- runtime: { baseUrl: "https://example.test", deviceToken: "present", dashboardUrl: "https://example.test" },
35
- diagnostics: null,
36
- fetch: async () => ({ ok: true, status: 200 }),
37
- paths: { trackerDir, configPath: path.join(trackerDir, "config.json") },
38
- });
39
- return report.checks.find((check) => check.id === "runtime.copy");
40
- }
41
-
42
- test("runtime-entry: the marker lives next to the copy, not inside bin/", () => {
43
- const trackerDir = path.join("home", ".tokenusage", "tracker");
44
- assert.equal(runtimeAppDir(trackerDir), path.join(trackerDir, "app"));
45
- assert.equal(runtimeBinPath(trackerDir), path.join(trackerDir, "app", "bin", "tracker.js"));
46
- assert.equal(
47
- runtimeReadyMarkerPath(runtimeAppDir(trackerDir)),
48
- path.join(trackerDir, "app", RUNTIME_READY_MARKER),
49
- );
50
- });
51
-
52
- test("doctor: no runtime copy is fine (hooks use the installed CLI)", async () => {
53
- const check = await withTrackerDir(({ trackerDir }) => runtimeCopyCheck(trackerDir));
54
- assert.equal(check.status, "ok");
55
- assert.match(check.detail, /installed CLI/);
56
- });
57
-
58
- test("doctor: a copy without its marker is reported as incomplete", async () => {
59
- await withTrackerDir(async ({ trackerDir }) => {
60
- const appDir = runtimeAppDir(trackerDir);
61
- fs.mkdirSync(path.join(appDir, "bin"), { recursive: true });
62
- fs.writeFileSync(path.join(appDir, "bin", "tracker.js"), "// half copy\n");
63
- const check = await runtimeCopyCheck(trackerDir);
64
- assert.equal(check.status, "warn");
65
- assert.match(check.detail, /incomplete/);
66
- });
67
- });
68
-
69
- test("doctor: a ready copy reports its version", async () => {
70
- await withTrackerDir(async ({ trackerDir }) => {
71
- const appDir = runtimeAppDir(trackerDir);
72
- fs.mkdirSync(path.join(appDir, "bin"), { recursive: true });
73
- fs.writeFileSync(path.join(appDir, "bin", "tracker.js"), "// copy\n");
74
- fs.writeFileSync(
75
- runtimeReadyMarkerPath(appDir),
76
- JSON.stringify({ version: CLI_VERSION, installedAt: "2026-10-28T00:00:00.000Z" }),
77
- );
78
- const check = await runtimeCopyCheck(trackerDir);
79
- assert.equal(check.status, "ok");
80
- assert.match(check.detail, new RegExp(`v${CLI_VERSION.replace(/\./g, "\\.")}`));
81
- assert.equal(check.meta.version, CLI_VERSION);
82
- });
83
- });
84
-
85
- test("doctor: a copy from an older CLI is flagged for a re-run", async () => {
86
- await withTrackerDir(async ({ trackerDir }) => {
87
- const appDir = runtimeAppDir(trackerDir);
88
- fs.mkdirSync(path.join(appDir, "bin"), { recursive: true });
89
- fs.writeFileSync(path.join(appDir, "bin", "tracker.js"), "// copy\n");
90
- fs.writeFileSync(runtimeReadyMarkerPath(appDir), JSON.stringify({ version: "0.0.1" }));
91
- const check = await runtimeCopyCheck(trackerDir);
92
- assert.equal(check.status, "warn");
93
- assert.match(check.detail, /older than the CLI/);
94
- });
95
- });
96
-
97
- test("notify handler: marker + installed-package fallback baked into the template", () => {
98
- const source = buildNotifyHandler({ trackerDir: path.join("C:", "home", ".tokenusage", "tracker") });
99
- assert.ok(source.includes("runtimeReadyPath"), "the template must probe the readiness marker");
100
- assert.ok(source.includes("packageEntryPath"), "the template must carry the installed CLI entry");
101
- assert.ok(
102
- source.includes("const hasLocalRuntime = fs.existsSync(runtimeReadyPath) && fs.existsSync(trackerBinPath);"),
103
- "both the launcher and the marker are required",
104
- );
105
- assert.ok(
106
- source.includes(": hasPackageEntry\n ? [process.execPath, packageEntryPath, ...syncArgs]"),
107
- "the middle tier runs the installed package without npx",
108
- );
109
- // The generated file must stay dependency-free: only node built-ins.
110
- assert.ok(!/require\(['"](?!node:)/.test(source), "notify.cjs must only require node: built-ins");
111
- });