tokenusage-cli 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.md +52 -11
- package/bin/tokenusage.js +14 -2
- package/package.json +1 -2
- package/src/cli.js +325 -78
- package/src/commands/device-login.js +3 -3
- package/src/commands/init.js +343 -153
- package/src/commands/pause.js +11 -23
- package/src/commands/reconcile.js +496 -0
- package/src/commands/sessions.js +57 -14
- package/src/commands/sync.js +69 -15
- package/src/commands/uninstall.js +6 -9
- package/src/commands/upload.js +147 -0
- package/src/lib/cli-errors.js +33 -0
- package/src/lib/cli-ui.js +93 -0
- package/src/lib/queue-row-normalize.js +55 -0
- package/src/lib/scheduled-task.js +597 -75
- package/src/lib/antigravity-paths.js +0 -48
- package/src/lib/async-queue-reader.js +0 -138
- package/src/lib/atomic-json.js +0 -27
- package/src/lib/bearer-token.js +0 -11
- package/src/lib/cloud-account.js +0 -541
- package/src/lib/codex-context-breakdown.js +0 -806
- package/src/lib/context-health.js +0 -124
- package/src/lib/grok-context-breakdown.js +0 -890
- package/src/lib/heatmap-compact.js +0 -114
- package/src/lib/local-api.js +0 -3574
- package/src/lib/outcomes-engine.js +0 -253
- package/src/lib/pet-packages.js +0 -722
- package/src/lib/skill-usage.js +0 -267
- package/src/lib/skills-manager.js +0 -1591
- package/src/lib/source-metadata.js +0 -48
- package/src/lib/subscription-manager.js +0 -293
- package/src/lib/wrapped-aggregator.js +0 -236
- package/test/audit-fixes.test.js +0 -249
- package/test/branding.test.js +0 -24
- package/test/cli-options.test.js +0 -43
- package/test/cli-smoke.test.js +0 -53
- package/test/cli-ui.test.js +0 -47
- package/test/collection-controls.test.js +0 -255
- package/test/ingest-sanitize.test.js +0 -162
- package/test/packaging.test.js +0 -107
- package/test/quiet-warnings.test.js +0 -53
- package/test/runtime-copy.test.js +0 -111
- package/test/session-upload.test.js +0 -427
- package/test/trae-db.test.js +0 -67
- package/test/zstd-native.test.js +0 -36
package/src/commands/pause.js
CHANGED
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
// `tokenusage pause` / `tokenusage resume` (0.2.0): the collection switch.
|
|
4
4
|
//
|
|
5
5
|
// pause writes the cross-platform flag in ~/.tokenusage (auto-flavored syncs
|
|
6
|
-
// — scheduled/notify/retry/watch — check it and exit silently) and
|
|
7
|
-
//
|
|
8
|
-
// firing. resume reverses both. The flag is the source
|
|
9
|
-
// toggle is a convenience, so a missing
|
|
6
|
+
// — scheduled/notify/retry/watch — check it and exit silently) and disables the
|
|
7
|
+
// registered auto-sync schedule (Task Scheduler / launchd / systemd --user) so
|
|
8
|
+
// the trigger itself stops firing. resume reverses both. The flag is the source
|
|
9
|
+
// of truth; the schedule toggle is a convenience, so a missing or unregistered
|
|
10
|
+
// schedule is reported, not an error.
|
|
10
11
|
|
|
11
12
|
const os = require("node:os");
|
|
12
13
|
|
|
@@ -16,36 +17,23 @@ const {
|
|
|
16
17
|
setCollectionPaused,
|
|
17
18
|
} = require("../lib/collection-pause");
|
|
18
19
|
const {
|
|
19
|
-
|
|
20
|
-
|
|
20
|
+
setAutoSyncEnabled,
|
|
21
|
+
formatScheduleToggle,
|
|
21
22
|
} = require("../lib/scheduled-task");
|
|
22
23
|
|
|
23
|
-
function describeTaskOutcome(outcome, enabled) {
|
|
24
|
-
if (outcome.changed) {
|
|
25
|
-
return `${TASK_NAME} ${enabled ? "enabled" : "disabled"}.`;
|
|
26
|
-
}
|
|
27
|
-
if (outcome.reason === "not_windows") {
|
|
28
|
-
return "(Windows scheduled task: n/a on this platform — the pause flag covers cron/launchd too).";
|
|
29
|
-
}
|
|
30
|
-
if (outcome.reason === "task_missing") {
|
|
31
|
-
return `${TASK_NAME} not found (nothing to ${enabled ? "enable" : "disable"}) — the pause flag still guards all auto syncs.`;
|
|
32
|
-
}
|
|
33
|
-
return `${TASK_NAME} could not be ${enabled ? "enabled" : "disabled"} (${outcome.detail || "unknown error"}) — the pause flag still guards all auto syncs.`;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
24
|
async function cmdPause() {
|
|
37
25
|
const { trackerDir } = await resolveTrackerPaths({ home: os.homedir() });
|
|
38
26
|
const alreadyPaused = isCollectionPaused(trackerDir);
|
|
39
27
|
|
|
40
28
|
setCollectionPaused(trackerDir, true);
|
|
41
|
-
const
|
|
29
|
+
const schedule = setAutoSyncEnabled(false);
|
|
42
30
|
|
|
43
31
|
process.stdout.write(
|
|
44
32
|
[
|
|
45
33
|
"Collection paused.",
|
|
46
34
|
"- Scheduled/notify/watch syncs will exit silently without uploading.",
|
|
47
35
|
"- Manual `tokenusage sync` still works when you explicitly run it.",
|
|
48
|
-
`- ${
|
|
36
|
+
`- ${formatScheduleToggle(schedule, false)}`,
|
|
49
37
|
alreadyPaused ? "(Already paused — state refreshed.)" : null,
|
|
50
38
|
"",
|
|
51
39
|
]
|
|
@@ -59,13 +47,13 @@ async function cmdResume() {
|
|
|
59
47
|
const wasPaused = isCollectionPaused(trackerDir);
|
|
60
48
|
|
|
61
49
|
setCollectionPaused(trackerDir, false);
|
|
62
|
-
const
|
|
50
|
+
const schedule = setAutoSyncEnabled(true);
|
|
63
51
|
|
|
64
52
|
process.stdout.write(
|
|
65
53
|
[
|
|
66
54
|
"Collection resumed.",
|
|
67
55
|
"- Auto syncs report runs again (empty runs included).",
|
|
68
|
-
`- ${
|
|
56
|
+
`- ${formatScheduleToggle(schedule, true)}`,
|
|
69
57
|
wasPaused ? null : "(Was not paused — nothing to do.)",
|
|
70
58
|
"",
|
|
71
59
|
]
|
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// `tokenusage reconcile` — compare this machine's local last-wins baseline
|
|
4
|
+
// against the hourly rows the cloud holds for THIS device.
|
|
5
|
+
//
|
|
6
|
+
// Baseline definition (AGENTS hard rule 8): the whole of queue.jsonl, last
|
|
7
|
+
// write wins per (hour_start, source, model) — the same rows `sync`/`upload`
|
|
8
|
+
// would send, read through the same normalize + sanitize pipeline
|
|
9
|
+
// (lib/queue-row-normalize.js, lib/ingest-sanitize.js). Server side is
|
|
10
|
+
// docs/cli-contract.md §2.13: per-device, verbatim rows, no account-level
|
|
11
|
+
// dedup and no derived cost.
|
|
12
|
+
//
|
|
13
|
+
// Read-only on both sides: nothing uploads, the queue is never rewritten and
|
|
14
|
+
// queue.state.json (the upload offset) is not touched. Drift is reported per
|
|
15
|
+
// key and exits 1 so a script can gate on it:
|
|
16
|
+
//
|
|
17
|
+
// missing_on_server local bucket the cloud never stored (pending/blocked upload)
|
|
18
|
+
// server_only cloud row without a local counterpart (pruned queue, or
|
|
19
|
+
// a row written under another machine identity)
|
|
20
|
+
// mismatch same key, different values (partial upload / legacy repair)
|
|
21
|
+
//
|
|
22
|
+
// Known residual: ingest dedupes a REPLAYED duplicate inside one request by
|
|
23
|
+
// taking the larger total_tokens, so a queue that carried the same key twice
|
|
24
|
+
// with a shrinking total could keep a stale cloud value. Such keys surface as
|
|
25
|
+
// `mismatch` for a human to judge — the command never writes.
|
|
26
|
+
|
|
27
|
+
const os = require("node:os");
|
|
28
|
+
const path = require("node:path");
|
|
29
|
+
const fs = require("node:fs/promises");
|
|
30
|
+
|
|
31
|
+
const { resolveTrackerPaths } = require("../lib/tracker-paths");
|
|
32
|
+
const { readJson } = require("../lib/fs");
|
|
33
|
+
const { resolveRuntimeConfig, assertInsforgeRuntime } = require("../lib/runtime-config");
|
|
34
|
+
const { functionUrlFor, fetchFunctionResponse } = require("../lib/function-url");
|
|
35
|
+
const { iterateQueueLines } = require("../lib/upload-queue");
|
|
36
|
+
const { normalizeQueueRow } = require("../lib/queue-row-normalize");
|
|
37
|
+
const { sanitizeBucketForUpload } = require("../lib/ingest-sanitize");
|
|
38
|
+
const { TOKEN_COLUMNS } = require("../lib/account-usage-dedup");
|
|
39
|
+
const { UsageError, unknownOptionError } = require("../lib/cli-errors");
|
|
40
|
+
|
|
41
|
+
const RECONCILE_SLUG = "token-usage-reconcile";
|
|
42
|
+
const DAY_MS = 86_400_000;
|
|
43
|
+
// Mirrors the server clamp (§2.13) so the local window can never be wider than
|
|
44
|
+
// what the cloud is willing to return in one paged scan.
|
|
45
|
+
const MAX_RANGE_DAYS = 366;
|
|
46
|
+
const DEFAULT_DAYS = 7;
|
|
47
|
+
const DEFAULT_LIMIT = 1000;
|
|
48
|
+
const MAX_LIMIT = 2000;
|
|
49
|
+
// Runaway guard: the server pages are ordered, so 1000 pages cover 1–2M rows
|
|
50
|
+
// (far beyond a year of one device's buckets). Hitting it means the window /
|
|
51
|
+
// limit combination is wrong, not that the data is that big.
|
|
52
|
+
const MAX_PAGES = 1000;
|
|
53
|
+
const TEXT_DRIFT_LIMIT = 25;
|
|
54
|
+
|
|
55
|
+
const DATE_ONLY = /^\d{4}-\d{2}-\d{2}$/;
|
|
56
|
+
|
|
57
|
+
function readValue(argv, index, flag) {
|
|
58
|
+
const value = argv[index + 1];
|
|
59
|
+
if (value == null || value === "" || value.startsWith("--")) {
|
|
60
|
+
throw new UsageError(`${flag} requires a value`, {
|
|
61
|
+
code: "MISSING_VALUE",
|
|
62
|
+
hint: "Run `tokenusage reconcile --help` for supported options.",
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
return value;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function parseArgs(argv = []) {
|
|
69
|
+
const out = { days: null, from: null, to: null, limit: null, json: false };
|
|
70
|
+
for (let i = 0; i < argv.length; i++) {
|
|
71
|
+
const a = argv[i];
|
|
72
|
+
if (a === "--json") out.json = true;
|
|
73
|
+
else if (a === "--days") {
|
|
74
|
+
out.days = readValue(argv, i, "--days");
|
|
75
|
+
i += 1;
|
|
76
|
+
} else if (a === "--from") {
|
|
77
|
+
out.from = readValue(argv, i, "--from");
|
|
78
|
+
i += 1;
|
|
79
|
+
} else if (a === "--to") {
|
|
80
|
+
out.to = readValue(argv, i, "--to");
|
|
81
|
+
i += 1;
|
|
82
|
+
} else if (a === "--limit") {
|
|
83
|
+
out.limit = readValue(argv, i, "--limit");
|
|
84
|
+
i += 1;
|
|
85
|
+
} else throw unknownOptionError(a, "reconcile");
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// A bare YYYY-MM-DD is a UTC calendar day: `--from 2026-10-25` starts at
|
|
91
|
+
// 00:00Z, `--to 2026-10-31` ends at the NEXT midnight (01:00 on 11-01), so the
|
|
92
|
+
// whole end day is inside the window. A full ISO instant is taken verbatim
|
|
93
|
+
// (exclusive end).
|
|
94
|
+
function parseBoundary(value, { end = false } = {}) {
|
|
95
|
+
if (DATE_ONLY.test(value)) {
|
|
96
|
+
const ms = Date.parse(`${value}T00:00:00.000Z`);
|
|
97
|
+
if (!Number.isFinite(ms)) {
|
|
98
|
+
throw new UsageError(`Invalid date: ${value}`, {
|
|
99
|
+
code: "WINDOW_INVALID",
|
|
100
|
+
hint: "Use a real UTC date in YYYY-MM-DD form (e.g. 2026-10-25).",
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
return end ? ms + DAY_MS : ms;
|
|
104
|
+
}
|
|
105
|
+
const ms = Date.parse(value);
|
|
106
|
+
if (!Number.isFinite(ms)) {
|
|
107
|
+
throw new UsageError(`Invalid date: ${value}`, {
|
|
108
|
+
code: "WINDOW_INVALID",
|
|
109
|
+
hint: "Use YYYY-MM-DD (UTC day) or an ISO timestamp such as 2026-10-25T00:00:00.000Z.",
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return ms;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function parseLimit(value) {
|
|
116
|
+
if (value == null) return DEFAULT_LIMIT;
|
|
117
|
+
const n = Number(value);
|
|
118
|
+
if (!Number.isFinite(n)) {
|
|
119
|
+
throw new UsageError(`Invalid --limit: ${value}`, {
|
|
120
|
+
code: "LIMIT_INVALID",
|
|
121
|
+
hint: `Use an integer between 1 and ${MAX_LIMIT}.`,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
return Math.min(Math.max(Math.trunc(n), 1), MAX_LIMIT);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function parseDays(value) {
|
|
128
|
+
if (value == null) return DEFAULT_DAYS;
|
|
129
|
+
const n = Number(value);
|
|
130
|
+
if (!Number.isFinite(n) || Math.trunc(n) !== n || n < 1 || n > MAX_RANGE_DAYS) {
|
|
131
|
+
throw new UsageError(`Invalid --days: ${value}`, {
|
|
132
|
+
code: "WINDOW_INVALID",
|
|
133
|
+
hint: `Use an integer between 1 and ${MAX_RANGE_DAYS}.`,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
return n;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Default window is the last N days ending NOW (`to` exclusive), so the
|
|
140
|
+
// current partial hour is included. The 366-day ceiling is applied locally too
|
|
141
|
+
// — the server would clamp it anyway, and reading keys outside it locally
|
|
142
|
+
// would report false `server_only` rows.
|
|
143
|
+
function resolveWindow(opts, nowMs) {
|
|
144
|
+
const explicit = opts.from != null || opts.to != null;
|
|
145
|
+
const fromMs = explicit && opts.from != null ? parseBoundary(opts.from) : null;
|
|
146
|
+
const toMs = explicit && opts.to != null ? parseBoundary(opts.to, { end: true }) : nowMs;
|
|
147
|
+
const effectiveFrom = fromMs == null ? toMs - parseDays(opts.days) * DAY_MS : fromMs;
|
|
148
|
+
|
|
149
|
+
if (effectiveFrom > toMs) {
|
|
150
|
+
throw new UsageError("--from must be before --to", {
|
|
151
|
+
code: "WINDOW_INVALID",
|
|
152
|
+
hint: "Check both dates: `--from` is inclusive, `--to` is the exclusive end.",
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
const clampedFrom = Math.max(effectiveFrom, toMs - MAX_RANGE_DAYS * DAY_MS);
|
|
156
|
+
return { fromMs: clampedFrom, toMs, clamped: clampedFrom !== effectiveFrom };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function bucketKey(hourMs, source, model) {
|
|
160
|
+
// JSON over a delimiter: model names may contain `|`.
|
|
161
|
+
return JSON.stringify([new Date(hourMs).toISOString(), source, model]);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Local baseline: every usage row in queue.jsonl, normalized/sanitized exactly
|
|
165
|
+
// like the upload path, keyed and filtered on the normalized hour. Control
|
|
166
|
+
// records (kind-bearing account session states / watermarks) are not buckets.
|
|
167
|
+
async function readLocalBaseline({ queuePath, fromMs, toMs }) {
|
|
168
|
+
let handle;
|
|
169
|
+
try {
|
|
170
|
+
handle = await fs.open(queuePath, "r");
|
|
171
|
+
} catch (error) {
|
|
172
|
+
if (error?.code === "ENOENT") return new Map();
|
|
173
|
+
throw error;
|
|
174
|
+
}
|
|
175
|
+
const buckets = new Map();
|
|
176
|
+
try {
|
|
177
|
+
const before = await handle.stat();
|
|
178
|
+
for await (const record of iterateQueueLines(handle, 0, before.size)) {
|
|
179
|
+
// A producer may be mid-write; an incomplete tail is not yet queued data.
|
|
180
|
+
if (!record.complete) break;
|
|
181
|
+
const line = record.bytes.toString("utf8");
|
|
182
|
+
if (!line.trim()) continue;
|
|
183
|
+
let bucket;
|
|
184
|
+
try {
|
|
185
|
+
bucket = JSON.parse(line);
|
|
186
|
+
} catch (_error) {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
if (!bucket || typeof bucket !== "object" || Array.isArray(bucket)) continue;
|
|
190
|
+
if (Object.hasOwn(bucket, "kind")) continue;
|
|
191
|
+
const hourMs = typeof bucket.hour_start === "string" ? Date.parse(bucket.hour_start) : NaN;
|
|
192
|
+
if (!Number.isFinite(hourMs) || hourMs < fromMs || hourMs >= toMs) continue;
|
|
193
|
+
const source = (typeof bucket.source === "string" ? bucket.source.trim().toLowerCase() : "") || "codex";
|
|
194
|
+
const model = (typeof bucket.model === "string" ? bucket.model.trim() : "") || "unknown";
|
|
195
|
+
const row = normalizeQueueRow({
|
|
196
|
+
...bucket,
|
|
197
|
+
source,
|
|
198
|
+
model,
|
|
199
|
+
hour_start: new Date(hourMs).toISOString(),
|
|
200
|
+
});
|
|
201
|
+
const sanitized = sanitizeBucketForUpload(row);
|
|
202
|
+
if (!sanitized) continue;
|
|
203
|
+
buckets.set(bucketKey(hourMs, source, model), sanitized);
|
|
204
|
+
}
|
|
205
|
+
} finally {
|
|
206
|
+
await handle.close();
|
|
207
|
+
}
|
|
208
|
+
return buckets;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
async function fetchPage({ url, deviceToken, anonKey, timeoutMs, fetchImpl }) {
|
|
212
|
+
const headers = { Accept: "application/json", Authorization: `Bearer ${deviceToken}` };
|
|
213
|
+
if (anonKey) headers.apikey = anonKey;
|
|
214
|
+
const res = await fetchFunctionResponse(
|
|
215
|
+
url,
|
|
216
|
+
{ method: "GET", headers, ...(timeoutMs > 0 ? { signal: AbortSignal.timeout(timeoutMs) } : {}) },
|
|
217
|
+
fetchImpl,
|
|
218
|
+
);
|
|
219
|
+
const text = await res.text();
|
|
220
|
+
let data = {};
|
|
221
|
+
try {
|
|
222
|
+
data = JSON.parse(text);
|
|
223
|
+
} catch (_error) {
|
|
224
|
+
data = {};
|
|
225
|
+
}
|
|
226
|
+
if (res.status === 401) {
|
|
227
|
+
throw new UsageError("The backend rejected this device token.", {
|
|
228
|
+
code: "CLOUD_DEVICE_TOKEN_REJECTED",
|
|
229
|
+
hint: "Re-pair this machine with `tokenusage device-login`, then retry.",
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
if (res.status === 403) {
|
|
233
|
+
throw new UsageError("This device is revoked, so its rows are not readable.", {
|
|
234
|
+
code: "CLOUD_DEVICE_REVOKED",
|
|
235
|
+
hint: "Re-pair this machine with `tokenusage device-login`, then retry.",
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
if (!res.ok) {
|
|
239
|
+
throw new Error(`reconcile failed (HTTP ${res.status}): ${text.slice(0, 300)}`);
|
|
240
|
+
}
|
|
241
|
+
return data;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async function fetchServerBuckets({ baseUrl, anonKey, deviceToken, from, to, limit, timeoutMs, fetchImpl }) {
|
|
245
|
+
const endpoint = functionUrlFor(baseUrl, RECONCILE_SLUG);
|
|
246
|
+
const buckets = new Map();
|
|
247
|
+
let deviceId = null;
|
|
248
|
+
let pages = 0;
|
|
249
|
+
let offset = 0;
|
|
250
|
+
let pageSize = limit;
|
|
251
|
+
for (; pages < MAX_PAGES; pages++) {
|
|
252
|
+
const url = new URL(endpoint);
|
|
253
|
+
url.searchParams.set("from", from);
|
|
254
|
+
url.searchParams.set("to", to);
|
|
255
|
+
url.searchParams.set("limit", String(limit));
|
|
256
|
+
url.searchParams.set("offset", String(offset));
|
|
257
|
+
const data = await fetchPage({ url: url.toString(), deviceToken, anonKey, timeoutMs, fetchImpl });
|
|
258
|
+
const rows = Array.isArray(data?.buckets) ? data.buckets : [];
|
|
259
|
+
// The server echoes its clamped limit; trust it for the end-of-pages test.
|
|
260
|
+
pageSize = Number.isFinite(Number(data?.limit)) ? Math.max(1, Number(data.limit)) : limit;
|
|
261
|
+
deviceId = typeof data?.device_id === "string" ? data.device_id : deviceId;
|
|
262
|
+
for (const row of rows) {
|
|
263
|
+
const hourMs = Date.parse(String(row?.hour_start));
|
|
264
|
+
if (!Number.isFinite(hourMs)) continue;
|
|
265
|
+
const source = String(row.source);
|
|
266
|
+
const model = String(row.model);
|
|
267
|
+
buckets.set(bucketKey(hourMs, source, model), row);
|
|
268
|
+
}
|
|
269
|
+
if (rows.length < pageSize) break;
|
|
270
|
+
offset += rows.length;
|
|
271
|
+
}
|
|
272
|
+
if (pages >= MAX_PAGES) {
|
|
273
|
+
throw new UsageError("Reconciliation window returned too many pages.", {
|
|
274
|
+
code: "WINDOW_TOO_LARGE",
|
|
275
|
+
hint: "Narrow the window with --from/--to or --days, or raise --limit.",
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
return { buckets, deviceId, pages: pages + 1 };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function valuesDiffer(local, server, column) {
|
|
282
|
+
if (column === "billable_total_tokens") {
|
|
283
|
+
if (local == null && server == null) return false;
|
|
284
|
+
if (local == null || server == null) return true;
|
|
285
|
+
return Number(local) !== Number(server);
|
|
286
|
+
}
|
|
287
|
+
if (column === "credits") {
|
|
288
|
+
return round6(Number(local ?? 0)) !== round6(Number(server ?? 0));
|
|
289
|
+
}
|
|
290
|
+
return Number(local ?? 0) !== Number(server ?? 0);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function round6(value) {
|
|
294
|
+
return Math.round((Number.isFinite(value) ? value : 0) * 1e6) / 1e6;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const COMPARED_COLUMNS = [...TOKEN_COLUMNS, "billable_total_tokens", "conversations", "credits"];
|
|
298
|
+
|
|
299
|
+
// Queue rows use the `conversation_count` wire alias (§2.4) while the hourly
|
|
300
|
+
// table stores `conversations`; read either name so a row that only carries the
|
|
301
|
+
// legacy one is not compared as 0.
|
|
302
|
+
function columnValue(row, column) {
|
|
303
|
+
if (column === "conversations" && row?.conversations === undefined) {
|
|
304
|
+
return row?.conversation_count;
|
|
305
|
+
}
|
|
306
|
+
return row?.[column];
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function describeRow(row) {
|
|
310
|
+
const out = {};
|
|
311
|
+
for (const column of COMPARED_COLUMNS) {
|
|
312
|
+
const value = columnValue(row, column);
|
|
313
|
+
if (column === "billable_total_tokens" && value == null) out[column] = null;
|
|
314
|
+
else if (column === "credits") out[column] = round6(Number(value ?? 0));
|
|
315
|
+
else out[column] = Number(value ?? 0);
|
|
316
|
+
}
|
|
317
|
+
return out;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function compareBuckets({ local, server }) {
|
|
321
|
+
const missingOnServer = [];
|
|
322
|
+
const mismatches = [];
|
|
323
|
+
const serverOnly = [];
|
|
324
|
+
|
|
325
|
+
for (const [key, localRow] of local) {
|
|
326
|
+
const hourStart = new Date(Date.parse(localRow.hour_start)).toISOString();
|
|
327
|
+
const base = { hour_start: hourStart, source: localRow.source, model: localRow.model };
|
|
328
|
+
const serverRow = server.get(key);
|
|
329
|
+
if (!serverRow) {
|
|
330
|
+
missingOnServer.push({ ...base, local: describeRow(localRow), server: null });
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
const fields = COMPARED_COLUMNS.filter((column) =>
|
|
334
|
+
valuesDiffer(columnValue(localRow, column), serverRow?.[column], column),
|
|
335
|
+
);
|
|
336
|
+
if (fields.length > 0) {
|
|
337
|
+
mismatches.push({
|
|
338
|
+
...base,
|
|
339
|
+
local: describeRow(localRow),
|
|
340
|
+
server: describeRow(serverRow),
|
|
341
|
+
fields,
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
for (const [key, serverRow] of server) {
|
|
347
|
+
if (local.has(key)) continue;
|
|
348
|
+
serverOnly.push({
|
|
349
|
+
hour_start: new Date(Date.parse(String(serverRow.hour_start))).toISOString(),
|
|
350
|
+
source: String(serverRow.source),
|
|
351
|
+
model: String(serverRow.model),
|
|
352
|
+
local: null,
|
|
353
|
+
server: describeRow(serverRow),
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const sortEntries = (entries) =>
|
|
358
|
+
entries.sort(
|
|
359
|
+
(a, b) =>
|
|
360
|
+
a.hour_start.localeCompare(b.hour_start) ||
|
|
361
|
+
a.source.localeCompare(b.source) ||
|
|
362
|
+
a.model.localeCompare(b.model),
|
|
363
|
+
);
|
|
364
|
+
|
|
365
|
+
return {
|
|
366
|
+
missing_on_server: sortEntries(missingOnServer),
|
|
367
|
+
server_only: sortEntries(serverOnly),
|
|
368
|
+
mismatch: sortEntries(mismatches),
|
|
369
|
+
counts: {
|
|
370
|
+
missing_on_server: missingOnServer.length,
|
|
371
|
+
server_only: serverOnly.length,
|
|
372
|
+
mismatch: mismatches.length,
|
|
373
|
+
},
|
|
374
|
+
drifted: missingOnServer.length + serverOnly.length + mismatches.length,
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function describeEntry(entry, isMismatch = false) {
|
|
379
|
+
const head = `${entry.hour_start} ${entry.source} ${entry.model}`;
|
|
380
|
+
if (isMismatch) {
|
|
381
|
+
const parts = entry.fields.map((field) =>
|
|
382
|
+
`${field} local ${entry.local[field] === null ? "null" : entry.local[field]} → cloud ${entry.server[field] === null ? "null" : entry.server[field]}`,
|
|
383
|
+
);
|
|
384
|
+
return ` ${head} (${parts.join(", ")})`;
|
|
385
|
+
}
|
|
386
|
+
const row = entry.local || entry.server;
|
|
387
|
+
return ` ${head} (total_tokens=${row ? row.total_tokens : 0})`;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function renderText(summary) {
|
|
391
|
+
const lines = [
|
|
392
|
+
`Reconcile window (UTC, hour_start in [from, to)):`,
|
|
393
|
+
` from ${summary.from}`,
|
|
394
|
+
` to ${summary.to}`,
|
|
395
|
+
summary.clamped ? " (from clamped to the server's 366-day ceiling)" : null,
|
|
396
|
+
`Local baseline: ${summary.local_buckets.toLocaleString("en-US")} bucket(s) from queue.jsonl (last-wins)`,
|
|
397
|
+
`Cloud device: ${summary.device_id || "unknown"} — ${summary.cloud_buckets.toLocaleString("en-US")} row(s) in ${summary.pages} page(s)`,
|
|
398
|
+
].filter(Boolean);
|
|
399
|
+
|
|
400
|
+
if (summary.drifted === 0) {
|
|
401
|
+
lines.push("Result: OK — every local bucket matches the cloud row.");
|
|
402
|
+
return `${lines.join("\n")}\n`;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
lines.push(
|
|
406
|
+
`Result: ${summary.drifted.toLocaleString("en-US")} drifted key(s) — ` +
|
|
407
|
+
`${summary.counts.missing_on_server} missing on server, ` +
|
|
408
|
+
`${summary.counts.server_only} server-only, ${summary.counts.mismatch} mismatched`,
|
|
409
|
+
);
|
|
410
|
+
const sections = [
|
|
411
|
+
["missing_on_server", summary.drift.missing_on_server, false],
|
|
412
|
+
["server_only", summary.drift.server_only, false],
|
|
413
|
+
["mismatch", summary.drift.mismatch, true],
|
|
414
|
+
];
|
|
415
|
+
for (const [label, entries, isMismatch] of sections) {
|
|
416
|
+
if (entries.length === 0) continue;
|
|
417
|
+
lines.push(`${label}:`);
|
|
418
|
+
for (const entry of entries.slice(0, TEXT_DRIFT_LIMIT)) {
|
|
419
|
+
lines.push(describeEntry(entry, isMismatch));
|
|
420
|
+
}
|
|
421
|
+
if (entries.length > TEXT_DRIFT_LIMIT) {
|
|
422
|
+
lines.push(` … and ${entries.length - TEXT_DRIFT_LIMIT} more (use --json for the full list)`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
lines.push(
|
|
426
|
+
"Hint: `tokenusage upload` pushes pending local buckets; `server_only` keys mean the",
|
|
427
|
+
"cloud holds rows this queue no longer contains. This command never writes.",
|
|
428
|
+
);
|
|
429
|
+
return `${lines.join("\n")}\n`;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
async function cmdReconcile(argv = [], options = {}) {
|
|
433
|
+
const opts = parseArgs(argv);
|
|
434
|
+
const home = options.home || os.homedir();
|
|
435
|
+
const nowMs = typeof options.now === "function" ? Number(options.now()) : Number(options.now ?? Date.now());
|
|
436
|
+
const fetchImpl = options.fetchImpl || fetch;
|
|
437
|
+
const { trackerDir } = await resolveTrackerPaths({ home });
|
|
438
|
+
const queuePath = path.join(trackerDir, "queue.jsonl");
|
|
439
|
+
|
|
440
|
+
const window = resolveWindow(opts, nowMs);
|
|
441
|
+
const from = new Date(window.fromMs).toISOString();
|
|
442
|
+
// Explicit `--to` is already an exclusive instant; a default `to` is "now".
|
|
443
|
+
const to = new Date(window.toMs).toISOString();
|
|
444
|
+
const limit = parseLimit(opts.limit);
|
|
445
|
+
|
|
446
|
+
const config = (await readJson(path.join(trackerDir, "config.json"))) || {};
|
|
447
|
+
const runtime = assertInsforgeRuntime(resolveRuntimeConfig({ config, env: process.env }));
|
|
448
|
+
if (!runtime.deviceToken) {
|
|
449
|
+
throw new UsageError("This machine is not paired with a dashboard yet.", {
|
|
450
|
+
code: "NO_DEVICE_TOKEN",
|
|
451
|
+
hint: "Run `tokenusage device-login` (or `tokenusage init`) first.",
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const local = await readLocalBaseline({ queuePath, fromMs: window.fromMs, toMs: window.toMs });
|
|
456
|
+
const { buckets: server, deviceId, pages } = await fetchServerBuckets({
|
|
457
|
+
baseUrl: runtime.baseUrl,
|
|
458
|
+
anonKey: runtime.anonKey,
|
|
459
|
+
deviceToken: runtime.deviceToken,
|
|
460
|
+
from,
|
|
461
|
+
to,
|
|
462
|
+
limit,
|
|
463
|
+
timeoutMs: runtime.httpTimeoutMs,
|
|
464
|
+
fetchImpl,
|
|
465
|
+
});
|
|
466
|
+
const compared = compareBuckets({ local, server });
|
|
467
|
+
|
|
468
|
+
const summary = {
|
|
469
|
+
status: compared.drifted === 0 ? "ok" : "drift",
|
|
470
|
+
from,
|
|
471
|
+
to,
|
|
472
|
+
clamped: window.clamped,
|
|
473
|
+
device_id: deviceId,
|
|
474
|
+
local_buckets: local.size,
|
|
475
|
+
cloud_buckets: server.size,
|
|
476
|
+
pages,
|
|
477
|
+
counts: compared.counts,
|
|
478
|
+
drifted: compared.drifted,
|
|
479
|
+
drift: {
|
|
480
|
+
missing_on_server: compared.missing_on_server,
|
|
481
|
+
server_only: compared.server_only,
|
|
482
|
+
mismatch: compared.mismatch,
|
|
483
|
+
},
|
|
484
|
+
};
|
|
485
|
+
|
|
486
|
+
// Output seam: tests render without hijacking process.stdout (which would
|
|
487
|
+
// swallow the node test runner's own protocol writes).
|
|
488
|
+
const write = options.output || ((text) => process.stdout.write(text));
|
|
489
|
+
if (opts.json) write(`${JSON.stringify(summary, null, 2)}\n`);
|
|
490
|
+
else write(renderText(summary));
|
|
491
|
+
|
|
492
|
+
if (compared.drifted > 0) process.exitCode = 1;
|
|
493
|
+
return summary;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
module.exports = { cmdReconcile, parseArgs, resolveWindow, readLocalBaseline, compareBuckets, RECONCILE_SLUG };
|
package/src/commands/sessions.js
CHANGED
|
@@ -1,29 +1,72 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
// `tokenusage sessions` — metadata-only efficiency analytics export.
|
|
4
|
+
//
|
|
5
|
+
// Strict parsing (was: unknown flags silently ignored, an invalid --format
|
|
6
|
+
// silently produced JSON, and --out used a non-atomic write). Analytics
|
|
7
|
+
// exports are easy to script, so a typo must fail instead of quietly
|
|
8
|
+
// exporting the wrong shape to the wrong file.
|
|
9
|
+
|
|
10
|
+
const { writeFileAtomic } = require("../lib/fs");
|
|
11
|
+
const { UsageError, unknownOptionError } = require("../lib/cli-errors");
|
|
4
12
|
const { buildSessionAnalytics, summarizeSessions, sessionsToCsv } = require("../lib/session-analytics");
|
|
5
13
|
const { buildGitOutcomes } = require("../lib/git-outcomes");
|
|
6
14
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
15
|
+
const FORMATS = new Set(["json", "csv"]);
|
|
16
|
+
const VALUE_FLAGS = new Set(["--format", "--out", "--from", "--to"]);
|
|
17
|
+
const BOOLEAN_FLAGS = new Set(["--csv", "--refresh", "--no-git"]);
|
|
18
|
+
|
|
19
|
+
function parseArgs(args = []) {
|
|
20
|
+
const out = { format: null, out: null, from: null, to: null, refresh: false, git: true };
|
|
21
|
+
for (let i = 0; i < args.length; i++) {
|
|
22
|
+
const a = args[i];
|
|
23
|
+
if (VALUE_FLAGS.has(a)) {
|
|
24
|
+
const value = args[i + 1];
|
|
25
|
+
if (value == null || value === "" || value.startsWith("--")) {
|
|
26
|
+
throw new UsageError(`${a} requires a value`, {
|
|
27
|
+
code: "MISSING_VALUE",
|
|
28
|
+
hint: "Run `tokenusage sessions --help` for supported options.",
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
i += 1;
|
|
32
|
+
if (a === "--format") out.format = value.trim().toLowerCase();
|
|
33
|
+
else if (a === "--out") out.out = value;
|
|
34
|
+
else if (a === "--from") out.from = value;
|
|
35
|
+
else out.to = value;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
if (BOOLEAN_FLAGS.has(a)) {
|
|
39
|
+
if (a === "--csv") out.format = "csv";
|
|
40
|
+
else if (a === "--refresh") out.refresh = true;
|
|
41
|
+
else out.git = false;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
throw unknownOptionError(a, "sessions");
|
|
45
|
+
}
|
|
46
|
+
if (out.format == null) out.format = "json";
|
|
47
|
+
if (!FORMATS.has(out.format)) {
|
|
48
|
+
throw new UsageError(`Unknown --format: ${out.format}`, {
|
|
49
|
+
code: "FORMAT_UNKNOWN",
|
|
50
|
+
hint: "Use `--format json` or `--format csv`.",
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
10
54
|
}
|
|
11
55
|
|
|
12
56
|
async function cmdSessions(args = []) {
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const
|
|
18
|
-
const content = format === "csv"
|
|
57
|
+
const opts = parseArgs(args);
|
|
58
|
+
const sessions = await buildSessionAnalytics({ force: opts.refresh });
|
|
59
|
+
if (opts.git) await buildGitOutcomes(sessions);
|
|
60
|
+
const summary = summarizeSessions(sessions, { from: opts.from || "", to: opts.to || "" });
|
|
61
|
+
const content = opts.format === "csv"
|
|
19
62
|
? sessionsToCsv(summary.sessions)
|
|
20
63
|
: `${JSON.stringify(summary, null, 2)}\n`;
|
|
21
|
-
if (out) {
|
|
22
|
-
|
|
23
|
-
process.stdout.write(`${out}\n`);
|
|
64
|
+
if (opts.out) {
|
|
65
|
+
await writeFileAtomic(opts.out, content, { mode: 0o600 });
|
|
66
|
+
process.stdout.write(`${opts.out}\n`);
|
|
24
67
|
} else {
|
|
25
68
|
process.stdout.write(content);
|
|
26
69
|
}
|
|
27
70
|
}
|
|
28
71
|
|
|
29
|
-
module.exports = { cmdSessions };
|
|
72
|
+
module.exports = { cmdSessions, parseArgs };
|