tokenusage-cli 0.5.5 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +7 -6
- package/package.json +1 -1
- package/src/cli.js +32 -5
- package/src/commands/device-login.js +16 -17
- package/src/commands/diagnostics.js +16 -16
- package/src/commands/doctor.js +14 -11
- package/src/commands/pause.js +34 -3
- package/src/commands/project-profiles.js +5 -2
- package/src/commands/sessions.js +18 -30
- package/src/commands/status.js +22 -16
- package/src/commands/sync.js +117 -3614
- package/src/commands/uninstall.js +10 -9
- package/src/commands/upload.js +18 -12
- package/src/lib/ark-agent-plan-limits.js +4 -13
- package/src/lib/ark-coding-plan-limits.js +4 -13
- package/src/lib/cli-flags.js +109 -0
- package/src/lib/commandcode-limits.js +11 -38
- package/src/lib/doctor.js +91 -1
- package/src/lib/fs.js +10 -0
- package/src/lib/grok-limits.js +5 -18
- package/src/lib/limits-core.js +64 -0
- package/src/lib/qoder-limits.js +4 -13
- package/src/lib/queue-backup.js +24 -0
- package/src/lib/queue-drain.js +388 -0
- package/src/lib/runtime-config.js +26 -7
- package/src/lib/usage-limits.js +22 -29
- package/src/lib/workbuddy-ipc.js +191 -0
- package/src/lib/workbuddy-limits.js +656 -0
- package/src/lib/zcode-limits.js +5 -18
- package/src/repairs/claude.js +442 -0
- package/src/repairs/codebuddy.js +246 -0
- package/src/repairs/codex.js +1023 -0
- package/src/repairs/droid.js +266 -0
- package/src/repairs/grok.js +352 -0
- package/src/repairs/legacy-migrations.js +135 -0
- package/src/repairs/mimo.js +161 -0
- package/src/repairs/workbuddy.js +209 -0
- package/src/repairs/zcode.js +647 -0
|
@@ -0,0 +1,647 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Zcode repair family, split out of commands/sync.js. These are one-shot
|
|
4
|
+
// history corrections: two queue+state rebuilds keyed by migration id
|
|
5
|
+
// (native-usage adoption, inclusive-token semantics) and the prune-window
|
|
6
|
+
// restore that re-appends pre-window peaks from queue backups. They mutate
|
|
7
|
+
// cursors/queue files in place and are idempotent via cursors.migrations.
|
|
8
|
+
|
|
9
|
+
const path = require("node:path");
|
|
10
|
+
const fs = require("node:fs/promises");
|
|
11
|
+
const fssync = require("node:fs");
|
|
12
|
+
|
|
13
|
+
const { ensureDir } = require("../lib/fs");
|
|
14
|
+
const { readSqliteJsonRows } = require("../lib/sqlite-reader");
|
|
15
|
+
const { backupExistingFile } = require("../lib/queue-backup");
|
|
16
|
+
const {
|
|
17
|
+
readZcodeDbMessages,
|
|
18
|
+
hasZcodeNativeUsageSchema,
|
|
19
|
+
hasZcodeCursorState,
|
|
20
|
+
coerceEpochMs,
|
|
21
|
+
toUtcHalfHourStart,
|
|
22
|
+
bucketKey,
|
|
23
|
+
} = require("../lib/rollout");
|
|
24
|
+
|
|
25
|
+
const ZCODE_NATIVE_USAGE_REPAIR_KEY = "zcodeNativeUsageRepair_2026_08";
|
|
26
|
+
const ZCODE_INCLUSIVE_TOKEN_REPAIR_KEY = "zcodeInclusiveTokenRepair_2026_09";
|
|
27
|
+
const ZCODE_PRUNE_WINDOW_RESTORE_KEY = "zcodePruneWindowRestore_2026_10";
|
|
28
|
+
async function resetUploadOffsetForZcodeRepair(queueStatePath, note) {
|
|
29
|
+
if (typeof queueStatePath !== "string" || !queueStatePath) return false;
|
|
30
|
+
let state = {};
|
|
31
|
+
try {
|
|
32
|
+
state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
|
|
33
|
+
} catch (_error) {
|
|
34
|
+
state = {};
|
|
35
|
+
}
|
|
36
|
+
state.offset = 0;
|
|
37
|
+
state.updatedAt = new Date().toISOString();
|
|
38
|
+
state.note = note;
|
|
39
|
+
await ensureDir(path.dirname(queueStatePath));
|
|
40
|
+
await fs.writeFile(queueStatePath, JSON.stringify(state, null, 2) + "\n", "utf8");
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function zcodeHourStartMs(hourStart) {
|
|
45
|
+
if (typeof hourStart !== "string" || !hourStart.trim()) return null;
|
|
46
|
+
const ms = Date.parse(hourStart);
|
|
47
|
+
return Number.isFinite(ms) ? ms : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// One restored queue row from a backup peak (repairZcodePruneWindowRestore).
|
|
51
|
+
// peak.totals comes from zcodeRowTotals, which fills the billable/cost columns
|
|
52
|
+
// older backup rows are missing — the restored row must match the live queue
|
|
53
|
+
// schema exactly (audit fix: 恢复行缺 billable/cost 字段).
|
|
54
|
+
function buildZcodeRestorationRow(peak) {
|
|
55
|
+
return {
|
|
56
|
+
source: "zcode",
|
|
57
|
+
model: peak.model,
|
|
58
|
+
hour_start: peak.hour_start,
|
|
59
|
+
...peak.totals,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Main bucket keys are `source|model|hour_start` — hour_start (an ISO string)
|
|
64
|
+
// never contains `|`, so it is everything after the LAST separator. Splitting
|
|
65
|
+
// on every `|` mis-parses models whose name contains one (audit fix).
|
|
66
|
+
function zcodeHourFromMainBucketKey(key) {
|
|
67
|
+
if (typeof key !== "string" || !key.startsWith("zcode|")) return null;
|
|
68
|
+
const last = key.lastIndexOf("|");
|
|
69
|
+
if (last <= "zcode|".length - 1) return null;
|
|
70
|
+
const hour = key.slice(last + 1);
|
|
71
|
+
return hour || null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function zcodeHourFromProjectBucketKey(key) {
|
|
75
|
+
if (typeof key !== "string") return null;
|
|
76
|
+
const marker = "|zcode|";
|
|
77
|
+
const idx = key.indexOf(marker);
|
|
78
|
+
if (idx < 0) return null;
|
|
79
|
+
const hour = key.slice(idx + marker.length);
|
|
80
|
+
return hour || null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function zcodeHourFromGroupQueuedKey(key) {
|
|
84
|
+
if (typeof key !== "string" || !key.startsWith("zcode|")) return null;
|
|
85
|
+
return key.slice("zcode|".length) || null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function zcodeHourIsPreWindow(hourStart, preserveBeforeMs) {
|
|
89
|
+
if (!Number.isFinite(preserveBeforeMs)) return false;
|
|
90
|
+
const hourMs = zcodeHourStartMs(hourStart);
|
|
91
|
+
return hourMs != null && hourMs < preserveBeforeMs;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Per-install rebuild floor. Prefer native model_usage MIN(started_at) when the
|
|
95
|
+
// native schema exists — an ancient legacy message does not prove continuous
|
|
96
|
+
// coverage through pruned native gaps (#828).
|
|
97
|
+
function resolveZcodeDbRebuildFloorMs(dbPath) {
|
|
98
|
+
if (!dbPath || !fssync.existsSync(dbPath)) return null;
|
|
99
|
+
if (hasZcodeNativeUsageSchema(dbPath)) {
|
|
100
|
+
try {
|
|
101
|
+
const rows = readSqliteJsonRows(
|
|
102
|
+
dbPath,
|
|
103
|
+
`SELECT MIN(started_at) AS floor
|
|
104
|
+
FROM model_usage
|
|
105
|
+
WHERE status = 'completed'
|
|
106
|
+
AND trim(model_id) != ''
|
|
107
|
+
AND (
|
|
108
|
+
input_tokens > 0 OR output_tokens > 0 OR reasoning_tokens > 0 OR
|
|
109
|
+
cache_creation_input_tokens > 0 OR cache_read_input_tokens > 0
|
|
110
|
+
)`,
|
|
111
|
+
{ label: "ZCode", timeout: 10_000, maxBuffer: 1024 * 1024, readOnly: true },
|
|
112
|
+
);
|
|
113
|
+
const floor = coerceEpochMs(Array.isArray(rows) ? rows[0]?.floor : null);
|
|
114
|
+
return floor > 0 ? floor : null;
|
|
115
|
+
} catch (_error) {
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
let messages;
|
|
120
|
+
try {
|
|
121
|
+
messages = readZcodeDbMessages(dbPath, { readOnly: true });
|
|
122
|
+
} catch (_error) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
let floor = Number.POSITIVE_INFINITY;
|
|
126
|
+
for (const message of Array.isArray(messages) ? messages : []) {
|
|
127
|
+
const ms = coerceEpochMs(
|
|
128
|
+
message?.timeUpdated ?? message?.data?.time?.completed ?? message?.data?.time?.created,
|
|
129
|
+
);
|
|
130
|
+
if (ms > 0) floor = Math.min(floor, ms);
|
|
131
|
+
}
|
|
132
|
+
return Number.isFinite(floor) ? floor : null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Shared wipe window across installs: the LATEST per-install floor (most
|
|
136
|
+
// conservative). Any listed install that exists but cannot report a floor
|
|
137
|
+
// defers the wipe — silently taking min() would erase another install's history.
|
|
138
|
+
function resolveZcodeRebuildFloorMs(dbPaths = []) {
|
|
139
|
+
const paths = [...new Set((Array.isArray(dbPaths) ? dbPaths : []).filter(Boolean))];
|
|
140
|
+
if (paths.length === 0) return null;
|
|
141
|
+
let latestFloor = 0;
|
|
142
|
+
for (const dbPath of paths) {
|
|
143
|
+
if (!fssync.existsSync(dbPath)) continue;
|
|
144
|
+
const floor = resolveZcodeDbRebuildFloorMs(dbPath);
|
|
145
|
+
if (!Number.isFinite(floor)) return null;
|
|
146
|
+
latestFloor = Math.max(latestFloor, floor);
|
|
147
|
+
}
|
|
148
|
+
return latestFloor > 0 ? latestFloor : null;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Wipe starts at the half-hour bucket that contains the first live message so
|
|
152
|
+
// that bucket is rebuilt (not preserved) and cannot be double-counted after
|
|
153
|
+
// cursors.zcode is cleared.
|
|
154
|
+
function zcodePreserveBeforeMsFromFloor(floorMs) {
|
|
155
|
+
if (!Number.isFinite(floorMs)) return null;
|
|
156
|
+
const iso = toUtcHalfHourStart(floorMs);
|
|
157
|
+
const ms = iso ? Date.parse(iso) : floorMs;
|
|
158
|
+
return Number.isFinite(ms) ? ms : null;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function hasZcodeCursorHistory(cursors) {
|
|
162
|
+
return hasZcodeCursorState(cursors);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function pruneZcodeMessageIndexForRebuild(messages, preserveBeforeMs) {
|
|
166
|
+
if (!messages || typeof messages !== "object") return 0;
|
|
167
|
+
let removed = 0;
|
|
168
|
+
for (const key of Object.keys(messages)) {
|
|
169
|
+
const entry = messages[key];
|
|
170
|
+
const attribution = typeof entry?.attribution === "string"
|
|
171
|
+
? entry.attribution.split(" ")[0]
|
|
172
|
+
: entry?.attribution?.bucketStart;
|
|
173
|
+
const bucketStart = typeof attribution === "string" ? attribution : "";
|
|
174
|
+
// Keep unattributed entries: older cursors omit attribution, and deleting
|
|
175
|
+
// them re-adds still-present pre-window legacy messages onto preserved
|
|
176
|
+
// buckets (5000→10000). Prefer a possible in-window under-correction over
|
|
177
|
+
// permanent double-count (#828).
|
|
178
|
+
if (!bucketStart) continue;
|
|
179
|
+
if (zcodeHourIsPreWindow(bucketStart, preserveBeforeMs)) continue;
|
|
180
|
+
delete messages[key];
|
|
181
|
+
removed += 1;
|
|
182
|
+
}
|
|
183
|
+
return removed;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function pruneZcodeMessageCursorForRebuild(cursors, preserveBeforeMs) {
|
|
187
|
+
if (!cursors || typeof cursors !== "object") return;
|
|
188
|
+
if (!Number.isFinite(preserveBeforeMs)) {
|
|
189
|
+
delete cursors.zcode;
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
const root = cursors.zcode;
|
|
193
|
+
if (!root || typeof root !== "object") {
|
|
194
|
+
delete cursors.zcode;
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
pruneZcodeMessageIndexForRebuild(root.messages, preserveBeforeMs);
|
|
198
|
+
for (const nestedKey of ["native", "wsl"]) {
|
|
199
|
+
const nested = root[nestedKey];
|
|
200
|
+
if (nested && typeof nested === "object") {
|
|
201
|
+
pruneZcodeMessageIndexForRebuild(nested.messages, preserveBeforeMs);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
const hasMessages = Object.keys(root.messages || {}).length > 0
|
|
205
|
+
|| Object.keys(root.native?.messages || {}).length > 0
|
|
206
|
+
|| Object.keys(root.wsl?.messages || {}).length > 0;
|
|
207
|
+
if (!hasMessages) delete cursors.zcode;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async function queueFileHasZcodeRows(filePath) {
|
|
211
|
+
if (typeof filePath !== "string" || !filePath) return false;
|
|
212
|
+
let raw;
|
|
213
|
+
try {
|
|
214
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
215
|
+
} catch (error) {
|
|
216
|
+
if (error?.code === "ENOENT") return false;
|
|
217
|
+
throw error;
|
|
218
|
+
}
|
|
219
|
+
for (const line of raw.split("\n")) {
|
|
220
|
+
if (!line.trim()) continue;
|
|
221
|
+
try {
|
|
222
|
+
const row = JSON.parse(line);
|
|
223
|
+
if (row?.source === "zcode") return true;
|
|
224
|
+
} catch (_error) {
|
|
225
|
+
// keep scanning
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function isCompletedZcodeMigration(entry) {
|
|
232
|
+
return Boolean(entry) && !(typeof entry === "object" && entry.skipped);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async function dropZcodeQueueRows(filePath, {
|
|
236
|
+
retainRetractions = false,
|
|
237
|
+
preserveBeforeMs = null,
|
|
238
|
+
} = {}) {
|
|
239
|
+
let raw;
|
|
240
|
+
try {
|
|
241
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
242
|
+
} catch (error) {
|
|
243
|
+
if (error?.code === "ENOENT") return { removed: 0, retractions: 0, preserved: 0 };
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
246
|
+
const kept = [];
|
|
247
|
+
const retractions = new Map();
|
|
248
|
+
let removed = 0;
|
|
249
|
+
let preserved = 0;
|
|
250
|
+
for (const line of raw.split("\n")) {
|
|
251
|
+
if (!line.trim()) continue;
|
|
252
|
+
let row;
|
|
253
|
+
try {
|
|
254
|
+
row = JSON.parse(line);
|
|
255
|
+
} catch (_error) {
|
|
256
|
+
kept.push(line);
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
if (row?.source === "zcode") {
|
|
260
|
+
const model = typeof row.model === "string" && row.model.trim()
|
|
261
|
+
? row.model.trim()
|
|
262
|
+
: "unknown";
|
|
263
|
+
const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
|
|
264
|
+
// #828: keep pre-window rows intact — the live DB cannot rebuild them.
|
|
265
|
+
if (zcodeHourIsPreWindow(hourStart, preserveBeforeMs)) {
|
|
266
|
+
kept.push(line);
|
|
267
|
+
preserved += 1;
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
removed += 1;
|
|
271
|
+
if (retainRetractions && hourStart) {
|
|
272
|
+
retractions.set(`${model}|${hourStart}`, {
|
|
273
|
+
source: "zcode",
|
|
274
|
+
model,
|
|
275
|
+
hour_start: hourStart,
|
|
276
|
+
input_tokens: 0,
|
|
277
|
+
cached_input_tokens: 0,
|
|
278
|
+
cache_creation_input_tokens: 0,
|
|
279
|
+
output_tokens: 0,
|
|
280
|
+
reasoning_output_tokens: 0,
|
|
281
|
+
total_tokens: 0,
|
|
282
|
+
conversation_count: 0,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
kept.push(line);
|
|
288
|
+
}
|
|
289
|
+
if (removed === 0) return { removed: 0, retractions: 0, preserved };
|
|
290
|
+
await backupExistingFile(filePath);
|
|
291
|
+
const tmp = `${filePath}.zcoderepair.${process.pid}.${Date.now()}`;
|
|
292
|
+
const rewritten = [
|
|
293
|
+
...kept,
|
|
294
|
+
...[...retractions.values()].map((row) => JSON.stringify(row)),
|
|
295
|
+
];
|
|
296
|
+
await fs.writeFile(tmp, rewritten.length ? rewritten.join("\n") + "\n" : "", "utf8");
|
|
297
|
+
await fs.rename(tmp, filePath);
|
|
298
|
+
return { removed, retractions: retractions.size, preserved };
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
async function repairZcodeUsageMigration({
|
|
302
|
+
migrationKey,
|
|
303
|
+
queueStateNote,
|
|
304
|
+
cursors,
|
|
305
|
+
queuePath,
|
|
306
|
+
queueStatePath,
|
|
307
|
+
projectQueuePath,
|
|
308
|
+
projectQueueStatePath,
|
|
309
|
+
rebuildFloorMs = null,
|
|
310
|
+
} = {}) {
|
|
311
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
312
|
+
const migrations = (cursors.migrations ||= {});
|
|
313
|
+
// Mirror Codex: a prior {skipped:true} must retry once the live ledger can
|
|
314
|
+
// establish a rebuild floor (or the pre-window history is gone).
|
|
315
|
+
const prior = migrations[migrationKey];
|
|
316
|
+
if (isCompletedZcodeMigration(prior)) return false;
|
|
317
|
+
|
|
318
|
+
const preserveBeforeMs = zcodePreserveBeforeMsFromFloor(rebuildFloorMs);
|
|
319
|
+
// Empty / unreadable live ledger + existing history => refuse the wipe. The
|
|
320
|
+
// forward reader still works; only this historical correction defers (#828).
|
|
321
|
+
if (preserveBeforeMs == null) {
|
|
322
|
+
const hasQueueHistory = await queueFileHasZcodeRows(queuePath)
|
|
323
|
+
|| await queueFileHasZcodeRows(projectQueuePath);
|
|
324
|
+
if (hasZcodeCursorHistory(cursors) || hasQueueHistory) {
|
|
325
|
+
migrations[migrationKey] = {
|
|
326
|
+
skipped: true,
|
|
327
|
+
reason: "zcode_rebuild_floor_unavailable",
|
|
328
|
+
at: new Date().toISOString(),
|
|
329
|
+
};
|
|
330
|
+
return false;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const mainRepair = await dropZcodeQueueRows(queuePath, {
|
|
335
|
+
retainRetractions: true,
|
|
336
|
+
preserveBeforeMs,
|
|
337
|
+
});
|
|
338
|
+
const projectRepair = projectQueuePath
|
|
339
|
+
? await dropZcodeQueueRows(projectQueuePath, { preserveBeforeMs })
|
|
340
|
+
: { removed: 0, retractions: 0, preserved: 0 };
|
|
341
|
+
|
|
342
|
+
const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : null;
|
|
343
|
+
if (hourly?.buckets) {
|
|
344
|
+
for (const key of Object.keys(hourly.buckets)) {
|
|
345
|
+
if (!key.startsWith("zcode|")) continue;
|
|
346
|
+
if (zcodeHourIsPreWindow(zcodeHourFromMainBucketKey(key), preserveBeforeMs)) continue;
|
|
347
|
+
delete hourly.buckets[key];
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (hourly?.groupQueued) {
|
|
351
|
+
for (const key of Object.keys(hourly.groupQueued)) {
|
|
352
|
+
if (!key.startsWith("zcode|")) continue;
|
|
353
|
+
if (zcodeHourIsPreWindow(zcodeHourFromGroupQueuedKey(key), preserveBeforeMs)) continue;
|
|
354
|
+
delete hourly.groupQueued[key];
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
const projectHourly = cursors.projectHourly && typeof cursors.projectHourly === "object"
|
|
358
|
+
? cursors.projectHourly
|
|
359
|
+
: null;
|
|
360
|
+
if (projectHourly?.buckets) {
|
|
361
|
+
for (const key of Object.keys(projectHourly.buckets)) {
|
|
362
|
+
if (!key.includes("|zcode|")) continue;
|
|
363
|
+
if (zcodeHourIsPreWindow(zcodeHourFromProjectBucketKey(key), preserveBeforeMs)) continue;
|
|
364
|
+
delete projectHourly.buckets[key];
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
// Keep pre-window message dedup so preserved buckets are not double-counted
|
|
368
|
+
// when legacy rows still exist in the live DB. Clear only in-window (and
|
|
369
|
+
// unattributed) cursor entries so the rescan can rebuild those hours.
|
|
370
|
+
pruneZcodeMessageCursorForRebuild(cursors, preserveBeforeMs);
|
|
371
|
+
|
|
372
|
+
if (mainRepair.removed > 0) {
|
|
373
|
+
await resetUploadOffsetForZcodeRepair(queueStatePath, queueStateNote);
|
|
374
|
+
}
|
|
375
|
+
if (projectRepair.removed > 0) {
|
|
376
|
+
await resetUploadOffsetForZcodeRepair(projectQueueStatePath, queueStateNote);
|
|
377
|
+
}
|
|
378
|
+
migrations[migrationKey] = {
|
|
379
|
+
appliedAt: new Date().toISOString(),
|
|
380
|
+
removedMain: mainRepair.removed,
|
|
381
|
+
removedProject: projectRepair.removed,
|
|
382
|
+
retractions: mainRepair.retractions,
|
|
383
|
+
preservedMain: mainRepair.preserved || 0,
|
|
384
|
+
preservedProject: projectRepair.preserved || 0,
|
|
385
|
+
rebuildFloorMs: preserveBeforeMs,
|
|
386
|
+
};
|
|
387
|
+
return true;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function zcodeRowTotals(row) {
|
|
391
|
+
return {
|
|
392
|
+
input_tokens: Number(row?.input_tokens) || 0,
|
|
393
|
+
cached_input_tokens: Number(row?.cached_input_tokens) || 0,
|
|
394
|
+
cache_creation_input_tokens: Number(row?.cache_creation_input_tokens) || 0,
|
|
395
|
+
output_tokens: Number(row?.output_tokens) || 0,
|
|
396
|
+
reasoning_output_tokens: Number(row?.reasoning_output_tokens) || 0,
|
|
397
|
+
total_tokens: Number(row?.total_tokens) || 0,
|
|
398
|
+
// Queue-schema completeness (audit fix): backup rows written by older
|
|
399
|
+
// versions may lack these two columns; restored rows must carry them or
|
|
400
|
+
// the restored (recovered) usage uploads with a shape the cloud and the
|
|
401
|
+
// local readers have to heal. billable falls back to total (zcode counts
|
|
402
|
+
// plan-included usage as billable), cost to 0 (read-side pricing recomputes).
|
|
403
|
+
billable_total_tokens: row?.billable_total_tokens == null
|
|
404
|
+
? Math.max(0, Number(row?.total_tokens) || 0)
|
|
405
|
+
: Math.max(0, Number(row.billable_total_tokens) || 0),
|
|
406
|
+
total_cost_usd: Math.max(0, Number(row?.total_cost_usd) || 0),
|
|
407
|
+
conversation_count: Number(row?.conversation_count) || 0,
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function zcodeTotalsDominate(candidate, baseline) {
|
|
412
|
+
if (!candidate) return false;
|
|
413
|
+
if (!baseline) return (candidate.total_tokens || 0) > 0;
|
|
414
|
+
return (candidate.total_tokens || 0) > (baseline.total_tokens || 0);
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
async function collectZcodeQueuePeaks(filePath, { beforeMs = null } = {}) {
|
|
418
|
+
const peaks = new Map();
|
|
419
|
+
let raw;
|
|
420
|
+
try {
|
|
421
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
422
|
+
} catch (error) {
|
|
423
|
+
if (error?.code === "ENOENT") return peaks;
|
|
424
|
+
throw error;
|
|
425
|
+
}
|
|
426
|
+
for (const line of raw.split("\n")) {
|
|
427
|
+
if (!line.trim()) continue;
|
|
428
|
+
let row;
|
|
429
|
+
try {
|
|
430
|
+
row = JSON.parse(line);
|
|
431
|
+
} catch (_error) {
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
if (row?.source !== "zcode") continue;
|
|
435
|
+
const model = typeof row.model === "string" && row.model.trim()
|
|
436
|
+
? row.model.trim()
|
|
437
|
+
: "unknown";
|
|
438
|
+
const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
|
|
439
|
+
if (!hourStart) continue;
|
|
440
|
+
if (beforeMs != null && !zcodeHourIsPreWindow(hourStart, beforeMs)) continue;
|
|
441
|
+
const key = `${model}|${hourStart}`;
|
|
442
|
+
const totals = zcodeRowTotals(row);
|
|
443
|
+
const prev = peaks.get(key);
|
|
444
|
+
if (!prev || zcodeTotalsDominate(totals, prev.totals)) {
|
|
445
|
+
peaks.set(key, { model, hour_start: hourStart, totals });
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
return peaks;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
// Last-wins per key (cloud replay semantics). A historical peak of 9000 followed
|
|
452
|
+
// by a repair retraction of 0 must report 0 so restore still re-appends.
|
|
453
|
+
async function collectZcodeQueueLatest(filePath, { beforeMs = null } = {}) {
|
|
454
|
+
const latest = new Map();
|
|
455
|
+
let raw;
|
|
456
|
+
try {
|
|
457
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
458
|
+
} catch (error) {
|
|
459
|
+
if (error?.code === "ENOENT") return latest;
|
|
460
|
+
throw error;
|
|
461
|
+
}
|
|
462
|
+
for (const line of raw.split("\n")) {
|
|
463
|
+
if (!line.trim()) continue;
|
|
464
|
+
let row;
|
|
465
|
+
try {
|
|
466
|
+
row = JSON.parse(line);
|
|
467
|
+
} catch (_error) {
|
|
468
|
+
continue;
|
|
469
|
+
}
|
|
470
|
+
if (row?.source !== "zcode") continue;
|
|
471
|
+
const model = typeof row.model === "string" && row.model.trim()
|
|
472
|
+
? row.model.trim()
|
|
473
|
+
: "unknown";
|
|
474
|
+
const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
|
|
475
|
+
if (!hourStart) continue;
|
|
476
|
+
if (beforeMs != null && !zcodeHourIsPreWindow(hourStart, beforeMs)) continue;
|
|
477
|
+
latest.set(`${model}|${hourStart}`, {
|
|
478
|
+
model,
|
|
479
|
+
hour_start: hourStart,
|
|
480
|
+
totals: zcodeRowTotals(row),
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
return latest;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
async function listZcodeQueueBackupPaths(queuePath) {
|
|
487
|
+
if (typeof queuePath !== "string" || !queuePath) return [];
|
|
488
|
+
const dir = path.dirname(queuePath);
|
|
489
|
+
const base = path.basename(queuePath);
|
|
490
|
+
let names;
|
|
491
|
+
try {
|
|
492
|
+
names = await fs.readdir(dir);
|
|
493
|
+
} catch (error) {
|
|
494
|
+
if (error?.code === "ENOENT") return [];
|
|
495
|
+
throw error;
|
|
496
|
+
}
|
|
497
|
+
return names
|
|
498
|
+
.filter((name) => name.startsWith(`${base}.bak.`))
|
|
499
|
+
.map((name) => path.join(dir, name))
|
|
500
|
+
.sort();
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
async function repairZcodePruneWindowRestore({
|
|
504
|
+
cursors,
|
|
505
|
+
queuePath,
|
|
506
|
+
queueStatePath,
|
|
507
|
+
dbPaths = [],
|
|
508
|
+
rebuildFloorMs: rebuildFloorMsOption = null,
|
|
509
|
+
} = {}) {
|
|
510
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
511
|
+
const migrations = (cursors.migrations ||= {});
|
|
512
|
+
const prior = migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY];
|
|
513
|
+
if (isCompletedZcodeMigration(prior)) return false;
|
|
514
|
+
|
|
515
|
+
const rebuildFloorMs = Number.isFinite(rebuildFloorMsOption)
|
|
516
|
+
? rebuildFloorMsOption
|
|
517
|
+
: resolveZcodeRebuildFloorMs(dbPaths);
|
|
518
|
+
if (!Number.isFinite(rebuildFloorMs)) {
|
|
519
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
520
|
+
skipped: true,
|
|
521
|
+
reason: "zcode_rebuild_floor_unavailable",
|
|
522
|
+
at: new Date().toISOString(),
|
|
523
|
+
};
|
|
524
|
+
return false;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
const backupPaths = await listZcodeQueueBackupPaths(queuePath);
|
|
528
|
+
if (backupPaths.length === 0) {
|
|
529
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
530
|
+
skipped: true,
|
|
531
|
+
reason: "zcode_queue_backup_missing",
|
|
532
|
+
at: new Date().toISOString(),
|
|
533
|
+
rebuildFloorMs,
|
|
534
|
+
};
|
|
535
|
+
return false;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const preserveBeforeMs = zcodePreserveBeforeMsFromFloor(rebuildFloorMs);
|
|
539
|
+
const backupPeaks = new Map();
|
|
540
|
+
for (const backupPath of backupPaths) {
|
|
541
|
+
const peaks = await collectZcodeQueuePeaks(backupPath, { beforeMs: preserveBeforeMs });
|
|
542
|
+
for (const [key, peak] of peaks) {
|
|
543
|
+
const prev = backupPeaks.get(key);
|
|
544
|
+
if (!prev || zcodeTotalsDominate(peak.totals, prev.totals)) {
|
|
545
|
+
backupPeaks.set(key, peak);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
if (backupPeaks.size === 0) {
|
|
550
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
551
|
+
appliedAt: new Date().toISOString(),
|
|
552
|
+
restored: 0,
|
|
553
|
+
rebuildFloorMs,
|
|
554
|
+
preserveBeforeMs,
|
|
555
|
+
note: "no_pre_window_peaks_in_backups",
|
|
556
|
+
};
|
|
557
|
+
return true;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
const liveLatest = await collectZcodeQueueLatest(queuePath, { beforeMs: preserveBeforeMs });
|
|
561
|
+
const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
|
|
562
|
+
hourly.buckets ||= {};
|
|
563
|
+
const restorations = [];
|
|
564
|
+
for (const [key, peak] of backupPeaks) {
|
|
565
|
+
const bucketKeyName = bucketKey("zcode", peak.model, peak.hour_start);
|
|
566
|
+
const live = liveLatest.get(key)?.totals || null;
|
|
567
|
+
// Compare against the latest queue emission (last-wins), not historical peaks
|
|
568
|
+
// or cursor buckets — a 9000→0 retraction must still restore.
|
|
569
|
+
if (!zcodeTotalsDominate(peak.totals, live)) {
|
|
570
|
+
const bucketTotals = hourly.buckets[bucketKeyName]?.totals
|
|
571
|
+
? zcodeRowTotals(hourly.buckets[bucketKeyName].totals)
|
|
572
|
+
: null;
|
|
573
|
+
if (bucketTotals && zcodeTotalsDominate(peak.totals, bucketTotals)) {
|
|
574
|
+
hourly.buckets[bucketKeyName] = {
|
|
575
|
+
...(hourly.buckets[bucketKeyName] || {}),
|
|
576
|
+
totals: { ...peak.totals },
|
|
577
|
+
hour_start: peak.hour_start,
|
|
578
|
+
source: "zcode",
|
|
579
|
+
model: peak.model,
|
|
580
|
+
};
|
|
581
|
+
}
|
|
582
|
+
continue;
|
|
583
|
+
}
|
|
584
|
+
restorations.push(peak);
|
|
585
|
+
hourly.buckets[bucketKeyName] = {
|
|
586
|
+
...(hourly.buckets[bucketKeyName] || {}),
|
|
587
|
+
totals: { ...peak.totals },
|
|
588
|
+
hour_start: peak.hour_start,
|
|
589
|
+
source: "zcode",
|
|
590
|
+
model: peak.model,
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
if (restorations.length === 0) {
|
|
595
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
596
|
+
appliedAt: new Date().toISOString(),
|
|
597
|
+
restored: 0,
|
|
598
|
+
rebuildFloorMs,
|
|
599
|
+
backupsConsidered: backupPaths.length,
|
|
600
|
+
note: "backup_peaks_not_higher",
|
|
601
|
+
};
|
|
602
|
+
return true;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
await backupExistingFile(queuePath);
|
|
606
|
+
const append = restorations.map((peak) => JSON.stringify(buildZcodeRestorationRow(peak)));
|
|
607
|
+
await fs.appendFile(queuePath, `${append.join("\n")}\n`, "utf8");
|
|
608
|
+
await resetUploadOffsetForZcodeRepair(
|
|
609
|
+
queueStatePath,
|
|
610
|
+
"reset_after_zcode_prune_window_restore_2026_10",
|
|
611
|
+
);
|
|
612
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
613
|
+
appliedAt: new Date().toISOString(),
|
|
614
|
+
restored: restorations.length,
|
|
615
|
+
rebuildFloorMs,
|
|
616
|
+
backupsConsidered: backupPaths.length,
|
|
617
|
+
};
|
|
618
|
+
return true;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
async function repairZcodeNativeUsageMigration(options = {}) {
|
|
622
|
+
return repairZcodeUsageMigration({
|
|
623
|
+
...options,
|
|
624
|
+
migrationKey: ZCODE_NATIVE_USAGE_REPAIR_KEY,
|
|
625
|
+
queueStateNote: "reset_after_zcode_native_usage_repair_2026_08",
|
|
626
|
+
});
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
async function repairZcodeInclusiveTokenMigration(options = {}) {
|
|
630
|
+
return repairZcodeUsageMigration({
|
|
631
|
+
...options,
|
|
632
|
+
migrationKey: ZCODE_INCLUSIVE_TOKEN_REPAIR_KEY,
|
|
633
|
+
queueStateNote: "reset_after_zcode_inclusive_token_repair_2026_09",
|
|
634
|
+
});
|
|
635
|
+
}
|
|
636
|
+
module.exports = {
|
|
637
|
+
ZCODE_NATIVE_USAGE_REPAIR_KEY,
|
|
638
|
+
ZCODE_INCLUSIVE_TOKEN_REPAIR_KEY,
|
|
639
|
+
ZCODE_PRUNE_WINDOW_RESTORE_KEY,
|
|
640
|
+
resolveZcodeDbRebuildFloorMs,
|
|
641
|
+
resolveZcodeRebuildFloorMs,
|
|
642
|
+
zcodeRowTotals,
|
|
643
|
+
buildZcodeRestorationRow,
|
|
644
|
+
repairZcodeNativeUsageMigration,
|
|
645
|
+
repairZcodeInclusiveTokenMigration,
|
|
646
|
+
repairZcodePruneWindowRestore,
|
|
647
|
+
};
|