tokenusage-cli 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +7 -6
- package/package.json +1 -1
- package/src/cli.js +32 -5
- package/src/commands/device-login.js +16 -17
- package/src/commands/diagnostics.js +16 -16
- package/src/commands/doctor.js +14 -11
- package/src/commands/pause.js +34 -3
- package/src/commands/project-profiles.js +5 -2
- package/src/commands/sessions.js +18 -30
- package/src/commands/status.js +22 -16
- package/src/commands/sync.js +117 -3614
- package/src/commands/uninstall.js +10 -9
- package/src/commands/upload.js +18 -12
- package/src/lib/ark-agent-plan-limits.js +4 -13
- package/src/lib/ark-coding-plan-limits.js +4 -13
- package/src/lib/cli-flags.js +109 -0
- package/src/lib/commandcode-limits.js +11 -38
- package/src/lib/doctor.js +91 -1
- package/src/lib/fs.js +10 -0
- package/src/lib/grok-limits.js +5 -18
- package/src/lib/limits-core.js +64 -0
- package/src/lib/qoder-limits.js +4 -13
- package/src/lib/queue-backup.js +24 -0
- package/src/lib/queue-drain.js +388 -0
- package/src/lib/usage-limits.js +22 -29
- package/src/lib/workbuddy-ipc.js +191 -0
- package/src/lib/workbuddy-limits.js +656 -0
- package/src/lib/zcode-limits.js +5 -18
- package/src/repairs/claude.js +442 -0
- package/src/repairs/codebuddy.js +246 -0
- package/src/repairs/codex.js +1023 -0
- package/src/repairs/droid.js +266 -0
- package/src/repairs/grok.js +352 -0
- package/src/repairs/legacy-migrations.js +135 -0
- package/src/repairs/mimo.js +161 -0
- package/src/repairs/workbuddy.js +209 -0
- package/src/repairs/zcode.js +647 -0
|
@@ -0,0 +1,442 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Claude history repairs, split out of commands/sync.js: the ground-truth
|
|
4
|
+
// rebuild (queue + hourly state from the raw transcript set), claude-mem
|
|
5
|
+
// observer-file re-inclusion and the claude-mem row relabel.
|
|
6
|
+
|
|
7
|
+
const fs = require("node:fs/promises");
|
|
8
|
+
const readline = require("node:readline");
|
|
9
|
+
const fssync = require("node:fs");
|
|
10
|
+
const path = require("node:path");
|
|
11
|
+
const {
|
|
12
|
+
computeClaudeGroundTruthBuckets,
|
|
13
|
+
} = require("../lib/claude-categorizer");
|
|
14
|
+
const {
|
|
15
|
+
ensureDir,
|
|
16
|
+
} = require("../lib/fs");
|
|
17
|
+
const {
|
|
18
|
+
bucketKey,
|
|
19
|
+
claudeMessageDedupKey,
|
|
20
|
+
totalsKey,
|
|
21
|
+
} = require("../lib/rollout");
|
|
22
|
+
|
|
23
|
+
const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
|
|
24
|
+
const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
|
|
25
|
+
const CLAUDE_GROUND_TRUTH_REPAIR_KEY = "claudeGroundTruthRepair_2026_05_v4";
|
|
26
|
+
|
|
27
|
+
async function repairClaudeQueueFromGroundTruth({
|
|
28
|
+
cursors,
|
|
29
|
+
queuePath,
|
|
30
|
+
queueStatePath = null,
|
|
31
|
+
projectQueuePath = null,
|
|
32
|
+
projectQueueStatePath = null,
|
|
33
|
+
rootDirs = null,
|
|
34
|
+
}) {
|
|
35
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
36
|
+
const migrations = (cursors.migrations ||= {});
|
|
37
|
+
if (migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY]) return false;
|
|
38
|
+
|
|
39
|
+
let result;
|
|
40
|
+
try {
|
|
41
|
+
// rootDirs must cover every install the incremental parser scans (native
|
|
42
|
+
// AND WSL): this repair's semantics are "disk is truth" — it clears all
|
|
43
|
+
// claude buckets and replaces claudeHashes wholesale, so scanning fewer
|
|
44
|
+
// roots than sync would silently drop the missing install's history.
|
|
45
|
+
result = await computeClaudeGroundTruthBuckets(
|
|
46
|
+
Array.isArray(rootDirs) && rootDirs.length > 0 ? { rootDirs } : {},
|
|
47
|
+
);
|
|
48
|
+
} catch (e) {
|
|
49
|
+
console.error("[sync] claude ground-truth repair: scan failed:", e?.message || e);
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
const { rows, seenHashes, fileList } = result;
|
|
53
|
+
|
|
54
|
+
// 1. Atomic rewrite of queue.jsonl: keep non-claude rows, drop existing
|
|
55
|
+
// claude/claude-mem rows, append truth rows. Atomic via tmp + rename.
|
|
56
|
+
let claudeRowsRemoved = 0;
|
|
57
|
+
if (typeof queuePath === "string" && queuePath) {
|
|
58
|
+
let raw = "";
|
|
59
|
+
try {
|
|
60
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
61
|
+
} catch (e) {
|
|
62
|
+
if (e?.code !== "ENOENT") throw e;
|
|
63
|
+
}
|
|
64
|
+
const keptLines = [];
|
|
65
|
+
for (const line of raw.split("\n")) {
|
|
66
|
+
if (!line.trim()) continue;
|
|
67
|
+
let row;
|
|
68
|
+
try {
|
|
69
|
+
row = JSON.parse(line);
|
|
70
|
+
} catch (_e) {
|
|
71
|
+
// Preserve unparseable lines verbatim — operator may want to
|
|
72
|
+
// recover them later.
|
|
73
|
+
keptLines.push(line);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (row?.source === "claude" || row?.source === "claude-mem") {
|
|
77
|
+
claudeRowsRemoved += 1;
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
keptLines.push(line);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const truthLines = rows.map((r) =>
|
|
84
|
+
JSON.stringify({
|
|
85
|
+
source: "claude",
|
|
86
|
+
model: r.model,
|
|
87
|
+
hour_start: r.hour_start,
|
|
88
|
+
input_tokens: r.input_tokens,
|
|
89
|
+
cached_input_tokens: r.cached_input_tokens,
|
|
90
|
+
cache_creation_input_tokens: r.cache_creation_input_tokens,
|
|
91
|
+
output_tokens: r.output_tokens,
|
|
92
|
+
reasoning_output_tokens: r.reasoning_output_tokens,
|
|
93
|
+
total_tokens: r.total_tokens,
|
|
94
|
+
billable_total_tokens: r.billable_total_tokens,
|
|
95
|
+
conversation_count: r.conversation_count,
|
|
96
|
+
}),
|
|
97
|
+
);
|
|
98
|
+
|
|
99
|
+
await ensureDir(path.dirname(queuePath));
|
|
100
|
+
const out = keptLines.concat(truthLines).join("\n") + "\n";
|
|
101
|
+
const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
|
|
102
|
+
await fs.writeFile(tmp, out, "utf8");
|
|
103
|
+
await fs.rename(tmp, queuePath);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 2. Reset cursors.hourly.buckets / groupQueued for source=claude (and the
|
|
107
|
+
// dead source=claude-mem buckets) so incremental sync's in-memory state
|
|
108
|
+
// matches the truth.
|
|
109
|
+
const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
|
|
110
|
+
hourly.buckets ||= {};
|
|
111
|
+
hourly.groupQueued ||= {};
|
|
112
|
+
|
|
113
|
+
let bucketsCleared = 0;
|
|
114
|
+
for (const k of Object.keys(hourly.buckets)) {
|
|
115
|
+
if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
|
|
116
|
+
delete hourly.buckets[k];
|
|
117
|
+
bucketsCleared += 1;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// Clear stale claude entries from groupQueued (left over by v2 repair).
|
|
121
|
+
// After v3 we never repopulate it for claude, so nothing should be added
|
|
122
|
+
// back during the per-model write loop below.
|
|
123
|
+
for (const k of Object.keys(hourly.groupQueued)) {
|
|
124
|
+
if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
|
|
125
|
+
delete hourly.groupQueued[k];
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Per-model claude buckets: set queuedKey but DO NOT touch
|
|
130
|
+
// hourly.groupQueued. groupQueued is used by enqueueTouchedBuckets to
|
|
131
|
+
// mark a (source, hour) as legacy-aggregate state; writing claude hours
|
|
132
|
+
// there would force every later sync to re-emit the hour as a single
|
|
133
|
+
// model=DEFAULT_MODEL aggregate row instead of touching only the bucket
|
|
134
|
+
// that actually changed. The original v2 release did write groupQueued
|
|
135
|
+
// here and was the cause of an unknown-bucket inflation regression.
|
|
136
|
+
for (const r of rows) {
|
|
137
|
+
const totals = {
|
|
138
|
+
input_tokens: r.input_tokens,
|
|
139
|
+
cached_input_tokens: r.cached_input_tokens,
|
|
140
|
+
cache_creation_input_tokens: r.cache_creation_input_tokens,
|
|
141
|
+
output_tokens: r.output_tokens,
|
|
142
|
+
reasoning_output_tokens: r.reasoning_output_tokens,
|
|
143
|
+
total_tokens: r.total_tokens,
|
|
144
|
+
billable_total_tokens: r.billable_total_tokens,
|
|
145
|
+
conversation_count: r.conversation_count,
|
|
146
|
+
};
|
|
147
|
+
const key = bucketKey("claude", r.model, r.hour_start);
|
|
148
|
+
hourly.buckets[key] = {
|
|
149
|
+
totals,
|
|
150
|
+
queuedKey: totalsKey(totals),
|
|
151
|
+
source: "claude",
|
|
152
|
+
hour_start: r.hour_start,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// 3. Reset per-file cursors so future incremental sync only reads genuinely
|
|
157
|
+
// new tail content. Format must match what rollout.js expects:
|
|
158
|
+
// { inode, offset, updatedAt }. Setting a plain integer here breaks
|
|
159
|
+
// the inode-equality check inside parseClaudeFile, which would treat
|
|
160
|
+
// the file as untracked and re-read it from byte 0 — silently doubling
|
|
161
|
+
// everything. (That was the actual cause of the regression after the
|
|
162
|
+
// first repair attempt.)
|
|
163
|
+
cursors.files ||= {};
|
|
164
|
+
let filesReset = 0;
|
|
165
|
+
const nowIso = new Date().toISOString();
|
|
166
|
+
for (const fp of fileList) {
|
|
167
|
+
let st;
|
|
168
|
+
try {
|
|
169
|
+
st = fssync.statSync(fp);
|
|
170
|
+
} catch (_e) {
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
cursors.files[fp] = {
|
|
174
|
+
inode: st.ino || 0,
|
|
175
|
+
offset: st.size,
|
|
176
|
+
updatedAt: nowIso,
|
|
177
|
+
};
|
|
178
|
+
filesReset += 1;
|
|
179
|
+
}
|
|
180
|
+
cursors.claudeHashes = seenHashes;
|
|
181
|
+
|
|
182
|
+
// 4. Reset cloud-upload offset so the corrected rows are re-sent. Other
|
|
183
|
+
// sources are upserted idempotently by the ingest endpoint, so this is
|
|
184
|
+
// safe — just costs one extra round of bandwidth.
|
|
185
|
+
if (typeof queueStatePath === "string" && queueStatePath) {
|
|
186
|
+
let uploadState = {};
|
|
187
|
+
try {
|
|
188
|
+
uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
|
|
189
|
+
} catch (_e) {
|
|
190
|
+
uploadState = {};
|
|
191
|
+
}
|
|
192
|
+
uploadState.offset = 0;
|
|
193
|
+
uploadState.updatedAt = new Date().toISOString();
|
|
194
|
+
uploadState.note = "reset_after_claude_repair_2026_05_v4";
|
|
195
|
+
await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// 5. Repair project queue. Historical claude rows in project.queue.jsonl
|
|
199
|
+
// were uniformly mis-attributed to project_key=
|
|
200
|
+
// "claude-mem/observer-sessions" (left over from the observer
|
|
201
|
+
// relabel migration). We can't reconstruct the true cwd-based
|
|
202
|
+
// project_key for each historical message reliably, so we drop every
|
|
203
|
+
// claude/claude-mem row from project.queue.jsonl and reset the
|
|
204
|
+
// matching cursors.projectHourly state. New claude usage will
|
|
205
|
+
// accumulate to the correct cwd-derived project_key going forward.
|
|
206
|
+
let projectRowsRemoved = 0;
|
|
207
|
+
let projectBucketsCleared = 0;
|
|
208
|
+
if (typeof projectQueuePath === "string" && projectQueuePath) {
|
|
209
|
+
let projRaw = "";
|
|
210
|
+
try {
|
|
211
|
+
projRaw = await fs.readFile(projectQueuePath, "utf8");
|
|
212
|
+
} catch (e) {
|
|
213
|
+
if (e?.code !== "ENOENT") throw e;
|
|
214
|
+
}
|
|
215
|
+
if (projRaw) {
|
|
216
|
+
const projKept = [];
|
|
217
|
+
for (const line of projRaw.split("\n")) {
|
|
218
|
+
if (!line.trim()) continue;
|
|
219
|
+
let row;
|
|
220
|
+
try {
|
|
221
|
+
row = JSON.parse(line);
|
|
222
|
+
} catch (_e) {
|
|
223
|
+
projKept.push(line);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
if (row?.source === "claude" || row?.source === "claude-mem") {
|
|
227
|
+
projectRowsRemoved += 1;
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
projKept.push(line);
|
|
231
|
+
}
|
|
232
|
+
await ensureDir(path.dirname(projectQueuePath));
|
|
233
|
+
const tmp = `${projectQueuePath}.tmp.${process.pid}.${Date.now()}`;
|
|
234
|
+
await fs.writeFile(tmp, projKept.join("\n") + "\n", "utf8");
|
|
235
|
+
await fs.rename(tmp, projectQueuePath);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Clear matching projectHourly state so the claude project buckets
|
|
239
|
+
// start fresh.
|
|
240
|
+
const projHourly = (cursors.projectHourly ||= { buckets: {} });
|
|
241
|
+
projHourly.buckets ||= {};
|
|
242
|
+
for (const k of Object.keys(projHourly.buckets)) {
|
|
243
|
+
const v = projHourly.buckets[k];
|
|
244
|
+
const src = v?.source || "";
|
|
245
|
+
if (src === "claude" || src === "claude-mem") {
|
|
246
|
+
delete projHourly.buckets[k];
|
|
247
|
+
projectBucketsCleared += 1;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// Reset project upload offset.
|
|
252
|
+
if (typeof projectQueueStatePath === "string" && projectQueueStatePath) {
|
|
253
|
+
let st = {};
|
|
254
|
+
try {
|
|
255
|
+
st = JSON.parse(await fs.readFile(projectQueueStatePath, "utf8"));
|
|
256
|
+
} catch (_e) {
|
|
257
|
+
st = {};
|
|
258
|
+
}
|
|
259
|
+
st.offset = 0;
|
|
260
|
+
st.updatedAt = new Date().toISOString();
|
|
261
|
+
st.note = "reset_after_claude_repair_2026_05_v6";
|
|
262
|
+
await fs.writeFile(projectQueueStatePath, JSON.stringify(st));
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY] = {
|
|
267
|
+
appliedAt: new Date().toISOString(),
|
|
268
|
+
bucketsWritten: rows.length,
|
|
269
|
+
bucketsCleared,
|
|
270
|
+
rowsRemoved: claudeRowsRemoved,
|
|
271
|
+
filesReset,
|
|
272
|
+
hashesRetained: seenHashes.length,
|
|
273
|
+
uploadOffsetReset: typeof queueStatePath === "string" && !!queueStatePath,
|
|
274
|
+
projectRowsRemoved,
|
|
275
|
+
projectBucketsCleared,
|
|
276
|
+
};
|
|
277
|
+
return true;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
async function reincludeClaudeMemObserverFiles({ cursors, claudeFiles, queuePath, queueStatePath }) {
|
|
281
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
282
|
+
const migrations = (cursors.migrations ||= {});
|
|
283
|
+
if (migrations[CLAUDE_MEM_OBSERVER_REINCLUDE_KEY]) return false;
|
|
284
|
+
|
|
285
|
+
const observerPaths = (Array.isArray(claudeFiles) ? claudeFiles : [])
|
|
286
|
+
.map((entry) => (typeof entry === "string" ? entry : entry?.path))
|
|
287
|
+
.filter((p) => typeof p === "string" && p.includes(CLAUDE_MEM_OBSERVER_PATH_SEGMENT));
|
|
288
|
+
|
|
289
|
+
if (!cursors.files || typeof cursors.files !== "object") {
|
|
290
|
+
cursors.files = {};
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
let filesReset = 0;
|
|
294
|
+
for (const filePath of observerPaths) {
|
|
295
|
+
if (cursors.files[filePath]) {
|
|
296
|
+
delete cursors.files[filePath];
|
|
297
|
+
filesReset += 1;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
const hashesToRemove = observerPaths.length > 0
|
|
302
|
+
? await collectClaudeMessageHashes(observerPaths)
|
|
303
|
+
: new Set();
|
|
304
|
+
let hashesRemoved = 0;
|
|
305
|
+
if (Array.isArray(cursors.claudeHashes) && hashesToRemove.size > 0) {
|
|
306
|
+
const nextHashes = [];
|
|
307
|
+
for (const hash of cursors.claudeHashes) {
|
|
308
|
+
if (hashesToRemove.has(hash)) {
|
|
309
|
+
hashesRemoved += 1;
|
|
310
|
+
continue;
|
|
311
|
+
}
|
|
312
|
+
nextHashes.push(hash);
|
|
313
|
+
}
|
|
314
|
+
cursors.claudeHashes = nextHashes;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
const queueRowsRelabeled = typeof queuePath === "string" && queuePath
|
|
318
|
+
? await relabelClaudeMemQueueRows(queuePath, queueStatePath)
|
|
319
|
+
: 0;
|
|
320
|
+
|
|
321
|
+
migrations[CLAUDE_MEM_OBSERVER_REINCLUDE_KEY] = {
|
|
322
|
+
appliedAt: new Date().toISOString(),
|
|
323
|
+
filesReset,
|
|
324
|
+
hashesRemoved,
|
|
325
|
+
queueRowsRelabeled,
|
|
326
|
+
};
|
|
327
|
+
return filesReset > 0 || hashesRemoved > 0 || queueRowsRelabeled > 0;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
async function relabelClaudeMemQueueRows(queuePath, queueStatePath = null) {
|
|
331
|
+
let raw;
|
|
332
|
+
try {
|
|
333
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
334
|
+
} catch (_e) {
|
|
335
|
+
return 0;
|
|
336
|
+
}
|
|
337
|
+
if (!raw || !raw.includes('"claude-mem"')) return 0;
|
|
338
|
+
|
|
339
|
+
// The cloud-upload cursor (queue.state.json `offset`) is a byte position in
|
|
340
|
+
// the pre-rewrite file. Relabeling shrinks rewritten lines ("claude-mem" →
|
|
341
|
+
// "claude"), so the old offset would land mid-line in the new file and the
|
|
342
|
+
// next drainQueueToCloud batch would skip part of a row (or a whole row).
|
|
343
|
+
// Track the old→new byte mapping while rewriting and remap the offset to
|
|
344
|
+
// the equivalent line boundary (same pattern as project-usage-purge.js).
|
|
345
|
+
let previousOffset = 0;
|
|
346
|
+
if (typeof queueStatePath === "string" && queueStatePath) {
|
|
347
|
+
try {
|
|
348
|
+
const st = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
|
|
349
|
+
const off = Number(st?.offset || 0);
|
|
350
|
+
if (Number.isFinite(off) && off > 0) previousOffset = off;
|
|
351
|
+
} catch (_e) {
|
|
352
|
+
previousOffset = 0;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const lines = raw.split("\n");
|
|
357
|
+
const out = [];
|
|
358
|
+
let relabeled = 0;
|
|
359
|
+
let inputOffset = 0;
|
|
360
|
+
let outputOffset = 0;
|
|
361
|
+
let nextOffset = 0;
|
|
362
|
+
for (let i = 0; i < lines.length; i++) {
|
|
363
|
+
const line = lines[i];
|
|
364
|
+
const isLast = i === lines.length - 1;
|
|
365
|
+
let outLine = line;
|
|
366
|
+
if (line) {
|
|
367
|
+
try {
|
|
368
|
+
const obj = JSON.parse(line);
|
|
369
|
+
if (obj && obj.source === "claude-mem") {
|
|
370
|
+
obj.source = "claude";
|
|
371
|
+
relabeled += 1;
|
|
372
|
+
outLine = JSON.stringify(obj);
|
|
373
|
+
}
|
|
374
|
+
} catch (_e) {
|
|
375
|
+
// keep malformed lines verbatim
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
out.push(outLine);
|
|
379
|
+
inputOffset += Buffer.byteLength(line, "utf8") + (isLast ? 0 : 1);
|
|
380
|
+
outputOffset += Buffer.byteLength(outLine, "utf8") + (isLast ? 0 : 1);
|
|
381
|
+
// Upload offsets always sit at line boundaries; a mid-line offset
|
|
382
|
+
// (corruption) rounds down to the previous boundary so no row is skipped
|
|
383
|
+
// — worst case a row is re-uploaded, and cloud ingest upserts by key.
|
|
384
|
+
if (inputOffset <= previousOffset) nextOffset = outputOffset;
|
|
385
|
+
}
|
|
386
|
+
if (relabeled === 0) return 0;
|
|
387
|
+
|
|
388
|
+
// Atomic rewrite: temp file in the same directory + rename, so a crash
|
|
389
|
+
// mid-write can never leave queue.jsonl truncated.
|
|
390
|
+
const tmpPath = `${queuePath}.tmp`;
|
|
391
|
+
await fs.writeFile(tmpPath, out.join("\n"), "utf8");
|
|
392
|
+
await fs.rename(tmpPath, queuePath);
|
|
393
|
+
|
|
394
|
+
if (typeof queueStatePath === "string" && queueStatePath && previousOffset > 0) {
|
|
395
|
+
let state = {};
|
|
396
|
+
try {
|
|
397
|
+
state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
|
|
398
|
+
if (!state || typeof state !== "object") state = {};
|
|
399
|
+
} catch (_e) {
|
|
400
|
+
state = {};
|
|
401
|
+
}
|
|
402
|
+
state.offset = nextOffset;
|
|
403
|
+
state.updatedAt = new Date().toISOString();
|
|
404
|
+
await fs.writeFile(queueStatePath, JSON.stringify(state), "utf8");
|
|
405
|
+
}
|
|
406
|
+
return relabeled;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
async function collectClaudeMessageHashes(filePaths) {
|
|
410
|
+
const hashes = new Set();
|
|
411
|
+
for (const filePath of filePaths) {
|
|
412
|
+
let stream;
|
|
413
|
+
try {
|
|
414
|
+
stream = fssync.createReadStream(filePath, { encoding: "utf8" });
|
|
415
|
+
} catch (_e) {
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
419
|
+
for await (const line of rl) {
|
|
420
|
+
if (!line.includes('"usage"')) continue;
|
|
421
|
+
let obj;
|
|
422
|
+
try {
|
|
423
|
+
obj = JSON.parse(line);
|
|
424
|
+
} catch (_e) {
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
const hash = claudeMessageDedupKey(obj);
|
|
428
|
+
if (hash) hashes.add(hash);
|
|
429
|
+
}
|
|
430
|
+
rl.close();
|
|
431
|
+
stream.close?.();
|
|
432
|
+
}
|
|
433
|
+
return hashes;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
module.exports = {
|
|
437
|
+
CLAUDE_GROUND_TRUTH_REPAIR_KEY,
|
|
438
|
+
CLAUDE_MEM_OBSERVER_PATH_SEGMENT,
|
|
439
|
+
CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
|
|
440
|
+
reincludeClaudeMemObserverFiles,
|
|
441
|
+
repairClaudeQueueFromGroundTruth,
|
|
442
|
+
};
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// CodeBuddy log/JSONL overlap repair (issue #403), split out of
|
|
4
|
+
// commands/sync.js. Rebuilds CodeBuddy history from the discoverable sources
|
|
5
|
+
// and retracts keys the rebuild cannot reproduce.
|
|
6
|
+
|
|
7
|
+
const fs = require("node:fs/promises");
|
|
8
|
+
const path = require("node:path");
|
|
9
|
+
|
|
10
|
+
const { ensureDir } = require("../lib/fs");
|
|
11
|
+
const {
|
|
12
|
+
bucketKey,
|
|
13
|
+
codebuddyJsonlHasUsage,
|
|
14
|
+
parseCodebuddyIncremental,
|
|
15
|
+
} = require("../lib/rollout");
|
|
16
|
+
|
|
17
|
+
const CODEBUDDY_LOG_JSONL_REPAIR_KEY = "codebuddyLogJsonlOverlapRepair_2026_08";
|
|
18
|
+
|
|
19
|
+
function codebuddyUsageRowKey(row) {
|
|
20
|
+
if (!row || row.source !== "codebuddy") return null;
|
|
21
|
+
const model = typeof row.model === "string" && row.model ? row.model : "unknown";
|
|
22
|
+
const hourStart = typeof row.hour_start === "string" && row.hour_start ? row.hour_start : null;
|
|
23
|
+
return hourStart ? bucketKey("codebuddy", model, hourStart) : null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function codebuddyZeroUsageRow(model, hourStart) {
|
|
27
|
+
return {
|
|
28
|
+
source: "codebuddy",
|
|
29
|
+
model: model || "unknown",
|
|
30
|
+
hour_start: hourStart,
|
|
31
|
+
input_tokens: 0,
|
|
32
|
+
cached_input_tokens: 0,
|
|
33
|
+
cache_creation_input_tokens: 0,
|
|
34
|
+
output_tokens: 0,
|
|
35
|
+
reasoning_output_tokens: 0,
|
|
36
|
+
total_tokens: 0,
|
|
37
|
+
billable_total_tokens: 0,
|
|
38
|
+
conversation_count: 0,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Issue #403 historical repair. Before the JSONL usage fix, CodeBuddy could
|
|
43
|
+
// ingest the same round-trip once from ~/.codebuddy/projects and once from the
|
|
44
|
+
// IDE extension log. The forward resolver now disables logs whenever any JSONL
|
|
45
|
+
// has usable rawUsage, but existing queues/hourly state need one guarded,
|
|
46
|
+
// atomic rebuild so the old double-count is not left visible forever.
|
|
47
|
+
async function repairCodebuddyLogJsonlOverlap({
|
|
48
|
+
cursors,
|
|
49
|
+
queuePath,
|
|
50
|
+
queueStatePath,
|
|
51
|
+
codebuddyFiles,
|
|
52
|
+
env = process.env,
|
|
53
|
+
} = {}) {
|
|
54
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
55
|
+
const migrations = (cursors.migrations ||= {});
|
|
56
|
+
const prior = migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY];
|
|
57
|
+
if (prior && !(typeof prior === "object" && prior.skipped)) return false;
|
|
58
|
+
|
|
59
|
+
const files = Array.isArray(codebuddyFiles) ? codebuddyFiles : [];
|
|
60
|
+
const jsonlFiles = files.filter((filePath) => typeof filePath === "string" && filePath.endsWith(".jsonl"));
|
|
61
|
+
const logFiles = files.filter((filePath) => typeof filePath === "string" && filePath.endsWith(".log"));
|
|
62
|
+
const hasDetailedJsonl = jsonlFiles.some((filePath) => codebuddyJsonlHasUsage(filePath));
|
|
63
|
+
|
|
64
|
+
const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
|
|
65
|
+
const liveHourlyKeys = new Set(
|
|
66
|
+
Object.keys(hourly.buckets || {}).filter((key) => key.startsWith("codebuddy|")),
|
|
67
|
+
);
|
|
68
|
+
let queueRaw = "";
|
|
69
|
+
try {
|
|
70
|
+
queueRaw = await fs.readFile(queuePath, "utf8");
|
|
71
|
+
} catch (e) {
|
|
72
|
+
if (e?.code !== "ENOENT") throw e;
|
|
73
|
+
}
|
|
74
|
+
const queueCodebuddyKeys = new Set();
|
|
75
|
+
const keptQueueLines = [];
|
|
76
|
+
for (const line of queueRaw.split("\n")) {
|
|
77
|
+
if (!line.trim()) continue;
|
|
78
|
+
let row;
|
|
79
|
+
try {
|
|
80
|
+
row = JSON.parse(line);
|
|
81
|
+
} catch (_e) {
|
|
82
|
+
keptQueueLines.push(line);
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
const key = codebuddyUsageRowKey(row);
|
|
86
|
+
if (!key) {
|
|
87
|
+
keptQueueLines.push(line);
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
queueCodebuddyKeys.add(key);
|
|
91
|
+
}
|
|
92
|
+
const liveKeys = new Set([...liveHourlyKeys, ...queueCodebuddyKeys]);
|
|
93
|
+
if (liveKeys.size === 0) {
|
|
94
|
+
migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
|
|
95
|
+
status: "noop",
|
|
96
|
+
appliedAt: new Date().toISOString(),
|
|
97
|
+
reason: "no-codebuddy-history",
|
|
98
|
+
};
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Keep this condition retryable when old CodeBuddy history exists but the
|
|
103
|
+
// extension log is temporarily rotated/missing. A permanent no-op here
|
|
104
|
+
// would close the repair window before a later full scan can rediscover it.
|
|
105
|
+
if (jsonlFiles.length === 0 || logFiles.length === 0 || !hasDetailedJsonl) {
|
|
106
|
+
migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
|
|
107
|
+
skipped: true,
|
|
108
|
+
reason: "no-mixed-detailed-sources",
|
|
109
|
+
at: new Date().toISOString(),
|
|
110
|
+
};
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// A rebuild from the currently discoverable JSONL is safe only if every
|
|
115
|
+
// file previously tailed by CodeBuddy is still available in this scan. If a
|
|
116
|
+
// rotated/deleted log is missing, retain the user's history and retry later
|
|
117
|
+
// rather than silently erasing an unreproducible bucket.
|
|
118
|
+
const discovered = new Set(files);
|
|
119
|
+
const priorFiles = cursors.codebuddy?.fileOffsets && typeof cursors.codebuddy.fileOffsets === "object"
|
|
120
|
+
? Object.keys(cursors.codebuddy.fileOffsets)
|
|
121
|
+
: [];
|
|
122
|
+
const missingPriorFile = priorFiles.find((filePath) => !discovered.has(filePath));
|
|
123
|
+
if (missingPriorFile) {
|
|
124
|
+
migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
|
|
125
|
+
skipped: true,
|
|
126
|
+
reason: "codebuddy_file_unreproducible",
|
|
127
|
+
filePath: missingPriorFile,
|
|
128
|
+
at: new Date().toISOString(),
|
|
129
|
+
};
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const tmpQueue = `${queuePath}.codebuddyrebuild.${process.pid}.${Date.now()}`;
|
|
134
|
+
const tmpCursors = {
|
|
135
|
+
version: 1,
|
|
136
|
+
hourly: { buckets: {}, groupQueued: {} },
|
|
137
|
+
codebuddy: {},
|
|
138
|
+
};
|
|
139
|
+
let rebuiltRows = [];
|
|
140
|
+
try {
|
|
141
|
+
await parseCodebuddyIncremental({
|
|
142
|
+
// Rebuild from both sources. The parser now preserves legacy-only log
|
|
143
|
+
// sessions while consuming mirrored log fingerprints against JSONL, so
|
|
144
|
+
// rebuilding JSONL alone would silently discard valid pre-rawUsage data.
|
|
145
|
+
projectFiles: files,
|
|
146
|
+
cursors: tmpCursors,
|
|
147
|
+
queuePath: tmpQueue,
|
|
148
|
+
env,
|
|
149
|
+
});
|
|
150
|
+
let rebuiltRaw = "";
|
|
151
|
+
try {
|
|
152
|
+
rebuiltRaw = await fs.readFile(tmpQueue, "utf8");
|
|
153
|
+
} catch (e) {
|
|
154
|
+
if (e?.code !== "ENOENT") throw e;
|
|
155
|
+
}
|
|
156
|
+
rebuiltRows = rebuiltRaw.split("\n").filter((line) => line.trim());
|
|
157
|
+
} catch (e) {
|
|
158
|
+
console.error("[sync] CodeBuddy log/JSONL repair: rebuild failed, leaving history untouched:", e?.message || e);
|
|
159
|
+
return false;
|
|
160
|
+
} finally {
|
|
161
|
+
await fs.rm(tmpQueue, { force: true }).catch(() => {});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const rebuiltKeys = new Set();
|
|
165
|
+
const validRebuiltRows = [];
|
|
166
|
+
for (const line of rebuiltRows) {
|
|
167
|
+
let row;
|
|
168
|
+
try { row = JSON.parse(line); } catch (_e) { continue; }
|
|
169
|
+
const key = codebuddyUsageRowKey(row);
|
|
170
|
+
if (!key) continue;
|
|
171
|
+
rebuiltKeys.add(key);
|
|
172
|
+
validRebuiltRows.push(line);
|
|
173
|
+
}
|
|
174
|
+
if (rebuiltKeys.size === 0) {
|
|
175
|
+
migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
|
|
176
|
+
skipped: true,
|
|
177
|
+
reason: "jsonl_rebuild_empty",
|
|
178
|
+
at: new Date().toISOString(),
|
|
179
|
+
};
|
|
180
|
+
return false;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const staleRetractions = [];
|
|
184
|
+
for (const key of liveKeys) {
|
|
185
|
+
if (rebuiltKeys.has(key)) continue;
|
|
186
|
+
const [, model, ...hourParts] = key.split("|");
|
|
187
|
+
staleRetractions.push(JSON.stringify(codebuddyZeroUsageRow(model, hourParts.join("|"))));
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const nextQueueLines = keptQueueLines.concat(staleRetractions, validRebuiltRows);
|
|
191
|
+
await ensureDir(path.dirname(queuePath));
|
|
192
|
+
const queueTmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
|
|
193
|
+
await fs.writeFile(queueTmp, nextQueueLines.length ? `${nextQueueLines.join("\n")}\n` : "", "utf8");
|
|
194
|
+
await fs.rename(queueTmp, queuePath);
|
|
195
|
+
|
|
196
|
+
const nextHourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
|
|
197
|
+
nextHourly.buckets ||= {};
|
|
198
|
+
nextHourly.groupQueued ||= {};
|
|
199
|
+
for (const key of Object.keys(nextHourly.buckets)) {
|
|
200
|
+
if (key.startsWith("codebuddy|")) delete nextHourly.buckets[key];
|
|
201
|
+
}
|
|
202
|
+
for (const key of Object.keys(nextHourly.groupQueued)) {
|
|
203
|
+
if (key.startsWith("codebuddy|")) delete nextHourly.groupQueued[key];
|
|
204
|
+
}
|
|
205
|
+
for (const [key, bucket] of Object.entries(tmpCursors.hourly.buckets || {})) {
|
|
206
|
+
if (key.startsWith("codebuddy|")) nextHourly.buckets[key] = bucket;
|
|
207
|
+
}
|
|
208
|
+
for (const [key, value] of Object.entries(tmpCursors.hourly.groupQueued || {})) {
|
|
209
|
+
if (key.startsWith("codebuddy|")) nextHourly.groupQueued[key] = value;
|
|
210
|
+
}
|
|
211
|
+
cursors.codebuddy = tmpCursors.codebuddy || {};
|
|
212
|
+
|
|
213
|
+
let uploadState = {};
|
|
214
|
+
try { uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8")); } catch (_e) {}
|
|
215
|
+
uploadState.offset = 0;
|
|
216
|
+
uploadState.updatedAt = new Date().toISOString();
|
|
217
|
+
uploadState.note = "reset_after_codebuddy_log_jsonl_repair_2026_08";
|
|
218
|
+
await ensureDir(path.dirname(queueStatePath));
|
|
219
|
+
await fs.writeFile(queueStatePath, JSON.stringify(uploadState, null, 2) + "\n", "utf8");
|
|
220
|
+
|
|
221
|
+
migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
|
|
222
|
+
status: "applied",
|
|
223
|
+
appliedAt: new Date().toISOString(),
|
|
224
|
+
jsonlFiles: jsonlFiles.length,
|
|
225
|
+
logFiles: logFiles.length,
|
|
226
|
+
previousBuckets: liveKeys.size,
|
|
227
|
+
rebuiltBuckets: rebuiltKeys.size,
|
|
228
|
+
retractedBuckets: staleRetractions.length,
|
|
229
|
+
};
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// WorkBuddy's 0.87.9 SQLite fallback treated the bounded session_usage.used
|
|
234
|
+
// context snapshot as cumulative input tokens. A current database proves that
|
|
235
|
+
// assumption false (`size` is a 192k context limit while trace input totals can
|
|
236
|
+
// exceed it), so rebuild historical WorkBuddy buckets from JSONL/trace sources
|
|
237
|
+
// once the new parser is available. The same missing-source guard used by the
|
|
238
|
+
// other data migrations keeps this recoverable: if a previously contributing
|
|
239
|
+
// file is gone, leave history untouched and stop adding new context snapshots.
|
|
240
|
+
|
|
241
|
+
module.exports = {
|
|
242
|
+
CODEBUDDY_LOG_JSONL_REPAIR_KEY,
|
|
243
|
+
codebuddyUsageRowKey,
|
|
244
|
+
codebuddyZeroUsageRow,
|
|
245
|
+
repairCodebuddyLogJsonlOverlap,
|
|
246
|
+
};
|