tokenusage-cli 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,442 @@
1
+ "use strict";
2
+
3
+ // Claude history repairs, split out of commands/sync.js: the ground-truth
4
+ // rebuild (queue + hourly state from the raw transcript set), claude-mem
5
+ // observer-file re-inclusion and the claude-mem row relabel.
6
+
7
+ const fs = require("node:fs/promises");
8
+ const readline = require("node:readline");
9
+ const fssync = require("node:fs");
10
+ const path = require("node:path");
11
+ const {
12
+ computeClaudeGroundTruthBuckets,
13
+ } = require("../lib/claude-categorizer");
14
+ const {
15
+ ensureDir,
16
+ } = require("../lib/fs");
17
+ const {
18
+ bucketKey,
19
+ claudeMessageDedupKey,
20
+ totalsKey,
21
+ } = require("../lib/rollout");
22
+
23
+ const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
24
+ const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
25
+ const CLAUDE_GROUND_TRUTH_REPAIR_KEY = "claudeGroundTruthRepair_2026_05_v4";
26
+
27
+ async function repairClaudeQueueFromGroundTruth({
28
+ cursors,
29
+ queuePath,
30
+ queueStatePath = null,
31
+ projectQueuePath = null,
32
+ projectQueueStatePath = null,
33
+ rootDirs = null,
34
+ }) {
35
+ if (!cursors || typeof cursors !== "object") return false;
36
+ const migrations = (cursors.migrations ||= {});
37
+ if (migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY]) return false;
38
+
39
+ let result;
40
+ try {
41
+ // rootDirs must cover every install the incremental parser scans (native
42
+ // AND WSL): this repair's semantics are "disk is truth" — it clears all
43
+ // claude buckets and replaces claudeHashes wholesale, so scanning fewer
44
+ // roots than sync would silently drop the missing install's history.
45
+ result = await computeClaudeGroundTruthBuckets(
46
+ Array.isArray(rootDirs) && rootDirs.length > 0 ? { rootDirs } : {},
47
+ );
48
+ } catch (e) {
49
+ console.error("[sync] claude ground-truth repair: scan failed:", e?.message || e);
50
+ return false;
51
+ }
52
+ const { rows, seenHashes, fileList } = result;
53
+
54
+ // 1. Atomic rewrite of queue.jsonl: keep non-claude rows, drop existing
55
+ // claude/claude-mem rows, append truth rows. Atomic via tmp + rename.
56
+ let claudeRowsRemoved = 0;
57
+ if (typeof queuePath === "string" && queuePath) {
58
+ let raw = "";
59
+ try {
60
+ raw = await fs.readFile(queuePath, "utf8");
61
+ } catch (e) {
62
+ if (e?.code !== "ENOENT") throw e;
63
+ }
64
+ const keptLines = [];
65
+ for (const line of raw.split("\n")) {
66
+ if (!line.trim()) continue;
67
+ let row;
68
+ try {
69
+ row = JSON.parse(line);
70
+ } catch (_e) {
71
+ // Preserve unparseable lines verbatim — operator may want to
72
+ // recover them later.
73
+ keptLines.push(line);
74
+ continue;
75
+ }
76
+ if (row?.source === "claude" || row?.source === "claude-mem") {
77
+ claudeRowsRemoved += 1;
78
+ continue;
79
+ }
80
+ keptLines.push(line);
81
+ }
82
+
83
+ const truthLines = rows.map((r) =>
84
+ JSON.stringify({
85
+ source: "claude",
86
+ model: r.model,
87
+ hour_start: r.hour_start,
88
+ input_tokens: r.input_tokens,
89
+ cached_input_tokens: r.cached_input_tokens,
90
+ cache_creation_input_tokens: r.cache_creation_input_tokens,
91
+ output_tokens: r.output_tokens,
92
+ reasoning_output_tokens: r.reasoning_output_tokens,
93
+ total_tokens: r.total_tokens,
94
+ billable_total_tokens: r.billable_total_tokens,
95
+ conversation_count: r.conversation_count,
96
+ }),
97
+ );
98
+
99
+ await ensureDir(path.dirname(queuePath));
100
+ const out = keptLines.concat(truthLines).join("\n") + "\n";
101
+ const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
102
+ await fs.writeFile(tmp, out, "utf8");
103
+ await fs.rename(tmp, queuePath);
104
+ }
105
+
106
+ // 2. Reset cursors.hourly.buckets / groupQueued for source=claude (and the
107
+ // dead source=claude-mem buckets) so incremental sync's in-memory state
108
+ // matches the truth.
109
+ const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
110
+ hourly.buckets ||= {};
111
+ hourly.groupQueued ||= {};
112
+
113
+ let bucketsCleared = 0;
114
+ for (const k of Object.keys(hourly.buckets)) {
115
+ if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
116
+ delete hourly.buckets[k];
117
+ bucketsCleared += 1;
118
+ }
119
+ }
120
+ // Clear stale claude entries from groupQueued (left over by v2 repair).
121
+ // After v3 we never repopulate it for claude, so nothing should be added
122
+ // back during the per-model write loop below.
123
+ for (const k of Object.keys(hourly.groupQueued)) {
124
+ if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
125
+ delete hourly.groupQueued[k];
126
+ }
127
+ }
128
+
129
+ // Per-model claude buckets: set queuedKey but DO NOT touch
130
+ // hourly.groupQueued. groupQueued is used by enqueueTouchedBuckets to
131
+ // mark a (source, hour) as legacy-aggregate state; writing claude hours
132
+ // there would force every later sync to re-emit the hour as a single
133
+ // model=DEFAULT_MODEL aggregate row instead of touching only the bucket
134
+ // that actually changed. The original v2 release did write groupQueued
135
+ // here and was the cause of an unknown-bucket inflation regression.
136
+ for (const r of rows) {
137
+ const totals = {
138
+ input_tokens: r.input_tokens,
139
+ cached_input_tokens: r.cached_input_tokens,
140
+ cache_creation_input_tokens: r.cache_creation_input_tokens,
141
+ output_tokens: r.output_tokens,
142
+ reasoning_output_tokens: r.reasoning_output_tokens,
143
+ total_tokens: r.total_tokens,
144
+ billable_total_tokens: r.billable_total_tokens,
145
+ conversation_count: r.conversation_count,
146
+ };
147
+ const key = bucketKey("claude", r.model, r.hour_start);
148
+ hourly.buckets[key] = {
149
+ totals,
150
+ queuedKey: totalsKey(totals),
151
+ source: "claude",
152
+ hour_start: r.hour_start,
153
+ };
154
+ }
155
+
156
+ // 3. Reset per-file cursors so future incremental sync only reads genuinely
157
+ // new tail content. Format must match what rollout.js expects:
158
+ // { inode, offset, updatedAt }. Setting a plain integer here breaks
159
+ // the inode-equality check inside parseClaudeFile, which would treat
160
+ // the file as untracked and re-read it from byte 0 — silently doubling
161
+ // everything. (That was the actual cause of the regression after the
162
+ // first repair attempt.)
163
+ cursors.files ||= {};
164
+ let filesReset = 0;
165
+ const nowIso = new Date().toISOString();
166
+ for (const fp of fileList) {
167
+ let st;
168
+ try {
169
+ st = fssync.statSync(fp);
170
+ } catch (_e) {
171
+ continue;
172
+ }
173
+ cursors.files[fp] = {
174
+ inode: st.ino || 0,
175
+ offset: st.size,
176
+ updatedAt: nowIso,
177
+ };
178
+ filesReset += 1;
179
+ }
180
+ cursors.claudeHashes = seenHashes;
181
+
182
+ // 4. Reset cloud-upload offset so the corrected rows are re-sent. Other
183
+ // sources are upserted idempotently by the ingest endpoint, so this is
184
+ // safe — just costs one extra round of bandwidth.
185
+ if (typeof queueStatePath === "string" && queueStatePath) {
186
+ let uploadState = {};
187
+ try {
188
+ uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
189
+ } catch (_e) {
190
+ uploadState = {};
191
+ }
192
+ uploadState.offset = 0;
193
+ uploadState.updatedAt = new Date().toISOString();
194
+ uploadState.note = "reset_after_claude_repair_2026_05_v4";
195
+ await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
196
+ }
197
+
198
+ // 5. Repair project queue. Historical claude rows in project.queue.jsonl
199
+ // were uniformly mis-attributed to project_key=
200
+ // "claude-mem/observer-sessions" (left over from the observer
201
+ // relabel migration). We can't reconstruct the true cwd-based
202
+ // project_key for each historical message reliably, so we drop every
203
+ // claude/claude-mem row from project.queue.jsonl and reset the
204
+ // matching cursors.projectHourly state. New claude usage will
205
+ // accumulate to the correct cwd-derived project_key going forward.
206
+ let projectRowsRemoved = 0;
207
+ let projectBucketsCleared = 0;
208
+ if (typeof projectQueuePath === "string" && projectQueuePath) {
209
+ let projRaw = "";
210
+ try {
211
+ projRaw = await fs.readFile(projectQueuePath, "utf8");
212
+ } catch (e) {
213
+ if (e?.code !== "ENOENT") throw e;
214
+ }
215
+ if (projRaw) {
216
+ const projKept = [];
217
+ for (const line of projRaw.split("\n")) {
218
+ if (!line.trim()) continue;
219
+ let row;
220
+ try {
221
+ row = JSON.parse(line);
222
+ } catch (_e) {
223
+ projKept.push(line);
224
+ continue;
225
+ }
226
+ if (row?.source === "claude" || row?.source === "claude-mem") {
227
+ projectRowsRemoved += 1;
228
+ continue;
229
+ }
230
+ projKept.push(line);
231
+ }
232
+ await ensureDir(path.dirname(projectQueuePath));
233
+ const tmp = `${projectQueuePath}.tmp.${process.pid}.${Date.now()}`;
234
+ await fs.writeFile(tmp, projKept.join("\n") + "\n", "utf8");
235
+ await fs.rename(tmp, projectQueuePath);
236
+ }
237
+
238
+ // Clear matching projectHourly state so the claude project buckets
239
+ // start fresh.
240
+ const projHourly = (cursors.projectHourly ||= { buckets: {} });
241
+ projHourly.buckets ||= {};
242
+ for (const k of Object.keys(projHourly.buckets)) {
243
+ const v = projHourly.buckets[k];
244
+ const src = v?.source || "";
245
+ if (src === "claude" || src === "claude-mem") {
246
+ delete projHourly.buckets[k];
247
+ projectBucketsCleared += 1;
248
+ }
249
+ }
250
+
251
+ // Reset project upload offset.
252
+ if (typeof projectQueueStatePath === "string" && projectQueueStatePath) {
253
+ let st = {};
254
+ try {
255
+ st = JSON.parse(await fs.readFile(projectQueueStatePath, "utf8"));
256
+ } catch (_e) {
257
+ st = {};
258
+ }
259
+ st.offset = 0;
260
+ st.updatedAt = new Date().toISOString();
261
+ st.note = "reset_after_claude_repair_2026_05_v6";
262
+ await fs.writeFile(projectQueueStatePath, JSON.stringify(st));
263
+ }
264
+ }
265
+
266
+ migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY] = {
267
+ appliedAt: new Date().toISOString(),
268
+ bucketsWritten: rows.length,
269
+ bucketsCleared,
270
+ rowsRemoved: claudeRowsRemoved,
271
+ filesReset,
272
+ hashesRetained: seenHashes.length,
273
+ uploadOffsetReset: typeof queueStatePath === "string" && !!queueStatePath,
274
+ projectRowsRemoved,
275
+ projectBucketsCleared,
276
+ };
277
+ return true;
278
+ }
279
+
280
+ async function reincludeClaudeMemObserverFiles({ cursors, claudeFiles, queuePath, queueStatePath }) {
281
+ if (!cursors || typeof cursors !== "object") return false;
282
+ const migrations = (cursors.migrations ||= {});
283
+ if (migrations[CLAUDE_MEM_OBSERVER_REINCLUDE_KEY]) return false;
284
+
285
+ const observerPaths = (Array.isArray(claudeFiles) ? claudeFiles : [])
286
+ .map((entry) => (typeof entry === "string" ? entry : entry?.path))
287
+ .filter((p) => typeof p === "string" && p.includes(CLAUDE_MEM_OBSERVER_PATH_SEGMENT));
288
+
289
+ if (!cursors.files || typeof cursors.files !== "object") {
290
+ cursors.files = {};
291
+ }
292
+
293
+ let filesReset = 0;
294
+ for (const filePath of observerPaths) {
295
+ if (cursors.files[filePath]) {
296
+ delete cursors.files[filePath];
297
+ filesReset += 1;
298
+ }
299
+ }
300
+
301
+ const hashesToRemove = observerPaths.length > 0
302
+ ? await collectClaudeMessageHashes(observerPaths)
303
+ : new Set();
304
+ let hashesRemoved = 0;
305
+ if (Array.isArray(cursors.claudeHashes) && hashesToRemove.size > 0) {
306
+ const nextHashes = [];
307
+ for (const hash of cursors.claudeHashes) {
308
+ if (hashesToRemove.has(hash)) {
309
+ hashesRemoved += 1;
310
+ continue;
311
+ }
312
+ nextHashes.push(hash);
313
+ }
314
+ cursors.claudeHashes = nextHashes;
315
+ }
316
+
317
+ const queueRowsRelabeled = typeof queuePath === "string" && queuePath
318
+ ? await relabelClaudeMemQueueRows(queuePath, queueStatePath)
319
+ : 0;
320
+
321
+ migrations[CLAUDE_MEM_OBSERVER_REINCLUDE_KEY] = {
322
+ appliedAt: new Date().toISOString(),
323
+ filesReset,
324
+ hashesRemoved,
325
+ queueRowsRelabeled,
326
+ };
327
+ return filesReset > 0 || hashesRemoved > 0 || queueRowsRelabeled > 0;
328
+ }
329
+
330
+ async function relabelClaudeMemQueueRows(queuePath, queueStatePath = null) {
331
+ let raw;
332
+ try {
333
+ raw = await fs.readFile(queuePath, "utf8");
334
+ } catch (_e) {
335
+ return 0;
336
+ }
337
+ if (!raw || !raw.includes('"claude-mem"')) return 0;
338
+
339
+ // The cloud-upload cursor (queue.state.json `offset`) is a byte position in
340
+ // the pre-rewrite file. Relabeling shrinks rewritten lines ("claude-mem" →
341
+ // "claude"), so the old offset would land mid-line in the new file and the
342
+ // next drainQueueToCloud batch would skip part of a row (or a whole row).
343
+ // Track the old→new byte mapping while rewriting and remap the offset to
344
+ // the equivalent line boundary (same pattern as project-usage-purge.js).
345
+ let previousOffset = 0;
346
+ if (typeof queueStatePath === "string" && queueStatePath) {
347
+ try {
348
+ const st = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
349
+ const off = Number(st?.offset || 0);
350
+ if (Number.isFinite(off) && off > 0) previousOffset = off;
351
+ } catch (_e) {
352
+ previousOffset = 0;
353
+ }
354
+ }
355
+
356
+ const lines = raw.split("\n");
357
+ const out = [];
358
+ let relabeled = 0;
359
+ let inputOffset = 0;
360
+ let outputOffset = 0;
361
+ let nextOffset = 0;
362
+ for (let i = 0; i < lines.length; i++) {
363
+ const line = lines[i];
364
+ const isLast = i === lines.length - 1;
365
+ let outLine = line;
366
+ if (line) {
367
+ try {
368
+ const obj = JSON.parse(line);
369
+ if (obj && obj.source === "claude-mem") {
370
+ obj.source = "claude";
371
+ relabeled += 1;
372
+ outLine = JSON.stringify(obj);
373
+ }
374
+ } catch (_e) {
375
+ // keep malformed lines verbatim
376
+ }
377
+ }
378
+ out.push(outLine);
379
+ inputOffset += Buffer.byteLength(line, "utf8") + (isLast ? 0 : 1);
380
+ outputOffset += Buffer.byteLength(outLine, "utf8") + (isLast ? 0 : 1);
381
+ // Upload offsets always sit at line boundaries; a mid-line offset
382
+ // (corruption) rounds down to the previous boundary so no row is skipped
383
+ // — worst case a row is re-uploaded, and cloud ingest upserts by key.
384
+ if (inputOffset <= previousOffset) nextOffset = outputOffset;
385
+ }
386
+ if (relabeled === 0) return 0;
387
+
388
+ // Atomic rewrite: temp file in the same directory + rename, so a crash
389
+ // mid-write can never leave queue.jsonl truncated.
390
+ const tmpPath = `${queuePath}.tmp`;
391
+ await fs.writeFile(tmpPath, out.join("\n"), "utf8");
392
+ await fs.rename(tmpPath, queuePath);
393
+
394
+ if (typeof queueStatePath === "string" && queueStatePath && previousOffset > 0) {
395
+ let state = {};
396
+ try {
397
+ state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
398
+ if (!state || typeof state !== "object") state = {};
399
+ } catch (_e) {
400
+ state = {};
401
+ }
402
+ state.offset = nextOffset;
403
+ state.updatedAt = new Date().toISOString();
404
+ await fs.writeFile(queueStatePath, JSON.stringify(state), "utf8");
405
+ }
406
+ return relabeled;
407
+ }
408
+
409
+ async function collectClaudeMessageHashes(filePaths) {
410
+ const hashes = new Set();
411
+ for (const filePath of filePaths) {
412
+ let stream;
413
+ try {
414
+ stream = fssync.createReadStream(filePath, { encoding: "utf8" });
415
+ } catch (_e) {
416
+ continue;
417
+ }
418
+ const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
419
+ for await (const line of rl) {
420
+ if (!line.includes('"usage"')) continue;
421
+ let obj;
422
+ try {
423
+ obj = JSON.parse(line);
424
+ } catch (_e) {
425
+ continue;
426
+ }
427
+ const hash = claudeMessageDedupKey(obj);
428
+ if (hash) hashes.add(hash);
429
+ }
430
+ rl.close();
431
+ stream.close?.();
432
+ }
433
+ return hashes;
434
+ }
435
+
436
+ module.exports = {
437
+ CLAUDE_GROUND_TRUTH_REPAIR_KEY,
438
+ CLAUDE_MEM_OBSERVER_PATH_SEGMENT,
439
+ CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
440
+ reincludeClaudeMemObserverFiles,
441
+ repairClaudeQueueFromGroundTruth,
442
+ };
@@ -0,0 +1,246 @@
1
+ "use strict";
2
+
3
+ // CodeBuddy log/JSONL overlap repair (issue #403), split out of
4
+ // commands/sync.js. Rebuilds CodeBuddy history from the discoverable sources
5
+ // and retracts keys the rebuild cannot reproduce.
6
+
7
+ const fs = require("node:fs/promises");
8
+ const path = require("node:path");
9
+
10
+ const { ensureDir } = require("../lib/fs");
11
+ const {
12
+ bucketKey,
13
+ codebuddyJsonlHasUsage,
14
+ parseCodebuddyIncremental,
15
+ } = require("../lib/rollout");
16
+
17
+ const CODEBUDDY_LOG_JSONL_REPAIR_KEY = "codebuddyLogJsonlOverlapRepair_2026_08";
18
+
19
+ function codebuddyUsageRowKey(row) {
20
+ if (!row || row.source !== "codebuddy") return null;
21
+ const model = typeof row.model === "string" && row.model ? row.model : "unknown";
22
+ const hourStart = typeof row.hour_start === "string" && row.hour_start ? row.hour_start : null;
23
+ return hourStart ? bucketKey("codebuddy", model, hourStart) : null;
24
+ }
25
+
26
+ function codebuddyZeroUsageRow(model, hourStart) {
27
+ return {
28
+ source: "codebuddy",
29
+ model: model || "unknown",
30
+ hour_start: hourStart,
31
+ input_tokens: 0,
32
+ cached_input_tokens: 0,
33
+ cache_creation_input_tokens: 0,
34
+ output_tokens: 0,
35
+ reasoning_output_tokens: 0,
36
+ total_tokens: 0,
37
+ billable_total_tokens: 0,
38
+ conversation_count: 0,
39
+ };
40
+ }
41
+
42
+ // Issue #403 historical repair. Before the JSONL usage fix, CodeBuddy could
43
+ // ingest the same round-trip once from ~/.codebuddy/projects and once from the
44
+ // IDE extension log. The forward resolver now disables logs whenever any JSONL
45
+ // has usable rawUsage, but existing queues/hourly state need one guarded,
46
+ // atomic rebuild so the old double-count is not left visible forever.
47
+ async function repairCodebuddyLogJsonlOverlap({
48
+ cursors,
49
+ queuePath,
50
+ queueStatePath,
51
+ codebuddyFiles,
52
+ env = process.env,
53
+ } = {}) {
54
+ if (!cursors || typeof cursors !== "object") return false;
55
+ const migrations = (cursors.migrations ||= {});
56
+ const prior = migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY];
57
+ if (prior && !(typeof prior === "object" && prior.skipped)) return false;
58
+
59
+ const files = Array.isArray(codebuddyFiles) ? codebuddyFiles : [];
60
+ const jsonlFiles = files.filter((filePath) => typeof filePath === "string" && filePath.endsWith(".jsonl"));
61
+ const logFiles = files.filter((filePath) => typeof filePath === "string" && filePath.endsWith(".log"));
62
+ const hasDetailedJsonl = jsonlFiles.some((filePath) => codebuddyJsonlHasUsage(filePath));
63
+
64
+ const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
65
+ const liveHourlyKeys = new Set(
66
+ Object.keys(hourly.buckets || {}).filter((key) => key.startsWith("codebuddy|")),
67
+ );
68
+ let queueRaw = "";
69
+ try {
70
+ queueRaw = await fs.readFile(queuePath, "utf8");
71
+ } catch (e) {
72
+ if (e?.code !== "ENOENT") throw e;
73
+ }
74
+ const queueCodebuddyKeys = new Set();
75
+ const keptQueueLines = [];
76
+ for (const line of queueRaw.split("\n")) {
77
+ if (!line.trim()) continue;
78
+ let row;
79
+ try {
80
+ row = JSON.parse(line);
81
+ } catch (_e) {
82
+ keptQueueLines.push(line);
83
+ continue;
84
+ }
85
+ const key = codebuddyUsageRowKey(row);
86
+ if (!key) {
87
+ keptQueueLines.push(line);
88
+ continue;
89
+ }
90
+ queueCodebuddyKeys.add(key);
91
+ }
92
+ const liveKeys = new Set([...liveHourlyKeys, ...queueCodebuddyKeys]);
93
+ if (liveKeys.size === 0) {
94
+ migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
95
+ status: "noop",
96
+ appliedAt: new Date().toISOString(),
97
+ reason: "no-codebuddy-history",
98
+ };
99
+ return false;
100
+ }
101
+
102
+ // Keep this condition retryable when old CodeBuddy history exists but the
103
+ // extension log is temporarily rotated/missing. A permanent no-op here
104
+ // would close the repair window before a later full scan can rediscover it.
105
+ if (jsonlFiles.length === 0 || logFiles.length === 0 || !hasDetailedJsonl) {
106
+ migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
107
+ skipped: true,
108
+ reason: "no-mixed-detailed-sources",
109
+ at: new Date().toISOString(),
110
+ };
111
+ return false;
112
+ }
113
+
114
+ // A rebuild from the currently discoverable JSONL is safe only if every
115
+ // file previously tailed by CodeBuddy is still available in this scan. If a
116
+ // rotated/deleted log is missing, retain the user's history and retry later
117
+ // rather than silently erasing an unreproducible bucket.
118
+ const discovered = new Set(files);
119
+ const priorFiles = cursors.codebuddy?.fileOffsets && typeof cursors.codebuddy.fileOffsets === "object"
120
+ ? Object.keys(cursors.codebuddy.fileOffsets)
121
+ : [];
122
+ const missingPriorFile = priorFiles.find((filePath) => !discovered.has(filePath));
123
+ if (missingPriorFile) {
124
+ migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
125
+ skipped: true,
126
+ reason: "codebuddy_file_unreproducible",
127
+ filePath: missingPriorFile,
128
+ at: new Date().toISOString(),
129
+ };
130
+ return false;
131
+ }
132
+
133
+ const tmpQueue = `${queuePath}.codebuddyrebuild.${process.pid}.${Date.now()}`;
134
+ const tmpCursors = {
135
+ version: 1,
136
+ hourly: { buckets: {}, groupQueued: {} },
137
+ codebuddy: {},
138
+ };
139
+ let rebuiltRows = [];
140
+ try {
141
+ await parseCodebuddyIncremental({
142
+ // Rebuild from both sources. The parser now preserves legacy-only log
143
+ // sessions while consuming mirrored log fingerprints against JSONL, so
144
+ // rebuilding JSONL alone would silently discard valid pre-rawUsage data.
145
+ projectFiles: files,
146
+ cursors: tmpCursors,
147
+ queuePath: tmpQueue,
148
+ env,
149
+ });
150
+ let rebuiltRaw = "";
151
+ try {
152
+ rebuiltRaw = await fs.readFile(tmpQueue, "utf8");
153
+ } catch (e) {
154
+ if (e?.code !== "ENOENT") throw e;
155
+ }
156
+ rebuiltRows = rebuiltRaw.split("\n").filter((line) => line.trim());
157
+ } catch (e) {
158
+ console.error("[sync] CodeBuddy log/JSONL repair: rebuild failed, leaving history untouched:", e?.message || e);
159
+ return false;
160
+ } finally {
161
+ await fs.rm(tmpQueue, { force: true }).catch(() => {});
162
+ }
163
+
164
+ const rebuiltKeys = new Set();
165
+ const validRebuiltRows = [];
166
+ for (const line of rebuiltRows) {
167
+ let row;
168
+ try { row = JSON.parse(line); } catch (_e) { continue; }
169
+ const key = codebuddyUsageRowKey(row);
170
+ if (!key) continue;
171
+ rebuiltKeys.add(key);
172
+ validRebuiltRows.push(line);
173
+ }
174
+ if (rebuiltKeys.size === 0) {
175
+ migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
176
+ skipped: true,
177
+ reason: "jsonl_rebuild_empty",
178
+ at: new Date().toISOString(),
179
+ };
180
+ return false;
181
+ }
182
+
183
+ const staleRetractions = [];
184
+ for (const key of liveKeys) {
185
+ if (rebuiltKeys.has(key)) continue;
186
+ const [, model, ...hourParts] = key.split("|");
187
+ staleRetractions.push(JSON.stringify(codebuddyZeroUsageRow(model, hourParts.join("|"))));
188
+ }
189
+
190
+ const nextQueueLines = keptQueueLines.concat(staleRetractions, validRebuiltRows);
191
+ await ensureDir(path.dirname(queuePath));
192
+ const queueTmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
193
+ await fs.writeFile(queueTmp, nextQueueLines.length ? `${nextQueueLines.join("\n")}\n` : "", "utf8");
194
+ await fs.rename(queueTmp, queuePath);
195
+
196
+ const nextHourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
197
+ nextHourly.buckets ||= {};
198
+ nextHourly.groupQueued ||= {};
199
+ for (const key of Object.keys(nextHourly.buckets)) {
200
+ if (key.startsWith("codebuddy|")) delete nextHourly.buckets[key];
201
+ }
202
+ for (const key of Object.keys(nextHourly.groupQueued)) {
203
+ if (key.startsWith("codebuddy|")) delete nextHourly.groupQueued[key];
204
+ }
205
+ for (const [key, bucket] of Object.entries(tmpCursors.hourly.buckets || {})) {
206
+ if (key.startsWith("codebuddy|")) nextHourly.buckets[key] = bucket;
207
+ }
208
+ for (const [key, value] of Object.entries(tmpCursors.hourly.groupQueued || {})) {
209
+ if (key.startsWith("codebuddy|")) nextHourly.groupQueued[key] = value;
210
+ }
211
+ cursors.codebuddy = tmpCursors.codebuddy || {};
212
+
213
+ let uploadState = {};
214
+ try { uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8")); } catch (_e) {}
215
+ uploadState.offset = 0;
216
+ uploadState.updatedAt = new Date().toISOString();
217
+ uploadState.note = "reset_after_codebuddy_log_jsonl_repair_2026_08";
218
+ await ensureDir(path.dirname(queueStatePath));
219
+ await fs.writeFile(queueStatePath, JSON.stringify(uploadState, null, 2) + "\n", "utf8");
220
+
221
+ migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
222
+ status: "applied",
223
+ appliedAt: new Date().toISOString(),
224
+ jsonlFiles: jsonlFiles.length,
225
+ logFiles: logFiles.length,
226
+ previousBuckets: liveKeys.size,
227
+ rebuiltBuckets: rebuiltKeys.size,
228
+ retractedBuckets: staleRetractions.length,
229
+ };
230
+ return true;
231
+ }
232
+
233
+ // WorkBuddy's 0.87.9 SQLite fallback treated the bounded session_usage.used
234
+ // context snapshot as cumulative input tokens. A current database proves that
235
+ // assumption false (`size` is a 192k context limit while trace input totals can
236
+ // exceed it), so rebuild historical WorkBuddy buckets from JSONL/trace sources
237
+ // once the new parser is available. The same missing-source guard used by the
238
+ // other data migrations keeps this recoverable: if a previously contributing
239
+ // file is gone, leave history untouched and stop adding new context snapshots.
240
+
241
+ module.exports = {
242
+ CODEBUDDY_LOG_JSONL_REPAIR_KEY,
243
+ codebuddyUsageRowKey,
244
+ codebuddyZeroUsageRow,
245
+ repairCodebuddyLogJsonlOverlap,
246
+ };