tokenusage-cli 0.5.5 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +44 -0
  2. package/README.md +7 -6
  3. package/package.json +1 -1
  4. package/src/cli.js +32 -5
  5. package/src/commands/device-login.js +16 -17
  6. package/src/commands/diagnostics.js +16 -16
  7. package/src/commands/doctor.js +14 -11
  8. package/src/commands/pause.js +34 -3
  9. package/src/commands/project-profiles.js +5 -2
  10. package/src/commands/sessions.js +18 -30
  11. package/src/commands/status.js +22 -16
  12. package/src/commands/sync.js +117 -3614
  13. package/src/commands/uninstall.js +10 -9
  14. package/src/commands/upload.js +18 -12
  15. package/src/lib/ark-agent-plan-limits.js +4 -13
  16. package/src/lib/ark-coding-plan-limits.js +4 -13
  17. package/src/lib/cli-flags.js +109 -0
  18. package/src/lib/commandcode-limits.js +11 -38
  19. package/src/lib/doctor.js +91 -1
  20. package/src/lib/fs.js +10 -0
  21. package/src/lib/grok-limits.js +5 -18
  22. package/src/lib/limits-core.js +64 -0
  23. package/src/lib/qoder-limits.js +4 -13
  24. package/src/lib/queue-backup.js +24 -0
  25. package/src/lib/queue-drain.js +388 -0
  26. package/src/lib/runtime-config.js +26 -7
  27. package/src/lib/usage-limits.js +22 -29
  28. package/src/lib/workbuddy-ipc.js +191 -0
  29. package/src/lib/workbuddy-limits.js +656 -0
  30. package/src/lib/zcode-limits.js +5 -18
  31. package/src/repairs/claude.js +442 -0
  32. package/src/repairs/codebuddy.js +246 -0
  33. package/src/repairs/codex.js +1023 -0
  34. package/src/repairs/droid.js +266 -0
  35. package/src/repairs/grok.js +352 -0
  36. package/src/repairs/legacy-migrations.js +135 -0
  37. package/src/repairs/mimo.js +161 -0
  38. package/src/repairs/workbuddy.js +209 -0
  39. package/src/repairs/zcode.js +647 -0
@@ -0,0 +1,266 @@
1
+ "use strict";
2
+
3
+ // Droid duplicate-session repair, split out of commands/sync.js: one Droid
4
+ // session could be counted twice when the same conversation appeared under two
5
+ // session ids, inflating history until this rebuild retracts the duplicate.
6
+
7
+ const fs = require("node:fs/promises");
8
+ const fssync = require("node:fs");
9
+ const path = require("node:path");
10
+ const {
11
+ ensureDir,
12
+ } = require("../lib/fs");
13
+ const {
14
+ backupExistingFile,
15
+ } = require("../lib/queue-backup");
16
+ const {
17
+ reincludeClaudeMemObserverFiles,
18
+ } = require("./claude");
19
+ const {
20
+ bucketKey,
21
+ droidSessionIdFromPath,
22
+ listDroidSettingsFiles,
23
+ parseDroidIncremental,
24
+ resolveDroidModel,
25
+ toUtcHalfHourStart,
26
+ } = require("../lib/rollout");
27
+
28
+ const DROID_DUP_SESSION_REPAIR_KEY = "droidDupSessionInflationRepair_2026_06";
29
+
30
+ async function repairDroidDuplicateSessionInflation({ cursors, queuePath, queueStatePath } = {}) {
31
+ if (!cursors || typeof cursors !== "object") return false;
32
+ const migrations = (cursors.migrations ||= {});
33
+ // Completed run → truthy non-skip sentinel (final). A {skipped:true} object would
34
+ // retry (codex skip-retry semantics; this repair only ever writes done/none).
35
+ const prior = migrations[DROID_DUP_SESSION_REPAIR_KEY];
36
+ if (prior && !(typeof prior === "object" && prior.skipped)) return false;
37
+
38
+ // Group current on-disk settings files by session id; collect duplicates.
39
+ let onDisk;
40
+ try {
41
+ onDisk = listDroidSettingsFiles(process.env);
42
+ } catch {
43
+ onDisk = [];
44
+ }
45
+ const bySession = new Map();
46
+ for (const fp of onDisk) {
47
+ const sid = droidSessionIdFromPath(fp);
48
+ if (!sid) continue;
49
+ if (!bySession.has(sid)) bySession.set(sid, []);
50
+ bySession.get(sid).push(fp);
51
+ }
52
+ const dupFiles = [];
53
+ const cleanFiles = [];
54
+ for (const group of bySession.values()) {
55
+ if (group.length > 1) dupFiles.push(...group);
56
+ else cleanFiles.push(...group);
57
+ }
58
+ if (dupFiles.length === 0) {
59
+ migrations[DROID_DUP_SESSION_REPAIR_KEY] = new Date().toISOString();
60
+ return false;
61
+ }
62
+
63
+ // Map on-disk settings files to the (droid, model, half-hour) bucket keys they
64
+ // emit under the parser's own keying (mirrors parseDroidIncremental).
65
+ const bucketKeysForFiles = (files) => {
66
+ const keys = new Set();
67
+ for (const fp of files) {
68
+ let mtimeMs = 0;
69
+ try {
70
+ mtimeMs = fssync.statSync(fp).mtimeMs;
71
+ } catch {
72
+ continue;
73
+ }
74
+ let settings;
75
+ try {
76
+ settings = JSON.parse(fssync.readFileSync(fp, "utf8"));
77
+ } catch {
78
+ continue;
79
+ }
80
+ if (!settings || typeof settings !== "object" || !settings.tokenUsage) continue;
81
+ const bucketStart = toUtcHalfHourStart(
82
+ new Date(mtimeMs || Date.now()).toISOString(),
83
+ );
84
+ if (!bucketStart) continue;
85
+ keys.add(bucketKey("droid", resolveDroidModel(settings, fp), bucketStart));
86
+ }
87
+ return keys;
88
+ };
89
+
90
+ // A bucket key is (source, model, half-hour) — it carries NO session identity.
91
+ // pollutedKeys are the buckets the duplicate files emit to; cleanKeys are buckets
92
+ // a NON-duplicate on-disk session emits to. When a clean session resolves to the
93
+ // same (model, half-hour) as a duplicate file, they collide on one key. We must
94
+ // NOT delete-and-replace such a shared bucket: the rebuild runs over duplicate
95
+ // files only, so replacing the bucket would erase the clean session's tokens
96
+ // (silent data loss). So repair ONLY buckets owned exclusively by duplicate
97
+ // sessions; leave shared buckets intact. Residual inflation in a rare shared
98
+ // bucket is visible and recoverable; destroying real data is not (this is the
99
+ // dedup-needs-identity-proof rule).
100
+ const pollutedKeys = bucketKeysForFiles(dupFiles);
101
+ const cleanKeys = bucketKeysForFiles(cleanFiles);
102
+ const repairKeys = new Set();
103
+ for (const k of pollutedKeys) if (!cleanKeys.has(k)) repairKeys.add(k);
104
+ if (repairKeys.size === 0) {
105
+ migrations[DROID_DUP_SESSION_REPAIR_KEY] = new Date().toISOString();
106
+ return false;
107
+ }
108
+
109
+ // Ground-truth rebuild into throwaway state over the DUPLICATE files only
110
+ // (parseDroidIncremental de-dupes its own input → canonical per session). On any
111
+ // throw, leave all state untouched and do NOT set the sentinel (retry next sync).
112
+ let rebuilt;
113
+ const tmpQueue = `${queuePath}.droidrebuild.${process.pid}.${Date.now()}`;
114
+ try {
115
+ const tmpCursors = { hourly: { buckets: {}, groupQueued: {} }, droid: {} };
116
+ await parseDroidIncremental({
117
+ settingsFiles: dupFiles,
118
+ cursors: tmpCursors,
119
+ queuePath: tmpQueue,
120
+ env: process.env,
121
+ prune: true,
122
+ });
123
+ let tmpRaw = "";
124
+ try {
125
+ tmpRaw = await fs.readFile(tmpQueue, "utf8");
126
+ } catch (e) {
127
+ if (e?.code !== "ENOENT") throw e;
128
+ }
129
+ rebuilt = {
130
+ buckets: tmpCursors.hourly.buckets || {},
131
+ sessionTotals: (tmpCursors.droid && tmpCursors.droid.sessionTotals) || {},
132
+ queueRows: tmpRaw.split("\n").filter((l) => l.trim()),
133
+ };
134
+ } catch (e) {
135
+ console.error(
136
+ "[sync] droid dup-session repair: rebuild failed, leaving all data untouched:",
137
+ e?.message || e,
138
+ );
139
+ return false;
140
+ } finally {
141
+ await fs.rm(tmpQueue, { force: true }).catch(() => {});
142
+ }
143
+
144
+ // Inflation present? Compare live vs rebuilt totals over the repair-scoped keys
145
+ // only. Fire only on live > rebuilt (real inflation) — never on a fresh install
146
+ // (live 0).
147
+ const liveBuckets = (cursors.hourly && cursors.hourly.buckets) || {};
148
+ let liveScoped = 0;
149
+ let rebuiltScoped = 0;
150
+ for (const key of repairKeys) {
151
+ liveScoped += Number(liveBuckets[key]?.totals?.total_tokens || 0);
152
+ rebuiltScoped += Number(rebuilt.buckets[key]?.totals?.total_tokens || 0);
153
+ }
154
+ if (liveScoped <= rebuiltScoped) {
155
+ migrations[DROID_DUP_SESSION_REPAIR_KEY] = new Date().toISOString();
156
+ return false;
157
+ }
158
+
159
+ // ── COMMIT (atomic) ──
160
+ await ensureDir(path.dirname(queuePath));
161
+ await backupExistingFile(queuePath);
162
+
163
+ // 1. queue.jsonl: keep every non-droid line verbatim (incl. unparseable) and
164
+ // every droid row whose bucket key is NOT in repairKeys (clean + shared +
165
+ // deleted-session history). Drop droid rows in repairKeys; append rebuilt rows.
166
+ let raw = "";
167
+ try {
168
+ raw = await fs.readFile(queuePath, "utf8");
169
+ } catch (e) {
170
+ if (e?.code !== "ENOENT") throw e;
171
+ }
172
+ // A queue line is in scope if it's a droid row whose bucket is in repairKeys.
173
+ // Unparseable / non-droid / clean / shared droid rows are kept verbatim.
174
+ const isRepairDroidRow = (line) => {
175
+ let row;
176
+ try {
177
+ row = JSON.parse(line);
178
+ } catch {
179
+ return false;
180
+ }
181
+ return (
182
+ row?.source === "droid" &&
183
+ repairKeys.has(bucketKey("droid", row.model, row.hour_start))
184
+ );
185
+ };
186
+ const kept = raw
187
+ .split("\n")
188
+ .filter((line) => line.trim() && !isRepairDroidRow(line));
189
+ const rebuiltRepairRows = rebuilt.queueRows.filter(isRepairDroidRow);
190
+ const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
191
+ await fs.writeFile(
192
+ tmp,
193
+ kept.concat(rebuiltRepairRows).join("\n") + "\n",
194
+ "utf8",
195
+ );
196
+ await fs.rename(tmp, queuePath);
197
+
198
+ // 2. live hourly buckets: delete repair-scoped droid keys, install the rebuilt
199
+ // buckets for those keys. Other droid buckets untouched.
200
+ const hourly = (cursors.hourly ||= { version: 3, buckets: {}, groupQueued: {} });
201
+ hourly.buckets ||= {};
202
+ hourly.groupQueued ||= {};
203
+ for (const key of repairKeys) {
204
+ delete hourly.buckets[key];
205
+ if (rebuilt.buckets[key]) hourly.buckets[key] = rebuilt.buckets[key];
206
+ }
207
+ // Defensive: droid never uses the legacy aggregate path, but drop any stale droid
208
+ // group markers so a repaired hour can't re-emit as model=unknown.
209
+ for (const gk of Object.keys(hourly.groupQueued)) {
210
+ if (gk.startsWith("droid|")) delete hourly.groupQueued[gk];
211
+ }
212
+
213
+ // 3. session cursor: overwrite ONLY the duplicate sessions with the ground-truth
214
+ // rebuild so the later same-sync droid parse short-circuits (mtime match) and
215
+ // emits nothing. Clean sessions' cursor entries are correct already — leave
216
+ // them, or the later parse would re-emit them from zero.
217
+ const droidState = (cursors.droid ||= {});
218
+ if (!droidState.sessionTotals || typeof droidState.sessionTotals !== "object") {
219
+ droidState.sessionTotals = {};
220
+ }
221
+ for (const sid of Object.keys(rebuilt.sessionTotals)) {
222
+ droidState.sessionTotals[sid] = rebuilt.sessionTotals[sid];
223
+ }
224
+ droidState.updatedAt = new Date().toISOString();
225
+
226
+ // 4. reset the cloud upload offset so corrected rows re-upload (idempotent upsert).
227
+ if (typeof queueStatePath === "string" && queueStatePath) {
228
+ let uploadState = {};
229
+ try {
230
+ uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
231
+ } catch {
232
+ uploadState = {};
233
+ }
234
+ uploadState.offset = 0;
235
+ uploadState.updatedAt = new Date().toISOString();
236
+ uploadState.note = "reset_after_droid_dup_session_2026_06";
237
+ await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
238
+ }
239
+
240
+ migrations[DROID_DUP_SESSION_REPAIR_KEY] = {
241
+ status: "done",
242
+ at: new Date().toISOString(),
243
+ keysRepaired: repairKeys.size,
244
+ keysSkippedSharedWithCleanSession: pollutedKeys.size - repairKeys.size,
245
+ liveBefore: liveScoped,
246
+ rebuiltAfter: rebuiltScoped,
247
+ deltaReclaimed: liveScoped - rebuiltScoped,
248
+ };
249
+ return true;
250
+ }
251
+
252
+ // One-time repair migration: rebuild source=claude rows in queue.jsonl from
253
+ // the actual jsonl files using ccusage's algorithm (msgId+reqId global
254
+ // dedup). Earlier `reincludeClaudeMemObserverFiles` versions (v1/v2/v3) each
255
+ // reset the hash set and re-read observer jsonls, which silently inflated
256
+ // queue.jsonl's claude totals by ~40%. We do an atomic rewrite — keep all
257
+ // non-claude rows verbatim, replace every claude/claude-mem row with the
258
+ // ground-truth set — then reset cursors so the next incremental sync stays
259
+ // in sync, and reset the cloud upload offset so the corrected rows actually
260
+ // reach the cloud (the ingest endpoint upserts by (source, model,
261
+ // hour_start), so re-uploading other sources is idempotent).
262
+
263
+ module.exports = {
264
+ DROID_DUP_SESSION_REPAIR_KEY,
265
+ repairDroidDuplicateSessionInflation,
266
+ };
@@ -0,0 +1,352 @@
1
+ "use strict";
2
+
3
+ // Grok append-only repair, split out of commands/sync.js. The pre-0.2 Grok
4
+ // reader replaced hourly rows instead of appending, so historical buckets were
5
+ // overwritten by whichever pass ran last. This rebuilds them from the persisted
6
+ // session snapshots (the only source that still knows the totals) and retracts
7
+ // keys the snapshots cannot reproduce; idempotent via cursors.grok.migrations.
8
+
9
+ const path = require("node:path");
10
+ const fs = require("node:fs/promises");
11
+
12
+ const { ensureDir } = require("../lib/fs");
13
+ const { backupExistingFile } = require("../lib/queue-backup");
14
+ const { bucketKey, totalsKey } = require("../lib/rollout");
15
+
16
+ const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
17
+
18
+ function normalizeGrokRepairSource(value) {
19
+ return typeof value === "string" ? value.trim().toLowerCase() : "";
20
+ }
21
+
22
+ function normalizeGrokRepairModel(value) {
23
+ return typeof value === "string" && value.trim() ? value.trim() : "grok-build";
24
+ }
25
+
26
+ function normalizeGrokRepairNumber(value) {
27
+ const n = Number(value);
28
+ return Number.isFinite(n) && n > 0 ? n : 0;
29
+ }
30
+
31
+ function toGrokRepairHalfHourStart(value) {
32
+ if (value == null) return null;
33
+ const millis =
34
+ typeof value === "number"
35
+ ? value < 10_000_000_000
36
+ ? value * 1000
37
+ : value
38
+ : Date.parse(String(value));
39
+ if (!Number.isFinite(millis)) return null;
40
+ const halfHourMs = 30 * 60 * 1000;
41
+ return new Date(Math.floor(millis / halfHourMs) * halfHourMs).toISOString();
42
+ }
43
+
44
+ function estimateGrokRepairTotals(totalTokens, conversationCount) {
45
+ const total = Math.trunc(normalizeGrokRepairNumber(totalTokens));
46
+ const inputTokens = Math.round(total * 0.8);
47
+ const outputTokens = Math.max(0, total - inputTokens);
48
+ return {
49
+ input_tokens: inputTokens,
50
+ cached_input_tokens: 0,
51
+ cache_creation_input_tokens: 0,
52
+ output_tokens: outputTokens,
53
+ reasoning_output_tokens: 0,
54
+ total_tokens: total,
55
+ billable_total_tokens: total,
56
+ conversation_count: Math.trunc(normalizeGrokRepairNumber(conversationCount)),
57
+ };
58
+ }
59
+
60
+ function addGrokRepairTotals(target, delta) {
61
+ target.input_tokens += delta.input_tokens;
62
+ target.cached_input_tokens += delta.cached_input_tokens;
63
+ target.cache_creation_input_tokens += delta.cache_creation_input_tokens;
64
+ target.output_tokens += delta.output_tokens;
65
+ target.reasoning_output_tokens += delta.reasoning_output_tokens;
66
+ target.total_tokens += delta.total_tokens;
67
+ target.billable_total_tokens += delta.billable_total_tokens;
68
+ target.conversation_count += delta.conversation_count;
69
+ }
70
+
71
+ function buildGrokRepairRowsFromSnapshots(sessionSnapshots) {
72
+ if (!sessionSnapshots || typeof sessionSnapshots !== "object") return [];
73
+
74
+ const buckets = new Map();
75
+ for (const snapshot of Object.values(sessionSnapshots)) {
76
+ if (!snapshot || typeof snapshot !== "object") continue;
77
+ const totalTokens = Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens));
78
+ if (totalTokens <= 0) continue;
79
+
80
+ const hourStart = toGrokRepairHalfHourStart(
81
+ snapshot.lastEventTimestamp || snapshot.updatedAt,
82
+ );
83
+ if (!hourStart) continue;
84
+
85
+ const model = normalizeGrokRepairModel(snapshot.model);
86
+ const key = bucketKey("grok", model, hourStart);
87
+ let totals = buckets.get(key);
88
+ if (!totals) {
89
+ totals = {
90
+ source: "grok",
91
+ model,
92
+ hour_start: hourStart,
93
+ input_tokens: 0,
94
+ cached_input_tokens: 0,
95
+ cache_creation_input_tokens: 0,
96
+ output_tokens: 0,
97
+ reasoning_output_tokens: 0,
98
+ total_tokens: 0,
99
+ billable_total_tokens: 0,
100
+ conversation_count: 0,
101
+ };
102
+ buckets.set(key, totals);
103
+ }
104
+ addGrokRepairTotals(
105
+ totals,
106
+ estimateGrokRepairTotals(totalTokens, snapshot.messageCount),
107
+ );
108
+ }
109
+
110
+ return Array.from(buckets.values()).sort((a, b) => {
111
+ const timeCompare = a.hour_start.localeCompare(b.hour_start);
112
+ return timeCompare || a.model.localeCompare(b.model);
113
+ });
114
+ }
115
+
116
+ function applyGrokRepairHourlyState(cursors, rows) {
117
+ const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
118
+ const buckets = hourly.buckets && typeof hourly.buckets === "object" ? hourly.buckets : {};
119
+ const groupQueued =
120
+ hourly.groupQueued && typeof hourly.groupQueued === "object" ? hourly.groupQueued : {};
121
+
122
+ for (const key of Object.keys(buckets)) {
123
+ if (key.startsWith("grok|")) {
124
+ delete buckets[key];
125
+ }
126
+ }
127
+ for (const key of Object.keys(groupQueued)) {
128
+ if (key.startsWith("grok|")) {
129
+ delete groupQueued[key];
130
+ }
131
+ }
132
+
133
+ for (const row of rows) {
134
+ const totals = {
135
+ input_tokens: row.input_tokens,
136
+ cached_input_tokens: row.cached_input_tokens,
137
+ cache_creation_input_tokens: row.cache_creation_input_tokens,
138
+ output_tokens: row.output_tokens,
139
+ reasoning_output_tokens: row.reasoning_output_tokens,
140
+ total_tokens: row.total_tokens,
141
+ billable_total_tokens: row.billable_total_tokens,
142
+ conversation_count: row.conversation_count,
143
+ };
144
+ buckets[bucketKey("grok", row.model, row.hour_start)] = {
145
+ totals,
146
+ queuedKey: totalsKey(totals),
147
+ source: "grok",
148
+ hour_start: row.hour_start,
149
+ };
150
+ }
151
+
152
+ cursors.hourly = {
153
+ ...hourly,
154
+ version: 3,
155
+ buckets,
156
+ groupQueued,
157
+ updatedAt: typeof hourly.updatedAt === "string" ? hourly.updatedAt : null,
158
+ };
159
+ }
160
+
161
+ async function resetGrokRepairUploadOffset(queueStatePath) {
162
+ if (typeof queueStatePath !== "string" || !queueStatePath) return false;
163
+ let state = {};
164
+ try {
165
+ state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
166
+ } catch (_e) {
167
+ state = {};
168
+ }
169
+ state.offset = 0;
170
+ state.updatedAt = new Date().toISOString();
171
+ state.note = "reset_after_grok_append_only_repair_2026_05_v4";
172
+ await ensureDir(path.dirname(queueStatePath));
173
+ await fs.writeFile(queueStatePath, JSON.stringify(state, null, 2) + "\n", "utf8");
174
+ return true;
175
+ }
176
+
177
+ function hasAppliedGrokRepairMigration(value) {
178
+ if (!value) return false;
179
+ if (value === true) return true;
180
+ if (value && typeof value === "object") {
181
+ if (value.status === "applied" || value.status === "noop") return true;
182
+ if (value.status) return false;
183
+ return value.rowsWritten != null || value.rowsRemoved != null;
184
+ }
185
+ return false;
186
+ }
187
+
188
+ function serializeGrokRepairRow(row) {
189
+ return JSON.stringify({
190
+ source: "grok",
191
+ model: normalizeGrokRepairModel(row.model),
192
+ hour_start: row.hour_start,
193
+ input_tokens: row.input_tokens || 0,
194
+ cached_input_tokens: row.cached_input_tokens || 0,
195
+ cache_creation_input_tokens: row.cache_creation_input_tokens || 0,
196
+ output_tokens: row.output_tokens || 0,
197
+ reasoning_output_tokens: row.reasoning_output_tokens || 0,
198
+ total_tokens: row.total_tokens || 0,
199
+ billable_total_tokens: row.billable_total_tokens || 0,
200
+ conversation_count: row.conversation_count || 0,
201
+ });
202
+ }
203
+
204
+ async function repairGrokQueueFromSessionSnapshots({ cursors, queuePath, queueStatePath } = {}) {
205
+ if (!cursors || typeof cursors !== "object") return false;
206
+ const grokState = (cursors.grok ||= {});
207
+ const migrations = (grokState.migrations ||= {});
208
+ if (hasAppliedGrokRepairMigration(migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY])) {
209
+ return false;
210
+ }
211
+
212
+ let raw = "";
213
+ try {
214
+ raw = await fs.readFile(queuePath, "utf8");
215
+ } catch (e) {
216
+ if (e?.code !== "ENOENT") throw e;
217
+ }
218
+
219
+ const latestGrokRows = new Map();
220
+ let existingGrokRows = 0;
221
+ for (const line of raw.split("\n")) {
222
+ if (!line.trim()) continue;
223
+ let row;
224
+ try {
225
+ row = JSON.parse(line);
226
+ } catch (_e) {
227
+ continue;
228
+ }
229
+
230
+ if (normalizeGrokRepairSource(row?.source) === "grok") {
231
+ const model = normalizeGrokRepairModel(row.model);
232
+ const hourStart = typeof row.hour_start === "string" ? row.hour_start : null;
233
+ if (!hourStart) continue;
234
+ existingGrokRows += 1;
235
+ latestGrokRows.set(bucketKey("grok", model, hourStart), {
236
+ ...row,
237
+ source: "grok",
238
+ model,
239
+ hour_start: hourStart,
240
+ });
241
+ }
242
+ }
243
+
244
+ if (existingGrokRows === 0) {
245
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
246
+ status: "noop",
247
+ appliedAt: new Date().toISOString(),
248
+ existingGrokRows: 0,
249
+ rowsWritten: 0,
250
+ snapshotsUsed: 0,
251
+ uploadOffsetReset: false,
252
+ };
253
+ return false;
254
+ }
255
+
256
+ const repairRows = buildGrokRepairRowsFromSnapshots(grokState.sessionSnapshots);
257
+ if (repairRows.length === 0) {
258
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
259
+ status: "skipped",
260
+ appliedAt: new Date().toISOString(),
261
+ reason: "missing-session-snapshots",
262
+ existingGrokRows,
263
+ rowsWritten: 0,
264
+ snapshotsUsed: 0,
265
+ uploadOffsetReset: false,
266
+ };
267
+ return false;
268
+ }
269
+
270
+ applyGrokRepairHourlyState(cursors, repairRows);
271
+
272
+ const repairLines = [];
273
+ const repairKeys = new Set();
274
+ for (const row of repairRows) {
275
+ const key = bucketKey("grok", row.model, row.hour_start);
276
+ repairKeys.add(key);
277
+ const current = latestGrokRows.get(key);
278
+ if (current && totalsKey(current) === totalsKey(row)) continue;
279
+ repairLines.push(serializeGrokRepairRow(row));
280
+ }
281
+
282
+ const zeroTotals = {
283
+ input_tokens: 0,
284
+ cached_input_tokens: 0,
285
+ cache_creation_input_tokens: 0,
286
+ output_tokens: 0,
287
+ reasoning_output_tokens: 0,
288
+ total_tokens: 0,
289
+ billable_total_tokens: 0,
290
+ conversation_count: 0,
291
+ };
292
+ let staleRowsRetracted = 0;
293
+ for (const [key, row] of latestGrokRows.entries()) {
294
+ if (repairKeys.has(key)) continue;
295
+ if (totalsKey(row) === totalsKey(zeroTotals)) continue;
296
+ staleRowsRetracted += 1;
297
+ repairLines.push(serializeGrokRepairRow({
298
+ ...zeroTotals,
299
+ model: row.model,
300
+ hour_start: row.hour_start,
301
+ }));
302
+ }
303
+
304
+ if (repairLines.length === 0) {
305
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
306
+ status: "noop",
307
+ appliedAt: new Date().toISOString(),
308
+ existingGrokRows,
309
+ rowsWritten: 0,
310
+ staleRowsRetracted,
311
+ snapshotsUsed: repairRows.length,
312
+ uploadOffsetReset: false,
313
+ };
314
+ return false;
315
+ }
316
+
317
+ await ensureDir(path.dirname(queuePath));
318
+ const queueBackupPath = await backupExistingFile(queuePath);
319
+ const queueStateBackupPath = await backupExistingFile(queueStatePath);
320
+ await fs.appendFile(queuePath, `${repairLines.join("\n")}\n`, "utf8");
321
+
322
+ const uploadOffsetReset = await resetGrokRepairUploadOffset(queueStatePath);
323
+ migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
324
+ status: "applied",
325
+ appliedAt: new Date().toISOString(),
326
+ existingGrokRows,
327
+ rowsWritten: repairLines.length,
328
+ staleRowsRetracted,
329
+ snapshotsUsed: Object.values(grokState.sessionSnapshots || {}).filter((snapshot) => {
330
+ if (!snapshot || typeof snapshot !== "object") return false;
331
+ if (Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens)) <= 0) return false;
332
+ return Boolean(toGrokRepairHalfHourStart(snapshot.lastEventTimestamp || snapshot.updatedAt));
333
+ }).length,
334
+ uploadOffsetReset,
335
+ queueBackupPath,
336
+ queueStateBackupPath,
337
+ };
338
+ return true;
339
+ }
340
+ module.exports = {
341
+ GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
342
+ normalizeGrokRepairSource,
343
+ normalizeGrokRepairModel,
344
+ normalizeGrokRepairNumber,
345
+ toGrokRepairHalfHourStart,
346
+ buildGrokRepairRowsFromSnapshots,
347
+ applyGrokRepairHourlyState,
348
+ resetGrokRepairUploadOffset,
349
+ hasAppliedGrokRepairMigration,
350
+ serializeGrokRepairRow,
351
+ repairGrokQueueFromSessionSnapshots,
352
+ };