tokenusage-cli 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +7 -6
- package/package.json +1 -1
- package/src/cli.js +32 -5
- package/src/commands/device-login.js +16 -17
- package/src/commands/diagnostics.js +16 -16
- package/src/commands/doctor.js +14 -11
- package/src/commands/pause.js +34 -3
- package/src/commands/project-profiles.js +5 -2
- package/src/commands/sessions.js +18 -30
- package/src/commands/status.js +22 -16
- package/src/commands/sync.js +117 -3614
- package/src/commands/uninstall.js +10 -9
- package/src/commands/upload.js +18 -12
- package/src/lib/ark-agent-plan-limits.js +4 -13
- package/src/lib/ark-coding-plan-limits.js +4 -13
- package/src/lib/cli-flags.js +109 -0
- package/src/lib/commandcode-limits.js +11 -38
- package/src/lib/doctor.js +91 -1
- package/src/lib/fs.js +10 -0
- package/src/lib/grok-limits.js +5 -18
- package/src/lib/limits-core.js +64 -0
- package/src/lib/qoder-limits.js +4 -13
- package/src/lib/queue-backup.js +24 -0
- package/src/lib/queue-drain.js +388 -0
- package/src/lib/usage-limits.js +22 -29
- package/src/lib/workbuddy-ipc.js +191 -0
- package/src/lib/workbuddy-limits.js +656 -0
- package/src/lib/zcode-limits.js +5 -18
- package/src/repairs/claude.js +442 -0
- package/src/repairs/codebuddy.js +246 -0
- package/src/repairs/codex.js +1023 -0
- package/src/repairs/droid.js +266 -0
- package/src/repairs/grok.js +352 -0
- package/src/repairs/legacy-migrations.js +135 -0
- package/src/repairs/mimo.js +161 -0
- package/src/repairs/workbuddy.js +209 -0
- package/src/repairs/zcode.js +647 -0
|
@@ -0,0 +1,1023 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Codex repair family, split out of commands/sync.js: the cold-scan
|
|
4
|
+
// double-count repair, the fork-replay inflation repair and the interleaved
|
|
5
|
+
// usage-lineage repair, plus the snapshot/lineage scanners and the max-totals
|
|
6
|
+
// helpers they share. All are one-shot corrections keyed by cursors.migrations.
|
|
7
|
+
|
|
8
|
+
const fs = require("node:fs/promises");
|
|
9
|
+
const fssync = require("node:fs");
|
|
10
|
+
const path = require("node:path");
|
|
11
|
+
const {
|
|
12
|
+
extractTokenCount,
|
|
13
|
+
} = require("../lib/codex-rollout-parser");
|
|
14
|
+
const {
|
|
15
|
+
consumeUsageDelta,
|
|
16
|
+
createUsageDeltaState,
|
|
17
|
+
} = require("../lib/codex-token-usage");
|
|
18
|
+
const {
|
|
19
|
+
isCodexSessionCursorPath,
|
|
20
|
+
} = require("../lib/cursor-store");
|
|
21
|
+
const {
|
|
22
|
+
ensureDir,
|
|
23
|
+
} = require("../lib/fs");
|
|
24
|
+
const {
|
|
25
|
+
physicalJsonlRecords,
|
|
26
|
+
} = require("../lib/jsonl-lines");
|
|
27
|
+
const {
|
|
28
|
+
codexSessionIdFromPath,
|
|
29
|
+
parseDroidIncremental,
|
|
30
|
+
parseRolloutIncremental,
|
|
31
|
+
readCodexRolloutSnapshot,
|
|
32
|
+
} = require("../lib/rollout");
|
|
33
|
+
|
|
34
|
+
const CODEX_RESCAN_DEDUP_REPAIR_KEY = "codexRescanDedupRepair_2026_06";
|
|
35
|
+
const CODEX_FORK_REPLAY_REPAIR_KEY = "codexForkReplayRepair_2026_07";
|
|
36
|
+
const CODEX_USAGE_LINEAGE_REPAIR_KEY = "codexUsageLineageRepair_2026_07_v2";
|
|
37
|
+
|
|
38
|
+
async function repairCodexRescanInflation({
|
|
39
|
+
cursors,
|
|
40
|
+
queuePath,
|
|
41
|
+
queueStatePath,
|
|
42
|
+
projectQueuePath,
|
|
43
|
+
projectQueueStatePath,
|
|
44
|
+
rolloutFiles,
|
|
45
|
+
expectedCodexFileSnapshots = null,
|
|
46
|
+
// The fork-replay repair (#169 follow-up) re-runs this exact rebuild under
|
|
47
|
+
// its own key: the rebuild always uses the CURRENT parser, so any parser fix
|
|
48
|
+
// shipped since the last run is applied to the rebuilt history.
|
|
49
|
+
migrationKey = CODEX_RESCAN_DEDUP_REPAIR_KEY,
|
|
50
|
+
uploadNote = "reset_after_codex_rescan_dedup_2026_06",
|
|
51
|
+
}) {
|
|
52
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
53
|
+
const migrations = (cursors.migrations ||= {});
|
|
54
|
+
// A COMPLETED run writes an ISO-string timestamp (final — never re-run). A
|
|
55
|
+
// prior SKIP writes an object {skipped:true} and MUST be retried: the skip
|
|
56
|
+
// condition can clear in a later version (e.g. v0.53.4 started scanning
|
|
57
|
+
// archived_sessions/, so a session that was "unscanned" under v0.53.3 is now
|
|
58
|
+
// found). Treating the skip sentinel as "done" is what left users like #187
|
|
59
|
+
// permanently stuck on the inflated value after upgrading (the key was truthy
|
|
60
|
+
// so the guard never got a second chance).
|
|
61
|
+
const priorRepair = migrations[migrationKey];
|
|
62
|
+
if (priorRepair && !(typeof priorRepair === "object" && priorRepair.skipped)) return false;
|
|
63
|
+
|
|
64
|
+
// Codex session files THIS sync discovered (source === "codex").
|
|
65
|
+
const codexFiles = [];
|
|
66
|
+
for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
|
|
67
|
+
const fp = typeof entry === "string" ? entry : entry?.path;
|
|
68
|
+
const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
|
|
69
|
+
if (fp && src === "codex") codexFiles.push(fp);
|
|
70
|
+
}
|
|
71
|
+
const codexFileSet = new Set(codexFiles);
|
|
72
|
+
const projectRepairEnabled = typeof projectQueuePath === "string" && projectQueuePath.length > 0;
|
|
73
|
+
|
|
74
|
+
// GUARD (data-loss prevention, ref the v6 ground-truth-repair incident): the
|
|
75
|
+
// rebuild can only reproduce buckets from the files this sync re-parses
|
|
76
|
+
// (codexFiles, now covering sessions/ AND archived_sessions/). If a session
|
|
77
|
+
// that previously contributed can no longer be reproduced, skip entirely —
|
|
78
|
+
// the forward dedup fix still stops NEW double-counting; only this historical
|
|
79
|
+
// correction defers.
|
|
80
|
+
//
|
|
81
|
+
// Reproducibility is keyed on the session UUID, NOT the exact cursor path:
|
|
82
|
+
// Codex-Manager MOVES a file sessions/ -> archived_sessions/ (path changes,
|
|
83
|
+
// UUID does not). A path-based check false-positives on every moved file as
|
|
84
|
+
// "missing" and skips forever (issue #187, easonlee05). Genuinely deleted
|
|
85
|
+
// sessions (no file with that UUID anywhere in the scan) still defer.
|
|
86
|
+
const scannedSessionIds = new Set();
|
|
87
|
+
for (const fp of codexFiles) {
|
|
88
|
+
const id = codexSessionIdFromPath(fp);
|
|
89
|
+
if (id) scannedSessionIds.add(id);
|
|
90
|
+
}
|
|
91
|
+
if (cursors.files && typeof cursors.files === "object") {
|
|
92
|
+
for (const fp of Object.keys(cursors.files)) {
|
|
93
|
+
if (!isCodexSessionCursorPath(fp)) continue;
|
|
94
|
+
if (codexFileSet.has(fp)) continue; // exact file re-scanned this run
|
|
95
|
+
const id = codexSessionIdFromPath(fp);
|
|
96
|
+
if (id && scannedSessionIds.has(id)) continue; // same session scanned elsewhere (moved)
|
|
97
|
+
migrations[migrationKey] = {
|
|
98
|
+
skipped: true,
|
|
99
|
+
reason: "codex_session_unreproducible",
|
|
100
|
+
at: new Date().toISOString(),
|
|
101
|
+
};
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ATOMIC REBUILD into a THROWAWAY cursors + queue: never touch the live
|
|
107
|
+
// buckets/queue until the rebuild has fully succeeded. If anything throws we
|
|
108
|
+
// return WITHOUT mutating any persistent state and the migration retries next
|
|
109
|
+
// sync (the key is not set). This is the crucial difference from a
|
|
110
|
+
// "clear-then-rely-on-the-later-parse" design: the later parse's failure is
|
|
111
|
+
// swallowed (warnProviderParseFailure) and cursors are saved regardless, which
|
|
112
|
+
// would permanently zero a user's codex history.
|
|
113
|
+
let rebuilt;
|
|
114
|
+
const tmpQueue = `${queuePath}.codexrebuild.${process.pid}.${Date.now()}`;
|
|
115
|
+
const tmpProjectQueue = projectRepairEnabled
|
|
116
|
+
? `${projectQueuePath}.codexrebuild.${process.pid}.${Date.now()}`
|
|
117
|
+
: null;
|
|
118
|
+
try {
|
|
119
|
+
const tmpCursors = {
|
|
120
|
+
version: 1,
|
|
121
|
+
files: {},
|
|
122
|
+
hourly: { buckets: {}, groupQueued: {} },
|
|
123
|
+
codexHashes: [],
|
|
124
|
+
};
|
|
125
|
+
await parseRolloutIncremental({
|
|
126
|
+
rolloutFiles: codexFiles.map((p) => ({ path: p, source: "codex" })),
|
|
127
|
+
cursors: tmpCursors,
|
|
128
|
+
queuePath: tmpQueue,
|
|
129
|
+
projectQueuePath: tmpProjectQueue,
|
|
130
|
+
invalidRecordPolicy: "throw",
|
|
131
|
+
});
|
|
132
|
+
let tmpRaw = "";
|
|
133
|
+
try {
|
|
134
|
+
tmpRaw = await fs.readFile(tmpQueue, "utf8");
|
|
135
|
+
} catch (e) {
|
|
136
|
+
if (e?.code !== "ENOENT") throw e;
|
|
137
|
+
}
|
|
138
|
+
let tmpProjectRaw = "";
|
|
139
|
+
if (tmpProjectQueue) {
|
|
140
|
+
try {
|
|
141
|
+
tmpProjectRaw = await fs.readFile(tmpProjectQueue, "utf8");
|
|
142
|
+
} catch (e) {
|
|
143
|
+
if (e?.code !== "ENOENT") throw e;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
rebuilt = {
|
|
147
|
+
buckets: tmpCursors.hourly.buckets || {},
|
|
148
|
+
groupQueued: tmpCursors.hourly.groupQueued || {},
|
|
149
|
+
codexHashes: Array.isArray(tmpCursors.codexHashes) ? tmpCursors.codexHashes : [],
|
|
150
|
+
codexCompactionResponseIds: Array.isArray(tmpCursors.codexCompactionResponseIds)
|
|
151
|
+
? tmpCursors.codexCompactionResponseIds
|
|
152
|
+
: [],
|
|
153
|
+
files: tmpCursors.files || {},
|
|
154
|
+
queueRows: tmpRaw.split("\n").filter((l) => l.trim()),
|
|
155
|
+
projectHourly: tmpCursors.projectHourly || null,
|
|
156
|
+
projectQueueRows: tmpProjectRaw.split("\n").filter((l) => l.trim()),
|
|
157
|
+
};
|
|
158
|
+
} catch (e) {
|
|
159
|
+
console.error(
|
|
160
|
+
"[sync] codex rescan repair: rebuild failed, leaving all data untouched:",
|
|
161
|
+
e?.message || e,
|
|
162
|
+
);
|
|
163
|
+
return false;
|
|
164
|
+
} finally {
|
|
165
|
+
await fs.rm(tmpQueue, { force: true }).catch(() => {});
|
|
166
|
+
if (tmpProjectQueue) await fs.rm(tmpProjectQueue, { force: true }).catch(() => {});
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// SANITY: codex files exist on disk but the rebuild produced no codex buckets
|
|
170
|
+
// → treat as a failed rebuild and skip (do NOT clear live data, do NOT set the
|
|
171
|
+
// key — retry next sync).
|
|
172
|
+
const rebuiltCodexKeys = Object.keys(rebuilt.buckets).filter((k) => k.startsWith("codex|"));
|
|
173
|
+
if (codexFiles.length > 0 && rebuiltCodexKeys.length === 0) {
|
|
174
|
+
console.error(
|
|
175
|
+
`[sync] codex rescan repair: rebuild produced 0 codex buckets from ${codexFiles.length} files — skipping to avoid data loss`,
|
|
176
|
+
);
|
|
177
|
+
return false;
|
|
178
|
+
}
|
|
179
|
+
if (projectRepairEnabled) {
|
|
180
|
+
const malformedProjectRows = await countMalformedCodexProjectQueueRows(projectQueuePath);
|
|
181
|
+
if (malformedProjectRows > 0) {
|
|
182
|
+
console.error(
|
|
183
|
+
`[sync] codex rescan repair: found ${malformedProjectRows} malformed codex project queue row(s) — skipping to avoid data loss`,
|
|
184
|
+
);
|
|
185
|
+
return false;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const existingProjectKeys = new Set([
|
|
189
|
+
...(await projectUsageKeysFromQueuePath(projectQueuePath, "codex")),
|
|
190
|
+
...projectUsageKeysFromState(cursors.projectHourly, "codex"),
|
|
191
|
+
]);
|
|
192
|
+
const rebuiltProjectKeys = new Set([
|
|
193
|
+
...projectUsageKeysFromQueueRows(rebuilt.projectQueueRows, "codex"),
|
|
194
|
+
...projectUsageKeysFromState(rebuilt.projectHourly, "codex"),
|
|
195
|
+
]);
|
|
196
|
+
const missingProjectKeys = [...existingProjectKeys].filter((key) => !rebuiltProjectKeys.has(key));
|
|
197
|
+
if (missingProjectKeys.length > 0) {
|
|
198
|
+
console.error(
|
|
199
|
+
`[sync] codex rescan repair: project rebuild missed ${missingProjectKeys.length} existing codex project bucket(s) — skipping to avoid data loss`,
|
|
200
|
+
);
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const existingProjectTotals = mergeMaxTotals(
|
|
205
|
+
await projectUsageTotalsFromQueuePath(projectQueuePath, "codex"),
|
|
206
|
+
projectUsageTotalsFromState(cursors.projectHourly, "codex"),
|
|
207
|
+
);
|
|
208
|
+
const rebuiltProjectTotals = mergeMaxTotals(
|
|
209
|
+
projectUsageTotalsFromQueueRows(rebuilt.projectQueueRows, "codex"),
|
|
210
|
+
projectUsageTotalsFromState(rebuilt.projectHourly, "codex"),
|
|
211
|
+
);
|
|
212
|
+
const existingMainHourTotals = mergeMaxTotals(
|
|
213
|
+
await mainUsageHourTotalsFromQueuePath(queuePath, "codex"),
|
|
214
|
+
mainUsageHourTotalsFromState(cursors.hourly, "codex"),
|
|
215
|
+
);
|
|
216
|
+
const rebuiltMainHourTotals = mergeMaxTotals(
|
|
217
|
+
mainUsageHourTotalsFromQueueRows(rebuilt.queueRows, "codex"),
|
|
218
|
+
mainUsageHourTotalsFromState({ buckets: rebuilt.buckets }, "codex"),
|
|
219
|
+
);
|
|
220
|
+
const partialProjectKeys = [];
|
|
221
|
+
for (const [key, existingTotal] of existingProjectTotals.entries()) {
|
|
222
|
+
const rebuiltTotal = rebuiltProjectTotals.get(key);
|
|
223
|
+
if (!Number.isFinite(rebuiltTotal) || rebuiltTotal >= existingTotal) continue;
|
|
224
|
+
const [, source, hourStart] = key.split("|");
|
|
225
|
+
const mainKey = `${source}|${hourStart}`;
|
|
226
|
+
const existingMainTotal = existingMainHourTotals.get(mainKey) || 0;
|
|
227
|
+
const rebuiltMainTotal = rebuiltMainHourTotals.get(mainKey) || 0;
|
|
228
|
+
if (rebuiltMainTotal >= existingMainTotal) partialProjectKeys.push(key);
|
|
229
|
+
}
|
|
230
|
+
if (partialProjectKeys.length > 0) {
|
|
231
|
+
console.error(
|
|
232
|
+
`[sync] codex rescan repair: project rebuild lowered ${partialProjectKeys.length} existing codex project bucket(s) without a matching main-bucket repair — skipping to avoid data loss`,
|
|
233
|
+
);
|
|
234
|
+
return false;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const rebuildValidation = expectedCodexFileSnapshots instanceof Map
|
|
239
|
+
? await validateCodexRebuildFileSnapshots(rebuilt.files, expectedCodexFileSnapshots)
|
|
240
|
+
: { ok: true };
|
|
241
|
+
if (!rebuildValidation.ok) {
|
|
242
|
+
console.error(
|
|
243
|
+
`[sync] codex rescan repair: ${rebuildValidation.reason} — skipping to avoid data loss`,
|
|
244
|
+
);
|
|
245
|
+
return false;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// COMMIT (only after a verified rebuild). A crash partway just leaves the
|
|
249
|
+
// migration to re-run next sync — re-rebuild + re-strip + re-commit converges.
|
|
250
|
+
//
|
|
251
|
+
// 1. queue.jsonl: drop the inflated codex rows, append the clean rebuilt ones
|
|
252
|
+
// (atomic tmp+rename). With no old-high codex rows left, the cloud ingest's
|
|
253
|
+
// within-batch MAX keeps nothing larger and its overwrite-upsert replaces
|
|
254
|
+
// the inflated cloud rows on the next upload.
|
|
255
|
+
if (typeof queuePath === "string" && queuePath) {
|
|
256
|
+
let raw = "";
|
|
257
|
+
try {
|
|
258
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
259
|
+
} catch (e) {
|
|
260
|
+
if (e?.code !== "ENOENT") throw e;
|
|
261
|
+
}
|
|
262
|
+
const kept = [];
|
|
263
|
+
for (const line of raw.split("\n")) {
|
|
264
|
+
if (!line.trim()) continue;
|
|
265
|
+
let row;
|
|
266
|
+
try {
|
|
267
|
+
row = JSON.parse(line);
|
|
268
|
+
} catch (_e) {
|
|
269
|
+
kept.push(line);
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
if (row?.source === "codex") continue;
|
|
273
|
+
kept.push(line);
|
|
274
|
+
}
|
|
275
|
+
await ensureDir(path.dirname(queuePath));
|
|
276
|
+
const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
|
|
277
|
+
await fs.writeFile(tmp, kept.concat(rebuilt.queueRows).join("\n") + "\n", "utf8");
|
|
278
|
+
await fs.rename(tmp, queuePath);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// 2. Swap the live codex hourly state for the rebuilt one, and install the
|
|
282
|
+
// rebuilt per-file cursors (offset at EOF) so the later parse in THIS sync
|
|
283
|
+
// does not re-read codex (which would re-inflate project buckets).
|
|
284
|
+
const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
|
|
285
|
+
hourly.buckets ||= {};
|
|
286
|
+
hourly.groupQueued ||= {};
|
|
287
|
+
for (const k of Object.keys(hourly.buckets)) {
|
|
288
|
+
if (k.startsWith("codex|")) delete hourly.buckets[k];
|
|
289
|
+
}
|
|
290
|
+
for (const k of Object.keys(hourly.groupQueued)) {
|
|
291
|
+
if (k.startsWith("codex|")) delete hourly.groupQueued[k];
|
|
292
|
+
}
|
|
293
|
+
for (const [k, v] of Object.entries(rebuilt.buckets)) {
|
|
294
|
+
if (k.startsWith("codex|")) hourly.buckets[k] = v;
|
|
295
|
+
}
|
|
296
|
+
for (const [k, v] of Object.entries(rebuilt.groupQueued)) {
|
|
297
|
+
if (k.startsWith("codex|")) hourly.groupQueued[k] = v;
|
|
298
|
+
}
|
|
299
|
+
cursors.files ||= {};
|
|
300
|
+
for (const fp of Object.keys(cursors.files)) {
|
|
301
|
+
if (isCodexSessionCursorPath(fp)) delete cursors.files[fp];
|
|
302
|
+
}
|
|
303
|
+
for (const [fp, v] of Object.entries(rebuilt.files)) {
|
|
304
|
+
cursors.files[fp] = v;
|
|
305
|
+
}
|
|
306
|
+
cursors.codexHashes = rebuilt.codexHashes;
|
|
307
|
+
cursors.codexCompactionResponseIds = rebuilt.codexCompactionResponseIds;
|
|
308
|
+
|
|
309
|
+
// 3. Project usage mirrors the main Codex repair: drop inflated Codex project
|
|
310
|
+
// rows, append the rebuilt rows, and swap only Codex project buckets. Project
|
|
311
|
+
// metadata is merged so visibility/purge state from non-Codex sources stays.
|
|
312
|
+
if (projectRepairEnabled) {
|
|
313
|
+
let projectRaw = "";
|
|
314
|
+
try {
|
|
315
|
+
projectRaw = await fs.readFile(projectQueuePath, "utf8");
|
|
316
|
+
} catch (e) {
|
|
317
|
+
if (e?.code !== "ENOENT") throw e;
|
|
318
|
+
}
|
|
319
|
+
const keptProjectRows = [];
|
|
320
|
+
for (const line of projectRaw.split("\n")) {
|
|
321
|
+
if (!line.trim()) continue;
|
|
322
|
+
let row;
|
|
323
|
+
try {
|
|
324
|
+
row = JSON.parse(line);
|
|
325
|
+
} catch (_e) {
|
|
326
|
+
keptProjectRows.push(line);
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
if (row?.source === "codex") continue;
|
|
330
|
+
keptProjectRows.push(line);
|
|
331
|
+
}
|
|
332
|
+
await ensureDir(path.dirname(projectQueuePath));
|
|
333
|
+
const tmp = `${projectQueuePath}.tmp.${process.pid}.${Date.now()}`;
|
|
334
|
+
await fs.writeFile(
|
|
335
|
+
tmp,
|
|
336
|
+
keptProjectRows.concat(rebuilt.projectQueueRows).join("\n") + "\n",
|
|
337
|
+
"utf8",
|
|
338
|
+
);
|
|
339
|
+
await fs.rename(tmp, projectQueuePath);
|
|
340
|
+
|
|
341
|
+
const projectHourly = (cursors.projectHourly ||= { version: 2, buckets: {}, projects: {} });
|
|
342
|
+
projectHourly.version = 2;
|
|
343
|
+
projectHourly.buckets ||= {};
|
|
344
|
+
projectHourly.projects ||= {};
|
|
345
|
+
for (const [key, bucket] of Object.entries(projectHourly.buckets)) {
|
|
346
|
+
const source = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
|
|
347
|
+
if (source === "codex") delete projectHourly.buckets[key];
|
|
348
|
+
}
|
|
349
|
+
const rebuiltProjectHourly = rebuilt.projectHourly || {};
|
|
350
|
+
for (const [key, bucket] of Object.entries(rebuiltProjectHourly.buckets || {})) {
|
|
351
|
+
const source = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
|
|
352
|
+
if (source === "codex") projectHourly.buckets[key] = bucket;
|
|
353
|
+
}
|
|
354
|
+
for (const [key, meta] of Object.entries(rebuiltProjectHourly.projects || {})) {
|
|
355
|
+
if (meta && typeof meta === "object") projectHourly.projects[key] = meta;
|
|
356
|
+
}
|
|
357
|
+
projectHourly.updatedAt = new Date().toISOString();
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// 4. Reset the cloud upload offset so the corrected queue re-uploads. Other
|
|
361
|
+
// sources re-upsert idempotently (last emission per key wins).
|
|
362
|
+
if (typeof queueStatePath === "string" && queueStatePath) {
|
|
363
|
+
let uploadState = {};
|
|
364
|
+
try {
|
|
365
|
+
uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
|
|
366
|
+
} catch (_e) {
|
|
367
|
+
uploadState = {};
|
|
368
|
+
}
|
|
369
|
+
uploadState.offset = 0;
|
|
370
|
+
uploadState.updatedAt = new Date().toISOString();
|
|
371
|
+
uploadState.note = uploadNote;
|
|
372
|
+
await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
|
|
373
|
+
}
|
|
374
|
+
if (projectRepairEnabled && typeof projectQueueStatePath === "string" && projectQueueStatePath) {
|
|
375
|
+
let uploadState = {};
|
|
376
|
+
try {
|
|
377
|
+
uploadState = JSON.parse(await fs.readFile(projectQueueStatePath, "utf8"));
|
|
378
|
+
} catch (_e) {
|
|
379
|
+
uploadState = {};
|
|
380
|
+
}
|
|
381
|
+
uploadState.offset = 0;
|
|
382
|
+
uploadState.updatedAt = new Date().toISOString();
|
|
383
|
+
uploadState.note = uploadNote;
|
|
384
|
+
await fs.writeFile(projectQueueStatePath, JSON.stringify(uploadState));
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
migrations[migrationKey] = new Date().toISOString();
|
|
388
|
+
return true;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// #169 follow-up: repair the historical inflation left by same-day forked
|
|
392
|
+
// Codex rollout replays (see CODEX_FORK_REPLAY_REPAIR_KEY). Delegates to the
|
|
393
|
+
// #187 guarded rebuild with the fork-repair key.
|
|
394
|
+
async function repairCodexForkReplayInflation({
|
|
395
|
+
cursors,
|
|
396
|
+
queuePath,
|
|
397
|
+
queueStatePath,
|
|
398
|
+
projectQueuePath,
|
|
399
|
+
projectQueueStatePath,
|
|
400
|
+
rolloutFiles,
|
|
401
|
+
// True when the #187 repair ran its rebuild earlier in THIS sync: that
|
|
402
|
+
// rebuild already used the fork-aware parser, so the history is clean and a
|
|
403
|
+
// second rebuild would be pure waste.
|
|
404
|
+
legacyRepairRan = false,
|
|
405
|
+
}) {
|
|
406
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
407
|
+
const migrations = (cursors.migrations ||= {});
|
|
408
|
+
// Same completed-vs-skipped semantics as the #187 repair: an ISO string is
|
|
409
|
+
// final, a {skipped:true} sentinel retries (the skip condition can clear).
|
|
410
|
+
const prior = migrations[CODEX_FORK_REPLAY_REPAIR_KEY];
|
|
411
|
+
if (prior && !(typeof prior === "object" && prior.skipped)) return false;
|
|
412
|
+
|
|
413
|
+
if (legacyRepairRan) {
|
|
414
|
+
migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = new Date().toISOString();
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// Pre-gate: fork phantom can only exist on installs that have (or had) a
|
|
419
|
+
// forked rollout. Scanning file heads is cheap; the rebuild re-parses every
|
|
420
|
+
// codex file. Files forked AFTER the parser fix shipped never accrue
|
|
421
|
+
// phantom, so a definitive "no forks" verdict may finalize — but ONLY when
|
|
422
|
+
// this scan can actually stand for the install's whole codex history:
|
|
423
|
+
// 1. every candidate head was readable (an unreadable candidate could be
|
|
424
|
+
// the one forked rollout), AND
|
|
425
|
+
// 2. every codex session recorded in cursors is covered by THIS scan's
|
|
426
|
+
// file list (same UUID-keyed reproducibility rule as the #187 guard).
|
|
427
|
+
// A forked rollout can be real yet absent from one run's rolloutFiles —
|
|
428
|
+
// WSL/native split probing, a temporarily missing ~/.codex, a deleted
|
|
429
|
+
// file whose phantom is already in history — and finalizing off such a
|
|
430
|
+
// partial view would permanently foreclose the repair this migration
|
|
431
|
+
// exists to perform, while the guarded repair itself would only have
|
|
432
|
+
// deferred retryably.
|
|
433
|
+
// Either failure defers with the retryable {skipped:true} sentinel.
|
|
434
|
+
const forkScan = await scanForForkedCodexRollout(rolloutFiles);
|
|
435
|
+
if (!forkScan.forked) {
|
|
436
|
+
if (forkScan.indeterminate) {
|
|
437
|
+
migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = {
|
|
438
|
+
skipped: true,
|
|
439
|
+
reason: "fork_scan_indeterminate",
|
|
440
|
+
at: new Date().toISOString(),
|
|
441
|
+
};
|
|
442
|
+
return false;
|
|
443
|
+
}
|
|
444
|
+
const scannedSessionIds = new Set();
|
|
445
|
+
const scannedPaths = new Set();
|
|
446
|
+
for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
|
|
447
|
+
const fp = typeof entry === "string" ? entry : entry?.path;
|
|
448
|
+
const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
|
|
449
|
+
if (!fp || src !== "codex") continue;
|
|
450
|
+
scannedPaths.add(fp);
|
|
451
|
+
const id = codexSessionIdFromPath(fp);
|
|
452
|
+
if (id) scannedSessionIds.add(id);
|
|
453
|
+
}
|
|
454
|
+
if (cursors.files && typeof cursors.files === "object") {
|
|
455
|
+
for (const fp of Object.keys(cursors.files)) {
|
|
456
|
+
if (!isCodexSessionCursorPath(fp)) continue;
|
|
457
|
+
if (scannedPaths.has(fp)) continue;
|
|
458
|
+
const id = codexSessionIdFromPath(fp);
|
|
459
|
+
if (id && scannedSessionIds.has(id)) continue;
|
|
460
|
+
migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = {
|
|
461
|
+
skipped: true,
|
|
462
|
+
reason: "codex_history_not_covered",
|
|
463
|
+
at: new Date().toISOString(),
|
|
464
|
+
};
|
|
465
|
+
return false;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = new Date().toISOString();
|
|
469
|
+
return false;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
return repairCodexRescanInflation({
|
|
473
|
+
cursors,
|
|
474
|
+
queuePath,
|
|
475
|
+
queueStatePath,
|
|
476
|
+
projectQueuePath,
|
|
477
|
+
projectQueueStatePath,
|
|
478
|
+
rolloutFiles,
|
|
479
|
+
migrationKey: CODEX_FORK_REPLAY_REPAIR_KEY,
|
|
480
|
+
uploadNote: "reset_after_codex_fork_replay_2026_07",
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
// Rebuild historical Codex usage only when a rollout proves that the old
|
|
485
|
+
// single-baseline parser crossed cumulative SessionState lineages. Detection is
|
|
486
|
+
// based on the token counter stream itself, not multi_agent_version metadata:
|
|
487
|
+
// affected legacy rollouts can predate that marker or carry it only in the
|
|
488
|
+
// middle of a large file. Every contributing rollout must also be valid and
|
|
489
|
+
// stable before the atomic rebuild can replace live history.
|
|
490
|
+
async function repairCodexInterleavedUsageInflation({
|
|
491
|
+
cursors,
|
|
492
|
+
queuePath,
|
|
493
|
+
queueStatePath,
|
|
494
|
+
projectQueuePath,
|
|
495
|
+
projectQueueStatePath,
|
|
496
|
+
rolloutFiles,
|
|
497
|
+
// Either older guarded repair rebuilt every Codex file with the current
|
|
498
|
+
// parser earlier in this sync, so a second full rebuild is unnecessary.
|
|
499
|
+
legacyRepairRan = false,
|
|
500
|
+
maxLineageScanBytes = Infinity,
|
|
501
|
+
}) {
|
|
502
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
503
|
+
const migrations = (cursors.migrations ||= {});
|
|
504
|
+
const prior = migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY];
|
|
505
|
+
if (prior && !(typeof prior === "object" && prior.skipped)) return false;
|
|
506
|
+
|
|
507
|
+
if (legacyRepairRan) {
|
|
508
|
+
migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = new Date().toISOString();
|
|
509
|
+
return false;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
const scan = await scanForInterleavedCodexUsage(rolloutFiles, {
|
|
513
|
+
maxLineageScanBytes,
|
|
514
|
+
});
|
|
515
|
+
if (scan.indeterminate) {
|
|
516
|
+
migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = {
|
|
517
|
+
skipped: true,
|
|
518
|
+
reason: "usage_lineage_scan_indeterminate",
|
|
519
|
+
at: new Date().toISOString(),
|
|
520
|
+
};
|
|
521
|
+
return false;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
if (!scan.affected) {
|
|
525
|
+
if (!isCodexHistoryCovered(cursors, rolloutFiles)) {
|
|
526
|
+
migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = {
|
|
527
|
+
skipped: true,
|
|
528
|
+
reason: "codex_history_not_covered",
|
|
529
|
+
at: new Date().toISOString(),
|
|
530
|
+
};
|
|
531
|
+
return false;
|
|
532
|
+
}
|
|
533
|
+
migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = new Date().toISOString();
|
|
534
|
+
return false;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
return repairCodexRescanInflation({
|
|
538
|
+
cursors,
|
|
539
|
+
queuePath,
|
|
540
|
+
queueStatePath,
|
|
541
|
+
projectQueuePath,
|
|
542
|
+
projectQueueStatePath,
|
|
543
|
+
rolloutFiles,
|
|
544
|
+
expectedCodexFileSnapshots: scan.fileSnapshots,
|
|
545
|
+
migrationKey: CODEX_USAGE_LINEAGE_REPAIR_KEY,
|
|
546
|
+
uploadNote: "reset_after_codex_usage_lineage_2026_07_v2",
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
async function scanForInterleavedCodexUsage(
|
|
551
|
+
rolloutFiles,
|
|
552
|
+
{ maxLineageScanBytes = Infinity } = {},
|
|
553
|
+
) {
|
|
554
|
+
let affected = false;
|
|
555
|
+
let remainingBytes = Number.isFinite(maxLineageScanBytes)
|
|
556
|
+
? Math.max(0, maxLineageScanBytes)
|
|
557
|
+
: Infinity;
|
|
558
|
+
const fileSnapshots = new Map();
|
|
559
|
+
const seenPaths = new Set();
|
|
560
|
+
|
|
561
|
+
for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
|
|
562
|
+
const fp = typeof entry === "string" ? entry : entry?.path;
|
|
563
|
+
const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
|
|
564
|
+
if (!fp || src !== "codex" || seenPaths.has(fp)) continue;
|
|
565
|
+
seenPaths.add(fp);
|
|
566
|
+
|
|
567
|
+
const lineage = await scanCodexUsageLineages(fp, remainingBytes);
|
|
568
|
+
if (lineage.indeterminate) {
|
|
569
|
+
return { affected: false, indeterminate: true, fileSnapshots: new Map() };
|
|
570
|
+
}
|
|
571
|
+
affected ||= lineage.affected;
|
|
572
|
+
fileSnapshots.set(fp, lineage.fileSnapshot);
|
|
573
|
+
if (Number.isFinite(remainingBytes)) {
|
|
574
|
+
remainingBytes = Math.max(0, remainingBytes - lineage.bytesRead);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
return { affected, indeterminate: false, fileSnapshots };
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
async function scanCodexUsageLineages(filePath, maxBytes = Infinity) {
|
|
581
|
+
let stream = null;
|
|
582
|
+
try {
|
|
583
|
+
const before = await readCodexFileSnapshot(filePath);
|
|
584
|
+
const state = createUsageDeltaState();
|
|
585
|
+
const byteLimit = Number.isFinite(maxBytes) ? Math.max(0, maxBytes) : Infinity;
|
|
586
|
+
let bytesRead = 0;
|
|
587
|
+
let affected = false;
|
|
588
|
+
const isCompressed = typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath);
|
|
589
|
+
if (isCompressed) {
|
|
590
|
+
// Honor the background-sync lineage budget against compressed size so a
|
|
591
|
+
// pending repair with large archives stays bounded and returns
|
|
592
|
+
// indeterminate until a full sync can finish the scan.
|
|
593
|
+
if (!(byteLimit > 0) || before.size > byteLimit) {
|
|
594
|
+
return { affected: false, indeterminate: true };
|
|
595
|
+
}
|
|
596
|
+
const snapshot = await readCodexRolloutSnapshot(filePath);
|
|
597
|
+
if (!snapshot) return { affected: false, indeterminate: true };
|
|
598
|
+
for (const raw of snapshot.lines) {
|
|
599
|
+
const line = typeof raw === "string" ? raw : "";
|
|
600
|
+
if (!line.trim()) continue;
|
|
601
|
+
let obj;
|
|
602
|
+
try {
|
|
603
|
+
obj = JSON.parse(line);
|
|
604
|
+
} catch (_e) {
|
|
605
|
+
return { affected: false, indeterminate: true };
|
|
606
|
+
}
|
|
607
|
+
const token = extractTokenCount(obj);
|
|
608
|
+
const info = token?.info;
|
|
609
|
+
if (!info || typeof info !== "object") continue;
|
|
610
|
+
consumeUsageDelta(state, info.last_token_usage, info.total_token_usage);
|
|
611
|
+
if (state.sawInterleaved || state.sawDivergentCumulative) {
|
|
612
|
+
affected = true;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
bytesRead = snapshot.size;
|
|
616
|
+
} else {
|
|
617
|
+
stream = fssync.createReadStream(filePath, { highWaterMark: 32 * 1024 });
|
|
618
|
+
for await (const record of physicalJsonlRecords(stream, {
|
|
619
|
+
maxPhysicalBytes: byteLimit,
|
|
620
|
+
})) {
|
|
621
|
+
const { line } = record;
|
|
622
|
+
bytesRead += record.physicalBytes;
|
|
623
|
+
if (bytesRead > byteLimit) return { affected: false, indeterminate: true };
|
|
624
|
+
if (!line.trim()) continue;
|
|
625
|
+
let obj;
|
|
626
|
+
try {
|
|
627
|
+
obj = JSON.parse(line);
|
|
628
|
+
} catch (_e) {
|
|
629
|
+
return { affected: false, indeterminate: true };
|
|
630
|
+
}
|
|
631
|
+
const token = extractTokenCount(obj);
|
|
632
|
+
const info = token?.info;
|
|
633
|
+
if (!info || typeof info !== "object") continue;
|
|
634
|
+
consumeUsageDelta(state, info.last_token_usage, info.total_token_usage);
|
|
635
|
+
if (state.sawInterleaved || state.sawDivergentCumulative) {
|
|
636
|
+
affected = true;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
const after = await readCodexFileSnapshot(filePath);
|
|
641
|
+
if (!sameCodexFileSnapshot(before, after)) {
|
|
642
|
+
return { affected: false, indeterminate: true };
|
|
643
|
+
}
|
|
644
|
+
return {
|
|
645
|
+
affected,
|
|
646
|
+
indeterminate: false,
|
|
647
|
+
bytesRead,
|
|
648
|
+
fileSnapshot: after,
|
|
649
|
+
};
|
|
650
|
+
} catch (_e) {
|
|
651
|
+
return { affected: false, indeterminate: true };
|
|
652
|
+
} finally {
|
|
653
|
+
if (stream) stream.destroy();
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
async function readCodexFileSnapshot(filePath) {
|
|
658
|
+
const stat = await fs.stat(filePath);
|
|
659
|
+
return {
|
|
660
|
+
size: Number(stat.size),
|
|
661
|
+
mtimeMs: Number(stat.mtimeMs),
|
|
662
|
+
ctimeMs: Number(stat.ctimeMs),
|
|
663
|
+
ino: Number(stat.ino),
|
|
664
|
+
dev: Number(stat.dev),
|
|
665
|
+
};
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
function sameCodexFileSnapshot(left, right) {
|
|
669
|
+
return Boolean(
|
|
670
|
+
left &&
|
|
671
|
+
right &&
|
|
672
|
+
left.size === right.size &&
|
|
673
|
+
left.mtimeMs === right.mtimeMs &&
|
|
674
|
+
left.ctimeMs === right.ctimeMs &&
|
|
675
|
+
left.ino === right.ino &&
|
|
676
|
+
left.dev === right.dev
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
async function validateCodexRebuildFileSnapshots(rebuiltFiles, expectedSnapshots) {
|
|
681
|
+
for (const [filePath, expected] of expectedSnapshots) {
|
|
682
|
+
let actual;
|
|
683
|
+
try {
|
|
684
|
+
actual = await readCodexFileSnapshot(filePath);
|
|
685
|
+
} catch (_e) {
|
|
686
|
+
return { ok: false, reason: "a scanned rollout became unreadable during rebuild" };
|
|
687
|
+
}
|
|
688
|
+
if (!sameCodexFileSnapshot(expected, actual)) {
|
|
689
|
+
return { ok: false, reason: "a scanned rollout changed during rebuild" };
|
|
690
|
+
}
|
|
691
|
+
const rebuiltCursor = rebuiltFiles?.[filePath];
|
|
692
|
+
if (!rebuiltCursor) {
|
|
693
|
+
return { ok: false, reason: "the rebuild omitted a scanned rollout cursor" };
|
|
694
|
+
}
|
|
695
|
+
if (Number(rebuiltCursor.offset) !== expected.size) {
|
|
696
|
+
return { ok: false, reason: "the rebuild did not consume a scanned rollout through EOF" };
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return { ok: true };
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
function isCodexHistoryCovered(cursors, rolloutFiles) {
|
|
703
|
+
const scannedPaths = new Set();
|
|
704
|
+
const scannedSessionIds = new Set();
|
|
705
|
+
for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
|
|
706
|
+
const fp = typeof entry === "string" ? entry : entry?.path;
|
|
707
|
+
const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
|
|
708
|
+
if (!fp || src !== "codex") continue;
|
|
709
|
+
scannedPaths.add(fp);
|
|
710
|
+
const id = codexSessionIdFromPath(fp);
|
|
711
|
+
if (id) scannedSessionIds.add(id);
|
|
712
|
+
}
|
|
713
|
+
if (!cursors.files || typeof cursors.files !== "object") return true;
|
|
714
|
+
for (const fp of Object.keys(cursors.files)) {
|
|
715
|
+
if (!isCodexSessionCursorPath(fp)) continue;
|
|
716
|
+
if (scannedPaths.has(fp)) continue;
|
|
717
|
+
const id = codexSessionIdFromPath(fp);
|
|
718
|
+
if (id && scannedSessionIds.has(id)) continue;
|
|
719
|
+
return false;
|
|
720
|
+
}
|
|
721
|
+
return true;
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// Scans codex rollout heads for the fork marker. The child session_meta (with
|
|
725
|
+
// forked_from_id) is the FIRST line of a forked rollout — observed at byte
|
|
726
|
+
// offset ≤169 across every local sample — so a bounded head read is
|
|
727
|
+
// sufficient. 64KB leaves ample margin for long base_instructions preceding
|
|
728
|
+
// it; a substring false positive (the marker quoted in unrelated head content)
|
|
729
|
+
// only costs one redundant guarded rebuild, never data. Tri-state result:
|
|
730
|
+
// {forked:true} on a hit, {forked:false, indeterminate:false} only when every
|
|
731
|
+
// candidate was inspected, {forked:false, indeterminate:true} when any head
|
|
732
|
+
// read failed — the caller must not finalize "no forks" off a failed read.
|
|
733
|
+
async function scanForForkedCodexRollout(rolloutFiles) {
|
|
734
|
+
const HEAD_BYTES = 65536;
|
|
735
|
+
let indeterminate = false;
|
|
736
|
+
for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
|
|
737
|
+
const fp = typeof entry === "string" ? entry : entry?.path;
|
|
738
|
+
const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
|
|
739
|
+
if (!fp || src !== "codex") continue;
|
|
740
|
+
let fh = null;
|
|
741
|
+
try {
|
|
742
|
+
if (typeof fp === "string" && /\.jsonl\.zstd?$/i.test(fp)) {
|
|
743
|
+
const snapshot = await readCodexRolloutSnapshot(fp);
|
|
744
|
+
if (!snapshot) {
|
|
745
|
+
indeterminate = true;
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
// Fork marker is in early session_meta; scan a bounded prefix of lines.
|
|
749
|
+
let head = "";
|
|
750
|
+
for (const raw of snapshot.lines) {
|
|
751
|
+
head += (typeof raw === "string" ? raw : "") + "\n";
|
|
752
|
+
if (head.length >= HEAD_BYTES) break;
|
|
753
|
+
}
|
|
754
|
+
if (head.includes('"forked_from_id"')) {
|
|
755
|
+
return { forked: true, indeterminate: false };
|
|
756
|
+
}
|
|
757
|
+
continue;
|
|
758
|
+
}
|
|
759
|
+
fh = await fs.open(fp, "r");
|
|
760
|
+
const buf = Buffer.alloc(HEAD_BYTES);
|
|
761
|
+
const { bytesRead } = await fh.read(buf, 0, HEAD_BYTES, 0);
|
|
762
|
+
if (buf.toString("utf8", 0, bytesRead).includes('"forked_from_id"')) {
|
|
763
|
+
return { forked: true, indeterminate: false };
|
|
764
|
+
}
|
|
765
|
+
} catch (_e) {
|
|
766
|
+
indeterminate = true;
|
|
767
|
+
} finally {
|
|
768
|
+
if (fh) await fh.close().catch(() => {});
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
return { forked: false, indeterminate };
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
async function projectUsageKeysFromQueuePath(queuePath, source) {
|
|
775
|
+
if (typeof queuePath !== "string" || !queuePath) return [];
|
|
776
|
+
let raw = "";
|
|
777
|
+
try {
|
|
778
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
779
|
+
} catch (e) {
|
|
780
|
+
if (e?.code !== "ENOENT") throw e;
|
|
781
|
+
return [];
|
|
782
|
+
}
|
|
783
|
+
return projectUsageKeysFromQueueRows(raw.split("\n").filter((line) => line.trim()), source);
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
function projectUsageKeysFromQueueRows(rows, source) {
|
|
787
|
+
const keys = [];
|
|
788
|
+
for (const line of Array.isArray(rows) ? rows : []) {
|
|
789
|
+
let row;
|
|
790
|
+
try {
|
|
791
|
+
row = JSON.parse(line);
|
|
792
|
+
} catch (_e) {
|
|
793
|
+
continue;
|
|
794
|
+
}
|
|
795
|
+
const key = projectUsageKeyFromFields({
|
|
796
|
+
projectKey: row?.project_key,
|
|
797
|
+
source: row?.source,
|
|
798
|
+
hourStart: row?.hour_start,
|
|
799
|
+
});
|
|
800
|
+
if (key && row?.source === source) keys.push(key);
|
|
801
|
+
}
|
|
802
|
+
return keys;
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function projectUsageKeysFromState(projectState, source) {
|
|
806
|
+
const buckets =
|
|
807
|
+
projectState && typeof projectState === "object" && projectState.buckets
|
|
808
|
+
? projectState.buckets
|
|
809
|
+
: {};
|
|
810
|
+
const keys = [];
|
|
811
|
+
for (const [key, bucket] of Object.entries(buckets)) {
|
|
812
|
+
const bucketSource = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
|
|
813
|
+
if (bucketSource !== source) continue;
|
|
814
|
+
const usageKey =
|
|
815
|
+
projectUsageKeyFromFields({
|
|
816
|
+
projectKey: bucket?.project_key,
|
|
817
|
+
source: bucketSource,
|
|
818
|
+
hourStart: bucket?.hour_start,
|
|
819
|
+
}) || key;
|
|
820
|
+
keys.push(usageKey);
|
|
821
|
+
}
|
|
822
|
+
return keys;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
function projectUsageKeyFromFields({ projectKey, source, hourStart }) {
|
|
826
|
+
if (
|
|
827
|
+
typeof projectKey !== "string" ||
|
|
828
|
+
typeof source !== "string" ||
|
|
829
|
+
typeof hourStart !== "string" ||
|
|
830
|
+
!projectKey ||
|
|
831
|
+
!source ||
|
|
832
|
+
!hourStart
|
|
833
|
+
) {
|
|
834
|
+
return null;
|
|
835
|
+
}
|
|
836
|
+
return `${projectKey}|${source}|${hourStart}`;
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
async function countMalformedCodexProjectQueueRows(queuePath) {
|
|
840
|
+
if (typeof queuePath !== "string" || !queuePath) return 0;
|
|
841
|
+
let raw = "";
|
|
842
|
+
try {
|
|
843
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
844
|
+
} catch (e) {
|
|
845
|
+
if (e?.code !== "ENOENT") throw e;
|
|
846
|
+
return 0;
|
|
847
|
+
}
|
|
848
|
+
let count = 0;
|
|
849
|
+
for (const line of raw.split("\n")) {
|
|
850
|
+
if (!line.trim()) continue;
|
|
851
|
+
let row;
|
|
852
|
+
try {
|
|
853
|
+
row = JSON.parse(line);
|
|
854
|
+
} catch (_e) {
|
|
855
|
+
continue;
|
|
856
|
+
}
|
|
857
|
+
if (row?.source !== "codex") continue;
|
|
858
|
+
const key = projectUsageKeyFromFields({
|
|
859
|
+
projectKey: row?.project_key,
|
|
860
|
+
source: row?.source,
|
|
861
|
+
hourStart: row?.hour_start,
|
|
862
|
+
});
|
|
863
|
+
if (!key) count += 1;
|
|
864
|
+
}
|
|
865
|
+
return count;
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
async function projectUsageTotalsFromQueuePath(queuePath, source) {
|
|
869
|
+
if (typeof queuePath !== "string" || !queuePath) return new Map();
|
|
870
|
+
let raw = "";
|
|
871
|
+
try {
|
|
872
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
873
|
+
} catch (e) {
|
|
874
|
+
if (e?.code !== "ENOENT") throw e;
|
|
875
|
+
return new Map();
|
|
876
|
+
}
|
|
877
|
+
return projectUsageTotalsFromQueueRows(raw.split("\n").filter((line) => line.trim()), source);
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
function projectUsageTotalsFromQueueRows(rows, source) {
|
|
881
|
+
const totals = new Map();
|
|
882
|
+
for (const line of Array.isArray(rows) ? rows : []) {
|
|
883
|
+
let row;
|
|
884
|
+
try {
|
|
885
|
+
row = JSON.parse(line);
|
|
886
|
+
} catch (_e) {
|
|
887
|
+
continue;
|
|
888
|
+
}
|
|
889
|
+
const key = projectUsageKeyFromFields({
|
|
890
|
+
projectKey: row?.project_key,
|
|
891
|
+
source: row?.source,
|
|
892
|
+
hourStart: row?.hour_start,
|
|
893
|
+
});
|
|
894
|
+
if (!key || row?.source !== source) continue;
|
|
895
|
+
setMaxTotal(totals, key, Number(row.total_tokens || 0));
|
|
896
|
+
}
|
|
897
|
+
return totals;
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
function projectUsageTotalsFromState(projectState, source) {
|
|
901
|
+
const buckets =
|
|
902
|
+
projectState && typeof projectState === "object" && projectState.buckets
|
|
903
|
+
? projectState.buckets
|
|
904
|
+
: {};
|
|
905
|
+
const totals = new Map();
|
|
906
|
+
for (const [key, bucket] of Object.entries(buckets)) {
|
|
907
|
+
const bucketSource = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
|
|
908
|
+
if (bucketSource !== source) continue;
|
|
909
|
+
const usageKey =
|
|
910
|
+
projectUsageKeyFromFields({
|
|
911
|
+
projectKey: bucket?.project_key,
|
|
912
|
+
source: bucketSource,
|
|
913
|
+
hourStart: bucket?.hour_start,
|
|
914
|
+
}) || key;
|
|
915
|
+
setMaxTotal(totals, usageKey, Number(bucket?.totals?.total_tokens || 0));
|
|
916
|
+
}
|
|
917
|
+
return totals;
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
async function mainUsageHourTotalsFromQueuePath(queuePath, source) {
|
|
921
|
+
if (typeof queuePath !== "string" || !queuePath) return new Map();
|
|
922
|
+
let raw = "";
|
|
923
|
+
try {
|
|
924
|
+
raw = await fs.readFile(queuePath, "utf8");
|
|
925
|
+
} catch (e) {
|
|
926
|
+
if (e?.code !== "ENOENT") throw e;
|
|
927
|
+
return new Map();
|
|
928
|
+
}
|
|
929
|
+
return mainUsageHourTotalsFromQueueRows(raw.split("\n").filter((line) => line.trim()), source);
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
function mainUsageHourTotalsFromQueueRows(rows, source) {
|
|
933
|
+
const modelTotals = new Map();
|
|
934
|
+
for (const line of Array.isArray(rows) ? rows : []) {
|
|
935
|
+
let row;
|
|
936
|
+
try {
|
|
937
|
+
row = JSON.parse(line);
|
|
938
|
+
} catch (_e) {
|
|
939
|
+
continue;
|
|
940
|
+
}
|
|
941
|
+
if (row?.source !== source || typeof row?.hour_start !== "string") continue;
|
|
942
|
+
const model = typeof row?.model === "string" && row.model ? row.model : "unknown";
|
|
943
|
+
setMaxTotal(modelTotals, `${row.source}|${model}|${row.hour_start}`, Number(row.total_tokens || 0));
|
|
944
|
+
}
|
|
945
|
+
return collapseModelTotalsByHour(modelTotals);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
function mainUsageHourTotalsFromState(hourlyState, source) {
|
|
949
|
+
const buckets =
|
|
950
|
+
hourlyState && typeof hourlyState === "object" && hourlyState.buckets
|
|
951
|
+
? hourlyState.buckets
|
|
952
|
+
: {};
|
|
953
|
+
const modelTotals = new Map();
|
|
954
|
+
for (const [key, bucket] of Object.entries(buckets)) {
|
|
955
|
+
const parts = key.split("|");
|
|
956
|
+
const bucketSource = typeof bucket?.source === "string" ? bucket.source : parts[0];
|
|
957
|
+
if (bucketSource !== source) continue;
|
|
958
|
+
const model = typeof bucket?.model === "string" && bucket.model ? bucket.model : parts[1] || "unknown";
|
|
959
|
+
const hourStart =
|
|
960
|
+
typeof bucket?.hour_start === "string" && bucket.hour_start ? bucket.hour_start : parts[2];
|
|
961
|
+
if (typeof hourStart !== "string" || !hourStart) continue;
|
|
962
|
+
setMaxTotal(modelTotals, `${bucketSource}|${model}|${hourStart}`, Number(bucket?.totals?.total_tokens || 0));
|
|
963
|
+
}
|
|
964
|
+
return collapseModelTotalsByHour(modelTotals);
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
function collapseModelTotalsByHour(modelTotals) {
|
|
968
|
+
const totals = new Map();
|
|
969
|
+
for (const [key, total] of modelTotals.entries()) {
|
|
970
|
+
const [source, , hourStart] = key.split("|");
|
|
971
|
+
const hourKey = `${source}|${hourStart}`;
|
|
972
|
+
totals.set(hourKey, (totals.get(hourKey) || 0) + total);
|
|
973
|
+
}
|
|
974
|
+
return totals;
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
function mergeMaxTotals(...maps) {
|
|
978
|
+
const merged = new Map();
|
|
979
|
+
for (const map of maps) {
|
|
980
|
+
if (!(map instanceof Map)) continue;
|
|
981
|
+
for (const [key, total] of map.entries()) {
|
|
982
|
+
setMaxTotal(merged, key, total);
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
return merged;
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
function setMaxTotal(map, key, total) {
|
|
989
|
+
if (!key || !Number.isFinite(total)) return;
|
|
990
|
+
const prev = map.get(key);
|
|
991
|
+
if (!Number.isFinite(prev) || total > prev) map.set(key, total);
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
// One-time repair (#204): when the SAME Droid session id existed in two folders
|
|
995
|
+
// under ~/.factory/sessions, parseDroidIncremental's cumulative-delta loop made the
|
|
996
|
+
// lower-count file look like a reset and re-emitted each duplicate's full total on
|
|
997
|
+
// EVERY sync, inflating one (droid, model, hour) bucket without bound (a real ~10M
|
|
998
|
+
// session showed as 40.06B). The forward fix is dedupeDroidSettingsFilesBySession
|
|
999
|
+
// inside the parser; this migration repairs already-polluted installs.
|
|
1000
|
+
//
|
|
1001
|
+
// SCOPE — strictly the duplicate sessions' buckets, and ONLY when duplicate files
|
|
1002
|
+
// still exist on disk:
|
|
1003
|
+
// * A from-zero rebuild cannot reconstruct Droid's historical per-sync bucket
|
|
1004
|
+
// distribution — settings.json carries only the CURRENT mtime, not per-turn
|
|
1005
|
+
// timestamps like Codex's jsonl. So we rebuild over the DUPLICATE files only,
|
|
1006
|
+
// and overwrite only the bucket keys those files map to (pollutedKeys) plus
|
|
1007
|
+
// the duplicate sessions' cursor entries. Every other droid bucket and cursor
|
|
1008
|
+
// — clean sessions AND deleted-session history — is left byte-for-byte intact,
|
|
1009
|
+
// so healthy history is never collapsed into the current half-hour.
|
|
1010
|
+
// * No session id has >1 file on disk → this bug never fired here: set the
|
|
1011
|
+
// sentinel, touch nothing.
|
|
1012
|
+
// * Fire only when live > rebuilt over pollutedKeys (actual inflation), so a
|
|
1013
|
+
// fresh install (live empty) is left to the normal same-sync parse.
|
|
1014
|
+
// Droid has no project dimension, so project.queue.jsonl is never involved.
|
|
1015
|
+
|
|
1016
|
+
module.exports = {
|
|
1017
|
+
CODEX_FORK_REPLAY_REPAIR_KEY,
|
|
1018
|
+
CODEX_RESCAN_DEDUP_REPAIR_KEY,
|
|
1019
|
+
CODEX_USAGE_LINEAGE_REPAIR_KEY,
|
|
1020
|
+
repairCodexForkReplayInflation,
|
|
1021
|
+
repairCodexInterleavedUsageInflation,
|
|
1022
|
+
repairCodexRescanInflation,
|
|
1023
|
+
};
|