tokenusage-cli 0.5.4 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,10 +3,57 @@ const path = require("node:path");
3
3
  const fs = require("node:fs/promises");
4
4
  const fssync = require("node:fs");
5
5
  const cp = require("node:child_process");
6
- const readline = require("node:readline");
7
6
  const { functionUrlFor, fetchFunctionResponse } = require("../lib/function-url");
8
7
  const { cloudUsageResetBlocksUpload, finishCloudDeviceUsageReset } = require("../lib/cloud-usage-reset");
9
8
  const { KIMI_MODEL_REPAIR_KEY, repairKimiModelAttribution } = require("../lib/kimi-model-repair");
9
+ const { backupExistingFile } = require("../lib/queue-backup");
10
+ const {
11
+ GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
12
+ repairGrokQueueFromSessionSnapshots,
13
+ } = require("../repairs/grok");
14
+ const { repairMimoClaudeMislabel } = require("../repairs/mimo");
15
+ const {
16
+ CURSOR_UNKNOWN_MIGRATION_KEY,
17
+ ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY,
18
+ migrateCursorUnknownBuckets,
19
+ migrateRolloutCumulativeDeltaBuckets,
20
+ } = require("../repairs/legacy-migrations");
21
+ const {
22
+ CODEBUDDY_LOG_JSONL_REPAIR_KEY,
23
+ repairCodebuddyLogJsonlOverlap,
24
+ } = require("../repairs/codebuddy");
25
+ const {
26
+ WORKBUDDY_CONTEXT_USAGE_REPAIR_KEY,
27
+ repairWorkbuddyContextUsage,
28
+ } = require("../repairs/workbuddy");
29
+ const {
30
+ INGEST_SLUG,
31
+ MAX_INGEST_BUCKETS,
32
+ UPLOAD_BUDGET_MS,
33
+ drainQueueToCloud,
34
+ readQueueBatch,
35
+ } = require("../lib/queue-drain");
36
+ const {
37
+ reincludeClaudeMemObserverFiles,
38
+ repairClaudeQueueFromGroundTruth,
39
+ } = require("../repairs/claude");
40
+ const {
41
+ repairDroidDuplicateSessionInflation,
42
+ } = require("../repairs/droid");
43
+ const {
44
+ repairCodexForkReplayInflation,
45
+ repairCodexInterleavedUsageInflation,
46
+ repairCodexRescanInflation,
47
+ } = require("../repairs/codex");
48
+ const {
49
+ ZCODE_PRUNE_WINDOW_RESTORE_KEY,
50
+ resolveZcodeRebuildFloorMs,
51
+ zcodeRowTotals,
52
+ buildZcodeRestorationRow,
53
+ repairZcodeNativeUsageMigration,
54
+ repairZcodeInclusiveTokenMigration,
55
+ repairZcodePruneWindowRestore,
56
+ } = require("../repairs/zcode");
10
57
  const { readSqliteJsonRows } = require("../lib/sqlite-reader");
11
58
  const { runtimeReadyMarkerPath, runtimeBinPath } = require("../lib/runtime-entry");
12
59
  const { cliEntryPath } = require("../lib/scheduled-task");
@@ -23,6 +70,7 @@ const {
23
70
  openLock,
24
71
  inspectLock,
25
72
  updateJsonLocked,
73
+ safeStatSize,
26
74
  } = require("../lib/fs");
27
75
  const { physicalJsonlRecords } = require("../lib/jsonl-lines");
28
76
  const { countRecordOnlyFiles, formatRecordOnlyWarning } = require("../lib/codex-usage-record");
@@ -178,7 +226,6 @@ const AUTO_UPLOAD_CONFIG = {
178
226
  maxBatchesSmall: 5,
179
227
  maxBatchesLarge: 100,
180
228
  };
181
- const UPLOAD_BUDGET_MS = 90_000;
182
229
  const {
183
230
  isCursorInstalled,
184
231
  extractCursorSessionToken,
@@ -216,12 +263,7 @@ const {
216
263
  createUsageDeltaState,
217
264
  } = require("../lib/codex-token-usage");
218
265
 
219
- const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
220
- const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
221
266
  const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
222
- const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
223
- const CODEBUDDY_LOG_JSONL_REPAIR_KEY = "codebuddyLogJsonlOverlapRepair_2026_08";
224
- const WORKBUDDY_CONTEXT_USAGE_REPAIR_KEY = "workbuddyContextUsageRepair_2026_08";
225
267
  const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
226
268
  // v1 had a cursor-format bug (wrote plain integer instead of {inode, offset,
227
269
  // updatedAt}), which made parseClaudeIncremental reread every jsonl from
@@ -318,18 +360,18 @@ const CODEX_COLD_SCAN_AUDIT_MAX_SYNCS = 288;
318
360
  // source=mimo (read the whole DB instead of only providerID=mimo rows). This
319
361
  // one-time repair purges all source=mimo data from the local queues + cursor
320
362
  // state so the next sync (providerID-filtered reader) rebuilds it correctly.
321
- const MIMO_PROVIDER_REPAIR_KEY = "mimoClaudeMislabelRepair_2026_06";
322
363
  const DSH_LEGACY_SOURCE_MIGRATION_KEY = "deepseekHarnessSourceMigration_2026_08";
323
- const ZCODE_NATIVE_USAGE_REPAIR_KEY = "zcodeNativeUsageRepair_2026_08";
324
- const ZCODE_INCLUSIVE_TOKEN_REPAIR_KEY = "zcodeInclusiveTokenRepair_2026_09";
364
+
325
365
  // #828: ZCode CLI prunes model_usage/turn_usage to ~30 days. The wipe+rescan
326
366
  // repairs above destroyed pre-window history for late upgraders (and anyone
327
367
  // whose prune line advanced before the one-shot key fired). This follow-up
328
368
  // restores pre-window peaks from queue.jsonl.bak.* when the live ledger is
329
369
  // lower, and the wipe helpers below now preserve hours older than the live
330
370
  // rebuild floor so a future re-key cannot repeat the loss.
331
- const ZCODE_PRUNE_WINDOW_RESTORE_KEY = "zcodePruneWindowRestore_2026_10";
332
371
  const AUTO_SYNC_SOURCE_ALIASES = new Map([
372
+ // The integration is labelled AStudio everywhere in the CLI; `acode` is only
373
+ // the collector id, so both spellings must reach the same scanner.
374
+ ["astudio", "acode"],
333
375
  ["code", "every-code"],
334
376
  ["commandcode", "command-code"],
335
377
  ["deepseek", "dsh"],
@@ -375,6 +417,10 @@ const AUTO_SYNC_SOURCES = new Set([
375
417
  "opencode",
376
418
  "openclaw",
377
419
  "pi",
420
+ // The parser existed but the id was missing here, so normalizeSyncSource()
421
+ // rejected `--source prime-agent` and every auto/background run skipped the
422
+ // source entirely.
423
+ "prime-agent",
378
424
  "qoder",
379
425
  "qoder-cn",
380
426
  "reasonix",
@@ -715,6 +761,16 @@ async function cmdSync(argv, context = {}) {
715
761
  // try/finally so an auxiliary diagnostic never strands the sync lock.
716
762
  await clearSyncSkip(trackerDir);
717
763
  progress = !opts.auto ? createProgress({ stream: process.stdout }) : null;
764
+ // `--timing`: coarse phase durations for the interactive summary. Purely
765
+ // diagnostic — nothing is persisted, and auto runs never print it.
766
+ const phaseTimings = opts.timing ? [] : null;
767
+ let phaseMarkMs = syncStartedAtMs;
768
+ const markPhase = (label) => {
769
+ if (!phaseTimings) return;
770
+ const now = Date.now();
771
+ phaseTimings.push({ label, ms: now - phaseMarkMs });
772
+ phaseMarkMs = now;
773
+ };
718
774
  // Every auto-flavored run (scheduled task, hooks, retry, watch) self-logs:
719
775
  // a hidden TTY-less task has no other trail, so --from-scheduled must not
720
776
  // fall through to the interactive stdout summary below.
@@ -868,13 +924,23 @@ async function cmdSync(argv, context = {}) {
868
924
  : null;
869
925
 
870
926
  const autoSourceScope = resolveAutoSourceScope(opts);
927
+ // `--source` on a manual run has to be a real scan filter. It used to only
928
+ // validate the id (and scope auto/notify retries through
929
+ // resolveAutoSourceScope), so `sync --source zed` still walked every
930
+ // provider tree and reported ~32k parsed files — the documented "scan a
931
+ // single provider" was true only for hooks, which always pass `--auto`.
932
+ const manualSourceScope = resolveManualSourceScope(opts);
871
933
  // --background controls local scan breadth; --drain only controls upload
872
934
  // depth and lock priority. Plain `sync --drain` remains a full source scan.
873
935
  const isBackgroundLightweightSync = opts.auto && opts.background;
874
936
  const isBackgroundAllLocalSync = isBackgroundLightweightSync && opts.allLocalSources;
875
- const isFullSourceScan = !autoSourceScope && !isBackgroundLightweightSync;
937
+ // A single-source run is not a full scan: the cross-source repairs below
938
+ // rebuild queue rows from the parsed file set, so feeding them one source's
939
+ // files could make them treat the other sources' buckets as stale.
940
+ const isFullSourceScan = !autoSourceScope && !manualSourceScope && !isBackgroundLightweightSync;
876
941
  const sourceAllowed = (...sources) => {
877
942
  if (Array.isArray(context.scanSources) && !sources.some(source => context.scanSources.includes(source))) return false;
943
+ if (manualSourceScope) return sources.includes(manualSourceScope);
878
944
  if (isBackgroundLightweightSync) {
879
945
  if (autoSourceScope) {
880
946
  return (
@@ -3886,6 +3952,8 @@ async function cmdSync(argv, context = {}) {
3886
3952
  if (uploadError) syncRun.setOutcome({ result: "partial", errorCode: "upload_failed" });
3887
3953
  }
3888
3954
 
3955
+ markPhase("scan + hourly upload");
3956
+
3889
3957
  // Session metadata upload (docs/cli-contract.md section 2.4 `sessions`):
3890
3958
  // strictly-newer-than-watermark increments, batches of <=200, best-effort —
3891
3959
  // an hourly-bucket sync must not be reported as failed by a sessions error.
@@ -4012,6 +4080,8 @@ async function cmdSync(argv, context = {}) {
4012
4080
  }
4013
4081
  }
4014
4082
 
4083
+ markPhase("sessions + projects");
4084
+
4015
4085
  if (!autoishSync) {
4016
4086
  const codexRecordOnlyWarning = formatRecordOnlyWarning(countRecordOnlyFiles(cursors));
4017
4087
  const deviceIdentity = buildDeviceIdentity({ config: runtimeConfig, hostname: os.hostname() });
@@ -4030,6 +4100,14 @@ async function cmdSync(argv, context = {}) {
4030
4100
  warning: codexRecordOnlyWarning,
4031
4101
  }).join("\n"),
4032
4102
  );
4103
+ if (phaseTimings) {
4104
+ process.stdout.write(
4105
+ // Start-to-finish total is intentionally the last column: the phases
4106
+ // above are sequential, so any gap shows up there.
4107
+ `- Timing: ${phaseTimings.map((p) => `${p.label} ${formatDuration(p.ms)}`).join(" | ")}` +
4108
+ ` | total ${formatDuration(Date.now() - syncStartedAtMs)}\n`,
4109
+ );
4110
+ }
4033
4111
  } else {
4034
4112
  autoLog(
4035
4113
  `Auto sync finished — parsed ${totalParsed} files, +${totalBuckets} buckets, ` +
@@ -4098,6 +4176,7 @@ function parseArgs(argv) {
4098
4176
  allLocalSources: false,
4099
4177
  repairGrok: false,
4100
4178
  noSessions: false,
4179
+ timing: false,
4101
4180
  };
4102
4181
  for (let i = 0; i < argv.length; i++) {
4103
4182
  const a = argv[i];
@@ -4118,11 +4197,20 @@ function parseArgs(argv) {
4118
4197
  else if (a === "--publish-account") out.publishAccount = true;
4119
4198
  else if (a === "--all-local-sources") out.allLocalSources = true;
4120
4199
  else if (a === "--repair-grok") out.repairGrok = true;
4200
+ else if (a === "--timing") out.timing = true;
4121
4201
  else throw unknownOptionError(a, "sync");
4122
4202
  }
4123
4203
  return out;
4124
4204
  }
4125
4205
 
4206
+ // Manual (`sync --source <id>`) scan scope. Kept separate from
4207
+ // resolveAutoSourceScope: that one deliberately returns null unless `--auto` is
4208
+ // set, which is how the flag ended up validated-but-unused on plain syncs.
4209
+ function resolveManualSourceScope(opts) {
4210
+ if (!opts || opts.auto) return null;
4211
+ return normalizeSyncSource(opts.source);
4212
+ }
4213
+
4126
4214
  function resolveAutoSourceScope(opts) {
4127
4215
  if (!opts?.auto) return null;
4128
4216
  if (opts.fromOpenclaw) return "openclaw";
@@ -4231,6 +4319,7 @@ module.exports = {
4231
4319
  readQueueBatch,
4232
4320
  drainQueueToCloud,
4233
4321
  normalizeSyncSource,
4322
+ resolveManualSourceScope,
4234
4323
  parseSyncSourceArg,
4235
4324
  formatSyncSourceList,
4236
4325
  zcodeRowTotals,
@@ -4564,14 +4653,6 @@ function normalizeIsoOrEpoch(value) {
4564
4653
  return dt.toISOString();
4565
4654
  }
4566
4655
 
4567
- async function safeStatSize(p) {
4568
- try {
4569
- const st = await fs.stat(p);
4570
- return st && st.isFile() ? st.size : 0;
4571
- } catch (_e) {
4572
- return 0;
4573
- }
4574
- }
4575
4656
 
4576
4657
  function deriveAutoSkipReason({ decision, state }) {
4577
4658
  if (!decision || decision.reason !== "throttled") return decision?.reason || "unknown";
@@ -4759,3615 +4840,37 @@ async function writeOpenclawSignal(trackerDir) {
4759
4840
  const AUTO_RETRY_FILENAME = "auto.retry.json";
4760
4841
  const AUTO_RETRY_MAX_DELAY_MS = 2 * 60 * 60 * 1000;
4761
4842
 
4762
- const INGEST_SLUG = "token-usage-ingest";
4763
4843
  const { buildSessionAnalytics } = require("../lib/session-analytics");
4764
4844
  const { uploadSessions } = require("../lib/session-upload");
4765
4845
  const { uploadProjectRepos } = require("../lib/project-repos");
4766
- const MAX_INGEST_BUCKETS = 500;
4767
4846
  // Sessions split batches on a 500 KiB byte budget (lib/session-upload.js);
4768
4847
  // the hourly drain used to rely on the row cap alone. Raw queue-line bytes
4769
4848
  // upper-bound the sanitized wire rows (the whitelist in ingest-sanitize only
4770
4849
  // shrinks them), so cutting on raw bytes keeps every request comfortably
4771
4850
  // under the contract's 1 MiB body cap (§1.2 → 413).
4772
- const MAX_BATCH_BYTES = 512 * 1024;
4773
-
4774
- async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200, maxDurationMs = UPLOAD_BUDGET_MS, canUpload = () => true, fetchImpl = fetch }) {
4775
- const state = (await readJson(queueStatePath)) || { offset: 0 };
4776
- let offset = Number(state.offset || 0);
4777
- let inserted = 0;
4778
- let skipped = 0;
4779
- let batches = 0;
4780
- let tokensIngested = 0;
4781
- let creditsIngested = 0;
4782
- let cloudAccess = null;
4783
- let rejectStrikes = readRejectStrikes(state);
4784
- const uploadId = crypto.randomUUID();
4785
- const deadline = Date.now() + maxDurationMs;
4786
- let batchBudget = Math.min(maxBatches, 100);
4787
- let requestSignal = null;
4788
- const resultSoFar = () => ({ inserted, skipped, batches, tokens: tokensIngested, credits: creditsIngested, ...(cloudAccess?.next_allowed_at ? { cloudAccess } : {}) });
4789
-
4790
- const queueSize = await safeStatSize(queuePath);
4791
- const limit = Math.min(Math.max(1, Math.floor(Number(batchSize || 200))), MAX_INGEST_BUCKETS);
4792
-
4793
- try {
4794
- for (let batch = 0; batch < batchBudget; batch++) {
4795
- if (Date.now() >= deadline) break;
4796
- if (offset >= queueSize) break;
4797
- const result = await readQueueBatch(queuePath, offset, limit);
4798
- // A states-only tail (e.g. a retry batch whose bucket rows already
4799
- // uploaded) is still a meaningful upload: it advances the canonical
4800
- // session corrections cloud-side.
4801
- if (result.buckets.length === 0 && result.sessionStates.length === 0) {
4802
- // Complete ignored control/malformed rows need no cloud request, but
4803
- // must not leave a permanent pending tail. Incomplete bytes stay put.
4804
- if (result.nextOffset > offset) {
4805
- offset = result.nextOffset;
4806
- state.offset = offset;
4807
- state.updatedAt = new Date().toISOString();
4808
- await writeJson(queueStatePath, state);
4809
- }
4810
- break;
4811
- }
4812
-
4813
- const root = baseUrl.replace(/\/$/, "");
4814
- const headers = {
4815
- "Content-Type": "application/json",
4816
- Accept: "application/json",
4817
- Authorization: `Bearer ${deviceToken}`,
4818
- };
4819
- if (anonKey) headers.apikey = anonKey;
4820
- // Re-read after parsing/each batch so switching off stops an active drain.
4821
- if (!canUpload()) break;
4822
- const remainingMs = deadline - Date.now();
4823
- if (remainingMs <= 0) break;
4824
- requestSignal = AbortSignal.timeout(Math.max(1, Math.ceil(remainingMs)));
4825
- const res = await fetchFunctionResponse(functionUrlFor(root, INGEST_SLUG), {
4826
- method: "POST",
4827
- headers,
4828
- signal: requestSignal,
4829
- body: JSON.stringify({
4830
- upload_id: uploadId,
4831
- hourly: result.buckets,
4832
- ...(result.sessionStates.length > 0
4833
- ? { account_session_states: result.sessionStates }
4834
- : {}),
4835
- }),
4836
- }, fetchImpl);
4837
-
4838
- const rawText = await res.text();
4839
- let data = {};
4840
- try { data = JSON.parse(rawText); } catch { data = {}; }
4841
- if (!res.ok) {
4842
- const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
4843
- err.status = res.status;
4844
- err.code = res.status === 401 ? "CLOUD_DEVICE_TOKEN_REJECTED"
4845
- : typeof data?.code === "string" ? data.code
4846
- : res.status === 403 ? "CLOUD_UPLOAD_FORBIDDEN" : "CLOUD_UPLOAD_FAILED";
4847
- const retryAfter = res.headers?.get?.("Retry-After") ?? null;
4848
- const retryAfterMs = parseRetryAfterMs(retryAfter);
4849
- if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
4850
- // 400/413 are deterministic client-shape rejections: retrying the
4851
- // identical batch can never succeed, and the offset only moves on
4852
- // success. Count strikes per offset (persisted, so the count survives
4853
- // process exits between scheduled syncs); after three, quarantine the
4854
- // batch and advance past it — a loud, inspectable drop instead of a
4855
- // silent stall. Quarantined rows stay in queue.jsonl, so
4856
- // `tokenusage reconcile` keeps reporting them as drift for a human
4857
- // to judge.
4858
- if (err.status === 400 || err.status === 413) {
4859
- const strikes = rejectStrikes.offset === offset ? rejectStrikes.count + 1 : 1;
4860
- rejectStrikes = { count: strikes, offset };
4861
- if (strikes >= REJECT_VALVE_LIMIT) {
4862
- const quarantinePath = `${queuePath}.quarantine.jsonl`;
4863
- const quarantined = await quarantineBatchAt({ queuePath, offset, limit, quarantinePath });
4864
- offset = quarantined.nextOffset;
4865
- state.offset = offset;
4866
- delete state.reject;
4867
- state.updatedAt = new Date().toISOString();
4868
- await writeJson(queueStatePath, state);
4869
- process.stderr.write(
4870
- `Upload escape valve: HTTP ${err.status} rejected the same batch ${strikes} times; ` +
4871
- `quarantined ${quarantined.rows} row(s) to ${quarantinePath} and advanced past them. ` +
4872
- `Inspect the quarantine file — \`tokenusage reconcile\` will flag these keys as drift.\n`,
4873
- );
4874
- continue;
4875
- }
4876
- state.reject = { count: strikes, offset };
4877
- state.updatedAt = new Date().toISOString();
4878
- await writeJson(queueStatePath, state);
4879
- }
4880
- throw err;
4881
- }
4882
-
4883
- if (!isConfirmedIngestResponse(data)) {
4884
- const err = new Error(
4885
- `Unconfirmed ingest response (HTTP ${res.status}): ${rawText.substring(0, 300)}`,
4886
- );
4887
- err.status = res.status;
4888
- err.code = "CLOUD_UPLOAD_UNCONFIRMED";
4889
- throw err;
4890
- }
4891
- cloudAccess = data;
4892
- if (data.upload_scope === "community") batchBudget = Math.min(batchBudget, 10);
4893
- inserted += Number(data?.inserted || 0);
4894
- skipped += Number(data?.skipped || 0);
4895
- // Client-side §2.11 telemetry: the total_tokens of the buckets this
4896
- // request carried (a larger replay may last-wins-override them later;
4897
- // this figure never participates in reconciliation). Credits sum the
4898
- // same way — the display-only dimension (docs/sources-capture-audit.md
4899
- // §2) — rounded to 6 decimals to keep float noise out of the ledger.
4900
- for (const bucketRow of result.buckets) {
4901
- tokensIngested += Number(bucketRow?.total_tokens) || 0;
4902
- creditsIngested =
4903
- Math.round((creditsIngested + (Number(bucketRow?.credits) || 0)) * 1e6) / 1e6;
4904
- }
4905
- batches += 1;
4906
-
4907
- offset = result.nextOffset;
4908
- state.offset = offset;
4909
- delete state.reject;
4910
- state.updatedAt = new Date().toISOString();
4911
- await writeJson(queueStatePath, state);
4912
- }
4913
- } catch (error) {
4914
- // A bounded drain can finish partially without discarding successful
4915
- // offsets or the server cooldown. Other failures retain their retry path.
4916
- const requestTimedOut = requestSignal?.aborted && requestSignal.reason?.name === "TimeoutError" &&
4917
- (error === requestSignal.reason || error.name === "TimeoutError" || error.name === "AbortError");
4918
- if (!requestTimedOut || batches === 0) {
4919
- error.uploadResult = resultSoFar();
4920
- throw error;
4921
- }
4922
- }
4923
- return resultSoFar();
4924
- }
4925
-
4926
- // A 200 is not proof the ingest committed: captive portals / reverse proxies
4927
- // return 200 HTML and maintenance pages return an empty body. Advancing the
4928
- // queue offset on those silently drops the batch forever, so the drain only
4929
- // accepts the contract's success shape (docs/cli-contract.md §2.4).
4930
- function isCount(value) {
4931
- return (
4932
- (typeof value === "number" || typeof value === "string") &&
4933
- value !== "" &&
4934
- Number.isFinite(Number(value))
4935
- );
4936
- }
4937
-
4938
- function isConfirmedIngestResponse(data) {
4939
- return Boolean(data) && typeof data === "object" && !Array.isArray(data) &&
4940
- isCount(data.inserted) && isCount(data.skipped);
4941
- }
4942
-
4943
- // Consecutive client-shape rejections (400/413) tolerated for one batch
4944
- // offset before the escape valve quarantines it. Three strikes mean the batch
4945
- // failed across three separate sync passes — no transient server state
4946
- // explains it.
4947
- const REJECT_VALVE_LIMIT = 3;
4948
-
4949
- function readRejectStrikes(state) {
4950
- const raw = state && typeof state.reject === "object" && state.reject !== null
4951
- ? state.reject
4952
- : {};
4953
- const count = Number(raw.count);
4954
- const offset = Number(raw.offset);
4955
-
4956
- return {
4957
- count: Number.isFinite(count) && count > 0 ? Math.floor(count) : 0,
4958
- offset: Number.isFinite(offset) ? offset : null,
4959
- };
4960
- }
4961
-
4962
- // Re-read the batch at `offset` through the exact same pipeline the upload
4963
- // used, append the rows verbatim to the quarantine file, and report where the
4964
- // offset should jump to. The queue itself is never rewritten: quarantined rows
4965
- // stay in queue.jsonl, so reconcile keeps flagging their keys as
4966
- // missing_on_server drift for a human to judge.
4967
- async function quarantineBatchAt({ queuePath, offset, limit, quarantinePath }) {
4968
- const result = await readQueueBatch(queuePath, offset, limit);
4969
- const rows = [...result.buckets, ...result.sessionStates];
4970
-
4971
- if (rows.length > 0) {
4972
- const payload = rows.map((row) => JSON.stringify(row)).join("\n") + "\n";
4973
- await fs.appendFile(quarantinePath, payload, "utf8");
4974
- }
4975
-
4976
- return { rows: rows.length, nextOffset: result.nextOffset };
4977
- }
4978
-
4979
- async function readQueueBatch(queuePath, startOffset, maxBuckets) {
4980
- const st = await fs.stat(queuePath).catch(() => null);
4981
- if (!st || !st.isFile()) return { buckets: [], sessionStates: [], nextOffset: startOffset };
4982
- if (startOffset >= st.size) return { buckets: [], sessionStates: [], nextOffset: startOffset };
4983
-
4984
- const handle = await fs.open(queuePath, "r");
4985
-
4986
- const bucketMap = new Map();
4987
- // Account session states (kind: "account_session_state") are queue
4988
- // control records, not usage rows: collect them separately so they ride
4989
- // the same upload (after the bucket rows in file order) without entering
4990
- // the bucket stream. Keyed by (source, session_id), use the strictly newer
4991
- // snapshot_verified_at and retain the first row on ties, matching account
4992
- // truth even after a clock rollback. Only one observation per session may
4993
- // reach the ingest batch (duplicates would affect one ON CONFLICT row twice).
4994
- // snapshot_verified_at rides along verbatim from the ORIGINAL fetch, so a
4995
- // transport retry never fakes logical freshness.
4996
- //
4997
- // Batching: session-state records count toward the SAME per-batch record
4998
- // cap (maxBuckets) as bucket rows. The ingest edge rejects a request with
4999
- // more than 500 states (HTTP 400); before they counted, a states-heavy
5000
- // queue (a fresh device's first 30-day TRAE sync appends one record per
5001
- // observed session) was read in ONE uncapped batch - the 400 left the
5002
- // queue offset untouched and every retry re-read the identical oversized
5003
- // request, a permanent upload failure. With the default batchSize of 200
5004
- // (and the MAX_INGEST_BUCKETS=500 clamp above), every request stays under
5005
- // the edge cap while batching follows the existing design.
5006
- const sessionStateMap = new Map();
5007
- const sessionStateStamps = new Map();
5008
- let offset = startOffset;
5009
- let linesRead = 0;
5010
- try {
5011
- let batchBytes = 0;
5012
- for await (const record of iterateQueueLines(handle, startOffset, st.size)) {
5013
- // A producer may still be writing the last row; retry it once complete.
5014
- if (!record.complete) break;
5015
- // Byte budget: cut before the row that would overflow, never on the
5016
- // batch's first row (a single oversized line cannot be split anyway,
5017
- // and the whitelist keeps its wire shape small).
5018
- if (linesRead > 0 && batchBytes + record.bytes.length > MAX_BATCH_BYTES) break;
5019
- const line = record.bytes.toString("utf8");
5020
- offset += record.bytes.length;
5021
- batchBytes += record.bytes.length;
5022
- if (!line.trim()) continue;
5023
- let bucket;
5024
- try {
5025
- bucket = JSON.parse(line);
5026
- } catch (_e) {
5027
- continue;
5028
- }
5029
- if (bucket?.kind === "account_session_state") {
5030
- if (typeof bucket.source === "string" && typeof bucket.session_id === "string" && bucket.session_id.trim()) {
5031
- const verifiedMs = sessionSnapshotMs(bucket);
5032
- // The edge validates before writing and 400s the WHOLE batch on one
5033
- // invalid state ("Invalid account session state"); the offset only
5034
- // advances on success, so uploading a poison row would stall the
5035
- // queue head-of-line forever. Drop it here instead — nothing is
5036
- // recorded for the key, so a later, valid snapshot for the same
5037
- // session is still admitted.
5038
- if (verifiedMs === null) continue;
5039
- const ssSource = bucket.source.trim().toLowerCase();
5040
- const stateKey = ssSource + "|" + bucket.session_id.trim();
5041
- const sessionState = {
5042
- source: ssSource,
5043
- session_id: bucket.session_id.trim(),
5044
- model: typeof bucket.model === "string" ? bucket.model : "",
5045
- bucket_start: typeof bucket.bucket_start === "string" ? bucket.bucket_start : "",
5046
- input_tokens: Number(bucket.input_tokens) || 0,
5047
- output_tokens: Number(bucket.output_tokens) || 0,
5048
- cached_input_tokens: Number(bucket.cached_input_tokens) || 0,
5049
- cache_creation_input_tokens: Number(bucket.cache_creation_input_tokens) || 0,
5050
- reasoning_output_tokens: Number(bucket.reasoning_output_tokens) || 0,
5051
- total_tokens: Number(bucket.total_tokens) || 0,
5052
- ...(typeof bucket.snapshot_verified_at === "string" && bucket.snapshot_verified_at
5053
- ? { snapshot_verified_at: bucket.snapshot_verified_at }
5054
- : {}),
5055
- };
5056
- const priorState = sessionStateMap.get(stateKey);
5057
- const priorVerifiedMs = sessionStateStamps.get(stateKey) ?? null;
5058
- if (!priorState || priorVerifiedMs === null || verifiedMs > priorVerifiedMs) {
5059
- sessionStateMap.set(stateKey, sessionState);
5060
- sessionStateStamps.set(stateKey, verifiedMs);
5061
- }
5062
- // Counted like a bucket row: the record rides THIS batch and its
5063
- // bytes are covered by nextOffset, so the next batch resumes right
5064
- // after it.
5065
- linesRead += 1;
5066
- if (linesRead >= maxBuckets) break;
5067
- }
5068
- continue;
5069
- }
5070
- // Pre-release watermark records from this feature branch's dev queues
5071
- // are obsolete control records: drop them instead of letting them enter
5072
- // the bucket stream (they carry no hour_start and would be skipped
5073
- // downstream anyway, but be explicit).
5074
- if (bucket?.kind === "account_sync_watermark") continue;
5075
- const hourStart = typeof bucket?.hour_start === "string" ? bucket.hour_start : null;
5076
- if (!hourStart || !Number.isFinite(Date.parse(hourStart))) continue;
5077
- const source = (typeof bucket?.source === "string" ? bucket.source.trim().toLowerCase() : "") || "codex";
5078
- const model = (typeof bucket?.model === "string" ? bucket.model.trim() : "") || "unknown";
5079
- bucket.source = source;
5080
- bucket.model = model;
5081
- // Apply the same legacy-row corrections every local reader applies (the
5082
- // project queue and the wrapped aggregator use this module too). Without
5083
- // this the cloud permanently kept the RAW legacy values — e.g. old Codex
5084
- // rows whose input_tokens still include cached tokens (6-7x inflated) —
5085
- // while local readers showed corrected numbers.
5086
- bucket = normalizeQueueRow(bucket);
5087
- // Contract-shape coercion, poison-row drop (see lib/ingest-sanitize.js):
5088
- // a fractional Grok counter or a negative OpenCode repair delta would
5089
- // otherwise 400 the whole ingest batch and stall the queue.
5090
- const sanitized = sanitizeBucketForUpload(bucket);
5091
- if (!sanitized) continue;
5092
- // Contract v2: project_key is part of the bucket identity. Split hours
5093
- // carry several rows with the same (source, model, hour) under different
5094
- // project keys (plus the '' residual); omitting it here collapsed them
5095
- // to the last row and then advanced the offset past the rest, losing the
5096
- // attributed usage permanently. New queue rows already carry the
5097
- // server-normalized basename (rollout's appendProjectSplitRows); old
5098
- // owner/repo rows stay distinct here and let the server's own basename
5099
- // dedupe pick the larger total instead of collapsing them locally.
5100
- const projectKey =
5101
- typeof sanitized.project_key === "string" ? sanitized.project_key.trim() : "";
5102
- bucketMap.set(`${source}|${model}|${hourStart}|${projectKey}`, sanitized);
5103
- linesRead += 1;
5104
- if (linesRead >= maxBuckets) break;
5105
- }
5106
-
5107
- } finally {
5108
- await handle.close();
5109
- }
5110
- return {
5111
- buckets: Array.from(bucketMap.values()),
5112
- sessionStates: Array.from(sessionStateMap.values()),
5113
- nextOffset: offset,
5114
- };
5115
- }
5116
-
5117
- function normalizeGrokRepairSource(value) {
5118
- return typeof value === "string" ? value.trim().toLowerCase() : "";
5119
- }
5120
-
5121
- function normalizeGrokRepairModel(value) {
5122
- return typeof value === "string" && value.trim() ? value.trim() : "grok-build";
5123
- }
5124
-
5125
- function normalizeGrokRepairNumber(value) {
5126
- const n = Number(value);
5127
- return Number.isFinite(n) && n > 0 ? n : 0;
5128
- }
5129
-
5130
- function toGrokRepairHalfHourStart(value) {
5131
- if (value == null) return null;
5132
- const millis =
5133
- typeof value === "number"
5134
- ? value < 10_000_000_000
5135
- ? value * 1000
5136
- : value
5137
- : Date.parse(String(value));
5138
- if (!Number.isFinite(millis)) return null;
5139
- const halfHourMs = 30 * 60 * 1000;
5140
- return new Date(Math.floor(millis / halfHourMs) * halfHourMs).toISOString();
5141
- }
5142
-
5143
- function estimateGrokRepairTotals(totalTokens, conversationCount) {
5144
- const total = Math.trunc(normalizeGrokRepairNumber(totalTokens));
5145
- const inputTokens = Math.round(total * 0.8);
5146
- const outputTokens = Math.max(0, total - inputTokens);
5147
- return {
5148
- input_tokens: inputTokens,
5149
- cached_input_tokens: 0,
5150
- cache_creation_input_tokens: 0,
5151
- output_tokens: outputTokens,
5152
- reasoning_output_tokens: 0,
5153
- total_tokens: total,
5154
- billable_total_tokens: total,
5155
- conversation_count: Math.trunc(normalizeGrokRepairNumber(conversationCount)),
5156
- };
5157
- }
5158
-
5159
- function addGrokRepairTotals(target, delta) {
5160
- target.input_tokens += delta.input_tokens;
5161
- target.cached_input_tokens += delta.cached_input_tokens;
5162
- target.cache_creation_input_tokens += delta.cache_creation_input_tokens;
5163
- target.output_tokens += delta.output_tokens;
5164
- target.reasoning_output_tokens += delta.reasoning_output_tokens;
5165
- target.total_tokens += delta.total_tokens;
5166
- target.billable_total_tokens += delta.billable_total_tokens;
5167
- target.conversation_count += delta.conversation_count;
5168
- }
5169
-
5170
- function buildGrokRepairRowsFromSnapshots(sessionSnapshots) {
5171
- if (!sessionSnapshots || typeof sessionSnapshots !== "object") return [];
5172
-
5173
- const buckets = new Map();
5174
- for (const snapshot of Object.values(sessionSnapshots)) {
5175
- if (!snapshot || typeof snapshot !== "object") continue;
5176
- const totalTokens = Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens));
5177
- if (totalTokens <= 0) continue;
5178
-
5179
- const hourStart = toGrokRepairHalfHourStart(
5180
- snapshot.lastEventTimestamp || snapshot.updatedAt,
5181
- );
5182
- if (!hourStart) continue;
5183
-
5184
- const model = normalizeGrokRepairModel(snapshot.model);
5185
- const key = bucketKey("grok", model, hourStart);
5186
- let totals = buckets.get(key);
5187
- if (!totals) {
5188
- totals = {
5189
- source: "grok",
5190
- model,
5191
- hour_start: hourStart,
5192
- input_tokens: 0,
5193
- cached_input_tokens: 0,
5194
- cache_creation_input_tokens: 0,
5195
- output_tokens: 0,
5196
- reasoning_output_tokens: 0,
5197
- total_tokens: 0,
5198
- billable_total_tokens: 0,
5199
- conversation_count: 0,
5200
- };
5201
- buckets.set(key, totals);
5202
- }
5203
- addGrokRepairTotals(
5204
- totals,
5205
- estimateGrokRepairTotals(totalTokens, snapshot.messageCount),
5206
- );
5207
- }
5208
-
5209
- return Array.from(buckets.values()).sort((a, b) => {
5210
- const timeCompare = a.hour_start.localeCompare(b.hour_start);
5211
- return timeCompare || a.model.localeCompare(b.model);
5212
- });
5213
- }
5214
-
5215
- function applyGrokRepairHourlyState(cursors, rows) {
5216
- const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
5217
- const buckets = hourly.buckets && typeof hourly.buckets === "object" ? hourly.buckets : {};
5218
- const groupQueued =
5219
- hourly.groupQueued && typeof hourly.groupQueued === "object" ? hourly.groupQueued : {};
5220
-
5221
- for (const key of Object.keys(buckets)) {
5222
- if (key.startsWith("grok|")) {
5223
- delete buckets[key];
5224
- }
5225
- }
5226
- for (const key of Object.keys(groupQueued)) {
5227
- if (key.startsWith("grok|")) {
5228
- delete groupQueued[key];
5229
- }
5230
- }
5231
-
5232
- for (const row of rows) {
5233
- const totals = {
5234
- input_tokens: row.input_tokens,
5235
- cached_input_tokens: row.cached_input_tokens,
5236
- cache_creation_input_tokens: row.cache_creation_input_tokens,
5237
- output_tokens: row.output_tokens,
5238
- reasoning_output_tokens: row.reasoning_output_tokens,
5239
- total_tokens: row.total_tokens,
5240
- billable_total_tokens: row.billable_total_tokens,
5241
- conversation_count: row.conversation_count,
5242
- };
5243
- buckets[bucketKey("grok", row.model, row.hour_start)] = {
5244
- totals,
5245
- queuedKey: totalsKey(totals),
5246
- source: "grok",
5247
- hour_start: row.hour_start,
5248
- };
5249
- }
5250
-
5251
- cursors.hourly = {
5252
- ...hourly,
5253
- version: 3,
5254
- buckets,
5255
- groupQueued,
5256
- updatedAt: typeof hourly.updatedAt === "string" ? hourly.updatedAt : null,
5257
- };
5258
- }
5259
-
5260
- async function resetGrokRepairUploadOffset(queueStatePath) {
5261
- if (typeof queueStatePath !== "string" || !queueStatePath) return false;
5262
- let state = {};
5263
- try {
5264
- state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
5265
- } catch (_e) {
5266
- state = {};
5267
- }
5268
- state.offset = 0;
5269
- state.updatedAt = new Date().toISOString();
5270
- state.note = "reset_after_grok_append_only_repair_2026_05_v4";
5271
- await ensureDir(path.dirname(queueStatePath));
5272
- await fs.writeFile(queueStatePath, JSON.stringify(state, null, 2) + "\n", "utf8");
5273
- return true;
5274
- }
5275
-
5276
- function hasAppliedGrokRepairMigration(value) {
5277
- if (!value) return false;
5278
- if (value === true) return true;
5279
- if (value && typeof value === "object") {
5280
- if (value.status === "applied" || value.status === "noop") return true;
5281
- if (value.status) return false;
5282
- return value.rowsWritten != null || value.rowsRemoved != null;
5283
- }
5284
- return false;
5285
- }
5286
-
5287
- function serializeGrokRepairRow(row) {
5288
- return JSON.stringify({
5289
- source: "grok",
5290
- model: normalizeGrokRepairModel(row.model),
5291
- hour_start: row.hour_start,
5292
- input_tokens: row.input_tokens || 0,
5293
- cached_input_tokens: row.cached_input_tokens || 0,
5294
- cache_creation_input_tokens: row.cache_creation_input_tokens || 0,
5295
- output_tokens: row.output_tokens || 0,
5296
- reasoning_output_tokens: row.reasoning_output_tokens || 0,
5297
- total_tokens: row.total_tokens || 0,
5298
- billable_total_tokens: row.billable_total_tokens || 0,
5299
- conversation_count: row.conversation_count || 0,
5300
- });
5301
- }
5302
-
5303
- async function backupExistingFile(filePath) {
5304
- if (typeof filePath !== "string" || !filePath) return null;
5305
- try {
5306
- const stat = await fs.stat(filePath);
5307
- if (!stat.isFile()) return null;
5308
- } catch (e) {
5309
- if (e?.code === "ENOENT" || e?.code === "ENOTDIR") return null;
5310
- throw e;
5311
- }
5312
- const stamp = new Date().toISOString().replace(/[:.]/g, "-");
5313
- const backupPath = `${filePath}.bak.${stamp}`;
5314
- await fs.copyFile(filePath, backupPath);
5315
- return backupPath;
5316
- }
5317
-
5318
- async function resetUploadOffsetForMimoRepair(queueStatePath) {
5319
- if (typeof queueStatePath !== "string" || !queueStatePath) return false;
5320
- let state = {};
5321
- try {
5322
- state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
5323
- } catch (_e) {
5324
- state = {};
5325
- }
5326
- state.offset = 0;
5327
- state.updatedAt = new Date().toISOString();
5328
- state.note = "reset_after_mimo_claude_mislabel_repair_2026_06";
5329
- await ensureDir(path.dirname(queueStatePath));
5330
- await fs.writeFile(queueStatePath, JSON.stringify(state, null, 2) + "\n", "utf8");
5331
- return true;
5332
- }
5333
-
5334
- // Remove every source=mimo row from a queue file (atomic rewrite, backed up
5335
- // first). Returns the number of rows removed. Non-JSON lines are preserved.
5336
- async function dropMimoQueueRows(filePath) {
5337
- let raw;
5338
- try {
5339
- raw = await fs.readFile(filePath, "utf8");
5340
- } catch (e) {
5341
- if (e?.code === "ENOENT") return 0;
5342
- throw e;
5343
- }
5344
- const kept = [];
5345
- let removed = 0;
5346
- for (const line of raw.split("\n")) {
5347
- if (!line.trim()) continue;
5348
- let row;
5349
- try {
5350
- row = JSON.parse(line);
5351
- } catch (_e) {
5352
- kept.push(line);
5353
- continue;
5354
- }
5355
- if (row && row.source === "mimo") {
5356
- removed += 1;
5357
- continue;
5358
- }
5359
- kept.push(line);
5360
- }
5361
- if (removed === 0) return 0;
5362
- await backupExistingFile(filePath);
5363
- const tmp = `${filePath}.mimorepair.${process.pid}.${Date.now()}`;
5364
- await fs.writeFile(tmp, kept.length ? kept.join("\n") + "\n" : "", "utf8");
5365
- await fs.rename(tmp, filePath);
5366
- return removed;
5367
- }
5368
4851
 
5369
- // One-time repair for the 0.57.0 mimo mislabel bug. Purges all source=mimo data
5370
- // (the mislabeled Claude/claude-mem mirror) from the local queues and cursor
5371
- // state, so the next sync re-parses mimocode.db with the providerID-filtered
5372
- // reader and rebuilds source=mimo from scratch — correct mimo-auto only. Cloud
5373
- // orphans (mimo rows already uploaded) are cleaned server-side separately.
5374
- async function repairMimoClaudeMislabel({
5375
- cursors,
5376
- queuePath,
5377
- queueStatePath,
5378
- projectQueuePath,
5379
- projectQueueStatePath,
5380
- } = {}) {
4852
+ async function applyCloudConversationsBackfill({ cursors, queueStatePath }) {
5381
4853
  if (!cursors || typeof cursors !== "object") return false;
5382
- const migrations = (cursors.migrations ||= {});
5383
- if (migrations[MIMO_PROVIDER_REPAIR_KEY]) return false;
5384
-
5385
- const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : null;
5386
- const hasMimoBucket =
5387
- hourly && hourly.buckets
5388
- ? Object.keys(hourly.buckets).some((k) => k.startsWith("mimo|"))
5389
- : false;
4854
+ cursors.migrations = cursors.migrations || {};
4855
+ if (cursors.migrations[CLOUD_CONVERSATIONS_BACKFILL_KEY]) return false;
5390
4856
 
5391
- // Nothing mimo-related anywhere → mark done so we don't re-scan every sync.
5392
- let mainRaw = null;
4857
+ // Reset ONLY the cloud upload offset. The queue file itself is untouched;
4858
+ // ingest upserts are idempotent per (user, device, hour, source, model),
4859
+ // so replaying the whole queue is safe — it costs upload batches, not
4860
+ // correctness. Project queue is never uploaded and is not touched.
4861
+ let prevOffset = 0;
5393
4862
  try {
5394
- mainRaw = await fs.readFile(queuePath, "utf8");
5395
- } catch (e) {
5396
- if (e?.code !== "ENOENT") throw e;
5397
- }
5398
- const hasMimoRow =
5399
- typeof mainRaw === "string" &&
5400
- mainRaw.split("\n").some((l) => {
5401
- if (!l.trim()) return false;
5402
- try {
5403
- return JSON.parse(l).source === "mimo";
5404
- } catch (_e) {
5405
- return false;
5406
- }
5407
- });
5408
-
5409
- if (!hasMimoBucket && !hasMimoRow && !cursors.mimo) {
5410
- migrations[MIMO_PROVIDER_REPAIR_KEY] = new Date().toISOString();
5411
- return false;
5412
- }
5413
-
5414
- // 1. Drop source=mimo rows from the main + project queues.
5415
- const removedMain = await dropMimoQueueRows(queuePath);
5416
- const removedProject =
5417
- typeof projectQueuePath === "string" && projectQueuePath
5418
- ? await dropMimoQueueRows(projectQueuePath)
5419
- : 0;
5420
-
5421
- // 2. Clear stale mimo buckets from the aggregation state (keys are
5422
- // `source|model|hour` for hourly, `projectKey|source|hour` for project).
5423
- if (hourly && hourly.buckets) {
5424
- for (const k of Object.keys(hourly.buckets)) {
5425
- if (k.startsWith("mimo|")) delete hourly.buckets[k];
5426
- }
5427
- }
5428
- if (hourly && hourly.groupQueued) {
5429
- for (const k of Object.keys(hourly.groupQueued)) {
5430
- if (k.startsWith("mimo|")) delete hourly.groupQueued[k];
5431
- }
4863
+ const st = (await readJson(queueStatePath)) || {};
4864
+ prevOffset = Number(st.offset || 0);
4865
+ } catch (_e) {
4866
+ /* missing state file — nothing to reset */
5432
4867
  }
5433
- const projectHourly =
5434
- cursors.projectHourly && typeof cursors.projectHourly === "object"
5435
- ? cursors.projectHourly
5436
- : null;
5437
- if (projectHourly && projectHourly.buckets) {
5438
- for (const k of Object.keys(projectHourly.buckets)) {
5439
- if (k.includes("|mimo|")) delete projectHourly.buckets[k];
5440
- }
4868
+ if (prevOffset > 0) {
4869
+ await writeJson(queueStatePath, { offset: 0, updatedAt: new Date().toISOString() });
5441
4870
  }
5442
-
5443
- // 3. Reset the mimo message index so the next sync re-parses the DB fresh.
5444
- delete cursors.mimo;
5445
-
5446
- // 4. Reset upload offsets — the queue rewrite changed byte offsets, so a full
5447
- // replay is required (cloud keeps latest per key; orphan mimo rows already
5448
- // uploaded are removed server-side).
5449
- if (removedMain > 0) await resetUploadOffsetForMimoRepair(queueStatePath);
5450
- if (removedProject > 0) await resetUploadOffsetForMimoRepair(projectQueueStatePath);
5451
-
5452
- migrations[MIMO_PROVIDER_REPAIR_KEY] = {
4871
+ cursors.migrations[CLOUD_CONVERSATIONS_BACKFILL_KEY] = {
5453
4872
  appliedAt: new Date().toISOString(),
5454
- removedMain,
5455
- removedProject,
5456
- };
5457
- return true;
5458
- }
5459
-
5460
- async function resetUploadOffsetForZcodeRepair(queueStatePath, note) {
5461
- if (typeof queueStatePath !== "string" || !queueStatePath) return false;
5462
- let state = {};
5463
- try {
5464
- state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
5465
- } catch (_error) {
5466
- state = {};
5467
- }
5468
- state.offset = 0;
5469
- state.updatedAt = new Date().toISOString();
5470
- state.note = note;
5471
- await ensureDir(path.dirname(queueStatePath));
5472
- await fs.writeFile(queueStatePath, JSON.stringify(state, null, 2) + "\n", "utf8");
5473
- return true;
5474
- }
5475
-
5476
- function zcodeHourStartMs(hourStart) {
5477
- if (typeof hourStart !== "string" || !hourStart.trim()) return null;
5478
- const ms = Date.parse(hourStart);
5479
- return Number.isFinite(ms) ? ms : null;
5480
- }
5481
-
5482
- // One restored queue row from a backup peak (repairZcodePruneWindowRestore).
5483
- // peak.totals comes from zcodeRowTotals, which fills the billable/cost columns
5484
- // older backup rows are missing — the restored row must match the live queue
5485
- // schema exactly (audit fix: 恢复行缺 billable/cost 字段).
5486
- function buildZcodeRestorationRow(peak) {
5487
- return {
5488
- source: "zcode",
5489
- model: peak.model,
5490
- hour_start: peak.hour_start,
5491
- ...peak.totals,
5492
- };
5493
- }
5494
-
5495
- // Main bucket keys are `source|model|hour_start` — hour_start (an ISO string)
5496
- // never contains `|`, so it is everything after the LAST separator. Splitting
5497
- // on every `|` mis-parses models whose name contains one (audit fix).
5498
- function zcodeHourFromMainBucketKey(key) {
5499
- if (typeof key !== "string" || !key.startsWith("zcode|")) return null;
5500
- const last = key.lastIndexOf("|");
5501
- if (last <= "zcode|".length - 1) return null;
5502
- const hour = key.slice(last + 1);
5503
- return hour || null;
5504
- }
5505
-
5506
- function zcodeHourFromProjectBucketKey(key) {
5507
- if (typeof key !== "string") return null;
5508
- const marker = "|zcode|";
5509
- const idx = key.indexOf(marker);
5510
- if (idx < 0) return null;
5511
- const hour = key.slice(idx + marker.length);
5512
- return hour || null;
5513
- }
5514
-
5515
- function zcodeHourFromGroupQueuedKey(key) {
5516
- if (typeof key !== "string" || !key.startsWith("zcode|")) return null;
5517
- return key.slice("zcode|".length) || null;
5518
- }
5519
-
5520
- function zcodeHourIsPreWindow(hourStart, preserveBeforeMs) {
5521
- if (!Number.isFinite(preserveBeforeMs)) return false;
5522
- const hourMs = zcodeHourStartMs(hourStart);
5523
- return hourMs != null && hourMs < preserveBeforeMs;
5524
- }
5525
-
5526
- // Per-install rebuild floor. Prefer native model_usage MIN(started_at) when the
5527
- // native schema exists — an ancient legacy message does not prove continuous
5528
- // coverage through pruned native gaps (#828).
5529
- function resolveZcodeDbRebuildFloorMs(dbPath) {
5530
- if (!dbPath || !fssync.existsSync(dbPath)) return null;
5531
- if (hasZcodeNativeUsageSchema(dbPath)) {
5532
- try {
5533
- const rows = readSqliteJsonRows(
5534
- dbPath,
5535
- `SELECT MIN(started_at) AS floor
5536
- FROM model_usage
5537
- WHERE status = 'completed'
5538
- AND trim(model_id) != ''
5539
- AND (
5540
- input_tokens > 0 OR output_tokens > 0 OR reasoning_tokens > 0 OR
5541
- cache_creation_input_tokens > 0 OR cache_read_input_tokens > 0
5542
- )`,
5543
- { label: "ZCode", timeout: 10_000, maxBuffer: 1024 * 1024, readOnly: true },
5544
- );
5545
- const floor = coerceEpochMs(Array.isArray(rows) ? rows[0]?.floor : null);
5546
- return floor > 0 ? floor : null;
5547
- } catch (_error) {
5548
- return null;
5549
- }
5550
- }
5551
- let messages;
5552
- try {
5553
- messages = readZcodeDbMessages(dbPath, { readOnly: true });
5554
- } catch (_error) {
5555
- return null;
5556
- }
5557
- let floor = Number.POSITIVE_INFINITY;
5558
- for (const message of Array.isArray(messages) ? messages : []) {
5559
- const ms = coerceEpochMs(
5560
- message?.timeUpdated ?? message?.data?.time?.completed ?? message?.data?.time?.created,
5561
- );
5562
- if (ms > 0) floor = Math.min(floor, ms);
5563
- }
5564
- return Number.isFinite(floor) ? floor : null;
5565
- }
5566
-
5567
- // Shared wipe window across installs: the LATEST per-install floor (most
5568
- // conservative). Any listed install that exists but cannot report a floor
5569
- // defers the wipe — silently taking min() would erase another install's history.
5570
- function resolveZcodeRebuildFloorMs(dbPaths = []) {
5571
- const paths = [...new Set((Array.isArray(dbPaths) ? dbPaths : []).filter(Boolean))];
5572
- if (paths.length === 0) return null;
5573
- let latestFloor = 0;
5574
- for (const dbPath of paths) {
5575
- if (!fssync.existsSync(dbPath)) continue;
5576
- const floor = resolveZcodeDbRebuildFloorMs(dbPath);
5577
- if (!Number.isFinite(floor)) return null;
5578
- latestFloor = Math.max(latestFloor, floor);
5579
- }
5580
- return latestFloor > 0 ? latestFloor : null;
5581
- }
5582
-
5583
- // Wipe starts at the half-hour bucket that contains the first live message so
5584
- // that bucket is rebuilt (not preserved) and cannot be double-counted after
5585
- // cursors.zcode is cleared.
5586
- function zcodePreserveBeforeMsFromFloor(floorMs) {
5587
- if (!Number.isFinite(floorMs)) return null;
5588
- const iso = toUtcHalfHourStart(floorMs);
5589
- const ms = iso ? Date.parse(iso) : floorMs;
5590
- return Number.isFinite(ms) ? ms : null;
5591
- }
5592
-
5593
- function hasZcodeCursorHistory(cursors) {
5594
- return hasZcodeCursorState(cursors);
5595
- }
5596
-
5597
- function pruneZcodeMessageIndexForRebuild(messages, preserveBeforeMs) {
5598
- if (!messages || typeof messages !== "object") return 0;
5599
- let removed = 0;
5600
- for (const key of Object.keys(messages)) {
5601
- const entry = messages[key];
5602
- const attribution = typeof entry?.attribution === "string"
5603
- ? entry.attribution.split(" ")[0]
5604
- : entry?.attribution?.bucketStart;
5605
- const bucketStart = typeof attribution === "string" ? attribution : "";
5606
- // Keep unattributed entries: older cursors omit attribution, and deleting
5607
- // them re-adds still-present pre-window legacy messages onto preserved
5608
- // buckets (5000→10000). Prefer a possible in-window under-correction over
5609
- // permanent double-count (#828).
5610
- if (!bucketStart) continue;
5611
- if (zcodeHourIsPreWindow(bucketStart, preserveBeforeMs)) continue;
5612
- delete messages[key];
5613
- removed += 1;
5614
- }
5615
- return removed;
5616
- }
5617
-
5618
- function pruneZcodeMessageCursorForRebuild(cursors, preserveBeforeMs) {
5619
- if (!cursors || typeof cursors !== "object") return;
5620
- if (!Number.isFinite(preserveBeforeMs)) {
5621
- delete cursors.zcode;
5622
- return;
5623
- }
5624
- const root = cursors.zcode;
5625
- if (!root || typeof root !== "object") {
5626
- delete cursors.zcode;
5627
- return;
5628
- }
5629
- pruneZcodeMessageIndexForRebuild(root.messages, preserveBeforeMs);
5630
- for (const nestedKey of ["native", "wsl"]) {
5631
- const nested = root[nestedKey];
5632
- if (nested && typeof nested === "object") {
5633
- pruneZcodeMessageIndexForRebuild(nested.messages, preserveBeforeMs);
5634
- }
5635
- }
5636
- const hasMessages = Object.keys(root.messages || {}).length > 0
5637
- || Object.keys(root.native?.messages || {}).length > 0
5638
- || Object.keys(root.wsl?.messages || {}).length > 0;
5639
- if (!hasMessages) delete cursors.zcode;
5640
- }
5641
-
5642
- async function queueFileHasZcodeRows(filePath) {
5643
- if (typeof filePath !== "string" || !filePath) return false;
5644
- let raw;
5645
- try {
5646
- raw = await fs.readFile(filePath, "utf8");
5647
- } catch (error) {
5648
- if (error?.code === "ENOENT") return false;
5649
- throw error;
5650
- }
5651
- for (const line of raw.split("\n")) {
5652
- if (!line.trim()) continue;
5653
- try {
5654
- const row = JSON.parse(line);
5655
- if (row?.source === "zcode") return true;
5656
- } catch (_error) {
5657
- // keep scanning
5658
- }
5659
- }
5660
- return false;
5661
- }
5662
-
5663
- function isCompletedZcodeMigration(entry) {
5664
- return Boolean(entry) && !(typeof entry === "object" && entry.skipped);
5665
- }
5666
-
5667
- async function dropZcodeQueueRows(filePath, {
5668
- retainRetractions = false,
5669
- preserveBeforeMs = null,
5670
- } = {}) {
5671
- let raw;
5672
- try {
5673
- raw = await fs.readFile(filePath, "utf8");
5674
- } catch (error) {
5675
- if (error?.code === "ENOENT") return { removed: 0, retractions: 0, preserved: 0 };
5676
- throw error;
5677
- }
5678
- const kept = [];
5679
- const retractions = new Map();
5680
- let removed = 0;
5681
- let preserved = 0;
5682
- for (const line of raw.split("\n")) {
5683
- if (!line.trim()) continue;
5684
- let row;
5685
- try {
5686
- row = JSON.parse(line);
5687
- } catch (_error) {
5688
- kept.push(line);
5689
- continue;
5690
- }
5691
- if (row?.source === "zcode") {
5692
- const model = typeof row.model === "string" && row.model.trim()
5693
- ? row.model.trim()
5694
- : "unknown";
5695
- const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
5696
- // #828: keep pre-window rows intact — the live DB cannot rebuild them.
5697
- if (zcodeHourIsPreWindow(hourStart, preserveBeforeMs)) {
5698
- kept.push(line);
5699
- preserved += 1;
5700
- continue;
5701
- }
5702
- removed += 1;
5703
- if (retainRetractions && hourStart) {
5704
- retractions.set(`${model}|${hourStart}`, {
5705
- source: "zcode",
5706
- model,
5707
- hour_start: hourStart,
5708
- input_tokens: 0,
5709
- cached_input_tokens: 0,
5710
- cache_creation_input_tokens: 0,
5711
- output_tokens: 0,
5712
- reasoning_output_tokens: 0,
5713
- total_tokens: 0,
5714
- conversation_count: 0,
5715
- });
5716
- }
5717
- continue;
5718
- }
5719
- kept.push(line);
5720
- }
5721
- if (removed === 0) return { removed: 0, retractions: 0, preserved };
5722
- await backupExistingFile(filePath);
5723
- const tmp = `${filePath}.zcoderepair.${process.pid}.${Date.now()}`;
5724
- const rewritten = [
5725
- ...kept,
5726
- ...[...retractions.values()].map((row) => JSON.stringify(row)),
5727
- ];
5728
- await fs.writeFile(tmp, rewritten.length ? rewritten.join("\n") + "\n" : "", "utf8");
5729
- await fs.rename(tmp, filePath);
5730
- return { removed, retractions: retractions.size, preserved };
5731
- }
5732
-
5733
- async function repairZcodeUsageMigration({
5734
- migrationKey,
5735
- queueStateNote,
5736
- cursors,
5737
- queuePath,
5738
- queueStatePath,
5739
- projectQueuePath,
5740
- projectQueueStatePath,
5741
- rebuildFloorMs = null,
5742
- } = {}) {
5743
- if (!cursors || typeof cursors !== "object") return false;
5744
- const migrations = (cursors.migrations ||= {});
5745
- // Mirror Codex: a prior {skipped:true} must retry once the live ledger can
5746
- // establish a rebuild floor (or the pre-window history is gone).
5747
- const prior = migrations[migrationKey];
5748
- if (isCompletedZcodeMigration(prior)) return false;
5749
-
5750
- const preserveBeforeMs = zcodePreserveBeforeMsFromFloor(rebuildFloorMs);
5751
- // Empty / unreadable live ledger + existing history => refuse the wipe. The
5752
- // forward reader still works; only this historical correction defers (#828).
5753
- if (preserveBeforeMs == null) {
5754
- const hasQueueHistory = await queueFileHasZcodeRows(queuePath)
5755
- || await queueFileHasZcodeRows(projectQueuePath);
5756
- if (hasZcodeCursorHistory(cursors) || hasQueueHistory) {
5757
- migrations[migrationKey] = {
5758
- skipped: true,
5759
- reason: "zcode_rebuild_floor_unavailable",
5760
- at: new Date().toISOString(),
5761
- };
5762
- return false;
5763
- }
5764
- }
5765
-
5766
- const mainRepair = await dropZcodeQueueRows(queuePath, {
5767
- retainRetractions: true,
5768
- preserveBeforeMs,
5769
- });
5770
- const projectRepair = projectQueuePath
5771
- ? await dropZcodeQueueRows(projectQueuePath, { preserveBeforeMs })
5772
- : { removed: 0, retractions: 0, preserved: 0 };
5773
-
5774
- const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : null;
5775
- if (hourly?.buckets) {
5776
- for (const key of Object.keys(hourly.buckets)) {
5777
- if (!key.startsWith("zcode|")) continue;
5778
- if (zcodeHourIsPreWindow(zcodeHourFromMainBucketKey(key), preserveBeforeMs)) continue;
5779
- delete hourly.buckets[key];
5780
- }
5781
- }
5782
- if (hourly?.groupQueued) {
5783
- for (const key of Object.keys(hourly.groupQueued)) {
5784
- if (!key.startsWith("zcode|")) continue;
5785
- if (zcodeHourIsPreWindow(zcodeHourFromGroupQueuedKey(key), preserveBeforeMs)) continue;
5786
- delete hourly.groupQueued[key];
5787
- }
5788
- }
5789
- const projectHourly = cursors.projectHourly && typeof cursors.projectHourly === "object"
5790
- ? cursors.projectHourly
5791
- : null;
5792
- if (projectHourly?.buckets) {
5793
- for (const key of Object.keys(projectHourly.buckets)) {
5794
- if (!key.includes("|zcode|")) continue;
5795
- if (zcodeHourIsPreWindow(zcodeHourFromProjectBucketKey(key), preserveBeforeMs)) continue;
5796
- delete projectHourly.buckets[key];
5797
- }
5798
- }
5799
- // Keep pre-window message dedup so preserved buckets are not double-counted
5800
- // when legacy rows still exist in the live DB. Clear only in-window (and
5801
- // unattributed) cursor entries so the rescan can rebuild those hours.
5802
- pruneZcodeMessageCursorForRebuild(cursors, preserveBeforeMs);
5803
-
5804
- if (mainRepair.removed > 0) {
5805
- await resetUploadOffsetForZcodeRepair(queueStatePath, queueStateNote);
5806
- }
5807
- if (projectRepair.removed > 0) {
5808
- await resetUploadOffsetForZcodeRepair(projectQueueStatePath, queueStateNote);
5809
- }
5810
- migrations[migrationKey] = {
5811
- appliedAt: new Date().toISOString(),
5812
- removedMain: mainRepair.removed,
5813
- removedProject: projectRepair.removed,
5814
- retractions: mainRepair.retractions,
5815
- preservedMain: mainRepair.preserved || 0,
5816
- preservedProject: projectRepair.preserved || 0,
5817
- rebuildFloorMs: preserveBeforeMs,
5818
- };
5819
- return true;
5820
- }
5821
-
5822
- function zcodeRowTotals(row) {
5823
- return {
5824
- input_tokens: Number(row?.input_tokens) || 0,
5825
- cached_input_tokens: Number(row?.cached_input_tokens) || 0,
5826
- cache_creation_input_tokens: Number(row?.cache_creation_input_tokens) || 0,
5827
- output_tokens: Number(row?.output_tokens) || 0,
5828
- reasoning_output_tokens: Number(row?.reasoning_output_tokens) || 0,
5829
- total_tokens: Number(row?.total_tokens) || 0,
5830
- // Queue-schema completeness (audit fix): backup rows written by older
5831
- // versions may lack these two columns; restored rows must carry them or
5832
- // the restored (recovered) usage uploads with a shape the cloud and the
5833
- // local readers have to heal. billable falls back to total (zcode counts
5834
- // plan-included usage as billable), cost to 0 (read-side pricing recomputes).
5835
- billable_total_tokens: row?.billable_total_tokens == null
5836
- ? Math.max(0, Number(row?.total_tokens) || 0)
5837
- : Math.max(0, Number(row.billable_total_tokens) || 0),
5838
- total_cost_usd: Math.max(0, Number(row?.total_cost_usd) || 0),
5839
- conversation_count: Number(row?.conversation_count) || 0,
5840
- };
5841
- }
5842
-
5843
- function zcodeTotalsDominate(candidate, baseline) {
5844
- if (!candidate) return false;
5845
- if (!baseline) return (candidate.total_tokens || 0) > 0;
5846
- return (candidate.total_tokens || 0) > (baseline.total_tokens || 0);
5847
- }
5848
-
5849
- async function collectZcodeQueuePeaks(filePath, { beforeMs = null } = {}) {
5850
- const peaks = new Map();
5851
- let raw;
5852
- try {
5853
- raw = await fs.readFile(filePath, "utf8");
5854
- } catch (error) {
5855
- if (error?.code === "ENOENT") return peaks;
5856
- throw error;
5857
- }
5858
- for (const line of raw.split("\n")) {
5859
- if (!line.trim()) continue;
5860
- let row;
5861
- try {
5862
- row = JSON.parse(line);
5863
- } catch (_error) {
5864
- continue;
5865
- }
5866
- if (row?.source !== "zcode") continue;
5867
- const model = typeof row.model === "string" && row.model.trim()
5868
- ? row.model.trim()
5869
- : "unknown";
5870
- const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
5871
- if (!hourStart) continue;
5872
- if (beforeMs != null && !zcodeHourIsPreWindow(hourStart, beforeMs)) continue;
5873
- const key = `${model}|${hourStart}`;
5874
- const totals = zcodeRowTotals(row);
5875
- const prev = peaks.get(key);
5876
- if (!prev || zcodeTotalsDominate(totals, prev.totals)) {
5877
- peaks.set(key, { model, hour_start: hourStart, totals });
5878
- }
5879
- }
5880
- return peaks;
5881
- }
5882
-
5883
- // Last-wins per key (cloud replay semantics). A historical peak of 9000 followed
5884
- // by a repair retraction of 0 must report 0 so restore still re-appends.
5885
- async function collectZcodeQueueLatest(filePath, { beforeMs = null } = {}) {
5886
- const latest = new Map();
5887
- let raw;
5888
- try {
5889
- raw = await fs.readFile(filePath, "utf8");
5890
- } catch (error) {
5891
- if (error?.code === "ENOENT") return latest;
5892
- throw error;
5893
- }
5894
- for (const line of raw.split("\n")) {
5895
- if (!line.trim()) continue;
5896
- let row;
5897
- try {
5898
- row = JSON.parse(line);
5899
- } catch (_error) {
5900
- continue;
5901
- }
5902
- if (row?.source !== "zcode") continue;
5903
- const model = typeof row.model === "string" && row.model.trim()
5904
- ? row.model.trim()
5905
- : "unknown";
5906
- const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
5907
- if (!hourStart) continue;
5908
- if (beforeMs != null && !zcodeHourIsPreWindow(hourStart, beforeMs)) continue;
5909
- latest.set(`${model}|${hourStart}`, {
5910
- model,
5911
- hour_start: hourStart,
5912
- totals: zcodeRowTotals(row),
5913
- });
5914
- }
5915
- return latest;
5916
- }
5917
-
5918
- async function listZcodeQueueBackupPaths(queuePath) {
5919
- if (typeof queuePath !== "string" || !queuePath) return [];
5920
- const dir = path.dirname(queuePath);
5921
- const base = path.basename(queuePath);
5922
- let names;
5923
- try {
5924
- names = await fs.readdir(dir);
5925
- } catch (error) {
5926
- if (error?.code === "ENOENT") return [];
5927
- throw error;
5928
- }
5929
- return names
5930
- .filter((name) => name.startsWith(`${base}.bak.`))
5931
- .map((name) => path.join(dir, name))
5932
- .sort();
5933
- }
5934
-
5935
- async function repairZcodePruneWindowRestore({
5936
- cursors,
5937
- queuePath,
5938
- queueStatePath,
5939
- dbPaths = [],
5940
- rebuildFloorMs: rebuildFloorMsOption = null,
5941
- } = {}) {
5942
- if (!cursors || typeof cursors !== "object") return false;
5943
- const migrations = (cursors.migrations ||= {});
5944
- const prior = migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY];
5945
- if (isCompletedZcodeMigration(prior)) return false;
5946
-
5947
- const rebuildFloorMs = Number.isFinite(rebuildFloorMsOption)
5948
- ? rebuildFloorMsOption
5949
- : resolveZcodeRebuildFloorMs(dbPaths);
5950
- if (!Number.isFinite(rebuildFloorMs)) {
5951
- migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
5952
- skipped: true,
5953
- reason: "zcode_rebuild_floor_unavailable",
5954
- at: new Date().toISOString(),
5955
- };
5956
- return false;
5957
- }
5958
-
5959
- const backupPaths = await listZcodeQueueBackupPaths(queuePath);
5960
- if (backupPaths.length === 0) {
5961
- migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
5962
- skipped: true,
5963
- reason: "zcode_queue_backup_missing",
5964
- at: new Date().toISOString(),
5965
- rebuildFloorMs,
5966
- };
5967
- return false;
5968
- }
5969
-
5970
- const preserveBeforeMs = zcodePreserveBeforeMsFromFloor(rebuildFloorMs);
5971
- const backupPeaks = new Map();
5972
- for (const backupPath of backupPaths) {
5973
- const peaks = await collectZcodeQueuePeaks(backupPath, { beforeMs: preserveBeforeMs });
5974
- for (const [key, peak] of peaks) {
5975
- const prev = backupPeaks.get(key);
5976
- if (!prev || zcodeTotalsDominate(peak.totals, prev.totals)) {
5977
- backupPeaks.set(key, peak);
5978
- }
5979
- }
5980
- }
5981
- if (backupPeaks.size === 0) {
5982
- migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
5983
- appliedAt: new Date().toISOString(),
5984
- restored: 0,
5985
- rebuildFloorMs,
5986
- preserveBeforeMs,
5987
- note: "no_pre_window_peaks_in_backups",
5988
- };
5989
- return true;
5990
- }
5991
-
5992
- const liveLatest = await collectZcodeQueueLatest(queuePath, { beforeMs: preserveBeforeMs });
5993
- const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
5994
- hourly.buckets ||= {};
5995
- const restorations = [];
5996
- for (const [key, peak] of backupPeaks) {
5997
- const bucketKeyName = bucketKey("zcode", peak.model, peak.hour_start);
5998
- const live = liveLatest.get(key)?.totals || null;
5999
- // Compare against the latest queue emission (last-wins), not historical peaks
6000
- // or cursor buckets — a 9000→0 retraction must still restore.
6001
- if (!zcodeTotalsDominate(peak.totals, live)) {
6002
- const bucketTotals = hourly.buckets[bucketKeyName]?.totals
6003
- ? zcodeRowTotals(hourly.buckets[bucketKeyName].totals)
6004
- : null;
6005
- if (bucketTotals && zcodeTotalsDominate(peak.totals, bucketTotals)) {
6006
- hourly.buckets[bucketKeyName] = {
6007
- ...(hourly.buckets[bucketKeyName] || {}),
6008
- totals: { ...peak.totals },
6009
- hour_start: peak.hour_start,
6010
- source: "zcode",
6011
- model: peak.model,
6012
- };
6013
- }
6014
- continue;
6015
- }
6016
- restorations.push(peak);
6017
- hourly.buckets[bucketKeyName] = {
6018
- ...(hourly.buckets[bucketKeyName] || {}),
6019
- totals: { ...peak.totals },
6020
- hour_start: peak.hour_start,
6021
- source: "zcode",
6022
- model: peak.model,
6023
- };
6024
- }
6025
-
6026
- if (restorations.length === 0) {
6027
- migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
6028
- appliedAt: new Date().toISOString(),
6029
- restored: 0,
6030
- rebuildFloorMs,
6031
- backupsConsidered: backupPaths.length,
6032
- note: "backup_peaks_not_higher",
6033
- };
6034
- return true;
6035
- }
6036
-
6037
- await backupExistingFile(queuePath);
6038
- const append = restorations.map((peak) => JSON.stringify(buildZcodeRestorationRow(peak)));
6039
- await fs.appendFile(queuePath, `${append.join("\n")}\n`, "utf8");
6040
- await resetUploadOffsetForZcodeRepair(
6041
- queueStatePath,
6042
- "reset_after_zcode_prune_window_restore_2026_10",
6043
- );
6044
- migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
6045
- appliedAt: new Date().toISOString(),
6046
- restored: restorations.length,
6047
- rebuildFloorMs,
6048
- backupsConsidered: backupPaths.length,
6049
- };
6050
- return true;
6051
- }
6052
-
6053
- async function repairZcodeNativeUsageMigration(options = {}) {
6054
- return repairZcodeUsageMigration({
6055
- ...options,
6056
- migrationKey: ZCODE_NATIVE_USAGE_REPAIR_KEY,
6057
- queueStateNote: "reset_after_zcode_native_usage_repair_2026_08",
6058
- });
6059
- }
6060
-
6061
- async function repairZcodeInclusiveTokenMigration(options = {}) {
6062
- return repairZcodeUsageMigration({
6063
- ...options,
6064
- migrationKey: ZCODE_INCLUSIVE_TOKEN_REPAIR_KEY,
6065
- queueStateNote: "reset_after_zcode_inclusive_token_repair_2026_09",
6066
- });
6067
- }
6068
-
6069
- async function repairGrokQueueFromSessionSnapshots({ cursors, queuePath, queueStatePath } = {}) {
6070
- if (!cursors || typeof cursors !== "object") return false;
6071
- const grokState = (cursors.grok ||= {});
6072
- const migrations = (grokState.migrations ||= {});
6073
- if (hasAppliedGrokRepairMigration(migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY])) {
6074
- return false;
6075
- }
6076
-
6077
- let raw = "";
6078
- try {
6079
- raw = await fs.readFile(queuePath, "utf8");
6080
- } catch (e) {
6081
- if (e?.code !== "ENOENT") throw e;
6082
- }
6083
-
6084
- const latestGrokRows = new Map();
6085
- let existingGrokRows = 0;
6086
- for (const line of raw.split("\n")) {
6087
- if (!line.trim()) continue;
6088
- let row;
6089
- try {
6090
- row = JSON.parse(line);
6091
- } catch (_e) {
6092
- continue;
6093
- }
6094
-
6095
- if (normalizeGrokRepairSource(row?.source) === "grok") {
6096
- const model = normalizeGrokRepairModel(row.model);
6097
- const hourStart = typeof row.hour_start === "string" ? row.hour_start : null;
6098
- if (!hourStart) continue;
6099
- existingGrokRows += 1;
6100
- latestGrokRows.set(bucketKey("grok", model, hourStart), {
6101
- ...row,
6102
- source: "grok",
6103
- model,
6104
- hour_start: hourStart,
6105
- });
6106
- }
6107
- }
6108
-
6109
- if (existingGrokRows === 0) {
6110
- migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
6111
- status: "noop",
6112
- appliedAt: new Date().toISOString(),
6113
- existingGrokRows: 0,
6114
- rowsWritten: 0,
6115
- snapshotsUsed: 0,
6116
- uploadOffsetReset: false,
6117
- };
6118
- return false;
6119
- }
6120
-
6121
- const repairRows = buildGrokRepairRowsFromSnapshots(grokState.sessionSnapshots);
6122
- if (repairRows.length === 0) {
6123
- migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
6124
- status: "skipped",
6125
- appliedAt: new Date().toISOString(),
6126
- reason: "missing-session-snapshots",
6127
- existingGrokRows,
6128
- rowsWritten: 0,
6129
- snapshotsUsed: 0,
6130
- uploadOffsetReset: false,
6131
- };
6132
- return false;
6133
- }
6134
-
6135
- applyGrokRepairHourlyState(cursors, repairRows);
6136
-
6137
- const repairLines = [];
6138
- const repairKeys = new Set();
6139
- for (const row of repairRows) {
6140
- const key = bucketKey("grok", row.model, row.hour_start);
6141
- repairKeys.add(key);
6142
- const current = latestGrokRows.get(key);
6143
- if (current && totalsKey(current) === totalsKey(row)) continue;
6144
- repairLines.push(serializeGrokRepairRow(row));
6145
- }
6146
-
6147
- const zeroTotals = {
6148
- input_tokens: 0,
6149
- cached_input_tokens: 0,
6150
- cache_creation_input_tokens: 0,
6151
- output_tokens: 0,
6152
- reasoning_output_tokens: 0,
6153
- total_tokens: 0,
6154
- billable_total_tokens: 0,
6155
- conversation_count: 0,
6156
- };
6157
- let staleRowsRetracted = 0;
6158
- for (const [key, row] of latestGrokRows.entries()) {
6159
- if (repairKeys.has(key)) continue;
6160
- if (totalsKey(row) === totalsKey(zeroTotals)) continue;
6161
- staleRowsRetracted += 1;
6162
- repairLines.push(serializeGrokRepairRow({
6163
- ...zeroTotals,
6164
- model: row.model,
6165
- hour_start: row.hour_start,
6166
- }));
6167
- }
6168
-
6169
- if (repairLines.length === 0) {
6170
- migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
6171
- status: "noop",
6172
- appliedAt: new Date().toISOString(),
6173
- existingGrokRows,
6174
- rowsWritten: 0,
6175
- staleRowsRetracted,
6176
- snapshotsUsed: repairRows.length,
6177
- uploadOffsetReset: false,
6178
- };
6179
- return false;
6180
- }
6181
-
6182
- await ensureDir(path.dirname(queuePath));
6183
- const queueBackupPath = await backupExistingFile(queuePath);
6184
- const queueStateBackupPath = await backupExistingFile(queueStatePath);
6185
- await fs.appendFile(queuePath, `${repairLines.join("\n")}\n`, "utf8");
6186
-
6187
- const uploadOffsetReset = await resetGrokRepairUploadOffset(queueStatePath);
6188
- migrations[GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY] = {
6189
- status: "applied",
6190
- appliedAt: new Date().toISOString(),
6191
- existingGrokRows,
6192
- rowsWritten: repairLines.length,
6193
- staleRowsRetracted,
6194
- snapshotsUsed: Object.values(grokState.sessionSnapshots || {}).filter((snapshot) => {
6195
- if (!snapshot || typeof snapshot !== "object") return false;
6196
- if (Math.trunc(normalizeGrokRepairNumber(snapshot.totalTokens)) <= 0) return false;
6197
- return Boolean(toGrokRepairHalfHourStart(snapshot.lastEventTimestamp || snapshot.updatedAt));
6198
- }).length,
6199
- uploadOffsetReset,
6200
- queueBackupPath,
6201
- queueStateBackupPath,
6202
- };
6203
- return true;
6204
- }
6205
-
6206
- async function applyCloudConversationsBackfill({ cursors, queueStatePath }) {
6207
- if (!cursors || typeof cursors !== "object") return false;
6208
- cursors.migrations = cursors.migrations || {};
6209
- if (cursors.migrations[CLOUD_CONVERSATIONS_BACKFILL_KEY]) return false;
6210
-
6211
- // Reset ONLY the cloud upload offset. The queue file itself is untouched;
6212
- // ingest upserts are idempotent per (user, device, hour, source, model),
6213
- // so replaying the whole queue is safe — it costs upload batches, not
6214
- // correctness. Project queue is never uploaded and is not touched.
6215
- let prevOffset = 0;
6216
- try {
6217
- const st = (await readJson(queueStatePath)) || {};
6218
- prevOffset = Number(st.offset || 0);
6219
- } catch (_e) {
6220
- /* missing state file — nothing to reset */
6221
- }
6222
- if (prevOffset > 0) {
6223
- await writeJson(queueStatePath, { offset: 0, updatedAt: new Date().toISOString() });
6224
- }
6225
- cursors.migrations[CLOUD_CONVERSATIONS_BACKFILL_KEY] = {
6226
- appliedAt: new Date().toISOString(),
6227
- previousOffset: prevOffset,
4873
+ previousOffset: prevOffset,
6228
4874
  };
6229
4875
  return prevOffset > 0;
6230
4876
  }
6231
-
6232
- async function migrateCursorUnknownBuckets({ cursors, queuePath }) {
6233
- if (!cursors || typeof cursors !== "object") return;
6234
- cursors.migrations = cursors.migrations || {};
6235
- if (cursors.migrations[CURSOR_UNKNOWN_MIGRATION_KEY]) return;
6236
-
6237
- const buckets = cursors.hourly?.buckets;
6238
- if (!buckets || typeof buckets !== "object") {
6239
- cursors.migrations[CURSOR_UNKNOWN_MIGRATION_KEY] = new Date().toISOString();
6240
- return;
6241
- }
6242
-
6243
- const retractions = [];
6244
- for (const key of Object.keys(buckets)) {
6245
- if (!key.startsWith("cursor|unknown|")) continue;
6246
- const hourStart = key.split("|").slice(2).join("|");
6247
- retractions.push(
6248
- JSON.stringify({
6249
- source: "cursor",
6250
- model: "unknown",
6251
- hour_start: hourStart,
6252
- input_tokens: 0,
6253
- cached_input_tokens: 0,
6254
- cache_creation_input_tokens: 0,
6255
- output_tokens: 0,
6256
- reasoning_output_tokens: 0,
6257
- total_tokens: 0,
6258
- conversation_count: 0,
6259
- }),
6260
- );
6261
- delete buckets[key];
6262
- }
6263
-
6264
- if (retractions.length > 0) {
6265
- await ensureDir(path.dirname(queuePath));
6266
- await fs.appendFile(queuePath, retractions.join("\n") + "\n");
6267
- if (cursors.cursorApi) {
6268
- cursors.cursorApi.lastRecordTimestamp = null;
6269
- }
6270
- }
6271
-
6272
- cursors.migrations[CURSOR_UNKNOWN_MIGRATION_KEY] = new Date().toISOString();
6273
- }
6274
-
6275
- async function migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rolloutFiles }) {
6276
- if (!cursors || typeof cursors !== "object") return;
6277
- cursors.migrations = cursors.migrations || {};
6278
- if (cursors.migrations[ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY]) return;
6279
-
6280
- const rolloutPathSources = new Map();
6281
- for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
6282
- const filePath = typeof entry === "string" ? entry : entry?.path;
6283
- const source = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
6284
- if (!filePath) continue;
6285
- if (source === "codex" || source === "every-code") {
6286
- rolloutPathSources.set(filePath, source);
6287
- }
6288
- }
6289
-
6290
- if (cursors.files && typeof cursors.files === "object") {
6291
- for (const filePath of rolloutPathSources.keys()) {
6292
- delete cursors.files[filePath];
6293
- }
6294
- }
6295
- // The migration clears Codex buckets and reparses the discovered corpus from
6296
- // byte zero. Persisted event keys belong to the cleared buckets, so retaining
6297
- // them can suppress the rebuild when a moved session still has an old path
6298
- // cursor. Rebuild the hash inventory together with the buckets. Counted
6299
- // compaction ids (#652) belong to the cleared buckets the same way.
6300
- cursors.codexHashes = [];
6301
- cursors.codexCompactionResponseIds = [];
6302
-
6303
- const buckets = cursors.hourly?.buckets;
6304
- const retractions = [];
6305
- if (buckets && typeof buckets === "object") {
6306
- for (const key of Object.keys(buckets)) {
6307
- const [source, model, ...hourParts] = key.split("|");
6308
- if (source !== "codex" && source !== "every-code") continue;
6309
- const hourStart = hourParts.join("|");
6310
- retractions.push(
6311
- JSON.stringify({
6312
- source,
6313
- model: model || "unknown",
6314
- hour_start: hourStart,
6315
- input_tokens: 0,
6316
- cached_input_tokens: 0,
6317
- cache_creation_input_tokens: 0,
6318
- output_tokens: 0,
6319
- reasoning_output_tokens: 0,
6320
- total_tokens: 0,
6321
- billable_total_tokens: 0,
6322
- conversation_count: 0,
6323
- }),
6324
- );
6325
- delete buckets[key];
6326
- }
6327
- }
6328
-
6329
- const groupQueued = cursors.hourly?.groupQueued;
6330
- if (groupQueued && typeof groupQueued === "object") {
6331
- for (const key of Object.keys(groupQueued)) {
6332
- if (key.startsWith("codex|") || key.startsWith("every-code|")) {
6333
- delete groupQueued[key];
6334
- }
6335
- }
6336
- }
6337
-
6338
- if (retractions.length > 0) {
6339
- await ensureDir(path.dirname(queuePath));
6340
- await fs.appendFile(queuePath, retractions.join("\n") + "\n");
6341
- }
6342
-
6343
- cursors.migrations[ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY] = new Date().toISOString();
6344
- }
6345
-
6346
- function codebuddyUsageRowKey(row) {
6347
- if (!row || row.source !== "codebuddy") return null;
6348
- const model = typeof row.model === "string" && row.model ? row.model : "unknown";
6349
- const hourStart = typeof row.hour_start === "string" && row.hour_start ? row.hour_start : null;
6350
- return hourStart ? bucketKey("codebuddy", model, hourStart) : null;
6351
- }
6352
-
6353
- function codebuddyZeroUsageRow(model, hourStart) {
6354
- return {
6355
- source: "codebuddy",
6356
- model: model || "unknown",
6357
- hour_start: hourStart,
6358
- input_tokens: 0,
6359
- cached_input_tokens: 0,
6360
- cache_creation_input_tokens: 0,
6361
- output_tokens: 0,
6362
- reasoning_output_tokens: 0,
6363
- total_tokens: 0,
6364
- billable_total_tokens: 0,
6365
- conversation_count: 0,
6366
- };
6367
- }
6368
-
6369
- // Issue #403 historical repair. Before the JSONL usage fix, CodeBuddy could
6370
- // ingest the same round-trip once from ~/.codebuddy/projects and once from the
6371
- // IDE extension log. The forward resolver now disables logs whenever any JSONL
6372
- // has usable rawUsage, but existing queues/hourly state need one guarded,
6373
- // atomic rebuild so the old double-count is not left visible forever.
6374
- async function repairCodebuddyLogJsonlOverlap({
6375
- cursors,
6376
- queuePath,
6377
- queueStatePath,
6378
- codebuddyFiles,
6379
- env = process.env,
6380
- } = {}) {
6381
- if (!cursors || typeof cursors !== "object") return false;
6382
- const migrations = (cursors.migrations ||= {});
6383
- const prior = migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY];
6384
- if (prior && !(typeof prior === "object" && prior.skipped)) return false;
6385
-
6386
- const files = Array.isArray(codebuddyFiles) ? codebuddyFiles : [];
6387
- const jsonlFiles = files.filter((filePath) => typeof filePath === "string" && filePath.endsWith(".jsonl"));
6388
- const logFiles = files.filter((filePath) => typeof filePath === "string" && filePath.endsWith(".log"));
6389
- const hasDetailedJsonl = jsonlFiles.some((filePath) => codebuddyJsonlHasUsage(filePath));
6390
-
6391
- const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
6392
- const liveHourlyKeys = new Set(
6393
- Object.keys(hourly.buckets || {}).filter((key) => key.startsWith("codebuddy|")),
6394
- );
6395
- let queueRaw = "";
6396
- try {
6397
- queueRaw = await fs.readFile(queuePath, "utf8");
6398
- } catch (e) {
6399
- if (e?.code !== "ENOENT") throw e;
6400
- }
6401
- const queueCodebuddyKeys = new Set();
6402
- const keptQueueLines = [];
6403
- for (const line of queueRaw.split("\n")) {
6404
- if (!line.trim()) continue;
6405
- let row;
6406
- try {
6407
- row = JSON.parse(line);
6408
- } catch (_e) {
6409
- keptQueueLines.push(line);
6410
- continue;
6411
- }
6412
- const key = codebuddyUsageRowKey(row);
6413
- if (!key) {
6414
- keptQueueLines.push(line);
6415
- continue;
6416
- }
6417
- queueCodebuddyKeys.add(key);
6418
- }
6419
- const liveKeys = new Set([...liveHourlyKeys, ...queueCodebuddyKeys]);
6420
- if (liveKeys.size === 0) {
6421
- migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
6422
- status: "noop",
6423
- appliedAt: new Date().toISOString(),
6424
- reason: "no-codebuddy-history",
6425
- };
6426
- return false;
6427
- }
6428
-
6429
- // Keep this condition retryable when old CodeBuddy history exists but the
6430
- // extension log is temporarily rotated/missing. A permanent no-op here
6431
- // would close the repair window before a later full scan can rediscover it.
6432
- if (jsonlFiles.length === 0 || logFiles.length === 0 || !hasDetailedJsonl) {
6433
- migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
6434
- skipped: true,
6435
- reason: "no-mixed-detailed-sources",
6436
- at: new Date().toISOString(),
6437
- };
6438
- return false;
6439
- }
6440
-
6441
- // A rebuild from the currently discoverable JSONL is safe only if every
6442
- // file previously tailed by CodeBuddy is still available in this scan. If a
6443
- // rotated/deleted log is missing, retain the user's history and retry later
6444
- // rather than silently erasing an unreproducible bucket.
6445
- const discovered = new Set(files);
6446
- const priorFiles = cursors.codebuddy?.fileOffsets && typeof cursors.codebuddy.fileOffsets === "object"
6447
- ? Object.keys(cursors.codebuddy.fileOffsets)
6448
- : [];
6449
- const missingPriorFile = priorFiles.find((filePath) => !discovered.has(filePath));
6450
- if (missingPriorFile) {
6451
- migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
6452
- skipped: true,
6453
- reason: "codebuddy_file_unreproducible",
6454
- filePath: missingPriorFile,
6455
- at: new Date().toISOString(),
6456
- };
6457
- return false;
6458
- }
6459
-
6460
- const tmpQueue = `${queuePath}.codebuddyrebuild.${process.pid}.${Date.now()}`;
6461
- const tmpCursors = {
6462
- version: 1,
6463
- hourly: { buckets: {}, groupQueued: {} },
6464
- codebuddy: {},
6465
- };
6466
- let rebuiltRows = [];
6467
- try {
6468
- await parseCodebuddyIncremental({
6469
- // Rebuild from both sources. The parser now preserves legacy-only log
6470
- // sessions while consuming mirrored log fingerprints against JSONL, so
6471
- // rebuilding JSONL alone would silently discard valid pre-rawUsage data.
6472
- projectFiles: files,
6473
- cursors: tmpCursors,
6474
- queuePath: tmpQueue,
6475
- env,
6476
- });
6477
- let rebuiltRaw = "";
6478
- try {
6479
- rebuiltRaw = await fs.readFile(tmpQueue, "utf8");
6480
- } catch (e) {
6481
- if (e?.code !== "ENOENT") throw e;
6482
- }
6483
- rebuiltRows = rebuiltRaw.split("\n").filter((line) => line.trim());
6484
- } catch (e) {
6485
- console.error("[sync] CodeBuddy log/JSONL repair: rebuild failed, leaving history untouched:", e?.message || e);
6486
- return false;
6487
- } finally {
6488
- await fs.rm(tmpQueue, { force: true }).catch(() => {});
6489
- }
6490
-
6491
- const rebuiltKeys = new Set();
6492
- const validRebuiltRows = [];
6493
- for (const line of rebuiltRows) {
6494
- let row;
6495
- try { row = JSON.parse(line); } catch (_e) { continue; }
6496
- const key = codebuddyUsageRowKey(row);
6497
- if (!key) continue;
6498
- rebuiltKeys.add(key);
6499
- validRebuiltRows.push(line);
6500
- }
6501
- if (rebuiltKeys.size === 0) {
6502
- migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
6503
- skipped: true,
6504
- reason: "jsonl_rebuild_empty",
6505
- at: new Date().toISOString(),
6506
- };
6507
- return false;
6508
- }
6509
-
6510
- const staleRetractions = [];
6511
- for (const key of liveKeys) {
6512
- if (rebuiltKeys.has(key)) continue;
6513
- const [, model, ...hourParts] = key.split("|");
6514
- staleRetractions.push(JSON.stringify(codebuddyZeroUsageRow(model, hourParts.join("|"))));
6515
- }
6516
-
6517
- const nextQueueLines = keptQueueLines.concat(staleRetractions, validRebuiltRows);
6518
- await ensureDir(path.dirname(queuePath));
6519
- const queueTmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
6520
- await fs.writeFile(queueTmp, nextQueueLines.length ? `${nextQueueLines.join("\n")}\n` : "", "utf8");
6521
- await fs.rename(queueTmp, queuePath);
6522
-
6523
- const nextHourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
6524
- nextHourly.buckets ||= {};
6525
- nextHourly.groupQueued ||= {};
6526
- for (const key of Object.keys(nextHourly.buckets)) {
6527
- if (key.startsWith("codebuddy|")) delete nextHourly.buckets[key];
6528
- }
6529
- for (const key of Object.keys(nextHourly.groupQueued)) {
6530
- if (key.startsWith("codebuddy|")) delete nextHourly.groupQueued[key];
6531
- }
6532
- for (const [key, bucket] of Object.entries(tmpCursors.hourly.buckets || {})) {
6533
- if (key.startsWith("codebuddy|")) nextHourly.buckets[key] = bucket;
6534
- }
6535
- for (const [key, value] of Object.entries(tmpCursors.hourly.groupQueued || {})) {
6536
- if (key.startsWith("codebuddy|")) nextHourly.groupQueued[key] = value;
6537
- }
6538
- cursors.codebuddy = tmpCursors.codebuddy || {};
6539
-
6540
- let uploadState = {};
6541
- try { uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8")); } catch (_e) {}
6542
- uploadState.offset = 0;
6543
- uploadState.updatedAt = new Date().toISOString();
6544
- uploadState.note = "reset_after_codebuddy_log_jsonl_repair_2026_08";
6545
- await ensureDir(path.dirname(queueStatePath));
6546
- await fs.writeFile(queueStatePath, JSON.stringify(uploadState, null, 2) + "\n", "utf8");
6547
-
6548
- migrations[CODEBUDDY_LOG_JSONL_REPAIR_KEY] = {
6549
- status: "applied",
6550
- appliedAt: new Date().toISOString(),
6551
- jsonlFiles: jsonlFiles.length,
6552
- logFiles: logFiles.length,
6553
- previousBuckets: liveKeys.size,
6554
- rebuiltBuckets: rebuiltKeys.size,
6555
- retractedBuckets: staleRetractions.length,
6556
- };
6557
- return true;
6558
- }
6559
-
6560
- // WorkBuddy's 0.87.9 SQLite fallback treated the bounded session_usage.used
6561
- // context snapshot as cumulative input tokens. A current database proves that
6562
- // assumption false (`size` is a 192k context limit while trace input totals can
6563
- // exceed it), so rebuild historical WorkBuddy buckets from JSONL/trace sources
6564
- // once the new parser is available. The same missing-source guard used by the
6565
- // other data migrations keeps this recoverable: if a previously contributing
6566
- // file is gone, leave history untouched and stop adding new context snapshots.
6567
- async function repairWorkbuddyContextUsage({
6568
- cursors,
6569
- queuePath,
6570
- queueStatePath,
6571
- workbuddyFiles,
6572
- env = process.env,
6573
- } = {}) {
6574
- if (!cursors || typeof cursors !== "object") return false;
6575
- const migrations = (cursors.migrations ||= {});
6576
- const prior = migrations[WORKBUDDY_CONTEXT_USAGE_REPAIR_KEY];
6577
- if (prior && !(typeof prior === "object" && prior.skipped)) return false;
6578
-
6579
- const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : {};
6580
- const liveHourlyKeys = new Set(
6581
- Object.keys(hourly.buckets || {}).filter((key) => key.startsWith("workbuddy|")),
6582
- );
6583
- let queueRaw = "";
6584
- try {
6585
- queueRaw = await fs.readFile(queuePath, "utf8");
6586
- } catch (e) {
6587
- if (e?.code !== "ENOENT") throw e;
6588
- }
6589
- const queueWorkbuddyKeys = new Set();
6590
- const keptQueueLines = [];
6591
- for (const line of queueRaw.split("\n")) {
6592
- if (!line.trim()) continue;
6593
- let row;
6594
- try { row = JSON.parse(line); } catch (_e) {
6595
- keptQueueLines.push(line);
6596
- continue;
6597
- }
6598
- if (row && row.source === "workbuddy" && typeof row.model === "string" && typeof row.hour_start === "string") {
6599
- queueWorkbuddyKeys.add(bucketKey("workbuddy", row.model || "unknown", row.hour_start));
6600
- } else {
6601
- keptQueueLines.push(line);
6602
- }
6603
- }
6604
- const liveKeys = new Set([...liveHourlyKeys, ...queueWorkbuddyKeys]);
6605
- if (liveKeys.size === 0) {
6606
- migrations[WORKBUDDY_CONTEXT_USAGE_REPAIR_KEY] = {
6607
- status: "noop",
6608
- appliedAt: new Date().toISOString(),
6609
- reason: "no-workbuddy-history",
6610
- };
6611
- return false;
6612
- }
6613
-
6614
- const files = Array.isArray(workbuddyFiles) ? workbuddyFiles : [];
6615
- const discoveredPaths = new Set(
6616
- files.map((entry) => typeof entry === "string" ? entry : entry?.path).filter(Boolean),
6617
- );
6618
- const priorFiles = cursors.workbuddy?.fileOffsets && typeof cursors.workbuddy.fileOffsets === "object"
6619
- ? Object.keys(cursors.workbuddy.fileOffsets)
6620
- : [];
6621
- if (files.length === 0 || priorFiles.some((filePath) => !discoveredPaths.has(filePath))) {
6622
- migrations[WORKBUDDY_CONTEXT_USAGE_REPAIR_KEY] = {
6623
- skipped: true,
6624
- reason: files.length === 0 ? "no-rebuild-sources" : "workbuddy_file_unreproducible",
6625
- at: new Date().toISOString(),
6626
- };
6627
- return false;
6628
- }
6629
-
6630
- const tmpQueue = `${queuePath}.workbuddyrebuild.${process.pid}.${Date.now()}`;
6631
- const tmpCursors = {
6632
- version: 1,
6633
- hourly: { buckets: {}, groupQueued: {} },
6634
- workbuddy: {},
6635
- };
6636
- let rebuiltRows = [];
6637
- try {
6638
- await parseWorkbuddyIncremental({
6639
- projectFiles: files,
6640
- cursors: tmpCursors,
6641
- queuePath: tmpQueue,
6642
- env,
6643
- });
6644
- let rebuiltRaw = "";
6645
- try { rebuiltRaw = await fs.readFile(tmpQueue, "utf8"); } catch (e) {
6646
- if (e?.code !== "ENOENT") throw e;
6647
- }
6648
- rebuiltRows = rebuiltRaw.split("\n").filter((line) => line.trim());
6649
- } catch (e) {
6650
- console.error("[sync] WorkBuddy context-usage repair: rebuild failed, leaving history untouched:", e?.message || e);
6651
- return false;
6652
- } finally {
6653
- await fs.rm(tmpQueue, { force: true }).catch(() => {});
6654
- }
6655
-
6656
- const rebuiltKeys = new Set();
6657
- const validRebuiltRows = [];
6658
- for (const line of rebuiltRows) {
6659
- let row;
6660
- try { row = JSON.parse(line); } catch (_e) { continue; }
6661
- if (!row || row.source !== "workbuddy" || typeof row.model !== "string" || typeof row.hour_start !== "string") continue;
6662
- rebuiltKeys.add(bucketKey("workbuddy", row.model || "unknown", row.hour_start));
6663
- validRebuiltRows.push(line);
6664
- }
6665
-
6666
- const staleRetractions = [];
6667
- for (const key of liveKeys) {
6668
- if (rebuiltKeys.has(key)) continue;
6669
- const [, model, ...hourParts] = key.split("|");
6670
- staleRetractions.push(JSON.stringify({
6671
- source: "workbuddy",
6672
- model: model || "unknown",
6673
- hour_start: hourParts.join("|"),
6674
- input_tokens: 0,
6675
- cached_input_tokens: 0,
6676
- cache_creation_input_tokens: 0,
6677
- output_tokens: 0,
6678
- reasoning_output_tokens: 0,
6679
- total_tokens: 0,
6680
- billable_total_tokens: 0,
6681
- conversation_count: 0,
6682
- }));
6683
- }
6684
-
6685
- const nextQueueLines = keptQueueLines.concat(staleRetractions, validRebuiltRows);
6686
- await ensureDir(path.dirname(queuePath));
6687
- const queueTmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
6688
- await fs.writeFile(queueTmp, nextQueueLines.length ? `${nextQueueLines.join("\n")}\n` : "", "utf8");
6689
- await fs.rename(queueTmp, queuePath);
6690
-
6691
- const nextHourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
6692
- nextHourly.buckets ||= {};
6693
- nextHourly.groupQueued ||= {};
6694
- for (const key of Object.keys(nextHourly.buckets)) {
6695
- if (key.startsWith("workbuddy|")) delete nextHourly.buckets[key];
6696
- }
6697
- for (const key of Object.keys(nextHourly.groupQueued)) {
6698
- if (key.startsWith("workbuddy|")) delete nextHourly.groupQueued[key];
6699
- }
6700
- for (const [key, bucket] of Object.entries(tmpCursors.hourly.buckets || {})) {
6701
- if (key.startsWith("workbuddy|")) nextHourly.buckets[key] = bucket;
6702
- }
6703
- for (const [key, value] of Object.entries(tmpCursors.hourly.groupQueued || {})) {
6704
- if (key.startsWith("workbuddy|")) nextHourly.groupQueued[key] = value;
6705
- }
6706
- // This migration rebuilds totals from a temp queue using empty cursors, so
6707
- // `tmpCursors.workbuddy` carries no project state. Replacing the whole object
6708
- // would drop the project cursor while leaving the published project buckets
6709
- // (cursors.projectHourly and the real project queue) untouched, and the next
6710
- // parse would re-attribute every historical session on top of them. The
6711
- // migration only changes totals, so keep the project cursor as it is.
6712
- const previousWorkbuddyState =
6713
- cursors.workbuddy && typeof cursors.workbuddy === "object" ? cursors.workbuddy : {};
6714
- cursors.workbuddy = {
6715
- ...(tmpCursors.workbuddy || {}),
6716
- ...(previousWorkbuddyState.projectFileOffsets
6717
- ? { projectFileOffsets: previousWorkbuddyState.projectFileOffsets }
6718
- : {}),
6719
- ...(Array.isArray(previousWorkbuddyState.projectSeenIds)
6720
- ? { projectSeenIds: previousWorkbuddyState.projectSeenIds }
6721
- : {}),
6722
- };
6723
-
6724
- let uploadState = {};
6725
- try { uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8")); } catch (_e) {}
6726
- uploadState.offset = 0;
6727
- uploadState.updatedAt = new Date().toISOString();
6728
- uploadState.note = "reset_after_workbuddy_context_usage_repair_2026_08";
6729
- await ensureDir(path.dirname(queueStatePath));
6730
- await fs.writeFile(queueStatePath, JSON.stringify(uploadState, null, 2) + "\n", "utf8");
6731
-
6732
- migrations[WORKBUDDY_CONTEXT_USAGE_REPAIR_KEY] = {
6733
- status: "applied",
6734
- appliedAt: new Date().toISOString(),
6735
- previousBuckets: liveKeys.size,
6736
- rebuiltBuckets: rebuiltKeys.size,
6737
- retractedBuckets: staleRetractions.length,
6738
- };
6739
- return true;
6740
- }
6741
-
6742
- // One-time repair (#187): rebuild codex hourly buckets that the inode-keyed
6743
- // re-scan double-counted before the codexHashes event-dedup landed, and push
6744
- // the corrected values to the cloud. Runs BEFORE the codex parse in the same
6745
- // sync: it clears codex hourly state + codexHashes so the parse rebuilds clean,
6746
- // atomically strips the inflated codex rows from queue.jsonl, and resets the
6747
- // upload offset so the re-uploaded clean rows overwrite the cloud (with no
6748
- // stale-high codex rows left in the queue, the ingest's within-batch MAX keeps
6749
- // nothing larger, so its overwrite-upsert replaces the inflated cloud rows).
6750
- //
6751
- // GUARDED against the v6 ground-truth-repair data-loss incident: a clear+reparse
6752
- // rebuilds codex buckets ONLY from the codex files this sync re-parses (now both
6753
- // sessions/ AND archived_sessions/), so if any codex file that previously
6754
- // contributed is gone from disk (genuinely deleted — Codex-Manager log rotation
6755
- // or user cleanup) the migration is skipped entirely — the forward dedup fix
6756
- // still prevents new double-counting; only this historical correction is deferred.
6757
- async function repairCodexRescanInflation({
6758
- cursors,
6759
- queuePath,
6760
- queueStatePath,
6761
- projectQueuePath,
6762
- projectQueueStatePath,
6763
- rolloutFiles,
6764
- expectedCodexFileSnapshots = null,
6765
- // The fork-replay repair (#169 follow-up) re-runs this exact rebuild under
6766
- // its own key: the rebuild always uses the CURRENT parser, so any parser fix
6767
- // shipped since the last run is applied to the rebuilt history.
6768
- migrationKey = CODEX_RESCAN_DEDUP_REPAIR_KEY,
6769
- uploadNote = "reset_after_codex_rescan_dedup_2026_06",
6770
- }) {
6771
- if (!cursors || typeof cursors !== "object") return false;
6772
- const migrations = (cursors.migrations ||= {});
6773
- // A COMPLETED run writes an ISO-string timestamp (final — never re-run). A
6774
- // prior SKIP writes an object {skipped:true} and MUST be retried: the skip
6775
- // condition can clear in a later version (e.g. v0.53.4 started scanning
6776
- // archived_sessions/, so a session that was "unscanned" under v0.53.3 is now
6777
- // found). Treating the skip sentinel as "done" is what left users like #187
6778
- // permanently stuck on the inflated value after upgrading (the key was truthy
6779
- // so the guard never got a second chance).
6780
- const priorRepair = migrations[migrationKey];
6781
- if (priorRepair && !(typeof priorRepair === "object" && priorRepair.skipped)) return false;
6782
-
6783
- // Codex session files THIS sync discovered (source === "codex").
6784
- const codexFiles = [];
6785
- for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
6786
- const fp = typeof entry === "string" ? entry : entry?.path;
6787
- const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
6788
- if (fp && src === "codex") codexFiles.push(fp);
6789
- }
6790
- const codexFileSet = new Set(codexFiles);
6791
- const projectRepairEnabled = typeof projectQueuePath === "string" && projectQueuePath.length > 0;
6792
-
6793
- // GUARD (data-loss prevention, ref the v6 ground-truth-repair incident): the
6794
- // rebuild can only reproduce buckets from the files this sync re-parses
6795
- // (codexFiles, now covering sessions/ AND archived_sessions/). If a session
6796
- // that previously contributed can no longer be reproduced, skip entirely —
6797
- // the forward dedup fix still stops NEW double-counting; only this historical
6798
- // correction defers.
6799
- //
6800
- // Reproducibility is keyed on the session UUID, NOT the exact cursor path:
6801
- // Codex-Manager MOVES a file sessions/ -> archived_sessions/ (path changes,
6802
- // UUID does not). A path-based check false-positives on every moved file as
6803
- // "missing" and skips forever (issue #187, easonlee05). Genuinely deleted
6804
- // sessions (no file with that UUID anywhere in the scan) still defer.
6805
- const scannedSessionIds = new Set();
6806
- for (const fp of codexFiles) {
6807
- const id = codexSessionIdFromPath(fp);
6808
- if (id) scannedSessionIds.add(id);
6809
- }
6810
- if (cursors.files && typeof cursors.files === "object") {
6811
- for (const fp of Object.keys(cursors.files)) {
6812
- if (!isCodexSessionCursorPath(fp)) continue;
6813
- if (codexFileSet.has(fp)) continue; // exact file re-scanned this run
6814
- const id = codexSessionIdFromPath(fp);
6815
- if (id && scannedSessionIds.has(id)) continue; // same session scanned elsewhere (moved)
6816
- migrations[migrationKey] = {
6817
- skipped: true,
6818
- reason: "codex_session_unreproducible",
6819
- at: new Date().toISOString(),
6820
- };
6821
- return false;
6822
- }
6823
- }
6824
-
6825
- // ATOMIC REBUILD into a THROWAWAY cursors + queue: never touch the live
6826
- // buckets/queue until the rebuild has fully succeeded. If anything throws we
6827
- // return WITHOUT mutating any persistent state and the migration retries next
6828
- // sync (the key is not set). This is the crucial difference from a
6829
- // "clear-then-rely-on-the-later-parse" design: the later parse's failure is
6830
- // swallowed (warnProviderParseFailure) and cursors are saved regardless, which
6831
- // would permanently zero a user's codex history.
6832
- let rebuilt;
6833
- const tmpQueue = `${queuePath}.codexrebuild.${process.pid}.${Date.now()}`;
6834
- const tmpProjectQueue = projectRepairEnabled
6835
- ? `${projectQueuePath}.codexrebuild.${process.pid}.${Date.now()}`
6836
- : null;
6837
- try {
6838
- const tmpCursors = {
6839
- version: 1,
6840
- files: {},
6841
- hourly: { buckets: {}, groupQueued: {} },
6842
- codexHashes: [],
6843
- };
6844
- await parseRolloutIncremental({
6845
- rolloutFiles: codexFiles.map((p) => ({ path: p, source: "codex" })),
6846
- cursors: tmpCursors,
6847
- queuePath: tmpQueue,
6848
- projectQueuePath: tmpProjectQueue,
6849
- invalidRecordPolicy: "throw",
6850
- });
6851
- let tmpRaw = "";
6852
- try {
6853
- tmpRaw = await fs.readFile(tmpQueue, "utf8");
6854
- } catch (e) {
6855
- if (e?.code !== "ENOENT") throw e;
6856
- }
6857
- let tmpProjectRaw = "";
6858
- if (tmpProjectQueue) {
6859
- try {
6860
- tmpProjectRaw = await fs.readFile(tmpProjectQueue, "utf8");
6861
- } catch (e) {
6862
- if (e?.code !== "ENOENT") throw e;
6863
- }
6864
- }
6865
- rebuilt = {
6866
- buckets: tmpCursors.hourly.buckets || {},
6867
- groupQueued: tmpCursors.hourly.groupQueued || {},
6868
- codexHashes: Array.isArray(tmpCursors.codexHashes) ? tmpCursors.codexHashes : [],
6869
- codexCompactionResponseIds: Array.isArray(tmpCursors.codexCompactionResponseIds)
6870
- ? tmpCursors.codexCompactionResponseIds
6871
- : [],
6872
- files: tmpCursors.files || {},
6873
- queueRows: tmpRaw.split("\n").filter((l) => l.trim()),
6874
- projectHourly: tmpCursors.projectHourly || null,
6875
- projectQueueRows: tmpProjectRaw.split("\n").filter((l) => l.trim()),
6876
- };
6877
- } catch (e) {
6878
- console.error(
6879
- "[sync] codex rescan repair: rebuild failed, leaving all data untouched:",
6880
- e?.message || e,
6881
- );
6882
- return false;
6883
- } finally {
6884
- await fs.rm(tmpQueue, { force: true }).catch(() => {});
6885
- if (tmpProjectQueue) await fs.rm(tmpProjectQueue, { force: true }).catch(() => {});
6886
- }
6887
-
6888
- // SANITY: codex files exist on disk but the rebuild produced no codex buckets
6889
- // → treat as a failed rebuild and skip (do NOT clear live data, do NOT set the
6890
- // key — retry next sync).
6891
- const rebuiltCodexKeys = Object.keys(rebuilt.buckets).filter((k) => k.startsWith("codex|"));
6892
- if (codexFiles.length > 0 && rebuiltCodexKeys.length === 0) {
6893
- console.error(
6894
- `[sync] codex rescan repair: rebuild produced 0 codex buckets from ${codexFiles.length} files — skipping to avoid data loss`,
6895
- );
6896
- return false;
6897
- }
6898
- if (projectRepairEnabled) {
6899
- const malformedProjectRows = await countMalformedCodexProjectQueueRows(projectQueuePath);
6900
- if (malformedProjectRows > 0) {
6901
- console.error(
6902
- `[sync] codex rescan repair: found ${malformedProjectRows} malformed codex project queue row(s) — skipping to avoid data loss`,
6903
- );
6904
- return false;
6905
- }
6906
-
6907
- const existingProjectKeys = new Set([
6908
- ...(await projectUsageKeysFromQueuePath(projectQueuePath, "codex")),
6909
- ...projectUsageKeysFromState(cursors.projectHourly, "codex"),
6910
- ]);
6911
- const rebuiltProjectKeys = new Set([
6912
- ...projectUsageKeysFromQueueRows(rebuilt.projectQueueRows, "codex"),
6913
- ...projectUsageKeysFromState(rebuilt.projectHourly, "codex"),
6914
- ]);
6915
- const missingProjectKeys = [...existingProjectKeys].filter((key) => !rebuiltProjectKeys.has(key));
6916
- if (missingProjectKeys.length > 0) {
6917
- console.error(
6918
- `[sync] codex rescan repair: project rebuild missed ${missingProjectKeys.length} existing codex project bucket(s) — skipping to avoid data loss`,
6919
- );
6920
- return false;
6921
- }
6922
-
6923
- const existingProjectTotals = mergeMaxTotals(
6924
- await projectUsageTotalsFromQueuePath(projectQueuePath, "codex"),
6925
- projectUsageTotalsFromState(cursors.projectHourly, "codex"),
6926
- );
6927
- const rebuiltProjectTotals = mergeMaxTotals(
6928
- projectUsageTotalsFromQueueRows(rebuilt.projectQueueRows, "codex"),
6929
- projectUsageTotalsFromState(rebuilt.projectHourly, "codex"),
6930
- );
6931
- const existingMainHourTotals = mergeMaxTotals(
6932
- await mainUsageHourTotalsFromQueuePath(queuePath, "codex"),
6933
- mainUsageHourTotalsFromState(cursors.hourly, "codex"),
6934
- );
6935
- const rebuiltMainHourTotals = mergeMaxTotals(
6936
- mainUsageHourTotalsFromQueueRows(rebuilt.queueRows, "codex"),
6937
- mainUsageHourTotalsFromState({ buckets: rebuilt.buckets }, "codex"),
6938
- );
6939
- const partialProjectKeys = [];
6940
- for (const [key, existingTotal] of existingProjectTotals.entries()) {
6941
- const rebuiltTotal = rebuiltProjectTotals.get(key);
6942
- if (!Number.isFinite(rebuiltTotal) || rebuiltTotal >= existingTotal) continue;
6943
- const [, source, hourStart] = key.split("|");
6944
- const mainKey = `${source}|${hourStart}`;
6945
- const existingMainTotal = existingMainHourTotals.get(mainKey) || 0;
6946
- const rebuiltMainTotal = rebuiltMainHourTotals.get(mainKey) || 0;
6947
- if (rebuiltMainTotal >= existingMainTotal) partialProjectKeys.push(key);
6948
- }
6949
- if (partialProjectKeys.length > 0) {
6950
- console.error(
6951
- `[sync] codex rescan repair: project rebuild lowered ${partialProjectKeys.length} existing codex project bucket(s) without a matching main-bucket repair — skipping to avoid data loss`,
6952
- );
6953
- return false;
6954
- }
6955
- }
6956
-
6957
- const rebuildValidation = expectedCodexFileSnapshots instanceof Map
6958
- ? await validateCodexRebuildFileSnapshots(rebuilt.files, expectedCodexFileSnapshots)
6959
- : { ok: true };
6960
- if (!rebuildValidation.ok) {
6961
- console.error(
6962
- `[sync] codex rescan repair: ${rebuildValidation.reason} — skipping to avoid data loss`,
6963
- );
6964
- return false;
6965
- }
6966
-
6967
- // COMMIT (only after a verified rebuild). A crash partway just leaves the
6968
- // migration to re-run next sync — re-rebuild + re-strip + re-commit converges.
6969
- //
6970
- // 1. queue.jsonl: drop the inflated codex rows, append the clean rebuilt ones
6971
- // (atomic tmp+rename). With no old-high codex rows left, the cloud ingest's
6972
- // within-batch MAX keeps nothing larger and its overwrite-upsert replaces
6973
- // the inflated cloud rows on the next upload.
6974
- if (typeof queuePath === "string" && queuePath) {
6975
- let raw = "";
6976
- try {
6977
- raw = await fs.readFile(queuePath, "utf8");
6978
- } catch (e) {
6979
- if (e?.code !== "ENOENT") throw e;
6980
- }
6981
- const kept = [];
6982
- for (const line of raw.split("\n")) {
6983
- if (!line.trim()) continue;
6984
- let row;
6985
- try {
6986
- row = JSON.parse(line);
6987
- } catch (_e) {
6988
- kept.push(line);
6989
- continue;
6990
- }
6991
- if (row?.source === "codex") continue;
6992
- kept.push(line);
6993
- }
6994
- await ensureDir(path.dirname(queuePath));
6995
- const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
6996
- await fs.writeFile(tmp, kept.concat(rebuilt.queueRows).join("\n") + "\n", "utf8");
6997
- await fs.rename(tmp, queuePath);
6998
- }
6999
-
7000
- // 2. Swap the live codex hourly state for the rebuilt one, and install the
7001
- // rebuilt per-file cursors (offset at EOF) so the later parse in THIS sync
7002
- // does not re-read codex (which would re-inflate project buckets).
7003
- const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
7004
- hourly.buckets ||= {};
7005
- hourly.groupQueued ||= {};
7006
- for (const k of Object.keys(hourly.buckets)) {
7007
- if (k.startsWith("codex|")) delete hourly.buckets[k];
7008
- }
7009
- for (const k of Object.keys(hourly.groupQueued)) {
7010
- if (k.startsWith("codex|")) delete hourly.groupQueued[k];
7011
- }
7012
- for (const [k, v] of Object.entries(rebuilt.buckets)) {
7013
- if (k.startsWith("codex|")) hourly.buckets[k] = v;
7014
- }
7015
- for (const [k, v] of Object.entries(rebuilt.groupQueued)) {
7016
- if (k.startsWith("codex|")) hourly.groupQueued[k] = v;
7017
- }
7018
- cursors.files ||= {};
7019
- for (const fp of Object.keys(cursors.files)) {
7020
- if (isCodexSessionCursorPath(fp)) delete cursors.files[fp];
7021
- }
7022
- for (const [fp, v] of Object.entries(rebuilt.files)) {
7023
- cursors.files[fp] = v;
7024
- }
7025
- cursors.codexHashes = rebuilt.codexHashes;
7026
- cursors.codexCompactionResponseIds = rebuilt.codexCompactionResponseIds;
7027
-
7028
- // 3. Project usage mirrors the main Codex repair: drop inflated Codex project
7029
- // rows, append the rebuilt rows, and swap only Codex project buckets. Project
7030
- // metadata is merged so visibility/purge state from non-Codex sources stays.
7031
- if (projectRepairEnabled) {
7032
- let projectRaw = "";
7033
- try {
7034
- projectRaw = await fs.readFile(projectQueuePath, "utf8");
7035
- } catch (e) {
7036
- if (e?.code !== "ENOENT") throw e;
7037
- }
7038
- const keptProjectRows = [];
7039
- for (const line of projectRaw.split("\n")) {
7040
- if (!line.trim()) continue;
7041
- let row;
7042
- try {
7043
- row = JSON.parse(line);
7044
- } catch (_e) {
7045
- keptProjectRows.push(line);
7046
- continue;
7047
- }
7048
- if (row?.source === "codex") continue;
7049
- keptProjectRows.push(line);
7050
- }
7051
- await ensureDir(path.dirname(projectQueuePath));
7052
- const tmp = `${projectQueuePath}.tmp.${process.pid}.${Date.now()}`;
7053
- await fs.writeFile(
7054
- tmp,
7055
- keptProjectRows.concat(rebuilt.projectQueueRows).join("\n") + "\n",
7056
- "utf8",
7057
- );
7058
- await fs.rename(tmp, projectQueuePath);
7059
-
7060
- const projectHourly = (cursors.projectHourly ||= { version: 2, buckets: {}, projects: {} });
7061
- projectHourly.version = 2;
7062
- projectHourly.buckets ||= {};
7063
- projectHourly.projects ||= {};
7064
- for (const [key, bucket] of Object.entries(projectHourly.buckets)) {
7065
- const source = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
7066
- if (source === "codex") delete projectHourly.buckets[key];
7067
- }
7068
- const rebuiltProjectHourly = rebuilt.projectHourly || {};
7069
- for (const [key, bucket] of Object.entries(rebuiltProjectHourly.buckets || {})) {
7070
- const source = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
7071
- if (source === "codex") projectHourly.buckets[key] = bucket;
7072
- }
7073
- for (const [key, meta] of Object.entries(rebuiltProjectHourly.projects || {})) {
7074
- if (meta && typeof meta === "object") projectHourly.projects[key] = meta;
7075
- }
7076
- projectHourly.updatedAt = new Date().toISOString();
7077
- }
7078
-
7079
- // 4. Reset the cloud upload offset so the corrected queue re-uploads. Other
7080
- // sources re-upsert idempotently (last emission per key wins).
7081
- if (typeof queueStatePath === "string" && queueStatePath) {
7082
- let uploadState = {};
7083
- try {
7084
- uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
7085
- } catch (_e) {
7086
- uploadState = {};
7087
- }
7088
- uploadState.offset = 0;
7089
- uploadState.updatedAt = new Date().toISOString();
7090
- uploadState.note = uploadNote;
7091
- await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
7092
- }
7093
- if (projectRepairEnabled && typeof projectQueueStatePath === "string" && projectQueueStatePath) {
7094
- let uploadState = {};
7095
- try {
7096
- uploadState = JSON.parse(await fs.readFile(projectQueueStatePath, "utf8"));
7097
- } catch (_e) {
7098
- uploadState = {};
7099
- }
7100
- uploadState.offset = 0;
7101
- uploadState.updatedAt = new Date().toISOString();
7102
- uploadState.note = uploadNote;
7103
- await fs.writeFile(projectQueueStatePath, JSON.stringify(uploadState));
7104
- }
7105
-
7106
- migrations[migrationKey] = new Date().toISOString();
7107
- return true;
7108
- }
7109
-
7110
- // #169 follow-up: repair the historical inflation left by same-day forked
7111
- // Codex rollout replays (see CODEX_FORK_REPLAY_REPAIR_KEY). Delegates to the
7112
- // #187 guarded rebuild with the fork-repair key.
7113
- async function repairCodexForkReplayInflation({
7114
- cursors,
7115
- queuePath,
7116
- queueStatePath,
7117
- projectQueuePath,
7118
- projectQueueStatePath,
7119
- rolloutFiles,
7120
- // True when the #187 repair ran its rebuild earlier in THIS sync: that
7121
- // rebuild already used the fork-aware parser, so the history is clean and a
7122
- // second rebuild would be pure waste.
7123
- legacyRepairRan = false,
7124
- }) {
7125
- if (!cursors || typeof cursors !== "object") return false;
7126
- const migrations = (cursors.migrations ||= {});
7127
- // Same completed-vs-skipped semantics as the #187 repair: an ISO string is
7128
- // final, a {skipped:true} sentinel retries (the skip condition can clear).
7129
- const prior = migrations[CODEX_FORK_REPLAY_REPAIR_KEY];
7130
- if (prior && !(typeof prior === "object" && prior.skipped)) return false;
7131
-
7132
- if (legacyRepairRan) {
7133
- migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = new Date().toISOString();
7134
- return false;
7135
- }
7136
-
7137
- // Pre-gate: fork phantom can only exist on installs that have (or had) a
7138
- // forked rollout. Scanning file heads is cheap; the rebuild re-parses every
7139
- // codex file. Files forked AFTER the parser fix shipped never accrue
7140
- // phantom, so a definitive "no forks" verdict may finalize — but ONLY when
7141
- // this scan can actually stand for the install's whole codex history:
7142
- // 1. every candidate head was readable (an unreadable candidate could be
7143
- // the one forked rollout), AND
7144
- // 2. every codex session recorded in cursors is covered by THIS scan's
7145
- // file list (same UUID-keyed reproducibility rule as the #187 guard).
7146
- // A forked rollout can be real yet absent from one run's rolloutFiles —
7147
- // WSL/native split probing, a temporarily missing ~/.codex, a deleted
7148
- // file whose phantom is already in history — and finalizing off such a
7149
- // partial view would permanently foreclose the repair this migration
7150
- // exists to perform, while the guarded repair itself would only have
7151
- // deferred retryably.
7152
- // Either failure defers with the retryable {skipped:true} sentinel.
7153
- const forkScan = await scanForForkedCodexRollout(rolloutFiles);
7154
- if (!forkScan.forked) {
7155
- if (forkScan.indeterminate) {
7156
- migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = {
7157
- skipped: true,
7158
- reason: "fork_scan_indeterminate",
7159
- at: new Date().toISOString(),
7160
- };
7161
- return false;
7162
- }
7163
- const scannedSessionIds = new Set();
7164
- const scannedPaths = new Set();
7165
- for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
7166
- const fp = typeof entry === "string" ? entry : entry?.path;
7167
- const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
7168
- if (!fp || src !== "codex") continue;
7169
- scannedPaths.add(fp);
7170
- const id = codexSessionIdFromPath(fp);
7171
- if (id) scannedSessionIds.add(id);
7172
- }
7173
- if (cursors.files && typeof cursors.files === "object") {
7174
- for (const fp of Object.keys(cursors.files)) {
7175
- if (!isCodexSessionCursorPath(fp)) continue;
7176
- if (scannedPaths.has(fp)) continue;
7177
- const id = codexSessionIdFromPath(fp);
7178
- if (id && scannedSessionIds.has(id)) continue;
7179
- migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = {
7180
- skipped: true,
7181
- reason: "codex_history_not_covered",
7182
- at: new Date().toISOString(),
7183
- };
7184
- return false;
7185
- }
7186
- }
7187
- migrations[CODEX_FORK_REPLAY_REPAIR_KEY] = new Date().toISOString();
7188
- return false;
7189
- }
7190
-
7191
- return repairCodexRescanInflation({
7192
- cursors,
7193
- queuePath,
7194
- queueStatePath,
7195
- projectQueuePath,
7196
- projectQueueStatePath,
7197
- rolloutFiles,
7198
- migrationKey: CODEX_FORK_REPLAY_REPAIR_KEY,
7199
- uploadNote: "reset_after_codex_fork_replay_2026_07",
7200
- });
7201
- }
7202
-
7203
- // Rebuild historical Codex usage only when a rollout proves that the old
7204
- // single-baseline parser crossed cumulative SessionState lineages. Detection is
7205
- // based on the token counter stream itself, not multi_agent_version metadata:
7206
- // affected legacy rollouts can predate that marker or carry it only in the
7207
- // middle of a large file. Every contributing rollout must also be valid and
7208
- // stable before the atomic rebuild can replace live history.
7209
- async function repairCodexInterleavedUsageInflation({
7210
- cursors,
7211
- queuePath,
7212
- queueStatePath,
7213
- projectQueuePath,
7214
- projectQueueStatePath,
7215
- rolloutFiles,
7216
- // Either older guarded repair rebuilt every Codex file with the current
7217
- // parser earlier in this sync, so a second full rebuild is unnecessary.
7218
- legacyRepairRan = false,
7219
- maxLineageScanBytes = Infinity,
7220
- }) {
7221
- if (!cursors || typeof cursors !== "object") return false;
7222
- const migrations = (cursors.migrations ||= {});
7223
- const prior = migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY];
7224
- if (prior && !(typeof prior === "object" && prior.skipped)) return false;
7225
-
7226
- if (legacyRepairRan) {
7227
- migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = new Date().toISOString();
7228
- return false;
7229
- }
7230
-
7231
- const scan = await scanForInterleavedCodexUsage(rolloutFiles, {
7232
- maxLineageScanBytes,
7233
- });
7234
- if (scan.indeterminate) {
7235
- migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = {
7236
- skipped: true,
7237
- reason: "usage_lineage_scan_indeterminate",
7238
- at: new Date().toISOString(),
7239
- };
7240
- return false;
7241
- }
7242
-
7243
- if (!scan.affected) {
7244
- if (!isCodexHistoryCovered(cursors, rolloutFiles)) {
7245
- migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = {
7246
- skipped: true,
7247
- reason: "codex_history_not_covered",
7248
- at: new Date().toISOString(),
7249
- };
7250
- return false;
7251
- }
7252
- migrations[CODEX_USAGE_LINEAGE_REPAIR_KEY] = new Date().toISOString();
7253
- return false;
7254
- }
7255
-
7256
- return repairCodexRescanInflation({
7257
- cursors,
7258
- queuePath,
7259
- queueStatePath,
7260
- projectQueuePath,
7261
- projectQueueStatePath,
7262
- rolloutFiles,
7263
- expectedCodexFileSnapshots: scan.fileSnapshots,
7264
- migrationKey: CODEX_USAGE_LINEAGE_REPAIR_KEY,
7265
- uploadNote: "reset_after_codex_usage_lineage_2026_07_v2",
7266
- });
7267
- }
7268
-
7269
- async function scanForInterleavedCodexUsage(
7270
- rolloutFiles,
7271
- { maxLineageScanBytes = Infinity } = {},
7272
- ) {
7273
- let affected = false;
7274
- let remainingBytes = Number.isFinite(maxLineageScanBytes)
7275
- ? Math.max(0, maxLineageScanBytes)
7276
- : Infinity;
7277
- const fileSnapshots = new Map();
7278
- const seenPaths = new Set();
7279
-
7280
- for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
7281
- const fp = typeof entry === "string" ? entry : entry?.path;
7282
- const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
7283
- if (!fp || src !== "codex" || seenPaths.has(fp)) continue;
7284
- seenPaths.add(fp);
7285
-
7286
- const lineage = await scanCodexUsageLineages(fp, remainingBytes);
7287
- if (lineage.indeterminate) {
7288
- return { affected: false, indeterminate: true, fileSnapshots: new Map() };
7289
- }
7290
- affected ||= lineage.affected;
7291
- fileSnapshots.set(fp, lineage.fileSnapshot);
7292
- if (Number.isFinite(remainingBytes)) {
7293
- remainingBytes = Math.max(0, remainingBytes - lineage.bytesRead);
7294
- }
7295
- }
7296
- return { affected, indeterminate: false, fileSnapshots };
7297
- }
7298
-
7299
- async function scanCodexUsageLineages(filePath, maxBytes = Infinity) {
7300
- let stream = null;
7301
- try {
7302
- const before = await readCodexFileSnapshot(filePath);
7303
- const state = createUsageDeltaState();
7304
- const byteLimit = Number.isFinite(maxBytes) ? Math.max(0, maxBytes) : Infinity;
7305
- let bytesRead = 0;
7306
- let affected = false;
7307
- const isCompressed = typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath);
7308
- if (isCompressed) {
7309
- // Honor the background-sync lineage budget against compressed size so a
7310
- // pending repair with large archives stays bounded and returns
7311
- // indeterminate until a full sync can finish the scan.
7312
- if (!(byteLimit > 0) || before.size > byteLimit) {
7313
- return { affected: false, indeterminate: true };
7314
- }
7315
- const snapshot = await readCodexRolloutSnapshot(filePath);
7316
- if (!snapshot) return { affected: false, indeterminate: true };
7317
- for (const raw of snapshot.lines) {
7318
- const line = typeof raw === "string" ? raw : "";
7319
- if (!line.trim()) continue;
7320
- let obj;
7321
- try {
7322
- obj = JSON.parse(line);
7323
- } catch (_e) {
7324
- return { affected: false, indeterminate: true };
7325
- }
7326
- const token = extractTokenCount(obj);
7327
- const info = token?.info;
7328
- if (!info || typeof info !== "object") continue;
7329
- consumeUsageDelta(state, info.last_token_usage, info.total_token_usage);
7330
- if (state.sawInterleaved || state.sawDivergentCumulative) {
7331
- affected = true;
7332
- }
7333
- }
7334
- bytesRead = snapshot.size;
7335
- } else {
7336
- stream = fssync.createReadStream(filePath, { highWaterMark: 32 * 1024 });
7337
- for await (const record of physicalJsonlRecords(stream, {
7338
- maxPhysicalBytes: byteLimit,
7339
- })) {
7340
- const { line } = record;
7341
- bytesRead += record.physicalBytes;
7342
- if (bytesRead > byteLimit) return { affected: false, indeterminate: true };
7343
- if (!line.trim()) continue;
7344
- let obj;
7345
- try {
7346
- obj = JSON.parse(line);
7347
- } catch (_e) {
7348
- return { affected: false, indeterminate: true };
7349
- }
7350
- const token = extractTokenCount(obj);
7351
- const info = token?.info;
7352
- if (!info || typeof info !== "object") continue;
7353
- consumeUsageDelta(state, info.last_token_usage, info.total_token_usage);
7354
- if (state.sawInterleaved || state.sawDivergentCumulative) {
7355
- affected = true;
7356
- }
7357
- }
7358
- }
7359
- const after = await readCodexFileSnapshot(filePath);
7360
- if (!sameCodexFileSnapshot(before, after)) {
7361
- return { affected: false, indeterminate: true };
7362
- }
7363
- return {
7364
- affected,
7365
- indeterminate: false,
7366
- bytesRead,
7367
- fileSnapshot: after,
7368
- };
7369
- } catch (_e) {
7370
- return { affected: false, indeterminate: true };
7371
- } finally {
7372
- if (stream) stream.destroy();
7373
- }
7374
- }
7375
-
7376
- async function readCodexFileSnapshot(filePath) {
7377
- const stat = await fs.stat(filePath);
7378
- return {
7379
- size: Number(stat.size),
7380
- mtimeMs: Number(stat.mtimeMs),
7381
- ctimeMs: Number(stat.ctimeMs),
7382
- ino: Number(stat.ino),
7383
- dev: Number(stat.dev),
7384
- };
7385
- }
7386
-
7387
- function sameCodexFileSnapshot(left, right) {
7388
- return Boolean(
7389
- left &&
7390
- right &&
7391
- left.size === right.size &&
7392
- left.mtimeMs === right.mtimeMs &&
7393
- left.ctimeMs === right.ctimeMs &&
7394
- left.ino === right.ino &&
7395
- left.dev === right.dev
7396
- );
7397
- }
7398
-
7399
- async function validateCodexRebuildFileSnapshots(rebuiltFiles, expectedSnapshots) {
7400
- for (const [filePath, expected] of expectedSnapshots) {
7401
- let actual;
7402
- try {
7403
- actual = await readCodexFileSnapshot(filePath);
7404
- } catch (_e) {
7405
- return { ok: false, reason: "a scanned rollout became unreadable during rebuild" };
7406
- }
7407
- if (!sameCodexFileSnapshot(expected, actual)) {
7408
- return { ok: false, reason: "a scanned rollout changed during rebuild" };
7409
- }
7410
- const rebuiltCursor = rebuiltFiles?.[filePath];
7411
- if (!rebuiltCursor) {
7412
- return { ok: false, reason: "the rebuild omitted a scanned rollout cursor" };
7413
- }
7414
- if (Number(rebuiltCursor.offset) !== expected.size) {
7415
- return { ok: false, reason: "the rebuild did not consume a scanned rollout through EOF" };
7416
- }
7417
- }
7418
- return { ok: true };
7419
- }
7420
-
7421
- function isCodexHistoryCovered(cursors, rolloutFiles) {
7422
- const scannedPaths = new Set();
7423
- const scannedSessionIds = new Set();
7424
- for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
7425
- const fp = typeof entry === "string" ? entry : entry?.path;
7426
- const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
7427
- if (!fp || src !== "codex") continue;
7428
- scannedPaths.add(fp);
7429
- const id = codexSessionIdFromPath(fp);
7430
- if (id) scannedSessionIds.add(id);
7431
- }
7432
- if (!cursors.files || typeof cursors.files !== "object") return true;
7433
- for (const fp of Object.keys(cursors.files)) {
7434
- if (!isCodexSessionCursorPath(fp)) continue;
7435
- if (scannedPaths.has(fp)) continue;
7436
- const id = codexSessionIdFromPath(fp);
7437
- if (id && scannedSessionIds.has(id)) continue;
7438
- return false;
7439
- }
7440
- return true;
7441
- }
7442
-
7443
- // Scans codex rollout heads for the fork marker. The child session_meta (with
7444
- // forked_from_id) is the FIRST line of a forked rollout — observed at byte
7445
- // offset ≤169 across every local sample — so a bounded head read is
7446
- // sufficient. 64KB leaves ample margin for long base_instructions preceding
7447
- // it; a substring false positive (the marker quoted in unrelated head content)
7448
- // only costs one redundant guarded rebuild, never data. Tri-state result:
7449
- // {forked:true} on a hit, {forked:false, indeterminate:false} only when every
7450
- // candidate was inspected, {forked:false, indeterminate:true} when any head
7451
- // read failed — the caller must not finalize "no forks" off a failed read.
7452
- async function scanForForkedCodexRollout(rolloutFiles) {
7453
- const HEAD_BYTES = 65536;
7454
- let indeterminate = false;
7455
- for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
7456
- const fp = typeof entry === "string" ? entry : entry?.path;
7457
- const src = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
7458
- if (!fp || src !== "codex") continue;
7459
- let fh = null;
7460
- try {
7461
- if (typeof fp === "string" && /\.jsonl\.zstd?$/i.test(fp)) {
7462
- const snapshot = await readCodexRolloutSnapshot(fp);
7463
- if (!snapshot) {
7464
- indeterminate = true;
7465
- continue;
7466
- }
7467
- // Fork marker is in early session_meta; scan a bounded prefix of lines.
7468
- let head = "";
7469
- for (const raw of snapshot.lines) {
7470
- head += (typeof raw === "string" ? raw : "") + "\n";
7471
- if (head.length >= HEAD_BYTES) break;
7472
- }
7473
- if (head.includes('"forked_from_id"')) {
7474
- return { forked: true, indeterminate: false };
7475
- }
7476
- continue;
7477
- }
7478
- fh = await fs.open(fp, "r");
7479
- const buf = Buffer.alloc(HEAD_BYTES);
7480
- const { bytesRead } = await fh.read(buf, 0, HEAD_BYTES, 0);
7481
- if (buf.toString("utf8", 0, bytesRead).includes('"forked_from_id"')) {
7482
- return { forked: true, indeterminate: false };
7483
- }
7484
- } catch (_e) {
7485
- indeterminate = true;
7486
- } finally {
7487
- if (fh) await fh.close().catch(() => {});
7488
- }
7489
- }
7490
- return { forked: false, indeterminate };
7491
- }
7492
-
7493
- async function projectUsageKeysFromQueuePath(queuePath, source) {
7494
- if (typeof queuePath !== "string" || !queuePath) return [];
7495
- let raw = "";
7496
- try {
7497
- raw = await fs.readFile(queuePath, "utf8");
7498
- } catch (e) {
7499
- if (e?.code !== "ENOENT") throw e;
7500
- return [];
7501
- }
7502
- return projectUsageKeysFromQueueRows(raw.split("\n").filter((line) => line.trim()), source);
7503
- }
7504
-
7505
- function projectUsageKeysFromQueueRows(rows, source) {
7506
- const keys = [];
7507
- for (const line of Array.isArray(rows) ? rows : []) {
7508
- let row;
7509
- try {
7510
- row = JSON.parse(line);
7511
- } catch (_e) {
7512
- continue;
7513
- }
7514
- const key = projectUsageKeyFromFields({
7515
- projectKey: row?.project_key,
7516
- source: row?.source,
7517
- hourStart: row?.hour_start,
7518
- });
7519
- if (key && row?.source === source) keys.push(key);
7520
- }
7521
- return keys;
7522
- }
7523
-
7524
- function projectUsageKeysFromState(projectState, source) {
7525
- const buckets =
7526
- projectState && typeof projectState === "object" && projectState.buckets
7527
- ? projectState.buckets
7528
- : {};
7529
- const keys = [];
7530
- for (const [key, bucket] of Object.entries(buckets)) {
7531
- const bucketSource = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
7532
- if (bucketSource !== source) continue;
7533
- const usageKey =
7534
- projectUsageKeyFromFields({
7535
- projectKey: bucket?.project_key,
7536
- source: bucketSource,
7537
- hourStart: bucket?.hour_start,
7538
- }) || key;
7539
- keys.push(usageKey);
7540
- }
7541
- return keys;
7542
- }
7543
-
7544
- function projectUsageKeyFromFields({ projectKey, source, hourStart }) {
7545
- if (
7546
- typeof projectKey !== "string" ||
7547
- typeof source !== "string" ||
7548
- typeof hourStart !== "string" ||
7549
- !projectKey ||
7550
- !source ||
7551
- !hourStart
7552
- ) {
7553
- return null;
7554
- }
7555
- return `${projectKey}|${source}|${hourStart}`;
7556
- }
7557
-
7558
- async function countMalformedCodexProjectQueueRows(queuePath) {
7559
- if (typeof queuePath !== "string" || !queuePath) return 0;
7560
- let raw = "";
7561
- try {
7562
- raw = await fs.readFile(queuePath, "utf8");
7563
- } catch (e) {
7564
- if (e?.code !== "ENOENT") throw e;
7565
- return 0;
7566
- }
7567
- let count = 0;
7568
- for (const line of raw.split("\n")) {
7569
- if (!line.trim()) continue;
7570
- let row;
7571
- try {
7572
- row = JSON.parse(line);
7573
- } catch (_e) {
7574
- continue;
7575
- }
7576
- if (row?.source !== "codex") continue;
7577
- const key = projectUsageKeyFromFields({
7578
- projectKey: row?.project_key,
7579
- source: row?.source,
7580
- hourStart: row?.hour_start,
7581
- });
7582
- if (!key) count += 1;
7583
- }
7584
- return count;
7585
- }
7586
-
7587
- async function projectUsageTotalsFromQueuePath(queuePath, source) {
7588
- if (typeof queuePath !== "string" || !queuePath) return new Map();
7589
- let raw = "";
7590
- try {
7591
- raw = await fs.readFile(queuePath, "utf8");
7592
- } catch (e) {
7593
- if (e?.code !== "ENOENT") throw e;
7594
- return new Map();
7595
- }
7596
- return projectUsageTotalsFromQueueRows(raw.split("\n").filter((line) => line.trim()), source);
7597
- }
7598
-
7599
- function projectUsageTotalsFromQueueRows(rows, source) {
7600
- const totals = new Map();
7601
- for (const line of Array.isArray(rows) ? rows : []) {
7602
- let row;
7603
- try {
7604
- row = JSON.parse(line);
7605
- } catch (_e) {
7606
- continue;
7607
- }
7608
- const key = projectUsageKeyFromFields({
7609
- projectKey: row?.project_key,
7610
- source: row?.source,
7611
- hourStart: row?.hour_start,
7612
- });
7613
- if (!key || row?.source !== source) continue;
7614
- setMaxTotal(totals, key, Number(row.total_tokens || 0));
7615
- }
7616
- return totals;
7617
- }
7618
-
7619
- function projectUsageTotalsFromState(projectState, source) {
7620
- const buckets =
7621
- projectState && typeof projectState === "object" && projectState.buckets
7622
- ? projectState.buckets
7623
- : {};
7624
- const totals = new Map();
7625
- for (const [key, bucket] of Object.entries(buckets)) {
7626
- const bucketSource = typeof bucket?.source === "string" ? bucket.source : key.split("|")[1];
7627
- if (bucketSource !== source) continue;
7628
- const usageKey =
7629
- projectUsageKeyFromFields({
7630
- projectKey: bucket?.project_key,
7631
- source: bucketSource,
7632
- hourStart: bucket?.hour_start,
7633
- }) || key;
7634
- setMaxTotal(totals, usageKey, Number(bucket?.totals?.total_tokens || 0));
7635
- }
7636
- return totals;
7637
- }
7638
-
7639
- async function mainUsageHourTotalsFromQueuePath(queuePath, source) {
7640
- if (typeof queuePath !== "string" || !queuePath) return new Map();
7641
- let raw = "";
7642
- try {
7643
- raw = await fs.readFile(queuePath, "utf8");
7644
- } catch (e) {
7645
- if (e?.code !== "ENOENT") throw e;
7646
- return new Map();
7647
- }
7648
- return mainUsageHourTotalsFromQueueRows(raw.split("\n").filter((line) => line.trim()), source);
7649
- }
7650
-
7651
- function mainUsageHourTotalsFromQueueRows(rows, source) {
7652
- const modelTotals = new Map();
7653
- for (const line of Array.isArray(rows) ? rows : []) {
7654
- let row;
7655
- try {
7656
- row = JSON.parse(line);
7657
- } catch (_e) {
7658
- continue;
7659
- }
7660
- if (row?.source !== source || typeof row?.hour_start !== "string") continue;
7661
- const model = typeof row?.model === "string" && row.model ? row.model : "unknown";
7662
- setMaxTotal(modelTotals, `${row.source}|${model}|${row.hour_start}`, Number(row.total_tokens || 0));
7663
- }
7664
- return collapseModelTotalsByHour(modelTotals);
7665
- }
7666
-
7667
- function mainUsageHourTotalsFromState(hourlyState, source) {
7668
- const buckets =
7669
- hourlyState && typeof hourlyState === "object" && hourlyState.buckets
7670
- ? hourlyState.buckets
7671
- : {};
7672
- const modelTotals = new Map();
7673
- for (const [key, bucket] of Object.entries(buckets)) {
7674
- const parts = key.split("|");
7675
- const bucketSource = typeof bucket?.source === "string" ? bucket.source : parts[0];
7676
- if (bucketSource !== source) continue;
7677
- const model = typeof bucket?.model === "string" && bucket.model ? bucket.model : parts[1] || "unknown";
7678
- const hourStart =
7679
- typeof bucket?.hour_start === "string" && bucket.hour_start ? bucket.hour_start : parts[2];
7680
- if (typeof hourStart !== "string" || !hourStart) continue;
7681
- setMaxTotal(modelTotals, `${bucketSource}|${model}|${hourStart}`, Number(bucket?.totals?.total_tokens || 0));
7682
- }
7683
- return collapseModelTotalsByHour(modelTotals);
7684
- }
7685
-
7686
- function collapseModelTotalsByHour(modelTotals) {
7687
- const totals = new Map();
7688
- for (const [key, total] of modelTotals.entries()) {
7689
- const [source, , hourStart] = key.split("|");
7690
- const hourKey = `${source}|${hourStart}`;
7691
- totals.set(hourKey, (totals.get(hourKey) || 0) + total);
7692
- }
7693
- return totals;
7694
- }
7695
-
7696
- function mergeMaxTotals(...maps) {
7697
- const merged = new Map();
7698
- for (const map of maps) {
7699
- if (!(map instanceof Map)) continue;
7700
- for (const [key, total] of map.entries()) {
7701
- setMaxTotal(merged, key, total);
7702
- }
7703
- }
7704
- return merged;
7705
- }
7706
-
7707
- function setMaxTotal(map, key, total) {
7708
- if (!key || !Number.isFinite(total)) return;
7709
- const prev = map.get(key);
7710
- if (!Number.isFinite(prev) || total > prev) map.set(key, total);
7711
- }
7712
-
7713
- // One-time repair (#204): when the SAME Droid session id existed in two folders
7714
- // under ~/.factory/sessions, parseDroidIncremental's cumulative-delta loop made the
7715
- // lower-count file look like a reset and re-emitted each duplicate's full total on
7716
- // EVERY sync, inflating one (droid, model, hour) bucket without bound (a real ~10M
7717
- // session showed as 40.06B). The forward fix is dedupeDroidSettingsFilesBySession
7718
- // inside the parser; this migration repairs already-polluted installs.
7719
- //
7720
- // SCOPE — strictly the duplicate sessions' buckets, and ONLY when duplicate files
7721
- // still exist on disk:
7722
- // * A from-zero rebuild cannot reconstruct Droid's historical per-sync bucket
7723
- // distribution — settings.json carries only the CURRENT mtime, not per-turn
7724
- // timestamps like Codex's jsonl. So we rebuild over the DUPLICATE files only,
7725
- // and overwrite only the bucket keys those files map to (pollutedKeys) plus
7726
- // the duplicate sessions' cursor entries. Every other droid bucket and cursor
7727
- // — clean sessions AND deleted-session history — is left byte-for-byte intact,
7728
- // so healthy history is never collapsed into the current half-hour.
7729
- // * No session id has >1 file on disk → this bug never fired here: set the
7730
- // sentinel, touch nothing.
7731
- // * Fire only when live > rebuilt over pollutedKeys (actual inflation), so a
7732
- // fresh install (live empty) is left to the normal same-sync parse.
7733
- // Droid has no project dimension, so project.queue.jsonl is never involved.
7734
- async function repairDroidDuplicateSessionInflation({ cursors, queuePath, queueStatePath } = {}) {
7735
- if (!cursors || typeof cursors !== "object") return false;
7736
- const migrations = (cursors.migrations ||= {});
7737
- // Completed run → truthy non-skip sentinel (final). A {skipped:true} object would
7738
- // retry (codex skip-retry semantics; this repair only ever writes done/none).
7739
- const prior = migrations[DROID_DUP_SESSION_REPAIR_KEY];
7740
- if (prior && !(typeof prior === "object" && prior.skipped)) return false;
7741
-
7742
- // Group current on-disk settings files by session id; collect duplicates.
7743
- let onDisk;
7744
- try {
7745
- onDisk = listDroidSettingsFiles(process.env);
7746
- } catch {
7747
- onDisk = [];
7748
- }
7749
- const bySession = new Map();
7750
- for (const fp of onDisk) {
7751
- const sid = droidSessionIdFromPath(fp);
7752
- if (!sid) continue;
7753
- if (!bySession.has(sid)) bySession.set(sid, []);
7754
- bySession.get(sid).push(fp);
7755
- }
7756
- const dupFiles = [];
7757
- const cleanFiles = [];
7758
- for (const group of bySession.values()) {
7759
- if (group.length > 1) dupFiles.push(...group);
7760
- else cleanFiles.push(...group);
7761
- }
7762
- if (dupFiles.length === 0) {
7763
- migrations[DROID_DUP_SESSION_REPAIR_KEY] = new Date().toISOString();
7764
- return false;
7765
- }
7766
-
7767
- // Map on-disk settings files to the (droid, model, half-hour) bucket keys they
7768
- // emit under the parser's own keying (mirrors parseDroidIncremental).
7769
- const bucketKeysForFiles = (files) => {
7770
- const keys = new Set();
7771
- for (const fp of files) {
7772
- let mtimeMs = 0;
7773
- try {
7774
- mtimeMs = fssync.statSync(fp).mtimeMs;
7775
- } catch {
7776
- continue;
7777
- }
7778
- let settings;
7779
- try {
7780
- settings = JSON.parse(fssync.readFileSync(fp, "utf8"));
7781
- } catch {
7782
- continue;
7783
- }
7784
- if (!settings || typeof settings !== "object" || !settings.tokenUsage) continue;
7785
- const bucketStart = toUtcHalfHourStart(
7786
- new Date(mtimeMs || Date.now()).toISOString(),
7787
- );
7788
- if (!bucketStart) continue;
7789
- keys.add(bucketKey("droid", resolveDroidModel(settings, fp), bucketStart));
7790
- }
7791
- return keys;
7792
- };
7793
-
7794
- // A bucket key is (source, model, half-hour) — it carries NO session identity.
7795
- // pollutedKeys are the buckets the duplicate files emit to; cleanKeys are buckets
7796
- // a NON-duplicate on-disk session emits to. When a clean session resolves to the
7797
- // same (model, half-hour) as a duplicate file, they collide on one key. We must
7798
- // NOT delete-and-replace such a shared bucket: the rebuild runs over duplicate
7799
- // files only, so replacing the bucket would erase the clean session's tokens
7800
- // (silent data loss). So repair ONLY buckets owned exclusively by duplicate
7801
- // sessions; leave shared buckets intact. Residual inflation in a rare shared
7802
- // bucket is visible and recoverable; destroying real data is not (this is the
7803
- // dedup-needs-identity-proof rule).
7804
- const pollutedKeys = bucketKeysForFiles(dupFiles);
7805
- const cleanKeys = bucketKeysForFiles(cleanFiles);
7806
- const repairKeys = new Set();
7807
- for (const k of pollutedKeys) if (!cleanKeys.has(k)) repairKeys.add(k);
7808
- if (repairKeys.size === 0) {
7809
- migrations[DROID_DUP_SESSION_REPAIR_KEY] = new Date().toISOString();
7810
- return false;
7811
- }
7812
-
7813
- // Ground-truth rebuild into throwaway state over the DUPLICATE files only
7814
- // (parseDroidIncremental de-dupes its own input → canonical per session). On any
7815
- // throw, leave all state untouched and do NOT set the sentinel (retry next sync).
7816
- let rebuilt;
7817
- const tmpQueue = `${queuePath}.droidrebuild.${process.pid}.${Date.now()}`;
7818
- try {
7819
- const tmpCursors = { hourly: { buckets: {}, groupQueued: {} }, droid: {} };
7820
- await parseDroidIncremental({
7821
- settingsFiles: dupFiles,
7822
- cursors: tmpCursors,
7823
- queuePath: tmpQueue,
7824
- env: process.env,
7825
- prune: true,
7826
- });
7827
- let tmpRaw = "";
7828
- try {
7829
- tmpRaw = await fs.readFile(tmpQueue, "utf8");
7830
- } catch (e) {
7831
- if (e?.code !== "ENOENT") throw e;
7832
- }
7833
- rebuilt = {
7834
- buckets: tmpCursors.hourly.buckets || {},
7835
- sessionTotals: (tmpCursors.droid && tmpCursors.droid.sessionTotals) || {},
7836
- queueRows: tmpRaw.split("\n").filter((l) => l.trim()),
7837
- };
7838
- } catch (e) {
7839
- console.error(
7840
- "[sync] droid dup-session repair: rebuild failed, leaving all data untouched:",
7841
- e?.message || e,
7842
- );
7843
- return false;
7844
- } finally {
7845
- await fs.rm(tmpQueue, { force: true }).catch(() => {});
7846
- }
7847
-
7848
- // Inflation present? Compare live vs rebuilt totals over the repair-scoped keys
7849
- // only. Fire only on live > rebuilt (real inflation) — never on a fresh install
7850
- // (live 0).
7851
- const liveBuckets = (cursors.hourly && cursors.hourly.buckets) || {};
7852
- let liveScoped = 0;
7853
- let rebuiltScoped = 0;
7854
- for (const key of repairKeys) {
7855
- liveScoped += Number(liveBuckets[key]?.totals?.total_tokens || 0);
7856
- rebuiltScoped += Number(rebuilt.buckets[key]?.totals?.total_tokens || 0);
7857
- }
7858
- if (liveScoped <= rebuiltScoped) {
7859
- migrations[DROID_DUP_SESSION_REPAIR_KEY] = new Date().toISOString();
7860
- return false;
7861
- }
7862
-
7863
- // ── COMMIT (atomic) ──
7864
- await ensureDir(path.dirname(queuePath));
7865
- await backupExistingFile(queuePath);
7866
-
7867
- // 1. queue.jsonl: keep every non-droid line verbatim (incl. unparseable) and
7868
- // every droid row whose bucket key is NOT in repairKeys (clean + shared +
7869
- // deleted-session history). Drop droid rows in repairKeys; append rebuilt rows.
7870
- let raw = "";
7871
- try {
7872
- raw = await fs.readFile(queuePath, "utf8");
7873
- } catch (e) {
7874
- if (e?.code !== "ENOENT") throw e;
7875
- }
7876
- // A queue line is in scope if it's a droid row whose bucket is in repairKeys.
7877
- // Unparseable / non-droid / clean / shared droid rows are kept verbatim.
7878
- const isRepairDroidRow = (line) => {
7879
- let row;
7880
- try {
7881
- row = JSON.parse(line);
7882
- } catch {
7883
- return false;
7884
- }
7885
- return (
7886
- row?.source === "droid" &&
7887
- repairKeys.has(bucketKey("droid", row.model, row.hour_start))
7888
- );
7889
- };
7890
- const kept = raw
7891
- .split("\n")
7892
- .filter((line) => line.trim() && !isRepairDroidRow(line));
7893
- const rebuiltRepairRows = rebuilt.queueRows.filter(isRepairDroidRow);
7894
- const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
7895
- await fs.writeFile(
7896
- tmp,
7897
- kept.concat(rebuiltRepairRows).join("\n") + "\n",
7898
- "utf8",
7899
- );
7900
- await fs.rename(tmp, queuePath);
7901
-
7902
- // 2. live hourly buckets: delete repair-scoped droid keys, install the rebuilt
7903
- // buckets for those keys. Other droid buckets untouched.
7904
- const hourly = (cursors.hourly ||= { version: 3, buckets: {}, groupQueued: {} });
7905
- hourly.buckets ||= {};
7906
- hourly.groupQueued ||= {};
7907
- for (const key of repairKeys) {
7908
- delete hourly.buckets[key];
7909
- if (rebuilt.buckets[key]) hourly.buckets[key] = rebuilt.buckets[key];
7910
- }
7911
- // Defensive: droid never uses the legacy aggregate path, but drop any stale droid
7912
- // group markers so a repaired hour can't re-emit as model=unknown.
7913
- for (const gk of Object.keys(hourly.groupQueued)) {
7914
- if (gk.startsWith("droid|")) delete hourly.groupQueued[gk];
7915
- }
7916
-
7917
- // 3. session cursor: overwrite ONLY the duplicate sessions with the ground-truth
7918
- // rebuild so the later same-sync droid parse short-circuits (mtime match) and
7919
- // emits nothing. Clean sessions' cursor entries are correct already — leave
7920
- // them, or the later parse would re-emit them from zero.
7921
- const droidState = (cursors.droid ||= {});
7922
- if (!droidState.sessionTotals || typeof droidState.sessionTotals !== "object") {
7923
- droidState.sessionTotals = {};
7924
- }
7925
- for (const sid of Object.keys(rebuilt.sessionTotals)) {
7926
- droidState.sessionTotals[sid] = rebuilt.sessionTotals[sid];
7927
- }
7928
- droidState.updatedAt = new Date().toISOString();
7929
-
7930
- // 4. reset the cloud upload offset so corrected rows re-upload (idempotent upsert).
7931
- if (typeof queueStatePath === "string" && queueStatePath) {
7932
- let uploadState = {};
7933
- try {
7934
- uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
7935
- } catch {
7936
- uploadState = {};
7937
- }
7938
- uploadState.offset = 0;
7939
- uploadState.updatedAt = new Date().toISOString();
7940
- uploadState.note = "reset_after_droid_dup_session_2026_06";
7941
- await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
7942
- }
7943
-
7944
- migrations[DROID_DUP_SESSION_REPAIR_KEY] = {
7945
- status: "done",
7946
- at: new Date().toISOString(),
7947
- keysRepaired: repairKeys.size,
7948
- keysSkippedSharedWithCleanSession: pollutedKeys.size - repairKeys.size,
7949
- liveBefore: liveScoped,
7950
- rebuiltAfter: rebuiltScoped,
7951
- deltaReclaimed: liveScoped - rebuiltScoped,
7952
- };
7953
- return true;
7954
- }
7955
-
7956
- // One-time repair migration: rebuild source=claude rows in queue.jsonl from
7957
- // the actual jsonl files using ccusage's algorithm (msgId+reqId global
7958
- // dedup). Earlier `reincludeClaudeMemObserverFiles` versions (v1/v2/v3) each
7959
- // reset the hash set and re-read observer jsonls, which silently inflated
7960
- // queue.jsonl's claude totals by ~40%. We do an atomic rewrite — keep all
7961
- // non-claude rows verbatim, replace every claude/claude-mem row with the
7962
- // ground-truth set — then reset cursors so the next incremental sync stays
7963
- // in sync, and reset the cloud upload offset so the corrected rows actually
7964
- // reach the cloud (the ingest endpoint upserts by (source, model,
7965
- // hour_start), so re-uploading other sources is idempotent).
7966
- async function repairClaudeQueueFromGroundTruth({
7967
- cursors,
7968
- queuePath,
7969
- queueStatePath = null,
7970
- projectQueuePath = null,
7971
- projectQueueStatePath = null,
7972
- rootDirs = null,
7973
- }) {
7974
- if (!cursors || typeof cursors !== "object") return false;
7975
- const migrations = (cursors.migrations ||= {});
7976
- if (migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY]) return false;
7977
-
7978
- let result;
7979
- try {
7980
- // rootDirs must cover every install the incremental parser scans (native
7981
- // AND WSL): this repair's semantics are "disk is truth" — it clears all
7982
- // claude buckets and replaces claudeHashes wholesale, so scanning fewer
7983
- // roots than sync would silently drop the missing install's history.
7984
- result = await computeClaudeGroundTruthBuckets(
7985
- Array.isArray(rootDirs) && rootDirs.length > 0 ? { rootDirs } : {},
7986
- );
7987
- } catch (e) {
7988
- console.error("[sync] claude ground-truth repair: scan failed:", e?.message || e);
7989
- return false;
7990
- }
7991
- const { rows, seenHashes, fileList } = result;
7992
-
7993
- // 1. Atomic rewrite of queue.jsonl: keep non-claude rows, drop existing
7994
- // claude/claude-mem rows, append truth rows. Atomic via tmp + rename.
7995
- let claudeRowsRemoved = 0;
7996
- if (typeof queuePath === "string" && queuePath) {
7997
- let raw = "";
7998
- try {
7999
- raw = await fs.readFile(queuePath, "utf8");
8000
- } catch (e) {
8001
- if (e?.code !== "ENOENT") throw e;
8002
- }
8003
- const keptLines = [];
8004
- for (const line of raw.split("\n")) {
8005
- if (!line.trim()) continue;
8006
- let row;
8007
- try {
8008
- row = JSON.parse(line);
8009
- } catch (_e) {
8010
- // Preserve unparseable lines verbatim — operator may want to
8011
- // recover them later.
8012
- keptLines.push(line);
8013
- continue;
8014
- }
8015
- if (row?.source === "claude" || row?.source === "claude-mem") {
8016
- claudeRowsRemoved += 1;
8017
- continue;
8018
- }
8019
- keptLines.push(line);
8020
- }
8021
-
8022
- const truthLines = rows.map((r) =>
8023
- JSON.stringify({
8024
- source: "claude",
8025
- model: r.model,
8026
- hour_start: r.hour_start,
8027
- input_tokens: r.input_tokens,
8028
- cached_input_tokens: r.cached_input_tokens,
8029
- cache_creation_input_tokens: r.cache_creation_input_tokens,
8030
- output_tokens: r.output_tokens,
8031
- reasoning_output_tokens: r.reasoning_output_tokens,
8032
- total_tokens: r.total_tokens,
8033
- billable_total_tokens: r.billable_total_tokens,
8034
- conversation_count: r.conversation_count,
8035
- }),
8036
- );
8037
-
8038
- await ensureDir(path.dirname(queuePath));
8039
- const out = keptLines.concat(truthLines).join("\n") + "\n";
8040
- const tmp = `${queuePath}.tmp.${process.pid}.${Date.now()}`;
8041
- await fs.writeFile(tmp, out, "utf8");
8042
- await fs.rename(tmp, queuePath);
8043
- }
8044
-
8045
- // 2. Reset cursors.hourly.buckets / groupQueued for source=claude (and the
8046
- // dead source=claude-mem buckets) so incremental sync's in-memory state
8047
- // matches the truth.
8048
- const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
8049
- hourly.buckets ||= {};
8050
- hourly.groupQueued ||= {};
8051
-
8052
- let bucketsCleared = 0;
8053
- for (const k of Object.keys(hourly.buckets)) {
8054
- if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
8055
- delete hourly.buckets[k];
8056
- bucketsCleared += 1;
8057
- }
8058
- }
8059
- // Clear stale claude entries from groupQueued (left over by v2 repair).
8060
- // After v3 we never repopulate it for claude, so nothing should be added
8061
- // back during the per-model write loop below.
8062
- for (const k of Object.keys(hourly.groupQueued)) {
8063
- if (k.startsWith("claude|") || k.startsWith("claude-mem|")) {
8064
- delete hourly.groupQueued[k];
8065
- }
8066
- }
8067
-
8068
- // Per-model claude buckets: set queuedKey but DO NOT touch
8069
- // hourly.groupQueued. groupQueued is used by enqueueTouchedBuckets to
8070
- // mark a (source, hour) as legacy-aggregate state; writing claude hours
8071
- // there would force every later sync to re-emit the hour as a single
8072
- // model=DEFAULT_MODEL aggregate row instead of touching only the bucket
8073
- // that actually changed. The original v2 release did write groupQueued
8074
- // here and was the cause of an unknown-bucket inflation regression.
8075
- for (const r of rows) {
8076
- const totals = {
8077
- input_tokens: r.input_tokens,
8078
- cached_input_tokens: r.cached_input_tokens,
8079
- cache_creation_input_tokens: r.cache_creation_input_tokens,
8080
- output_tokens: r.output_tokens,
8081
- reasoning_output_tokens: r.reasoning_output_tokens,
8082
- total_tokens: r.total_tokens,
8083
- billable_total_tokens: r.billable_total_tokens,
8084
- conversation_count: r.conversation_count,
8085
- };
8086
- const key = bucketKey("claude", r.model, r.hour_start);
8087
- hourly.buckets[key] = {
8088
- totals,
8089
- queuedKey: totalsKey(totals),
8090
- source: "claude",
8091
- hour_start: r.hour_start,
8092
- };
8093
- }
8094
-
8095
- // 3. Reset per-file cursors so future incremental sync only reads genuinely
8096
- // new tail content. Format must match what rollout.js expects:
8097
- // { inode, offset, updatedAt }. Setting a plain integer here breaks
8098
- // the inode-equality check inside parseClaudeFile, which would treat
8099
- // the file as untracked and re-read it from byte 0 — silently doubling
8100
- // everything. (That was the actual cause of the regression after the
8101
- // first repair attempt.)
8102
- cursors.files ||= {};
8103
- let filesReset = 0;
8104
- const nowIso = new Date().toISOString();
8105
- for (const fp of fileList) {
8106
- let st;
8107
- try {
8108
- st = fssync.statSync(fp);
8109
- } catch (_e) {
8110
- continue;
8111
- }
8112
- cursors.files[fp] = {
8113
- inode: st.ino || 0,
8114
- offset: st.size,
8115
- updatedAt: nowIso,
8116
- };
8117
- filesReset += 1;
8118
- }
8119
- cursors.claudeHashes = seenHashes;
8120
-
8121
- // 4. Reset cloud-upload offset so the corrected rows are re-sent. Other
8122
- // sources are upserted idempotently by the ingest endpoint, so this is
8123
- // safe — just costs one extra round of bandwidth.
8124
- if (typeof queueStatePath === "string" && queueStatePath) {
8125
- let uploadState = {};
8126
- try {
8127
- uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
8128
- } catch (_e) {
8129
- uploadState = {};
8130
- }
8131
- uploadState.offset = 0;
8132
- uploadState.updatedAt = new Date().toISOString();
8133
- uploadState.note = "reset_after_claude_repair_2026_05_v4";
8134
- await fs.writeFile(queueStatePath, JSON.stringify(uploadState));
8135
- }
8136
-
8137
- // 5. Repair project queue. Historical claude rows in project.queue.jsonl
8138
- // were uniformly mis-attributed to project_key=
8139
- // "claude-mem/observer-sessions" (left over from the observer
8140
- // relabel migration). We can't reconstruct the true cwd-based
8141
- // project_key for each historical message reliably, so we drop every
8142
- // claude/claude-mem row from project.queue.jsonl and reset the
8143
- // matching cursors.projectHourly state. New claude usage will
8144
- // accumulate to the correct cwd-derived project_key going forward.
8145
- let projectRowsRemoved = 0;
8146
- let projectBucketsCleared = 0;
8147
- if (typeof projectQueuePath === "string" && projectQueuePath) {
8148
- let projRaw = "";
8149
- try {
8150
- projRaw = await fs.readFile(projectQueuePath, "utf8");
8151
- } catch (e) {
8152
- if (e?.code !== "ENOENT") throw e;
8153
- }
8154
- if (projRaw) {
8155
- const projKept = [];
8156
- for (const line of projRaw.split("\n")) {
8157
- if (!line.trim()) continue;
8158
- let row;
8159
- try {
8160
- row = JSON.parse(line);
8161
- } catch (_e) {
8162
- projKept.push(line);
8163
- continue;
8164
- }
8165
- if (row?.source === "claude" || row?.source === "claude-mem") {
8166
- projectRowsRemoved += 1;
8167
- continue;
8168
- }
8169
- projKept.push(line);
8170
- }
8171
- await ensureDir(path.dirname(projectQueuePath));
8172
- const tmp = `${projectQueuePath}.tmp.${process.pid}.${Date.now()}`;
8173
- await fs.writeFile(tmp, projKept.join("\n") + "\n", "utf8");
8174
- await fs.rename(tmp, projectQueuePath);
8175
- }
8176
-
8177
- // Clear matching projectHourly state so the claude project buckets
8178
- // start fresh.
8179
- const projHourly = (cursors.projectHourly ||= { buckets: {} });
8180
- projHourly.buckets ||= {};
8181
- for (const k of Object.keys(projHourly.buckets)) {
8182
- const v = projHourly.buckets[k];
8183
- const src = v?.source || "";
8184
- if (src === "claude" || src === "claude-mem") {
8185
- delete projHourly.buckets[k];
8186
- projectBucketsCleared += 1;
8187
- }
8188
- }
8189
-
8190
- // Reset project upload offset.
8191
- if (typeof projectQueueStatePath === "string" && projectQueueStatePath) {
8192
- let st = {};
8193
- try {
8194
- st = JSON.parse(await fs.readFile(projectQueueStatePath, "utf8"));
8195
- } catch (_e) {
8196
- st = {};
8197
- }
8198
- st.offset = 0;
8199
- st.updatedAt = new Date().toISOString();
8200
- st.note = "reset_after_claude_repair_2026_05_v6";
8201
- await fs.writeFile(projectQueueStatePath, JSON.stringify(st));
8202
- }
8203
- }
8204
-
8205
- migrations[CLAUDE_GROUND_TRUTH_REPAIR_KEY] = {
8206
- appliedAt: new Date().toISOString(),
8207
- bucketsWritten: rows.length,
8208
- bucketsCleared,
8209
- rowsRemoved: claudeRowsRemoved,
8210
- filesReset,
8211
- hashesRetained: seenHashes.length,
8212
- uploadOffsetReset: typeof queueStatePath === "string" && !!queueStatePath,
8213
- projectRowsRemoved,
8214
- projectBucketsCleared,
8215
- };
8216
- return true;
8217
- }
8218
-
8219
- async function reincludeClaudeMemObserverFiles({ cursors, claudeFiles, queuePath, queueStatePath }) {
8220
- if (!cursors || typeof cursors !== "object") return false;
8221
- const migrations = (cursors.migrations ||= {});
8222
- if (migrations[CLAUDE_MEM_OBSERVER_REINCLUDE_KEY]) return false;
8223
-
8224
- const observerPaths = (Array.isArray(claudeFiles) ? claudeFiles : [])
8225
- .map((entry) => (typeof entry === "string" ? entry : entry?.path))
8226
- .filter((p) => typeof p === "string" && p.includes(CLAUDE_MEM_OBSERVER_PATH_SEGMENT));
8227
-
8228
- if (!cursors.files || typeof cursors.files !== "object") {
8229
- cursors.files = {};
8230
- }
8231
-
8232
- let filesReset = 0;
8233
- for (const filePath of observerPaths) {
8234
- if (cursors.files[filePath]) {
8235
- delete cursors.files[filePath];
8236
- filesReset += 1;
8237
- }
8238
- }
8239
-
8240
- const hashesToRemove = observerPaths.length > 0
8241
- ? await collectClaudeMessageHashes(observerPaths)
8242
- : new Set();
8243
- let hashesRemoved = 0;
8244
- if (Array.isArray(cursors.claudeHashes) && hashesToRemove.size > 0) {
8245
- const nextHashes = [];
8246
- for (const hash of cursors.claudeHashes) {
8247
- if (hashesToRemove.has(hash)) {
8248
- hashesRemoved += 1;
8249
- continue;
8250
- }
8251
- nextHashes.push(hash);
8252
- }
8253
- cursors.claudeHashes = nextHashes;
8254
- }
8255
-
8256
- const queueRowsRelabeled = typeof queuePath === "string" && queuePath
8257
- ? await relabelClaudeMemQueueRows(queuePath, queueStatePath)
8258
- : 0;
8259
-
8260
- migrations[CLAUDE_MEM_OBSERVER_REINCLUDE_KEY] = {
8261
- appliedAt: new Date().toISOString(),
8262
- filesReset,
8263
- hashesRemoved,
8264
- queueRowsRelabeled,
8265
- };
8266
- return filesReset > 0 || hashesRemoved > 0 || queueRowsRelabeled > 0;
8267
- }
8268
-
8269
- async function relabelClaudeMemQueueRows(queuePath, queueStatePath = null) {
8270
- let raw;
8271
- try {
8272
- raw = await fs.readFile(queuePath, "utf8");
8273
- } catch (_e) {
8274
- return 0;
8275
- }
8276
- if (!raw || !raw.includes('"claude-mem"')) return 0;
8277
-
8278
- // The cloud-upload cursor (queue.state.json `offset`) is a byte position in
8279
- // the pre-rewrite file. Relabeling shrinks rewritten lines ("claude-mem" →
8280
- // "claude"), so the old offset would land mid-line in the new file and the
8281
- // next drainQueueToCloud batch would skip part of a row (or a whole row).
8282
- // Track the old→new byte mapping while rewriting and remap the offset to
8283
- // the equivalent line boundary (same pattern as project-usage-purge.js).
8284
- let previousOffset = 0;
8285
- if (typeof queueStatePath === "string" && queueStatePath) {
8286
- try {
8287
- const st = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
8288
- const off = Number(st?.offset || 0);
8289
- if (Number.isFinite(off) && off > 0) previousOffset = off;
8290
- } catch (_e) {
8291
- previousOffset = 0;
8292
- }
8293
- }
8294
-
8295
- const lines = raw.split("\n");
8296
- const out = [];
8297
- let relabeled = 0;
8298
- let inputOffset = 0;
8299
- let outputOffset = 0;
8300
- let nextOffset = 0;
8301
- for (let i = 0; i < lines.length; i++) {
8302
- const line = lines[i];
8303
- const isLast = i === lines.length - 1;
8304
- let outLine = line;
8305
- if (line) {
8306
- try {
8307
- const obj = JSON.parse(line);
8308
- if (obj && obj.source === "claude-mem") {
8309
- obj.source = "claude";
8310
- relabeled += 1;
8311
- outLine = JSON.stringify(obj);
8312
- }
8313
- } catch (_e) {
8314
- // keep malformed lines verbatim
8315
- }
8316
- }
8317
- out.push(outLine);
8318
- inputOffset += Buffer.byteLength(line, "utf8") + (isLast ? 0 : 1);
8319
- outputOffset += Buffer.byteLength(outLine, "utf8") + (isLast ? 0 : 1);
8320
- // Upload offsets always sit at line boundaries; a mid-line offset
8321
- // (corruption) rounds down to the previous boundary so no row is skipped
8322
- // — worst case a row is re-uploaded, and cloud ingest upserts by key.
8323
- if (inputOffset <= previousOffset) nextOffset = outputOffset;
8324
- }
8325
- if (relabeled === 0) return 0;
8326
-
8327
- // Atomic rewrite: temp file in the same directory + rename, so a crash
8328
- // mid-write can never leave queue.jsonl truncated.
8329
- const tmpPath = `${queuePath}.tmp`;
8330
- await fs.writeFile(tmpPath, out.join("\n"), "utf8");
8331
- await fs.rename(tmpPath, queuePath);
8332
-
8333
- if (typeof queueStatePath === "string" && queueStatePath && previousOffset > 0) {
8334
- let state = {};
8335
- try {
8336
- state = JSON.parse(await fs.readFile(queueStatePath, "utf8"));
8337
- if (!state || typeof state !== "object") state = {};
8338
- } catch (_e) {
8339
- state = {};
8340
- }
8341
- state.offset = nextOffset;
8342
- state.updatedAt = new Date().toISOString();
8343
- await fs.writeFile(queueStatePath, JSON.stringify(state), "utf8");
8344
- }
8345
- return relabeled;
8346
- }
8347
-
8348
- async function collectClaudeMessageHashes(filePaths) {
8349
- const hashes = new Set();
8350
- for (const filePath of filePaths) {
8351
- let stream;
8352
- try {
8353
- stream = fssync.createReadStream(filePath, { encoding: "utf8" });
8354
- } catch (_e) {
8355
- continue;
8356
- }
8357
- const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
8358
- for await (const line of rl) {
8359
- if (!line.includes('"usage"')) continue;
8360
- let obj;
8361
- try {
8362
- obj = JSON.parse(line);
8363
- } catch (_e) {
8364
- continue;
8365
- }
8366
- const hash = claudeMessageDedupKey(obj);
8367
- if (hash) hashes.add(hash);
8368
- }
8369
- rl.close();
8370
- stream.close?.();
8371
- }
8372
- return hashes;
8373
- }