tokentracker-cli 1.4.3 → 1.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +1 -0
- package/README.ja.md +1 -0
- package/README.ko.md +1 -0
- package/README.md +5 -0
- package/README.zh-CN.md +1 -0
- package/dashboard/dist/assets/{AchievementBadge-3IYidbrI.js → AchievementBadge-BmWRsZMx.js} +1 -1
- package/dashboard/dist/assets/{AchievementsPage-DmZ_ojyM.js → AchievementsPage-CyIP2nck.js} +2 -2
- package/dashboard/dist/assets/{AchievementsSection-DIYrrzhw.js → AchievementsSection-BeNyJ0LW.js} +1 -1
- package/dashboard/dist/assets/{BadgeDetailModal-DMMExCe4.js → BadgeDetailModal-DIFnFzkv.js} +1 -1
- package/dashboard/dist/assets/{BotAnimated-y6vZNRvC.js → BotAnimated-CKVeJHit.js} +1 -1
- package/dashboard/dist/assets/{Card-LJX3G3zo.js → Card-lQmyKbh4.js} +1 -1
- package/dashboard/dist/assets/{CloudCheckoutPage-BGBgSXNE.js → CloudCheckoutPage-B_8y6dyX.js} +1 -1
- package/dashboard/dist/assets/{CloudPage-Dij2GTXi.js → CloudPage-BOQjUGAt.js} +2 -2
- package/dashboard/dist/assets/{CommandPalette-BIky6Fk0.js → CommandPalette-BV9LqokI.js} +1 -1
- package/dashboard/dist/assets/{CommunityStatsModal-DbM99i0s.js → CommunityStatsModal-WELxKEFc.js} +1 -1
- package/dashboard/dist/assets/{ConfirmModal-C_NH8vgO.js → ConfirmModal-B8H5o8c_.js} +1 -1
- package/dashboard/dist/assets/{Controls-BEAj_kvr.js → Controls-9ynPx9rF.js} +1 -1
- package/dashboard/dist/assets/{DashboardPage-DO7h2dfu.js → DashboardPage-B8z-U63p.js} +1 -1
- package/dashboard/dist/assets/{DateRangePopover-PQXB8v0k.js → DateRangePopover-DEqWsV_8.js} +1 -1
- package/dashboard/dist/assets/{DevicePage-0JZ3OIWZ.js → DevicePage-CUOlUySM.js} +1 -1
- package/dashboard/dist/assets/{DialogClose-BpWoD5h9.js → DialogClose-Dlg52bL6.js} +1 -1
- package/dashboard/dist/assets/DialogDescription-vuQFBlM_.js +1 -0
- package/dashboard/dist/assets/{DialogTitle-DoQgdh5w.js → DialogTitle-CduwXjqH.js} +1 -1
- package/dashboard/dist/assets/{ElectricLogo-0O87VjKv.js → ElectricLogo-DFt7_F2f.js} +1 -1
- package/dashboard/dist/assets/{FadeIn-9ph1h_AG.js → FadeIn-DlyusE6n.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-DCZ92xbH.js → HeaderGithubStar-tUtJoFHh.js} +1 -1
- package/dashboard/dist/assets/{HoverTooltip-CCKdmuOP.js → HoverTooltip-v-zkCqsX.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-CpyhpmdS.js → IpCheckPage-C5Y1uVEr.js} +1 -1
- package/dashboard/dist/assets/{LandingPage-spvsPkYy.js → LandingPage-DzOJOAj7.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardPage-2nJ08ohw.js → LeaderboardPage-BBZ_6tCw.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProBadge-D6FSL3Qy.js → LeaderboardProBadge-DakoXIQn.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardProfileModal-DD1jX-rD.js → LeaderboardProfileModal-CINpES_x.js} +3 -3
- package/dashboard/dist/assets/{LeaderboardProfilePage-DYqRrntL.js → LeaderboardProfilePage-B6ISf-1l.js} +1 -1
- package/dashboard/dist/assets/{LimitsPage-DXHikunk.js → LimitsPage-UVX730CQ.js} +8 -8
- package/dashboard/dist/assets/{LocalOnlyNotice-BURiQAtP.js → LocalOnlyNotice-BJSsmXF1.js} +1 -1
- package/dashboard/dist/assets/{LoginCard-Cv2ijoJx.js → LoginCard-Dggv2aGv.js} +1 -1
- package/dashboard/dist/assets/{LoginModal-DyOideDA.js → LoginModal-B8qDd2kY.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-BKNvqvYi.js → LoginPage-Dv66U8yJ.js} +1 -1
- package/dashboard/dist/assets/{PetPage-BBmozE0L.js → PetPage-CNwrQdKM.js} +2 -2
- package/dashboard/dist/assets/{PopoverPopup-DDnISLKM.js → PopoverPopup-CqmvpYOi.js} +1 -1
- package/dashboard/dist/assets/{ProviderIcon-Da90GDcM.js → ProviderIcon-BF2Ch7wi.js} +1 -1
- package/dashboard/dist/assets/{ResetPasswordPage-DsETr50j.js → ResetPasswordPage-B0R8VUU3.js} +1 -1
- package/dashboard/dist/assets/{Select-pZDawyxt.js → Select-idAiBnJG.js} +1 -1
- package/dashboard/dist/assets/{SelectItemText-D5wuuN7G.js → SelectItemText-DPmmPY_D.js} +1 -1
- package/dashboard/dist/assets/{SelfHostPage-sm-PARYj.js → SelfHostPage-oR9cPK02.js} +1 -1
- package/dashboard/dist/assets/{ServiceStatusPage-D4f8h0sL.js → ServiceStatusPage-CdV9gfGF.js} +1 -1
- package/dashboard/dist/assets/{SessionsPage-xVLsTN4m.js → SessionsPage-BkD34VlL.js} +1 -1
- package/dashboard/dist/assets/{SettingsPage-Dt9iFheY.js → SettingsPage-DsthWCyM.js} +3 -3
- package/dashboard/dist/assets/{SkillsPage-BdWalPpd.js → SkillsPage-CzMk9s3x.js} +1 -1
- package/dashboard/dist/assets/{TokenGalaxy-d2_ml0tE.js → TokenGalaxy-DQ6o8YBp.js} +1 -1
- package/dashboard/dist/assets/{TooltipPopup-B4masxDW.js → TooltipPopup-CMQjA-HE.js} +1 -1
- package/dashboard/dist/assets/{TrendMonitor-CceYpa2O.js → TrendMonitor-DrV1m8E7.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-CSN0829H.js → WidgetsPage-Cc16WM0Y.js} +1 -1
- package/dashboard/dist/assets/{WrappedPage-gP4WfO_m.js → WrappedPage-pMwbM4R7.js} +1 -1
- package/dashboard/dist/assets/{__vite-browser-external-BtJVmciH.js → __vite-browser-external-DCy_YLfe.js} +3 -3
- package/dashboard/dist/assets/{agent-logos-wxccxfWz.js → agent-logos-Zw7LNkMO.js} +1 -1
- package/dashboard/dist/assets/{arrow-left-Bd5C77R2.js → arrow-left-DycYjE-k.js} +1 -1
- package/dashboard/dist/assets/{arrow-right-C1RqrK4A.js → arrow-right-uNxLHoJ0.js} +1 -1
- package/dashboard/dist/assets/{arrow-up-right-B_b2i8hn.js → arrow-up-right-CmdJ7gd7.js} +1 -1
- package/dashboard/dist/assets/{calendar-days-C8eYygme.js → calendar-days-CJ33_t5O.js} +1 -1
- package/dashboard/dist/assets/{check-BlBHefQ7.js → check-Dbubk4Il.js} +1 -1
- package/dashboard/dist/assets/{chevron-down-mjUspqZv.js → chevron-down-BwvBVAOs.js} +1 -1
- package/dashboard/dist/assets/{cloud-checkout-CEkMnWXf.js → cloud-checkout-IG-IOpeP.js} +1 -1
- package/dashboard/dist/assets/{composite-CzXFY9IC.js → composite-IIgJPPIw.js} +1 -1
- package/dashboard/dist/assets/{download-CQvg59w1.js → download-DxtH7w6a.js} +1 -1
- package/dashboard/dist/assets/{external-link-CDYfIZ3Z.js → external-link-evKf3mnD.js} +1 -1
- package/dashboard/dist/assets/{flame-C9NXQRil.js → flame-D2qOE_W4.js} +1 -1
- package/dashboard/dist/assets/{format-tokens-fKy_3qpW.js → format-tokens-CqDmm-sK.js} +1 -1
- package/dashboard/dist/assets/{gift-WmV4SBGR.js → gift-Ge11HK1x.js} +1 -1
- package/dashboard/dist/assets/{host-mode-C5MvZ7SV.js → host-mode-8_TNh7yE.js} +1 -1
- package/dashboard/dist/assets/{icons-B1EtsaZs.js → icons-OmfWvE5K.js} +1 -1
- package/dashboard/dist/assets/{index-9EZ3o2RV.js → index-BYuEzP9p.js} +1 -1
- package/dashboard/dist/assets/{index-BEecBW06.js → index-BsYtyo4U.js} +1 -1
- package/dashboard/dist/assets/{index-DXNwBFNi.js → index-tbe1aERA.js} +1 -1
- package/dashboard/dist/assets/{info-CWDhggIW.js → info-xhrso9aO.js} +1 -1
- package/dashboard/dist/assets/{link-2-DEWNkK7s.js → link-2-BgNpTaO4.js} +1 -1
- package/dashboard/dist/assets/{loader-circle-DotT5gVp.js → loader-circle-DRooZ0e5.js} +1 -1
- package/dashboard/dist/assets/{modifiers.esm-qZnGgJAz.js → modifiers.esm-D-QAEpOZ.js} +1 -1
- package/dashboard/dist/assets/{monitor-smartphone-DOP5eqBD.js → monitor-smartphone-Bwu7__2B.js} +1 -1
- package/dashboard/dist/assets/{play-BUsybi9m.js → play-CBPvTiWM.js} +1 -1
- package/dashboard/dist/assets/{plus-DAjojh_m.js → plus-B4lFIyP8.js} +1 -1
- package/dashboard/dist/assets/{provider-display-BCQQYPWD.js → provider-display-CQlWusXH.js} +1 -1
- package/dashboard/dist/assets/{react-BgpqhoCj.js → react-DDOtS4Bd.js} +1 -1
- package/dashboard/dist/assets/{safePolygon-C-PFpZSA.js → safePolygon-BGfBVKoy.js} +1 -1
- package/dashboard/dist/assets/{search-W1A-6ugo.js → search-DVyNq72J.js} +1 -1
- package/dashboard/dist/assets/{server-CE7pFjDr.js → server-EpUTEsOf.js} +1 -1
- package/dashboard/dist/assets/{shield-check-KL5xc2FR.js → shield-check-DRg8cPKV.js} +1 -1
- package/dashboard/dist/assets/{skills-api-DL_zLjKA.js → skills-api-y42dEF96.js} +1 -1
- package/dashboard/dist/assets/{sortable.esm-CehJQvFH.js → sortable.esm-D2S1gXCK.js} +1 -1
- package/dashboard/dist/assets/{store-BHAXkHPR.js → store-DfGPS2IR.js} +1 -1
- package/dashboard/dist/assets/{terminal-Br5vyD6Q.js → terminal-hXIrrL2b.js} +1 -1
- package/dashboard/dist/assets/{trash-CzaCJP6G.js → trash-cjaASRQl.js} +1 -1
- package/dashboard/dist/assets/{use-community-stats-DbJrecQw.js → use-community-stats-B-nipjWt.js} +1 -1
- package/dashboard/dist/assets/{use-latest-request-guard-DGhwmlX8.js → use-latest-request-guard-DIEYrSzn.js} +1 -1
- package/dashboard/dist/assets/{use-limits-display-prefs-T7KWAWZ6.js → use-limits-display-prefs-BdHDwBBC.js} +1 -1
- package/dashboard/dist/assets/{use-native-settings-MZJhS328.js → use-native-settings-7w71_h44.js} +1 -1
- package/dashboard/dist/assets/{use-ordered-list-BDN24weA.js → use-ordered-list-CSMpK4ar.js} +1 -1
- package/dashboard/dist/assets/use-reduced-motion-HUGf223R.js +1 -0
- package/dashboard/dist/assets/{use-session-efficiency-pref-DUu16y8Z.js → use-session-efficiency-pref-BagBVWW6.js} +1 -1
- package/dashboard/dist/assets/{use-transform-BczE46xm.js → use-transform-sjGGXCNn.js} +1 -1
- package/dashboard/dist/assets/use-usage-limits-Nml-wUB-.js +1 -0
- package/dashboard/dist/assets/{useCurrency-BgZmWbkx.js → useCurrency-DAKIY1nR.js} +1 -1
- package/dashboard/dist/assets/{usePositioner-dWENyY8z.js → usePositioner-B7vVQ7th.js} +1 -1
- package/dashboard/dist/assets/{useTokenFormat-D7s1-Zuk.js → useTokenFormat-gtgw8k2Z.js} +1 -1
- package/dashboard/dist/assets/{zap-Dr__bKlq.js → zap-DFWOiTJ1.js} +1 -1
- package/dashboard/dist/index.html +1 -1
- package/dashboard/dist/ip-check.html +1 -1
- package/dashboard/dist/leaderboard.html +1 -1
- package/dashboard/dist/share.html +1 -1
- package/package.json +1 -1
- package/src/commands/sync.js +571 -34
- package/src/lib/async-queue-reader.js +37 -16
- package/src/lib/codex-context-breakdown.js +8 -1
- package/src/lib/codex-rollout-parser.js +65 -5
- package/src/lib/grok-limits.js +56 -0
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +690 -60
- package/src/lib/session-analytics.js +1 -1
- package/src/lib/usage-limits.js +33 -1
- package/dashboard/dist/assets/DialogDescription-ne_rYSD7.js +0 -1
- package/dashboard/dist/assets/use-reduced-motion-CfCY-K-a.js +0 -1
- package/dashboard/dist/assets/use-usage-limits-CjyDfH-M.js +0 -1
package/src/commands/sync.js
CHANGED
|
@@ -7,6 +7,7 @@ const readline = require("node:readline");
|
|
|
7
7
|
const { functionUrlFor, fetchFunctionResponse } = require("../lib/function-url");
|
|
8
8
|
const { cloudUsageResetBlocksUpload, finishCloudDeviceUsageReset } = require("../lib/cloud-usage-reset");
|
|
9
9
|
const { KIMI_MODEL_REPAIR_KEY, repairKimiModelAttribution } = require("../lib/kimi-model-repair");
|
|
10
|
+
const { readSqliteJsonRows } = require("../lib/sqlite-reader");
|
|
10
11
|
|
|
11
12
|
const { resolveInstallPaths, resolveZcodeNativeDbPath, resolveMimoNativeDbPath, ensureFlatCursor } = require("../lib/install-resolver");
|
|
12
13
|
const { multiInstallParse, mergeBothFileSources } = require("../lib/multi-install-parser");
|
|
@@ -26,6 +27,7 @@ const { countRecordOnlyFiles, formatRecordOnlyWarning } = require("../lib/codex-
|
|
|
26
27
|
const {
|
|
27
28
|
listRolloutFiles,
|
|
28
29
|
listRolloutFilesDeep,
|
|
30
|
+
readCodexRolloutSnapshot,
|
|
29
31
|
codexSessionIdFromPath,
|
|
30
32
|
filterColdCodexRolloutFiles,
|
|
31
33
|
listClaudeProjectFiles,
|
|
@@ -149,6 +151,7 @@ const {
|
|
|
149
151
|
toUtcHalfHourStart,
|
|
150
152
|
totalsKey,
|
|
151
153
|
claudeMessageDedupKey,
|
|
154
|
+
coerceEpochMs,
|
|
152
155
|
hasZcodeCursorState,
|
|
153
156
|
resolveZcodeInstallPaths,
|
|
154
157
|
} = require("../lib/rollout");
|
|
@@ -311,6 +314,13 @@ const MIMO_PROVIDER_REPAIR_KEY = "mimoClaudeMislabelRepair_2026_06";
|
|
|
311
314
|
const DSH_LEGACY_SOURCE_MIGRATION_KEY = "deepseekHarnessSourceMigration_2026_08";
|
|
312
315
|
const ZCODE_NATIVE_USAGE_REPAIR_KEY = "zcodeNativeUsageRepair_2026_08";
|
|
313
316
|
const ZCODE_INCLUSIVE_TOKEN_REPAIR_KEY = "zcodeInclusiveTokenRepair_2026_09";
|
|
317
|
+
// #828: ZCode CLI prunes model_usage/turn_usage to ~30 days. The wipe+rescan
|
|
318
|
+
// repairs above destroyed pre-window history for late upgraders (and anyone
|
|
319
|
+
// whose prune line advanced before the one-shot key fired). This follow-up
|
|
320
|
+
// restores pre-window peaks from queue.jsonl.bak.* when the live ledger is
|
|
321
|
+
// lower, and the wipe helpers below now preserve hours older than the live
|
|
322
|
+
// rebuild floor so a future re-key cannot repeat the loss.
|
|
323
|
+
const ZCODE_PRUNE_WINDOW_RESTORE_KEY = "zcodePruneWindowRestore_2026_10";
|
|
314
324
|
const AUTO_SYNC_SOURCE_ALIASES = new Map([
|
|
315
325
|
["code", "every-code"],
|
|
316
326
|
["commandcode", "command-code"],
|
|
@@ -1769,6 +1779,8 @@ async function cmdSync(argv, context = {}) {
|
|
|
1769
1779
|
const nativeUsageAvailable = [zcodePaths.native, zcodePaths.wsl]
|
|
1770
1780
|
.filter(Boolean)
|
|
1771
1781
|
.some((dbPath) => hasZcodeNativeUsageSchema(dbPath));
|
|
1782
|
+
const zcodeDbPaths = [zcodePaths.native, zcodePaths.wsl].filter(Boolean);
|
|
1783
|
+
const zcodeRebuildFloorMs = resolveZcodeRebuildFloorMs(zcodeDbPaths);
|
|
1772
1784
|
if (nativeUsageAvailable) {
|
|
1773
1785
|
await repairZcodeNativeUsageMigration({
|
|
1774
1786
|
cursors,
|
|
@@ -1776,6 +1788,7 @@ async function cmdSync(argv, context = {}) {
|
|
|
1776
1788
|
queueStatePath,
|
|
1777
1789
|
projectQueuePath,
|
|
1778
1790
|
projectQueueStatePath,
|
|
1791
|
+
rebuildFloorMs: zcodeRebuildFloorMs,
|
|
1779
1792
|
});
|
|
1780
1793
|
}
|
|
1781
1794
|
await repairZcodeInclusiveTokenMigration({
|
|
@@ -1784,6 +1797,13 @@ async function cmdSync(argv, context = {}) {
|
|
|
1784
1797
|
queueStatePath,
|
|
1785
1798
|
projectQueuePath,
|
|
1786
1799
|
projectQueueStatePath,
|
|
1800
|
+
rebuildFloorMs: zcodeRebuildFloorMs,
|
|
1801
|
+
});
|
|
1802
|
+
await repairZcodePruneWindowRestore({
|
|
1803
|
+
cursors,
|
|
1804
|
+
queuePath,
|
|
1805
|
+
queueStatePath,
|
|
1806
|
+
dbPaths: zcodeDbPaths,
|
|
1787
1807
|
});
|
|
1788
1808
|
zcodeResult = await multiInstallParse({
|
|
1789
1809
|
paths: zcodePaths, parserFn: parseOpencodeDbForInstall, providerName: "zcode",
|
|
@@ -2634,7 +2654,7 @@ async function cmdSync(argv, context = {}) {
|
|
|
2634
2654
|
// usage rides on function_call records too (not only assistant messages) and
|
|
2635
2655
|
// sub-agent logs nest one level deeper, so the resolver recurses. See the
|
|
2636
2656
|
// parser comment in rollout.js for the cache-aware token math.
|
|
2637
|
-
let workbuddyResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
2657
|
+
let workbuddyResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0, projectCursorUpdated: false };
|
|
2638
2658
|
const workbuddyFiles = sourceAllowed("workbuddy")
|
|
2639
2659
|
? mergeBothFileSources({ resolveFiles: resolveWorkbuddyProjectFiles, env: process.env })
|
|
2640
2660
|
: [];
|
|
@@ -2660,6 +2680,7 @@ async function cmdSync(argv, context = {}) {
|
|
|
2660
2680
|
projectFiles: workbuddyFiles,
|
|
2661
2681
|
cursors,
|
|
2662
2682
|
queuePath,
|
|
2683
|
+
projectQueuePath,
|
|
2663
2684
|
env: process.env,
|
|
2664
2685
|
onProgress: (p) => {
|
|
2665
2686
|
if (!progress?.enabled) return;
|
|
@@ -2679,7 +2700,7 @@ async function cmdSync(argv, context = {}) {
|
|
|
2679
2700
|
// only the home directory differs, so it reuses the WorkBuddy parser and
|
|
2680
2701
|
// gets its own source key (and cursor namespace) so the two builds stay
|
|
2681
2702
|
// separable instead of merging into one provider row.
|
|
2682
|
-
let workbuddyAiResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
2703
|
+
let workbuddyAiResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0, projectCursorUpdated: false };
|
|
2683
2704
|
const workbuddyAiFiles = sourceAllowed("workbuddy-ai")
|
|
2684
2705
|
? mergeBothFileSources({
|
|
2685
2706
|
resolveFiles: (resolveEnv) => resolveWorkbuddyProjectFiles(resolveEnv, "workbuddy-ai"),
|
|
@@ -2695,6 +2716,7 @@ async function cmdSync(argv, context = {}) {
|
|
|
2695
2716
|
projectFiles: workbuddyAiFiles,
|
|
2696
2717
|
cursors,
|
|
2697
2718
|
queuePath,
|
|
2719
|
+
projectQueuePath,
|
|
2698
2720
|
env: process.env,
|
|
2699
2721
|
source: "workbuddy-ai",
|
|
2700
2722
|
onProgress: (p) => {
|
|
@@ -3469,6 +3491,10 @@ async function cmdSync(argv, context = {}) {
|
|
|
3469
3491
|
cursorStore.requiresCommit !== true &&
|
|
3470
3492
|
totalParsed === (commandCodeResult.cursorUnchanged ? commandCodeResult.recordsProcessed : 0) &&
|
|
3471
3493
|
totalBuckets === 0 &&
|
|
3494
|
+
!(workbuddyResult.projectBucketsQueued > 0) &&
|
|
3495
|
+
!(workbuddyAiResult.projectBucketsQueued > 0) &&
|
|
3496
|
+
!workbuddyResult.projectCursorUpdated &&
|
|
3497
|
+
!workbuddyAiResult.projectCursorUpdated &&
|
|
3472
3498
|
!(grokResult.projectBucketsQueued > 0) &&
|
|
3473
3499
|
!(commandCodeResult.projectBucketsQueued > 0) &&
|
|
3474
3500
|
!commandCodeResult.schemaMigrated &&
|
|
@@ -3892,6 +3918,9 @@ module.exports = {
|
|
|
3892
3918
|
repairMimoClaudeMislabel,
|
|
3893
3919
|
repairZcodeNativeUsageMigration,
|
|
3894
3920
|
repairZcodeInclusiveTokenMigration,
|
|
3921
|
+
repairZcodePruneWindowRestore,
|
|
3922
|
+
resolveZcodeRebuildFloorMs,
|
|
3923
|
+
ZCODE_PRUNE_WINDOW_RESTORE_KEY,
|
|
3895
3924
|
reincludeClaudeMemObserverFiles,
|
|
3896
3925
|
repairGrokQueueFromSessionSnapshots,
|
|
3897
3926
|
applyCloudConversationsBackfill,
|
|
@@ -4936,17 +4965,194 @@ async function resetUploadOffsetForZcodeRepair(queueStatePath, note) {
|
|
|
4936
4965
|
return true;
|
|
4937
4966
|
}
|
|
4938
4967
|
|
|
4939
|
-
|
|
4968
|
+
function zcodeHourStartMs(hourStart) {
|
|
4969
|
+
if (typeof hourStart !== "string" || !hourStart.trim()) return null;
|
|
4970
|
+
const ms = Date.parse(hourStart);
|
|
4971
|
+
return Number.isFinite(ms) ? ms : null;
|
|
4972
|
+
}
|
|
4973
|
+
|
|
4974
|
+
function zcodeHourFromMainBucketKey(key) {
|
|
4975
|
+
if (typeof key !== "string" || !key.startsWith("zcode|")) return null;
|
|
4976
|
+
const parts = key.split("|");
|
|
4977
|
+
return parts.length >= 3 ? parts.slice(2).join("|") : null;
|
|
4978
|
+
}
|
|
4979
|
+
|
|
4980
|
+
function zcodeHourFromProjectBucketKey(key) {
|
|
4981
|
+
if (typeof key !== "string") return null;
|
|
4982
|
+
const marker = "|zcode|";
|
|
4983
|
+
const idx = key.indexOf(marker);
|
|
4984
|
+
if (idx < 0) return null;
|
|
4985
|
+
const hour = key.slice(idx + marker.length);
|
|
4986
|
+
return hour || null;
|
|
4987
|
+
}
|
|
4988
|
+
|
|
4989
|
+
function zcodeHourFromGroupQueuedKey(key) {
|
|
4990
|
+
if (typeof key !== "string" || !key.startsWith("zcode|")) return null;
|
|
4991
|
+
return key.slice("zcode|".length) || null;
|
|
4992
|
+
}
|
|
4993
|
+
|
|
4994
|
+
function zcodeHourIsPreWindow(hourStart, preserveBeforeMs) {
|
|
4995
|
+
if (!Number.isFinite(preserveBeforeMs)) return false;
|
|
4996
|
+
const hourMs = zcodeHourStartMs(hourStart);
|
|
4997
|
+
return hourMs != null && hourMs < preserveBeforeMs;
|
|
4998
|
+
}
|
|
4999
|
+
|
|
5000
|
+
// Per-install rebuild floor. Prefer native model_usage MIN(started_at) when the
|
|
5001
|
+
// native schema exists — an ancient legacy message does not prove continuous
|
|
5002
|
+
// coverage through pruned native gaps (#828).
|
|
5003
|
+
function resolveZcodeDbRebuildFloorMs(dbPath) {
|
|
5004
|
+
if (!dbPath || !fssync.existsSync(dbPath)) return null;
|
|
5005
|
+
if (hasZcodeNativeUsageSchema(dbPath)) {
|
|
5006
|
+
try {
|
|
5007
|
+
const rows = readSqliteJsonRows(
|
|
5008
|
+
dbPath,
|
|
5009
|
+
`SELECT MIN(started_at) AS floor
|
|
5010
|
+
FROM model_usage
|
|
5011
|
+
WHERE status = 'completed'
|
|
5012
|
+
AND trim(model_id) != ''
|
|
5013
|
+
AND (
|
|
5014
|
+
input_tokens > 0 OR output_tokens > 0 OR reasoning_tokens > 0 OR
|
|
5015
|
+
cache_creation_input_tokens > 0 OR cache_read_input_tokens > 0
|
|
5016
|
+
)`,
|
|
5017
|
+
{ label: "ZCode", timeout: 10_000, maxBuffer: 1024 * 1024, readOnly: true },
|
|
5018
|
+
);
|
|
5019
|
+
const floor = coerceEpochMs(Array.isArray(rows) ? rows[0]?.floor : null);
|
|
5020
|
+
return floor > 0 ? floor : null;
|
|
5021
|
+
} catch (_error) {
|
|
5022
|
+
return null;
|
|
5023
|
+
}
|
|
5024
|
+
}
|
|
5025
|
+
let messages;
|
|
5026
|
+
try {
|
|
5027
|
+
messages = readZcodeDbMessages(dbPath, { readOnly: true });
|
|
5028
|
+
} catch (_error) {
|
|
5029
|
+
return null;
|
|
5030
|
+
}
|
|
5031
|
+
let floor = Number.POSITIVE_INFINITY;
|
|
5032
|
+
for (const message of Array.isArray(messages) ? messages : []) {
|
|
5033
|
+
const ms = coerceEpochMs(
|
|
5034
|
+
message?.timeUpdated ?? message?.data?.time?.completed ?? message?.data?.time?.created,
|
|
5035
|
+
);
|
|
5036
|
+
if (ms > 0) floor = Math.min(floor, ms);
|
|
5037
|
+
}
|
|
5038
|
+
return Number.isFinite(floor) ? floor : null;
|
|
5039
|
+
}
|
|
5040
|
+
|
|
5041
|
+
// Shared wipe window across installs: the LATEST per-install floor (most
|
|
5042
|
+
// conservative). Any listed install that exists but cannot report a floor
|
|
5043
|
+
// defers the wipe — silently taking min() would erase another install's history.
|
|
5044
|
+
function resolveZcodeRebuildFloorMs(dbPaths = []) {
|
|
5045
|
+
const paths = [...new Set((Array.isArray(dbPaths) ? dbPaths : []).filter(Boolean))];
|
|
5046
|
+
if (paths.length === 0) return null;
|
|
5047
|
+
let latestFloor = 0;
|
|
5048
|
+
for (const dbPath of paths) {
|
|
5049
|
+
if (!fssync.existsSync(dbPath)) continue;
|
|
5050
|
+
const floor = resolveZcodeDbRebuildFloorMs(dbPath);
|
|
5051
|
+
if (!Number.isFinite(floor)) return null;
|
|
5052
|
+
latestFloor = Math.max(latestFloor, floor);
|
|
5053
|
+
}
|
|
5054
|
+
return latestFloor > 0 ? latestFloor : null;
|
|
5055
|
+
}
|
|
5056
|
+
|
|
5057
|
+
// Wipe starts at the half-hour bucket that contains the first live message so
|
|
5058
|
+
// that bucket is rebuilt (not preserved) and cannot be double-counted after
|
|
5059
|
+
// cursors.zcode is cleared.
|
|
5060
|
+
function zcodePreserveBeforeMsFromFloor(floorMs) {
|
|
5061
|
+
if (!Number.isFinite(floorMs)) return null;
|
|
5062
|
+
const iso = toUtcHalfHourStart(floorMs);
|
|
5063
|
+
const ms = iso ? Date.parse(iso) : floorMs;
|
|
5064
|
+
return Number.isFinite(ms) ? ms : null;
|
|
5065
|
+
}
|
|
5066
|
+
|
|
5067
|
+
function hasZcodeCursorHistory(cursors) {
|
|
5068
|
+
return hasZcodeCursorState(cursors);
|
|
5069
|
+
}
|
|
5070
|
+
|
|
5071
|
+
function pruneZcodeMessageIndexForRebuild(messages, preserveBeforeMs) {
|
|
5072
|
+
if (!messages || typeof messages !== "object") return 0;
|
|
5073
|
+
let removed = 0;
|
|
5074
|
+
for (const key of Object.keys(messages)) {
|
|
5075
|
+
const entry = messages[key];
|
|
5076
|
+
const attribution = typeof entry?.attribution === "string"
|
|
5077
|
+
? entry.attribution.split(" ")[0]
|
|
5078
|
+
: entry?.attribution?.bucketStart;
|
|
5079
|
+
const bucketStart = typeof attribution === "string" ? attribution : "";
|
|
5080
|
+
// Keep unattributed entries: older cursors omit attribution, and deleting
|
|
5081
|
+
// them re-adds still-present pre-window legacy messages onto preserved
|
|
5082
|
+
// buckets (5000→10000). Prefer a possible in-window under-correction over
|
|
5083
|
+
// permanent double-count (#828).
|
|
5084
|
+
if (!bucketStart) continue;
|
|
5085
|
+
if (zcodeHourIsPreWindow(bucketStart, preserveBeforeMs)) continue;
|
|
5086
|
+
delete messages[key];
|
|
5087
|
+
removed += 1;
|
|
5088
|
+
}
|
|
5089
|
+
return removed;
|
|
5090
|
+
}
|
|
5091
|
+
|
|
5092
|
+
function pruneZcodeMessageCursorForRebuild(cursors, preserveBeforeMs) {
|
|
5093
|
+
if (!cursors || typeof cursors !== "object") return;
|
|
5094
|
+
if (!Number.isFinite(preserveBeforeMs)) {
|
|
5095
|
+
delete cursors.zcode;
|
|
5096
|
+
return;
|
|
5097
|
+
}
|
|
5098
|
+
const root = cursors.zcode;
|
|
5099
|
+
if (!root || typeof root !== "object") {
|
|
5100
|
+
delete cursors.zcode;
|
|
5101
|
+
return;
|
|
5102
|
+
}
|
|
5103
|
+
pruneZcodeMessageIndexForRebuild(root.messages, preserveBeforeMs);
|
|
5104
|
+
for (const nestedKey of ["native", "wsl"]) {
|
|
5105
|
+
const nested = root[nestedKey];
|
|
5106
|
+
if (nested && typeof nested === "object") {
|
|
5107
|
+
pruneZcodeMessageIndexForRebuild(nested.messages, preserveBeforeMs);
|
|
5108
|
+
}
|
|
5109
|
+
}
|
|
5110
|
+
const hasMessages = Object.keys(root.messages || {}).length > 0
|
|
5111
|
+
|| Object.keys(root.native?.messages || {}).length > 0
|
|
5112
|
+
|| Object.keys(root.wsl?.messages || {}).length > 0;
|
|
5113
|
+
if (!hasMessages) delete cursors.zcode;
|
|
5114
|
+
}
|
|
5115
|
+
|
|
5116
|
+
async function queueFileHasZcodeRows(filePath) {
|
|
5117
|
+
if (typeof filePath !== "string" || !filePath) return false;
|
|
4940
5118
|
let raw;
|
|
4941
5119
|
try {
|
|
4942
5120
|
raw = await fs.readFile(filePath, "utf8");
|
|
4943
5121
|
} catch (error) {
|
|
4944
|
-
if (error?.code === "ENOENT") return
|
|
5122
|
+
if (error?.code === "ENOENT") return false;
|
|
5123
|
+
throw error;
|
|
5124
|
+
}
|
|
5125
|
+
for (const line of raw.split("\n")) {
|
|
5126
|
+
if (!line.trim()) continue;
|
|
5127
|
+
try {
|
|
5128
|
+
const row = JSON.parse(line);
|
|
5129
|
+
if (row?.source === "zcode") return true;
|
|
5130
|
+
} catch (_error) {
|
|
5131
|
+
// keep scanning
|
|
5132
|
+
}
|
|
5133
|
+
}
|
|
5134
|
+
return false;
|
|
5135
|
+
}
|
|
5136
|
+
|
|
5137
|
+
function isCompletedZcodeMigration(entry) {
|
|
5138
|
+
return Boolean(entry) && !(typeof entry === "object" && entry.skipped);
|
|
5139
|
+
}
|
|
5140
|
+
|
|
5141
|
+
async function dropZcodeQueueRows(filePath, {
|
|
5142
|
+
retainRetractions = false,
|
|
5143
|
+
preserveBeforeMs = null,
|
|
5144
|
+
} = {}) {
|
|
5145
|
+
let raw;
|
|
5146
|
+
try {
|
|
5147
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
5148
|
+
} catch (error) {
|
|
5149
|
+
if (error?.code === "ENOENT") return { removed: 0, retractions: 0, preserved: 0 };
|
|
4945
5150
|
throw error;
|
|
4946
5151
|
}
|
|
4947
5152
|
const kept = [];
|
|
4948
5153
|
const retractions = new Map();
|
|
4949
5154
|
let removed = 0;
|
|
5155
|
+
let preserved = 0;
|
|
4950
5156
|
for (const line of raw.split("\n")) {
|
|
4951
5157
|
if (!line.trim()) continue;
|
|
4952
5158
|
let row;
|
|
@@ -4957,11 +5163,17 @@ async function dropZcodeQueueRows(filePath, { retainRetractions = false } = {})
|
|
|
4957
5163
|
continue;
|
|
4958
5164
|
}
|
|
4959
5165
|
if (row?.source === "zcode") {
|
|
4960
|
-
removed += 1;
|
|
4961
5166
|
const model = typeof row.model === "string" && row.model.trim()
|
|
4962
5167
|
? row.model.trim()
|
|
4963
5168
|
: "unknown";
|
|
4964
5169
|
const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
|
|
5170
|
+
// #828: keep pre-window rows intact — the live DB cannot rebuild them.
|
|
5171
|
+
if (zcodeHourIsPreWindow(hourStart, preserveBeforeMs)) {
|
|
5172
|
+
kept.push(line);
|
|
5173
|
+
preserved += 1;
|
|
5174
|
+
continue;
|
|
5175
|
+
}
|
|
5176
|
+
removed += 1;
|
|
4965
5177
|
if (retainRetractions && hourStart) {
|
|
4966
5178
|
retractions.set(`${model}|${hourStart}`, {
|
|
4967
5179
|
source: "zcode",
|
|
@@ -4980,7 +5192,7 @@ async function dropZcodeQueueRows(filePath, { retainRetractions = false } = {})
|
|
|
4980
5192
|
}
|
|
4981
5193
|
kept.push(line);
|
|
4982
5194
|
}
|
|
4983
|
-
if (removed === 0) return { removed: 0, retractions: 0 };
|
|
5195
|
+
if (removed === 0) return { removed: 0, retractions: 0, preserved };
|
|
4984
5196
|
await backupExistingFile(filePath);
|
|
4985
5197
|
const tmp = `${filePath}.zcoderepair.${process.pid}.${Date.now()}`;
|
|
4986
5198
|
const rewritten = [
|
|
@@ -4989,7 +5201,7 @@ async function dropZcodeQueueRows(filePath, { retainRetractions = false } = {})
|
|
|
4989
5201
|
];
|
|
4990
5202
|
await fs.writeFile(tmp, rewritten.length ? rewritten.join("\n") + "\n" : "", "utf8");
|
|
4991
5203
|
await fs.rename(tmp, filePath);
|
|
4992
|
-
return { removed, retractions: retractions.size };
|
|
5204
|
+
return { removed, retractions: retractions.size, preserved };
|
|
4993
5205
|
}
|
|
4994
5206
|
|
|
4995
5207
|
async function repairZcodeUsageMigration({
|
|
@@ -5000,25 +5212,52 @@ async function repairZcodeUsageMigration({
|
|
|
5000
5212
|
queueStatePath,
|
|
5001
5213
|
projectQueuePath,
|
|
5002
5214
|
projectQueueStatePath,
|
|
5215
|
+
rebuildFloorMs = null,
|
|
5003
5216
|
} = {}) {
|
|
5004
5217
|
if (!cursors || typeof cursors !== "object") return false;
|
|
5005
5218
|
const migrations = (cursors.migrations ||= {});
|
|
5006
|
-
|
|
5219
|
+
// Mirror Codex: a prior {skipped:true} must retry once the live ledger can
|
|
5220
|
+
// establish a rebuild floor (or the pre-window history is gone).
|
|
5221
|
+
const prior = migrations[migrationKey];
|
|
5222
|
+
if (isCompletedZcodeMigration(prior)) return false;
|
|
5223
|
+
|
|
5224
|
+
const preserveBeforeMs = zcodePreserveBeforeMsFromFloor(rebuildFloorMs);
|
|
5225
|
+
// Empty / unreadable live ledger + existing history => refuse the wipe. The
|
|
5226
|
+
// forward reader still works; only this historical correction defers (#828).
|
|
5227
|
+
if (preserveBeforeMs == null) {
|
|
5228
|
+
const hasQueueHistory = await queueFileHasZcodeRows(queuePath)
|
|
5229
|
+
|| await queueFileHasZcodeRows(projectQueuePath);
|
|
5230
|
+
if (hasZcodeCursorHistory(cursors) || hasQueueHistory) {
|
|
5231
|
+
migrations[migrationKey] = {
|
|
5232
|
+
skipped: true,
|
|
5233
|
+
reason: "zcode_rebuild_floor_unavailable",
|
|
5234
|
+
at: new Date().toISOString(),
|
|
5235
|
+
};
|
|
5236
|
+
return false;
|
|
5237
|
+
}
|
|
5238
|
+
}
|
|
5007
5239
|
|
|
5008
|
-
const mainRepair = await dropZcodeQueueRows(queuePath, {
|
|
5240
|
+
const mainRepair = await dropZcodeQueueRows(queuePath, {
|
|
5241
|
+
retainRetractions: true,
|
|
5242
|
+
preserveBeforeMs,
|
|
5243
|
+
});
|
|
5009
5244
|
const projectRepair = projectQueuePath
|
|
5010
|
-
? await dropZcodeQueueRows(projectQueuePath)
|
|
5011
|
-
: { removed: 0, retractions: 0 };
|
|
5245
|
+
? await dropZcodeQueueRows(projectQueuePath, { preserveBeforeMs })
|
|
5246
|
+
: { removed: 0, retractions: 0, preserved: 0 };
|
|
5012
5247
|
|
|
5013
5248
|
const hourly = cursors.hourly && typeof cursors.hourly === "object" ? cursors.hourly : null;
|
|
5014
5249
|
if (hourly?.buckets) {
|
|
5015
5250
|
for (const key of Object.keys(hourly.buckets)) {
|
|
5016
|
-
if (key.startsWith("zcode|"))
|
|
5251
|
+
if (!key.startsWith("zcode|")) continue;
|
|
5252
|
+
if (zcodeHourIsPreWindow(zcodeHourFromMainBucketKey(key), preserveBeforeMs)) continue;
|
|
5253
|
+
delete hourly.buckets[key];
|
|
5017
5254
|
}
|
|
5018
5255
|
}
|
|
5019
5256
|
if (hourly?.groupQueued) {
|
|
5020
5257
|
for (const key of Object.keys(hourly.groupQueued)) {
|
|
5021
|
-
if (key.startsWith("zcode|"))
|
|
5258
|
+
if (!key.startsWith("zcode|")) continue;
|
|
5259
|
+
if (zcodeHourIsPreWindow(zcodeHourFromGroupQueuedKey(key), preserveBeforeMs)) continue;
|
|
5260
|
+
delete hourly.groupQueued[key];
|
|
5022
5261
|
}
|
|
5023
5262
|
}
|
|
5024
5263
|
const projectHourly = cursors.projectHourly && typeof cursors.projectHourly === "object"
|
|
@@ -5026,10 +5265,15 @@ async function repairZcodeUsageMigration({
|
|
|
5026
5265
|
: null;
|
|
5027
5266
|
if (projectHourly?.buckets) {
|
|
5028
5267
|
for (const key of Object.keys(projectHourly.buckets)) {
|
|
5029
|
-
if (key.includes("|zcode|"))
|
|
5268
|
+
if (!key.includes("|zcode|")) continue;
|
|
5269
|
+
if (zcodeHourIsPreWindow(zcodeHourFromProjectBucketKey(key), preserveBeforeMs)) continue;
|
|
5270
|
+
delete projectHourly.buckets[key];
|
|
5030
5271
|
}
|
|
5031
5272
|
}
|
|
5032
|
-
|
|
5273
|
+
// Keep pre-window message dedup so preserved buckets are not double-counted
|
|
5274
|
+
// when legacy rows still exist in the live DB. Clear only in-window (and
|
|
5275
|
+
// unattributed) cursor entries so the rescan can rebuild those hours.
|
|
5276
|
+
pruneZcodeMessageCursorForRebuild(cursors, preserveBeforeMs);
|
|
5033
5277
|
|
|
5034
5278
|
if (mainRepair.removed > 0) {
|
|
5035
5279
|
await resetUploadOffsetForZcodeRepair(queueStatePath, queueStateNote);
|
|
@@ -5042,6 +5286,236 @@ async function repairZcodeUsageMigration({
|
|
|
5042
5286
|
removedMain: mainRepair.removed,
|
|
5043
5287
|
removedProject: projectRepair.removed,
|
|
5044
5288
|
retractions: mainRepair.retractions,
|
|
5289
|
+
preservedMain: mainRepair.preserved || 0,
|
|
5290
|
+
preservedProject: projectRepair.preserved || 0,
|
|
5291
|
+
rebuildFloorMs: preserveBeforeMs,
|
|
5292
|
+
};
|
|
5293
|
+
return true;
|
|
5294
|
+
}
|
|
5295
|
+
|
|
5296
|
+
function zcodeRowTotals(row) {
|
|
5297
|
+
return {
|
|
5298
|
+
input_tokens: Number(row?.input_tokens) || 0,
|
|
5299
|
+
cached_input_tokens: Number(row?.cached_input_tokens) || 0,
|
|
5300
|
+
cache_creation_input_tokens: Number(row?.cache_creation_input_tokens) || 0,
|
|
5301
|
+
output_tokens: Number(row?.output_tokens) || 0,
|
|
5302
|
+
reasoning_output_tokens: Number(row?.reasoning_output_tokens) || 0,
|
|
5303
|
+
total_tokens: Number(row?.total_tokens) || 0,
|
|
5304
|
+
conversation_count: Number(row?.conversation_count) || 0,
|
|
5305
|
+
};
|
|
5306
|
+
}
|
|
5307
|
+
|
|
5308
|
+
function zcodeTotalsDominate(candidate, baseline) {
|
|
5309
|
+
if (!candidate) return false;
|
|
5310
|
+
if (!baseline) return (candidate.total_tokens || 0) > 0;
|
|
5311
|
+
return (candidate.total_tokens || 0) > (baseline.total_tokens || 0);
|
|
5312
|
+
}
|
|
5313
|
+
|
|
5314
|
+
async function collectZcodeQueuePeaks(filePath, { beforeMs = null } = {}) {
|
|
5315
|
+
const peaks = new Map();
|
|
5316
|
+
let raw;
|
|
5317
|
+
try {
|
|
5318
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
5319
|
+
} catch (error) {
|
|
5320
|
+
if (error?.code === "ENOENT") return peaks;
|
|
5321
|
+
throw error;
|
|
5322
|
+
}
|
|
5323
|
+
for (const line of raw.split("\n")) {
|
|
5324
|
+
if (!line.trim()) continue;
|
|
5325
|
+
let row;
|
|
5326
|
+
try {
|
|
5327
|
+
row = JSON.parse(line);
|
|
5328
|
+
} catch (_error) {
|
|
5329
|
+
continue;
|
|
5330
|
+
}
|
|
5331
|
+
if (row?.source !== "zcode") continue;
|
|
5332
|
+
const model = typeof row.model === "string" && row.model.trim()
|
|
5333
|
+
? row.model.trim()
|
|
5334
|
+
: "unknown";
|
|
5335
|
+
const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
|
|
5336
|
+
if (!hourStart) continue;
|
|
5337
|
+
if (beforeMs != null && !zcodeHourIsPreWindow(hourStart, beforeMs)) continue;
|
|
5338
|
+
const key = `${model}|${hourStart}`;
|
|
5339
|
+
const totals = zcodeRowTotals(row);
|
|
5340
|
+
const prev = peaks.get(key);
|
|
5341
|
+
if (!prev || zcodeTotalsDominate(totals, prev.totals)) {
|
|
5342
|
+
peaks.set(key, { model, hour_start: hourStart, totals });
|
|
5343
|
+
}
|
|
5344
|
+
}
|
|
5345
|
+
return peaks;
|
|
5346
|
+
}
|
|
5347
|
+
|
|
5348
|
+
// Last-wins per key (cloud replay semantics). A historical peak of 9000 followed
|
|
5349
|
+
// by a repair retraction of 0 must report 0 so restore still re-appends.
|
|
5350
|
+
async function collectZcodeQueueLatest(filePath, { beforeMs = null } = {}) {
|
|
5351
|
+
const latest = new Map();
|
|
5352
|
+
let raw;
|
|
5353
|
+
try {
|
|
5354
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
5355
|
+
} catch (error) {
|
|
5356
|
+
if (error?.code === "ENOENT") return latest;
|
|
5357
|
+
throw error;
|
|
5358
|
+
}
|
|
5359
|
+
for (const line of raw.split("\n")) {
|
|
5360
|
+
if (!line.trim()) continue;
|
|
5361
|
+
let row;
|
|
5362
|
+
try {
|
|
5363
|
+
row = JSON.parse(line);
|
|
5364
|
+
} catch (_error) {
|
|
5365
|
+
continue;
|
|
5366
|
+
}
|
|
5367
|
+
if (row?.source !== "zcode") continue;
|
|
5368
|
+
const model = typeof row.model === "string" && row.model.trim()
|
|
5369
|
+
? row.model.trim()
|
|
5370
|
+
: "unknown";
|
|
5371
|
+
const hourStart = typeof row.hour_start === "string" ? row.hour_start : "";
|
|
5372
|
+
if (!hourStart) continue;
|
|
5373
|
+
if (beforeMs != null && !zcodeHourIsPreWindow(hourStart, beforeMs)) continue;
|
|
5374
|
+
latest.set(`${model}|${hourStart}`, {
|
|
5375
|
+
model,
|
|
5376
|
+
hour_start: hourStart,
|
|
5377
|
+
totals: zcodeRowTotals(row),
|
|
5378
|
+
});
|
|
5379
|
+
}
|
|
5380
|
+
return latest;
|
|
5381
|
+
}
|
|
5382
|
+
|
|
5383
|
+
async function listZcodeQueueBackupPaths(queuePath) {
|
|
5384
|
+
if (typeof queuePath !== "string" || !queuePath) return [];
|
|
5385
|
+
const dir = path.dirname(queuePath);
|
|
5386
|
+
const base = path.basename(queuePath);
|
|
5387
|
+
let names;
|
|
5388
|
+
try {
|
|
5389
|
+
names = await fs.readdir(dir);
|
|
5390
|
+
} catch (error) {
|
|
5391
|
+
if (error?.code === "ENOENT") return [];
|
|
5392
|
+
throw error;
|
|
5393
|
+
}
|
|
5394
|
+
return names
|
|
5395
|
+
.filter((name) => name.startsWith(`${base}.bak.`))
|
|
5396
|
+
.map((name) => path.join(dir, name))
|
|
5397
|
+
.sort();
|
|
5398
|
+
}
|
|
5399
|
+
|
|
5400
|
+
async function repairZcodePruneWindowRestore({
|
|
5401
|
+
cursors,
|
|
5402
|
+
queuePath,
|
|
5403
|
+
queueStatePath,
|
|
5404
|
+
dbPaths = [],
|
|
5405
|
+
rebuildFloorMs: rebuildFloorMsOption = null,
|
|
5406
|
+
} = {}) {
|
|
5407
|
+
if (!cursors || typeof cursors !== "object") return false;
|
|
5408
|
+
const migrations = (cursors.migrations ||= {});
|
|
5409
|
+
const prior = migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY];
|
|
5410
|
+
if (isCompletedZcodeMigration(prior)) return false;
|
|
5411
|
+
|
|
5412
|
+
const rebuildFloorMs = Number.isFinite(rebuildFloorMsOption)
|
|
5413
|
+
? rebuildFloorMsOption
|
|
5414
|
+
: resolveZcodeRebuildFloorMs(dbPaths);
|
|
5415
|
+
if (!Number.isFinite(rebuildFloorMs)) {
|
|
5416
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
5417
|
+
skipped: true,
|
|
5418
|
+
reason: "zcode_rebuild_floor_unavailable",
|
|
5419
|
+
at: new Date().toISOString(),
|
|
5420
|
+
};
|
|
5421
|
+
return false;
|
|
5422
|
+
}
|
|
5423
|
+
|
|
5424
|
+
const backupPaths = await listZcodeQueueBackupPaths(queuePath);
|
|
5425
|
+
if (backupPaths.length === 0) {
|
|
5426
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
5427
|
+
skipped: true,
|
|
5428
|
+
reason: "zcode_queue_backup_missing",
|
|
5429
|
+
at: new Date().toISOString(),
|
|
5430
|
+
rebuildFloorMs,
|
|
5431
|
+
};
|
|
5432
|
+
return false;
|
|
5433
|
+
}
|
|
5434
|
+
|
|
5435
|
+
const preserveBeforeMs = zcodePreserveBeforeMsFromFloor(rebuildFloorMs);
|
|
5436
|
+
const backupPeaks = new Map();
|
|
5437
|
+
for (const backupPath of backupPaths) {
|
|
5438
|
+
const peaks = await collectZcodeQueuePeaks(backupPath, { beforeMs: preserveBeforeMs });
|
|
5439
|
+
for (const [key, peak] of peaks) {
|
|
5440
|
+
const prev = backupPeaks.get(key);
|
|
5441
|
+
if (!prev || zcodeTotalsDominate(peak.totals, prev.totals)) {
|
|
5442
|
+
backupPeaks.set(key, peak);
|
|
5443
|
+
}
|
|
5444
|
+
}
|
|
5445
|
+
}
|
|
5446
|
+
if (backupPeaks.size === 0) {
|
|
5447
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
5448
|
+
appliedAt: new Date().toISOString(),
|
|
5449
|
+
restored: 0,
|
|
5450
|
+
rebuildFloorMs,
|
|
5451
|
+
preserveBeforeMs,
|
|
5452
|
+
note: "no_pre_window_peaks_in_backups",
|
|
5453
|
+
};
|
|
5454
|
+
return true;
|
|
5455
|
+
}
|
|
5456
|
+
|
|
5457
|
+
const liveLatest = await collectZcodeQueueLatest(queuePath, { beforeMs: preserveBeforeMs });
|
|
5458
|
+
const hourly = (cursors.hourly ||= { buckets: {}, groupQueued: {} });
|
|
5459
|
+
hourly.buckets ||= {};
|
|
5460
|
+
const restorations = [];
|
|
5461
|
+
for (const [key, peak] of backupPeaks) {
|
|
5462
|
+
const bucketKeyName = bucketKey("zcode", peak.model, peak.hour_start);
|
|
5463
|
+
const live = liveLatest.get(key)?.totals || null;
|
|
5464
|
+
// Compare against the latest queue emission (last-wins), not historical peaks
|
|
5465
|
+
// or cursor buckets — a 9000→0 retraction must still restore.
|
|
5466
|
+
if (!zcodeTotalsDominate(peak.totals, live)) {
|
|
5467
|
+
const bucketTotals = hourly.buckets[bucketKeyName]?.totals
|
|
5468
|
+
? zcodeRowTotals(hourly.buckets[bucketKeyName].totals)
|
|
5469
|
+
: null;
|
|
5470
|
+
if (bucketTotals && zcodeTotalsDominate(peak.totals, bucketTotals)) {
|
|
5471
|
+
hourly.buckets[bucketKeyName] = {
|
|
5472
|
+
...(hourly.buckets[bucketKeyName] || {}),
|
|
5473
|
+
totals: { ...peak.totals },
|
|
5474
|
+
hour_start: peak.hour_start,
|
|
5475
|
+
source: "zcode",
|
|
5476
|
+
model: peak.model,
|
|
5477
|
+
};
|
|
5478
|
+
}
|
|
5479
|
+
continue;
|
|
5480
|
+
}
|
|
5481
|
+
restorations.push(peak);
|
|
5482
|
+
hourly.buckets[bucketKeyName] = {
|
|
5483
|
+
...(hourly.buckets[bucketKeyName] || {}),
|
|
5484
|
+
totals: { ...peak.totals },
|
|
5485
|
+
hour_start: peak.hour_start,
|
|
5486
|
+
source: "zcode",
|
|
5487
|
+
model: peak.model,
|
|
5488
|
+
};
|
|
5489
|
+
}
|
|
5490
|
+
|
|
5491
|
+
if (restorations.length === 0) {
|
|
5492
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
5493
|
+
appliedAt: new Date().toISOString(),
|
|
5494
|
+
restored: 0,
|
|
5495
|
+
rebuildFloorMs,
|
|
5496
|
+
backupsConsidered: backupPaths.length,
|
|
5497
|
+
note: "backup_peaks_not_higher",
|
|
5498
|
+
};
|
|
5499
|
+
return true;
|
|
5500
|
+
}
|
|
5501
|
+
|
|
5502
|
+
await backupExistingFile(queuePath);
|
|
5503
|
+
const append = restorations.map((peak) => JSON.stringify({
|
|
5504
|
+
source: "zcode",
|
|
5505
|
+
model: peak.model,
|
|
5506
|
+
hour_start: peak.hour_start,
|
|
5507
|
+
...peak.totals,
|
|
5508
|
+
}));
|
|
5509
|
+
await fs.appendFile(queuePath, `${append.join("\n")}\n`, "utf8");
|
|
5510
|
+
await resetUploadOffsetForZcodeRepair(
|
|
5511
|
+
queueStatePath,
|
|
5512
|
+
"reset_after_zcode_prune_window_restore_2026_10",
|
|
5513
|
+
);
|
|
5514
|
+
migrations[ZCODE_PRUNE_WINDOW_RESTORE_KEY] = {
|
|
5515
|
+
appliedAt: new Date().toISOString(),
|
|
5516
|
+
restored: restorations.length,
|
|
5517
|
+
rebuildFloorMs,
|
|
5518
|
+
backupsConsidered: backupPaths.length,
|
|
5045
5519
|
};
|
|
5046
5520
|
return true;
|
|
5047
5521
|
}
|
|
@@ -5699,7 +6173,23 @@ async function repairWorkbuddyContextUsage({
|
|
|
5699
6173
|
for (const [key, value] of Object.entries(tmpCursors.hourly.groupQueued || {})) {
|
|
5700
6174
|
if (key.startsWith("workbuddy|")) nextHourly.groupQueued[key] = value;
|
|
5701
6175
|
}
|
|
5702
|
-
|
|
6176
|
+
// This migration rebuilds totals from a temp queue using empty cursors, so
|
|
6177
|
+
// `tmpCursors.workbuddy` carries no project state. Replacing the whole object
|
|
6178
|
+
// would drop the project cursor while leaving the published project buckets
|
|
6179
|
+
// (cursors.projectHourly and the real project queue) untouched, and the next
|
|
6180
|
+
// parse would re-attribute every historical session on top of them. The
|
|
6181
|
+
// migration only changes totals, so keep the project cursor as it is.
|
|
6182
|
+
const previousWorkbuddyState =
|
|
6183
|
+
cursors.workbuddy && typeof cursors.workbuddy === "object" ? cursors.workbuddy : {};
|
|
6184
|
+
cursors.workbuddy = {
|
|
6185
|
+
...(tmpCursors.workbuddy || {}),
|
|
6186
|
+
...(previousWorkbuddyState.projectFileOffsets
|
|
6187
|
+
? { projectFileOffsets: previousWorkbuddyState.projectFileOffsets }
|
|
6188
|
+
: {}),
|
|
6189
|
+
...(Array.isArray(previousWorkbuddyState.projectSeenIds)
|
|
6190
|
+
? { projectSeenIds: previousWorkbuddyState.projectSeenIds }
|
|
6191
|
+
: {}),
|
|
6192
|
+
};
|
|
5703
6193
|
|
|
5704
6194
|
let uploadState = {};
|
|
5705
6195
|
try { uploadState = JSON.parse(await fs.readFile(queueStatePath, "utf8")); } catch (_e) {}
|
|
@@ -6284,26 +6774,56 @@ async function scanCodexUsageLineages(filePath, maxBytes = Infinity) {
|
|
|
6284
6774
|
const byteLimit = Number.isFinite(maxBytes) ? Math.max(0, maxBytes) : Infinity;
|
|
6285
6775
|
let bytesRead = 0;
|
|
6286
6776
|
let affected = false;
|
|
6287
|
-
|
|
6288
|
-
|
|
6289
|
-
|
|
6290
|
-
|
|
6291
|
-
|
|
6292
|
-
|
|
6293
|
-
if (bytesRead > byteLimit) return { affected: false, indeterminate: true };
|
|
6294
|
-
if (!line.trim()) continue;
|
|
6295
|
-
let obj;
|
|
6296
|
-
try {
|
|
6297
|
-
obj = JSON.parse(line);
|
|
6298
|
-
} catch (_e) {
|
|
6777
|
+
const isCompressed = typeof filePath === "string" && /\.jsonl\.zstd?$/i.test(filePath);
|
|
6778
|
+
if (isCompressed) {
|
|
6779
|
+
// Honor the background-sync lineage budget against compressed size so a
|
|
6780
|
+
// pending repair with large archives stays bounded and returns
|
|
6781
|
+
// indeterminate until a full sync can finish the scan.
|
|
6782
|
+
if (!(byteLimit > 0) || before.size > byteLimit) {
|
|
6299
6783
|
return { affected: false, indeterminate: true };
|
|
6300
6784
|
}
|
|
6301
|
-
const
|
|
6302
|
-
|
|
6303
|
-
|
|
6304
|
-
|
|
6305
|
-
|
|
6306
|
-
|
|
6785
|
+
const snapshot = await readCodexRolloutSnapshot(filePath);
|
|
6786
|
+
if (!snapshot) return { affected: false, indeterminate: true };
|
|
6787
|
+
for (const raw of snapshot.lines) {
|
|
6788
|
+
const line = typeof raw === "string" ? raw : "";
|
|
6789
|
+
if (!line.trim()) continue;
|
|
6790
|
+
let obj;
|
|
6791
|
+
try {
|
|
6792
|
+
obj = JSON.parse(line);
|
|
6793
|
+
} catch (_e) {
|
|
6794
|
+
return { affected: false, indeterminate: true };
|
|
6795
|
+
}
|
|
6796
|
+
const token = extractTokenCount(obj);
|
|
6797
|
+
const info = token?.info;
|
|
6798
|
+
if (!info || typeof info !== "object") continue;
|
|
6799
|
+
consumeUsageDelta(state, info.last_token_usage, info.total_token_usage);
|
|
6800
|
+
if (state.sawInterleaved || state.sawDivergentCumulative) {
|
|
6801
|
+
affected = true;
|
|
6802
|
+
}
|
|
6803
|
+
}
|
|
6804
|
+
bytesRead = snapshot.size;
|
|
6805
|
+
} else {
|
|
6806
|
+
stream = fssync.createReadStream(filePath, { highWaterMark: 32 * 1024 });
|
|
6807
|
+
for await (const record of physicalJsonlRecords(stream, {
|
|
6808
|
+
maxPhysicalBytes: byteLimit,
|
|
6809
|
+
})) {
|
|
6810
|
+
const { line } = record;
|
|
6811
|
+
bytesRead += record.physicalBytes;
|
|
6812
|
+
if (bytesRead > byteLimit) return { affected: false, indeterminate: true };
|
|
6813
|
+
if (!line.trim()) continue;
|
|
6814
|
+
let obj;
|
|
6815
|
+
try {
|
|
6816
|
+
obj = JSON.parse(line);
|
|
6817
|
+
} catch (_e) {
|
|
6818
|
+
return { affected: false, indeterminate: true };
|
|
6819
|
+
}
|
|
6820
|
+
const token = extractTokenCount(obj);
|
|
6821
|
+
const info = token?.info;
|
|
6822
|
+
if (!info || typeof info !== "object") continue;
|
|
6823
|
+
consumeUsageDelta(state, info.last_token_usage, info.total_token_usage);
|
|
6824
|
+
if (state.sawInterleaved || state.sawDivergentCumulative) {
|
|
6825
|
+
affected = true;
|
|
6826
|
+
}
|
|
6307
6827
|
}
|
|
6308
6828
|
}
|
|
6309
6829
|
const after = await readCodexFileSnapshot(filePath);
|
|
@@ -6408,6 +6928,23 @@ async function scanForForkedCodexRollout(rolloutFiles) {
|
|
|
6408
6928
|
if (!fp || src !== "codex") continue;
|
|
6409
6929
|
let fh = null;
|
|
6410
6930
|
try {
|
|
6931
|
+
if (typeof fp === "string" && /\.jsonl\.zstd?$/i.test(fp)) {
|
|
6932
|
+
const snapshot = await readCodexRolloutSnapshot(fp);
|
|
6933
|
+
if (!snapshot) {
|
|
6934
|
+
indeterminate = true;
|
|
6935
|
+
continue;
|
|
6936
|
+
}
|
|
6937
|
+
// Fork marker is in early session_meta; scan a bounded prefix of lines.
|
|
6938
|
+
let head = "";
|
|
6939
|
+
for (const raw of snapshot.lines) {
|
|
6940
|
+
head += (typeof raw === "string" ? raw : "") + "\n";
|
|
6941
|
+
if (head.length >= HEAD_BYTES) break;
|
|
6942
|
+
}
|
|
6943
|
+
if (head.includes('"forked_from_id"')) {
|
|
6944
|
+
return { forked: true, indeterminate: false };
|
|
6945
|
+
}
|
|
6946
|
+
continue;
|
|
6947
|
+
}
|
|
6411
6948
|
fh = await fs.open(fp, "r");
|
|
6412
6949
|
const buf = Buffer.alloc(HEAD_BYTES);
|
|
6413
6950
|
const { bytesRead } = await fh.read(buf, 0, HEAD_BYTES, 0);
|