tokentracker-cli 0.96.1 → 0.96.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +34 -25
- package/README.ja.md +38 -27
- package/README.ko.md +39 -28
- package/README.md +36 -28
- package/README.zh-CN.md +36 -25
- package/dashboard/dist/assets/{AchievementBadge-DKyElm0T.js → AchievementBadge-BX4iBBYM.js} +1 -1
- package/dashboard/dist/assets/{AchievementsPage-BvCx3wND.js → AchievementsPage-DRYRuCbh.js} +2 -2
- package/dashboard/dist/assets/{AchievementsSection-DN69LzXX.js → AchievementsSection-BM_JWWhJ.js} +1 -1
- package/dashboard/dist/assets/{BadgeDetailModal-DGiKdZfz.js → BadgeDetailModal-DxoxXst2.js} +1 -1
- package/dashboard/dist/assets/{BotAnimated-zN8rH7_K.js → BotAnimated-BVpFxG8u.js} +1 -1
- package/dashboard/dist/assets/{Card-COWo-ClV.js → Card-BJJs701t.js} +1 -1
- package/dashboard/dist/assets/{ClawdAnimated-BWsB7Eat.js → ClawdAnimated-46wPZ_Tc.js} +2 -2
- package/dashboard/dist/assets/{CommandPalette-BfvIwuoJ.js → CommandPalette-B_zkc7IU.js} +1 -1
- package/dashboard/dist/assets/{CommunityStatsModal-BQWg7R_y.js → CommunityStatsModal-M2k5rIf1.js} +1 -1
- package/dashboard/dist/assets/{ConfirmModal-C7ABSI-O.js → ConfirmModal-DBvOTcAQ.js} +1 -1
- package/dashboard/dist/assets/{Controls-vMzEZzej.js → Controls-BCkjPMSH.js} +1 -1
- package/dashboard/dist/assets/DashboardPage-3bHhou09.js +1 -0
- package/dashboard/dist/assets/{DevicePage-3ejUBLWM.js → DevicePage-DFBUce8C.js} +1 -1
- package/dashboard/dist/assets/DialogClose-kvDd8h3V.js +1 -0
- package/dashboard/dist/assets/DialogDescription-BMyOicfz.js +1 -0
- package/dashboard/dist/assets/DialogTitle-DL5nvD31.js +1 -0
- package/dashboard/dist/assets/{FadeIn-D7Jcfz1N.js → FadeIn-B-Bg6SEy.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-D--Rx5BS.js → HeaderGithubStar-BExz0hh4.js} +1 -1
- package/dashboard/dist/assets/{HoverTooltip-n1W0z3Wr.js → HoverTooltip-Dj5zemYT.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-D202tl9B.js → IpCheckPage-DN7FuJ_v.js} +1 -1
- package/dashboard/dist/assets/{LandingPage-D6yL2EZo.js → LandingPage-9dKgSBRy.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardAvatar-ChFg2mtA.js → LeaderboardAvatar-00V95XIg.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-BXHwGs8r.js → LeaderboardPage-DkKfUt0Z.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProfileModal-B5LoOpCk.js → LeaderboardProfileModal-C9sIHBgN.js} +4 -4
- package/dashboard/dist/assets/{LeaderboardProfilePage-jsukhRz8.js → LeaderboardProfilePage-BAiLMsEb.js} +1 -1
- package/dashboard/dist/assets/LimitsPage-BuenTaHa.js +8 -0
- package/dashboard/dist/assets/{LocalOnlyNotice-Du0lz657.js → LocalOnlyNotice-CKhY27Lr.js} +1 -1
- package/dashboard/dist/assets/{LoginCard-BFEX9x7-.js → LoginCard-BbvoBx5N.js} +1 -1
- package/dashboard/dist/assets/{LoginModal-DqC7_d35.js → LoginModal-P7zIkBhO.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-Dzx8N7HE.js → LoginPage-Bo0bR26x.js} +1 -1
- package/dashboard/dist/assets/PetPage-C6i9bSSl.js +1 -0
- package/dashboard/dist/assets/PopoverPopup-h0gf36vg.js +1 -0
- package/dashboard/dist/assets/ProviderIcon-Dm8fwPdj.js +1 -0
- package/dashboard/dist/assets/{ResetPasswordPage-x_GR79sR.js → ResetPasswordPage-DvSJ8kg0.js} +1 -1
- package/dashboard/dist/assets/{SegmentedControl-BxDiDV4R.js → SegmentedControl-CP7oiGZE.js} +1 -1
- package/dashboard/dist/assets/Select-iwP92x9C.js +1 -0
- package/dashboard/dist/assets/SelectItemText-DzljA15_.js +1 -0
- package/dashboard/dist/assets/{ServiceStatusPage-BVQuVdjg.js → ServiceStatusPage-uD4D0Tid.js} +1 -1
- package/dashboard/dist/assets/{SessionsPage-DE39wNIO.js → SessionsPage-Cmb1ssNP.js} +2 -2
- package/dashboard/dist/assets/SettingsPage-D8ryg1Pv.js +2 -0
- package/dashboard/dist/assets/SkillsPage-eXI09q03.js +1 -0
- package/dashboard/dist/assets/{TokenGalaxy-BPM779Wb.js → TokenGalaxy-f2djrlUv.js} +1 -1
- package/dashboard/dist/assets/ToolbarRootContext-CDxCPYLv.js +1 -0
- package/dashboard/dist/assets/{TrendMonitor-BkNy-o1D.js → TrendMonitor-C419X22G.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-Bpg6H-ZB.js → WidgetsPage-AYq__pOO.js} +1 -1
- package/dashboard/dist/assets/{WrappedPage-C6-NM-RO.js → WrappedPage-Bsm9k5XI.js} +1 -1
- package/dashboard/dist/assets/{__vite-browser-external-CvMMIjhi.css → __vite-browser-external-5P7me0Qq.css} +1 -1
- package/dashboard/dist/assets/__vite-browser-external-BDkeh90N.js +45 -0
- package/dashboard/dist/assets/{agent-logos-DpbkRqXT.js → agent-logos-B6Vg-4_1.js} +1 -1
- package/dashboard/dist/assets/{arrow-up-right-BjOtWUVs.js → arrow-up-right-DIQXwaOD.js} +1 -1
- package/dashboard/dist/assets/{check-BlxnX5jZ.js → check-CgD_bV39.js} +1 -1
- package/dashboard/dist/assets/composite-BIgEqWZD.js +1 -0
- package/dashboard/dist/assets/{copy-CIsWjLY5.js → copy-DlM1BHdp.js} +1 -1
- package/dashboard/dist/assets/{download-QDaQbQdd.js → download-CQFG-VIY.js} +1 -1
- package/dashboard/dist/assets/{external-link-DPV4VU3m.js → external-link-BQyDqGxz.js} +1 -1
- package/dashboard/dist/assets/{flame-B_X-7VNp.js → flame-BamzyXzF.js} +1 -1
- package/dashboard/dist/assets/{format-tokens-DEx2TJj8.js → format-tokens-C4G-_xYt.js} +1 -1
- package/dashboard/dist/assets/{icons-983RyGSl.js → icons-Ntscvl1a.js} +1 -1
- package/dashboard/dist/assets/{index-qtonw_x4.js → index-CUrcPqgp.js} +1 -1
- package/dashboard/dist/assets/{index-BONYA8HZ.js → index-DGxgsNoQ.js} +1 -1
- package/dashboard/dist/assets/{index-Cjs5stC0.js → index-FRIt5mlv.js} +1 -1
- package/dashboard/dist/assets/{info-DpVOhXZI.js → info-CXytWcZi.js} +1 -1
- package/dashboard/dist/assets/{link-2-CBro-upz.js → link-2-Dexoo_jO.js} +1 -1
- package/dashboard/dist/assets/{loader-circle-wYFdSyOd.js → loader-circle-Cn0JHc8t.js} +1 -1
- package/dashboard/dist/assets/{maximize-2-DhTyQ4ly.js → maximize-2-DLh7gYuP.js} +1 -1
- package/dashboard/dist/assets/{modifiers.esm-CPljwVhN.js → modifiers.esm-DOh_xtyt.js} +1 -1
- package/dashboard/dist/assets/module-DUSXqJtA.js +3 -0
- package/dashboard/dist/assets/{monitor-smartphone-D_wr-HOR.js → monitor-smartphone-CpqS9Ior.js} +1 -1
- package/dashboard/dist/assets/{plus-BNjfkBEF.js → plus-bfQV7nyv.js} +1 -1
- package/dashboard/dist/assets/provider-display-Bf9pW06U.js +42 -0
- package/dashboard/dist/assets/{react-yHv9MTUA.js → react-BmFZKA_b.js} +1 -1
- package/dashboard/dist/assets/{search-Dfak92ZA.js → search-l3jJ2fzC.js} +1 -1
- package/dashboard/dist/assets/{skills-api-By2PdNa1.js → skills-api-CWmLnaxn.js} +1 -1
- package/dashboard/dist/assets/{sortable.esm-B9xy66_v.js → sortable.esm-DLgyXLHM.js} +1 -1
- package/dashboard/dist/assets/store-JM-a8ElZ.js +1 -0
- package/dashboard/dist/assets/{terminal-pbXWBmZM.js → terminal-6aY2Sn8P.js} +1 -1
- package/dashboard/dist/assets/{trash-2-CboDPwbW.js → trash-D5oon5pT.js} +1 -1
- package/dashboard/dist/assets/{use-community-stats-D6HTdnHq.js → use-community-stats-ZWUMFwzc.js} +1 -1
- package/dashboard/dist/assets/{use-latest-request-guard-hwSr9Cam.js → use-latest-request-guard-BDaXr4gK.js} +1 -1
- package/dashboard/dist/assets/use-limits-display-prefs-yXXToNIc.js +1 -0
- package/dashboard/dist/assets/{use-ordered-list-OtfOqrNb.js → use-ordered-list-C6RG8N6c.js} +1 -1
- package/dashboard/dist/assets/use-reduced-motion-BqDKim_3.js +1 -0
- package/dashboard/dist/assets/{use-session-efficiency-pref-BScvtS8F.js → use-session-efficiency-pref-DpgZO_Lx.js} +1 -1
- package/dashboard/dist/assets/{use-transform-Cy1ykjEs.js → use-transform-D42P7jkd.js} +1 -1
- package/dashboard/dist/assets/use-usage-limits-CV6puFwq.js +1 -0
- package/dashboard/dist/assets/{useCurrency-VqF3cy1C.js → useCurrency-GWBmOrCR.js} +1 -1
- package/dashboard/dist/assets/{useTokenFormat-Cgub4MXu.js → useTokenFormat-Beybkb0v.js} +1 -1
- package/dashboard/dist/assets/{zap-DStzvQJI.js → zap-CEq2616W.js} +1 -1
- package/dashboard/dist/brand-logos/acode-dark.png +0 -0
- package/dashboard/dist/brand-logos/acode.png +0 -0
- package/dashboard/dist/brand-logos/commandcode.svg +1 -0
- package/dashboard/dist/brand-logos/devin.svg +6 -0
- package/dashboard/dist/index.html +20 -16
- package/dashboard/dist/ip-check.html +2 -2
- package/dashboard/dist/leaderboard.html +2 -2
- package/dashboard/dist/llms.txt +4 -2
- package/dashboard/dist/share.html +2 -2
- package/package.json +2 -2
- package/src/commands/init.js +114 -4
- package/src/commands/status.js +37 -0
- package/src/commands/sync.js +92 -10
- package/src/commands/uninstall.js +23 -0
- package/src/lib/codex-config.js +42 -0
- package/src/lib/codex-rollout-parser.js +2 -2
- package/src/lib/commandcode-limits.js +317 -0
- package/src/lib/devin-limits.js +249 -0
- package/src/lib/devin-usage.js +212 -0
- package/src/lib/local-api.js +14 -0
- package/src/lib/passive-mode.js +13 -1
- package/src/lib/pricing/curated-overrides.json +78 -1
- package/src/lib/pricing/index.js +11 -9
- package/src/lib/pricing/matcher.js +28 -0
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +1473 -147
- package/src/lib/session-analytics.js +2 -2
- package/src/lib/skills-manager.js +5 -0
- package/src/lib/subscriptions.js +70 -16
- package/src/lib/usage-limits.js +240 -49
- package/dashboard/dist/assets/DashboardPage-QBhbCMZF.js +0 -1
- package/dashboard/dist/assets/DialogClose-D_dvtfGV.js +0 -1
- package/dashboard/dist/assets/DialogDescription-ryGYdQjX.js +0 -1
- package/dashboard/dist/assets/DialogTitle-_ahf0z8b.js +0 -1
- package/dashboard/dist/assets/LimitsPage-PvqkOOwp.js +0 -6
- package/dashboard/dist/assets/PetPage-CNlWHF4z.js +0 -1
- package/dashboard/dist/assets/PopoverPopup-D9jadD3j.js +0 -1
- package/dashboard/dist/assets/ProviderIcon-Caz5PuAm.js +0 -1
- package/dashboard/dist/assets/Select-B5o-1sEl.js +0 -1
- package/dashboard/dist/assets/SelectItemText-CiA6L7Wk.js +0 -1
- package/dashboard/dist/assets/SettingsPage-DJrb_fsK.js +0 -2
- package/dashboard/dist/assets/SkillsPage-COk-YQQ5.js +0 -1
- package/dashboard/dist/assets/ToolbarRootContext-CozJT6Vs.js +0 -1
- package/dashboard/dist/assets/__vite-browser-external-BGnq8Ebm.js +0 -45
- package/dashboard/dist/assets/composite-Dxmivlk4.js +0 -1
- package/dashboard/dist/assets/limits-providers-BUoquSA-.js +0 -1
- package/dashboard/dist/assets/module-xpSFyNBK.js +0 -3
- package/dashboard/dist/assets/provider-display-BOBtUyFi.js +0 -42
- package/dashboard/dist/assets/store-oQZR4d9M.js +0 -1
- package/dashboard/dist/assets/use-limits-display-prefs-B-pbwUS8.js +0 -1
- package/dashboard/dist/assets/use-reduced-motion-DGXcZBX3.js +0 -1
- package/dashboard/dist/assets/use-usage-limits-CSMb2zYW.js +0 -1
package/src/lib/rollout.js
CHANGED
|
@@ -21,6 +21,11 @@ const {
|
|
|
21
21
|
currentCodexModel,
|
|
22
22
|
snapshotCodexModelAttributionState,
|
|
23
23
|
} = require("./codex-model-attribution");
|
|
24
|
+
const {
|
|
25
|
+
DEVIN_TABLE_PROBE_SQL,
|
|
26
|
+
devinUsageSql,
|
|
27
|
+
buildDevinUsageEvents,
|
|
28
|
+
} = require("./devin-usage");
|
|
24
29
|
const { USD_TICKS_PER_USD, normalizeGrokUsage } = require("./grok-usage");
|
|
25
30
|
|
|
26
31
|
const DEFAULT_SOURCE = "codex";
|
|
@@ -259,15 +264,96 @@ async function parseRolloutIncremental({
|
|
|
259
264
|
typeof codexEventStore.add === "function"
|
|
260
265
|
? codexEventStore
|
|
261
266
|
: null;
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
cursors
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
267
|
+
const createRolloutEventDedup = ({ cursorKey, externalEventStore = null }) => {
|
|
268
|
+
const previousHashes = Array.isArray(cursors?.[cursorKey]) ? cursors[cursorKey] : [];
|
|
269
|
+
if (!externalEventStore && !Array.isArray(cursors[cursorKey])) {
|
|
270
|
+
cursors[cursorKey] = previousHashes;
|
|
271
|
+
}
|
|
272
|
+
let seenEvents = null;
|
|
273
|
+
let newEventKeys = null;
|
|
274
|
+
let newEventKeySet = null;
|
|
275
|
+
let appendOnlyEvents = null;
|
|
276
|
+
let historicalEvents = null;
|
|
277
|
+
|
|
278
|
+
const ensureNewEventKeys = () => {
|
|
279
|
+
if (!newEventKeys) newEventKeys = [];
|
|
280
|
+
if (!newEventKeySet) newEventKeySet = new Set();
|
|
281
|
+
};
|
|
282
|
+
const recordNewEvent = (key) => {
|
|
283
|
+
ensureNewEventKeys();
|
|
284
|
+
if (newEventKeySet.has(key)) return;
|
|
285
|
+
newEventKeySet.add(key);
|
|
286
|
+
newEventKeys.push(key);
|
|
287
|
+
if (seenEvents) seenEvents.add(key);
|
|
288
|
+
externalEventStore?.add(key);
|
|
289
|
+
};
|
|
290
|
+
const getAppendOnlyEvents = () => {
|
|
291
|
+
if (!appendOnlyEvents) {
|
|
292
|
+
appendOnlyEvents = {
|
|
293
|
+
has(key) {
|
|
294
|
+
return Boolean(newEventKeySet?.has(key) || seenEvents?.has(key));
|
|
295
|
+
},
|
|
296
|
+
add: recordNewEvent,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
return appendOnlyEvents;
|
|
300
|
+
};
|
|
301
|
+
const getHistoricalEvents = () => {
|
|
302
|
+
if (externalEventStore) {
|
|
303
|
+
if (!historicalEvents) {
|
|
304
|
+
historicalEvents = {
|
|
305
|
+
has(key) {
|
|
306
|
+
return Boolean(newEventKeySet?.has(key) || externalEventStore.has(key));
|
|
307
|
+
},
|
|
308
|
+
add: recordNewEvent,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
return historicalEvents;
|
|
312
|
+
}
|
|
313
|
+
if (!seenEvents) {
|
|
314
|
+
seenEvents = new Set(previousHashes);
|
|
315
|
+
for (const key of newEventKeys || []) seenEvents.add(key);
|
|
316
|
+
if (syncDiagnostics && cursorKey === "codexHashes") {
|
|
317
|
+
syncDiagnostics.hash_set_constructions += 1;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
if (!historicalEvents) {
|
|
321
|
+
historicalEvents = {
|
|
322
|
+
has: (key) => seenEvents.has(key),
|
|
323
|
+
add: recordNewEvent,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
return historicalEvents;
|
|
327
|
+
};
|
|
328
|
+
|
|
329
|
+
return {
|
|
330
|
+
getAppendOnlyEvents,
|
|
331
|
+
getHistoricalEvents,
|
|
332
|
+
persist() {
|
|
333
|
+
if (!externalEventStore) {
|
|
334
|
+
for (const key of newEventKeys || []) previousHashes.push(key);
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
size() {
|
|
338
|
+
return externalEventStore
|
|
339
|
+
? Number(externalEventStore.size || 0)
|
|
340
|
+
: Array.isArray(cursors[cursorKey])
|
|
341
|
+
? cursors[cursorKey].length
|
|
342
|
+
: previousHashes.length;
|
|
343
|
+
},
|
|
344
|
+
};
|
|
345
|
+
};
|
|
346
|
+
const codexEventDedup = createRolloutEventDedup({
|
|
347
|
+
cursorKey: "codexHashes",
|
|
348
|
+
externalEventStore: externalCodexEventStore,
|
|
349
|
+
});
|
|
350
|
+
let acodeEventDedup = null;
|
|
351
|
+
const getAcodeEventDedup = () => {
|
|
352
|
+
if (!acodeEventDedup) {
|
|
353
|
+
acodeEventDedup = createRolloutEventDedup({ cursorKey: "acodeHashes" });
|
|
354
|
+
}
|
|
355
|
+
return acodeEventDedup;
|
|
356
|
+
};
|
|
271
357
|
let cursorSessionPaths = null;
|
|
272
358
|
if (syncDiagnostics) {
|
|
273
359
|
const codexParseCandidates = (Array.isArray(rolloutFiles) ? rolloutFiles : []).reduce((count, entry) => {
|
|
@@ -290,65 +376,9 @@ async function parseRolloutIncremental({
|
|
|
290
376
|
hash_set_constructions: 0,
|
|
291
377
|
hash_array_materializations: 0,
|
|
292
378
|
hash_array_materialized_items: 0,
|
|
293
|
-
codex_hash_count:
|
|
294
|
-
? Number(externalCodexEventStore.size || 0)
|
|
295
|
-
: prevCodexHashes.length,
|
|
379
|
+
codex_hash_count: codexEventDedup.size(),
|
|
296
380
|
});
|
|
297
381
|
}
|
|
298
|
-
const ensureNewCodexEventKeys = () => {
|
|
299
|
-
if (!newCodexEventKeys) newCodexEventKeys = [];
|
|
300
|
-
if (!newCodexEventKeySet) newCodexEventKeySet = new Set();
|
|
301
|
-
};
|
|
302
|
-
const recordNewCodexEvent = (key) => {
|
|
303
|
-
ensureNewCodexEventKeys();
|
|
304
|
-
if (newCodexEventKeySet.has(key)) return;
|
|
305
|
-
newCodexEventKeySet.add(key);
|
|
306
|
-
newCodexEventKeys.push(key);
|
|
307
|
-
if (seenCodexEvents) seenCodexEvents.add(key);
|
|
308
|
-
externalCodexEventStore?.add(key);
|
|
309
|
-
};
|
|
310
|
-
const getAppendOnlyCodexEvents = () => {
|
|
311
|
-
if (!appendOnlyCodexEvents) {
|
|
312
|
-
appendOnlyCodexEvents = {
|
|
313
|
-
has(key) {
|
|
314
|
-
return Boolean(
|
|
315
|
-
newCodexEventKeySet?.has(key) ||
|
|
316
|
-
seenCodexEvents?.has(key)
|
|
317
|
-
);
|
|
318
|
-
},
|
|
319
|
-
add: recordNewCodexEvent,
|
|
320
|
-
};
|
|
321
|
-
}
|
|
322
|
-
return appendOnlyCodexEvents;
|
|
323
|
-
};
|
|
324
|
-
const getHistoricalCodexEvents = () => {
|
|
325
|
-
if (externalCodexEventStore) {
|
|
326
|
-
if (!historicalCodexEvents) {
|
|
327
|
-
historicalCodexEvents = {
|
|
328
|
-
has(key) {
|
|
329
|
-
return Boolean(
|
|
330
|
-
newCodexEventKeySet?.has(key) ||
|
|
331
|
-
externalCodexEventStore.has(key)
|
|
332
|
-
);
|
|
333
|
-
},
|
|
334
|
-
add: recordNewCodexEvent,
|
|
335
|
-
};
|
|
336
|
-
}
|
|
337
|
-
return historicalCodexEvents;
|
|
338
|
-
}
|
|
339
|
-
if (!seenCodexEvents) {
|
|
340
|
-
seenCodexEvents = new Set(prevCodexHashes);
|
|
341
|
-
for (const key of newCodexEventKeys || []) seenCodexEvents.add(key);
|
|
342
|
-
if (syncDiagnostics) syncDiagnostics.hash_set_constructions += 1;
|
|
343
|
-
}
|
|
344
|
-
if (!historicalCodexEvents) {
|
|
345
|
-
historicalCodexEvents = {
|
|
346
|
-
has: (key) => seenCodexEvents.has(key),
|
|
347
|
-
add: recordNewCodexEvent,
|
|
348
|
-
};
|
|
349
|
-
}
|
|
350
|
-
return historicalCodexEvents;
|
|
351
|
-
};
|
|
352
382
|
const getCursorSessionPaths = () => {
|
|
353
383
|
if (cursorSessionPaths) return cursorSessionPaths;
|
|
354
384
|
cursorSessionPaths = new Map();
|
|
@@ -360,7 +390,7 @@ async function parseRolloutIncremental({
|
|
|
360
390
|
}
|
|
361
391
|
return cursorSessionPaths;
|
|
362
392
|
};
|
|
363
|
-
const
|
|
393
|
+
const needsHistoricalRolloutDedup = ({
|
|
364
394
|
filePath,
|
|
365
395
|
prev,
|
|
366
396
|
sameInode,
|
|
@@ -426,7 +456,7 @@ async function parseRolloutIncremental({
|
|
|
426
456
|
const prevOffset = sameInode ? prev.offset || 0 : 0;
|
|
427
457
|
const truncated = sameInode && prevOffset > st.size;
|
|
428
458
|
const rebuildingCodexBaseline = Boolean(
|
|
429
|
-
fileSource === DEFAULT_SOURCE &&
|
|
459
|
+
(fileSource === DEFAULT_SOURCE || fileSource === "acode") &&
|
|
430
460
|
sameInode &&
|
|
431
461
|
!truncated &&
|
|
432
462
|
prevOffset > 0 &&
|
|
@@ -445,7 +475,8 @@ async function parseRolloutIncremental({
|
|
|
445
475
|
? prev.modelAttributionState || null
|
|
446
476
|
: null;
|
|
447
477
|
|
|
448
|
-
const codexProjectFastPath =
|
|
478
|
+
const codexProjectFastPath =
|
|
479
|
+
projectEnabled && (fileSource === DEFAULT_SOURCE || fileSource === "acode");
|
|
449
480
|
const projectOffset = sameInode && !truncated ? Number(prev.projectOffset || 0) : 0;
|
|
450
481
|
const projectUpToDate =
|
|
451
482
|
codexProjectFastPath &&
|
|
@@ -496,8 +527,13 @@ async function parseRolloutIncremental({
|
|
|
496
527
|
if (syncDiagnostics && !projectContextOnlyScan && fileSource === DEFAULT_SOURCE) {
|
|
497
528
|
syncDiagnostics.content_files_read += 1;
|
|
498
529
|
}
|
|
499
|
-
const
|
|
500
|
-
?
|
|
530
|
+
const rolloutEventDedup = fileSource === DEFAULT_SOURCE
|
|
531
|
+
? codexEventDedup
|
|
532
|
+
: fileSource === "acode"
|
|
533
|
+
? getAcodeEventDedup()
|
|
534
|
+
: null;
|
|
535
|
+
const codexEventTracker = rolloutEventDedup
|
|
536
|
+
? (needsHistoricalRolloutDedup({
|
|
501
537
|
filePath,
|
|
502
538
|
prev,
|
|
503
539
|
sameInode,
|
|
@@ -505,8 +541,8 @@ async function parseRolloutIncremental({
|
|
|
505
541
|
startOffset,
|
|
506
542
|
rebuildingBaseline: rebuildingCodexBaseline,
|
|
507
543
|
})
|
|
508
|
-
?
|
|
509
|
-
:
|
|
544
|
+
? rolloutEventDedup.getHistoricalEvents
|
|
545
|
+
: rolloutEventDedup.getAppendOnlyEvents)
|
|
510
546
|
: null;
|
|
511
547
|
const result = projectContextOnlyScan
|
|
512
548
|
? await scanRolloutProjectFileContexts({
|
|
@@ -587,15 +623,10 @@ async function parseRolloutIncremental({
|
|
|
587
623
|
const projectBucketsQueued = projectEnabled
|
|
588
624
|
? await enqueueTouchedProjectBuckets({ projectQueuePath, projectState, projectTouchedBuckets })
|
|
589
625
|
: 0;
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
}
|
|
626
|
+
codexEventDedup.persist();
|
|
627
|
+
acodeEventDedup?.persist();
|
|
593
628
|
if (syncDiagnostics) {
|
|
594
|
-
syncDiagnostics.codex_hash_count =
|
|
595
|
-
? Number(externalCodexEventStore.size || 0)
|
|
596
|
-
: Array.isArray(cursors.codexHashes)
|
|
597
|
-
? cursors.codexHashes.length
|
|
598
|
-
: prevCodexHashes.length;
|
|
629
|
+
syncDiagnostics.codex_hash_count = codexEventDedup.size();
|
|
599
630
|
}
|
|
600
631
|
hourlyState.updatedAt = new Date().toISOString();
|
|
601
632
|
cursors.hourly = hourlyState;
|
|
@@ -2193,7 +2224,7 @@ async function parseRolloutFile({
|
|
|
2193
2224
|
if (!delta || isAllZeroUsage(delta)) continue;
|
|
2194
2225
|
delta.conversation_count = 1;
|
|
2195
2226
|
|
|
2196
|
-
// Forked Codex rollouts replay the parent session's entire token history
|
|
2227
|
+
// Forked Codex and Acode rollouts replay the parent session's entire token history
|
|
2197
2228
|
// into the child file the moment the fork is created. The date guard below
|
|
2198
2229
|
// (current_date < rolloutDate) catches cross-day forks, but same-day forks
|
|
2199
2230
|
// share the parent's current_date and slip through it. The replay is written
|
|
@@ -2209,9 +2240,13 @@ async function parseRolloutFile({
|
|
|
2209
2240
|
// counted (a bounded, <1% residual over-count); dropping real usage is the
|
|
2210
2241
|
// worse failure, so we bias against it. `usageDeltaState` is already advanced above,
|
|
2211
2242
|
// keeping the cumulative lineage correct for the live turns we keep.
|
|
2212
|
-
// Scoped to forked
|
|
2243
|
+
// Scoped to forked Codex/Acode rollouts. (issue #169 follow-up.)
|
|
2213
2244
|
let forkedReplaySkip = false;
|
|
2214
|
-
if (
|
|
2245
|
+
if (
|
|
2246
|
+
isForkedRollout &&
|
|
2247
|
+
(source === DEFAULT_SOURCE || source === "acode") &&
|
|
2248
|
+
replayPrefixActive
|
|
2249
|
+
) {
|
|
2215
2250
|
const tokenMs = Date.parse(tokenTimestamp);
|
|
2216
2251
|
// Fail open on anything the burst heuristic was not measured against:
|
|
2217
2252
|
// an unparseable timestamp or a backwards clock step permanently ends
|
|
@@ -2221,13 +2256,13 @@ async function parseRolloutFile({
|
|
|
2221
2256
|
if (!Number.isFinite(tokenMs) || (prevForkedTokenMs !== null && tokenMs < prevForkedTokenMs)) {
|
|
2222
2257
|
replayPrefixActive = false;
|
|
2223
2258
|
} else {
|
|
2224
|
-
if (prevForkedTokenMs !== null && tokenMs - prevForkedTokenMs >=
|
|
2259
|
+
if (prevForkedTokenMs !== null && tokenMs - prevForkedTokenMs >= FORK_REPLAY_GAP_MS) {
|
|
2225
2260
|
replayPrefixActive = false;
|
|
2226
2261
|
}
|
|
2227
2262
|
forkedReplaySkip =
|
|
2228
2263
|
replayPrefixActive &&
|
|
2229
2264
|
prevForkedTokenMs !== null &&
|
|
2230
|
-
tokenMs - prevForkedTokenMs <
|
|
2265
|
+
tokenMs - prevForkedTokenMs < FORK_REPLAY_GAP_MS;
|
|
2231
2266
|
prevForkedTokenMs = tokenMs;
|
|
2232
2267
|
}
|
|
2233
2268
|
}
|
|
@@ -2253,14 +2288,12 @@ async function parseRolloutFile({
|
|
|
2253
2288
|
// are still counted. Key = sessionUUID:eventTimestamp (both stable across the
|
|
2254
2289
|
// rewrite and across a sessions/ -> archived_sessions/ move).
|
|
2255
2290
|
//
|
|
2256
|
-
//
|
|
2257
|
-
//
|
|
2258
|
-
// their own model re-alignment that legitimately re-reads prior events, which
|
|
2259
|
-
// this dedup would otherwise suppress.
|
|
2291
|
+
// Apply event deduplication only to Codex and Acode. Every Code and other sources
|
|
2292
|
+
// reread events to realign models.
|
|
2260
2293
|
const codexEvents = typeof seenCodexEvents === "function"
|
|
2261
2294
|
? seenCodexEvents()
|
|
2262
2295
|
: seenCodexEvents;
|
|
2263
|
-
if (codexEvents && source === "codex") {
|
|
2296
|
+
if (codexEvents && (source === "codex" || source === "acode")) {
|
|
2264
2297
|
const dedupKey = `${sessionId || filePath}:${tokenTimestamp}`;
|
|
2265
2298
|
if (codexEvents.has(dedupKey)) continue;
|
|
2266
2299
|
codexEvents.add(dedupKey);
|
|
@@ -3781,7 +3814,7 @@ function normalizeIsoDate(value) {
|
|
|
3781
3814
|
// toward the low end because merging a real fast turn (under-count) is worse than
|
|
3782
3815
|
// leaving a slow replay counted (bounded over-count). See the skip site in
|
|
3783
3816
|
// parseRolloutFile. (issue #169 follow-up.)
|
|
3784
|
-
const
|
|
3817
|
+
const FORK_REPLAY_GAP_MS = 500;
|
|
3785
3818
|
|
|
3786
3819
|
function isForkedReplayToken({ isForkedRollout, rolloutDate, currentDate }) {
|
|
3787
3820
|
return Boolean(isForkedRollout && rolloutDate && currentDate && currentDate < rolloutDate);
|
|
@@ -4458,6 +4491,9 @@ function claudeMessageDedupKey(obj) {
|
|
|
4458
4491
|
function normalizeClaudeUsage(u) {
|
|
4459
4492
|
const inputTokens = toNonNegativeInt(u?.input_tokens);
|
|
4460
4493
|
const outputTokens = toNonNegativeInt(u?.output_tokens);
|
|
4494
|
+
const reasoningTokens = Math.min(outputTokens, toNonNegativeInt(
|
|
4495
|
+
u?.output_tokens_details?.thinking_tokens ?? u?.output_tokens_details?.reasoning_tokens,
|
|
4496
|
+
));
|
|
4461
4497
|
const cacheCreation = toNonNegativeInt(u?.cache_creation_input_tokens);
|
|
4462
4498
|
const cacheRead = toNonNegativeInt(u?.cache_read_input_tokens);
|
|
4463
4499
|
const totalTokens = inputTokens + outputTokens + cacheCreation + cacheRead;
|
|
@@ -4465,8 +4501,9 @@ function normalizeClaudeUsage(u) {
|
|
|
4465
4501
|
input_tokens: inputTokens,
|
|
4466
4502
|
cached_input_tokens: cacheRead,
|
|
4467
4503
|
cache_creation_input_tokens: cacheCreation,
|
|
4468
|
-
output_tokens
|
|
4469
|
-
|
|
4504
|
+
// Claude rows price reasoning separately; it is already included in usage.output_tokens.
|
|
4505
|
+
output_tokens: outputTokens - reasoningTokens,
|
|
4506
|
+
reasoning_output_tokens: reasoningTokens,
|
|
4470
4507
|
total_tokens: totalTokens,
|
|
4471
4508
|
};
|
|
4472
4509
|
}
|
|
@@ -12867,6 +12904,395 @@ async function parseAnythingllmIncremental({
|
|
|
12867
12904
|
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
12868
12905
|
}
|
|
12869
12906
|
|
|
12907
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
12908
|
+
// Devin (Cognition — devin.ai CLI)
|
|
12909
|
+
//
|
|
12910
|
+
// Data: SQLite at
|
|
12911
|
+
// macOS/Linux: $XDG_DATA_HOME/devin/cli/sessions.db (~/.local/share)
|
|
12912
|
+
// Windows: no evidenced native location; the CLI's XDG data dir inside a
|
|
12913
|
+
// WSL distro is probed via the \\wsl$ UNC bridge
|
|
12914
|
+
// Override: $TOKENTRACKER_DEVIN_DB
|
|
12915
|
+
//
|
|
12916
|
+
// Devin rewrites history aggressively: replay, compaction and forks copy
|
|
12917
|
+
// message_nodes rows, so several nodes share one chat_message
|
|
12918
|
+
// `.metadata.request_id` (verified on CLI 3000.10.21: ~2 duplicated nodes per
|
|
12919
|
+
// request). request_id is the billing identity — copies must not add usage.
|
|
12920
|
+
// Aggregation is a full projection + per-request reconcile keyed by
|
|
12921
|
+
// request_id: every fingerprint change re-reads the usage-only rows, the
|
|
12922
|
+
// ledger in cursors.devin.requests subtracts a request's prior contribution
|
|
12923
|
+
// before adding its current one, and deleted/compacted history keeps its
|
|
12924
|
+
// (non-refunded) ledger entry. There is no row_id high-water mark because
|
|
12925
|
+
// in-place metric corrections cannot be detected by one.
|
|
12926
|
+
//
|
|
12927
|
+
// Attribution recorded at first observation is authoritative: a request's
|
|
12928
|
+
// owning session, resolved project and conversation share live in the ledger,
|
|
12929
|
+
// so deleting the original node while a fork copy survives never migrates its
|
|
12930
|
+
// spend or refunds its conversation. A fork made only of copies pays nothing;
|
|
12931
|
+
// the first genuinely new request in a session pays its single
|
|
12932
|
+
// conversation_count once — the set of sessions that already paid is derived
|
|
12933
|
+
// from the ledger entries themselves, not persisted separately.
|
|
12934
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
12935
|
+
|
|
12936
|
+
const DEVIN_SOURCE = "devin";
|
|
12937
|
+
|
|
12938
|
+
function resolveDevinDbPath(env = process.env, deps = {}) {
|
|
12939
|
+
const override =
|
|
12940
|
+
typeof env.TOKENTRACKER_DEVIN_DB === "string" && env.TOKENTRACKER_DEVIN_DB.trim();
|
|
12941
|
+
if (override) return path.resolve(env.TOKENTRACKER_DEVIN_DB.trim());
|
|
12942
|
+
const home = env.HOME || env.USERPROFILE || os.homedir();
|
|
12943
|
+
const xdgDataHome =
|
|
12944
|
+
typeof env.XDG_DATA_HOME === "string" && env.XDG_DATA_HOME.trim()
|
|
12945
|
+
? path.resolve(env.XDG_DATA_HOME.trim())
|
|
12946
|
+
: path.join(home, ".local", "share");
|
|
12947
|
+
if (process.platform !== "win32") {
|
|
12948
|
+
return path.join(xdgDataHome, "devin", "cli", "sessions.db");
|
|
12949
|
+
}
|
|
12950
|
+
// Devin CLI is not known to write a native-Windows data dir; on Windows its
|
|
12951
|
+
// sessions.db lives under the distro's XDG home, reachable over \\wsl$.
|
|
12952
|
+
const wslDir = wsl.shouldProbeWsl(env)
|
|
12953
|
+
? wsl.discoverWslHome(".local/share/devin/cli", { ...deps, env })
|
|
12954
|
+
: null;
|
|
12955
|
+
const wslValue = wslDir ? path.join(wslDir, "sessions.db") : null;
|
|
12956
|
+
const paths = resolveInstallPaths({ nativeValue: null, wslValue }, env, deps);
|
|
12957
|
+
return paths.native || paths.wsl;
|
|
12958
|
+
}
|
|
12959
|
+
|
|
12960
|
+
function devinSqliteFingerprint(dbPath) {
|
|
12961
|
+
const fingerprint = sqliteSidecarFingerprint(dbPath);
|
|
12962
|
+
// -shm is reader bookkeeping: opening a WAL database can bump it without any
|
|
12963
|
+
// content change, so it must not force a rescan (same convention as Unsloth).
|
|
12964
|
+
delete fingerprint["-shm"];
|
|
12965
|
+
return fingerprint;
|
|
12966
|
+
}
|
|
12967
|
+
|
|
12968
|
+
// Request ids and conversation keys are untrusted strings — normalize the
|
|
12969
|
+
// persisted maps into null-prototype dictionaries so a literal "__proto__"
|
|
12970
|
+
// key stays an own entry that round-trips through cursors.json instead of
|
|
12971
|
+
// silently mutating the prototype chain (and re-adding that request's usage
|
|
12972
|
+
// on every rescan).
|
|
12973
|
+
function devinStringMap(value) {
|
|
12974
|
+
const dict = Object.create(null);
|
|
12975
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
12976
|
+
for (const key of Object.keys(value)) dict[key] = value[key];
|
|
12977
|
+
}
|
|
12978
|
+
return dict;
|
|
12979
|
+
}
|
|
12980
|
+
|
|
12981
|
+
// Stage only the bucket objects this parser can mutate. The shared
|
|
12982
|
+
// normalizers copy the bucket maps but alias each bucket/totals object, and
|
|
12983
|
+
// the enqueue helpers stamp queuedKey before appendFile runs — so a failed
|
|
12984
|
+
// append used to leave the caller's published state polluted. Only
|
|
12985
|
+
// devin-owned entries get private copies; every other provider's buckets
|
|
12986
|
+
// stay shared read-only references.
|
|
12987
|
+
function stageDevinBuckets(buckets, isDevinBucket) {
|
|
12988
|
+
const staged = {};
|
|
12989
|
+
for (const [key, bucket] of Object.entries(buckets || {})) {
|
|
12990
|
+
staged[key] =
|
|
12991
|
+
bucket && typeof bucket === "object" && isDevinBucket(key, bucket)
|
|
12992
|
+
? {
|
|
12993
|
+
...bucket,
|
|
12994
|
+
totals:
|
|
12995
|
+
bucket.totals && typeof bucket.totals === "object"
|
|
12996
|
+
? { ...bucket.totals }
|
|
12997
|
+
: bucket.totals,
|
|
12998
|
+
}
|
|
12999
|
+
: bucket;
|
|
13000
|
+
}
|
|
13001
|
+
return staged;
|
|
13002
|
+
}
|
|
13003
|
+
|
|
13004
|
+
async function readDevinUsageRows(dbPath, sqliteOptions = {}) {
|
|
13005
|
+
if (!dbPath || !fssync.existsSync(dbPath)) return [];
|
|
13006
|
+
const options = {
|
|
13007
|
+
label: "Devin",
|
|
13008
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
13009
|
+
timeout: 30_000,
|
|
13010
|
+
readOnly: true,
|
|
13011
|
+
throwOnReadFailure: true,
|
|
13012
|
+
...sqliteOptions,
|
|
13013
|
+
};
|
|
13014
|
+
let snapshot = null;
|
|
13015
|
+
let effectiveDbPath = dbPath;
|
|
13016
|
+
if (isUncPath(dbPath)) {
|
|
13017
|
+
try {
|
|
13018
|
+
snapshot = snapshotSqliteDb(dbPath);
|
|
13019
|
+
effectiveDbPath = snapshot.path;
|
|
13020
|
+
} catch (_e) { }
|
|
13021
|
+
}
|
|
13022
|
+
try {
|
|
13023
|
+
const tables = new Set(
|
|
13024
|
+
(await readSqliteJsonRowsAsync(effectiveDbPath, DEVIN_TABLE_PROBE_SQL, options))
|
|
13025
|
+
.map((row) => row?.name)
|
|
13026
|
+
.filter(Boolean),
|
|
13027
|
+
);
|
|
13028
|
+
if (!tables.has("message_nodes")) return [];
|
|
13029
|
+
return await readSqliteJsonRowsAsync(
|
|
13030
|
+
effectiveDbPath,
|
|
13031
|
+
devinUsageSql({ hasSessionsTable: tables.has("sessions") }),
|
|
13032
|
+
options,
|
|
13033
|
+
);
|
|
13034
|
+
} finally {
|
|
13035
|
+
if (snapshot) snapshot.cleanup();
|
|
13036
|
+
}
|
|
13037
|
+
}
|
|
13038
|
+
|
|
13039
|
+
async function parseDevinIncremental({
|
|
13040
|
+
dbPath,
|
|
13041
|
+
cursors,
|
|
13042
|
+
queuePath,
|
|
13043
|
+
projectQueuePath,
|
|
13044
|
+
onProgress,
|
|
13045
|
+
env,
|
|
13046
|
+
sqliteOptions,
|
|
13047
|
+
publicRepoResolver,
|
|
13048
|
+
} = {}) {
|
|
13049
|
+
await ensureDir(path.dirname(queuePath));
|
|
13050
|
+
const resolvedDb = dbPath || resolveDevinDbPath(env || process.env);
|
|
13051
|
+
const priorState =
|
|
13052
|
+
cursors.devin && typeof cursors.devin === "object" ? cursors.devin : {};
|
|
13053
|
+
const requests = devinStringMap(priorState.requests);
|
|
13054
|
+
if (!resolvedDb || !fssync.existsSync(resolvedDb)) {
|
|
13055
|
+
const nextState = { ...priorState, requests, updatedAt: new Date().toISOString() };
|
|
13056
|
+
delete nextState.conversations;
|
|
13057
|
+
cursors.devin = nextState;
|
|
13058
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0 };
|
|
13059
|
+
}
|
|
13060
|
+
|
|
13061
|
+
// Cheap unchanged check: skip all SQL work when neither the DB nor its WAL
|
|
13062
|
+
// moved since the last published state.
|
|
13063
|
+
const initialFingerprint = devinSqliteFingerprint(resolvedDb);
|
|
13064
|
+
if (sameSqliteFingerprint(initialFingerprint, priorState.fingerprint)) {
|
|
13065
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0 };
|
|
13066
|
+
}
|
|
13067
|
+
|
|
13068
|
+
const rows = await readDevinUsageRows(resolvedDb, sqliteOptions);
|
|
13069
|
+
const { events } = buildDevinUsageEvents(rows);
|
|
13070
|
+
|
|
13071
|
+
// Stage the normalized working states: bucket-map copies alias the
|
|
13072
|
+
// caller's published bucket objects, so give only the devin-owned entries
|
|
13073
|
+
// (plus the flat groupQueued map) private copies. Reconciliation and
|
|
13074
|
+
// enqueue mutations then land on staged state and are published only after
|
|
13075
|
+
// both queue appends below succeed — a failed append leaves `cursors`
|
|
13076
|
+
// untouched so a retry re-derives the same contribution and latest-wins
|
|
13077
|
+
// rows recover either queue. Unrelated providers' buckets stay shared
|
|
13078
|
+
// read-only references (this parser never writes their keys).
|
|
13079
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
13080
|
+
hourlyState.buckets = stageDevinBuckets(
|
|
13081
|
+
hourlyState.buckets,
|
|
13082
|
+
(key) =>
|
|
13083
|
+
(normalizeSourceInput(parseBucketKey(key).source) || DEFAULT_SOURCE) ===
|
|
13084
|
+
DEVIN_SOURCE,
|
|
13085
|
+
);
|
|
13086
|
+
hourlyState.groupQueued =
|
|
13087
|
+
hourlyState.groupQueued && typeof hourlyState.groupQueued === "object"
|
|
13088
|
+
? { ...hourlyState.groupQueued }
|
|
13089
|
+
: {};
|
|
13090
|
+
const touchedBuckets = new Set();
|
|
13091
|
+
const projectEnabled =
|
|
13092
|
+
typeof projectQueuePath === "string" && projectQueuePath.length > 0;
|
|
13093
|
+
const projectState = projectEnabled
|
|
13094
|
+
? normalizeProjectState(cursors?.projectHourly)
|
|
13095
|
+
: null;
|
|
13096
|
+
if (projectState) {
|
|
13097
|
+
// Project bucket keys are `projectKey|source|hourStart`; this parser only
|
|
13098
|
+
// ever looks up keys whose middle segment is the devin source.
|
|
13099
|
+
projectState.buckets = stageDevinBuckets(projectState.buckets, (key) => {
|
|
13100
|
+
const last = key.lastIndexOf(BUCKET_SEPARATOR);
|
|
13101
|
+
const prev = last > 0 ? key.lastIndexOf(BUCKET_SEPARATOR, last - 1) : -1;
|
|
13102
|
+
const source = prev >= 0 ? key.slice(prev + 1, last) : "";
|
|
13103
|
+
return (
|
|
13104
|
+
(normalizeSourceInput(source) || DEFAULT_SOURCE) === DEVIN_SOURCE
|
|
13105
|
+
);
|
|
13106
|
+
});
|
|
13107
|
+
}
|
|
13108
|
+
const projectTouchedBuckets = projectEnabled ? new Set() : null;
|
|
13109
|
+
const projectMetaCache = projectEnabled ? new Map() : null;
|
|
13110
|
+
const publicRepoCache = projectEnabled ? new Map() : null;
|
|
13111
|
+
const projectContextBySession = projectEnabled ? new Map() : null;
|
|
13112
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
13113
|
+
let eventsAggregated = 0;
|
|
13114
|
+
|
|
13115
|
+
// The counted-conversation index is derived from the retained ledger: the
|
|
13116
|
+
// entry that paid a conversation's +1 keeps that share in its own totals,
|
|
13117
|
+
// even after every copy of the request vanished — no second persisted map.
|
|
13118
|
+
const countedConversations = new Set();
|
|
13119
|
+
for (const [requestId, entry] of Object.entries(requests)) {
|
|
13120
|
+
const totals = entry && typeof entry === "object" ? entry.totals : null;
|
|
13121
|
+
if (totals && Number.isSafeInteger(totals.conversation_count) && totals.conversation_count >= 1) {
|
|
13122
|
+
countedConversations.add(
|
|
13123
|
+
typeof entry.sessionId === "string" && entry.sessionId
|
|
13124
|
+
? entry.sessionId
|
|
13125
|
+
: `request:${requestId}`,
|
|
13126
|
+
);
|
|
13127
|
+
}
|
|
13128
|
+
}
|
|
13129
|
+
|
|
13130
|
+
for (let index = 0; index < events.length; index++) {
|
|
13131
|
+
const event = events[index];
|
|
13132
|
+
const bucketStart = toUtcHalfHourStart(new Date(event.tsMs).toISOString());
|
|
13133
|
+
if (!bucketStart) continue;
|
|
13134
|
+
|
|
13135
|
+
const previous = requests[event.requestId];
|
|
13136
|
+
const previousTotals =
|
|
13137
|
+
previous?.totals && typeof previous.totals === "object" ? previous.totals : null;
|
|
13138
|
+
|
|
13139
|
+
// Ownership recorded at first observation is authoritative. For a known
|
|
13140
|
+
// request the ledger's session/project/conversation share survive copy
|
|
13141
|
+
// deletion and fork timelines; for a new request the canonical retained
|
|
13142
|
+
// record supplies them exactly once.
|
|
13143
|
+
let projectKey = previous ? previous.projectKey || null : null;
|
|
13144
|
+
let projectRef = previous ? previous.projectRef || null : null;
|
|
13145
|
+
if (previous) {
|
|
13146
|
+
const priorConv = previousTotals ? previousTotals.conversation_count : 0;
|
|
13147
|
+
event.totals.conversation_count =
|
|
13148
|
+
Number.isSafeInteger(priorConv) && priorConv >= 0 ? priorConv : 0;
|
|
13149
|
+
} else {
|
|
13150
|
+
if (projectEnabled && event.sessionId) {
|
|
13151
|
+
let context = projectContextBySession.get(event.sessionId);
|
|
13152
|
+
if (context === undefined) {
|
|
13153
|
+
const startDir = event.workingDirectory
|
|
13154
|
+
? wsl.mapWslCwdToUnc(event.workingDirectory, resolvedDb)
|
|
13155
|
+
: null;
|
|
13156
|
+
context = startDir
|
|
13157
|
+
? await resolveProjectContextForPath({
|
|
13158
|
+
startDir,
|
|
13159
|
+
projectMetaCache,
|
|
13160
|
+
publicRepoCache,
|
|
13161
|
+
publicRepoResolver,
|
|
13162
|
+
projectState,
|
|
13163
|
+
})
|
|
13164
|
+
: null;
|
|
13165
|
+
projectContextBySession.set(event.sessionId, context || null);
|
|
13166
|
+
}
|
|
13167
|
+
projectKey = context?.projectKey || null;
|
|
13168
|
+
projectRef = context?.projectRef || null;
|
|
13169
|
+
}
|
|
13170
|
+
// A fork made only of copied requests pays no conversation of its own;
|
|
13171
|
+
// the first genuinely new request in a conversation pays it once.
|
|
13172
|
+
const convKey = event.sessionId || `request:${event.requestId}`;
|
|
13173
|
+
if (countedConversations.has(convKey)) {
|
|
13174
|
+
event.totals.conversation_count = 0;
|
|
13175
|
+
} else {
|
|
13176
|
+
event.totals.conversation_count = 1;
|
|
13177
|
+
countedConversations.add(convKey);
|
|
13178
|
+
}
|
|
13179
|
+
}
|
|
13180
|
+
|
|
13181
|
+
const unchanged =
|
|
13182
|
+
previousTotals &&
|
|
13183
|
+
totalsKey(previousTotals) === totalsKey(event.totals) &&
|
|
13184
|
+
previous.bucketStart === bucketStart &&
|
|
13185
|
+
previous.model === event.model;
|
|
13186
|
+
if (!unchanged) {
|
|
13187
|
+
if (previousTotals && previous.bucketStart && previous.model) {
|
|
13188
|
+
const oldBucket = getHourlyBucket(
|
|
13189
|
+
hourlyState,
|
|
13190
|
+
DEVIN_SOURCE,
|
|
13191
|
+
previous.model,
|
|
13192
|
+
previous.bucketStart,
|
|
13193
|
+
);
|
|
13194
|
+
subtractTotals(oldBucket.totals, previousTotals);
|
|
13195
|
+
touchedBuckets.add(
|
|
13196
|
+
bucketKey(DEVIN_SOURCE, previous.model, previous.bucketStart),
|
|
13197
|
+
);
|
|
13198
|
+
if (projectEnabled && previous.projectKey) {
|
|
13199
|
+
const oldProjectBucket = getProjectBucket(
|
|
13200
|
+
projectState,
|
|
13201
|
+
previous.projectKey,
|
|
13202
|
+
DEVIN_SOURCE,
|
|
13203
|
+
previous.bucketStart,
|
|
13204
|
+
previous.projectRef || null,
|
|
13205
|
+
);
|
|
13206
|
+
subtractTotals(oldProjectBucket.totals, previousTotals);
|
|
13207
|
+
projectTouchedBuckets.add(
|
|
13208
|
+
projectBucketKey(previous.projectKey, DEVIN_SOURCE, previous.bucketStart),
|
|
13209
|
+
);
|
|
13210
|
+
}
|
|
13211
|
+
}
|
|
13212
|
+
|
|
13213
|
+
const bucket = getHourlyBucket(hourlyState, DEVIN_SOURCE, event.model, bucketStart);
|
|
13214
|
+
addTotals(bucket.totals, event.totals);
|
|
13215
|
+
touchedBuckets.add(bucketKey(DEVIN_SOURCE, event.model, bucketStart));
|
|
13216
|
+
if (projectEnabled && projectKey) {
|
|
13217
|
+
const projectBucket = getProjectBucket(
|
|
13218
|
+
projectState,
|
|
13219
|
+
projectKey,
|
|
13220
|
+
DEVIN_SOURCE,
|
|
13221
|
+
bucketStart,
|
|
13222
|
+
projectRef,
|
|
13223
|
+
);
|
|
13224
|
+
addTotals(projectBucket.totals, event.totals);
|
|
13225
|
+
projectTouchedBuckets.add(
|
|
13226
|
+
projectBucketKey(projectKey, DEVIN_SOURCE, bucketStart),
|
|
13227
|
+
);
|
|
13228
|
+
}
|
|
13229
|
+
// The ledger records only what was added: owning session, model,
|
|
13230
|
+
// bucket, totals and the resolved project identity — never request text
|
|
13231
|
+
// or raw working paths.
|
|
13232
|
+
requests[event.requestId] = {
|
|
13233
|
+
sessionId:
|
|
13234
|
+
previous && typeof previous.sessionId === "string" && previous.sessionId
|
|
13235
|
+
? previous.sessionId
|
|
13236
|
+
: event.sessionId,
|
|
13237
|
+
model: event.model,
|
|
13238
|
+
bucketStart,
|
|
13239
|
+
totals: event.totals,
|
|
13240
|
+
projectKey,
|
|
13241
|
+
projectRef,
|
|
13242
|
+
updatedAt: new Date().toISOString(),
|
|
13243
|
+
};
|
|
13244
|
+
eventsAggregated += 1;
|
|
13245
|
+
}
|
|
13246
|
+
if (cb) {
|
|
13247
|
+
cb({
|
|
13248
|
+
index: index + 1,
|
|
13249
|
+
total: events.length,
|
|
13250
|
+
recordsProcessed: index + 1,
|
|
13251
|
+
eventsAggregated,
|
|
13252
|
+
bucketsQueued: touchedBuckets.size,
|
|
13253
|
+
});
|
|
13254
|
+
}
|
|
13255
|
+
}
|
|
13256
|
+
|
|
13257
|
+
const bucketsQueued = await enqueueTouchedBuckets({
|
|
13258
|
+
queuePath,
|
|
13259
|
+
hourlyState,
|
|
13260
|
+
touchedBuckets,
|
|
13261
|
+
});
|
|
13262
|
+
const projectBucketsQueued = projectEnabled
|
|
13263
|
+
? await enqueueTouchedProjectBuckets({
|
|
13264
|
+
projectQueuePath,
|
|
13265
|
+
projectState,
|
|
13266
|
+
projectTouchedBuckets,
|
|
13267
|
+
})
|
|
13268
|
+
: 0;
|
|
13269
|
+
// If Devin wrote to the DB/WAL while we read, publish the pre-read
|
|
13270
|
+
// fingerprint so the next sync re-reads instead of acknowledging a snapshot
|
|
13271
|
+
// it never saw (same convention as parseUnslothIncremental).
|
|
13272
|
+
const finalFingerprint = devinSqliteFingerprint(resolvedDb);
|
|
13273
|
+
const updatedAt = new Date().toISOString();
|
|
13274
|
+
hourlyState.updatedAt = updatedAt;
|
|
13275
|
+
cursors.hourly = hourlyState;
|
|
13276
|
+
cursors.devin = {
|
|
13277
|
+
version: 1,
|
|
13278
|
+
requests,
|
|
13279
|
+
fingerprint: sameSqliteFingerprint(initialFingerprint, finalFingerprint)
|
|
13280
|
+
? finalFingerprint
|
|
13281
|
+
: initialFingerprint,
|
|
13282
|
+
updatedAt,
|
|
13283
|
+
};
|
|
13284
|
+
if (projectState) {
|
|
13285
|
+
projectState.updatedAt = updatedAt;
|
|
13286
|
+
cursors.projectHourly = projectState;
|
|
13287
|
+
}
|
|
13288
|
+
return {
|
|
13289
|
+
recordsProcessed: rows.length,
|
|
13290
|
+
eventsAggregated,
|
|
13291
|
+
bucketsQueued,
|
|
13292
|
+
projectBucketsQueued,
|
|
13293
|
+
};
|
|
13294
|
+
}
|
|
13295
|
+
|
|
12870
13296
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
12871
13297
|
// Goose (Block AI agent — github.com/block/goose)
|
|
12872
13298
|
//
|
|
@@ -14620,6 +15046,8 @@ async function parsePiLikeIncremental({
|
|
|
14620
15046
|
sessionFiles,
|
|
14621
15047
|
cursors,
|
|
14622
15048
|
queuePath,
|
|
15049
|
+
projectQueuePath,
|
|
15050
|
+
publicRepoResolver,
|
|
14623
15051
|
onProgress,
|
|
14624
15052
|
env,
|
|
14625
15053
|
defaultModel,
|
|
@@ -14629,6 +15057,7 @@ async function parsePiLikeIncremental({
|
|
|
14629
15057
|
sourceForProvider,
|
|
14630
15058
|
} = {}) {
|
|
14631
15059
|
await ensureDir(path.dirname(queuePath));
|
|
15060
|
+
const projectEnabled = typeof projectQueuePath === "string" && projectQueuePath.length > 0;
|
|
14632
15061
|
const providerState = cursors[stateKey] && typeof cursors[stateKey] === "object"
|
|
14633
15062
|
? cursors[stateKey]
|
|
14634
15063
|
: {};
|
|
@@ -14638,6 +15067,9 @@ async function parsePiLikeIncremental({
|
|
|
14638
15067
|
? { ...providerState.fileOffsets }
|
|
14639
15068
|
: {};
|
|
14640
15069
|
|
|
15070
|
+
const projectSeenIds = new Set(Array.isArray(providerState.projectSeenIds) ? providerState.projectSeenIds : []);
|
|
15071
|
+
const projectFileOffsets = { ...(providerState.projectFileOffsets || {}) };
|
|
15072
|
+
|
|
14641
15073
|
const files = Array.isArray(sessionFiles)
|
|
14642
15074
|
? sessionFiles
|
|
14643
15075
|
: resolveSessionFiles(env || process.env);
|
|
@@ -14650,11 +15082,33 @@ async function parsePiLikeIncremental({
|
|
|
14650
15082
|
fileOffsets,
|
|
14651
15083
|
updatedAt: new Date().toISOString(),
|
|
14652
15084
|
};
|
|
14653
|
-
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
15085
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0 };
|
|
14654
15086
|
}
|
|
14655
15087
|
|
|
14656
15088
|
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
14657
15089
|
const touchedBuckets = new Set();
|
|
15090
|
+
const projectState = projectEnabled ? normalizeProjectState(cursors?.projectHourly) : null;
|
|
15091
|
+
const projectTouchedBuckets = projectEnabled ? new Set() : null;
|
|
15092
|
+
const projectMetaCache = projectEnabled ? new Map() : null;
|
|
15093
|
+
const publicRepoCache = projectEnabled ? new Map() : null;
|
|
15094
|
+
// Both queues must publish before cursor progress is acknowledged. Normalized
|
|
15095
|
+
// bucket maps still alias their values, so stage this provider's buckets to
|
|
15096
|
+
// keep a failed project append from mutating the caller's aggregate state.
|
|
15097
|
+
const family = sourceForProvider(null);
|
|
15098
|
+
const ownsSource = (source) => source === family || source.startsWith(`${family}-`);
|
|
15099
|
+
hourlyState.groupQueued = { ...(hourlyState.groupQueued || {}) };
|
|
15100
|
+
for (const [key, bucket] of Object.entries(hourlyState.buckets)) {
|
|
15101
|
+
if (ownsSource(parseBucketKey(key).source || "")) {
|
|
15102
|
+
hourlyState.buckets[key] = { ...bucket, totals: { ...bucket.totals } };
|
|
15103
|
+
}
|
|
15104
|
+
}
|
|
15105
|
+
if (projectState) {
|
|
15106
|
+
for (const [key, bucket] of Object.entries(projectState.buckets)) {
|
|
15107
|
+
if (ownsSource(bucket.source || "")) {
|
|
15108
|
+
projectState.buckets[key] = { ...bucket, totals: { ...bucket.totals } };
|
|
15109
|
+
}
|
|
15110
|
+
}
|
|
15111
|
+
}
|
|
14658
15112
|
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
14659
15113
|
let recordsProcessed = 0;
|
|
14660
15114
|
let eventsAggregated = 0;
|
|
@@ -14793,26 +15247,166 @@ async function parsePiLikeIncremental({
|
|
|
14793
15247
|
};
|
|
14794
15248
|
}
|
|
14795
15249
|
|
|
14796
|
-
|
|
14797
|
-
|
|
14798
|
-
|
|
14799
|
-
|
|
14800
|
-
|
|
14801
|
-
|
|
15250
|
+
// Project attribution has an independent cursor so upgrading an existing
|
|
15251
|
+
// installation can backfill already-consumed Pi sessions without adding
|
|
15252
|
+
// those messages to the total-usage buckets a second time. Current Pi
|
|
15253
|
+
// session headers persist the real cwd; unlike the encoded session folder,
|
|
15254
|
+
// it is lossless even when path components contain dashes.
|
|
15255
|
+
if (projectEnabled) {
|
|
15256
|
+
for (const filePath of files) {
|
|
15257
|
+
let stat;
|
|
15258
|
+
try { stat = fssync.statSync(filePath); } catch { continue; }
|
|
15259
|
+
|
|
15260
|
+
const prevEntry = projectFileOffsets[filePath] || {};
|
|
15261
|
+
const prevSize = Number(prevEntry.size) || 0;
|
|
15262
|
+
const prevIno = prevEntry.ino;
|
|
15263
|
+
const inodeChanged = typeof prevIno === "number" && prevIno !== stat.ino;
|
|
15264
|
+
const startOffset = stat.size < prevSize || inodeChanged ? 0 : prevSize;
|
|
15265
|
+
if (stat.size <= startOffset) continue;
|
|
15266
|
+
|
|
15267
|
+
const cwd = await resolveOmpFileCwd(filePath);
|
|
15268
|
+
const projectContext = cwd
|
|
15269
|
+
? await resolveProjectContextForPath({
|
|
15270
|
+
startDir: wsl.mapWslCwdToUnc(cwd, filePath),
|
|
15271
|
+
projectMetaCache,
|
|
15272
|
+
publicRepoCache,
|
|
15273
|
+
publicRepoResolver,
|
|
15274
|
+
projectState,
|
|
15275
|
+
})
|
|
15276
|
+
: null;
|
|
15277
|
+
const projectRef = projectContext?.projectRef || null;
|
|
15278
|
+
const projectKey = projectContext?.projectKey || null;
|
|
15279
|
+
|
|
15280
|
+
let lastCompleteOffset = startOffset;
|
|
15281
|
+
if (projectKey && projectRef) {
|
|
15282
|
+
let stream;
|
|
15283
|
+
try {
|
|
15284
|
+
stream = fssync.createReadStream(filePath, {
|
|
15285
|
+
encoding: "utf8",
|
|
15286
|
+
start: startOffset,
|
|
15287
|
+
});
|
|
15288
|
+
} catch {
|
|
15289
|
+
continue;
|
|
15290
|
+
}
|
|
15291
|
+
let streamedBytes = 0;
|
|
15292
|
+
stream.on("data", (chunk) => {
|
|
15293
|
+
const newlineIndex = chunk.lastIndexOf("\n");
|
|
15294
|
+
if (newlineIndex !== -1) {
|
|
15295
|
+
lastCompleteOffset = startOffset + streamedBytes
|
|
15296
|
+
+ Buffer.byteLength(chunk.slice(0, newlineIndex + 1), "utf8");
|
|
15297
|
+
}
|
|
15298
|
+
streamedBytes += Buffer.byteLength(chunk, "utf8");
|
|
15299
|
+
});
|
|
15300
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
15301
|
+
for await (const line of rl) {
|
|
15302
|
+
if (!line || !line.trim()) continue;
|
|
15303
|
+
let entry;
|
|
15304
|
+
try { entry = JSON.parse(line); } catch { continue; }
|
|
15305
|
+
const msg = entry?.type === "message" ? entry.message : null;
|
|
15306
|
+
const usage = msg?.role === "assistant" ? msg.usage : null;
|
|
15307
|
+
if (!usage || typeof usage !== "object") continue;
|
|
15308
|
+
|
|
15309
|
+
const entryId = typeof entry.id === "string" && entry.id ? entry.id : null;
|
|
15310
|
+
if (!entryId || projectSeenIds.has(entryId)) continue;
|
|
15311
|
+
|
|
15312
|
+
const input = toNonNegativeInt(usage.input);
|
|
15313
|
+
const output = toNonNegativeInt(usage.output);
|
|
15314
|
+
const cacheRead = toNonNegativeInt(usage.cacheRead);
|
|
15315
|
+
const cacheWrite = toNonNegativeInt(usage.cacheWrite);
|
|
15316
|
+
const reasoningTokens = toNonNegativeInt(usage.reasoningTokens);
|
|
15317
|
+
if (
|
|
15318
|
+
input === 0 &&
|
|
15319
|
+
output === 0 &&
|
|
15320
|
+
cacheRead === 0 &&
|
|
15321
|
+
cacheWrite === 0 &&
|
|
15322
|
+
reasoningTokens === 0
|
|
15323
|
+
) {
|
|
15324
|
+
projectSeenIds.add(entryId);
|
|
15325
|
+
continue;
|
|
15326
|
+
}
|
|
15327
|
+
|
|
15328
|
+
let tsMs = null;
|
|
15329
|
+
if (Number.isFinite(Number(msg.timestamp)) && Number(msg.timestamp) > 0) {
|
|
15330
|
+
tsMs = Number(msg.timestamp);
|
|
15331
|
+
} else if (typeof entry.timestamp === "string" && entry.timestamp) {
|
|
15332
|
+
const parsed = Date.parse(entry.timestamp);
|
|
15333
|
+
if (Number.isFinite(parsed) && parsed > 0) tsMs = parsed;
|
|
15334
|
+
}
|
|
15335
|
+
const bucketStart = tsMs == null
|
|
15336
|
+
? null
|
|
15337
|
+
: toUtcHalfHourStart(new Date(tsMs).toISOString());
|
|
15338
|
+
if (!bucketStart) {
|
|
15339
|
+
projectSeenIds.add(entryId);
|
|
15340
|
+
continue;
|
|
15341
|
+
}
|
|
15342
|
+
|
|
15343
|
+
const totalTokens =
|
|
15344
|
+
Number.isFinite(Number(usage.totalTokens)) && Number(usage.totalTokens) > 0
|
|
15345
|
+
? toNonNegativeInt(usage.totalTokens)
|
|
15346
|
+
: input + output + cacheRead + cacheWrite + reasoningTokens;
|
|
15347
|
+
const delta = {
|
|
15348
|
+
input_tokens: input,
|
|
15349
|
+
cached_input_tokens: cacheRead,
|
|
15350
|
+
cache_creation_input_tokens: cacheWrite,
|
|
15351
|
+
output_tokens: output,
|
|
15352
|
+
reasoning_output_tokens: reasoningTokens,
|
|
15353
|
+
total_tokens: totalTokens,
|
|
15354
|
+
conversation_count: 1,
|
|
15355
|
+
};
|
|
15356
|
+
const projectBucket = getProjectBucket(
|
|
15357
|
+
projectState,
|
|
15358
|
+
projectKey,
|
|
15359
|
+
sourceForProvider(msg.provider),
|
|
15360
|
+
bucketStart,
|
|
15361
|
+
projectRef,
|
|
15362
|
+
);
|
|
15363
|
+
addTotals(projectBucket.totals, delta);
|
|
15364
|
+
projectTouchedBuckets.add(projectBucketKey(projectKey, sourceForProvider(msg.provider), bucketStart));
|
|
15365
|
+
projectSeenIds.add(entryId);
|
|
15366
|
+
}
|
|
15367
|
+
}
|
|
15368
|
+
|
|
15369
|
+
let postStat = stat;
|
|
15370
|
+
try { postStat = fssync.statSync(filePath); } catch {}
|
|
15371
|
+
projectFileOffsets[filePath] = {
|
|
15372
|
+
size: Math.min(lastCompleteOffset, postStat.size),
|
|
15373
|
+
mtimeMs: postStat.mtimeMs,
|
|
15374
|
+
ino: postStat.ino,
|
|
15375
|
+
};
|
|
15376
|
+
}
|
|
15377
|
+
}
|
|
15378
|
+
|
|
15379
|
+
const seenArr = Array.from(seenIds);
|
|
15380
|
+
const cappedSeen =
|
|
15381
|
+
seenArr.length > 10_000 ? seenArr.slice(seenArr.length - 10_000) : seenArr;
|
|
15382
|
+
|
|
15383
|
+
const bucketsQueued = await enqueueTouchedBuckets({
|
|
15384
|
+
queuePath,
|
|
14802
15385
|
hourlyState,
|
|
14803
15386
|
touchedBuckets,
|
|
14804
15387
|
});
|
|
15388
|
+
const projectBucketsQueued = projectEnabled
|
|
15389
|
+
? await enqueueTouchedProjectBuckets({ projectQueuePath, projectState, projectTouchedBuckets })
|
|
15390
|
+
: 0;
|
|
14805
15391
|
const updatedAt = new Date().toISOString();
|
|
14806
15392
|
hourlyState.updatedAt = updatedAt;
|
|
14807
15393
|
cursors.hourly = hourlyState;
|
|
15394
|
+
if (projectState) {
|
|
15395
|
+
projectState.updatedAt = updatedAt;
|
|
15396
|
+
cursors.projectHourly = projectState;
|
|
15397
|
+
}
|
|
14808
15398
|
cursors[stateKey] = {
|
|
14809
15399
|
...providerState,
|
|
14810
15400
|
seenIds: cappedSeen,
|
|
14811
15401
|
fileOffsets,
|
|
15402
|
+
...(projectEnabled ? {
|
|
15403
|
+
projectSeenIds: Array.from(projectSeenIds).slice(-10_000),
|
|
15404
|
+
projectFileOffsets,
|
|
15405
|
+
} : {}),
|
|
14812
15406
|
updatedAt,
|
|
14813
15407
|
};
|
|
14814
15408
|
|
|
14815
|
-
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
15409
|
+
return { recordsProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
|
|
14816
15410
|
}
|
|
14817
15411
|
|
|
14818
15412
|
async function parsePiIncremental(options = {}) {
|
|
@@ -18666,6 +19260,9 @@ async function parseAntigravityIncremental({
|
|
|
18666
19260
|
const initialContextTokens = sameFile ? Number(prev.contextTokens || 0) : 0;
|
|
18667
19261
|
const initialPrevContext = sameFile ? Number(prev.previousContextTokens || 0) : 0;
|
|
18668
19262
|
const initialModel = sameFile && typeof prev.currentModel === "string" ? prev.currentModel : null;
|
|
19263
|
+
const initialLastPlannerModel =
|
|
19264
|
+
sameFile && typeof prev.lastPlannerModel === "string" ? prev.lastPlannerModel : null;
|
|
19265
|
+
const initialUsageSource = sameFile && typeof prev.usageSource === "string" ? prev.usageSource : null;
|
|
18669
19266
|
|
|
18670
19267
|
const projectContext = projectEnabled
|
|
18671
19268
|
? await resolveProjectContextForFile({
|
|
@@ -18685,6 +19282,8 @@ async function parseAntigravityIncremental({
|
|
|
18685
19282
|
initialContextTokens,
|
|
18686
19283
|
initialPrevContext,
|
|
18687
19284
|
initialModel,
|
|
19285
|
+
initialLastPlannerModel,
|
|
19286
|
+
initialUsageSource,
|
|
18688
19287
|
hourlyState,
|
|
18689
19288
|
touchedBuckets,
|
|
18690
19289
|
source: fileSource,
|
|
@@ -18702,6 +19301,8 @@ async function parseAntigravityIncremental({
|
|
|
18702
19301
|
contextTokens: result.contextTokens,
|
|
18703
19302
|
previousContextTokens: result.previousContextTokens,
|
|
18704
19303
|
currentModel: result.currentModel,
|
|
19304
|
+
lastPlannerModel: result.lastPlannerModel,
|
|
19305
|
+
usageSource: result.usageSource,
|
|
18705
19306
|
updatedAt: new Date().toISOString(),
|
|
18706
19307
|
};
|
|
18707
19308
|
|
|
@@ -18734,12 +19335,121 @@ async function parseAntigravityIncremental({
|
|
|
18734
19335
|
return { filesProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
|
|
18735
19336
|
}
|
|
18736
19337
|
|
|
19338
|
+
function decodeAntigravityVarint(buf, offset) {
|
|
19339
|
+
let res = 0;
|
|
19340
|
+
let shift = 0;
|
|
19341
|
+
while (offset < buf.length) {
|
|
19342
|
+
const b = buf[offset++];
|
|
19343
|
+
res += (b & 0x7f) * 2 ** shift;
|
|
19344
|
+
if (!(b & 0x80)) break;
|
|
19345
|
+
shift += 7;
|
|
19346
|
+
}
|
|
19347
|
+
return [res, offset];
|
|
19348
|
+
}
|
|
19349
|
+
|
|
19350
|
+
function findAntigravityProtoFields(buf) {
|
|
19351
|
+
const fields = [];
|
|
19352
|
+
let offset = 0;
|
|
19353
|
+
while (offset < buf.length) {
|
|
19354
|
+
const [tag, next] = decodeAntigravityVarint(buf, offset);
|
|
19355
|
+
offset = next;
|
|
19356
|
+
const fieldNum = tag >> 3;
|
|
19357
|
+
const wireType = tag & 7;
|
|
19358
|
+
if (wireType === 0) {
|
|
19359
|
+
const [val, vNext] = decodeAntigravityVarint(buf, offset);
|
|
19360
|
+
offset = vNext;
|
|
19361
|
+
fields.push({ num: fieldNum, val });
|
|
19362
|
+
} else if (wireType === 2) {
|
|
19363
|
+
const [len, lNext] = decodeAntigravityVarint(buf, offset);
|
|
19364
|
+
offset = lNext;
|
|
19365
|
+
fields.push({ num: fieldNum, val: buf.subarray(offset, offset + len) });
|
|
19366
|
+
offset += len;
|
|
19367
|
+
} else if (wireType === 1) {
|
|
19368
|
+
offset += 8;
|
|
19369
|
+
} else if (wireType === 5) {
|
|
19370
|
+
offset += 4;
|
|
19371
|
+
} else {
|
|
19372
|
+
break;
|
|
19373
|
+
}
|
|
19374
|
+
}
|
|
19375
|
+
return fields;
|
|
19376
|
+
}
|
|
19377
|
+
|
|
19378
|
+
function extractAntigravityGenInfo(buf) {
|
|
19379
|
+
const root = findAntigravityProtoFields(buf);
|
|
19380
|
+
const f1 = root.find((f) => f.num === 1)?.val;
|
|
19381
|
+
if (!f1) return null;
|
|
19382
|
+
|
|
19383
|
+
const inner = findAntigravityProtoFields(f1);
|
|
19384
|
+
let model = null;
|
|
19385
|
+
const f19 = inner.find((f) => f.num === 19)?.val;
|
|
19386
|
+
if (f19) model = Buffer.from(f19).toString("utf8").trim();
|
|
19387
|
+
|
|
19388
|
+
let contextTokens = 0;
|
|
19389
|
+
const f9 = inner.find((f) => f.num === 9)?.val;
|
|
19390
|
+
if (f9) {
|
|
19391
|
+
const f10 = findAntigravityProtoFields(f9).find((f) => f.num === 10)?.val;
|
|
19392
|
+
if (f10) {
|
|
19393
|
+
const tok = findAntigravityProtoFields(f10).find((f) => f.num === 1)?.val;
|
|
19394
|
+
if (Number.isFinite(tok)) contextTokens = tok;
|
|
19395
|
+
}
|
|
19396
|
+
}
|
|
19397
|
+
|
|
19398
|
+
let lastStepIndex = null;
|
|
19399
|
+
for (const f of inner) {
|
|
19400
|
+
if (f.num !== 20 || !f.val) continue;
|
|
19401
|
+
const kv = findAntigravityProtoFields(f.val);
|
|
19402
|
+
const k = kv.find((x) => x.num === 1)?.val;
|
|
19403
|
+
const v = kv.find((x) => x.num === 2)?.val;
|
|
19404
|
+
if (k && Buffer.from(k).toString("utf8") === "last_step_index" && v) {
|
|
19405
|
+
const parsed = parseInt(Buffer.from(v).toString("utf8"), 10);
|
|
19406
|
+
if (Number.isFinite(parsed)) lastStepIndex = parsed;
|
|
19407
|
+
}
|
|
19408
|
+
}
|
|
19409
|
+
|
|
19410
|
+
return { model, contextTokens, lastStepIndex };
|
|
19411
|
+
}
|
|
19412
|
+
|
|
19413
|
+
function resolveAntigravityDbPath(transcriptPath) {
|
|
19414
|
+
if (!transcriptPath || typeof transcriptPath !== "string") return null;
|
|
19415
|
+
const m = transcriptPath.match(/^(.*)[/\\]brain[/\\]([^/\\]+)[/\\]\.system_generated[/\\]logs[/\\]transcript.*\.jsonl$/);
|
|
19416
|
+
if (!m) return null;
|
|
19417
|
+
return path.join(m[1], "conversations", `${m[2]}.db`);
|
|
19418
|
+
}
|
|
19419
|
+
|
|
19420
|
+
function readAntigravityConversationDb(dbPath) {
|
|
19421
|
+
if (!dbPath) return null;
|
|
19422
|
+
try {
|
|
19423
|
+
const rows = readSqliteJsonRows(
|
|
19424
|
+
dbPath,
|
|
19425
|
+
"SELECT idx, quote(data) as hex FROM gen_metadata ORDER BY idx",
|
|
19426
|
+
{ readOnly: true },
|
|
19427
|
+
);
|
|
19428
|
+
if (!Array.isArray(rows) || rows.length === 0) return null;
|
|
19429
|
+
|
|
19430
|
+
const stepMap = new Map();
|
|
19431
|
+
for (const r of rows) {
|
|
19432
|
+
if (!r || typeof r.hex !== "string" || !r.hex.startsWith("X'")) continue;
|
|
19433
|
+
const buf = Buffer.from(r.hex.slice(2, -1), "hex");
|
|
19434
|
+
const info = extractAntigravityGenInfo(buf);
|
|
19435
|
+
if (info && info.lastStepIndex != null && info.contextTokens > 0) {
|
|
19436
|
+
stepMap.set(info.lastStepIndex + 1, info);
|
|
19437
|
+
}
|
|
19438
|
+
}
|
|
19439
|
+
return stepMap.size > 0 ? stepMap : null;
|
|
19440
|
+
} catch (_) {
|
|
19441
|
+
return null;
|
|
19442
|
+
}
|
|
19443
|
+
}
|
|
19444
|
+
|
|
18737
19445
|
async function parseAntigravityFile({
|
|
18738
19446
|
filePath,
|
|
18739
19447
|
lastLine,
|
|
18740
19448
|
initialContextTokens,
|
|
18741
19449
|
initialPrevContext,
|
|
18742
19450
|
initialModel,
|
|
19451
|
+
initialLastPlannerModel,
|
|
19452
|
+
initialUsageSource,
|
|
18743
19453
|
hourlyState,
|
|
18744
19454
|
touchedBuckets,
|
|
18745
19455
|
source,
|
|
@@ -18756,6 +19466,8 @@ async function parseAntigravityFile({
|
|
|
18756
19466
|
contextTokens: 0,
|
|
18757
19467
|
previousContextTokens: 0,
|
|
18758
19468
|
currentModel: null,
|
|
19469
|
+
lastPlannerModel: null,
|
|
19470
|
+
usageSource: "estimated",
|
|
18759
19471
|
};
|
|
18760
19472
|
}
|
|
18761
19473
|
|
|
@@ -18764,6 +19476,8 @@ async function parseAntigravityFile({
|
|
|
18764
19476
|
.map((line) => line.trim())
|
|
18765
19477
|
.filter(Boolean);
|
|
18766
19478
|
let eventsAggregated = 0;
|
|
19479
|
+
const dbPath = resolveAntigravityDbPath(filePath);
|
|
19480
|
+
const stepMap = dbPath ? readAntigravityConversationDb(dbPath) : null;
|
|
18767
19481
|
// Resume cached context-token total + model so historical lines (i < lastLine)
|
|
18768
19482
|
// don't need to be re-tokenized on every sync. Falls back to a full re-walk
|
|
18769
19483
|
// when the cached state is missing (legacy cursor) or the file rotated.
|
|
@@ -18772,7 +19486,9 @@ async function parseAntigravityFile({
|
|
|
18772
19486
|
const cachedTokens = Number.isFinite(initialContextTokens) ? initialContextTokens : 0;
|
|
18773
19487
|
const cachedPrev = Number.isFinite(initialPrevContext) ? initialPrevContext : 0;
|
|
18774
19488
|
const cachedModel = typeof initialModel === "string" ? initialModel : null;
|
|
18775
|
-
const
|
|
19489
|
+
const sqliteCursor = initialUsageSource === "sqlite";
|
|
19490
|
+
const resumed =
|
|
19491
|
+
canResume && (cachedTokens > 0 || cachedModel !== null) && (!stepMap || sqliteCursor);
|
|
18776
19492
|
const scanStart = resumed ? lastLine : 0;
|
|
18777
19493
|
let currentModel = resumed ? cachedModel : null;
|
|
18778
19494
|
if (!currentModel) {
|
|
@@ -18783,6 +19499,11 @@ async function parseAntigravityFile({
|
|
|
18783
19499
|
// tokens accumulated AFTER that point count as new input on the next planner
|
|
18784
19500
|
// call — prevents O(N²) double-counting of the full history every turn.
|
|
18785
19501
|
let previousContextTokens = resumed ? cachedPrev : 0;
|
|
19502
|
+
let lastPlannerModel = null;
|
|
19503
|
+
if (resumed) {
|
|
19504
|
+
lastPlannerModel =
|
|
19505
|
+
typeof initialLastPlannerModel === "string" ? initialLastPlannerModel : cachedModel;
|
|
19506
|
+
}
|
|
18786
19507
|
let lastCompletedLine = Math.min(Number.isFinite(lastLine) ? lastLine : 0, lines.length);
|
|
18787
19508
|
|
|
18788
19509
|
for (let i = scanStart; i < lines.length; i++) {
|
|
@@ -18805,9 +19526,27 @@ async function parseAntigravityFile({
|
|
|
18805
19526
|
}
|
|
18806
19527
|
|
|
18807
19528
|
const eventContextTokens = antigravityContextTokens(parsed);
|
|
19529
|
+
const dbTurn =
|
|
19530
|
+
parsed.type === "PLANNER_RESPONSE" && stepMap && Number.isFinite(parsed.step_index)
|
|
19531
|
+
? stepMap.get(parsed.step_index)
|
|
19532
|
+
: null;
|
|
19533
|
+
const dbContextTokens = dbTurn && dbTurn.contextTokens > 0 ? dbTurn.contextTokens : 0;
|
|
19534
|
+
if (dbTurn && dbTurn.model) {
|
|
19535
|
+
const norm = normalizeAntigravityTranscriptModel(dbTurn.model);
|
|
19536
|
+
if (norm) currentModel = norm;
|
|
19537
|
+
}
|
|
18808
19538
|
|
|
18809
19539
|
if (!isNewEvent) {
|
|
18810
|
-
|
|
19540
|
+
if (parsed.type === "PLANNER_RESPONSE") {
|
|
19541
|
+
if (dbContextTokens > 0) {
|
|
19542
|
+
contextTokens = dbContextTokens;
|
|
19543
|
+
}
|
|
19544
|
+
previousContextTokens = contextTokens;
|
|
19545
|
+
lastPlannerModel = currentModel;
|
|
19546
|
+
contextTokens += eventContextTokens;
|
|
19547
|
+
} else {
|
|
19548
|
+
contextTokens += eventContextTokens;
|
|
19549
|
+
}
|
|
18811
19550
|
lastCompletedLine = i + 1;
|
|
18812
19551
|
continue;
|
|
18813
19552
|
}
|
|
@@ -18834,7 +19573,14 @@ async function parseAntigravityFile({
|
|
|
18834
19573
|
const content = typeof parsed.content === "string" ? parsed.content : "";
|
|
18835
19574
|
const thinking = typeof parsed.thinking === "string" ? parsed.thinking : "";
|
|
18836
19575
|
|
|
19576
|
+
if (dbContextTokens > 0) {
|
|
19577
|
+
contextTokens = dbContextTokens;
|
|
19578
|
+
}
|
|
19579
|
+
if (lastPlannerModel && model !== lastPlannerModel) {
|
|
19580
|
+
previousContextTokens = 0;
|
|
19581
|
+
}
|
|
18837
19582
|
const inputDelta = Math.max(0, contextTokens - previousContextTokens);
|
|
19583
|
+
|
|
18838
19584
|
const outputTokens =
|
|
18839
19585
|
antigravityValueTokens(content) + antigravityValueTokens(parsed.tool_calls);
|
|
18840
19586
|
const reasoningTokens = antigravityValueTokens(thinking);
|
|
@@ -18842,8 +19588,6 @@ async function parseAntigravityFile({
|
|
|
18842
19588
|
delta.input_tokens = inputDelta;
|
|
18843
19589
|
delta.output_tokens = outputTokens;
|
|
18844
19590
|
delta.reasoning_output_tokens = reasoningTokens;
|
|
18845
|
-
// Match the mainstream convention (Codebuddy / Kilocode / OMP / Hermes):
|
|
18846
|
-
// total_tokens = sum of every token column. No cache columns here.
|
|
18847
19591
|
delta.total_tokens = inputDelta + outputTokens + reasoningTokens;
|
|
18848
19592
|
delta.billable_total_tokens = delta.total_tokens;
|
|
18849
19593
|
delta.conversation_count = 1;
|
|
@@ -18877,6 +19621,7 @@ async function parseAntigravityFile({
|
|
|
18877
19621
|
// so they MUST be billed as input on the next planner — don't fold them into
|
|
18878
19622
|
// previousContextTokens or that history vanishes from the totals.
|
|
18879
19623
|
previousContextTokens = contextTokens;
|
|
19624
|
+
lastPlannerModel = model;
|
|
18880
19625
|
contextTokens += eventContextTokens;
|
|
18881
19626
|
lastCompletedLine = i + 1;
|
|
18882
19627
|
}
|
|
@@ -18887,6 +19632,8 @@ async function parseAntigravityFile({
|
|
|
18887
19632
|
contextTokens,
|
|
18888
19633
|
previousContextTokens,
|
|
18889
19634
|
currentModel,
|
|
19635
|
+
lastPlannerModel,
|
|
19636
|
+
usageSource: stepMap ? "sqlite" : "estimated",
|
|
18890
19637
|
};
|
|
18891
19638
|
}
|
|
18892
19639
|
|
|
@@ -19625,9 +20372,10 @@ async function parseTraeCnApiIncremental({
|
|
|
19625
20372
|
// built-in zlib zstd first, `@mongodb-js/zstd` as the Node 20 fallback —
|
|
19626
20373
|
// enforcing a cumulative plaintext bound as we go.
|
|
19627
20374
|
// Dedup is a per-file `lastSeq` watermark — seq
|
|
19628
|
-
// is monotonic within
|
|
19629
|
-
// and skips everything at or below the watermark
|
|
19630
|
-
//
|
|
20375
|
+
// is monotonic within an append-only session log, so a grow re-reads the file
|
|
20376
|
+
// and skips everything at or below the watermark. Torn tails never advance the
|
|
20377
|
+
// watermark until their JSON record is complete; format replacements are
|
|
20378
|
+
// reconciled through per-session contribution ledgers.
|
|
19631
20379
|
const DSH_SESSION_LOG_MAX_BYTES = 64 * 1024 * 1024;
|
|
19632
20380
|
const DSH_SESSION_TEXT_MAX_BYTES = 128 * 1024 * 1024;
|
|
19633
20381
|
const DSH_SOURCE = "dsh";
|
|
@@ -19677,13 +20425,20 @@ function resolveDshHomes(env = process.env, deps = {}) {
|
|
|
19677
20425
|
return [...new Set([resolved.native, resolved.wsl].filter(Boolean))];
|
|
19678
20426
|
}
|
|
19679
20427
|
|
|
20428
|
+
const DSH_SESSION_LOG_PATTERN = /^session(?:\.v\d+)?\.jsonl(?:\.zstd)?$/;
|
|
20429
|
+
|
|
19680
20430
|
function isDshSessionLogName(name) {
|
|
19681
|
-
return name === "
|
|
20431
|
+
return typeof name === "string" && DSH_SESSION_LOG_PATTERN.test(name);
|
|
20432
|
+
}
|
|
20433
|
+
|
|
20434
|
+
function parseDshVersion(name) {
|
|
20435
|
+
const match = typeof name === "string" ? name.match(/\.v(\d+)\.jsonl/) : null;
|
|
20436
|
+
return match ? parseInt(match[1], 10) : 0;
|
|
19682
20437
|
}
|
|
19683
20438
|
|
|
19684
20439
|
// Walk the harness sessions root for per-session log artifacts. The tree is
|
|
19685
|
-
// `<sessions-root>/<project-key>/<session-id>/session.jsonl[.zstd]`; only
|
|
19686
|
-
//
|
|
20440
|
+
// `<sessions-root>/<project-key>/<session-id>/session[.v3].jsonl[.zstd]`; only
|
|
20441
|
+
// the matching leaf names are collected so unrelated harness files are ignored.
|
|
19687
20442
|
async function resolveDshSessionFiles(env = process.env, deps = {}) {
|
|
19688
20443
|
const out = [];
|
|
19689
20444
|
const seen = new Set();
|
|
@@ -19707,7 +20462,7 @@ async function resolveDshSessionFiles(env = process.env, deps = {}) {
|
|
|
19707
20462
|
} else if (transcripts.length > 1) {
|
|
19708
20463
|
// Harness itself rejects mixed encodings in one root. For a passive
|
|
19709
20464
|
// reader, choose the actively-written artifact instead of counting the
|
|
19710
|
-
// same session twice; a tie prefers the
|
|
20465
|
+
// same session twice; a tie prefers the higher format version, then zstd.
|
|
19711
20466
|
const ranked = await Promise.all(transcripts.map(async (artifact) => {
|
|
19712
20467
|
const full = path.join(sessionDir, artifact.name);
|
|
19713
20468
|
const handle = await fs.open(full, "r").catch(() => null);
|
|
@@ -19725,6 +20480,7 @@ async function resolveDshSessionFiles(env = process.env, deps = {}) {
|
|
|
19725
20480
|
}));
|
|
19726
20481
|
ranked.sort((left, right) =>
|
|
19727
20482
|
right.mtimeMs - left.mtimeMs ||
|
|
20483
|
+
parseDshVersion(right.name) - parseDshVersion(left.name) ||
|
|
19728
20484
|
Number(right.name.endsWith(".zstd")) - Number(left.name.endsWith(".zstd")),
|
|
19729
20485
|
);
|
|
19730
20486
|
selected = ranked[0]?.full || null;
|
|
@@ -19915,6 +20671,161 @@ function sameDshSessionMetadata(previous, current) {
|
|
|
19915
20671
|
);
|
|
19916
20672
|
}
|
|
19917
20673
|
|
|
20674
|
+
function dshSessionDirectoryKey(filePath) {
|
|
20675
|
+
return typeof filePath === "string" ? path.dirname(path.resolve(filePath)) : null;
|
|
20676
|
+
}
|
|
20677
|
+
|
|
20678
|
+
function normalizeDshContributions(value) {
|
|
20679
|
+
const normalized = {};
|
|
20680
|
+
if (!value || typeof value !== "object") return normalized;
|
|
20681
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
20682
|
+
if (!entry || typeof entry !== "object" || !entry.totals) continue;
|
|
20683
|
+
const model = normalizeModelInput(entry.model);
|
|
20684
|
+
const bucketStart = typeof entry.bucketStart === "string" ? entry.bucketStart : null;
|
|
20685
|
+
if (!model || !bucketStart) continue;
|
|
20686
|
+
normalized[key] = {
|
|
20687
|
+
model,
|
|
20688
|
+
bucketStart,
|
|
20689
|
+
totals: cloneTotals(entry.totals),
|
|
20690
|
+
};
|
|
20691
|
+
}
|
|
20692
|
+
return normalized;
|
|
20693
|
+
}
|
|
20694
|
+
|
|
20695
|
+
function storedDshContributions(state) {
|
|
20696
|
+
if (
|
|
20697
|
+
!state ||
|
|
20698
|
+
typeof state !== "object" ||
|
|
20699
|
+
!Object.prototype.hasOwnProperty.call(state, "contributions") ||
|
|
20700
|
+
!state.contributions ||
|
|
20701
|
+
typeof state.contributions !== "object"
|
|
20702
|
+
) {
|
|
20703
|
+
return null;
|
|
20704
|
+
}
|
|
20705
|
+
const normalized = normalizeDshContributions(state.contributions);
|
|
20706
|
+
// A partially written or hand-edited ledger is not a safe subtraction
|
|
20707
|
+
// baseline. Treat it as absent so migration defers instead of silently
|
|
20708
|
+
// retracting only some of a session's contribution.
|
|
20709
|
+
if (Object.keys(normalized).length !== Object.keys(state.contributions).length) {
|
|
20710
|
+
return null;
|
|
20711
|
+
}
|
|
20712
|
+
return normalized;
|
|
20713
|
+
}
|
|
20714
|
+
|
|
20715
|
+
function addDshContribution(contributions, model, bucketStart, totals) {
|
|
20716
|
+
const key = bucketKey(DSH_SOURCE, model, bucketStart);
|
|
20717
|
+
let contribution = contributions[key];
|
|
20718
|
+
if (!contribution) {
|
|
20719
|
+
contribution = {
|
|
20720
|
+
model,
|
|
20721
|
+
bucketStart,
|
|
20722
|
+
totals: initTotals(),
|
|
20723
|
+
};
|
|
20724
|
+
contributions[key] = contribution;
|
|
20725
|
+
}
|
|
20726
|
+
addTotals(contribution.totals, totals);
|
|
20727
|
+
}
|
|
20728
|
+
|
|
20729
|
+
function dshContributionsFromDeltas(deltas) {
|
|
20730
|
+
const contributions = {};
|
|
20731
|
+
for (const delta of deltas || []) {
|
|
20732
|
+
const bucketStart = toUtcHalfHourStart(delta.timeMs);
|
|
20733
|
+
if (!bucketStart || !delta.model || !delta.totals) continue;
|
|
20734
|
+
addDshContribution(contributions, delta.model, bucketStart, delta.totals);
|
|
20735
|
+
}
|
|
20736
|
+
return contributions;
|
|
20737
|
+
}
|
|
20738
|
+
|
|
20739
|
+
function legacyDshContributionsFromSnapshot(snapshot, previousState) {
|
|
20740
|
+
if (
|
|
20741
|
+
!snapshot?.text ||
|
|
20742
|
+
!previousState ||
|
|
20743
|
+
typeof previousState !== "object" ||
|
|
20744
|
+
!Number.isSafeInteger(previousState.lastSeq) ||
|
|
20745
|
+
previousState.lastSeq < -1 ||
|
|
20746
|
+
previousState.inode !== snapshot.inode ||
|
|
20747
|
+
!Number.isFinite(previousState.size) ||
|
|
20748
|
+
snapshot.size < previousState.size
|
|
20749
|
+
) {
|
|
20750
|
+
return null;
|
|
20751
|
+
}
|
|
20752
|
+
const parsed = extractDshSessionUsage(snapshot.text, -1);
|
|
20753
|
+
if (!parsed.complete || parsed.sessionId !== previousState.sessionId) return null;
|
|
20754
|
+
// The pre-ledger parser replayed unknown-sequence usage on every pass, so
|
|
20755
|
+
// there is no safe prefix boundary for such a record. Defer instead.
|
|
20756
|
+
if (parsed.deltas.some((delta) => !Number.isSafeInteger(delta.seq))) return null;
|
|
20757
|
+
return dshContributionsFromDeltas(
|
|
20758
|
+
parsed.deltas.filter((delta) => delta.seq <= previousState.lastSeq),
|
|
20759
|
+
);
|
|
20760
|
+
}
|
|
20761
|
+
|
|
20762
|
+
function dshContributionsCoverPrior(prior, candidate) {
|
|
20763
|
+
for (const [key, previous] of Object.entries(prior || {})) {
|
|
20764
|
+
const current = candidate?.[key];
|
|
20765
|
+
if (!current?.totals || !previous?.totals) return false;
|
|
20766
|
+
for (const field of [
|
|
20767
|
+
"input_tokens",
|
|
20768
|
+
"cached_input_tokens",
|
|
20769
|
+
"cache_creation_input_tokens",
|
|
20770
|
+
"output_tokens",
|
|
20771
|
+
"reasoning_output_tokens",
|
|
20772
|
+
"total_tokens",
|
|
20773
|
+
"billable_total_tokens",
|
|
20774
|
+
"total_cost_usd",
|
|
20775
|
+
"conversation_count",
|
|
20776
|
+
]) {
|
|
20777
|
+
const available = Number(current.totals[field] || 0);
|
|
20778
|
+
const required = Number(previous.totals[field] || 0);
|
|
20779
|
+
if (!Number.isFinite(available) || !Number.isFinite(required) || available < required) {
|
|
20780
|
+
return false;
|
|
20781
|
+
}
|
|
20782
|
+
}
|
|
20783
|
+
}
|
|
20784
|
+
return true;
|
|
20785
|
+
}
|
|
20786
|
+
|
|
20787
|
+
function dshContributionsFitHourlyState(hourlyState, contributions) {
|
|
20788
|
+
for (const contribution of Object.values(contributions || {})) {
|
|
20789
|
+
if (!contribution?.model || !contribution.bucketStart || !contribution.totals) continue;
|
|
20790
|
+
const key = bucketKey(DSH_SOURCE, contribution.model, contribution.bucketStart);
|
|
20791
|
+
const bucket = hourlyState?.buckets?.[key];
|
|
20792
|
+
if (!bucket?.totals) return false;
|
|
20793
|
+
for (const field of [
|
|
20794
|
+
"input_tokens",
|
|
20795
|
+
"cached_input_tokens",
|
|
20796
|
+
"cache_creation_input_tokens",
|
|
20797
|
+
"output_tokens",
|
|
20798
|
+
"reasoning_output_tokens",
|
|
20799
|
+
"total_tokens",
|
|
20800
|
+
"billable_total_tokens",
|
|
20801
|
+
"total_cost_usd",
|
|
20802
|
+
"conversation_count",
|
|
20803
|
+
]) {
|
|
20804
|
+
const available = Number(bucket.totals[field] || 0);
|
|
20805
|
+
const required = Number(contribution.totals[field] || 0);
|
|
20806
|
+
if (!Number.isFinite(available) || !Number.isFinite(required) || available < required) {
|
|
20807
|
+
return false;
|
|
20808
|
+
}
|
|
20809
|
+
}
|
|
20810
|
+
}
|
|
20811
|
+
return true;
|
|
20812
|
+
}
|
|
20813
|
+
|
|
20814
|
+
function applyDshContributions({ hourlyState, touchedBuckets, contributions, subtract = false }) {
|
|
20815
|
+
for (const contribution of Object.values(contributions || {})) {
|
|
20816
|
+
if (!contribution?.model || !contribution.bucketStart || !contribution.totals) continue;
|
|
20817
|
+
const bucket = getHourlyBucket(
|
|
20818
|
+
hourlyState,
|
|
20819
|
+
DSH_SOURCE,
|
|
20820
|
+
contribution.model,
|
|
20821
|
+
contribution.bucketStart,
|
|
20822
|
+
);
|
|
20823
|
+
if (subtract) subtractTotals(bucket.totals, contribution.totals);
|
|
20824
|
+
else addTotals(bucket.totals, contribution.totals);
|
|
20825
|
+
touchedBuckets.add(bucketKey(DSH_SOURCE, contribution.model, contribution.bucketStart));
|
|
20826
|
+
}
|
|
20827
|
+
}
|
|
20828
|
+
|
|
19918
20829
|
// Open, inspect and read through one handle so a path replacement cannot make
|
|
19919
20830
|
// the metadata describe a different file from the bytes we parse.
|
|
19920
20831
|
async function readDshSessionSnapshot(
|
|
@@ -20128,25 +21039,149 @@ function parseDshUsageSlice(raw) {
|
|
|
20128
21039
|
};
|
|
20129
21040
|
}
|
|
20130
21041
|
|
|
21042
|
+
function isCompleteDshJsonLine(raw) {
|
|
21043
|
+
const text = String(raw || "").trim();
|
|
21044
|
+
if (!text || text[0] !== "{") return false;
|
|
21045
|
+
let index = 0;
|
|
21046
|
+
const maxDepth = 256;
|
|
21047
|
+
const hex = (char) => /[0-9a-f]/i.test(char || "");
|
|
21048
|
+
const skipWhitespace = () => {
|
|
21049
|
+
while (index < text.length && /\s/.test(text[index])) index += 1;
|
|
21050
|
+
};
|
|
21051
|
+
const parseString = () => {
|
|
21052
|
+
if (text[index] !== '"') return false;
|
|
21053
|
+
index += 1;
|
|
21054
|
+
while (index < text.length) {
|
|
21055
|
+
const char = text[index++];
|
|
21056
|
+
if (char === '"') return true;
|
|
21057
|
+
if (char.charCodeAt(0) < 0x20) return false;
|
|
21058
|
+
if (char !== "\\") continue;
|
|
21059
|
+
if (index >= text.length) return false;
|
|
21060
|
+
const escaped = text[index++];
|
|
21061
|
+
if (escaped === "u") {
|
|
21062
|
+
if (index + 4 > text.length || ![...text.slice(index, index + 4)].every(hex)) return false;
|
|
21063
|
+
index += 4;
|
|
21064
|
+
} else if (!'"\\/bfnrt'.includes(escaped)) {
|
|
21065
|
+
return false;
|
|
21066
|
+
}
|
|
21067
|
+
}
|
|
21068
|
+
return false;
|
|
21069
|
+
};
|
|
21070
|
+
const parseNumber = () => {
|
|
21071
|
+
const start = index;
|
|
21072
|
+
if (text[index] === "-") index += 1;
|
|
21073
|
+
if (text[index] === "0") {
|
|
21074
|
+
index += 1;
|
|
21075
|
+
} else if (/[1-9]/.test(text[index] || "")) {
|
|
21076
|
+
while (/[0-9]/.test(text[index] || "")) index += 1;
|
|
21077
|
+
} else {
|
|
21078
|
+
return false;
|
|
21079
|
+
}
|
|
21080
|
+
if (text[index] === ".") {
|
|
21081
|
+
index += 1;
|
|
21082
|
+
const fractionStart = index;
|
|
21083
|
+
while (/[0-9]/.test(text[index] || "")) index += 1;
|
|
21084
|
+
if (index === fractionStart) return false;
|
|
21085
|
+
}
|
|
21086
|
+
if (text[index] === "e" || text[index] === "E") {
|
|
21087
|
+
index += 1;
|
|
21088
|
+
if (text[index] === "+" || text[index] === "-") index += 1;
|
|
21089
|
+
const exponentStart = index;
|
|
21090
|
+
while (/[0-9]/.test(text[index] || "")) index += 1;
|
|
21091
|
+
if (index === exponentStart) return false;
|
|
21092
|
+
}
|
|
21093
|
+
return index > start;
|
|
21094
|
+
};
|
|
21095
|
+
const parseValue = (depth) => {
|
|
21096
|
+
if (depth > maxDepth) return false;
|
|
21097
|
+
skipWhitespace();
|
|
21098
|
+
const char = text[index];
|
|
21099
|
+
if (char === '"') return parseString();
|
|
21100
|
+
if (char === "{") {
|
|
21101
|
+
index += 1;
|
|
21102
|
+
skipWhitespace();
|
|
21103
|
+
if (text[index] === "}") {
|
|
21104
|
+
index += 1;
|
|
21105
|
+
return true;
|
|
21106
|
+
}
|
|
21107
|
+
while (index < text.length) {
|
|
21108
|
+
skipWhitespace();
|
|
21109
|
+
if (!parseString()) return false;
|
|
21110
|
+
skipWhitespace();
|
|
21111
|
+
if (text[index++] !== ":") return false;
|
|
21112
|
+
if (!parseValue(depth + 1)) return false;
|
|
21113
|
+
skipWhitespace();
|
|
21114
|
+
if (text[index] === "}") {
|
|
21115
|
+
index += 1;
|
|
21116
|
+
return true;
|
|
21117
|
+
}
|
|
21118
|
+
if (text[index++] !== ",") return false;
|
|
21119
|
+
}
|
|
21120
|
+
return false;
|
|
21121
|
+
}
|
|
21122
|
+
if (char === "[") {
|
|
21123
|
+
index += 1;
|
|
21124
|
+
skipWhitespace();
|
|
21125
|
+
if (text[index] === "]") {
|
|
21126
|
+
index += 1;
|
|
21127
|
+
return true;
|
|
21128
|
+
}
|
|
21129
|
+
while (index < text.length) {
|
|
21130
|
+
if (!parseValue(depth + 1)) return false;
|
|
21131
|
+
skipWhitespace();
|
|
21132
|
+
if (text[index] === "]") {
|
|
21133
|
+
index += 1;
|
|
21134
|
+
return true;
|
|
21135
|
+
}
|
|
21136
|
+
if (text[index++] !== ",") return false;
|
|
21137
|
+
skipWhitespace();
|
|
21138
|
+
}
|
|
21139
|
+
return false;
|
|
21140
|
+
}
|
|
21141
|
+
for (const literal of ["true", "false", "null"]) {
|
|
21142
|
+
if (text.startsWith(literal, index)) {
|
|
21143
|
+
index += literal.length;
|
|
21144
|
+
return true;
|
|
21145
|
+
}
|
|
21146
|
+
}
|
|
21147
|
+
return parseNumber();
|
|
21148
|
+
};
|
|
21149
|
+
|
|
21150
|
+
if (!parseValue(0)) return false;
|
|
21151
|
+
skipWhitespace();
|
|
21152
|
+
return index === text.length;
|
|
21153
|
+
}
|
|
21154
|
+
|
|
20131
21155
|
// Parse one session log's plaintext into usage deltas, skipping events whose
|
|
20132
21156
|
// seq is at or below the watermark. Returns the deltas (each carrying the
|
|
20133
|
-
// model and epoch-ms timestamp), the highest seq seen,
|
|
21157
|
+
// sequence, model and epoch-ms timestamp), the highest complete seq seen, the
|
|
21158
|
+
// session id, and whether every non-empty JSONL line was complete.
|
|
20134
21159
|
function extractDshSessionUsage(text, lastSeq = -1) {
|
|
20135
21160
|
const deltas = [];
|
|
20136
21161
|
const watermark = Number.isFinite(lastSeq) ? lastSeq : -1;
|
|
20137
21162
|
let maxSeq = watermark;
|
|
20138
21163
|
let sessionId = null;
|
|
20139
21164
|
let headerModel = null;
|
|
21165
|
+
let complete = true;
|
|
20140
21166
|
|
|
20141
21167
|
const lines = String(text || "").split("\n");
|
|
20142
21168
|
for (let i = 0; i < lines.length; i++) {
|
|
20143
21169
|
const line = lines[i];
|
|
20144
21170
|
if (!line || !line.trim()) continue;
|
|
21171
|
+
if (!isCompleteDshJsonLine(line)) {
|
|
21172
|
+
complete = false;
|
|
21173
|
+
// A later event cannot make this gap safe to acknowledge. Stop at the
|
|
21174
|
+
// complete prefix so a repaired record is retried before seq advances.
|
|
21175
|
+
break;
|
|
21176
|
+
}
|
|
20145
21177
|
const eventType = parseDshJsonString(findDshJsonProperty(line, "type"));
|
|
20146
21178
|
if (!eventType) continue;
|
|
20147
21179
|
|
|
20148
|
-
if (eventType === "session") {
|
|
20149
|
-
const id =
|
|
21180
|
+
if (eventType === "session" || eventType === "session/start") {
|
|
21181
|
+
const id =
|
|
21182
|
+
parseDshJsonString(findDshJsonProperty(line, "id")) ||
|
|
21183
|
+
parseDshJsonString(findDshJsonProperty(findDshJsonProperty(line, "data"), "id")) ||
|
|
21184
|
+
parseDshJsonString(findDshJsonProperty(findDshJsonProperty(line, "data"), "sessionId"));
|
|
20150
21185
|
if (id) sessionId = id;
|
|
20151
21186
|
continue;
|
|
20152
21187
|
}
|
|
@@ -20167,114 +21202,397 @@ function extractDshSessionUsage(text, lastSeq = -1) {
|
|
|
20167
21202
|
continue;
|
|
20168
21203
|
}
|
|
20169
21204
|
|
|
20170
|
-
if (eventType !== "assistant/message") continue;
|
|
21205
|
+
if (eventType !== "assistant/message" && eventType !== "message/assistant") continue;
|
|
20171
21206
|
if (seqKnown && seq <= watermark) continue;
|
|
20172
21207
|
|
|
20173
21208
|
const message = findDshJsonProperty(data, "message");
|
|
20174
|
-
const source = findDshJsonProperty(message, "source");
|
|
21209
|
+
const source = message ? findDshJsonProperty(message, "source") : null;
|
|
20175
21210
|
const model = normalizeDshModelName(
|
|
20176
|
-
parseDshJsonString(findDshJsonProperty(source, "model"))
|
|
21211
|
+
(source && parseDshJsonString(findDshJsonProperty(source, "model"))) ||
|
|
21212
|
+
parseDshJsonString(findDshJsonProperty(data, "model")),
|
|
20177
21213
|
) || headerModel;
|
|
20178
21214
|
const totals = dshUsageToTotals(
|
|
20179
|
-
parseDshUsageSlice(
|
|
21215
|
+
parseDshUsageSlice(
|
|
21216
|
+
findDshJsonProperty(data, "usage") || findDshJsonProperty(line, "usage"),
|
|
21217
|
+
),
|
|
20180
21218
|
);
|
|
20181
21219
|
if (!model || !totals) continue;
|
|
20182
21220
|
|
|
20183
|
-
const timeMs =
|
|
21221
|
+
const timeMs =
|
|
21222
|
+
parseDshJsonNumber(findDshJsonProperty(line, "time")) ||
|
|
21223
|
+
parseDshJsonNumber(findDshJsonProperty(line, "timestamp")) ||
|
|
21224
|
+
parseDshJsonNumber(findDshJsonProperty(data, "time")) ||
|
|
21225
|
+
parseDshJsonNumber(findDshJsonProperty(data, "timestamp"));
|
|
20184
21226
|
if (!Number.isFinite(timeMs) || timeMs <= 0) continue;
|
|
20185
21227
|
|
|
20186
|
-
deltas.push({ model, timeMs, totals });
|
|
21228
|
+
deltas.push({ seq: seqKnown ? seq : null, model, timeMs, totals });
|
|
20187
21229
|
}
|
|
20188
21230
|
|
|
20189
|
-
return { deltas, maxSeq, sessionId };
|
|
21231
|
+
return { deltas, maxSeq, sessionId, complete };
|
|
20190
21232
|
}
|
|
20191
21233
|
|
|
20192
21234
|
// Incremental parser entrypoint. Mirrors the other passive JSONL readers:
|
|
20193
21235
|
// identity-check each file (inode/size/mtime), re-read + re-parse only when it
|
|
20194
21236
|
// changed, dedup via the per-file seq watermark, accumulate into hourly
|
|
20195
|
-
// buckets, then flush touched buckets to the queue.
|
|
21237
|
+
// buckets, then flush touched buckets to the queue. Session contribution ledgers
|
|
21238
|
+
// make artifact-path migrations replace one session instead of resetting DSH.
|
|
20196
21239
|
async function parseDshIncremental({ sessionFiles, cursors, queuePath, onProgress }) {
|
|
20197
21240
|
await ensureDir(path.dirname(queuePath));
|
|
20198
21241
|
if (!cursors || typeof cursors !== "object") cursors = {};
|
|
20199
21242
|
if (!cursors.dsh || typeof cursors.dsh !== "object") cursors.dsh = {};
|
|
20200
21243
|
const dshState = cursors.dsh;
|
|
20201
|
-
const
|
|
21244
|
+
const storedFileState =
|
|
20202
21245
|
dshState.files && typeof dshState.files === "object" ? dshState.files : {};
|
|
21246
|
+
let fileState = { ...storedFileState };
|
|
21247
|
+
const storedSessionState =
|
|
21248
|
+
dshState.sessions && typeof dshState.sessions === "object" ? dshState.sessions : {};
|
|
21249
|
+
const sessionState = { ...storedSessionState };
|
|
20203
21250
|
|
|
20204
21251
|
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
21252
|
+
// normalizeHourlyState preserves bucket objects for the other incremental
|
|
21253
|
+
// parsers. DSH needs transactional ownership because replacement validation
|
|
21254
|
+
// can defer after inspecting several files and queue append may fail.
|
|
21255
|
+
hourlyState.groupQueued = { ...(hourlyState.groupQueued || {}) };
|
|
21256
|
+
for (const [key, bucket] of Object.entries(hourlyState.buckets || {})) {
|
|
21257
|
+
hourlyState.buckets[key] = {
|
|
21258
|
+
...(bucket && typeof bucket === "object" ? bucket : {}),
|
|
21259
|
+
totals: {
|
|
21260
|
+
...initTotals(),
|
|
21261
|
+
...(bucket?.totals && typeof bucket.totals === "object" ? bucket.totals : {}),
|
|
21262
|
+
},
|
|
21263
|
+
};
|
|
21264
|
+
}
|
|
20205
21265
|
const touchedBuckets = new Set();
|
|
21266
|
+
const deferredMigrationPaths = new Map();
|
|
20206
21267
|
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
20207
21268
|
|
|
20208
|
-
const files = Array.isArray(sessionFiles)
|
|
20209
|
-
|
|
21269
|
+
const files = Array.isArray(sessionFiles)
|
|
21270
|
+
? sessionFiles.filter((filePath) => typeof filePath === "string")
|
|
21271
|
+
: [];
|
|
21272
|
+
const presentFiles = new Set(files);
|
|
20210
21273
|
const total = files.length;
|
|
21274
|
+
const deferredFilePaths = new Set();
|
|
20211
21275
|
let recordsProcessed = 0;
|
|
20212
21276
|
let eventsAggregated = 0;
|
|
20213
21277
|
|
|
21278
|
+
const staleFileForSession = (sessionId, currentPath) => {
|
|
21279
|
+
if (!sessionId) return null;
|
|
21280
|
+
for (const [filePath, state] of Object.entries(fileState)) {
|
|
21281
|
+
if (
|
|
21282
|
+
filePath !== currentPath &&
|
|
21283
|
+
!presentFiles.has(filePath) &&
|
|
21284
|
+
(state?.sessionId === sessionId ||
|
|
21285
|
+
(!state?.sessionId &&
|
|
21286
|
+
dshSessionDirectoryKey(filePath) === dshSessionDirectoryKey(currentPath)))
|
|
21287
|
+
) {
|
|
21288
|
+
return filePath;
|
|
21289
|
+
}
|
|
21290
|
+
}
|
|
21291
|
+
return null;
|
|
21292
|
+
};
|
|
21293
|
+
|
|
21294
|
+
const deferStaleMigrationPaths = (currentPath, reason) => {
|
|
21295
|
+
let deferred = 0;
|
|
21296
|
+
for (const oldPath of Object.keys(fileState)) {
|
|
21297
|
+
if (
|
|
21298
|
+
!presentFiles.has(oldPath) &&
|
|
21299
|
+
dshSessionDirectoryKey(oldPath) === dshSessionDirectoryKey(currentPath)
|
|
21300
|
+
) {
|
|
21301
|
+
if (!deferredMigrationPaths.has(oldPath)) deferred += 1;
|
|
21302
|
+
deferredFilePaths.add(oldPath);
|
|
21303
|
+
deferredMigrationPaths.set(oldPath, reason);
|
|
21304
|
+
}
|
|
21305
|
+
}
|
|
21306
|
+
return deferred;
|
|
21307
|
+
};
|
|
21308
|
+
|
|
21309
|
+
const reportProgress = (idx, recordsProcessed, eventsAggregated, total) => {
|
|
21310
|
+
if (cb) {
|
|
21311
|
+
cb({
|
|
21312
|
+
index: idx + 1,
|
|
21313
|
+
total,
|
|
21314
|
+
recordsProcessed,
|
|
21315
|
+
eventsAggregated,
|
|
21316
|
+
bucketsQueued: touchedBuckets.size,
|
|
21317
|
+
deferredMigrations: deferredMigrationPaths.size,
|
|
21318
|
+
});
|
|
21319
|
+
}
|
|
21320
|
+
};
|
|
21321
|
+
|
|
20214
21322
|
for (let idx = 0; idx < files.length; idx++) {
|
|
20215
21323
|
const filePath = files[idx];
|
|
20216
21324
|
const prev = fileState[filePath] || null;
|
|
21325
|
+
const previousSessionId = typeof prev?.sessionId === "string" ? prev.sessionId : null;
|
|
21326
|
+
const previousSession = previousSessionId ? sessionState[previousSessionId] : null;
|
|
21327
|
+
const previousFileContributions = storedDshContributions(prev);
|
|
21328
|
+
const previousSessionContributions = storedDshContributions(previousSession);
|
|
21329
|
+
const needsLedgerBackfill = Boolean(
|
|
21330
|
+
previousSessionId && !previousSessionContributions && !previousFileContributions,
|
|
21331
|
+
);
|
|
20217
21332
|
let snapshot;
|
|
20218
21333
|
let parsed;
|
|
21334
|
+
let fullParsed = null;
|
|
21335
|
+
let fileReset = false;
|
|
21336
|
+
let sessionChanged = false;
|
|
21337
|
+
|
|
20219
21338
|
try {
|
|
20220
|
-
snapshot = await readDshSessionSnapshot(filePath, {
|
|
20221
|
-
|
|
20222
|
-
|
|
20223
|
-
|
|
20224
|
-
|
|
21339
|
+
snapshot = await readDshSessionSnapshot(filePath, {
|
|
21340
|
+
previous: needsLedgerBackfill ? null : prev,
|
|
21341
|
+
});
|
|
21342
|
+
if (!snapshot) {
|
|
21343
|
+
// A resolver-selected replacement can disappear between discovery and
|
|
21344
|
+
// open. Preserve the old path so a later retry cannot re-add its full
|
|
21345
|
+
// contribution. This is deliberately separate from a thrown read
|
|
21346
|
+
// error because a missing path returns null from the snapshot helper.
|
|
21347
|
+
deferStaleMigrationPaths(filePath, "replacement-missing");
|
|
21348
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21349
|
+
continue;
|
|
21350
|
+
}
|
|
21351
|
+
if (snapshot.unchanged && !needsLedgerBackfill) {
|
|
21352
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
20225
21353
|
continue;
|
|
20226
21354
|
}
|
|
20227
21355
|
|
|
20228
|
-
const
|
|
21356
|
+
const previousHasInode = Number.isFinite(prev?.inode);
|
|
21357
|
+
const previousHasSize = Number.isFinite(prev?.size);
|
|
21358
|
+
const previousHasMtime = Number.isFinite(prev?.mtimeMs);
|
|
21359
|
+
fileReset = Boolean(
|
|
21360
|
+
prev &&
|
|
21361
|
+
(
|
|
21362
|
+
(previousHasInode && snapshot.inode !== prev.inode) ||
|
|
21363
|
+
(previousHasSize && snapshot.size < prev.size) ||
|
|
21364
|
+
(
|
|
21365
|
+
previousHasSize &&
|
|
21366
|
+
previousHasMtime &&
|
|
21367
|
+
snapshot.size <= prev.size &&
|
|
21368
|
+
snapshot.mtimeMs !== prev.mtimeMs
|
|
21369
|
+
)
|
|
21370
|
+
),
|
|
21371
|
+
);
|
|
21372
|
+
const lastSeq = fileReset || !Number.isFinite(prev?.lastSeq) ? -1 : prev.lastSeq;
|
|
20229
21373
|
parsed = extractDshSessionUsage(snapshot.text, lastSeq);
|
|
20230
|
-
|
|
20231
|
-
|
|
21374
|
+
sessionChanged = Boolean(
|
|
21375
|
+
prev &&
|
|
21376
|
+
parsed.sessionId &&
|
|
21377
|
+
parsed.sessionId !== previousSessionId,
|
|
21378
|
+
);
|
|
21379
|
+
if (sessionChanged) parsed = extractDshSessionUsage(snapshot.text, -1);
|
|
21380
|
+
if (!prev || needsLedgerBackfill || sessionChanged || fileReset) {
|
|
21381
|
+
fullParsed = extractDshSessionUsage(snapshot.text, -1);
|
|
21382
|
+
if (!prev || sessionChanged || fileReset) parsed = fullParsed;
|
|
20232
21383
|
}
|
|
21384
|
+
if (!parsed.complete) snapshot.mtimeMs = -1;
|
|
20233
21385
|
} catch (error) {
|
|
20234
21386
|
if (process.env.TOKENTRACKER_DEBUG) {
|
|
20235
21387
|
process.stderr.write(`[dsh] skipped ${filePath}: ${error?.message || error}\n`);
|
|
20236
21388
|
}
|
|
20237
|
-
|
|
20238
|
-
|
|
20239
|
-
|
|
21389
|
+
// A replacement artifact may have an old path that is no longer in the
|
|
21390
|
+
// selected file list. Keep that cursor until the replacement is readable
|
|
21391
|
+
// so a failed migration cannot turn into an untracked double-count.
|
|
21392
|
+
deferStaleMigrationPaths(filePath, "replacement-read-failed");
|
|
21393
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
20240
21394
|
continue;
|
|
20241
21395
|
}
|
|
20242
21396
|
|
|
21397
|
+
const sessionId = typeof parsed.sessionId === "string" ? parsed.sessionId : null;
|
|
21398
|
+
const stalePathWithoutId = sessionId
|
|
21399
|
+
? null
|
|
21400
|
+
: Object.keys(fileState).find(
|
|
21401
|
+
(oldPath) =>
|
|
21402
|
+
!presentFiles.has(oldPath) &&
|
|
21403
|
+
dshSessionDirectoryKey(oldPath) === dshSessionDirectoryKey(filePath),
|
|
21404
|
+
);
|
|
21405
|
+
if (stalePathWithoutId) {
|
|
21406
|
+
deferredFilePaths.add(stalePathWithoutId);
|
|
21407
|
+
deferredMigrationPaths.set(stalePathWithoutId, "replacement-session-id-missing");
|
|
21408
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21409
|
+
continue;
|
|
21410
|
+
}
|
|
21411
|
+
const session = sessionId ? sessionState[sessionId] : null;
|
|
21412
|
+
const stalePath = staleFileForSession(sessionId, filePath);
|
|
21413
|
+
const previousPath =
|
|
21414
|
+
session?.lastPath && session.lastPath !== filePath ? session.lastPath : stalePath;
|
|
21415
|
+
const resetCurrentFile = Boolean(fileReset && !sessionChanged);
|
|
21416
|
+
const replacementPath = previousPath || (resetCurrentFile ? filePath : null);
|
|
21417
|
+
const sessionContributions = storedDshContributions(session);
|
|
21418
|
+
const staleContributions = storedDshContributions(stalePath && fileState[stalePath]);
|
|
21419
|
+
const fileContributions = storedDshContributions(prev);
|
|
21420
|
+
let oldContributions = sessionContributions || staleContributions || fileContributions;
|
|
21421
|
+
|
|
21422
|
+
// A pure rename preserves the same physical bytes and metadata, so an old
|
|
21423
|
+
// cursor can safely adopt the new path without relying on rewritten seqs.
|
|
21424
|
+
const previousState = previousPath ? fileState[previousPath] : null;
|
|
21425
|
+
if (
|
|
21426
|
+
sessionId &&
|
|
21427
|
+
previousPath &&
|
|
21428
|
+
!oldContributions &&
|
|
21429
|
+
!prev &&
|
|
21430
|
+
parsed.complete &&
|
|
21431
|
+
previousState &&
|
|
21432
|
+
previousState.inode === snapshot.inode &&
|
|
21433
|
+
previousState.size === snapshot.size &&
|
|
21434
|
+
previousState.mtimeMs === snapshot.mtimeMs
|
|
21435
|
+
) {
|
|
21436
|
+
// A pure rename preserves the same bytes. Reusing the parsed full
|
|
21437
|
+
// contribution is safe here because replacement below subtracts and
|
|
21438
|
+
// re-adds it; no sequence watermark is transferred.
|
|
21439
|
+
oldContributions = dshContributionsFromDeltas(parsed.deltas);
|
|
21440
|
+
}
|
|
21441
|
+
|
|
21442
|
+
// Cursors written before the contribution ledger existed can still be
|
|
21443
|
+
// reconciled when the old artifact remains available and its same-inode
|
|
21444
|
+
// growth follows the Harness append-only contract. Reconstruct only the
|
|
21445
|
+
// prefix through the old watermark; using the whole current artifact would
|
|
21446
|
+
// subtract events that were appended after the old cursor was written.
|
|
21447
|
+
if (sessionId && previousPath && !oldContributions) {
|
|
21448
|
+
const oldState = fileState[previousPath];
|
|
21449
|
+
const oldSnapshot = await readDshSessionSnapshot(previousPath).catch(() => null);
|
|
21450
|
+
oldContributions = legacyDshContributionsFromSnapshot(oldSnapshot, oldState);
|
|
21451
|
+
}
|
|
21452
|
+
const replaceSession = Boolean(
|
|
21453
|
+
sessionId &&
|
|
21454
|
+
oldContributions &&
|
|
21455
|
+
(!prev || resetCurrentFile || (previousPath && previousPath !== filePath)),
|
|
21456
|
+
);
|
|
21457
|
+
const oldContributionCount = Object.keys(oldContributions || {}).length;
|
|
21458
|
+
if (sessionId && replacementPath && !oldContributions) {
|
|
21459
|
+
deferredFilePaths.add(replacementPath);
|
|
21460
|
+
deferredMigrationPaths.set(replacementPath, "legacy-baseline-unavailable");
|
|
21461
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21462
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21463
|
+
continue;
|
|
21464
|
+
}
|
|
21465
|
+
if (
|
|
21466
|
+
replaceSession &&
|
|
21467
|
+
(!parsed.complete || (oldContributionCount > 0 && parsed.deltas.length === 0))
|
|
21468
|
+
) {
|
|
21469
|
+
// A readable header-only or torn replacement is not authoritative. Keep
|
|
21470
|
+
// the old contribution until a complete replacement with usage arrives.
|
|
21471
|
+
deferredFilePaths.add(replacementPath);
|
|
21472
|
+
deferredMigrationPaths.set(
|
|
21473
|
+
replacementPath,
|
|
21474
|
+
parsed.complete ? "replacement-has-no-usage" : "replacement-incomplete",
|
|
21475
|
+
);
|
|
21476
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21477
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21478
|
+
continue;
|
|
21479
|
+
}
|
|
21480
|
+
const candidateContributions = dshContributionsFromDeltas(parsed.deltas);
|
|
21481
|
+
if (
|
|
21482
|
+
replaceSession &&
|
|
21483
|
+
oldContributionCount > 0 &&
|
|
21484
|
+
!dshContributionsCoverPrior(oldContributions, candidateContributions)
|
|
21485
|
+
) {
|
|
21486
|
+
// A complete-looking replacement that drops a previously counted bucket
|
|
21487
|
+
// is still unsafe. Format migration should preserve prior usage; defer
|
|
21488
|
+
// until a candidate with a verifiable superset arrives.
|
|
21489
|
+
deferredFilePaths.add(replacementPath);
|
|
21490
|
+
deferredMigrationPaths.set(replacementPath, "replacement-drops-prior-usage");
|
|
21491
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21492
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21493
|
+
continue;
|
|
21494
|
+
}
|
|
21495
|
+
if (replaceSession && !dshContributionsFitHourlyState(hourlyState, oldContributions)) {
|
|
21496
|
+
// Never let subtractTotals clamp away another session's history when the
|
|
21497
|
+
// persisted ledger and hourly bucket disagree. Defer for a repairable,
|
|
21498
|
+
// visible retry instead.
|
|
21499
|
+
deferredFilePaths.add(replacementPath);
|
|
21500
|
+
deferredMigrationPaths.set(replacementPath, "stored-contribution-not-in-bucket");
|
|
21501
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21502
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21503
|
+
continue;
|
|
21504
|
+
}
|
|
21505
|
+
|
|
21506
|
+
if (replaceSession) {
|
|
21507
|
+
applyDshContributions({
|
|
21508
|
+
hourlyState,
|
|
21509
|
+
touchedBuckets,
|
|
21510
|
+
contributions: oldContributions,
|
|
21511
|
+
subtract: true,
|
|
21512
|
+
});
|
|
21513
|
+
if (previousPath && !presentFiles.has(previousPath)) delete fileState[previousPath];
|
|
21514
|
+
}
|
|
21515
|
+
|
|
21516
|
+
let nextContributions = replaceSession
|
|
21517
|
+
? {}
|
|
21518
|
+
: sessionContributions || fileContributions || {};
|
|
21519
|
+
if (fullParsed) {
|
|
21520
|
+
nextContributions = dshContributionsFromDeltas(fullParsed.deltas);
|
|
21521
|
+
}
|
|
21522
|
+
|
|
20243
21523
|
for (const delta of parsed.deltas) {
|
|
20244
21524
|
const bucketStart = toUtcHalfHourStart(delta.timeMs);
|
|
20245
21525
|
if (!bucketStart) continue;
|
|
20246
21526
|
const bucket = getHourlyBucket(hourlyState, DSH_SOURCE, delta.model, bucketStart);
|
|
20247
21527
|
addTotals(bucket.totals, delta.totals);
|
|
20248
21528
|
touchedBuckets.add(bucketKey(DSH_SOURCE, delta.model, bucketStart));
|
|
21529
|
+
if (!fullParsed) addDshContribution(nextContributions, delta.model, bucketStart, delta.totals);
|
|
20249
21530
|
eventsAggregated += 1;
|
|
20250
21531
|
}
|
|
20251
21532
|
|
|
21533
|
+
const updatedAt = new Date().toISOString();
|
|
20252
21534
|
fileState[filePath] = {
|
|
20253
21535
|
inode: snapshot.inode,
|
|
20254
21536
|
size: snapshot.size,
|
|
20255
21537
|
mtimeMs: snapshot.mtimeMs,
|
|
20256
|
-
sessionId
|
|
21538
|
+
sessionId,
|
|
20257
21539
|
lastSeq: parsed.maxSeq,
|
|
20258
|
-
|
|
21540
|
+
contributions: nextContributions,
|
|
21541
|
+
updatedAt,
|
|
20259
21542
|
};
|
|
21543
|
+
if (sessionId) {
|
|
21544
|
+
sessionState[sessionId] = {
|
|
21545
|
+
lastPath: filePath,
|
|
21546
|
+
contributions: nextContributions,
|
|
21547
|
+
updatedAt,
|
|
21548
|
+
};
|
|
21549
|
+
}
|
|
20260
21550
|
recordsProcessed += 1;
|
|
20261
21551
|
|
|
20262
|
-
|
|
20263
|
-
cb({ index: idx + 1, total, recordsProcessed, eventsAggregated, bucketsQueued: touchedBuckets.size });
|
|
20264
|
-
}
|
|
21552
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
20265
21553
|
}
|
|
20266
21554
|
|
|
21555
|
+
const nowMs = Date.now();
|
|
20267
21556
|
for (const filePath of Object.keys(fileState)) {
|
|
20268
|
-
if (
|
|
21557
|
+
if (presentFiles.has(filePath) || deferredFilePaths.has(filePath)) continue;
|
|
21558
|
+
const state = fileState[filePath];
|
|
21559
|
+
const legacySessionId = typeof state?.sessionId === "string" ? state.sessionId : null;
|
|
21560
|
+
// Retain pre-ledger identity until a replacement is verified. Its counted
|
|
21561
|
+
// usage survives file deletion, so elapsed time cannot authorize replay.
|
|
21562
|
+
// Current ledgers retain the same identity in sessionState instead.
|
|
21563
|
+
if (legacySessionId && !storedDshContributions(state) && !sessionState[legacySessionId]) {
|
|
21564
|
+
if (!Number.isFinite(Number(state?.missingSince))) {
|
|
21565
|
+
fileState[filePath] = { ...state, missingSince: nowMs };
|
|
21566
|
+
}
|
|
21567
|
+
continue;
|
|
21568
|
+
}
|
|
21569
|
+
delete fileState[filePath];
|
|
20269
21570
|
}
|
|
20270
21571
|
|
|
20271
21572
|
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
20272
|
-
|
|
21573
|
+
const updatedAt = new Date().toISOString();
|
|
21574
|
+
hourlyState.updatedAt = updatedAt;
|
|
20273
21575
|
cursors.hourly = hourlyState;
|
|
20274
21576
|
dshState.files = fileState;
|
|
20275
|
-
dshState.
|
|
21577
|
+
dshState.sessions = sessionState;
|
|
21578
|
+
if (deferredMigrationPaths.size > 0) {
|
|
21579
|
+
const reasons = [...new Set(deferredMigrationPaths.values())].sort();
|
|
21580
|
+
dshState.deferredMigrations = {
|
|
21581
|
+
count: deferredMigrationPaths.size,
|
|
21582
|
+
reasons,
|
|
21583
|
+
updatedAt,
|
|
21584
|
+
};
|
|
21585
|
+
} else {
|
|
21586
|
+
delete dshState.deferredMigrations;
|
|
21587
|
+
}
|
|
21588
|
+
dshState.updatedAt = updatedAt;
|
|
20276
21589
|
|
|
20277
|
-
return {
|
|
21590
|
+
return {
|
|
21591
|
+
recordsProcessed,
|
|
21592
|
+
eventsAggregated,
|
|
21593
|
+
bucketsQueued,
|
|
21594
|
+
deferredMigrations: deferredMigrationPaths.size,
|
|
21595
|
+
};
|
|
20278
21596
|
}
|
|
20279
21597
|
|
|
20280
21598
|
|
|
@@ -20401,6 +21719,9 @@ module.exports = {
|
|
|
20401
21719
|
parseAnythingllmTimestamp,
|
|
20402
21720
|
readAnythingllmUsageRows,
|
|
20403
21721
|
parseAnythingllmIncremental,
|
|
21722
|
+
resolveDevinDbPath,
|
|
21723
|
+
readDevinUsageRows,
|
|
21724
|
+
parseDevinIncremental,
|
|
20404
21725
|
resolveGooseDbPath,
|
|
20405
21726
|
parseGooseModelName,
|
|
20406
21727
|
parseGooseCreatedAt,
|
|
@@ -20469,6 +21790,9 @@ module.exports = {
|
|
|
20469
21790
|
parseAntigravityIncremental,
|
|
20470
21791
|
estimateAntigravityTokens,
|
|
20471
21792
|
isCjkCodePoint,
|
|
21793
|
+
resolveAntigravityDbPath,
|
|
21794
|
+
extractAntigravityGenInfo,
|
|
21795
|
+
readAntigravityConversationDb,
|
|
20472
21796
|
|
|
20473
21797
|
// Trae SOLO (ByteDance AI IDE)
|
|
20474
21798
|
resolveTraePath,
|
|
@@ -20479,6 +21803,8 @@ module.exports = {
|
|
|
20479
21803
|
resolveDshHome,
|
|
20480
21804
|
resolveDshHomes,
|
|
20481
21805
|
resolveDshSessionFiles,
|
|
21806
|
+
isDshSessionLogName,
|
|
21807
|
+
parseDshVersion,
|
|
20482
21808
|
readDshSessionText,
|
|
20483
21809
|
decodeDshZstd,
|
|
20484
21810
|
inspectDshZstdFrames,
|