tokentracker-cli 0.96.2 → 0.96.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +34 -25
- package/README.ja.md +38 -27
- package/README.ko.md +39 -28
- package/README.md +36 -28
- package/README.zh-CN.md +36 -25
- package/dashboard/dist/assets/{AchievementBadge-CgEitoX1.js → AchievementBadge-CCcvmz6R.js} +1 -1
- package/dashboard/dist/assets/{AchievementsPage-dnnea0JH.js → AchievementsPage-C5jdktOK.js} +2 -2
- package/dashboard/dist/assets/{AchievementsSection-CJqG0UP7.js → AchievementsSection-BFMtUzn1.js} +1 -1
- package/dashboard/dist/assets/{BadgeDetailModal-BLV6SJoI.js → BadgeDetailModal-Be8gM7tM.js} +1 -1
- package/dashboard/dist/assets/{BotAnimated-Cm78yEbz.js → BotAnimated-CXTGR71z.js} +1 -1
- package/dashboard/dist/assets/{Card-IPGSYT6O.js → Card-CsUsp3CV.js} +1 -1
- package/dashboard/dist/assets/{ClawdAnimated-04w1U1sm.js → ClawdAnimated-BEjpZjEE.js} +2 -2
- package/dashboard/dist/assets/{CommandPalette-gSSa1V_s.js → CommandPalette-CHc_Z1r8.js} +1 -1
- package/dashboard/dist/assets/{CommunityStatsModal-CICwUz14.js → CommunityStatsModal-Gs6kZ_Mp.js} +1 -1
- package/dashboard/dist/assets/{ConfirmModal-DaWC0cY7.js → ConfirmModal-niHkpv2O.js} +1 -1
- package/dashboard/dist/assets/{Controls-BDihA03F.js → Controls-X2Z1UNUx.js} +1 -1
- package/dashboard/dist/assets/DashboardPage-B66s3Exi.js +1 -0
- package/dashboard/dist/assets/{DevicePage-BPpo_eiY.js → DevicePage-DQdLdEI8.js} +1 -1
- package/dashboard/dist/assets/DialogClose-CCUjTxl8.js +1 -0
- package/dashboard/dist/assets/DialogDescription-CjMjYG3N.js +1 -0
- package/dashboard/dist/assets/DialogTitle-CgkUGAag.js +1 -0
- package/dashboard/dist/assets/{FadeIn-DmI9PRLa.js → FadeIn-D4QD1bPo.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-CBO4cY7i.js → HeaderGithubStar-D0my-Z3a.js} +1 -1
- package/dashboard/dist/assets/{HoverTooltip-CPFGhi9K.js → HoverTooltip-nSWgrKlK.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-Do7w_6z6.js → IpCheckPage-4hfqHAgi.js} +1 -1
- package/dashboard/dist/assets/{LandingPage-at_Ti1ZE.js → LandingPage-wIc4tVRI.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardAvatar-BH1cnKPp.js → LeaderboardAvatar-DGlN8Vy8.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-Cd30aXwL.js → LeaderboardPage-CybuzE87.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProfileModal-CVsti_HN.js → LeaderboardProfileModal-D9la3tNd.js} +4 -4
- package/dashboard/dist/assets/{LeaderboardProfilePage-BfSpDM9e.js → LeaderboardProfilePage-gxXSeJYb.js} +1 -1
- package/dashboard/dist/assets/LimitsPage-CZH3G54Y.js +8 -0
- package/dashboard/dist/assets/{LocalOnlyNotice-9-vRgsdn.js → LocalOnlyNotice-Ckc7zohC.js} +1 -1
- package/dashboard/dist/assets/{LoginCard-DM_ZdFNl.js → LoginCard-BqeRHqWK.js} +1 -1
- package/dashboard/dist/assets/{LoginModal-BaNMznwh.js → LoginModal-BjH6aIBG.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-BMMthUf4.js → LoginPage-BOyPLNU3.js} +1 -1
- package/dashboard/dist/assets/PetPage-BXRXG40D.js +1 -0
- package/dashboard/dist/assets/PopoverPopup-Dad1ZYVd.js +1 -0
- package/dashboard/dist/assets/ProviderIcon-BDmrNJSl.js +1 -0
- package/dashboard/dist/assets/{ResetPasswordPage-Ca4SULSV.js → ResetPasswordPage-CZpE6qfa.js} +1 -1
- package/dashboard/dist/assets/{SegmentedControl-Bbsn-qvZ.js → SegmentedControl-CE1hrQFi.js} +1 -1
- package/dashboard/dist/assets/Select-YVj7u6Nv.js +1 -0
- package/dashboard/dist/assets/SelectItemText-BMYGgQ3o.js +1 -0
- package/dashboard/dist/assets/{ServiceStatusPage-BPS8KJ3a.js → ServiceStatusPage-HehjDbZg.js} +1 -1
- package/dashboard/dist/assets/{SessionsPage-C1yt3TMZ.js → SessionsPage-Chvu4YqZ.js} +2 -2
- package/dashboard/dist/assets/SettingsPage-BXjS7u9_.js +2 -0
- package/dashboard/dist/assets/SkillsPage-q3Rjlj9h.js +1 -0
- package/dashboard/dist/assets/{TokenGalaxy-DKvbJbnv.js → TokenGalaxy-BqZyj82r.js} +1 -1
- package/dashboard/dist/assets/ToolbarRootContext-Ch8ueI_C.js +1 -0
- package/dashboard/dist/assets/{TrendMonitor-YF5C_-L1.js → TrendMonitor-C2aTL9p1.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-BKV3DgXg.js → WidgetsPage-DYaIai8j.js} +1 -1
- package/dashboard/dist/assets/{WrappedPage-s4IRoLf7.js → WrappedPage-51B0v-Sr.js} +1 -1
- package/dashboard/dist/assets/{__vite-browser-external-CvMMIjhi.css → __vite-browser-external-5P7me0Qq.css} +1 -1
- package/dashboard/dist/assets/__vite-browser-external-Cp7dNuCm.js +45 -0
- package/dashboard/dist/assets/{agent-logos-Hbt-wyl2.js → agent-logos-DXrPSAdX.js} +1 -1
- package/dashboard/dist/assets/{arrow-up-right-KeIpKlw3.js → arrow-up-right-BFkf7l0U.js} +1 -1
- package/dashboard/dist/assets/{check-_QKyY1v_.js → check--N8wmgIJ.js} +1 -1
- package/dashboard/dist/assets/composite-zF5Pj94e.js +1 -0
- package/dashboard/dist/assets/{copy-DpLM3QMD.js → copy-BG0JMr6A.js} +1 -1
- package/dashboard/dist/assets/{download-C4NX-JnW.js → download-CPFJf-Sf.js} +1 -1
- package/dashboard/dist/assets/{external-link-CG-7FcOF.js → external-link-D2yF0kgP.js} +1 -1
- package/dashboard/dist/assets/{flame-CkWid7Ji.js → flame-l460AFx9.js} +1 -1
- package/dashboard/dist/assets/{format-tokens-CbwWhioZ.js → format-tokens-DKO4ljuH.js} +1 -1
- package/dashboard/dist/assets/{icons-BZq17rU3.js → icons-RoxRGkz4.js} +1 -1
- package/dashboard/dist/assets/{index-CR9tDiiC.js → index--ouT5L4N.js} +1 -1
- package/dashboard/dist/assets/{index-DDGLE0dz.js → index-DhyLCvvv.js} +1 -1
- package/dashboard/dist/assets/{index-CSwEiUn4.js → index-PTJ4368-.js} +1 -1
- package/dashboard/dist/assets/{info-mkeRjzqc.js → info-DNWMO4Ca.js} +1 -1
- package/dashboard/dist/assets/{link-2-BjMw8csC.js → link-2-DzWXAz1h.js} +1 -1
- package/dashboard/dist/assets/{loader-circle-cGWOJnWn.js → loader-circle-BzGMqXNu.js} +1 -1
- package/dashboard/dist/assets/{maximize-2-Cg_S73tD.js → maximize-2-Dkpyk4x3.js} +1 -1
- package/dashboard/dist/assets/{modifiers.esm-DNxTWxId.js → modifiers.esm-lggHUQpW.js} +1 -1
- package/dashboard/dist/assets/module-DUSXqJtA.js +3 -0
- package/dashboard/dist/assets/{monitor-smartphone-BvE9Tt2U.js → monitor-smartphone-Ck0-gz9l.js} +1 -1
- package/dashboard/dist/assets/{plus-BC2wdPRr.js → plus-qTJpnn43.js} +1 -1
- package/dashboard/dist/assets/provider-display-psY26qmp.js +42 -0
- package/dashboard/dist/assets/{react-D0kcslTz.js → react-CYS24KmX.js} +1 -1
- package/dashboard/dist/assets/{search-BuBNF3E3.js → search-CUzE6zxr.js} +1 -1
- package/dashboard/dist/assets/{skills-api-DxAN5pNx.js → skills-api-D9SzxPFG.js} +1 -1
- package/dashboard/dist/assets/{sortable.esm-DoqMcEX-.js → sortable.esm-Bl2hAUyR.js} +1 -1
- package/dashboard/dist/assets/store-mHSJAJtE.js +1 -0
- package/dashboard/dist/assets/{terminal-COWFUJdU.js → terminal-BlQNEvn1.js} +1 -1
- package/dashboard/dist/assets/{trash-2-Cqjki3RI.js → trash-CHdgmOyk.js} +1 -1
- package/dashboard/dist/assets/{use-community-stats-DsrlOPDM.js → use-community-stats-DUiCs91N.js} +1 -1
- package/dashboard/dist/assets/{use-latest-request-guard-ryRFL2gx.js → use-latest-request-guard-ehXfkvuU.js} +1 -1
- package/dashboard/dist/assets/use-limits-display-prefs-CcwwQ0Fq.js +1 -0
- package/dashboard/dist/assets/{use-ordered-list-D-Q86R1b.js → use-ordered-list-CCOtI3Wa.js} +1 -1
- package/dashboard/dist/assets/use-reduced-motion-CDlahPqe.js +1 -0
- package/dashboard/dist/assets/{use-session-efficiency-pref-CrNz45y7.js → use-session-efficiency-pref-CYRDpcc_.js} +1 -1
- package/dashboard/dist/assets/{use-transform-CoGM21px.js → use-transform-btLlTJpj.js} +1 -1
- package/dashboard/dist/assets/use-usage-limits-BGr5sR-c.js +1 -0
- package/dashboard/dist/assets/{useCurrency-CgCfDktb.js → useCurrency-CNutuEKd.js} +1 -1
- package/dashboard/dist/assets/{useTokenFormat-QKtzCKTL.js → useTokenFormat-C0bAJFMC.js} +1 -1
- package/dashboard/dist/assets/{zap-CTIobzgU.js → zap-B2fbyjbc.js} +1 -1
- package/dashboard/dist/brand-logos/acode-dark.png +0 -0
- package/dashboard/dist/brand-logos/acode.png +0 -0
- package/dashboard/dist/brand-logos/devin.svg +6 -0
- package/dashboard/dist/index.html +20 -16
- package/dashboard/dist/ip-check.html +2 -2
- package/dashboard/dist/leaderboard.html +2 -2
- package/dashboard/dist/llms.txt +4 -2
- package/dashboard/dist/share.html +2 -2
- package/package.json +2 -2
- package/src/commands/init.js +114 -4
- package/src/commands/status.js +37 -0
- package/src/commands/sync.js +92 -10
- package/src/commands/uninstall.js +23 -0
- package/src/lib/codex-config.js +42 -0
- package/src/lib/devin-limits.js +249 -0
- package/src/lib/devin-usage.js +212 -0
- package/src/lib/local-api.js +19 -0
- package/src/lib/passive-mode.js +13 -1
- package/src/lib/pricing/curated-overrides.json +67 -1
- package/src/lib/pricing/index.js +5 -2
- package/src/lib/pricing/matcher.js +28 -0
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +1310 -144
- package/src/lib/skills-manager.js +231 -30
- package/src/lib/subscriptions.js +70 -16
- package/src/lib/usage-limits.js +199 -49
- package/dashboard/dist/assets/DashboardPage-hV7wncKF.js +0 -1
- package/dashboard/dist/assets/DialogClose-C_dF-r3s.js +0 -1
- package/dashboard/dist/assets/DialogDescription-GClOSCaE.js +0 -1
- package/dashboard/dist/assets/DialogTitle-Buh_5688.js +0 -1
- package/dashboard/dist/assets/LimitsPage-_-BadDAj.js +0 -8
- package/dashboard/dist/assets/PetPage-Iq8Hswai.js +0 -1
- package/dashboard/dist/assets/PopoverPopup-DXr3JxVV.js +0 -1
- package/dashboard/dist/assets/ProviderIcon-sb2keSY2.js +0 -1
- package/dashboard/dist/assets/Select-vD-yj-qa.js +0 -1
- package/dashboard/dist/assets/SelectItemText-CRtFEuvg.js +0 -1
- package/dashboard/dist/assets/SettingsPage-CWVHrMer.js +0 -2
- package/dashboard/dist/assets/SkillsPage-DGWEH1xm.js +0 -1
- package/dashboard/dist/assets/ToolbarRootContext-D8yov1dW.js +0 -1
- package/dashboard/dist/assets/__vite-browser-external-_LUatzwU.js +0 -45
- package/dashboard/dist/assets/composite-BidfcMeb.js +0 -1
- package/dashboard/dist/assets/limits-providers-DRkcWQup.js +0 -1
- package/dashboard/dist/assets/module-xpSFyNBK.js +0 -3
- package/dashboard/dist/assets/provider-display-B7GggHQy.js +0 -42
- package/dashboard/dist/assets/store-Cg1HeUFM.js +0 -1
- package/dashboard/dist/assets/use-limits-display-prefs-DkSuu12m.js +0 -1
- package/dashboard/dist/assets/use-reduced-motion-D7NNqavo.js +0 -1
- package/dashboard/dist/assets/use-usage-limits-C_6L1sO5.js +0 -1
package/src/lib/rollout.js
CHANGED
|
@@ -21,6 +21,11 @@ const {
|
|
|
21
21
|
currentCodexModel,
|
|
22
22
|
snapshotCodexModelAttributionState,
|
|
23
23
|
} = require("./codex-model-attribution");
|
|
24
|
+
const {
|
|
25
|
+
DEVIN_TABLE_PROBE_SQL,
|
|
26
|
+
devinUsageSql,
|
|
27
|
+
buildDevinUsageEvents,
|
|
28
|
+
} = require("./devin-usage");
|
|
24
29
|
const { USD_TICKS_PER_USD, normalizeGrokUsage } = require("./grok-usage");
|
|
25
30
|
|
|
26
31
|
const DEFAULT_SOURCE = "codex";
|
|
@@ -259,15 +264,96 @@ async function parseRolloutIncremental({
|
|
|
259
264
|
typeof codexEventStore.add === "function"
|
|
260
265
|
? codexEventStore
|
|
261
266
|
: null;
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
cursors
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
267
|
+
const createRolloutEventDedup = ({ cursorKey, externalEventStore = null }) => {
|
|
268
|
+
const previousHashes = Array.isArray(cursors?.[cursorKey]) ? cursors[cursorKey] : [];
|
|
269
|
+
if (!externalEventStore && !Array.isArray(cursors[cursorKey])) {
|
|
270
|
+
cursors[cursorKey] = previousHashes;
|
|
271
|
+
}
|
|
272
|
+
let seenEvents = null;
|
|
273
|
+
let newEventKeys = null;
|
|
274
|
+
let newEventKeySet = null;
|
|
275
|
+
let appendOnlyEvents = null;
|
|
276
|
+
let historicalEvents = null;
|
|
277
|
+
|
|
278
|
+
const ensureNewEventKeys = () => {
|
|
279
|
+
if (!newEventKeys) newEventKeys = [];
|
|
280
|
+
if (!newEventKeySet) newEventKeySet = new Set();
|
|
281
|
+
};
|
|
282
|
+
const recordNewEvent = (key) => {
|
|
283
|
+
ensureNewEventKeys();
|
|
284
|
+
if (newEventKeySet.has(key)) return;
|
|
285
|
+
newEventKeySet.add(key);
|
|
286
|
+
newEventKeys.push(key);
|
|
287
|
+
if (seenEvents) seenEvents.add(key);
|
|
288
|
+
externalEventStore?.add(key);
|
|
289
|
+
};
|
|
290
|
+
const getAppendOnlyEvents = () => {
|
|
291
|
+
if (!appendOnlyEvents) {
|
|
292
|
+
appendOnlyEvents = {
|
|
293
|
+
has(key) {
|
|
294
|
+
return Boolean(newEventKeySet?.has(key) || seenEvents?.has(key));
|
|
295
|
+
},
|
|
296
|
+
add: recordNewEvent,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
return appendOnlyEvents;
|
|
300
|
+
};
|
|
301
|
+
const getHistoricalEvents = () => {
|
|
302
|
+
if (externalEventStore) {
|
|
303
|
+
if (!historicalEvents) {
|
|
304
|
+
historicalEvents = {
|
|
305
|
+
has(key) {
|
|
306
|
+
return Boolean(newEventKeySet?.has(key) || externalEventStore.has(key));
|
|
307
|
+
},
|
|
308
|
+
add: recordNewEvent,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
return historicalEvents;
|
|
312
|
+
}
|
|
313
|
+
if (!seenEvents) {
|
|
314
|
+
seenEvents = new Set(previousHashes);
|
|
315
|
+
for (const key of newEventKeys || []) seenEvents.add(key);
|
|
316
|
+
if (syncDiagnostics && cursorKey === "codexHashes") {
|
|
317
|
+
syncDiagnostics.hash_set_constructions += 1;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
if (!historicalEvents) {
|
|
321
|
+
historicalEvents = {
|
|
322
|
+
has: (key) => seenEvents.has(key),
|
|
323
|
+
add: recordNewEvent,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
return historicalEvents;
|
|
327
|
+
};
|
|
328
|
+
|
|
329
|
+
return {
|
|
330
|
+
getAppendOnlyEvents,
|
|
331
|
+
getHistoricalEvents,
|
|
332
|
+
persist() {
|
|
333
|
+
if (!externalEventStore) {
|
|
334
|
+
for (const key of newEventKeys || []) previousHashes.push(key);
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
size() {
|
|
338
|
+
return externalEventStore
|
|
339
|
+
? Number(externalEventStore.size || 0)
|
|
340
|
+
: Array.isArray(cursors[cursorKey])
|
|
341
|
+
? cursors[cursorKey].length
|
|
342
|
+
: previousHashes.length;
|
|
343
|
+
},
|
|
344
|
+
};
|
|
345
|
+
};
|
|
346
|
+
const codexEventDedup = createRolloutEventDedup({
|
|
347
|
+
cursorKey: "codexHashes",
|
|
348
|
+
externalEventStore: externalCodexEventStore,
|
|
349
|
+
});
|
|
350
|
+
let acodeEventDedup = null;
|
|
351
|
+
const getAcodeEventDedup = () => {
|
|
352
|
+
if (!acodeEventDedup) {
|
|
353
|
+
acodeEventDedup = createRolloutEventDedup({ cursorKey: "acodeHashes" });
|
|
354
|
+
}
|
|
355
|
+
return acodeEventDedup;
|
|
356
|
+
};
|
|
271
357
|
let cursorSessionPaths = null;
|
|
272
358
|
if (syncDiagnostics) {
|
|
273
359
|
const codexParseCandidates = (Array.isArray(rolloutFiles) ? rolloutFiles : []).reduce((count, entry) => {
|
|
@@ -290,65 +376,9 @@ async function parseRolloutIncremental({
|
|
|
290
376
|
hash_set_constructions: 0,
|
|
291
377
|
hash_array_materializations: 0,
|
|
292
378
|
hash_array_materialized_items: 0,
|
|
293
|
-
codex_hash_count:
|
|
294
|
-
? Number(externalCodexEventStore.size || 0)
|
|
295
|
-
: prevCodexHashes.length,
|
|
379
|
+
codex_hash_count: codexEventDedup.size(),
|
|
296
380
|
});
|
|
297
381
|
}
|
|
298
|
-
const ensureNewCodexEventKeys = () => {
|
|
299
|
-
if (!newCodexEventKeys) newCodexEventKeys = [];
|
|
300
|
-
if (!newCodexEventKeySet) newCodexEventKeySet = new Set();
|
|
301
|
-
};
|
|
302
|
-
const recordNewCodexEvent = (key) => {
|
|
303
|
-
ensureNewCodexEventKeys();
|
|
304
|
-
if (newCodexEventKeySet.has(key)) return;
|
|
305
|
-
newCodexEventKeySet.add(key);
|
|
306
|
-
newCodexEventKeys.push(key);
|
|
307
|
-
if (seenCodexEvents) seenCodexEvents.add(key);
|
|
308
|
-
externalCodexEventStore?.add(key);
|
|
309
|
-
};
|
|
310
|
-
const getAppendOnlyCodexEvents = () => {
|
|
311
|
-
if (!appendOnlyCodexEvents) {
|
|
312
|
-
appendOnlyCodexEvents = {
|
|
313
|
-
has(key) {
|
|
314
|
-
return Boolean(
|
|
315
|
-
newCodexEventKeySet?.has(key) ||
|
|
316
|
-
seenCodexEvents?.has(key)
|
|
317
|
-
);
|
|
318
|
-
},
|
|
319
|
-
add: recordNewCodexEvent,
|
|
320
|
-
};
|
|
321
|
-
}
|
|
322
|
-
return appendOnlyCodexEvents;
|
|
323
|
-
};
|
|
324
|
-
const getHistoricalCodexEvents = () => {
|
|
325
|
-
if (externalCodexEventStore) {
|
|
326
|
-
if (!historicalCodexEvents) {
|
|
327
|
-
historicalCodexEvents = {
|
|
328
|
-
has(key) {
|
|
329
|
-
return Boolean(
|
|
330
|
-
newCodexEventKeySet?.has(key) ||
|
|
331
|
-
externalCodexEventStore.has(key)
|
|
332
|
-
);
|
|
333
|
-
},
|
|
334
|
-
add: recordNewCodexEvent,
|
|
335
|
-
};
|
|
336
|
-
}
|
|
337
|
-
return historicalCodexEvents;
|
|
338
|
-
}
|
|
339
|
-
if (!seenCodexEvents) {
|
|
340
|
-
seenCodexEvents = new Set(prevCodexHashes);
|
|
341
|
-
for (const key of newCodexEventKeys || []) seenCodexEvents.add(key);
|
|
342
|
-
if (syncDiagnostics) syncDiagnostics.hash_set_constructions += 1;
|
|
343
|
-
}
|
|
344
|
-
if (!historicalCodexEvents) {
|
|
345
|
-
historicalCodexEvents = {
|
|
346
|
-
has: (key) => seenCodexEvents.has(key),
|
|
347
|
-
add: recordNewCodexEvent,
|
|
348
|
-
};
|
|
349
|
-
}
|
|
350
|
-
return historicalCodexEvents;
|
|
351
|
-
};
|
|
352
382
|
const getCursorSessionPaths = () => {
|
|
353
383
|
if (cursorSessionPaths) return cursorSessionPaths;
|
|
354
384
|
cursorSessionPaths = new Map();
|
|
@@ -360,7 +390,7 @@ async function parseRolloutIncremental({
|
|
|
360
390
|
}
|
|
361
391
|
return cursorSessionPaths;
|
|
362
392
|
};
|
|
363
|
-
const
|
|
393
|
+
const needsHistoricalRolloutDedup = ({
|
|
364
394
|
filePath,
|
|
365
395
|
prev,
|
|
366
396
|
sameInode,
|
|
@@ -426,7 +456,7 @@ async function parseRolloutIncremental({
|
|
|
426
456
|
const prevOffset = sameInode ? prev.offset || 0 : 0;
|
|
427
457
|
const truncated = sameInode && prevOffset > st.size;
|
|
428
458
|
const rebuildingCodexBaseline = Boolean(
|
|
429
|
-
fileSource === DEFAULT_SOURCE &&
|
|
459
|
+
(fileSource === DEFAULT_SOURCE || fileSource === "acode") &&
|
|
430
460
|
sameInode &&
|
|
431
461
|
!truncated &&
|
|
432
462
|
prevOffset > 0 &&
|
|
@@ -445,7 +475,8 @@ async function parseRolloutIncremental({
|
|
|
445
475
|
? prev.modelAttributionState || null
|
|
446
476
|
: null;
|
|
447
477
|
|
|
448
|
-
const codexProjectFastPath =
|
|
478
|
+
const codexProjectFastPath =
|
|
479
|
+
projectEnabled && (fileSource === DEFAULT_SOURCE || fileSource === "acode");
|
|
449
480
|
const projectOffset = sameInode && !truncated ? Number(prev.projectOffset || 0) : 0;
|
|
450
481
|
const projectUpToDate =
|
|
451
482
|
codexProjectFastPath &&
|
|
@@ -496,8 +527,13 @@ async function parseRolloutIncremental({
|
|
|
496
527
|
if (syncDiagnostics && !projectContextOnlyScan && fileSource === DEFAULT_SOURCE) {
|
|
497
528
|
syncDiagnostics.content_files_read += 1;
|
|
498
529
|
}
|
|
499
|
-
const
|
|
500
|
-
?
|
|
530
|
+
const rolloutEventDedup = fileSource === DEFAULT_SOURCE
|
|
531
|
+
? codexEventDedup
|
|
532
|
+
: fileSource === "acode"
|
|
533
|
+
? getAcodeEventDedup()
|
|
534
|
+
: null;
|
|
535
|
+
const codexEventTracker = rolloutEventDedup
|
|
536
|
+
? (needsHistoricalRolloutDedup({
|
|
501
537
|
filePath,
|
|
502
538
|
prev,
|
|
503
539
|
sameInode,
|
|
@@ -505,8 +541,8 @@ async function parseRolloutIncremental({
|
|
|
505
541
|
startOffset,
|
|
506
542
|
rebuildingBaseline: rebuildingCodexBaseline,
|
|
507
543
|
})
|
|
508
|
-
?
|
|
509
|
-
:
|
|
544
|
+
? rolloutEventDedup.getHistoricalEvents
|
|
545
|
+
: rolloutEventDedup.getAppendOnlyEvents)
|
|
510
546
|
: null;
|
|
511
547
|
const result = projectContextOnlyScan
|
|
512
548
|
? await scanRolloutProjectFileContexts({
|
|
@@ -587,15 +623,10 @@ async function parseRolloutIncremental({
|
|
|
587
623
|
const projectBucketsQueued = projectEnabled
|
|
588
624
|
? await enqueueTouchedProjectBuckets({ projectQueuePath, projectState, projectTouchedBuckets })
|
|
589
625
|
: 0;
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
}
|
|
626
|
+
codexEventDedup.persist();
|
|
627
|
+
acodeEventDedup?.persist();
|
|
593
628
|
if (syncDiagnostics) {
|
|
594
|
-
syncDiagnostics.codex_hash_count =
|
|
595
|
-
? Number(externalCodexEventStore.size || 0)
|
|
596
|
-
: Array.isArray(cursors.codexHashes)
|
|
597
|
-
? cursors.codexHashes.length
|
|
598
|
-
: prevCodexHashes.length;
|
|
629
|
+
syncDiagnostics.codex_hash_count = codexEventDedup.size();
|
|
599
630
|
}
|
|
600
631
|
hourlyState.updatedAt = new Date().toISOString();
|
|
601
632
|
cursors.hourly = hourlyState;
|
|
@@ -2193,7 +2224,7 @@ async function parseRolloutFile({
|
|
|
2193
2224
|
if (!delta || isAllZeroUsage(delta)) continue;
|
|
2194
2225
|
delta.conversation_count = 1;
|
|
2195
2226
|
|
|
2196
|
-
// Forked Codex rollouts replay the parent session's entire token history
|
|
2227
|
+
// Forked Codex and Acode rollouts replay the parent session's entire token history
|
|
2197
2228
|
// into the child file the moment the fork is created. The date guard below
|
|
2198
2229
|
// (current_date < rolloutDate) catches cross-day forks, but same-day forks
|
|
2199
2230
|
// share the parent's current_date and slip through it. The replay is written
|
|
@@ -2209,9 +2240,13 @@ async function parseRolloutFile({
|
|
|
2209
2240
|
// counted (a bounded, <1% residual over-count); dropping real usage is the
|
|
2210
2241
|
// worse failure, so we bias against it. `usageDeltaState` is already advanced above,
|
|
2211
2242
|
// keeping the cumulative lineage correct for the live turns we keep.
|
|
2212
|
-
// Scoped to forked
|
|
2243
|
+
// Scoped to forked Codex/Acode rollouts. (issue #169 follow-up.)
|
|
2213
2244
|
let forkedReplaySkip = false;
|
|
2214
|
-
if (
|
|
2245
|
+
if (
|
|
2246
|
+
isForkedRollout &&
|
|
2247
|
+
(source === DEFAULT_SOURCE || source === "acode") &&
|
|
2248
|
+
replayPrefixActive
|
|
2249
|
+
) {
|
|
2215
2250
|
const tokenMs = Date.parse(tokenTimestamp);
|
|
2216
2251
|
// Fail open on anything the burst heuristic was not measured against:
|
|
2217
2252
|
// an unparseable timestamp or a backwards clock step permanently ends
|
|
@@ -2221,13 +2256,13 @@ async function parseRolloutFile({
|
|
|
2221
2256
|
if (!Number.isFinite(tokenMs) || (prevForkedTokenMs !== null && tokenMs < prevForkedTokenMs)) {
|
|
2222
2257
|
replayPrefixActive = false;
|
|
2223
2258
|
} else {
|
|
2224
|
-
if (prevForkedTokenMs !== null && tokenMs - prevForkedTokenMs >=
|
|
2259
|
+
if (prevForkedTokenMs !== null && tokenMs - prevForkedTokenMs >= FORK_REPLAY_GAP_MS) {
|
|
2225
2260
|
replayPrefixActive = false;
|
|
2226
2261
|
}
|
|
2227
2262
|
forkedReplaySkip =
|
|
2228
2263
|
replayPrefixActive &&
|
|
2229
2264
|
prevForkedTokenMs !== null &&
|
|
2230
|
-
tokenMs - prevForkedTokenMs <
|
|
2265
|
+
tokenMs - prevForkedTokenMs < FORK_REPLAY_GAP_MS;
|
|
2231
2266
|
prevForkedTokenMs = tokenMs;
|
|
2232
2267
|
}
|
|
2233
2268
|
}
|
|
@@ -2253,14 +2288,12 @@ async function parseRolloutFile({
|
|
|
2253
2288
|
// are still counted. Key = sessionUUID:eventTimestamp (both stable across the
|
|
2254
2289
|
// rewrite and across a sessions/ -> archived_sessions/ move).
|
|
2255
2290
|
//
|
|
2256
|
-
//
|
|
2257
|
-
//
|
|
2258
|
-
// their own model re-alignment that legitimately re-reads prior events, which
|
|
2259
|
-
// this dedup would otherwise suppress.
|
|
2291
|
+
// Apply event deduplication only to Codex and Acode. Every Code and other sources
|
|
2292
|
+
// reread events to realign models.
|
|
2260
2293
|
const codexEvents = typeof seenCodexEvents === "function"
|
|
2261
2294
|
? seenCodexEvents()
|
|
2262
2295
|
: seenCodexEvents;
|
|
2263
|
-
if (codexEvents && source === "codex") {
|
|
2296
|
+
if (codexEvents && (source === "codex" || source === "acode")) {
|
|
2264
2297
|
const dedupKey = `${sessionId || filePath}:${tokenTimestamp}`;
|
|
2265
2298
|
if (codexEvents.has(dedupKey)) continue;
|
|
2266
2299
|
codexEvents.add(dedupKey);
|
|
@@ -3781,7 +3814,7 @@ function normalizeIsoDate(value) {
|
|
|
3781
3814
|
// toward the low end because merging a real fast turn (under-count) is worse than
|
|
3782
3815
|
// leaving a slow replay counted (bounded over-count). See the skip site in
|
|
3783
3816
|
// parseRolloutFile. (issue #169 follow-up.)
|
|
3784
|
-
const
|
|
3817
|
+
const FORK_REPLAY_GAP_MS = 500;
|
|
3785
3818
|
|
|
3786
3819
|
function isForkedReplayToken({ isForkedRollout, rolloutDate, currentDate }) {
|
|
3787
3820
|
return Boolean(isForkedRollout && rolloutDate && currentDate && currentDate < rolloutDate);
|
|
@@ -12871,6 +12904,395 @@ async function parseAnythingllmIncremental({
|
|
|
12871
12904
|
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
12872
12905
|
}
|
|
12873
12906
|
|
|
12907
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
12908
|
+
// Devin (Cognition — devin.ai CLI)
|
|
12909
|
+
//
|
|
12910
|
+
// Data: SQLite at
|
|
12911
|
+
// macOS/Linux: $XDG_DATA_HOME/devin/cli/sessions.db (~/.local/share)
|
|
12912
|
+
// Windows: no evidenced native location; the CLI's XDG data dir inside a
|
|
12913
|
+
// WSL distro is probed via the \\wsl$ UNC bridge
|
|
12914
|
+
// Override: $TOKENTRACKER_DEVIN_DB
|
|
12915
|
+
//
|
|
12916
|
+
// Devin rewrites history aggressively: replay, compaction and forks copy
|
|
12917
|
+
// message_nodes rows, so several nodes share one chat_message
|
|
12918
|
+
// `.metadata.request_id` (verified on CLI 3000.10.21: ~2 duplicated nodes per
|
|
12919
|
+
// request). request_id is the billing identity — copies must not add usage.
|
|
12920
|
+
// Aggregation is a full projection + per-request reconcile keyed by
|
|
12921
|
+
// request_id: every fingerprint change re-reads the usage-only rows, the
|
|
12922
|
+
// ledger in cursors.devin.requests subtracts a request's prior contribution
|
|
12923
|
+
// before adding its current one, and deleted/compacted history keeps its
|
|
12924
|
+
// (non-refunded) ledger entry. There is no row_id high-water mark because
|
|
12925
|
+
// in-place metric corrections cannot be detected by one.
|
|
12926
|
+
//
|
|
12927
|
+
// Attribution recorded at first observation is authoritative: a request's
|
|
12928
|
+
// owning session, resolved project and conversation share live in the ledger,
|
|
12929
|
+
// so deleting the original node while a fork copy survives never migrates its
|
|
12930
|
+
// spend or refunds its conversation. A fork made only of copies pays nothing;
|
|
12931
|
+
// the first genuinely new request in a session pays its single
|
|
12932
|
+
// conversation_count once — the set of sessions that already paid is derived
|
|
12933
|
+
// from the ledger entries themselves, not persisted separately.
|
|
12934
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
12935
|
+
|
|
12936
|
+
const DEVIN_SOURCE = "devin";
|
|
12937
|
+
|
|
12938
|
+
function resolveDevinDbPath(env = process.env, deps = {}) {
|
|
12939
|
+
const override =
|
|
12940
|
+
typeof env.TOKENTRACKER_DEVIN_DB === "string" && env.TOKENTRACKER_DEVIN_DB.trim();
|
|
12941
|
+
if (override) return path.resolve(env.TOKENTRACKER_DEVIN_DB.trim());
|
|
12942
|
+
const home = env.HOME || env.USERPROFILE || os.homedir();
|
|
12943
|
+
const xdgDataHome =
|
|
12944
|
+
typeof env.XDG_DATA_HOME === "string" && env.XDG_DATA_HOME.trim()
|
|
12945
|
+
? path.resolve(env.XDG_DATA_HOME.trim())
|
|
12946
|
+
: path.join(home, ".local", "share");
|
|
12947
|
+
if (process.platform !== "win32") {
|
|
12948
|
+
return path.join(xdgDataHome, "devin", "cli", "sessions.db");
|
|
12949
|
+
}
|
|
12950
|
+
// Devin CLI is not known to write a native-Windows data dir; on Windows its
|
|
12951
|
+
// sessions.db lives under the distro's XDG home, reachable over \\wsl$.
|
|
12952
|
+
const wslDir = wsl.shouldProbeWsl(env)
|
|
12953
|
+
? wsl.discoverWslHome(".local/share/devin/cli", { ...deps, env })
|
|
12954
|
+
: null;
|
|
12955
|
+
const wslValue = wslDir ? path.join(wslDir, "sessions.db") : null;
|
|
12956
|
+
const paths = resolveInstallPaths({ nativeValue: null, wslValue }, env, deps);
|
|
12957
|
+
return paths.native || paths.wsl;
|
|
12958
|
+
}
|
|
12959
|
+
|
|
12960
|
+
function devinSqliteFingerprint(dbPath) {
|
|
12961
|
+
const fingerprint = sqliteSidecarFingerprint(dbPath);
|
|
12962
|
+
// -shm is reader bookkeeping: opening a WAL database can bump it without any
|
|
12963
|
+
// content change, so it must not force a rescan (same convention as Unsloth).
|
|
12964
|
+
delete fingerprint["-shm"];
|
|
12965
|
+
return fingerprint;
|
|
12966
|
+
}
|
|
12967
|
+
|
|
12968
|
+
// Request ids and conversation keys are untrusted strings — normalize the
|
|
12969
|
+
// persisted maps into null-prototype dictionaries so a literal "__proto__"
|
|
12970
|
+
// key stays an own entry that round-trips through cursors.json instead of
|
|
12971
|
+
// silently mutating the prototype chain (and re-adding that request's usage
|
|
12972
|
+
// on every rescan).
|
|
12973
|
+
function devinStringMap(value) {
|
|
12974
|
+
const dict = Object.create(null);
|
|
12975
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
12976
|
+
for (const key of Object.keys(value)) dict[key] = value[key];
|
|
12977
|
+
}
|
|
12978
|
+
return dict;
|
|
12979
|
+
}
|
|
12980
|
+
|
|
12981
|
+
// Stage only the bucket objects this parser can mutate. The shared
|
|
12982
|
+
// normalizers copy the bucket maps but alias each bucket/totals object, and
|
|
12983
|
+
// the enqueue helpers stamp queuedKey before appendFile runs — so a failed
|
|
12984
|
+
// append used to leave the caller's published state polluted. Only
|
|
12985
|
+
// devin-owned entries get private copies; every other provider's buckets
|
|
12986
|
+
// stay shared read-only references.
|
|
12987
|
+
function stageDevinBuckets(buckets, isDevinBucket) {
|
|
12988
|
+
const staged = {};
|
|
12989
|
+
for (const [key, bucket] of Object.entries(buckets || {})) {
|
|
12990
|
+
staged[key] =
|
|
12991
|
+
bucket && typeof bucket === "object" && isDevinBucket(key, bucket)
|
|
12992
|
+
? {
|
|
12993
|
+
...bucket,
|
|
12994
|
+
totals:
|
|
12995
|
+
bucket.totals && typeof bucket.totals === "object"
|
|
12996
|
+
? { ...bucket.totals }
|
|
12997
|
+
: bucket.totals,
|
|
12998
|
+
}
|
|
12999
|
+
: bucket;
|
|
13000
|
+
}
|
|
13001
|
+
return staged;
|
|
13002
|
+
}
|
|
13003
|
+
|
|
13004
|
+
async function readDevinUsageRows(dbPath, sqliteOptions = {}) {
|
|
13005
|
+
if (!dbPath || !fssync.existsSync(dbPath)) return [];
|
|
13006
|
+
const options = {
|
|
13007
|
+
label: "Devin",
|
|
13008
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
13009
|
+
timeout: 30_000,
|
|
13010
|
+
readOnly: true,
|
|
13011
|
+
throwOnReadFailure: true,
|
|
13012
|
+
...sqliteOptions,
|
|
13013
|
+
};
|
|
13014
|
+
let snapshot = null;
|
|
13015
|
+
let effectiveDbPath = dbPath;
|
|
13016
|
+
if (isUncPath(dbPath)) {
|
|
13017
|
+
try {
|
|
13018
|
+
snapshot = snapshotSqliteDb(dbPath);
|
|
13019
|
+
effectiveDbPath = snapshot.path;
|
|
13020
|
+
} catch (_e) { }
|
|
13021
|
+
}
|
|
13022
|
+
try {
|
|
13023
|
+
const tables = new Set(
|
|
13024
|
+
(await readSqliteJsonRowsAsync(effectiveDbPath, DEVIN_TABLE_PROBE_SQL, options))
|
|
13025
|
+
.map((row) => row?.name)
|
|
13026
|
+
.filter(Boolean),
|
|
13027
|
+
);
|
|
13028
|
+
if (!tables.has("message_nodes")) return [];
|
|
13029
|
+
return await readSqliteJsonRowsAsync(
|
|
13030
|
+
effectiveDbPath,
|
|
13031
|
+
devinUsageSql({ hasSessionsTable: tables.has("sessions") }),
|
|
13032
|
+
options,
|
|
13033
|
+
);
|
|
13034
|
+
} finally {
|
|
13035
|
+
if (snapshot) snapshot.cleanup();
|
|
13036
|
+
}
|
|
13037
|
+
}
|
|
13038
|
+
|
|
13039
|
+
async function parseDevinIncremental({
|
|
13040
|
+
dbPath,
|
|
13041
|
+
cursors,
|
|
13042
|
+
queuePath,
|
|
13043
|
+
projectQueuePath,
|
|
13044
|
+
onProgress,
|
|
13045
|
+
env,
|
|
13046
|
+
sqliteOptions,
|
|
13047
|
+
publicRepoResolver,
|
|
13048
|
+
} = {}) {
|
|
13049
|
+
await ensureDir(path.dirname(queuePath));
|
|
13050
|
+
const resolvedDb = dbPath || resolveDevinDbPath(env || process.env);
|
|
13051
|
+
const priorState =
|
|
13052
|
+
cursors.devin && typeof cursors.devin === "object" ? cursors.devin : {};
|
|
13053
|
+
const requests = devinStringMap(priorState.requests);
|
|
13054
|
+
if (!resolvedDb || !fssync.existsSync(resolvedDb)) {
|
|
13055
|
+
const nextState = { ...priorState, requests, updatedAt: new Date().toISOString() };
|
|
13056
|
+
delete nextState.conversations;
|
|
13057
|
+
cursors.devin = nextState;
|
|
13058
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0 };
|
|
13059
|
+
}
|
|
13060
|
+
|
|
13061
|
+
// Cheap unchanged check: skip all SQL work when neither the DB nor its WAL
|
|
13062
|
+
// moved since the last published state.
|
|
13063
|
+
const initialFingerprint = devinSqliteFingerprint(resolvedDb);
|
|
13064
|
+
if (sameSqliteFingerprint(initialFingerprint, priorState.fingerprint)) {
|
|
13065
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0 };
|
|
13066
|
+
}
|
|
13067
|
+
|
|
13068
|
+
const rows = await readDevinUsageRows(resolvedDb, sqliteOptions);
|
|
13069
|
+
const { events } = buildDevinUsageEvents(rows);
|
|
13070
|
+
|
|
13071
|
+
// Stage the normalized working states: bucket-map copies alias the
|
|
13072
|
+
// caller's published bucket objects, so give only the devin-owned entries
|
|
13073
|
+
// (plus the flat groupQueued map) private copies. Reconciliation and
|
|
13074
|
+
// enqueue mutations then land on staged state and are published only after
|
|
13075
|
+
// both queue appends below succeed — a failed append leaves `cursors`
|
|
13076
|
+
// untouched so a retry re-derives the same contribution and latest-wins
|
|
13077
|
+
// rows recover either queue. Unrelated providers' buckets stay shared
|
|
13078
|
+
// read-only references (this parser never writes their keys).
|
|
13079
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
13080
|
+
hourlyState.buckets = stageDevinBuckets(
|
|
13081
|
+
hourlyState.buckets,
|
|
13082
|
+
(key) =>
|
|
13083
|
+
(normalizeSourceInput(parseBucketKey(key).source) || DEFAULT_SOURCE) ===
|
|
13084
|
+
DEVIN_SOURCE,
|
|
13085
|
+
);
|
|
13086
|
+
hourlyState.groupQueued =
|
|
13087
|
+
hourlyState.groupQueued && typeof hourlyState.groupQueued === "object"
|
|
13088
|
+
? { ...hourlyState.groupQueued }
|
|
13089
|
+
: {};
|
|
13090
|
+
const touchedBuckets = new Set();
|
|
13091
|
+
const projectEnabled =
|
|
13092
|
+
typeof projectQueuePath === "string" && projectQueuePath.length > 0;
|
|
13093
|
+
const projectState = projectEnabled
|
|
13094
|
+
? normalizeProjectState(cursors?.projectHourly)
|
|
13095
|
+
: null;
|
|
13096
|
+
if (projectState) {
|
|
13097
|
+
// Project bucket keys are `projectKey|source|hourStart`; this parser only
|
|
13098
|
+
// ever looks up keys whose middle segment is the devin source.
|
|
13099
|
+
projectState.buckets = stageDevinBuckets(projectState.buckets, (key) => {
|
|
13100
|
+
const last = key.lastIndexOf(BUCKET_SEPARATOR);
|
|
13101
|
+
const prev = last > 0 ? key.lastIndexOf(BUCKET_SEPARATOR, last - 1) : -1;
|
|
13102
|
+
const source = prev >= 0 ? key.slice(prev + 1, last) : "";
|
|
13103
|
+
return (
|
|
13104
|
+
(normalizeSourceInput(source) || DEFAULT_SOURCE) === DEVIN_SOURCE
|
|
13105
|
+
);
|
|
13106
|
+
});
|
|
13107
|
+
}
|
|
13108
|
+
const projectTouchedBuckets = projectEnabled ? new Set() : null;
|
|
13109
|
+
const projectMetaCache = projectEnabled ? new Map() : null;
|
|
13110
|
+
const publicRepoCache = projectEnabled ? new Map() : null;
|
|
13111
|
+
const projectContextBySession = projectEnabled ? new Map() : null;
|
|
13112
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
13113
|
+
let eventsAggregated = 0;
|
|
13114
|
+
|
|
13115
|
+
// The counted-conversation index is derived from the retained ledger: the
|
|
13116
|
+
// entry that paid a conversation's +1 keeps that share in its own totals,
|
|
13117
|
+
// even after every copy of the request vanished — no second persisted map.
|
|
13118
|
+
const countedConversations = new Set();
|
|
13119
|
+
for (const [requestId, entry] of Object.entries(requests)) {
|
|
13120
|
+
const totals = entry && typeof entry === "object" ? entry.totals : null;
|
|
13121
|
+
if (totals && Number.isSafeInteger(totals.conversation_count) && totals.conversation_count >= 1) {
|
|
13122
|
+
countedConversations.add(
|
|
13123
|
+
typeof entry.sessionId === "string" && entry.sessionId
|
|
13124
|
+
? entry.sessionId
|
|
13125
|
+
: `request:${requestId}`,
|
|
13126
|
+
);
|
|
13127
|
+
}
|
|
13128
|
+
}
|
|
13129
|
+
|
|
13130
|
+
for (let index = 0; index < events.length; index++) {
|
|
13131
|
+
const event = events[index];
|
|
13132
|
+
const bucketStart = toUtcHalfHourStart(new Date(event.tsMs).toISOString());
|
|
13133
|
+
if (!bucketStart) continue;
|
|
13134
|
+
|
|
13135
|
+
const previous = requests[event.requestId];
|
|
13136
|
+
const previousTotals =
|
|
13137
|
+
previous?.totals && typeof previous.totals === "object" ? previous.totals : null;
|
|
13138
|
+
|
|
13139
|
+
// Ownership recorded at first observation is authoritative. For a known
|
|
13140
|
+
// request the ledger's session/project/conversation share survive copy
|
|
13141
|
+
// deletion and fork timelines; for a new request the canonical retained
|
|
13142
|
+
// record supplies them exactly once.
|
|
13143
|
+
let projectKey = previous ? previous.projectKey || null : null;
|
|
13144
|
+
let projectRef = previous ? previous.projectRef || null : null;
|
|
13145
|
+
if (previous) {
|
|
13146
|
+
const priorConv = previousTotals ? previousTotals.conversation_count : 0;
|
|
13147
|
+
event.totals.conversation_count =
|
|
13148
|
+
Number.isSafeInteger(priorConv) && priorConv >= 0 ? priorConv : 0;
|
|
13149
|
+
} else {
|
|
13150
|
+
if (projectEnabled && event.sessionId) {
|
|
13151
|
+
let context = projectContextBySession.get(event.sessionId);
|
|
13152
|
+
if (context === undefined) {
|
|
13153
|
+
const startDir = event.workingDirectory
|
|
13154
|
+
? wsl.mapWslCwdToUnc(event.workingDirectory, resolvedDb)
|
|
13155
|
+
: null;
|
|
13156
|
+
context = startDir
|
|
13157
|
+
? await resolveProjectContextForPath({
|
|
13158
|
+
startDir,
|
|
13159
|
+
projectMetaCache,
|
|
13160
|
+
publicRepoCache,
|
|
13161
|
+
publicRepoResolver,
|
|
13162
|
+
projectState,
|
|
13163
|
+
})
|
|
13164
|
+
: null;
|
|
13165
|
+
projectContextBySession.set(event.sessionId, context || null);
|
|
13166
|
+
}
|
|
13167
|
+
projectKey = context?.projectKey || null;
|
|
13168
|
+
projectRef = context?.projectRef || null;
|
|
13169
|
+
}
|
|
13170
|
+
// A fork made only of copied requests pays no conversation of its own;
|
|
13171
|
+
// the first genuinely new request in a conversation pays it once.
|
|
13172
|
+
const convKey = event.sessionId || `request:${event.requestId}`;
|
|
13173
|
+
if (countedConversations.has(convKey)) {
|
|
13174
|
+
event.totals.conversation_count = 0;
|
|
13175
|
+
} else {
|
|
13176
|
+
event.totals.conversation_count = 1;
|
|
13177
|
+
countedConversations.add(convKey);
|
|
13178
|
+
}
|
|
13179
|
+
}
|
|
13180
|
+
|
|
13181
|
+
const unchanged =
|
|
13182
|
+
previousTotals &&
|
|
13183
|
+
totalsKey(previousTotals) === totalsKey(event.totals) &&
|
|
13184
|
+
previous.bucketStart === bucketStart &&
|
|
13185
|
+
previous.model === event.model;
|
|
13186
|
+
if (!unchanged) {
|
|
13187
|
+
if (previousTotals && previous.bucketStart && previous.model) {
|
|
13188
|
+
const oldBucket = getHourlyBucket(
|
|
13189
|
+
hourlyState,
|
|
13190
|
+
DEVIN_SOURCE,
|
|
13191
|
+
previous.model,
|
|
13192
|
+
previous.bucketStart,
|
|
13193
|
+
);
|
|
13194
|
+
subtractTotals(oldBucket.totals, previousTotals);
|
|
13195
|
+
touchedBuckets.add(
|
|
13196
|
+
bucketKey(DEVIN_SOURCE, previous.model, previous.bucketStart),
|
|
13197
|
+
);
|
|
13198
|
+
if (projectEnabled && previous.projectKey) {
|
|
13199
|
+
const oldProjectBucket = getProjectBucket(
|
|
13200
|
+
projectState,
|
|
13201
|
+
previous.projectKey,
|
|
13202
|
+
DEVIN_SOURCE,
|
|
13203
|
+
previous.bucketStart,
|
|
13204
|
+
previous.projectRef || null,
|
|
13205
|
+
);
|
|
13206
|
+
subtractTotals(oldProjectBucket.totals, previousTotals);
|
|
13207
|
+
projectTouchedBuckets.add(
|
|
13208
|
+
projectBucketKey(previous.projectKey, DEVIN_SOURCE, previous.bucketStart),
|
|
13209
|
+
);
|
|
13210
|
+
}
|
|
13211
|
+
}
|
|
13212
|
+
|
|
13213
|
+
const bucket = getHourlyBucket(hourlyState, DEVIN_SOURCE, event.model, bucketStart);
|
|
13214
|
+
addTotals(bucket.totals, event.totals);
|
|
13215
|
+
touchedBuckets.add(bucketKey(DEVIN_SOURCE, event.model, bucketStart));
|
|
13216
|
+
if (projectEnabled && projectKey) {
|
|
13217
|
+
const projectBucket = getProjectBucket(
|
|
13218
|
+
projectState,
|
|
13219
|
+
projectKey,
|
|
13220
|
+
DEVIN_SOURCE,
|
|
13221
|
+
bucketStart,
|
|
13222
|
+
projectRef,
|
|
13223
|
+
);
|
|
13224
|
+
addTotals(projectBucket.totals, event.totals);
|
|
13225
|
+
projectTouchedBuckets.add(
|
|
13226
|
+
projectBucketKey(projectKey, DEVIN_SOURCE, bucketStart),
|
|
13227
|
+
);
|
|
13228
|
+
}
|
|
13229
|
+
// The ledger records only what was added: owning session, model,
|
|
13230
|
+
// bucket, totals and the resolved project identity — never request text
|
|
13231
|
+
// or raw working paths.
|
|
13232
|
+
requests[event.requestId] = {
|
|
13233
|
+
sessionId:
|
|
13234
|
+
previous && typeof previous.sessionId === "string" && previous.sessionId
|
|
13235
|
+
? previous.sessionId
|
|
13236
|
+
: event.sessionId,
|
|
13237
|
+
model: event.model,
|
|
13238
|
+
bucketStart,
|
|
13239
|
+
totals: event.totals,
|
|
13240
|
+
projectKey,
|
|
13241
|
+
projectRef,
|
|
13242
|
+
updatedAt: new Date().toISOString(),
|
|
13243
|
+
};
|
|
13244
|
+
eventsAggregated += 1;
|
|
13245
|
+
}
|
|
13246
|
+
if (cb) {
|
|
13247
|
+
cb({
|
|
13248
|
+
index: index + 1,
|
|
13249
|
+
total: events.length,
|
|
13250
|
+
recordsProcessed: index + 1,
|
|
13251
|
+
eventsAggregated,
|
|
13252
|
+
bucketsQueued: touchedBuckets.size,
|
|
13253
|
+
});
|
|
13254
|
+
}
|
|
13255
|
+
}
|
|
13256
|
+
|
|
13257
|
+
const bucketsQueued = await enqueueTouchedBuckets({
|
|
13258
|
+
queuePath,
|
|
13259
|
+
hourlyState,
|
|
13260
|
+
touchedBuckets,
|
|
13261
|
+
});
|
|
13262
|
+
const projectBucketsQueued = projectEnabled
|
|
13263
|
+
? await enqueueTouchedProjectBuckets({
|
|
13264
|
+
projectQueuePath,
|
|
13265
|
+
projectState,
|
|
13266
|
+
projectTouchedBuckets,
|
|
13267
|
+
})
|
|
13268
|
+
: 0;
|
|
13269
|
+
// If Devin wrote to the DB/WAL while we read, publish the pre-read
|
|
13270
|
+
// fingerprint so the next sync re-reads instead of acknowledging a snapshot
|
|
13271
|
+
// it never saw (same convention as parseUnslothIncremental).
|
|
13272
|
+
const finalFingerprint = devinSqliteFingerprint(resolvedDb);
|
|
13273
|
+
const updatedAt = new Date().toISOString();
|
|
13274
|
+
hourlyState.updatedAt = updatedAt;
|
|
13275
|
+
cursors.hourly = hourlyState;
|
|
13276
|
+
cursors.devin = {
|
|
13277
|
+
version: 1,
|
|
13278
|
+
requests,
|
|
13279
|
+
fingerprint: sameSqliteFingerprint(initialFingerprint, finalFingerprint)
|
|
13280
|
+
? finalFingerprint
|
|
13281
|
+
: initialFingerprint,
|
|
13282
|
+
updatedAt,
|
|
13283
|
+
};
|
|
13284
|
+
if (projectState) {
|
|
13285
|
+
projectState.updatedAt = updatedAt;
|
|
13286
|
+
cursors.projectHourly = projectState;
|
|
13287
|
+
}
|
|
13288
|
+
return {
|
|
13289
|
+
recordsProcessed: rows.length,
|
|
13290
|
+
eventsAggregated,
|
|
13291
|
+
bucketsQueued,
|
|
13292
|
+
projectBucketsQueued,
|
|
13293
|
+
};
|
|
13294
|
+
}
|
|
13295
|
+
|
|
12874
13296
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
12875
13297
|
// Goose (Block AI agent — github.com/block/goose)
|
|
12876
13298
|
//
|
|
@@ -14624,6 +15046,8 @@ async function parsePiLikeIncremental({
|
|
|
14624
15046
|
sessionFiles,
|
|
14625
15047
|
cursors,
|
|
14626
15048
|
queuePath,
|
|
15049
|
+
projectQueuePath,
|
|
15050
|
+
publicRepoResolver,
|
|
14627
15051
|
onProgress,
|
|
14628
15052
|
env,
|
|
14629
15053
|
defaultModel,
|
|
@@ -14633,6 +15057,7 @@ async function parsePiLikeIncremental({
|
|
|
14633
15057
|
sourceForProvider,
|
|
14634
15058
|
} = {}) {
|
|
14635
15059
|
await ensureDir(path.dirname(queuePath));
|
|
15060
|
+
const projectEnabled = typeof projectQueuePath === "string" && projectQueuePath.length > 0;
|
|
14636
15061
|
const providerState = cursors[stateKey] && typeof cursors[stateKey] === "object"
|
|
14637
15062
|
? cursors[stateKey]
|
|
14638
15063
|
: {};
|
|
@@ -14642,6 +15067,9 @@ async function parsePiLikeIncremental({
|
|
|
14642
15067
|
? { ...providerState.fileOffsets }
|
|
14643
15068
|
: {};
|
|
14644
15069
|
|
|
15070
|
+
const projectSeenIds = new Set(Array.isArray(providerState.projectSeenIds) ? providerState.projectSeenIds : []);
|
|
15071
|
+
const projectFileOffsets = { ...(providerState.projectFileOffsets || {}) };
|
|
15072
|
+
|
|
14645
15073
|
const files = Array.isArray(sessionFiles)
|
|
14646
15074
|
? sessionFiles
|
|
14647
15075
|
: resolveSessionFiles(env || process.env);
|
|
@@ -14654,11 +15082,33 @@ async function parsePiLikeIncremental({
|
|
|
14654
15082
|
fileOffsets,
|
|
14655
15083
|
updatedAt: new Date().toISOString(),
|
|
14656
15084
|
};
|
|
14657
|
-
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
15085
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0, projectBucketsQueued: 0 };
|
|
14658
15086
|
}
|
|
14659
15087
|
|
|
14660
15088
|
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
14661
15089
|
const touchedBuckets = new Set();
|
|
15090
|
+
const projectState = projectEnabled ? normalizeProjectState(cursors?.projectHourly) : null;
|
|
15091
|
+
const projectTouchedBuckets = projectEnabled ? new Set() : null;
|
|
15092
|
+
const projectMetaCache = projectEnabled ? new Map() : null;
|
|
15093
|
+
const publicRepoCache = projectEnabled ? new Map() : null;
|
|
15094
|
+
// Both queues must publish before cursor progress is acknowledged. Normalized
|
|
15095
|
+
// bucket maps still alias their values, so stage this provider's buckets to
|
|
15096
|
+
// keep a failed project append from mutating the caller's aggregate state.
|
|
15097
|
+
const family = sourceForProvider(null);
|
|
15098
|
+
const ownsSource = (source) => source === family || source.startsWith(`${family}-`);
|
|
15099
|
+
hourlyState.groupQueued = { ...(hourlyState.groupQueued || {}) };
|
|
15100
|
+
for (const [key, bucket] of Object.entries(hourlyState.buckets)) {
|
|
15101
|
+
if (ownsSource(parseBucketKey(key).source || "")) {
|
|
15102
|
+
hourlyState.buckets[key] = { ...bucket, totals: { ...bucket.totals } };
|
|
15103
|
+
}
|
|
15104
|
+
}
|
|
15105
|
+
if (projectState) {
|
|
15106
|
+
for (const [key, bucket] of Object.entries(projectState.buckets)) {
|
|
15107
|
+
if (ownsSource(bucket.source || "")) {
|
|
15108
|
+
projectState.buckets[key] = { ...bucket, totals: { ...bucket.totals } };
|
|
15109
|
+
}
|
|
15110
|
+
}
|
|
15111
|
+
}
|
|
14662
15112
|
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
14663
15113
|
let recordsProcessed = 0;
|
|
14664
15114
|
let eventsAggregated = 0;
|
|
@@ -14797,26 +15247,166 @@ async function parsePiLikeIncremental({
|
|
|
14797
15247
|
};
|
|
14798
15248
|
}
|
|
14799
15249
|
|
|
14800
|
-
|
|
14801
|
-
|
|
14802
|
-
|
|
14803
|
-
|
|
14804
|
-
|
|
14805
|
-
|
|
14806
|
-
|
|
14807
|
-
|
|
14808
|
-
|
|
15250
|
+
// Project attribution has an independent cursor so upgrading an existing
|
|
15251
|
+
// installation can backfill already-consumed Pi sessions without adding
|
|
15252
|
+
// those messages to the total-usage buckets a second time. Current Pi
|
|
15253
|
+
// session headers persist the real cwd; unlike the encoded session folder,
|
|
15254
|
+
// it is lossless even when path components contain dashes.
|
|
15255
|
+
if (projectEnabled) {
|
|
15256
|
+
for (const filePath of files) {
|
|
15257
|
+
let stat;
|
|
15258
|
+
try { stat = fssync.statSync(filePath); } catch { continue; }
|
|
15259
|
+
|
|
15260
|
+
const prevEntry = projectFileOffsets[filePath] || {};
|
|
15261
|
+
const prevSize = Number(prevEntry.size) || 0;
|
|
15262
|
+
const prevIno = prevEntry.ino;
|
|
15263
|
+
const inodeChanged = typeof prevIno === "number" && prevIno !== stat.ino;
|
|
15264
|
+
const startOffset = stat.size < prevSize || inodeChanged ? 0 : prevSize;
|
|
15265
|
+
if (stat.size <= startOffset) continue;
|
|
15266
|
+
|
|
15267
|
+
const cwd = await resolveOmpFileCwd(filePath);
|
|
15268
|
+
const projectContext = cwd
|
|
15269
|
+
? await resolveProjectContextForPath({
|
|
15270
|
+
startDir: wsl.mapWslCwdToUnc(cwd, filePath),
|
|
15271
|
+
projectMetaCache,
|
|
15272
|
+
publicRepoCache,
|
|
15273
|
+
publicRepoResolver,
|
|
15274
|
+
projectState,
|
|
15275
|
+
})
|
|
15276
|
+
: null;
|
|
15277
|
+
const projectRef = projectContext?.projectRef || null;
|
|
15278
|
+
const projectKey = projectContext?.projectKey || null;
|
|
15279
|
+
|
|
15280
|
+
let lastCompleteOffset = startOffset;
|
|
15281
|
+
if (projectKey && projectRef) {
|
|
15282
|
+
let stream;
|
|
15283
|
+
try {
|
|
15284
|
+
stream = fssync.createReadStream(filePath, {
|
|
15285
|
+
encoding: "utf8",
|
|
15286
|
+
start: startOffset,
|
|
15287
|
+
});
|
|
15288
|
+
} catch {
|
|
15289
|
+
continue;
|
|
15290
|
+
}
|
|
15291
|
+
let streamedBytes = 0;
|
|
15292
|
+
stream.on("data", (chunk) => {
|
|
15293
|
+
const newlineIndex = chunk.lastIndexOf("\n");
|
|
15294
|
+
if (newlineIndex !== -1) {
|
|
15295
|
+
lastCompleteOffset = startOffset + streamedBytes
|
|
15296
|
+
+ Buffer.byteLength(chunk.slice(0, newlineIndex + 1), "utf8");
|
|
15297
|
+
}
|
|
15298
|
+
streamedBytes += Buffer.byteLength(chunk, "utf8");
|
|
15299
|
+
});
|
|
15300
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
15301
|
+
for await (const line of rl) {
|
|
15302
|
+
if (!line || !line.trim()) continue;
|
|
15303
|
+
let entry;
|
|
15304
|
+
try { entry = JSON.parse(line); } catch { continue; }
|
|
15305
|
+
const msg = entry?.type === "message" ? entry.message : null;
|
|
15306
|
+
const usage = msg?.role === "assistant" ? msg.usage : null;
|
|
15307
|
+
if (!usage || typeof usage !== "object") continue;
|
|
15308
|
+
|
|
15309
|
+
const entryId = typeof entry.id === "string" && entry.id ? entry.id : null;
|
|
15310
|
+
if (!entryId || projectSeenIds.has(entryId)) continue;
|
|
15311
|
+
|
|
15312
|
+
const input = toNonNegativeInt(usage.input);
|
|
15313
|
+
const output = toNonNegativeInt(usage.output);
|
|
15314
|
+
const cacheRead = toNonNegativeInt(usage.cacheRead);
|
|
15315
|
+
const cacheWrite = toNonNegativeInt(usage.cacheWrite);
|
|
15316
|
+
const reasoningTokens = toNonNegativeInt(usage.reasoningTokens);
|
|
15317
|
+
if (
|
|
15318
|
+
input === 0 &&
|
|
15319
|
+
output === 0 &&
|
|
15320
|
+
cacheRead === 0 &&
|
|
15321
|
+
cacheWrite === 0 &&
|
|
15322
|
+
reasoningTokens === 0
|
|
15323
|
+
) {
|
|
15324
|
+
projectSeenIds.add(entryId);
|
|
15325
|
+
continue;
|
|
15326
|
+
}
|
|
15327
|
+
|
|
15328
|
+
let tsMs = null;
|
|
15329
|
+
if (Number.isFinite(Number(msg.timestamp)) && Number(msg.timestamp) > 0) {
|
|
15330
|
+
tsMs = Number(msg.timestamp);
|
|
15331
|
+
} else if (typeof entry.timestamp === "string" && entry.timestamp) {
|
|
15332
|
+
const parsed = Date.parse(entry.timestamp);
|
|
15333
|
+
if (Number.isFinite(parsed) && parsed > 0) tsMs = parsed;
|
|
15334
|
+
}
|
|
15335
|
+
const bucketStart = tsMs == null
|
|
15336
|
+
? null
|
|
15337
|
+
: toUtcHalfHourStart(new Date(tsMs).toISOString());
|
|
15338
|
+
if (!bucketStart) {
|
|
15339
|
+
projectSeenIds.add(entryId);
|
|
15340
|
+
continue;
|
|
15341
|
+
}
|
|
15342
|
+
|
|
15343
|
+
const totalTokens =
|
|
15344
|
+
Number.isFinite(Number(usage.totalTokens)) && Number(usage.totalTokens) > 0
|
|
15345
|
+
? toNonNegativeInt(usage.totalTokens)
|
|
15346
|
+
: input + output + cacheRead + cacheWrite + reasoningTokens;
|
|
15347
|
+
const delta = {
|
|
15348
|
+
input_tokens: input,
|
|
15349
|
+
cached_input_tokens: cacheRead,
|
|
15350
|
+
cache_creation_input_tokens: cacheWrite,
|
|
15351
|
+
output_tokens: output,
|
|
15352
|
+
reasoning_output_tokens: reasoningTokens,
|
|
15353
|
+
total_tokens: totalTokens,
|
|
15354
|
+
conversation_count: 1,
|
|
15355
|
+
};
|
|
15356
|
+
const projectBucket = getProjectBucket(
|
|
15357
|
+
projectState,
|
|
15358
|
+
projectKey,
|
|
15359
|
+
sourceForProvider(msg.provider),
|
|
15360
|
+
bucketStart,
|
|
15361
|
+
projectRef,
|
|
15362
|
+
);
|
|
15363
|
+
addTotals(projectBucket.totals, delta);
|
|
15364
|
+
projectTouchedBuckets.add(projectBucketKey(projectKey, sourceForProvider(msg.provider), bucketStart));
|
|
15365
|
+
projectSeenIds.add(entryId);
|
|
15366
|
+
}
|
|
15367
|
+
}
|
|
15368
|
+
|
|
15369
|
+
let postStat = stat;
|
|
15370
|
+
try { postStat = fssync.statSync(filePath); } catch {}
|
|
15371
|
+
projectFileOffsets[filePath] = {
|
|
15372
|
+
size: Math.min(lastCompleteOffset, postStat.size),
|
|
15373
|
+
mtimeMs: postStat.mtimeMs,
|
|
15374
|
+
ino: postStat.ino,
|
|
15375
|
+
};
|
|
15376
|
+
}
|
|
15377
|
+
}
|
|
15378
|
+
|
|
15379
|
+
const seenArr = Array.from(seenIds);
|
|
15380
|
+
const cappedSeen =
|
|
15381
|
+
seenArr.length > 10_000 ? seenArr.slice(seenArr.length - 10_000) : seenArr;
|
|
15382
|
+
|
|
15383
|
+
const bucketsQueued = await enqueueTouchedBuckets({
|
|
15384
|
+
queuePath,
|
|
15385
|
+
hourlyState,
|
|
15386
|
+
touchedBuckets,
|
|
15387
|
+
});
|
|
15388
|
+
const projectBucketsQueued = projectEnabled
|
|
15389
|
+
? await enqueueTouchedProjectBuckets({ projectQueuePath, projectState, projectTouchedBuckets })
|
|
15390
|
+
: 0;
|
|
14809
15391
|
const updatedAt = new Date().toISOString();
|
|
14810
15392
|
hourlyState.updatedAt = updatedAt;
|
|
14811
15393
|
cursors.hourly = hourlyState;
|
|
15394
|
+
if (projectState) {
|
|
15395
|
+
projectState.updatedAt = updatedAt;
|
|
15396
|
+
cursors.projectHourly = projectState;
|
|
15397
|
+
}
|
|
14812
15398
|
cursors[stateKey] = {
|
|
14813
15399
|
...providerState,
|
|
14814
15400
|
seenIds: cappedSeen,
|
|
14815
15401
|
fileOffsets,
|
|
15402
|
+
...(projectEnabled ? {
|
|
15403
|
+
projectSeenIds: Array.from(projectSeenIds).slice(-10_000),
|
|
15404
|
+
projectFileOffsets,
|
|
15405
|
+
} : {}),
|
|
14816
15406
|
updatedAt,
|
|
14817
15407
|
};
|
|
14818
15408
|
|
|
14819
|
-
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
15409
|
+
return { recordsProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
|
|
14820
15410
|
}
|
|
14821
15411
|
|
|
14822
15412
|
async function parsePiIncremental(options = {}) {
|
|
@@ -19782,9 +20372,10 @@ async function parseTraeCnApiIncremental({
|
|
|
19782
20372
|
// built-in zlib zstd first, `@mongodb-js/zstd` as the Node 20 fallback —
|
|
19783
20373
|
// enforcing a cumulative plaintext bound as we go.
|
|
19784
20374
|
// Dedup is a per-file `lastSeq` watermark — seq
|
|
19785
|
-
// is monotonic within
|
|
19786
|
-
// and skips everything at or below the watermark
|
|
19787
|
-
//
|
|
20375
|
+
// is monotonic within an append-only session log, so a grow re-reads the file
|
|
20376
|
+
// and skips everything at or below the watermark. Torn tails never advance the
|
|
20377
|
+
// watermark until their JSON record is complete; format replacements are
|
|
20378
|
+
// reconciled through per-session contribution ledgers.
|
|
19788
20379
|
const DSH_SESSION_LOG_MAX_BYTES = 64 * 1024 * 1024;
|
|
19789
20380
|
const DSH_SESSION_TEXT_MAX_BYTES = 128 * 1024 * 1024;
|
|
19790
20381
|
const DSH_SOURCE = "dsh";
|
|
@@ -19834,13 +20425,20 @@ function resolveDshHomes(env = process.env, deps = {}) {
|
|
|
19834
20425
|
return [...new Set([resolved.native, resolved.wsl].filter(Boolean))];
|
|
19835
20426
|
}
|
|
19836
20427
|
|
|
20428
|
+
const DSH_SESSION_LOG_PATTERN = /^session(?:\.v\d+)?\.jsonl(?:\.zstd)?$/;
|
|
20429
|
+
|
|
19837
20430
|
function isDshSessionLogName(name) {
|
|
19838
|
-
return name === "
|
|
20431
|
+
return typeof name === "string" && DSH_SESSION_LOG_PATTERN.test(name);
|
|
20432
|
+
}
|
|
20433
|
+
|
|
20434
|
+
function parseDshVersion(name) {
|
|
20435
|
+
const match = typeof name === "string" ? name.match(/\.v(\d+)\.jsonl/) : null;
|
|
20436
|
+
return match ? parseInt(match[1], 10) : 0;
|
|
19839
20437
|
}
|
|
19840
20438
|
|
|
19841
20439
|
// Walk the harness sessions root for per-session log artifacts. The tree is
|
|
19842
|
-
// `<sessions-root>/<project-key>/<session-id>/session.jsonl[.zstd]`; only
|
|
19843
|
-
//
|
|
20440
|
+
// `<sessions-root>/<project-key>/<session-id>/session[.v3].jsonl[.zstd]`; only
|
|
20441
|
+
// the matching leaf names are collected so unrelated harness files are ignored.
|
|
19844
20442
|
async function resolveDshSessionFiles(env = process.env, deps = {}) {
|
|
19845
20443
|
const out = [];
|
|
19846
20444
|
const seen = new Set();
|
|
@@ -19864,7 +20462,7 @@ async function resolveDshSessionFiles(env = process.env, deps = {}) {
|
|
|
19864
20462
|
} else if (transcripts.length > 1) {
|
|
19865
20463
|
// Harness itself rejects mixed encodings in one root. For a passive
|
|
19866
20464
|
// reader, choose the actively-written artifact instead of counting the
|
|
19867
|
-
// same session twice; a tie prefers the
|
|
20465
|
+
// same session twice; a tie prefers the higher format version, then zstd.
|
|
19868
20466
|
const ranked = await Promise.all(transcripts.map(async (artifact) => {
|
|
19869
20467
|
const full = path.join(sessionDir, artifact.name);
|
|
19870
20468
|
const handle = await fs.open(full, "r").catch(() => null);
|
|
@@ -19882,6 +20480,7 @@ async function resolveDshSessionFiles(env = process.env, deps = {}) {
|
|
|
19882
20480
|
}));
|
|
19883
20481
|
ranked.sort((left, right) =>
|
|
19884
20482
|
right.mtimeMs - left.mtimeMs ||
|
|
20483
|
+
parseDshVersion(right.name) - parseDshVersion(left.name) ||
|
|
19885
20484
|
Number(right.name.endsWith(".zstd")) - Number(left.name.endsWith(".zstd")),
|
|
19886
20485
|
);
|
|
19887
20486
|
selected = ranked[0]?.full || null;
|
|
@@ -20072,6 +20671,161 @@ function sameDshSessionMetadata(previous, current) {
|
|
|
20072
20671
|
);
|
|
20073
20672
|
}
|
|
20074
20673
|
|
|
20674
|
+
function dshSessionDirectoryKey(filePath) {
|
|
20675
|
+
return typeof filePath === "string" ? path.dirname(path.resolve(filePath)) : null;
|
|
20676
|
+
}
|
|
20677
|
+
|
|
20678
|
+
function normalizeDshContributions(value) {
|
|
20679
|
+
const normalized = {};
|
|
20680
|
+
if (!value || typeof value !== "object") return normalized;
|
|
20681
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
20682
|
+
if (!entry || typeof entry !== "object" || !entry.totals) continue;
|
|
20683
|
+
const model = normalizeModelInput(entry.model);
|
|
20684
|
+
const bucketStart = typeof entry.bucketStart === "string" ? entry.bucketStart : null;
|
|
20685
|
+
if (!model || !bucketStart) continue;
|
|
20686
|
+
normalized[key] = {
|
|
20687
|
+
model,
|
|
20688
|
+
bucketStart,
|
|
20689
|
+
totals: cloneTotals(entry.totals),
|
|
20690
|
+
};
|
|
20691
|
+
}
|
|
20692
|
+
return normalized;
|
|
20693
|
+
}
|
|
20694
|
+
|
|
20695
|
+
function storedDshContributions(state) {
|
|
20696
|
+
if (
|
|
20697
|
+
!state ||
|
|
20698
|
+
typeof state !== "object" ||
|
|
20699
|
+
!Object.prototype.hasOwnProperty.call(state, "contributions") ||
|
|
20700
|
+
!state.contributions ||
|
|
20701
|
+
typeof state.contributions !== "object"
|
|
20702
|
+
) {
|
|
20703
|
+
return null;
|
|
20704
|
+
}
|
|
20705
|
+
const normalized = normalizeDshContributions(state.contributions);
|
|
20706
|
+
// A partially written or hand-edited ledger is not a safe subtraction
|
|
20707
|
+
// baseline. Treat it as absent so migration defers instead of silently
|
|
20708
|
+
// retracting only some of a session's contribution.
|
|
20709
|
+
if (Object.keys(normalized).length !== Object.keys(state.contributions).length) {
|
|
20710
|
+
return null;
|
|
20711
|
+
}
|
|
20712
|
+
return normalized;
|
|
20713
|
+
}
|
|
20714
|
+
|
|
20715
|
+
function addDshContribution(contributions, model, bucketStart, totals) {
|
|
20716
|
+
const key = bucketKey(DSH_SOURCE, model, bucketStart);
|
|
20717
|
+
let contribution = contributions[key];
|
|
20718
|
+
if (!contribution) {
|
|
20719
|
+
contribution = {
|
|
20720
|
+
model,
|
|
20721
|
+
bucketStart,
|
|
20722
|
+
totals: initTotals(),
|
|
20723
|
+
};
|
|
20724
|
+
contributions[key] = contribution;
|
|
20725
|
+
}
|
|
20726
|
+
addTotals(contribution.totals, totals);
|
|
20727
|
+
}
|
|
20728
|
+
|
|
20729
|
+
function dshContributionsFromDeltas(deltas) {
|
|
20730
|
+
const contributions = {};
|
|
20731
|
+
for (const delta of deltas || []) {
|
|
20732
|
+
const bucketStart = toUtcHalfHourStart(delta.timeMs);
|
|
20733
|
+
if (!bucketStart || !delta.model || !delta.totals) continue;
|
|
20734
|
+
addDshContribution(contributions, delta.model, bucketStart, delta.totals);
|
|
20735
|
+
}
|
|
20736
|
+
return contributions;
|
|
20737
|
+
}
|
|
20738
|
+
|
|
20739
|
+
function legacyDshContributionsFromSnapshot(snapshot, previousState) {
|
|
20740
|
+
if (
|
|
20741
|
+
!snapshot?.text ||
|
|
20742
|
+
!previousState ||
|
|
20743
|
+
typeof previousState !== "object" ||
|
|
20744
|
+
!Number.isSafeInteger(previousState.lastSeq) ||
|
|
20745
|
+
previousState.lastSeq < -1 ||
|
|
20746
|
+
previousState.inode !== snapshot.inode ||
|
|
20747
|
+
!Number.isFinite(previousState.size) ||
|
|
20748
|
+
snapshot.size < previousState.size
|
|
20749
|
+
) {
|
|
20750
|
+
return null;
|
|
20751
|
+
}
|
|
20752
|
+
const parsed = extractDshSessionUsage(snapshot.text, -1);
|
|
20753
|
+
if (!parsed.complete || parsed.sessionId !== previousState.sessionId) return null;
|
|
20754
|
+
// The pre-ledger parser replayed unknown-sequence usage on every pass, so
|
|
20755
|
+
// there is no safe prefix boundary for such a record. Defer instead.
|
|
20756
|
+
if (parsed.deltas.some((delta) => !Number.isSafeInteger(delta.seq))) return null;
|
|
20757
|
+
return dshContributionsFromDeltas(
|
|
20758
|
+
parsed.deltas.filter((delta) => delta.seq <= previousState.lastSeq),
|
|
20759
|
+
);
|
|
20760
|
+
}
|
|
20761
|
+
|
|
20762
|
+
function dshContributionsCoverPrior(prior, candidate) {
|
|
20763
|
+
for (const [key, previous] of Object.entries(prior || {})) {
|
|
20764
|
+
const current = candidate?.[key];
|
|
20765
|
+
if (!current?.totals || !previous?.totals) return false;
|
|
20766
|
+
for (const field of [
|
|
20767
|
+
"input_tokens",
|
|
20768
|
+
"cached_input_tokens",
|
|
20769
|
+
"cache_creation_input_tokens",
|
|
20770
|
+
"output_tokens",
|
|
20771
|
+
"reasoning_output_tokens",
|
|
20772
|
+
"total_tokens",
|
|
20773
|
+
"billable_total_tokens",
|
|
20774
|
+
"total_cost_usd",
|
|
20775
|
+
"conversation_count",
|
|
20776
|
+
]) {
|
|
20777
|
+
const available = Number(current.totals[field] || 0);
|
|
20778
|
+
const required = Number(previous.totals[field] || 0);
|
|
20779
|
+
if (!Number.isFinite(available) || !Number.isFinite(required) || available < required) {
|
|
20780
|
+
return false;
|
|
20781
|
+
}
|
|
20782
|
+
}
|
|
20783
|
+
}
|
|
20784
|
+
return true;
|
|
20785
|
+
}
|
|
20786
|
+
|
|
20787
|
+
function dshContributionsFitHourlyState(hourlyState, contributions) {
|
|
20788
|
+
for (const contribution of Object.values(contributions || {})) {
|
|
20789
|
+
if (!contribution?.model || !contribution.bucketStart || !contribution.totals) continue;
|
|
20790
|
+
const key = bucketKey(DSH_SOURCE, contribution.model, contribution.bucketStart);
|
|
20791
|
+
const bucket = hourlyState?.buckets?.[key];
|
|
20792
|
+
if (!bucket?.totals) return false;
|
|
20793
|
+
for (const field of [
|
|
20794
|
+
"input_tokens",
|
|
20795
|
+
"cached_input_tokens",
|
|
20796
|
+
"cache_creation_input_tokens",
|
|
20797
|
+
"output_tokens",
|
|
20798
|
+
"reasoning_output_tokens",
|
|
20799
|
+
"total_tokens",
|
|
20800
|
+
"billable_total_tokens",
|
|
20801
|
+
"total_cost_usd",
|
|
20802
|
+
"conversation_count",
|
|
20803
|
+
]) {
|
|
20804
|
+
const available = Number(bucket.totals[field] || 0);
|
|
20805
|
+
const required = Number(contribution.totals[field] || 0);
|
|
20806
|
+
if (!Number.isFinite(available) || !Number.isFinite(required) || available < required) {
|
|
20807
|
+
return false;
|
|
20808
|
+
}
|
|
20809
|
+
}
|
|
20810
|
+
}
|
|
20811
|
+
return true;
|
|
20812
|
+
}
|
|
20813
|
+
|
|
20814
|
+
function applyDshContributions({ hourlyState, touchedBuckets, contributions, subtract = false }) {
|
|
20815
|
+
for (const contribution of Object.values(contributions || {})) {
|
|
20816
|
+
if (!contribution?.model || !contribution.bucketStart || !contribution.totals) continue;
|
|
20817
|
+
const bucket = getHourlyBucket(
|
|
20818
|
+
hourlyState,
|
|
20819
|
+
DSH_SOURCE,
|
|
20820
|
+
contribution.model,
|
|
20821
|
+
contribution.bucketStart,
|
|
20822
|
+
);
|
|
20823
|
+
if (subtract) subtractTotals(bucket.totals, contribution.totals);
|
|
20824
|
+
else addTotals(bucket.totals, contribution.totals);
|
|
20825
|
+
touchedBuckets.add(bucketKey(DSH_SOURCE, contribution.model, contribution.bucketStart));
|
|
20826
|
+
}
|
|
20827
|
+
}
|
|
20828
|
+
|
|
20075
20829
|
// Open, inspect and read through one handle so a path replacement cannot make
|
|
20076
20830
|
// the metadata describe a different file from the bytes we parse.
|
|
20077
20831
|
async function readDshSessionSnapshot(
|
|
@@ -20285,25 +21039,149 @@ function parseDshUsageSlice(raw) {
|
|
|
20285
21039
|
};
|
|
20286
21040
|
}
|
|
20287
21041
|
|
|
21042
|
+
function isCompleteDshJsonLine(raw) {
|
|
21043
|
+
const text = String(raw || "").trim();
|
|
21044
|
+
if (!text || text[0] !== "{") return false;
|
|
21045
|
+
let index = 0;
|
|
21046
|
+
const maxDepth = 256;
|
|
21047
|
+
const hex = (char) => /[0-9a-f]/i.test(char || "");
|
|
21048
|
+
const skipWhitespace = () => {
|
|
21049
|
+
while (index < text.length && /\s/.test(text[index])) index += 1;
|
|
21050
|
+
};
|
|
21051
|
+
const parseString = () => {
|
|
21052
|
+
if (text[index] !== '"') return false;
|
|
21053
|
+
index += 1;
|
|
21054
|
+
while (index < text.length) {
|
|
21055
|
+
const char = text[index++];
|
|
21056
|
+
if (char === '"') return true;
|
|
21057
|
+
if (char.charCodeAt(0) < 0x20) return false;
|
|
21058
|
+
if (char !== "\\") continue;
|
|
21059
|
+
if (index >= text.length) return false;
|
|
21060
|
+
const escaped = text[index++];
|
|
21061
|
+
if (escaped === "u") {
|
|
21062
|
+
if (index + 4 > text.length || ![...text.slice(index, index + 4)].every(hex)) return false;
|
|
21063
|
+
index += 4;
|
|
21064
|
+
} else if (!'"\\/bfnrt'.includes(escaped)) {
|
|
21065
|
+
return false;
|
|
21066
|
+
}
|
|
21067
|
+
}
|
|
21068
|
+
return false;
|
|
21069
|
+
};
|
|
21070
|
+
const parseNumber = () => {
|
|
21071
|
+
const start = index;
|
|
21072
|
+
if (text[index] === "-") index += 1;
|
|
21073
|
+
if (text[index] === "0") {
|
|
21074
|
+
index += 1;
|
|
21075
|
+
} else if (/[1-9]/.test(text[index] || "")) {
|
|
21076
|
+
while (/[0-9]/.test(text[index] || "")) index += 1;
|
|
21077
|
+
} else {
|
|
21078
|
+
return false;
|
|
21079
|
+
}
|
|
21080
|
+
if (text[index] === ".") {
|
|
21081
|
+
index += 1;
|
|
21082
|
+
const fractionStart = index;
|
|
21083
|
+
while (/[0-9]/.test(text[index] || "")) index += 1;
|
|
21084
|
+
if (index === fractionStart) return false;
|
|
21085
|
+
}
|
|
21086
|
+
if (text[index] === "e" || text[index] === "E") {
|
|
21087
|
+
index += 1;
|
|
21088
|
+
if (text[index] === "+" || text[index] === "-") index += 1;
|
|
21089
|
+
const exponentStart = index;
|
|
21090
|
+
while (/[0-9]/.test(text[index] || "")) index += 1;
|
|
21091
|
+
if (index === exponentStart) return false;
|
|
21092
|
+
}
|
|
21093
|
+
return index > start;
|
|
21094
|
+
};
|
|
21095
|
+
const parseValue = (depth) => {
|
|
21096
|
+
if (depth > maxDepth) return false;
|
|
21097
|
+
skipWhitespace();
|
|
21098
|
+
const char = text[index];
|
|
21099
|
+
if (char === '"') return parseString();
|
|
21100
|
+
if (char === "{") {
|
|
21101
|
+
index += 1;
|
|
21102
|
+
skipWhitespace();
|
|
21103
|
+
if (text[index] === "}") {
|
|
21104
|
+
index += 1;
|
|
21105
|
+
return true;
|
|
21106
|
+
}
|
|
21107
|
+
while (index < text.length) {
|
|
21108
|
+
skipWhitespace();
|
|
21109
|
+
if (!parseString()) return false;
|
|
21110
|
+
skipWhitespace();
|
|
21111
|
+
if (text[index++] !== ":") return false;
|
|
21112
|
+
if (!parseValue(depth + 1)) return false;
|
|
21113
|
+
skipWhitespace();
|
|
21114
|
+
if (text[index] === "}") {
|
|
21115
|
+
index += 1;
|
|
21116
|
+
return true;
|
|
21117
|
+
}
|
|
21118
|
+
if (text[index++] !== ",") return false;
|
|
21119
|
+
}
|
|
21120
|
+
return false;
|
|
21121
|
+
}
|
|
21122
|
+
if (char === "[") {
|
|
21123
|
+
index += 1;
|
|
21124
|
+
skipWhitespace();
|
|
21125
|
+
if (text[index] === "]") {
|
|
21126
|
+
index += 1;
|
|
21127
|
+
return true;
|
|
21128
|
+
}
|
|
21129
|
+
while (index < text.length) {
|
|
21130
|
+
if (!parseValue(depth + 1)) return false;
|
|
21131
|
+
skipWhitespace();
|
|
21132
|
+
if (text[index] === "]") {
|
|
21133
|
+
index += 1;
|
|
21134
|
+
return true;
|
|
21135
|
+
}
|
|
21136
|
+
if (text[index++] !== ",") return false;
|
|
21137
|
+
skipWhitespace();
|
|
21138
|
+
}
|
|
21139
|
+
return false;
|
|
21140
|
+
}
|
|
21141
|
+
for (const literal of ["true", "false", "null"]) {
|
|
21142
|
+
if (text.startsWith(literal, index)) {
|
|
21143
|
+
index += literal.length;
|
|
21144
|
+
return true;
|
|
21145
|
+
}
|
|
21146
|
+
}
|
|
21147
|
+
return parseNumber();
|
|
21148
|
+
};
|
|
21149
|
+
|
|
21150
|
+
if (!parseValue(0)) return false;
|
|
21151
|
+
skipWhitespace();
|
|
21152
|
+
return index === text.length;
|
|
21153
|
+
}
|
|
21154
|
+
|
|
20288
21155
|
// Parse one session log's plaintext into usage deltas, skipping events whose
|
|
20289
21156
|
// seq is at or below the watermark. Returns the deltas (each carrying the
|
|
20290
|
-
// model and epoch-ms timestamp), the highest seq seen,
|
|
21157
|
+
// sequence, model and epoch-ms timestamp), the highest complete seq seen, the
|
|
21158
|
+
// session id, and whether every non-empty JSONL line was complete.
|
|
20291
21159
|
function extractDshSessionUsage(text, lastSeq = -1) {
|
|
20292
21160
|
const deltas = [];
|
|
20293
21161
|
const watermark = Number.isFinite(lastSeq) ? lastSeq : -1;
|
|
20294
21162
|
let maxSeq = watermark;
|
|
20295
21163
|
let sessionId = null;
|
|
20296
21164
|
let headerModel = null;
|
|
21165
|
+
let complete = true;
|
|
20297
21166
|
|
|
20298
21167
|
const lines = String(text || "").split("\n");
|
|
20299
21168
|
for (let i = 0; i < lines.length; i++) {
|
|
20300
21169
|
const line = lines[i];
|
|
20301
21170
|
if (!line || !line.trim()) continue;
|
|
21171
|
+
if (!isCompleteDshJsonLine(line)) {
|
|
21172
|
+
complete = false;
|
|
21173
|
+
// A later event cannot make this gap safe to acknowledge. Stop at the
|
|
21174
|
+
// complete prefix so a repaired record is retried before seq advances.
|
|
21175
|
+
break;
|
|
21176
|
+
}
|
|
20302
21177
|
const eventType = parseDshJsonString(findDshJsonProperty(line, "type"));
|
|
20303
21178
|
if (!eventType) continue;
|
|
20304
21179
|
|
|
20305
|
-
if (eventType === "session") {
|
|
20306
|
-
const id =
|
|
21180
|
+
if (eventType === "session" || eventType === "session/start") {
|
|
21181
|
+
const id =
|
|
21182
|
+
parseDshJsonString(findDshJsonProperty(line, "id")) ||
|
|
21183
|
+
parseDshJsonString(findDshJsonProperty(findDshJsonProperty(line, "data"), "id")) ||
|
|
21184
|
+
parseDshJsonString(findDshJsonProperty(findDshJsonProperty(line, "data"), "sessionId"));
|
|
20307
21185
|
if (id) sessionId = id;
|
|
20308
21186
|
continue;
|
|
20309
21187
|
}
|
|
@@ -20324,114 +21202,397 @@ function extractDshSessionUsage(text, lastSeq = -1) {
|
|
|
20324
21202
|
continue;
|
|
20325
21203
|
}
|
|
20326
21204
|
|
|
20327
|
-
if (eventType !== "assistant/message") continue;
|
|
21205
|
+
if (eventType !== "assistant/message" && eventType !== "message/assistant") continue;
|
|
20328
21206
|
if (seqKnown && seq <= watermark) continue;
|
|
20329
21207
|
|
|
20330
21208
|
const message = findDshJsonProperty(data, "message");
|
|
20331
|
-
const source = findDshJsonProperty(message, "source");
|
|
21209
|
+
const source = message ? findDshJsonProperty(message, "source") : null;
|
|
20332
21210
|
const model = normalizeDshModelName(
|
|
20333
|
-
parseDshJsonString(findDshJsonProperty(source, "model"))
|
|
21211
|
+
(source && parseDshJsonString(findDshJsonProperty(source, "model"))) ||
|
|
21212
|
+
parseDshJsonString(findDshJsonProperty(data, "model")),
|
|
20334
21213
|
) || headerModel;
|
|
20335
21214
|
const totals = dshUsageToTotals(
|
|
20336
|
-
parseDshUsageSlice(
|
|
21215
|
+
parseDshUsageSlice(
|
|
21216
|
+
findDshJsonProperty(data, "usage") || findDshJsonProperty(line, "usage"),
|
|
21217
|
+
),
|
|
20337
21218
|
);
|
|
20338
21219
|
if (!model || !totals) continue;
|
|
20339
21220
|
|
|
20340
|
-
const timeMs =
|
|
21221
|
+
const timeMs =
|
|
21222
|
+
parseDshJsonNumber(findDshJsonProperty(line, "time")) ||
|
|
21223
|
+
parseDshJsonNumber(findDshJsonProperty(line, "timestamp")) ||
|
|
21224
|
+
parseDshJsonNumber(findDshJsonProperty(data, "time")) ||
|
|
21225
|
+
parseDshJsonNumber(findDshJsonProperty(data, "timestamp"));
|
|
20341
21226
|
if (!Number.isFinite(timeMs) || timeMs <= 0) continue;
|
|
20342
21227
|
|
|
20343
|
-
deltas.push({ model, timeMs, totals });
|
|
21228
|
+
deltas.push({ seq: seqKnown ? seq : null, model, timeMs, totals });
|
|
20344
21229
|
}
|
|
20345
21230
|
|
|
20346
|
-
return { deltas, maxSeq, sessionId };
|
|
21231
|
+
return { deltas, maxSeq, sessionId, complete };
|
|
20347
21232
|
}
|
|
20348
21233
|
|
|
20349
21234
|
// Incremental parser entrypoint. Mirrors the other passive JSONL readers:
|
|
20350
21235
|
// identity-check each file (inode/size/mtime), re-read + re-parse only when it
|
|
20351
21236
|
// changed, dedup via the per-file seq watermark, accumulate into hourly
|
|
20352
|
-
// buckets, then flush touched buckets to the queue.
|
|
21237
|
+
// buckets, then flush touched buckets to the queue. Session contribution ledgers
|
|
21238
|
+
// make artifact-path migrations replace one session instead of resetting DSH.
|
|
20353
21239
|
async function parseDshIncremental({ sessionFiles, cursors, queuePath, onProgress }) {
|
|
20354
21240
|
await ensureDir(path.dirname(queuePath));
|
|
20355
21241
|
if (!cursors || typeof cursors !== "object") cursors = {};
|
|
20356
21242
|
if (!cursors.dsh || typeof cursors.dsh !== "object") cursors.dsh = {};
|
|
20357
21243
|
const dshState = cursors.dsh;
|
|
20358
|
-
const
|
|
21244
|
+
const storedFileState =
|
|
20359
21245
|
dshState.files && typeof dshState.files === "object" ? dshState.files : {};
|
|
21246
|
+
let fileState = { ...storedFileState };
|
|
21247
|
+
const storedSessionState =
|
|
21248
|
+
dshState.sessions && typeof dshState.sessions === "object" ? dshState.sessions : {};
|
|
21249
|
+
const sessionState = { ...storedSessionState };
|
|
20360
21250
|
|
|
20361
21251
|
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
21252
|
+
// normalizeHourlyState preserves bucket objects for the other incremental
|
|
21253
|
+
// parsers. DSH needs transactional ownership because replacement validation
|
|
21254
|
+
// can defer after inspecting several files and queue append may fail.
|
|
21255
|
+
hourlyState.groupQueued = { ...(hourlyState.groupQueued || {}) };
|
|
21256
|
+
for (const [key, bucket] of Object.entries(hourlyState.buckets || {})) {
|
|
21257
|
+
hourlyState.buckets[key] = {
|
|
21258
|
+
...(bucket && typeof bucket === "object" ? bucket : {}),
|
|
21259
|
+
totals: {
|
|
21260
|
+
...initTotals(),
|
|
21261
|
+
...(bucket?.totals && typeof bucket.totals === "object" ? bucket.totals : {}),
|
|
21262
|
+
},
|
|
21263
|
+
};
|
|
21264
|
+
}
|
|
20362
21265
|
const touchedBuckets = new Set();
|
|
21266
|
+
const deferredMigrationPaths = new Map();
|
|
20363
21267
|
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
20364
21268
|
|
|
20365
|
-
const files = Array.isArray(sessionFiles)
|
|
20366
|
-
|
|
21269
|
+
const files = Array.isArray(sessionFiles)
|
|
21270
|
+
? sessionFiles.filter((filePath) => typeof filePath === "string")
|
|
21271
|
+
: [];
|
|
21272
|
+
const presentFiles = new Set(files);
|
|
20367
21273
|
const total = files.length;
|
|
21274
|
+
const deferredFilePaths = new Set();
|
|
20368
21275
|
let recordsProcessed = 0;
|
|
20369
21276
|
let eventsAggregated = 0;
|
|
20370
21277
|
|
|
21278
|
+
const staleFileForSession = (sessionId, currentPath) => {
|
|
21279
|
+
if (!sessionId) return null;
|
|
21280
|
+
for (const [filePath, state] of Object.entries(fileState)) {
|
|
21281
|
+
if (
|
|
21282
|
+
filePath !== currentPath &&
|
|
21283
|
+
!presentFiles.has(filePath) &&
|
|
21284
|
+
(state?.sessionId === sessionId ||
|
|
21285
|
+
(!state?.sessionId &&
|
|
21286
|
+
dshSessionDirectoryKey(filePath) === dshSessionDirectoryKey(currentPath)))
|
|
21287
|
+
) {
|
|
21288
|
+
return filePath;
|
|
21289
|
+
}
|
|
21290
|
+
}
|
|
21291
|
+
return null;
|
|
21292
|
+
};
|
|
21293
|
+
|
|
21294
|
+
const deferStaleMigrationPaths = (currentPath, reason) => {
|
|
21295
|
+
let deferred = 0;
|
|
21296
|
+
for (const oldPath of Object.keys(fileState)) {
|
|
21297
|
+
if (
|
|
21298
|
+
!presentFiles.has(oldPath) &&
|
|
21299
|
+
dshSessionDirectoryKey(oldPath) === dshSessionDirectoryKey(currentPath)
|
|
21300
|
+
) {
|
|
21301
|
+
if (!deferredMigrationPaths.has(oldPath)) deferred += 1;
|
|
21302
|
+
deferredFilePaths.add(oldPath);
|
|
21303
|
+
deferredMigrationPaths.set(oldPath, reason);
|
|
21304
|
+
}
|
|
21305
|
+
}
|
|
21306
|
+
return deferred;
|
|
21307
|
+
};
|
|
21308
|
+
|
|
21309
|
+
const reportProgress = (idx, recordsProcessed, eventsAggregated, total) => {
|
|
21310
|
+
if (cb) {
|
|
21311
|
+
cb({
|
|
21312
|
+
index: idx + 1,
|
|
21313
|
+
total,
|
|
21314
|
+
recordsProcessed,
|
|
21315
|
+
eventsAggregated,
|
|
21316
|
+
bucketsQueued: touchedBuckets.size,
|
|
21317
|
+
deferredMigrations: deferredMigrationPaths.size,
|
|
21318
|
+
});
|
|
21319
|
+
}
|
|
21320
|
+
};
|
|
21321
|
+
|
|
20371
21322
|
for (let idx = 0; idx < files.length; idx++) {
|
|
20372
21323
|
const filePath = files[idx];
|
|
20373
21324
|
const prev = fileState[filePath] || null;
|
|
21325
|
+
const previousSessionId = typeof prev?.sessionId === "string" ? prev.sessionId : null;
|
|
21326
|
+
const previousSession = previousSessionId ? sessionState[previousSessionId] : null;
|
|
21327
|
+
const previousFileContributions = storedDshContributions(prev);
|
|
21328
|
+
const previousSessionContributions = storedDshContributions(previousSession);
|
|
21329
|
+
const needsLedgerBackfill = Boolean(
|
|
21330
|
+
previousSessionId && !previousSessionContributions && !previousFileContributions,
|
|
21331
|
+
);
|
|
20374
21332
|
let snapshot;
|
|
20375
21333
|
let parsed;
|
|
21334
|
+
let fullParsed = null;
|
|
21335
|
+
let fileReset = false;
|
|
21336
|
+
let sessionChanged = false;
|
|
21337
|
+
|
|
20376
21338
|
try {
|
|
20377
|
-
snapshot = await readDshSessionSnapshot(filePath, {
|
|
20378
|
-
|
|
20379
|
-
|
|
20380
|
-
|
|
20381
|
-
|
|
21339
|
+
snapshot = await readDshSessionSnapshot(filePath, {
|
|
21340
|
+
previous: needsLedgerBackfill ? null : prev,
|
|
21341
|
+
});
|
|
21342
|
+
if (!snapshot) {
|
|
21343
|
+
// A resolver-selected replacement can disappear between discovery and
|
|
21344
|
+
// open. Preserve the old path so a later retry cannot re-add its full
|
|
21345
|
+
// contribution. This is deliberately separate from a thrown read
|
|
21346
|
+
// error because a missing path returns null from the snapshot helper.
|
|
21347
|
+
deferStaleMigrationPaths(filePath, "replacement-missing");
|
|
21348
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21349
|
+
continue;
|
|
21350
|
+
}
|
|
21351
|
+
if (snapshot.unchanged && !needsLedgerBackfill) {
|
|
21352
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
20382
21353
|
continue;
|
|
20383
21354
|
}
|
|
20384
21355
|
|
|
20385
|
-
const
|
|
21356
|
+
const previousHasInode = Number.isFinite(prev?.inode);
|
|
21357
|
+
const previousHasSize = Number.isFinite(prev?.size);
|
|
21358
|
+
const previousHasMtime = Number.isFinite(prev?.mtimeMs);
|
|
21359
|
+
fileReset = Boolean(
|
|
21360
|
+
prev &&
|
|
21361
|
+
(
|
|
21362
|
+
(previousHasInode && snapshot.inode !== prev.inode) ||
|
|
21363
|
+
(previousHasSize && snapshot.size < prev.size) ||
|
|
21364
|
+
(
|
|
21365
|
+
previousHasSize &&
|
|
21366
|
+
previousHasMtime &&
|
|
21367
|
+
snapshot.size <= prev.size &&
|
|
21368
|
+
snapshot.mtimeMs !== prev.mtimeMs
|
|
21369
|
+
)
|
|
21370
|
+
),
|
|
21371
|
+
);
|
|
21372
|
+
const lastSeq = fileReset || !Number.isFinite(prev?.lastSeq) ? -1 : prev.lastSeq;
|
|
20386
21373
|
parsed = extractDshSessionUsage(snapshot.text, lastSeq);
|
|
20387
|
-
|
|
20388
|
-
|
|
21374
|
+
sessionChanged = Boolean(
|
|
21375
|
+
prev &&
|
|
21376
|
+
parsed.sessionId &&
|
|
21377
|
+
parsed.sessionId !== previousSessionId,
|
|
21378
|
+
);
|
|
21379
|
+
if (sessionChanged) parsed = extractDshSessionUsage(snapshot.text, -1);
|
|
21380
|
+
if (!prev || needsLedgerBackfill || sessionChanged || fileReset) {
|
|
21381
|
+
fullParsed = extractDshSessionUsage(snapshot.text, -1);
|
|
21382
|
+
if (!prev || sessionChanged || fileReset) parsed = fullParsed;
|
|
20389
21383
|
}
|
|
21384
|
+
if (!parsed.complete) snapshot.mtimeMs = -1;
|
|
20390
21385
|
} catch (error) {
|
|
20391
21386
|
if (process.env.TOKENTRACKER_DEBUG) {
|
|
20392
21387
|
process.stderr.write(`[dsh] skipped ${filePath}: ${error?.message || error}\n`);
|
|
20393
21388
|
}
|
|
20394
|
-
|
|
20395
|
-
|
|
20396
|
-
|
|
21389
|
+
// A replacement artifact may have an old path that is no longer in the
|
|
21390
|
+
// selected file list. Keep that cursor until the replacement is readable
|
|
21391
|
+
// so a failed migration cannot turn into an untracked double-count.
|
|
21392
|
+
deferStaleMigrationPaths(filePath, "replacement-read-failed");
|
|
21393
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
20397
21394
|
continue;
|
|
20398
21395
|
}
|
|
20399
21396
|
|
|
21397
|
+
const sessionId = typeof parsed.sessionId === "string" ? parsed.sessionId : null;
|
|
21398
|
+
const stalePathWithoutId = sessionId
|
|
21399
|
+
? null
|
|
21400
|
+
: Object.keys(fileState).find(
|
|
21401
|
+
(oldPath) =>
|
|
21402
|
+
!presentFiles.has(oldPath) &&
|
|
21403
|
+
dshSessionDirectoryKey(oldPath) === dshSessionDirectoryKey(filePath),
|
|
21404
|
+
);
|
|
21405
|
+
if (stalePathWithoutId) {
|
|
21406
|
+
deferredFilePaths.add(stalePathWithoutId);
|
|
21407
|
+
deferredMigrationPaths.set(stalePathWithoutId, "replacement-session-id-missing");
|
|
21408
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21409
|
+
continue;
|
|
21410
|
+
}
|
|
21411
|
+
const session = sessionId ? sessionState[sessionId] : null;
|
|
21412
|
+
const stalePath = staleFileForSession(sessionId, filePath);
|
|
21413
|
+
const previousPath =
|
|
21414
|
+
session?.lastPath && session.lastPath !== filePath ? session.lastPath : stalePath;
|
|
21415
|
+
const resetCurrentFile = Boolean(fileReset && !sessionChanged);
|
|
21416
|
+
const replacementPath = previousPath || (resetCurrentFile ? filePath : null);
|
|
21417
|
+
const sessionContributions = storedDshContributions(session);
|
|
21418
|
+
const staleContributions = storedDshContributions(stalePath && fileState[stalePath]);
|
|
21419
|
+
const fileContributions = storedDshContributions(prev);
|
|
21420
|
+
let oldContributions = sessionContributions || staleContributions || fileContributions;
|
|
21421
|
+
|
|
21422
|
+
// A pure rename preserves the same physical bytes and metadata, so an old
|
|
21423
|
+
// cursor can safely adopt the new path without relying on rewritten seqs.
|
|
21424
|
+
const previousState = previousPath ? fileState[previousPath] : null;
|
|
21425
|
+
if (
|
|
21426
|
+
sessionId &&
|
|
21427
|
+
previousPath &&
|
|
21428
|
+
!oldContributions &&
|
|
21429
|
+
!prev &&
|
|
21430
|
+
parsed.complete &&
|
|
21431
|
+
previousState &&
|
|
21432
|
+
previousState.inode === snapshot.inode &&
|
|
21433
|
+
previousState.size === snapshot.size &&
|
|
21434
|
+
previousState.mtimeMs === snapshot.mtimeMs
|
|
21435
|
+
) {
|
|
21436
|
+
// A pure rename preserves the same bytes. Reusing the parsed full
|
|
21437
|
+
// contribution is safe here because replacement below subtracts and
|
|
21438
|
+
// re-adds it; no sequence watermark is transferred.
|
|
21439
|
+
oldContributions = dshContributionsFromDeltas(parsed.deltas);
|
|
21440
|
+
}
|
|
21441
|
+
|
|
21442
|
+
// Cursors written before the contribution ledger existed can still be
|
|
21443
|
+
// reconciled when the old artifact remains available and its same-inode
|
|
21444
|
+
// growth follows the Harness append-only contract. Reconstruct only the
|
|
21445
|
+
// prefix through the old watermark; using the whole current artifact would
|
|
21446
|
+
// subtract events that were appended after the old cursor was written.
|
|
21447
|
+
if (sessionId && previousPath && !oldContributions) {
|
|
21448
|
+
const oldState = fileState[previousPath];
|
|
21449
|
+
const oldSnapshot = await readDshSessionSnapshot(previousPath).catch(() => null);
|
|
21450
|
+
oldContributions = legacyDshContributionsFromSnapshot(oldSnapshot, oldState);
|
|
21451
|
+
}
|
|
21452
|
+
const replaceSession = Boolean(
|
|
21453
|
+
sessionId &&
|
|
21454
|
+
oldContributions &&
|
|
21455
|
+
(!prev || resetCurrentFile || (previousPath && previousPath !== filePath)),
|
|
21456
|
+
);
|
|
21457
|
+
const oldContributionCount = Object.keys(oldContributions || {}).length;
|
|
21458
|
+
if (sessionId && replacementPath && !oldContributions) {
|
|
21459
|
+
deferredFilePaths.add(replacementPath);
|
|
21460
|
+
deferredMigrationPaths.set(replacementPath, "legacy-baseline-unavailable");
|
|
21461
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21462
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21463
|
+
continue;
|
|
21464
|
+
}
|
|
21465
|
+
if (
|
|
21466
|
+
replaceSession &&
|
|
21467
|
+
(!parsed.complete || (oldContributionCount > 0 && parsed.deltas.length === 0))
|
|
21468
|
+
) {
|
|
21469
|
+
// A readable header-only or torn replacement is not authoritative. Keep
|
|
21470
|
+
// the old contribution until a complete replacement with usage arrives.
|
|
21471
|
+
deferredFilePaths.add(replacementPath);
|
|
21472
|
+
deferredMigrationPaths.set(
|
|
21473
|
+
replacementPath,
|
|
21474
|
+
parsed.complete ? "replacement-has-no-usage" : "replacement-incomplete",
|
|
21475
|
+
);
|
|
21476
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21477
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21478
|
+
continue;
|
|
21479
|
+
}
|
|
21480
|
+
const candidateContributions = dshContributionsFromDeltas(parsed.deltas);
|
|
21481
|
+
if (
|
|
21482
|
+
replaceSession &&
|
|
21483
|
+
oldContributionCount > 0 &&
|
|
21484
|
+
!dshContributionsCoverPrior(oldContributions, candidateContributions)
|
|
21485
|
+
) {
|
|
21486
|
+
// A complete-looking replacement that drops a previously counted bucket
|
|
21487
|
+
// is still unsafe. Format migration should preserve prior usage; defer
|
|
21488
|
+
// until a candidate with a verifiable superset arrives.
|
|
21489
|
+
deferredFilePaths.add(replacementPath);
|
|
21490
|
+
deferredMigrationPaths.set(replacementPath, "replacement-drops-prior-usage");
|
|
21491
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21492
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21493
|
+
continue;
|
|
21494
|
+
}
|
|
21495
|
+
if (replaceSession && !dshContributionsFitHourlyState(hourlyState, oldContributions)) {
|
|
21496
|
+
// Never let subtractTotals clamp away another session's history when the
|
|
21497
|
+
// persisted ledger and hourly bucket disagree. Defer for a repairable,
|
|
21498
|
+
// visible retry instead.
|
|
21499
|
+
deferredFilePaths.add(replacementPath);
|
|
21500
|
+
deferredMigrationPaths.set(replacementPath, "stored-contribution-not-in-bucket");
|
|
21501
|
+
if (resetCurrentFile) fileState[filePath] = { ...prev, mtimeMs: -1 };
|
|
21502
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
21503
|
+
continue;
|
|
21504
|
+
}
|
|
21505
|
+
|
|
21506
|
+
if (replaceSession) {
|
|
21507
|
+
applyDshContributions({
|
|
21508
|
+
hourlyState,
|
|
21509
|
+
touchedBuckets,
|
|
21510
|
+
contributions: oldContributions,
|
|
21511
|
+
subtract: true,
|
|
21512
|
+
});
|
|
21513
|
+
if (previousPath && !presentFiles.has(previousPath)) delete fileState[previousPath];
|
|
21514
|
+
}
|
|
21515
|
+
|
|
21516
|
+
let nextContributions = replaceSession
|
|
21517
|
+
? {}
|
|
21518
|
+
: sessionContributions || fileContributions || {};
|
|
21519
|
+
if (fullParsed) {
|
|
21520
|
+
nextContributions = dshContributionsFromDeltas(fullParsed.deltas);
|
|
21521
|
+
}
|
|
21522
|
+
|
|
20400
21523
|
for (const delta of parsed.deltas) {
|
|
20401
21524
|
const bucketStart = toUtcHalfHourStart(delta.timeMs);
|
|
20402
21525
|
if (!bucketStart) continue;
|
|
20403
21526
|
const bucket = getHourlyBucket(hourlyState, DSH_SOURCE, delta.model, bucketStart);
|
|
20404
21527
|
addTotals(bucket.totals, delta.totals);
|
|
20405
21528
|
touchedBuckets.add(bucketKey(DSH_SOURCE, delta.model, bucketStart));
|
|
21529
|
+
if (!fullParsed) addDshContribution(nextContributions, delta.model, bucketStart, delta.totals);
|
|
20406
21530
|
eventsAggregated += 1;
|
|
20407
21531
|
}
|
|
20408
21532
|
|
|
21533
|
+
const updatedAt = new Date().toISOString();
|
|
20409
21534
|
fileState[filePath] = {
|
|
20410
21535
|
inode: snapshot.inode,
|
|
20411
21536
|
size: snapshot.size,
|
|
20412
21537
|
mtimeMs: snapshot.mtimeMs,
|
|
20413
|
-
sessionId
|
|
21538
|
+
sessionId,
|
|
20414
21539
|
lastSeq: parsed.maxSeq,
|
|
20415
|
-
|
|
21540
|
+
contributions: nextContributions,
|
|
21541
|
+
updatedAt,
|
|
20416
21542
|
};
|
|
21543
|
+
if (sessionId) {
|
|
21544
|
+
sessionState[sessionId] = {
|
|
21545
|
+
lastPath: filePath,
|
|
21546
|
+
contributions: nextContributions,
|
|
21547
|
+
updatedAt,
|
|
21548
|
+
};
|
|
21549
|
+
}
|
|
20417
21550
|
recordsProcessed += 1;
|
|
20418
21551
|
|
|
20419
|
-
|
|
20420
|
-
cb({ index: idx + 1, total, recordsProcessed, eventsAggregated, bucketsQueued: touchedBuckets.size });
|
|
20421
|
-
}
|
|
21552
|
+
reportProgress(idx, recordsProcessed, eventsAggregated, total);
|
|
20422
21553
|
}
|
|
20423
21554
|
|
|
21555
|
+
const nowMs = Date.now();
|
|
20424
21556
|
for (const filePath of Object.keys(fileState)) {
|
|
20425
|
-
if (
|
|
21557
|
+
if (presentFiles.has(filePath) || deferredFilePaths.has(filePath)) continue;
|
|
21558
|
+
const state = fileState[filePath];
|
|
21559
|
+
const legacySessionId = typeof state?.sessionId === "string" ? state.sessionId : null;
|
|
21560
|
+
// Retain pre-ledger identity until a replacement is verified. Its counted
|
|
21561
|
+
// usage survives file deletion, so elapsed time cannot authorize replay.
|
|
21562
|
+
// Current ledgers retain the same identity in sessionState instead.
|
|
21563
|
+
if (legacySessionId && !storedDshContributions(state) && !sessionState[legacySessionId]) {
|
|
21564
|
+
if (!Number.isFinite(Number(state?.missingSince))) {
|
|
21565
|
+
fileState[filePath] = { ...state, missingSince: nowMs };
|
|
21566
|
+
}
|
|
21567
|
+
continue;
|
|
21568
|
+
}
|
|
21569
|
+
delete fileState[filePath];
|
|
20426
21570
|
}
|
|
20427
21571
|
|
|
20428
21572
|
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
20429
|
-
|
|
21573
|
+
const updatedAt = new Date().toISOString();
|
|
21574
|
+
hourlyState.updatedAt = updatedAt;
|
|
20430
21575
|
cursors.hourly = hourlyState;
|
|
20431
21576
|
dshState.files = fileState;
|
|
20432
|
-
dshState.
|
|
21577
|
+
dshState.sessions = sessionState;
|
|
21578
|
+
if (deferredMigrationPaths.size > 0) {
|
|
21579
|
+
const reasons = [...new Set(deferredMigrationPaths.values())].sort();
|
|
21580
|
+
dshState.deferredMigrations = {
|
|
21581
|
+
count: deferredMigrationPaths.size,
|
|
21582
|
+
reasons,
|
|
21583
|
+
updatedAt,
|
|
21584
|
+
};
|
|
21585
|
+
} else {
|
|
21586
|
+
delete dshState.deferredMigrations;
|
|
21587
|
+
}
|
|
21588
|
+
dshState.updatedAt = updatedAt;
|
|
20433
21589
|
|
|
20434
|
-
return {
|
|
21590
|
+
return {
|
|
21591
|
+
recordsProcessed,
|
|
21592
|
+
eventsAggregated,
|
|
21593
|
+
bucketsQueued,
|
|
21594
|
+
deferredMigrations: deferredMigrationPaths.size,
|
|
21595
|
+
};
|
|
20435
21596
|
}
|
|
20436
21597
|
|
|
20437
21598
|
|
|
@@ -20558,6 +21719,9 @@ module.exports = {
|
|
|
20558
21719
|
parseAnythingllmTimestamp,
|
|
20559
21720
|
readAnythingllmUsageRows,
|
|
20560
21721
|
parseAnythingllmIncremental,
|
|
21722
|
+
resolveDevinDbPath,
|
|
21723
|
+
readDevinUsageRows,
|
|
21724
|
+
parseDevinIncremental,
|
|
20561
21725
|
resolveGooseDbPath,
|
|
20562
21726
|
parseGooseModelName,
|
|
20563
21727
|
parseGooseCreatedAt,
|
|
@@ -20639,6 +21803,8 @@ module.exports = {
|
|
|
20639
21803
|
resolveDshHome,
|
|
20640
21804
|
resolveDshHomes,
|
|
20641
21805
|
resolveDshSessionFiles,
|
|
21806
|
+
isDshSessionLogName,
|
|
21807
|
+
parseDshVersion,
|
|
20642
21808
|
readDshSessionText,
|
|
20643
21809
|
decodeDshZstd,
|
|
20644
21810
|
inspectDshZstdFrames,
|