tokentracker-cli 0.93.7 → 0.94.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +1 -1
- package/README.ja.md +1 -1
- package/README.ko.md +1 -1
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/dashboard/dist/assets/{AchievementBadge-DqvblIQY.js → AchievementBadge-DnmgEa4q.js} +1 -1
- package/dashboard/dist/assets/{AchievementsPage-C-xm27pB.js → AchievementsPage-B7YTbAuK.js} +2 -2
- package/dashboard/dist/assets/{AchievementsSection-D8Oe2FGJ.js → AchievementsSection-CwgT5sqH.js} +1 -1
- package/dashboard/dist/assets/{BadgeDetailModal-BJOcOnHo.js → BadgeDetailModal-Dets6jzW.js} +1 -1
- package/dashboard/dist/assets/{BotAnimated-6fB046ld.js → BotAnimated-5KwUvC9F.js} +1 -1
- package/dashboard/dist/assets/{Card-BKbWtBaL.js → Card-jf9w-4-N.js} +1 -1
- package/dashboard/dist/assets/{ClawdAnimated-DbpeAldp.js → ClawdAnimated-CGxDVQ0c.js} +2 -2
- package/dashboard/dist/assets/{CommandPalette-Jte7TBon.js → CommandPalette-B1bUUf32.js} +1 -1
- package/dashboard/dist/assets/{CommunityStatsModal-DDm7tCXO.js → CommunityStatsModal-DRDpZycV.js} +1 -1
- package/dashboard/dist/assets/{ConfirmModal-CrIjcrpZ.js → ConfirmModal-BR-aIRH9.js} +1 -1
- package/dashboard/dist/assets/{Controls-BWJtMkIy.js → Controls-CbHsJAqV.js} +1 -1
- package/dashboard/dist/assets/{DashboardPage-Bcq4EwG5.js → DashboardPage-B5XMQoxy.js} +1 -1
- package/dashboard/dist/assets/{DevicePage-C9NcPZGX.js → DevicePage-DDlicLLn.js} +1 -1
- package/dashboard/dist/assets/{DialogClose-CY-LguUG.js → DialogClose-B89uLXqp.js} +1 -1
- package/dashboard/dist/assets/DialogDescription-DiVbwvTR.js +1 -0
- package/dashboard/dist/assets/{DialogTitle-BkAmJrQ_.js → DialogTitle-jOyycYKF.js} +1 -1
- package/dashboard/dist/assets/{FadeIn-U-bxcM-y.js → FadeIn-BEND42ze.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-4HUCxVhg.js → HeaderGithubStar-Dacl1WGS.js} +1 -1
- package/dashboard/dist/assets/{HoverTooltip-BXcXOCs8.js → HoverTooltip-B2Mh2LTi.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-Cppkc-gE.js → IpCheckPage-CPEuRhiU.js} +1 -1
- package/dashboard/dist/assets/{LandingPage-D3KXfYKE.js → LandingPage-BYWNR_dx.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardAvatar-CdsHg6L0.js → LeaderboardAvatar-B9_5LDVj.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-niat0TZW.js → LeaderboardPage-_Hl1NaIa.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProfileModal-DgjhhEbB.js → LeaderboardProfileModal-Cydp7CyA.js} +3 -3
- package/dashboard/dist/assets/{LeaderboardProfilePage-CZej8gpc.js → LeaderboardProfilePage-Dz0ecujs.js} +1 -1
- package/dashboard/dist/assets/LimitsPage-CmdT2qOZ.js +5 -0
- package/dashboard/dist/assets/{LocalOnlyNotice-DQswDMSD.js → LocalOnlyNotice-DhuPGUkw.js} +1 -1
- package/dashboard/dist/assets/{LoginCard-glPyVqYh.js → LoginCard-DQtHaX_5.js} +1 -1
- package/dashboard/dist/assets/{LoginModal-slsdEXC7.js → LoginModal-DvpPRnYV.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-DX5QT917.js → LoginPage-DOjfmdwb.js} +1 -1
- package/dashboard/dist/assets/{PetPage-B8U3CJwt.js → PetPage-Dd6U9fh2.js} +1 -1
- package/dashboard/dist/assets/{PopoverPopup-BVPcQAio.js → PopoverPopup-B1V01u10.js} +1 -1
- package/dashboard/dist/assets/{ProviderIcon-CljIacVj.js → ProviderIcon-BLRe55c7.js} +1 -1
- package/dashboard/dist/assets/{ResetPasswordPage-DTBmQk2y.js → ResetPasswordPage-BJ95vy1S.js} +1 -1
- package/dashboard/dist/assets/{SegmentedControl-C6npelCT.js → SegmentedControl-DacS_m9M.js} +1 -1
- package/dashboard/dist/assets/{Select-D1elkRBG.js → Select-bQDO5BEA.js} +1 -1
- package/dashboard/dist/assets/{SelectItemText-3N-hvYuD.js → SelectItemText-CUGrs9CY.js} +1 -1
- package/dashboard/dist/assets/{ServiceStatusPage-C5ZK9X2o.js → ServiceStatusPage-CsERvmVC.js} +1 -1
- package/dashboard/dist/assets/SessionsPage-0VvqEfPx.js +2 -0
- package/dashboard/dist/assets/SettingsPage-9EHh2HfD.js +2 -0
- package/dashboard/dist/assets/{SkillsPage-Cwkevv29.js → SkillsPage-B1AcLX41.js} +1 -1
- package/dashboard/dist/assets/{TokenGalaxy-a1-E6_jp.js → TokenGalaxy-DSKTHQSs.js} +1 -1
- package/dashboard/dist/assets/{ToolbarRootContext-Cm5cQb9Z.js → ToolbarRootContext-DP4mQLbq.js} +1 -1
- package/dashboard/dist/assets/{TrendMonitor-DoNDyarF.js → TrendMonitor-rabtcE5w.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-QMTmKnRl.js → WidgetsPage-Cr6Wg9QX.js} +1 -1
- package/dashboard/dist/assets/{WrappedPage-C9rJcY2o.js → WrappedPage-DqQvRcnA.js} +1 -1
- package/dashboard/dist/assets/__vite-browser-external-BHz2N7Ih.css +1 -0
- package/dashboard/dist/assets/{__vite-browser-external-CsbVaBbw.js → __vite-browser-external-C8KwSyTz.js} +12 -12
- package/dashboard/dist/assets/{agent-logos-B39DKy3X.js → agent-logos-BjIh_mqf.js} +1 -1
- package/dashboard/dist/assets/{arrow-up-right-Cw5mMBik.js → arrow-up-right-BIhbN0NO.js} +1 -1
- package/dashboard/dist/assets/{check-DrQztnq1.js → check-U-wNzPvd.js} +1 -1
- package/dashboard/dist/assets/{chevron-right-CEelysET.js → chevron-right-Du-QVNj4.js} +1 -1
- package/dashboard/dist/assets/{composite-MhgOkYfv.js → composite-BR91CxNl.js} +1 -1
- package/dashboard/dist/assets/{copy-Bouz-qQR.js → copy-Bw3hXBCu.js} +1 -1
- package/dashboard/dist/assets/{download-BugS2cl5.js → download-WOqcQzvK.js} +1 -1
- package/dashboard/dist/assets/{external-link-Ceqto-f4.js → external-link-BZtf6WfS.js} +1 -1
- package/dashboard/dist/assets/{flame-B67y0kLz.js → flame-B5kp2VI9.js} +1 -1
- package/dashboard/dist/assets/{format-tokens-CR8njNJs.js → format-tokens-BOCnuMuJ.js} +1 -1
- package/dashboard/dist/assets/{icons-DFj5ctTx.js → icons-CA3FiIa_.js} +1 -1
- package/dashboard/dist/assets/{index-ZGfRrxuq.js → index-DQaanK52.js} +1 -1
- package/dashboard/dist/assets/{index-CrymIhtC.js → index-DjWC63Bs.js} +1 -1
- package/dashboard/dist/assets/{index-Bx6EP36K.js → index-mEG0HZ-a.js} +1 -1
- package/dashboard/dist/assets/{info-C6XoktLa.js → info-Djer9i3-.js} +1 -1
- package/dashboard/dist/assets/{limits-providers-B9L6hIZT.js → limits-providers-F-RnNFAb.js} +1 -1
- package/dashboard/dist/assets/{link-2-DKG2j43e.js → link-2-DV4ocD0l.js} +1 -1
- package/dashboard/dist/assets/{loader-circle-DB2txq05.js → loader-circle-DJK72c-o.js} +1 -1
- package/dashboard/dist/assets/{maximize-2-3-dz0brp.js → maximize-2-BwBA_lMm.js} +1 -1
- package/dashboard/dist/assets/{modifiers.esm-D0KjKi1o.js → modifiers.esm-B3sl4qyZ.js} +1 -1
- package/dashboard/dist/assets/{monitor-smartphone-Ds8H_QW9.js → monitor-smartphone-BUTpM4ng.js} +1 -1
- package/dashboard/dist/assets/{plus-BDsmAe4Z.js → plus-BA9RP5L6.js} +1 -1
- package/dashboard/dist/assets/{provider-display-DRz643iD.js → provider-display-DqsJzGSr.js} +1 -1
- package/dashboard/dist/assets/{react-D3Afmy9b.js → react-BYw5AVSz.js} +1 -1
- package/dashboard/dist/assets/{search-CO2Xf-Kg.js → search-BzcnFWVD.js} +1 -1
- package/dashboard/dist/assets/{skills-api-C1BZuUIq.js → skills-api-vAME1P7G.js} +1 -1
- package/dashboard/dist/assets/{sortable.esm-DutNEc4F.js → sortable.esm-BpEiRqqI.js} +1 -1
- package/dashboard/dist/assets/{store-a-oUpY-g.js → store-GeT5fhRX.js} +1 -1
- package/dashboard/dist/assets/{terminal-DAePk67T.js → terminal-CZCMYQ8K.js} +1 -1
- package/dashboard/dist/assets/{trash-2-bnFmtRSG.js → trash-2-DzlWEwKO.js} +1 -1
- package/dashboard/dist/assets/{use-community-stats-PJBeXyek.js → use-community-stats-DOvJfj-i.js} +1 -1
- package/dashboard/dist/assets/use-limits-display-prefs-BmLs4HyN.js +1 -0
- package/dashboard/dist/assets/{use-ordered-list-BaVvCzZ9.js → use-ordered-list-BgozDSSV.js} +1 -1
- package/dashboard/dist/assets/use-reduced-motion-Bc3RFJdg.js +1 -0
- package/dashboard/dist/assets/{use-session-efficiency-pref-COTG-zJs.js → use-session-efficiency-pref-HP1ckVkS.js} +1 -1
- package/dashboard/dist/assets/{use-transform-BQEPyChL.js → use-transform-B4VSogOx.js} +1 -1
- package/dashboard/dist/assets/{use-usage-limits-LR2gHJJn.js → use-usage-limits-DNNhD2H6.js} +1 -1
- package/dashboard/dist/assets/{useCurrency-CeqKrzFz.js → useCurrency-CSuzCBAL.js} +1 -1
- package/dashboard/dist/assets/{useTokenFormat-D0IMlZlO.js → useTokenFormat-ClnL9OFe.js} +1 -1
- package/dashboard/dist/assets/{zap-BBn6G5_E.js → zap-C_T9CCiS.js} +1 -1
- package/dashboard/dist/index.html +2 -2
- package/dashboard/dist/ip-check.html +2 -2
- package/dashboard/dist/leaderboard.html +2 -2
- package/dashboard/dist/share.html +2 -2
- package/package.json +1 -1
- package/src/commands/status.js +3 -1
- package/src/commands/sync.js +8 -3
- package/src/lib/codex-rollout-parser.js +35 -4
- package/src/lib/grok-context-breakdown.js +24 -33
- package/src/lib/grok-usage.js +95 -0
- package/src/lib/local-api.js +2 -0
- package/src/lib/pricing/index.js +10 -0
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +817 -239
- package/src/lib/session-analytics.js +305 -50
- package/src/lib/telemetry.js +2 -2
- package/src/lib/usage-limits.js +20 -10
- package/dashboard/dist/assets/DialogDescription-DPyhpdeR.js +0 -1
- package/dashboard/dist/assets/LimitsPage-BXzzX87I.js +0 -5
- package/dashboard/dist/assets/SessionsPage-CJ0JvE8-.js +0 -2
- package/dashboard/dist/assets/SettingsPage-C32uYzqG.js +0 -2
- package/dashboard/dist/assets/__vite-browser-external-D95WhsZe.css +0 -1
- package/dashboard/dist/assets/use-limits-display-prefs-Blvn-7CB.js +0 -1
- package/dashboard/dist/assets/use-reduced-motion-f51jgK_S.js +0 -1
package/src/lib/rollout.js
CHANGED
|
@@ -15,6 +15,7 @@ const {
|
|
|
15
15
|
createUsageDeltaState,
|
|
16
16
|
snapshotUsageBaselines,
|
|
17
17
|
} = require("./codex-token-usage");
|
|
18
|
+
const { USD_TICKS_PER_USD, normalizeGrokUsage } = require("./grok-usage");
|
|
18
19
|
|
|
19
20
|
const DEFAULT_SOURCE = "codex";
|
|
20
21
|
const DEFAULT_MODEL = "unknown";
|
|
@@ -2887,6 +2888,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
2887
2888
|
reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
|
|
2888
2889
|
total_tokens: zeroTotals.total_tokens,
|
|
2889
2890
|
billable_total_tokens: zeroTotals.billable_total_tokens,
|
|
2891
|
+
total_cost_usd: zeroTotals.total_cost_usd,
|
|
2890
2892
|
conversation_count: zeroTotals.conversation_count,
|
|
2891
2893
|
}),
|
|
2892
2894
|
);
|
|
@@ -2905,10 +2907,12 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
2905
2907
|
hour_start: group.hourStart,
|
|
2906
2908
|
input_tokens: zeroTotals.input_tokens,
|
|
2907
2909
|
cached_input_tokens: zeroTotals.cached_input_tokens,
|
|
2910
|
+
cache_creation_input_tokens: zeroTotals.cache_creation_input_tokens,
|
|
2908
2911
|
output_tokens: zeroTotals.output_tokens,
|
|
2909
2912
|
reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
|
|
2910
2913
|
total_tokens: zeroTotals.total_tokens,
|
|
2911
2914
|
billable_total_tokens: zeroTotals.billable_total_tokens,
|
|
2915
|
+
total_cost_usd: zeroTotals.total_cost_usd,
|
|
2912
2916
|
conversation_count: zeroTotals.conversation_count,
|
|
2913
2917
|
}),
|
|
2914
2918
|
);
|
|
@@ -2923,7 +2927,8 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
2923
2927
|
totals = cloneTotals(bucket.totals);
|
|
2924
2928
|
addTotals(totals, unknownBucket.totals);
|
|
2925
2929
|
}
|
|
2926
|
-
const
|
|
2930
|
+
const usagePrecision = bucket.usage_precision || null;
|
|
2931
|
+
const key = usagePrecision ? `${totalsKey(totals)}|${usagePrecision}` : totalsKey(totals);
|
|
2927
2932
|
if (bucket.queuedKey === key) continue;
|
|
2928
2933
|
toAppend.push(
|
|
2929
2934
|
JSON.stringify({
|
|
@@ -2937,6 +2942,8 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
2937
2942
|
reasoning_output_tokens: totals.reasoning_output_tokens,
|
|
2938
2943
|
total_tokens: totals.total_tokens,
|
|
2939
2944
|
billable_total_tokens: totals.billable_total_tokens ?? totals.total_tokens,
|
|
2945
|
+
total_cost_usd: totals.total_cost_usd || 0,
|
|
2946
|
+
usage_precision: usagePrecision || undefined,
|
|
2940
2947
|
conversation_count: totals.conversation_count,
|
|
2941
2948
|
}),
|
|
2942
2949
|
);
|
|
@@ -2960,10 +2967,12 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
2960
2967
|
hour_start: group.hourStart,
|
|
2961
2968
|
input_tokens: zeroTotals.input_tokens,
|
|
2962
2969
|
cached_input_tokens: zeroTotals.cached_input_tokens,
|
|
2970
|
+
cache_creation_input_tokens: zeroTotals.cache_creation_input_tokens,
|
|
2963
2971
|
output_tokens: zeroTotals.output_tokens,
|
|
2964
2972
|
reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
|
|
2965
2973
|
total_tokens: zeroTotals.total_tokens,
|
|
2966
2974
|
billable_total_tokens: zeroTotals.billable_total_tokens,
|
|
2975
|
+
total_cost_usd: zeroTotals.total_cost_usd,
|
|
2967
2976
|
conversation_count: zeroTotals.conversation_count,
|
|
2968
2977
|
}),
|
|
2969
2978
|
);
|
|
@@ -2987,6 +2996,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
2987
2996
|
reasoning_output_tokens: zeroTotals.reasoning_output_tokens,
|
|
2988
2997
|
total_tokens: zeroTotals.total_tokens,
|
|
2989
2998
|
billable_total_tokens: zeroTotals.billable_total_tokens,
|
|
2999
|
+
total_cost_usd: zeroTotals.total_cost_usd,
|
|
2990
3000
|
conversation_count: zeroTotals.conversation_count,
|
|
2991
3001
|
}),
|
|
2992
3002
|
);
|
|
@@ -2994,7 +3004,10 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
2994
3004
|
}
|
|
2995
3005
|
}
|
|
2996
3006
|
if (unknownBucket) unknownBucket.alignedModel = nextAligned;
|
|
2997
|
-
const
|
|
3007
|
+
const usagePrecision = unknownBucket.usage_precision || null;
|
|
3008
|
+
const key = usagePrecision
|
|
3009
|
+
? `${totalsKey(unknownBucket.totals)}|${usagePrecision}`
|
|
3010
|
+
: totalsKey(unknownBucket.totals);
|
|
2998
3011
|
const outputKey = outputModel === DEFAULT_MODEL ? key : `${key}|${outputModel}`;
|
|
2999
3012
|
if (unknownBucket.queuedKey === outputKey) continue;
|
|
3000
3013
|
toAppend.push(
|
|
@@ -3009,6 +3022,8 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
3009
3022
|
reasoning_output_tokens: unknownBucket.totals.reasoning_output_tokens,
|
|
3010
3023
|
total_tokens: unknownBucket.totals.total_tokens,
|
|
3011
3024
|
billable_total_tokens: unknownBucket.totals.billable_total_tokens ?? unknownBucket.totals.total_tokens,
|
|
3025
|
+
total_cost_usd: unknownBucket.totals.total_cost_usd || 0,
|
|
3026
|
+
usage_precision: usagePrecision || undefined,
|
|
3012
3027
|
conversation_count: unknownBucket.totals.conversation_count,
|
|
3013
3028
|
}),
|
|
3014
3029
|
);
|
|
@@ -3056,6 +3071,7 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
|
|
|
3056
3071
|
reasoning_output_tokens: group.totals.reasoning_output_tokens,
|
|
3057
3072
|
total_tokens: group.totals.total_tokens,
|
|
3058
3073
|
billable_total_tokens: group.totals.billable_total_tokens ?? group.totals.total_tokens,
|
|
3074
|
+
total_cost_usd: group.totals.total_cost_usd || 0,
|
|
3059
3075
|
conversation_count: group.totals.conversation_count,
|
|
3060
3076
|
}),
|
|
3061
3077
|
);
|
|
@@ -3111,6 +3127,7 @@ async function enqueueTouchedProjectBuckets({
|
|
|
3111
3127
|
reasoning_output_tokens: totals.reasoning_output_tokens,
|
|
3112
3128
|
total_tokens: totals.total_tokens,
|
|
3113
3129
|
billable_total_tokens: totals.billable_total_tokens ?? totals.total_tokens,
|
|
3130
|
+
total_cost_usd: totals.total_cost_usd || 0,
|
|
3114
3131
|
conversation_count: totals.conversation_count,
|
|
3115
3132
|
}),
|
|
3116
3133
|
);
|
|
@@ -3285,6 +3302,7 @@ function normalizeOpencodeState(raw) {
|
|
|
3285
3302
|
const messages = state.messages && typeof state.messages === "object" ? state.messages : {};
|
|
3286
3303
|
return {
|
|
3287
3304
|
messages,
|
|
3305
|
+
dbCursor: state.dbCursor && typeof state.dbCursor === "object" ? state.dbCursor : null,
|
|
3288
3306
|
updatedAt: typeof state.updatedAt === "string" ? state.updatedAt : null,
|
|
3289
3307
|
};
|
|
3290
3308
|
}
|
|
@@ -3362,13 +3380,20 @@ function deriveOpencodeMessageFingerprint({ msg, totals, source }) {
|
|
|
3362
3380
|
// Rebuilt per parse run. Pre-#426 entries are fingerprinted as they are read;
|
|
3363
3381
|
// the first claims ownership and later cross-session matches are retracted from
|
|
3364
3382
|
// persisted buckets once. Tombstoned copies never claim ownership themselves.
|
|
3365
|
-
function buildOpencodeFingerprintIndex(messageIndex) {
|
|
3383
|
+
function buildOpencodeFingerprintIndex(messageIndex, wantedFingerprints = null) {
|
|
3366
3384
|
const byFingerprint = new Map();
|
|
3367
3385
|
if (!messageIndex || typeof messageIndex !== "object") return byFingerprint;
|
|
3368
|
-
for (const
|
|
3386
|
+
for (const key in messageIndex) {
|
|
3387
|
+
const entry = messageIndex[key];
|
|
3369
3388
|
if (entry?.dedupedForkCopy === true) continue;
|
|
3370
3389
|
const fingerprint = entry && typeof entry.fingerprint === "string" ? entry.fingerprint : null;
|
|
3371
|
-
if (
|
|
3390
|
+
if (
|
|
3391
|
+
fingerprint &&
|
|
3392
|
+
(!wantedFingerprints || wantedFingerprints.has(fingerprint)) &&
|
|
3393
|
+
!byFingerprint.has(fingerprint)
|
|
3394
|
+
) {
|
|
3395
|
+
byFingerprint.set(fingerprint, key);
|
|
3396
|
+
}
|
|
3372
3397
|
}
|
|
3373
3398
|
return byFingerprint;
|
|
3374
3399
|
}
|
|
@@ -3393,6 +3418,52 @@ function isOpencodeForkCopy(fingerprintIndex, fingerprint, messageKey) {
|
|
|
3393
3418
|
return ownerSession !== session;
|
|
3394
3419
|
}
|
|
3395
3420
|
|
|
3421
|
+
function normalizeOpencodeAttribution(raw) {
|
|
3422
|
+
if (typeof raw === "string") {
|
|
3423
|
+
const [bucketStart = "", model = "", projectKey = "", projectRef = ""] = raw.split("\t");
|
|
3424
|
+
if (!bucketStart || !model) return null;
|
|
3425
|
+
return {
|
|
3426
|
+
bucketStart,
|
|
3427
|
+
model,
|
|
3428
|
+
projectKey: projectKey || null,
|
|
3429
|
+
projectRef: projectRef || null,
|
|
3430
|
+
};
|
|
3431
|
+
}
|
|
3432
|
+
if (!raw || typeof raw !== "object") return null;
|
|
3433
|
+
const bucketStart = typeof raw.bucketStart === "string" ? raw.bucketStart : "";
|
|
3434
|
+
const model = typeof raw.model === "string" ? raw.model : "";
|
|
3435
|
+
if (!bucketStart || !model) return null;
|
|
3436
|
+
return {
|
|
3437
|
+
bucketStart,
|
|
3438
|
+
model,
|
|
3439
|
+
projectKey: typeof raw.projectKey === "string" ? raw.projectKey : null,
|
|
3440
|
+
projectRef: typeof raw.projectRef === "string" ? raw.projectRef : null,
|
|
3441
|
+
};
|
|
3442
|
+
}
|
|
3443
|
+
|
|
3444
|
+
function encodeOpencodeAttribution(raw) {
|
|
3445
|
+
const attribution = normalizeOpencodeAttribution(raw);
|
|
3446
|
+
if (!attribution) return null;
|
|
3447
|
+
return [
|
|
3448
|
+
attribution.bucketStart,
|
|
3449
|
+
attribution.model,
|
|
3450
|
+
attribution.projectKey || "",
|
|
3451
|
+
attribution.projectRef || "",
|
|
3452
|
+
].join("\t");
|
|
3453
|
+
}
|
|
3454
|
+
|
|
3455
|
+
function sameOpencodeAttribution(a, b) {
|
|
3456
|
+
const left = normalizeOpencodeAttribution(a);
|
|
3457
|
+
const right = normalizeOpencodeAttribution(b);
|
|
3458
|
+
if (!left || !right) return left === right;
|
|
3459
|
+
return (
|
|
3460
|
+
left.bucketStart === right.bucketStart &&
|
|
3461
|
+
left.model === right.model &&
|
|
3462
|
+
left.projectKey === right.projectKey &&
|
|
3463
|
+
left.projectRef === right.projectRef
|
|
3464
|
+
);
|
|
3465
|
+
}
|
|
3466
|
+
|
|
3396
3467
|
// Persist a message's snapshot + fingerprint and claim the fingerprint for it.
|
|
3397
3468
|
// Writes only when something actually changed so a steady-state sync leaves the
|
|
3398
3469
|
// cursor untouched. A falsy `fingerprint` means "unknown" and preserves whatever
|
|
@@ -3404,6 +3475,7 @@ function recordOpencodeMessage({
|
|
|
3404
3475
|
messageKey,
|
|
3405
3476
|
totals,
|
|
3406
3477
|
fingerprint,
|
|
3478
|
+
attribution,
|
|
3407
3479
|
dedupedForkCopy = false,
|
|
3408
3480
|
}) {
|
|
3409
3481
|
if (!messageIndex || !messageKey) return;
|
|
@@ -3411,10 +3483,13 @@ function recordOpencodeMessage({
|
|
|
3411
3483
|
const prevTotals = prev && typeof prev.lastTotals === "object" ? prev.lastTotals : null;
|
|
3412
3484
|
const prevFingerprint = prev && typeof prev.fingerprint === "string" ? prev.fingerprint : null;
|
|
3413
3485
|
const nextFingerprint = fingerprint || prevFingerprint;
|
|
3486
|
+
const prevAttribution = normalizeOpencodeAttribution(prev?.attribution);
|
|
3487
|
+
const nextAttribution = normalizeOpencodeAttribution(attribution) || prevAttribution;
|
|
3414
3488
|
const prevDeduped = prev?.dedupedForkCopy === true;
|
|
3415
3489
|
if (
|
|
3416
3490
|
sameGeminiTotals(totals, prevTotals) &&
|
|
3417
3491
|
prevFingerprint === nextFingerprint &&
|
|
3492
|
+
sameOpencodeAttribution(prevAttribution, nextAttribution) &&
|
|
3418
3493
|
prevDeduped === Boolean(dedupedForkCopy)
|
|
3419
3494
|
) return;
|
|
3420
3495
|
|
|
@@ -3425,6 +3500,7 @@ function recordOpencodeMessage({
|
|
|
3425
3500
|
}
|
|
3426
3501
|
const entry = { lastTotals: totals, updatedAt: new Date().toISOString() };
|
|
3427
3502
|
if (nextFingerprint) entry.fingerprint = nextFingerprint;
|
|
3503
|
+
if (nextAttribution) entry.attribution = encodeOpencodeAttribution(nextAttribution);
|
|
3428
3504
|
if (dedupedForkCopy) entry.dedupedForkCopy = true;
|
|
3429
3505
|
messageIndex[messageKey] = entry;
|
|
3430
3506
|
if (
|
|
@@ -3437,41 +3513,79 @@ function recordOpencodeMessage({
|
|
|
3437
3513
|
}
|
|
3438
3514
|
}
|
|
3439
3515
|
|
|
3440
|
-
function
|
|
3441
|
-
|
|
3516
|
+
function subtractCountedOpencodeMessage({
|
|
3517
|
+
attribution,
|
|
3442
3518
|
totals,
|
|
3443
3519
|
source,
|
|
3444
3520
|
hourlyState,
|
|
3445
3521
|
touchedBuckets,
|
|
3446
3522
|
projectState,
|
|
3447
3523
|
projectTouchedBuckets,
|
|
3448
|
-
projectRef,
|
|
3449
|
-
projectKey,
|
|
3450
3524
|
}) {
|
|
3451
|
-
const
|
|
3452
|
-
if (!
|
|
3453
|
-
const bucketStart = toUtcHalfHourStart(new Date(timestampMs).toISOString());
|
|
3454
|
-
if (!bucketStart) return false;
|
|
3455
|
-
const { modelId: repairModelId } = normalizeOpencodeModelFields(msg);
|
|
3456
|
-
const model = repairModelId || DEFAULT_MODEL;
|
|
3525
|
+
const countedAt = normalizeOpencodeAttribution(attribution);
|
|
3526
|
+
if (!countedAt) return false;
|
|
3457
3527
|
const counted = { ...totals, conversation_count: 1 };
|
|
3458
|
-
const bucket = getHourlyBucket(
|
|
3528
|
+
const bucket = getHourlyBucket(
|
|
3529
|
+
hourlyState,
|
|
3530
|
+
source,
|
|
3531
|
+
countedAt.model,
|
|
3532
|
+
countedAt.bucketStart,
|
|
3533
|
+
);
|
|
3459
3534
|
subtractTotals(bucket.totals, counted);
|
|
3460
|
-
touchedBuckets.add(bucketKey(source, model, bucketStart));
|
|
3461
|
-
if (projectKey && projectState && projectTouchedBuckets) {
|
|
3535
|
+
touchedBuckets.add(bucketKey(source, countedAt.model, countedAt.bucketStart));
|
|
3536
|
+
if (countedAt.projectKey && projectState && projectTouchedBuckets) {
|
|
3462
3537
|
const projectBucket = getProjectBucket(
|
|
3463
3538
|
projectState,
|
|
3464
|
-
projectKey,
|
|
3539
|
+
countedAt.projectKey,
|
|
3465
3540
|
source,
|
|
3466
|
-
bucketStart,
|
|
3467
|
-
projectRef,
|
|
3541
|
+
countedAt.bucketStart,
|
|
3542
|
+
countedAt.projectRef,
|
|
3468
3543
|
);
|
|
3469
3544
|
subtractTotals(projectBucket.totals, counted);
|
|
3470
|
-
projectTouchedBuckets.add(
|
|
3545
|
+
projectTouchedBuckets.add(
|
|
3546
|
+
projectBucketKey(countedAt.projectKey, source, countedAt.bucketStart),
|
|
3547
|
+
);
|
|
3471
3548
|
}
|
|
3472
3549
|
return true;
|
|
3473
3550
|
}
|
|
3474
3551
|
|
|
3552
|
+
function repairCountedOpencodeForkCopy({
|
|
3553
|
+
msg,
|
|
3554
|
+
attribution,
|
|
3555
|
+
totals,
|
|
3556
|
+
source,
|
|
3557
|
+
hourlyState,
|
|
3558
|
+
touchedBuckets,
|
|
3559
|
+
projectState,
|
|
3560
|
+
projectTouchedBuckets,
|
|
3561
|
+
projectRef,
|
|
3562
|
+
projectKey,
|
|
3563
|
+
}) {
|
|
3564
|
+
let countedAt = normalizeOpencodeAttribution(attribution);
|
|
3565
|
+
if (!countedAt) {
|
|
3566
|
+
const timestampMs = coerceEpochMs(msg?.time?.completed) || coerceEpochMs(msg?.time?.created);
|
|
3567
|
+
if (!timestampMs) return false;
|
|
3568
|
+
const bucketStart = toUtcHalfHourStart(new Date(timestampMs).toISOString());
|
|
3569
|
+
if (!bucketStart) return false;
|
|
3570
|
+
const { modelId: repairModelId } = normalizeOpencodeModelFields(msg);
|
|
3571
|
+
countedAt = {
|
|
3572
|
+
bucketStart,
|
|
3573
|
+
model: repairModelId || DEFAULT_MODEL,
|
|
3574
|
+
projectKey,
|
|
3575
|
+
projectRef,
|
|
3576
|
+
};
|
|
3577
|
+
}
|
|
3578
|
+
return subtractCountedOpencodeMessage({
|
|
3579
|
+
attribution: countedAt,
|
|
3580
|
+
totals,
|
|
3581
|
+
source,
|
|
3582
|
+
hourlyState,
|
|
3583
|
+
touchedBuckets,
|
|
3584
|
+
projectState,
|
|
3585
|
+
projectTouchedBuckets,
|
|
3586
|
+
});
|
|
3587
|
+
}
|
|
3588
|
+
|
|
3475
3589
|
function getHourlyBucket(state, source, model, hourStart) {
|
|
3476
3590
|
const buckets = state.buckets;
|
|
3477
3591
|
const normalizedSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
|
|
@@ -3538,6 +3652,7 @@ function initTotals() {
|
|
|
3538
3652
|
reasoning_output_tokens: 0,
|
|
3539
3653
|
total_tokens: 0,
|
|
3540
3654
|
billable_total_tokens: 0,
|
|
3655
|
+
total_cost_usd: 0,
|
|
3541
3656
|
conversation_count: 0,
|
|
3542
3657
|
};
|
|
3543
3658
|
}
|
|
@@ -3550,6 +3665,9 @@ function addTotals(target, delta) {
|
|
|
3550
3665
|
target.reasoning_output_tokens += delta.reasoning_output_tokens || 0;
|
|
3551
3666
|
target.total_tokens += delta.total_tokens || 0;
|
|
3552
3667
|
target.billable_total_tokens += delta.billable_total_tokens ?? delta.total_tokens ?? 0;
|
|
3668
|
+
target.total_cost_usd = Math.round(
|
|
3669
|
+
((target.total_cost_usd || 0) + (delta.total_cost_usd || 0)) * USD_TICKS_PER_USD,
|
|
3670
|
+
) / USD_TICKS_PER_USD;
|
|
3553
3671
|
target.conversation_count += delta.conversation_count || 0;
|
|
3554
3672
|
}
|
|
3555
3673
|
|
|
@@ -3574,6 +3692,12 @@ function subtractTotals(target, totals) {
|
|
|
3574
3692
|
target.billable_total_tokens -
|
|
3575
3693
|
(totals.billable_total_tokens ?? totals.total_tokens ?? 0),
|
|
3576
3694
|
);
|
|
3695
|
+
target.total_cost_usd = Math.max(
|
|
3696
|
+
0,
|
|
3697
|
+
Math.round(
|
|
3698
|
+
((target.total_cost_usd || 0) - (totals.total_cost_usd || 0)) * USD_TICKS_PER_USD,
|
|
3699
|
+
) / USD_TICKS_PER_USD,
|
|
3700
|
+
);
|
|
3577
3701
|
target.conversation_count = Math.max(
|
|
3578
3702
|
0,
|
|
3579
3703
|
target.conversation_count - (totals.conversation_count || 0),
|
|
@@ -3589,6 +3713,7 @@ function totalsKey(totals) {
|
|
|
3589
3713
|
totals.reasoning_output_tokens || 0,
|
|
3590
3714
|
totals.total_tokens || 0,
|
|
3591
3715
|
totals.billable_total_tokens ?? totals.total_tokens ?? 0,
|
|
3716
|
+
totals.total_cost_usd || 0,
|
|
3592
3717
|
totals.conversation_count || 0,
|
|
3593
3718
|
].join("|");
|
|
3594
3719
|
}
|
|
@@ -4185,10 +4310,6 @@ function diffGeminiTotals(current, previous) {
|
|
|
4185
4310
|
const totalReset = (current.total_tokens || 0) < (previous.total_tokens || 0);
|
|
4186
4311
|
if (totalReset) return current;
|
|
4187
4312
|
|
|
4188
|
-
// Must include cache_creation_input_tokens in both the equality check and
|
|
4189
|
-
// the delta — OpenCode routes through this diff and its cache.write number
|
|
4190
|
-
// would otherwise be permanently reported as zero. Gemini itself always
|
|
4191
|
-
// emits cache_creation=0 so the extra field is a no-op for Gemini.
|
|
4192
4313
|
const delta = {
|
|
4193
4314
|
input_tokens: Math.max(0, (current.input_tokens || 0) - (previous.input_tokens || 0)),
|
|
4194
4315
|
cached_input_tokens: Math.max(
|
|
@@ -4210,6 +4331,29 @@ function diffGeminiTotals(current, previous) {
|
|
|
4210
4331
|
return isAllZeroUsage(delta) ? null : delta;
|
|
4211
4332
|
}
|
|
4212
4333
|
|
|
4334
|
+
// OpenCode rows are authoritative snapshots of one message and can be
|
|
4335
|
+
// corrected downward or move tokens between cache/output columns. Signed
|
|
4336
|
+
// deltas replace the prior contribution instead of treating a correction as a
|
|
4337
|
+
// fresh cumulative reset. Gemini keeps its provider-specific reset behavior.
|
|
4338
|
+
function diffOpencodeTotals(current, previous) {
|
|
4339
|
+
if (!current || typeof current !== "object") return null;
|
|
4340
|
+
if (!previous || typeof previous !== "object") return current;
|
|
4341
|
+
if (sameGeminiTotals(current, previous)) return null;
|
|
4342
|
+
|
|
4343
|
+
const delta = {};
|
|
4344
|
+
for (const key of [
|
|
4345
|
+
"input_tokens",
|
|
4346
|
+
"cached_input_tokens",
|
|
4347
|
+
"cache_creation_input_tokens",
|
|
4348
|
+
"output_tokens",
|
|
4349
|
+
"reasoning_output_tokens",
|
|
4350
|
+
"total_tokens",
|
|
4351
|
+
]) {
|
|
4352
|
+
delta[key] = Number(current[key] || 0) - Number(previous[key] || 0);
|
|
4353
|
+
}
|
|
4354
|
+
return isAllZeroUsage(delta) ? null : delta;
|
|
4355
|
+
}
|
|
4356
|
+
|
|
4213
4357
|
function extractTokenCount(obj) {
|
|
4214
4358
|
const payload = obj?.payload;
|
|
4215
4359
|
if (!payload) return null;
|
|
@@ -4391,15 +4535,73 @@ async function walkOpencodeMessages(dir, out) {
|
|
|
4391
4535
|
// both "is there v2 data?" and "which session table exists?" in one round-trip.
|
|
4392
4536
|
// ---------------------------------------------------------------------------
|
|
4393
4537
|
|
|
4394
|
-
const
|
|
4395
|
-
|
|
4396
|
-
|
|
4538
|
+
const OPENCODE_DB_CURSOR_VERSION = 1;
|
|
4539
|
+
|
|
4540
|
+
function normalizeOpencodeDbWatermark(raw) {
|
|
4541
|
+
if (!raw || typeof raw !== "object") return null;
|
|
4542
|
+
const maxRowId = Math.max(0, Math.floor(Number(raw.maxRowId) || 0));
|
|
4543
|
+
const maxUpdatedAt = Math.max(0, Math.floor(Number(raw.maxUpdatedAt) || 0));
|
|
4544
|
+
const anchor = typeof raw.anchor === "string" && raw.anchor ? raw.anchor : null;
|
|
4545
|
+
return maxRowId || maxUpdatedAt ? { maxRowId, maxUpdatedAt, anchor } : null;
|
|
4546
|
+
}
|
|
4547
|
+
|
|
4548
|
+
function opencodeDbIdentity(dbPath) {
|
|
4549
|
+
try {
|
|
4550
|
+
const stat = fssync.statSync(dbPath);
|
|
4551
|
+
const resolved = path.resolve(dbPath);
|
|
4552
|
+
return {
|
|
4553
|
+
pathHash: crypto.createHash("sha256").update(resolved).digest("hex"),
|
|
4554
|
+
dev: String(stat.dev || 0),
|
|
4555
|
+
ino: String(stat.ino || 0),
|
|
4556
|
+
};
|
|
4557
|
+
} catch (_e) {
|
|
4558
|
+
return null;
|
|
4559
|
+
}
|
|
4560
|
+
}
|
|
4561
|
+
|
|
4562
|
+
function sameOpencodeDbIdentity(a, b) {
|
|
4563
|
+
return Boolean(
|
|
4564
|
+
a &&
|
|
4565
|
+
b &&
|
|
4566
|
+
a.pathHash === b.pathHash &&
|
|
4567
|
+
a.dev === b.dev &&
|
|
4568
|
+
a.ino === b.ino,
|
|
4569
|
+
);
|
|
4570
|
+
}
|
|
4571
|
+
|
|
4572
|
+
function opencodeDbRowAnchor(row) {
|
|
4573
|
+
if (!row || typeof row !== "object") return null;
|
|
4574
|
+
const rowId = Math.max(0, Math.floor(Number(row.row_id) || 0));
|
|
4575
|
+
const id = typeof row.id === "string" ? row.id : "";
|
|
4576
|
+
const created = Math.max(0, Math.floor(Number(row.time_created) || 0));
|
|
4577
|
+
if (!rowId || !id) return null;
|
|
4578
|
+
return crypto.createHash("sha256").update(`${rowId}\0${id}\0${created}`).digest("base64url");
|
|
4579
|
+
}
|
|
4580
|
+
|
|
4581
|
+
function opencodeDbIncrementalPredicate(alias, watermark) {
|
|
4582
|
+
const cursor = normalizeOpencodeDbWatermark(watermark);
|
|
4583
|
+
if (!cursor) return "";
|
|
4584
|
+
const clauses = [];
|
|
4585
|
+
if (cursor.maxRowId) clauses.push(`${alias}rowid > ${cursor.maxRowId}`);
|
|
4586
|
+
// Re-read the boundary timestamp so simultaneous updates cannot fall through
|
|
4587
|
+
// a strict greater-than watermark. The duplicate is removed by messageIndex.
|
|
4588
|
+
if (cursor.maxUpdatedAt) clauses.push(`${alias}time_updated >= ${cursor.maxUpdatedAt}`);
|
|
4589
|
+
return clauses.length > 0 ? ` AND (${clauses.join(" OR ")})` : "";
|
|
4590
|
+
}
|
|
4591
|
+
|
|
4592
|
+
function buildV1Sql(watermark = null) {
|
|
4593
|
+
return (
|
|
4594
|
+
`SELECT rowid AS row_id, id, session_id, time_updated, data FROM message ` +
|
|
4595
|
+
`WHERE json_extract(data, '$.role') = 'assistant'` +
|
|
4596
|
+
`${opencodeDbIncrementalPredicate("", watermark)} ORDER BY time_created ASC`
|
|
4597
|
+
);
|
|
4598
|
+
}
|
|
4397
4599
|
|
|
4398
4600
|
// Build the v2 query. When a session table exists the LEFT JOIN restores the
|
|
4399
4601
|
// project directory for downstream attribution; when it does not (type B
|
|
4400
4602
|
// without session_v2, or an exotic fork) the join and directory column are
|
|
4401
4603
|
// omitted and tokens are counted as-is.
|
|
4402
|
-
function buildV2Sql(sessionTable) {
|
|
4604
|
+
function buildV2Sql(sessionTable, watermark = null) {
|
|
4403
4605
|
const joinClause = sessionTable
|
|
4404
4606
|
? `LEFT JOIN ${sessionTable} s ON s.id = sm.session_id `
|
|
4405
4607
|
: "";
|
|
@@ -4407,10 +4609,11 @@ function buildV2Sql(sessionTable) {
|
|
|
4407
4609
|
? `s.directory AS directory, `
|
|
4408
4610
|
: "";
|
|
4409
4611
|
return (
|
|
4410
|
-
`SELECT sm.id AS id, sm.session_id AS session_id, sm.time_updated AS time_updated, ` +
|
|
4612
|
+
`SELECT sm.rowid AS row_id, sm.id AS id, sm.session_id AS session_id, sm.time_updated AS time_updated, ` +
|
|
4411
4613
|
`${directorySelect}sm.data AS data ` +
|
|
4412
4614
|
`FROM session_message sm ${joinClause}` +
|
|
4413
|
-
`WHERE sm.type = 'assistant'
|
|
4615
|
+
`WHERE sm.type = 'assistant'` +
|
|
4616
|
+
`${opencodeDbIncrementalPredicate("sm.", watermark)} ORDER BY sm.time_created ASC`
|
|
4414
4617
|
);
|
|
4415
4618
|
}
|
|
4416
4619
|
|
|
@@ -4447,8 +4650,13 @@ function opencodeMessageProvider(data) {
|
|
|
4447
4650
|
return data?.providerID || data?.model?.providerID || "";
|
|
4448
4651
|
}
|
|
4449
4652
|
|
|
4450
|
-
function
|
|
4451
|
-
if (!dbPath || !fssync.existsSync(dbPath)) return [];
|
|
4653
|
+
function readOpencodeDbMessagesIncremental(dbPath, previousCursor = null, sqliteOptions = {}) {
|
|
4654
|
+
if (!dbPath || !fssync.existsSync(dbPath)) return { messages: [], cursor: null };
|
|
4655
|
+
|
|
4656
|
+
const identity = opencodeDbIdentity(dbPath);
|
|
4657
|
+
const canResume =
|
|
4658
|
+
previousCursor?.version === OPENCODE_DB_CURSOR_VERSION &&
|
|
4659
|
+
sameOpencodeDbIdentity(previousCursor.identity, identity);
|
|
4452
4660
|
|
|
4453
4661
|
let snapshot = null;
|
|
4454
4662
|
let effectiveDbPath = dbPath;
|
|
@@ -4483,7 +4691,9 @@ function readOpencodeDbMessages(dbPath, sqliteOptions = {}) {
|
|
|
4483
4691
|
const hasTokens =
|
|
4484
4692
|
toNonNegativeInt(tokens.input) > 0 ||
|
|
4485
4693
|
toNonNegativeInt(tokens.output) > 0 ||
|
|
4486
|
-
toNonNegativeInt(tokens.reasoning) > 0
|
|
4694
|
+
toNonNegativeInt(tokens.reasoning) > 0 ||
|
|
4695
|
+
toNonNegativeInt(tokens.cache?.read) > 0 ||
|
|
4696
|
+
toNonNegativeInt(tokens.cache?.write) > 0;
|
|
4487
4697
|
if (!hasTokens) continue;
|
|
4488
4698
|
if (isV2 && typeof row.directory === "string" && row.directory.trim()) {
|
|
4489
4699
|
data.path = { ...(typeof data.path === "object" && data.path ? data.path : {}), cwd: row.directory };
|
|
@@ -4512,6 +4722,74 @@ function readOpencodeDbMessages(dbPath, sqliteOptions = {}) {
|
|
|
4512
4722
|
}
|
|
4513
4723
|
};
|
|
4514
4724
|
|
|
4725
|
+
const readMaxRowId = (table) => {
|
|
4726
|
+
const rows = readGeneration(`SELECT MAX(rowid) AS max_row_id FROM ${table}`);
|
|
4727
|
+
if (!rows) return null;
|
|
4728
|
+
return Math.max(0, Math.floor(Number(rows[0]?.max_row_id) || 0));
|
|
4729
|
+
};
|
|
4730
|
+
|
|
4731
|
+
const readRowAnchor = (table, rowId) => {
|
|
4732
|
+
if (!rowId) return null;
|
|
4733
|
+
const rows = readGeneration(
|
|
4734
|
+
`SELECT rowid AS row_id, id, time_created FROM ${table} WHERE rowid = ${rowId}`,
|
|
4735
|
+
);
|
|
4736
|
+
return rows ? opencodeDbRowAnchor(rows[0]) : null;
|
|
4737
|
+
};
|
|
4738
|
+
|
|
4739
|
+
const readCursorGeneration = ({ table, isV2, sql }) => {
|
|
4740
|
+
const previous = canResume
|
|
4741
|
+
? normalizeOpencodeDbWatermark(previousCursor?.[isV2 ? "v2" : "v1"])
|
|
4742
|
+
: null;
|
|
4743
|
+
let forceReplay = false;
|
|
4744
|
+
let lastRows = [];
|
|
4745
|
+
|
|
4746
|
+
// Each query uses a separate SQLite reader. Verify the same immutable head
|
|
4747
|
+
// row before and after the data query so an in-place database replacement
|
|
4748
|
+
// cannot make us persist a cursor from a generation we did not read.
|
|
4749
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
4750
|
+
const currentMaxRowId = readMaxRowId(table);
|
|
4751
|
+
if (currentMaxRowId === null) return null;
|
|
4752
|
+
const currentAnchor = readRowAnchor(table, currentMaxRowId);
|
|
4753
|
+
const previousAnchor = previous
|
|
4754
|
+
? readRowAnchor(table, previous.maxRowId)
|
|
4755
|
+
: null;
|
|
4756
|
+
const watermark =
|
|
4757
|
+
!forceReplay &&
|
|
4758
|
+
previous?.anchor &&
|
|
4759
|
+
currentMaxRowId >= previous.maxRowId &&
|
|
4760
|
+
previousAnchor === previous.anchor
|
|
4761
|
+
? previous
|
|
4762
|
+
: null;
|
|
4763
|
+
const rows = readGeneration(sql(watermark));
|
|
4764
|
+
if (!rows) return null;
|
|
4765
|
+
lastRows = rows;
|
|
4766
|
+
|
|
4767
|
+
if (readRowAnchor(table, currentMaxRowId) !== currentAnchor) {
|
|
4768
|
+
forceReplay = true;
|
|
4769
|
+
continue;
|
|
4770
|
+
}
|
|
4771
|
+
|
|
4772
|
+
let maxUpdatedAt = watermark?.maxUpdatedAt || 0;
|
|
4773
|
+
for (const row of rows) {
|
|
4774
|
+
maxUpdatedAt = Math.max(maxUpdatedAt, Math.floor(Number(row?.time_updated) || 0));
|
|
4775
|
+
}
|
|
4776
|
+
return {
|
|
4777
|
+
rows,
|
|
4778
|
+
// Advance only to the head whose anchor bracketed this query. Rows
|
|
4779
|
+
// appended concurrently may be returned now and harmlessly reread on
|
|
4780
|
+
// the next sync; advancing to them would require an unverified anchor.
|
|
4781
|
+
cursor: {
|
|
4782
|
+
maxRowId: currentMaxRowId,
|
|
4783
|
+
maxUpdatedAt,
|
|
4784
|
+
anchor: currentAnchor,
|
|
4785
|
+
},
|
|
4786
|
+
};
|
|
4787
|
+
}
|
|
4788
|
+
|
|
4789
|
+
// Keep parsed usage but refuse to advance while the database is unstable.
|
|
4790
|
+
return { rows: lastRows, cursor: null };
|
|
4791
|
+
};
|
|
4792
|
+
|
|
4515
4793
|
try {
|
|
4516
4794
|
// Combined probe drives the orchestration: v1 always runs (it is the
|
|
4517
4795
|
// baseline every database carries or degrades to), v2 runs only when the
|
|
@@ -4519,21 +4797,48 @@ function readOpencodeDbMessages(dbPath, sqliteOptions = {}) {
|
|
|
4519
4797
|
// against a non-existent session table on type-A databases.
|
|
4520
4798
|
const probe = detectOpencodeMessageLayout(effectiveDbPath, sqliteOptions);
|
|
4521
4799
|
const out = [];
|
|
4800
|
+
const nextCursor = {
|
|
4801
|
+
version: OPENCODE_DB_CURSOR_VERSION,
|
|
4802
|
+
identity,
|
|
4803
|
+
v1: null,
|
|
4804
|
+
v2: null,
|
|
4805
|
+
};
|
|
4522
4806
|
|
|
4523
|
-
const
|
|
4524
|
-
|
|
4807
|
+
const v1 = readCursorGeneration({
|
|
4808
|
+
table: "message",
|
|
4809
|
+
isV2: false,
|
|
4810
|
+
sql: buildV1Sql,
|
|
4811
|
+
});
|
|
4812
|
+
if (v1) {
|
|
4813
|
+
appendRows(v1.rows, false, out);
|
|
4814
|
+
nextCursor.v1 = v1.cursor;
|
|
4815
|
+
}
|
|
4525
4816
|
|
|
4526
4817
|
if (probe?.hasRows) {
|
|
4527
|
-
const
|
|
4528
|
-
|
|
4818
|
+
const v2 = readCursorGeneration({
|
|
4819
|
+
table: "session_message",
|
|
4820
|
+
isV2: true,
|
|
4821
|
+
sql: (watermark) => buildV2Sql(probe.sessionTable, watermark),
|
|
4822
|
+
});
|
|
4823
|
+
if (v2) {
|
|
4824
|
+
appendRows(v2.rows, true, out);
|
|
4825
|
+
nextCursor.v2 = v2.cursor;
|
|
4826
|
+
}
|
|
4529
4827
|
}
|
|
4530
4828
|
|
|
4531
|
-
return
|
|
4829
|
+
return {
|
|
4830
|
+
messages: out,
|
|
4831
|
+
cursor: nextCursor.v1 || nextCursor.v2 ? nextCursor : null,
|
|
4832
|
+
};
|
|
4532
4833
|
} finally {
|
|
4533
4834
|
if (snapshot) snapshot.cleanup();
|
|
4534
4835
|
}
|
|
4535
4836
|
}
|
|
4536
4837
|
|
|
4838
|
+
function readOpencodeDbMessages(dbPath, sqliteOptions = {}) {
|
|
4839
|
+
return readOpencodeDbMessagesIncremental(dbPath, null, sqliteOptions).messages;
|
|
4840
|
+
}
|
|
4841
|
+
|
|
4537
4842
|
// mimocode mirrors the user's Claude Code + claude-mem history into its own
|
|
4538
4843
|
// `message` table — via an explicit `claude_import` AND a live observer /
|
|
4539
4844
|
// session sync — so the overwhelming majority of rows are anthropic-endpoint
|
|
@@ -4791,6 +5096,7 @@ function readZcodeDbMessages(dbPath, sqliteOptions = {}) {
|
|
|
4791
5096
|
async function parseOpencodeDbIncremental({
|
|
4792
5097
|
dbMessages,
|
|
4793
5098
|
dbPath,
|
|
5099
|
+
dbCursor,
|
|
4794
5100
|
cursors,
|
|
4795
5101
|
queuePath,
|
|
4796
5102
|
projectQueuePath,
|
|
@@ -4815,9 +5121,20 @@ async function parseOpencodeDbIncremental({
|
|
|
4815
5121
|
const cursorNamespace = typeof cursorKey === "string" && cursorKey.length > 0 ? cursorKey : "opencode";
|
|
4816
5122
|
const opencodeState = normalizeOpencodeState(cursors?.[cursorNamespace]);
|
|
4817
5123
|
const messageIndex = opencodeState.messages;
|
|
4818
|
-
const fingerprintIndex = buildOpencodeFingerprintIndex(messageIndex);
|
|
4819
5124
|
const touchedBuckets = new Set();
|
|
4820
5125
|
const defaultSource = normalizeSourceInput(source) || "opencode";
|
|
5126
|
+
const candidateFingerprints = new Set();
|
|
5127
|
+
for (const entry of messages) {
|
|
5128
|
+
const totals = normalizeOpencodeTokens(entry?.data?.tokens);
|
|
5129
|
+
const fingerprint = totals
|
|
5130
|
+
? deriveOpencodeMessageFingerprint({ msg: entry.data, totals, source: defaultSource })
|
|
5131
|
+
: null;
|
|
5132
|
+
if (fingerprint) candidateFingerprints.add(fingerprint);
|
|
5133
|
+
}
|
|
5134
|
+
// A hook-triggered sync normally carries one changed row. Restrict the
|
|
5135
|
+
// historical fingerprint map to fingerprints that can actually match this
|
|
5136
|
+
// batch instead of duplicating the whole long-lived message index in memory.
|
|
5137
|
+
const fingerprintIndex = buildOpencodeFingerprintIndex(messageIndex, candidateFingerprints);
|
|
4821
5138
|
|
|
4822
5139
|
for (let idx = 0; idx < messages.length; idx++) {
|
|
4823
5140
|
const entry = messages[idx];
|
|
@@ -4867,6 +5184,7 @@ async function parseOpencodeDbIncremental({
|
|
|
4867
5184
|
if (lastTotals && prev?.dedupedForkCopy !== true) {
|
|
4868
5185
|
repairCountedOpencodeForkCopy({
|
|
4869
5186
|
msg,
|
|
5187
|
+
attribution: prev?.attribution,
|
|
4870
5188
|
totals: lastTotals,
|
|
4871
5189
|
source: defaultSource,
|
|
4872
5190
|
hourlyState,
|
|
@@ -4890,17 +5208,58 @@ async function parseOpencodeDbIncremental({
|
|
|
4890
5208
|
}
|
|
4891
5209
|
|
|
4892
5210
|
const effectiveLastTotals = prev?.dedupedForkCopy === true ? null : lastTotals;
|
|
4893
|
-
const delta =
|
|
4894
|
-
|
|
5211
|
+
const delta = diffOpencodeTotals(currentTotals, effectiveLastTotals);
|
|
5212
|
+
const timestampMs = coerceEpochMs(msg?.time?.completed) || coerceEpochMs(msg?.time?.created);
|
|
5213
|
+
if (!timestampMs) {
|
|
5214
|
+
messagesProcessed += 1;
|
|
5215
|
+
continue;
|
|
5216
|
+
}
|
|
5217
|
+
|
|
5218
|
+
const tsIso = new Date(timestampMs).toISOString();
|
|
5219
|
+
const bucketStart = toUtcHalfHourStart(tsIso);
|
|
5220
|
+
if (!bucketStart) {
|
|
5221
|
+
messagesProcessed += 1;
|
|
5222
|
+
continue;
|
|
5223
|
+
}
|
|
5224
|
+
|
|
5225
|
+
const { modelId: dbModelId } = normalizeOpencodeModelFields(msg);
|
|
5226
|
+
const model = dbModelId || DEFAULT_MODEL;
|
|
5227
|
+
const projectContext = projectEnabled
|
|
5228
|
+
? await resolveProjectContextForDb({
|
|
5229
|
+
msg,
|
|
5230
|
+
dbPath,
|
|
5231
|
+
projectMetaCache,
|
|
5232
|
+
publicRepoCache,
|
|
5233
|
+
publicRepoResolver,
|
|
5234
|
+
projectState,
|
|
5235
|
+
})
|
|
5236
|
+
: null;
|
|
5237
|
+
const attribution = {
|
|
5238
|
+
bucketStart,
|
|
5239
|
+
model,
|
|
5240
|
+
projectKey: projectContext?.projectKey || null,
|
|
5241
|
+
projectRef: projectContext?.projectRef || null,
|
|
5242
|
+
};
|
|
5243
|
+
|
|
5244
|
+
const previousAttribution = effectiveLastTotals
|
|
5245
|
+
? normalizeOpencodeAttribution(prev?.attribution)
|
|
5246
|
+
: null;
|
|
5247
|
+
const moved = Boolean(
|
|
5248
|
+
effectiveLastTotals &&
|
|
5249
|
+
previousAttribution &&
|
|
5250
|
+
!sameOpencodeAttribution(previousAttribution, attribution),
|
|
5251
|
+
);
|
|
5252
|
+
if ((!delta || isAllZeroUsage(delta)) && !moved) {
|
|
4895
5253
|
// Refresh the index even without a delta: normalization may have changed,
|
|
4896
|
-
// and pre-#426 entries need their fingerprint backfilled
|
|
4897
|
-
//
|
|
5254
|
+
// and pre-#426 entries need their fingerprint backfilled. Preserve an
|
|
5255
|
+
// existing contribution location without bloating every legacy cursor.
|
|
4898
5256
|
recordOpencodeMessage({
|
|
4899
5257
|
messageIndex,
|
|
4900
5258
|
fingerprintIndex,
|
|
4901
5259
|
messageKey,
|
|
4902
5260
|
totals: currentTotals,
|
|
4903
5261
|
fingerprint,
|
|
5262
|
+
attribution: prev?.attribution ? attribution : null,
|
|
4904
5263
|
dedupedForkCopy: false,
|
|
4905
5264
|
});
|
|
4906
5265
|
messagesProcessed += 1;
|
|
@@ -4915,49 +5274,37 @@ async function parseOpencodeDbIncremental({
|
|
|
4915
5274
|
}
|
|
4916
5275
|
continue;
|
|
4917
5276
|
}
|
|
4918
|
-
delta.conversation_count = 1;
|
|
4919
5277
|
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
continue;
|
|
5278
|
+
if (moved) {
|
|
5279
|
+
subtractCountedOpencodeMessage({
|
|
5280
|
+
attribution: previousAttribution,
|
|
5281
|
+
totals: effectiveLastTotals,
|
|
5282
|
+
source: defaultSource,
|
|
5283
|
+
hourlyState,
|
|
5284
|
+
touchedBuckets,
|
|
5285
|
+
projectState,
|
|
5286
|
+
projectTouchedBuckets,
|
|
5287
|
+
});
|
|
4931
5288
|
}
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
5289
|
+
const contribution = moved
|
|
5290
|
+
? { ...currentTotals, conversation_count: 1 }
|
|
5291
|
+
: { ...delta, conversation_count: effectiveLastTotals ? 0 : 1 };
|
|
4935
5292
|
const bucket = getHourlyBucket(hourlyState, defaultSource, model, bucketStart);
|
|
4936
|
-
addTotals(bucket.totals,
|
|
5293
|
+
addTotals(bucket.totals, contribution);
|
|
4937
5294
|
touchedBuckets.add(bucketKey(defaultSource, model, bucketStart));
|
|
4938
5295
|
|
|
4939
|
-
if (
|
|
4940
|
-
const
|
|
4941
|
-
msg,
|
|
4942
|
-
dbPath,
|
|
4943
|
-
projectMetaCache,
|
|
4944
|
-
publicRepoCache,
|
|
4945
|
-
publicRepoResolver,
|
|
5296
|
+
if (attribution.projectKey && projectState && projectTouchedBuckets) {
|
|
5297
|
+
const projectBucket = getProjectBucket(
|
|
4946
5298
|
projectState,
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
projectRef,
|
|
4957
|
-
);
|
|
4958
|
-
addTotals(projectBucket.totals, delta);
|
|
4959
|
-
projectTouchedBuckets.add(projectBucketKey(projectKey, defaultSource, bucketStart));
|
|
4960
|
-
}
|
|
5299
|
+
attribution.projectKey,
|
|
5300
|
+
defaultSource,
|
|
5301
|
+
bucketStart,
|
|
5302
|
+
attribution.projectRef,
|
|
5303
|
+
);
|
|
5304
|
+
addTotals(projectBucket.totals, contribution);
|
|
5305
|
+
projectTouchedBuckets.add(
|
|
5306
|
+
projectBucketKey(attribution.projectKey, defaultSource, bucketStart),
|
|
5307
|
+
);
|
|
4961
5308
|
}
|
|
4962
5309
|
|
|
4963
5310
|
recordOpencodeMessage({
|
|
@@ -4966,10 +5313,11 @@ async function parseOpencodeDbIncremental({
|
|
|
4966
5313
|
messageKey,
|
|
4967
5314
|
totals: currentTotals,
|
|
4968
5315
|
fingerprint,
|
|
5316
|
+
attribution,
|
|
4969
5317
|
dedupedForkCopy: false,
|
|
4970
5318
|
});
|
|
4971
5319
|
messagesProcessed += 1;
|
|
4972
|
-
eventsAggregated += 1;
|
|
5320
|
+
if (delta && !isAllZeroUsage(delta)) eventsAggregated += 1;
|
|
4973
5321
|
|
|
4974
5322
|
if (cb) {
|
|
4975
5323
|
cb({
|
|
@@ -4988,6 +5336,7 @@ async function parseOpencodeDbIncremental({
|
|
|
4988
5336
|
: 0;
|
|
4989
5337
|
hourlyState.updatedAt = new Date().toISOString();
|
|
4990
5338
|
cursors.hourly = hourlyState;
|
|
5339
|
+
if (dbCursor && typeof dbCursor === "object") opencodeState.dbCursor = dbCursor;
|
|
4991
5340
|
opencodeState.updatedAt = new Date().toISOString();
|
|
4992
5341
|
cursors[cursorNamespace] = opencodeState;
|
|
4993
5342
|
if (projectState) {
|
|
@@ -13542,7 +13891,7 @@ async function parseCraftIncremental({
|
|
|
13542
13891
|
// COPILOT_OTEL_ENABLED=true
|
|
13543
13892
|
// COPILOT_OTEL_EXPORTER_TYPE=file
|
|
13544
13893
|
// COPILOT_OTEL_FILE_EXPORTER_PATH=$HOME/.copilot/otel/copilot-otel-...jsonl
|
|
13545
|
-
// We scan
|
|
13894
|
+
// We scan both known default directories plus the env-overridden path.
|
|
13546
13895
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
13547
13896
|
|
|
13548
13897
|
function resolveCopilotOtelPaths(env = process.env) {
|
|
@@ -13557,12 +13906,19 @@ function resolveCopilotOtelPaths(env = process.env) {
|
|
|
13557
13906
|
} catch (_e) {}
|
|
13558
13907
|
};
|
|
13559
13908
|
if (process.platform !== "win32" || wsl.shouldProbeNative(env)) {
|
|
13560
|
-
|
|
13909
|
+
for (const dir of [
|
|
13910
|
+
path.join(home, ".copilot", "otel"),
|
|
13911
|
+
path.join(home, ".copilot-otel"),
|
|
13912
|
+
]) {
|
|
13913
|
+
scanDir(dir);
|
|
13914
|
+
}
|
|
13561
13915
|
}
|
|
13562
13916
|
if (process.platform === "win32") {
|
|
13563
13917
|
if (wsl.shouldProbeWsl(env)) {
|
|
13564
|
-
const
|
|
13565
|
-
|
|
13918
|
+
for (const providerDir of [".copilot/otel", ".copilot-otel"]) {
|
|
13919
|
+
const wslDir = wsl.discoverWslHome(providerDir, { env });
|
|
13920
|
+
if (wslDir) scanDir(wslDir);
|
|
13921
|
+
}
|
|
13566
13922
|
}
|
|
13567
13923
|
}
|
|
13568
13924
|
const explicit = env.COPILOT_OTEL_FILE_EXPORTER_PATH;
|
|
@@ -13604,7 +13960,7 @@ function pickCopilotModel(attrs) {
|
|
|
13604
13960
|
return null;
|
|
13605
13961
|
}
|
|
13606
13962
|
|
|
13607
|
-
const COPILOT_PARSER_VERSION =
|
|
13963
|
+
const COPILOT_PARSER_VERSION = 3;
|
|
13608
13964
|
const COPILOT_USAGE_CLAIM_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
13609
13965
|
const COPILOT_USAGE_CLAIM_MAX_ENTRIES = 10_000;
|
|
13610
13966
|
|
|
@@ -13661,14 +14017,168 @@ function incrementMapCount(map, key, amount = 1) {
|
|
|
13661
14017
|
map.set(key, (map.get(key) || 0) + amount);
|
|
13662
14018
|
}
|
|
13663
14019
|
|
|
13664
|
-
function
|
|
14020
|
+
function getCopilotResponseId(attrs = {}) {
|
|
14021
|
+
const responseId = attrs["gen_ai.response.id"];
|
|
14022
|
+
return typeof responseId === "string" && responseId.trim() ? responseId.trim() : "";
|
|
14023
|
+
}
|
|
14024
|
+
|
|
14025
|
+
// v2 preferred spanContext for every OTEL envelope. Chat-extension LogRecords
|
|
14026
|
+
// can share that nested context across several model requests, so keep the old
|
|
14027
|
+
// key only for the one envelope that owns top-level traceId/spanId: CLI spans.
|
|
14028
|
+
function getCopilotLegacyDedupKey(record, attrs = record?.attributes || {}) {
|
|
13665
14029
|
const traceId = record?.traceId || record?.spanContext?.traceId || "";
|
|
13666
14030
|
const spanId = record?.spanId || record?.spanContext?.spanId || "";
|
|
13667
|
-
const responseId =
|
|
13668
|
-
typeof attrs["gen_ai.response.id"] === "string" ? attrs["gen_ai.response.id"] : "";
|
|
14031
|
+
const responseId = getCopilotResponseId(attrs);
|
|
13669
14032
|
return traceId && spanId ? `${traceId}:${spanId}` : responseId ? `resp:${responseId}` : null;
|
|
13670
14033
|
}
|
|
13671
14034
|
|
|
14035
|
+
function getCopilotDedupKey(record, attrs = record?.attributes || {}) {
|
|
14036
|
+
const responseId = getCopilotResponseId(attrs);
|
|
14037
|
+
if (!isCopilotV1ChatSpan(record)) {
|
|
14038
|
+
return responseId ? `resp:${responseId}` : null;
|
|
14039
|
+
}
|
|
14040
|
+
|
|
14041
|
+
const traceId = record?.traceId || "";
|
|
14042
|
+
const spanId = record?.spanId || "";
|
|
14043
|
+
return traceId && spanId
|
|
14044
|
+
? `${traceId}:${spanId}`
|
|
14045
|
+
: responseId
|
|
14046
|
+
? `resp:${responseId}`
|
|
14047
|
+
: null;
|
|
14048
|
+
}
|
|
14049
|
+
|
|
14050
|
+
function copilotOtelAggregateKey(model, bucketStart) {
|
|
14051
|
+
return JSON.stringify([model, bucketStart]);
|
|
14052
|
+
}
|
|
14053
|
+
|
|
14054
|
+
function addCopilotOtelAggregate(aggregates, model, bucketStart, delta) {
|
|
14055
|
+
const key = copilotOtelAggregateKey(model, bucketStart);
|
|
14056
|
+
let totals = aggregates.get(key);
|
|
14057
|
+
if (!totals) {
|
|
14058
|
+
totals = initTotals();
|
|
14059
|
+
aggregates.set(key, totals);
|
|
14060
|
+
}
|
|
14061
|
+
addTotals(totals, delta);
|
|
14062
|
+
}
|
|
14063
|
+
|
|
14064
|
+
function copilotTotalsCover(existing, required) {
|
|
14065
|
+
if (!existing || typeof existing !== "object") return false;
|
|
14066
|
+
for (const field of [
|
|
14067
|
+
"input_tokens",
|
|
14068
|
+
"cached_input_tokens",
|
|
14069
|
+
"cache_creation_input_tokens",
|
|
14070
|
+
"output_tokens",
|
|
14071
|
+
"reasoning_output_tokens",
|
|
14072
|
+
"total_tokens",
|
|
14073
|
+
"billable_total_tokens",
|
|
14074
|
+
"conversation_count",
|
|
14075
|
+
]) {
|
|
14076
|
+
const actual = Number(existing[field] || 0);
|
|
14077
|
+
const needed = Number(required?.[field] || 0);
|
|
14078
|
+
if (!Number.isFinite(actual) || !Number.isFinite(needed) || actual < needed) {
|
|
14079
|
+
return false;
|
|
14080
|
+
}
|
|
14081
|
+
}
|
|
14082
|
+
return true;
|
|
14083
|
+
}
|
|
14084
|
+
|
|
14085
|
+
function extractCopilotOtelUsage(record) {
|
|
14086
|
+
if (!isCopilotChatSpan(record)) return null;
|
|
14087
|
+
|
|
14088
|
+
const attrs = record.attributes || {};
|
|
14089
|
+
const inputRaw = toNonNegativeInt(attrs["gen_ai.usage.input_tokens"]);
|
|
14090
|
+
const output = toNonNegativeInt(attrs["gen_ai.usage.output_tokens"]);
|
|
14091
|
+
const cacheRead = toNonNegativeInt(
|
|
14092
|
+
attrs["gen_ai.usage.cache_read.input_tokens"] ??
|
|
14093
|
+
attrs["gen_ai.usage.cache_read_input_tokens"] ??
|
|
14094
|
+
attrs["gen_ai.usage.cached_input_tokens"],
|
|
14095
|
+
);
|
|
14096
|
+
// Copilot CLI: cache_write.input_tokens; Copilot Chat extension: cache_creation.input_tokens
|
|
14097
|
+
const cacheWrite = toNonNegativeInt(
|
|
14098
|
+
attrs["gen_ai.usage.cache_write.input_tokens"] ??
|
|
14099
|
+
attrs["gen_ai.usage.cache_creation.input_tokens"] ??
|
|
14100
|
+
attrs["gen_ai.usage.cache_write_input_tokens"] ??
|
|
14101
|
+
attrs["gen_ai.usage.cache_creation_input_tokens"],
|
|
14102
|
+
);
|
|
14103
|
+
// Copilot CLI: reasoning.output_tokens; Copilot Chat extension: reasoning_tokens
|
|
14104
|
+
const reasoning = toNonNegativeInt(
|
|
14105
|
+
attrs["gen_ai.usage.reasoning.output_tokens"] ??
|
|
14106
|
+
attrs["gen_ai.usage.reasoning_tokens"] ??
|
|
14107
|
+
attrs["gen_ai.usage.reasoning_output_tokens"],
|
|
14108
|
+
);
|
|
14109
|
+
const reasoningClamped = Math.min(reasoning, output);
|
|
14110
|
+
const outputWithoutReasoning = Math.max(0, output - reasoningClamped);
|
|
14111
|
+
const cliSpan = isCopilotV1ChatSpan(record);
|
|
14112
|
+
// CLI input includes both cache reads and writes. Chat-extension LogRecords
|
|
14113
|
+
// expose cache creation separately, so preserve their existing input-minus-read semantics.
|
|
14114
|
+
const cacheReadClamped = Math.min(cacheRead, inputRaw);
|
|
14115
|
+
const cacheWriteClamped = Math.min(
|
|
14116
|
+
cacheWrite,
|
|
14117
|
+
Math.max(0, inputRaw - cacheReadClamped),
|
|
14118
|
+
);
|
|
14119
|
+
const cacheWriteForAccounting = cliSpan ? cacheWriteClamped : cacheWrite;
|
|
14120
|
+
const input = Math.max(
|
|
14121
|
+
0,
|
|
14122
|
+
inputRaw - cacheReadClamped - (cliSpan ? cacheWriteClamped : 0),
|
|
14123
|
+
);
|
|
14124
|
+
const totalInteresting =
|
|
14125
|
+
input +
|
|
14126
|
+
outputWithoutReasoning +
|
|
14127
|
+
cacheReadClamped +
|
|
14128
|
+
cacheWriteForAccounting +
|
|
14129
|
+
reasoningClamped;
|
|
14130
|
+
if (totalInteresting === 0) return null;
|
|
14131
|
+
|
|
14132
|
+
// CLI Span uses endTime/startTime; Chat extension LogRecord uses hrTime/hrTimeObserved.
|
|
14133
|
+
const tsMs =
|
|
14134
|
+
copilotOtelTimeToMs(record.endTime) ||
|
|
14135
|
+
copilotOtelTimeToMs(record.startTime) ||
|
|
14136
|
+
copilotOtelTimeToMs(record.hrTime) ||
|
|
14137
|
+
copilotOtelTimeToMs(record.hrTimeObserved);
|
|
14138
|
+
if (!tsMs) return null;
|
|
14139
|
+
const bucketStart = toUtcHalfHourStart(new Date(tsMs).toISOString());
|
|
14140
|
+
if (!bucketStart) return null;
|
|
14141
|
+
|
|
14142
|
+
const model =
|
|
14143
|
+
normalizeCopilotAppModel(pickCopilotModel(attrs)) ||
|
|
14144
|
+
COPILOT_APP_DEFAULT_MODEL;
|
|
14145
|
+
const cliSessionId =
|
|
14146
|
+
typeof attrs["gen_ai.conversation.id"] === "string"
|
|
14147
|
+
? attrs["gen_ai.conversation.id"].trim()
|
|
14148
|
+
: "";
|
|
14149
|
+
const matchBase = {
|
|
14150
|
+
sessionId: cliSessionId,
|
|
14151
|
+
model,
|
|
14152
|
+
output: outputWithoutReasoning,
|
|
14153
|
+
cacheRead: cacheReadClamped,
|
|
14154
|
+
cacheWrite: cacheWriteClamped,
|
|
14155
|
+
reasoning: reasoningClamped,
|
|
14156
|
+
tsMs,
|
|
14157
|
+
};
|
|
14158
|
+
return {
|
|
14159
|
+
bucketStart,
|
|
14160
|
+
cliSpan,
|
|
14161
|
+
cliSessionId,
|
|
14162
|
+
delta: {
|
|
14163
|
+
input_tokens: input,
|
|
14164
|
+
cached_input_tokens: cacheReadClamped,
|
|
14165
|
+
cache_creation_input_tokens: cacheWriteForAccounting,
|
|
14166
|
+
output_tokens: outputWithoutReasoning,
|
|
14167
|
+
reasoning_output_tokens: reasoningClamped,
|
|
14168
|
+
total_tokens:
|
|
14169
|
+
input +
|
|
14170
|
+
outputWithoutReasoning +
|
|
14171
|
+
cacheReadClamped +
|
|
14172
|
+
cacheWriteForAccounting +
|
|
14173
|
+
reasoningClamped,
|
|
14174
|
+
conversation_count: 1,
|
|
14175
|
+
},
|
|
14176
|
+
matchBase,
|
|
14177
|
+
model,
|
|
14178
|
+
tsMs,
|
|
14179
|
+
};
|
|
14180
|
+
}
|
|
14181
|
+
|
|
13672
14182
|
function copilotUsageMatchKey({
|
|
13673
14183
|
sessionId,
|
|
13674
14184
|
model,
|
|
@@ -13728,6 +14238,141 @@ function createCopilotStoreUsageMatcher(events) {
|
|
|
13728
14238
|
};
|
|
13729
14239
|
}
|
|
13730
14240
|
|
|
14241
|
+
// v2 may already have advanced the file cursor while collapsing several Chat
|
|
14242
|
+
// extension LogRecords that shared one nested spanContext. Recompute only that
|
|
14243
|
+
// envelope's historical contribution and apply the delta to the existing
|
|
14244
|
+
// Copilot buckets. CLI spans are deliberately left alone: their v2 key was the
|
|
14245
|
+
// correct top-level traceId:spanId key, and session-store adoption can coexist
|
|
14246
|
+
// with the OTEL parser.
|
|
14247
|
+
async function migrateCopilotChatLogRecordDedup({
|
|
14248
|
+
files,
|
|
14249
|
+
fileOffsets,
|
|
14250
|
+
hourlyState,
|
|
14251
|
+
touchedBuckets,
|
|
14252
|
+
seenIds,
|
|
14253
|
+
} = {}) {
|
|
14254
|
+
const trackedFiles = Object.keys(fileOffsets || {}).filter(
|
|
14255
|
+
(filePath) => Number(fileOffsets[filePath]?.size) > 0,
|
|
14256
|
+
);
|
|
14257
|
+
if (trackedFiles.length === 0) {
|
|
14258
|
+
return { applied: true, changed: false };
|
|
14259
|
+
}
|
|
14260
|
+
|
|
14261
|
+
const availableFiles = new Set(Array.isArray(files) ? files : []);
|
|
14262
|
+
if (trackedFiles.some((filePath) => !availableFiles.has(filePath))) {
|
|
14263
|
+
return { applied: false, reason: "a previously parsed OTEL file is unavailable" };
|
|
14264
|
+
}
|
|
14265
|
+
|
|
14266
|
+
const oldSeen = new Set();
|
|
14267
|
+
const newSeen = new Set();
|
|
14268
|
+
const oldTotals = new Map();
|
|
14269
|
+
const newTotals = new Map();
|
|
14270
|
+
|
|
14271
|
+
for (const filePath of files) {
|
|
14272
|
+
if (!Object.prototype.hasOwnProperty.call(fileOffsets, filePath)) continue;
|
|
14273
|
+
const prevEntry = fileOffsets[filePath] || {};
|
|
14274
|
+
const prevSize = Number(prevEntry.size) || 0;
|
|
14275
|
+
if (prevSize <= 0) continue;
|
|
14276
|
+
|
|
14277
|
+
let stat;
|
|
14278
|
+
try {
|
|
14279
|
+
stat = fssync.statSync(filePath);
|
|
14280
|
+
} catch (_e) {
|
|
14281
|
+
return { applied: false, reason: "a previously parsed OTEL file is unreadable" };
|
|
14282
|
+
}
|
|
14283
|
+
if (
|
|
14284
|
+
stat.size < prevSize ||
|
|
14285
|
+
(typeof prevEntry.ino === "number" && stat.ino !== prevEntry.ino)
|
|
14286
|
+
) {
|
|
14287
|
+
return { applied: false, reason: "a previously parsed OTEL file changed during migration" };
|
|
14288
|
+
}
|
|
14289
|
+
|
|
14290
|
+
let stream;
|
|
14291
|
+
try {
|
|
14292
|
+
stream = fssync.createReadStream(filePath, {
|
|
14293
|
+
encoding: "utf8",
|
|
14294
|
+
start: 0,
|
|
14295
|
+
end: prevSize - 1,
|
|
14296
|
+
});
|
|
14297
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
14298
|
+
try {
|
|
14299
|
+
for await (const line of rl) {
|
|
14300
|
+
if (!line || !line.trim()) continue;
|
|
14301
|
+
let record;
|
|
14302
|
+
try {
|
|
14303
|
+
record = JSON.parse(line);
|
|
14304
|
+
} catch (_e) {
|
|
14305
|
+
continue;
|
|
14306
|
+
}
|
|
14307
|
+
const usage = extractCopilotOtelUsage(record);
|
|
14308
|
+
if (!usage || usage.cliSpan) continue;
|
|
14309
|
+
|
|
14310
|
+
const oldKey = getCopilotLegacyDedupKey(record, record.attributes || {});
|
|
14311
|
+
const oldDuplicate = oldKey && oldSeen.has(oldKey);
|
|
14312
|
+
if (oldKey) oldSeen.add(oldKey);
|
|
14313
|
+
if (!oldDuplicate) {
|
|
14314
|
+
addCopilotOtelAggregate(
|
|
14315
|
+
oldTotals,
|
|
14316
|
+
usage.model,
|
|
14317
|
+
usage.bucketStart,
|
|
14318
|
+
usage.delta,
|
|
14319
|
+
);
|
|
14320
|
+
}
|
|
14321
|
+
|
|
14322
|
+
const newKey = getCopilotDedupKey(record, record.attributes || {});
|
|
14323
|
+
const newDuplicate = newKey && newSeen.has(newKey);
|
|
14324
|
+
if (newKey) newSeen.add(newKey);
|
|
14325
|
+
if (!newDuplicate) {
|
|
14326
|
+
addCopilotOtelAggregate(
|
|
14327
|
+
newTotals,
|
|
14328
|
+
usage.model,
|
|
14329
|
+
usage.bucketStart,
|
|
14330
|
+
usage.delta,
|
|
14331
|
+
);
|
|
14332
|
+
}
|
|
14333
|
+
}
|
|
14334
|
+
} finally {
|
|
14335
|
+
rl.close();
|
|
14336
|
+
}
|
|
14337
|
+
} catch (_e) {
|
|
14338
|
+
return { applied: false, reason: "an OTEL file could not be scanned" };
|
|
14339
|
+
} finally {
|
|
14340
|
+
stream?.destroy();
|
|
14341
|
+
}
|
|
14342
|
+
}
|
|
14343
|
+
|
|
14344
|
+
const keys = new Set([...oldTotals.keys(), ...newTotals.keys()]);
|
|
14345
|
+
for (const key of keys) {
|
|
14346
|
+
const [model, bucketStart] = JSON.parse(key);
|
|
14347
|
+
const oldUsage = oldTotals.get(key) || initTotals();
|
|
14348
|
+
if (!copilotTotalsCover(
|
|
14349
|
+
hourlyState.buckets[bucketKey("copilot", model, bucketStart)]?.totals,
|
|
14350
|
+
oldUsage,
|
|
14351
|
+
)) {
|
|
14352
|
+
return {
|
|
14353
|
+
applied: false,
|
|
14354
|
+
reason: "existing Copilot buckets do not cover the old OTEL contribution",
|
|
14355
|
+
};
|
|
14356
|
+
}
|
|
14357
|
+
}
|
|
14358
|
+
|
|
14359
|
+
let changed = false;
|
|
14360
|
+
for (const key of keys) {
|
|
14361
|
+
const [model, bucketStart] = JSON.parse(key);
|
|
14362
|
+
const oldUsage = oldTotals.get(key) || initTotals();
|
|
14363
|
+
const newUsage = newTotals.get(key) || initTotals();
|
|
14364
|
+
if (totalsKey(oldUsage) === totalsKey(newUsage)) continue;
|
|
14365
|
+
const bucket = getHourlyBucket(hourlyState, "copilot", model, bucketStart);
|
|
14366
|
+
subtractTotals(bucket.totals, oldUsage);
|
|
14367
|
+
addTotals(bucket.totals, newUsage);
|
|
14368
|
+
touchedBuckets.add(bucketKey("copilot", model, bucketStart));
|
|
14369
|
+
changed = true;
|
|
14370
|
+
}
|
|
14371
|
+
|
|
14372
|
+
for (const id of newSeen) seenIds.add(id);
|
|
14373
|
+
return { applied: true, changed };
|
|
14374
|
+
}
|
|
14375
|
+
|
|
13731
14376
|
// Migration helper: stream the bytes v1 already saw (0 -> prevSize), classify
|
|
13732
14377
|
// whether the file contains old CLI spans v1 processed, and whether it also
|
|
13733
14378
|
// contains v2-only chat records v1 skipped. Mixed files must be replayed, but
|
|
@@ -13841,12 +14486,18 @@ async function parseCopilotIncremental({
|
|
|
13841
14486
|
seenIds.size > 0 || Object.keys(fileOffsetsRaw).length > 0;
|
|
13842
14487
|
let usageClaimsComplete =
|
|
13843
14488
|
copilotState.usageClaimsComplete === true || !hadPriorUsageHistory;
|
|
14489
|
+
const files = Array.isArray(otelPaths) && otelPaths.length > 0
|
|
14490
|
+
? otelPaths
|
|
14491
|
+
: resolveCopilotOtelPaths(env || process.env);
|
|
14492
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
14493
|
+
const touchedBuckets = new Set();
|
|
13844
14494
|
const migrationSkipLineHashes = new Map();
|
|
14495
|
+
let cursorVersion = COPILOT_PARSER_VERSION;
|
|
13845
14496
|
// One-shot v1->v2 migration:
|
|
13846
14497
|
// - pure v2-only files: clear offset and re-read all skipped Chat records
|
|
13847
14498
|
// - pure v1 CLI files: preserve offset to avoid replaying history beyond seenIds
|
|
13848
14499
|
// - mixed files: clear offset, but skip old v1 CLI lines by hash during replay
|
|
13849
|
-
if (priorVersion <
|
|
14500
|
+
if (priorVersion < 2) {
|
|
13850
14501
|
for (const filePath of Object.keys(fileOffsets)) {
|
|
13851
14502
|
const prevSize = Number(fileOffsets[filePath]?.size) || 0;
|
|
13852
14503
|
const scan = await scanCopilotV1MigrationFile(filePath, prevSize);
|
|
@@ -13859,13 +14510,33 @@ async function parseCopilotIncremental({
|
|
|
13859
14510
|
}
|
|
13860
14511
|
}
|
|
13861
14512
|
|
|
13862
|
-
|
|
13863
|
-
|
|
13864
|
-
|
|
14513
|
+
// v2 used spanContext as the fallback key for Chat-extension LogRecords.
|
|
14514
|
+
// Those records can share one context across multiple model requests, so
|
|
14515
|
+
// repair the already-counted file prefix before switching to response.id.
|
|
14516
|
+
// If the prefix cannot be verified, leave the cursor at v2 and retry on the
|
|
14517
|
+
// next sync rather than replaying it with a different deduplication scheme.
|
|
14518
|
+
if (priorVersion === 2) {
|
|
14519
|
+
const migration = await migrateCopilotChatLogRecordDedup({
|
|
14520
|
+
files,
|
|
14521
|
+
fileOffsets,
|
|
14522
|
+
hourlyState,
|
|
14523
|
+
touchedBuckets,
|
|
14524
|
+
seenIds,
|
|
14525
|
+
});
|
|
14526
|
+
if (!migration.applied) {
|
|
14527
|
+
return {
|
|
14528
|
+
recordsProcessed: 0,
|
|
14529
|
+
eventsAggregated: 0,
|
|
14530
|
+
bucketsQueued: 0,
|
|
14531
|
+
usageClaims: recentOtelUsageEvents,
|
|
14532
|
+
};
|
|
14533
|
+
}
|
|
14534
|
+
}
|
|
14535
|
+
|
|
13865
14536
|
if (files.length === 0) {
|
|
13866
14537
|
cursors.copilot = {
|
|
13867
14538
|
...copilotState,
|
|
13868
|
-
version:
|
|
14539
|
+
version: cursorVersion,
|
|
13869
14540
|
seenIds: Array.from(seenIds),
|
|
13870
14541
|
fileOffsets,
|
|
13871
14542
|
recentUsageEvents: recentOtelUsageEvents,
|
|
@@ -13879,9 +14550,6 @@ async function parseCopilotIncremental({
|
|
|
13879
14550
|
usageClaims: recentOtelUsageEvents,
|
|
13880
14551
|
};
|
|
13881
14552
|
}
|
|
13882
|
-
|
|
13883
|
-
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
13884
|
-
const touchedBuckets = new Set();
|
|
13885
14553
|
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
13886
14554
|
let recordsProcessed = 0;
|
|
13887
14555
|
let eventsAggregated = 0;
|
|
@@ -13941,117 +14609,39 @@ async function parseCopilotIncremental({
|
|
|
13941
14609
|
// gen_ai.response.id is per-LLM-call unique.
|
|
13942
14610
|
const dedupKey = getCopilotDedupKey(record, attrs);
|
|
13943
14611
|
if (dedupKey && seenIds.has(dedupKey)) continue;
|
|
13944
|
-
|
|
14612
|
+
const cliSpan = isCopilotV1ChatSpan(record);
|
|
14613
|
+
if (skipCliSpans && cliSpan) {
|
|
13945
14614
|
if (dedupKey) seenIds.add(dedupKey);
|
|
13946
14615
|
continue;
|
|
13947
14616
|
}
|
|
13948
14617
|
|
|
13949
|
-
const
|
|
13950
|
-
|
|
13951
|
-
|
|
13952
|
-
attrs["gen_ai.usage.cache_read.input_tokens"] ??
|
|
13953
|
-
attrs["gen_ai.usage.cache_read_input_tokens"] ??
|
|
13954
|
-
attrs["gen_ai.usage.cached_input_tokens"],
|
|
13955
|
-
);
|
|
13956
|
-
// Copilot CLI: cache_write.input_tokens; Copilot Chat extension: cache_creation.input_tokens
|
|
13957
|
-
const cacheWrite = toNonNegativeInt(
|
|
13958
|
-
attrs["gen_ai.usage.cache_write.input_tokens"] ??
|
|
13959
|
-
attrs["gen_ai.usage.cache_creation.input_tokens"] ??
|
|
13960
|
-
attrs["gen_ai.usage.cache_write_input_tokens"] ??
|
|
13961
|
-
attrs["gen_ai.usage.cache_creation_input_tokens"],
|
|
13962
|
-
);
|
|
13963
|
-
// Copilot CLI: reasoning.output_tokens; Copilot Chat extension: reasoning_tokens
|
|
13964
|
-
const reasoning = toNonNegativeInt(
|
|
13965
|
-
attrs["gen_ai.usage.reasoning.output_tokens"] ??
|
|
13966
|
-
attrs["gen_ai.usage.reasoning_tokens"] ??
|
|
13967
|
-
attrs["gen_ai.usage.reasoning_output_tokens"],
|
|
13968
|
-
);
|
|
13969
|
-
const reasoningClamped = Math.min(reasoning, output);
|
|
13970
|
-
const outputWithoutReasoning = Math.max(0, output - reasoningClamped);
|
|
13971
|
-
const cliSpan = isCopilotV1ChatSpan(record);
|
|
13972
|
-
// CLI input includes both cache reads and writes. Chat-extension
|
|
13973
|
-
// LogRecords expose cache creation separately, so preserve their existing
|
|
13974
|
-
// input-minus-read semantics.
|
|
13975
|
-
const cacheReadClamped = Math.min(cacheRead, inputRaw);
|
|
13976
|
-
const cacheWriteClamped = Math.min(
|
|
13977
|
-
cacheWrite,
|
|
13978
|
-
Math.max(0, inputRaw - cacheReadClamped),
|
|
13979
|
-
);
|
|
13980
|
-
const cacheWriteForAccounting = cliSpan ? cacheWriteClamped : cacheWrite;
|
|
13981
|
-
const input = Math.max(
|
|
13982
|
-
0,
|
|
13983
|
-
inputRaw - cacheReadClamped - (cliSpan ? cacheWriteClamped : 0),
|
|
13984
|
-
);
|
|
13985
|
-
const totalInteresting =
|
|
13986
|
-
input +
|
|
13987
|
-
outputWithoutReasoning +
|
|
13988
|
-
cacheReadClamped +
|
|
13989
|
-
cacheWriteForAccounting +
|
|
13990
|
-
reasoningClamped;
|
|
13991
|
-
if (totalInteresting === 0) continue;
|
|
13992
|
-
|
|
13993
|
-
// CLI Span uses endTime/startTime; Chat extension LogRecord uses hrTime/hrTimeObserved.
|
|
13994
|
-
const tsMs =
|
|
13995
|
-
copilotOtelTimeToMs(record.endTime) ||
|
|
13996
|
-
copilotOtelTimeToMs(record.startTime) ||
|
|
13997
|
-
copilotOtelTimeToMs(record.hrTime) ||
|
|
13998
|
-
copilotOtelTimeToMs(record.hrTimeObserved);
|
|
13999
|
-
if (!tsMs) continue;
|
|
14000
|
-
const tsIso = new Date(tsMs).toISOString();
|
|
14001
|
-
const bucketStart = toUtcHalfHourStart(tsIso);
|
|
14002
|
-
if (!bucketStart) continue;
|
|
14003
|
-
|
|
14004
|
-
const model =
|
|
14005
|
-
normalizeCopilotAppModel(pickCopilotModel(attrs)) ||
|
|
14006
|
-
COPILOT_APP_DEFAULT_MODEL;
|
|
14007
|
-
const cliSessionId =
|
|
14008
|
-
typeof attrs["gen_ai.conversation.id"] === "string"
|
|
14009
|
-
? attrs["gen_ai.conversation.id"].trim()
|
|
14010
|
-
: "";
|
|
14011
|
-
if (cliSpan && !cliSessionId) usageClaimsComplete = false;
|
|
14012
|
-
const matchBase = {
|
|
14013
|
-
sessionId: cliSessionId,
|
|
14014
|
-
model,
|
|
14015
|
-
output: outputWithoutReasoning,
|
|
14016
|
-
cacheRead: cacheReadClamped,
|
|
14017
|
-
cacheWrite: cacheWriteClamped,
|
|
14018
|
-
reasoning: reasoningClamped,
|
|
14019
|
-
tsMs,
|
|
14020
|
-
};
|
|
14618
|
+
const usage = extractCopilotOtelUsage(record);
|
|
14619
|
+
if (!usage) continue;
|
|
14620
|
+
if (usage.cliSpan && !usage.cliSessionId) usageClaimsComplete = false;
|
|
14021
14621
|
const matchedStoreUsage =
|
|
14022
|
-
|
|
14622
|
+
usage.cliSpan &&
|
|
14023
14623
|
storeUsageMatcher.consume({
|
|
14024
|
-
...matchBase,
|
|
14025
|
-
input,
|
|
14624
|
+
...usage.matchBase,
|
|
14625
|
+
input: usage.delta.input_tokens,
|
|
14026
14626
|
});
|
|
14027
14627
|
if (matchedStoreUsage) {
|
|
14028
14628
|
if (dedupKey) seenIds.add(dedupKey);
|
|
14029
14629
|
continue;
|
|
14030
14630
|
}
|
|
14031
14631
|
|
|
14032
|
-
const
|
|
14033
|
-
|
|
14034
|
-
|
|
14035
|
-
|
|
14036
|
-
|
|
14037
|
-
|
|
14038
|
-
|
|
14039
|
-
|
|
14040
|
-
outputWithoutReasoning +
|
|
14041
|
-
cacheReadClamped +
|
|
14042
|
-
cacheWriteForAccounting +
|
|
14043
|
-
reasoningClamped,
|
|
14044
|
-
conversation_count: 1,
|
|
14045
|
-
};
|
|
14046
|
-
|
|
14047
|
-
const bucket = getHourlyBucket(hourlyState, "copilot", model, bucketStart);
|
|
14048
|
-
addTotals(bucket.totals, delta);
|
|
14049
|
-
touchedBuckets.add(bucketKey("copilot", model, bucketStart));
|
|
14632
|
+
const bucket = getHourlyBucket(
|
|
14633
|
+
hourlyState,
|
|
14634
|
+
"copilot",
|
|
14635
|
+
usage.model,
|
|
14636
|
+
usage.bucketStart,
|
|
14637
|
+
);
|
|
14638
|
+
addTotals(bucket.totals, usage.delta);
|
|
14639
|
+
touchedBuckets.add(bucketKey("copilot", usage.model, usage.bucketStart));
|
|
14050
14640
|
eventsAggregated++;
|
|
14051
|
-
if (cliSpan && cliSessionId) {
|
|
14641
|
+
if (usage.cliSpan && usage.cliSessionId) {
|
|
14052
14642
|
recentOtelUsageEvents.push({
|
|
14053
|
-
...matchBase,
|
|
14054
|
-
input,
|
|
14643
|
+
...usage.matchBase,
|
|
14644
|
+
input: usage.delta.input_tokens,
|
|
14055
14645
|
firstSeenAtMs: claimNowMs,
|
|
14056
14646
|
});
|
|
14057
14647
|
}
|
|
@@ -14093,7 +14683,7 @@ async function parseCopilotIncremental({
|
|
|
14093
14683
|
cursors.hourly = hourlyState;
|
|
14094
14684
|
cursors.copilot = {
|
|
14095
14685
|
...copilotState,
|
|
14096
|
-
version:
|
|
14686
|
+
version: cursorVersion,
|
|
14097
14687
|
seenIds: cappedSeen,
|
|
14098
14688
|
fileOffsets,
|
|
14099
14689
|
recentUsageEvents: retainedRecentOtelUsageEvents,
|
|
@@ -15534,14 +16124,14 @@ async function parseCopilotAppDbIncremental({
|
|
|
15534
16124
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
15535
16125
|
// Grok Build (xAI) — passive reader for ~/.grok/sessions/**/updates.jsonl + signals.json
|
|
15536
16126
|
// Triggered either by full scan in sync or by the SessionEnd hook writing a signal.
|
|
15537
|
-
//
|
|
15538
|
-
//
|
|
15539
|
-
//
|
|
16127
|
+
// turn_completed.usage exposes the reported input/output/cache/reasoning split
|
|
16128
|
+
// and, on current Grok builds, exact server cost ticks. Older/partial sessions
|
|
16129
|
+
// without that event retain the explicitly isolated context-watermark fallback.
|
|
15540
16130
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
15541
16131
|
|
|
15542
16132
|
const GROK_ESTIMATED_INPUT_RATIO = 0.8;
|
|
15543
|
-
//
|
|
15544
|
-
const GROK_CURSOR_VERSION =
|
|
16133
|
+
// v5: split cache creation + reasoning correctly and retain reported cost.
|
|
16134
|
+
const GROK_CURSOR_VERSION = 5;
|
|
15545
16135
|
|
|
15546
16136
|
function resolveGrokBuildHome(env = process.env) {
|
|
15547
16137
|
if (env.TOKENTRACKER_GROK_HOME) return env.TOKENTRACKER_GROK_HOME;
|
|
@@ -15784,37 +16374,13 @@ function canonicalizeGrokUsageModel(model) {
|
|
|
15784
16374
|
}
|
|
15785
16375
|
|
|
15786
16376
|
function normalizeGrokTurnUsage(usage, model, timestamp, eventId) {
|
|
15787
|
-
|
|
15788
|
-
|
|
15789
|
-
usage.inputTokens ?? usage.input_tokens,
|
|
15790
|
-
);
|
|
15791
|
-
const output = normalizeNonNegativeNumber(
|
|
15792
|
-
usage.outputTokens ?? usage.output_tokens,
|
|
15793
|
-
);
|
|
15794
|
-
const cached = normalizeNonNegativeNumber(
|
|
15795
|
-
usage.cachedReadTokens ??
|
|
15796
|
-
usage.cache_read_input_tokens ??
|
|
15797
|
-
usage.cached_input_tokens,
|
|
15798
|
-
);
|
|
15799
|
-
const reasoning = normalizeNonNegativeNumber(
|
|
15800
|
-
usage.reasoningTokens ?? usage.reasoning_output_tokens,
|
|
15801
|
-
);
|
|
15802
|
-
// Grok reports inputTokens as the full prompt (including cache hits). Split
|
|
15803
|
-
// so pricing can apply cache_read rates correctly.
|
|
15804
|
-
const nonCachedInput = Math.max(0, inputRaw - cached);
|
|
15805
|
-
let total = normalizeNonNegativeNumber(usage.totalTokens ?? usage.total_tokens);
|
|
15806
|
-
if (total <= 0) {
|
|
15807
|
-
total = inputRaw + output + reasoning;
|
|
15808
|
-
}
|
|
15809
|
-
if (total <= 0 && nonCachedInput <= 0 && cached <= 0 && output <= 0) return null;
|
|
16377
|
+
const normalized = normalizeGrokUsage(usage);
|
|
16378
|
+
if (!normalized) return null;
|
|
15810
16379
|
return {
|
|
15811
|
-
|
|
15812
|
-
|
|
15813
|
-
|
|
15814
|
-
|
|
15815
|
-
reasoning_output_tokens: reasoning,
|
|
15816
|
-
total_tokens: total > 0 ? total : nonCachedInput + cached + output + reasoning,
|
|
15817
|
-
billable_total_tokens: total > 0 ? total : nonCachedInput + cached + output + reasoning,
|
|
16380
|
+
...normalized,
|
|
16381
|
+
// A missing/partial cost must fall back to model pricing, not turn into a
|
|
16382
|
+
// falsely precise $0 row in the hourly queue.
|
|
16383
|
+
total_cost_usd: normalized.total_cost_usd ?? 0,
|
|
15818
16384
|
conversation_count: 1,
|
|
15819
16385
|
model: canonicalizeGrokUsageModel(model),
|
|
15820
16386
|
timestamp,
|
|
@@ -15978,6 +16544,12 @@ function estimateGrokTokenDelta(totalTokens, conversationCount, options = {}) {
|
|
|
15978
16544
|
};
|
|
15979
16545
|
}
|
|
15980
16546
|
|
|
16547
|
+
function mergeGrokUsagePrecision(current, next) {
|
|
16548
|
+
if (!current) return next;
|
|
16549
|
+
if (!next || current === next) return current;
|
|
16550
|
+
return "mixed";
|
|
16551
|
+
}
|
|
16552
|
+
|
|
15981
16553
|
function clearGrokHourlyBuckets(hourlyState) {
|
|
15982
16554
|
if (!hourlyState || typeof hourlyState !== "object") return;
|
|
15983
16555
|
const buckets = hourlyState.buckets && typeof hourlyState.buckets === "object" ? hourlyState.buckets : null;
|
|
@@ -16106,9 +16678,9 @@ async function parseGrokBuildIncremental({
|
|
|
16106
16678
|
const prevVersion = Number(grokState.version) || 0;
|
|
16107
16679
|
const needsTurnUsageMigration = prevVersion < GROK_CURSOR_VERSION;
|
|
16108
16680
|
|
|
16109
|
-
// v3 and earlier treated context-window totalTokens as cumulative spend
|
|
16110
|
-
//
|
|
16111
|
-
// turn_completed.usage
|
|
16681
|
+
// v3 and earlier treated context-window totalTokens as cumulative spend;
|
|
16682
|
+
// v4 still overlapped output/reasoning, discarded cache creation, and ignored
|
|
16683
|
+
// provider-reported cost. Rebuild from turn_completed.usage for v5.
|
|
16112
16684
|
//
|
|
16113
16685
|
// Drop prior watermark totals / updateOffsets so files are re-read from byte 0,
|
|
16114
16686
|
// but keep legacySeen markers from seenSessions so the one-shot baseline
|
|
@@ -16232,9 +16804,10 @@ async function parseGrokBuildIncremental({
|
|
|
16232
16804
|
reasoning_output_tokens: event.reasoning_output_tokens,
|
|
16233
16805
|
total_tokens: event.total_tokens,
|
|
16234
16806
|
billable_total_tokens: event.billable_total_tokens,
|
|
16807
|
+
total_cost_usd: event.total_cost_usd,
|
|
16235
16808
|
conversation_count: event.conversation_count || 1,
|
|
16236
16809
|
};
|
|
16237
|
-
pendingBucketDeltas.push({ model: eventModel, hourStartStr, delta });
|
|
16810
|
+
pendingBucketDeltas.push({ model: eventModel, hourStartStr, delta, usagePrecision: "reported" });
|
|
16238
16811
|
cumulativeTotal += event.total_tokens;
|
|
16239
16812
|
tokenDeltaForSession += event.total_tokens;
|
|
16240
16813
|
finalTouchedHourStart = hourStartStr;
|
|
@@ -16261,7 +16834,7 @@ async function parseGrokBuildIncremental({
|
|
|
16261
16834
|
toUtcHalfHourStart(Date.now());
|
|
16262
16835
|
if (!hourStartStr) continue;
|
|
16263
16836
|
const delta = estimateGrokTokenDelta(deltaTokens, 0, { allowZeroConversationCount: true });
|
|
16264
|
-
pendingBucketDeltas.push({ model, hourStartStr, delta });
|
|
16837
|
+
pendingBucketDeltas.push({ model, hourStartStr, delta, usagePrecision: "estimated" });
|
|
16265
16838
|
tokenDeltaForSession += deltaTokens;
|
|
16266
16839
|
finalTouchedHourStart = hourStartStr;
|
|
16267
16840
|
source = "updates";
|
|
@@ -16274,7 +16847,7 @@ async function parseGrokBuildIncremental({
|
|
|
16274
16847
|
const hourStartStr = toUtcHalfHourStart(lastActive) || toUtcHalfHourStart(Date.now());
|
|
16275
16848
|
if (hourStartStr) {
|
|
16276
16849
|
const delta = estimateGrokTokenDelta(deltaTokens, 0, { allowZeroConversationCount: true });
|
|
16277
|
-
pendingBucketDeltas.push({ model, hourStartStr, delta });
|
|
16850
|
+
pendingBucketDeltas.push({ model, hourStartStr, delta, usagePrecision: "estimated" });
|
|
16278
16851
|
tokenDeltaForSession += deltaTokens;
|
|
16279
16852
|
finalTouchedHourStart = hourStartStr;
|
|
16280
16853
|
source = "signals";
|
|
@@ -16292,6 +16865,10 @@ async function parseGrokBuildIncremental({
|
|
|
16292
16865
|
for (const pending of pendingBucketDeltas) {
|
|
16293
16866
|
const bucket = getHourlyBucket(hourlyState, "grok", pending.model, pending.hourStartStr);
|
|
16294
16867
|
addTotals(bucket.totals, pending.delta);
|
|
16868
|
+
bucket.usage_precision = mergeGrokUsagePrecision(
|
|
16869
|
+
bucket.usage_precision,
|
|
16870
|
+
pending.usagePrecision,
|
|
16871
|
+
);
|
|
16295
16872
|
touchedBuckets.add(bucketKey("grok", pending.model, pending.hourStartStr));
|
|
16296
16873
|
eventsAggregated++;
|
|
16297
16874
|
}
|
|
@@ -16530,7 +17107,7 @@ async function parseGrokBuildIncremental({
|
|
|
16530
17107
|
? { ...grokState.migrations }
|
|
16531
17108
|
: {};
|
|
16532
17109
|
if (needsTurnUsageMigration) {
|
|
16533
|
-
migrations.
|
|
17110
|
+
migrations.turnUsageV5 = {
|
|
16534
17111
|
appliedAt: new Date().toISOString(),
|
|
16535
17112
|
fromVersion: prevVersion,
|
|
16536
17113
|
toVersion: GROK_CURSOR_VERSION,
|
|
@@ -18281,6 +18858,7 @@ module.exports = {
|
|
|
18281
18858
|
listGeminiSessionFiles,
|
|
18282
18859
|
listOpencodeMessageFiles,
|
|
18283
18860
|
readOpencodeDbMessages,
|
|
18861
|
+
readOpencodeDbMessagesIncremental,
|
|
18284
18862
|
readMimoDbMessages,
|
|
18285
18863
|
readZcodeDbMessages,
|
|
18286
18864
|
hasZcodeNativeUsageSchema,
|