tokentracker-cli 0.77.1 → 0.77.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +3 -3
- package/README.ja.md +3 -3
- package/README.ko.md +3 -3
- package/README.md +3 -3
- package/README.zh-CN.md +3 -3
- package/dashboard/dist/assets/{ActivityHeatmap-DTZh551Z.js → ActivityHeatmap-CH6m_ZJM.js} +1 -1
- package/dashboard/dist/assets/{Card-DP2HT77t.js → Card-D_aKGDr7.js} +1 -1
- package/dashboard/dist/assets/{ClawdAnimated-ScYRJSG0.js → ClawdAnimated-vRiBiNhb.js} +1 -1
- package/dashboard/dist/assets/{CommunityStatsModal-CU46JMkg.js → CommunityStatsModal-CjetJZ-5.js} +1 -1
- package/dashboard/dist/assets/{ConfirmModal-BYvz0dzg.js → ConfirmModal-C6bZw6FZ.js} +1 -1
- package/dashboard/dist/assets/{Controls-DPElSq0S.js → Controls-Cj22oGL3.js} +1 -1
- package/dashboard/dist/assets/{DashboardPage-CN0buIso.js → DashboardPage-DPxiQkQw.js} +1 -1
- package/dashboard/dist/assets/{DevicePage-BL9ew9-m.js → DevicePage-YAo4ert4.js} +1 -1
- package/dashboard/dist/assets/{DialogClose-cqXrG903.js → DialogClose-hhTDeRp1.js} +1 -1
- package/dashboard/dist/assets/DialogDescription-gxEpcUbJ.js +1 -0
- package/dashboard/dist/assets/{DialogTitle-Oqnn_dIL.js → DialogTitle-CA5znwms.js} +1 -1
- package/dashboard/dist/assets/{FadeIn-CFLwnHqt.js → FadeIn--ff4ZbC-.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-B397ELF4.js → HeaderGithubStar-CTZvanhz.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-CamsHe6o.js → IpCheckPage-Bg47gNVu.js} +1 -1
- package/dashboard/dist/assets/LandingPage-DZ8Mz38n.js +4314 -0
- package/dashboard/dist/assets/{LeaderboardAvatar-B39rNGCV.js → LeaderboardAvatar-3T57GKKK.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-BFUOxX6P.js → LeaderboardPage-wP_yCU9I.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProfileModal-BIe6gBNu.js → LeaderboardProfileModal-CfVMENby.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardProfilePage-DeVBEYtg.js → LeaderboardProfilePage-C4bGXJCE.js} +1 -1
- package/dashboard/dist/assets/{LimitsPage-6I5EU7qe.js → LimitsPage-BAQ10X55.js} +1 -1
- package/dashboard/dist/assets/{LocalOnlyNotice-AyYKZloH.js → LocalOnlyNotice-gtHCn7Mx.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-D8smF1gU.js → LoginPage-TXvIBzoW.js} +1 -1
- package/dashboard/dist/assets/{PetPage-BlMYvJn2.js → PetPage-Dj1jrxYe.js} +1 -1
- package/dashboard/dist/assets/{PopoverPopup-DRWhz-bY.js → PopoverPopup-CQxxgUXk.js} +1 -1
- package/dashboard/dist/assets/{ResetPasswordPage-CzSqJfuA.js → ResetPasswordPage-tN0tmSfP.js} +1 -1
- package/dashboard/dist/assets/{Select-Cb8n7kQO.js → Select-DnbHYPS4.js} +1 -1
- package/dashboard/dist/assets/{SelectItemText-DEYadcIK.js → SelectItemText-2LjzGvo5.js} +1 -1
- package/dashboard/dist/assets/{SettingsPage-DlHXcFdJ.js → SettingsPage-DosKS_L6.js} +1 -1
- package/dashboard/dist/assets/{SkillsPage--Kcv1pxG.js → SkillsPage-DaCJ25vT.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-DwtJ9CKU.js → WidgetsPage-CCGemQVD.js} +1 -1
- package/dashboard/dist/assets/{WrappedPage-BuP9KXqF.js → WrappedPage-YxshTScX.js} +1 -1
- package/dashboard/dist/assets/{agent-logos-DbwDXgg1.js → agent-logos-SeuxwQsp.js} +1 -1
- package/dashboard/dist/assets/{arrow-up-right-cFMBPEVS.js → arrow-up-right-Bm3t-iA0.js} +1 -1
- package/dashboard/dist/assets/check-D8Owfjyw.js +1 -0
- package/dashboard/dist/assets/{copy-DYx8X3ZO.js → copy-DUNz-xWm.js} +1 -1
- package/dashboard/dist/assets/{download-B-hXZWKj.js → download-COSLX73O.js} +1 -1
- package/dashboard/dist/assets/{format-tokens-CngV51wN.js → format-tokens-CR28AiQO.js} +1 -1
- package/dashboard/dist/assets/{info-qVF2hV50.js → info-Cm7V_KDO.js} +1 -1
- package/dashboard/dist/assets/{link-2-DR-yGZFS.js → link-2-CYqDGrkv.js} +1 -1
- package/dashboard/dist/assets/{loader-circle-0k9uueb5.js → loader-circle-6XAdy2ec.js} +1 -1
- package/dashboard/dist/assets/{main-CU04bNYH.js → main-BCstrDYE.js} +3 -3
- package/dashboard/dist/assets/main-BYdJ5vWR.css +1 -0
- package/dashboard/dist/assets/{trash-2-DpxBaV_1.js → trash-2-DKLewOQb.js} +1 -1
- package/dashboard/dist/assets/{use-community-stats-Cbij-V8j.js → use-community-stats-AdPaun74.js} +1 -1
- package/dashboard/dist/assets/{use-limits-display-prefs-q0TuryM0.js → use-limits-display-prefs-Bau4ykMw.js} +1 -1
- package/dashboard/dist/assets/{use-native-settings-BCzogs1L.js → use-native-settings-CidwlprT.js} +1 -1
- package/dashboard/dist/assets/{use-ordered-list-CH-J7c04.js → use-ordered-list-BVaEJaWa.js} +1 -1
- package/dashboard/dist/assets/{use-quality-per-dollar-pref-DKLmkwxe.js → use-quality-per-dollar-pref-Htlc-uFe.js} +1 -1
- package/dashboard/dist/assets/{use-usage-limits-DrmF6I_t.js → use-usage-limits-BnzWTJQS.js} +1 -1
- package/dashboard/dist/assets/{useCurrency-DwygZqrs.js → useCurrency-bd6TLRsc.js} +1 -1
- package/dashboard/dist/assets/{useScrollLock-DVCKuErg.js → useScrollLock-ajoWrfku.js} +1 -1
- package/dashboard/dist/index.html +2 -2
- package/dashboard/dist/share.html +2 -2
- package/package.json +1 -1
- package/src/commands/status.js +174 -18
- package/src/commands/sync.js +336 -21
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +1311 -38
- package/dashboard/dist/assets/DialogDescription-Bo0iWtYz.js +0 -1
- package/dashboard/dist/assets/LandingPage-D4ygTHwO.js +0 -4312
- package/dashboard/dist/assets/check-t_ltRp2A.js +0 -1
- package/dashboard/dist/assets/main-B8lT0e78.css +0 -1
package/src/lib/rollout.js
CHANGED
|
@@ -3729,7 +3729,11 @@ async function parseKiroIncremental({ dbPath, jsonlPath, cursors, queuePath, onP
|
|
|
3729
3729
|
jsonl: { lastLine: jsonlResult.lineCount, updatedAt: new Date().toISOString() },
|
|
3730
3730
|
updatedAt: new Date().toISOString(),
|
|
3731
3731
|
};
|
|
3732
|
-
return {
|
|
3732
|
+
return {
|
|
3733
|
+
recordsProcessed: 0,
|
|
3734
|
+
eventsAggregated: 0,
|
|
3735
|
+
bucketsQueued: 0,
|
|
3736
|
+
};
|
|
3733
3737
|
}
|
|
3734
3738
|
} else {
|
|
3735
3739
|
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
@@ -4212,7 +4216,11 @@ async function parseHermesIncremental({ hermesPath, dbPath, cursors, queuePath,
|
|
|
4212
4216
|
updatedAt, // Update the overall profile state timestamp even if the DB doesn't exist for the fast-path check
|
|
4213
4217
|
};
|
|
4214
4218
|
|
|
4215
|
-
return {
|
|
4219
|
+
return {
|
|
4220
|
+
recordsProcessed,
|
|
4221
|
+
eventsAggregated,
|
|
4222
|
+
bucketsQueued,
|
|
4223
|
+
};
|
|
4216
4224
|
}
|
|
4217
4225
|
|
|
4218
4226
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -4993,7 +5001,11 @@ async function parseKiroCliFromSessionFiles({
|
|
|
4993
5001
|
fileOffsets,
|
|
4994
5002
|
updatedAt: new Date().toISOString(),
|
|
4995
5003
|
};
|
|
4996
|
-
return {
|
|
5004
|
+
return {
|
|
5005
|
+
recordsProcessed: 0,
|
|
5006
|
+
eventsAggregated: 0,
|
|
5007
|
+
bucketsQueued: 0,
|
|
5008
|
+
};
|
|
4997
5009
|
}
|
|
4998
5010
|
|
|
4999
5011
|
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
@@ -5124,7 +5136,11 @@ async function parseKiroCliFromSessionFiles({
|
|
|
5124
5136
|
cursors.hourly = hourlyState;
|
|
5125
5137
|
cursors.kiroCli = { ...kiroCliState, seenIds: cappedSeen, fileOffsets, updatedAt };
|
|
5126
5138
|
|
|
5127
|
-
return {
|
|
5139
|
+
return {
|
|
5140
|
+
recordsProcessed,
|
|
5141
|
+
eventsAggregated,
|
|
5142
|
+
bucketsQueued,
|
|
5143
|
+
};
|
|
5128
5144
|
}
|
|
5129
5145
|
|
|
5130
5146
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -9264,6 +9280,46 @@ function pickCopilotModel(attrs) {
|
|
|
9264
9280
|
}
|
|
9265
9281
|
|
|
9266
9282
|
const COPILOT_PARSER_VERSION = 2;
|
|
9283
|
+
const COPILOT_USAGE_CLAIM_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
9284
|
+
const COPILOT_USAGE_CLAIM_MAX_ENTRIES = 10_000;
|
|
9285
|
+
|
|
9286
|
+
function pruneCopilotUsageClaims(
|
|
9287
|
+
events,
|
|
9288
|
+
nowMs = Date.now(),
|
|
9289
|
+
maxEntries = COPILOT_USAGE_CLAIM_MAX_ENTRIES,
|
|
9290
|
+
) {
|
|
9291
|
+
const cutoffMs = nowMs - COPILOT_USAGE_CLAIM_RETENTION_MS;
|
|
9292
|
+
const retained = [];
|
|
9293
|
+
for (const event of Array.isArray(events) ? events : []) {
|
|
9294
|
+
if (!event || typeof event !== "object" || event.consumed === true) {
|
|
9295
|
+
continue;
|
|
9296
|
+
}
|
|
9297
|
+
const storedFirstSeenAtMs = Number(event.firstSeenAtMs);
|
|
9298
|
+
const firstSeenAtMs =
|
|
9299
|
+
Number.isFinite(storedFirstSeenAtMs) && storedFirstSeenAtMs > 0
|
|
9300
|
+
? storedFirstSeenAtMs
|
|
9301
|
+
: nowMs;
|
|
9302
|
+
if (firstSeenAtMs < cutoffMs) continue;
|
|
9303
|
+
retained.push(
|
|
9304
|
+
firstSeenAtMs === storedFirstSeenAtMs
|
|
9305
|
+
? event
|
|
9306
|
+
: { ...event, firstSeenAtMs },
|
|
9307
|
+
);
|
|
9308
|
+
}
|
|
9309
|
+
return retained.length > maxEntries
|
|
9310
|
+
? retained.slice(retained.length - maxEntries)
|
|
9311
|
+
: retained;
|
|
9312
|
+
}
|
|
9313
|
+
|
|
9314
|
+
function replaceCopilotUsageClaims(
|
|
9315
|
+
target,
|
|
9316
|
+
nowMs = Date.now(),
|
|
9317
|
+
maxEntries = COPILOT_USAGE_CLAIM_MAX_ENTRIES,
|
|
9318
|
+
) {
|
|
9319
|
+
if (!Array.isArray(target)) return;
|
|
9320
|
+
const retained = pruneCopilotUsageClaims(target, nowMs, maxEntries);
|
|
9321
|
+
target.splice(0, target.length, ...retained);
|
|
9322
|
+
}
|
|
9267
9323
|
|
|
9268
9324
|
function isCopilotV1ChatSpan(record) {
|
|
9269
9325
|
if (!record || record.type !== "span") return false;
|
|
@@ -9288,6 +9344,65 @@ function getCopilotDedupKey(record, attrs = record?.attributes || {}) {
|
|
|
9288
9344
|
return traceId && spanId ? `${traceId}:${spanId}` : responseId ? `resp:${responseId}` : null;
|
|
9289
9345
|
}
|
|
9290
9346
|
|
|
9347
|
+
function copilotUsageMatchKey({
|
|
9348
|
+
sessionId,
|
|
9349
|
+
model,
|
|
9350
|
+
input,
|
|
9351
|
+
output,
|
|
9352
|
+
cacheRead,
|
|
9353
|
+
cacheWrite,
|
|
9354
|
+
reasoning,
|
|
9355
|
+
}) {
|
|
9356
|
+
return [
|
|
9357
|
+
sessionId || "",
|
|
9358
|
+
normalizeCopilotAppModel(model) || COPILOT_APP_DEFAULT_MODEL,
|
|
9359
|
+
toNonNegativeInt(input),
|
|
9360
|
+
toNonNegativeInt(output),
|
|
9361
|
+
toNonNegativeInt(cacheRead),
|
|
9362
|
+
toNonNegativeInt(cacheWrite),
|
|
9363
|
+
toNonNegativeInt(reasoning),
|
|
9364
|
+
].join("|");
|
|
9365
|
+
}
|
|
9366
|
+
|
|
9367
|
+
function createCopilotStoreUsageMatcher(events) {
|
|
9368
|
+
const candidates = new Map();
|
|
9369
|
+
for (const event of Array.isArray(events) ? events : []) {
|
|
9370
|
+
if (!event || typeof event !== "object") continue;
|
|
9371
|
+
if (event.consumed === true) continue;
|
|
9372
|
+
const tsMs = Number(event.tsMs);
|
|
9373
|
+
if (!Number.isFinite(tsMs) || tsMs <= 0) continue;
|
|
9374
|
+
const key = copilotUsageMatchKey(event);
|
|
9375
|
+
const list = candidates.get(key) || [];
|
|
9376
|
+
list.push({ tsMs, event });
|
|
9377
|
+
candidates.set(key, list);
|
|
9378
|
+
}
|
|
9379
|
+
for (const list of candidates.values()) {
|
|
9380
|
+
list.sort((a, b) => a.tsMs - b.tsMs);
|
|
9381
|
+
}
|
|
9382
|
+
return {
|
|
9383
|
+
consume(event, toleranceMs = 2_000) {
|
|
9384
|
+
const tsMs = Number(event?.tsMs);
|
|
9385
|
+
if (!Number.isFinite(tsMs) || tsMs <= 0) return false;
|
|
9386
|
+
const key = copilotUsageMatchKey(event || {});
|
|
9387
|
+
const list = candidates.get(key);
|
|
9388
|
+
if (!list || list.length === 0) return false;
|
|
9389
|
+
let bestIndex = -1;
|
|
9390
|
+
let bestDistance = Infinity;
|
|
9391
|
+
for (let i = 0; i < list.length; i++) {
|
|
9392
|
+
const distance = Math.abs(list[i].tsMs - tsMs);
|
|
9393
|
+
if (distance <= toleranceMs && distance < bestDistance) {
|
|
9394
|
+
bestIndex = i;
|
|
9395
|
+
bestDistance = distance;
|
|
9396
|
+
}
|
|
9397
|
+
}
|
|
9398
|
+
if (bestIndex < 0) return false;
|
|
9399
|
+
const [matched] = list.splice(bestIndex, 1);
|
|
9400
|
+
matched.event.consumed = true;
|
|
9401
|
+
return true;
|
|
9402
|
+
},
|
|
9403
|
+
};
|
|
9404
|
+
}
|
|
9405
|
+
|
|
9291
9406
|
// Migration helper: stream the bytes v1 already saw (0 -> prevSize), classify
|
|
9292
9407
|
// whether the file contains old CLI spans v1 processed, and whether it also
|
|
9293
9408
|
// contains v2-only chat records v1 skipped. Mixed files must be replayed, but
|
|
@@ -9334,7 +9449,54 @@ async function scanCopilotV1MigrationFile(filePath, maxBytes) {
|
|
|
9334
9449
|
return result;
|
|
9335
9450
|
}
|
|
9336
9451
|
|
|
9337
|
-
async function
|
|
9452
|
+
async function copilotOtelCursorHasLegacyCliUsage(cursors) {
|
|
9453
|
+
const state =
|
|
9454
|
+
cursors?.copilot && typeof cursors.copilot === "object"
|
|
9455
|
+
? cursors.copilot
|
|
9456
|
+
: {};
|
|
9457
|
+
if (state.legacyCliHistory === true) return true;
|
|
9458
|
+
if (state.usageClaimsComplete === true) return false;
|
|
9459
|
+
const offsets =
|
|
9460
|
+
state.fileOffsets && typeof state.fileOffsets === "object"
|
|
9461
|
+
? state.fileOffsets
|
|
9462
|
+
: {};
|
|
9463
|
+
const entries = Object.entries(offsets).filter(
|
|
9464
|
+
([, entry]) => Number(entry?.size) > 0,
|
|
9465
|
+
);
|
|
9466
|
+
const hasSeenIds =
|
|
9467
|
+
Array.isArray(state.seenIds) && state.seenIds.length > 0;
|
|
9468
|
+
if (entries.length === 0) return hasSeenIds;
|
|
9469
|
+
for (const [filePath, entry] of entries) {
|
|
9470
|
+
let stat;
|
|
9471
|
+
try {
|
|
9472
|
+
stat = fssync.statSync(filePath);
|
|
9473
|
+
} catch (_e) {
|
|
9474
|
+
return true;
|
|
9475
|
+
}
|
|
9476
|
+
if (
|
|
9477
|
+
stat.size < Number(entry?.size) ||
|
|
9478
|
+
(typeof entry?.ino === "number" && stat.ino !== entry.ino)
|
|
9479
|
+
) {
|
|
9480
|
+
return true;
|
|
9481
|
+
}
|
|
9482
|
+
const scan = await scanCopilotV1MigrationFile(
|
|
9483
|
+
filePath,
|
|
9484
|
+
Number(entry?.size) || 0,
|
|
9485
|
+
);
|
|
9486
|
+
if (scan.v1Processed) return true;
|
|
9487
|
+
}
|
|
9488
|
+
return false;
|
|
9489
|
+
}
|
|
9490
|
+
|
|
9491
|
+
async function parseCopilotIncremental({
|
|
9492
|
+
otelPaths,
|
|
9493
|
+
cursors,
|
|
9494
|
+
queuePath,
|
|
9495
|
+
onProgress,
|
|
9496
|
+
env,
|
|
9497
|
+
skipCliSpans = false,
|
|
9498
|
+
storeUsageEvents,
|
|
9499
|
+
} = {}) {
|
|
9338
9500
|
await ensureDir(path.dirname(queuePath));
|
|
9339
9501
|
const copilotState = cursors.copilot && typeof cursors.copilot === "object" ? cursors.copilot : {};
|
|
9340
9502
|
const seenIds = new Set(Array.isArray(copilotState.seenIds) ? copilotState.seenIds : []);
|
|
@@ -9344,6 +9506,16 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9344
9506
|
? copilotState.fileOffsets
|
|
9345
9507
|
: {};
|
|
9346
9508
|
const fileOffsets = { ...fileOffsetsRaw };
|
|
9509
|
+
const claimNowMs = Date.now();
|
|
9510
|
+
const recentOtelUsageEvents = pruneCopilotUsageClaims(
|
|
9511
|
+
copilotState.recentUsageEvents,
|
|
9512
|
+
claimNowMs,
|
|
9513
|
+
);
|
|
9514
|
+
replaceCopilotUsageClaims(storeUsageEvents, claimNowMs, Infinity);
|
|
9515
|
+
const hadPriorUsageHistory =
|
|
9516
|
+
seenIds.size > 0 || Object.keys(fileOffsetsRaw).length > 0;
|
|
9517
|
+
let usageClaimsComplete =
|
|
9518
|
+
copilotState.usageClaimsComplete === true || !hadPriorUsageHistory;
|
|
9347
9519
|
const migrationSkipLineHashes = new Map();
|
|
9348
9520
|
// One-shot v1->v2 migration:
|
|
9349
9521
|
// - pure v2-only files: clear offset and re-read all skipped Chat records
|
|
@@ -9371,9 +9543,16 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9371
9543
|
version: COPILOT_PARSER_VERSION,
|
|
9372
9544
|
seenIds: Array.from(seenIds),
|
|
9373
9545
|
fileOffsets,
|
|
9546
|
+
recentUsageEvents: recentOtelUsageEvents,
|
|
9547
|
+
usageClaimsComplete,
|
|
9374
9548
|
updatedAt: new Date().toISOString(),
|
|
9375
9549
|
};
|
|
9376
|
-
return {
|
|
9550
|
+
return {
|
|
9551
|
+
recordsProcessed: 0,
|
|
9552
|
+
eventsAggregated: 0,
|
|
9553
|
+
bucketsQueued: 0,
|
|
9554
|
+
usageClaims: recentOtelUsageEvents,
|
|
9555
|
+
};
|
|
9377
9556
|
}
|
|
9378
9557
|
|
|
9379
9558
|
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
@@ -9381,6 +9560,7 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9381
9560
|
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
9382
9561
|
let recordsProcessed = 0;
|
|
9383
9562
|
let eventsAggregated = 0;
|
|
9563
|
+
const storeUsageMatcher = createCopilotStoreUsageMatcher(storeUsageEvents);
|
|
9384
9564
|
|
|
9385
9565
|
for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
|
|
9386
9566
|
const filePath = files[fileIdx];
|
|
@@ -9436,6 +9616,10 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9436
9616
|
// gen_ai.response.id is per-LLM-call unique.
|
|
9437
9617
|
const dedupKey = getCopilotDedupKey(record, attrs);
|
|
9438
9618
|
if (dedupKey && seenIds.has(dedupKey)) continue;
|
|
9619
|
+
if (skipCliSpans && isCopilotV1ChatSpan(record)) {
|
|
9620
|
+
if (dedupKey) seenIds.add(dedupKey);
|
|
9621
|
+
continue;
|
|
9622
|
+
}
|
|
9439
9623
|
|
|
9440
9624
|
const inputRaw = toNonNegativeInt(attrs["gen_ai.usage.input_tokens"]);
|
|
9441
9625
|
const output = toNonNegativeInt(attrs["gen_ai.usage.output_tokens"]);
|
|
@@ -9495,6 +9679,30 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9495
9679
|
const model =
|
|
9496
9680
|
normalizeCopilotAppModel(pickCopilotModel(attrs)) ||
|
|
9497
9681
|
COPILOT_APP_DEFAULT_MODEL;
|
|
9682
|
+
const cliSessionId =
|
|
9683
|
+
typeof attrs["gen_ai.conversation.id"] === "string"
|
|
9684
|
+
? attrs["gen_ai.conversation.id"].trim()
|
|
9685
|
+
: "";
|
|
9686
|
+
if (cliSpan && !cliSessionId) usageClaimsComplete = false;
|
|
9687
|
+
const matchBase = {
|
|
9688
|
+
sessionId: cliSessionId,
|
|
9689
|
+
model,
|
|
9690
|
+
output: outputWithoutReasoning,
|
|
9691
|
+
cacheRead: cacheReadClamped,
|
|
9692
|
+
cacheWrite: cacheWriteClamped,
|
|
9693
|
+
reasoning: reasoningClamped,
|
|
9694
|
+
tsMs,
|
|
9695
|
+
};
|
|
9696
|
+
const matchedStoreUsage =
|
|
9697
|
+
isCopilotV1ChatSpan(record) &&
|
|
9698
|
+
storeUsageMatcher.consume({
|
|
9699
|
+
...matchBase,
|
|
9700
|
+
input,
|
|
9701
|
+
});
|
|
9702
|
+
if (matchedStoreUsage) {
|
|
9703
|
+
if (dedupKey) seenIds.add(dedupKey);
|
|
9704
|
+
continue;
|
|
9705
|
+
}
|
|
9498
9706
|
|
|
9499
9707
|
const delta = {
|
|
9500
9708
|
input_tokens: input,
|
|
@@ -9515,6 +9723,13 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9515
9723
|
addTotals(bucket.totals, delta);
|
|
9516
9724
|
touchedBuckets.add(bucketKey("copilot", model, bucketStart));
|
|
9517
9725
|
eventsAggregated++;
|
|
9726
|
+
if (cliSpan && cliSessionId) {
|
|
9727
|
+
recentOtelUsageEvents.push({
|
|
9728
|
+
...matchBase,
|
|
9729
|
+
input,
|
|
9730
|
+
firstSeenAtMs: claimNowMs,
|
|
9731
|
+
});
|
|
9732
|
+
}
|
|
9518
9733
|
if (dedupKey) seenIds.add(dedupKey);
|
|
9519
9734
|
|
|
9520
9735
|
if (cb) {
|
|
@@ -9542,6 +9757,10 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9542
9757
|
// Cap dedup set to last 10k IDs to bound state size
|
|
9543
9758
|
const seenArr = Array.from(seenIds);
|
|
9544
9759
|
const cappedSeen = seenArr.length > 10_000 ? seenArr.slice(seenArr.length - 10_000) : seenArr;
|
|
9760
|
+
const retainedRecentOtelUsageEvents = pruneCopilotUsageClaims(
|
|
9761
|
+
recentOtelUsageEvents,
|
|
9762
|
+
);
|
|
9763
|
+
replaceCopilotUsageClaims(storeUsageEvents, Date.now(), Infinity);
|
|
9545
9764
|
|
|
9546
9765
|
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
9547
9766
|
const updatedAt = new Date().toISOString();
|
|
@@ -9552,10 +9771,914 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9552
9771
|
version: COPILOT_PARSER_VERSION,
|
|
9553
9772
|
seenIds: cappedSeen,
|
|
9554
9773
|
fileOffsets,
|
|
9774
|
+
recentUsageEvents: retainedRecentOtelUsageEvents,
|
|
9775
|
+
usageClaimsComplete,
|
|
9555
9776
|
updatedAt,
|
|
9556
9777
|
};
|
|
9557
9778
|
|
|
9558
|
-
return {
|
|
9779
|
+
return {
|
|
9780
|
+
recordsProcessed,
|
|
9781
|
+
eventsAggregated,
|
|
9782
|
+
bucketsQueued,
|
|
9783
|
+
usageClaims: recentOtelUsageEvents,
|
|
9784
|
+
};
|
|
9785
|
+
}
|
|
9786
|
+
|
|
9787
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
9788
|
+
// GitHub Copilot local runtime — session-store.db assistant_usage_events
|
|
9789
|
+
//
|
|
9790
|
+
// Copilot CLI 1.0.70+ persists one row per LLM request here. The Copilot App
|
|
9791
|
+
// uses the same runtime, so this is the only local source that remains complete
|
|
9792
|
+
// when one session moves App -> CLI -> App (or the reverse). data.db only keeps
|
|
9793
|
+
// the App-owned portion of those sessions.
|
|
9794
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
9795
|
+
|
|
9796
|
+
const COPILOT_STORE_CURSOR_VERSION = 2;
|
|
9797
|
+
const COPILOT_STORE_LEGACY_MODEL = "github-copilot-legacy";
|
|
9798
|
+
|
|
9799
|
+
function resolveCopilotDbPaths({ fileName, overrideEnvKey }, env = process.env) {
|
|
9800
|
+
const home =
|
|
9801
|
+
process.platform === "win32"
|
|
9802
|
+
? env.USERPROFILE || env.HOME || require("node:os").homedir()
|
|
9803
|
+
: env.HOME || require("node:os").homedir();
|
|
9804
|
+
const paths = new Set();
|
|
9805
|
+
const addDbPath = (dbPath) => {
|
|
9806
|
+
const normalized = normalizeCopilotDbPath(dbPath, env);
|
|
9807
|
+
if (normalized) paths.add(normalized);
|
|
9808
|
+
};
|
|
9809
|
+
const addHome = (homePath) => {
|
|
9810
|
+
const normalizedHome = normalizeCopilotDbPath(homePath, env);
|
|
9811
|
+
if (normalizedHome) addDbPath(path.join(normalizedHome, fileName));
|
|
9812
|
+
};
|
|
9813
|
+
|
|
9814
|
+
if (typeof env[overrideEnvKey] === "string" && env[overrideEnvKey].trim()) {
|
|
9815
|
+
addDbPath(env[overrideEnvKey]);
|
|
9816
|
+
}
|
|
9817
|
+
if (typeof env.COPILOT_HOME === "string" && env.COPILOT_HOME.trim()) {
|
|
9818
|
+
addHome(env.COPILOT_HOME);
|
|
9819
|
+
}
|
|
9820
|
+
|
|
9821
|
+
addDbPath(path.join(home, ".copilot", fileName));
|
|
9822
|
+
|
|
9823
|
+
return Array.from(paths).sort();
|
|
9824
|
+
}
|
|
9825
|
+
|
|
9826
|
+
function resolveCopilotSessionStorePaths(env = process.env) {
|
|
9827
|
+
return resolveCopilotDbPaths(
|
|
9828
|
+
{
|
|
9829
|
+
fileName: "session-store.db",
|
|
9830
|
+
overrideEnvKey: "TOKENTRACKER_COPILOT_SESSION_STORE_DB",
|
|
9831
|
+
},
|
|
9832
|
+
env,
|
|
9833
|
+
);
|
|
9834
|
+
}
|
|
9835
|
+
|
|
9836
|
+
function copilotStoreRowKey(row) {
|
|
9837
|
+
if (!row) return null;
|
|
9838
|
+
return crypto
|
|
9839
|
+
.createHash("sha256")
|
|
9840
|
+
.update(
|
|
9841
|
+
JSON.stringify([
|
|
9842
|
+
row.max_id ?? row.id,
|
|
9843
|
+
row.session_id,
|
|
9844
|
+
row.model,
|
|
9845
|
+
row.input_tokens,
|
|
9846
|
+
row.output_tokens,
|
|
9847
|
+
row.cache_read_tokens,
|
|
9848
|
+
row.cache_write_tokens,
|
|
9849
|
+
row.reasoning_tokens,
|
|
9850
|
+
row.created_at,
|
|
9851
|
+
]),
|
|
9852
|
+
)
|
|
9853
|
+
.digest("hex");
|
|
9854
|
+
}
|
|
9855
|
+
|
|
9856
|
+
function copilotStoreEventFingerprint(row) {
|
|
9857
|
+
const sessionId =
|
|
9858
|
+
typeof row?.session_id === "string" ? row.session_id.trim() : "";
|
|
9859
|
+
if (!sessionId) return null;
|
|
9860
|
+
const normalized = normalizeCopilotSessionStoreUsage(row);
|
|
9861
|
+
const timestamp =
|
|
9862
|
+
parseCopilotAppTimestamp(row?.created_at) ||
|
|
9863
|
+
`invalid:${String(row?.created_at ?? "")}`;
|
|
9864
|
+
return crypto
|
|
9865
|
+
.createHash("sha256")
|
|
9866
|
+
.update(
|
|
9867
|
+
JSON.stringify([
|
|
9868
|
+
sessionId,
|
|
9869
|
+
normalizeCopilotAppModel(row?.model) || COPILOT_APP_DEFAULT_MODEL,
|
|
9870
|
+
normalized.input_tokens,
|
|
9871
|
+
normalized.output_tokens,
|
|
9872
|
+
normalized.cached_input_tokens,
|
|
9873
|
+
normalized.cache_creation_input_tokens,
|
|
9874
|
+
normalized.reasoning_output_tokens,
|
|
9875
|
+
timestamp,
|
|
9876
|
+
]),
|
|
9877
|
+
)
|
|
9878
|
+
.digest("hex");
|
|
9879
|
+
}
|
|
9880
|
+
|
|
9881
|
+
function normalizeCopilotFingerprintCounts(value) {
|
|
9882
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
9883
|
+
const result = {};
|
|
9884
|
+
for (const [fingerprint, count] of Object.entries(value)) {
|
|
9885
|
+
const normalized = toNonNegativeInt(count);
|
|
9886
|
+
if (/^[a-f0-9]{64}$/.test(fingerprint) && normalized > 0) {
|
|
9887
|
+
result[fingerprint] = normalized;
|
|
9888
|
+
}
|
|
9889
|
+
}
|
|
9890
|
+
return result;
|
|
9891
|
+
}
|
|
9892
|
+
|
|
9893
|
+
function normalizeCopilotPendingMalformedEvents(value) {
|
|
9894
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
9895
|
+
const result = {};
|
|
9896
|
+
for (const [id, fingerprint] of Object.entries(value)) {
|
|
9897
|
+
const normalizedId = toNonNegativeInt(id);
|
|
9898
|
+
if (normalizedId > 0) {
|
|
9899
|
+
result[normalizedId] =
|
|
9900
|
+
typeof fingerprint === "string" ? fingerprint : "";
|
|
9901
|
+
}
|
|
9902
|
+
}
|
|
9903
|
+
return result;
|
|
9904
|
+
}
|
|
9905
|
+
|
|
9906
|
+
function incrementCopilotFingerprintCount(counts, fingerprint) {
|
|
9907
|
+
if (!fingerprint) return;
|
|
9908
|
+
counts[fingerprint] = toNonNegativeInt(counts[fingerprint]) + 1;
|
|
9909
|
+
}
|
|
9910
|
+
|
|
9911
|
+
function latestCopilotTimestamp(first, second) {
|
|
9912
|
+
const values = [first, second]
|
|
9913
|
+
.map((value) => parseCopilotAppTimestamp(value))
|
|
9914
|
+
.filter(Boolean);
|
|
9915
|
+
if (values.length === 0) return null;
|
|
9916
|
+
return values.reduce((latest, value) => (value > latest ? value : latest));
|
|
9917
|
+
}
|
|
9918
|
+
|
|
9919
|
+
function omitCopilotLegacyCatchupState(state) {
|
|
9920
|
+
const clean = { ...(state || {}) };
|
|
9921
|
+
delete clean.pendingCatchupMaxId;
|
|
9922
|
+
delete clean.pendingCatchupFingerprint;
|
|
9923
|
+
delete clean.pendingLegacyCatchup;
|
|
9924
|
+
return clean;
|
|
9925
|
+
}
|
|
9926
|
+
|
|
9927
|
+
function readCopilotSessionStoreMetadata(
|
|
9928
|
+
dbPath,
|
|
9929
|
+
sqliteOptions = {},
|
|
9930
|
+
cursorId = 0,
|
|
9931
|
+
) {
|
|
9932
|
+
const tables = readSqliteJsonRows(
|
|
9933
|
+
dbPath,
|
|
9934
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('assistant_usage_events', 'schema_version')",
|
|
9935
|
+
{
|
|
9936
|
+
label: "GitHub Copilot session store",
|
|
9937
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
9938
|
+
timeout: 10_000,
|
|
9939
|
+
readOnly: true,
|
|
9940
|
+
throwOnReadFailure: true,
|
|
9941
|
+
...sqliteOptions,
|
|
9942
|
+
},
|
|
9943
|
+
);
|
|
9944
|
+
const names = new Set(tables.map((row) => row?.name).filter(Boolean));
|
|
9945
|
+
if (!names.has("assistant_usage_events")) {
|
|
9946
|
+
return { active: false, schemaVersion: null, maxId: 0 };
|
|
9947
|
+
}
|
|
9948
|
+
const maxRows = readSqliteJsonRows(
|
|
9949
|
+
dbPath,
|
|
9950
|
+
`
|
|
9951
|
+
SELECT
|
|
9952
|
+
id AS max_id,
|
|
9953
|
+
session_id,
|
|
9954
|
+
model,
|
|
9955
|
+
input_tokens,
|
|
9956
|
+
output_tokens,
|
|
9957
|
+
cache_read_tokens,
|
|
9958
|
+
cache_write_tokens,
|
|
9959
|
+
reasoning_tokens,
|
|
9960
|
+
created_at
|
|
9961
|
+
FROM assistant_usage_events
|
|
9962
|
+
ORDER BY id DESC
|
|
9963
|
+
LIMIT 1
|
|
9964
|
+
`.trim(),
|
|
9965
|
+
{
|
|
9966
|
+
label: "GitHub Copilot session store",
|
|
9967
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
9968
|
+
timeout: 10_000,
|
|
9969
|
+
readOnly: true,
|
|
9970
|
+
throwOnReadFailure: true,
|
|
9971
|
+
...sqliteOptions,
|
|
9972
|
+
},
|
|
9973
|
+
);
|
|
9974
|
+
let schemaVersion = null;
|
|
9975
|
+
if (names.has("schema_version")) {
|
|
9976
|
+
const versionRows = readSqliteJsonRows(
|
|
9977
|
+
dbPath,
|
|
9978
|
+
"SELECT version FROM schema_version LIMIT 1",
|
|
9979
|
+
{
|
|
9980
|
+
label: "GitHub Copilot session store",
|
|
9981
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
9982
|
+
timeout: 10_000,
|
|
9983
|
+
readOnly: true,
|
|
9984
|
+
throwOnReadFailure: true,
|
|
9985
|
+
...sqliteOptions,
|
|
9986
|
+
},
|
|
9987
|
+
);
|
|
9988
|
+
schemaVersion = toNonNegativeInt(versionRows[0]?.version);
|
|
9989
|
+
}
|
|
9990
|
+
let cursorRowKey = null;
|
|
9991
|
+
const normalizedCursorId = toNonNegativeInt(cursorId);
|
|
9992
|
+
if (normalizedCursorId > 0) {
|
|
9993
|
+
if (normalizedCursorId === toNonNegativeInt(maxRows[0]?.max_id)) {
|
|
9994
|
+
cursorRowKey = copilotStoreRowKey(maxRows[0]);
|
|
9995
|
+
} else {
|
|
9996
|
+
const cursorRows = readSqliteJsonRows(
|
|
9997
|
+
dbPath,
|
|
9998
|
+
`
|
|
9999
|
+
SELECT
|
|
10000
|
+
id,
|
|
10001
|
+
session_id,
|
|
10002
|
+
model,
|
|
10003
|
+
input_tokens,
|
|
10004
|
+
output_tokens,
|
|
10005
|
+
cache_read_tokens,
|
|
10006
|
+
cache_write_tokens,
|
|
10007
|
+
reasoning_tokens,
|
|
10008
|
+
created_at
|
|
10009
|
+
FROM assistant_usage_events
|
|
10010
|
+
WHERE id = ${normalizedCursorId}
|
|
10011
|
+
LIMIT 1
|
|
10012
|
+
`.trim(),
|
|
10013
|
+
{
|
|
10014
|
+
label: "GitHub Copilot session store",
|
|
10015
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
10016
|
+
timeout: 10_000,
|
|
10017
|
+
readOnly: true,
|
|
10018
|
+
throwOnReadFailure: true,
|
|
10019
|
+
...sqliteOptions,
|
|
10020
|
+
},
|
|
10021
|
+
);
|
|
10022
|
+
cursorRowKey = copilotStoreRowKey(cursorRows[0]);
|
|
10023
|
+
}
|
|
10024
|
+
}
|
|
10025
|
+
return {
|
|
10026
|
+
active: true,
|
|
10027
|
+
schemaVersion,
|
|
10028
|
+
maxId: toNonNegativeInt(maxRows[0]?.max_id),
|
|
10029
|
+
lastRowKey: copilotStoreRowKey(maxRows[0]),
|
|
10030
|
+
lastEventAt: parseCopilotAppTimestamp(maxRows[0]?.created_at),
|
|
10031
|
+
cursorRowKey,
|
|
10032
|
+
};
|
|
10033
|
+
}
|
|
10034
|
+
|
|
10035
|
+
function describeCopilotSessionStoreDb(dbPath, sqliteOptions = {}) {
|
|
10036
|
+
let snap = null;
|
|
10037
|
+
try {
|
|
10038
|
+
snap = snapshotSqliteDb(dbPath);
|
|
10039
|
+
const metadata = readCopilotSessionStoreMetadata(snap.path, sqliteOptions);
|
|
10040
|
+
if (!metadata.active) {
|
|
10041
|
+
return {
|
|
10042
|
+
path: dbPath,
|
|
10043
|
+
active: false,
|
|
10044
|
+
schemaVersion: metadata.schemaVersion,
|
|
10045
|
+
eventCount: 0,
|
|
10046
|
+
lastEventId: 0,
|
|
10047
|
+
lastEventAt: null,
|
|
10048
|
+
};
|
|
10049
|
+
}
|
|
10050
|
+
const countRows = readSqliteJsonRows(
|
|
10051
|
+
snap.path,
|
|
10052
|
+
"SELECT COUNT(*) AS event_count FROM assistant_usage_events",
|
|
10053
|
+
{
|
|
10054
|
+
label: "GitHub Copilot session store",
|
|
10055
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
10056
|
+
timeout: 10_000,
|
|
10057
|
+
readOnly: true,
|
|
10058
|
+
throwOnReadFailure: true,
|
|
10059
|
+
...sqliteOptions,
|
|
10060
|
+
},
|
|
10061
|
+
);
|
|
10062
|
+
return {
|
|
10063
|
+
path: dbPath,
|
|
10064
|
+
active: true,
|
|
10065
|
+
schemaVersion: metadata.schemaVersion,
|
|
10066
|
+
eventCount: toNonNegativeInt(countRows[0]?.event_count),
|
|
10067
|
+
lastEventId: metadata.maxId,
|
|
10068
|
+
lastEventAt: metadata.lastEventAt,
|
|
10069
|
+
};
|
|
10070
|
+
} finally {
|
|
10071
|
+
if (snap) snap.cleanup();
|
|
10072
|
+
}
|
|
10073
|
+
}
|
|
10074
|
+
|
|
10075
|
+
function readCopilotSessionStoreUsageRowsWhere(
|
|
10076
|
+
dbPath,
|
|
10077
|
+
whereClause,
|
|
10078
|
+
sqliteOptions = {},
|
|
10079
|
+
) {
|
|
10080
|
+
return readSqliteJsonRows(
|
|
10081
|
+
dbPath,
|
|
10082
|
+
`
|
|
10083
|
+
SELECT
|
|
10084
|
+
id,
|
|
10085
|
+
session_id,
|
|
10086
|
+
model,
|
|
10087
|
+
input_tokens,
|
|
10088
|
+
output_tokens,
|
|
10089
|
+
cache_read_tokens,
|
|
10090
|
+
cache_write_tokens,
|
|
10091
|
+
reasoning_tokens,
|
|
10092
|
+
token_details_json,
|
|
10093
|
+
created_at
|
|
10094
|
+
FROM assistant_usage_events
|
|
10095
|
+
WHERE ${whereClause}
|
|
10096
|
+
ORDER BY id ASC
|
|
10097
|
+
`.trim(),
|
|
10098
|
+
{
|
|
10099
|
+
label: "GitHub Copilot session store",
|
|
10100
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
10101
|
+
timeout: 60_000,
|
|
10102
|
+
readOnly: true,
|
|
10103
|
+
throwOnReadFailure: true,
|
|
10104
|
+
...sqliteOptions,
|
|
10105
|
+
},
|
|
10106
|
+
);
|
|
10107
|
+
}
|
|
10108
|
+
|
|
10109
|
+
function readCopilotSessionStoreUsageRows(dbPath, lastId, sqliteOptions = {}) {
|
|
10110
|
+
return readCopilotSessionStoreUsageRowsWhere(
|
|
10111
|
+
dbPath,
|
|
10112
|
+
`id > ${toNonNegativeInt(lastId)}`,
|
|
10113
|
+
sqliteOptions,
|
|
10114
|
+
);
|
|
10115
|
+
}
|
|
10116
|
+
|
|
10117
|
+
function readCopilotSessionStoreUsageRowsByIds(
|
|
10118
|
+
dbPath,
|
|
10119
|
+
ids,
|
|
10120
|
+
sqliteOptions = {},
|
|
10121
|
+
) {
|
|
10122
|
+
const normalizedIds = [
|
|
10123
|
+
...new Set(
|
|
10124
|
+
(Array.isArray(ids) ? ids : [])
|
|
10125
|
+
.map(toNonNegativeInt)
|
|
10126
|
+
.filter((id) => id > 0),
|
|
10127
|
+
),
|
|
10128
|
+
];
|
|
10129
|
+
const rows = [];
|
|
10130
|
+
for (let start = 0; start < normalizedIds.length; start += 500) {
|
|
10131
|
+
const chunk = normalizedIds.slice(start, start + 500);
|
|
10132
|
+
rows.push(
|
|
10133
|
+
...readCopilotSessionStoreUsageRowsWhere(
|
|
10134
|
+
dbPath,
|
|
10135
|
+
`id IN (${chunk.join(",")})`,
|
|
10136
|
+
sqliteOptions,
|
|
10137
|
+
),
|
|
10138
|
+
);
|
|
10139
|
+
}
|
|
10140
|
+
return rows.sort((a, b) => toNonNegativeInt(a?.id) - toNonNegativeInt(b?.id));
|
|
10141
|
+
}
|
|
10142
|
+
|
|
10143
|
+
function parseCopilotStoreTokenDetails(raw) {
|
|
10144
|
+
if (typeof raw !== "string" || !raw.trim()) return null;
|
|
10145
|
+
let entries;
|
|
10146
|
+
try {
|
|
10147
|
+
entries = JSON.parse(raw);
|
|
10148
|
+
} catch (_e) {
|
|
10149
|
+
return null;
|
|
10150
|
+
}
|
|
10151
|
+
if (!Array.isArray(entries)) return null;
|
|
10152
|
+
const totals = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
|
|
10153
|
+
let recognized = 0;
|
|
10154
|
+
for (const entry of entries) {
|
|
10155
|
+
if (!entry || typeof entry !== "object") continue;
|
|
10156
|
+
const count = toNonNegativeInt(entry.tokenCount);
|
|
10157
|
+
switch (entry.tokenType) {
|
|
10158
|
+
case "input":
|
|
10159
|
+
totals.input += count;
|
|
10160
|
+
recognized++;
|
|
10161
|
+
break;
|
|
10162
|
+
case "cache_read":
|
|
10163
|
+
totals.cacheRead += count;
|
|
10164
|
+
recognized++;
|
|
10165
|
+
break;
|
|
10166
|
+
case "cache_write":
|
|
10167
|
+
totals.cacheWrite += count;
|
|
10168
|
+
recognized++;
|
|
10169
|
+
break;
|
|
10170
|
+
case "output":
|
|
10171
|
+
totals.output += count;
|
|
10172
|
+
recognized++;
|
|
10173
|
+
break;
|
|
10174
|
+
default:
|
|
10175
|
+
break;
|
|
10176
|
+
}
|
|
10177
|
+
}
|
|
10178
|
+
return recognized > 0 ? totals : null;
|
|
10179
|
+
}
|
|
10180
|
+
|
|
10181
|
+
function copilotStoreRecentEvent(row, firstSeenAtMs = Date.now()) {
|
|
10182
|
+
const tsIso = parseCopilotAppTimestamp(row?.created_at);
|
|
10183
|
+
const tsMs = tsIso ? Date.parse(tsIso) : NaN;
|
|
10184
|
+
const sessionId =
|
|
10185
|
+
typeof row?.session_id === "string" ? row.session_id.trim() : "";
|
|
10186
|
+
if (!sessionId || !Number.isFinite(tsMs) || tsMs <= 0) return null;
|
|
10187
|
+
const normalized = normalizeCopilotSessionStoreUsage(row);
|
|
10188
|
+
return {
|
|
10189
|
+
id: toNonNegativeInt(row?.id),
|
|
10190
|
+
sessionId,
|
|
10191
|
+
model: normalizeCopilotAppModel(row?.model) || COPILOT_APP_DEFAULT_MODEL,
|
|
10192
|
+
input: normalized.input_tokens,
|
|
10193
|
+
output: normalized.output_tokens,
|
|
10194
|
+
cacheRead: normalized.cached_input_tokens,
|
|
10195
|
+
cacheWrite: normalized.cache_creation_input_tokens,
|
|
10196
|
+
reasoning: normalized.reasoning_output_tokens,
|
|
10197
|
+
tsMs,
|
|
10198
|
+
firstSeenAtMs,
|
|
10199
|
+
};
|
|
10200
|
+
}
|
|
10201
|
+
|
|
10202
|
+
function normalizeCopilotSessionStoreUsage(row) {
|
|
10203
|
+
const inputRaw = toNonNegativeInt(row?.input_tokens);
|
|
10204
|
+
const outputRaw = toNonNegativeInt(row?.output_tokens);
|
|
10205
|
+
const reasoning = toNonNegativeInt(row?.reasoning_tokens);
|
|
10206
|
+
const details = parseCopilotStoreTokenDetails(row?.token_details_json);
|
|
10207
|
+
let input;
|
|
10208
|
+
let cacheRead;
|
|
10209
|
+
let cacheWrite;
|
|
10210
|
+
let output;
|
|
10211
|
+
let precision = "fallback";
|
|
10212
|
+
|
|
10213
|
+
if (
|
|
10214
|
+
details &&
|
|
10215
|
+
details.input + details.cacheRead + details.cacheWrite === inputRaw &&
|
|
10216
|
+
details.output === outputRaw
|
|
10217
|
+
) {
|
|
10218
|
+
input = details.input;
|
|
10219
|
+
cacheRead = details.cacheRead;
|
|
10220
|
+
cacheWrite = details.cacheWrite;
|
|
10221
|
+
output = details.output;
|
|
10222
|
+
precision = "exact";
|
|
10223
|
+
} else {
|
|
10224
|
+
cacheRead = Math.min(toNonNegativeInt(row?.cache_read_tokens), inputRaw);
|
|
10225
|
+
cacheWrite = Math.min(
|
|
10226
|
+
toNonNegativeInt(row?.cache_write_tokens),
|
|
10227
|
+
Math.max(0, inputRaw - cacheRead),
|
|
10228
|
+
);
|
|
10229
|
+
input = Math.max(0, inputRaw - cacheRead - cacheWrite);
|
|
10230
|
+
output = outputRaw;
|
|
10231
|
+
}
|
|
10232
|
+
|
|
10233
|
+
const reasoningClamped = Math.min(reasoning, output);
|
|
10234
|
+
const outputWithoutReasoning = Math.max(0, output - reasoningClamped);
|
|
10235
|
+
return {
|
|
10236
|
+
input_tokens: input,
|
|
10237
|
+
cached_input_tokens: cacheRead,
|
|
10238
|
+
cache_creation_input_tokens: cacheWrite,
|
|
10239
|
+
output_tokens: outputWithoutReasoning,
|
|
10240
|
+
reasoning_output_tokens: reasoningClamped,
|
|
10241
|
+
total_tokens:
|
|
10242
|
+
input + cacheRead + cacheWrite + outputWithoutReasoning + reasoningClamped,
|
|
10243
|
+
precision,
|
|
10244
|
+
};
|
|
10245
|
+
}
|
|
10246
|
+
|
|
10247
|
+
async function parseCopilotSessionStoreIncremental({
|
|
10248
|
+
dbPath,
|
|
10249
|
+
dbPaths,
|
|
10250
|
+
cursors,
|
|
10251
|
+
queuePath,
|
|
10252
|
+
onProgress,
|
|
10253
|
+
env,
|
|
10254
|
+
sqliteOptions,
|
|
10255
|
+
backfillOnFirstRun = false,
|
|
10256
|
+
excludeSessionIdsOnFirstRun,
|
|
10257
|
+
excludeSessionTotalsOnFirstRun,
|
|
10258
|
+
expectedFingerprints,
|
|
10259
|
+
otelUsageEvents,
|
|
10260
|
+
} = {}) {
|
|
10261
|
+
await ensureDir(path.dirname(queuePath));
|
|
10262
|
+
const paths = Array.isArray(dbPaths)
|
|
10263
|
+
? dbPaths
|
|
10264
|
+
: dbPath
|
|
10265
|
+
? [dbPath]
|
|
10266
|
+
: resolveCopilotSessionStorePaths(env || process.env);
|
|
10267
|
+
const storeState =
|
|
10268
|
+
cursors.copilotStore && typeof cursors.copilotStore === "object"
|
|
10269
|
+
? cursors.copilotStore
|
|
10270
|
+
: {};
|
|
10271
|
+
const dbStates =
|
|
10272
|
+
storeState.dbs && typeof storeState.dbs === "object" ? { ...storeState.dbs } : {};
|
|
10273
|
+
let uniquePaths = uniqueCopilotDbPaths(
|
|
10274
|
+
paths,
|
|
10275
|
+
env || process.env,
|
|
10276
|
+
Object.keys(dbStates),
|
|
10277
|
+
);
|
|
10278
|
+
const migratedAliasPaths = coalesceCopilotDbStatesByIdentity(
|
|
10279
|
+
dbStates,
|
|
10280
|
+
uniquePaths,
|
|
10281
|
+
mergeCopilotStoreAliasStates,
|
|
10282
|
+
);
|
|
10283
|
+
uniquePaths = uniquePaths.filter(
|
|
10284
|
+
(dbPath) => !migratedAliasPaths.has(dbPath),
|
|
10285
|
+
);
|
|
10286
|
+
const seenSessions = new Set(
|
|
10287
|
+
Array.isArray(storeState.seenSessions) ? storeState.seenSessions : [],
|
|
10288
|
+
);
|
|
10289
|
+
const excludedFirstRunSessions = new Set(
|
|
10290
|
+
Array.isArray(excludeSessionIdsOnFirstRun)
|
|
10291
|
+
? excludeSessionIdsOnFirstRun.filter((value) => typeof value === "string" && value)
|
|
10292
|
+
: [],
|
|
10293
|
+
);
|
|
10294
|
+
const excludedFirstRunTotals =
|
|
10295
|
+
excludeSessionTotalsOnFirstRun &&
|
|
10296
|
+
typeof excludeSessionTotalsOnFirstRun === "object" &&
|
|
10297
|
+
!Array.isArray(excludeSessionTotalsOnFirstRun)
|
|
10298
|
+
? excludeSessionTotalsOnFirstRun
|
|
10299
|
+
: {};
|
|
10300
|
+
const legacyResiduals = new Map();
|
|
10301
|
+
const claimNowMs = Date.now();
|
|
10302
|
+
const recentEvents = pruneCopilotUsageClaims(
|
|
10303
|
+
storeState.recentEvents,
|
|
10304
|
+
claimNowMs,
|
|
10305
|
+
);
|
|
10306
|
+
replaceCopilotUsageClaims(otelUsageEvents, claimNowMs, Infinity);
|
|
10307
|
+
const otelUsageMatcher = createCopilotStoreUsageMatcher(otelUsageEvents);
|
|
10308
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
10309
|
+
const touchedBuckets = new Set();
|
|
10310
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
10311
|
+
const updatedAt = new Date(claimNowMs).toISOString();
|
|
10312
|
+
let activeDbCount = 0;
|
|
10313
|
+
let adoptedThisRun = false;
|
|
10314
|
+
let recordsProcessed = 0;
|
|
10315
|
+
let eventsAggregated = 0;
|
|
10316
|
+
let dbErrors = 0;
|
|
10317
|
+
const canonicalDbPaths = [];
|
|
10318
|
+
const globalAdoptionPending = storeState.active !== true;
|
|
10319
|
+
|
|
10320
|
+
for (const resolvedDb of uniquePaths) {
|
|
10321
|
+
const dbState =
|
|
10322
|
+
dbStates[resolvedDb] && typeof dbStates[resolvedDb] === "object"
|
|
10323
|
+
? dbStates[resolvedDb]
|
|
10324
|
+
: {};
|
|
10325
|
+
const seenEventCounts = normalizeCopilotFingerprintCounts(
|
|
10326
|
+
dbState.seenEventCounts,
|
|
10327
|
+
);
|
|
10328
|
+
let pendingMalformedEvents = normalizeCopilotPendingMalformedEvents(
|
|
10329
|
+
dbState.pendingMalformedEvents,
|
|
10330
|
+
);
|
|
10331
|
+
let fingerprint = null;
|
|
10332
|
+
let snap = null;
|
|
10333
|
+
try {
|
|
10334
|
+
fingerprint = sqliteSidecarFingerprint(resolvedDb);
|
|
10335
|
+
const expectedFingerprint = expectedFingerprints?.[resolvedDb];
|
|
10336
|
+
if (
|
|
10337
|
+
expectedFingerprint &&
|
|
10338
|
+
!sameSqliteFingerprint(fingerprint, expectedFingerprint)
|
|
10339
|
+
) {
|
|
10340
|
+
dbStates[resolvedDb] = {
|
|
10341
|
+
...omitCopilotLegacyCatchupState(dbState),
|
|
10342
|
+
adoptionDeferredAt: updatedAt,
|
|
10343
|
+
updatedAt,
|
|
10344
|
+
};
|
|
10345
|
+
continue;
|
|
10346
|
+
}
|
|
10347
|
+
const currentIno = fingerprint?.db?.ino;
|
|
10348
|
+
const priorLastId = toNonNegativeInt(dbState.lastId);
|
|
10349
|
+
snap = snapshotSqliteDb(resolvedDb);
|
|
10350
|
+
const metadata = readCopilotSessionStoreMetadata(
|
|
10351
|
+
snap.path,
|
|
10352
|
+
sqliteOptions,
|
|
10353
|
+
priorLastId,
|
|
10354
|
+
);
|
|
10355
|
+
if (!metadata.active) {
|
|
10356
|
+
dbStates[resolvedDb] = {
|
|
10357
|
+
...omitCopilotLegacyCatchupState(dbState),
|
|
10358
|
+
schemaVersion: metadata.schemaVersion,
|
|
10359
|
+
lastError: null,
|
|
10360
|
+
lastErrorAt: null,
|
|
10361
|
+
updatedAt,
|
|
10362
|
+
};
|
|
10363
|
+
continue;
|
|
10364
|
+
}
|
|
10365
|
+
|
|
10366
|
+
const inodeChanged =
|
|
10367
|
+
typeof dbState.dbIno === "number" &&
|
|
10368
|
+
typeof currentIno === "number" &&
|
|
10369
|
+
dbState.dbIno !== currentIno;
|
|
10370
|
+
const cursorRowChanged =
|
|
10371
|
+
priorLastId > 0 &&
|
|
10372
|
+
typeof dbState.lastRowKey === "string" &&
|
|
10373
|
+
dbState.lastRowKey !== metadata.cursorRowKey;
|
|
10374
|
+
const idRegressed = metadata.maxId < priorLastId;
|
|
10375
|
+
const resetDetected =
|
|
10376
|
+
Boolean(dbState.adoptedAt) && (idRegressed || cursorRowChanged);
|
|
10377
|
+
const needsFingerprintSeed =
|
|
10378
|
+
Boolean(dbState.adoptedAt) &&
|
|
10379
|
+
priorLastId > 0 &&
|
|
10380
|
+
Object.keys(seenEventCounts).length === 0;
|
|
10381
|
+
const resetWithoutFingerprintHistory =
|
|
10382
|
+
resetDetected && needsFingerprintSeed;
|
|
10383
|
+
const reconcileAllRows =
|
|
10384
|
+
(resetDetected || inodeChanged) && !resetWithoutFingerprintHistory;
|
|
10385
|
+
const needsAdoption =
|
|
10386
|
+
globalAdoptionPending ||
|
|
10387
|
+
!dbState.adoptedAt ||
|
|
10388
|
+
resetDetected;
|
|
10389
|
+
const baselineOnly =
|
|
10390
|
+
resetWithoutFingerprintHistory ||
|
|
10391
|
+
(needsAdoption && !resetDetected && !backfillOnFirstRun);
|
|
10392
|
+
const incrementalRows = readCopilotSessionStoreUsageRows(
|
|
10393
|
+
snap.path,
|
|
10394
|
+
reconcileAllRows || baselineOnly || needsFingerprintSeed ? 0 : priorLastId,
|
|
10395
|
+
sqliteOptions,
|
|
10396
|
+
);
|
|
10397
|
+
const pendingRetryRows = resetDetected || reconcileAllRows
|
|
10398
|
+
? []
|
|
10399
|
+
: readCopilotSessionStoreUsageRowsByIds(
|
|
10400
|
+
snap.path,
|
|
10401
|
+
Object.keys(pendingMalformedEvents),
|
|
10402
|
+
sqliteOptions,
|
|
10403
|
+
);
|
|
10404
|
+
const pendingRetryIds = new Set(
|
|
10405
|
+
pendingRetryRows.map((row) => toNonNegativeInt(row?.id)),
|
|
10406
|
+
);
|
|
10407
|
+
if (resetDetected || reconcileAllRows) {
|
|
10408
|
+
pendingMalformedEvents = {};
|
|
10409
|
+
} else {
|
|
10410
|
+
for (const pendingId of Object.keys(pendingMalformedEvents)) {
|
|
10411
|
+
if (!pendingRetryIds.has(toNonNegativeInt(pendingId))) {
|
|
10412
|
+
delete pendingMalformedEvents[pendingId];
|
|
10413
|
+
}
|
|
10414
|
+
}
|
|
10415
|
+
}
|
|
10416
|
+
const rowsById = new Map();
|
|
10417
|
+
for (const row of [...pendingRetryRows, ...incrementalRows]) {
|
|
10418
|
+
rowsById.set(toNonNegativeInt(row?.id), row);
|
|
10419
|
+
}
|
|
10420
|
+
const rows = Array.from(rowsById.values()).sort(
|
|
10421
|
+
(a, b) => toNonNegativeInt(a?.id) - toNonNegativeInt(b?.id),
|
|
10422
|
+
);
|
|
10423
|
+
// Reindex/reset can rewrite every SQLite id. Subtract the exact metadata
|
|
10424
|
+
// multiset already observed so only genuinely new requests are emitted.
|
|
10425
|
+
const historicalFingerprintCounts = reconcileAllRows
|
|
10426
|
+
? new Map(
|
|
10427
|
+
Object.entries(seenEventCounts).map(([eventFingerprint, count]) => [
|
|
10428
|
+
eventFingerprint,
|
|
10429
|
+
toNonNegativeInt(count),
|
|
10430
|
+
]),
|
|
10431
|
+
)
|
|
10432
|
+
: null;
|
|
10433
|
+
if (needsAdoption) adoptedThisRun = true;
|
|
10434
|
+
if (expectedFingerprint) {
|
|
10435
|
+
const finalFingerprint = sqliteSidecarFingerprint(resolvedDb);
|
|
10436
|
+
if (!sameSqliteFingerprint(finalFingerprint, expectedFingerprint)) {
|
|
10437
|
+
dbStates[resolvedDb] = {
|
|
10438
|
+
...omitCopilotLegacyCatchupState(dbState),
|
|
10439
|
+
adoptionDeferredAt: updatedAt,
|
|
10440
|
+
updatedAt,
|
|
10441
|
+
};
|
|
10442
|
+
continue;
|
|
10443
|
+
}
|
|
10444
|
+
fingerprint = finalFingerprint;
|
|
10445
|
+
}
|
|
10446
|
+
let lastId =
|
|
10447
|
+
reconcileAllRows || baselineOnly ? metadata.maxId : priorLastId;
|
|
10448
|
+
for (const row of rows) {
|
|
10449
|
+
recordsProcessed++;
|
|
10450
|
+
const rowId = toNonNegativeInt(row?.id);
|
|
10451
|
+
lastId = Math.max(lastId, rowId);
|
|
10452
|
+
const sessionId =
|
|
10453
|
+
typeof row?.session_id === "string" ? row.session_id.trim() : "";
|
|
10454
|
+
if (!sessionId) {
|
|
10455
|
+
if (rowId > 0) pendingMalformedEvents[rowId] = "";
|
|
10456
|
+
continue;
|
|
10457
|
+
}
|
|
10458
|
+
const eventFingerprint = copilotStoreEventFingerprint(row);
|
|
10459
|
+
if (historicalFingerprintCounts && eventFingerprint) {
|
|
10460
|
+
const historicalCount = toNonNegativeInt(
|
|
10461
|
+
historicalFingerprintCounts.get(eventFingerprint),
|
|
10462
|
+
);
|
|
10463
|
+
if (historicalCount > 0) {
|
|
10464
|
+
historicalFingerprintCounts.set(
|
|
10465
|
+
eventFingerprint,
|
|
10466
|
+
historicalCount - 1,
|
|
10467
|
+
);
|
|
10468
|
+
if (!parseCopilotAppTimestamp(row?.created_at) && rowId > 0) {
|
|
10469
|
+
pendingMalformedEvents[rowId] = eventFingerprint;
|
|
10470
|
+
}
|
|
10471
|
+
seenSessions.add(sessionId);
|
|
10472
|
+
continue;
|
|
10473
|
+
}
|
|
10474
|
+
}
|
|
10475
|
+
const seedOnly =
|
|
10476
|
+
needsFingerprintSeed && rowId <= priorLastId;
|
|
10477
|
+
const recentEvent = copilotStoreRecentEvent(row, claimNowMs);
|
|
10478
|
+
if (pendingRetryIds.has(rowId) && !recentEvent) {
|
|
10479
|
+
if (seedOnly) {
|
|
10480
|
+
incrementCopilotFingerprintCount(
|
|
10481
|
+
seenEventCounts,
|
|
10482
|
+
eventFingerprint,
|
|
10483
|
+
);
|
|
10484
|
+
}
|
|
10485
|
+
pendingMalformedEvents[rowId] = eventFingerprint || "";
|
|
10486
|
+
seenSessions.add(sessionId);
|
|
10487
|
+
continue;
|
|
10488
|
+
}
|
|
10489
|
+
if (rowId > 0) delete pendingMalformedEvents[rowId];
|
|
10490
|
+
const matchedOtelUsage =
|
|
10491
|
+
recentEvent && otelUsageMatcher.consume(recentEvent);
|
|
10492
|
+
if (matchedOtelUsage) recentEvent.consumed = true;
|
|
10493
|
+
if (baselineOnly || seedOnly) {
|
|
10494
|
+
incrementCopilotFingerprintCount(seenEventCounts, eventFingerprint);
|
|
10495
|
+
if (!recentEvent && rowId > 0) {
|
|
10496
|
+
pendingMalformedEvents[rowId] = eventFingerprint || "";
|
|
10497
|
+
}
|
|
10498
|
+
seenSessions.add(sessionId);
|
|
10499
|
+
continue;
|
|
10500
|
+
}
|
|
10501
|
+
incrementCopilotFingerprintCount(seenEventCounts, eventFingerprint);
|
|
10502
|
+
if (needsAdoption && backfillOnFirstRun && excludedFirstRunSessions.has(sessionId)) {
|
|
10503
|
+
if (
|
|
10504
|
+
!matchedOtelUsage &&
|
|
10505
|
+
Object.prototype.hasOwnProperty.call(
|
|
10506
|
+
excludedFirstRunTotals,
|
|
10507
|
+
sessionId,
|
|
10508
|
+
)
|
|
10509
|
+
) {
|
|
10510
|
+
const normalized = normalizeCopilotSessionStoreUsage(row);
|
|
10511
|
+
const priorResidual = legacyResiduals.get(sessionId) || {
|
|
10512
|
+
storeTokens: 0,
|
|
10513
|
+
lastEventAt: null,
|
|
10514
|
+
};
|
|
10515
|
+
priorResidual.storeTokens += normalized.total_tokens;
|
|
10516
|
+
priorResidual.lastEventAt = latestCopilotTimestamp(
|
|
10517
|
+
priorResidual.lastEventAt,
|
|
10518
|
+
row?.created_at,
|
|
10519
|
+
);
|
|
10520
|
+
legacyResiduals.set(sessionId, priorResidual);
|
|
10521
|
+
}
|
|
10522
|
+
if (recentEvent && !matchedOtelUsage) recentEvents.push(recentEvent);
|
|
10523
|
+
seenSessions.add(sessionId);
|
|
10524
|
+
continue;
|
|
10525
|
+
}
|
|
10526
|
+
if (matchedOtelUsage) {
|
|
10527
|
+
seenSessions.add(sessionId);
|
|
10528
|
+
continue;
|
|
10529
|
+
}
|
|
10530
|
+
if (recentEvent) recentEvents.push(recentEvent);
|
|
10531
|
+
const normalized = normalizeCopilotSessionStoreUsage(row);
|
|
10532
|
+
if (normalized.total_tokens <= 0) {
|
|
10533
|
+
seenSessions.add(sessionId);
|
|
10534
|
+
continue;
|
|
10535
|
+
}
|
|
10536
|
+
const tsIso = parseCopilotAppTimestamp(row.created_at);
|
|
10537
|
+
const bucketStart = tsIso ? toUtcHalfHourStart(tsIso) : null;
|
|
10538
|
+
if (!bucketStart) {
|
|
10539
|
+
if (rowId > 0) {
|
|
10540
|
+
pendingMalformedEvents[rowId] = eventFingerprint || "";
|
|
10541
|
+
}
|
|
10542
|
+
seenSessions.add(sessionId);
|
|
10543
|
+
continue;
|
|
10544
|
+
}
|
|
10545
|
+
const model =
|
|
10546
|
+
normalizeCopilotAppModel(row.model) || COPILOT_APP_DEFAULT_MODEL;
|
|
10547
|
+
const delta = {
|
|
10548
|
+
input_tokens: normalized.input_tokens,
|
|
10549
|
+
cached_input_tokens: normalized.cached_input_tokens,
|
|
10550
|
+
cache_creation_input_tokens: normalized.cache_creation_input_tokens,
|
|
10551
|
+
output_tokens: normalized.output_tokens,
|
|
10552
|
+
reasoning_output_tokens: normalized.reasoning_output_tokens,
|
|
10553
|
+
total_tokens: normalized.total_tokens,
|
|
10554
|
+
conversation_count: seenSessions.has(sessionId) ? 0 : 1,
|
|
10555
|
+
};
|
|
10556
|
+
const bucket = getHourlyBucket(hourlyState, "copilot", model, bucketStart);
|
|
10557
|
+
addTotals(bucket.totals, delta);
|
|
10558
|
+
touchedBuckets.add(bucketKey("copilot", model, bucketStart));
|
|
10559
|
+
seenSessions.add(sessionId);
|
|
10560
|
+
eventsAggregated++;
|
|
10561
|
+
if (cb) {
|
|
10562
|
+
cb({
|
|
10563
|
+
index: recordsProcessed,
|
|
10564
|
+
total: rows.length,
|
|
10565
|
+
recordsProcessed,
|
|
10566
|
+
eventsAggregated,
|
|
10567
|
+
bucketsQueued: touchedBuckets.size,
|
|
10568
|
+
});
|
|
10569
|
+
}
|
|
10570
|
+
}
|
|
10571
|
+
|
|
10572
|
+
dbStates[resolvedDb] = {
|
|
10573
|
+
...omitCopilotLegacyCatchupState(dbState),
|
|
10574
|
+
adoptedAt: dbState.adoptedAt || updatedAt,
|
|
10575
|
+
resetAt: resetDetected ? updatedAt : dbState.resetAt || null,
|
|
10576
|
+
schemaVersion: metadata.schemaVersion,
|
|
10577
|
+
lastRowKey: metadata.lastRowKey,
|
|
10578
|
+
lastId: Math.max(lastId, metadata.maxId),
|
|
10579
|
+
lastEventAt: latestCopilotTimestamp(
|
|
10580
|
+
dbState.lastEventAt,
|
|
10581
|
+
metadata.lastEventAt,
|
|
10582
|
+
),
|
|
10583
|
+
seenEventCounts,
|
|
10584
|
+
pendingMalformedEvents,
|
|
10585
|
+
malformedEventCount: Object.keys(pendingMalformedEvents).length,
|
|
10586
|
+
resetGapEventCount:
|
|
10587
|
+
toNonNegativeInt(dbState.resetGapEventCount) +
|
|
10588
|
+
(resetWithoutFingerprintHistory ? rows.length : 0),
|
|
10589
|
+
dbIno: typeof currentIno === "number" ? currentIno : null,
|
|
10590
|
+
lastDbFingerprint: fingerprint,
|
|
10591
|
+
lastError: null,
|
|
10592
|
+
lastErrorAt: null,
|
|
10593
|
+
updatedAt,
|
|
10594
|
+
};
|
|
10595
|
+
canonicalDbPaths.push(resolvedDb);
|
|
10596
|
+
activeDbCount++;
|
|
10597
|
+
} catch (err) {
|
|
10598
|
+
if (err?.code !== "ENOENT") dbErrors++;
|
|
10599
|
+
dbStates[resolvedDb] = {
|
|
10600
|
+
...omitCopilotLegacyCatchupState(dbState),
|
|
10601
|
+
lastError: err && err.message ? err.message : String(err),
|
|
10602
|
+
lastErrorAt: updatedAt,
|
|
10603
|
+
updatedAt,
|
|
10604
|
+
};
|
|
10605
|
+
} finally {
|
|
10606
|
+
if (snap) snap.cleanup();
|
|
10607
|
+
}
|
|
10608
|
+
}
|
|
10609
|
+
|
|
10610
|
+
for (const [sessionId, residual] of legacyResiduals) {
|
|
10611
|
+
const baseline = excludedFirstRunTotals[sessionId] || {};
|
|
10612
|
+
const appTotals = normalizeCopilotAppTokenDelta(baseline, {
|
|
10613
|
+
input: 0,
|
|
10614
|
+
output: 0,
|
|
10615
|
+
cached: 0,
|
|
10616
|
+
reasoning: 0,
|
|
10617
|
+
});
|
|
10618
|
+
const residualTokens = Math.max(
|
|
10619
|
+
0,
|
|
10620
|
+
toNonNegativeInt(residual.storeTokens) - appTotals.total_tokens,
|
|
10621
|
+
);
|
|
10622
|
+
if (residualTokens <= 0) continue;
|
|
10623
|
+
const timestamp = latestCopilotTimestamp(
|
|
10624
|
+
residual.lastEventAt,
|
|
10625
|
+
baseline.updatedAt,
|
|
10626
|
+
);
|
|
10627
|
+
const bucketStart = timestamp ? toUtcHalfHourStart(timestamp) : null;
|
|
10628
|
+
if (!bucketStart) continue;
|
|
10629
|
+
const bucket = getHourlyBucket(
|
|
10630
|
+
hourlyState,
|
|
10631
|
+
"copilot",
|
|
10632
|
+
COPILOT_STORE_LEGACY_MODEL,
|
|
10633
|
+
bucketStart,
|
|
10634
|
+
);
|
|
10635
|
+
addTotals(bucket.totals, {
|
|
10636
|
+
input_tokens: residualTokens,
|
|
10637
|
+
cached_input_tokens: 0,
|
|
10638
|
+
cache_creation_input_tokens: 0,
|
|
10639
|
+
output_tokens: 0,
|
|
10640
|
+
reasoning_output_tokens: 0,
|
|
10641
|
+
total_tokens: residualTokens,
|
|
10642
|
+
conversation_count: 0,
|
|
10643
|
+
});
|
|
10644
|
+
touchedBuckets.add(
|
|
10645
|
+
bucketKey("copilot", COPILOT_STORE_LEGACY_MODEL, bucketStart),
|
|
10646
|
+
);
|
|
10647
|
+
eventsAggregated++;
|
|
10648
|
+
}
|
|
10649
|
+
|
|
10650
|
+
const bucketsQueued = await enqueueTouchedBuckets({
|
|
10651
|
+
queuePath,
|
|
10652
|
+
hourlyState,
|
|
10653
|
+
touchedBuckets,
|
|
10654
|
+
});
|
|
10655
|
+
const allCurrentPathsHealthy =
|
|
10656
|
+
uniquePaths.length > 0 &&
|
|
10657
|
+
activeDbCount === uniquePaths.length &&
|
|
10658
|
+
dbErrors === 0;
|
|
10659
|
+
const canonicalActive = storeState.active === true || allCurrentPathsHealthy;
|
|
10660
|
+
hourlyState.updatedAt = updatedAt;
|
|
10661
|
+
cursors.hourly = hourlyState;
|
|
10662
|
+
cursors.copilotStore = {
|
|
10663
|
+
...omitCopilotLegacyCatchupState(storeState),
|
|
10664
|
+
version: COPILOT_STORE_CURSOR_VERSION,
|
|
10665
|
+
active: canonicalActive,
|
|
10666
|
+
dbs: dbStates,
|
|
10667
|
+
seenSessions: Array.from(seenSessions),
|
|
10668
|
+
recentEvents: pruneCopilotUsageClaims(recentEvents),
|
|
10669
|
+
updatedAt,
|
|
10670
|
+
};
|
|
10671
|
+
return {
|
|
10672
|
+
active: canonicalActive,
|
|
10673
|
+
healthy: allCurrentPathsHealthy,
|
|
10674
|
+
adoptedThisRun,
|
|
10675
|
+
canonicalDbPaths,
|
|
10676
|
+
recordsProcessed,
|
|
10677
|
+
eventsAggregated,
|
|
10678
|
+
bucketsQueued,
|
|
10679
|
+
dbErrors,
|
|
10680
|
+
usageClaims: recentEvents,
|
|
10681
|
+
};
|
|
9559
10682
|
}
|
|
9560
10683
|
|
|
9561
10684
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -9567,9 +10690,9 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9567
10690
|
// there before the child has produced any tokens, so parsing event history would
|
|
9568
10691
|
// double-count parent usage. The only accounting source here is positive deltas
|
|
9569
10692
|
// from the `sessions.total_*` counters. Those deltas feed the same aggregate
|
|
9570
|
-
// `source="copilot"` pipeline
|
|
9571
|
-
//
|
|
9572
|
-
//
|
|
10693
|
+
// `source="copilot"` pipeline during legacy adoption. Once session-store is
|
|
10694
|
+
// canonical, this parser runs observe-only so a temporary store outage cannot
|
|
10695
|
+
// create duplicate fallback writes when the store returns.
|
|
9573
10696
|
// Project attribution and per-conversation detail are intentionally absent:
|
|
9574
10697
|
// data.db's summary columns do not expose privacy-safe project/message metadata.
|
|
9575
10698
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -9577,44 +10700,169 @@ async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgre
|
|
|
9577
10700
|
const COPILOT_APP_CURSOR_VERSION = 1;
|
|
9578
10701
|
const COPILOT_APP_DEFAULT_MODEL = "github-copilot";
|
|
9579
10702
|
|
|
9580
|
-
function
|
|
10703
|
+
function normalizeCopilotDbPath(dbPath, env = process.env) {
|
|
9581
10704
|
if (typeof dbPath !== "string" || !dbPath.trim()) return null;
|
|
9582
|
-
const
|
|
9583
|
-
|
|
10705
|
+
const raw = dbPath.trim();
|
|
10706
|
+
const home =
|
|
10707
|
+
process.platform === "win32"
|
|
10708
|
+
? env.USERPROFILE || env.HOME || require("node:os").homedir()
|
|
10709
|
+
: env.HOME || require("node:os").homedir();
|
|
10710
|
+
const expanded =
|
|
10711
|
+
raw === "~"
|
|
10712
|
+
? home
|
|
10713
|
+
: raw.startsWith("~/") || raw.startsWith("~\\")
|
|
10714
|
+
? path.join(home, raw.slice(2))
|
|
10715
|
+
: raw;
|
|
10716
|
+
const normalized =
|
|
10717
|
+
path.isAbsolute(expanded) ? path.normalize(expanded) : path.resolve(expanded);
|
|
10718
|
+
return normalized;
|
|
9584
10719
|
}
|
|
9585
10720
|
|
|
9586
|
-
function
|
|
9587
|
-
const
|
|
9588
|
-
const
|
|
9589
|
-
const
|
|
9590
|
-
|
|
9591
|
-
|
|
10721
|
+
function mergeCopilotStoreAliasStates(primary = {}, alias = {}) {
|
|
10722
|
+
const primaryLastId = toNonNegativeInt(primary?.lastId);
|
|
10723
|
+
const aliasLastId = toNonNegativeInt(alias?.lastId);
|
|
10724
|
+
const primaryUpdatedAt = Date.parse(primary?.updatedAt || "") || 0;
|
|
10725
|
+
const aliasUpdatedAt = Date.parse(alias?.updatedAt || "") || 0;
|
|
10726
|
+
const newer =
|
|
10727
|
+
aliasLastId > primaryLastId ||
|
|
10728
|
+
(aliasLastId === primaryLastId && aliasUpdatedAt > primaryUpdatedAt)
|
|
10729
|
+
? alias
|
|
10730
|
+
: primary;
|
|
10731
|
+
const seenEventCounts = normalizeCopilotFingerprintCounts(
|
|
10732
|
+
primary?.seenEventCounts,
|
|
10733
|
+
);
|
|
10734
|
+
for (const [fingerprint, count] of Object.entries(
|
|
10735
|
+
normalizeCopilotFingerprintCounts(alias?.seenEventCounts),
|
|
10736
|
+
)) {
|
|
10737
|
+
seenEventCounts[fingerprint] = Math.max(
|
|
10738
|
+
toNonNegativeInt(seenEventCounts[fingerprint]),
|
|
10739
|
+
count,
|
|
10740
|
+
);
|
|
10741
|
+
}
|
|
10742
|
+
const pendingMalformedEvents = {
|
|
10743
|
+
...normalizeCopilotPendingMalformedEvents(
|
|
10744
|
+
primary?.pendingMalformedEvents,
|
|
10745
|
+
),
|
|
10746
|
+
...normalizeCopilotPendingMalformedEvents(
|
|
10747
|
+
alias?.pendingMalformedEvents,
|
|
10748
|
+
),
|
|
9592
10749
|
};
|
|
9593
|
-
|
|
9594
|
-
|
|
9595
|
-
|
|
10750
|
+
return {
|
|
10751
|
+
...primary,
|
|
10752
|
+
...newer,
|
|
10753
|
+
seenEventCounts,
|
|
10754
|
+
pendingMalformedEvents,
|
|
10755
|
+
malformedEventCount: Object.keys(pendingMalformedEvents).length,
|
|
10756
|
+
resetGapEventCount: Math.max(
|
|
10757
|
+
toNonNegativeInt(primary?.resetGapEventCount),
|
|
10758
|
+
toNonNegativeInt(alias?.resetGapEventCount),
|
|
10759
|
+
),
|
|
9596
10760
|
};
|
|
10761
|
+
}
|
|
9597
10762
|
|
|
9598
|
-
|
|
9599
|
-
|
|
10763
|
+
function coalesceCopilotDbStatesByIdentity(
|
|
10764
|
+
dbStates,
|
|
10765
|
+
selectedPaths,
|
|
10766
|
+
mergeStates,
|
|
10767
|
+
) {
|
|
10768
|
+
if (!dbStates || typeof dbStates !== "object") return new Set();
|
|
10769
|
+
const selectedByIdentity = new Map();
|
|
10770
|
+
const selectedByInode = new Map();
|
|
10771
|
+
for (const selectedPath of selectedPaths) {
|
|
10772
|
+
try {
|
|
10773
|
+
const stat = fssync.statSync(selectedPath);
|
|
10774
|
+
selectedByIdentity.set(`${stat.dev}:${stat.ino}`, selectedPath);
|
|
10775
|
+
if (selectedByInode.has(stat.ino)) selectedByInode.set(stat.ino, null);
|
|
10776
|
+
else selectedByInode.set(stat.ino, selectedPath);
|
|
10777
|
+
} catch (_e) {}
|
|
9600
10778
|
}
|
|
9601
|
-
|
|
9602
|
-
|
|
10779
|
+
const migratedPaths = new Set();
|
|
10780
|
+
for (const [statePath, state] of Object.entries({ ...dbStates })) {
|
|
10781
|
+
let dev = null;
|
|
10782
|
+
let ino = null;
|
|
10783
|
+
try {
|
|
10784
|
+
const stat = fssync.statSync(statePath);
|
|
10785
|
+
dev = stat.dev;
|
|
10786
|
+
ino = stat.ino;
|
|
10787
|
+
} catch (_e) {
|
|
10788
|
+
dev = state?.lastDbFingerprint?.db?.dev;
|
|
10789
|
+
ino = state?.lastDbFingerprint?.db?.ino;
|
|
10790
|
+
}
|
|
10791
|
+
const selectedPath =
|
|
10792
|
+
Number.isFinite(dev) && Number.isFinite(ino)
|
|
10793
|
+
? selectedByIdentity.get(`${dev}:${ino}`)
|
|
10794
|
+
: Number.isFinite(ino)
|
|
10795
|
+
? selectedByInode.get(ino)
|
|
10796
|
+
: null;
|
|
10797
|
+
if (!selectedPath || selectedPath === statePath) continue;
|
|
10798
|
+
dbStates[selectedPath] = mergeStates(
|
|
10799
|
+
dbStates[selectedPath],
|
|
10800
|
+
state,
|
|
10801
|
+
);
|
|
10802
|
+
delete dbStates[statePath];
|
|
10803
|
+
migratedPaths.add(statePath);
|
|
9603
10804
|
}
|
|
10805
|
+
return migratedPaths;
|
|
10806
|
+
}
|
|
9604
10807
|
|
|
9605
|
-
|
|
9606
|
-
|
|
9607
|
-
|
|
9608
|
-
|
|
9609
|
-
|
|
9610
|
-
|
|
9611
|
-
|
|
9612
|
-
|
|
9613
|
-
|
|
9614
|
-
|
|
10808
|
+
function uniqueCopilotDbPaths(
|
|
10809
|
+
paths,
|
|
10810
|
+
env = process.env,
|
|
10811
|
+
preferredPaths = [],
|
|
10812
|
+
) {
|
|
10813
|
+
const pathKey = (dbPath) =>
|
|
10814
|
+
process.platform === "win32" ? dbPath.toLowerCase() : dbPath;
|
|
10815
|
+
const preferredKeys = new Set(
|
|
10816
|
+
(Array.isArray(preferredPaths) ? preferredPaths : [])
|
|
10817
|
+
.map((candidate) => normalizeCopilotDbPath(candidate, env))
|
|
10818
|
+
.filter(Boolean)
|
|
10819
|
+
.map(pathKey),
|
|
10820
|
+
);
|
|
10821
|
+
const candidates = (Array.isArray(paths) ? paths : [])
|
|
10822
|
+
.map((candidate, index) => ({
|
|
10823
|
+
index,
|
|
10824
|
+
path: normalizeCopilotDbPath(candidate, env),
|
|
10825
|
+
}))
|
|
10826
|
+
.filter((candidate) => candidate.path)
|
|
10827
|
+
.sort((a, b) => {
|
|
10828
|
+
const preferredDiff =
|
|
10829
|
+
Number(preferredKeys.has(pathKey(b.path))) -
|
|
10830
|
+
Number(preferredKeys.has(pathKey(a.path)));
|
|
10831
|
+
return preferredDiff || a.index - b.index;
|
|
10832
|
+
});
|
|
10833
|
+
const result = [];
|
|
10834
|
+
const identities = new Set();
|
|
10835
|
+
for (const candidate of candidates) {
|
|
10836
|
+
const normalized = candidate.path;
|
|
10837
|
+
let identity =
|
|
10838
|
+
process.platform === "win32"
|
|
10839
|
+
? `path:${normalized.toLowerCase()}`
|
|
10840
|
+
: `path:${normalized}`;
|
|
10841
|
+
try {
|
|
10842
|
+
const stat = fssync.statSync(normalized);
|
|
10843
|
+
if (
|
|
10844
|
+
Number.isFinite(stat.dev) &&
|
|
10845
|
+
Number.isFinite(stat.ino) &&
|
|
10846
|
+
stat.ino > 0
|
|
10847
|
+
) {
|
|
10848
|
+
identity = `inode:${stat.dev}:${stat.ino}`;
|
|
10849
|
+
}
|
|
10850
|
+
} catch (_e) {}
|
|
10851
|
+
if (identities.has(identity)) continue;
|
|
10852
|
+
identities.add(identity);
|
|
10853
|
+
result.push(normalized);
|
|
9615
10854
|
}
|
|
10855
|
+
return result;
|
|
10856
|
+
}
|
|
9616
10857
|
|
|
9617
|
-
|
|
10858
|
+
function resolveCopilotAppDbPaths(env = process.env) {
|
|
10859
|
+
return resolveCopilotDbPaths(
|
|
10860
|
+
{
|
|
10861
|
+
fileName: "data.db",
|
|
10862
|
+
overrideEnvKey: "TOKENTRACKER_COPILOT_APP_DB",
|
|
10863
|
+
},
|
|
10864
|
+
env,
|
|
10865
|
+
);
|
|
9618
10866
|
}
|
|
9619
10867
|
|
|
9620
10868
|
function resolveCopilotAppDbPath(env = process.env) {
|
|
@@ -9723,6 +10971,7 @@ function sqliteSidecarFingerprint(dbPath) {
|
|
|
9723
10971
|
out[suffix || "db"] = {
|
|
9724
10972
|
size: stat.size,
|
|
9725
10973
|
mtimeMs: stat.mtimeMs,
|
|
10974
|
+
dev: stat.dev,
|
|
9726
10975
|
ino: stat.ino,
|
|
9727
10976
|
};
|
|
9728
10977
|
} catch (e) {
|
|
@@ -9736,6 +10985,10 @@ function sameSqliteFingerprint(a, b) {
|
|
|
9736
10985
|
return JSON.stringify(a || {}) === JSON.stringify(b || {});
|
|
9737
10986
|
}
|
|
9738
10987
|
|
|
10988
|
+
function getCopilotSqliteFingerprint(dbPath) {
|
|
10989
|
+
return sqliteSidecarFingerprint(dbPath);
|
|
10990
|
+
}
|
|
10991
|
+
|
|
9739
10992
|
function copilotAppBaselineTotal(value) {
|
|
9740
10993
|
if (!value || typeof value !== "object") return 0;
|
|
9741
10994
|
return (
|
|
@@ -9801,6 +11054,7 @@ async function parseCopilotAppDbIncremental({
|
|
|
9801
11054
|
onProgress,
|
|
9802
11055
|
env,
|
|
9803
11056
|
sqliteOptions,
|
|
11057
|
+
observeOnly = false,
|
|
9804
11058
|
} = {}) {
|
|
9805
11059
|
await ensureDir(path.dirname(queuePath));
|
|
9806
11060
|
const paths = Array.isArray(dbPaths) && dbPaths.length > 0
|
|
@@ -9808,11 +11062,15 @@ async function parseCopilotAppDbIncremental({
|
|
|
9808
11062
|
: dbPath
|
|
9809
11063
|
? [dbPath]
|
|
9810
11064
|
: resolveCopilotAppDbPaths(env || process.env);
|
|
9811
|
-
const uniquePaths = [...new Set(paths.map((p) => normalizeCopilotAppDbPath(p, env || process.env)).filter(Boolean))];
|
|
9812
11065
|
const appState =
|
|
9813
11066
|
cursors.copilotApp && typeof cursors.copilotApp === "object" ? cursors.copilotApp : {};
|
|
9814
11067
|
const dbStates =
|
|
9815
11068
|
appState.dbs && typeof appState.dbs === "object" ? { ...appState.dbs } : {};
|
|
11069
|
+
const uniquePaths = uniqueCopilotDbPaths(
|
|
11070
|
+
paths,
|
|
11071
|
+
env || process.env,
|
|
11072
|
+
Object.keys(dbStates),
|
|
11073
|
+
);
|
|
9816
11074
|
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
9817
11075
|
const touchedBuckets = new Set();
|
|
9818
11076
|
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
@@ -9895,6 +11153,10 @@ async function parseCopilotAppDbIncremental({
|
|
|
9895
11153
|
sessionTotals[sessionId] = { ...curr, model, updatedAt: rowUpdatedAt };
|
|
9896
11154
|
continue;
|
|
9897
11155
|
}
|
|
11156
|
+
if (observeOnly) {
|
|
11157
|
+
sessionTotals[sessionId] = { ...curr, model, updatedAt: rowUpdatedAt };
|
|
11158
|
+
continue;
|
|
11159
|
+
}
|
|
9898
11160
|
|
|
9899
11161
|
const bucketStart = toUtcHalfHourStart(rowUpdatedAt);
|
|
9900
11162
|
if (!bucketStart) {
|
|
@@ -10859,8 +12121,16 @@ module.exports = {
|
|
|
10859
12121
|
probeWslDistros,
|
|
10860
12122
|
discoverWslHermesHome,
|
|
10861
12123
|
resolveCopilotOtelPaths,
|
|
12124
|
+
normalizeCopilotDbPath,
|
|
12125
|
+
uniqueCopilotDbPaths,
|
|
12126
|
+
coalesceCopilotDbStatesByIdentity,
|
|
12127
|
+
resolveCopilotSessionStorePaths,
|
|
12128
|
+
getCopilotSqliteFingerprint,
|
|
12129
|
+
describeCopilotSessionStoreDb,
|
|
10862
12130
|
resolveCopilotAppDbPath,
|
|
10863
12131
|
resolveCopilotAppDbPaths,
|
|
12132
|
+
readCopilotSessionStoreUsageRows,
|
|
12133
|
+
normalizeCopilotSessionStoreUsage,
|
|
10864
12134
|
readCopilotAppSessionsFromSqlite,
|
|
10865
12135
|
parseRolloutIncremental,
|
|
10866
12136
|
parseClaudeIncremental,
|
|
@@ -10874,7 +12144,10 @@ module.exports = {
|
|
|
10874
12144
|
gooseInstallOwnsCursor,
|
|
10875
12145
|
zedInstallOwnsCursor,
|
|
10876
12146
|
hermesInstallOwnsCursor,
|
|
12147
|
+
copilotOtelCursorHasLegacyCliUsage,
|
|
12148
|
+
pruneCopilotUsageClaims,
|
|
10877
12149
|
parseCopilotIncremental,
|
|
12150
|
+
parseCopilotSessionStoreIncremental,
|
|
10878
12151
|
parseCopilotAppDbIncremental,
|
|
10879
12152
|
resolveKimiWireFiles,
|
|
10880
12153
|
resolveKimiDefaultModel,
|