tokentracker-cli 0.5.65 → 0.5.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/{Card-CzO2gq_D.js → Card-DSx0j4mK.js} +1 -1
- package/dashboard/dist/assets/{DashboardPage-BE7gGFUL.js → DashboardPage-1P9-gqhu.js} +1 -1
- package/dashboard/dist/assets/{FadeIn-CTN2r1YX.js → FadeIn-DL1zAbdt.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-D7eagOPA.js → IpCheckPage-BM3t1yfV.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-B-N6D-VW.js → LeaderboardPage-BZvpMVkN.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProfilePage-f5R4Ny08.js → LeaderboardProfilePage-aO-byJ7B.js} +1 -1
- package/dashboard/dist/assets/LimitsPage-CwUkpJp_.js +2 -0
- package/dashboard/dist/assets/{SettingsPage-DWr_7D_U.js → SettingsPage-DhirGYnP.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-OKjKynuO.js → WidgetsPage-P9a72ksF.js} +1 -1
- package/dashboard/dist/assets/{download-B0JjZrok.js → download-DTMGheZ-.js} +1 -1
- package/dashboard/dist/assets/leaderboard-columns-DVxnNSJP.js +1 -0
- package/dashboard/dist/assets/main-DHsb2wcd.css +1 -0
- package/dashboard/dist/assets/{main-DDaoGWcU.js → main-XWVaG1Bp.js} +20 -15
- package/dashboard/dist/assets/use-limits-display-prefs-B4BIb5sR.js +1 -0
- package/dashboard/dist/assets/{use-usage-limits-BLkA4tpF.js → use-usage-limits-DOtf83Kb.js} +1 -1
- package/dashboard/dist/brand-logos/copilot.svg +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dashboard/dist/share.html +2 -2
- package/package.json +1 -1
- package/src/commands/status.js +29 -0
- package/src/commands/sync.js +81 -3
- package/src/lib/cursor-config.js +44 -45
- package/src/lib/rollout.js +192 -0
- package/src/lib/usage-limits.js +148 -1
- package/dashboard/dist/assets/LimitsPage-ynP3bW3P.js +0 -1
- package/dashboard/dist/assets/leaderboard-columns-CHbtsVUx.js +0 -1
- package/dashboard/dist/assets/main-DYCIFB2M.css +0 -1
- package/dashboard/dist/assets/use-limits-display-prefs-DPtzcPRA.js +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{r as a}from"./main-XWVaG1Bp.js";const d=["claude","codex","cursor","gemini","kiro","copilot","antigravity"],v={claude:"Claude",codex:"Codex",cursor:"Cursor",gemini:"Gemini",kiro:"Kiro",copilot:"GitHub Copilot",antigravity:"Antigravity"},k={claude:"/brand-logos/claude-code.svg",codex:"/brand-logos/codex.svg",cursor:"/brand-logos/cursor.svg",gemini:"/brand-logos/gemini.svg",kiro:"/brand-logos/kiro.svg",copilot:"/brand-logos/copilot.svg",antigravity:"/brand-logos/antigravity.svg"},l="tt.limits.providerOrder",g="tt.limits.providerVisibility";function y(){if(typeof window>"u")return[...d];try{const r=window.localStorage.getItem(l);if(!r)return[...d];const s=JSON.parse(r);if(!Array.isArray(s))return[...d];const n=s.filter(c=>d.includes(c));for(const c of d)n.includes(c)||n.push(c);return n}catch{return[...d]}}function m(){const r=Object.fromEntries(d.map(s=>[s,!0]));if(typeof window>"u")return r;try{const s=window.localStorage.getItem(g);if(!s)return r;const n=JSON.parse(s);if(!n||typeof n!="object")return r;const c={...r};for(const u of d)typeof n[u]=="boolean"&&(c[u]=n[u]);return c}catch{return r}}function C(){const[r,s]=a.useState(y),[n,c]=a.useState(m);a.useEffect(()=>{if(!(typeof window>"u"))try{window.localStorage.setItem(l,JSON.stringify(r))}catch{}},[r]),a.useEffect(()=>{if(!(typeof window>"u"))try{window.localStorage.setItem(g,JSON.stringify(n))}catch{}},[n]),a.useEffect(()=>{if(typeof window>"u")return;const o=t=>{t.key===l&&s(y()),t.key===g&&c(m())};return window.addEventListener("storage",o),()=>window.removeEventListener("storage",o)},[]);const u=a.useCallback(o=>{c(t=>({...t,[o]:!t[o]}))},[]),b=a.useCallback(o=>{s(t=>{const e=t.indexOf(o);if(e<=0)return t;const i=[...t];return[i[e-1],i[e]]=[i[e],i[e-1]],i})},[]),p=a.useCallback(o=>{s(t=>{const e=t.indexOf(o);if(e<0||e>=t.length-1)return t;const i=[...t];return[i[e],i[e+1]]=[i[e+1],i[e]],i})},[]),O=a.useCallback((o,t)=>{o!==t&&s(e=>{const i=e.indexOf(o),w=e.indexOf(t);if(i<0||w<0)return e;const f=[...e],[E]=f.splice(i,1);return f.splice(w,0,E),f})},[]),x=a.useCallback(()=>{s([...d]),c(Object.fromEntries(d.map(o=>[o,!0])))},[]),I=a.useMemo(()=>r.filter(o=>n[o]!==!1),[r,n]);return{order:r,visibility:n,visibleOrdered:I,toggle:u,moveUp:b,moveDown:p,moveToward:O,reset:x}}export{k as L,v as a,C as u};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{r,aM as l}from"./main-
|
|
1
|
+
import{r,aM as l}from"./main-XWVaG1Bp.js";function y(i){const[o,a]=r.useState(null),[u,s]=r.useState(null),[c,f]=r.useState(!0),n=!!i?.initialRefresh,g=r.useCallback(async()=>{try{const e=await l({refresh:!0});a(e&&typeof e=="object"?e:null),s(null)}catch(e){s(e?.message||String(e))}},[]);return r.useEffect(()=>{let e=!1;return(async()=>{try{const t=await l(n?{refresh:!0}:{});if(e)return;a(t&&typeof t=="object"?t:null),s(null)}catch(t){if(e)return;s(t?.message||String(t))}finally{e||f(!1)}})(),()=>{e=!0}},[n]),{data:o,error:u,isLoading:c,refresh:g}}export{y as u};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>GithubCopilot</title><path d="M19.245 5.364c1.322 1.36 1.877 3.216 2.11 5.817.622 0 1.2.135 1.592.654l.73.964c.21.278.323.61.323.955v2.62c0 .339-.173.669-.453.868C20.239 19.602 16.157 21.5 12 21.5c-4.6 0-9.205-2.583-11.547-4.258-.28-.2-.452-.53-.453-.868v-2.62c0-.345.113-.679.321-.956l.73-.963c.392-.517.974-.654 1.593-.654l.029-.297c.25-2.446.81-4.213 2.082-5.52 2.461-2.54 5.71-2.851 7.146-2.864h.198c1.436.013 4.685.323 7.146 2.864zm-7.244 4.328c-.284 0-.613.016-.962.05-.123.447-.305.85-.57 1.108-1.05 1.023-2.316 1.18-2.994 1.18-.638 0-1.306-.13-1.851-.464-.516.165-1.012.403-1.044.996a65.882 65.882 0 00-.063 2.884l-.002.48c-.002.563-.005 1.126-.013 1.69.002.326.204.63.51.765 2.482 1.102 4.83 1.657 6.99 1.657 2.156 0 4.504-.555 6.985-1.657a.854.854 0 00.51-.766c.03-1.682.006-3.372-.076-5.053-.031-.596-.528-.83-1.046-.996-.546.333-1.212.464-1.85.464-.677 0-1.942-.157-2.993-1.18-.266-.258-.447-.661-.57-1.108-.32-.032-.64-.049-.96-.05zm-2.525 4.013c.539 0 .976.426.976.95v1.753c0 .525-.437.95-.976.95a.964.964 0 01-.976-.95v-1.752c0-.525.437-.951.976-.951zm5 0c.539 0 .976.426.976.95v1.753c0 .525-.437.95-.976.95a.964.964 0 01-.976-.95v-1.752c0-.525.437-.951.976-.951zM7.635 5.087c-1.05.102-1.935.438-2.385.906-.975 1.037-.765 3.668-.21 4.224.405.394 1.17.657 1.995.657h.09c.649-.013 1.785-.176 2.73-1.11.435-.41.705-1.433.675-2.47-.03-.834-.27-1.52-.63-1.813-.39-.336-1.275-.482-2.265-.394zm6.465.394c-.36.292-.6.98-.63 1.813-.03 1.037.24 2.06.675 2.47.968.957 2.136 1.104 2.776 1.11h.044c.825 0 1.59-.263 1.995-.657.555-.556.765-3.187-.21-4.224-.45-.468-1.335-.804-2.385-.906-.99-.088-1.875.058-2.265.394zM12 7.615c-.24 0-.525.015-.84.044.03.16.045.336.06.526l-.001.159a2.94 2.94 0 01-.014.25c.225-.022.425-.027.612-.028h.366c.187 0 .387.006.612.028-.015-.146-.015-.277-.015-.409.015-.19.03-.365.06-.526a9.29 9.29 0 00-.84-.044z"></path></svg>
|
|
@@ -135,8 +135,8 @@
|
|
|
135
135
|
]
|
|
136
136
|
}
|
|
137
137
|
</script>
|
|
138
|
-
<script type="module" crossorigin src="/assets/main-
|
|
139
|
-
<link rel="stylesheet" crossorigin href="/assets/main-
|
|
138
|
+
<script type="module" crossorigin src="/assets/main-XWVaG1Bp.js"></script>
|
|
139
|
+
<link rel="stylesheet" crossorigin href="/assets/main-DHsb2wcd.css">
|
|
140
140
|
</head>
|
|
141
141
|
<body>
|
|
142
142
|
<main class="aeo-seed-content" aria-label="Token Tracker AI-readable summary">
|
|
@@ -51,8 +51,8 @@
|
|
|
51
51
|
"description": "Shareable Token Tracker dashboard snapshot."
|
|
52
52
|
}
|
|
53
53
|
</script>
|
|
54
|
-
<script type="module" crossorigin src="/assets/main-
|
|
55
|
-
<link rel="stylesheet" crossorigin href="/assets/main-
|
|
54
|
+
<script type="module" crossorigin src="/assets/main-XWVaG1Bp.js"></script>
|
|
55
|
+
<link rel="stylesheet" crossorigin href="/assets/main-DHsb2wcd.css">
|
|
56
56
|
</head>
|
|
57
57
|
<body>
|
|
58
58
|
<main class="aeo-seed-content" aria-label="Token Tracker share page summary">
|
package/package.json
CHANGED
package/src/commands/status.js
CHANGED
|
@@ -13,6 +13,7 @@ const {
|
|
|
13
13
|
} = require("../lib/gemini-config");
|
|
14
14
|
const { resolveOpencodeConfigDir, isOpencodePluginInstalled } = require("../lib/opencode-config");
|
|
15
15
|
const { collectLocalSubscriptions } = require("../lib/subscriptions");
|
|
16
|
+
const { describeCopilotOtelStatus, readCopilotOauthToken } = require("../lib/usage-limits");
|
|
16
17
|
const { normalizeState: normalizeUploadState } = require("../lib/upload-throttle");
|
|
17
18
|
const { collectTrackerDiagnostics } = require("../lib/diagnostics");
|
|
18
19
|
const { probeOpenclawHookState } = require("../lib/openclaw-hook");
|
|
@@ -111,6 +112,10 @@ async function cmdStatus(argv = []) {
|
|
|
111
112
|
const subscriptionLines =
|
|
112
113
|
subscriptions.length > 0 ? subscriptions.map(formatSubscriptionLine) : [];
|
|
113
114
|
|
|
115
|
+
const copilotToken = readCopilotOauthToken({ home });
|
|
116
|
+
const copilotOtel = describeCopilotOtelStatus({ home, env: process.env });
|
|
117
|
+
const copilotLines = formatCopilotLines({ token: copilotToken, otel: copilotOtel });
|
|
118
|
+
|
|
114
119
|
process.stdout.write(
|
|
115
120
|
[
|
|
116
121
|
"Status:",
|
|
@@ -133,6 +138,7 @@ async function cmdStatus(argv = []) {
|
|
|
133
138
|
`- Opencode plugin: ${opencodePluginConfigured ? "set" : "unset"}`,
|
|
134
139
|
`- OpenClaw session plugin: ${openclawSessionPluginState?.configured ? "set" : "unset"}`,
|
|
135
140
|
`- OpenClaw hook (legacy): ${openclawHookState?.configured ? "set" : "unset"}`,
|
|
141
|
+
...copilotLines,
|
|
136
142
|
...subscriptionLines,
|
|
137
143
|
"",
|
|
138
144
|
]
|
|
@@ -141,6 +147,29 @@ async function cmdStatus(argv = []) {
|
|
|
141
147
|
);
|
|
142
148
|
}
|
|
143
149
|
|
|
150
|
+
function formatCopilotLines({ token, otel }) {
|
|
151
|
+
if (!token && !otel.otel_has_files) return [];
|
|
152
|
+
const limitsState = token ? "set (via GitHub OAuth)" : "unset (no Copilot OAuth token found)";
|
|
153
|
+
const usageState = otel.otel_has_files
|
|
154
|
+
? `set (${otel.otel_path || otel.otel_default_dir})`
|
|
155
|
+
: otel.otel_enabled
|
|
156
|
+
? "enabled but no files yet"
|
|
157
|
+
: "unset (OTEL export not enabled)";
|
|
158
|
+
const lines = [
|
|
159
|
+
`- GitHub Copilot limits: ${limitsState}`,
|
|
160
|
+
`- GitHub Copilot usage (OTEL): ${usageState}`,
|
|
161
|
+
];
|
|
162
|
+
if (!otel.otel_has_files) {
|
|
163
|
+
lines.push(
|
|
164
|
+
" To track Copilot token usage, add to your shell profile:",
|
|
165
|
+
" export COPILOT_OTEL_ENABLED=true",
|
|
166
|
+
" export COPILOT_OTEL_EXPORTER_TYPE=file",
|
|
167
|
+
` export COPILOT_OTEL_FILE_EXPORTER_PATH="${otel.otel_default_dir}/copilot-otel-$(date +%Y%m%d).jsonl"`,
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
return lines;
|
|
171
|
+
}
|
|
172
|
+
|
|
144
173
|
function formatSubscriptionLine(entry = {}) {
|
|
145
174
|
const tool = String(entry.tool || "");
|
|
146
175
|
const provider = String(entry.provider || "");
|
package/src/commands/sync.js
CHANGED
|
@@ -14,6 +14,7 @@ const {
|
|
|
14
14
|
resolveKiroDbPath,
|
|
15
15
|
resolveKiroJsonlPath,
|
|
16
16
|
resolveHermesDbPath,
|
|
17
|
+
resolveCopilotOtelPaths,
|
|
17
18
|
parseRolloutIncremental,
|
|
18
19
|
parseClaudeIncremental,
|
|
19
20
|
parseGeminiIncremental,
|
|
@@ -23,6 +24,7 @@ const {
|
|
|
23
24
|
parseCursorApiIncremental,
|
|
24
25
|
parseKiroIncremental,
|
|
25
26
|
parseHermesIncremental,
|
|
27
|
+
parseCopilotIncremental,
|
|
26
28
|
} = require("../lib/rollout");
|
|
27
29
|
const { createProgress, renderBar, formatNumber, formatBytes } = require("../lib/progress");
|
|
28
30
|
const {
|
|
@@ -39,6 +41,8 @@ const { purgeProjectUsage } = require("../lib/project-usage-purge");
|
|
|
39
41
|
const { resolveTrackerPaths } = require("../lib/tracker-paths");
|
|
40
42
|
const { resolveRuntimeConfig } = require("../lib/runtime-config");
|
|
41
43
|
|
|
44
|
+
const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
|
|
45
|
+
|
|
42
46
|
async function cmdSync(argv) {
|
|
43
47
|
const opts = parseArgs(argv);
|
|
44
48
|
const home = os.homedir();
|
|
@@ -259,6 +263,13 @@ async function cmdSync(argv) {
|
|
|
259
263
|
}
|
|
260
264
|
|
|
261
265
|
// ── Cursor (API-based) ──
|
|
266
|
+
// One-time migration: earlier CLI versions mis-parsed the Cursor CSV after
|
|
267
|
+
// Cursor inserted new "Cloud Agent ID"/"Automation ID" columns, writing
|
|
268
|
+
// cursor records under model="unknown". Purge those local buckets, emit
|
|
269
|
+
// zero retractions so the cloud upserts overwrite them to zero, and reset
|
|
270
|
+
// the incremental cursor so the fixed parser re-fetches all affected rows.
|
|
271
|
+
await migrateCursorUnknownBuckets({ cursors, queuePath });
|
|
272
|
+
|
|
262
273
|
let cursorResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
263
274
|
if (isCursorInstalled({ home })) {
|
|
264
275
|
const cursorAuth = extractCursorSessionToken({ home });
|
|
@@ -343,6 +354,28 @@ async function cmdSync(argv) {
|
|
|
343
354
|
});
|
|
344
355
|
}
|
|
345
356
|
|
|
357
|
+
// ── GitHub Copilot CLI (OTEL JSONL files) ──
|
|
358
|
+
let copilotResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
359
|
+
const copilotPaths = resolveCopilotOtelPaths(process.env);
|
|
360
|
+
if (copilotPaths.length > 0) {
|
|
361
|
+
if (progress?.enabled) {
|
|
362
|
+
progress.start(`Parsing Copilot ${renderBar(0)} | buckets 0`);
|
|
363
|
+
}
|
|
364
|
+
copilotResult = await parseCopilotIncremental({
|
|
365
|
+
otelPaths: copilotPaths,
|
|
366
|
+
cursors,
|
|
367
|
+
queuePath,
|
|
368
|
+
env: process.env,
|
|
369
|
+
onProgress: (p) => {
|
|
370
|
+
if (!progress?.enabled) return;
|
|
371
|
+
const pct = p.total > 0 ? p.index / p.total : 1;
|
|
372
|
+
progress.update(
|
|
373
|
+
`Parsing Copilot ${renderBar(pct)} ${formatNumber(p.index)}/${formatNumber(p.total)} files | buckets ${formatNumber(p.bucketsQueued)}`,
|
|
374
|
+
);
|
|
375
|
+
},
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
|
|
346
379
|
if (cursors?.projectHourly?.projects && projectQueuePath && projectQueueStatePath) {
|
|
347
380
|
for (const [projectKey, meta] of Object.entries(cursors.projectHourly.projects)) {
|
|
348
381
|
if (!meta || typeof meta !== "object") continue;
|
|
@@ -419,7 +452,8 @@ async function cmdSync(argv) {
|
|
|
419
452
|
opencodeResult.filesProcessed +
|
|
420
453
|
cursorResult.recordsProcessed +
|
|
421
454
|
kiroResult.recordsProcessed +
|
|
422
|
-
hermesResult.recordsProcessed
|
|
455
|
+
hermesResult.recordsProcessed +
|
|
456
|
+
copilotResult.recordsProcessed;
|
|
423
457
|
const totalBuckets =
|
|
424
458
|
parseResult.bucketsQueued +
|
|
425
459
|
openclawResult.bucketsQueued +
|
|
@@ -428,7 +462,8 @@ async function cmdSync(argv) {
|
|
|
428
462
|
opencodeResult.bucketsQueued +
|
|
429
463
|
cursorResult.bucketsQueued +
|
|
430
464
|
kiroResult.bucketsQueued +
|
|
431
|
-
hermesResult.bucketsQueued
|
|
465
|
+
hermesResult.bucketsQueued +
|
|
466
|
+
copilotResult.bucketsQueued;
|
|
432
467
|
process.stdout.write(
|
|
433
468
|
[
|
|
434
469
|
"Sync finished:",
|
|
@@ -473,7 +508,7 @@ function parseArgs(argv) {
|
|
|
473
508
|
return out;
|
|
474
509
|
}
|
|
475
510
|
|
|
476
|
-
module.exports = { cmdSync };
|
|
511
|
+
module.exports = { cmdSync, migrateCursorUnknownBuckets, CURSOR_UNKNOWN_MIGRATION_KEY };
|
|
477
512
|
|
|
478
513
|
function normalizeString(value) {
|
|
479
514
|
if (typeof value !== "string") return null;
|
|
@@ -838,3 +873,46 @@ async function readQueueBatch(queuePath, startOffset, maxBuckets) {
|
|
|
838
873
|
stream.close?.();
|
|
839
874
|
return { buckets: Array.from(bucketMap.values()), nextOffset: offset };
|
|
840
875
|
}
|
|
876
|
+
|
|
877
|
+
async function migrateCursorUnknownBuckets({ cursors, queuePath }) {
|
|
878
|
+
if (!cursors || typeof cursors !== "object") return;
|
|
879
|
+
cursors.migrations = cursors.migrations || {};
|
|
880
|
+
if (cursors.migrations[CURSOR_UNKNOWN_MIGRATION_KEY]) return;
|
|
881
|
+
|
|
882
|
+
const buckets = cursors.hourly?.buckets;
|
|
883
|
+
if (!buckets || typeof buckets !== "object") {
|
|
884
|
+
cursors.migrations[CURSOR_UNKNOWN_MIGRATION_KEY] = new Date().toISOString();
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
const retractions = [];
|
|
889
|
+
for (const key of Object.keys(buckets)) {
|
|
890
|
+
if (!key.startsWith("cursor|unknown|")) continue;
|
|
891
|
+
const hourStart = key.split("|").slice(2).join("|");
|
|
892
|
+
retractions.push(
|
|
893
|
+
JSON.stringify({
|
|
894
|
+
source: "cursor",
|
|
895
|
+
model: "unknown",
|
|
896
|
+
hour_start: hourStart,
|
|
897
|
+
input_tokens: 0,
|
|
898
|
+
cached_input_tokens: 0,
|
|
899
|
+
cache_creation_input_tokens: 0,
|
|
900
|
+
output_tokens: 0,
|
|
901
|
+
reasoning_output_tokens: 0,
|
|
902
|
+
total_tokens: 0,
|
|
903
|
+
conversation_count: 0,
|
|
904
|
+
}),
|
|
905
|
+
);
|
|
906
|
+
delete buckets[key];
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
if (retractions.length > 0) {
|
|
910
|
+
await ensureDir(path.dirname(queuePath));
|
|
911
|
+
await fs.appendFile(queuePath, retractions.join("\n") + "\n");
|
|
912
|
+
if (cursors.cursorApi) {
|
|
913
|
+
cursors.cursorApi.lastRecordTimestamp = null;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
cursors.migrations[CURSOR_UNKNOWN_MIGRATION_KEY] = new Date().toISOString();
|
|
918
|
+
}
|
package/src/lib/cursor-config.js
CHANGED
|
@@ -220,62 +220,61 @@ function fetchUrlRaw({ urlStr, cookie, timeoutMs }) {
|
|
|
220
220
|
/**
|
|
221
221
|
* Parse Cursor usage CSV into structured records.
|
|
222
222
|
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
223
|
+
* Column order has changed multiple times (e.g. new "Cloud Agent ID",
|
|
224
|
+
* "Automation ID" columns inserted before "Kind"). Resolve columns by
|
|
225
|
+
* header name instead of fixed index so the parser keeps working across
|
|
226
|
+
* future Cursor updates.
|
|
226
227
|
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
228
|
+
* Known required columns: Date, Model, Input (w/ Cache Write),
|
|
229
|
+
* Input (w/o Cache Write), Cache Read, Output Tokens, Total Tokens, Cost.
|
|
230
|
+
* Optional: Kind, Max Mode.
|
|
230
231
|
*/
|
|
231
232
|
function parseCursorCsv(csvText) {
|
|
232
233
|
const lines = csvText.split("\n").filter((l) => l.trim().length > 0);
|
|
233
234
|
if (lines.length < 2) return [];
|
|
234
235
|
|
|
235
|
-
const
|
|
236
|
-
const
|
|
236
|
+
const headerFields = parseCsvLine(lines[0]).map((f) => stripQuotes(f));
|
|
237
|
+
const columnIndex = new Map();
|
|
238
|
+
for (let i = 0; i < headerFields.length; i++) {
|
|
239
|
+
columnIndex.set(headerFields[i], i);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const dateIdx = columnIndex.get("Date");
|
|
243
|
+
const modelIdx = columnIndex.get("Model");
|
|
244
|
+
const inputWithIdx = columnIndex.get("Input (w/ Cache Write)");
|
|
245
|
+
const inputWithoutIdx = columnIndex.get("Input (w/o Cache Write)");
|
|
246
|
+
const cacheReadIdx = columnIndex.get("Cache Read");
|
|
247
|
+
const outputIdx = columnIndex.get("Output Tokens");
|
|
248
|
+
const totalIdx = columnIndex.get("Total Tokens");
|
|
249
|
+
const costIdx = columnIndex.get("Cost");
|
|
250
|
+
const kindIdx = columnIndex.get("Kind");
|
|
251
|
+
const maxModeIdx = columnIndex.get("Max Mode");
|
|
252
|
+
|
|
253
|
+
const required = [dateIdx, modelIdx, inputWithIdx, inputWithoutIdx, cacheReadIdx, outputIdx, totalIdx, costIdx];
|
|
254
|
+
if (required.some((idx) => idx === undefined)) return [];
|
|
255
|
+
|
|
256
|
+
const minFields = Math.max(...required) + 1;
|
|
237
257
|
|
|
238
258
|
const records = [];
|
|
239
259
|
for (let i = 1; i < lines.length; i++) {
|
|
240
260
|
const fields = parseCsvLine(lines[i]);
|
|
241
|
-
if (!fields || fields.length <
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
totalTokens: toNum(fields[8]),
|
|
258
|
-
cost: toFloat(fields[9]),
|
|
259
|
-
};
|
|
260
|
-
} else {
|
|
261
|
-
// Date,Model,Input (w/ Cache Write),Input (w/o Cache Write),Cache Read,Output Tokens,Total Tokens,Cost,Cost to you
|
|
262
|
-
const inputWithCache = toNum(fields[2]);
|
|
263
|
-
const inputWithoutCache = toNum(fields[3]);
|
|
264
|
-
record = {
|
|
265
|
-
date: stripQuotes(fields[0]),
|
|
266
|
-
kind: "unknown",
|
|
267
|
-
model: stripQuotes(fields[1]),
|
|
268
|
-
maxMode: "No",
|
|
269
|
-
inputTokens: inputWithoutCache,
|
|
270
|
-
cacheWriteTokens: Math.max(0, inputWithCache - inputWithoutCache),
|
|
271
|
-
cacheReadTokens: toNum(fields[4]),
|
|
272
|
-
outputTokens: toNum(fields[5]),
|
|
273
|
-
totalTokens: toNum(fields[6]),
|
|
274
|
-
cost: toFloat(fields[7]),
|
|
275
|
-
};
|
|
276
|
-
}
|
|
261
|
+
if (!fields || fields.length < minFields) continue;
|
|
262
|
+
|
|
263
|
+
const inputWithCache = toNum(fields[inputWithIdx]);
|
|
264
|
+
const inputWithoutCache = toNum(fields[inputWithoutIdx]);
|
|
265
|
+
const record = {
|
|
266
|
+
date: stripQuotes(fields[dateIdx]),
|
|
267
|
+
kind: kindIdx !== undefined ? stripQuotes(fields[kindIdx]) : "unknown",
|
|
268
|
+
model: stripQuotes(fields[modelIdx]),
|
|
269
|
+
maxMode: maxModeIdx !== undefined ? stripQuotes(fields[maxModeIdx]) : "No",
|
|
270
|
+
inputTokens: inputWithoutCache,
|
|
271
|
+
cacheWriteTokens: Math.max(0, inputWithCache - inputWithoutCache),
|
|
272
|
+
cacheReadTokens: toNum(fields[cacheReadIdx]),
|
|
273
|
+
outputTokens: toNum(fields[outputIdx]),
|
|
274
|
+
totalTokens: toNum(fields[totalIdx]),
|
|
275
|
+
cost: toFloat(fields[costIdx]),
|
|
276
|
+
};
|
|
277
277
|
|
|
278
|
-
// Skip records with no tokens
|
|
279
278
|
if (record.totalTokens <= 0 && record.inputTokens <= 0 && record.outputTokens <= 0) continue;
|
|
280
279
|
|
|
281
280
|
records.push(record);
|
package/src/lib/rollout.js
CHANGED
|
@@ -2999,6 +2999,196 @@ async function parseHermesIncremental({ dbPath, cursors, queuePath, onProgress }
|
|
|
2999
2999
|
return { recordsProcessed: rows.length, eventsAggregated, bucketsQueued };
|
|
3000
3000
|
}
|
|
3001
3001
|
|
|
3002
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
3003
|
+
// GitHub Copilot CLI — OpenTelemetry JSONL exporter
|
|
3004
|
+
// User must opt in by setting:
|
|
3005
|
+
// COPILOT_OTEL_ENABLED=true
|
|
3006
|
+
// COPILOT_OTEL_EXPORTER_TYPE=file
|
|
3007
|
+
// COPILOT_OTEL_FILE_EXPORTER_PATH=$HOME/.copilot/otel/copilot-otel-...jsonl
|
|
3008
|
+
// We scan the default directory plus the env-overridden path.
|
|
3009
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
3010
|
+
|
|
3011
|
+
function resolveCopilotOtelPaths(env = process.env) {
|
|
3012
|
+
const home = require("node:os").homedir();
|
|
3013
|
+
const paths = new Set();
|
|
3014
|
+
const defaultDir = path.join(home, ".copilot", "otel");
|
|
3015
|
+
if (fssync.existsSync(defaultDir)) {
|
|
3016
|
+
try {
|
|
3017
|
+
for (const entry of fssync.readdirSync(defaultDir)) {
|
|
3018
|
+
if (entry.endsWith(".jsonl")) paths.add(path.join(defaultDir, entry));
|
|
3019
|
+
}
|
|
3020
|
+
} catch (_e) {}
|
|
3021
|
+
}
|
|
3022
|
+
const explicit = env.COPILOT_OTEL_FILE_EXPORTER_PATH;
|
|
3023
|
+
if (typeof explicit === "string" && explicit.trim() && fssync.existsSync(explicit)) {
|
|
3024
|
+
paths.add(explicit);
|
|
3025
|
+
}
|
|
3026
|
+
return Array.from(paths).sort();
|
|
3027
|
+
}
|
|
3028
|
+
|
|
3029
|
+
function isCopilotChatSpan(record) {
|
|
3030
|
+
if (!record || record.type !== "span") return false;
|
|
3031
|
+
const opName = record?.attributes?.["gen_ai.operation.name"];
|
|
3032
|
+
if (opName === "chat") return true;
|
|
3033
|
+
if (typeof record.name === "string" && record.name.startsWith("chat ")) return true;
|
|
3034
|
+
return false;
|
|
3035
|
+
}
|
|
3036
|
+
|
|
3037
|
+
function copilotOtelTimeToMs(value) {
|
|
3038
|
+
if (!Array.isArray(value) || value.length < 2) return null;
|
|
3039
|
+
const seconds = Number(value[0]);
|
|
3040
|
+
const nanos = Number(value[1]);
|
|
3041
|
+
if (!Number.isFinite(seconds)) return null;
|
|
3042
|
+
const ns = Number.isFinite(nanos) ? nanos : 0;
|
|
3043
|
+
return Math.round(seconds * 1000 + ns / 1_000_000);
|
|
3044
|
+
}
|
|
3045
|
+
|
|
3046
|
+
function pickCopilotModel(attrs) {
|
|
3047
|
+
const candidates = [attrs?.["gen_ai.response.model"], attrs?.["gen_ai.request.model"]];
|
|
3048
|
+
for (const c of candidates) {
|
|
3049
|
+
if (typeof c === "string" && c.trim()) return c.trim();
|
|
3050
|
+
}
|
|
3051
|
+
return null;
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
async function parseCopilotIncremental({ otelPaths, cursors, queuePath, onProgress, env } = {}) {
|
|
3055
|
+
await ensureDir(path.dirname(queuePath));
|
|
3056
|
+
const copilotState = cursors.copilot && typeof cursors.copilot === "object" ? cursors.copilot : {};
|
|
3057
|
+
const seenIds = new Set(Array.isArray(copilotState.seenIds) ? copilotState.seenIds : []);
|
|
3058
|
+
const fileOffsets =
|
|
3059
|
+
copilotState.fileOffsets && typeof copilotState.fileOffsets === "object"
|
|
3060
|
+
? { ...copilotState.fileOffsets }
|
|
3061
|
+
: {};
|
|
3062
|
+
|
|
3063
|
+
const files = Array.isArray(otelPaths) && otelPaths.length > 0
|
|
3064
|
+
? otelPaths
|
|
3065
|
+
: resolveCopilotOtelPaths(env || process.env);
|
|
3066
|
+
if (files.length === 0) {
|
|
3067
|
+
cursors.copilot = { ...copilotState, seenIds: Array.from(seenIds), fileOffsets, updatedAt: new Date().toISOString() };
|
|
3068
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
3072
|
+
const touchedBuckets = new Set();
|
|
3073
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
3074
|
+
let recordsProcessed = 0;
|
|
3075
|
+
let eventsAggregated = 0;
|
|
3076
|
+
|
|
3077
|
+
for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
|
|
3078
|
+
const filePath = files[fileIdx];
|
|
3079
|
+
let stat;
|
|
3080
|
+
try {
|
|
3081
|
+
stat = fssync.statSync(filePath);
|
|
3082
|
+
} catch (_e) {
|
|
3083
|
+
continue;
|
|
3084
|
+
}
|
|
3085
|
+
const prevEntry = fileOffsets[filePath] || {};
|
|
3086
|
+
const prevSize = Number(prevEntry.size) || 0;
|
|
3087
|
+
const prevIno = prevEntry.ino;
|
|
3088
|
+
// Re-read from start if (a) file shrunk (truncate/rewrite in place) or
|
|
3089
|
+
// (b) inode changed (rotator deleted + recreated at same path). Without
|
|
3090
|
+
// the inode check, a rotator producing a same-or-larger file would leave
|
|
3091
|
+
// the old offset stuck and skip the new file's prefix forever.
|
|
3092
|
+
const inodeChanged = typeof prevIno === "number" && prevIno !== stat.ino;
|
|
3093
|
+
const startOffset = stat.size < prevSize || inodeChanged ? 0 : prevSize;
|
|
3094
|
+
if (stat.size <= startOffset) continue;
|
|
3095
|
+
|
|
3096
|
+
let stream;
|
|
3097
|
+
try {
|
|
3098
|
+
stream = fssync.createReadStream(filePath, { encoding: "utf8", start: startOffset });
|
|
3099
|
+
} catch (_e) {
|
|
3100
|
+
continue;
|
|
3101
|
+
}
|
|
3102
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
3103
|
+
|
|
3104
|
+
for await (const line of rl) {
|
|
3105
|
+
if (!line || !line.trim()) continue;
|
|
3106
|
+
let record;
|
|
3107
|
+
try {
|
|
3108
|
+
record = JSON.parse(line);
|
|
3109
|
+
} catch (_e) {
|
|
3110
|
+
continue;
|
|
3111
|
+
}
|
|
3112
|
+
recordsProcessed++;
|
|
3113
|
+
if (!isCopilotChatSpan(record)) continue;
|
|
3114
|
+
|
|
3115
|
+
const traceId = record?.traceId || "";
|
|
3116
|
+
const spanId = record?.spanId || "";
|
|
3117
|
+
const dedupKey = traceId && spanId ? `${traceId}:${spanId}` : null;
|
|
3118
|
+
if (dedupKey && seenIds.has(dedupKey)) continue;
|
|
3119
|
+
|
|
3120
|
+
const attrs = record.attributes || {};
|
|
3121
|
+
const inputRaw = toNonNegativeInt(attrs["gen_ai.usage.input_tokens"]);
|
|
3122
|
+
const output = toNonNegativeInt(attrs["gen_ai.usage.output_tokens"]);
|
|
3123
|
+
const cacheRead = toNonNegativeInt(attrs["gen_ai.usage.cache_read.input_tokens"]);
|
|
3124
|
+
const cacheWrite = toNonNegativeInt(attrs["gen_ai.usage.cache_write.input_tokens"]);
|
|
3125
|
+
const reasoning = toNonNegativeInt(attrs["gen_ai.usage.reasoning.output_tokens"]);
|
|
3126
|
+
// OTEL input_tokens INCLUDES cache_read — subtract per project convention
|
|
3127
|
+
const cacheReadClamped = Math.min(cacheRead, inputRaw);
|
|
3128
|
+
const input = Math.max(0, inputRaw - cacheReadClamped);
|
|
3129
|
+
const totalInteresting = input + output + cacheReadClamped + cacheWrite + reasoning;
|
|
3130
|
+
// Drop empty rows unless cache-only
|
|
3131
|
+
if (totalInteresting === 0) continue;
|
|
3132
|
+
|
|
3133
|
+
const tsMs = copilotOtelTimeToMs(record.endTime) || copilotOtelTimeToMs(record.startTime);
|
|
3134
|
+
if (!tsMs) continue;
|
|
3135
|
+
const tsIso = new Date(tsMs).toISOString();
|
|
3136
|
+
const bucketStart = toUtcHalfHourStart(tsIso);
|
|
3137
|
+
if (!bucketStart) continue;
|
|
3138
|
+
|
|
3139
|
+
const model = normalizeModelInput(pickCopilotModel(attrs)) || "github-copilot";
|
|
3140
|
+
|
|
3141
|
+
const delta = {
|
|
3142
|
+
input_tokens: input,
|
|
3143
|
+
cached_input_tokens: cacheReadClamped,
|
|
3144
|
+
cache_creation_input_tokens: cacheWrite,
|
|
3145
|
+
output_tokens: output,
|
|
3146
|
+
reasoning_output_tokens: reasoning,
|
|
3147
|
+
total_tokens: input + output + cacheReadClamped + cacheWrite + reasoning,
|
|
3148
|
+
conversation_count: 1,
|
|
3149
|
+
};
|
|
3150
|
+
|
|
3151
|
+
const bucket = getHourlyBucket(hourlyState, "copilot", model, bucketStart);
|
|
3152
|
+
addTotals(bucket.totals, delta);
|
|
3153
|
+
touchedBuckets.add(bucketKey("copilot", model, bucketStart));
|
|
3154
|
+
eventsAggregated++;
|
|
3155
|
+
if (dedupKey) seenIds.add(dedupKey);
|
|
3156
|
+
|
|
3157
|
+
if (cb) {
|
|
3158
|
+
cb({
|
|
3159
|
+
index: fileIdx + 1,
|
|
3160
|
+
total: files.length,
|
|
3161
|
+
recordsProcessed,
|
|
3162
|
+
eventsAggregated,
|
|
3163
|
+
bucketsQueued: touchedBuckets.size,
|
|
3164
|
+
});
|
|
3165
|
+
}
|
|
3166
|
+
}
|
|
3167
|
+
|
|
3168
|
+
// Re-stat after readline drains: file may have been appended during the
|
|
3169
|
+
// parse loop. Without this, those new lines would be replayed next run
|
|
3170
|
+
// (dedup catches records with traceId+spanId, but spans missing either
|
|
3171
|
+
// would be double-counted).
|
|
3172
|
+
let postStat = stat;
|
|
3173
|
+
try {
|
|
3174
|
+
postStat = fssync.statSync(filePath);
|
|
3175
|
+
} catch (_e) {}
|
|
3176
|
+
fileOffsets[filePath] = { size: postStat.size, mtimeMs: postStat.mtimeMs, ino: postStat.ino };
|
|
3177
|
+
}
|
|
3178
|
+
|
|
3179
|
+
// Cap dedup set to last 10k IDs to bound state size
|
|
3180
|
+
const seenArr = Array.from(seenIds);
|
|
3181
|
+
const cappedSeen = seenArr.length > 10_000 ? seenArr.slice(seenArr.length - 10_000) : seenArr;
|
|
3182
|
+
|
|
3183
|
+
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
3184
|
+
const updatedAt = new Date().toISOString();
|
|
3185
|
+
hourlyState.updatedAt = updatedAt;
|
|
3186
|
+
cursors.hourly = hourlyState;
|
|
3187
|
+
cursors.copilot = { ...copilotState, seenIds: cappedSeen, fileOffsets, updatedAt };
|
|
3188
|
+
|
|
3189
|
+
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
3190
|
+
}
|
|
3191
|
+
|
|
3002
3192
|
module.exports = {
|
|
3003
3193
|
listRolloutFiles,
|
|
3004
3194
|
listClaudeProjectFiles,
|
|
@@ -3008,6 +3198,7 @@ module.exports = {
|
|
|
3008
3198
|
resolveKiroDbPath,
|
|
3009
3199
|
resolveKiroJsonlPath,
|
|
3010
3200
|
resolveHermesDbPath,
|
|
3201
|
+
resolveCopilotOtelPaths,
|
|
3011
3202
|
parseRolloutIncremental,
|
|
3012
3203
|
parseClaudeIncremental,
|
|
3013
3204
|
parseGeminiIncremental,
|
|
@@ -3017,4 +3208,5 @@ module.exports = {
|
|
|
3017
3208
|
parseCursorApiIncremental,
|
|
3018
3209
|
parseKiroIncremental,
|
|
3019
3210
|
parseHermesIncremental,
|
|
3211
|
+
parseCopilotIncremental,
|
|
3020
3212
|
};
|