pi-tokenburn 0.5.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -4
- package/extensions/command/completions.ts +80 -0
- package/extensions/command/handlers.ts +22 -0
- package/extensions/command/host.ts +25 -0
- package/extensions/command/parse.ts +173 -0
- package/extensions/core/aggregate.ts +155 -0
- package/extensions/core/config.ts +157 -0
- package/extensions/core/format.ts +12 -0
- package/extensions/core/paint.ts +35 -0
- package/extensions/core/parser.ts +261 -0
- package/extensions/core/select.ts +105 -0
- package/extensions/core/time.ts +25 -0
- package/extensions/core/types.ts +43 -0
- package/extensions/features/breakdown/commands.ts +32 -0
- package/extensions/features/breakdown/query.ts +138 -0
- package/extensions/features/breakdown/table.ts +100 -0
- package/extensions/features/budgets/budgets.ts +105 -0
- package/extensions/features/budgets/commands.ts +20 -0
- package/extensions/features/export/commands.ts +19 -0
- package/extensions/features/export/export.ts +61 -0
- package/extensions/features/filter/commands.ts +18 -0
- package/extensions/features/live/coalescer.ts +71 -0
- package/extensions/features/pricing/commands.ts +30 -0
- package/extensions/features/pricing/report.ts +59 -0
- package/extensions/features/report/chart.ts +96 -0
- package/extensions/features/report/commands.ts +33 -0
- package/extensions/features/report/report.ts +47 -0
- package/extensions/features/status-line/commands.ts +78 -0
- package/extensions/features/status-line/render.ts +43 -0
- package/extensions/features/status-line/status.ts +35 -0
- package/extensions/tokenburn.ts +78 -1030
- package/package.json +5 -3
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/** Config file (tokenburn.json), path resolution, budgets. */
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
|
5
|
+
import { sanitizeFilter, sanitizePrices, type PriceMode, type PriceTable, type UsageFilter } from "./select.ts";
|
|
6
|
+
import { STATUS_WINDOWS, type PeriodStats, type Window } from "./types.ts";
|
|
7
|
+
|
|
8
|
+
export interface TokenBurnConfig {
|
|
9
|
+
/** Master switch. When false nothing runs in the background and the status line / panel are cleared. */
|
|
10
|
+
enabled: boolean;
|
|
11
|
+
/** Refresh after each assistant message (coalesced), not only when a turn ends. */
|
|
12
|
+
liveRefresh: boolean;
|
|
13
|
+
/** Trailing delay for live refreshes, in ms (250 to 60000). */
|
|
14
|
+
refreshMs: number;
|
|
15
|
+
/** Count cache read/write tokens in totals (matches the tokenburn CLI). */
|
|
16
|
+
includeCache: boolean;
|
|
17
|
+
/** Color the report chart (set false if your terminal garbles ANSI in notifications). */
|
|
18
|
+
chartColor: boolean;
|
|
19
|
+
showStatus: boolean;
|
|
20
|
+
showWidget: boolean;
|
|
21
|
+
statusWindow: Window;
|
|
22
|
+
/** Show the burn rate (tokens/hour averaged over the status window) in the status line. */
|
|
23
|
+
showRate: boolean;
|
|
24
|
+
/** Show the cache hit ratio of the status window in the status line. */
|
|
25
|
+
showHit: boolean;
|
|
26
|
+
/** Show the estimated cache savings of the status window in the status line. */
|
|
27
|
+
showSaved: boolean;
|
|
28
|
+
/** Notify once when a budget reaches this percentage (1 to 100); 0 turns warnings off. */
|
|
29
|
+
warnPct: number;
|
|
30
|
+
/** Only count usage matching these (case-insensitive substrings). Empty = everything. */
|
|
31
|
+
filter: UsageFilter;
|
|
32
|
+
/** Price table in $ per million tokens, keyed by `model` or `provider/model`. */
|
|
33
|
+
prices: PriceTable;
|
|
34
|
+
/** `fill`: use the table only where the log has no cost. `override`: table wins over the log. */
|
|
35
|
+
priceMode: PriceMode;
|
|
36
|
+
budgets: {
|
|
37
|
+
dayTokens?: number;
|
|
38
|
+
weekTokens?: number;
|
|
39
|
+
monthTokens?: number;
|
|
40
|
+
dayCost?: number;
|
|
41
|
+
weekCost?: number;
|
|
42
|
+
monthCost?: number;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export const DEFAULT_CONFIG: TokenBurnConfig = {
|
|
47
|
+
enabled: true,
|
|
48
|
+
liveRefresh: true,
|
|
49
|
+
refreshMs: 750,
|
|
50
|
+
includeCache: true,
|
|
51
|
+
chartColor: true,
|
|
52
|
+
showStatus: true,
|
|
53
|
+
showWidget: false,
|
|
54
|
+
statusWindow: "today",
|
|
55
|
+
showRate: true,
|
|
56
|
+
showHit: true,
|
|
57
|
+
showSaved: true,
|
|
58
|
+
warnPct: 80,
|
|
59
|
+
filter: {},
|
|
60
|
+
prices: {},
|
|
61
|
+
priceMode: "fill",
|
|
62
|
+
budgets: {},
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** Where pi keeps its config: PI_CODING_AGENT_DIR, then $XDG_CONFIG_HOME/pi/agent, then ~/.pi/agent. */
|
|
66
|
+
export function resolveAgentDir(env: Record<string, string | undefined>, home: string): string {
|
|
67
|
+
if (env.PI_CODING_AGENT_DIR) return env.PI_CODING_AGENT_DIR;
|
|
68
|
+
if (env.XDG_CONFIG_HOME) return join(env.XDG_CONFIG_HOME, "pi", "agent");
|
|
69
|
+
return join(home, ".pi", "agent");
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Pi session logs: TOKENBURN_PI_SESSIONS, PI_CODING_AGENT_SESSION_DIR, then <agent dir>/sessions. */
|
|
73
|
+
export function resolveSessionsDir(env: Record<string, string | undefined>, home: string): string {
|
|
74
|
+
if (env.TOKENBURN_PI_SESSIONS) return env.TOKENBURN_PI_SESSIONS;
|
|
75
|
+
if (env.PI_CODING_AGENT_SESSION_DIR) return env.PI_CODING_AGENT_SESSION_DIR;
|
|
76
|
+
return join(resolveAgentDir(env, home), "sessions");
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export const getConfigDir = () => resolveAgentDir(process.env, homedir());
|
|
80
|
+
|
|
81
|
+
export const getSessionsDir = () => resolveSessionsDir(process.env, homedir());
|
|
82
|
+
|
|
83
|
+
export async function loadConfig(): Promise<TokenBurnConfig> {
|
|
84
|
+
try {
|
|
85
|
+
const raw = await readFile(join(getConfigDir(), "tokenburn.json"), "utf8");
|
|
86
|
+
const parsed = JSON.parse(raw);
|
|
87
|
+
return {
|
|
88
|
+
enabled: typeof parsed.enabled === "boolean" ? parsed.enabled : DEFAULT_CONFIG.enabled,
|
|
89
|
+
liveRefresh: typeof parsed.liveRefresh === "boolean" ? parsed.liveRefresh : DEFAULT_CONFIG.liveRefresh,
|
|
90
|
+
refreshMs: sanitizeRefreshMs(parsed.refreshMs),
|
|
91
|
+
includeCache: typeof parsed.includeCache === "boolean" ? parsed.includeCache : DEFAULT_CONFIG.includeCache,
|
|
92
|
+
chartColor: typeof parsed.chartColor === "boolean" ? parsed.chartColor : DEFAULT_CONFIG.chartColor,
|
|
93
|
+
showStatus: typeof parsed.showStatus === "boolean" ? parsed.showStatus : DEFAULT_CONFIG.showStatus,
|
|
94
|
+
showWidget: typeof parsed.showWidget === "boolean" ? parsed.showWidget : DEFAULT_CONFIG.showWidget,
|
|
95
|
+
statusWindow: (STATUS_WINDOWS as readonly string[]).includes(parsed.statusWindow)
|
|
96
|
+
? parsed.statusWindow
|
|
97
|
+
: DEFAULT_CONFIG.statusWindow,
|
|
98
|
+
showRate: typeof parsed.showRate === "boolean" ? parsed.showRate : DEFAULT_CONFIG.showRate,
|
|
99
|
+
showHit: typeof parsed.showHit === "boolean" ? parsed.showHit : DEFAULT_CONFIG.showHit,
|
|
100
|
+
showSaved: typeof parsed.showSaved === "boolean" ? parsed.showSaved : DEFAULT_CONFIG.showSaved,
|
|
101
|
+
warnPct: sanitizeWarnPct(parsed.warnPct),
|
|
102
|
+
filter: sanitizeFilter(parsed.filter),
|
|
103
|
+
prices: sanitizePrices(parsed.prices),
|
|
104
|
+
priceMode: parsed.priceMode === "override" ? "override" : "fill",
|
|
105
|
+
budgets: sanitizeBudgets(parsed.budgets),
|
|
106
|
+
};
|
|
107
|
+
} catch {
|
|
108
|
+
return freshConfig();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Live-refresh delay: a finite number clamped to 250..60000 ms, else the default. */
|
|
113
|
+
export function sanitizeRefreshMs(value: unknown): number {
|
|
114
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_CONFIG.refreshMs;
|
|
115
|
+
return Math.min(60_000, Math.max(250, Math.round(value)));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Warn threshold: a finite number 1..100, 0 = off, anything else the default. */
|
|
119
|
+
export function sanitizeWarnPct(value: unknown): number {
|
|
120
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_CONFIG.warnPct;
|
|
121
|
+
return value <= 0 ? 0 : Math.min(100, Math.round(value));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const BUDGET_KEYS = ["dayTokens", "weekTokens", "monthTokens", "dayCost", "weekCost", "monthCost"] as const;
|
|
125
|
+
|
|
126
|
+
/** Keep only known budget keys holding positive, finite numbers (the file is hand-editable). */
|
|
127
|
+
export function sanitizeBudgets(value: unknown): TokenBurnConfig["budgets"] {
|
|
128
|
+
const out: TokenBurnConfig["budgets"] = {};
|
|
129
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return out;
|
|
130
|
+
for (const key of BUDGET_KEYS) {
|
|
131
|
+
const n = (value as Record<string, unknown>)[key];
|
|
132
|
+
if (typeof n === "number" && Number.isFinite(n) && n > 0) out[key] = n;
|
|
133
|
+
}
|
|
134
|
+
return out;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** A config that shares no mutable state with DEFAULT_CONFIG. */
|
|
138
|
+
export function freshConfig(): TokenBurnConfig {
|
|
139
|
+
return { ...DEFAULT_CONFIG, budgets: {}, filter: {}, prices: {} };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export async function saveConfig(cfg: TokenBurnConfig): Promise<void> {
|
|
143
|
+
await mkdir(getConfigDir(), { recursive: true });
|
|
144
|
+
await writeFile(join(getConfigDir(), "tokenburn.json"), JSON.stringify(cfg, null, 2) + "\n", "utf8");
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export function budgetsFor(cfg: TokenBurnConfig, w: Window): { tokens?: number; cost?: number } {
|
|
148
|
+
const b = cfg.budgets;
|
|
149
|
+
if (w === "today") return { tokens: b.dayTokens, cost: b.dayCost };
|
|
150
|
+
if (w === "week") return { tokens: b.weekTokens, cost: b.weekCost };
|
|
151
|
+
if (w === "month") return { tokens: b.monthTokens, cost: b.monthCost };
|
|
152
|
+
return {};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function isOverBudget(p: PeriodStats, b: { tokens?: number; cost?: number }): boolean {
|
|
156
|
+
return (b.tokens !== undefined && p.tokens > b.tokens) || (b.cost !== undefined && p.cost > b.cost);
|
|
157
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Number and cost formatting. */
|
|
2
|
+
|
|
3
|
+
export function formatNum(n: number): string {
|
|
4
|
+
if (n >= 1_000_000_000) return (n / 1_000_000_000).toFixed(2) + "B";
|
|
5
|
+
if (n >= 1_000_000) return (n / 1_000_000).toFixed(2) + "M";
|
|
6
|
+
if (n >= 1_000) return (n / 1_000).toFixed(1) + "k";
|
|
7
|
+
return n.toString();
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function formatCost(c: number): string {
|
|
11
|
+
return `$${c.toFixed(2)}`;
|
|
12
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/** Terminal paint helpers shared by charts and tables: bars, sparklines, theme colors. */
|
|
2
|
+
|
|
3
|
+
const EIGHTHS = ["", "▏", "▎", "▍", "▌", "▋", "▊", "▉"];
|
|
4
|
+
export const SPARK = "▁▂▃▄▅▆▇█";
|
|
5
|
+
|
|
6
|
+
/** Horizontal bar of `value/max` over `width` cells, with 1/8-cell precision. */
|
|
7
|
+
export function bar(value: number, max: number, width: number): string {
|
|
8
|
+
if (!(max > 0) || !(value > 0) || !(width > 0)) return "";
|
|
9
|
+
// Capped at the full width: a value above `max` (or Infinity) must not build a giant string.
|
|
10
|
+
const eighths = Math.min(Math.floor(width) * 8, Math.max(1, Math.round((value / max) * width * 8)));
|
|
11
|
+
const full = Math.floor(eighths / 8);
|
|
12
|
+
return "█".repeat(full) + EIGHTHS[eighths % 8];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function sparkline(values: number[]): string {
|
|
16
|
+
const max = Math.max(...values, 0);
|
|
17
|
+
if (max <= 0) return SPARK[0].repeat(values.length);
|
|
18
|
+
return values.map((v) => SPARK[Math.min(7, Math.round((v / max) * 7))]).join("");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type Paint = (kind: "bar" | "now" | "over" | "axis" | "budget" | "dim" | "title", text: string) => string;
|
|
22
|
+
|
|
23
|
+
/** Map chart roles to pi theme colors. */
|
|
24
|
+
export function themePaint(theme: { fg: (c: any, t: string) => string }): Paint {
|
|
25
|
+
const color: Record<string, string> = {
|
|
26
|
+
bar: "accent",
|
|
27
|
+
now: "success",
|
|
28
|
+
over: "error",
|
|
29
|
+
axis: "dim",
|
|
30
|
+
budget: "warning",
|
|
31
|
+
dim: "dim",
|
|
32
|
+
title: "accent",
|
|
33
|
+
};
|
|
34
|
+
return (kind, text) => theme.fg(color[kind], text);
|
|
35
|
+
}
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session-log parser with a per-file incremental cache. Pi session logs are append-only
|
|
3
|
+
* JSONL, so after the first parse only the appended bytes are read.
|
|
4
|
+
*/
|
|
5
|
+
import { readdirSync, statSync } from "node:fs";
|
|
6
|
+
import { open, type FileHandle } from "node:fs/promises";
|
|
7
|
+
import { basename, dirname, join } from "node:path";
|
|
8
|
+
|
|
9
|
+
export interface Turn {
|
|
10
|
+
ts: number;
|
|
11
|
+
input: number;
|
|
12
|
+
output: number;
|
|
13
|
+
cacheRead: number;
|
|
14
|
+
cacheWrite: number;
|
|
15
|
+
cost: number;
|
|
16
|
+
/** Cost split, used to estimate what prompt caching saved. */
|
|
17
|
+
costInput: number;
|
|
18
|
+
costCacheRead: number;
|
|
19
|
+
provider: string;
|
|
20
|
+
model: string;
|
|
21
|
+
/** Working directory of the session (falls back to the session directory name). */
|
|
22
|
+
project: string;
|
|
23
|
+
/** Session file path. */
|
|
24
|
+
session: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Per-file context for a parse: where the file is and the cwd from its header line. */
|
|
28
|
+
interface Meta {
|
|
29
|
+
file: string;
|
|
30
|
+
cwd?: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** What was read from disk, for tests and tuning: refreshes must cost O(new bytes). */
|
|
34
|
+
export const stats = { bytesRead: 0, fullParses: 0, incrementalParses: 0 };
|
|
35
|
+
|
|
36
|
+
export function resetStats(): void {
|
|
37
|
+
stats.bytesRead = 0;
|
|
38
|
+
stats.fullParses = 0;
|
|
39
|
+
stats.incrementalParses = 0;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Per-session-file parse state. Pi session logs are append-only JSONL, so after the first
|
|
44
|
+
* parse only the bytes added since then are read. The state is dropped (full reparse) when
|
|
45
|
+
* the file shrank, was rewritten in place, or its head / last-parsed bytes no longer match.
|
|
46
|
+
*/
|
|
47
|
+
interface FileEntry {
|
|
48
|
+
size: number;
|
|
49
|
+
mtimeMs: number;
|
|
50
|
+
/** End of the last COMPLETE (newline-terminated) line that was parsed. */
|
|
51
|
+
offset: number;
|
|
52
|
+
turns: Turn[];
|
|
53
|
+
/** Turn from an unterminated final line; counted now, re-parsed once the line completes. */
|
|
54
|
+
tail: Turn[];
|
|
55
|
+
cwd?: string;
|
|
56
|
+
/** First bytes of the file, to notice a rewrite. */
|
|
57
|
+
head: Buffer;
|
|
58
|
+
/** The bytes just before `offset`, same purpose. */
|
|
59
|
+
guard: Buffer;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const FILE_CACHE = new Map<string, FileEntry>();
|
|
63
|
+
const GUARD_BYTES = 64;
|
|
64
|
+
const HEAD_BYTES = 256;
|
|
65
|
+
export const CHUNK_BYTES = 1 << 20;
|
|
66
|
+
|
|
67
|
+
/** A Turn from one JSONL line, or undefined when it carries no usage. */
|
|
68
|
+
/** A finite number, else 0 (logs can hold 1e999, strings, junk). */
|
|
69
|
+
const finite = (v: unknown): number => {
|
|
70
|
+
const n = Number(v);
|
|
71
|
+
return Number.isFinite(n) ? n : 0;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
function turnFromLine(line: Buffer, meta: Meta): Turn | undefined {
|
|
75
|
+
if (meta.cwd === undefined && line.includes('"type":"session"')) {
|
|
76
|
+
try {
|
|
77
|
+
const h = JSON.parse(line.toString("utf8"));
|
|
78
|
+
if (h.type === "session" && typeof h.cwd === "string") meta.cwd = h.cwd;
|
|
79
|
+
} catch {
|
|
80
|
+
/* not a header */
|
|
81
|
+
}
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
|
84
|
+
if (!line.includes('"usage"')) return undefined; // most lines: skip without a JSON parse
|
|
85
|
+
try {
|
|
86
|
+
const obj = JSON.parse(line.toString("utf8"));
|
|
87
|
+
const usage = obj.message?.usage;
|
|
88
|
+
if (!obj.timestamp || !usage) return undefined;
|
|
89
|
+
const ts = new Date(obj.timestamp).getTime();
|
|
90
|
+
if (!Number.isFinite(ts)) return undefined;
|
|
91
|
+
return {
|
|
92
|
+
ts,
|
|
93
|
+
input: finite(usage.input),
|
|
94
|
+
output: finite(usage.output),
|
|
95
|
+
cacheRead: finite(usage.cacheRead),
|
|
96
|
+
cacheWrite: finite(usage.cacheWrite),
|
|
97
|
+
cost: finite(usage.cost?.total),
|
|
98
|
+
costInput: finite(usage.cost?.input),
|
|
99
|
+
costCacheRead: finite(usage.cost?.cacheRead),
|
|
100
|
+
provider: String(obj.message?.provider ?? "unknown"),
|
|
101
|
+
model: String(obj.message?.model ?? "unknown"),
|
|
102
|
+
project: meta.cwd ?? basename(dirname(meta.file)),
|
|
103
|
+
session: meta.file,
|
|
104
|
+
};
|
|
105
|
+
} catch {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function readAt(fd: FileHandle, buf: Buffer, position: number): Promise<number> {
|
|
111
|
+
const { bytesRead } = await fd.read(buf, 0, buf.length, position);
|
|
112
|
+
stats.bytesRead += bytesRead;
|
|
113
|
+
return bytesRead;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Parse complete lines in [start, end). Returns their turns, where they stop, and the unterminated rest. */
|
|
117
|
+
async function scan(
|
|
118
|
+
fd: FileHandle,
|
|
119
|
+
start: number,
|
|
120
|
+
end: number,
|
|
121
|
+
chunkSize: number,
|
|
122
|
+
meta: Meta,
|
|
123
|
+
): Promise<{ turns: Turn[]; consumed: number; rest: Buffer }> {
|
|
124
|
+
const turns: Turn[] = [];
|
|
125
|
+
const buf = Buffer.allocUnsafe(Math.min(chunkSize, Math.max(1, end - start)));
|
|
126
|
+
let pos = start;
|
|
127
|
+
let consumed = start;
|
|
128
|
+
let carry: Buffer = Buffer.alloc(0);
|
|
129
|
+
while (pos < end) {
|
|
130
|
+
const want = Math.min(buf.length, end - pos);
|
|
131
|
+
const n = await readAt(fd, want === buf.length ? buf : buf.subarray(0, want), pos);
|
|
132
|
+
if (n === 0) break;
|
|
133
|
+
const base = pos - carry.length;
|
|
134
|
+
const data = carry.length > 0 ? Buffer.concat([carry, buf.subarray(0, n)]) : buf.subarray(0, n);
|
|
135
|
+
let lineStart = 0;
|
|
136
|
+
for (let nl = data.indexOf(10, lineStart); nl !== -1; nl = data.indexOf(10, lineStart)) {
|
|
137
|
+
const t = turnFromLine(data.subarray(lineStart, nl), meta);
|
|
138
|
+
if (t) turns.push(t);
|
|
139
|
+
lineStart = nl + 1;
|
|
140
|
+
}
|
|
141
|
+
if (lineStart > 0) consumed = base + lineStart;
|
|
142
|
+
carry = Buffer.from(data.subarray(lineStart)); // copy: `buf` is reused
|
|
143
|
+
pos += n;
|
|
144
|
+
}
|
|
145
|
+
return { turns, consumed, rest: carry };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function bytesAt(fd: FileHandle, position: number, length: number): Promise<Buffer> {
|
|
149
|
+
if (length <= 0) return Buffer.alloc(0);
|
|
150
|
+
const buf = Buffer.alloc(length);
|
|
151
|
+
const n = await readAt(fd, buf, position);
|
|
152
|
+
return buf.subarray(0, n);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Is the prefix we parsed earlier still what is on disk? */
|
|
156
|
+
async function stillAppendOnly(fd: FileHandle, prev: FileEntry): Promise<boolean> {
|
|
157
|
+
const head = await bytesAt(fd, 0, prev.head.length);
|
|
158
|
+
if (!head.equals(prev.head)) return false;
|
|
159
|
+
const guard = await bytesAt(fd, prev.offset - prev.guard.length, prev.guard.length);
|
|
160
|
+
return guard.equals(prev.guard);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async function refreshEntry(
|
|
164
|
+
file: string,
|
|
165
|
+
st: { size: number; mtimeMs: number },
|
|
166
|
+
prev: FileEntry | undefined,
|
|
167
|
+
chunkSize: number,
|
|
168
|
+
): Promise<FileEntry | undefined> {
|
|
169
|
+
let fd: FileHandle;
|
|
170
|
+
try {
|
|
171
|
+
fd = await open(file, "r");
|
|
172
|
+
} catch {
|
|
173
|
+
return undefined;
|
|
174
|
+
}
|
|
175
|
+
try {
|
|
176
|
+
const grew = prev !== undefined && st.size > prev.size;
|
|
177
|
+
const incremental = grew && (await stillAppendOnly(fd, prev!));
|
|
178
|
+
const start = incremental ? prev!.offset : 0;
|
|
179
|
+
const turns = incremental ? prev!.turns : [];
|
|
180
|
+
if (incremental) stats.incrementalParses++;
|
|
181
|
+
else stats.fullParses++;
|
|
182
|
+
|
|
183
|
+
const meta: Meta = { file, cwd: incremental ? prev!.cwd : undefined };
|
|
184
|
+
const r = await scan(fd, start, st.size, chunkSize, meta);
|
|
185
|
+
for (const t of r.turns) turns.push(t);
|
|
186
|
+
const tailTurn = r.rest.length > 0 ? turnFromLine(r.rest, meta) : undefined;
|
|
187
|
+
return {
|
|
188
|
+
size: st.size,
|
|
189
|
+
mtimeMs: st.mtimeMs,
|
|
190
|
+
offset: r.consumed,
|
|
191
|
+
turns,
|
|
192
|
+
tail: tailTurn ? [tailTurn] : [],
|
|
193
|
+
cwd: meta.cwd,
|
|
194
|
+
head: incremental ? prev!.head : await bytesAt(fd, 0, Math.min(HEAD_BYTES, st.size)),
|
|
195
|
+
guard: await bytesAt(fd, Math.max(0, r.consumed - GUARD_BYTES), Math.min(GUARD_BYTES, r.consumed)),
|
|
196
|
+
};
|
|
197
|
+
} catch {
|
|
198
|
+
return undefined;
|
|
199
|
+
} finally {
|
|
200
|
+
await fd.close();
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Size and mtime of a file, or undefined when it cannot be stat'ed. */
|
|
205
|
+
export function fileStat(file: string): { size: number; mtimeMs: number } | undefined {
|
|
206
|
+
try {
|
|
207
|
+
const st = statSync(file);
|
|
208
|
+
return { size: st.size, mtimeMs: st.mtimeMs };
|
|
209
|
+
} catch {
|
|
210
|
+
return undefined;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function listSessionFiles(root: string): string[] {
|
|
215
|
+
const files: string[] = [];
|
|
216
|
+
const walk = (dir: string) => {
|
|
217
|
+
let entries;
|
|
218
|
+
try {
|
|
219
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
220
|
+
} catch {
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
for (const e of entries) {
|
|
224
|
+
const full = join(dir, e.name);
|
|
225
|
+
if (e.isDirectory()) walk(full);
|
|
226
|
+
else if (e.isFile() && e.name.endsWith(".jsonl")) files.push(full);
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
walk(root);
|
|
230
|
+
return files;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Bring the per-file cache up to date (reading only what changed) and return every file's turns. */
|
|
234
|
+
export async function loadAll(sessionsRoot: string, chunkSize = CHUNK_BYTES): Promise<Turn[][]> {
|
|
235
|
+
const files = listSessionFiles(sessionsRoot);
|
|
236
|
+
const live = new Set(files);
|
|
237
|
+
for (const key of FILE_CACHE.keys()) if (!live.has(key)) FILE_CACHE.delete(key);
|
|
238
|
+
|
|
239
|
+
const out: Turn[][] = [];
|
|
240
|
+
for (const file of files) {
|
|
241
|
+
const st = fileStat(file);
|
|
242
|
+
if (st === undefined) continue; // vanished between listing and stat
|
|
243
|
+
let entry = FILE_CACHE.get(file);
|
|
244
|
+
if (!entry || entry.size !== st.size || entry.mtimeMs !== st.mtimeMs) {
|
|
245
|
+
const fresh = await refreshEntry(file, st, entry, chunkSize);
|
|
246
|
+
if (fresh) FILE_CACHE.set(file, fresh);
|
|
247
|
+
else FILE_CACHE.delete(file);
|
|
248
|
+
entry = fresh;
|
|
249
|
+
}
|
|
250
|
+
if (entry) out.push(entry.tail.length > 0 ? entry.turns.concat(entry.tail) : entry.turns);
|
|
251
|
+
}
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** Test hooks: parse with a tiny chunk size (chunk boundaries) / refresh one file directly. */
|
|
256
|
+
export const __loadAllForTests = loadAll;
|
|
257
|
+
export const __refreshEntryForTests = refreshEntry;
|
|
258
|
+
|
|
259
|
+
export function clearCache(): void {
|
|
260
|
+
FILE_CACHE.clear();
|
|
261
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage filter + price table. Costs normally come straight from the session logs (pi computes
|
|
3
|
+
* them from its own model prices). The price table lets you fill in models pi had no price for
|
|
4
|
+
* (`fill`) or correct stale prices (`override`).
|
|
5
|
+
*/
|
|
6
|
+
import type { Turn } from "./parser.ts";
|
|
7
|
+
|
|
8
|
+
export interface UsageFilter {
|
|
9
|
+
provider?: string;
|
|
10
|
+
model?: string;
|
|
11
|
+
project?: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** $ per million tokens. */
|
|
15
|
+
export interface Price {
|
|
16
|
+
input: number;
|
|
17
|
+
output: number;
|
|
18
|
+
cacheRead: number;
|
|
19
|
+
cacheWrite: number;
|
|
20
|
+
}
|
|
21
|
+
export type PriceTable = Record<string, Price>;
|
|
22
|
+
export type PriceMode = "fill" | "override";
|
|
23
|
+
|
|
24
|
+
/** Options every collector accepts. */
|
|
25
|
+
export interface UsageOptions {
|
|
26
|
+
filter?: UsageFilter;
|
|
27
|
+
prices?: PriceTable;
|
|
28
|
+
priceMode?: PriceMode;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const FILTER_KEYS = ["provider", "model", "project"] as const;
|
|
32
|
+
export type FilterKey = (typeof FILTER_KEYS)[number];
|
|
33
|
+
export const FILTER_KEYS_LIST: readonly FilterKey[] = FILTER_KEYS;
|
|
34
|
+
|
|
35
|
+
/** Keep only known keys holding non-empty strings (the file is hand-editable). */
|
|
36
|
+
export function sanitizeFilter(value: unknown): UsageFilter {
|
|
37
|
+
const out: UsageFilter = {};
|
|
38
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return out;
|
|
39
|
+
for (const k of FILTER_KEYS) {
|
|
40
|
+
const v = (value as Record<string, unknown>)[k];
|
|
41
|
+
if (typeof v === "string" && v.trim() !== "") out[k] = v.trim();
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const num = (v: unknown) => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined);
|
|
47
|
+
|
|
48
|
+
/** Keep only entries with a finite, non-negative input and output price. */
|
|
49
|
+
export function sanitizePrices(value: unknown): PriceTable {
|
|
50
|
+
const out: PriceTable = {};
|
|
51
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return out;
|
|
52
|
+
for (const [name, p] of Object.entries(value as Record<string, unknown>)) {
|
|
53
|
+
if (typeof p !== "object" || p === null) continue;
|
|
54
|
+
const r = p as Record<string, unknown>;
|
|
55
|
+
const input = num(r.input);
|
|
56
|
+
const output = num(r.output);
|
|
57
|
+
if (input === undefined || output === undefined) continue;
|
|
58
|
+
out[name] = { input, output, cacheRead: num(r.cacheRead) ?? input, cacheWrite: num(r.cacheWrite) ?? input };
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export const hasFilter = (f: UsageFilter | undefined) => !!f && (!!f.provider || !!f.model || !!f.project);
|
|
64
|
+
|
|
65
|
+
/** Human text for an active filter, e.g. `provider=anthropic model=opus`. */
|
|
66
|
+
export function describeFilter(f: UsageFilter | undefined): string {
|
|
67
|
+
return FILTER_KEYS.filter((k) => f?.[k]).map((k) => `${k}=${f![k]}`).join(" ");
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function matchesFilter(t: Turn, f: UsageFilter | undefined): boolean {
|
|
71
|
+
if (!f) return true;
|
|
72
|
+
const has = (hay: string, needle: string | undefined) => !needle || hay.toLowerCase().includes(needle.toLowerCase());
|
|
73
|
+
return has(t.provider, f.provider) && has(t.model, f.model) && has(t.project, f.project);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function priceFor(t: Turn, prices: PriceTable): Price | undefined {
|
|
77
|
+
return prices[`${t.provider}/${t.model}`] ?? prices[t.model];
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export type CostSource = "log" | "table";
|
|
81
|
+
|
|
82
|
+
/** The turn with the price table applied (same object when it does not apply). */
|
|
83
|
+
export function applyPricing(t: Turn, prices: PriceTable | undefined, mode: PriceMode = "fill"): { turn: Turn; source: CostSource } {
|
|
84
|
+
if (!prices) return { turn: t, source: "log" };
|
|
85
|
+
const p = priceFor(t, prices);
|
|
86
|
+
if (!p || (mode === "fill" && t.cost > 0)) return { turn: t, source: "log" };
|
|
87
|
+
const costInput = (t.input * p.input) / 1e6;
|
|
88
|
+
const costCacheRead = (t.cacheRead * p.cacheRead) / 1e6;
|
|
89
|
+
const cost = costInput + costCacheRead + (t.output * p.output) / 1e6 + (t.cacheWrite * p.cacheWrite) / 1e6;
|
|
90
|
+
return { turn: { ...t, cost, costInput, costCacheRead }, source: "table" };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Filter then price every turn. Returns the input array untouched when there is nothing to do. */
|
|
94
|
+
export function selectTurns(turns: Turn[], opts: UsageOptions | undefined): Turn[] {
|
|
95
|
+
if (!opts) return turns;
|
|
96
|
+
const filter = hasFilter(opts.filter) ? opts.filter : undefined;
|
|
97
|
+
const prices = opts.prices && Object.keys(opts.prices).length > 0 ? opts.prices : undefined;
|
|
98
|
+
if (!filter && !prices) return turns;
|
|
99
|
+
const out: Turn[] = [];
|
|
100
|
+
for (const t of turns) {
|
|
101
|
+
if (!matchesFilter(t, filter)) continue;
|
|
102
|
+
out.push(prices ? applyPricing(t, prices, opts.priceMode).turn : t);
|
|
103
|
+
}
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Local-time buckets (hour / day / Monday-week / month / year). */
|
|
2
|
+
|
|
3
|
+
export type Unit = "day" | "week" | "month" | "year";
|
|
4
|
+
|
|
5
|
+
/** Local-time start of the day / Monday-week / month containing `ts`. */
|
|
6
|
+
export function bucketStart(ts: number, unit: Unit): number {
|
|
7
|
+
const d = new Date(ts);
|
|
8
|
+
if (unit === "day") return new Date(d.getFullYear(), d.getMonth(), d.getDate()).getTime();
|
|
9
|
+
if (unit === "week") {
|
|
10
|
+
const dow = (d.getDay() + 6) % 7; // Monday = 0
|
|
11
|
+
return new Date(d.getFullYear(), d.getMonth(), d.getDate() - dow).getTime();
|
|
12
|
+
}
|
|
13
|
+
if (unit === "year") return new Date(d.getFullYear(), 0, 1).getTime();
|
|
14
|
+
return new Date(d.getFullYear(), d.getMonth(), 1).getTime();
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const pad2 = (n: number) => String(n).padStart(2, "0");
|
|
18
|
+
|
|
19
|
+
export function bucketLabel(start: number, unit: Unit): string {
|
|
20
|
+
const d = new Date(start);
|
|
21
|
+
if (unit === "year") return String(d.getFullYear());
|
|
22
|
+
if (unit === "month") return `${d.getFullYear()}-${pad2(d.getMonth() + 1)}`;
|
|
23
|
+
const base = `${pad2(d.getMonth() + 1)}-${pad2(d.getDate())}`;
|
|
24
|
+
return unit === "day" ? `${base} ${["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][d.getDay()]}` : `${base} wk`;
|
|
25
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/** Shared types. */
|
|
2
|
+
|
|
3
|
+
export interface TokenUsage {
|
|
4
|
+
input: number;
|
|
5
|
+
output: number;
|
|
6
|
+
cacheRead: number;
|
|
7
|
+
cacheWrite: number;
|
|
8
|
+
cost: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface PeriodStats {
|
|
12
|
+
/** input + output (+ cache read/write when includeCache). */
|
|
13
|
+
tokens: number;
|
|
14
|
+
input: number;
|
|
15
|
+
output: number;
|
|
16
|
+
cacheRead: number;
|
|
17
|
+
cacheWrite: number;
|
|
18
|
+
cost: number;
|
|
19
|
+
sessions: number;
|
|
20
|
+
/** What cache reads would have cost as plain input vs what they cost (turns with a known input price). */
|
|
21
|
+
cacheBase: number;
|
|
22
|
+
cacheActual: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface TokenBurnReport {
|
|
26
|
+
/** Rolling last 60 minutes (burn rate). */
|
|
27
|
+
hour: PeriodStats;
|
|
28
|
+
today: PeriodStats;
|
|
29
|
+
week: PeriodStats;
|
|
30
|
+
month: PeriodStats;
|
|
31
|
+
year: PeriodStats;
|
|
32
|
+
total: PeriodStats;
|
|
33
|
+
/** Timestamp of the earliest turn counted (undefined when there is none). */
|
|
34
|
+
firstTs?: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export const STATUS_WINDOWS = ["hour", "today", "week", "month", "year", "total"] as const;
|
|
38
|
+
export type Window = (typeof STATUS_WINDOWS)[number];
|
|
39
|
+
|
|
40
|
+
export interface Rate {
|
|
41
|
+
tokensPerHour: number;
|
|
42
|
+
costPerHour: number;
|
|
43
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** Commands: usage tables (models / providers / projects / sessions) and the current session. */
|
|
2
|
+
import { collectUsage } from "../../core/aggregate.ts";
|
|
3
|
+
import { getSessionsDir } from "../../core/config.ts";
|
|
4
|
+
import { themePaint } from "../../core/paint.ts";
|
|
5
|
+
import { describeFilter } from "../../core/select.ts";
|
|
6
|
+
import type { HandlersFor } from "../../command/host.ts";
|
|
7
|
+
import { collectGroups } from "./query.ts";
|
|
8
|
+
import { renderTable, sessionSummary } from "./table.ts";
|
|
9
|
+
|
|
10
|
+
export const breakdownCommands: HandlersFor<"table" | "session"> = {
|
|
11
|
+
async table(cmd, { config, usageOpts, ctx }) {
|
|
12
|
+
const opts = usageOpts(cmd.filter);
|
|
13
|
+
const rows = await collectGroups(getSessionsDir(), cmd.by, cmd.period, new Date(), config.includeCache, opts);
|
|
14
|
+
const paint = config.chartColor ? themePaint(ctx.ui.theme) : undefined;
|
|
15
|
+
ctx.ui.notify(renderTable(cmd.by, rows, cmd.period, { limit: cmd.limit, filterText: describeFilter(opts.filter), paint }).join("\n"), "info");
|
|
16
|
+
},
|
|
17
|
+
|
|
18
|
+
async session(_cmd, { config, usageOpts, ctx }) {
|
|
19
|
+
const file = ctx.sessionManager?.getSessionFile?.();
|
|
20
|
+
if (!file) {
|
|
21
|
+
ctx.ui.notify("tokenburn: this session has no log file yet (in-memory session)", "info");
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
const root = getSessionsDir();
|
|
25
|
+
const now = new Date();
|
|
26
|
+
const [rows, report] = await Promise.all([
|
|
27
|
+
collectGroups(root, "session", "all", now, config.includeCache, usageOpts()),
|
|
28
|
+
collectUsage(root, now, config.includeCache, usageOpts()),
|
|
29
|
+
]);
|
|
30
|
+
ctx.ui.notify(sessionSummary(file, rows.find((r) => r.key === file), report.today.cost), "info");
|
|
31
|
+
},
|
|
32
|
+
};
|