pi-metrics 0.2.0 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/README.zh-CN.md +11 -3
- package/locales/index.json +28 -0
- package/package.json +12 -7
- package/src/index.ts +4 -1
- package/src/tps.ts +519 -0
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
# pi-
|
|
1
|
+
# pi-metrics
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Session metrics for the [Pi coding agent](https://github.com/earendil-works/pi): a live elapsed timer, per-turn summaries, and token-generation telemetry.
|
|
4
4
|
|
|
5
5
|
[中文文档](./README.zh-CN.md)
|
|
6
6
|
|
|
@@ -9,17 +9,25 @@ Heads-up session timing for the [Pi coding agent](https://github.com/earendil-wo
|
|
|
9
9
|
- While the agent is working, the spinner shows the **total elapsed time since you sent the message** (for example `⏱ 47s`). It keeps counting across turns instead of resetting per turn.
|
|
10
10
|
- When each turn ends, a dim line shows that turn's precise duration (`⏱ Turn elapsed 8.2s`).
|
|
11
11
|
- When the agent fully settles (`agent_settled` — including auto-retries, compaction continuations, or Esc interruption), a final line shows the **total elapsed time from message send to stop** (`⏱ Total elapsed 18.9s`).
|
|
12
|
+
- After each LLM turn, a notification reports TPS, TTFT, token counts, generation time, stalls, and blended cost when available.
|
|
13
|
+
- Telemetry is persisted as `tps` custom session entries and restored after session resume or `/tree` navigation.
|
|
14
|
+
- Metrics are exposed through session entries and notifications; this package does not register export slash commands.
|
|
15
|
+
|
|
16
|
+
## Migration from pi-tps
|
|
17
|
+
|
|
18
|
+
The TPS implementation is maintained in this package. Remove the standalone `npm:@monotykamary/pi-tps` entry from Pi settings before enabling this package, otherwise both extensions will record duplicate `tps` entries and notifications.
|
|
12
19
|
|
|
13
20
|
## Install
|
|
14
21
|
|
|
15
22
|
```bash
|
|
16
|
-
pi install npm:pi-
|
|
23
|
+
pi install npm:pi-metrics
|
|
17
24
|
```
|
|
18
25
|
|
|
19
26
|
## How it works
|
|
20
27
|
|
|
21
28
|
- The total timer starts on the `input` event (the moment you submit a message) and ends on `agent_settled`, so multi-turn tool calls, automatic retries, and queued continuations are all covered. Steer/follow-up messages sent mid-run do not reset the start point.
|
|
22
29
|
- In non-TUI mode (rpc/print) the timer and notifications are disabled.
|
|
30
|
+
- The shared Neuralwatt cost listener is unsubscribed during `session_shutdown`, and deferred rehydration notifications are cancelled during reload/session changes.
|
|
23
31
|
|
|
24
32
|
## Localization
|
|
25
33
|
|
package/README.zh-CN.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
# pi-
|
|
1
|
+
# pi-metrics
|
|
2
2
|
|
|
3
|
-
面向 [Pi coding agent](https://github.com/earendil-works/pi)
|
|
3
|
+
面向 [Pi coding agent](https://github.com/earendil-works/pi) 的会话指标扩展:working spinner 实时计时,并提供每轮 TPS、TTFT 和 token 使用量。
|
|
4
4
|
|
|
5
5
|
[English](./README.md)
|
|
6
6
|
|
|
@@ -9,17 +9,25 @@
|
|
|
9
9
|
- 工作期间 spinner 显示**从发出消息起的全程耗时**(如 `⏱ 47s`),跨轮次持续累加,不再每轮回零。
|
|
10
10
|
- 每个轮次结束时插入一条灰色文本,显示该轮精确耗时(`⏱ 本轮耗时 8.2s`)。
|
|
11
11
|
- AI 完全停止时(`agent_settled`,覆盖自动重试、compaction 续跑以及 Esc 中断)追加一行**从发出消息到停止的总耗时**(`⏱ 总耗时 18.9s`)。
|
|
12
|
+
- 每轮 LLM 调用结束后显示 TPS、TTFT、token 数、生成耗时、stall 和可用的综合成本。
|
|
13
|
+
- Telemetry 以 `tps` custom session entry 持久化,并在恢复 session 或 `/tree` 后恢复显示。
|
|
14
|
+
- Metrics 通过 session entry 和通知提供;本包不再注册导出类斜杠命令。
|
|
15
|
+
|
|
16
|
+
## 从 pi-tps 迁移
|
|
17
|
+
|
|
18
|
+
TPS 实现现在由本包维护。启用本包前,请从 Pi 配置中移除独立的 `npm:@monotykamary/pi-tps`,否则两个扩展会重复写入 `tps` 条目并重复通知。
|
|
12
19
|
|
|
13
20
|
## 安装
|
|
14
21
|
|
|
15
22
|
```bash
|
|
16
|
-
pi install npm:pi-
|
|
23
|
+
pi install npm:pi-metrics
|
|
17
24
|
```
|
|
18
25
|
|
|
19
26
|
## 实现说明
|
|
20
27
|
|
|
21
28
|
- 总耗时以 `input` 事件(用户提交消息的时刻)为起点、`agent_settled` 为终点,因此多轮工具调用、自动重试和队列续跑都计入同一次总耗时;运行中发送的 steer/followUp 消息不会重置起点。
|
|
22
29
|
- 非 TUI 模式(rpc/print)下不启动定时器、不发送通知。
|
|
30
|
+
- Neuralwatt 成本监听器会在 `session_shutdown` 时取消订阅,恢复通知的延迟定时器也会在 reload/session 切换时清理。
|
|
23
31
|
|
|
24
32
|
## 国际化
|
|
25
33
|
|
package/locales/index.json
CHANGED
|
@@ -10,5 +10,33 @@
|
|
|
10
10
|
"elapsedTotal": {
|
|
11
11
|
"zh-CN": "⏱ 总耗时 {value}",
|
|
12
12
|
"en-US": "⏱ Total elapsed {value}"
|
|
13
|
+
},
|
|
14
|
+
"tpsValue": {
|
|
15
|
+
"zh-CN": "TPS {value} tok/s",
|
|
16
|
+
"en-US": "TPS {value} tok/s"
|
|
17
|
+
},
|
|
18
|
+
"tpsUnknown": {
|
|
19
|
+
"zh-CN": "TPS —",
|
|
20
|
+
"en-US": "TPS —"
|
|
21
|
+
},
|
|
22
|
+
"tpsTtft": {
|
|
23
|
+
"zh-CN": "TTFT {value}",
|
|
24
|
+
"en-US": "TTFT {value}"
|
|
25
|
+
},
|
|
26
|
+
"tpsInput": {
|
|
27
|
+
"zh-CN": "in {value}",
|
|
28
|
+
"en-US": "in {value}"
|
|
29
|
+
},
|
|
30
|
+
"tpsOutput": {
|
|
31
|
+
"zh-CN": "out {value}",
|
|
32
|
+
"en-US": "out {value}"
|
|
33
|
+
},
|
|
34
|
+
"tpsStall": {
|
|
35
|
+
"zh-CN": "stall {value}×{count}",
|
|
36
|
+
"en-US": "stall {value}×{count}"
|
|
37
|
+
},
|
|
38
|
+
"tpsRate": {
|
|
39
|
+
"zh-CN": "${value}/M",
|
|
40
|
+
"en-US": "${value}/M"
|
|
13
41
|
}
|
|
14
42
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-metrics",
|
|
3
|
-
"version": "0.2.
|
|
4
|
-
"description": "Pi metrics extension:
|
|
3
|
+
"version": "0.2.3",
|
|
4
|
+
"description": "Pi metrics extension: elapsed-time HUD and resilient TPS telemetry",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./index.ts",
|
|
7
7
|
"exports": {
|
|
@@ -15,14 +15,15 @@
|
|
|
15
15
|
"README.zh-CN.md"
|
|
16
16
|
],
|
|
17
17
|
"scripts": {
|
|
18
|
-
"test": "tsx --test tests
|
|
18
|
+
"test": "tsx --test tests/*.test.ts",
|
|
19
19
|
"typecheck": "tsc --noEmit --pretty false",
|
|
20
20
|
"build": "npm run typecheck",
|
|
21
21
|
"check": "npm run typecheck && npm test && npm pack --dry-run --json > /dev/null"
|
|
22
22
|
},
|
|
23
23
|
"pi": {
|
|
24
24
|
"extensions": [
|
|
25
|
-
"./index.ts"
|
|
25
|
+
"./index.ts",
|
|
26
|
+
"../pi-extensions-i18n/index.ts"
|
|
26
27
|
]
|
|
27
28
|
},
|
|
28
29
|
"engines": {
|
|
@@ -47,13 +48,17 @@
|
|
|
47
48
|
"pi-extension",
|
|
48
49
|
"coding-agent",
|
|
49
50
|
"hud",
|
|
50
|
-
"elapsed-time"
|
|
51
|
+
"elapsed-time",
|
|
52
|
+
"tokens-per-second",
|
|
53
|
+
"telemetry"
|
|
51
54
|
],
|
|
52
55
|
"peerDependencies": {
|
|
53
56
|
"@earendil-works/pi-ai": ">=0.80.0 <0.81.0",
|
|
54
57
|
"@earendil-works/pi-coding-agent": ">=0.80.0 <0.81.0",
|
|
55
|
-
"@earendil-works/pi-tui": ">=0.80.0 <0.81.0"
|
|
56
|
-
|
|
58
|
+
"@earendil-works/pi-tui": ">=0.80.0 <0.81.0"
|
|
59
|
+
},
|
|
60
|
+
"dependencies": {
|
|
61
|
+
"pi-extensions-i18n": "^0.3.1"
|
|
57
62
|
},
|
|
58
63
|
"devDependencies": {
|
|
59
64
|
"@earendil-works/pi-ai": "0.80.10",
|
package/src/index.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import turnElapsed from "./turn-elapsed.ts";
|
|
3
|
+
import tps from "./tps.ts";
|
|
3
4
|
|
|
4
5
|
export default function piHud(pi: ExtensionAPI): void {
|
|
5
|
-
// 后续 HUD 类功能(如 token 用量、上下文水位等)在此注册
|
|
6
6
|
turnElapsed(pi);
|
|
7
|
+
tps(pi);
|
|
7
8
|
}
|
|
8
9
|
|
|
9
10
|
export { default as turnElapsed } from "./turn-elapsed.ts";
|
|
11
|
+
export { default as tps } from "./tps.ts";
|
|
10
12
|
export * from "./format-utils.ts";
|
|
13
|
+
export * from "./tps.ts";
|
package/src/tps.ts
ADDED
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token generation metrics for Pi.
|
|
3
|
+
*
|
|
4
|
+
* This is the TPS portion of pi-tps, maintained inside pi-metrics so the
|
|
5
|
+
* elapsed-time HUD and generation telemetry share one lifecycle.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { performance } from "node:perf_hooks";
|
|
9
|
+
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
|
10
|
+
import type {
|
|
11
|
+
ExtensionAPI,
|
|
12
|
+
ExtensionContext,
|
|
13
|
+
} from "@earendil-works/pi-coding-agent";
|
|
14
|
+
import { i18n } from "./i18n.ts";
|
|
15
|
+
|
|
16
|
+
interface TurnStartEvent {
|
|
17
|
+
type: "turn_start";
|
|
18
|
+
turnIndex: number;
|
|
19
|
+
timestamp: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface TurnEndEvent {
|
|
23
|
+
type: "turn_end";
|
|
24
|
+
turnIndex: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface MessageEvent {
|
|
28
|
+
type: string;
|
|
29
|
+
message: unknown;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
interface SessionTreeEvent {
|
|
33
|
+
type: "session_tree";
|
|
34
|
+
newLeafId: string | null;
|
|
35
|
+
oldLeafId: string | null;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface ToolExecutionStartEvent {
|
|
39
|
+
type: "tool_execution_start";
|
|
40
|
+
toolCallId: string;
|
|
41
|
+
toolName: string;
|
|
42
|
+
args: unknown;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface TurnTelemetry {
|
|
46
|
+
model: { provider: string; modelId: string };
|
|
47
|
+
tokens: {
|
|
48
|
+
input: number;
|
|
49
|
+
output: number;
|
|
50
|
+
cacheRead: number;
|
|
51
|
+
cacheWrite: number;
|
|
52
|
+
total: number;
|
|
53
|
+
};
|
|
54
|
+
timing: {
|
|
55
|
+
ttftMs: number | null;
|
|
56
|
+
totalMs: number;
|
|
57
|
+
generationMs: number;
|
|
58
|
+
streamMs: number | null;
|
|
59
|
+
stallMs: number;
|
|
60
|
+
stallCount: number;
|
|
61
|
+
messageCount: number;
|
|
62
|
+
};
|
|
63
|
+
tps: number | null;
|
|
64
|
+
isPrimaryBranch: boolean;
|
|
65
|
+
cost: {
|
|
66
|
+
input: number;
|
|
67
|
+
output: number;
|
|
68
|
+
cacheRead: number;
|
|
69
|
+
cacheWrite: number;
|
|
70
|
+
total: number;
|
|
71
|
+
} | null;
|
|
72
|
+
rateUsdPerMTokens: number | null;
|
|
73
|
+
timestamp: number;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
interface TurnTiming {
|
|
77
|
+
turnIndex: number;
|
|
78
|
+
turnStartMs: number;
|
|
79
|
+
turnStartTimestamp: number;
|
|
80
|
+
lastUpdateMs: number;
|
|
81
|
+
firstTokenMs: number | null;
|
|
82
|
+
currentMessageStartMs: number | null;
|
|
83
|
+
assistantMessages: AssistantMessage[];
|
|
84
|
+
totalGenerationMs: number;
|
|
85
|
+
updateCount: number;
|
|
86
|
+
firstStreamUpdateMs: number | null;
|
|
87
|
+
lastStreamUpdateMs: number;
|
|
88
|
+
stallMs: number;
|
|
89
|
+
stallCount: number;
|
|
90
|
+
inStall: boolean;
|
|
91
|
+
messageCount: number;
|
|
92
|
+
isToolCall: boolean;
|
|
93
|
+
isPrimaryBranch: boolean;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
interface SessionEntryLike {
|
|
97
|
+
id: string;
|
|
98
|
+
parentId?: string | null;
|
|
99
|
+
type: string;
|
|
100
|
+
customType?: string;
|
|
101
|
+
data?: unknown;
|
|
102
|
+
timestamp?: number | string;
|
|
103
|
+
[key: string]: unknown;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const STALL_THRESHOLD_MS = 500;
|
|
107
|
+
const NEURALWATT_ENERGY_EVENT = "neuralwatt:turn-energy";
|
|
108
|
+
|
|
109
|
+
export function formatNumber(num: number): string {
|
|
110
|
+
if (num < 1_000) return String(num);
|
|
111
|
+
|
|
112
|
+
const [value, suffix] = num >= 1_000_000_000
|
|
113
|
+
? [num / 1_000_000_000, "B"]
|
|
114
|
+
: num >= 1_000_000
|
|
115
|
+
? [num / 1_000_000, "M"]
|
|
116
|
+
: [num / 1_000, "K"];
|
|
117
|
+
const formatted = value.toFixed(1);
|
|
118
|
+
return formatted.endsWith(".0") ? `${value.toFixed(0)}${suffix}` : `${formatted}${suffix}`;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function formatDuration(totalSeconds: number): string {
|
|
122
|
+
if (totalSeconds < 60) return `${totalSeconds.toFixed(1)}s`;
|
|
123
|
+
|
|
124
|
+
const units = [
|
|
125
|
+
["y", 365 * 24 * 60 * 60],
|
|
126
|
+
["mo", 30 * 24 * 60 * 60],
|
|
127
|
+
["w", 7 * 24 * 60 * 60],
|
|
128
|
+
["d", 24 * 60 * 60],
|
|
129
|
+
["h", 60 * 60],
|
|
130
|
+
["m", 60],
|
|
131
|
+
["s", 1],
|
|
132
|
+
] as const;
|
|
133
|
+
const parts: Array<{ value: number; label: string }> = [];
|
|
134
|
+
let remaining = Math.round(totalSeconds);
|
|
135
|
+
|
|
136
|
+
for (const [label, seconds] of units) {
|
|
137
|
+
if (remaining >= seconds) {
|
|
138
|
+
parts.push({ value: Math.floor(remaining / seconds), label });
|
|
139
|
+
remaining %= seconds;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (parts.length === 1) {
|
|
144
|
+
const first = units.findIndex(([label]) => label === parts[0].label);
|
|
145
|
+
let next = first + 1;
|
|
146
|
+
if (parts[0].label === "mo") next++;
|
|
147
|
+
if (parts[0].label === "y") next += 2;
|
|
148
|
+
if (next < units.length) parts.push({ value: 0, label: units[next][0] });
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return parts.slice(0, 2).map(({ value, label }) => `${value}${label}`).join(" ");
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export function computeRateUsdPerM(costUsd: number | null, totalTokens: number): number | null {
|
|
155
|
+
if (costUsd === null || !Number.isFinite(costUsd) || costUsd < 0) return null;
|
|
156
|
+
if (!Number.isFinite(totalTokens) || totalTokens <= 0) return null;
|
|
157
|
+
const rate = costUsd / (totalTokens / 1_000_000);
|
|
158
|
+
return Number.isFinite(rate) && rate >= 0 ? Math.round(rate * 100) / 100 : null;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function isAssistantMessage(message: unknown): message is AssistantMessage {
|
|
162
|
+
if (!message || typeof message !== "object") return false;
|
|
163
|
+
const candidate = message as Record<string, unknown>;
|
|
164
|
+
if (candidate.role !== "assistant" || typeof candidate.usage !== "object" || candidate.usage === null) {
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
const usage = candidate.usage as Record<string, unknown>;
|
|
168
|
+
return typeof usage.input === "number" && typeof usage.output === "number";
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function findEnergyCostFromSession(ctx: ExtensionContext, turnStartTimestamp: number): number | null {
|
|
172
|
+
const entries = ctx.sessionManager?.getEntries?.() as SessionEntryLike[] | undefined;
|
|
173
|
+
if (!entries) return null;
|
|
174
|
+
|
|
175
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
176
|
+
const entry = entries[i];
|
|
177
|
+
if (entry.type !== "custom" || entry.customType !== "neuralwatt-energy") continue;
|
|
178
|
+
const entryTimestamp = parseEntryTimestamp(entry.timestamp);
|
|
179
|
+
if (Number.isFinite(entryTimestamp) && entryTimestamp < turnStartTimestamp) return null;
|
|
180
|
+
const data = entry.data as Record<string, unknown> | null | undefined;
|
|
181
|
+
const cost = data?.cost_usd;
|
|
182
|
+
if (typeof cost === "number" && Number.isFinite(cost) && cost >= 0) return cost;
|
|
183
|
+
}
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function parseEntryTimestamp(value: number | string | undefined): number {
|
|
188
|
+
if (typeof value === "number") return value;
|
|
189
|
+
if (typeof value !== "string") return Number.NaN;
|
|
190
|
+
const numeric = Number(value);
|
|
191
|
+
return Number.isFinite(numeric) ? numeric : Date.parse(value);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function buildTelemetry(
|
|
195
|
+
timing: TurnTiming,
|
|
196
|
+
turnEndMs: number,
|
|
197
|
+
billedCost: number | null,
|
|
198
|
+
): TurnTelemetry | null {
|
|
199
|
+
let input = 0;
|
|
200
|
+
let output = 0;
|
|
201
|
+
let cacheRead = 0;
|
|
202
|
+
let cacheWrite = 0;
|
|
203
|
+
let totalTokens = 0;
|
|
204
|
+
let costInput = 0;
|
|
205
|
+
let costOutput = 0;
|
|
206
|
+
let costCacheRead = 0;
|
|
207
|
+
let costCacheWrite = 0;
|
|
208
|
+
let costTotal = 0;
|
|
209
|
+
let hasCost = false;
|
|
210
|
+
let model: { provider: string; modelId: string } | null = null;
|
|
211
|
+
|
|
212
|
+
for (const message of timing.assistantMessages) {
|
|
213
|
+
const usage = message.usage;
|
|
214
|
+
input += usage.input || 0;
|
|
215
|
+
output += usage.output || 0;
|
|
216
|
+
cacheRead += usage.cacheRead || 0;
|
|
217
|
+
cacheWrite += usage.cacheWrite || 0;
|
|
218
|
+
totalTokens += usage.totalTokens || 0;
|
|
219
|
+
if (usage.cost) {
|
|
220
|
+
costInput += usage.cost.input || 0;
|
|
221
|
+
costOutput += usage.cost.output || 0;
|
|
222
|
+
costCacheRead += usage.cost.cacheRead || 0;
|
|
223
|
+
costCacheWrite += usage.cost.cacheWrite || 0;
|
|
224
|
+
costTotal += usage.cost.total || 0;
|
|
225
|
+
hasCost = true;
|
|
226
|
+
}
|
|
227
|
+
if (!model && message.provider && message.model) {
|
|
228
|
+
model = { provider: message.provider, modelId: message.model };
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (output <= 0 || timing.firstTokenMs === null || !model) return null;
|
|
233
|
+
|
|
234
|
+
const totalMs = turnEndMs - timing.turnStartMs;
|
|
235
|
+
const streamMs = timing.updateCount > 0 && timing.firstStreamUpdateMs !== null
|
|
236
|
+
? timing.lastStreamUpdateMs - timing.firstStreamUpdateMs
|
|
237
|
+
: null;
|
|
238
|
+
const averageGap = streamMs !== null && timing.updateCount > 1
|
|
239
|
+
? streamMs / (timing.updateCount - 1)
|
|
240
|
+
: 0;
|
|
241
|
+
|
|
242
|
+
const primary =
|
|
243
|
+
streamMs !== null &&
|
|
244
|
+
streamMs >= 1 &&
|
|
245
|
+
timing.updateCount >= 5 &&
|
|
246
|
+
averageGap >= 1 &&
|
|
247
|
+
timing.stallMs < streamMs &&
|
|
248
|
+
streamMs - timing.stallMs >= 200 &&
|
|
249
|
+
timing.stallMs < streamMs - timing.stallMs;
|
|
250
|
+
|
|
251
|
+
let tps: number | null;
|
|
252
|
+
let isPrimaryBranch = false;
|
|
253
|
+
if (primary) {
|
|
254
|
+
tps = Math.round((output / ((streamMs! - timing.stallMs) / 1000)) * 10) / 10;
|
|
255
|
+
isPrimaryBranch = true;
|
|
256
|
+
} else if (timing.updateCount >= 2 && timing.totalGenerationMs >= 200) {
|
|
257
|
+
let effectiveMs = timing.totalGenerationMs - timing.stallMs;
|
|
258
|
+
if (effectiveMs < 200 || timing.stallMs > timing.totalGenerationMs * 0.85) {
|
|
259
|
+
effectiveMs = Math.max(timing.totalGenerationMs - timing.stallMs / 2, 200);
|
|
260
|
+
} else {
|
|
261
|
+
effectiveMs = Math.max(effectiveMs, 200);
|
|
262
|
+
}
|
|
263
|
+
tps = Math.round((output / (effectiveMs / 1000)) * 10) / 10;
|
|
264
|
+
} else {
|
|
265
|
+
tps = null;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (tps !== null && tps > 10_000) {
|
|
269
|
+
tps = null;
|
|
270
|
+
isPrimaryBranch = false;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const listPriceCost = hasCost && Number.isFinite(costTotal) && costTotal > 0 ? costTotal : null;
|
|
274
|
+
const effectiveCost = billedCost ?? listPriceCost;
|
|
275
|
+
return {
|
|
276
|
+
model,
|
|
277
|
+
tokens: { input, output, cacheRead, cacheWrite, total: totalTokens },
|
|
278
|
+
timing: {
|
|
279
|
+
ttftMs: timing.firstTokenMs - timing.turnStartMs,
|
|
280
|
+
totalMs,
|
|
281
|
+
generationMs: timing.totalGenerationMs,
|
|
282
|
+
streamMs,
|
|
283
|
+
stallMs: timing.stallMs,
|
|
284
|
+
stallCount: timing.stallCount,
|
|
285
|
+
messageCount: timing.messageCount,
|
|
286
|
+
},
|
|
287
|
+
tps,
|
|
288
|
+
isPrimaryBranch,
|
|
289
|
+
cost: listPriceCost === null
|
|
290
|
+
? null
|
|
291
|
+
: { input: costInput, output: costOutput, cacheRead: costCacheRead, cacheWrite: costCacheWrite, total: costTotal },
|
|
292
|
+
rateUsdPerMTokens: computeRateUsdPerM(effectiveCost, totalTokens),
|
|
293
|
+
timestamp: Date.now(),
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function composeDisplayString(telemetry: TurnTelemetry): string {
|
|
298
|
+
const parts = [
|
|
299
|
+
telemetry.tps === null
|
|
300
|
+
? i18n.t("tpsUnknown")
|
|
301
|
+
: i18n.t("tpsValue", { value: telemetry.tps.toFixed(1) }),
|
|
302
|
+
];
|
|
303
|
+
if (telemetry.timing.ttftMs !== null) {
|
|
304
|
+
parts.push(i18n.t("tpsTtft", { value: formatDuration(telemetry.timing.ttftMs / 1000) }));
|
|
305
|
+
}
|
|
306
|
+
parts.push(formatDuration(telemetry.timing.totalMs / 1000));
|
|
307
|
+
parts.push(i18n.t("tpsInput", { value: formatNumber(telemetry.tokens.input) }));
|
|
308
|
+
parts.push(i18n.t("tpsOutput", { value: formatNumber(telemetry.tokens.output) }));
|
|
309
|
+
if (telemetry.timing.stallMs > 0) {
|
|
310
|
+
parts.push(i18n.t("tpsStall", {
|
|
311
|
+
value: formatDuration(telemetry.timing.stallMs / 1000),
|
|
312
|
+
count: telemetry.timing.stallCount,
|
|
313
|
+
}));
|
|
314
|
+
}
|
|
315
|
+
if (telemetry.rateUsdPerMTokens !== null) {
|
|
316
|
+
parts.push(i18n.t("tpsRate", { value: telemetry.rateUsdPerMTokens.toFixed(2) }));
|
|
317
|
+
}
|
|
318
|
+
return parts.join(" · ");
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function restoreTPSNotification(
|
|
322
|
+
ctx: ExtensionContext,
|
|
323
|
+
schedule: (callback: () => void) => void,
|
|
324
|
+
): void {
|
|
325
|
+
if (!ctx.hasUI) return;
|
|
326
|
+
const entries = ctx.sessionManager.getBranch() as SessionEntryLike[];
|
|
327
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
328
|
+
const entry = entries[i];
|
|
329
|
+
if (entry.type !== "custom" || entry.customType !== "tps") continue;
|
|
330
|
+
const data = entry.data as Record<string, unknown> | null | undefined;
|
|
331
|
+
if (!data) continue;
|
|
332
|
+
if (typeof data.model === "object" && data.model !== null) {
|
|
333
|
+
schedule(() => ctx.ui.notify(composeDisplayString(data as unknown as TurnTelemetry), "info"));
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
if (typeof data.message === "string") {
|
|
337
|
+
schedule(() => ctx.ui.notify(data.message as string, "info"));
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export default function tpsExtension(pi: ExtensionAPI): void {
|
|
344
|
+
let currentTiming: TurnTiming | null = null;
|
|
345
|
+
let pendingNeuralwattBilledCost: { turnIndex: number; costUsd: number } | null = null;
|
|
346
|
+
let lastCommittedTurn: {
|
|
347
|
+
turnIndex: number;
|
|
348
|
+
telemetry: TurnTelemetry;
|
|
349
|
+
billedApplied: boolean;
|
|
350
|
+
ctx: ExtensionContext;
|
|
351
|
+
} | null = null;
|
|
352
|
+
const tpsCaps = new Map<string, number>();
|
|
353
|
+
const restoreTimers = new Set<ReturnType<typeof setTimeout>>();
|
|
354
|
+
let unsubscribeNeuralwatt: (() => void) | undefined;
|
|
355
|
+
|
|
356
|
+
const clearState = () => {
|
|
357
|
+
currentTiming = null;
|
|
358
|
+
pendingNeuralwattBilledCost = null;
|
|
359
|
+
lastCommittedTurn = null;
|
|
360
|
+
for (const timer of restoreTimers) clearTimeout(timer);
|
|
361
|
+
restoreTimers.clear();
|
|
362
|
+
};
|
|
363
|
+
|
|
364
|
+
const scheduleRestore = (callback: () => void) => {
|
|
365
|
+
const timer = setTimeout(() => {
|
|
366
|
+
restoreTimers.delete(timer);
|
|
367
|
+
callback();
|
|
368
|
+
}, 0);
|
|
369
|
+
restoreTimers.add(timer);
|
|
370
|
+
};
|
|
371
|
+
|
|
372
|
+
unsubscribeNeuralwatt = pi.events?.on(NEURALWATT_ENERGY_EVENT, (payload: unknown) => {
|
|
373
|
+
if (!payload || typeof payload !== "object") return;
|
|
374
|
+
const data = payload as Record<string, unknown>;
|
|
375
|
+
const turnIndex = typeof data.turnIndex === "number" ? data.turnIndex : null;
|
|
376
|
+
const costUsd = typeof data.costUsd === "number" ? data.costUsd : null;
|
|
377
|
+
if (turnIndex === null || costUsd === null || !Number.isFinite(costUsd) || costUsd < 0) return;
|
|
378
|
+
|
|
379
|
+
if (currentTiming) {
|
|
380
|
+
if (currentTiming.turnIndex === turnIndex) {
|
|
381
|
+
pendingNeuralwattBilledCost = { turnIndex, costUsd };
|
|
382
|
+
}
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
const committed = lastCommittedTurn;
|
|
387
|
+
if (!committed || committed.billedApplied || committed.turnIndex !== turnIndex) return;
|
|
388
|
+
committed.billedApplied = true;
|
|
389
|
+
const correctedRate = computeRateUsdPerM(costUsd, committed.telemetry.tokens.total);
|
|
390
|
+
if (correctedRate === null || correctedRate === committed.telemetry.rateUsdPerMTokens) return;
|
|
391
|
+
const corrected = { ...committed.telemetry, rateUsdPerMTokens: correctedRate };
|
|
392
|
+
committed.telemetry = corrected;
|
|
393
|
+
pi.appendEntry("tps", corrected);
|
|
394
|
+
pi.events?.emit("tps:telemetry", corrected);
|
|
395
|
+
if (committed.ctx.hasUI) committed.ctx.ui.notify(composeDisplayString(corrected), "info");
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
pi.on("session_shutdown", () => {
|
|
399
|
+
unsubscribeNeuralwatt?.();
|
|
400
|
+
unsubscribeNeuralwatt = undefined;
|
|
401
|
+
clearState();
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
pi.on("session_start", (_event, ctx) => {
|
|
405
|
+
clearState();
|
|
406
|
+
restoreTPSNotification(ctx, scheduleRestore);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
pi.on("session_tree", (_event: SessionTreeEvent, ctx) => {
|
|
410
|
+
pendingNeuralwattBilledCost = null;
|
|
411
|
+
lastCommittedTurn = null;
|
|
412
|
+
restoreTPSNotification(ctx, scheduleRestore);
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
pi.on("turn_start", (event: TurnStartEvent) => {
|
|
416
|
+
pendingNeuralwattBilledCost = null;
|
|
417
|
+
lastCommittedTurn = null;
|
|
418
|
+
currentTiming = {
|
|
419
|
+
turnIndex: event.turnIndex,
|
|
420
|
+
turnStartMs: performance.now(),
|
|
421
|
+
turnStartTimestamp: typeof event.timestamp === "number" ? event.timestamp : Date.now(),
|
|
422
|
+
lastUpdateMs: performance.now(),
|
|
423
|
+
firstTokenMs: null,
|
|
424
|
+
currentMessageStartMs: null,
|
|
425
|
+
assistantMessages: [],
|
|
426
|
+
totalGenerationMs: 0,
|
|
427
|
+
updateCount: 0,
|
|
428
|
+
firstStreamUpdateMs: null,
|
|
429
|
+
lastStreamUpdateMs: 0,
|
|
430
|
+
stallMs: 0,
|
|
431
|
+
stallCount: 0,
|
|
432
|
+
inStall: false,
|
|
433
|
+
messageCount: 0,
|
|
434
|
+
isToolCall: false,
|
|
435
|
+
isPrimaryBranch: false,
|
|
436
|
+
};
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
pi.on("message_start", (event: MessageEvent) => {
|
|
440
|
+
if (!currentTiming || !isAssistantMessage(event.message)) return;
|
|
441
|
+
const now = performance.now();
|
|
442
|
+
currentTiming.currentMessageStartMs = now;
|
|
443
|
+
currentTiming.messageCount++;
|
|
444
|
+
currentTiming.lastUpdateMs = now;
|
|
445
|
+
currentTiming.inStall = false;
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
pi.on("message_update", (event: MessageEvent) => {
|
|
449
|
+
if (!currentTiming || !isAssistantMessage(event.message)) return;
|
|
450
|
+
const now = performance.now();
|
|
451
|
+
if (currentTiming.firstTokenMs === null) {
|
|
452
|
+
currentTiming.firstTokenMs = now;
|
|
453
|
+
currentTiming.lastUpdateMs = now;
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
currentTiming.updateCount++;
|
|
458
|
+
if (currentTiming.firstStreamUpdateMs === null) currentTiming.firstStreamUpdateMs = now;
|
|
459
|
+
currentTiming.lastStreamUpdateMs = now;
|
|
460
|
+
const gap = now - currentTiming.lastUpdateMs;
|
|
461
|
+
if (gap >= STALL_THRESHOLD_MS) {
|
|
462
|
+
if (!currentTiming.inStall) currentTiming.stallCount++;
|
|
463
|
+
currentTiming.inStall = true;
|
|
464
|
+
currentTiming.stallMs += gap;
|
|
465
|
+
} else {
|
|
466
|
+
currentTiming.inStall = false;
|
|
467
|
+
}
|
|
468
|
+
currentTiming.lastUpdateMs = now;
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
pi.on("tool_execution_start", (_event: ToolExecutionStartEvent) => {
|
|
472
|
+
if (currentTiming) currentTiming.isToolCall = true;
|
|
473
|
+
});
|
|
474
|
+
|
|
475
|
+
pi.on("message_end", (event: MessageEvent) => {
|
|
476
|
+
if (!currentTiming || !isAssistantMessage(event.message)) return;
|
|
477
|
+
const now = performance.now();
|
|
478
|
+
if (currentTiming.currentMessageStartMs !== null) {
|
|
479
|
+
currentTiming.totalGenerationMs += now - currentTiming.currentMessageStartMs;
|
|
480
|
+
currentTiming.currentMessageStartMs = null;
|
|
481
|
+
}
|
|
482
|
+
currentTiming.assistantMessages.push(event.message);
|
|
483
|
+
currentTiming.lastUpdateMs = now;
|
|
484
|
+
});
|
|
485
|
+
|
|
486
|
+
pi.on("turn_end", (event: TurnEndEvent, ctx: ExtensionContext) => {
|
|
487
|
+
if (!currentTiming) return;
|
|
488
|
+
const timing = currentTiming;
|
|
489
|
+
currentTiming = null;
|
|
490
|
+
let billedCost = pendingNeuralwattBilledCost?.turnIndex === event.turnIndex
|
|
491
|
+
? pendingNeuralwattBilledCost.costUsd
|
|
492
|
+
: null;
|
|
493
|
+
pendingNeuralwattBilledCost = null;
|
|
494
|
+
if (billedCost === null) billedCost = findEnergyCostFromSession(ctx, timing.turnStartTimestamp);
|
|
495
|
+
const telemetry = buildTelemetry(timing, performance.now(), billedCost);
|
|
496
|
+
if (!telemetry) return;
|
|
497
|
+
|
|
498
|
+
const modelKey = `${telemetry.model.provider}:${telemetry.model.modelId}`;
|
|
499
|
+
if (telemetry.isPrimaryBranch && telemetry.tps !== null) {
|
|
500
|
+
const currentCap = tpsCaps.get(modelKey);
|
|
501
|
+
if (currentCap === undefined || telemetry.tps > currentCap) tpsCaps.set(modelKey, telemetry.tps);
|
|
502
|
+
}
|
|
503
|
+
if (timing.isToolCall && telemetry.tps !== null) {
|
|
504
|
+
const cap = tpsCaps.get(modelKey);
|
|
505
|
+
telemetry.tps = cap === undefined ? null : Math.min(telemetry.tps, cap);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
lastCommittedTurn = {
|
|
509
|
+
turnIndex: event.turnIndex,
|
|
510
|
+
telemetry,
|
|
511
|
+
billedApplied: billedCost !== null,
|
|
512
|
+
ctx,
|
|
513
|
+
};
|
|
514
|
+
pi.appendEntry("tps", telemetry);
|
|
515
|
+
pi.events?.emit("tps:telemetry", telemetry);
|
|
516
|
+
if (ctx.hasUI) ctx.ui.notify(composeDisplayString(telemetry), "info");
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
}
|