pi-smart-router 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/command-formatters.ts +93 -0
- package/.pi/extensions/smart-router/commands.ts +24 -2
- package/.pi/extensions/smart-router/extension-setup.ts +4 -3
- package/.pi/extensions/smart-router/index.ts +2 -0
- package/.pi/extensions/smart-router/types.ts +1 -0
- package/README.md +31 -10
- package/config/benchmark-profiles.json +2 -2
- package/dist/infrastructure/telemetry/session-stats.d.ts +78 -0
- package/dist/infrastructure/telemetry/session-stats.d.ts.map +1 -0
- package/dist/infrastructure/telemetry/session-stats.js +187 -0
- package/dist/infrastructure/telemetry/session-stats.js.map +1 -0
- package/package.json +7 -4
- package/src/infrastructure/telemetry/session-stats.ts +290 -0
|
@@ -4,9 +4,14 @@ import {
|
|
|
4
4
|
} from '../../../src/infrastructure/telemetry/telemetry-limits.js';
|
|
5
5
|
import type {
|
|
6
6
|
ModelProfile,
|
|
7
|
+
PriceCatalog,
|
|
7
8
|
RoutingDecision,
|
|
8
9
|
RoutingTelemetry,
|
|
9
10
|
} from '../../../src/domain/types/index.js';
|
|
11
|
+
import {
|
|
12
|
+
aggregateSessionStatsFromFleet,
|
|
13
|
+
type SessionStatsSnapshot,
|
|
14
|
+
} from '../../../src/infrastructure/telemetry/session-stats.js';
|
|
10
15
|
import { SMART_ROUTER_USAGE } from './commands.js';
|
|
11
16
|
import {
|
|
12
17
|
DEFAULT_TELEMETRY_CONTRIB_EXPORT_LIMIT,
|
|
@@ -19,6 +24,8 @@ import {
|
|
|
19
24
|
import { formatPricingStalenessLine } from './pricing-lifecycle.js';
|
|
20
25
|
import type { FleetMode, SmartRouterCommand, SmartRouterRuntime } from './types.js';
|
|
21
26
|
|
|
27
|
+
export type { SessionStatsSnapshot };
|
|
28
|
+
|
|
22
29
|
/** Opaque / virtual auto ids that hide the concrete delegated fleet model (SP-178). */
|
|
23
30
|
function isBareOrSmartRouterAuto(modelId: string): boolean {
|
|
24
31
|
return modelId === 'auto' || modelId === 'smart-router/auto';
|
|
@@ -149,6 +156,10 @@ export function parseSmartRouterArgs(args: string): SmartRouterCommand {
|
|
|
149
156
|
return { command: 'history', limit: parseHistoryLimit(tokens[1]) };
|
|
150
157
|
}
|
|
151
158
|
|
|
159
|
+
if (tokens[0] === 'stats') {
|
|
160
|
+
return { command: 'stats', limit: parseHistoryLimit(tokens[1]) };
|
|
161
|
+
}
|
|
162
|
+
|
|
152
163
|
if (tokens[0] === 'mode' && (tokens[1] === 'scoped' || tokens[1] === 'all')) {
|
|
153
164
|
return { command: 'mode', mode: tokens[1] };
|
|
154
165
|
}
|
|
@@ -245,4 +256,86 @@ export function formatHistoryMessage(
|
|
|
245
256
|
.join('\n');
|
|
246
257
|
}
|
|
247
258
|
|
|
259
|
+
function formatUsd(value: number): string {
|
|
260
|
+
if (!Number.isFinite(value)) {
|
|
261
|
+
return 'n/a';
|
|
262
|
+
}
|
|
263
|
+
if (Math.abs(value) >= 1) {
|
|
264
|
+
return `$${value.toFixed(4)}`;
|
|
265
|
+
}
|
|
266
|
+
return `$${value.toFixed(6)}`;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function formatShare(value: number | null): string {
|
|
270
|
+
if (value === null) {
|
|
271
|
+
return 'n/a';
|
|
272
|
+
}
|
|
273
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function formatMean(value: number | null, suffix: string): string {
|
|
277
|
+
if (value === null) {
|
|
278
|
+
return 'n/a';
|
|
279
|
+
}
|
|
280
|
+
if (suffix === 'ms') {
|
|
281
|
+
return `${value.toFixed(1)}${suffix}`;
|
|
282
|
+
}
|
|
283
|
+
return `${formatUsd(value)}${suffix}`;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Operator-facing text for `/smart-router stats` (privacy-safe aggregates only).
|
|
288
|
+
*/
|
|
289
|
+
export function formatStatsMessage(
|
|
290
|
+
entries: readonly RoutingTelemetry[],
|
|
291
|
+
options?: {
|
|
292
|
+
fleet?: readonly ModelProfile[];
|
|
293
|
+
priceCatalog?: PriceCatalog | null;
|
|
294
|
+
},
|
|
295
|
+
): string {
|
|
296
|
+
const snapshot = aggregateSessionStatsFromFleet(
|
|
297
|
+
entries,
|
|
298
|
+
options?.fleet,
|
|
299
|
+
options?.priceCatalog,
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
if (snapshot.entry_count === 0) {
|
|
303
|
+
return 'No routing stats yet (empty telemetry window).';
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const lines = [
|
|
307
|
+
`Entries: ${snapshot.entry_count}`,
|
|
308
|
+
`Cost: total ${formatUsd(snapshot.total_cost_usd)} | mean ${formatMean(snapshot.mean_cost_usd, '')}`,
|
|
309
|
+
`Latency: total ${snapshot.total_latency_ms.toFixed(0)}ms | mean ${formatMean(snapshot.mean_latency_ms, 'ms')}`,
|
|
310
|
+
`Planning delegate share: ${formatShare(snapshot.planning_delegate_share)} (direct ${formatShare(snapshot.direct_share)})`,
|
|
311
|
+
`Local vs cloud (when known): local ${formatShare(snapshot.local_share)} | cloud ${formatShare(snapshot.cloud_share)}`,
|
|
312
|
+
'Role cost breakdown:',
|
|
313
|
+
` primary (pin path): ${snapshot.role_cost.primary.count} | ${formatUsd(snapshot.role_cost.primary.total_cost_usd)}`,
|
|
314
|
+
` planning_delegate: ${snapshot.role_cost.planning_delegate.count} | ${formatUsd(snapshot.role_cost.planning_delegate.total_cost_usd)}`,
|
|
315
|
+
` other: ${snapshot.role_cost.other.count} | ${formatUsd(snapshot.role_cost.other.total_cost_usd)}`,
|
|
316
|
+
];
|
|
317
|
+
|
|
318
|
+
if (snapshot.frontier_savings_usd !== undefined) {
|
|
319
|
+
lines.push(
|
|
320
|
+
`Vs always-frontier savings (est.): ${formatUsd(snapshot.frontier_savings_usd)}`,
|
|
321
|
+
' formula: sum max(0, tokens/1e6 * frontier_cost_per_1m - estimated_cost_usd); omitted when prices missing',
|
|
322
|
+
);
|
|
323
|
+
} else {
|
|
324
|
+
lines.push('Vs always-frontier savings: (omitted — frontier prices unavailable)');
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
return lines.join('\n');
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** JSON snapshot helper for automation (same aggregate as formatStatsMessage). */
|
|
331
|
+
export function buildStatsSnapshot(
|
|
332
|
+
entries: readonly RoutingTelemetry[],
|
|
333
|
+
options?: {
|
|
334
|
+
fleet?: readonly ModelProfile[];
|
|
335
|
+
priceCatalog?: PriceCatalog | null;
|
|
336
|
+
},
|
|
337
|
+
): SessionStatsSnapshot {
|
|
338
|
+
return aggregateSessionStatsFromFleet(entries, options?.fleet, options?.priceCatalog);
|
|
339
|
+
}
|
|
340
|
+
|
|
248
341
|
export type { FleetMode };
|
|
@@ -2,6 +2,7 @@ import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
4
|
formatHistoryMessage,
|
|
5
|
+
formatStatsMessage,
|
|
5
6
|
formatStatusMessage,
|
|
6
7
|
parseSmartRouterArgs,
|
|
7
8
|
} from './command-formatters.js';
|
|
@@ -13,13 +14,14 @@ import { FLEET_MODE_ENTRY_TYPE } from './session-lifecycle.js';
|
|
|
13
14
|
import type { SmartRouterRuntime } from './types.js';
|
|
14
15
|
|
|
15
16
|
export const SMART_ROUTER_USAGE =
|
|
16
|
-
'/smart-router [status] | history [limit] | mode scoped|all | pricing refresh | export dataset [--limit N] | export telemetry-contrib [--limit N] | feedback good|bad | unpin';
|
|
17
|
+
'/smart-router [status] | history [limit] | stats [limit] | mode scoped|all | pricing refresh | export dataset [--limit N] | export telemetry-contrib [--limit N] | feedback good|bad | unpin';
|
|
17
18
|
|
|
18
19
|
type CompletionItem = { value: string; label: string };
|
|
19
20
|
|
|
20
21
|
const TOP_LEVEL: CompletionItem[] = [
|
|
21
22
|
{ value: 'status', label: 'Show last routing decision' },
|
|
22
23
|
{ value: 'history', label: 'Show recent routing history' },
|
|
24
|
+
{ value: 'stats', label: 'Show session stats + role cost breakdown' },
|
|
23
25
|
{ value: 'mode', label: 'Switch fleet mode (scoped or all)' },
|
|
24
26
|
{ value: 'pricing', label: 'Manage pricing catalog' },
|
|
25
27
|
{ value: 'export', label: 'Export opt-in routing dataset' },
|
|
@@ -52,6 +54,8 @@ export const SMART_ROUTER_FULL_INVOCATIONS = [
|
|
|
52
54
|
'status',
|
|
53
55
|
'history',
|
|
54
56
|
'history 10',
|
|
57
|
+
'stats',
|
|
58
|
+
'stats 50',
|
|
55
59
|
'mode scoped',
|
|
56
60
|
'mode all',
|
|
57
61
|
'pricing refresh',
|
|
@@ -100,6 +104,10 @@ export function getSmartRouterArgumentCompletions(prefix: string): CompletionIte
|
|
|
100
104
|
return [{ value: 'history', label: 'Show recent routing history' }];
|
|
101
105
|
}
|
|
102
106
|
|
|
107
|
+
if (tokens[0] === 'stats') {
|
|
108
|
+
return [{ value: 'stats', label: 'Show session stats + role cost breakdown' }];
|
|
109
|
+
}
|
|
110
|
+
|
|
103
111
|
const firstToken = tokens[0] ?? '';
|
|
104
112
|
const filtered = filterByPrefix(TOP_LEVEL, firstToken);
|
|
105
113
|
return filtered.length > 0 ? filtered : null;
|
|
@@ -143,7 +151,7 @@ export function registerSmartRouterCommand(
|
|
|
143
151
|
): void {
|
|
144
152
|
pi.registerCommand('smart-router', {
|
|
145
153
|
description:
|
|
146
|
-
'Show routing status/history, switch fleet mode (scoped|all), refresh pricing, or export dataset',
|
|
154
|
+
'Show routing status/history/stats, switch fleet mode (scoped|all), refresh pricing, or export dataset',
|
|
147
155
|
getArgumentCompletions: getSmartRouterArgumentCompletions,
|
|
148
156
|
handler: async (args, ctx) => {
|
|
149
157
|
try {
|
|
@@ -167,6 +175,20 @@ export function registerSmartRouterCommand(
|
|
|
167
175
|
return;
|
|
168
176
|
}
|
|
169
177
|
|
|
178
|
+
if (parsed.command === 'stats') {
|
|
179
|
+
throwIfCommandAborted(signal);
|
|
180
|
+
const rows = await runtime.store.listTelemetry({ limit: parsed.limit });
|
|
181
|
+
throwIfCommandAborted(signal);
|
|
182
|
+
ctx.ui.notify(
|
|
183
|
+
formatStatsMessage(rows, {
|
|
184
|
+
fleet: runtime.streamDeps.fleet,
|
|
185
|
+
priceCatalog: runtime.priceCatalog,
|
|
186
|
+
}),
|
|
187
|
+
'info',
|
|
188
|
+
);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
|
|
170
192
|
if (parsed.command === 'pricing') {
|
|
171
193
|
throwIfCommandAborted(signal);
|
|
172
194
|
const { modelCount, lastUpdated } = await refreshPricingCatalog(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
|
-
AuthStorage,
|
|
3
2
|
ModelRegistry,
|
|
3
|
+
ModelRuntime,
|
|
4
4
|
type ExtensionAPI,
|
|
5
5
|
} from '@earendil-works/pi-coding-agent';
|
|
6
6
|
|
|
@@ -35,8 +35,9 @@ export async function createSmartRouterRuntime(cwd: string): Promise<{
|
|
|
35
35
|
runtime: SmartRouterRuntime;
|
|
36
36
|
datasetNotify: DatasetNotify;
|
|
37
37
|
}> {
|
|
38
|
-
|
|
39
|
-
const
|
|
38
|
+
// Placeholder until session_start / commands bind ctx.modelRegistry (SP-087).
|
|
39
|
+
const modelRuntime = await ModelRuntime.create();
|
|
40
|
+
const modelRegistry = new ModelRegistry(modelRuntime);
|
|
40
41
|
const hydraMatcher = await initHydraMatcher();
|
|
41
42
|
const store = createExtensionStore(cwd);
|
|
42
43
|
const operatorConfig = resolveOperatorConfigFromEnv();
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
} from './fleet-bootstrap.js';
|
|
37
37
|
import {
|
|
38
38
|
formatHistoryMessage,
|
|
39
|
+
formatStatsMessage,
|
|
39
40
|
formatStatusMessage,
|
|
40
41
|
parseSmartRouterArgs,
|
|
41
42
|
resolveHistoryModelId,
|
|
@@ -82,6 +83,7 @@ export {
|
|
|
82
83
|
formatLmuStatus,
|
|
83
84
|
formatPricingStalenessLine,
|
|
84
85
|
formatHistoryMessage,
|
|
86
|
+
formatStatsMessage,
|
|
85
87
|
formatStatusMessage,
|
|
86
88
|
getDatasetExportPath,
|
|
87
89
|
getRouterStateDbPath,
|
|
@@ -34,6 +34,7 @@ export type FleetMode = 'scoped' | 'all';
|
|
|
34
34
|
export type SmartRouterCommand =
|
|
35
35
|
| { command: 'status' }
|
|
36
36
|
| { command: 'history'; limit: number }
|
|
37
|
+
| { command: 'stats'; limit: number }
|
|
37
38
|
| { command: 'mode'; mode: FleetMode }
|
|
38
39
|
| { command: 'pricing'; subcommand: 'refresh' }
|
|
39
40
|
| { command: 'export'; subcommand: 'dataset'; limit: number }
|
package/README.md
CHANGED
|
@@ -111,7 +111,7 @@ After installing via `pi install npm:pi-smart-router` (or from clone — see bel
|
|
|
111
111
|
|
|
112
112
|
1. Authenticate providers (`/login`) and enable models in your scoped list if you use one (`/scoped-models`)
|
|
113
113
|
2. `/model smart-router/auto` — every turn runs through the routing pipeline
|
|
114
|
-
3. `/smart-router status
|
|
114
|
+
3. `/smart-router status`, `/smart-router history`, or `/smart-router stats` — inspect routing decisions and window aggregates
|
|
115
115
|
|
|
116
116
|
Set `SMART_ROUTER_LOG_ROUTING=1` before starting pi to print each routing decision to stderr (see [Environment variables](#environment-variables)).
|
|
117
117
|
|
|
@@ -127,24 +127,43 @@ After auth or model list changes, restart pi or run `/reload`.
|
|
|
127
127
|
|
|
128
128
|
### Develop from clone
|
|
129
129
|
|
|
130
|
-
|
|
130
|
+
Requires **Pi ≥ 0.80.8** (`pi.minPiVersion`). The extension loads TypeScript from this repo via pi’s loader — no `npm run build` needed for dogfooding.
|
|
131
|
+
|
|
132
|
+
**Recommended (works from any cwd):** install the clone as a **path package** and remove any published npm copy. npm and path packages have different identities; leaving both installed can double-load and you may keep running a stale npm tarball.
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
# From anywhere — remove published package if present
|
|
136
|
+
pi remove npm:pi-smart-router
|
|
137
|
+
|
|
138
|
+
# Prefer an absolute path (relative paths are resolved from the install cwd first).
|
|
139
|
+
# Pi stores a path relative to ~/.pi/agent/ in settings (same pattern as other local packages).
|
|
140
|
+
pi install /path/to/pi-smart-router
|
|
141
|
+
|
|
142
|
+
# Pi does not run npm install for path packages
|
|
143
|
+
cd /path/to/pi-smart-router && npm install
|
|
144
|
+
|
|
145
|
+
pi list # must show this clone path, not ~/.pi/agent/npm/.../pi-smart-router
|
|
146
|
+
pi --list-models | grep smart-router
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
**Alternative (repo-root discovery only):** with the npm package removed, start `pi` with cwd at this repo root. Project-local extensions under `.pi/extensions/` load only after the project is trusted — without trust, `smart-router` never appears in `/scoped-models` or `/model`.
|
|
131
150
|
|
|
132
151
|
**On first run**, pi prompts you to trust the project when it detects `.pi/extensions/`. Accept the prompt.
|
|
133
152
|
|
|
134
153
|
**Later or missed prompt:** run `/trust` inside pi to save a trust decision for this directory (or its parent) to `~/.pi/agent/trust.json`. Trust on a **parent folder** (for example `~/Documents/github`) applies to this repo as well. After `/trust`, **restart pi** — the current session is not reloaded automatically.
|
|
135
154
|
|
|
136
|
-
**Verify the extension loaded
|
|
155
|
+
**Verify the extension loaded:**
|
|
137
156
|
|
|
138
157
|
```bash
|
|
139
|
-
cd pi-smart-router
|
|
140
158
|
pi --list-models | grep smart-router
|
|
141
159
|
```
|
|
142
160
|
|
|
143
161
|
You should see `smart-router auto`. If the line is missing:
|
|
144
162
|
|
|
145
|
-
1. Confirm `pi`
|
|
146
|
-
2. Confirm
|
|
147
|
-
3.
|
|
163
|
+
1. Confirm `pi list` points at this clone (path package) or that cwd is the repo root (discovery mode).
|
|
164
|
+
2. Confirm no `npm:pi-smart-router` entry remains in settings while developing from clone.
|
|
165
|
+
3. Confirm the project is trusted if relying on `.pi/extensions/` discovery (`/trust`, or check `~/.pi/agent/trust.json`).
|
|
166
|
+
4. Restart pi or run `/reload` after trusting or changing packages.
|
|
148
167
|
|
|
149
168
|
Non-interactive one-shot checks can pass `--approve` to trust project-local resources for that run only.
|
|
150
169
|
|
|
@@ -180,7 +199,7 @@ pi exposes two different **auto** models. They are easy to confuse but play diff
|
|
|
180
199
|
**When to use `smart-router/auto`:**
|
|
181
200
|
|
|
182
201
|
- You want cost/capability-aware model selection across your full authenticated fleet
|
|
183
|
-
- You rely on session pinning, failover, or `/smart-router status` / `history` telemetry
|
|
202
|
+
- You rely on session pinning, failover, or `/smart-router status` / `history` / `stats` telemetry
|
|
184
203
|
- Tool-heavy sessions with Gemini economical models work via in-repo replay repair; add `cursor/auto` for unrepairable Google replay edge cases (see [pi-smart-router#85](https://github.com/beettlle/pi-smart-router/issues/85))
|
|
185
204
|
|
|
186
205
|
Cursor models (`cursor/*`, `composer-*`, and the opaque fleet id `default`) map to **frontier-cloud** tier in `pi-model-mapper.ts` so HyDRA can score them against Gemini and Claude instead of treating them as unknown economical models ([pi-smart-router#40](https://github.com/beettlle/pi-smart-router/issues/40), [pi-smart-router#70](https://github.com/beettlle/pi-smart-router/issues/70)). Related: [pi-smart-router#23](https://github.com/beettlle/pi-smart-router/issues/23) (turn envelope / pin order), [pi-smart-router#37](https://github.com/beettlle/pi-smart-router/issues/37) (Gemini `thought_signature` errors).
|
|
@@ -198,6 +217,7 @@ Cursor models bill against your **Cursor Pro subscription quota**, not per-token
|
|
|
198
217
|
| `/smart-router` | Same as `status` (default when no subcommand is given) |
|
|
199
218
|
| `/smart-router status` | Show fleet mode, fleet size, pricing freshness/staleness, and the last routing decision (stage, tier, selected model, latency) |
|
|
200
219
|
| `/smart-router history` | Show recent routing telemetry from SQLite (default limit; optional numeric limit, e.g. `/smart-router history 20`). Displays the concrete delegated model id (never bare virtual `auto`) |
|
|
220
|
+
| `/smart-router stats` | Privacy-safe session/window aggregates from routing telemetry: count, mean cost/latency, planning_delegate vs direct share, local vs cloud when distinguishable, and role cost breakdown (primary pin path / planning_delegate / other). Optional vs-always-frontier savings when frontier fleet prices exist (omitted otherwise). Optional numeric limit, e.g. `/smart-router stats 50` |
|
|
201
221
|
| `/smart-router mode scoped` | Route only among pi's **enabled model patterns** (default) |
|
|
202
222
|
| `/smart-router mode all` | Route among **all authenticated models** in the registry |
|
|
203
223
|
| `/smart-router pricing refresh` | Manually fetch LiteLLM pricing from `LITELLM_PRICING_URL`, persist to SQLite, and rebuild the fleet with updated rates |
|
|
@@ -1149,10 +1169,11 @@ npm run release:check
|
|
|
1149
1169
|
npm pack --dry-run
|
|
1150
1170
|
```
|
|
1151
1171
|
|
|
1152
|
-
**Post-publish smoke (manual, macOS):** CI does not run `pi install`. After publish:
|
|
1172
|
+
**Post-publish smoke (manual, macOS):** CI does not run `pi install`. After publish (use the version just published; remove any path-package clone entry first if dogfooding):
|
|
1153
1173
|
|
|
1154
1174
|
```bash
|
|
1155
|
-
pi
|
|
1175
|
+
pi remove ../../Documents/github/pi-smart-router # if path dogfood was installed
|
|
1176
|
+
pi install npm:pi-smart-router@<published-version>
|
|
1156
1177
|
pi --list-models | grep smart-router
|
|
1157
1178
|
# in pi: /model smart-router/auto, /smart-router status
|
|
1158
1179
|
```
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
"livecodebench": "https://livecodebench.github.io/leaderboard.html",
|
|
8
8
|
"bfcl": "https://gorilla.cs.berkeley.edu/leaderboard.html"
|
|
9
9
|
},
|
|
10
|
-
"scrape_date": "2026-07-
|
|
11
|
-
"catalog_freeze_date": "2026-07-
|
|
10
|
+
"scrape_date": "2026-07-16",
|
|
11
|
+
"catalog_freeze_date": "2026-07-16"
|
|
12
12
|
},
|
|
13
13
|
"aliases": {
|
|
14
14
|
"claude-3-5-sonnet": "claude-sonnet-4-6",
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Privacy-safe session / window stats over RoutingTelemetry (SP-207 / #118).
|
|
3
|
+
*
|
|
4
|
+
* Aggregates only numeric and categorical telemetry fields — never prompt,
|
|
5
|
+
* message, or tool-argument bodies.
|
|
6
|
+
*/
|
|
7
|
+
import type { ModelProfile, PriceCatalog, RoutingTelemetry, Tier } from '../../domain/types/index.js';
|
|
8
|
+
export type RoleCostBucket = 'primary' | 'planning_delegate' | 'other';
|
|
9
|
+
export type DeploymentClass = 'local' | 'cloud' | 'unknown';
|
|
10
|
+
export interface RoleBucketStats {
|
|
11
|
+
readonly count: number;
|
|
12
|
+
readonly total_cost_usd: number;
|
|
13
|
+
}
|
|
14
|
+
export interface RoleCostBreakdown {
|
|
15
|
+
readonly primary: RoleBucketStats;
|
|
16
|
+
readonly planning_delegate: RoleBucketStats;
|
|
17
|
+
readonly other: RoleBucketStats;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Compact JSON snapshot for automation / MCP (llm-use `stats_snapshot` analog).
|
|
21
|
+
* Optional `frontier_savings_usd` is omitted when prices are unavailable (fail closed).
|
|
22
|
+
*/
|
|
23
|
+
export interface SessionStatsSnapshot {
|
|
24
|
+
readonly entry_count: number;
|
|
25
|
+
readonly total_cost_usd: number;
|
|
26
|
+
readonly mean_cost_usd: number | null;
|
|
27
|
+
readonly total_latency_ms: number;
|
|
28
|
+
readonly mean_latency_ms: number | null;
|
|
29
|
+
/** Share of entries with planning_delegate_path === 'delegate' (0–1), null if empty. */
|
|
30
|
+
readonly planning_delegate_share: number | null;
|
|
31
|
+
/** Share of entries with non-delegate path (direct / none / null). */
|
|
32
|
+
readonly direct_share: number | null;
|
|
33
|
+
/** Share classified as local (zero-tier) when distinguishable; null if none classified. */
|
|
34
|
+
readonly local_share: number | null;
|
|
35
|
+
/** Share classified as cloud when distinguishable; null if none classified. */
|
|
36
|
+
readonly cloud_share: number | null;
|
|
37
|
+
readonly role_cost: RoleCostBreakdown;
|
|
38
|
+
/**
|
|
39
|
+
* Estimated USD saved vs always-frontier baseline.
|
|
40
|
+
* Formula: sum over entries with token counts of
|
|
41
|
+
* max(0, tokens/1e6 * frontier_cost_per_1m - estimated_cost_usd).
|
|
42
|
+
* Omitted when frontier price inputs are missing (fail closed).
|
|
43
|
+
*/
|
|
44
|
+
readonly frontier_savings_usd?: number;
|
|
45
|
+
}
|
|
46
|
+
export interface AggregateSessionStatsOptions {
|
|
47
|
+
/**
|
|
48
|
+
* USD per 1M tokens for the always-frontier baseline.
|
|
49
|
+
* When absent / non-finite / ≤0, `frontier_savings_usd` is omitted.
|
|
50
|
+
*/
|
|
51
|
+
readonly frontier_cost_per_1m?: number;
|
|
52
|
+
/** Optional model_id → tier map (e.g. from fleet) for local vs cloud. */
|
|
53
|
+
readonly tier_by_model_id?: ReadonlyMap<string, Tier>;
|
|
54
|
+
}
|
|
55
|
+
/** Mutually exclusive role for cost bucketing. */
|
|
56
|
+
export declare function classifyRoleCostBucket(entry: RoutingTelemetry): RoleCostBucket;
|
|
57
|
+
export declare function classifyDeployment(entry: RoutingTelemetry, tierByModelId?: ReadonlyMap<string, Tier>): DeploymentClass;
|
|
58
|
+
/**
|
|
59
|
+
* Resolve always-frontier cost/1M from fleet + optional catalog.
|
|
60
|
+
* Returns undefined when no positive frontier price is available (fail closed).
|
|
61
|
+
*/
|
|
62
|
+
export declare function resolveFrontierCostPer1M(fleet?: readonly ModelProfile[], catalog?: PriceCatalog | null): number | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Pure aggregate over routing telemetry for operator stats.
|
|
65
|
+
* Does not read or emit prompt/message/tool bodies.
|
|
66
|
+
*/
|
|
67
|
+
export declare function aggregateSessionStats(entries: readonly RoutingTelemetry[], options?: AggregateSessionStatsOptions): SessionStatsSnapshot;
|
|
68
|
+
/**
|
|
69
|
+
* Optional vs-always-frontier savings. Returns undefined when price input is
|
|
70
|
+
* missing or non-positive (fail closed) or when no entry has token counts.
|
|
71
|
+
*/
|
|
72
|
+
export declare function estimateFrontierSavingsUsd(entries: readonly RoutingTelemetry[], frontierCostPer1M: number | undefined): number | undefined;
|
|
73
|
+
/** Convenience: aggregate with fleet/catalog-derived tier map + frontier price. */
|
|
74
|
+
export declare function aggregateSessionStatsFromFleet(entries: readonly RoutingTelemetry[], fleet?: readonly ModelProfile[], catalog?: PriceCatalog | null): SessionStatsSnapshot;
|
|
75
|
+
/** Keys that must never appear on a stats snapshot (privacy). */
|
|
76
|
+
export declare const SESSION_STATS_FORBIDDEN_KEYS: readonly ["prompt", "prompt_text", "messages", "content", "tool_calls", "tool_args", "pepper"];
|
|
77
|
+
export declare function assertSessionStatsPrivacySafe(snapshot: SessionStatsSnapshot): void;
|
|
78
|
+
//# sourceMappingURL=session-stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-stats.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/telemetry/session-stats.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,YAAY,EAEZ,YAAY,EACZ,gBAAgB,EAChB,IAAI,EACL,MAAM,6BAA6B,CAAC;AAErC,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,mBAAmB,GAAG,OAAO,CAAC;AAEvE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAE5D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,iBAAiB,EAAE,eAAe,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,wFAAwF;IACxF,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,sEAAsE;IACtE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,2FAA2F;IAC3F,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,+EAA+E;IAC/E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,4BAA4B;IAC3C;;;OAGG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC,yEAAyE;IACzE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;CACvD;AAED,kDAAkD;AAClD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,gBAAgB,GAAG,cAAc,CAQ9E;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,gBAAgB,EACvB,aAAa,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,GACxC,eAAe,CAUjB;AAMD;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,CAAC,EAAE,SAAS,YAAY,EAAE,EAC/B,OAAO,CAAC,EAAE,YAAY,GAAG,IAAI,GAC5B,MAAM,GAAG,SAAS,CA6BpB;AAcD;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,OAAO,GAAE,4BAAiC,GACzC,oBAAoB,CA+DtB;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,iBAAiB,EAAE,MAAM,GAAG,SAAS,GACpC,MAAM,GAAG,SAAS,CA4BpB;AAED,mFAAmF;AACnF,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,KAAK,CAAC,EAAE,SAAS,YAAY,EAAE,EAC/B,OAAO,CAAC,EAAE,YAAY,GAAG,IAAI,GAC5B,oBAAoB,CAOtB;AAED,iEAAiE;AACjE,eAAO,MAAM,4BAA4B,gGAQ/B,CAAC;AAEX,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAOlF"}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Privacy-safe session / window stats over RoutingTelemetry (SP-207 / #118).
|
|
3
|
+
*
|
|
4
|
+
* Aggregates only numeric and categorical telemetry fields — never prompt,
|
|
5
|
+
* message, or tool-argument bodies.
|
|
6
|
+
*/
|
|
7
|
+
/** Mutually exclusive role for cost bucketing. */
|
|
8
|
+
export function classifyRoleCostBucket(entry) {
|
|
9
|
+
if (entry.planning_delegate_path === 'delegate') {
|
|
10
|
+
return 'planning_delegate';
|
|
11
|
+
}
|
|
12
|
+
if (entry.pin_reason != null) {
|
|
13
|
+
return 'primary';
|
|
14
|
+
}
|
|
15
|
+
return 'other';
|
|
16
|
+
}
|
|
17
|
+
export function classifyDeployment(entry, tierByModelId) {
|
|
18
|
+
const fromFleet = tierByModelId?.get(entry.selected_model_id);
|
|
19
|
+
const tier = fromFleet ?? entry.tier_hint;
|
|
20
|
+
if (tier === 'zero-tier') {
|
|
21
|
+
return 'local';
|
|
22
|
+
}
|
|
23
|
+
if (tier === 'economical-cloud' || tier === 'frontier-cloud') {
|
|
24
|
+
return 'cloud';
|
|
25
|
+
}
|
|
26
|
+
return 'unknown';
|
|
27
|
+
}
|
|
28
|
+
function isPlanningDelegate(path) {
|
|
29
|
+
return path === 'delegate';
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Resolve always-frontier cost/1M from fleet + optional catalog.
|
|
33
|
+
* Returns undefined when no positive frontier price is available (fail closed).
|
|
34
|
+
*/
|
|
35
|
+
export function resolveFrontierCostPer1M(fleet, catalog) {
|
|
36
|
+
const candidates = [];
|
|
37
|
+
if (fleet) {
|
|
38
|
+
for (const model of fleet) {
|
|
39
|
+
if (model.tier !== 'frontier-cloud') {
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
const cost = model.pricing.fallback_cost_per_1m;
|
|
43
|
+
if (Number.isFinite(cost) && cost > 0) {
|
|
44
|
+
candidates.push(cost);
|
|
45
|
+
}
|
|
46
|
+
if (catalog) {
|
|
47
|
+
const key = model.pricing.registry_key ?? model.id;
|
|
48
|
+
const fromCatalog = catalog.user_overrides[key] ?? catalog.registry_snapshot[key];
|
|
49
|
+
if (fromCatalog !== undefined && Number.isFinite(fromCatalog) && fromCatalog > 0) {
|
|
50
|
+
candidates.push(fromCatalog);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (candidates.length === 0) {
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
// Upper-bound “always frontier” baseline: most expensive known frontier rate.
|
|
59
|
+
return Math.max(...candidates);
|
|
60
|
+
}
|
|
61
|
+
function buildTierMap(fleet) {
|
|
62
|
+
if (!fleet || fleet.length === 0) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
const map = new Map();
|
|
66
|
+
for (const model of fleet) {
|
|
67
|
+
map.set(model.id, model.tier);
|
|
68
|
+
map.set(`${model.provider}/${model.id}`, model.tier);
|
|
69
|
+
}
|
|
70
|
+
return map;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Pure aggregate over routing telemetry for operator stats.
|
|
74
|
+
* Does not read or emit prompt/message/tool bodies.
|
|
75
|
+
*/
|
|
76
|
+
export function aggregateSessionStats(entries, options = {}) {
|
|
77
|
+
const tierByModelId = options.tier_by_model_id;
|
|
78
|
+
const roleAccum = {
|
|
79
|
+
primary: { count: 0, total_cost_usd: 0 },
|
|
80
|
+
planning_delegate: { count: 0, total_cost_usd: 0 },
|
|
81
|
+
other: { count: 0, total_cost_usd: 0 },
|
|
82
|
+
};
|
|
83
|
+
let totalCost = 0;
|
|
84
|
+
let totalLatency = 0;
|
|
85
|
+
let delegateCount = 0;
|
|
86
|
+
let localCount = 0;
|
|
87
|
+
let cloudCount = 0;
|
|
88
|
+
let classifiedDeployment = 0;
|
|
89
|
+
for (const entry of entries) {
|
|
90
|
+
const cost = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
91
|
+
const latency = Number.isFinite(entry.routing_latency_ms) ? entry.routing_latency_ms : 0;
|
|
92
|
+
totalCost += cost;
|
|
93
|
+
totalLatency += latency;
|
|
94
|
+
if (isPlanningDelegate(entry.planning_delegate_path)) {
|
|
95
|
+
delegateCount += 1;
|
|
96
|
+
}
|
|
97
|
+
const role = classifyRoleCostBucket(entry);
|
|
98
|
+
roleAccum[role].count += 1;
|
|
99
|
+
roleAccum[role].total_cost_usd += cost;
|
|
100
|
+
const deployment = classifyDeployment(entry, tierByModelId);
|
|
101
|
+
if (deployment === 'local') {
|
|
102
|
+
localCount += 1;
|
|
103
|
+
classifiedDeployment += 1;
|
|
104
|
+
}
|
|
105
|
+
else if (deployment === 'cloud') {
|
|
106
|
+
cloudCount += 1;
|
|
107
|
+
classifiedDeployment += 1;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const n = entries.length;
|
|
111
|
+
const snapshot = {
|
|
112
|
+
entry_count: n,
|
|
113
|
+
total_cost_usd: totalCost,
|
|
114
|
+
mean_cost_usd: n > 0 ? totalCost / n : null,
|
|
115
|
+
total_latency_ms: totalLatency,
|
|
116
|
+
mean_latency_ms: n > 0 ? totalLatency / n : null,
|
|
117
|
+
planning_delegate_share: n > 0 ? delegateCount / n : null,
|
|
118
|
+
direct_share: n > 0 ? (n - delegateCount) / n : null,
|
|
119
|
+
local_share: classifiedDeployment > 0 ? localCount / classifiedDeployment : null,
|
|
120
|
+
cloud_share: classifiedDeployment > 0 ? cloudCount / classifiedDeployment : null,
|
|
121
|
+
role_cost: {
|
|
122
|
+
primary: { ...roleAccum.primary },
|
|
123
|
+
planning_delegate: { ...roleAccum.planning_delegate },
|
|
124
|
+
other: { ...roleAccum.other },
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
const frontierSavings = estimateFrontierSavingsUsd(entries, options.frontier_cost_per_1m);
|
|
128
|
+
if (frontierSavings !== undefined) {
|
|
129
|
+
return { ...snapshot, frontier_savings_usd: frontierSavings };
|
|
130
|
+
}
|
|
131
|
+
return snapshot;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Optional vs-always-frontier savings. Returns undefined when price input is
|
|
135
|
+
* missing or non-positive (fail closed) or when no entry has token counts.
|
|
136
|
+
*/
|
|
137
|
+
export function estimateFrontierSavingsUsd(entries, frontierCostPer1M) {
|
|
138
|
+
if (frontierCostPer1M === undefined ||
|
|
139
|
+
!Number.isFinite(frontierCostPer1M) ||
|
|
140
|
+
frontierCostPer1M <= 0) {
|
|
141
|
+
return undefined;
|
|
142
|
+
}
|
|
143
|
+
let savings = 0;
|
|
144
|
+
let counted = 0;
|
|
145
|
+
for (const entry of entries) {
|
|
146
|
+
const tokens = entry.estimated_input_tokens;
|
|
147
|
+
if (tokens === null || !Number.isFinite(tokens) || tokens < 0) {
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const actual = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
151
|
+
const frontierCost = (tokens / 1_000_000) * frontierCostPer1M;
|
|
152
|
+
savings += Math.max(0, frontierCost - actual);
|
|
153
|
+
counted += 1;
|
|
154
|
+
}
|
|
155
|
+
if (counted === 0) {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
return savings;
|
|
159
|
+
}
|
|
160
|
+
/** Convenience: aggregate with fleet/catalog-derived tier map + frontier price. */
|
|
161
|
+
export function aggregateSessionStatsFromFleet(entries, fleet, catalog) {
|
|
162
|
+
const tierMap = buildTierMap(fleet);
|
|
163
|
+
const frontierCost = resolveFrontierCostPer1M(fleet, catalog);
|
|
164
|
+
return aggregateSessionStats(entries, {
|
|
165
|
+
...(tierMap ? { tier_by_model_id: tierMap } : {}),
|
|
166
|
+
...(frontierCost !== undefined ? { frontier_cost_per_1m: frontierCost } : {}),
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
/** Keys that must never appear on a stats snapshot (privacy). */
|
|
170
|
+
export const SESSION_STATS_FORBIDDEN_KEYS = [
|
|
171
|
+
'prompt',
|
|
172
|
+
'prompt_text',
|
|
173
|
+
'messages',
|
|
174
|
+
'content',
|
|
175
|
+
'tool_calls',
|
|
176
|
+
'tool_args',
|
|
177
|
+
'pepper',
|
|
178
|
+
];
|
|
179
|
+
export function assertSessionStatsPrivacySafe(snapshot) {
|
|
180
|
+
const json = JSON.stringify(snapshot);
|
|
181
|
+
for (const key of SESSION_STATS_FORBIDDEN_KEYS) {
|
|
182
|
+
if (json.includes(`"${key}"`)) {
|
|
183
|
+
throw new Error(`Stats snapshot contains forbidden privacy key: ${key}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
//# sourceMappingURL=session-stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-stats.js","sourceRoot":"","sources":["../../../src/infrastructure/telemetry/session-stats.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA+DH,kDAAkD;AAClD,MAAM,UAAU,sBAAsB,CAAC,KAAuB;IAC5D,IAAI,KAAK,CAAC,sBAAsB,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,KAAuB,EACvB,aAAyC;IAEzC,MAAM,SAAS,GAAG,aAAa,EAAE,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;IAC1C,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QACzB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;QAC7D,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAiC;IAC3D,OAAO,IAAI,KAAK,UAAU,CAAC;AAC7B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CACtC,KAA+B,EAC/B,OAA6B;IAE7B,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;gBACpC,SAAS;YACX,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC;YAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,EAAE,CAAC;gBACnD,MAAM,WAAW,GACf,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAChE,IAAI,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;oBACjF,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8EAA8E;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,YAAY,CAAC,KAA+B;IACnD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAgB,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAAoC,EACpC,UAAwC,EAAE;IAE1C,MAAM,aAAa,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC/C,MAAM,SAAS,GAAG;QAChB,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QACxC,iBAAiB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QAClD,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;KACvC,CAAC;IAEF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAE7B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QACzF,SAAS,IAAI,IAAI,CAAC;QAClB,YAAY,IAAI,OAAO,CAAC;QAExB,IAAI,kBAAkB,CAAC,KAAK,CAAC,sBAAsB,CAAC,EAAE,CAAC;YACrD,aAAa,IAAI,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,IAAI,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAC3C,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC3B,SAAS,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,IAAI,CAAC;QAEvC,MAAM,UAAU,GAAG,kBAAkB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QAC5D,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YAC3B,UAAU,IAAI,CAAC,CAAC;YAChB,oBAAoB,IAAI,CAAC,CAAC;QAC5B,CAAC;aAAM,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YAClC,UAAU,IAAI,CAAC,CAAC;YAChB,oBAAoB,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,QAAQ,GAAyB;QACrC,WAAW,EAAE,CAAC;QACd,cAAc,EAAE,SAAS;QACzB,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAC3C,gBAAgB,EAAE,YAAY;QAC9B,eAAe,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAChD,uBAAuB,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QACzD,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QACpD,WAAW,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,oBAAoB,CAAC,CAAC,CAAC,IAAI;QAChF,WAAW,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,oBAAoB,CAAC,CAAC,CAAC,IAAI;QAChF,SAAS,EAAE;YACT,OAAO,EAAE,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;YACjC,iBAAiB,EAAE,EAAE,GAAG,SAAS,CAAC,iBAAiB,EAAE;YACrD,KAAK,EAAE,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE;SAC9B;KACF,CAAC;IAEF,MAAM,eAAe,GAAG,0BAA0B,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC1F,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,EAAE,GAAG,QAAQ,EAAE,oBAAoB,EAAE,eAAe,EAAE,CAAC;IAChE,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,0BAA0B,CACxC,OAAoC,EACpC,iBAAqC;IAErC,IACE,iBAAiB,KAAK,SAAS;QAC/B,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACnC,iBAAiB,IAAI,CAAC,EACtB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,sBAAsB,CAAC;QAC5C,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9D,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,MAAM,YAAY,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,iBAAiB,CAAC;QAC9D,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,CAAC;IACf,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,8BAA8B,CAC5C,OAAoC,EACpC,KAA+B,EAC/B,OAA6B;IAE7B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,YAAY,GAAG,wBAAwB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9D,OAAO,qBAAqB,CAAC,OAAO,EAAE;QACpC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9E,CAAC,CAAC;AACL,CAAC;AAED,iEAAiE;AACjE,MAAM,CAAC,MAAM,4BAA4B,GAAG;IAC1C,QAAQ;IACR,aAAa;IACb,UAAU;IACV,SAAS;IACT,YAAY;IACZ,WAAW;IACX,QAAQ;CACA,CAAC;AAEX,MAAM,UAAU,6BAA6B,CAAC,QAA8B;IAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,4BAA4B,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,kDAAkD,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-smart-router",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.2",
|
|
4
4
|
"description": "Auto-model router middleware for the pi.dev coding agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -88,6 +88,9 @@
|
|
|
88
88
|
"routing:test-projection": "npm run build && node --experimental-strip-types scripts/test-hydra-projection.ts",
|
|
89
89
|
"benchmark:encoder": "tsx scripts/benchmark-encoder-latency.ts",
|
|
90
90
|
"qa:shadow-dogfood": "bash scripts/qa/shadow-dogfood-session.sh",
|
|
91
|
+
"qa:count-labeled-econ": "tsx scripts/qa/count-labeled-econ.ts",
|
|
92
|
+
"qa:export-dogfood-snapshot": "tsx scripts/qa/export-dogfood-snapshot.ts",
|
|
93
|
+
"qa:dogfood-gather": "bash scripts/qa/dogfood-gather.sh",
|
|
91
94
|
"verify:ci": "npm run build && npm run typecheck && npm run lint && npm test && npm run coverage:check"
|
|
92
95
|
},
|
|
93
96
|
"peerDependencies": {
|
|
@@ -103,7 +106,7 @@
|
|
|
103
106
|
}
|
|
104
107
|
},
|
|
105
108
|
"devDependencies": {
|
|
106
|
-
"@earendil-works/pi-coding-agent": "^0.80.
|
|
109
|
+
"@earendil-works/pi-coding-agent": "^0.80.10",
|
|
107
110
|
"@types/better-sqlite3": "^7.6.13",
|
|
108
111
|
"@types/node": "^22.15.21",
|
|
109
112
|
"@typescript-eslint/eslint-plugin": "^8.62.1",
|
|
@@ -117,7 +120,7 @@
|
|
|
117
120
|
"vitest": "^3.2.3"
|
|
118
121
|
},
|
|
119
122
|
"dependencies": {
|
|
120
|
-
"@earendil-works/pi-ai": "
|
|
123
|
+
"@earendil-works/pi-ai": "^0.80.10",
|
|
121
124
|
"@huggingface/transformers": "^4.2.0",
|
|
122
125
|
"better-sqlite3": "^12.11.1",
|
|
123
126
|
"yaml": "^2.9.0",
|
|
@@ -133,6 +136,6 @@
|
|
|
133
136
|
"skills": [
|
|
134
137
|
"./skills"
|
|
135
138
|
],
|
|
136
|
-
"minPiVersion": "0.80.
|
|
139
|
+
"minPiVersion": "0.80.8"
|
|
137
140
|
}
|
|
138
141
|
}
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Privacy-safe session / window stats over RoutingTelemetry (SP-207 / #118).
|
|
3
|
+
*
|
|
4
|
+
* Aggregates only numeric and categorical telemetry fields — never prompt,
|
|
5
|
+
* message, or tool-argument bodies.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type {
|
|
9
|
+
ModelProfile,
|
|
10
|
+
PlanningDelegatePath,
|
|
11
|
+
PriceCatalog,
|
|
12
|
+
RoutingTelemetry,
|
|
13
|
+
Tier,
|
|
14
|
+
} from '../../domain/types/index.js';
|
|
15
|
+
|
|
16
|
+
export type RoleCostBucket = 'primary' | 'planning_delegate' | 'other';
|
|
17
|
+
|
|
18
|
+
export type DeploymentClass = 'local' | 'cloud' | 'unknown';
|
|
19
|
+
|
|
20
|
+
export interface RoleBucketStats {
|
|
21
|
+
readonly count: number;
|
|
22
|
+
readonly total_cost_usd: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface RoleCostBreakdown {
|
|
26
|
+
readonly primary: RoleBucketStats;
|
|
27
|
+
readonly planning_delegate: RoleBucketStats;
|
|
28
|
+
readonly other: RoleBucketStats;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Compact JSON snapshot for automation / MCP (llm-use `stats_snapshot` analog).
|
|
33
|
+
* Optional `frontier_savings_usd` is omitted when prices are unavailable (fail closed).
|
|
34
|
+
*/
|
|
35
|
+
export interface SessionStatsSnapshot {
|
|
36
|
+
readonly entry_count: number;
|
|
37
|
+
readonly total_cost_usd: number;
|
|
38
|
+
readonly mean_cost_usd: number | null;
|
|
39
|
+
readonly total_latency_ms: number;
|
|
40
|
+
readonly mean_latency_ms: number | null;
|
|
41
|
+
/** Share of entries with planning_delegate_path === 'delegate' (0–1), null if empty. */
|
|
42
|
+
readonly planning_delegate_share: number | null;
|
|
43
|
+
/** Share of entries with non-delegate path (direct / none / null). */
|
|
44
|
+
readonly direct_share: number | null;
|
|
45
|
+
/** Share classified as local (zero-tier) when distinguishable; null if none classified. */
|
|
46
|
+
readonly local_share: number | null;
|
|
47
|
+
/** Share classified as cloud when distinguishable; null if none classified. */
|
|
48
|
+
readonly cloud_share: number | null;
|
|
49
|
+
readonly role_cost: RoleCostBreakdown;
|
|
50
|
+
/**
|
|
51
|
+
* Estimated USD saved vs always-frontier baseline.
|
|
52
|
+
* Formula: sum over entries with token counts of
|
|
53
|
+
* max(0, tokens/1e6 * frontier_cost_per_1m - estimated_cost_usd).
|
|
54
|
+
* Omitted when frontier price inputs are missing (fail closed).
|
|
55
|
+
*/
|
|
56
|
+
readonly frontier_savings_usd?: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface AggregateSessionStatsOptions {
|
|
60
|
+
/**
|
|
61
|
+
* USD per 1M tokens for the always-frontier baseline.
|
|
62
|
+
* When absent / non-finite / ≤0, `frontier_savings_usd` is omitted.
|
|
63
|
+
*/
|
|
64
|
+
readonly frontier_cost_per_1m?: number;
|
|
65
|
+
/** Optional model_id → tier map (e.g. from fleet) for local vs cloud. */
|
|
66
|
+
readonly tier_by_model_id?: ReadonlyMap<string, Tier>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Mutually exclusive role for cost bucketing. */
|
|
70
|
+
export function classifyRoleCostBucket(entry: RoutingTelemetry): RoleCostBucket {
|
|
71
|
+
if (entry.planning_delegate_path === 'delegate') {
|
|
72
|
+
return 'planning_delegate';
|
|
73
|
+
}
|
|
74
|
+
if (entry.pin_reason != null) {
|
|
75
|
+
return 'primary';
|
|
76
|
+
}
|
|
77
|
+
return 'other';
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function classifyDeployment(
|
|
81
|
+
entry: RoutingTelemetry,
|
|
82
|
+
tierByModelId?: ReadonlyMap<string, Tier>,
|
|
83
|
+
): DeploymentClass {
|
|
84
|
+
const fromFleet = tierByModelId?.get(entry.selected_model_id);
|
|
85
|
+
const tier = fromFleet ?? entry.tier_hint;
|
|
86
|
+
if (tier === 'zero-tier') {
|
|
87
|
+
return 'local';
|
|
88
|
+
}
|
|
89
|
+
if (tier === 'economical-cloud' || tier === 'frontier-cloud') {
|
|
90
|
+
return 'cloud';
|
|
91
|
+
}
|
|
92
|
+
return 'unknown';
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function isPlanningDelegate(path: PlanningDelegatePath | null): boolean {
|
|
96
|
+
return path === 'delegate';
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Resolve always-frontier cost/1M from fleet + optional catalog.
|
|
101
|
+
* Returns undefined when no positive frontier price is available (fail closed).
|
|
102
|
+
*/
|
|
103
|
+
export function resolveFrontierCostPer1M(
|
|
104
|
+
fleet?: readonly ModelProfile[],
|
|
105
|
+
catalog?: PriceCatalog | null,
|
|
106
|
+
): number | undefined {
|
|
107
|
+
const candidates: number[] = [];
|
|
108
|
+
|
|
109
|
+
if (fleet) {
|
|
110
|
+
for (const model of fleet) {
|
|
111
|
+
if (model.tier !== 'frontier-cloud') {
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
const cost = model.pricing.fallback_cost_per_1m;
|
|
115
|
+
if (Number.isFinite(cost) && cost > 0) {
|
|
116
|
+
candidates.push(cost);
|
|
117
|
+
}
|
|
118
|
+
if (catalog) {
|
|
119
|
+
const key = model.pricing.registry_key ?? model.id;
|
|
120
|
+
const fromCatalog =
|
|
121
|
+
catalog.user_overrides[key] ?? catalog.registry_snapshot[key];
|
|
122
|
+
if (fromCatalog !== undefined && Number.isFinite(fromCatalog) && fromCatalog > 0) {
|
|
123
|
+
candidates.push(fromCatalog);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (candidates.length === 0) {
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Upper-bound “always frontier” baseline: most expensive known frontier rate.
|
|
134
|
+
return Math.max(...candidates);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function buildTierMap(fleet?: readonly ModelProfile[]): ReadonlyMap<string, Tier> | undefined {
|
|
138
|
+
if (!fleet || fleet.length === 0) {
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
const map = new Map<string, Tier>();
|
|
142
|
+
for (const model of fleet) {
|
|
143
|
+
map.set(model.id, model.tier);
|
|
144
|
+
map.set(`${model.provider}/${model.id}`, model.tier);
|
|
145
|
+
}
|
|
146
|
+
return map;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Pure aggregate over routing telemetry for operator stats.
|
|
151
|
+
* Does not read or emit prompt/message/tool bodies.
|
|
152
|
+
*/
|
|
153
|
+
export function aggregateSessionStats(
|
|
154
|
+
entries: readonly RoutingTelemetry[],
|
|
155
|
+
options: AggregateSessionStatsOptions = {},
|
|
156
|
+
): SessionStatsSnapshot {
|
|
157
|
+
const tierByModelId = options.tier_by_model_id;
|
|
158
|
+
const roleAccum = {
|
|
159
|
+
primary: { count: 0, total_cost_usd: 0 },
|
|
160
|
+
planning_delegate: { count: 0, total_cost_usd: 0 },
|
|
161
|
+
other: { count: 0, total_cost_usd: 0 },
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
let totalCost = 0;
|
|
165
|
+
let totalLatency = 0;
|
|
166
|
+
let delegateCount = 0;
|
|
167
|
+
let localCount = 0;
|
|
168
|
+
let cloudCount = 0;
|
|
169
|
+
let classifiedDeployment = 0;
|
|
170
|
+
|
|
171
|
+
for (const entry of entries) {
|
|
172
|
+
const cost = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
173
|
+
const latency = Number.isFinite(entry.routing_latency_ms) ? entry.routing_latency_ms : 0;
|
|
174
|
+
totalCost += cost;
|
|
175
|
+
totalLatency += latency;
|
|
176
|
+
|
|
177
|
+
if (isPlanningDelegate(entry.planning_delegate_path)) {
|
|
178
|
+
delegateCount += 1;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const role = classifyRoleCostBucket(entry);
|
|
182
|
+
roleAccum[role].count += 1;
|
|
183
|
+
roleAccum[role].total_cost_usd += cost;
|
|
184
|
+
|
|
185
|
+
const deployment = classifyDeployment(entry, tierByModelId);
|
|
186
|
+
if (deployment === 'local') {
|
|
187
|
+
localCount += 1;
|
|
188
|
+
classifiedDeployment += 1;
|
|
189
|
+
} else if (deployment === 'cloud') {
|
|
190
|
+
cloudCount += 1;
|
|
191
|
+
classifiedDeployment += 1;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const n = entries.length;
|
|
196
|
+
const snapshot: SessionStatsSnapshot = {
|
|
197
|
+
entry_count: n,
|
|
198
|
+
total_cost_usd: totalCost,
|
|
199
|
+
mean_cost_usd: n > 0 ? totalCost / n : null,
|
|
200
|
+
total_latency_ms: totalLatency,
|
|
201
|
+
mean_latency_ms: n > 0 ? totalLatency / n : null,
|
|
202
|
+
planning_delegate_share: n > 0 ? delegateCount / n : null,
|
|
203
|
+
direct_share: n > 0 ? (n - delegateCount) / n : null,
|
|
204
|
+
local_share: classifiedDeployment > 0 ? localCount / classifiedDeployment : null,
|
|
205
|
+
cloud_share: classifiedDeployment > 0 ? cloudCount / classifiedDeployment : null,
|
|
206
|
+
role_cost: {
|
|
207
|
+
primary: { ...roleAccum.primary },
|
|
208
|
+
planning_delegate: { ...roleAccum.planning_delegate },
|
|
209
|
+
other: { ...roleAccum.other },
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
const frontierSavings = estimateFrontierSavingsUsd(entries, options.frontier_cost_per_1m);
|
|
214
|
+
if (frontierSavings !== undefined) {
|
|
215
|
+
return { ...snapshot, frontier_savings_usd: frontierSavings };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return snapshot;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Optional vs-always-frontier savings. Returns undefined when price input is
|
|
223
|
+
* missing or non-positive (fail closed) or when no entry has token counts.
|
|
224
|
+
*/
|
|
225
|
+
export function estimateFrontierSavingsUsd(
|
|
226
|
+
entries: readonly RoutingTelemetry[],
|
|
227
|
+
frontierCostPer1M: number | undefined,
|
|
228
|
+
): number | undefined {
|
|
229
|
+
if (
|
|
230
|
+
frontierCostPer1M === undefined ||
|
|
231
|
+
!Number.isFinite(frontierCostPer1M) ||
|
|
232
|
+
frontierCostPer1M <= 0
|
|
233
|
+
) {
|
|
234
|
+
return undefined;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
let savings = 0;
|
|
238
|
+
let counted = 0;
|
|
239
|
+
|
|
240
|
+
for (const entry of entries) {
|
|
241
|
+
const tokens = entry.estimated_input_tokens;
|
|
242
|
+
if (tokens === null || !Number.isFinite(tokens) || tokens < 0) {
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
const actual = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
246
|
+
const frontierCost = (tokens / 1_000_000) * frontierCostPer1M;
|
|
247
|
+
savings += Math.max(0, frontierCost - actual);
|
|
248
|
+
counted += 1;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
if (counted === 0) {
|
|
252
|
+
return undefined;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
return savings;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Convenience: aggregate with fleet/catalog-derived tier map + frontier price. */
|
|
259
|
+
export function aggregateSessionStatsFromFleet(
|
|
260
|
+
entries: readonly RoutingTelemetry[],
|
|
261
|
+
fleet?: readonly ModelProfile[],
|
|
262
|
+
catalog?: PriceCatalog | null,
|
|
263
|
+
): SessionStatsSnapshot {
|
|
264
|
+
const tierMap = buildTierMap(fleet);
|
|
265
|
+
const frontierCost = resolveFrontierCostPer1M(fleet, catalog);
|
|
266
|
+
return aggregateSessionStats(entries, {
|
|
267
|
+
...(tierMap ? { tier_by_model_id: tierMap } : {}),
|
|
268
|
+
...(frontierCost !== undefined ? { frontier_cost_per_1m: frontierCost } : {}),
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** Keys that must never appear on a stats snapshot (privacy). */
|
|
273
|
+
export const SESSION_STATS_FORBIDDEN_KEYS = [
|
|
274
|
+
'prompt',
|
|
275
|
+
'prompt_text',
|
|
276
|
+
'messages',
|
|
277
|
+
'content',
|
|
278
|
+
'tool_calls',
|
|
279
|
+
'tool_args',
|
|
280
|
+
'pepper',
|
|
281
|
+
] as const;
|
|
282
|
+
|
|
283
|
+
export function assertSessionStatsPrivacySafe(snapshot: SessionStatsSnapshot): void {
|
|
284
|
+
const json = JSON.stringify(snapshot);
|
|
285
|
+
for (const key of SESSION_STATS_FORBIDDEN_KEYS) {
|
|
286
|
+
if (json.includes(`"${key}"`)) {
|
|
287
|
+
throw new Error(`Stats snapshot contains forbidden privacy key: ${key}`);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|